Logstash geoip случайные сбои

Question

Я использую logstash для обработки журналов из dnsmasq. В ответах DNS я использую фильтр geoip для обогащения запроса информацией о местоположении. К сожалению, некоторые запросы обогащены информацией о геолокации, а другие нет.

код геолокации:

 geoip { 
     source => "serverip" 
     } 

Список образцов IP-адресов без геолокации

104.156.81.217 
104.156.85.217 
104.16.92.65 
104.16.93.65 
104.16.94.65 
104.16.95.65 
104.16.96.65 
104.20.5.131 
104.20.6.131 
104.20.77.18 
104.20.78.18 
104.244.43.135 
104.244.43.167 
104.244.43.231 
104.244.43.39 
104.244.43.7 
104.28.30.27 
104.28.31.27 
104.40.196.5 
104.41.231.130 
104.45.95.112 
104.47.151.128 
104.71.97.80 
104.84.200.206 
104.90.129.122 
104.90.176.199 
104.90.176.77 
104.94.60.210 
104.98.119.204 
104.98.150.212 
162.255.119.124 
185.118.208.20 
185.19.196.101 
185.54.150.54 
185.63.147.12 
191.232.139.13 
191.233.80.151 
191.239.8.125 
192.229.233.25 
23.101.51.170 
23.196.235.245 
23.196.247.114 
23.196.249.86 
23.196.255.139 
23.197.0.60 
23.199.209.223 
23.235.33.217 
23.235.37.217 
23.97.173.24 

Успешная запись:

{ 
     "message" => "May 27 18:17:16 dnsmasq[385]: reply www.google.com is 216.58.213.228", 
     "@version" => "1", 
     "@timestamp" => "2016-05-27T18:17:17.147Z", 
      "path" => "/var/log/dnsmasq.log", 
      "host" => "dns", 
      "type" => "dnsmasq", 
    "reqtimestamp" => "May 27 18:17:16", 
     "program" => "dnsmasq", 
      "pid" => "385", 
      "action" => "reply", 
      "domain" => "www.google.com", 
     "function" => "is", 
     "serverip" => "216.58.213.228", 
      "geoip" => { 
         "ip" => "216.58.213.228", 
      "country_code2" => "US", 
      "country_code3" => "USA", 
      "country_name" => "United States", 
      "continent_code" => "NA", 
      "region_name" => "CA", 
       "city_name" => "Mountain View", 
      "postal_code" => "94043", 
       "latitude" => 37.41919999999999, 
       "longitude" => -122.0574, 
       "dma_code" => 807, 
       "area_code" => 650, 
       "timezone" => "America/Los_Angeles", 
     "real_region_name" => "California", 
       "location" => [ 
      [0] -122.0574, 
      [1] 37.41919999999999 
     ] 
    } 
} 

Ошибка входа:

{ 
     "message" => "May 27 18:15:50 dnsmasq[385]: reply e5884.d.akamaiedge.net is 23.197.8.251", 
     "@version" => "1", 
     "@timestamp" => "2016-05-27T18:15:51.697Z", 
      "path" => "/var/log/dnsmasq.log", 
      "host" => "dns", 
      "type" => "dnsmasq", 
    "reqtimestamp" => "May 27 18:15:50", 
     "program" => "dnsmasq", 
      "pid" => "385", 
      "action" => "reply", 
      "domain" => "e5884.d.akamaiedge.net", 
     "function" => "is", 
     "serverip" => "23.197.8.251" 
} 

Полная конфигурация Logstash:

input { 
    file { 
    path => "/var/log/dnsmasq.log" 
    start_position => "beginning" 
    type => "dnsmasq" 
    } 
} 

# Mar 15 20:13:05 dnsmasq[346]: query[A] imap.gmail.com from 192.168.0.140 
# Mar 2 20:38:45 dnsmasq-dhcp[11856]: DHCPACK(eth0) 192.168.0.152 60:67:20:72:df:00 E0199149 
# Mar 15 21:55:34 dnsmasq-dhcp[346]: 3806132383 DHCPACK(eth0) 192.168.0.80 04:0c:ce:d1:af:18 Air-de-irobot 
# Mar 16 08:54:31 dnsmasq-dhcp[346]: 4280587370 DHCPACK(eth0) 192.168.0.158 48:9d:24:ae:0e:00 BB-JP 
# Mar 16 08:18:49 dnsmasq[346]: /etc/pihole/gravity.list ssl.google-analytics.com is 192.168.0.2 

filter { 
    if [type] == "dnsmasq" { 
    grok { 
     match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num})?%{NOTSPACE:action} %{IP:clientip} %{MAC:clientmac} ?(%{HOSTNAME:clientname})?"] 
     match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num})?%{USER:action}?(\[%{USER:subaction}\])? %{NOTSPACE:domain} %{NOTSPACE:function} %{IP:clientip}"] 

     match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:action} %{DATA:data}"] 
    } 

    if [action] =~ "DHCPACK" { 
     if ![clientname] { 
     mutate { 
      add_field => { "clientname" => "No name" } 
     } 
     } 
     aggregate { 
     task_id => "%{clientip}" 
     code => "map['clientmac'] = event['clientmac']; map['clientname'] = event['clientname'];" 
     map_action => "create_or_update" 
     # timeout = 0 sets the timeout to the default value 1800 seconds. 
     timeout => 172800 
     } 
    } else if [action] == "query" { 
     aggregate { 
     task_id => "%{clientip}" 
     code => "event['clientmac'] = map['clientmac']; event['clientname'] = map['clientname']" 
     map_action => "update" 
     } 
     if ![clientname] { 
     mutate { 
      add_field => { "clientname" => "%{clientip}" } 
     } 
     } 
     if ![clientmac] { 
     mutate { 
      add_field => { "clientmac" => "%{clientip}" } 
     } 
     } 
    } else if [action] == "reply" { 
     mutate { 
      rename => { "clientip" => "serverip" } 
     } 
     geoip { 
     source => "serverip" 
     } 
    } else 
    { 
     drop{} 
    } 
    } 
} 
output { 
# elasticsearch { hosts => ["localhost:9200"] } 
stdout { codec => rubydebug } 
} 

Answer 1

версия последняя Logstash (2.3.2) расслоения logstash geoip filter 2.0.7, который, в свою очередь, включает в себя базу данных GeoIP старого Maxmind от 2013 (GeoLiteCity-2013-01-18.dat), как видно из отладочный лог

Using geoip database {:path=>"/usr/local/Cellar/logstash/2.3.2/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-2.0.7/vendor/GeoLiteCity-2013-01-18.dat", :level=>:info, :file=>"logstash/filters/geoip.rb", :line=>"97", :method=>"register"} 

Maxmind имеет новую услугу GeoIP2, что вы можете попробовать here и которые будут правильно геокодировать все IP-адреса из списка выше.

Что касается Logstash geoip, то переход от GeoIP к GeoIP2 has been made in March и будет доступен в next Logstash 5.0 version.