Logstash geoip 随机失败

Question

我正在使用 logstash 处理来自 dnsmasq 的日志。在 DNS 响应中，我使用 'geoip' 过滤器来丰富包含位置信息的请求。遗憾的是，一些请求包含了地理定位信息，而另一些则没有。

地理位置代码：

      geoip {
        source => "serverip"
      }

没有地理定位的示例 IP 列表

104.156.81.217
104.156.85.217
104.16.92.65
104.16.93.65
104.16.94.65
104.16.95.65
104.16.96.65
104.20.5.131
104.20.6.131
104.20.77.18
104.20.78.18
104.244.43.135
104.244.43.167
104.244.43.231
104.244.43.39
104.244.43.7
104.28.30.27
104.28.31.27
104.40.196.5
104.41.231.130
104.45.95.112
104.47.151.128
104.71.97.80
104.84.200.206
104.90.129.122
104.90.176.199
104.90.176.77
104.94.60.210
104.98.119.204
104.98.150.212
162.255.119.124
185.118.208.20
185.19.196.101
185.54.150.54
185.63.147.12
191.232.139.13
191.233.80.151
191.239.8.125
192.229.233.25
23.101.51.170
23.196.235.245
23.196.247.114
23.196.249.86
23.196.255.139
23.197.0.60
23.199.209.223
23.235.33.217
23.235.37.217
23.97.173.24

成功进入：

{
         "message" => "May 27 18:17:16 dnsmasq[385]: reply www.google.com is 216.58.213.228",
        "@version" => "1",
      "@timestamp" => "2016-05-27T18:17:17.147Z",
            "path" => "/var/log/dnsmasq.log",
            "host" => "dns",
            "type" => "dnsmasq",
    "reqtimestamp" => "May 27 18:17:16",
         "program" => "dnsmasq",
             "pid" => "385",
          "action" => "reply",
          "domain" => "www.google.com",
        "function" => "is",
        "serverip" => "216.58.213.228",
           "geoip" => {
                      "ip" => "216.58.213.228",
           "country_code2" => "US",
           "country_code3" => "USA",
            "country_name" => "United States",
          "continent_code" => "NA",
             "region_name" => "CA",
               "city_name" => "Mountain View",
             "postal_code" => "94043",
                "latitude" => 37.41919999999999,
               "longitude" => -122.0574,
                "dma_code" => 807,
               "area_code" => 650,
                "timezone" => "America/Los_Angeles",
        "real_region_name" => "California",
                "location" => [
            [0] -122.0574,
            [1] 37.41919999999999
        ]
    }
}

输入失败：

{
         "message" => "May 27 18:15:50 dnsmasq[385]: reply e5884.d.akamaiedge.net is 23.197.8.251",
        "@version" => "1",
      "@timestamp" => "2016-05-27T18:15:51.697Z",
            "path" => "/var/log/dnsmasq.log",
            "host" => "dns",
            "type" => "dnsmasq",
    "reqtimestamp" => "May 27 18:15:50",
         "program" => "dnsmasq",
             "pid" => "385",
          "action" => "reply",
          "domain" => "e5884.d.akamaiedge.net",
        "function" => "is",
        "serverip" => "23.197.8.251"
}

完成 Logstash 配置：

input {
  file {
    path => "/var/log/dnsmasq.log"
    start_position => "beginning"
    type => "dnsmasq"
  }
}

# Mar 15 20:13:05 dnsmasq[346]: query[A] imap.gmail.com from 192.168.0.140
# Mar  2 20:38:45 dnsmasq-dhcp[11856]: DHCPACK(eth0) 192.168.0.152 60:67:20:72:df:00 E0199149
# Mar 15 21:55:34 dnsmasq-dhcp[346]: 3806132383 DHCPACK(eth0) 192.168.0.80 04:0c:ce:d1:af:18 Air-de-irobot
# Mar 16 08:54:31 dnsmasq-dhcp[346]: 4280587370 DHCPACK(eth0) 192.168.0.158 48:9d:24:ae:0e:00 BB-JP
# Mar 16 08:18:49 dnsmasq[346]: /etc/pihole/gravity.list ssl.google-analytics.com is 192.168.0.2

filter {
  if [type] == "dnsmasq" {
    grok {
      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{NOTSPACE:action} %{IP:clientip} %{MAC:clientmac} ?(%{HOSTNAME:clientname})?"]
      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{USER:action}?(\[%{USER:subaction}\])? %{NOTSPACE:domain} %{NOTSPACE:function} %{IP:clientip}"]

      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:action} %{DATA:data}"]
    }

    if [action] =~ "DHCPACK" {
      if ![clientname] {
        mutate {
          add_field => { "clientname" => "No name" }
        }
      }
      aggregate {
        task_id => "%{clientip}"
        code => "map['clientmac'] = event['clientmac']; map['clientname'] = event['clientname'];"
        map_action => "create_or_update"
        # timeout = 0 sets the timeout to the default value 1800 seconds.
        timeout => 172800
      }
    } else if [action] == "query" {
      aggregate {
        task_id => "%{clientip}"
        code => "event['clientmac'] = map['clientmac']; event['clientname'] = map['clientname']"
        map_action => "update"
      }
      if ![clientname] {
        mutate {
          add_field => { "clientname" => "%{clientip}" }
        }
      }
      if ![clientmac] {
        mutate {
          add_field => { "clientmac" => "%{clientip}" }
        }
      }
    } else if [action] == "reply" {
        mutate {
          rename => { "clientip" => "serverip" }
        }
      geoip {
        source => "serverip"
      }
    } else
    {
      drop{}
    }
  }
}
output {
# elasticsearch { hosts => ["localhost:9200"] }
 stdout { codec => rubydebug }
}

Answer 1

最新的 Logstash 版本 (2.3.2) 捆绑了 logstash geoip filter 2.0.7，后者又包含 2013 年的旧 Maxmind 的 GeoIP 数据库 (GeoLiteCity-2013-01-18.dat)，从调试中可以看出日志

Using geoip database {:path=>"/usr/local/Cellar/logstash/2.3.2/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-2.0.7/vendor/GeoLiteCity-2013-01-18.dat", :level=>:info, :file=>"logstash/filters/geoip.rb", :line=>"97", :method=>"register"}

Maxmind 有一个新的 GeoIP2 服务，您可以尝试 here，它将正确地对您上面列表中的所有 IP 进行地理编码。

就 Logstash geoip 而言，从 GeoIP 切换到 GeoIP2 has been made in March and will be available in the next Logstash 5.0 version。

Logstash geoip 随机失败

Logstash geoip random failures

geoip

elasticsearch

logstash

logstash-configuration

地理位置代码：

没有地理定位的示例 IP 列表

成功进入：

输入失败：

完成 Logstash 配置：