配置 Logstash 以解码其自身的事件格式 JSON
Configuring Logstash to Decode Its Own Event Format JSON
我有一个 java webbapp 的日志文件，它是使用 SLF4J、Logback 和用于 logstash 1.4.2 的 logstash-logback-encoder 创建的。虽然各种配置已成功从日志中检索数据，但 none 实际上导致返回正确的 json。根据我阅读过的每份指南，以下配置应该有效，但没有。
日志样本
{"@timestamp":"2015-02-04T00:03:43.178+00:00","@version":1,"message":"No token was found, creating new token.","logger_name":"com.company.ws.service.AuthService","thread_name":"ajp-nio-8009-exec-10","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.199+00:00","@version":1,"message":"5f8aaebd-4274-4f00-a2eb-7b2350231ef2","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-1","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.199+00:00","@version":1,"message":"36","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-1","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"5f8aaebd-4274-4f00-a2eb-7b2350231ef2","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"36","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"135a2411-ac96-492b-94e9-df6b65974f9f","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"36","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.219+00:00","@version":1,"message":"is string","logger_name":"com.company.jaxrs.parameter.RestParameterFactory","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
/etc/logstash/conf.d/01-lumberjack-input.conf
input {
lumberjack {
port => 5000
type => "logs"
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}
/etc/logstash/conf.d/10-syslog.conf
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST$
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
else if [type] == "json" {
source => "message"
}
/etc/logstash/conf.d/30-lumberjack-output.conf
output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}
/etc/logstash-forwarder（其他机器）
{
"network": {
"servers": [ "utility.company.com:5000" ],
"timeout": 15,
"ssl ca": "/etc/pki/tls/certs/logstash-forwarder.crt"
},
"files": [
{
"paths": ["/company/apache-tomcat-8.0.9/logs/vhost1.log"],
"fields": { "type": "json" }
}
]
}
我在 Kibana 中找到的最好的 returns（如果有的话 returns）看起来像这样：
{
"_index": "logstash-2015.02.04",
"_type": "json",
"_id": "8l1rDYTZSceBCklFxAuvAg",
"_score": null,
"_source": {
"message": "{\"@timestamp\":\"2015-02-04T06:03:18.794+00:00\",\"@version\":1,\"message\":\"Attribute Count 1\",\"logger_name\":\"com.company.ws.service.ReportSearchService\",\"thread_name\":\"ajp-nio-8009-exec-1\",\"level\":\"INFO\",\"level_value\":20000,\"HOSTNAME\":\"development.company.com\"}",
"@version": "1",
"@timestamp": "2015-02-04T06:13:10.685Z",
"type": "json",
"file": "/company/apache-tomcat-8.0.9/logs/vhost1.log",
"host": "development.company.com",
"offset": "4907321"
},
"sort": [
1423030390685,
1423030390685
]
}
显然，json 转换逻辑运行不正常，所以我错过了什么？
使用 this guide 配置 ELK 堆栈。
这看起来很可疑：
else if [type] == "json" {
source => "message"
}
如果这确实是您的配置文件中的内容，我不明白为什么 Logstash 不抱怨它。它应该是这样的：
else if [type] == "json" {
json {
source => "message"
}
}
或者，如果通过 lumberjack 协议收到的所有消息都是 JSON 消息，您可以使用 json 编解码器作为 lumberjack 输入。
我有一个 java webbapp 的日志文件，它是使用 SLF4J、Logback 和用于 logstash 1.4.2 的 logstash-logback-encoder 创建的。虽然各种配置已成功从日志中检索数据，但 none 实际上导致返回正确的 json。根据我阅读过的每份指南，以下配置应该有效，但没有。
日志样本
{"@timestamp":"2015-02-04T00:03:43.178+00:00","@version":1,"message":"No token was found, creating new token.","logger_name":"com.company.ws.service.AuthService","thread_name":"ajp-nio-8009-exec-10","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.199+00:00","@version":1,"message":"5f8aaebd-4274-4f00-a2eb-7b2350231ef2","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-1","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.199+00:00","@version":1,"message":"36","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-1","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"5f8aaebd-4274-4f00-a2eb-7b2350231ef2","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"36","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"135a2411-ac96-492b-94e9-df6b65974f9f","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.218+00:00","@version":1,"message":"36","logger_name":"com.company.jaxrs.provider.ParamTest","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
{"@timestamp":"2015-02-04T00:03:43.219+00:00","@version":1,"message":"is string","logger_name":"com.company.jaxrs.parameter.RestParameterFactory","thread_name":"ajp-nio-8009-exec-3","level":"INFO","level_value":20000,"HOSTNAME":"development.company.com"}
/etc/logstash/conf.d/01-lumberjack-input.conf
input {
lumberjack {
port => 5000
type => "logs"
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}
/etc/logstash/conf.d/10-syslog.conf
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST$
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
else if [type] == "json" {
source => "message"
}
/etc/logstash/conf.d/30-lumberjack-output.conf
output {
elasticsearch { host => localhost }
stdout { codec => rubydebug }
}
/etc/logstash-forwarder（其他机器）
{
"network": {
"servers": [ "utility.company.com:5000" ],
"timeout": 15,
"ssl ca": "/etc/pki/tls/certs/logstash-forwarder.crt"
},
"files": [
{
"paths": ["/company/apache-tomcat-8.0.9/logs/vhost1.log"],
"fields": { "type": "json" }
}
]
}
我在 Kibana 中找到的最好的 returns（如果有的话 returns）看起来像这样：
{
"_index": "logstash-2015.02.04",
"_type": "json",
"_id": "8l1rDYTZSceBCklFxAuvAg",
"_score": null,
"_source": {
"message": "{\"@timestamp\":\"2015-02-04T06:03:18.794+00:00\",\"@version\":1,\"message\":\"Attribute Count 1\",\"logger_name\":\"com.company.ws.service.ReportSearchService\",\"thread_name\":\"ajp-nio-8009-exec-1\",\"level\":\"INFO\",\"level_value\":20000,\"HOSTNAME\":\"development.company.com\"}",
"@version": "1",
"@timestamp": "2015-02-04T06:13:10.685Z",
"type": "json",
"file": "/company/apache-tomcat-8.0.9/logs/vhost1.log",
"host": "development.company.com",
"offset": "4907321"
},
"sort": [
1423030390685,
1423030390685
]
}
显然，json 转换逻辑运行不正常，所以我错过了什么？
使用 this guide 配置 ELK 堆栈。
这看起来很可疑：
else if [type] == "json" {
source => "message"
}
如果这确实是您的配置文件中的内容，我不明白为什么 Logstash 不抱怨它。它应该是这样的：
else if [type] == "json" {
json {
source => "message"
}
}
或者，如果通过 lumberjack 协议收到的所有消息都是 JSON 消息，您可以使用 json 编解码器作为 lumberjack 输入。