Serilog Elasicsearch Sink - 忽略自定义索引模板映射

Question

我在 ASP.NET Core 3.1 中使用 Serilog elasticsearch 接收器（版本 8.4.1，elastic 7.8.0），配置如下：

{
  "Serilog": {
    "MinimumLevel": {
      "Default": "Information",
      "Override": {
        "Serilog.AspNetCore": "Information"
      }
    },
    "Enrich": [ "FromLogContext" ],
    "WriteTo": [
      {
        "Name": "Logger",
        "Args": {
          "configureLogger": {
            "Filter": [
              {
                "Name": "ByIncludingOnly",
                "Args": {
                  "expression": "SourceContext = 'Serilog.AspNetCore.RequestLoggingMiddleware'"
                }
              }
            ],
            "WriteTo": [
              {
                "Name": "Elasticsearch",
                "Args": {
                  "nodeUris": "http://localhost:9200",
                  "indexFormat": "request-logs-{0:yyyy.MM.dd}",
                  "period": 1,
                  "connectionTimeout": 5,
                  "typeName":  "_doc",
                  "inlineFields": true,
                  "restrictToMinimumLevel": "Information"
                }
              }
            ]
          }
        }
      }
    ]
  }
}

我正在使用 Serilog.AspNetCore

的 RequestLogging

app.UseSerilogRequestLogging();

并使用需要地理空间和 IP 属性的特殊映射的对象丰富自定义中间件中的 IDiagnosticContext。

HttpDiagnostics diagnostics = new HttpDiagnostics
{
  Host = host.ToString(),
  IsHttps = isHttps,
  LocalIp = localIpAddress,
  LocalPort = localPort,
  Protocol = protocol,
  RemoteIp = remoteIpAddress,
  RemotePort = remotePort,
  RequestContentLength = requestContentLength,
  RequestContentType = requestContentType,
  Scheme = scheme,
  UserAgent = userAgent,
  ResponseContentLength = responseContentLength,
  ResponseContentType = responseContentType
};

this.diagnosticContext.Set("Http", diagnostics, true);

我最终在 logevent 中正确映射字段的方法是：使用我的自定义对象重塑 LogEvent 类型，并使用 NEST 客户端为索引模板创建映射。

ElasticClient client = new ElasticClient(new Uri(settings.Uri));
PutIndexTemplateResponse response = client.Indices.PutTemplate(
  settings.Name,
  p => p.IndexPatterns(settings.IndexPattern)
        .Settings(s => s.DefaultPipeline("geoip"))
        .Map<SerilogDiagnosticsLogEvent>(m => m.AutoMap()));

索引模板映射创建正确

{
  "_doc": {
    "properties": {
      "traceId": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "level": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "requestMethod": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "message": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "sourceContext": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "parentId": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "elapsed": {
        "type": "double"
      },
      "spanId": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "@timestamp": {
        "type": "date"
      },
      "requestId": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "http": {
        "type": "object",
        "properties": {
          "responseContentLength": {
            "type": "long"
          },
          "requestContentLength": {
            "type": "long"
          },
          "geoIp": {
            "type": "object",
            "properties": {
              "cityName": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "type": "text"
              },
              "countryIsoCode": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "type": "text"
              },
              "regionName": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "type": "text"
              },
              "location": {
                "type": "geo_point"
              },
              "continentName": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "type": "text"
              }
            }
          },
          "remoteIp": {
            "type": "ip"
          },
          "localPort": {
            "type": "integer"
          },
          "scheme": {
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"
              }
            },
            "type": "text"
          },
          "remotePort": {
            "type": "integer"
          },
          "userAgent": {
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"
              }
            },
            "type": "text"
          },
          "protocol": {
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"
              }
            },
            "type": "text"
          },
          "responseContentType": {
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"
              }
            },
            "type": "text"
          },
          "host": {
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"
              }
            },
            "type": "text"
          },
          "isHttps": {
            "type": "boolean"
          },
          "localIp": {
            "type": "ip"
          },
          "requestContentType": {
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"
              }
            },
            "type": "text"
          }
        }
      },
      "connectionId": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "messageTemplate": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "requestPath": {
        "fields": {
          "keyword": {
            "ignore_above": 256,
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "statusCode": {
        "type": "integer"
      }
    }
  }
}

所以现在我希望 Serilog 索引的文档使用我创建的索引模板的映射。但实际发生的是，创建的索引没有使用模板中的正确映射。因此 IP 字段被索引为字符串，这导致 GeoIp 管道无法处理这些字段。

现在我在问自己：我的配置有问题吗？ Serilog 是否总是使用 THEIR 映射为 logevents 建立索引？如何将正确的映射应用于 logevent 中的自定义属性？

Answer 1

因此，正如本 Github 问题中所述 https://github.com/serilog/serilog-sinks-elasticsearch/issues/366 ...

原来使用 NEST 客户端创建的 index-template 映射使用 camel-case 作为属性名称，LogEvent 上的自定义属性为 pascal-case，这触发了 elasticsearch dynamic-mapping 并导致重复的映射条目。

编辑

您可以通过使用映射属性注释模型的属性并将名称设置为 pascal-case.

来解决此问题

Serilog Elasicsearch Sink - 忽略自定义索引模板映射

Serilog Elasicsearch Sink - custom index template mapping is ignored

serilog

elasticsearch-net