正在将 JSON 文件上传到 elasticsearch/kibana

Question

此代码正在将 access.log 日志转换为 JSON 格式。我正在尝试以索引也创建的方式将它上传到 elasticsearch。我该怎么做？

import json 
import re
import logging
import ecs_logging
import time
import sys, requests, os
import time
from datetime import datetime
from elasticsearch import Elasticsearch
from pprint import pprint


res = requests.get('http://localhost:9200')
print (res.content)

es = Elasticsearch([{'host': 'localhost', 'port': '9200'}])


i = 0
result = {}

with open('access.log') as f:
    lines = f.readlines()

regex = '(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<dateandtime>.*)\] \"(?P<httpstatus>(GET|POST) .+ HTTP\/1\.1)\" (?P<returnstatus>\d{3} \d+) (\".*\")(?P<browserinfo>.*)\"'

for line in lines:

    r = re.match(regex,line)
    
    if r != None:
        result[i] = {'IP address': r.group('ipaddress'), 'Time Stamp': r.group('dateandtime'), 
                     'HTTP status': r.group('httpstatus'), 'Return status': 
                     r.group('returnstatus'), 'Browser Info': r.group('browserinfo')}
        i += 1
    
print(result)

with open('data.json', 'w') as fp:
    json.dump(result, fp) ```

Answer 1

修改了您的代码，希望这对您有所帮助，如果您启用了身份验证，那么我们应该提到相同的内容，并且正如您所要求的那样，它将创建一个名为“my-index”的索引。

import json 
import re
import logging
import ecs_logging
import time
import sys, requests, os
import time
from datetime import datetime
from elasticsearch import Elasticsearch, helpers
from pprint import pprint
import json
from datetime import datetime
from time import sleep
from requests.auth import HTTPBasicAuth

client = Elasticsearch(["localhost:9200"], http_auth=('elastic', '<your-es-pwd>'))
#no authentication
#client = Elasticsearch(["localhost:9200"], http_auth=('elastic', '<your-es-pwd>'))

i = 0
result = {}

with open('access.log') as f:
    lines = f.readlines()

regex = '(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<dateandtime>.*)\] \"(?P<httpstatus>(GET|POST) .+ HTTP\/1\.1)\" (?P<returnstatus>\d{3} \d+) (\".*\")(?P<browserinfo>.*)\"'

for line in lines:

    r = re.match(regex,line)
    
    if r != None:
        result[i] = {'IP address': r.group('ipaddress'), 'Time Stamp': r.group('dateandtime'), 
                     'HTTP status': r.group('httpstatus'), 'Return status': 
                     r.group('returnstatus'), 'Browser Info': r.group('browserinfo')}
        i += 1


with open('data.json', 'w') as fp:
    json.dump(result, fp)
    
directory = '.'


def load_json(directory):
    data = []
    for filename in os.listdir(directory):
        if filename.endswith('data.json'):
            with open(filename, 'r') as open_file:
                json_data = json.load(open_file)
                for i,k in enumerate(json_data.keys()):
                    data.append({
                        "_index": "my-index",
                         "_type": "my-type",
                        "_id": i,
                        "_source": json_data[k]
                    })
    return data

helpers.bulk(client, load_json('.'))

res = requests.get('http://localhost:9200', auth = HTTPBasicAuth('elastic', '<your-password'))
#no authentication enabled
#res = requests.get('http://localhost:9200')
print(res.content)

正在将 JSON 文件上传到 elasticsearch/kibana

Uploading JSON file to elasticsearch/kibana

python

elasticsearch

kibana