Clikhouse + Amazon SNS 通知

Question

我尝试向 ClickHouse 插入 Amazon SNS 通知 eventype = Open，Json 架构很复杂，所以我不知道如何创建我的 table（嵌套在嵌套...)

{
  "eventType":"Open",
  "mail":{
    "commonHeaders":{
      "from":[
        "sender@example.com"
      ],
      "messageId":"EXAMPLE7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000",
      "subject":"Message sent from Amazon SES",
      "to":[
        "recipient@example.com"
      ]
    },
    "destination":[
      "recipient@example.com"
    ],
    "headers ":[
      {
        "name":"X-SES-CONFIGURATION-SET",
        "value":"ConfigSet"
      },
      {
        "name":"X-SES-MESSAGE-TAGS",
        "value":"myCustomTag1=myCustomValue1, myCustomTag2=myCustomValue2"
      },
      {
        "name":"From",
        "value":"sender@example.com"
      },
      {
        "name":"To",
        "value":"recipient@example.com"
      },
      {
        "name":"Subject",
        "value":"Message sent from Amazon SES"
      },
      {
        "name":"MIME-Version",
        "value":"1.0"
      },
      {
        "name":"Content-Type",
        "value":"multipart/alternative; boundary=\"XBoundary\""
      }
    ],
    "headersTruncated":false,
    "messageId":"EXAMPLE7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000",
    "sendingAccountId":"123456789012",
    "source":"sender@example.com",
    "tags":{
      "myCustomTag1":[
        "myCustomValue1"
      ],
      "myCustomTag2":[
        "myCustomValue2"
      ],
      "ses:caller-identity":[
        "ses-user"
      ],
      "ses:configuration-set":[
        "ConfigSet"
      ],
      "ses:from-domain":[
        "example.com"
      ],
      "ses:source-ip":[
        "192.0.2.0"
      ]
    },
    "timestamp":"2017-08-09T21:59:49.927Z"
  },
  "open":{
    "ipAddress":"192.0.2.1",
    "timestamp":"2017-08-09T22:00:19.652Z",
    "userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60"
  }
}

我尝试了 INSERT INTO Open FORMAT JSONEachRow 和 INSERT INTO Open FORMAT JSONCompact 但没有用。

谢谢。

Answer 1

您应该将 JSON 转换为更简单的形式，无需嵌套并使用 JSONEachRow。

或将数据作为 JSONAsString 插入 CH 并使用 JSONExtract

进行转换

create table i(J String) Engine=Null;
create table f(a String, i Int64, f Float64) Engine=MergeTree order by a;

create materialized view vv to f
as select (JSONExtract(J, 'Tuple(String,Tuple(Int64,Float64))') as x),
        x.1 as a,
        x.2.1 as i,
        x.2.2 as f
from i;

echo '{"s": "val1", "b2": {"i": 42, "f": 0.1}}' |clickhouse-client -q "insert into i format JSONAsString"

select * from f
┌─a────┬──i─┬───f─┐
│ val1 │ 42 │ 0.1 │
└──────┴────┴─────┘

Clikhouse + Amazon SNS 通知

Clikhouse + Amazon SNS notification

json

amazon-web-services

amazon-sns

clickhouse