当 json 中的字段不总是相同时的 Avro 模式
Avro Schema when no of fields in json are not always same
我正在使用融合的 kafka 平台。为了向主题生成消息,我给出了一个包含 16 个字段的 Avro 模式。现在传入的记录将仅包含来自这 16 个字段的数据,但不是全部。正如 discussion 中指出的那样,为每个字段使用默认值没有帮助。
这个问题的解决方案是什么?
示例代码:
var KafkaRest = require('kafka-rest');
var AvroSchemais = new KafkaRest.AvroSchema({
"name": "Mydata",
"type": "record",
"fields": [
{ "name": "id", "type": "string" },
{"name" : "data",
"type" : {
"type" : "array",
"items" :{
"name":"manyfields",
"type":"record",
"fields" : [
{"name" : "ip",
"type" : "string",
"default" : "NONE"},
{"name" : "iptime",
"type" : "string",
"default" : "NONE"},
{"name" : "mcc",
"type" : "string",
"default" : "NONE"},
{"name" : "mnc",
"type" : "string",
"default" : "NONE"},
{"name" : "cid",
"type" : "string",
"default" : "NONE"},
{"name" : "lac",
"type" : "string",
"default" : "NONE"}
]}}}]});
topic.produce(AvroSchema, {'id':'abcd','data': [{"ip":"12.12.12.12","lac":"1234"}]},function(err, res){
if (err){console.log(err);}
else{console.log(res);}
});
错误:
message: 'Conversion of JSON to Avro failed: Failed to convert JSON to Avro: Expected field name not found: iptime'
感谢任何帮助!!
从阅读 Avro specification 来看,默认值似乎应该让你做你想做的事:
default: A default value for this field, used when reading instances that lack this field
也许您使用的解串器没有为 JSON 编码实现此功能。同时,您可以使用 avsc
通过自动填充缺失的字段来解决此问题:
var KafkaRest = require('kafka-rest'),
avro = avsc;
var attrs = {
"name": "Mydata",
"type": "record",
"fields": [ /* ... */ ]
};
var AvroSchema = new KafkaRest.AvroSchema(attrs);
var type = avro.parse(attrs);
var withoutDefaults = {'id': 'abcd','data': [{"ip":"12.12.12.12", "lac":"1234"}]};
var withDefaults = type.clone(withoutDefaults); // All defaults are present here.
topic.produce(AvroSchema, withDefaults, function (err, res){
if (err) { console.log(err); }
else { console.log(res); }
});
我正在使用融合的 kafka 平台。为了向主题生成消息,我给出了一个包含 16 个字段的 Avro 模式。现在传入的记录将仅包含来自这 16 个字段的数据,但不是全部。正如 discussion 中指出的那样,为每个字段使用默认值没有帮助。 这个问题的解决方案是什么?
示例代码:
var KafkaRest = require('kafka-rest');
var AvroSchemais = new KafkaRest.AvroSchema({
"name": "Mydata",
"type": "record",
"fields": [
{ "name": "id", "type": "string" },
{"name" : "data",
"type" : {
"type" : "array",
"items" :{
"name":"manyfields",
"type":"record",
"fields" : [
{"name" : "ip",
"type" : "string",
"default" : "NONE"},
{"name" : "iptime",
"type" : "string",
"default" : "NONE"},
{"name" : "mcc",
"type" : "string",
"default" : "NONE"},
{"name" : "mnc",
"type" : "string",
"default" : "NONE"},
{"name" : "cid",
"type" : "string",
"default" : "NONE"},
{"name" : "lac",
"type" : "string",
"default" : "NONE"}
]}}}]});
topic.produce(AvroSchema, {'id':'abcd','data': [{"ip":"12.12.12.12","lac":"1234"}]},function(err, res){
if (err){console.log(err);}
else{console.log(res);}
});
错误:
message: 'Conversion of JSON to Avro failed: Failed to convert JSON to Avro: Expected field name not found: iptime'
感谢任何帮助!!
从阅读 Avro specification 来看,默认值似乎应该让你做你想做的事:
default: A default value for this field, used when reading instances that lack this field
也许您使用的解串器没有为 JSON 编码实现此功能。同时,您可以使用 avsc
通过自动填充缺失的字段来解决此问题:
var KafkaRest = require('kafka-rest'),
avro = avsc;
var attrs = {
"name": "Mydata",
"type": "record",
"fields": [ /* ... */ ]
};
var AvroSchema = new KafkaRest.AvroSchema(attrs);
var type = avro.parse(attrs);
var withoutDefaults = {'id': 'abcd','data': [{"ip":"12.12.12.12", "lac":"1234"}]};
var withDefaults = type.clone(withoutDefaults); // All defaults are present here.
topic.produce(AvroSchema, withDefaults, function (err, res){
if (err) { console.log(err); }
else { console.log(res); }
});