使用 sqlalchemy marshmallow 难以序列化地理列类型
Difficulty serializing Geography column type using sqlalchemy marshmallow
我尝试使用 Marshmallow 对 SQLAlchemy 对象进行反序列化和序列化，但是 运行 在处理 ORM 中的地理字段时遇到了问题。
首先是模型：
class Address(db.Model, TableColumnsBase):
__tablename__ = 'address'
addressLine1 = db.Column(String(255), nullable=True, info="Street address, company name, c/o")
addressLine2 = db.Column(String(255), nullable=True, info="Apartment, suite, unit, building floor, etc")
countryCode = db.Column(String(255), nullable=True, info="Country code such as AU, US etc")
suburb = db.Column(String(255), nullable=True, info="Users suburb such as Elizabeth Bay")
postcode = db.Column(String(32), nullable=True, info="Users postcode such as 2011 for Elizabeth Bay")
state = db.Column(String(64), info="State for user such as NSW")
user_presence = one_to_many('UserPresence', backref = 'user', lazy='select', cascade='all, delete-orphan')
location = Column(Geography(geometry_type='POINT', srid=4326))
discriminator = Column('type', String(50))
__mapper_args__ = {'polymorphic_on': discriminator}
def as_physical(self):
pa = { # TODO - stub, make it proper
"latitude": 2.1,
"longitude": 1.1,
"unitNumber": '1',
"streetNumber": self.addressLine1,
"streetName": self.addressLine2,
"streetType": 'street',
"suburb": self.suburb,
"postcode": self.postcode
}
return pa
其次是棉花糖架构：
class AddressSchema(ModelSchema):
class Meta:
model = Address
sqla_session
然后尝试运行时出错：
/Users/james/.virtualenvs/trustmile-api-p2710/bin/python /Users/james/Documents/workspace/trustmile-backend/trustmile/tests/postgis_scratch.py
Traceback (most recent call last):
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/tests/postgis_scratch.py", line 1, in <module>
from app import db
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/__init__.py", line 54, in <module>
from app.api.consumer_v1 import bp as blueprint
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/__init__.py", line 4, in <module>
import courier_v1
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/courier_v1/__init__.py", line 5, in <module>
from .routes import routes
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/courier_v1/routes.py", line 10, in <module>
from .api.nearestNeighbours_latitude_longitude import NearestneighboursLatitudeLongitude
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/courier_v1/api/__init__.py", line 5, in <module>
from app.ops import requesthandler
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/ops/requesthandler.py", line 1, in <module>
from app.ops.consumer_operations import cons_ops, ConsumerOperationsFactory
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/ops/consumer_operations.py", line 19, in <module>
from app.users.serialize import ApplicationInstallationSchema, UserAddressSchema, ConsumerUserSchema, CourierUserSchema
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/users/serialize.py", line 40, in <module>
class AddressSchema(ModelSchema):
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow/schema.py", line 116, in __new__
dict_cls=dict_cls
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/schema.py", line 57, in get_declared_fields
declared_fields = mcs.get_fields(converter, opts)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/schema.py", line 90, in get_fields
include_fk=opts.include_fk,
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 77, in fields_for_model
field = self.property2field(prop)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 95, in property2field
field_class = self._get_field_class_for_property(prop)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 153, in _get_field_class_for_property
field_cls = self._get_field_class_for_column(column)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 123, in _get_field_class_for_column
return self._get_field_class_for_data_type(column.type)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 145, in _get_field_class_for_data_type
'Could not find field column of type {0}.'.format(types[0]))
marshmallow_sqlalchemy.exceptions.ModelConversionError: Could not find field column of type <class 'geoalchemy2.types.Geography'>.
我在位置字段上尝试了一些覆盖，但无济于事。非常感谢任何帮助。
2016 年 1 月 24 日更新
这是我的代码，因为它具有更简单的模型：
SQLAlchemy 模型：
class Location(db.Model, TableColumnsBase):
__tablename__ = "location"
loc = db.Column(Geography(geometry_type='POINT', srid=4326))
marshmallow_sqlalchemy 模式对象
class LocationSchema(ModelSchema):
loc = GeographySerializationField(attribute='loc')
class Meta:
model = Location
sqla_session = db.session
model_converter = GeoConverter
根据建议的其他管道：
class GeoConverter(ModelConverter):
SQLA_TYPE_MAPPING = ModelConverter.SQLA_TYPE_MAPPING.copy()
SQLA_TYPE_MAPPING.update({
Geography: fields.Str
})
class GeographySerializationField(fields.Field):
def _serialize(self, value, attr, obj):
if value is None:
return value
else:
if isinstance(value, Geography):
return json.dumps({'latitude': db.session.scalar(geo_funcs.ST_X(value)), 'longitude': db.session.scalar(geo_funcs.ST_Y(value))})
else:
return None
def _deserialize(self, value, attr, data):
"""Deserialize value. Concrete :class:`Field` classes should implement this method.
:param value: The value to be deserialized.
:param str attr: The attribute/key in `data` to be deserialized.
:param dict data: The raw input data passed to the `Schema.load`.
:raise ValidationError: In case of formatting or validation failure.
:return: The deserialized value.
.. versionchanged:: 2.0.0
Added ``attr`` and ``data`` parameters.
"""
if value is None:
return value
else:
if isinstance(value, Geography):
return {'latitude': db.session.scalar(geo_funcs.ST_X(value)), 'longitude': db.session.scalar(geo_funcs.ST_Y(value))}
else:
return None
在运行宁代码中：
from app.users.serialize import *
from app.model.meta.schema import Location
l = LocationSchema()
loc = Location(27.685994, 85.317815)
r = l.load(loc)
print r
我得到的结果是：
UnmarshalResult(data={}, errors={u'_schema': [u'Invalid input type.']})
您可以覆盖 ModelConverter class 并为您的地理字段指定自定义映射。请在此处查看 Jair Perrut 的回答
from marshmallow_sqlalchemy import ModelConverter
from marshmallow import fields
class GeoConverter(ModelConverter):
SQLA_TYPE_MAPPING = ModelConverter.SQLA_TYPE_MAPPING.copy()
SQLA_TYPE_MAPPING.update({
Geography: fields.Str
})
class Meta:
model = Address
sqla_session = session
model_converter = GeoConverter
解决了，首先是误用了方法，但答案是：
对于序列化：
class GeoConverter(ModelConverter):
SQLA_TYPE_MAPPING = ModelConverter.SQLA_TYPE_MAPPING.copy()
SQLA_TYPE_MAPPING.update({
Geography: fields.Str
})
class GeographySerializationField(fields.String):
def _serialize(self, value, attr, obj):
if value is None:
return value
else:
if attr == 'loc':
return {'latitude': db.session.scalar(geo_funcs.ST_X(value)), 'longitude': db.session.scalar(geo_funcs.ST_Y(value))}
else:
return None
def _deserialize(self, value, attr, data):
"""Deserialize value. Concrete :class:`Field` classes should implement this method.
:param value: The value to be deserialized.
:param str attr: The attribute/key in `data` to be deserialized.
:param dict data: The raw input data passed to the `Schema.load`.
:raise ValidationError: In case of formatting or validation failure.
:return: The deserialized value.
.. versionchanged:: 2.0.0
Added ``attr`` and ``data`` parameters.
"""
if value is None:
return value
else:
if attr == 'loc':
return WKTGeographyElement('POINT({0} {1})'.format(str(value.get('longitude')), str(value.get('latitude'))))
else:
return None
架构定义：
class LocationSchema(ModelSchema):
loc = GeographySerializationField(attribute='loc')
id = fields.UUID(Location, attribute='id')
class Meta:
model = Location
sqla_session = db.session
model_converter = GeoConverter
测试：
from app.users.serialize import *
from app.model.meta.schema import Location
from app import db
l = LocationSchema()
loc = Location(27.685994, 85.317815)
db.session.add(loc)
db.session.commit()
r = l.dump(loc).data
print r
loc_obj = l.load(r)
print loc_obj.data
我尝试使用 Marshmallow 对 SQLAlchemy 对象进行反序列化和序列化，但是 运行 在处理 ORM 中的地理字段时遇到了问题。
首先是模型：
class Address(db.Model, TableColumnsBase):
__tablename__ = 'address'
addressLine1 = db.Column(String(255), nullable=True, info="Street address, company name, c/o")
addressLine2 = db.Column(String(255), nullable=True, info="Apartment, suite, unit, building floor, etc")
countryCode = db.Column(String(255), nullable=True, info="Country code such as AU, US etc")
suburb = db.Column(String(255), nullable=True, info="Users suburb such as Elizabeth Bay")
postcode = db.Column(String(32), nullable=True, info="Users postcode such as 2011 for Elizabeth Bay")
state = db.Column(String(64), info="State for user such as NSW")
user_presence = one_to_many('UserPresence', backref = 'user', lazy='select', cascade='all, delete-orphan')
location = Column(Geography(geometry_type='POINT', srid=4326))
discriminator = Column('type', String(50))
__mapper_args__ = {'polymorphic_on': discriminator}
def as_physical(self):
pa = { # TODO - stub, make it proper
"latitude": 2.1,
"longitude": 1.1,
"unitNumber": '1',
"streetNumber": self.addressLine1,
"streetName": self.addressLine2,
"streetType": 'street',
"suburb": self.suburb,
"postcode": self.postcode
}
return pa
其次是棉花糖架构：
class AddressSchema(ModelSchema):
class Meta:
model = Address
sqla_session
然后尝试运行时出错：
/Users/james/.virtualenvs/trustmile-api-p2710/bin/python /Users/james/Documents/workspace/trustmile-backend/trustmile/tests/postgis_scratch.py
Traceback (most recent call last):
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/tests/postgis_scratch.py", line 1, in <module>
from app import db
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/__init__.py", line 54, in <module>
from app.api.consumer_v1 import bp as blueprint
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/__init__.py", line 4, in <module>
import courier_v1
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/courier_v1/__init__.py", line 5, in <module>
from .routes import routes
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/courier_v1/routes.py", line 10, in <module>
from .api.nearestNeighbours_latitude_longitude import NearestneighboursLatitudeLongitude
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/api/courier_v1/api/__init__.py", line 5, in <module>
from app.ops import requesthandler
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/ops/requesthandler.py", line 1, in <module>
from app.ops.consumer_operations import cons_ops, ConsumerOperationsFactory
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/ops/consumer_operations.py", line 19, in <module>
from app.users.serialize import ApplicationInstallationSchema, UserAddressSchema, ConsumerUserSchema, CourierUserSchema
File "/Users/james/Documents/workspace/trustmile-backend/trustmile/app/users/serialize.py", line 40, in <module>
class AddressSchema(ModelSchema):
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow/schema.py", line 116, in __new__
dict_cls=dict_cls
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/schema.py", line 57, in get_declared_fields
declared_fields = mcs.get_fields(converter, opts)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/schema.py", line 90, in get_fields
include_fk=opts.include_fk,
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 77, in fields_for_model
field = self.property2field(prop)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 95, in property2field
field_class = self._get_field_class_for_property(prop)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 153, in _get_field_class_for_property
field_cls = self._get_field_class_for_column(column)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 123, in _get_field_class_for_column
return self._get_field_class_for_data_type(column.type)
File "/Users/james/.virtualenvs/trustmile-api-p2710/lib/python2.7/site-packages/marshmallow_sqlalchemy/convert.py", line 145, in _get_field_class_for_data_type
'Could not find field column of type {0}.'.format(types[0]))
marshmallow_sqlalchemy.exceptions.ModelConversionError: Could not find field column of type <class 'geoalchemy2.types.Geography'>.
我在位置字段上尝试了一些覆盖，但无济于事。非常感谢任何帮助。
2016 年 1 月 24 日更新
这是我的代码，因为它具有更简单的模型：
SQLAlchemy 模型：
class Location(db.Model, TableColumnsBase):
__tablename__ = "location"
loc = db.Column(Geography(geometry_type='POINT', srid=4326))
marshmallow_sqlalchemy 模式对象
class LocationSchema(ModelSchema):
loc = GeographySerializationField(attribute='loc')
class Meta:
model = Location
sqla_session = db.session
model_converter = GeoConverter
根据建议的其他管道：
class GeoConverter(ModelConverter):
SQLA_TYPE_MAPPING = ModelConverter.SQLA_TYPE_MAPPING.copy()
SQLA_TYPE_MAPPING.update({
Geography: fields.Str
})
class GeographySerializationField(fields.Field):
def _serialize(self, value, attr, obj):
if value is None:
return value
else:
if isinstance(value, Geography):
return json.dumps({'latitude': db.session.scalar(geo_funcs.ST_X(value)), 'longitude': db.session.scalar(geo_funcs.ST_Y(value))})
else:
return None
def _deserialize(self, value, attr, data):
"""Deserialize value. Concrete :class:`Field` classes should implement this method.
:param value: The value to be deserialized.
:param str attr: The attribute/key in `data` to be deserialized.
:param dict data: The raw input data passed to the `Schema.load`.
:raise ValidationError: In case of formatting or validation failure.
:return: The deserialized value.
.. versionchanged:: 2.0.0
Added ``attr`` and ``data`` parameters.
"""
if value is None:
return value
else:
if isinstance(value, Geography):
return {'latitude': db.session.scalar(geo_funcs.ST_X(value)), 'longitude': db.session.scalar(geo_funcs.ST_Y(value))}
else:
return None
在运行宁代码中：
from app.users.serialize import *
from app.model.meta.schema import Location
l = LocationSchema()
loc = Location(27.685994, 85.317815)
r = l.load(loc)
print r
我得到的结果是：
UnmarshalResult(data={}, errors={u'_schema': [u'Invalid input type.']})
您可以覆盖 ModelConverter class 并为您的地理字段指定自定义映射。请在此处查看 Jair Perrut 的回答
from marshmallow_sqlalchemy import ModelConverter
from marshmallow import fields
class GeoConverter(ModelConverter):
SQLA_TYPE_MAPPING = ModelConverter.SQLA_TYPE_MAPPING.copy()
SQLA_TYPE_MAPPING.update({
Geography: fields.Str
})
class Meta:
model = Address
sqla_session = session
model_converter = GeoConverter
解决了，首先是误用了方法，但答案是：
对于序列化：
class GeoConverter(ModelConverter):
SQLA_TYPE_MAPPING = ModelConverter.SQLA_TYPE_MAPPING.copy()
SQLA_TYPE_MAPPING.update({
Geography: fields.Str
})
class GeographySerializationField(fields.String):
def _serialize(self, value, attr, obj):
if value is None:
return value
else:
if attr == 'loc':
return {'latitude': db.session.scalar(geo_funcs.ST_X(value)), 'longitude': db.session.scalar(geo_funcs.ST_Y(value))}
else:
return None
def _deserialize(self, value, attr, data):
"""Deserialize value. Concrete :class:`Field` classes should implement this method.
:param value: The value to be deserialized.
:param str attr: The attribute/key in `data` to be deserialized.
:param dict data: The raw input data passed to the `Schema.load`.
:raise ValidationError: In case of formatting or validation failure.
:return: The deserialized value.
.. versionchanged:: 2.0.0
Added ``attr`` and ``data`` parameters.
"""
if value is None:
return value
else:
if attr == 'loc':
return WKTGeographyElement('POINT({0} {1})'.format(str(value.get('longitude')), str(value.get('latitude'))))
else:
return None
架构定义：
class LocationSchema(ModelSchema):
loc = GeographySerializationField(attribute='loc')
id = fields.UUID(Location, attribute='id')
class Meta:
model = Location
sqla_session = db.session
model_converter = GeoConverter
测试：
from app.users.serialize import *
from app.model.meta.schema import Location
from app import db
l = LocationSchema()
loc = Location(27.685994, 85.317815)
db.session.add(loc)
db.session.commit()
r = l.dump(loc).data
print r
loc_obj = l.load(r)
print loc_obj.data