Python 多处理管理器和代理；为什么我的自定义 class 不能通过网络共享？
Python multiprocessing manager and proxy; why does my custom class not share over a network?
所以我试图通过网络共享对自定义 class 对象的访问，方法是让主机为它提供服务，多个客户端连接并更新它。似乎标准的多处理库具有针对此用例的管理器和代理。
我相信通过阅读多处理 docs (and other here)，在主机端我需要一个从多处理 BaseManager class 派生的自定义管理器，然后使用自定义代理注册我的自定义 class派生自多处理 BaseProxy class。我相信在客户端我需要连接到管理器并使用代理与我的自定义 class 交互。
我遇到的问题是，在我的自定义 class 的主机版本上更改的属性在客户端上是看不到的，反之亦然。看来我在实现上做错了什么或者误解了docs/source。有人可以帮忙提供一个工作示例并解释解决方案吗？
这是我的最小示例。
运行 一个终端的管理员：
>> python -i manager.py
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# shared class
class Foo:
"""my custom shared class"""
def __init__(self):
self.x = None
def get_x(self):
return self.x
def set_x(self, value):
self.x = value
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
# manager, run manager server for shared class and set data
print(f'manager running on proc {os.getpid()}')
MyBaseManager.register("Foo", Foo, FooProxy)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.start()
print(f'manager server running on proc {m._process.pid}')
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get copy of managed class proxy and set value to 10
f = m.Foo()
print(f'x={f.get_x()} => should be None')
f.set_x(10) # set x value to 10
print(f'x={f.get_x()} => should be 10')
运行 其他终端的客户端
>> python -i client.py
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
# client, connect to manager server and get data from shared class
print(f'client running on proc {os.getpid()}')
MyBaseManager.register("Foo", None, FooProxy)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.connect()
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get copy of managed class proxy and get value, should be 10
f = m.Foo()
print(f'x={f.get_x()} => should be 10')
你会看到客户端 return x=None 应该是 10。我没有发现明显的异常并尝试了 Ubuntu 18 和 OSX 并获得相同的行为。
请注意我仅限于 python 3.6 并且在我的生产系统上使用标准库。我还成功地尝试了多处理文档中的其他示例，因此相信这不是 computer/networking 问题。
谢谢
客户端有一个不同的共享 Foo 实例。您可以将该实例传递给其他进程并查看它是否正常工作。下面我将它传递给另外两个进程：一个打印当前值并更改它，另一个等待片刻，然后打印第一个工人更新的值：
import multiprocessing as mp
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
import os
import time
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
def worker1(f):
print(f'worker pid={os.getpid()} x={f.get_x()}')
f.set_x(5)
def worker2(f):
time.sleep(1)
print(f'worker pid={os.getpid()} x={f.get_x()}')
if __name__ == '__main__':
# client, connect to manager server and get data from shared class
print(f'client running on proc {os.getpid()}')
MyBaseManager.register("Foo", None, FooProxy)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.connect()
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get copy of managed class proxy and get value, should be 10
f = m.Foo()
f.set_x(7)
print(f'client x={f.get_x()}')
mp.Process(target=worker1,args=(f,)).start()
mp.Process(target=worker2,args=(f,)).start()
来自py -i client.py的输出：
client running on proc 11332
(proxy) <__main__.MyBaseManager object at 0x000001EB9AC0AFA0>
(referant) <__main__.MyBaseManager object at 0x000001EB9AC0AFA0>
client x=7
>>> worker pid=2560 x=7
worker pid=11444 x=5
如果你想要一个单例 Foo，你可以公开一个 get_foo 函数来获得一个通用的全局 Foo:
manager.py:
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
import os
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# shared class
class Foo:
"""my custom shared class"""
def __init__(self):
self.x = None
def get_x(self):
return self.x
def set_x(self, value):
self.x = value
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
## Global Foo and function to get the instance.
global_foo = Foo()
def get_foo():
return global_foo
# custom shared class manager
class MyBaseManager(BaseManager):
pass
if __name__ == '__main__':
# manager, run manager server for shared class and set data
print(f'manager running on proc {os.getpid()}')
MyBaseManager.register("Foo", Foo, FooProxy)
MyBaseManager.register("get_foo", get_foo)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.start()
print(f'manager server running on proc {m._process.pid}')
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get global instance and set value to 10
f = m.get_foo()
print(f'x={f.get_x()} => should be None')
f.set_x(10) # set x value to 10
print(f'x={f.get_x()} => should be 10')
client.py:
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
import os
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
# client, connect to manager server and get data from shared class
print(f'client running on proc {os.getpid()}')
MyBaseManager.register("Foo", None, FooProxy)
MyBaseManager.register("get_foo")
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.connect()
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get global instance and get value, should be 10
f = m.get_foo()
print(f'x={f.get_x()} => should be 10')
客户端输出：
client running on proc 7180
(proxy) <__main__.MyBaseManager object at 0x000001A3F0267610>
(referant) <__main__.MyBaseManager object at 0x000001A3F0267610>
x=10 => should be 10
所以我试图通过网络共享对自定义 class 对象的访问，方法是让主机为它提供服务，多个客户端连接并更新它。似乎标准的多处理库具有针对此用例的管理器和代理。
我相信通过阅读多处理 docs (and other
我遇到的问题是，在我的自定义 class 的主机版本上更改的属性在客户端上是看不到的，反之亦然。看来我在实现上做错了什么或者误解了docs/source。有人可以帮忙提供一个工作示例并解释解决方案吗？
这是我的最小示例。
运行 一个终端的管理员：
>> python -i manager.py
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# shared class
class Foo:
"""my custom shared class"""
def __init__(self):
self.x = None
def get_x(self):
return self.x
def set_x(self, value):
self.x = value
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
# manager, run manager server for shared class and set data
print(f'manager running on proc {os.getpid()}')
MyBaseManager.register("Foo", Foo, FooProxy)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.start()
print(f'manager server running on proc {m._process.pid}')
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get copy of managed class proxy and set value to 10
f = m.Foo()
print(f'x={f.get_x()} => should be None')
f.set_x(10) # set x value to 10
print(f'x={f.get_x()} => should be 10')
运行 其他终端的客户端
>> python -i client.py
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
# client, connect to manager server and get data from shared class
print(f'client running on proc {os.getpid()}')
MyBaseManager.register("Foo", None, FooProxy)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.connect()
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get copy of managed class proxy and get value, should be 10
f = m.Foo()
print(f'x={f.get_x()} => should be 10')
你会看到客户端 return x=None 应该是 10。我没有发现明显的异常并尝试了 Ubuntu 18 和 OSX 并获得相同的行为。
请注意我仅限于 python 3.6 并且在我的生产系统上使用标准库。我还成功地尝试了多处理文档中的其他示例，因此相信这不是 computer/networking 问题。
谢谢
客户端有一个不同的共享 Foo 实例。您可以将该实例传递给其他进程并查看它是否正常工作。下面我将它传递给另外两个进程：一个打印当前值并更改它，另一个等待片刻，然后打印第一个工人更新的值：
import multiprocessing as mp
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
import os
import time
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
def worker1(f):
print(f'worker pid={os.getpid()} x={f.get_x()}')
f.set_x(5)
def worker2(f):
time.sleep(1)
print(f'worker pid={os.getpid()} x={f.get_x()}')
if __name__ == '__main__':
# client, connect to manager server and get data from shared class
print(f'client running on proc {os.getpid()}')
MyBaseManager.register("Foo", None, FooProxy)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.connect()
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get copy of managed class proxy and get value, should be 10
f = m.Foo()
f.set_x(7)
print(f'client x={f.get_x()}')
mp.Process(target=worker1,args=(f,)).start()
mp.Process(target=worker2,args=(f,)).start()
来自py -i client.py的输出：
client running on proc 11332
(proxy) <__main__.MyBaseManager object at 0x000001EB9AC0AFA0>
(referant) <__main__.MyBaseManager object at 0x000001EB9AC0AFA0>
client x=7
>>> worker pid=2560 x=7
worker pid=11444 x=5
如果你想要一个单例 Foo，你可以公开一个 get_foo 函数来获得一个通用的全局 Foo:
manager.py:
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
import os
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# shared class
class Foo:
"""my custom shared class"""
def __init__(self):
self.x = None
def get_x(self):
return self.x
def set_x(self, value):
self.x = value
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
## Global Foo and function to get the instance.
global_foo = Foo()
def get_foo():
return global_foo
# custom shared class manager
class MyBaseManager(BaseManager):
pass
if __name__ == '__main__':
# manager, run manager server for shared class and set data
print(f'manager running on proc {os.getpid()}')
MyBaseManager.register("Foo", Foo, FooProxy)
MyBaseManager.register("get_foo", get_foo)
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.start()
print(f'manager server running on proc {m._process.pid}')
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get global instance and set value to 10
f = m.get_foo()
print(f'x={f.get_x()} => should be None')
f.set_x(10) # set x value to 10
print(f'x={f.get_x()} => should be 10')
client.py:
from multiprocessing.managers import BaseManager, BaseProxy, MakeProxyType
import os
IP = 'localhost'
PORT = 50000
KEY = b'abracadabra'
# proxy class
FooProxyBase = MakeProxyType('FooProxyBase', ('get_x', 'set_x'))
class FooProxy(FooProxyBase):
"""shared class proxy"""
def get_x(self):
return self._callmethod('get_x')
def set_x(self, value):
return self._callmethod('set_x', (value,))
# custom shared class manager
class MyBaseManager(BaseManager):
pass
# client, connect to manager server and get data from shared class
print(f'client running on proc {os.getpid()}')
MyBaseManager.register("Foo", None, FooProxy)
MyBaseManager.register("get_foo")
m = MyBaseManager(address=(IP, PORT), authkey=KEY)
m.connect()
print(f'(proxy) {str(m)}')
print(f'(referant) {repr(m)}')
# get global instance and get value, should be 10
f = m.get_foo()
print(f'x={f.get_x()} => should be 10')
客户端输出：
client running on proc 7180
(proxy) <__main__.MyBaseManager object at 0x000001A3F0267610>
(referant) <__main__.MyBaseManager object at 0x000001A3F0267610>
x=10 => should be 10