python attribute error : can't pickle local object. Using multiprocessing
python attribute error : can't pickle local object. Using multiprocessing
我正在编写一个旨在包装 C 代码以开始相位计采集的程序。
这段代码应该是 运行 同时在多个进程上，所以我使用 multiprocessing.
我几乎阅读了所有有关此主题的已回答问题，但我认为我遗漏了一些东西。如果有人能帮我解决这个问题，我会很高兴。
我有以下错误：
wdir='/home/castaing/Documents/LISA/lisa_zifo_monitoring/Python
Drivers/Phasemeter/Py_Code') Traceback (most recent call last):
File "/home/castaing/Documents/LISA/lisa_zifo_monitoring/Python
Drivers/Phasemeter/Py_Code/DriverPhasemeter.py", line 297, in
Process.map(start,Phasemeter.phase_list)
File
"/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/pool.py",
line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File
"/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/pool.py",
line 657, in get
raise self._value
File
"/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/pool.py",
line 431, in _handle_tasks
put(task)
File
"/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/connection.py",
line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File
"/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/reduction.py",
line 51, in dumps
cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object
'CDLL.init.._FuncPtr'
我试图定义一个全局函数来调用相位计对象的方法，如下所示。一开始我在相位计对象中有一个 multistart 方法，代码是：
def multistart(self) :
with multiprocessing.Pool(len(Phasemeter.phase_list)) as Process :
Process.map(lambda x :x.start,Phasemeter.phase_list)
这是代码（我只放了与我相关的部分）：
#%% Initialization
#%% Function definition
#Fix pickle problem ?
def start(Phasemeter_object):
Phasemeter_object.start()
#%% Class definiton
class Phasemeter :
# List of all phasemeters objects, accessed by calling Phasemeter.phase_list
phase_list=[]
#%% Initialization
def __init__(self,*args,**kwargs) :
#%% Robustness. Check type of passed arguments
#%% Path setting, parsing config file
#%% Option handling
#%% Debug, used only if verbose argument is passed in start method
#%% Defining path to shared object file
self.path=os.path.abspath(os.path.join(os.getcwd(),
self.path_to_so_file))
# LIBC is now an object and its method are C Code's functions
self.LIBC = ctypes.CDLL(self.path)
# Settings C library's signature datatype with ctypes data structre tool
# INFO: To see all datas types that can be transmited to C Code
# read ctypes documentation
# First argument is int : argc
# Second argument is string array : argv
# Third is a string : path_to_log_file
self.LIBC.lisaf_phasemeter_main.argtypes= [ctypes.c_int,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_char_p,]
# Return type is int : Return code
self.LIBC.lisaf_phasemeter_main.restypes = [ctypes.c_int,]
# Add object to phase_list, list used in multistart method
Phasemeter.phase_list.append(self)
#%% Start
def start(self):
#%% Marshalling data for C library
# Create a string array with option list length size
self.c_char_pointer_array = ctypes.c_char_p * len(self.options)
# Encode option list
self.encoded_options = [str.encode(str(i)) for i in self.options ]
# Fill the string array with encoded strings
# REMINDER: C code only understand encoded strings
self.encoded_options = self.c_char_pointer_array (*self.encoded_options)
#%% Calling C library wihth encoded options
# If the logfile option is activated then the encoded
# string is transmited
if self.encoded_path_to_log_file :
self.status = self.LIBC.lisaf_phasemeter_main(
len(self.encoded_options),
self.encoded_options,
self.encoded_path_to_log_file)
# Otherwise None pointer is transmited
else :
self.status = self.LIBC.lisaf_phasemeter_main(len(self.encoded_options),
self.encoded_options,None)
#%% Multistart
if __name__ == '__main__' :
# This function is used to start acquisition on multiple phasemeters
my_phase = Phasemeter(name="PH1")
my_phase = Phasemeter(name="PH2")
with multiprocessing.Pool(len(Phasemeter.phase_list)) as Process :
Process.map(start,Phasemeter.phase_list)
您通过删除 lambda 函数解决了一半的问题。但现在你还在“泡菜”中。在您的 Phasemeter.__init__ 方法中，您有：
self.LIBC = ctypes.CDLL(self.path)
该代码在主进程中执行，而方法 Phasemeter.start 在另一个进程中执行。因此，尝试使用 pickle 从一个地址 space serialize/deserialize self.LIBC 到另一个地址，但失败了。我建议推迟设置此属性直到 start 方法，这样它就不必被腌制：
import multiprocessing
#%% Initialization
#%% Function definition
#Fix pickle problem ?
def start(Phasemeter_object):
Phasemeter_object.start()
#%% Class definiton
class Phasemeter :
# List of all phasemeters objects, accessed by calling Phasemeter.phase_list
phase_list=[]
#%% Initialization
def __init__(self,*args,**kwargs) :
#%% Robustness. Check type of passed arguments
#%% Path setting, parsing config file
#%% Option handling
#%% Debug, used only if verbose argument is passed in start method
#%% Defining path to shared object file
self.path=os.path.abspath(os.path.join(os.getcwd(),
self.path_to_so_file))
# Add object to phase_list, list used in multistart method
Phasemeter.phase_list.append(self)
#%% Start
def start(self):
# LIBC is now an object and its method are C Code's functions
self.LIBC = ctypes.CDLL(self.path)
# Settings C library's signature datatype with ctypes data structre tool
# INFO: To see all datas types that can be transmited to C Code
# read ctypes documentation
# First argument is int : argc
# Second argument is string array : argv
# Third is a string : path_to_log_file
self.LIBC.lisaf_phasemeter_main.argtypes= [ctypes.c_int,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_char_p,]
# Return type is int : Return code
self.LIBC.lisaf_phasemeter_main.restypes = [ctypes.c_int,]
#%% Marshalling data for C library
# Create a string array with option list length size
self.c_char_pointer_array = ctypes.c_char_p * len(self.options)
# Encode option list
self.encoded_options = [str.encode(str(i)) for i in self.options ]
# Fill the string array with encoded strings
# REMINDER: C code only understand encoded strings
self.encoded_options = self.c_char_pointer_array (*self.encoded_options)
#%% Calling C library wihth encoded options
# If the logfile option is activated then the encoded
# string is transmited
if self.encoded_path_to_log_file :
self.status = self.LIBC.lisaf_phasemeter_main(
len(self.encoded_options),
self.encoded_options,
self.encoded_path_to_log_file)
# Otherwise None pointer is transmited
else :
self.status = self.LIBC.lisaf_phasemeter_main(len(self.encoded_options),
self.encoded_options,None)
#%% Multistart
if __name__ == '__main__' :
# This function is used to start acquisition on multiple phasemeters
my_phase = Phasemeter(name="PH1")
my_phase = Phasemeter(name="PH2")
with multiprocessing.Pool(len(Phasemeter.phase_list)) as Process :
Process.map(start,Phasemeter.phase_list)
作为奖励，您并行化了更多代码。
我正在编写一个旨在包装 C 代码以开始相位计采集的程序。
这段代码应该是 运行 同时在多个进程上，所以我使用 multiprocessing.
我几乎阅读了所有有关此主题的已回答问题，但我认为我遗漏了一些东西。如果有人能帮我解决这个问题，我会很高兴。
我有以下错误：
wdir='/home/castaing/Documents/LISA/lisa_zifo_monitoring/Python Drivers/Phasemeter/Py_Code') Traceback (most recent call last):
File "/home/castaing/Documents/LISA/lisa_zifo_monitoring/Python Drivers/Phasemeter/Py_Code/DriverPhasemeter.py", line 297, in Process.map(start,Phasemeter.phase_list)
File "/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/pool.py", line 268, in map return self._map_async(func, iterable, mapstar, chunksize).get()
File "/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/pool.py", line 657, in get raise self._value
File "/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/pool.py", line 431, in _handle_tasks put(task)
File "/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/connection.py", line 206, in send self._send_bytes(_ForkingPickler.dumps(obj))
File "/home/castaing/anaconda3/envs/LISA/lib/python3.7/multiprocessing/reduction.py", line 51, in dumps cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object 'CDLL.init.._FuncPtr'
我试图定义一个全局函数来调用相位计对象的方法，如下所示。一开始我在相位计对象中有一个 multistart 方法，代码是：
def multistart(self) :
with multiprocessing.Pool(len(Phasemeter.phase_list)) as Process :
Process.map(lambda x :x.start,Phasemeter.phase_list)
这是代码（我只放了与我相关的部分）：
#%% Initialization
#%% Function definition
#Fix pickle problem ?
def start(Phasemeter_object):
Phasemeter_object.start()
#%% Class definiton
class Phasemeter :
# List of all phasemeters objects, accessed by calling Phasemeter.phase_list
phase_list=[]
#%% Initialization
def __init__(self,*args,**kwargs) :
#%% Robustness. Check type of passed arguments
#%% Path setting, parsing config file
#%% Option handling
#%% Debug, used only if verbose argument is passed in start method
#%% Defining path to shared object file
self.path=os.path.abspath(os.path.join(os.getcwd(),
self.path_to_so_file))
# LIBC is now an object and its method are C Code's functions
self.LIBC = ctypes.CDLL(self.path)
# Settings C library's signature datatype with ctypes data structre tool
# INFO: To see all datas types that can be transmited to C Code
# read ctypes documentation
# First argument is int : argc
# Second argument is string array : argv
# Third is a string : path_to_log_file
self.LIBC.lisaf_phasemeter_main.argtypes= [ctypes.c_int,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_char_p,]
# Return type is int : Return code
self.LIBC.lisaf_phasemeter_main.restypes = [ctypes.c_int,]
# Add object to phase_list, list used in multistart method
Phasemeter.phase_list.append(self)
#%% Start
def start(self):
#%% Marshalling data for C library
# Create a string array with option list length size
self.c_char_pointer_array = ctypes.c_char_p * len(self.options)
# Encode option list
self.encoded_options = [str.encode(str(i)) for i in self.options ]
# Fill the string array with encoded strings
# REMINDER: C code only understand encoded strings
self.encoded_options = self.c_char_pointer_array (*self.encoded_options)
#%% Calling C library wihth encoded options
# If the logfile option is activated then the encoded
# string is transmited
if self.encoded_path_to_log_file :
self.status = self.LIBC.lisaf_phasemeter_main(
len(self.encoded_options),
self.encoded_options,
self.encoded_path_to_log_file)
# Otherwise None pointer is transmited
else :
self.status = self.LIBC.lisaf_phasemeter_main(len(self.encoded_options),
self.encoded_options,None)
#%% Multistart
if __name__ == '__main__' :
# This function is used to start acquisition on multiple phasemeters
my_phase = Phasemeter(name="PH1")
my_phase = Phasemeter(name="PH2")
with multiprocessing.Pool(len(Phasemeter.phase_list)) as Process :
Process.map(start,Phasemeter.phase_list)
您通过删除 lambda 函数解决了一半的问题。但现在你还在“泡菜”中。在您的 Phasemeter.__init__ 方法中，您有：
self.LIBC = ctypes.CDLL(self.path)
该代码在主进程中执行，而方法 Phasemeter.start 在另一个进程中执行。因此，尝试使用 pickle 从一个地址 space serialize/deserialize self.LIBC 到另一个地址，但失败了。我建议推迟设置此属性直到 start 方法，这样它就不必被腌制：
import multiprocessing
#%% Initialization
#%% Function definition
#Fix pickle problem ?
def start(Phasemeter_object):
Phasemeter_object.start()
#%% Class definiton
class Phasemeter :
# List of all phasemeters objects, accessed by calling Phasemeter.phase_list
phase_list=[]
#%% Initialization
def __init__(self,*args,**kwargs) :
#%% Robustness. Check type of passed arguments
#%% Path setting, parsing config file
#%% Option handling
#%% Debug, used only if verbose argument is passed in start method
#%% Defining path to shared object file
self.path=os.path.abspath(os.path.join(os.getcwd(),
self.path_to_so_file))
# Add object to phase_list, list used in multistart method
Phasemeter.phase_list.append(self)
#%% Start
def start(self):
# LIBC is now an object and its method are C Code's functions
self.LIBC = ctypes.CDLL(self.path)
# Settings C library's signature datatype with ctypes data structre tool
# INFO: To see all datas types that can be transmited to C Code
# read ctypes documentation
# First argument is int : argc
# Second argument is string array : argv
# Third is a string : path_to_log_file
self.LIBC.lisaf_phasemeter_main.argtypes= [ctypes.c_int,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_char_p,]
# Return type is int : Return code
self.LIBC.lisaf_phasemeter_main.restypes = [ctypes.c_int,]
#%% Marshalling data for C library
# Create a string array with option list length size
self.c_char_pointer_array = ctypes.c_char_p * len(self.options)
# Encode option list
self.encoded_options = [str.encode(str(i)) for i in self.options ]
# Fill the string array with encoded strings
# REMINDER: C code only understand encoded strings
self.encoded_options = self.c_char_pointer_array (*self.encoded_options)
#%% Calling C library wihth encoded options
# If the logfile option is activated then the encoded
# string is transmited
if self.encoded_path_to_log_file :
self.status = self.LIBC.lisaf_phasemeter_main(
len(self.encoded_options),
self.encoded_options,
self.encoded_path_to_log_file)
# Otherwise None pointer is transmited
else :
self.status = self.LIBC.lisaf_phasemeter_main(len(self.encoded_options),
self.encoded_options,None)
#%% Multistart
if __name__ == '__main__' :
# This function is used to start acquisition on multiple phasemeters
my_phase = Phasemeter(name="PH1")
my_phase = Phasemeter(name="PH2")
with multiprocessing.Pool(len(Phasemeter.phase_list)) as Process :
Process.map(start,Phasemeter.phase_list)
作为奖励，您并行化了更多代码。