无法在我的 PyQT5 应用程序中使用 joblib 运行 方法并行，
Not able to run methods parallel using joblib in my PyQT5 application,
我对 PyQT5 并行编程还很陌生。我有 10 个方法想在我的 GUI 中并行 运行，我在我使用的 Joblib 上中止了。
我得到 TypeError: cannot unpack non-iterable function object。我尝试使用枚举器，但仍然中止。
我的错误信息：
joblib.externals.loky.process_executor._RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 418, in _process_worker
r = call_item()
File "/usr/local/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 272, in __call__
return self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 567, in __call__
return self.func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 225, in __call__
for func, args, kwargs in self.items]
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 225, in <listcomp>
for func, args, kwargs in self.items]
TypeError: cannot unpack non-iterable function object
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "job_parallel.py", line 73, in button_click_parallel
Parallel(n_jobs=8)(delayed(command_list[i]()) for i in range(10))
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 934, in __call__
self.retrieve()
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 833, in retrieve
self._output.extend(job.get(timeout=self.timeout))
File "/usr/local/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 521, in wrap_future_result
return future.result(timeout=timeout)
File "/usr/local/Cellar/python/3.7.2/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 432, in result
return self.__get_result()
File "/usr/local/Cellar/python/3.7.2/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
TypeError: cannot unpack non-iterable function object
Abort trap: 6
我的代码：
#!/usr/bin/env python3
import sys
from PyQt5 import QtCore, QtWidgets
from joblib import Parallel, delayed
import threading
class Window(QtWidgets.QMainWindow):
#uart_list = [1,2,3,4,5]
def __init__(self):
super(Window, self).__init__()
self.x_size = 800
self.y_size = 650
self.setFixedSize(self.x_size, self.y_size)
self.setWindowTitle("Test")
buttons = list()
self.button1 = QtWidgets.QPushButton("ONE", self)
self.button1.move(100,100)
buttons.append(self.button1)
self.button2 = QtWidgets.QPushButton("TWO", self)
self.button2.move(100,150)
buttons.append(self.button2)
for button in buttons:
button.clicked.connect(self.button_click_parallel)
self.show()
def run_iot_uart1(self):
print("Job 1")
def run_iot_uart2(self):
print("Job 2")
def run_iot_uart3(self):
print("Job 3")
def run_iot_uart4(self):
print("Job 4")
def run_iot_uart5(self):
print("Job 5")
def run_iot_uart6(self):
print("Job 6")
def run_iot_uart7(self):
print("Job 7")
def run_iot_uart8(self):
print("Job 8")
def run_iot_uart9(self):
print("Job 9")
def run_iot_uart10(self):
print("Job 10")
def button_click_parallel(self): # Click any button, process all UART in parallel
command_list=[self.run_iot_uart1, self.run_iot_uart2, self.run_iot_uart3,
self.run_iot_uart4, self.run_iot_uart5, self.run_iot_uart6,
self.run_iot_uart7, self.run_iot_uart8, self.run_iot_uart9,
self.run_iot_uart10]
Parallel(n_jobs=10)(delayed(command_list[i]()) for i in range(10))
sys.stdout.flush()
def run():
QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
app = QtWidgets.QApplication(sys.argv)
GUI = Window()
sys.exit(app.exec_())
run()
问题似乎出在您的 Parallel 设置中。我以前没有自己使用过这个库，但是文档给出了以下示例，其中 collect 是一个函数。
Parallel(n_jobs=2)(
delayed(collect)(i)
for i in range(5)
)
此处 函数 被传递给 delayed，并且该函数的参数作为参数传递给该调用的结果。即
delayed_function = delayed(function)
delayed_function(arg)
在您的示例代码中，您通过 command_list[i] 获取函数，然后立即调用它 command_list[i]()，然后传递该函数的 result（a字符串) 到 delayed.
Parallel(n_jobs=10)(
delayed(command_list[i]())
for i in range(10)
)
我想你要找的是
Parallel(n_jobs=10)(
delayed(command_list[i])()
for i in range(10)
)
无法腌制 x
对您的代码进行更改后，我仍然收到以下错误 —
File "/usr/local/lib/python3.6/site-packages/joblib/parallel.py", line 183, in
delayed
pickle.dumps(function)
TypeError: can't pickle Window objects
这是因为 Parallel 使用 pickle（一种 Python 序列化格式）将函数和参数发送给执行程序。不幸的是，它看起来好像 QWindow（可能还有其他 Qt 对象？）不可腌制。因为你的 window 方法需要访问 self 这需要被腌制。
解决方案是将这些方法移动到 a) 可腌制的 class，b) 到函数。下面我做了第二个
#!/usr/bin/env python3
import sys
from PyQt5 import QtCore, QtWidgets
from joblib import Parallel, delayed
import threading
def run_iot_uart1():
print("Job 1")
def run_iot_uart2():
print("Job 2")
def run_iot_uart3():
print("Job 3")
def run_iot_uart4():
print("Job 4")
def run_iot_uart5():
print("Job 5")
def run_iot_uart6():
print("Job 6")
def run_iot_uart7():
print("Job 7")
def run_iot_uart8():
print("Job 8")
def run_iot_uart9():
print("Job 9")
def run_iot_uart10():
print("Job 10")
class Window(QtWidgets.QMainWindow):
#uart_list = [1,2,3,4,5]
def __init__(self):
super(Window, self).__init__()
self.x_size = 800
self.y_size = 650
self.setFixedSize(self.x_size, self.y_size)
self.setWindowTitle("Test")
buttons = list()
self.button1 = QtWidgets.QPushButton("ONE", self)
self.button1.move(100,100)
buttons.append(self.button1)
self.button2 = QtWidgets.QPushButton("TWO", self)
self.button2.move(100,150)
buttons.append(self.button2)
for button in buttons:
button.clicked.connect(self.button_click_parallel)
self.show()
def button_click_parallel(self): # Click any button, process all UART in parallel
command_list=[run_iot_uart1, run_iot_uart2, run_iot_uart3,
run_iot_uart4, run_iot_uart5, run_iot_uart6,
run_iot_uart7, run_iot_uart8, run_iot_uart9,
run_iot_uart10]
Parallel(n_jobs=10)(delayed(command_list[i])() for i in range(10))
sys.stdout.flush()
def run():
QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
app = QtWidgets.QApplication(sys.argv)
GUI = Window()
sys.exit(app.exec_())
run()
运行 按下按钮 "ONE" 我得到以下输出。
martin@Martins-MacBook-Pro temp $ python3 parallel.py
Job 1
Job 3
Job 2
Job 4
Job 5
Job 6
Job 7
Job 8
Job 9
Job 10
我对 PyQT5 并行编程还很陌生。我有 10 个方法想在我的 GUI 中并行 运行，我在我使用的 Joblib 上中止了。
我得到 TypeError: cannot unpack non-iterable function object。我尝试使用枚举器，但仍然中止。
我的错误信息：
joblib.externals.loky.process_executor._RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 418, in _process_worker
r = call_item()
File "/usr/local/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 272, in __call__
return self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 567, in __call__
return self.func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 225, in __call__
for func, args, kwargs in self.items]
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 225, in <listcomp>
for func, args, kwargs in self.items]
TypeError: cannot unpack non-iterable function object
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "job_parallel.py", line 73, in button_click_parallel
Parallel(n_jobs=8)(delayed(command_list[i]()) for i in range(10))
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 934, in __call__
self.retrieve()
File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 833, in retrieve
self._output.extend(job.get(timeout=self.timeout))
File "/usr/local/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 521, in wrap_future_result
return future.result(timeout=timeout)
File "/usr/local/Cellar/python/3.7.2/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 432, in result
return self.__get_result()
File "/usr/local/Cellar/python/3.7.2/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
TypeError: cannot unpack non-iterable function object
Abort trap: 6
我的代码：
#!/usr/bin/env python3
import sys
from PyQt5 import QtCore, QtWidgets
from joblib import Parallel, delayed
import threading
class Window(QtWidgets.QMainWindow):
#uart_list = [1,2,3,4,5]
def __init__(self):
super(Window, self).__init__()
self.x_size = 800
self.y_size = 650
self.setFixedSize(self.x_size, self.y_size)
self.setWindowTitle("Test")
buttons = list()
self.button1 = QtWidgets.QPushButton("ONE", self)
self.button1.move(100,100)
buttons.append(self.button1)
self.button2 = QtWidgets.QPushButton("TWO", self)
self.button2.move(100,150)
buttons.append(self.button2)
for button in buttons:
button.clicked.connect(self.button_click_parallel)
self.show()
def run_iot_uart1(self):
print("Job 1")
def run_iot_uart2(self):
print("Job 2")
def run_iot_uart3(self):
print("Job 3")
def run_iot_uart4(self):
print("Job 4")
def run_iot_uart5(self):
print("Job 5")
def run_iot_uart6(self):
print("Job 6")
def run_iot_uart7(self):
print("Job 7")
def run_iot_uart8(self):
print("Job 8")
def run_iot_uart9(self):
print("Job 9")
def run_iot_uart10(self):
print("Job 10")
def button_click_parallel(self): # Click any button, process all UART in parallel
command_list=[self.run_iot_uart1, self.run_iot_uart2, self.run_iot_uart3,
self.run_iot_uart4, self.run_iot_uart5, self.run_iot_uart6,
self.run_iot_uart7, self.run_iot_uart8, self.run_iot_uart9,
self.run_iot_uart10]
Parallel(n_jobs=10)(delayed(command_list[i]()) for i in range(10))
sys.stdout.flush()
def run():
QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
app = QtWidgets.QApplication(sys.argv)
GUI = Window()
sys.exit(app.exec_())
run()
问题似乎出在您的 Parallel 设置中。我以前没有自己使用过这个库，但是文档给出了以下示例，其中 collect 是一个函数。
Parallel(n_jobs=2)(
delayed(collect)(i)
for i in range(5)
)
此处 函数 被传递给 delayed，并且该函数的参数作为参数传递给该调用的结果。即
delayed_function = delayed(function)
delayed_function(arg)
在您的示例代码中，您通过 command_list[i] 获取函数，然后立即调用它 command_list[i]()，然后传递该函数的 result（a字符串) 到 delayed.
Parallel(n_jobs=10)(
delayed(command_list[i]())
for i in range(10)
)
我想你要找的是
Parallel(n_jobs=10)(
delayed(command_list[i])()
for i in range(10)
)
无法腌制 x
对您的代码进行更改后，我仍然收到以下错误 —
File "/usr/local/lib/python3.6/site-packages/joblib/parallel.py", line 183, in
delayed
pickle.dumps(function)
TypeError: can't pickle Window objects
这是因为 Parallel 使用 pickle（一种 Python 序列化格式）将函数和参数发送给执行程序。不幸的是，它看起来好像 QWindow（可能还有其他 Qt 对象？）不可腌制。因为你的 window 方法需要访问 self 这需要被腌制。
解决方案是将这些方法移动到 a) 可腌制的 class，b) 到函数。下面我做了第二个
#!/usr/bin/env python3
import sys
from PyQt5 import QtCore, QtWidgets
from joblib import Parallel, delayed
import threading
def run_iot_uart1():
print("Job 1")
def run_iot_uart2():
print("Job 2")
def run_iot_uart3():
print("Job 3")
def run_iot_uart4():
print("Job 4")
def run_iot_uart5():
print("Job 5")
def run_iot_uart6():
print("Job 6")
def run_iot_uart7():
print("Job 7")
def run_iot_uart8():
print("Job 8")
def run_iot_uart9():
print("Job 9")
def run_iot_uart10():
print("Job 10")
class Window(QtWidgets.QMainWindow):
#uart_list = [1,2,3,4,5]
def __init__(self):
super(Window, self).__init__()
self.x_size = 800
self.y_size = 650
self.setFixedSize(self.x_size, self.y_size)
self.setWindowTitle("Test")
buttons = list()
self.button1 = QtWidgets.QPushButton("ONE", self)
self.button1.move(100,100)
buttons.append(self.button1)
self.button2 = QtWidgets.QPushButton("TWO", self)
self.button2.move(100,150)
buttons.append(self.button2)
for button in buttons:
button.clicked.connect(self.button_click_parallel)
self.show()
def button_click_parallel(self): # Click any button, process all UART in parallel
command_list=[run_iot_uart1, run_iot_uart2, run_iot_uart3,
run_iot_uart4, run_iot_uart5, run_iot_uart6,
run_iot_uart7, run_iot_uart8, run_iot_uart9,
run_iot_uart10]
Parallel(n_jobs=10)(delayed(command_list[i])() for i in range(10))
sys.stdout.flush()
def run():
QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
app = QtWidgets.QApplication(sys.argv)
GUI = Window()
sys.exit(app.exec_())
run()
运行 按下按钮 "ONE" 我得到以下输出。
martin@Martins-MacBook-Pro temp $ python3 parallel.py
Job 1
Job 3
Job 2
Job 4
Job 5
Job 6
Job 7
Job 8
Job 9
Job 10