在 if 语句检查 Python 池上休眠线程
Sleep the thread on if statement check, Python Pool
我有一个代码可以读取一个非常大的文本文件并处理池中的每一行。
在 elif 的情况下,我需要让整个过程休眠 120 秒,换句话说,我希望创建的所有其他池都暂停。但是 120 秒后所有池都应该恢复工作。
代码的功能类似于:
from multiprocessing import Pool
import sys
sys.tracebacklimit = 0
def req(line):
if "@" not in line:
# (some function for processing here)
return line
elif "somestring" in line:
#HERE I NEED TO SLEEP ALL POOLS
else:
# (some function for processing)
return line
if __name__ == "__main__":
pool = Pool(20)
with open("list.txt") as source_file:
# chunk the work into batches of 20 lines at a time
pool.map(req, source_file, 35)
正如 @abarnert 所说,您应该使用 Event
对象,如下所示:
from multiprocessing import Pool
import sys
from threading import Event, Timer
sys.tracebacklimit = 0
# Setup clojure environment
def reqgen():
ev_stop = Event()
def req(line):
# Wait at the start
if ev_stop.is_set():
ev_stop.wait()
if "somestring" in line:
#HERE I NEED TO SLEEP ALL POOLS
# Clear the internal flag, make all workers await
ev_stop.clear()
# An alarm to reset the internal flag,
# which will make all workers get back to work
Timer(120, lambda: ev_stop.set()).start()
# Regular work
return req(line)
else:
# (some function for processing)
return line
return req
if __name__ == "__main__":
pool = Pool(20)
with open("list.txt") as source_file:
# chunk the work into batches of 20 lines at a time
pool.map(reqgen(), source_file, 35)
我有一个代码可以读取一个非常大的文本文件并处理池中的每一行。
在 elif 的情况下,我需要让整个过程休眠 120 秒,换句话说,我希望创建的所有其他池都暂停。但是 120 秒后所有池都应该恢复工作。
代码的功能类似于:
from multiprocessing import Pool
import sys
sys.tracebacklimit = 0
def req(line):
if "@" not in line:
# (some function for processing here)
return line
elif "somestring" in line:
#HERE I NEED TO SLEEP ALL POOLS
else:
# (some function for processing)
return line
if __name__ == "__main__":
pool = Pool(20)
with open("list.txt") as source_file:
# chunk the work into batches of 20 lines at a time
pool.map(req, source_file, 35)
正如 @abarnert 所说,您应该使用 Event
对象,如下所示:
from multiprocessing import Pool
import sys
from threading import Event, Timer
sys.tracebacklimit = 0
# Setup clojure environment
def reqgen():
ev_stop = Event()
def req(line):
# Wait at the start
if ev_stop.is_set():
ev_stop.wait()
if "somestring" in line:
#HERE I NEED TO SLEEP ALL POOLS
# Clear the internal flag, make all workers await
ev_stop.clear()
# An alarm to reset the internal flag,
# which will make all workers get back to work
Timer(120, lambda: ev_stop.set()).start()
# Regular work
return req(line)
else:
# (some function for processing)
return line
return req
if __name__ == "__main__":
pool = Pool(20)
with open("list.txt") as source_file:
# chunk the work into batches of 20 lines at a time
pool.map(reqgen(), source_file, 35)