使用 python 中的子进程重定向 stdout 非常慢

Question

当我在 Python 中使用子进程重定向标准输出时，我的吞吐量非常慢。我做错了吗？

基本上，我通过管道将外部程序的标准输出放入队列中。然后在另一个函数中，我在控制台打印它。

这是一个使用 hexdump 生成随机输出的示例代码：

from subprocess import Popen, PIPE
from queue import Queue
import sys
from threading import Thread, Event
import threading

class Buffer(Queue):

    def __init__(self, *args, **kwargs):
        Queue.__init__(self, *args, **kwargs)

    def write(self, line):
        self.put_nowait(line)
        self.join()

    def read(self):
        element = self.get_nowait()
        self.task_done()
        return element

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.returncode is None:
        for line in hexdump.stdout.readlines(8192):
            buffer.write(line)
            if stopped.is_set():
                hexdump.terminate()
                hexdump.wait()
                print('process terminated.')
                break

def read_output(buffer, stopped):
    while not stopped.is_set():
        while not buffer.empty():
            output = buffer.read()
            print('********* output: {}'.format(output))
            sys.stdout.flush()
    print('stopped')
    sys.stdout.flush()


buffer = Buffer()
stopped = Event()


generate_random_output = Thread(target=write_output, args=(buffer, stopped))
generate_random_output.name = 'generate_random_output'
generate_random_output.start()

process_output = Thread(target=read_output, args=(buffer, stopped))
process_output.name = 'process_output'
process_output.start()

try:
    while True:
        continue
except KeyboardInterrupt:
    stopped.set()
    generate_random_output.join()
    process_output.join()
    print('finished generating')
    print('finished processing')

如有任何帮助，我将不胜感激。

Answer 1

不要将输出重定向到队列 - 直接处理它：

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.poll() is None:
        while not stopped.is_set():
            for line in iter(hexdump.stdout.readline, b''):
                print('********* output: %s' % line.decode(), end='')
                sys.stdout.flush()

        hexdump.terminate()
        hexdump.wait()
        print('process terminated.')
        break

使用 python 中的子进程重定向 stdout 非常慢

Redirecting stdout with subprocess in python is very slow

python

queue

subprocess

stdout

throughput