如何改进我的 python openCV 视频流？
How can I improve my python openCV video-stream?
我一直在做一个项目，我使用 raspberry pi 将实时视频发送到我的服务器。这有点管用，但不是我想要的。
问题主要是速度。现在我可以以大约 3.5 FPS 的速度发送 640x480 视频流和大约 0.5 FPS 的 1920x1080 视频流，这太糟糕了。由于我不是专业人士，我认为应该有一种方法可以改进我的代码。
发件人（Raspberry pi）：
def send_stream():
connection = True
while connection:
ret,frame = cap.read()
if ret:
# You might want to enable this while testing.
# cv2.imshow('camera', frame)
b_frame = pickle.dumps(frame)
b_size = len(b_frame)
try:
s.sendall(struct.pack("<L", b_size) + b_frame)
except socket.error:
print("Socket Error!")
connection = False
else:
print("Received no frame from camera, exiting.")
exit()
接收方（服务器）：
def recv_stream(self):
payload_size = struct.calcsize("<L")
data = b''
while True:
try:
start_time = datetime.datetime.now()
# keep receiving data until it gets the size of the msg.
while len(data) < payload_size:
data += self.connection.recv(4096)
# Get the frame size and remove it from the data.
frame_size = struct.unpack("<L", data[:payload_size])[0]
data = data[payload_size:]
# Keep receiving data until the frame size is reached.
while len(data) < frame_size:
data += self.connection.recv(32768)
# Cut the frame to the beginning of the next frame.
frame_data = data[:frame_size]
data = data[frame_size:]
frame = pickle.loads(frame_data)
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
end_time = datetime.datetime.now()
fps = 1/(end_time-start_time).total_seconds()
print("Fps: ",round(fps,2))
self.detect_motion(frame,fps)
self.current_frame = frame
except (socket.error,socket.timeout) as e:
# The timeout got reached or the client disconnected. Clean up the mess.
print("Cleaning up: ",e)
try:
self.connection.close()
except socket.error:
pass
self.is_connected = False
break
一个潜在的原因可能是 I/O 读取帧时的延迟。由于 cv2.VideoCapture().read() 是一个阻塞操作，主程序会暂停，直到从相机设备读取一帧并返回。一种提高性能的方法是生成另一个线程来处理 并行 中的抓取帧，而不是依赖单个线程按 顺序 顺序抓取帧.我们可以通过创建一个新线程来提高性能，该线程仅轮询新帧，而主线程处理 processing/graphing 最近的帧。
您当前的方法（顺序）：
线程 1：抓取框架 -> 处理框架 -> 绘图
建议的方法（并行）：
线程 1：抓取帧
from threading import Thread
import time
def get_frames():
while True:
ret, frame = cap.read()
time.sleep(.01)
thread_frames = Thread(target=self.get_frames, args=())
thread_frames.daemon = True
thread_frames.start()
线程 2：过程框架 -> 情节
def process_frames():
while True:
# Grab most recent frame
# Process/plot frame
...
通过使用单独的线程，您的程序将是并行的，因为总会有一个帧准备好进行处理，而不必等待帧被读入才能完成处理。
注意： 此方法将在减少 I/O 延迟的基础上提高性能。这并不是 FPS 的真正增加，因为它 延迟的显着减少 （帧始终可用于处理；我们不需要轮询相机设备并等待I/O 完成）。
在网上搜索了很久之后，我找到了一个可以将 fps 翻倍的快速解决方案（这仍然太低了：1.1 fps @1080p）。我所做的是停止使用 pickle 并改用 base64。显然酸洗图像只需要一段时间。不管怎样，这是我的新代码：
发件人（Raspberry pi）：
def send_stream():
global connected
connection = True
while connection:
if last_frame is not None:
# You might want to uncomment these lines while testing.
# cv2.imshow('camera', frame)
# cv2.waitKey(1)
frame = last_frame
# The old pickling method.
#b_frame = pickle.dumps(frame)
encoded, buffer = cv2.imencode('.jpg', frame)
b_frame = base64.b64encode(buffer)
b_size = len(b_frame)
print("Frame size = ",b_size)
try:
s.sendall(struct.pack("<L", b_size) + b_frame)
except socket.error:
print("Socket Error!")
connection = False
connected = False
s.close()
return "Socket Error"
else:
return "Received no frame from camera"
接收方（服务器）：
def recv_stream(self):
payload_size = struct.calcsize("<L")
data = b''
while True:
try:
start_time = datetime.datetime.now()
# keep receiving data until it gets the size of the msg.
while len(data) < payload_size:
data += self.connection.recv(4096)
# Get the frame size and remove it from the data.
frame_size = struct.unpack("<L", data[:payload_size])[0]
data = data[payload_size:]
# Keep receiving data until the frame size is reached.
while len(data) < frame_size:
data += self.connection.recv(131072)
# Cut the frame to the beginning of the next frame.
frame_data = data[:frame_size]
data = data[frame_size:]
# using the old pickling method.
# frame = pickle.loads(frame_data)
# Converting the image to be sent.
img = base64.b64decode(frame_data)
npimg = np.fromstring(img, dtype=np.uint8)
frame = cv2.imdecode(npimg, 1)
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
end_time = datetime.datetime.now()
fps = 1/(end_time-start_time).total_seconds()
print("Fps: ",round(fps,2))
self.detect_motion(frame,fps)
self.current_frame = frame
except (socket.error,socket.timeout) as e:
# The timeout got reached or the client disconnected. Clean up the mess.
print("Cleaning up: ",e)
try:
self.connection.close()
except socket.error:
pass
self.is_connected = False
break
我还增加了数据包大小，这在测试时从我的本地机器发送到我的本地机器时增加了 fps，但是这在使用 raspberry pi 时没有任何改变。
你可以在我的github上看到完整的代码：https://github.com/Ruud14/SecurityCamera
我一直在做一个项目，我使用 raspberry pi 将实时视频发送到我的服务器。这有点管用，但不是我想要的。 问题主要是速度。现在我可以以大约 3.5 FPS 的速度发送 640x480 视频流和大约 0.5 FPS 的 1920x1080 视频流，这太糟糕了。由于我不是专业人士，我认为应该有一种方法可以改进我的代码。
发件人（Raspberry pi）：
def send_stream():
connection = True
while connection:
ret,frame = cap.read()
if ret:
# You might want to enable this while testing.
# cv2.imshow('camera', frame)
b_frame = pickle.dumps(frame)
b_size = len(b_frame)
try:
s.sendall(struct.pack("<L", b_size) + b_frame)
except socket.error:
print("Socket Error!")
connection = False
else:
print("Received no frame from camera, exiting.")
exit()
接收方（服务器）：
def recv_stream(self):
payload_size = struct.calcsize("<L")
data = b''
while True:
try:
start_time = datetime.datetime.now()
# keep receiving data until it gets the size of the msg.
while len(data) < payload_size:
data += self.connection.recv(4096)
# Get the frame size and remove it from the data.
frame_size = struct.unpack("<L", data[:payload_size])[0]
data = data[payload_size:]
# Keep receiving data until the frame size is reached.
while len(data) < frame_size:
data += self.connection.recv(32768)
# Cut the frame to the beginning of the next frame.
frame_data = data[:frame_size]
data = data[frame_size:]
frame = pickle.loads(frame_data)
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
end_time = datetime.datetime.now()
fps = 1/(end_time-start_time).total_seconds()
print("Fps: ",round(fps,2))
self.detect_motion(frame,fps)
self.current_frame = frame
except (socket.error,socket.timeout) as e:
# The timeout got reached or the client disconnected. Clean up the mess.
print("Cleaning up: ",e)
try:
self.connection.close()
except socket.error:
pass
self.is_connected = False
break
一个潜在的原因可能是 I/O 读取帧时的延迟。由于 cv2.VideoCapture().read() 是一个阻塞操作，主程序会暂停，直到从相机设备读取一帧并返回。一种提高性能的方法是生成另一个线程来处理 并行 中的抓取帧，而不是依赖单个线程按 顺序 顺序抓取帧.我们可以通过创建一个新线程来提高性能，该线程仅轮询新帧，而主线程处理 processing/graphing 最近的帧。
您当前的方法（顺序）：
线程 1：抓取框架 -> 处理框架 -> 绘图
建议的方法（并行）：
线程 1：抓取帧
from threading import Thread
import time
def get_frames():
while True:
ret, frame = cap.read()
time.sleep(.01)
thread_frames = Thread(target=self.get_frames, args=())
thread_frames.daemon = True
thread_frames.start()
线程 2：过程框架 -> 情节
def process_frames():
while True:
# Grab most recent frame
# Process/plot frame
...
通过使用单独的线程，您的程序将是并行的，因为总会有一个帧准备好进行处理，而不必等待帧被读入才能完成处理。
注意： 此方法将在减少 I/O 延迟的基础上提高性能。这并不是 FPS 的真正增加，因为它 延迟的显着减少 （帧始终可用于处理；我们不需要轮询相机设备并等待I/O 完成）。
在网上搜索了很久之后，我找到了一个可以将 fps 翻倍的快速解决方案（这仍然太低了：1.1 fps @1080p）。我所做的是停止使用 pickle 并改用 base64。显然酸洗图像只需要一段时间。不管怎样，这是我的新代码：
发件人（Raspberry pi）：
def send_stream():
global connected
connection = True
while connection:
if last_frame is not None:
# You might want to uncomment these lines while testing.
# cv2.imshow('camera', frame)
# cv2.waitKey(1)
frame = last_frame
# The old pickling method.
#b_frame = pickle.dumps(frame)
encoded, buffer = cv2.imencode('.jpg', frame)
b_frame = base64.b64encode(buffer)
b_size = len(b_frame)
print("Frame size = ",b_size)
try:
s.sendall(struct.pack("<L", b_size) + b_frame)
except socket.error:
print("Socket Error!")
connection = False
connected = False
s.close()
return "Socket Error"
else:
return "Received no frame from camera"
接收方（服务器）：
def recv_stream(self):
payload_size = struct.calcsize("<L")
data = b''
while True:
try:
start_time = datetime.datetime.now()
# keep receiving data until it gets the size of the msg.
while len(data) < payload_size:
data += self.connection.recv(4096)
# Get the frame size and remove it from the data.
frame_size = struct.unpack("<L", data[:payload_size])[0]
data = data[payload_size:]
# Keep receiving data until the frame size is reached.
while len(data) < frame_size:
data += self.connection.recv(131072)
# Cut the frame to the beginning of the next frame.
frame_data = data[:frame_size]
data = data[frame_size:]
# using the old pickling method.
# frame = pickle.loads(frame_data)
# Converting the image to be sent.
img = base64.b64decode(frame_data)
npimg = np.fromstring(img, dtype=np.uint8)
frame = cv2.imdecode(npimg, 1)
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
end_time = datetime.datetime.now()
fps = 1/(end_time-start_time).total_seconds()
print("Fps: ",round(fps,2))
self.detect_motion(frame,fps)
self.current_frame = frame
except (socket.error,socket.timeout) as e:
# The timeout got reached or the client disconnected. Clean up the mess.
print("Cleaning up: ",e)
try:
self.connection.close()
except socket.error:
pass
self.is_connected = False
break
我还增加了数据包大小，这在测试时从我的本地机器发送到我的本地机器时增加了 fps，但是这在使用 raspberry pi 时没有任何改变。
你可以在我的github上看到完整的代码：https://github.com/Ruud14/SecurityCamera