对于捕获的帧，GStreamer 时间戳 (PTS) 不是单调递增的
GStreamer timestamps (PTS) are not monotonically increasing for captured frames
我编写了一些代码来使用 GStreamer 1.0 (PyGST) 从网络摄像头捕获帧。知道捕获的确切时间对我来说很重要。为此，我设置了 v4l2src 属性 do-timestamp，并使用 appsink 将缓冲区 PTS 写入文本文件。
但是，时间戳并不是单调递增的。例如。第 16 帧的时间戳为 0.88199 秒，第 17 帧的时间戳为 0.77462 秒，即比前一帧早 0.10737 秒。 （我有一个显示问题的图表，但缺乏 post 它所必需的声誉。）
捕获的GstBuffer的PTS并不总是单调递增的，这对吗？如果这不是正常行为，有人知道我搞砸了什么吗？
我使用罗技 c920 网络摄像头。这些帧在相机上采用 h.264 编码。代码大致如下所示：
import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, Gtk
GObject.threads_init()
Gst.init(None)
class Webcam:
def __init__(self, video_dev='/dev/video0', fps=30):
ts_log_fname = 'webcam_timestamps.log'
vid_fname = 'webcam.mkv'
self.ts_log = open(ts_log_fname, 'w')
self.ts_log.write('video filename: %s\n '
'\nframe_number, cam_running_ts\n' % vid_fname)
self.n_frames = 0
# Create GStreamer pipline
self.pipeline = Gst.Pipeline()
# Create bus to get events from GStreamer pipeline
self.bus = self.pipeline.get_bus()
self.bus.add_signal_watch()
self.bus.connect('message::error', self.on_error)
self.bus.enable_sync_message_emission()
self.bus.connect('sync-message::element', self.on_sync_message)
###########################
# Callable function
###########################
def on_new_sample(appsink):
"""
Function called from the pipeline by appsink.
Writes the timestampes of frame capture to a log file.
"""
# Get the buffer
smp = appsink.emit('pull-sample')
buf = smp.get_buffer()
# Nanoseconds to seconds
timestamp = np.float64(1e-9) * buf.pts
self.n_frames += 1
self.ts_log.write('%d,%0.9f\n' % (self.n_frames, timestamp))
return False
###########################
# Create GStreamer elements
###########################
# Video source:
self.v4l2src0 = Gst.ElementFactory.make('v4l2src', None)
self.v4l2src0.set_property('device', video_dev)
self.v4l2src0.set_property('do-timestamp', 'true')
# Video source filters:
vid0caps = Gst.Caps.from_string('video/x-h264,width=%d,height=%d,'
'framerate=%d/1' % (1280, 720, fps))
self.vid0filter = Gst.ElementFactory.make('capsfilter', None)
self.vid0filter.set_property('caps', vid0caps)
# Parse video:
self.vid0parse = Gst.ElementFactory.make('h264parse', None)
# Split:
self.tee0 = Gst.ElementFactory.make('tee', None)
self.tee0.set_property('name', 't0')
####
# Display branch
####
# Decode
self.vid0decode = Gst.ElementFactory.make('avdec_h264', None)
# Scale to display size:
self.disp0scale = Gst.ElementFactory.make('videoscale', None)
# Display filter caps:
disp0caps = Gst.Caps.from_string('video/x-raw,width=%d,height=%d' %
(800, 600))
# Sinks:
self.disp0sink = Gst.ElementFactory.make('autovideosink', None)
self.disp0sink.set_property('filter-caps', disp0caps)
####
# File branch
####
self.mux = Gst.ElementFactory.make('matroskamux', None)
self.file0sink = Gst.ElementFactory.make('filesink', None)
self.file0sink.set_property('location', vid_fname)
self.file0sink.set_property('sync', False)
####
# Timestamp branch
####
# Create appsink
self.ts0sink = Gst.ElementFactory.make('appsink', None)
# Setting properties of appsink
ts0caps = vid0caps # use same caps as for camera
self.ts0sink.set_property('caps', ts0caps)
self.ts0sink.set_property("max-buffers", 20) # Limit memory usage
# Tell sink to emit signals
self.ts0sink.set_property('emit-signals', True)
self.ts0sink.set_property('sync', False) # No sync
# Connect appsink to my function (writing timestamps)
self.ts0sink.connect('new-sample', on_new_sample)
self.queue0 = Gst.ElementFactory.make('queue', None)
self.queue1 = Gst.ElementFactory.make('queue', None)
self.disp_queue = Gst.ElementFactory.make('queue', None)
self.file_queue = Gst.ElementFactory.make('queue', None)
self.ts_queue = Gst.ElementFactory.make('queue', None)
# Add elements to the pipeline
self.pipeline.add(self.v4l2src0)
self.pipeline.add(self.vid0filter)
self.pipeline.add(self.vid0parse)
self.pipeline.add(self.tee0)
self.pipeline.add(self.vid0decode)
self.pipeline.add(self.disp0scale)
self.pipeline.add(self.disp0sink)
self.pipeline.add(self.mux)
self.pipeline.add(self.file0sink)
self.pipeline.add(self.ts0sink)
self.pipeline.add(self.queue0)
self.pipeline.add(self.queue1)
self.pipeline.add(self.disp_queue)
self.pipeline.add(self.file_queue)
self.pipeline.add(self.ts_queue)
###############
# Link elements
###############
# video source
if not self.v4l2src0.link(self.vid0filter):
print('video source to video filter link failed')
if not self.vid0filter.link(self.vid0parse):
print('video filter to video parse link failed')
if not self.vid0parse.link(self.tee0):
print('video parse to tee link failed')
# tee
if not self.tee0.link(self.disp_queue):
print('tee to display queue link failed')
if not self.tee0.link(self.file_queue):
print('tee to file queue link failed')
if not self.tee0.link(self.ts_queue):
print('tee to ts queue link failed')
# video display sink
if not self.disp_queue.link(self.vid0decode):
print('dispaly queue to video decode link failed')
if not self.vid0decode.link(self.disp0scale):
print('decode to videoscale link failed')
if not self.disp0scale.link(self.queue0):
print('disp0scale to queue0 link failed')
if not self.queue0.link_filtered(self.disp0sink, disp0caps):
print('queue0 to display-sink link failed')
# file sink
if not self.file_queue.link(self.mux):
print('file queue to mux link failed')
if not self.mux.link(self.queue1):
print('mux to queue1 link failed')
if not self.queue1.link(self.file0sink):
print('queue1 to file-sink link failed')
# timestamp sink
if not self.ts_queue.link(self.ts0sink):
print('ts queue to ts-sink link failed')
def run(self):
self.offset_t = datetime.now().timestamp() - self.t_start
self.pipeline.set_state(Gst.State.PLAYING)
def quit(self):
self.pipeline.set_state(Gst.State.NULL)
self.ts_log.close()
def on_sync_message(self, bus, msg):
if msg.get_structure().get_name() == 'prepare-window-handle':
msg.src.set_property('force-aspect-ratio', True)
def on_error(self, bus, msg):
print('on_error():', msg.parse_error())
正如 mpr 所建议的，流式传输 h.264 时会出现非单调时间戳。它似乎也只发生在流的开头。流式传输原始视频时问题消失。
大的 300 毫秒，每 30 帧左右跳跃一次似乎是罗技 C920 相机特有的。当我使用板载笔记本电脑摄像头时，跳跃更小，为 130 毫秒，而且更少，每 85 帧左右。
我知道这有点晚了 kalleknast，你有没有设法得到时间戳？我将 do-timestamp=True 用于 rtspsrc，但没有用。
如果有人试图修复来自 RTSP 摄像机（30 FPS）的传入流的 PTS 时间戳，这里是允许您即时更改 PTS 的代码。我花了 2 天时间弄明白了，希望它能为其他人节省一些时间：
#!/usr/bin/python3
import sys
import logging
import numpy as np
import gi
gi.require_version("Gst", "1.0")
gi.require_version("GstApp", "1.0")
from gi.repository import Gst, GstApp, GObject, GLib
Gst.init(None)
last_pts = 0
class Main:
def __init__(self):
self.signed = None
self.depth = None
self.rate = None
self.channels = None
self.pipeline = Gst.parse_launch("rtspsrc latency=2000 location=rtspt://guest:gstreamer1@sala.ath.cx:5540/Streaming/Channels/101 ! rtph264depay ! h264parse ! avdec_h264 ! identity name=adjust ! queue2 ! watchdog timeout=10000 ! autovideosink")
src_element = self.pipeline.get_by_name('adjust')
#get the static source pad of the element
srcpad = src_element.get_static_pad('src')
#add the probe to the pad obtained in previous solution
probeID = srcpad.add_probe(Gst.PadProbeType.BUFFER, self.adjust_pts)
# The MainLoop
self.mainloop = GLib.MainLoop()
# And off we go!
self.pipeline.set_state(Gst.State.PLAYING)
self.mainloop.run()
def adjust_pts(self, pad, info):
global last_pts
# 33333333 nanoseconds is 0.03 seconds per frame (at 30 FPS)
new_pts = last_pts + 33333333
info.get_buffer().pts = new_pts
last_pts = new_pts
return Gst.PadProbeReturn.OK
def on_eos(self, bus, msg):
logging.debug('on_eos')
self.mainloop.quit()
def on_error(self, bus, msg):
error = msg.parse_error()
print('on_error:', error[1])
self.mainloop.quit()
class Error(Exception):
def __init__(self, message, detail=None):
global last_detail
self.message = message
if detail:
self.detail = detail
else:
self.detail = last_detail
logging.debug('audio: Error %s %s', self.message, self.detail)
def __str__(self):
return '%s - %s' % (self.message, self.detail)
Main()
我编写了一些代码来使用 GStreamer 1.0 (PyGST) 从网络摄像头捕获帧。知道捕获的确切时间对我来说很重要。为此，我设置了 v4l2src 属性 do-timestamp，并使用 appsink 将缓冲区 PTS 写入文本文件。
但是，时间戳并不是单调递增的。例如。第 16 帧的时间戳为 0.88199 秒，第 17 帧的时间戳为 0.77462 秒，即比前一帧早 0.10737 秒。 （我有一个显示问题的图表，但缺乏 post 它所必需的声誉。）
捕获的GstBuffer的PTS并不总是单调递增的，这对吗？如果这不是正常行为，有人知道我搞砸了什么吗？
我使用罗技 c920 网络摄像头。这些帧在相机上采用 h.264 编码。代码大致如下所示：
import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, Gtk
GObject.threads_init()
Gst.init(None)
class Webcam:
def __init__(self, video_dev='/dev/video0', fps=30):
ts_log_fname = 'webcam_timestamps.log'
vid_fname = 'webcam.mkv'
self.ts_log = open(ts_log_fname, 'w')
self.ts_log.write('video filename: %s\n '
'\nframe_number, cam_running_ts\n' % vid_fname)
self.n_frames = 0
# Create GStreamer pipline
self.pipeline = Gst.Pipeline()
# Create bus to get events from GStreamer pipeline
self.bus = self.pipeline.get_bus()
self.bus.add_signal_watch()
self.bus.connect('message::error', self.on_error)
self.bus.enable_sync_message_emission()
self.bus.connect('sync-message::element', self.on_sync_message)
###########################
# Callable function
###########################
def on_new_sample(appsink):
"""
Function called from the pipeline by appsink.
Writes the timestampes of frame capture to a log file.
"""
# Get the buffer
smp = appsink.emit('pull-sample')
buf = smp.get_buffer()
# Nanoseconds to seconds
timestamp = np.float64(1e-9) * buf.pts
self.n_frames += 1
self.ts_log.write('%d,%0.9f\n' % (self.n_frames, timestamp))
return False
###########################
# Create GStreamer elements
###########################
# Video source:
self.v4l2src0 = Gst.ElementFactory.make('v4l2src', None)
self.v4l2src0.set_property('device', video_dev)
self.v4l2src0.set_property('do-timestamp', 'true')
# Video source filters:
vid0caps = Gst.Caps.from_string('video/x-h264,width=%d,height=%d,'
'framerate=%d/1' % (1280, 720, fps))
self.vid0filter = Gst.ElementFactory.make('capsfilter', None)
self.vid0filter.set_property('caps', vid0caps)
# Parse video:
self.vid0parse = Gst.ElementFactory.make('h264parse', None)
# Split:
self.tee0 = Gst.ElementFactory.make('tee', None)
self.tee0.set_property('name', 't0')
####
# Display branch
####
# Decode
self.vid0decode = Gst.ElementFactory.make('avdec_h264', None)
# Scale to display size:
self.disp0scale = Gst.ElementFactory.make('videoscale', None)
# Display filter caps:
disp0caps = Gst.Caps.from_string('video/x-raw,width=%d,height=%d' %
(800, 600))
# Sinks:
self.disp0sink = Gst.ElementFactory.make('autovideosink', None)
self.disp0sink.set_property('filter-caps', disp0caps)
####
# File branch
####
self.mux = Gst.ElementFactory.make('matroskamux', None)
self.file0sink = Gst.ElementFactory.make('filesink', None)
self.file0sink.set_property('location', vid_fname)
self.file0sink.set_property('sync', False)
####
# Timestamp branch
####
# Create appsink
self.ts0sink = Gst.ElementFactory.make('appsink', None)
# Setting properties of appsink
ts0caps = vid0caps # use same caps as for camera
self.ts0sink.set_property('caps', ts0caps)
self.ts0sink.set_property("max-buffers", 20) # Limit memory usage
# Tell sink to emit signals
self.ts0sink.set_property('emit-signals', True)
self.ts0sink.set_property('sync', False) # No sync
# Connect appsink to my function (writing timestamps)
self.ts0sink.connect('new-sample', on_new_sample)
self.queue0 = Gst.ElementFactory.make('queue', None)
self.queue1 = Gst.ElementFactory.make('queue', None)
self.disp_queue = Gst.ElementFactory.make('queue', None)
self.file_queue = Gst.ElementFactory.make('queue', None)
self.ts_queue = Gst.ElementFactory.make('queue', None)
# Add elements to the pipeline
self.pipeline.add(self.v4l2src0)
self.pipeline.add(self.vid0filter)
self.pipeline.add(self.vid0parse)
self.pipeline.add(self.tee0)
self.pipeline.add(self.vid0decode)
self.pipeline.add(self.disp0scale)
self.pipeline.add(self.disp0sink)
self.pipeline.add(self.mux)
self.pipeline.add(self.file0sink)
self.pipeline.add(self.ts0sink)
self.pipeline.add(self.queue0)
self.pipeline.add(self.queue1)
self.pipeline.add(self.disp_queue)
self.pipeline.add(self.file_queue)
self.pipeline.add(self.ts_queue)
###############
# Link elements
###############
# video source
if not self.v4l2src0.link(self.vid0filter):
print('video source to video filter link failed')
if not self.vid0filter.link(self.vid0parse):
print('video filter to video parse link failed')
if not self.vid0parse.link(self.tee0):
print('video parse to tee link failed')
# tee
if not self.tee0.link(self.disp_queue):
print('tee to display queue link failed')
if not self.tee0.link(self.file_queue):
print('tee to file queue link failed')
if not self.tee0.link(self.ts_queue):
print('tee to ts queue link failed')
# video display sink
if not self.disp_queue.link(self.vid0decode):
print('dispaly queue to video decode link failed')
if not self.vid0decode.link(self.disp0scale):
print('decode to videoscale link failed')
if not self.disp0scale.link(self.queue0):
print('disp0scale to queue0 link failed')
if not self.queue0.link_filtered(self.disp0sink, disp0caps):
print('queue0 to display-sink link failed')
# file sink
if not self.file_queue.link(self.mux):
print('file queue to mux link failed')
if not self.mux.link(self.queue1):
print('mux to queue1 link failed')
if not self.queue1.link(self.file0sink):
print('queue1 to file-sink link failed')
# timestamp sink
if not self.ts_queue.link(self.ts0sink):
print('ts queue to ts-sink link failed')
def run(self):
self.offset_t = datetime.now().timestamp() - self.t_start
self.pipeline.set_state(Gst.State.PLAYING)
def quit(self):
self.pipeline.set_state(Gst.State.NULL)
self.ts_log.close()
def on_sync_message(self, bus, msg):
if msg.get_structure().get_name() == 'prepare-window-handle':
msg.src.set_property('force-aspect-ratio', True)
def on_error(self, bus, msg):
print('on_error():', msg.parse_error())
正如 mpr 所建议的，流式传输 h.264 时会出现非单调时间戳。它似乎也只发生在流的开头。流式传输原始视频时问题消失。
大的 300 毫秒，每 30 帧左右跳跃一次似乎是罗技 C920 相机特有的。当我使用板载笔记本电脑摄像头时，跳跃更小，为 130 毫秒，而且更少，每 85 帧左右。
我知道这有点晚了 kalleknast，你有没有设法得到时间戳？我将 do-timestamp=True 用于 rtspsrc，但没有用。
如果有人试图修复来自 RTSP 摄像机（30 FPS）的传入流的 PTS 时间戳，这里是允许您即时更改 PTS 的代码。我花了 2 天时间弄明白了，希望它能为其他人节省一些时间：
#!/usr/bin/python3
import sys
import logging
import numpy as np
import gi
gi.require_version("Gst", "1.0")
gi.require_version("GstApp", "1.0")
from gi.repository import Gst, GstApp, GObject, GLib
Gst.init(None)
last_pts = 0
class Main:
def __init__(self):
self.signed = None
self.depth = None
self.rate = None
self.channels = None
self.pipeline = Gst.parse_launch("rtspsrc latency=2000 location=rtspt://guest:gstreamer1@sala.ath.cx:5540/Streaming/Channels/101 ! rtph264depay ! h264parse ! avdec_h264 ! identity name=adjust ! queue2 ! watchdog timeout=10000 ! autovideosink")
src_element = self.pipeline.get_by_name('adjust')
#get the static source pad of the element
srcpad = src_element.get_static_pad('src')
#add the probe to the pad obtained in previous solution
probeID = srcpad.add_probe(Gst.PadProbeType.BUFFER, self.adjust_pts)
# The MainLoop
self.mainloop = GLib.MainLoop()
# And off we go!
self.pipeline.set_state(Gst.State.PLAYING)
self.mainloop.run()
def adjust_pts(self, pad, info):
global last_pts
# 33333333 nanoseconds is 0.03 seconds per frame (at 30 FPS)
new_pts = last_pts + 33333333
info.get_buffer().pts = new_pts
last_pts = new_pts
return Gst.PadProbeReturn.OK
def on_eos(self, bus, msg):
logging.debug('on_eos')
self.mainloop.quit()
def on_error(self, bus, msg):
error = msg.parse_error()
print('on_error:', error[1])
self.mainloop.quit()
class Error(Exception):
def __init__(self, message, detail=None):
global last_detail
self.message = message
if detail:
self.detail = detail
else:
self.detail = last_detail
logging.debug('audio: Error %s %s', self.message, self.detail)
def __str__(self):
return '%s - %s' % (self.message, self.detail)
Main()