网络摄像机捕获

IP camera capture

我正在尝试捕获直接连接到 nVidia Jetson TK1 中的迷你 PCIe 双千兆位扩展卡的两个 IP 摄像机的流。

我通过以下命令使用 gstreamer 捕获了两个摄像头的流:

gst-launch-0.10 rtspsrc location=rtsp://admin:123456@192.168.0.123:554/mpeg4cif latency=0 ! decodebin ! ffmpegcolorspace ! autovideosink rtspsrc location=rtsp://admin:123456@192.168.2.254:554/mpeg4cif latency=0 ! decodebin ! ffmpegcolorspace ! autovideosink

每个摄像头显示一个 window,但在捕获开始时给出此输出:

    WARNING: from element /GstPipeline:pipeline0/GstAutoVideoSink:autovideosink1/GstXvImageSink:autovideosink1-actual-sink-xvimage: A lot of buffers are being dropped.
Additional debug info:
gstbasesink.c(2875): gst_base_sink_is_too_late (): /GstPipeline:pipeline0/GstAutoVideoSink:autovideosink1/GstXvImageSink:autovideosink1-actual-sink-xvimage:
There may be a timestamping problem, or this computer is too slow.
---> TVMR: Video-conferencing detected !!!!!!!!!

流播放良好,"good" 摄像机之间也同步,但过了一会儿,突然其中一个摄像机停止,通常几秒钟后另一个也停止。使用像 Wireshark 这样的接口嗅探器,我可以检查 rtsp 数据包是否仍在从相机发送。

我的目的是使用这个相机将它们用作使用 openCV 的立体相机。我可以使用以下函数使用 OpenCV 捕获流:

camera[0].open("rtsp://admin:123456@192.168.2.254:554/mpeg4cif");//right
camera[1].open("rtsp://admin:123456@192.168.0.123:554/mpeg4cif");//left

它随机开始捕捉好坏,同步与否,延迟与否,但一段时间后无法使用捕获的图像,您可以在图像中观察到:

而运行openCV程序的输出通常是这样的:(我复制了最完整的一个)

[h264 @ 0x1b9580] slice type too large (2) at 0 23
[h264 @ 0x1b9580] decode_slice_header error

[h264 @ 0x1b1160] left block unavailable for requested intra mode at 0 6
[h264 @ 0x1b1160] error while decoding MB 0 6, bytestream (-1)

[h264 @ 0x1b1160] mmco: unref short failure

[h264 @ 0x1b9580] too many reference frames

[h264 @ 0x1b1160] pps_id (-1) out of range

使用的摄像头是两个SIP-1080J模块。

有谁知道如何使用 openCV 实现良好的捕获?首先摆脱那些 h264 消息并在程序执行时拥有稳定的图像。

如果没有,我如何使用 gstreamer 改进管道和缓冲区,以便在不突然停止流的情况下进行良好的捕获?。虽然我从来没有使用 gstreamer 通过 openCV 进行捕获,但也许有一天我会知道如何做并解决这个问题。

非常感谢。

经过几天的深入搜索和尝试,我直接打开使用gstreamer-0.10 API。首先,我通过 http://docs.gstreamer.com/pages/viewpage.action?pageId=327735

中的教程学习了如何使用它

对于大多数教程,您只需要安装 libgstreamer0.10-dev 和一些其他包。我全部安装了:

sudo apt-get install libgstreamer0*

然后将你想尝试的例子的代码复制到一个.c文件中,然后从终端在.c文件所在的文件夹中输入(在一些例子中你必须添加更多的库到pkg-config ):

gcc basic-tutorial-1.c $(pkg-config --cflags --libs gstreamer-0.10) -o basic-tutorial-1.c

在那之后我并没有感到迷茫,我开始尝试混合一些c 和c++ 代码。您可以使用适当的 g++ 命令或 CMakeLists.txt 或您想要的方式编译它...因为我正在使用 nVidia Jetson TK1 进行开发,所以我使用 Nsight Eclipse Edition 并且我需要配置项目属性正确使用 gstreamer-0.10 库和 openCV 库。

混合一些代码,最后我能够实时捕获我的两个 IP 摄像机的流,没有明显的延迟,没有任何帧中的错误解码,并且两个流同步。我还没有解决的唯一问题是获得彩色帧而不是灰度帧(我尝试使用其他 CV_ 值 "segmentation fault" 结果):

v = Mat(Size(640, 360),CV_8U, (char*)GST_BUFFER_DATA(gstImageBuffer));

接下来是完整代码,我使用 gstreamer 捕获,将捕获转换为 openCV Mat 对象,然后显示它。该代码仅用于捕获一台 IP 摄像机。您可以复制同时捕获多个相机的对象和方法。

#include <opencv2/core/core.hpp>
#include <opencv2/contrib/contrib.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/video/video.hpp>

#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include <gst/app/gstappbuffer.h>
#include <glib.h>

#define DEFAULT_LATENCY_MS  1

using namespace cv;

typedef struct _vc_cfg_data {
    char server_ip_addr[100];
} vc_cfg_data;

typedef struct _vc_gst_data {
    GMainLoop *loop;
    GMainContext *context;
    GstElement *pipeline;
    GstElement *rtspsrc,*depayloader, *decoder, *converter, *sink;
    GstPad *recv_rtp_src_pad;
} vc_gst_data;

typedef struct _vc_data {
    vc_gst_data gst_data;
    vc_cfg_data cfg;
} vc_data;

/* Global data */
vc_data app_data;

static void vc_pad_added_handler (GstElement *src, GstPad *new_pad, vc_data *data);


#define VC_CHECK_ELEMENT_ERROR(e, name) \
if (!e) { \
g_printerr ("Element %s could not be created. Exiting.\n", name); \
return -1; \
}

/*******************************************************************************
Gstreamer pipeline creation and init
*******************************************************************************/
int vc_gst_pipeline_init(vc_data *data)
{
    GstStateChangeReturn ret;

    // Template
    GstPadTemplate* rtspsrc_pad_template;

    // Create a new GMainLoop
    data->gst_data.loop = g_main_loop_new (NULL, FALSE);
    data->gst_data.context = g_main_loop_get_context(data->gst_data.loop);

    // Create gstreamer elements
    data->gst_data.pipeline = gst_pipeline_new ("videoclient");
    VC_CHECK_ELEMENT_ERROR(data->gst_data.pipeline, "pipeline");

    //RTP UDP Source - for received RTP messages
    data->gst_data.rtspsrc = gst_element_factory_make ("rtspsrc", "rtspsrc");
    VC_CHECK_ELEMENT_ERROR(data->gst_data.rtspsrc,"rtspsrc");

    printf("URL: %s\n",data->cfg.server_ip_addr);
    g_print ("Setting RTSP source properties: \n");
    g_object_set (G_OBJECT (data->gst_data.rtspsrc), "location", data->cfg.server_ip_addr, "latency", DEFAULT_LATENCY_MS, NULL);

    //RTP H.264 Depayloader
    data->gst_data.depayloader = gst_element_factory_make ("rtph264depay","depayloader");
    VC_CHECK_ELEMENT_ERROR(data->gst_data.depayloader,"rtph264depay");

    //ffmpeg decoder
    data->gst_data.decoder = gst_element_factory_make ("ffdec_h264", "decoder");
    VC_CHECK_ELEMENT_ERROR(data->gst_data.decoder,"ffdec_h264");

    data->gst_data.converter = gst_element_factory_make ("ffmpegcolorspace", "converter");
    VC_CHECK_ELEMENT_ERROR(data->gst_data.converter,"ffmpegcolorspace");

    // i.MX Video sink
    data->gst_data.sink = gst_element_factory_make ("appsink", "sink");
    VC_CHECK_ELEMENT_ERROR(data->gst_data.sink,"appsink");
    gst_app_sink_set_max_buffers((GstAppSink*)data->gst_data.sink, 1);
    gst_app_sink_set_drop ((GstAppSink*)data->gst_data.sink, TRUE);
    g_object_set (G_OBJECT (data->gst_data.sink),"sync", FALSE, NULL);

    //Request pads from rtpbin, starting with the RTP receive sink pad,
    //This pad receives RTP data from the network (rtp-udpsrc).
    rtspsrc_pad_template = gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (data->gst_data.rtspsrc),"recv_rtp_src_0");

    // Use the template to request the pad
    data->gst_data.recv_rtp_src_pad = gst_element_request_pad (data->gst_data.rtspsrc, rtspsrc_pad_template,
    "recv_rtp_src_0", NULL);

    // Print the name for confirmation
    g_print ("A new pad %s was created\n",
    gst_pad_get_name (data->gst_data.recv_rtp_src_pad));

    // Add elements into the pipeline
    g_print(" Adding elements to pipeline...\n");
    gst_bin_add_many (GST_BIN (data->gst_data.pipeline),
            data->gst_data.rtspsrc,
            data->gst_data.depayloader,
            data->gst_data.decoder,
            data->gst_data.converter,
            data->gst_data.sink,
        NULL);

    // Link some of the elements together
    g_print(" Linking some elements ...\n");
    if(!gst_element_link_many (data->gst_data.depayloader, data->gst_data.decoder, data->gst_data.converter, data->gst_data.sink, NULL))
        g_print("Error: could not link all elements\n");

    // Connect to the pad-added signal for the rtpbin. This allows us to link
    //the dynamic RTP source pad to the depayloader when it is created.
    if(!g_signal_connect (data->gst_data.rtspsrc, "pad-added",
    G_CALLBACK (vc_pad_added_handler), data))
        g_print("Error: could not add signal handler\n");

    // Set the pipeline to "playing" state
    g_print ("Now playing A\n");
    ret = gst_element_set_state (data->gst_data.pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) {
        g_printerr ("Unable to set the pipeline A to the playing state.\n");
        gst_object_unref (data->gst_data.pipeline);
        return -1;
    }

    return 0;
}

static void vc_pad_added_handler (GstElement *src, GstPad *new_pad, vc_data *data) {
    GstPad *sink_pad = gst_element_get_static_pad (data->gst_data.depayloader, "sink");
    GstPadLinkReturn ret;
    GstCaps *new_pad_caps = NULL;
    GstStructure *new_pad_struct = NULL;
    const gchar *new_pad_type = NULL;
    g_print ("Received new pad '%s' from '%s':\n", GST_PAD_NAME (new_pad), GST_ELEMENT_NAME (src));

    /* Check the new pad's name */
    if (!g_str_has_prefix (GST_PAD_NAME (new_pad), "recv_rtp_src_")) {
        g_print (" It is not the right pad. Need recv_rtp_src_. Ignoring.\n");
        goto exit;
    }

    /* If our converter is already linked, we have nothing to do here */
    if (gst_pad_is_linked (sink_pad)) {
        g_print (" Sink pad from %s already linked. Ignoring.\n", GST_ELEMENT_NAME (src));
        goto exit;
    }

    /* Check the new pad's type */
    new_pad_caps = gst_pad_get_caps (new_pad);
    new_pad_struct = gst_caps_get_structure (new_pad_caps, 0);
    new_pad_type = gst_structure_get_name (new_pad_struct);

    /* Attempt the link */
    ret = gst_pad_link (new_pad, sink_pad);
    if (GST_PAD_LINK_FAILED (ret)) {
        g_print (" Type is '%s' but link failed.\n", new_pad_type);
    } else {
        g_print (" Link succeeded (type '%s').\n", new_pad_type);
    }

    exit:
    /* Unreference the new pad's caps, if we got them */
    if (new_pad_caps != NULL)
        gst_caps_unref (new_pad_caps);
    /* Unreference the sink pad */
    gst_object_unref (sink_pad);
}



int vc_gst_pipeline_clean(vc_data *data) {
    GstStateChangeReturn ret;
    GstStateChangeReturn ret2;

    /* Cleanup Gstreamer */
    if(!data->gst_data.pipeline)
        return 0;

    /* Send the main loop a quit signal */
    g_main_loop_quit(data->gst_data.loop);
    g_main_loop_unref(data->gst_data.loop);
    ret = gst_element_set_state (data->gst_data.pipeline, GST_STATE_NULL);
    if (ret == GST_STATE_CHANGE_FAILURE) {
        g_printerr ("Unable to set the pipeline A to the NULL state.\n");
        gst_object_unref (data->gst_data.pipeline);
        return -1;
    }

    g_print ("Deleting pipeline\n");
    gst_object_unref (GST_OBJECT (data->gst_data.pipeline));
    /* Zero out the structure */
    memset(&data->gst_data, 0, sizeof(vc_gst_data));
    return 0;
}


void handleKey(char key)
{
    switch (key)
    {
    case 27:

        break;
    }
}


int vc_mainloop(vc_data* data)
{

    GstBuffer *gstImageBuffer;

    Mat v;

    namedWindow("view",WINDOW_NORMAL);

    while (1) {

        gstImageBuffer = gst_app_sink_pull_buffer((GstAppSink*)data->gst_data.sink);

        if (gstImageBuffer != NULL )
        {
                v = Mat(Size(640, 360),CV_8U, (char*)GST_BUFFER_DATA(gstImageBuffer));

                imshow("view", v);

                handleKey((char)waitKey(3));

                gst_buffer_unref(gstImageBuffer);
        }else{
            g_print("gsink buffer didn't return buffer.");
        }
    }
    return 0;
}


int main (int argc, char *argv[])
{
    setenv("DISPLAY", ":0", 0);

    strcpy(app_data.cfg.server_ip_addr, "rtsp://admin:123456@192.168.0.123:554/mpeg4cif");

    gst_init (&argc, &argv);

    if(vc_gst_pipeline_init(&app_data) == -1) {
        printf("Gstreamer pipeline creation and init failed\n");
        goto cleanup;
    }

    vc_mainloop(&app_data);

    printf ("Returned, stopping playback\n");
    cleanup:
    return vc_gst_pipeline_clean(&app_data);
    return  0;
}

希望对您有所帮助!! ;)

uri = 'rtsp://admin:123456@192.168.0.123:554/mpeg4cif'
gst_str = ("rtspsrc location={} latency={} ! rtph264depay ! h264parse ! omxh264dec ! nvvidconv ! video/x-raw, width=(int){}, height=(int){}, format=(string)BGRx ! videoconvert ! appsink sync=false").format(uri, 200, 3072, 2048) 


cap= cv2.VideoCapture(gst_str,cv2.CAP_GSTREAMER)
while(True):
    _,frame = cap.read()
    if frame is None:
         break
    cv2.imshow("",frame)
    cv2.waitKey(0)
cap.release()
cv2.destroyAllWindows()