从 RTSP MJPEG 流构建 JPEG 图像

Question

我正在尝试从 IP 摄像机的 RTSP MJPEG 流构建 JPEG 图像。我正在使用 CURL to get the RTSP data and then I am following the RTP Payload Format for JPEG-compressed video RFC

我已经成功地形成了其中没有量化表的 JPEG 图像。但是对于 128 类型的 Q，我的算法无法正确 assemble JPEG 图像。

我知道有像 ffmpeg 和 gstreamer 这样的库可以执行这种解码，但由于其他一些问题我不能使用它们。我也看过其他帖子，我认为我正在正确解码流。但是 JPEG 解压缩后图像看起来像这样。背景图片看似正确，但图片上散布着不正确的水平条纹。

这是我正在使用的代码片段：

    while(!firstPacketFound) {
        std::cout <<  "-----------RTP Packet: " << index++ << " starting at " << startOfRTPPacketIterator << "---------"<< endl;
        isFirstPacket = false; isLastPacket = false;
        // 0 - LibCurl header
        // 1 - channel identifier
        // 2, 3 - packet length
        uint16_T packet_length_msb = f[ startOfRTPPacketIterator+2];
        uint16_T packet_length_lsb = f[ startOfRTPPacketIterator+3];
        packet_length = packet_length_msb << 8 | packet_length_lsb;
        std::cout << "Packet Size: " << packet_length << endl;
        // 4 - RTP Header - version + pasdding - generally 80
        // 5 - Marker Bit + Payload  26 for JPEG
        uint8_T rtp1 = f[ startOfRTPPacketIterator+4];
        uint8_T payloadandmarkerbit = f[startOfRTPPacketIterator+5];
        uint8_T markerbit =(payloadandmarkerbit >> 7) ;
        uint8_T payload = payloadandmarkerbit & 0x7F;
        std::cout << "Marker byte : " << int(payloadandmarkerbit) << endl;
        if( payload != 26) {
            cout << "NOT JPEG packet" << endl;
        }
        // 6 & 7 - sequence 
        // 8, 9, 10,11 - timestamp
        //12, 13, 14, 15 - ssrc
        uint16_t sequenceNumber_m = f[ startOfRTPPacketIterator+6];
        uint16_t sequenceNumber_l = f[ startOfRTPPacketIterator+7];
        uint16_t sequenceNumber = (sequenceNumber_m << 8) | sequenceNumber_l;
        std::cout << "Sequence Number : " << sequenceNumber <<endl;
        
        uint32_t timestamp1 = f[ startOfRTPPacketIterator+8];
        uint32_t timestamp2 = f[ startOfRTPPacketIterator+9];
        uint32_t timestamp3 = f[ startOfRTPPacketIterator+10];
        uint32_t timestamp4 = f[ startOfRTPPacketIterator+11];
        uint32_t timestamp = (timestamp1 <<24) | (timestamp2 <<16) | (timestamp3<<8) | timestamp4;
         std::cout << "Timestamp : " << timestamp <<endl;
        
        //--------------------JPEG Header ----------------------/
        // 16 - type specific
        // 17, 18, 19, - framgemnt offset
        // 20 - type
        // 21 - Q
        // 22 - Width
        //23- Height
        uint32_t framgent_offset_1 = f[startOfRTPPacketIterator+17];
        uint32_t framgent_offset_2 = f[startOfRTPPacketIterator+18];
        uint32_t framgent_offset_3 = f[startOfRTPPacketIterator+19];
        uint32_t framgent_offset = ((framgent_offset_1 << 16) | (framgent_offset_2 << 8) | (framgent_offset_3));
        if (framgent_offset == 0)
        {
            isFirstPacket = true;
        }

        cout << "Fragment Offset: " << framgent_offset <<endl;
        uint8_t type_specific = f[ startOfRTPPacketIterator+16];
        cout << "type_specific: " << int(type_specific) <<endl;
        uint8_t type = f[startOfRTPPacketIterator+20];
        cout << "type : " << int(type) <<endl;
        uint8_t Q = f[ startOfRTPPacketIterator+21];
        cout << "Q: " << int(Q) <<endl;
        uint8_t width = f[ startOfRTPPacketIterator+22];
        cout << "width : " << int(width*8) <<endl;
        uint8_t height = f[ startOfRTPPacketIterator+23];
        cout << "height : " << int(height*8) <<endl;

        // Check for restart markers
        if ( type > 63) {
        // For 64-127, restart markers are present in the JPEG data
        //------------------Restart Marker Header------------------/
        // 24, 25 - restart interval
        // 26, 27 - F+L+ Restart Count
        uint16_T restart_interval_m = uint16_T(f[startOfRTPPacketIterator+24]);
        uint16_T restart_interval_l = uint16_T(f[startOfRTPPacketIterator+25]);
        restart_interval = (restart_interval_m <<8 ) | restart_interval_l;
        uint16_T restart_count_m = uint16_T(f[startOfRTPPacketIterator+26]);
        uint16_T restart_count_l = uint16_T(f[startOfRTPPacketIterator+27]);
        uint16_T restart_count = ((restart_count_m <<8 ) | restart_count_l) & 0x3F;
     
        
        uint8_T F = restart_count_m  >> 7;
        uint8_T L = (restart_count_m & 0x40) >> 6;
        cout << "restart_interval : " << restart_interval <<endl;
        cout << "restart_count : " << restart_count <<endl;
        cout << "F : " << int(F) <<endl;
        cout << "L : " << int(L) <<endl;
        }

        //Check for quantization table in the first packet of the frame
        if ( Q > 127) {
                        //--------------Quantization Header -----------------------/
        // 28 - MBZ
        // 29- Precision 
        // 30, 31 - length
        // ...Quantization data
        uint8_t MBZ = f[startOfRTPPacketIterator+28];
        uint8_t precision = f[startOfRTPPacketIterator+29];
        uint16_T qauntization_data_length = uint16_T(f[startOfRTPPacketIterator+30]) << 8 | uint16_T(f[startOfRTPPacketIterator+31])  ;

        cout << "MBZ : " << int(MBZ) << endl;
        cout << "precision : " << int(precision) << endl;
        cout << "qauntization_data_length : " << qauntization_data_length << endl;
        }

        int indexOfPayload;
        indexOfPayload = 28 ;
      
        
     
        
       if (isFirstPacket) {
           cout << "First packet found" << endl;
           memcpy(lqt,&f[startOfRTPPacketIterator+32],64);
           memcpy(cqt,&f[startOfRTPPacketIterator+32+65],64);
           //  MakeTables(Q,lqt,cqt);
           unsigned char * p = new unsigned char[3000];
           int sizeOfHeaders =  MakeHeaders(p,type,width, height, lqt, cqt, restart_interval);
           cout << "Size of headers:"<< sizeOfHeaders <<endl;
           jpegPayload.insert(jpegPayload.end(),p,p+sizeOfHeaders+1);
           delete p;
            indexOfPayload = 32 +128 ;
         
       }
         int laststartOfRTPPacketIterator = startOfRTPPacketIterator;
          // Look for start of frame marker
         std::vector<uint8_t>::iterator startOfRTPPacketIterator2;
       // startOfRTPPacketIterator2 = std::find(f.begin() + startOfRTPPacketIterator +packet_length ,f.end(),0x24);
         startOfRTPPacketIterator2 = std::search(f.begin() + startOfRTPPacketIterator +packet_length,f.end(),startOfStreamMarker.begin(),startOfStreamMarker.begin()+2);
         startOfRTPPacketIterator = (int)std::distance(f.begin(),startOfRTPPacketIterator2);
    
         cout << "Adding payload data" << endl;
         cout << "Starting at " << (laststartOfRTPPacketIterator+indexOfPayload)  << endl;
         cout << "Ending at " << startOfRTPPacketIterator  << endl;
         cout << "Size of vector before : " <<jpegPayload.size() <<endl;
        // jpegPayload.reserve(framgent_offset);
       jpegPayload.insert(jpegPayload.end(), &f[laststartOfRTPPacketIterator+indexOfPayload], &f[startOfRTPPacketIterator]);
        cout << "Size of vector after : " <<jpegPayload.size() <<endl;
       
        if (markerbit) {
            firstPacketFound = true;
            cout << "Last packet found" << endl;
            isLastPacket = true;
        
            FILE * image = fopen("image.jpeg","wb");
            fwrite(&jpegPayload[0],sizeof(char), jpegPayload.size(), image);
            fclose(image);            

        }

知道哪里出了问题吗？

我只从第一个 RTP 数据包复制量化表。这是我从 RFC 中解释的。对吗
我是应该使用 Fragment Offset 字段来构建我的图像，还是只保留附加负载数据直到我命中下一个数据包。
数据包超过Libcurl数据包的长度。这是预期的吗。

谢谢！

更新： 我还注意到可以使用 UDP 流的应用程序，而我将传输设置为 TCP - const char *transport = "RTP/AVP/TCP;unicast;interleaved=0-1";

这会导致任何差异吗？有的相机只支持UDP不支持TCP吗

Answer 1

我从 curl 获取的数据中有额外的 0x0d。所有额外的 0x0d 都在 0x0a 之前。我删除了 0x0a 之前的所有 0x0d，然后我的算法运行良好。我通过比较 curl 和 wire shark 给出的数据的输出来解决这个问题。这似乎是卷曲插入额外字符的错误。

从 RTSP MJPEG 流构建 JPEG 图像

Constructing a JPEG image from RTSP MJPEG stream

c

curl

mjpeg

rtsp

ip-camera