Twitch 的 RTMP 广播包体结构
RTMP Broadcast packet body structure for Twitch
我目前正在做一个类似于 OBS 的项目，我正在捕获屏幕数据，使用 x264 库对其进行编码，然后将其广播到 twitch 服务器。
目前，服务器正在接受数据，但没有播放视频 - 它会缓冲片刻，然后 returns 错误代码“2000：网络错误”
像OBS Classic一样，我将x264提供的每个NAL按其类型划分，然后对每个进行更改
int frame_size = x264_encoder_encode(encoder, &nals, &num_nals, &pic_in, &pic_out);
//sort the NAL's into their types and make necessary adjustments
int timeOffset = int(pic_out.i_pts - pic_out.i_dts);
timeOffset = htonl(timeOffset);//host to network translation, ensure the bytes are in the right format
BYTE *timeOffsetAddr = ((BYTE*)&timeOffset) + 1;
videoSection sect;
bool foundFrame = false;
uint8_t * spsPayload = NULL;
int spsSize = 0;
for (int i = 0; i<num_nals; i++) {
//std::cout << "VideoEncoder: EncodedImages Size: " << encodedImages->size() << std::endl;
x264_nal_t &nal = nals[i];
//std::cout << "NAL is:" << nal.i_type << std::endl;
//need to account for pps/sps, seems to always be the first frame sent
if (nal.i_type == NAL_SPS) {
spsSize = nal.i_payload;
spsPayload = (uint8_t*)malloc(spsSize);
memcpy(spsPayload, nal.p_payload, spsSize);
} else if (nal.i_type == NAL_PPS){
//pps always happens after sps
if (spsPayload == NULL) {
std::cout << "VideoEncoder: critical error, sps not set" << std::endl;
}
uint8_t * payload = (uint8_t*)malloc(nal.i_payload + spsSize);
memcpy(payload, spsPayload, spsSize);
memcpy(payload, nal.p_payload + spsSize, nal.i_payload);
sect = { nal.i_payload + spsSize, payload, nal.i_type };
encodedImages->push(sect);
} else if (nal.i_type == NAL_SEI || nal.i_type == NAL_FILLER) {
//these need some bytes at the start removed
BYTE *skip = nal.p_payload;
while (*(skip++) != 0x1);
int skipBytes = (int)(skip - nal.p_payload);
int newPayloadSize = (nal.i_payload - skipBytes);
uint8_t * payload = (uint8_t*)malloc(newPayloadSize);
memcpy(payload, nal.p_payload + skipBytes, newPayloadSize);
sect = { newPayloadSize, payload, nal.i_type };
encodedImages->push(sect);
} else if (nal.i_type == NAL_SLICE_IDR || nal.i_type == NAL_SLICE) {
//these packets need an additional section at the start
BYTE *skip = nal.p_payload;
while (*(skip++) != 0x1);
int skipBytes = (int)(skip - nal.p_payload);
std::vector<BYTE> bodyData;
if (!foundFrame) {
if (nal.i_type == NAL_SLICE_IDR) { bodyData.push_back(0x17); } else { bodyData.push_back(0x27); } //add a 17 or a 27 as appropriate
bodyData.push_back(1);
bodyData.push_back(*timeOffsetAddr);
foundFrame = true;
}
//put into the payload the bodyData followed by the nal payload
uint8_t * bodyDataPayload = (uint8_t*)malloc(bodyData.size());
memcpy(bodyDataPayload, bodyData.data(), bodyData.size() * sizeof(BYTE));
int newPayloadSize = (nal.i_payload - skipBytes);
uint8_t * payload = (uint8_t*)malloc(newPayloadSize + sizeof(bodyDataPayload));
memcpy(payload, bodyDataPayload, sizeof(bodyDataPayload));
memcpy(payload + sizeof(bodyDataPayload), nal.p_payload + skipBytes, newPayloadSize);
int totalSize = newPayloadSize + sizeof(bodyDataPayload);
sect = { totalSize, payload, nal.i_type };
encodedImages->push(sect);
} else {
std::cout << "VideoEncoder: Nal type did not match expected" << std::endl;
continue;
}
}
然后将 NAL 有效负载数据放入队列缓冲区中的结构 VideoSection
//used to transfer encoded data
struct videoSection {
int frameSize;
uint8_t* payload;
int type;
};
之后它被广播公司接收，又做了一些更改，然后我调用 rtmp_send()
videoSection sect = encodedImages->front();
encodedImages->pop();
//std::cout << "Broadcaster: Frame Size: " << sect.frameSize << std::endl;
//two methods of sending RTMP data, _sendpacket and _write. Using sendpacket for greater control
RTMPPacket * packet;
unsigned char* buf = (unsigned char*)sect.payload;
int type = buf[0]&0x1f; //I believe &0x1f sets a 32bit limit
int len = sect.frameSize;
long timeOffset = GetTickCount() - rtmp_start_time;
//assign space packet will need
packet = (RTMPPacket *)malloc(sizeof(RTMPPacket)+RTMP_MAX_HEADER_SIZE + len + 9);
memset(packet, 0, sizeof(RTMPPacket) + RTMP_MAX_HEADER_SIZE);
packet->m_body = (char *)packet + sizeof(RTMPPacket) + RTMP_MAX_HEADER_SIZE;
packet->m_nBodySize = len + 9;
//std::cout << "Broadcaster: Packet Size: " << sizeof(RTMPPacket) + RTMP_MAX_HEADER_SIZE + len + 9 << std::endl;
//std::cout << "Broadcaster: Packet Body Size: " << len + 9 << std::endl;
//set body to point to the packetbody
unsigned char *body = (unsigned char *)packet->m_body;
memset(body, 0, len + 9);
//NAL_SLICE_IDR represents keyframe
//first element determines packet type
body[0] = 0x27;//inter-frame h.264
if (sect.type == NAL_SLICE_IDR) {
body[0] = 0x17; //h.264 codec id
}
//-------------------------------------------------------------------------------
//this section taken from
//in an effort to understand packet format. it does not resolve my previous issues formatting the data for twitch to play it
//sets body to be NAL unit
body[1] = 0x01;
body[2] = 0x00;
body[3] = 0x00;
body[4] = 0x00;
//>> is a shift right
//shift len to the right, and AND it
/*body[5] = (len >> 24) & 0xff;
body[6] = (len >> 16) & 0xff;
body[7] = (len >> 8) & 0xff;
body[8] = (len) & 0xff;*/
//end code sourced from
//-------------------------------------------------------------------------------
//copy from buffer into rest of body
memcpy(&body[9], buf, len);
//DEBUG
//save individual packet body to a file with name rtmp[packetnum]
//determine why some packets do not have 0x27 or 0x17 at the start
//still happening, makes no sense given the above code
/*std::string fileLocation = "rtmp" + std::to_string(packCount++);
std::cout << fileLocation << std::endl;
const char * charConversion = fileLocation.c_str();
FILE* saveFile = NULL;
saveFile = fopen(charConversion, "w+b");//open as write and binary
if (!fwrite(body, len + 9, 1, saveFile)) {
std::cout << "VideoEncoder: Error while trying to write to file" << std::endl;
}
fclose(saveFile);*/
//END DEBUG
//other packet details
packet->m_hasAbsTimestamp = 0;
packet->m_packetType = RTMP_PACKET_TYPE_VIDEO;
if (rtmp != NULL) {
packet->m_nInfoField2 = rtmp->m_stream_id;
}
packet->m_nChannel = 0x04;
packet->m_headerType = RTMP_PACKET_SIZE_LARGE;
packet->m_nTimeStamp = timeOffset;
//send the packet
if (rtmp != NULL) {
RTMP_SendPacket(rtmp, packet, TRUE);
}
我可以看到 Twitch 正在以稳定的 3kbps 的速度在检查器中接收数据。所以我确定我在发送数据之前调整数据的方式有问题。谁能告诉我我在这里做错了什么？
问题甚至在您包含的代码之前就开始了。配置x264时一定要设置：
b_aud = 0;
b_repeat_headers = 0;
b_annexb = 0;
这会告诉x264生成rtmp需要的格式，然后你可以跳过所有的per-nal预处理。
对于 sps/pps 使用 x264_encoder_headers 在 x264_encoder_open 之后检索它们。将它们编码到 "extradata" 缓冲区中，如此处 Possible Locations for Sequence/Picture Parameter Set(s) for H.264 Stream 所述。在发送任何帧之前，此额外数据进入 rtmp "sequence header" 数据包。在您的情况下相应地设置 AVCPacketType 的帧 body[1]，0 用于序列头 1 用于其他所有内容，
body[0] = 0x27;
body[1] = 0;
body[2] = 0;
body[3] = 0;
body[4] = 0;
memcpy(&body[5], extradata, extradata_size);
如果你有 b 帧，body[2] 到 body[4] 必须设置为帧 cts (pts - dts)。如果要将其设置为零，请为基线配置文件配置 x264，但这会导致图像质量下降。使用x264_encoder_encode中的return代码作为帧大小，一口气写完整个帧。
int frame_size = x264_encoder_encode(encoder, &nals, &num_nals, &pic_in, &pic_out);
if(frame_size) {
int cts = pic_out->i_pts - pic_out->i_dts;
body[0] = pic_out->b_keyframe ? 0x27 : 0x17;
body[1] = 1;
body[2] = cts>>16;
body[3] = cts>>8;
body[4] = cts;
memcpy(&body[5], nals->p_payload, frame_size);
}
最后，Twitch 还要求您发送 AAC 音频流。并确保将关键帧间隔设置为 2 秒。
我目前正在做一个类似于 OBS 的项目，我正在捕获屏幕数据，使用 x264 库对其进行编码，然后将其广播到 twitch 服务器。
目前，服务器正在接受数据，但没有播放视频 - 它会缓冲片刻，然后 returns 错误代码“2000：网络错误”
像OBS Classic一样，我将x264提供的每个NAL按其类型划分，然后对每个进行更改
int frame_size = x264_encoder_encode(encoder, &nals, &num_nals, &pic_in, &pic_out);
//sort the NAL's into their types and make necessary adjustments
int timeOffset = int(pic_out.i_pts - pic_out.i_dts);
timeOffset = htonl(timeOffset);//host to network translation, ensure the bytes are in the right format
BYTE *timeOffsetAddr = ((BYTE*)&timeOffset) + 1;
videoSection sect;
bool foundFrame = false;
uint8_t * spsPayload = NULL;
int spsSize = 0;
for (int i = 0; i<num_nals; i++) {
//std::cout << "VideoEncoder: EncodedImages Size: " << encodedImages->size() << std::endl;
x264_nal_t &nal = nals[i];
//std::cout << "NAL is:" << nal.i_type << std::endl;
//need to account for pps/sps, seems to always be the first frame sent
if (nal.i_type == NAL_SPS) {
spsSize = nal.i_payload;
spsPayload = (uint8_t*)malloc(spsSize);
memcpy(spsPayload, nal.p_payload, spsSize);
} else if (nal.i_type == NAL_PPS){
//pps always happens after sps
if (spsPayload == NULL) {
std::cout << "VideoEncoder: critical error, sps not set" << std::endl;
}
uint8_t * payload = (uint8_t*)malloc(nal.i_payload + spsSize);
memcpy(payload, spsPayload, spsSize);
memcpy(payload, nal.p_payload + spsSize, nal.i_payload);
sect = { nal.i_payload + spsSize, payload, nal.i_type };
encodedImages->push(sect);
} else if (nal.i_type == NAL_SEI || nal.i_type == NAL_FILLER) {
//these need some bytes at the start removed
BYTE *skip = nal.p_payload;
while (*(skip++) != 0x1);
int skipBytes = (int)(skip - nal.p_payload);
int newPayloadSize = (nal.i_payload - skipBytes);
uint8_t * payload = (uint8_t*)malloc(newPayloadSize);
memcpy(payload, nal.p_payload + skipBytes, newPayloadSize);
sect = { newPayloadSize, payload, nal.i_type };
encodedImages->push(sect);
} else if (nal.i_type == NAL_SLICE_IDR || nal.i_type == NAL_SLICE) {
//these packets need an additional section at the start
BYTE *skip = nal.p_payload;
while (*(skip++) != 0x1);
int skipBytes = (int)(skip - nal.p_payload);
std::vector<BYTE> bodyData;
if (!foundFrame) {
if (nal.i_type == NAL_SLICE_IDR) { bodyData.push_back(0x17); } else { bodyData.push_back(0x27); } //add a 17 or a 27 as appropriate
bodyData.push_back(1);
bodyData.push_back(*timeOffsetAddr);
foundFrame = true;
}
//put into the payload the bodyData followed by the nal payload
uint8_t * bodyDataPayload = (uint8_t*)malloc(bodyData.size());
memcpy(bodyDataPayload, bodyData.data(), bodyData.size() * sizeof(BYTE));
int newPayloadSize = (nal.i_payload - skipBytes);
uint8_t * payload = (uint8_t*)malloc(newPayloadSize + sizeof(bodyDataPayload));
memcpy(payload, bodyDataPayload, sizeof(bodyDataPayload));
memcpy(payload + sizeof(bodyDataPayload), nal.p_payload + skipBytes, newPayloadSize);
int totalSize = newPayloadSize + sizeof(bodyDataPayload);
sect = { totalSize, payload, nal.i_type };
encodedImages->push(sect);
} else {
std::cout << "VideoEncoder: Nal type did not match expected" << std::endl;
continue;
}
}
然后将 NAL 有效负载数据放入队列缓冲区中的结构 VideoSection
//used to transfer encoded data
struct videoSection {
int frameSize;
uint8_t* payload;
int type;
};
之后它被广播公司接收，又做了一些更改，然后我调用 rtmp_send()
videoSection sect = encodedImages->front();
encodedImages->pop();
//std::cout << "Broadcaster: Frame Size: " << sect.frameSize << std::endl;
//two methods of sending RTMP data, _sendpacket and _write. Using sendpacket for greater control
RTMPPacket * packet;
unsigned char* buf = (unsigned char*)sect.payload;
int type = buf[0]&0x1f; //I believe &0x1f sets a 32bit limit
int len = sect.frameSize;
long timeOffset = GetTickCount() - rtmp_start_time;
//assign space packet will need
packet = (RTMPPacket *)malloc(sizeof(RTMPPacket)+RTMP_MAX_HEADER_SIZE + len + 9);
memset(packet, 0, sizeof(RTMPPacket) + RTMP_MAX_HEADER_SIZE);
packet->m_body = (char *)packet + sizeof(RTMPPacket) + RTMP_MAX_HEADER_SIZE;
packet->m_nBodySize = len + 9;
//std::cout << "Broadcaster: Packet Size: " << sizeof(RTMPPacket) + RTMP_MAX_HEADER_SIZE + len + 9 << std::endl;
//std::cout << "Broadcaster: Packet Body Size: " << len + 9 << std::endl;
//set body to point to the packetbody
unsigned char *body = (unsigned char *)packet->m_body;
memset(body, 0, len + 9);
//NAL_SLICE_IDR represents keyframe
//first element determines packet type
body[0] = 0x27;//inter-frame h.264
if (sect.type == NAL_SLICE_IDR) {
body[0] = 0x17; //h.264 codec id
}
//-------------------------------------------------------------------------------
//this section taken from
//in an effort to understand packet format. it does not resolve my previous issues formatting the data for twitch to play it
//sets body to be NAL unit
body[1] = 0x01;
body[2] = 0x00;
body[3] = 0x00;
body[4] = 0x00;
//>> is a shift right
//shift len to the right, and AND it
/*body[5] = (len >> 24) & 0xff;
body[6] = (len >> 16) & 0xff;
body[7] = (len >> 8) & 0xff;
body[8] = (len) & 0xff;*/
//end code sourced from
//-------------------------------------------------------------------------------
//copy from buffer into rest of body
memcpy(&body[9], buf, len);
//DEBUG
//save individual packet body to a file with name rtmp[packetnum]
//determine why some packets do not have 0x27 or 0x17 at the start
//still happening, makes no sense given the above code
/*std::string fileLocation = "rtmp" + std::to_string(packCount++);
std::cout << fileLocation << std::endl;
const char * charConversion = fileLocation.c_str();
FILE* saveFile = NULL;
saveFile = fopen(charConversion, "w+b");//open as write and binary
if (!fwrite(body, len + 9, 1, saveFile)) {
std::cout << "VideoEncoder: Error while trying to write to file" << std::endl;
}
fclose(saveFile);*/
//END DEBUG
//other packet details
packet->m_hasAbsTimestamp = 0;
packet->m_packetType = RTMP_PACKET_TYPE_VIDEO;
if (rtmp != NULL) {
packet->m_nInfoField2 = rtmp->m_stream_id;
}
packet->m_nChannel = 0x04;
packet->m_headerType = RTMP_PACKET_SIZE_LARGE;
packet->m_nTimeStamp = timeOffset;
//send the packet
if (rtmp != NULL) {
RTMP_SendPacket(rtmp, packet, TRUE);
}
我可以看到 Twitch 正在以稳定的 3kbps 的速度在检查器中接收数据。所以我确定我在发送数据之前调整数据的方式有问题。谁能告诉我我在这里做错了什么？
问题甚至在您包含的代码之前就开始了。配置x264时一定要设置：
b_aud = 0;
b_repeat_headers = 0;
b_annexb = 0;
这会告诉x264生成rtmp需要的格式，然后你可以跳过所有的per-nal预处理。
对于 sps/pps 使用 x264_encoder_headers 在 x264_encoder_open 之后检索它们。将它们编码到 "extradata" 缓冲区中，如此处 Possible Locations for Sequence/Picture Parameter Set(s) for H.264 Stream 所述。在发送任何帧之前，此额外数据进入 rtmp "sequence header" 数据包。在您的情况下相应地设置 AVCPacketType 的帧 body[1]，0 用于序列头 1 用于其他所有内容，
body[0] = 0x27;
body[1] = 0;
body[2] = 0;
body[3] = 0;
body[4] = 0;
memcpy(&body[5], extradata, extradata_size);
body[2] 到 body[4] 必须设置为帧 cts (pts - dts)。如果要将其设置为零，请为基线配置文件配置 x264，但这会导致图像质量下降。使用x264_encoder_encode中的return代码作为帧大小，一口气写完整个帧。
int frame_size = x264_encoder_encode(encoder, &nals, &num_nals, &pic_in, &pic_out);
if(frame_size) {
int cts = pic_out->i_pts - pic_out->i_dts;
body[0] = pic_out->b_keyframe ? 0x27 : 0x17;
body[1] = 1;
body[2] = cts>>16;
body[3] = cts>>8;
body[4] = cts;
memcpy(&body[5], nals->p_payload, frame_size);
}
最后，Twitch 还要求您发送 AAC 音频流。并确保将关键帧间隔设置为 2 秒。