使用 MediaTranscoder 对多个帧进行编码

Question

我正在尝试使用 Store App 中的 Media Foundation 将来自捕获设备的一系列视频帧从 IYUV 转换为 H264。 MediaTranscoder 仅适用于第一帧，然后关闭并关闭 MediaStreamSource。我怎样才能让它不断地要求新的帧进行编码？这是相关代码：

int InitEncoder(const VideoCodec* inst) {

    auto encoderProps = VideoEncodingProperties::CreateUncompressed(MediaEncodingSubtypes::Iyuv, inst->width, inst->height);
    streamDescriptor_ = ref new VideoStreamDescriptor(encoderProps);
    mediaStreamSource_ = ref new MediaStreamSource(streamDescriptor_);

    mediaStreamSource_->Starting += 
    ref new TypedEventHandler<MediaStreamSource ^, MediaStreamSourceStartingEventArgs ^>
    ([=](MediaStreamSource ^source, MediaStreamSourceStartingEventArgs ^ args)
    {
        args->Request->GetDeferral()->Complete();
    });

    mediaStreamSource_->SwitchStreamsRequested += 
    ref new TypedEventHandler<MediaStreamSource ^, MediaStreamSourceSwitchStreamsRequestedEventArgs ^>
    ([=](MediaStreamSource ^source,     MediaStreamSourceSwitchStreamsRequestedEventArgs ^args)
    {
        OutputDebugString(L"Media stream source switch stream requested.\n");
    });

    mediaStreamSource_->Closed += 
    ref new TypedEventHandler<MediaStreamSource ^, MediaStreamSourceClosedEventArgs ^>
    ([=](MediaStreamSource ^source, MediaStreamSourceClosedEventArgs ^args)
    {
        OutputDebugString(L"Media stream source closed event, reason: ");
    });

    mediaStreamSource_->SampleRequested +=
    ref new TypedEventHandler<MediaStreamSource ^, MediaStreamSourceSampleRequestedEventArgs ^>
    ([=](MediaStreamSource ^source, MediaStreamSourceSampleRequestedEventArgs ^args){
    if (framesToEncode_.size() > 0) {
        FeedOneSample(args->Request);        
    }
    else {
        sampleRequest_ = args->Request;
        args->Request->ReportSampleProgress(1);
        isSampleRequestPending_ = true;
    }
    });

    mediaTranscoder_ = ref new MediaTranscoder();
    mediaTranscoder_->VideoProcessingAlgorithm = MediaVideoProcessingAlgorithm::Default;
    mediaTranscoder_->HardwareAccelerationEnabled = true;    

    encodedStream_ = ref new InMemoryRandomAccessStream();

    h264EncodingProfile_ = MediaEncodingProfile::CreateMp4(VideoEncodingQuality::HD720p);
    h264EncodingProfile_->Audio = nullptr;
    h264EncodingProfile_->Container = nullptr;

    auto prepareTranscodeResultTask = mediaTranscoder_->PrepareMediaStreamSourceTranscodeAsync(
    mediaStreamSource_,
    encodedStream_,
    h264EncodingProfile_);

    auto prepareTranscodeResultContinuationTask = create_task(prepareTranscodeResultTask).then(
    [=](PrepareTranscodeResult ^prepTransResult) {
    prepareTranscodeResult_ = prepTransResult;
    if (!prepTransResult->CanTranscode) {
        TranscodeFailureReason failureReason = TranscodeFailureReason::None;
        failureReason = prepTransResult->FailureReason;
    }
    });
    prepareTranscodeResultContinuationTask.then([this]()
    {  
        if (prepareTranscodeResult_->CanTranscode) {
            inited_ = true;
        }
        return 0;
    });

    return 0;
}

int EncodeFrame(const VideoFrame& frame) {

    if (!inited_)
    {
        return -1;
    }

    framesToEncode_.push_back(newFrame);

    if (framesToEncode_.size() == 1 && !transcodeStarted_)
    {
        auto transcodeOp = prepareTranscodeResult_->TranscodeAsync();
        OnTranscodeAsync(transcodeOp);
        transcodeStarted_ = true;
    }
    if (isSampleRequestPending_ && framesToEncode_.size() > 0)
    {
        FeedOneSample(sampleRequest_);
        isSampleRequestPending_ = false;
    }

    return 0;
}

void OnTranscodeAsync(
IAsyncActionWithProgress<double> ^actionProgress) {
    auto transcodeContinuation = create_task(actionProgress);
    transcodeContinuation.then([=]() {
        OutputDebugString(L"Transcoding frame complete.\n");
        OutputDebugString(L"Size of encoded stream is: ");
        OutputDebugString(encodedStream_->Size.ToString()->Data());
        OutputDebugString(L"\n");
    });
}

void FeedOneSample(MediaStreamSourceSampleRequest ^sampleRequest)
{
    const VideoFrame* frame = framesToEncode_.front();
    framesToEncode_.pop_front();
    sampleRequest->Sample = TransformToMediaStreamSample(frame);
    sampleRequest->GetDeferral()->Complete();
}

MediaStreamSample^ TransformToMediaStreamSample(const VideoFrame* frame)
{
    unsigned int totalSize = frame->allocated_size(kYPlane) +
        frame->allocated_size(kUPlane) +
        frame->allocated_size(kVPlane);
    const uint8_t *bytes = frame->buffer(kYPlane);
    unsigned char *unChars = const_cast<unsigned char*>(bytes);
    auto finalArray = ref new Platform::Array<unsigned char>(unChars, totalSize);
    auto dataWriter = ref new DataWriter();
    dataWriter->WriteBytes(finalArray);
    auto timeSpan = TimeSpan();
    timeSpan.Duration = frame->timestamp();
    auto sample = MediaStreamSample::CreateFromBuffer(dataWriter->DetachBuffer(), timeSpan);

    return sample;
}

.h文件对应的部分是：

private:
    bool inited_;
    bool transcodeStarted_;
    bool isSampleRequestPending_;
    std::list<const I420VideoFrame*> framesToEncode_;
    EncodedImageCallback* encoded_complete_callback_;
    MediaStreamSource ^mediaStreamSource_;
    MediaTranscoder ^mediaTranscoder_;
    InMemoryRandomAccessStream ^encodedStream_;
    MediaEncodingProfile ^h264EncodingProfile_;
    PrepareTranscodeResult ^prepareTranscodeResult_;
    MediaStreamSourceSampleRequest ^sampleRequest_;
    VideoStreamDescriptor ^streamDescriptor_;

    void FeedOneSample(MediaStreamSourceSampleRequest ^sampleRequest);
    MediaStreamSample^ TransformToMediaStreamSample(const I420VideoFrame* frame);

如果有用，我可以提供使用 MFTrace 捕获的日志。

Answer 1

所以最后，在更了解 Microsoft Media Foundation 内部的人的帮助下，发现了错误。 GetDeferral() 应该在没有可用帧时调用，而不是行 args->Request->ReportSampleProgress(1); 此调用告诉转码器它必须等待一段时间才能获得新帧。此外，请求必须在调用 GetDefferal() 之后存储并稍后使用：

MediaStreamSourceSampleRequestDeferral ^sampleRequestDefferal_;

然后，当有可用帧时，将使用新帧设置 SampleRequest，并且必须调用 sampleRequestDefferal_->Complete()。

通过这些修改，转码器可以连续工作。

使用 MediaTranscoder 对多个帧进行编码

Encode multiple frames with MediaTranscoder

c++-cx

ms-media-foundation