媒体编解码器解码器中的 BufferInfo.presentationTimeUs 时间戳乱序
Out of order BufferInfo.presentationTimeUs timestamps in mediacodec decoder
我变得不规律 newBufferInfo.presentationTimeUs 因为如果我放 Thread.sleep 来减慢播放速度，就会丢掉很多帧。
实际上，对于 Surface，帧时间戳会自动与系统时间戳同步而不休眠，但是它不适用于将输出提供给 OpenGLES。 https://developer.android.com/reference/android/media/MediaCodec#releaseOutputBuffer(int,%20long)
我以为 mExtractor.getSampleTime() 是问题所在，但即使删除它，问题仍然存在。
package com.example.app;
import android.graphics.PixelFormat;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;
import java.io.IOException;
import java.nio.ByteBuffer;
public final class MediaCodecDecoder {
private static final String VIDEO = "video/";
private static final String TAG = "MediaCodecDecoder";
private final Surface mSurface;
private final String mClipPath;
private SurfaceHolder mSurfaceHolder = null;
private MediaCodec mVideoDecoder;
private MediaExtractor mExtractor;
private Boolean mVideoDecoderRunning = false;
private Thread mVideoDecoderThread;
private int mDropCount;
private int mRenderCount;
private int mFramerate = 30;
public MediaCodecDecoder(SurfaceHolder surfaceHolder, Surface surface, String clipPath) {
this.mClipPath = clipPath;
this.mSurface = surface;
this.mSurfaceHolder = surfaceHolder;
}
public void start() {
mDropCount = 0;
mRenderCount = 0;
mExtractor = new MediaExtractor();
try {
mExtractor.setDataSource(mClipPath);
} catch (IOException e) {
e.printStackTrace();
}
for (int index = 0; index <= mExtractor.getTrackCount(); index++) {
MediaFormat format = mExtractor.getTrackFormat(index);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime != null && mime.startsWith(VIDEO)) {
mExtractor.selectTrack(index);
try {
mVideoDecoder = MediaCodec.createDecoderByType(mime);
} catch (IOException e) {
e.printStackTrace();
}
try {
Log.i(TAG, "format : " + format);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 65536);
mFramerate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
mVideoDecoder.configure(format, mSurface, null, 0 /* Decode */);
} catch (Exception e) {
e.printStackTrace();
}
break;
}
}
mVideoDecoder.start();
mVideoDecoderThread = new videoDecoderHandler(false);
mVideoDecoderThread.start();
}
public void stop() {
mVideoDecoderRunning = false;
try {
mVideoDecoderThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
// Clear the surface
if (mSurfaceHolder != null) {
mSurfaceHolder.setFormat(PixelFormat.TRANSPARENT);
mSurfaceHolder.setFormat(PixelFormat.OPAQUE);
}
Log.v("Extended Stats", "drop Count: " + mDropCount);
Log.v("Extended Stats", "rendered Count: " + mRenderCount);
Log.v("Extended Stats", "Total Frames Decoded: " + (mRenderCount + mDropCount));
}
class videoDecoderHandler extends Thread {
boolean endOfStream;
public videoDecoderHandler(boolean endOfStream) {
this.endOfStream = endOfStream;
}
//method where the thread execution will start
public void run() {
//logic to execute in a thread
mVideoDecoderRunning = true;
MediaCodec.BufferInfo newBufferInfo = new MediaCodec.BufferInfo();
ByteBuffer[] inputBuffers = mVideoDecoder.getInputBuffers();
ByteBuffer[] outputBuffers = mVideoDecoder.getOutputBuffers();
long startNs = System.nanoTime();
int generateIndex = 0;
while (mVideoDecoderRunning) {
int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[index];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (mExtractor.advance() && sampleSize > 0) {
Log.d(TAG, "index " + index + " mFramerate " + mFramerate + " mExtractor.getSampleTime() " +
mExtractor.getSampleTime() + " PresentationTime " +
(startNs / 1000 + computePresentationTime(generateIndex, mFramerate)));
// mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
computePresentationTime(generateIndex, 30), 0);
generateIndex++;
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mVideoDecoder.queueInputBuffer(
index,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM
);
}
}
int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000);
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = mVideoDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " +
mVideoDecoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.d(TAG, "INFO_TRY_AGAIN_LATER");
break;
default:
Log.d(TAG, "outIndex " + outIndex + " newBufferInfo.presentationTimeUs " +
newBufferInfo.presentationTimeUs);
boolean render = newBufferInfo.size != 0;
long currentNs = System.nanoTime();
if (!render) {
mVideoDecoder.releaseOutputBuffer(outIndex, false);
} else if (currentNs / 1000 - newBufferInfo.presentationTimeUs > 30000) {
mVideoDecoder.releaseOutputBuffer(outIndex, false);
mDropCount++; // drop if more than 30ms late
} else {
try {
if ((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000 > 0) {
Thread.sleep((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
mVideoDecoder.releaseOutputBuffer(outIndex, true);
mRenderCount++;
}
}
// All decoded frames have been rendered, we can stop playing now
if ((newBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
if (endOfStream) {
break;
}
}
mVideoDecoder.stop();
mVideoDecoder.release();
mExtractor.release();
}
}
/**
* Generates the presentation time for frame N, in microseconds.
*/
private static long computePresentationTime(int frameIndex, int FRAME_RATE) {
return (132 + Long.valueOf(frameIndex) * 1000000
/ Long.valueOf(FRAME_RATE));
}
}
这个class获取mp4视频clipPath，并有start和stop函数开始和停止解码mp4视频到传递的surface.
这是日志
04-21 20:49:35.875 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9666666 PresentationTime 362231984
04-21 20:49:35.879 3595 3971 D MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 361965317
04-21 20:49:35.884 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9800000 PresentationTime 362265317
04-21 20:49:35.888 3595 3971 D MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 362131984
04-21 20:49:35.894 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.962 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 9766666 PresentationTime 362298650
04-21 20:49:35.969 3595 3971 D MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 362098650
04-21 20:49:35.973 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9900000 PresentationTime 362331984
04-21 20:49:35.988 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.990 3595 3971 D MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 362165317
04-21 20:49:35.993 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.000 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9866666 PresentationTime 362365317
04-21 20:49:36.003 3595 3971 D MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 362065317
04-21 20:49:36.005 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9833333 PresentationTime 362398650
04-21 20:49:36.007 3595 3971 D MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 362265317
04-21 20:49:36.013 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.096 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 10033333 PresentationTime 362431984
04-21 20:49:36.102 3595 3971 D MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 362231984
04-21 20:49:36.108 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9966666 PresentationTime 362465317
04-21 20:49:36.111 3595 3971 D MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 362198650
04-21 20:49:40.614 3595 3595 V Extended Stats: drop Count: 232
04-21 20:49:40.614 3595 3595 V Extended Stats: rendered Count: 192
04-21 20:49:40.615 3595 3595 V Extended Stats: Total Frames Decoded: 424
这是带有
的日志
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
// mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
// computePresentationTime(generateIndex, 30), 0);
2021-11-09 15:36:33.898 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 64798066 PresentationTime 907904455
2021-11-09 15:36:33.899 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 907735489
2021-11-09 15:36:33.910 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 64898166 PresentationTime 907937788
2021-11-09 15:36:33.912 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 907902323
2021-11-09 15:36:33.969 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 64864800 PresentationTime 907971122
2021-11-09 15:36:33.970 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 907802223
2021-11-09 15:36:33.973 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 64964900 PresentationTime 908004455
2021-11-09 15:36:33.975 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 907935690
2021-11-09 15:36:33.999 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 64931533 PresentationTime 908037788
2021-11-09 15:36:34.002 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 11 newBufferInfo.presentationTimeUs 907868956
2021-11-09 15:36:34.004 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 5 mFramerate 30 mExtractor.getSampleTime() 65031633 PresentationTime 908071122
2021-11-09 15:36:34.005 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 908002423
2021-11-09 15:36:34.065 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 6 mFramerate 30 mExtractor.getSampleTime() 64998266 PresentationTime 908104455
2021-11-09 15:36:34.068 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 908069156
2021-11-09 15:36:34.131 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 7 mFramerate 30 mExtractor.getSampleTime() 65098366 PresentationTime 908137788
2021-11-09 15:36:34.132 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 907969056
2021-11-09 15:36:34.133 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 8 mFramerate 30 mExtractor.getSampleTime() 65065000 PresentationTime 908171122
2021-11-09 15:36:34.134 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 908135890
2021-11-09 15:36:34.209 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 9 mFramerate 30 mExtractor.getSampleTime() 65165099 PresentationTime 908204455
2021-11-09 15:36:34.211 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 908035790
2021-11-09 15:36:34.213 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 10 mFramerate 30 mExtractor.getSampleTime() 65131733 PresentationTime 908237788
2021-11-09 15:36:34.215 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 908202623
2021-11-09 15:36:34.270 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 11 mFramerate 30 mExtractor.getSampleTime() 65231833 PresentationTime 908271122
2021-11-09 15:36:34.273 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 16 newBufferInfo.presentationTimeUs 908102523
2021-11-09 15:36:34.275 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 65198466 PresentationTime 908304455
2021-11-09 15:36:34.277 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 908269356
2021-11-09 15:36:34.331 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 65265199 PresentationTime 908337788
2021-11-09 15:36:34.333 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 908169256
2021-11-09 15:36:34.334 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 65331933 PresentationTime 908371122
2021-11-09 15:36:34.335 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 908336089
2021-11-09 15:36:34.398 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 65298566 PresentationTime 908404455
2021-11-09 15:36:34.401 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 908235990
2021-11-09 15:36:34.402 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 65398666 PresentationTime 908437788
2021-11-09 15:36:34.403 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 908402823
2021-11-09 15:36:34.544 6051-6098/org.codeaurora.qmedia2 D/SurfaceUtils: disconnecting from surface 0x733202f010, reason disconnectFromSurface
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: drop Count: 909
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: rendered Count: 1044
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: Total Frames Decoded: 1953
这是带有
的日志
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, mExtractor.getSampleTime(), 0);
2021-02-23 02:36:10.112 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 15515500
2021-02-23 02:36:10.119 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 15415400
2021-02-23 02:36:10.178 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 15582233
2021-02-23 02:36:10.183 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 15482133
2021-02-23 02:36:10.245 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 15648966
2021-02-23 02:36:10.252 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 15548866
2021-02-23 02:36:10.278 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 15682333
2021-02-23 02:36:10.283 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 15615600
2021-02-23 02:36:10.345 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 1 newBufferInfo.presentationTimeUs 15749066
2021-02-23 02:36:10.412 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 15815800
2021-02-23 02:36:10.430 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 15715700
2021-02-23 02:36:10.478 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 15882533
2021-02-23 02:36:10.482 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 15782433
2021-02-23 02:36:10.545 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 15949266
2021-02-23 02:36:10.550 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 15849166
2021-02-23 02:36:10.612 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 16015999
默认情况下 int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000); 应该按照我在文档中检查过的显示顺序（即时间戳顺序）输出帧，但时间戳是乱序的。
出于某种原因，即使时间戳乱序，视频播放看起来也很流畅，我感到非常困惑。
我注意到您的代码存在一些问题。
首先，您不应该自己计算呈现时间，因为如果视频有 B 帧，则帧的呈现时间可能并不总是按递增顺序排列。导致帧时间戳似乎乱了。
https://ottverse.com/i-p-b-frames-idr-keyframes-differences-usecases/
其次，您不应根据将帧传递给解码器的时间来丢弃帧。解码器有时需要在输出新帧之前传递多个帧，因此解码器解码特定帧可能需要一些时间。
相反，您应该根据渲染的第一帧来计算它。
// when passing frame to decoder
sampleTime = mExtractor.getSampleTime()
if (firstFrame) {
firstSampleTime = sampleTime
}
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, sampleTime,0);
// when receiving the frame from decoder
currrent = System.nanoTime()/ 1000
if (firstFrame) {
firstFrameReceived = current
}
receivedDelta = current - firstFrame
presentationTimeDelta = mBufferInfo.presentationTimeUs - firstSampleTime
if (receivedDelta - presentatinTimeDelta > 30ms) {
// Drop frame
}
我想通了这个问题。它在 dequeueInputBuffer 部分。
我在 if 语句中预先做了 mExtractor.advance()。
正确的实现方式如下
int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[index];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (sampleSize >= 0) {
mVideoDecoder.queueInputBuffer(index, 0, sampleSize,
mExtractor.getSampleTime(), 0);
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mVideoDecoder.queueInputBuffer(index, 0, 0,
0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
mExtractor.advance();
}
我变得不规律 newBufferInfo.presentationTimeUs 因为如果我放 Thread.sleep 来减慢播放速度，就会丢掉很多帧。
实际上，对于 Surface，帧时间戳会自动与系统时间戳同步而不休眠，但是它不适用于将输出提供给 OpenGLES。 https://developer.android.com/reference/android/media/MediaCodec#releaseOutputBuffer(int,%20long)
我以为 mExtractor.getSampleTime() 是问题所在，但即使删除它，问题仍然存在。
package com.example.app;
import android.graphics.PixelFormat;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;
import java.io.IOException;
import java.nio.ByteBuffer;
public final class MediaCodecDecoder {
private static final String VIDEO = "video/";
private static final String TAG = "MediaCodecDecoder";
private final Surface mSurface;
private final String mClipPath;
private SurfaceHolder mSurfaceHolder = null;
private MediaCodec mVideoDecoder;
private MediaExtractor mExtractor;
private Boolean mVideoDecoderRunning = false;
private Thread mVideoDecoderThread;
private int mDropCount;
private int mRenderCount;
private int mFramerate = 30;
public MediaCodecDecoder(SurfaceHolder surfaceHolder, Surface surface, String clipPath) {
this.mClipPath = clipPath;
this.mSurface = surface;
this.mSurfaceHolder = surfaceHolder;
}
public void start() {
mDropCount = 0;
mRenderCount = 0;
mExtractor = new MediaExtractor();
try {
mExtractor.setDataSource(mClipPath);
} catch (IOException e) {
e.printStackTrace();
}
for (int index = 0; index <= mExtractor.getTrackCount(); index++) {
MediaFormat format = mExtractor.getTrackFormat(index);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime != null && mime.startsWith(VIDEO)) {
mExtractor.selectTrack(index);
try {
mVideoDecoder = MediaCodec.createDecoderByType(mime);
} catch (IOException e) {
e.printStackTrace();
}
try {
Log.i(TAG, "format : " + format);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 65536);
mFramerate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
mVideoDecoder.configure(format, mSurface, null, 0 /* Decode */);
} catch (Exception e) {
e.printStackTrace();
}
break;
}
}
mVideoDecoder.start();
mVideoDecoderThread = new videoDecoderHandler(false);
mVideoDecoderThread.start();
}
public void stop() {
mVideoDecoderRunning = false;
try {
mVideoDecoderThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
// Clear the surface
if (mSurfaceHolder != null) {
mSurfaceHolder.setFormat(PixelFormat.TRANSPARENT);
mSurfaceHolder.setFormat(PixelFormat.OPAQUE);
}
Log.v("Extended Stats", "drop Count: " + mDropCount);
Log.v("Extended Stats", "rendered Count: " + mRenderCount);
Log.v("Extended Stats", "Total Frames Decoded: " + (mRenderCount + mDropCount));
}
class videoDecoderHandler extends Thread {
boolean endOfStream;
public videoDecoderHandler(boolean endOfStream) {
this.endOfStream = endOfStream;
}
//method where the thread execution will start
public void run() {
//logic to execute in a thread
mVideoDecoderRunning = true;
MediaCodec.BufferInfo newBufferInfo = new MediaCodec.BufferInfo();
ByteBuffer[] inputBuffers = mVideoDecoder.getInputBuffers();
ByteBuffer[] outputBuffers = mVideoDecoder.getOutputBuffers();
long startNs = System.nanoTime();
int generateIndex = 0;
while (mVideoDecoderRunning) {
int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[index];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (mExtractor.advance() && sampleSize > 0) {
Log.d(TAG, "index " + index + " mFramerate " + mFramerate + " mExtractor.getSampleTime() " +
mExtractor.getSampleTime() + " PresentationTime " +
(startNs / 1000 + computePresentationTime(generateIndex, mFramerate)));
// mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
computePresentationTime(generateIndex, 30), 0);
generateIndex++;
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mVideoDecoder.queueInputBuffer(
index,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM
);
}
}
int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000);
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = mVideoDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " +
mVideoDecoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.d(TAG, "INFO_TRY_AGAIN_LATER");
break;
default:
Log.d(TAG, "outIndex " + outIndex + " newBufferInfo.presentationTimeUs " +
newBufferInfo.presentationTimeUs);
boolean render = newBufferInfo.size != 0;
long currentNs = System.nanoTime();
if (!render) {
mVideoDecoder.releaseOutputBuffer(outIndex, false);
} else if (currentNs / 1000 - newBufferInfo.presentationTimeUs > 30000) {
mVideoDecoder.releaseOutputBuffer(outIndex, false);
mDropCount++; // drop if more than 30ms late
} else {
try {
if ((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000 > 0) {
Thread.sleep((newBufferInfo.presentationTimeUs - currentNs / 1000) / 1000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
mVideoDecoder.releaseOutputBuffer(outIndex, true);
mRenderCount++;
}
}
// All decoded frames have been rendered, we can stop playing now
if ((newBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
if (endOfStream) {
break;
}
}
mVideoDecoder.stop();
mVideoDecoder.release();
mExtractor.release();
}
}
/**
* Generates the presentation time for frame N, in microseconds.
*/
private static long computePresentationTime(int frameIndex, int FRAME_RATE) {
return (132 + Long.valueOf(frameIndex) * 1000000
/ Long.valueOf(FRAME_RATE));
}
}
这个class获取mp4视频clipPath，并有start和stop函数开始和停止解码mp4视频到传递的surface.
这是日志
04-21 20:49:35.875 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9666666 PresentationTime 362231984
04-21 20:49:35.879 3595 3971 D MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 361965317
04-21 20:49:35.884 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9800000 PresentationTime 362265317
04-21 20:49:35.888 3595 3971 D MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 362131984
04-21 20:49:35.894 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.962 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 9766666 PresentationTime 362298650
04-21 20:49:35.969 3595 3971 D MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 362098650
04-21 20:49:35.973 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9900000 PresentationTime 362331984
04-21 20:49:35.988 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:35.990 3595 3971 D MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 362165317
04-21 20:49:35.993 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.000 3595 3971 D MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 9866666 PresentationTime 362365317
04-21 20:49:36.003 3595 3971 D MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 362065317
04-21 20:49:36.005 3595 3971 D MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 9833333 PresentationTime 362398650
04-21 20:49:36.007 3595 3971 D MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 362265317
04-21 20:49:36.013 1124 3960 E OMX-VDEC-1080P: Unable to convey fps info to driver, performance might be affected
04-21 20:49:36.096 3595 3971 D MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 10033333 PresentationTime 362431984
04-21 20:49:36.102 3595 3971 D MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 362231984
04-21 20:49:36.108 3595 3971 D MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 9966666 PresentationTime 362465317
04-21 20:49:36.111 3595 3971 D MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 362198650
04-21 20:49:40.614 3595 3595 V Extended Stats: drop Count: 232
04-21 20:49:40.614 3595 3595 V Extended Stats: rendered Count: 192
04-21 20:49:40.615 3595 3595 V Extended Stats: Total Frames Decoded: 424
这是带有
的日志 mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 + mExtractor.getSampleTime(), 0);
// mVideoDecoder.queueInputBuffer(index, 0, sampleSize, startNs / 1000 +
// computePresentationTime(generateIndex, 30), 0);
2021-11-09 15:36:33.898 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 64798066 PresentationTime 907904455
2021-11-09 15:36:33.899 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 907735489
2021-11-09 15:36:33.910 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 64898166 PresentationTime 907937788
2021-11-09 15:36:33.912 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 907902323
2021-11-09 15:36:33.969 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 64864800 PresentationTime 907971122
2021-11-09 15:36:33.970 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 907802223
2021-11-09 15:36:33.973 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 64964900 PresentationTime 908004455
2021-11-09 15:36:33.975 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 907935690
2021-11-09 15:36:33.999 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 64931533 PresentationTime 908037788
2021-11-09 15:36:34.002 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 11 newBufferInfo.presentationTimeUs 907868956
2021-11-09 15:36:34.004 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 5 mFramerate 30 mExtractor.getSampleTime() 65031633 PresentationTime 908071122
2021-11-09 15:36:34.005 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 908002423
2021-11-09 15:36:34.065 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 6 mFramerate 30 mExtractor.getSampleTime() 64998266 PresentationTime 908104455
2021-11-09 15:36:34.068 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 908069156
2021-11-09 15:36:34.131 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 7 mFramerate 30 mExtractor.getSampleTime() 65098366 PresentationTime 908137788
2021-11-09 15:36:34.132 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 907969056
2021-11-09 15:36:34.133 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 8 mFramerate 30 mExtractor.getSampleTime() 65065000 PresentationTime 908171122
2021-11-09 15:36:34.134 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 908135890
2021-11-09 15:36:34.209 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 9 mFramerate 30 mExtractor.getSampleTime() 65165099 PresentationTime 908204455
2021-11-09 15:36:34.211 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 908035790
2021-11-09 15:36:34.213 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 10 mFramerate 30 mExtractor.getSampleTime() 65131733 PresentationTime 908237788
2021-11-09 15:36:34.215 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 908202623
2021-11-09 15:36:34.270 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 11 mFramerate 30 mExtractor.getSampleTime() 65231833 PresentationTime 908271122
2021-11-09 15:36:34.273 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 16 newBufferInfo.presentationTimeUs 908102523
2021-11-09 15:36:34.275 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 0 mFramerate 30 mExtractor.getSampleTime() 65198466 PresentationTime 908304455
2021-11-09 15:36:34.277 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 908269356
2021-11-09 15:36:34.331 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 1 mFramerate 30 mExtractor.getSampleTime() 65265199 PresentationTime 908337788
2021-11-09 15:36:34.333 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 908169256
2021-11-09 15:36:34.334 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 2 mFramerate 30 mExtractor.getSampleTime() 65331933 PresentationTime 908371122
2021-11-09 15:36:34.335 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 908336089
2021-11-09 15:36:34.398 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 3 mFramerate 30 mExtractor.getSampleTime() 65298566 PresentationTime 908404455
2021-11-09 15:36:34.401 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 908235990
2021-11-09 15:36:34.402 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: index 4 mFramerate 30 mExtractor.getSampleTime() 65398666 PresentationTime 908437788
2021-11-09 15:36:34.403 6051-6106/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 908402823
2021-11-09 15:36:34.544 6051-6098/org.codeaurora.qmedia2 D/SurfaceUtils: disconnecting from surface 0x733202f010, reason disconnectFromSurface
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: drop Count: 909
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: rendered Count: 1044
2021-11-09 15:36:34.581 6051-6051/org.codeaurora.qmedia2 V/Extended Stats: Total Frames Decoded: 1953
这是带有
的日志
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, mExtractor.getSampleTime(), 0);
2021-02-23 02:36:10.112 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 15515500
2021-02-23 02:36:10.119 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 10 newBufferInfo.presentationTimeUs 15415400
2021-02-23 02:36:10.178 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 4 newBufferInfo.presentationTimeUs 15582233
2021-02-23 02:36:10.183 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 8 newBufferInfo.presentationTimeUs 15482133
2021-02-23 02:36:10.245 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 14 newBufferInfo.presentationTimeUs 15648966
2021-02-23 02:36:10.252 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 6 newBufferInfo.presentationTimeUs 15548866
2021-02-23 02:36:10.278 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 12 newBufferInfo.presentationTimeUs 15682333
2021-02-23 02:36:10.283 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 3 newBufferInfo.presentationTimeUs 15615600
2021-02-23 02:36:10.345 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 1 newBufferInfo.presentationTimeUs 15749066
2021-02-23 02:36:10.412 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 9 newBufferInfo.presentationTimeUs 15815800
2021-02-23 02:36:10.430 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 2 newBufferInfo.presentationTimeUs 15715700
2021-02-23 02:36:10.478 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 7 newBufferInfo.presentationTimeUs 15882533
2021-02-23 02:36:10.482 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 15 newBufferInfo.presentationTimeUs 15782433
2021-02-23 02:36:10.545 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 5 newBufferInfo.presentationTimeUs 15949266
2021-02-23 02:36:10.550 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 13 newBufferInfo.presentationTimeUs 15849166
2021-02-23 02:36:10.612 5149-5331/org.codeaurora.qmedia2 D/MediaCodecDecoder: outIndex 0 newBufferInfo.presentationTimeUs 16015999
默认情况下 int outIndex = mVideoDecoder.dequeueOutputBuffer(newBufferInfo, 1000); 应该按照我在文档中检查过的显示顺序（即时间戳顺序）输出帧，但时间戳是乱序的。
出于某种原因，即使时间戳乱序，视频播放看起来也很流畅，我感到非常困惑。
我注意到您的代码存在一些问题。
首先，您不应该自己计算呈现时间，因为如果视频有 B 帧，则帧的呈现时间可能并不总是按递增顺序排列。导致帧时间戳似乎乱了。
https://ottverse.com/i-p-b-frames-idr-keyframes-differences-usecases/
其次，您不应根据将帧传递给解码器的时间来丢弃帧。解码器有时需要在输出新帧之前传递多个帧，因此解码器解码特定帧可能需要一些时间。 相反，您应该根据渲染的第一帧来计算它。
// when passing frame to decoder
sampleTime = mExtractor.getSampleTime()
if (firstFrame) {
firstSampleTime = sampleTime
}
mVideoDecoder.queueInputBuffer(index, 0, sampleSize, sampleTime,0);
// when receiving the frame from decoder
currrent = System.nanoTime()/ 1000
if (firstFrame) {
firstFrameReceived = current
}
receivedDelta = current - firstFrame
presentationTimeDelta = mBufferInfo.presentationTimeUs - firstSampleTime
if (receivedDelta - presentatinTimeDelta > 30ms) {
// Drop frame
}
我想通了这个问题。它在 dequeueInputBuffer 部分。
我在 if 语句中预先做了 mExtractor.advance()。
正确的实现方式如下
int index = mVideoDecoder.dequeueInputBuffer(1000);
if (index >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[index];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (sampleSize >= 0) {
mVideoDecoder.queueInputBuffer(index, 0, sampleSize,
mExtractor.getSampleTime(), 0);
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mVideoDecoder.queueInputBuffer(index, 0, 0,
0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
mExtractor.advance();
}