使用 google/grafika 编码视频文件时延迟帧
Delay frame while encoding video file using google/grafika
我正在使用 google/grafika's 示例来解码、转换和编码回文件视频剪辑。转换是缩小和平移，它是通过存储在 Texture2dProgram 中的着色器完成的。我的主要 activity 是基于 CameraCaptureActivity。问题是我同时将两个视频并排放置在单个纹理上。我想将其中之一延迟给定数量的帧。另请注意，我在编码时不需要显示预览。
到目前为止，我最好的想法是在逐帧前进时更改时间戳。在 TextureMovieEncoder 中，我正在发送有关帧的信息，包括必须将它们放置在结果视频中的时间戳。它发生在 frameAvailiable()，我在这里一次发送有关两帧（左和右）的信息。这个想法是增加其中一个的时间戳。问题是结果视频是失真的，所以我不知道我的方法是否可行。 TextureMovieEncoder 发布在下方。
package com.android.grafika;
import android.graphics.SurfaceTexture;
import android.opengl.EGLContext;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.util.Log;
import com.android.grafika.gles.EglCore;
import com.android.grafika.gles.FullFrameRect;
import com.android.grafika.gles.Texture2dProgram;
import com.android.grafika.gles.WindowSurface;
import java.io.File;
import java.io.IOException;
import java.lang.ref.WeakReference;
/**
* Encode a movie from frames rendered from an external texture image.
* <p>
* The object wraps an encoder running on a dedicated thread. The various control messages
* may be sent from arbitrary threads (typically the app UI thread). The encoder thread
* manages both sides of the encoder (feeding and draining); the only external input is
* the GL texture.
* <p>
* The design is complicated slightly by the need to create an EGL context that shares state
* with a view that gets restarted if (say) the device orientation changes. When the view
* in question is a GLSurfaceView, we don't have full control over the EGL context creation
* on that side, so we have to bend a bit backwards here.
* <p>
* To use:
* <ul>
* <li>create TextureMovieEncoder object
* <li>create an EncoderConfig
* <li>call TextureMovieEncoder#startRecording() with the config
* <li>call TextureMovieEncoder#setTextureId() with the texture object that receives frames
* <li>for each frame, after latching it with SurfaceTexture#updateTexImage(),
* call TextureMovieEncoder#frameAvailable().
* </ul>
*
* TODOO: tweak the API (esp. textureId) so it's less awkward for simple use cases.
*/
public class TextureMovieEncoder implements Runnable {
private static final String TAG = MainActivity.TAG;
private static final boolean VERBOSE = false;
private static final long timestampCorrection = 1000000000;
private long timestampCorected;
private static final int MSG_START_RECORDING = 0;
private static final int MSG_STOP_RECORDING = 1;
private static final int MSG_FRAME_AVAILABLE = 2;
private static final int MSG_SET_TEXTURE_ID = 3;
private static final int MSG_UPDATE_SHARED_CONTEXT = 4;
private static final int MSG_QUIT = 5;
private boolean measure_started = false;
private long startTime = -1;
private int cycle = 0;
private long handleFrameTime = 0;
private long last_timestamp = -1;
private float [] transform;
private long last_orig_timestamp = -1;
public long getFrame() {
return frame;
}
private long frame = 0;
private long average_diff = 0;
private long step = 40000000;
private long actTimestamp = 0;
private boolean shouldStop = false;
public void setmSpeedCallback(SpeedControlCallback mSpeedCallback) {
this.mSpeedCallback = mSpeedCallback;
}
private SpeedControlCallback mSpeedCallback;
// ----- accessed exclusively by encoder thread -----
private WindowSurface mInputWindowSurface;
private EglCore mEglCore;
private FullFrameRect mFullScreen;
private int mTextureId;
private VideoEncoderCore mVideoEncoder;
// ----- accessed by multiple threads -----
private volatile EncoderHandler mHandler;
private Object mReadyFence = new Object(); // guards ready/running
private boolean mReady;
private boolean mRunning;
/**
* Encoder configuration.
* <p>
* Object is immutable, which means we can safely pass it between threads without
* explicit synchronization (and don't need to worry about it getting tweaked out from
* under us).
* <p>
* TODO: make frame rate and iframe interval configurable? Maybe use builder pattern
* with reasonable defaults for those and bit rate.
*/
public static class EncoderConfig {
final File mOutputFile;
final int mWidth;
final int mHeight;
final int mBitRate;
final EGLContext mEglContext;
public EncoderConfig(File outputFile, int width, int height, int bitRate,
EGLContext sharedEglContext) {
mOutputFile = outputFile;
mWidth = width;
mHeight = height;
mBitRate = bitRate;
mEglContext = sharedEglContext;
}
@Override
public String toString() {
return "EncoderConfig: " + mWidth + "x" + mHeight + " @" + mBitRate +
" to '" + mOutputFile.toString() + "' ctxt=" + mEglContext;
}
}
/**
* Tells the video recorder to start recording. (Call from non-encoder thread.)
* <p>
* Creates a new thread, which will create an encoder using the provided configuration.
* <p>
* Returns after the recorder thread has started and is ready to accept Messages. The
* encoder may not yet be fully configured.
*/
public void startRecording(EncoderConfig config) {
Log.d(TAG, "Encoder: startRecording()");
synchronized (mReadyFence) {
if (mRunning) {
Log.w(TAG, "Encoder thread already running");
return;
}
mRunning = true;
new Thread(this, "TextureMovieEncoder").start();
while (!mReady) {
try {
mReadyFence.wait();
} catch (InterruptedException ie) {
// ignore
}
}
}
mHandler.sendMessage(mHandler.obtainMessage(MSG_START_RECORDING, config));
}
/**
* Tells the video recorder to stop recording. (Call from non-encoder thread.)
* <p>
* Returns immediately; the encoder/muxer may not yet be finished creating the movie.
* <p>
* TODO: have the encoder thread invoke a callback on the UI thread just before it shuts down
* so we can provide reasonable status UI (and let the caller know that movie encoding
* has completed).
*/
public void stopRecording() {
//mHandler.sendMessage(mHandler.obtainMessage(MSG_STOP_RECORDING));
//mHandler.sendMessage(mHandler.obtainMessage(MSG_QUIT));
// We don't know when these will actually finish (or even start). We don't want to
// delay the UI thread though, so we return immediately.
shouldStop = true;
Log.d(TAG, "Shout down flag set up.");
}
/**
* Returns true if recording has been started.
*/
public boolean isRecording() {
synchronized (mReadyFence) {
return mRunning;
}
}
/**
* Tells the video recorder to refresh its EGL surface. (Call from non-encoder thread.)
*/
public void updateSharedContext(EGLContext sharedContext) {
mHandler.sendMessage(mHandler.obtainMessage(MSG_UPDATE_SHARED_CONTEXT, sharedContext));
}
/**
* Tells the video recorder that a new frame is available. (Call from non-encoder thread.)
* <p>
* This function sends a message and returns immediately. This isn't sufficient -- we
* don't want the caller to latch a new frame until we're done with this one -- but we
* can get away with it so long as the input frame rate is reasonable and the encoder
* thread doesn't stall.
* <p>
* TODO: either block here until the texture has been rendered onto the encoder surface,
* or have a separate "block if still busy" method that the caller can execute immediately
* before it calls updateTexImage(). The latter is preferred because we don't want to
* stall the caller while this thread does work.
*/
public void frameAvailable(SurfaceTexture st) {
synchronized (mReadyFence) {
if (!mReady) {
return;
}
}
transform = new float[16]; // TODOO - avoid alloc every frame
st.getTransformMatrix(transform);
long timestamp = st.getTimestamp();
// if first frame
if (last_timestamp < 0) {
if (!measure_started) {
startTime = System.currentTimeMillis();
measure_started = true;
}
last_timestamp = timestamp;
last_orig_timestamp = timestamp;
}
else {
// HARDCODED FRAME NUMBER :(
// if playback finished or frame number reached
if ((frame == 200) || shouldStop) {
if (measure_started) {
long stopTime = System.currentTimeMillis();
long elapsedTime = stopTime - startTime;
Log.d(TAG, "Rendering time: " + (double)elapsedTime * 0.001 + "[s]");
Log.d(TAG, "HandlingFrame time: " + (double)(stopTime - handleFrameTime) * 0.001 + "[s]");
measure_started = false;
}
mHandler.sendMessage(mHandler.obtainMessage(MSG_STOP_RECORDING));
mHandler.sendMessage(mHandler.obtainMessage(MSG_QUIT));
return;
}
else if (timestamp == 0) {
// Seeing this after device is toggled off/on with power button. The
// first frame back has a zero timestamp.
//
// MPEG4Writer thinks this is cause to abort() in native code, so it's very
// important that we just ignore the frame.
Log.w(TAG, "HEY: got SurfaceTexture with timestamp of zero");
return;
}
// this is workaround for duplicated timestamp
// might cause troubles with some videos
else if ((timestamp == last_orig_timestamp)) {
return;
}
else {
frame++;
mHandler.sendMessage(mHandler.obtainMessage(MSG_FRAME_AVAILABLE,
(int) (actTimestamp >> 32), (int) actTimestamp, transform));
timestampCorected = actTimestamp + timestampCorrection;
mHandler.sendMessage(mHandler.obtainMessage(MSG_FRAME_AVAILABLE,
(int) (timestampCorected >> 32), (int) timestampCorected, transform));
actTimestamp += step;
}
last_orig_timestamp = timestamp;
}
}
/**
* Calculates 'average' diffrence between frames.
* Result is based on first 50 frames.
* Shuld be called in frameAvailiable.
*
* @param timestamp actual frame timestamp
*/
private void calcAndShowAverageDiff(long timestamp) {
if ((frame < 50) && (frame > 0)) {
average_diff += timestamp - last_timestamp;
last_timestamp = timestamp;
}
if (frame == 50) {
average_diff /= frame;
Log.d(TAG, "Average timestamp difference: " + Long.toString(average_diff));
}
}
/**
* Tells the video recorder what texture name to use. This is the external texture that
* we're receiving camera previews in. (Call from non-encoder thread.)
* <p>
* TODOO: do something less clumsy
*/
public void setTextureId(int id) {
synchronized (mReadyFence) {
if (!mReady) {
return;
}
}
mHandler.sendMessage(mHandler.obtainMessage(MSG_SET_TEXTURE_ID, id, 0, null));
}
/**
* Encoder thread entry point. Establishes Looper/Handler and waits for messages.
* <p>
* @see java.lang.Thread#run()
*/
@Override
public void run() {
// Establish a Looper for this thread, and define a Handler for it.
Looper.prepare();
synchronized (mReadyFence) {
mHandler = new EncoderHandler(this);
mReady = true;
mReadyFence.notify();
}
Looper.loop();
Log.d(TAG, "Encoder thread exiting");
synchronized (mReadyFence) {
mReady = mRunning = false;
mHandler = null;
}
}
/**
* Handles encoder state change requests. The handler is created on the encoder thread.
*/
private static class EncoderHandler extends Handler {
private WeakReference<TextureMovieEncoder> mWeakEncoder;
public EncoderHandler(TextureMovieEncoder encoder) {
mWeakEncoder = new WeakReference<TextureMovieEncoder>(encoder);
}
@Override // runs on encoder thread
public void handleMessage(Message inputMessage) {
int what = inputMessage.what;
Object obj = inputMessage.obj;
TextureMovieEncoder encoder = mWeakEncoder.get();
if (encoder == null) {
Log.w(TAG, "EncoderHandler.handleMessage: encoder is null");
return;
}
switch (what) {
case MSG_START_RECORDING:
encoder.handleStartRecording((EncoderConfig) obj);
break;
case MSG_STOP_RECORDING:
encoder.handleStopRecording();
break;
case MSG_FRAME_AVAILABLE:
long timestamp = (((long) inputMessage.arg1) << 32) |
(((long) inputMessage.arg2) & 0xffffffffL);
encoder.handleFrameAvailable((float[]) obj, timestamp);
break;
case MSG_SET_TEXTURE_ID:
encoder.handleSetTexture(inputMessage.arg1);
break;
case MSG_UPDATE_SHARED_CONTEXT:
encoder.handleUpdateSharedContext((EGLContext) inputMessage.obj);
break;
case MSG_QUIT:
Looper.myLooper().quit();
break;
default:
throw new RuntimeException("Unhandled msg what=" + what);
}
}
}
/**
* Starts recording.
*/
private void handleStartRecording(EncoderConfig config) {
Log.d(TAG, "handleStartRecording " + config);
prepareEncoder(config.mEglContext, config.mWidth, config.mHeight, config.mBitRate,
config.mOutputFile);
}
/**
* Handles notification of an available frame.
* <p>
* The texture is rendered onto the encoder's input surface, along with a moving
* box (just because we can).
* <p>
* @param transform The texture transform, from SurfaceTexture.
* @param timestampNanos The frame's timestamp, from SurfaceTexture.
*/
private void handleFrameAvailable(float[] transform, long timestampNanos) {
if (VERBOSE) Log.d(TAG, "handleFrameAvailable tr=" + transform);
if (cycle == 1) {
mVideoEncoder.drainEncoder(false);
mFullScreen.drawFrame(mTextureId, transform, 1.0f);
}
else {
mFullScreen.drawFrame(mTextureId, transform, -1.0f);
}
mInputWindowSurface.setPresentationTime(timestampNanos);
mInputWindowSurface.swapBuffers();
if (cycle == 1) {
mSpeedCallback.setCanRelease(true);
cycle = 0;
} else
cycle++;
}
/**
* Handles a request to stop encoding.
*/
private void handleStopRecording() {
Log.d(TAG, "handleStopRecording");
mVideoEncoder.drainEncoder(true);
releaseEncoder();
}
/**
* Sets the texture name that SurfaceTexture will use when frames are received.
*/
private void handleSetTexture(int id) {
//Log.d(TAG, "handleSetTexture " + id);
mTextureId = id;
}
/**
* Tears down the EGL surface and context we've been using to feed the MediaCodec input
* surface, and replaces it with a new one that shares with the new context.
* <p>
* This is useful if the old context we were sharing with went away (maybe a GLSurfaceView
* that got torn down) and we need to hook up with the new one.
*/
private void handleUpdateSharedContext(EGLContext newSharedContext) {
Log.d(TAG, "handleUpdatedSharedContext " + newSharedContext);
// Release the EGLSurface and EGLContext.
mInputWindowSurface.releaseEglSurface();
mFullScreen.release(false);
mEglCore.release();
// Create a new EGLContext and recreate the window surface.
mEglCore = new EglCore(newSharedContext, EglCore.FLAG_RECORDABLE);
mInputWindowSurface.recreate(mEglCore);
mInputWindowSurface.makeCurrent();
// Create new programs and such for the new context.
mFullScreen = new FullFrameRect(
new Texture2dProgram(Texture2dProgram.ProgramType.TEXTURE_SBS));
}
private void prepareEncoder(EGLContext sharedContext, int width, int height, int bitRate,
File outputFile) {
try {
mVideoEncoder = new VideoEncoderCore(width, height, bitRate, outputFile);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
mEglCore = new EglCore(sharedContext, EglCore.FLAG_RECORDABLE);
mInputWindowSurface = new WindowSurface(mEglCore, mVideoEncoder.getInputSurface(), true);
mInputWindowSurface.makeCurrent();
mFullScreen = new FullFrameRect(
new Texture2dProgram(Texture2dProgram.ProgramType.TEXTURE_SBS));
}
private void releaseEncoder() {
mVideoEncoder.release();
if (mInputWindowSurface != null) {
mInputWindowSurface.release();
mInputWindowSurface = null;
}
if (mFullScreen != null) {
mFullScreen.release(false);
mFullScreen = null;
}
if (mEglCore != null) {
mEglCore.release();
mEglCore = null;
}
}
/**
* Draws a box, with position offset.
*/
private void drawBox(int posn) {
final int width = mInputWindowSurface.getWidth();
int xpos = (posn * 4) % (width - 50);
GLES20.glEnable(GLES20.GL_SCISSOR_TEST);
GLES20.glScissor(xpos, 0, 100, 100);
GLES20.glClearColor(1.0f, 0.0f, 1.0f, 1.0f);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glDisable(GLES20.GL_SCISSOR_TEST);
}
}
我的想法可行吗？或者是否有 better/correct 延迟其中一个视频的方法？
看来我第一个切换时间戳的想法是无效的。
按照Fadden的建议，我使用两个解码器成功地创建了延迟。我修改了 grafika 的 MoviePlayer 的代码，因此它包含两对提取器解码器。解码器具有单独的输出纹理。提取循环在单独的线程中 运行。我认为这种方法会导致性能大幅下降，但似乎并没有发生，性能仍然可以满足我的需求。
我正在使用 google/grafika's 示例来解码、转换和编码回文件视频剪辑。转换是缩小和平移，它是通过存储在 Texture2dProgram 中的着色器完成的。我的主要 activity 是基于 CameraCaptureActivity。问题是我同时将两个视频并排放置在单个纹理上。我想将其中之一延迟给定数量的帧。另请注意，我在编码时不需要显示预览。
到目前为止，我最好的想法是在逐帧前进时更改时间戳。在 TextureMovieEncoder 中，我正在发送有关帧的信息，包括必须将它们放置在结果视频中的时间戳。它发生在 frameAvailiable()，我在这里一次发送有关两帧（左和右）的信息。这个想法是增加其中一个的时间戳。问题是结果视频是失真的，所以我不知道我的方法是否可行。 TextureMovieEncoder 发布在下方。
package com.android.grafika;
import android.graphics.SurfaceTexture;
import android.opengl.EGLContext;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.util.Log;
import com.android.grafika.gles.EglCore;
import com.android.grafika.gles.FullFrameRect;
import com.android.grafika.gles.Texture2dProgram;
import com.android.grafika.gles.WindowSurface;
import java.io.File;
import java.io.IOException;
import java.lang.ref.WeakReference;
/**
* Encode a movie from frames rendered from an external texture image.
* <p>
* The object wraps an encoder running on a dedicated thread. The various control messages
* may be sent from arbitrary threads (typically the app UI thread). The encoder thread
* manages both sides of the encoder (feeding and draining); the only external input is
* the GL texture.
* <p>
* The design is complicated slightly by the need to create an EGL context that shares state
* with a view that gets restarted if (say) the device orientation changes. When the view
* in question is a GLSurfaceView, we don't have full control over the EGL context creation
* on that side, so we have to bend a bit backwards here.
* <p>
* To use:
* <ul>
* <li>create TextureMovieEncoder object
* <li>create an EncoderConfig
* <li>call TextureMovieEncoder#startRecording() with the config
* <li>call TextureMovieEncoder#setTextureId() with the texture object that receives frames
* <li>for each frame, after latching it with SurfaceTexture#updateTexImage(),
* call TextureMovieEncoder#frameAvailable().
* </ul>
*
* TODOO: tweak the API (esp. textureId) so it's less awkward for simple use cases.
*/
public class TextureMovieEncoder implements Runnable {
private static final String TAG = MainActivity.TAG;
private static final boolean VERBOSE = false;
private static final long timestampCorrection = 1000000000;
private long timestampCorected;
private static final int MSG_START_RECORDING = 0;
private static final int MSG_STOP_RECORDING = 1;
private static final int MSG_FRAME_AVAILABLE = 2;
private static final int MSG_SET_TEXTURE_ID = 3;
private static final int MSG_UPDATE_SHARED_CONTEXT = 4;
private static final int MSG_QUIT = 5;
private boolean measure_started = false;
private long startTime = -1;
private int cycle = 0;
private long handleFrameTime = 0;
private long last_timestamp = -1;
private float [] transform;
private long last_orig_timestamp = -1;
public long getFrame() {
return frame;
}
private long frame = 0;
private long average_diff = 0;
private long step = 40000000;
private long actTimestamp = 0;
private boolean shouldStop = false;
public void setmSpeedCallback(SpeedControlCallback mSpeedCallback) {
this.mSpeedCallback = mSpeedCallback;
}
private SpeedControlCallback mSpeedCallback;
// ----- accessed exclusively by encoder thread -----
private WindowSurface mInputWindowSurface;
private EglCore mEglCore;
private FullFrameRect mFullScreen;
private int mTextureId;
private VideoEncoderCore mVideoEncoder;
// ----- accessed by multiple threads -----
private volatile EncoderHandler mHandler;
private Object mReadyFence = new Object(); // guards ready/running
private boolean mReady;
private boolean mRunning;
/**
* Encoder configuration.
* <p>
* Object is immutable, which means we can safely pass it between threads without
* explicit synchronization (and don't need to worry about it getting tweaked out from
* under us).
* <p>
* TODO: make frame rate and iframe interval configurable? Maybe use builder pattern
* with reasonable defaults for those and bit rate.
*/
public static class EncoderConfig {
final File mOutputFile;
final int mWidth;
final int mHeight;
final int mBitRate;
final EGLContext mEglContext;
public EncoderConfig(File outputFile, int width, int height, int bitRate,
EGLContext sharedEglContext) {
mOutputFile = outputFile;
mWidth = width;
mHeight = height;
mBitRate = bitRate;
mEglContext = sharedEglContext;
}
@Override
public String toString() {
return "EncoderConfig: " + mWidth + "x" + mHeight + " @" + mBitRate +
" to '" + mOutputFile.toString() + "' ctxt=" + mEglContext;
}
}
/**
* Tells the video recorder to start recording. (Call from non-encoder thread.)
* <p>
* Creates a new thread, which will create an encoder using the provided configuration.
* <p>
* Returns after the recorder thread has started and is ready to accept Messages. The
* encoder may not yet be fully configured.
*/
public void startRecording(EncoderConfig config) {
Log.d(TAG, "Encoder: startRecording()");
synchronized (mReadyFence) {
if (mRunning) {
Log.w(TAG, "Encoder thread already running");
return;
}
mRunning = true;
new Thread(this, "TextureMovieEncoder").start();
while (!mReady) {
try {
mReadyFence.wait();
} catch (InterruptedException ie) {
// ignore
}
}
}
mHandler.sendMessage(mHandler.obtainMessage(MSG_START_RECORDING, config));
}
/**
* Tells the video recorder to stop recording. (Call from non-encoder thread.)
* <p>
* Returns immediately; the encoder/muxer may not yet be finished creating the movie.
* <p>
* TODO: have the encoder thread invoke a callback on the UI thread just before it shuts down
* so we can provide reasonable status UI (and let the caller know that movie encoding
* has completed).
*/
public void stopRecording() {
//mHandler.sendMessage(mHandler.obtainMessage(MSG_STOP_RECORDING));
//mHandler.sendMessage(mHandler.obtainMessage(MSG_QUIT));
// We don't know when these will actually finish (or even start). We don't want to
// delay the UI thread though, so we return immediately.
shouldStop = true;
Log.d(TAG, "Shout down flag set up.");
}
/**
* Returns true if recording has been started.
*/
public boolean isRecording() {
synchronized (mReadyFence) {
return mRunning;
}
}
/**
* Tells the video recorder to refresh its EGL surface. (Call from non-encoder thread.)
*/
public void updateSharedContext(EGLContext sharedContext) {
mHandler.sendMessage(mHandler.obtainMessage(MSG_UPDATE_SHARED_CONTEXT, sharedContext));
}
/**
* Tells the video recorder that a new frame is available. (Call from non-encoder thread.)
* <p>
* This function sends a message and returns immediately. This isn't sufficient -- we
* don't want the caller to latch a new frame until we're done with this one -- but we
* can get away with it so long as the input frame rate is reasonable and the encoder
* thread doesn't stall.
* <p>
* TODO: either block here until the texture has been rendered onto the encoder surface,
* or have a separate "block if still busy" method that the caller can execute immediately
* before it calls updateTexImage(). The latter is preferred because we don't want to
* stall the caller while this thread does work.
*/
public void frameAvailable(SurfaceTexture st) {
synchronized (mReadyFence) {
if (!mReady) {
return;
}
}
transform = new float[16]; // TODOO - avoid alloc every frame
st.getTransformMatrix(transform);
long timestamp = st.getTimestamp();
// if first frame
if (last_timestamp < 0) {
if (!measure_started) {
startTime = System.currentTimeMillis();
measure_started = true;
}
last_timestamp = timestamp;
last_orig_timestamp = timestamp;
}
else {
// HARDCODED FRAME NUMBER :(
// if playback finished or frame number reached
if ((frame == 200) || shouldStop) {
if (measure_started) {
long stopTime = System.currentTimeMillis();
long elapsedTime = stopTime - startTime;
Log.d(TAG, "Rendering time: " + (double)elapsedTime * 0.001 + "[s]");
Log.d(TAG, "HandlingFrame time: " + (double)(stopTime - handleFrameTime) * 0.001 + "[s]");
measure_started = false;
}
mHandler.sendMessage(mHandler.obtainMessage(MSG_STOP_RECORDING));
mHandler.sendMessage(mHandler.obtainMessage(MSG_QUIT));
return;
}
else if (timestamp == 0) {
// Seeing this after device is toggled off/on with power button. The
// first frame back has a zero timestamp.
//
// MPEG4Writer thinks this is cause to abort() in native code, so it's very
// important that we just ignore the frame.
Log.w(TAG, "HEY: got SurfaceTexture with timestamp of zero");
return;
}
// this is workaround for duplicated timestamp
// might cause troubles with some videos
else if ((timestamp == last_orig_timestamp)) {
return;
}
else {
frame++;
mHandler.sendMessage(mHandler.obtainMessage(MSG_FRAME_AVAILABLE,
(int) (actTimestamp >> 32), (int) actTimestamp, transform));
timestampCorected = actTimestamp + timestampCorrection;
mHandler.sendMessage(mHandler.obtainMessage(MSG_FRAME_AVAILABLE,
(int) (timestampCorected >> 32), (int) timestampCorected, transform));
actTimestamp += step;
}
last_orig_timestamp = timestamp;
}
}
/**
* Calculates 'average' diffrence between frames.
* Result is based on first 50 frames.
* Shuld be called in frameAvailiable.
*
* @param timestamp actual frame timestamp
*/
private void calcAndShowAverageDiff(long timestamp) {
if ((frame < 50) && (frame > 0)) {
average_diff += timestamp - last_timestamp;
last_timestamp = timestamp;
}
if (frame == 50) {
average_diff /= frame;
Log.d(TAG, "Average timestamp difference: " + Long.toString(average_diff));
}
}
/**
* Tells the video recorder what texture name to use. This is the external texture that
* we're receiving camera previews in. (Call from non-encoder thread.)
* <p>
* TODOO: do something less clumsy
*/
public void setTextureId(int id) {
synchronized (mReadyFence) {
if (!mReady) {
return;
}
}
mHandler.sendMessage(mHandler.obtainMessage(MSG_SET_TEXTURE_ID, id, 0, null));
}
/**
* Encoder thread entry point. Establishes Looper/Handler and waits for messages.
* <p>
* @see java.lang.Thread#run()
*/
@Override
public void run() {
// Establish a Looper for this thread, and define a Handler for it.
Looper.prepare();
synchronized (mReadyFence) {
mHandler = new EncoderHandler(this);
mReady = true;
mReadyFence.notify();
}
Looper.loop();
Log.d(TAG, "Encoder thread exiting");
synchronized (mReadyFence) {
mReady = mRunning = false;
mHandler = null;
}
}
/**
* Handles encoder state change requests. The handler is created on the encoder thread.
*/
private static class EncoderHandler extends Handler {
private WeakReference<TextureMovieEncoder> mWeakEncoder;
public EncoderHandler(TextureMovieEncoder encoder) {
mWeakEncoder = new WeakReference<TextureMovieEncoder>(encoder);
}
@Override // runs on encoder thread
public void handleMessage(Message inputMessage) {
int what = inputMessage.what;
Object obj = inputMessage.obj;
TextureMovieEncoder encoder = mWeakEncoder.get();
if (encoder == null) {
Log.w(TAG, "EncoderHandler.handleMessage: encoder is null");
return;
}
switch (what) {
case MSG_START_RECORDING:
encoder.handleStartRecording((EncoderConfig) obj);
break;
case MSG_STOP_RECORDING:
encoder.handleStopRecording();
break;
case MSG_FRAME_AVAILABLE:
long timestamp = (((long) inputMessage.arg1) << 32) |
(((long) inputMessage.arg2) & 0xffffffffL);
encoder.handleFrameAvailable((float[]) obj, timestamp);
break;
case MSG_SET_TEXTURE_ID:
encoder.handleSetTexture(inputMessage.arg1);
break;
case MSG_UPDATE_SHARED_CONTEXT:
encoder.handleUpdateSharedContext((EGLContext) inputMessage.obj);
break;
case MSG_QUIT:
Looper.myLooper().quit();
break;
default:
throw new RuntimeException("Unhandled msg what=" + what);
}
}
}
/**
* Starts recording.
*/
private void handleStartRecording(EncoderConfig config) {
Log.d(TAG, "handleStartRecording " + config);
prepareEncoder(config.mEglContext, config.mWidth, config.mHeight, config.mBitRate,
config.mOutputFile);
}
/**
* Handles notification of an available frame.
* <p>
* The texture is rendered onto the encoder's input surface, along with a moving
* box (just because we can).
* <p>
* @param transform The texture transform, from SurfaceTexture.
* @param timestampNanos The frame's timestamp, from SurfaceTexture.
*/
private void handleFrameAvailable(float[] transform, long timestampNanos) {
if (VERBOSE) Log.d(TAG, "handleFrameAvailable tr=" + transform);
if (cycle == 1) {
mVideoEncoder.drainEncoder(false);
mFullScreen.drawFrame(mTextureId, transform, 1.0f);
}
else {
mFullScreen.drawFrame(mTextureId, transform, -1.0f);
}
mInputWindowSurface.setPresentationTime(timestampNanos);
mInputWindowSurface.swapBuffers();
if (cycle == 1) {
mSpeedCallback.setCanRelease(true);
cycle = 0;
} else
cycle++;
}
/**
* Handles a request to stop encoding.
*/
private void handleStopRecording() {
Log.d(TAG, "handleStopRecording");
mVideoEncoder.drainEncoder(true);
releaseEncoder();
}
/**
* Sets the texture name that SurfaceTexture will use when frames are received.
*/
private void handleSetTexture(int id) {
//Log.d(TAG, "handleSetTexture " + id);
mTextureId = id;
}
/**
* Tears down the EGL surface and context we've been using to feed the MediaCodec input
* surface, and replaces it with a new one that shares with the new context.
* <p>
* This is useful if the old context we were sharing with went away (maybe a GLSurfaceView
* that got torn down) and we need to hook up with the new one.
*/
private void handleUpdateSharedContext(EGLContext newSharedContext) {
Log.d(TAG, "handleUpdatedSharedContext " + newSharedContext);
// Release the EGLSurface and EGLContext.
mInputWindowSurface.releaseEglSurface();
mFullScreen.release(false);
mEglCore.release();
// Create a new EGLContext and recreate the window surface.
mEglCore = new EglCore(newSharedContext, EglCore.FLAG_RECORDABLE);
mInputWindowSurface.recreate(mEglCore);
mInputWindowSurface.makeCurrent();
// Create new programs and such for the new context.
mFullScreen = new FullFrameRect(
new Texture2dProgram(Texture2dProgram.ProgramType.TEXTURE_SBS));
}
private void prepareEncoder(EGLContext sharedContext, int width, int height, int bitRate,
File outputFile) {
try {
mVideoEncoder = new VideoEncoderCore(width, height, bitRate, outputFile);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
mEglCore = new EglCore(sharedContext, EglCore.FLAG_RECORDABLE);
mInputWindowSurface = new WindowSurface(mEglCore, mVideoEncoder.getInputSurface(), true);
mInputWindowSurface.makeCurrent();
mFullScreen = new FullFrameRect(
new Texture2dProgram(Texture2dProgram.ProgramType.TEXTURE_SBS));
}
private void releaseEncoder() {
mVideoEncoder.release();
if (mInputWindowSurface != null) {
mInputWindowSurface.release();
mInputWindowSurface = null;
}
if (mFullScreen != null) {
mFullScreen.release(false);
mFullScreen = null;
}
if (mEglCore != null) {
mEglCore.release();
mEglCore = null;
}
}
/**
* Draws a box, with position offset.
*/
private void drawBox(int posn) {
final int width = mInputWindowSurface.getWidth();
int xpos = (posn * 4) % (width - 50);
GLES20.glEnable(GLES20.GL_SCISSOR_TEST);
GLES20.glScissor(xpos, 0, 100, 100);
GLES20.glClearColor(1.0f, 0.0f, 1.0f, 1.0f);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glDisable(GLES20.GL_SCISSOR_TEST);
}
}
我的想法可行吗？或者是否有 better/correct 延迟其中一个视频的方法？
看来我第一个切换时间戳的想法是无效的。
按照Fadden的建议，我使用两个解码器成功地创建了延迟。我修改了 grafika 的 MoviePlayer 的代码，因此它包含两对提取器解码器。解码器具有单独的输出纹理。提取循环在单独的线程中 运行。我认为这种方法会导致性能大幅下降，但似乎并没有发生，性能仍然可以满足我的需求。