点击以使用 javacv 像在 vine 中一样进行记录
Tap to record like in vine using javacv
我正在尝试像在 vine 中一样实现点击录制功能。 javacv 中提供的处理记录（不是触摸记录）的示例是https://github.com/bytedeco/javacv/blob/master/samples/RecordActivity.java。我正在尝试修改它，以便在 onPreviewFrame 方法中仅当用户将手指放在屏幕上时才将帧添加到缓冲区。然后尝试在 stopRecording 方法中将这些帧组合成最终视频。
问题是，如果我按照下面的代码片段（在 stopRecording 方法中）设置时间戳
if (t > recorder.getTimestamp())
{
recorder.setTimestamp(t);
}
行为如下
案例一
如果我点击屏幕录制 2 秒，然后将手指从屏幕上移开 3 秒，然后再次将手指放回屏幕上录制另外 4 秒，那么生成的视频就像，
第 1 2 秒视频已录制内容。接下来的 3 秒（手指离开屏幕的时间）。视频只显示手指最后放在屏幕上时记录的最后一帧。然后视频记录了接下来 4 秒的视频内容。因此，当手指从屏幕上移开时，处理视频录制似乎存在问题。
案例二
接下来我在stopRecording方法中删除了设置时间戳到记录器的代码（上面给出的代码片段）。
现在生成的视频（对于案例 1 中尝试的相同步骤）不包含手指离开屏幕时的中间 3 秒（这是必需的）。但是视频正在以更快的速度播放。所以看来我们需要设置时间戳才能让视频正常播放。
我的 activity 的完整代码如下。 （请注意，视频录制主要由 onPreviewFrame 和 stopRecording 方法处理）
public class TouchToRecordActivity extends Activity implements OnClickListener, View.OnTouchListener {
private final static String CLASS_LABEL = "TouchToRecordActivity";
private final static String LOG_TAG = CLASS_LABEL;
private String ffmpeg_link = "/mnt/sdcard/stream.mp4";
long startTime = 0;
boolean recording = false;
boolean rec = false;
private FFmpegFrameRecorder recorder;
private boolean isPreviewOn = false;
private int sampleAudioRateInHz = 44100;
private int imageWidth = 640;
private int imageHeight = 480;
private int destWidth = 480;
private int frameRate = 30;
/* audio data getting thread */
private AudioRecord audioRecord;
private AudioRecordRunnable audioRecordRunnable;
private Thread audioThread;
volatile boolean runAudioThread = true;
/* video data getting thread */
private Camera cameraDevice;
private CameraView cameraView;
private Frame yuvImage = null;
/* layout setting */
private final int bg_screen_bx = 232;
private final int bg_screen_by = 128;
private final int bg_screen_width = 700;
private final int bg_screen_height = 500;
private final int bg_width = 1123;
private final int bg_height = 715;
private final int live_width = 640;
private final int live_height = 480;
private int screenWidth, screenHeight;
private Button btnRecorderControl;
/* The number of seconds in the continuous record loop (or 0 to disable loop). */
final int RECORD_LENGTH = 20;
Frame[] images;
long[] timestamps;
ShortBuffer[] samples;
int imagesIndex, samplesIndex;
long firstTime = 0;
long startPauseTime = 0;
long totalPauseTime = 0;
long pausedTime = 0;
long stopPauseTime = 0;
long totalTime = 0;
long totalRecordedTS = 0;
private TextView txtTimer;
private Handler mHandler = new Handler();
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE);
setContentView(R.layout.touch_main);
initLayout();
}
@Override
protected void onDestroy() {
super.onDestroy();
recording = false;
if (cameraView != null) {
cameraView.stopPreview();
}
if (cameraDevice != null) {
cameraDevice.stopPreview();
cameraDevice.release();
cameraDevice = null;
}
}
private void initLayout() {
/* get size of screen */
Display display = ((WindowManager) getSystemService(Context.WINDOW_SERVICE)).getDefaultDisplay();
screenWidth = display.getWidth();
screenHeight = display.getHeight();
RelativeLayout.LayoutParams layoutParam = null;
LayoutInflater myInflate = null;
myInflate = (LayoutInflater) getSystemService(Context.LAYOUT_INFLATER_SERVICE);
RelativeLayout topLayout = new RelativeLayout(this);
setContentView(topLayout);
LinearLayout preViewLayout = (LinearLayout) myInflate.inflate(R.layout.touch_main, null);
layoutParam = new RelativeLayout.LayoutParams(screenWidth, screenHeight);
topLayout.addView(preViewLayout, layoutParam);
txtTimer = (TextView) preViewLayout.findViewById(R.id.txtTimer);
/* add control button: start and stop */
btnRecorderControl = (Button) findViewById(R.id.recorder_control);
btnRecorderControl.setText("Start");
btnRecorderControl.setOnClickListener(this);
/* add camera view */
int display_width_d = (int) (1.0 * bg_screen_width * screenWidth / bg_width);
int display_height_d = (int) (1.0 * bg_screen_height * screenHeight / bg_height);
int prev_rw, prev_rh;
if (1.0 * display_width_d / display_height_d > 1.0 * live_width / live_height) {
prev_rh = display_height_d;
prev_rw = (int) (1.0 * display_height_d * live_width / live_height);
} else {
prev_rw = display_width_d;
prev_rh = (int) (1.0 * display_width_d * live_height / live_width);
}
layoutParam = new RelativeLayout.LayoutParams(prev_rw, prev_rh);
layoutParam.topMargin = (int) (1.0 * bg_screen_by * screenHeight / bg_height);
layoutParam.leftMargin = (int) (1.0 * bg_screen_bx * screenWidth / bg_width);
cameraDevice = Camera.open();
Log.i(LOG_TAG, "cameara open");
cameraView = new CameraView(this, cameraDevice);
topLayout.addView(cameraView, layoutParam);
topLayout.setOnTouchListener(this);
Log.i(LOG_TAG, "cameara preview start: OK");
}
//---------------------------------------
// initialize ffmpeg_recorder
//---------------------------------------
private void initRecorder() {
Log.w(LOG_TAG, "init recorder");
if (RECORD_LENGTH > 0) {
imagesIndex = 0;
images = new Frame[RECORD_LENGTH * frameRate];
timestamps = new long[images.length];
for (int i = 0; i < images.length; i++) {
images[i] = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
timestamps[i] = -1;
}
} else if (yuvImage == null) {
yuvImage = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
Log.i(LOG_TAG, "create yuvImage");
}
Log.i(LOG_TAG, "ffmpeg_url: " + ffmpeg_link);
recorder = new FFmpegFrameRecorder(ffmpeg_link, destWidth, imageHeight, 1);
recorder.setFormat("mp4");
recorder.setVideoCodecName("libx264");
recorder.setSampleRate(sampleAudioRateInHz);
// Set in the surface changed method
recorder.setFrameRate(frameRate);
Log.i(LOG_TAG, "recorder initialize success");
audioRecordRunnable = new AudioRecordRunnable();
audioThread = new Thread(audioRecordRunnable);
runAudioThread = true;
}
public void startRecording() {
initRecorder();
mHandler.removeCallbacks(mUpdateTimeTask);
mHandler.postDelayed(mUpdateTimeTask, 100);
try {
recorder.start();
startTime = System.currentTimeMillis();
recording = true;
audioThread.start();
} catch (FFmpegFrameRecorder.Exception e) {
e.printStackTrace();
}
}
public void stopRecording() {
runAudioThread = false;
try {
audioThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
audioRecordRunnable = null;
audioThread = null;
if (recorder != null && recording) {
if (RECORD_LENGTH > 0) {
Log.v(LOG_TAG, "Writing frames");
try {
int firstIndex = imagesIndex % samples.length;
int lastIndex = (imagesIndex - 1) % images.length;
if (imagesIndex <= images.length) {
firstIndex = 0;
lastIndex = imagesIndex - 1;
}
if ((startTime = timestamps[lastIndex] - RECORD_LENGTH * 1000000L) < 0) {
startTime = 0;
}
if (lastIndex < firstIndex) {
lastIndex += images.length;
}
int videoCounter = 0;
for (int i = firstIndex; i <= lastIndex; i++) {
if (timestamps[i] == -1) {
Log.v(LOG_TAG, "frame not recorded");
}
if (timestamps[i] != -1) {
long t = timestamps[i % timestamps.length] - startTime;
if (t >= 0) {
videoCounter++;
/*if (((i % images.length) != 0) && images[i % images.length] != images[(i % images.length) - 1]) {
if (t > recorder.getTimestamp()) {
recorder.setTimestamp(t);
}*/
Log.v(LOG_TAG, "imageIndex=" + (i % images.length));
recorder.record(images[i % images.length]);
/* }*/
Log.v(LOG_TAG, "videoCounter=" + videoCounter);
}
}
}
firstIndex = samplesIndex % samples.length;
lastIndex = (samplesIndex - 1) % samples.length;
if (samplesIndex <= samples.length) {
firstIndex = 0;
lastIndex = samplesIndex - 1;
}
if (lastIndex < firstIndex) {
lastIndex += samples.length;
}
for (int i = firstIndex; i <= lastIndex; i++) {
if (timestamps[i] != -1) {
recorder.recordSamples(samples[i % samples.length]);
}
}
} catch (FFmpegFrameRecorder.Exception e) {
Log.v(LOG_TAG, e.getMessage());
e.printStackTrace();
}
}
recording = false;
Log.v(LOG_TAG, "Finishing recording, calling stop and release on recorder");
try {
recorder.stop();
recorder.release();
} catch (FFmpegFrameRecorder.Exception e) {
e.printStackTrace();
}
recorder = null;
}
}
@Override
public boolean onKeyDown(int keyCode, KeyEvent event) {
if (keyCode == KeyEvent.KEYCODE_BACK) {
if (recording) {
stopRecording();
}
finish();
return true;
}
return super.onKeyDown(keyCode, event);
}
@Override
public boolean onTouch(View view, MotionEvent motionEvent) {
switch (motionEvent.getAction()) {
case MotionEvent.ACTION_DOWN:
Log.v(LOG_TAG, "ACTION_DOWN" + recording);
if (!recording) {
startRecording();
} else {
stopPauseTime = System.currentTimeMillis();
totalPauseTime = stopPauseTime - startPauseTime - ((long) (1.0 / (double) frameRate) * 1000);
pausedTime += totalPauseTime;
}
rec = true;
setTotalVideoTime();
btnRecorderControl.setText(getResources().getString(R.string.stop));
break;
case MotionEvent.ACTION_MOVE:
rec = true;
setTotalVideoTime();
break;
case MotionEvent.ACTION_UP:
Log.v(LOG_TAG, "ACTION_UP");
rec = false;
startPauseTime = System.currentTimeMillis();
break;
}
return true;
}
private Runnable mUpdateTimeTask = new Runnable() {
public void run() {
if (recording) {
setTotalVideoTime();
}
mHandler.postDelayed(this, 500);
}
};
private synchronized void setTotalVideoTime() {
totalTime = System.currentTimeMillis() - firstTime - pausedTime - ((long) (1.0 / (double) frameRate) * 1000);
if (totalTime > 0)
txtTimer.setText(getRecordingTimeFromMillis(totalTime));
}
private String getRecordingTimeFromMillis(long millis) {
String strRecordingTime = null;
int seconds = (int) (millis / 1000);
int minutes = seconds / 60;
int hours = minutes / 60;
if (hours >= 0 && hours < 10)
strRecordingTime = "0" + hours + ":";
else
strRecordingTime = hours + ":";
if (hours > 0)
minutes = minutes % 60;
if (minutes >= 0 && minutes < 10)
strRecordingTime += "0" + minutes + ":";
else
strRecordingTime += minutes + ":";
seconds = seconds % 60;
if (seconds >= 0 && seconds < 10)
strRecordingTime += "0" + seconds;
else
strRecordingTime += seconds;
return strRecordingTime;
}
//---------------------------------------------
// audio thread, gets and encodes audio data
//---------------------------------------------
class AudioRecordRunnable implements Runnable {
@Override
public void run() {
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO);
// Audio
int bufferSize;
ShortBuffer audioData;
int bufferReadResult;
bufferSize = AudioRecord.getMinBufferSize(sampleAudioRateInHz,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT);
audioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC, sampleAudioRateInHz,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize);
if (RECORD_LENGTH > 0) {
samplesIndex = 0;
samples = new ShortBuffer[RECORD_LENGTH * sampleAudioRateInHz * 2 / bufferSize + 1];
for (int i = 0; i < samples.length; i++) {
samples[i] = ShortBuffer.allocate(bufferSize);
}
} else {
audioData = ShortBuffer.allocate(bufferSize);
}
Log.d(LOG_TAG, "audioRecord.startRecording()");
audioRecord.startRecording();
/* ffmpeg_audio encoding loop */
while (runAudioThread) {
if (RECORD_LENGTH > 0) {
audioData = samples[samplesIndex++ % samples.length];
audioData.position(0).limit(0);
}
//Log.v(LOG_TAG,"recording? " + recording);
bufferReadResult = audioRecord.read(audioData.array(), 0, audioData.capacity());
audioData.limit(bufferReadResult);
if (bufferReadResult > 0) {
Log.v(LOG_TAG, "bufferReadResult: " + bufferReadResult);
// If "recording" isn't true when start this thread, it never get's set according to this if statement...!!!
// Why? Good question...
if (recording && rec) {
Log.v(LOG_TAG, "Recording audio");
if (RECORD_LENGTH <= 0) try {
recorder.recordSamples(audioData);
//Log.v(LOG_TAG,"recording " + 1024*i + " to " + 1024*i+1024);
} catch (FFmpegFrameRecorder.Exception e) {
Log.v(LOG_TAG, e.getMessage());
e.printStackTrace();
}
}
}
}
Log.v(LOG_TAG, "AudioThread Finished, release audioRecord");
/* encoding finish, release recorder */
if (audioRecord != null) {
audioRecord.stop();
audioRecord.release();
audioRecord = null;
Log.v(LOG_TAG, "audioRecord released");
}
}
}
//---------------------------------------------
// camera thread, gets and encodes video data
//---------------------------------------------
class CameraView extends SurfaceView implements SurfaceHolder.Callback, PreviewCallback {
private SurfaceHolder mHolder;
private Camera mCamera;
public CameraView(Context context, Camera camera) {
super(context);
Log.w("camera", "camera view");
mCamera = camera;
mHolder = getHolder();
mHolder.addCallback(CameraView.this);
mHolder.setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS);
mCamera.setPreviewCallback(CameraView.this);
}
@Override
public void surfaceCreated(SurfaceHolder holder) {
try {
stopPreview();
mCamera.setPreviewDisplay(holder);
} catch (IOException exception) {
mCamera.release();
mCamera = null;
}
}
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
stopPreview();
Camera.Parameters camParams = mCamera.getParameters();
List<Camera.Size> sizes = camParams.getSupportedPreviewSizes();
// Sort the list in ascending order
Collections.sort(sizes, new Comparator<Camera.Size>() {
public int compare(final Camera.Size a, final Camera.Size b) {
return a.width * a.height - b.width * b.height;
}
});
camParams.setPreviewSize(imageWidth, imageHeight);
Log.v(LOG_TAG, "Setting imageWidth: " + imageWidth + " imageHeight: " + imageHeight + " frameRate: " + frameRate);
camParams.setPreviewFrameRate(frameRate);
Log.v(LOG_TAG, "Preview Framerate: " + camParams.getPreviewFrameRate());
mCamera.setParameters(camParams);
List<Camera.Size> videoSizes = mCamera.getParameters().getSupportedVideoSizes();
// Set the holder (which might have changed) again
try {
mCamera.setPreviewDisplay(holder);
mCamera.setPreviewCallback(CameraView.this);
startPreview();
} catch (Exception e) {
Log.e(LOG_TAG, "Could not set preview display in surfaceChanged");
}
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
try {
mHolder.addCallback(null);
mCamera.setPreviewCallback(null);
} catch (RuntimeException e) {
// The camera has probably just been released, ignore.
}
}
public void startPreview() {
if (!isPreviewOn && mCamera != null) {
isPreviewOn = true;
mCamera.startPreview();
}
}
public void stopPreview() {
if (isPreviewOn && mCamera != null) {
isPreviewOn = false;
mCamera.stopPreview();
}
}
@Override
public void onPreviewFrame(byte[] data, Camera camera) {
if (audioRecord == null || audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
startTime = System.currentTimeMillis();
return;
}
if (RECORD_LENGTH > 0) {
int i = imagesIndex++ % images.length;
Log.v(LOG_TAG, "recording:" + recording + "rec:" + rec);
if (recording && rec) {
yuvImage = images[i];
timestamps[i] = 1000 * (System.currentTimeMillis() - startTime);
totalRecordedTS++;
} else {
Log.v(LOG_TAG, "recording is paused");
yuvImage = null;
timestamps[i] = -1;
}
}
/* get video data */
if (yuvImage != null && recording && rec) {
if (data.length != imageWidth * imageHeight) {
Camera.Size sz = camera.getParameters().getPreviewSize();
imageWidth = sz.width;
imageHeight = sz.height;
destWidth = imageHeight;
Log.v(LOG_TAG, "data length:" + data.length);
}
ByteBuffer bb = (ByteBuffer) yuvImage.image[0].position(0); // resets the buffer
int start = 2 * ((imageWidth - destWidth) / 4); // this must be even
for (int row = 0; row < imageHeight * 3 / 2; row++) {
bb.put(data, start, destWidth);
start += imageWidth;
}
}
}
}
@Override
public void onClick(View v) {
if (!recording) {
startRecording();
Log.w(LOG_TAG, "Start Button Pushed");
btnRecorderControl.setText("Stop");
} else {
// This will trigger the audio recording loop to stop and then set isRecorderStart = false;
stopRecording();
Log.w(LOG_TAG, "Stop Button Pushed");
btnRecorderControl.setText("Start");
}
}}
根据 Alex Cohn 的建议进行的更改
建议 1 - 估计平均帧率
public void stopRecording() {
..............................
if (((i % images.length) != 0) && images[i % images.length] != images[(i % images.length) - 1]) {
if (t > recorder.getTimestamp()) {
t += 1000000 / frameRate;
recorder.setTimestamp(t);
}
recorder.record(images[i % images.length]);
}
..........................................
}
所做的更改是添加 t += 1000000 / frameRate；但这会导致视频在手指离开屏幕时部分冻结（如上述案例 1）。
建议2-修改onPreviewFrame()
long[] timestampsForRecorder;
private void initRecorder() {
Log.w(LOG_TAG, "init recorder");
if (RECORD_LENGTH > 0) {
.......................................................
timestampsForRecorder = new long[images.length];
for (int i = 0; i < images.length; i++) {
images[i] = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
timestamps[i] = -1;
timestampsForRecorder[i] = -1;
}
} else if (yuvImage == null) {
yuvImage = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
Log.i(LOG_TAG, "create yuvImage");
}
...................................................
}
@Override
public void onPreviewFrame(byte[] data, Camera camera) {
if (audioRecord == null || audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
startTime = SystemClock.elapsedRealtime();
return;
}
if (RECORD_LENGTH > 0) {
int i = imagesIndex++ % images.length;
Log.v(LOG_TAG, "recording:" + recording + "rec:" + rec);
if (recording && rec) {
yuvImage = images[i];
long thisFrameTime = SystemClock.elapsedRealtime();
timestamps[i] = thisFrameTime;
long lastFrameTime = timestamps[(int) (imagesIndex == 0 ? startTime : ((imagesIndex-1) % images.length))];
Log.v(LOG_TAG, "lastFrameTime:" + lastFrameTime+",stopPauseTime:" + stopPauseTime);
if (lastFrameTime > stopPauseTime) {
timestampsForRecorder[i] = 1000 * (thisFrameTime - Math.max(stopPauseTime, lastFrameTime));
}
}
}
.....................................................
}
public void stopRecording() {
.......................................................
if (recorder != null && recording) {
if (RECORD_LENGTH > 0) {
Log.v(LOG_TAG, "Writing frames");
try {
int firstIndex = imagesIndex % samples.length;
int lastIndex = (imagesIndex - 1) % images.length;
if (imagesIndex <= images.length) {
firstIndex = 0;
lastIndex = imagesIndex - 1;
}
if ((startTime = timestampsForRecorder[lastIndex] - RECORD_LENGTH * 1000000L) < 0) {
startTime = 0;
}
if (lastIndex < firstIndex) {
lastIndex += images.length;
}
for (int i = firstIndex; i <= lastIndex; i++) {
if (timestampsForRecorder[i] != -1) {
long t = timestampsForRecorder[i % timestampsForRecorder.length] - startTime;
if (t >= 0) {
if (((i % images.length) != 0) && images[i % images.length] != images[(i % images.length) - 1]) {
if (t > recorder.getTimestamp()) {
recorder.setTimestamp(t);
}
Log.v(LOG_TAG, "imageIndex=" + (i % images.length));
recorder.record(images[i % images.length]);
}
}
}
}
.............................................
} catch (FFmpegFrameRecorder.Exception e) {
.................................
}
}
...........................................
}
}
用这个录制的视频出现了上述案例2的问题。即，它正在以更快的速度播放
简单（但不精确）的解决方案是估计平均帧速率，并使用 t += 1000000/average_fps; recorder.setTimestamp(t); 而不是查看实际时间戳。
为了更准确，您可以将onPreviewFrame()更改如下：
long thisFrameTime = SystemClock.elapsedRealtime();
timestamps[i] = thisFrameTime;
long lastFrameTime = timestamps[imagesIndex < 2 ? startTime : (imagesIndex-2) % images.length)];
if (lastFrameTime > stopPauseTime) {
timestampsForRecorder[i] = 1000 * (thisFrameTime - Math.max(stopPauseTime, lastFrameTime));
}
您可以将第二个数组 timestampsForRecorder 直接提供给记录器。
请注意，到处使用 SystemClock.elapsedRealtime() 更安全：
This clock is guaranteed to be monotonic, and continues to tick even when the CPU is in power saving modes, so is the recommend basis for general purpose interval timing.
我正在尝试像在 vine 中一样实现点击录制功能。 javacv 中提供的处理记录（不是触摸记录）的示例是https://github.com/bytedeco/javacv/blob/master/samples/RecordActivity.java。我正在尝试修改它，以便在 onPreviewFrame 方法中仅当用户将手指放在屏幕上时才将帧添加到缓冲区。然后尝试在 stopRecording 方法中将这些帧组合成最终视频。
问题是，如果我按照下面的代码片段（在 stopRecording 方法中）设置时间戳
if (t > recorder.getTimestamp())
{
recorder.setTimestamp(t);
}
行为如下
案例一
如果我点击屏幕录制 2 秒，然后将手指从屏幕上移开 3 秒，然后再次将手指放回屏幕上录制另外 4 秒，那么生成的视频就像，
第 1 2 秒视频已录制内容。接下来的 3 秒（手指离开屏幕的时间）。视频只显示手指最后放在屏幕上时记录的最后一帧。然后视频记录了接下来 4 秒的视频内容。因此，当手指从屏幕上移开时，处理视频录制似乎存在问题。
案例二
接下来我在stopRecording方法中删除了设置时间戳到记录器的代码（上面给出的代码片段）。
现在生成的视频（对于案例 1 中尝试的相同步骤）不包含手指离开屏幕时的中间 3 秒（这是必需的）。但是视频正在以更快的速度播放。所以看来我们需要设置时间戳才能让视频正常播放。
我的 activity 的完整代码如下。 （请注意，视频录制主要由 onPreviewFrame 和 stopRecording 方法处理）
public class TouchToRecordActivity extends Activity implements OnClickListener, View.OnTouchListener {
private final static String CLASS_LABEL = "TouchToRecordActivity";
private final static String LOG_TAG = CLASS_LABEL;
private String ffmpeg_link = "/mnt/sdcard/stream.mp4";
long startTime = 0;
boolean recording = false;
boolean rec = false;
private FFmpegFrameRecorder recorder;
private boolean isPreviewOn = false;
private int sampleAudioRateInHz = 44100;
private int imageWidth = 640;
private int imageHeight = 480;
private int destWidth = 480;
private int frameRate = 30;
/* audio data getting thread */
private AudioRecord audioRecord;
private AudioRecordRunnable audioRecordRunnable;
private Thread audioThread;
volatile boolean runAudioThread = true;
/* video data getting thread */
private Camera cameraDevice;
private CameraView cameraView;
private Frame yuvImage = null;
/* layout setting */
private final int bg_screen_bx = 232;
private final int bg_screen_by = 128;
private final int bg_screen_width = 700;
private final int bg_screen_height = 500;
private final int bg_width = 1123;
private final int bg_height = 715;
private final int live_width = 640;
private final int live_height = 480;
private int screenWidth, screenHeight;
private Button btnRecorderControl;
/* The number of seconds in the continuous record loop (or 0 to disable loop). */
final int RECORD_LENGTH = 20;
Frame[] images;
long[] timestamps;
ShortBuffer[] samples;
int imagesIndex, samplesIndex;
long firstTime = 0;
long startPauseTime = 0;
long totalPauseTime = 0;
long pausedTime = 0;
long stopPauseTime = 0;
long totalTime = 0;
long totalRecordedTS = 0;
private TextView txtTimer;
private Handler mHandler = new Handler();
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE);
setContentView(R.layout.touch_main);
initLayout();
}
@Override
protected void onDestroy() {
super.onDestroy();
recording = false;
if (cameraView != null) {
cameraView.stopPreview();
}
if (cameraDevice != null) {
cameraDevice.stopPreview();
cameraDevice.release();
cameraDevice = null;
}
}
private void initLayout() {
/* get size of screen */
Display display = ((WindowManager) getSystemService(Context.WINDOW_SERVICE)).getDefaultDisplay();
screenWidth = display.getWidth();
screenHeight = display.getHeight();
RelativeLayout.LayoutParams layoutParam = null;
LayoutInflater myInflate = null;
myInflate = (LayoutInflater) getSystemService(Context.LAYOUT_INFLATER_SERVICE);
RelativeLayout topLayout = new RelativeLayout(this);
setContentView(topLayout);
LinearLayout preViewLayout = (LinearLayout) myInflate.inflate(R.layout.touch_main, null);
layoutParam = new RelativeLayout.LayoutParams(screenWidth, screenHeight);
topLayout.addView(preViewLayout, layoutParam);
txtTimer = (TextView) preViewLayout.findViewById(R.id.txtTimer);
/* add control button: start and stop */
btnRecorderControl = (Button) findViewById(R.id.recorder_control);
btnRecorderControl.setText("Start");
btnRecorderControl.setOnClickListener(this);
/* add camera view */
int display_width_d = (int) (1.0 * bg_screen_width * screenWidth / bg_width);
int display_height_d = (int) (1.0 * bg_screen_height * screenHeight / bg_height);
int prev_rw, prev_rh;
if (1.0 * display_width_d / display_height_d > 1.0 * live_width / live_height) {
prev_rh = display_height_d;
prev_rw = (int) (1.0 * display_height_d * live_width / live_height);
} else {
prev_rw = display_width_d;
prev_rh = (int) (1.0 * display_width_d * live_height / live_width);
}
layoutParam = new RelativeLayout.LayoutParams(prev_rw, prev_rh);
layoutParam.topMargin = (int) (1.0 * bg_screen_by * screenHeight / bg_height);
layoutParam.leftMargin = (int) (1.0 * bg_screen_bx * screenWidth / bg_width);
cameraDevice = Camera.open();
Log.i(LOG_TAG, "cameara open");
cameraView = new CameraView(this, cameraDevice);
topLayout.addView(cameraView, layoutParam);
topLayout.setOnTouchListener(this);
Log.i(LOG_TAG, "cameara preview start: OK");
}
//---------------------------------------
// initialize ffmpeg_recorder
//---------------------------------------
private void initRecorder() {
Log.w(LOG_TAG, "init recorder");
if (RECORD_LENGTH > 0) {
imagesIndex = 0;
images = new Frame[RECORD_LENGTH * frameRate];
timestamps = new long[images.length];
for (int i = 0; i < images.length; i++) {
images[i] = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
timestamps[i] = -1;
}
} else if (yuvImage == null) {
yuvImage = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
Log.i(LOG_TAG, "create yuvImage");
}
Log.i(LOG_TAG, "ffmpeg_url: " + ffmpeg_link);
recorder = new FFmpegFrameRecorder(ffmpeg_link, destWidth, imageHeight, 1);
recorder.setFormat("mp4");
recorder.setVideoCodecName("libx264");
recorder.setSampleRate(sampleAudioRateInHz);
// Set in the surface changed method
recorder.setFrameRate(frameRate);
Log.i(LOG_TAG, "recorder initialize success");
audioRecordRunnable = new AudioRecordRunnable();
audioThread = new Thread(audioRecordRunnable);
runAudioThread = true;
}
public void startRecording() {
initRecorder();
mHandler.removeCallbacks(mUpdateTimeTask);
mHandler.postDelayed(mUpdateTimeTask, 100);
try {
recorder.start();
startTime = System.currentTimeMillis();
recording = true;
audioThread.start();
} catch (FFmpegFrameRecorder.Exception e) {
e.printStackTrace();
}
}
public void stopRecording() {
runAudioThread = false;
try {
audioThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
audioRecordRunnable = null;
audioThread = null;
if (recorder != null && recording) {
if (RECORD_LENGTH > 0) {
Log.v(LOG_TAG, "Writing frames");
try {
int firstIndex = imagesIndex % samples.length;
int lastIndex = (imagesIndex - 1) % images.length;
if (imagesIndex <= images.length) {
firstIndex = 0;
lastIndex = imagesIndex - 1;
}
if ((startTime = timestamps[lastIndex] - RECORD_LENGTH * 1000000L) < 0) {
startTime = 0;
}
if (lastIndex < firstIndex) {
lastIndex += images.length;
}
int videoCounter = 0;
for (int i = firstIndex; i <= lastIndex; i++) {
if (timestamps[i] == -1) {
Log.v(LOG_TAG, "frame not recorded");
}
if (timestamps[i] != -1) {
long t = timestamps[i % timestamps.length] - startTime;
if (t >= 0) {
videoCounter++;
/*if (((i % images.length) != 0) && images[i % images.length] != images[(i % images.length) - 1]) {
if (t > recorder.getTimestamp()) {
recorder.setTimestamp(t);
}*/
Log.v(LOG_TAG, "imageIndex=" + (i % images.length));
recorder.record(images[i % images.length]);
/* }*/
Log.v(LOG_TAG, "videoCounter=" + videoCounter);
}
}
}
firstIndex = samplesIndex % samples.length;
lastIndex = (samplesIndex - 1) % samples.length;
if (samplesIndex <= samples.length) {
firstIndex = 0;
lastIndex = samplesIndex - 1;
}
if (lastIndex < firstIndex) {
lastIndex += samples.length;
}
for (int i = firstIndex; i <= lastIndex; i++) {
if (timestamps[i] != -1) {
recorder.recordSamples(samples[i % samples.length]);
}
}
} catch (FFmpegFrameRecorder.Exception e) {
Log.v(LOG_TAG, e.getMessage());
e.printStackTrace();
}
}
recording = false;
Log.v(LOG_TAG, "Finishing recording, calling stop and release on recorder");
try {
recorder.stop();
recorder.release();
} catch (FFmpegFrameRecorder.Exception e) {
e.printStackTrace();
}
recorder = null;
}
}
@Override
public boolean onKeyDown(int keyCode, KeyEvent event) {
if (keyCode == KeyEvent.KEYCODE_BACK) {
if (recording) {
stopRecording();
}
finish();
return true;
}
return super.onKeyDown(keyCode, event);
}
@Override
public boolean onTouch(View view, MotionEvent motionEvent) {
switch (motionEvent.getAction()) {
case MotionEvent.ACTION_DOWN:
Log.v(LOG_TAG, "ACTION_DOWN" + recording);
if (!recording) {
startRecording();
} else {
stopPauseTime = System.currentTimeMillis();
totalPauseTime = stopPauseTime - startPauseTime - ((long) (1.0 / (double) frameRate) * 1000);
pausedTime += totalPauseTime;
}
rec = true;
setTotalVideoTime();
btnRecorderControl.setText(getResources().getString(R.string.stop));
break;
case MotionEvent.ACTION_MOVE:
rec = true;
setTotalVideoTime();
break;
case MotionEvent.ACTION_UP:
Log.v(LOG_TAG, "ACTION_UP");
rec = false;
startPauseTime = System.currentTimeMillis();
break;
}
return true;
}
private Runnable mUpdateTimeTask = new Runnable() {
public void run() {
if (recording) {
setTotalVideoTime();
}
mHandler.postDelayed(this, 500);
}
};
private synchronized void setTotalVideoTime() {
totalTime = System.currentTimeMillis() - firstTime - pausedTime - ((long) (1.0 / (double) frameRate) * 1000);
if (totalTime > 0)
txtTimer.setText(getRecordingTimeFromMillis(totalTime));
}
private String getRecordingTimeFromMillis(long millis) {
String strRecordingTime = null;
int seconds = (int) (millis / 1000);
int minutes = seconds / 60;
int hours = minutes / 60;
if (hours >= 0 && hours < 10)
strRecordingTime = "0" + hours + ":";
else
strRecordingTime = hours + ":";
if (hours > 0)
minutes = minutes % 60;
if (minutes >= 0 && minutes < 10)
strRecordingTime += "0" + minutes + ":";
else
strRecordingTime += minutes + ":";
seconds = seconds % 60;
if (seconds >= 0 && seconds < 10)
strRecordingTime += "0" + seconds;
else
strRecordingTime += seconds;
return strRecordingTime;
}
//---------------------------------------------
// audio thread, gets and encodes audio data
//---------------------------------------------
class AudioRecordRunnable implements Runnable {
@Override
public void run() {
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO);
// Audio
int bufferSize;
ShortBuffer audioData;
int bufferReadResult;
bufferSize = AudioRecord.getMinBufferSize(sampleAudioRateInHz,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT);
audioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC, sampleAudioRateInHz,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize);
if (RECORD_LENGTH > 0) {
samplesIndex = 0;
samples = new ShortBuffer[RECORD_LENGTH * sampleAudioRateInHz * 2 / bufferSize + 1];
for (int i = 0; i < samples.length; i++) {
samples[i] = ShortBuffer.allocate(bufferSize);
}
} else {
audioData = ShortBuffer.allocate(bufferSize);
}
Log.d(LOG_TAG, "audioRecord.startRecording()");
audioRecord.startRecording();
/* ffmpeg_audio encoding loop */
while (runAudioThread) {
if (RECORD_LENGTH > 0) {
audioData = samples[samplesIndex++ % samples.length];
audioData.position(0).limit(0);
}
//Log.v(LOG_TAG,"recording? " + recording);
bufferReadResult = audioRecord.read(audioData.array(), 0, audioData.capacity());
audioData.limit(bufferReadResult);
if (bufferReadResult > 0) {
Log.v(LOG_TAG, "bufferReadResult: " + bufferReadResult);
// If "recording" isn't true when start this thread, it never get's set according to this if statement...!!!
// Why? Good question...
if (recording && rec) {
Log.v(LOG_TAG, "Recording audio");
if (RECORD_LENGTH <= 0) try {
recorder.recordSamples(audioData);
//Log.v(LOG_TAG,"recording " + 1024*i + " to " + 1024*i+1024);
} catch (FFmpegFrameRecorder.Exception e) {
Log.v(LOG_TAG, e.getMessage());
e.printStackTrace();
}
}
}
}
Log.v(LOG_TAG, "AudioThread Finished, release audioRecord");
/* encoding finish, release recorder */
if (audioRecord != null) {
audioRecord.stop();
audioRecord.release();
audioRecord = null;
Log.v(LOG_TAG, "audioRecord released");
}
}
}
//---------------------------------------------
// camera thread, gets and encodes video data
//---------------------------------------------
class CameraView extends SurfaceView implements SurfaceHolder.Callback, PreviewCallback {
private SurfaceHolder mHolder;
private Camera mCamera;
public CameraView(Context context, Camera camera) {
super(context);
Log.w("camera", "camera view");
mCamera = camera;
mHolder = getHolder();
mHolder.addCallback(CameraView.this);
mHolder.setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS);
mCamera.setPreviewCallback(CameraView.this);
}
@Override
public void surfaceCreated(SurfaceHolder holder) {
try {
stopPreview();
mCamera.setPreviewDisplay(holder);
} catch (IOException exception) {
mCamera.release();
mCamera = null;
}
}
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
stopPreview();
Camera.Parameters camParams = mCamera.getParameters();
List<Camera.Size> sizes = camParams.getSupportedPreviewSizes();
// Sort the list in ascending order
Collections.sort(sizes, new Comparator<Camera.Size>() {
public int compare(final Camera.Size a, final Camera.Size b) {
return a.width * a.height - b.width * b.height;
}
});
camParams.setPreviewSize(imageWidth, imageHeight);
Log.v(LOG_TAG, "Setting imageWidth: " + imageWidth + " imageHeight: " + imageHeight + " frameRate: " + frameRate);
camParams.setPreviewFrameRate(frameRate);
Log.v(LOG_TAG, "Preview Framerate: " + camParams.getPreviewFrameRate());
mCamera.setParameters(camParams);
List<Camera.Size> videoSizes = mCamera.getParameters().getSupportedVideoSizes();
// Set the holder (which might have changed) again
try {
mCamera.setPreviewDisplay(holder);
mCamera.setPreviewCallback(CameraView.this);
startPreview();
} catch (Exception e) {
Log.e(LOG_TAG, "Could not set preview display in surfaceChanged");
}
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
try {
mHolder.addCallback(null);
mCamera.setPreviewCallback(null);
} catch (RuntimeException e) {
// The camera has probably just been released, ignore.
}
}
public void startPreview() {
if (!isPreviewOn && mCamera != null) {
isPreviewOn = true;
mCamera.startPreview();
}
}
public void stopPreview() {
if (isPreviewOn && mCamera != null) {
isPreviewOn = false;
mCamera.stopPreview();
}
}
@Override
public void onPreviewFrame(byte[] data, Camera camera) {
if (audioRecord == null || audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
startTime = System.currentTimeMillis();
return;
}
if (RECORD_LENGTH > 0) {
int i = imagesIndex++ % images.length;
Log.v(LOG_TAG, "recording:" + recording + "rec:" + rec);
if (recording && rec) {
yuvImage = images[i];
timestamps[i] = 1000 * (System.currentTimeMillis() - startTime);
totalRecordedTS++;
} else {
Log.v(LOG_TAG, "recording is paused");
yuvImage = null;
timestamps[i] = -1;
}
}
/* get video data */
if (yuvImage != null && recording && rec) {
if (data.length != imageWidth * imageHeight) {
Camera.Size sz = camera.getParameters().getPreviewSize();
imageWidth = sz.width;
imageHeight = sz.height;
destWidth = imageHeight;
Log.v(LOG_TAG, "data length:" + data.length);
}
ByteBuffer bb = (ByteBuffer) yuvImage.image[0].position(0); // resets the buffer
int start = 2 * ((imageWidth - destWidth) / 4); // this must be even
for (int row = 0; row < imageHeight * 3 / 2; row++) {
bb.put(data, start, destWidth);
start += imageWidth;
}
}
}
}
@Override
public void onClick(View v) {
if (!recording) {
startRecording();
Log.w(LOG_TAG, "Start Button Pushed");
btnRecorderControl.setText("Stop");
} else {
// This will trigger the audio recording loop to stop and then set isRecorderStart = false;
stopRecording();
Log.w(LOG_TAG, "Stop Button Pushed");
btnRecorderControl.setText("Start");
}
}}
根据 Alex Cohn 的建议进行的更改
建议 1 - 估计平均帧率
public void stopRecording() {
..............................
if (((i % images.length) != 0) && images[i % images.length] != images[(i % images.length) - 1]) {
if (t > recorder.getTimestamp()) {
t += 1000000 / frameRate;
recorder.setTimestamp(t);
}
recorder.record(images[i % images.length]);
}
..........................................
}
所做的更改是添加 t += 1000000 / frameRate；但这会导致视频在手指离开屏幕时部分冻结（如上述案例 1）。
建议2-修改onPreviewFrame()
long[] timestampsForRecorder;
private void initRecorder() {
Log.w(LOG_TAG, "init recorder");
if (RECORD_LENGTH > 0) {
.......................................................
timestampsForRecorder = new long[images.length];
for (int i = 0; i < images.length; i++) {
images[i] = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
timestamps[i] = -1;
timestampsForRecorder[i] = -1;
}
} else if (yuvImage == null) {
yuvImage = new Frame(destWidth, imageHeight, Frame.DEPTH_UBYTE, 2);
Log.i(LOG_TAG, "create yuvImage");
}
...................................................
}
@Override
public void onPreviewFrame(byte[] data, Camera camera) {
if (audioRecord == null || audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
startTime = SystemClock.elapsedRealtime();
return;
}
if (RECORD_LENGTH > 0) {
int i = imagesIndex++ % images.length;
Log.v(LOG_TAG, "recording:" + recording + "rec:" + rec);
if (recording && rec) {
yuvImage = images[i];
long thisFrameTime = SystemClock.elapsedRealtime();
timestamps[i] = thisFrameTime;
long lastFrameTime = timestamps[(int) (imagesIndex == 0 ? startTime : ((imagesIndex-1) % images.length))];
Log.v(LOG_TAG, "lastFrameTime:" + lastFrameTime+",stopPauseTime:" + stopPauseTime);
if (lastFrameTime > stopPauseTime) {
timestampsForRecorder[i] = 1000 * (thisFrameTime - Math.max(stopPauseTime, lastFrameTime));
}
}
}
.....................................................
}
public void stopRecording() {
.......................................................
if (recorder != null && recording) {
if (RECORD_LENGTH > 0) {
Log.v(LOG_TAG, "Writing frames");
try {
int firstIndex = imagesIndex % samples.length;
int lastIndex = (imagesIndex - 1) % images.length;
if (imagesIndex <= images.length) {
firstIndex = 0;
lastIndex = imagesIndex - 1;
}
if ((startTime = timestampsForRecorder[lastIndex] - RECORD_LENGTH * 1000000L) < 0) {
startTime = 0;
}
if (lastIndex < firstIndex) {
lastIndex += images.length;
}
for (int i = firstIndex; i <= lastIndex; i++) {
if (timestampsForRecorder[i] != -1) {
long t = timestampsForRecorder[i % timestampsForRecorder.length] - startTime;
if (t >= 0) {
if (((i % images.length) != 0) && images[i % images.length] != images[(i % images.length) - 1]) {
if (t > recorder.getTimestamp()) {
recorder.setTimestamp(t);
}
Log.v(LOG_TAG, "imageIndex=" + (i % images.length));
recorder.record(images[i % images.length]);
}
}
}
}
.............................................
} catch (FFmpegFrameRecorder.Exception e) {
.................................
}
}
...........................................
}
}
用这个录制的视频出现了上述案例2的问题。即，它正在以更快的速度播放
简单（但不精确）的解决方案是估计平均帧速率，并使用 t += 1000000/average_fps; recorder.setTimestamp(t); 而不是查看实际时间戳。
为了更准确，您可以将onPreviewFrame()更改如下：
long thisFrameTime = SystemClock.elapsedRealtime();
timestamps[i] = thisFrameTime;
long lastFrameTime = timestamps[imagesIndex < 2 ? startTime : (imagesIndex-2) % images.length)];
if (lastFrameTime > stopPauseTime) {
timestampsForRecorder[i] = 1000 * (thisFrameTime - Math.max(stopPauseTime, lastFrameTime));
}
您可以将第二个数组 timestampsForRecorder 直接提供给记录器。
请注意，到处使用 SystemClock.elapsedRealtime() 更安全：
This clock is guaranteed to be monotonic, and continues to tick even when the CPU is in power saving modes, so is the recommend basis for general purpose interval timing.