Android: 运行无限循环中的图像分类器

Question

我想运行在后台线程的无限循环中使用图像分类器。该函数应在启动应用程序后立即调用。我想将预先录制的视频中的当前帧提供给分类器，该视频同时在 UI-thread 中播放，因此后台线程应该在完成后告诉 UI-thread，这样我就可以提供它与当前帧和 re运行分类器。

public class MainActivity extends AppCompatActivity {
    private static final String TAG = "MainActivity";

    private VideoView videoView;
    private ImageView imageView;
    private Uri uri_video;
    private MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
    private MediaController mMediaController;

    private static volatile int currentPosition;
    private static volatile Bitmap mBitmap;

    private final Object lock = new Object();
    private volatile boolean runClassifier = false;

    private HandlerThread backgroundThread;
    private Handler backgroundHandler;
    private static final String HANDLE_THREAD_NAME = "ClassifierBackground";


    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        imageView = findViewById(R.id.imageView);
        uri_video = Uri.parse("android.resource://" + getPackageName() + "/" + R.raw.kim);
        mediaMetadataRetriever = new MediaMetadataRetriever();
        mediaMetadataRetriever.setDataSource(getApplication(), uri_video);
        videoView = findViewById(R.id.videoView);
        videoView.setVideoURI(uri_video);
        mMediaController = new MediaController(this);
        videoView.setMediaController(mMediaController);
        videoView.setOnPreparedListener(MyVideoViewPreparedListener);
        videoView.start();

        startBackgroundThread();
    }


    /** Starts a background thread and its {@link Handler}. */
    private void startBackgroundThread() {
        backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
        backgroundThread.start();
        backgroundHandler = new Handler(backgroundThread.getLooper());
        synchronized (lock) {
            runClassifier = true;
        }
        backgroundHandler.post(periodicClassify);
    }

    /** Stops the background thread and its {@link Handler}. */
    private void stopBackgroundThread() {
        backgroundThread.quitSafely();
        try {
            backgroundThread.join();
            backgroundThread = null;
            backgroundHandler = null;
            synchronized (lock) {
                runClassifier = false;
            }
        } catch (InterruptedException e) {
            Log.e(TAG, "Interrupted when stopping background thread", e);
        }
    }

    private Runnable periodicClassify =
            new Runnable() {
                @Override
                public void run() {
                    synchronized (lock) {
                        if (runClassifier) {
                            //  classifyFrame(); // This will be implemented later
                            Log.d(TAG, "run: Classifier is running");
                            SystemClock.sleep(100); // Instead I simulate the classifier via sleep
                        }
                    }
                    runOnUiThread(new Runnable() {
                        @Override
                        public void run() {
                            setImageViewToCurrentFrame();
                        }
                    });
                    backgroundHandler.post(periodicClassify);
                }
            };


    private void setImageViewToCurrentFrame(){
        currentPosition = videoView.getCurrentPosition(); //in millisecond
        mBitmap = mediaMetadataRetriever
                .getFrameAtTime(currentPosition * 1000); //unit in microsecond
        imageView.setImageBitmap(mBitmap);
    }


    MediaPlayer.OnPreparedListener MyVideoViewPreparedListener =
            new MediaPlayer.OnPreparedListener() {

                @Override
                public void onPrepared(MediaPlayer mp) {

                    long duration = videoView.getDuration(); //in millisecond
                    Toast.makeText(MainActivity.this,
                            "Duration: " + duration + " (ms)",
                            Toast.LENGTH_LONG).show();
                    setImageViewToCurrentFrame();
                }
            };


    @Override
    protected void onDestroy() {
        super.onDestroy();
        stopBackgroundThread();
    }

编辑 1：
我从 these videos 那里得到了一些关于如何做的粗略想法。似乎我需要一个 backgroundThread (HandlerThread)，它有一个 backgroundHandler (Handler) 来与 UI-thread 通信，还需要一个 Looper 来保持后台线程处于活动状态。 setImageViewToCurrentFrame 使用 videoView.getCurrentPosition() 更新 mBitmap。

然而，与分类器的运行时间（SystemClock.sleep(100) 需要 100 毫秒）相比，更新速度非常慢（>10 秒）。

编辑2： 问题似乎出在 ImageView 的性能上，它似乎更新得很慢。用 TextView 替换它，使后台线程和 UI-thread 保持同步。我现在会寻找除 ImageView 以外的其他解决方案

Answer 1

这是我的解决方案的要点。需要在 SystemClock.sleep 部分适合实际的图像分类器。诀窍是使用 TextureView 而不是 ImageView 或 VideoView，因为它更快更灵活

@Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.video_playback);
    initializeBottomSheet();


    uri_video = Uri.parse("android.resource://" + getPackageName() + "/" + R.raw.my_demo_video);

    textureView = findViewById(R.id.textureView);
    textureView.setSurfaceTextureListener(this);

    mediaPlayer = new MediaPlayer();
    assert textureView != null;

    startBackgroundThread();
}




/** Starts a background thread and its {@link Handler}. */
private void startBackgroundThread() {
    backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
    backgroundThread.start();
    backgroundHandler = new Handler(backgroundThread.getLooper());
    synchronized (lock) {
        runClassifier = true;
    }
    backgroundHandler.post(periodicClassify);
}

/** Stops the background thread and its {@link Handler}. */
private void stopBackgroundThread() {
    backgroundThread.quitSafely();
    try {
        backgroundThread.join();
        backgroundThread = null;
        backgroundHandler = null;
        synchronized (lock) {
            runClassifier = false;
        }
    } catch (InterruptedException e) {
        Log.e(TAG, "Interrupted when stopping background thread", e);
    }
}

private Runnable periodicClassify =
        new Runnable() {
            @Override
            public void run() {
                // Get current frame from video playback
                mBitmap = textureView.getBitmap();
                if (classifier != null && mBitmap != null) {
                    Log.d(TAG, "Classifier: Start thread");
                    SystemClock.sleep(3000); // Instead I simulate the classifier via sleep
                }
                backgroundHandler.post(periodicClassify);
            }
        };


@Override
public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height) {
    Surface surface = new Surface(surfaceTexture);
    Context context = getApplicationContext();
    try {
        mediaPlayer.setDataSource(context, uri_video);
        mediaPlayer.setSurface(surface);
        mediaPlayer.prepareAsync();
        mediaPlayer.setOnPreparedListener(new MediaPlayer.OnPreparedListener(){

            @Override
            public void onPrepared(MediaPlayer mediaPlayer) {
                mediaPlayer.start();

                textureView.setOnClickListener(new View.OnClickListener() {
                    @Override
                    public void onClick(View view) {
                        if (mediaPlayer.isPlaying()){
                            mediaPlayer.pause();
                        }else{
                            mediaPlayer.start();
                        }
                    }
                });
            }
        });
    } catch (IOException e) {
        e.printStackTrace();
    }



}



@Override
protected void onDestroy() {
    super.onDestroy();
    stopBackgroundThread();
    if(mediaPlayer != null){
        mediaPlayer.stop();
        mediaPlayer.release();
        mediaPlayer = null;
    }
}

@Override
protected void onPause() {
    super.onPause();
    if(mediaPlayer != null && mediaPlayer.isPlaying()){
        mediaPlayer.pause();
    }
}

@Override
protected void onResume() {
    super.onResume();
    if(mediaPlayer != null && mediaPlayer.isPlaying()){
        mediaPlayer.start();
    }
    startBackgroundThread();
}

Android: 运行无限循环中的图像分类器

Android: Run image classifier in infinite loop

android

android-handler

Android: 运行 无限循环中的图像分类器

Android: Run image classifier in infinite loop

android

android-handler

Android: 运行无限循环中的图像分类器