Android CameraX Analyzer 格式为 YUV_420_888 的图像到 OpenCV Mat
Android CameraX Analyzer Image with format YUV_420_888 to OpenCV Mat
使用 Android CameraX Analyzer ImageProxy
在后台使用 ImageReader
默认 YUV_420_888
图像格式。
我想在 OpenCV 中转换它 Mat
以便在我的分析仪中使用 OpenCV:
override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
try {
imageProxy.image?.let {
// ImageProxy uses an ImageReader under the hood:
// https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
// That has a default format of YUV_420_888 if not changed that's the default
// Android camera format.
// https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
// https://developer.android.com/reference/android/media/ImageReader.html
// Sanity check
if (it.format == ImageFormat.YUV_420_888
&& it.planes.size == 3
) {
// TODO - convert ImageProxy.image to Mat
} else {
// Manage other image formats
// TODO - https://developer.android.com/reference/android/media/Image.html
}
}
} catch (ise: IllegalStateException) {
ise.printStackTrace()
}
}
我该怎么做?
查看 OpenCV JavaCamera2Frame class 在其 GitHub 存储库中,您可以编写一个 Image
扩展函数,如下所示:
(移植到 Kotlin)
// Ported from opencv private class JavaCamera2Frame
fun Image.yuvToRgba(): Mat {
val rgbaMat = Mat()
if (format == ImageFormat.YUV_420_888
&& planes.size == 3) {
val chromaPixelStride = planes[1].pixelStride
if (chromaPixelStride == 2) { // Chroma channels are interleaved
assert(planes[0].pixelStride == 1)
assert(planes[2].pixelStride == 2)
val yPlane = planes[0].buffer
val uvPlane1 = planes[1].buffer
val uvPlane2 = planes[2].buffer
val yMat = Mat(height, width, CvType.CV_8UC1, yPlane)
val uvMat1 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane1)
val uvMat2 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane2)
val addrDiff = uvMat2.dataAddr() - uvMat1.dataAddr()
if (addrDiff > 0) {
assert(addrDiff == 1L)
Imgproc.cvtColorTwoPlane(yMat, uvMat1, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV12)
} else {
assert(addrDiff == -1L)
Imgproc.cvtColorTwoPlane(yMat, uvMat2, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV21)
}
} else { // Chroma channels are not interleaved
val yuvBytes = ByteArray(width * (height + height / 2))
val yPlane = planes[0].buffer
val uPlane = planes[1].buffer
val vPlane = planes[2].buffer
yPlane.get(yuvBytes, 0, width * height)
val chromaRowStride = planes[1].rowStride
val chromaRowPadding = chromaRowStride - width / 2
var offset = width * height
if (chromaRowPadding == 0) {
// When the row stride of the chroma channels equals their width, we can copy
// the entire channels in one go
uPlane.get(yuvBytes, offset, width * height / 4)
offset += width * height / 4
vPlane.get(yuvBytes, offset, width * height / 4)
} else {
// When not equal, we need to copy the channels row by row
for (i in 0 until height / 2) {
uPlane.get(yuvBytes, offset, width / 2)
offset += width / 2
if (i < height / 2 - 1) {
uPlane.position(uPlane.position() + chromaRowPadding)
}
}
for (i in 0 until height / 2) {
vPlane.get(yuvBytes, offset, width / 2)
offset += width / 2
if (i < height / 2 - 1) {
vPlane.position(vPlane.position() + chromaRowPadding)
}
}
}
val yuvMat = Mat(height + height / 2, width, CvType.CV_8UC1)
yuvMat.put(0, 0, yuvBytes)
Imgproc.cvtColor(yuvMat, rgbaMat, Imgproc.COLOR_YUV2RGBA_I420, 4)
}
}
return rgbaMat
}
所以你可以这样写:
override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
try {
imageProxy.image?.let {
// ImageProxy uses an ImageReader under the hood:
// https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
// That has a default format of YUV_420_888 if not changed that's the default
// Android camera format.
// https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
// https://developer.android.com/reference/android/media/ImageReader.html
// Sanity check
if (it.format == ImageFormat.YUV_420_888
&& it.planes.size == 3
) {
val rgbaMat = it.yuvToRgba()
} else {
// Manage other image formats
// TODO - https://developer.android.com/reference/android/media/Image.html
}
}
} catch (ise: IllegalStateException) {
ise.printStackTrace()
}
}
@shadowsheep 解决方案就可以了,如果你需要得到 OpenCV 垫。
但是如果您想获取位图并且不想将 opencv 库添加到您的项目中,您可以查看 android/camera-samples repo
中的 RenderScript 解决方案
我还在 github 制作了单个 Java 文件库。如果您想获得正确的 ByteBuffer 而无需任何行或像素步幅以进行进一步处理(例如使用神经网络引擎),这将很有用。
我也compared所有这些方法。 OpenCV 是最快的。
private Mat convertYUVtoMat(@NonNull Image img) {
byte[] nv21;
ByteBuffer yBuffer = img.getPlanes()[0].getBuffer();
ByteBuffer uBuffer = img.getPlanes()[1].getBuffer();
ByteBuffer vBuffer = img.getPlanes()[2].getBuffer();
int ySize = yBuffer.remaining();
int uSize = uBuffer.remaining();
int vSize = vBuffer.remaining();
nv21 = new byte[ySize + uSize + vSize];
yBuffer.get(nv21, 0, ySize);
vBuffer.get(nv21, ySize, vSize);
uBuffer.get(nv21, ySize + vSize, uSize);
Mat yuv = new Mat(img.getHeight() + img.getHeight()/2, img.getWidth(), CvType.CV_8UC1);
yuv.put(0, 0, nv21);
Mat rgb = new Mat();
Imgproc.cvtColor(yuv, rgb, Imgproc.COLOR_YUV2RGB_NV21, 3);
Core.rotate(rgb, rgb, Core.ROTATE_90_CLOCKWISE);
return rgb;
}
此方法将 Camerax API YUV_420_888 图像转换为 OpenCV 的 Mat (RGB) 对象。
(2021 年工作)
使用 Android CameraX Analyzer ImageProxy
在后台使用 ImageReader
默认 YUV_420_888
图像格式。
我想在 OpenCV 中转换它 Mat
以便在我的分析仪中使用 OpenCV:
override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
try {
imageProxy.image?.let {
// ImageProxy uses an ImageReader under the hood:
// https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
// That has a default format of YUV_420_888 if not changed that's the default
// Android camera format.
// https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
// https://developer.android.com/reference/android/media/ImageReader.html
// Sanity check
if (it.format == ImageFormat.YUV_420_888
&& it.planes.size == 3
) {
// TODO - convert ImageProxy.image to Mat
} else {
// Manage other image formats
// TODO - https://developer.android.com/reference/android/media/Image.html
}
}
} catch (ise: IllegalStateException) {
ise.printStackTrace()
}
}
我该怎么做?
查看 OpenCV JavaCamera2Frame class 在其 GitHub 存储库中,您可以编写一个 Image
扩展函数,如下所示:
(移植到 Kotlin)
// Ported from opencv private class JavaCamera2Frame
fun Image.yuvToRgba(): Mat {
val rgbaMat = Mat()
if (format == ImageFormat.YUV_420_888
&& planes.size == 3) {
val chromaPixelStride = planes[1].pixelStride
if (chromaPixelStride == 2) { // Chroma channels are interleaved
assert(planes[0].pixelStride == 1)
assert(planes[2].pixelStride == 2)
val yPlane = planes[0].buffer
val uvPlane1 = planes[1].buffer
val uvPlane2 = planes[2].buffer
val yMat = Mat(height, width, CvType.CV_8UC1, yPlane)
val uvMat1 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane1)
val uvMat2 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane2)
val addrDiff = uvMat2.dataAddr() - uvMat1.dataAddr()
if (addrDiff > 0) {
assert(addrDiff == 1L)
Imgproc.cvtColorTwoPlane(yMat, uvMat1, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV12)
} else {
assert(addrDiff == -1L)
Imgproc.cvtColorTwoPlane(yMat, uvMat2, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV21)
}
} else { // Chroma channels are not interleaved
val yuvBytes = ByteArray(width * (height + height / 2))
val yPlane = planes[0].buffer
val uPlane = planes[1].buffer
val vPlane = planes[2].buffer
yPlane.get(yuvBytes, 0, width * height)
val chromaRowStride = planes[1].rowStride
val chromaRowPadding = chromaRowStride - width / 2
var offset = width * height
if (chromaRowPadding == 0) {
// When the row stride of the chroma channels equals their width, we can copy
// the entire channels in one go
uPlane.get(yuvBytes, offset, width * height / 4)
offset += width * height / 4
vPlane.get(yuvBytes, offset, width * height / 4)
} else {
// When not equal, we need to copy the channels row by row
for (i in 0 until height / 2) {
uPlane.get(yuvBytes, offset, width / 2)
offset += width / 2
if (i < height / 2 - 1) {
uPlane.position(uPlane.position() + chromaRowPadding)
}
}
for (i in 0 until height / 2) {
vPlane.get(yuvBytes, offset, width / 2)
offset += width / 2
if (i < height / 2 - 1) {
vPlane.position(vPlane.position() + chromaRowPadding)
}
}
}
val yuvMat = Mat(height + height / 2, width, CvType.CV_8UC1)
yuvMat.put(0, 0, yuvBytes)
Imgproc.cvtColor(yuvMat, rgbaMat, Imgproc.COLOR_YUV2RGBA_I420, 4)
}
}
return rgbaMat
}
所以你可以这样写:
override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
try {
imageProxy.image?.let {
// ImageProxy uses an ImageReader under the hood:
// https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
// That has a default format of YUV_420_888 if not changed that's the default
// Android camera format.
// https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
// https://developer.android.com/reference/android/media/ImageReader.html
// Sanity check
if (it.format == ImageFormat.YUV_420_888
&& it.planes.size == 3
) {
val rgbaMat = it.yuvToRgba()
} else {
// Manage other image formats
// TODO - https://developer.android.com/reference/android/media/Image.html
}
}
} catch (ise: IllegalStateException) {
ise.printStackTrace()
}
}
@shadowsheep 解决方案就可以了,如果你需要得到 OpenCV 垫。
但是如果您想获取位图并且不想将 opencv 库添加到您的项目中,您可以查看 android/camera-samples repo
中的 RenderScript 解决方案我还在 github 制作了单个 Java 文件库。如果您想获得正确的 ByteBuffer 而无需任何行或像素步幅以进行进一步处理(例如使用神经网络引擎),这将很有用。
我也compared所有这些方法。 OpenCV 是最快的。
private Mat convertYUVtoMat(@NonNull Image img) {
byte[] nv21;
ByteBuffer yBuffer = img.getPlanes()[0].getBuffer();
ByteBuffer uBuffer = img.getPlanes()[1].getBuffer();
ByteBuffer vBuffer = img.getPlanes()[2].getBuffer();
int ySize = yBuffer.remaining();
int uSize = uBuffer.remaining();
int vSize = vBuffer.remaining();
nv21 = new byte[ySize + uSize + vSize];
yBuffer.get(nv21, 0, ySize);
vBuffer.get(nv21, ySize, vSize);
uBuffer.get(nv21, ySize + vSize, uSize);
Mat yuv = new Mat(img.getHeight() + img.getHeight()/2, img.getWidth(), CvType.CV_8UC1);
yuv.put(0, 0, nv21);
Mat rgb = new Mat();
Imgproc.cvtColor(yuv, rgb, Imgproc.COLOR_YUV2RGB_NV21, 3);
Core.rotate(rgb, rgb, Core.ROTATE_90_CLOCKWISE);
return rgb;
}
此方法将 Camerax API YUV_420_888 图像转换为 OpenCV 的 Mat (RGB) 对象。 (2021 年工作)