如何重新映射 YUY2 图像?

How to Remap a YUY2 image?

我有一个 YUY2 数据格式的失真图像,YUY2 属于 YUV 4:2:2 家族(不是 4:2:0)。

我有 mapxmapy (height-720, width-1280),我从

cv::fisheye::initUndistortRectifyMap(K, D, cv::Mat::eye(3, 3, CV_64F), new_K, Size, CV_32FC1, mapx, mapy);

如何才能拥有不失真的YUY2? 我的最终目标是拥有不失真的 YUY2(不是 BGR)。

我想执行以下步骤:

cv::cvtColor(YUY, BGR, cv::COLOR_YUV2BGR_YUY2);
\ then perform remapping
\ and convert back to YUY

但是BGR2YUY_YUY2没有转换。

有没有更聪明的方法?

您可以将 YUV 4:2:2 转换为 YUV 4:4:4,不失真 4:4:4,然后再转换回 4:2:2。

转换阶段说明:
YUV422 -> YUV444 -> remap(YUV444) -> YUV422

我找不到用于从 YUV 4:2:2 转换为 YUV 4:4:4 的 OpenCV 函数。

通过简单的 for 循环实现转换非常简单:

//Convert YUYV to YUV (y,u,v,y,u,v,y,u,v...)
//The conversion is performed by duplicating each U and V element twice (equivalent to resize with nearest neighbor interpolation).
//The input type is CV_8UC1 (considered to be Grayscale image).
//The output type is CV_8UC3 (considered to be colored image with 3 channels).
static cv::Mat convertYuyv422toYuv444(const cv::Mat yuyv)
{
    int rows = yuyv.rows;
    int src_cols = yuyv.cols;
    size_t src_step = yuyv.step;
    const unsigned char *I = (unsigned char*)yuyv.data; //Pointer to source image.

    int dst_cols = src_cols / 2;
    cv::Mat yuv = cv::Mat(rows, dst_cols, CV_8UC3);
    size_t dst_step = yuv.step;
    unsigned char *J = (unsigned char*)yuv.data;    //Pointer to destination image.

    for (int y = 0; y < rows; y++)
    {
        const unsigned char *I0 = I + y*src_step;   //Points the beginning for the source row.
        unsigned char *J0 = J + y*dst_step;         //Points the beginning for the destination row.

        int srcx = 0;
        int dstx = 0;

        //yuyv -> yuvyuv
        //Convert 2 pixels per iteration
        for (int x = 0; x < src_cols / 2; x += 2)
        {
            unsigned char y0 = I0[srcx];
            unsigned char u0 = I0[srcx + 1];
            unsigned char y1 = I0[srcx + 2];
            unsigned char v0 = I0[srcx + 3];

            J0[dstx] = y0;
            J0[dstx + 1] = u0;
            J0[dstx + 2] = v0;
            J0[dstx + 3] = y1;
            J0[dstx + 4] = u0;  //Duplicate U
            J0[dstx + 5] = v0;  //Duplicate V

            srcx += 4;  //Source has 2 elements per pixel
            dstx += 6;  //Destination has 3 elements per pixel
        }
    }

    return yuv;
}

转换只是将每个 U 和 V 元素复制两次。
这不是最好的方法,但它被认为足够好。
复制 U 和 V 相当于使用最近邻插值调整大小。


要将 YUV 4:4:4 转换回 YUV 4:2:2,您可以使用以下 post:
中(我的)的代码示例 .


现有的优化库支持各种颜色格式转换。

libswscale 例如,但我认为这对您的需求来说有点矫枉过正...


测试:

为了测试,我使用了您之前的输入形式 post(以及我的回答):

因为我没有 YUYV 图像,所以我使用 FFmpeg(命令行)创建了一个:

ffmpeg -i input_image.jpg -codec rawvideo -pix_fmt yuyv422 input_image_yuyv.yuv

我使用 MATLAB 代码将原始 input_image_yuyv.yuv 转换为 PNG。

MATLAB 实现以两种方式将 4:2:2 转换为 4:4:4,并验证复制 U 和 V 等同于使用最近邻插值调整大小。

MATLAB 代码也用于验证 C++ 实现的正确性。

I = imread('input_image.jpg');
[rows, cols, ch] = size(I); % rows = 1280, cols = 720

% Read the YUYV to 2560x720 matrix from a binary file
f = fopen('input_image_yuyv.yuv', 'r');
YUYV = fread(f, [cols*2, rows], '*uint8')';
fclose(f);

% Write YUYV to PNG image - to be used as C++ input.
imwrite(YUYV, 'YUYV.png');

%figure;imshow(YUYV);title('YUYV');impixelinfo
Y = YUYV(:, 1:2:end); % 1280x720
U = YUYV(:, 2:4:end); % 640x720
V = YUYV(:, 4:4:end); % 640x720

% figure;imshow(Y);title('in Y');impixelinfo
% figure;imshow(U);title('in U');impixelinfo
% figure;imshow(V);title('in V');impixelinfo

% Convert U and V to 4:4:4 format using imresize with Nearest Neighbor interpolation method (used as reference).
refU2 = imresize(U, [rows, cols], 'nearest');
refV2 = imresize(V, [rows, cols], 'nearest');
% figure;imshow(U2);title('reference inU full');impixelinfo
% figure;imshow(V2);title('reference inV full');impixelinfo

% Resize x2 in the horizontal axis by simple duplication:
U2 = zeros(rows, cols, 'uint8');
U2(:, 1:2:end) = U;
U2(:, 2:2:end) = U;
V2 = zeros(rows, cols, 'uint8');
V2(:, 1:2:end) = V;
V2(:, 2:2:end) = V;

% Verify that the simple duplication is equivalent to resize with Nearest Neighbor interpolation:
% display(isequal(U2, refU2) && isequal(V2, refV2)) % Equal!!!

% Build YUV444 3840x720 matrix:
YUV444 = zeros(rows, cols*3, 'uint8');
YUV444(:, 1:3:end) = Y;
YUV444(:, 2:3:end) = U2;
YUV444(:, 3:3:end) = V2;
%figure;imshow(YUV444);title('YUV444');impixelinfo

% Write the YUV444 image to binary file (used as reference for C++ implementation)
f = fopen('image_yuv444.yuv', 'w');
fwrite(f, YUV444', 'uint8');
fclose(f);

imwrite(YUV444, 'matlabYUV444.png');


% Test output (after executing C++ code).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
c_YUV444 = imread('yuv444.png');
display(isequal(YUV444, c_YUV444));
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

以下代码是main()的C++实现:

int main()
{
    cv::Mat yuyv = cv::imread("YUYV.png", cv::IMREAD_GRAYSCALE);    //Read YUYV.png (created using MATLAB) as Grayscale

    cv::Mat yuv = convertYuyv422toYuv444(yuyv); //Convet yuyv to yuv (y,u,v,y,u,v...)

    //cv::imshow("yuyv", yuyv);
    cv::imwrite("yuv444.png", yuv); //Store YUV image for testing.

    //
    //remap the YUV 4:4:4
    ///////////////////////////////////////////////////////////////////////////////
    int W = 1280, H = 720;  //Assume resolution of Y plane is 1280x720
    cv::Mat mapx;
    cv::Mat mapy;
    cv::Mat dst_yuv;

    cv::Matx33d K = cv::Matx33d(541.2152931632737, 0.0, 661.7479652584254,
        0.0, 541.0606969363056, 317.4524205037745,
        0.0, 0.0, 1.0);
    cv::Vec4d D = cv::Vec4d(-0.042166406281296365, -0.001223961942208027, -0.0017036710622692108, 0.00023929900459453295);
    cv::Size newSize = cv::Size(3400, 1940);
    cv::Matx33d new_K;
    cv::fisheye::estimateNewCameraMatrixForUndistortRectify(K, D, cv::Size(W, H), cv::Mat::eye(3, 3, CV_64F), new_K, 1, newSize);    // W,H are the distorted image size
    cv::fisheye::initUndistortRectifyMap(K, D, cv::Mat::eye(3, 3, CV_64F), new_K, newSize, CV_16SC2, mapx, mapy);

    cv::remap(yuv, dst_yuv, mapx, mapy, cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0, 128, 128));
    ///////////////////////////////////////////////////////////////////////////////

    //Convert for BGR - just for dispaly
    cv::Mat dst_bgr;
    cv::cvtColor(dst_yuv, dst_bgr, cv::COLOR_YUV2BGR);

    cv::imshow("yuv", yuv);
    cv::imshow("dst_yuv", dst_yuv);
    cv::imshow("dst_bgr", dst_bgr);
    cv::waitKey(0);
    cv::destroyAllWindows();

    cv::imwrite("dst_bgr.png", dst_bgr);    //Store BGR image for testing.

    return 0;
}

注:
使用 remapcv::BORDER_CONSTANTcv::Scalar(0, 128, 128):

cv::remap(yuv, dst_yuv, mapx, mapy, cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0, 128, 128));

C++ 结果(转换为 BGR 后):

我尝试修改 mapx 和 mapy 使其适用于 YUV422。就计算时间而言,结果非常好。实时只需要一次重新映射。但是质量不是最好的。

然后我通过 libswscale 尝试了 YUV422 -> YUV444 -> remap(YUV444) -> YUV422,但是 YUV 转换再次需要时间。

终于开发了用于YUV转换的cuda内核。我附在下面。

//  nvcc -c -o colorConversion.o colorConversion.cu `pkg-config --libs --cflags opencv4`
//  /usr/bin/g++ -g -O3 /home/jai/vscode/opencvCUDA/cuda3.cpp -o /home/jai/vscode/opencvCUDA/cuda3 colorConversion.o `pkg-config --libs --cflags opencv4` `pkg-config --libs --cflags gstreamer-1.0` `pkg-config --libs --cflags cuda-11.3` `pkg-config --libs --cflags cudart-11.3`

#include "colorConversion.h"

__global__ void kernel_YUY422toYUY(cv::cuda::PtrStepSz<uchar2> YUV422, cv::cuda::PtrStepSz<uchar3> YUV)
{
    int i = blockIdx.y;         // row
    int j = blockDim.x * blockIdx.x + threadIdx.x; // col
    if (threadIdx.x & 1) {  // odd  1,3,5
      // YUV[i * step3 + 3 * j]     = YUV422[i * step2 + 2 * j];         // Y0
      // YUV[i * step3 + 3 * j + 1] = YUV422[i * step2 + 2 * j - 1]; // Y0
      // YUV[i * step3 + 3 * j + 2] = YUV422[i * step2 + 2 * j + 1]; // Y0
      YUV(i, j).x = YUV422(i, j).x;
      YUV(i, j).y = YUV422(i, j - 1).y;
      YUV(i, j).z = YUV422(i, j).y;
    } else {    // even 0,2,4,
      // YUV[i * step3 + 3 * j]     = YUV422[i * step2 + 2 * j];         // Y0
      //  YUV[i * step3 + 3 * j + 1] = YUV422[i * step2 + 2 * j + 1]; // U0
       // YUV[i * step3 + 3 * j + 2] = YUV422[i * step2 + 2 * j + 3]; // V0
       YUV(i, j).x = YUV422(i, j).x;
       YUV(i, j).y = YUV422(i, j).y;
       YUV(i, j).z = YUV422(i, j+1).y;
    }
}

void YUY422toYUY(const cv::cuda::GpuMat &YUV422gpu, cv::cuda::GpuMat &YUVgpu)
{
    kernel_YUY422toYUY<<<dim3(2, YUVgpu.rows), dim3(YUVgpu.cols / 2)>>>(YUV422gpu, YUVgpu);
    //cudaSafeCall(cudaGetLastError());
}

__global__ void kernel_YUYtoYUY422(cv::cuda::PtrStepSz<uchar3> YUV, cv::cuda::PtrStepSz<uchar2> YUV422)
{
    int i = blockIdx.x;         // row
    int j = threadIdx.x*2; // col

    YUV422(i, j).x = YUV(i, j).x;
    YUV422(i, j).y = (YUV(i, j).y + YUV(i, j+1).y)/2;

    YUV422(i, j+1).x = YUV(i, j+1).x;
    YUV422(i, j+1).y = (YUV(i, j).z + YUV(i, j+1).z)/2;
}

void YUYtoYUY422(const cv::cuda::GpuMat &YUVgpu, cv::cuda::GpuMat &YUV422gpu)
{
    kernel_YUYtoYUY422<<<dim3(YUV422gpu.rows), dim3(YUV422gpu.cols / 2)>>>(YUVgpu, YUV422gpu);
    //cudaSafeCall(cudaGetLastError());
}

然后我使用以下代码行再次使用 CUDA 进行重新映射:

YUV422GPU.upload(YUV422);  // YUV422 #channel = 2
YUV1.create(H, W, CV_8UC3);
YUV2.create(H, W, CV_8UC3);

YUY422toYUY(YUV422GPU, YUV1);
cv::cuda::remap(YUV1, YUV2, mapxGPU, mapyGPU, interpolationMethod); // YUV remap
YUYtoYUY422(YUV2, YUV422GPU);
YUV422GPU.download(dst);    // dst is the final YUV422. 2 channel image