如何在 openCV/C++ 中将 png 文件中的单词转换为字符串？

Question

所以我的项目是创建一个程序，根据用户点击图像的位置检测拼字板上的单词。我的问题如下：

如何将检测到的字母存入strings/words？（即依次检查每个轮廓，并根据它是否接近另一个轮廓然后将其添加为字符串的下一个元素）

程序根据力矩和轮廓区域绘制每个字母周围的轮廓。当前代码中的任何更正将不胜感激。它有效，但显然它可以写得更好。以下是获得的图片：

二进制：

颜色：

#include <opencv2\core.hpp> //cv::Mat etc
#include <opencv2\imgproc.hpp> // algorithms 
#include <opencv2\highgui.hpp> // input, output 
#include <iostream>
#include <fstream>

using namespace std;
using namespace cv;

Mat image, labimg, perspective, gaussian, binary, gray, erodedimg, dilatedimg, newmat, newmat2, transformed, transformed2;
Mat_<Point> capturePoint; // point coordinates, global variable;

vector< vector <Point> > contours; // Vector for storing contour
vector< Vec4i > hierarchy;
int largest_contour_index = 0;
double largest_area = 0;

void on_mouse(int e, int x, int y, int d, void *ptr)
{
    if (e == EVENT_LBUTTONDOWN)
    {
        Point*p = (Point*)ptr;
        p->x = x;
        p->y = y;

        //Point center = *p;
        cout << *p << endl;

        //destroyWindow("My Window");
    }
}

void myperspective()
{
    Mat dst = image.clone(); //(image.rows, image.cols, CV_8UC1, Scalar::all(0)); //create destination image
    findContours(binary.clone(), contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE, Point(0,0)); // Find the contours in the image
    for (int i = 0; i < contours.size(); i++) 
    {
        double a = contourArea(contours[i], false);  //  Find the area of contour
        if (a > largest_area) 
        {
            largest_area = a;
            largest_contour_index = i;                //Store the index of largest contour
        }
    }

    drawContours(dst, contours, largest_contour_index, Scalar(0, 255, 0), 2, 8, hierarchy);
    //imshow("Display window", dst);
    //waitKey(0);

    vector<vector<Point> > contours_poly(1);
    approxPolyDP(Mat(contours[largest_contour_index]), contours_poly[0], 10, true);
    Point2f output_points[4];
    Point2f input_points[4];

    cout << contours_poly[0].size();

    int min_x = image.cols;
    int min_y = image.rows;
    int max_x = 0;
    int max_y = 0;
    int myxmin = image.cols;
    int myxmin2 = image.cols;
    int myxmax = 0;
    int myymin = image.rows;

    for (int k = 0; k < contours_poly[0].size(); k++)
    {
        if (contours_poly[0][k].x < min_x && contours_poly[0][k].y < min_y)
        {
            min_x = contours_poly[0][k].x;
            min_y = contours_poly[0][k].y;
        }
        else if (contours_poly[0][k].x <= min_x && contours_poly[0][k].y > max_y)
        {
            //min_x = contours_poly[0][k].x;
            max_y = contours_poly[0][k].y;
            myxmin2 = contours_poly[0][k].x;
        }
        else if (contours_poly[0][k].x > max_x && contours_poly[0][k].y > max_y)
        {
            max_x = contours_poly[0][k].x;
            //max_y = contours_poly[0][k].y;
        }
        else if (contours_poly[0][k].x < max_x && contours_poly[0][k].y <= (min_y+20) && contours_poly[0][k].x > myxmax)
        {
            myxmax = contours_poly[0][k].x;
        }
        else if (contours_poly[0][k].x > min_x && contours_poly[0][k].y <= min_y && contours_poly[0][k].x < myxmin)
        {
            myxmin = contours_poly[0][k].x;
            myymin = contours_poly[0][k].y;
        }
        else
        {
            cout << "NA" << endl;
        }
    }

    input_points[0] = Point(myxmin, myymin);
    input_points[1] = Point(myxmax, min_y);
    input_points[2] = Point(myxmin2, max_y);
    input_points[3] = Point(max_x-30, max_y+5);

    output_points[0] = Point(0, 0);
    output_points[1] = Point(image.cols, 0);
    output_points[2] = Point(0, image.rows);
    output_points[3] = Point(image.cols, image.rows);

    Mat transmtx = getPerspectiveTransform(input_points, output_points);
    transformed = Mat::zeros(image.rows, image.cols, CV_8UC3);
    transformed2 = Mat::zeros(image.rows, image.cols, CV_8UC3);
    warpPerspective(binary, transformed, transmtx, image.size());
    warpPerspective(image, transformed2, transmtx, image.size());
}

int main(int argc, char** argv)
{
    image = cv::imread("ideal.png"); // Read image from file
    namedWindow("Display window", WINDOW_NORMAL);

    cv::GaussianBlur(image, gaussian, Size(3, 3), 0);
    cvtColor(gaussian, gray, CV_BGR2GRAY);

    cv::adaptiveThreshold(gray, binary, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY_INV, 7, 2);
    //createTrackbar("Threshold:", "Display window2", &thresh_elem, max_thresh_size, Threshold);
    //createTrackbar("Kernel:", "Display window2", &kernel_size, max_kernel_size, Threshold);
    //imshow("Display window", binary);
    //waitKey(0);

    myperspective();

    vector<vector<Point>>lettercontours;
    vector<Vec4i> hierarchyoflet;

    findContours(transformed, lettercontours, hierarchyoflet, CV_RETR_LIST, CV_CHAIN_APPROX_TC89_L1);
    int idx = 0;
    vector<Point>contour;

    //double table[100][7];
    fstream humom("humombase.txt", ios::out);

    double hu[7];

    for (int i = 0; idx >= 0; idx = hierarchyoflet[idx][0], i++)
    {
        double area = contourArea(lettercontours[i]);
        Moments lettermoments = moments(lettercontours[i]);
        HuMoments(lettermoments, hu);
        humom << hu[0] << " " << hu[1] << " " << hu[2] << " " << hu[3] << " " << hu[4] << " " << hu[5] << " " << hu[6] << "\n";
        if (area > 500 && area < 1800 && hu[0] > 0.16292 && hu[0] < 0.18292 && hu[1]> 0.000534 && hu[1] < 0.000665 && hu[2]> 0.000162 && hu[2] < 0.000339)
        { 
            cout << "D"; 
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.257 && hu[0] < 0.283 && hu[1]> 0.0423 && hu[1] < 0.0473 && hu[2]> 1.044e-05 && hu[2] < 8.475e-05)
        {
            cout << "I";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
         else if (area > 500 && area < 1800 && hu[0] > 0.243 && hu[0] < 0.263 && hu[1]> 0.000837 && hu[1] < 0.000857 && hu[2]> 0.000136 && hu[2] < 0.000156)
        {
            cout << "S";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.287 && hu[0] < 0.3 && hu[1]> 0.009 && hu[1] < 0.015 && hu[2]> 0.019 && hu[2] < 0.021)
        {
            cout << "T";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.28 && hu[0] < 0.3 && hu[1]> 0.002 && hu[1] < 0.003 && hu[2]> 0.0005 && hu[2] < 0.0007)
        {
            cout << "U";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.191 && hu[0] < 0.197 && hu[1]> 0.0001 && hu[1] < 0.005 && hu[2]> 4.39e-05 && hu[2] < 0.0003)
        {
            cout << "R";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.156 && hu[0] < 0.176 && hu[1]> 4.606e-05 && hu[1] < 4.806e-05 && hu[2]> 4.89e-05 && hu[2] < 5e-05)
        {
            cout << "B";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.24 && hu[0] < 0.28 && hu[1]> 0.001 && hu[1] < 0.004 && hu[2]> 0 && hu[2] < 0.0002)
        {
            cout << "E";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.15 && hu[0] < 0.25 && hu[1]> 0.0004 && hu[1] < 0.0006 && hu[2]> 0.004 && hu[2] < 0.006)
        {
            cout << "A";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.33 && hu[0] < 0.35 && hu[1]> 0.02 && hu[1] < 0.04 && hu[2]> 0.008 && hu[2] < 0.01)
        {
            cout << "M";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.29 && hu[0] < 0.32 && hu[1]> 0.027 && hu[1] < 0.034 && hu[2]> 0.009 && hu[2] < 0.015)
        {
            cout << "L";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.23 && hu[0] < 0.25 && hu[1]> 0.0009 && hu[1] < 0.0015 && hu[2]> 0.0005 && hu[2] < 0.0007)
        {
            cout << "K";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.27 && hu[0] < 0.29 && hu[1]> 0.0002 && hu[1] < 0.0003 && hu[2]> 0.019 && hu[2] < 0.021)
        {
            cout << "Y";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.15 && hu[0] < 0.17 && hu[1]> 0.001 && hu[1] < 0.002 && hu[2]> 2.2e-07 && hu[2] < 2.5e-07)
        {
            cout << "O";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.28 && hu[0] < 0.30 && hu[1]> 0.022 && hu[1] < 0.024 && hu[2]> 0.002 && hu[2] < 0.0025)
        {
            cout << "W";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else
        {
            drawContours(transformed, lettercontours, idx, Scalar(0,0,0), CV_FILLED, 8, hierarchyoflet);
        }
    }

    Mat element = getStructuringElement(MORPH_CROSS, Size(3, 3), Point(1, 1));
    for (int o = 0; o < 1; o++)
    {
        erode(transformed, erodedimg, element);
        dilate(erodedimg, dilatedimg, element);
        transformed = dilatedimg;
    }

    imshow("Display window", transformed);
    waitKey(0);
    imshow("Display window", transformed2);

    namedWindow("Display window2", WINDOW_NORMAL);
    Point p;
    setMouseCallback("Display window2", on_mouse, &p); //set the callback function for any mouse event
    imshow("Display window2", transformed2);
    waitKey(0); //wait till any key is pressed 
    return 0;
}

Answer 1

我将从检测网格开始。

找到 4 个完整的角单元格

所以让 (x1,y1)...(x4,y4) 成为找到的完整边形成矩形的中心点。 nx=12 , ny=12 是单元格中这些点之间的距离。假设 (x1,y1) 有网格位置 (1,1) 和 (x2,y2) (1+nx,1) 所以：
```
screen(x,y) grid(i,j)
x1,y1        1, 1
x2,y2       13, 1
x3,y3        1,13
x4,y4       13,13
```
interpolate/fit网格线

所以对于单元格 (i,j) 到屏幕 (x,y) 的转换，我们可以这样写：
```
a=x1+(x3-x1)*(j-j1); // x position of (i1,j)
b=x2+(x4-x2)*(j-j2); // x position of (i2,j)
x=a +(b - a)*(i-i1)/(i2-i1) // x position of (i,j)
a=y1+(y2-y1)*(i-i1); // y position of (i,j1)
b=y3+(y4-y3)*(i-i3); // y position of (i,j3)
y=a +(b - a)*(j-j1)/(j3-j1) // x position of (i,j)
```
希望我没有犯任何错误（因为我是直接在 SO 编辑器中得出的）...现在您知道每个单元格的中心位置。单元格 (i,j) 的角是使用 (i+/-0.5,j+/-0.5) 坐标通过此转换计算的...

这是简单的bi-linear插值，如果失真太大，您也可以使用bi-cubic或任何其他插值。

如果您没有完整的单元格形成矩形的角，您可以使用 4 条线覆盖尽可能大的区域。等式会有点变化，但它是可行的。
创建在地图和屏幕位置之间转换的函数

注意空单元格与在地面板上偏移的填充单元格不同。如果您还需要反向转换 (x,y)->(i,j) 那么您可以按照与 #2 相同的方式进行，只需交换 i,j 和 x,y ...

您可以使用它仅在所需的单元格而不是整个图像中查找轮廓，从而消除轮廓分组的问题。

您还可以剪切每个单元格的右下部分，以忽略轮廓中的数字分数。我不使用 OpenCV 所以如果你在将它包含到 OpenCV 代码中时遇到问题你总是可以生成 ROI 从这个应该很容易在 OpenCV 中使用到 select 任何操作的特定区域的映射。

对于粗略的字符检测，您需要某种 OCR，它不是 OpenCV 的一部分（至少据我所知).很少有像 Tesseract 这样的开源 OCR。您也可以尝试更简单的方法，例如相关 QA:

OCR and character similarity

如何在 openCV/C++ 中将 png 文件中的单词转换为字符串？

How to convert words from png file to string in openCV/C++?

c++

opencv

image-processing

image-recognition

contour