使用opencv裁剪汽车文档中的方框

Question

我是 opencv 的新手，想用它裁剪图像的一部分，然后使用 tesseract 读取它们。我不确定裁剪我需要的所有必要框的最佳方法是什么。

这是我需要转换的文档的简单示例：

有什么建议最好吗？

我尝试使用 ORB 和以下图像作为模板：

但是没有成功。

在模板上，一些线条被选为关键点，但在图像上我要处理的主要是文本而不是线条。这是一个糟糕的模板吗？我需要先处理图像吗？

和我的代码：

Feature2D f2d = ORB.create(5000); // SIFT.create(1000);

MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
Mat descriptors1 = new Mat();
Mat mask1 = new Mat();
f2d.detectAndCompute(img1, mask1, keypoints1, descriptors1);

MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
Mat descriptors2 = new Mat();
Mat mask2 = new Mat();
f2d.detectAndCompute(img2, mask2, keypoints2, descriptors2);

DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING); 
MatOfDMatch matches = new MatOfDMatch();
matcher.match(descriptors1, descriptors2, matches);


Mat outputImg = new Mat();
MatOfByte drawnMatches = new MatOfByte();
Features2d.drawMatches(img1, keypoints1, img2, keypoints2, matches, outputImg, new Scalar(0, 255, 0), new Scalar(255, 0, 0), drawnMatches, Features2d.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS);

Answer 1

我可以通过使用一个包含表单中所有不变文本的模板来获得好的结果。此外，创建 2 个模板（每页 1 个）并使用 SIFT 代替 ORB 也有很大帮助。

这是我的解决方案：

public static Mat matchTEmplateSIFT(Mat img1, Mat template, boolean showKeypoints, boolean drawMatchs) {
  Feature2D f2d = SIFT.create(15000);
  DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_SL2); // or FLANNBASED for better performance
  return matchTEmplate(img1, template, f2d, matcher);
}

public static Mat matchTEmplate(Mat baseImage, Mat template, Feature2D f2d, DescriptorMatcher matcher) {
  int dilateSize = 5;
  Mat scene = dilateBitwise(dilateSize, baseImage.clone());
  template = dilateBitwise(dilateSize, template.clone());

  MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
  Mat descriptors1 = new Mat();
  f2d.detectAndCompute(scene, new Mat(), keypoints1, descriptors1);

  MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
  Mat descriptors2 = new Mat();
  f2d.detectAndCompute(template, new Mat(), keypoints2, descriptors2);

  List<MatOfDMatch> matches = new ArrayList<>();
  matcher.knnMatch(descriptors1, descriptors2, matches, 2);

  MatOfDMatch goodMatches = getBestMatches(matches);
  Mat result = transformAndWarp(baseImage, template, keypoints1, keypoints2, goodMatches);

  return result;
}

private static Mat transformAndWarp(Mat baseImage, Mat template, MatOfKeyPoint keypoints1, MatOfKeyPoint keypoints2, MatOfDMatch goodMatches) {
  Mat H = findHomographyMatrix(keypoints1, keypoints2, goodMatches);
  perspectiveTransform(template, H);
  Mat result = new Mat();
  Imgproc.warpPerspective(baseImage, result, H, new Size(template.cols(), template.rows()));
  return result;
}

private static void perspectiveTransform(Mat template, Mat H) {
  Mat obj_corners = new Mat(4, 1, CvType.CV_32FC2);
  obj_corners.put(0, 0, new double[]{0, 0});
  obj_corners.put(0, 0, new double[]{template.cols(), 0});
  obj_corners.put(0, 0, new double[]{template.cols(), template.rows()});
  obj_corners.put(0, 0, new double[]{0, template.rows()});

  Mat scene_corners = new Mat(4, 1, CvType.CV_32FC2);

  Core.perspectiveTransform(obj_corners, scene_corners, H);
}

private static Mat findHomographyMatrix(MatOfKeyPoint keypoints1, MatOfKeyPoint keypoints2, MatOfDMatch goodMatches) {
  LinkedList<Point> templateList = new LinkedList<>();
  LinkedList<Point> sceneList = new LinkedList<>();

  List<KeyPoint> templateKeyPoints = keypoints1.toList();
  List<KeyPoint> sceneKeypoints = keypoints2.toList();

  for (int i = 0; i < goodMatches.toList().size(); i++) {
     templateList.addLast(templateKeyPoints.get(goodMatches.toList().get(i).queryIdx).pt);
     sceneList.addLast(sceneKeypoints.get(goodMatches.toList().get(i).trainIdx).pt);
  }

  MatOfPoint2f templateMat = new MatOfPoint2f(); 
  templateMat.fromList(templateList);

  MatOfPoint2f sceneMat = new MatOfPoint2f();
  sceneMat.fromList(sceneList);

  return Calib3d.findHomography(templateMat, sceneMat, Calib3d.RANSAC);
}

// https://docs.opencv.org/3.4/d5/d6f/tutorial_feature_flann_matcher.html
private static MatOfDMatch getBestMatches(List<MatOfDMatch> knnMatches) {
  //-- Filter matches using the Lowe's ratio test
  float ratioThresh = 0.5f;
  List<DMatch> listOfGoodMatches = new ArrayList<>();
  for (int i = 0; i < knnMatches.size(); i++) {
     if (knnMatches.get(i).rows() > 1) {
        DMatch[] matches = knnMatches.get(i).toArray();
        if (matches[0].distance < ratioThresh * matches[1].distance) {
           listOfGoodMatches.add(matches[0]);
        }
     }
  }
  MatOfDMatch matOfDMatch = new MatOfDMatch();
  matOfDMatch.fromList(listOfGoodMatches);
  return matOfDMatch;

}

使用opencv裁剪汽车文档中的方框

Use opencv to crop boxes in car documents

opencv

tesseract

image-processing