opencv 中的 StereoBM class 是否对输入图像或帧进行校正？
Does the StereoBM class in opencv do rectification of the input images or frames?
作为我项目的一部分，我正在使用 SteroBM class 进行立体视觉。我正在使用来自 2 个网络摄像头的输入帧和 运行 对输入帧灰度帧进行立体块匹配计算而不进行校正。我得到的输出与基本事实相去甚远（非常不完整）。我想知道，是不是因为我没有对输入帧进行整改。此外，我选择的基线保持在 20 厘米。我使用的是opencv-3.2.0版本的c++。
我是运行的代码如下。
#include <opencv2/core.hpp>
#include <opencv2/opencv.hpp>
#include </home/eswar/softwares/opencv_contrib-3.2.0/modules/contrib_world/include/opencv2/contrib_world.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgproc.hpp>
#include <stdio.h>
#include <iostream>
using namespace std;
using namespace cv;
int main()
{
//initialize and allocate memory to load the video stream from camera
VideoCapture camera0(0);
VideoCapture camera1(1);
if( !camera0.isOpened() ) return 1;
if( !camera1.isOpened() ) return 1;
Mat frame0,frame1;
Mat frame0gray,frame1gray;
Mat dispbm,dispsgbm;
Mat dispnorm_bm,dispnorm_sgbm;
Mat falseColorsMap, sfalseColorsMap;
int ndisparities = 16*5; /**< Range of disparity */
int SADWindowSize = 21; /**< Size of the block window. Must be odd */
Ptr<StereoBM> sbm = StereoBM::create( ndisparities, SADWindowSize );
Ptr<StereoSGBM> sgbm = StereoSGBM::create(0, //int minDisparity
96, //int numDisparities
5, //int SADWindowSize
600, //int P1 = 0
2400, //int P2 = 0
10, //int disp12MaxDiff = 0
16, //int preFilterCap = 0
2, //int uniquenessRatio = 0
20, //int speckleWindowSize = 0
30, //int speckleRange = 0
true); //bool fullDP = false
//-- Check its extreme values
double minVal; double maxVal;
while(true)
{
//grab and retrieve each frames of the video sequentially
camera0 >> frame0;
camera1 >> frame1;
imshow("Video0", frame0);
imshow("Video1", frame1);
cvtColor(frame0,frame0gray,CV_BGR2GRAY);
cvtColor(frame1,frame1gray,CV_BGR2GRAY);
sbm->compute( frame0gray, frame1gray, dispbm );
minMaxLoc( dispbm, &minVal, &maxVal );
dispbm.convertTo( dispnorm_bm, CV_8UC1, 255/(maxVal - minVal));
sgbm->compute(frame0gray, frame1gray, dispsgbm);
minMaxLoc( dispsgbm, &minVal, &maxVal );
dispsgbm.convertTo( dispnorm_sgbm, CV_8UC1, 255/(maxVal - minVal));
imshow( "BM", dispnorm_bm);
imshow( "SGBM",dispnorm_sgbm);
//wait for 40 milliseconds
int c = cvWaitKey(40);
//exit the loop if user press "Esc" key (ASCII value of "Esc" is 27)
if(27 == char(c)) break;
}
return 0;
}
尽管在代码中您看到也使用了块匹配，但请忽略它，因为它给出的输出更差。我发现 SGBM 输出更接近真实情况，因此我决定对其进行改进。但是，如果对如何改进块匹配结果有任何帮助。太棒了，我当然会很感激。
SGBM 技术的输出图像深度图像看起来像。
不，StereoBM不做校正，只是块匹配和一些预处理和post处理，但是opencv提供相机校准和校正检查的功能this link
在 opencv 示例中也有针对此过程的现成示例，因此不必从头开始编写代码。
关于结果，StereoBM是基于SAD算法（local stereo-matching），鲁棒性不强，你可以试试wls filter，可以显着提高你的结果。
StereoSGBM是基于SGM算法（实际上与原论文介绍的略有不同）的半全局算法，在视差图生成中考虑全局优化，产生的视差更好但速度较慢。
如上所示，我尝试对帧进行校正。代码如下。
#include <opencv2/core.hpp>
#include <opencv2/opencv.hpp>
#include </home/eswar/softwares/opencv_contrib-3.2.0/modules/contrib_world /include/opencv2/contrib_world.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgproc.hpp>
#include <stdio.h>
#include <iostream>
#include <opencv2/xfeatures2d/nonfree.hpp>
using namespace std;
using namespace cv;
using namespace cv::xfeatures2d;
int main()
{
//initialize and allocate memory to load the video stream from camera
VideoCapture camera0(0);
VideoCapture camera1(1);
int count=0;
Mat loRes, hiRes;
if( !camera0.isOpened() ) return 1;
if( !camera1.isOpened() ) return 1;
camera0.set(CV_CAP_PROP_FRAME_WIDTH, 400);
camera0.set(CV_CAP_PROP_FRAME_HEIGHT, 400);
camera1.set(CV_CAP_PROP_FRAME_WIDTH, 400);
camera1.set(CV_CAP_PROP_FRAME_HEIGHT, 400);
Mat frame0,frame1;
Mat frame0gray,frame1gray;
Mat dispbm,dispsgbm,disparity,disparity1;
Mat dispnorm_bm,dispnorm_sgbm;
Mat falseColorsMap, sfalseColorsMap,falsemap;
Mat img_matches;
Mat H1,H2;
int ndisparities = 96; /**< Range of disparity */
int SADWindowSize = 7;
Ptr<StereoBM> sbm = StereoBM::create( ndisparities, SADWindowSize );
Ptr<StereoSGBM> sgbm = StereoSGBM::create(-3, //int minDisparity
96, //int numDisparities
7, //int SADWindowSize
60, //int P1 = 0
2400, //int P2 = 0
90, //int disp12MaxDiff = 0
16, //int preFilterCap = 0
1, //int uniquenessRatio = 0
60, //int speckleWindowSize = 0
20, //int speckleRange = 0
true); //bool fullDP = false
//-- Check its extreme values
double minVal; double maxVal;
double max_dist = 0;
double min_dist = 100;
int minHessian = 630;
Ptr<Feature2D> f2d = SIFT::create();
vector<KeyPoint> keypoints_1, keypoints_2;
Ptr<Feature2D> fd = SIFT::create();
Mat descriptors_1, descriptors_2;
BFMatcher matcher(NORM_L2, true); //BFMatcher matcher(NORM_L2);
vector< DMatch > matches;
vector< DMatch > good_matches;
vector<Point2f>imgpts1,imgpts2;
vector<uchar> status;
while(true)
{
//grab and retrieve each frames of the video sequentially
camera0 >> frame0;
camera1 >> frame1;
imshow("Video0", frame0);
imshow("Video1", frame1);
cvtColor(frame0,frame0gray,CV_BGR2GRAY);
cvtColor(frame1,frame1gray,CV_BGR2GRAY);
sbm->compute( frame0gray, frame1gray, dispbm );
minMaxLoc( dispbm, &minVal, &maxVal );
dispbm.convertTo( dispnorm_bm, CV_8UC1, 255/(maxVal - minVal));
sgbm->compute(frame0gray, frame1gray, dispsgbm);
minMaxLoc( dispsgbm, &minVal, &maxVal );
dispsgbm.convertTo( dispnorm_sgbm, CV_8UC1, 255/(maxVal - minVal));
applyColorMap(dispnorm_bm, falseColorsMap, cv::COLORMAP_JET);
applyColorMap(dispnorm_sgbm, sfalseColorsMap, cv::COLORMAP_JET);
f2d->detect( frame0gray, keypoints_1 );
f2d->detect( frame1gray, keypoints_2 );
//-- Step 2: Calculate descriptors (feature vectors)
fd->compute( frame0gray, keypoints_1, descriptors_1 );
fd->compute( frame1gray, keypoints_2, descriptors_2 );
//-- Step 3: Matching descriptor vectors with a brute force matcher
matcher.match( descriptors_1, descriptors_2, matches );
drawMatches(frame0gray, keypoints_1, frame1gray, keypoints_2, matches, img_matches);
imshow("matches", img_matches);
//-- Quick calculation of max and min distances between keypoints
for( int i = 0; i < matches.size(); i++ )
{ double dist = matches[i].distance;
if( dist < min_dist ) min_dist = dist;
if( dist > max_dist ) max_dist = dist;
}
for( int i = 0; i < matches.size(); i++ )
{
if( matches[i].distance <= max(4.5*min_dist, 0.02) ){
good_matches.push_back( matches[i]);
imgpts1.push_back(keypoints_1[matches[i].queryIdx].pt);
imgpts2.push_back(keypoints_2[matches[i].trainIdx].pt);
}
}
Mat F = findFundamentalMat(imgpts1, imgpts2, cv::FM_RANSAC, 3., 0.99, status); //FM_RANSAC
stereoRectifyUncalibrated(imgpts1, imgpts1, F, frame0gray.size(), H1, H2);
Mat rectified1(frame0gray.size(), frame0gray.type());
warpPerspective(frame0gray, rectified1, H1, frame0gray.size());
Mat rectified2(frame1gray.size(), frame1gray.type());
warpPerspective(frame1gray, rectified2, H2, frame1gray.size());
sgbm->compute(rectified1, rectified2, disparity);
minMaxLoc( disparity, &minVal, &maxVal );
disparity.convertTo( disparity1, CV_8UC1, 255/(maxVal - minVal));
applyColorMap(disparity1, falsemap, cv::COLORMAP_JET);
imshow("disparity_rectified_color", falsemap);
imshow( "BM", falseColorsMap);
imshow( "CSGBM",sfalseColorsMap);
//wait for 40 milliseconds
int c = cvWaitKey(40);
//exit the loop if user press "Esc" key (ASCII value of "Esc" is 27)
if(27 == char(c)) break;
}
return 0;
}
现在输出又不是那么好，但比上次有所改进。然而，似乎一直存在一个问题，在上图中也可以看到。输出图像的左侧有一个全黑区域。它不应该这样正确。
如何解决这个问题呢？
任何帮助表示赞赏。
作为我项目的一部分，我正在使用 SteroBM class 进行立体视觉。我正在使用来自 2 个网络摄像头的输入帧和 运行 对输入帧灰度帧进行立体块匹配计算而不进行校正。我得到的输出与基本事实相去甚远（非常不完整）。我想知道，是不是因为我没有对输入帧进行整改。此外，我选择的基线保持在 20 厘米。我使用的是opencv-3.2.0版本的c++。
我是运行的代码如下。
#include <opencv2/core.hpp>
#include <opencv2/opencv.hpp>
#include </home/eswar/softwares/opencv_contrib-3.2.0/modules/contrib_world/include/opencv2/contrib_world.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgproc.hpp>
#include <stdio.h>
#include <iostream>
using namespace std;
using namespace cv;
int main()
{
//initialize and allocate memory to load the video stream from camera
VideoCapture camera0(0);
VideoCapture camera1(1);
if( !camera0.isOpened() ) return 1;
if( !camera1.isOpened() ) return 1;
Mat frame0,frame1;
Mat frame0gray,frame1gray;
Mat dispbm,dispsgbm;
Mat dispnorm_bm,dispnorm_sgbm;
Mat falseColorsMap, sfalseColorsMap;
int ndisparities = 16*5; /**< Range of disparity */
int SADWindowSize = 21; /**< Size of the block window. Must be odd */
Ptr<StereoBM> sbm = StereoBM::create( ndisparities, SADWindowSize );
Ptr<StereoSGBM> sgbm = StereoSGBM::create(0, //int minDisparity
96, //int numDisparities
5, //int SADWindowSize
600, //int P1 = 0
2400, //int P2 = 0
10, //int disp12MaxDiff = 0
16, //int preFilterCap = 0
2, //int uniquenessRatio = 0
20, //int speckleWindowSize = 0
30, //int speckleRange = 0
true); //bool fullDP = false
//-- Check its extreme values
double minVal; double maxVal;
while(true)
{
//grab and retrieve each frames of the video sequentially
camera0 >> frame0;
camera1 >> frame1;
imshow("Video0", frame0);
imshow("Video1", frame1);
cvtColor(frame0,frame0gray,CV_BGR2GRAY);
cvtColor(frame1,frame1gray,CV_BGR2GRAY);
sbm->compute( frame0gray, frame1gray, dispbm );
minMaxLoc( dispbm, &minVal, &maxVal );
dispbm.convertTo( dispnorm_bm, CV_8UC1, 255/(maxVal - minVal));
sgbm->compute(frame0gray, frame1gray, dispsgbm);
minMaxLoc( dispsgbm, &minVal, &maxVal );
dispsgbm.convertTo( dispnorm_sgbm, CV_8UC1, 255/(maxVal - minVal));
imshow( "BM", dispnorm_bm);
imshow( "SGBM",dispnorm_sgbm);
//wait for 40 milliseconds
int c = cvWaitKey(40);
//exit the loop if user press "Esc" key (ASCII value of "Esc" is 27)
if(27 == char(c)) break;
}
return 0;
}
尽管在代码中您看到也使用了块匹配，但请忽略它，因为它给出的输出更差。我发现 SGBM 输出更接近真实情况，因此我决定对其进行改进。但是，如果对如何改进块匹配结果有任何帮助。太棒了，我当然会很感激。
SGBM 技术的输出图像深度图像看起来像。
不，StereoBM不做校正，只是块匹配和一些预处理和post处理，但是opencv提供相机校准和校正检查的功能this link
在 opencv 示例中也有针对此过程的现成示例，因此不必从头开始编写代码。
关于结果，StereoBM是基于SAD算法（local stereo-matching），鲁棒性不强，你可以试试wls filter，可以显着提高你的结果。
StereoSGBM是基于SGM算法（实际上与原论文介绍的略有不同）的半全局算法，在视差图生成中考虑全局优化，产生的视差更好但速度较慢。
如上所示，我尝试对帧进行校正。代码如下。
#include <opencv2/core.hpp>
#include <opencv2/opencv.hpp>
#include </home/eswar/softwares/opencv_contrib-3.2.0/modules/contrib_world /include/opencv2/contrib_world.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgproc.hpp>
#include <stdio.h>
#include <iostream>
#include <opencv2/xfeatures2d/nonfree.hpp>
using namespace std;
using namespace cv;
using namespace cv::xfeatures2d;
int main()
{
//initialize and allocate memory to load the video stream from camera
VideoCapture camera0(0);
VideoCapture camera1(1);
int count=0;
Mat loRes, hiRes;
if( !camera0.isOpened() ) return 1;
if( !camera1.isOpened() ) return 1;
camera0.set(CV_CAP_PROP_FRAME_WIDTH, 400);
camera0.set(CV_CAP_PROP_FRAME_HEIGHT, 400);
camera1.set(CV_CAP_PROP_FRAME_WIDTH, 400);
camera1.set(CV_CAP_PROP_FRAME_HEIGHT, 400);
Mat frame0,frame1;
Mat frame0gray,frame1gray;
Mat dispbm,dispsgbm,disparity,disparity1;
Mat dispnorm_bm,dispnorm_sgbm;
Mat falseColorsMap, sfalseColorsMap,falsemap;
Mat img_matches;
Mat H1,H2;
int ndisparities = 96; /**< Range of disparity */
int SADWindowSize = 7;
Ptr<StereoBM> sbm = StereoBM::create( ndisparities, SADWindowSize );
Ptr<StereoSGBM> sgbm = StereoSGBM::create(-3, //int minDisparity
96, //int numDisparities
7, //int SADWindowSize
60, //int P1 = 0
2400, //int P2 = 0
90, //int disp12MaxDiff = 0
16, //int preFilterCap = 0
1, //int uniquenessRatio = 0
60, //int speckleWindowSize = 0
20, //int speckleRange = 0
true); //bool fullDP = false
//-- Check its extreme values
double minVal; double maxVal;
double max_dist = 0;
double min_dist = 100;
int minHessian = 630;
Ptr<Feature2D> f2d = SIFT::create();
vector<KeyPoint> keypoints_1, keypoints_2;
Ptr<Feature2D> fd = SIFT::create();
Mat descriptors_1, descriptors_2;
BFMatcher matcher(NORM_L2, true); //BFMatcher matcher(NORM_L2);
vector< DMatch > matches;
vector< DMatch > good_matches;
vector<Point2f>imgpts1,imgpts2;
vector<uchar> status;
while(true)
{
//grab and retrieve each frames of the video sequentially
camera0 >> frame0;
camera1 >> frame1;
imshow("Video0", frame0);
imshow("Video1", frame1);
cvtColor(frame0,frame0gray,CV_BGR2GRAY);
cvtColor(frame1,frame1gray,CV_BGR2GRAY);
sbm->compute( frame0gray, frame1gray, dispbm );
minMaxLoc( dispbm, &minVal, &maxVal );
dispbm.convertTo( dispnorm_bm, CV_8UC1, 255/(maxVal - minVal));
sgbm->compute(frame0gray, frame1gray, dispsgbm);
minMaxLoc( dispsgbm, &minVal, &maxVal );
dispsgbm.convertTo( dispnorm_sgbm, CV_8UC1, 255/(maxVal - minVal));
applyColorMap(dispnorm_bm, falseColorsMap, cv::COLORMAP_JET);
applyColorMap(dispnorm_sgbm, sfalseColorsMap, cv::COLORMAP_JET);
f2d->detect( frame0gray, keypoints_1 );
f2d->detect( frame1gray, keypoints_2 );
//-- Step 2: Calculate descriptors (feature vectors)
fd->compute( frame0gray, keypoints_1, descriptors_1 );
fd->compute( frame1gray, keypoints_2, descriptors_2 );
//-- Step 3: Matching descriptor vectors with a brute force matcher
matcher.match( descriptors_1, descriptors_2, matches );
drawMatches(frame0gray, keypoints_1, frame1gray, keypoints_2, matches, img_matches);
imshow("matches", img_matches);
//-- Quick calculation of max and min distances between keypoints
for( int i = 0; i < matches.size(); i++ )
{ double dist = matches[i].distance;
if( dist < min_dist ) min_dist = dist;
if( dist > max_dist ) max_dist = dist;
}
for( int i = 0; i < matches.size(); i++ )
{
if( matches[i].distance <= max(4.5*min_dist, 0.02) ){
good_matches.push_back( matches[i]);
imgpts1.push_back(keypoints_1[matches[i].queryIdx].pt);
imgpts2.push_back(keypoints_2[matches[i].trainIdx].pt);
}
}
Mat F = findFundamentalMat(imgpts1, imgpts2, cv::FM_RANSAC, 3., 0.99, status); //FM_RANSAC
stereoRectifyUncalibrated(imgpts1, imgpts1, F, frame0gray.size(), H1, H2);
Mat rectified1(frame0gray.size(), frame0gray.type());
warpPerspective(frame0gray, rectified1, H1, frame0gray.size());
Mat rectified2(frame1gray.size(), frame1gray.type());
warpPerspective(frame1gray, rectified2, H2, frame1gray.size());
sgbm->compute(rectified1, rectified2, disparity);
minMaxLoc( disparity, &minVal, &maxVal );
disparity.convertTo( disparity1, CV_8UC1, 255/(maxVal - minVal));
applyColorMap(disparity1, falsemap, cv::COLORMAP_JET);
imshow("disparity_rectified_color", falsemap);
imshow( "BM", falseColorsMap);
imshow( "CSGBM",sfalseColorsMap);
//wait for 40 milliseconds
int c = cvWaitKey(40);
//exit the loop if user press "Esc" key (ASCII value of "Esc" is 27)
if(27 == char(c)) break;
}
return 0;
}
现在输出又不是那么好，但比上次有所改进。然而，似乎一直存在一个问题，在上图中也可以看到。输出图像的左侧有一个全黑区域。它不应该这样正确。 如何解决这个问题呢？ 任何帮助表示赞赏。