Mastering OPENCV CH4:SFM Details (i)

The structure is restored from the motion so that the image geometry can be extracted better by moving the camera. In order to use monocular cameras, a discrete and sparse set of video frames, rather than continuous video streams, is used in the book. This provides ease of pairing in the back 22 loop combination.

Main content:
1: Estimate the motion posture of the camera from two images.
2: Refactoring the scene
3: Refactoring from view
4: Refactoring Refinement
5: Visualize three-dimensional point clouds

I: Assume that a calibrated camera -a previously calibrated camera-is used. The previous blog also mentions how to calibrate the camera. Therefore, we assume that the internal parameters of the camera exist and materialize into the K matrix, the K matrix is a result of the camera calibration process output.
Initialize the camera calibration parameters within the program:

//load calibration matrix Cv::filestorage FS; if (Fs.open (imgs_path_+ "\\out_camera_data.yml", Cv::filestorage::read))//Open correction parameter file {fs["Camera_matrix"]>>
        ; Cam_matrix;
    fs["Distortion_coefficients"]>>distortion_coeff;
        } else {//If there is no calibration file, combine a calibration internal parameter cv::size imgs_size = Imgs_[0].size ();
        Double max_w_h = max (imgs_size.height,imgs_size.width);
                                                Cam_matrix = (cv::mat_<double> (3,3) << max_w_h, 0, imgs_size.width/2.0,          0, Max_w_h, imgs_size.height/2.0, 0,
        0, 1);
    Distortion_coeff = Cv::mat_<double>::zeros (1,4);
    } K = Cam_matrix; Cv::invert (K, KINV);
    The internal parameters are taken back Distortion_coeff.convertto (DISTCOEFF_32F,CV_32FC1); K.convertto (K_32F,CV_32FC1); 

If there is a calibration file, import from the file, if there is no calibration file, the combination of a camera with the reference K, according to the size of the image can be combined, the distortion parameter is set to 0, the camera to take the inverse of the KINV, converted into 32FC1 accuracy.

II: Get images
In the book, when acquiring an image, the image in the directory is read in a given directory, and stored in a std::vector& IMGs . We can also read each other, or intercept video frames.
I think this is a regular piece of code that is placed below for random invocation later.

Given the directory path, read the directory image file, save the picture name, set the image scaling ratio void Open_imgs_dir (char* dir_name, std::vector<cv::mat>& images, std::
    vector<std::string>& images_names, double downscale_factor) {if (Dir_name = = NULL) {return;
    } string dir_name_ = string (dir_name);

Vector<string> Files_;
    #ifndef WIN32//open A directory The POSIX//linux or MacOS Read the image below, need to include # include <dirent.h> DIR *DP;     
    struct Dirent *ep;

    DP = Opendir (dir_name);
                if (DP! = NULL) {while (ep = Readdir (DP)) {if (ep->d_name[0]! = '. ')
        Files_.push_back (Ep->d_name);
    } (void) Closedir (DP);
        } else {Cerr << ("couldn ' t Open the Directory");
    Return
    } #else//open a directory the WIN32 HANDLE hfind = Invalid_handle_value;

    Win32_find_data Fdata; if (Dir_name_[dir_name_.size ()-1] = = ' \ \ ' | | dir_name_[dir_name_.size ()-1] = = '/') {dir_name_ = Dir_name_.sUbstr (0,dir_name_.size ()-1); 
    } hfind = FindFirstFile (String (dir_name_). Append ("\\*"). C_str (), &fdata); 
                if (hfind! = Invalid_handle_value) {do {if (strcmp (Fdata.cfilename, ".")!! = 0 && strcmp (Fdata.cfilename, "...")! = 0) {if (Fdata.dwfileattributes & File_a
                Ttribute_directory) {continue;//a diretory} else
                {Files_.push_back (fdata.cfilename);
    }}} while (FindNextFile (hfind, &fdata)! = 0);
        } else {Cerr << "can ' t open directory\n";
    Return
        } if (GetLastError ()! = error_no_more_files) {findclose (hfind);
        Cerr << "Some other error with opening directory:" << GetLastError () << Endl;
    Return
    } findclose (Hfind); Hfind = Invalid_handle_value; #endif for (unsigned int i=0; i<files_.size (); i++) {if (files_[i][0] = = '. ' | |! ( Hasendinglower (Files_[i], "jpg") | |
        Hasendinglower (Files_[i], "PNG"))) {continue;
        } Cv::mat m_ = Cv::imread (String (dir_name_). Append ("/"). Append (Files_[i])); if (downscale_factor! = 1.0) Cv::resize (M_,m_,size (), downscale_factor,downscale_factor);//whether to scale images _names.push_back (Files_[i]);//Store picture name Images.push_back (m_);//Save Picture as Mat}}

This piece of code calls two functions for image name stitching.

BOOL Hasendinglower (String const &fullstring_, String const &_ending)
{
    string fullstring = Fullstring_ , ending = _ending;
    Transform (Fullstring_.begin (), Fullstring_.end (), Fullstring.begin (),:: ToLower); Put in the last
    return hasending (fullstring,ending);
}

BOOL Hasending (std::string const &fullstring, std::string const &ending)
{
    if (fullstring.length () >= ending.length ()) {
        return (0 = = Fullstring.compare (Fullstring.length ()-Ending.length (), Ending.length (), Ending));
    } else {
        return false;
    }
}

The next step is to make sure that the image is 8uc3 and then grayscale, which is the preprocessing step.

//ensure that the image is cv_8uc3 for (unsigned int i=0; i ());//cv::vec3b is the substitution of 8UC3, typedef vec<uchar, 3> vec3b; if (!imgs_[i].empty ()) {if (imgs_[i].type () = = CV_8UC1) {cvtcolor (Imgs_[i], imgs_orig[i], CV
            _GRAY2BGR); } else if (imgs_[i].type () = = CV_32FC3 | | imgs_[i].type () = = CV_64FC3) {Imgs_[i].convertto (imgs_orig[i],c
            v_8uc3,255.0);
            } else {Imgs_[i].copyto (imgs_orig[i]);
        }} imgs.push_back (Cv::mat ());

        Cvtcolor (Imgs_orig[i],imgs[i], cv_bgr2gray);
        Defines vector<std::vector<cv::keypoint> > Imgpts,imgpts_good for saving image feature points. Imgpts.push_back (std::vector<cv::keypoint> ());//Imgpts_good.push_back (std::vector<cv::keypoint> ())
        ;
    Std::cout << "."; }

III: Feature Extraction
With the image, the image is matched, the basic matrix of two images is computed and the essence matrix is calculated. This is also the first essential part of SFM.
The base matrix (denoted by F) and the eigen matrix (denoted by e). The eigen matrices are assumed to be used by calibrated cameras, which are very similar. The OPENCV function only allows us to find the underlying matrix through the Findfundamentalmat function. However, it is very simple to use the calibration matrix (calibration matrix) k to derive the underlying matrix from the Eigen matrix, as follows:

mat_<double> E = k.t () * F * K; According to HZ (9.12)

The Eigen matrix, which is a 3x3-sized matrix, uses X ' ex=0 to impose a constraint between one point in the image and the other in the image, where X is a point in the image one, and X ' is the corresponding point in the image two. This is very useful because we are going to see. Another important fact that we use is that the Eigen matrix is all we need to recover two cameras for our image, despite the lack of a scaling factor, but we'll get it later. So, if we get the eigen matrix, we know where each camera is in space, and we know where they are looking, and if we have enough of these constrained equations, then we can simply calculate the matrix. Simple because each equation can be used to solve a small part of the matrix. In fact, OpenCV allows us to calculate it using only 7 point pairs, but we want to get more point pairs coming to a robust solution.

In computer vision, feature extraction and description sub-match is a fundamental process and is used in many ways to perform a variety of operations. For example, detecting the position and orientation of a target in an image, or finding a similar image in a big data image by giving a query image. Essentially, extracting means selecting points in an image, making good features available, and calculating a descriptor for them. A descriptor is a vector of multiple data used to describe the surrounding environment around a feature point in an image. Different methods have different lengths and data types to represent the description sub-vectors. A match is a descriptor that uses it to find a corresponding set of features from another image. OPENCV provides a very simple and efficient way to support feature extraction and matching. More information about feature matching can be found in Chapter 3 small (none) markup augmented reality.

This book uses three kinds of feature extraction methods: General rich feature extraction, based on the feature extraction of GPU, so that the optical flow method for feature extraction.

1: General feature-rich extraction

detector = featuredetector::create ("Pyramidfast");
Extractor = Descriptorextractor::create ("ORB");

Std::cout << "--------------------extract feature points for all images-------------------\ n";
    Detector->detect (IMGs, imgpts);//feature point detection for all images
    Extractor->compute (IMGs, imgpts, descriptors);// Computes the description of all feature points
    std::cout << "-------------------------------------done----------------------------------- \ n ";

2: Using rich feature extraction from the GPU

Extractor = new SURF ();//This class can directly extract the image's feature points and compute descriptors.

std::cout << "--------------------extract feature points for all images (GPU)-------------------\ n";

Imgpts.resize (Imgs_.size ());
Descriptors.resize (Imgs_.size ());

Cv_profile ("Extract", for
    (int img_i=0;img_i
 
3: Optical Flow method:
Optical flow is the process of matching a point from one image selection to another, assuming that the two images are part of a video sequence and that they are very similar to each other. Most optical flow methods compare a small area, called a search window or block, that surrounds each point in image A and every point in the same area of image B. Following a very common rule in computer vision, called Luminance constant constraint (brightness constancy constraint) (and other names), these small pieces of the image do not change much from one image to another, so their amplitude difference is close to 0. In addition to matching blocks, the updated optical flow method uses some additional methods to get better results. One way to do this is to use the image pyramid, which is an increasingly small size (size) version of the image, which takes into account the work from rough to delicate-a very useful technique in computer vision. Another method is to define a global constraint on a flow field, assuming that the points are close to each other and move in the same direction.

In order to be compatible with feature extraction methods with rich features, the fast feature extraction algorithm is used to compute only feature points. Then the feature matching is performed by using the optical flow method.

Detect keypoints for all images
    fastfeaturedetector ffd;  densefeaturedetector FfD;
    Ffd.detect (IMGs, imgpts);
 
IV: Feature Matching
After feature extraction is complete, feature matching is necessary. The use of OpenMP for parallel for operations in the book requires the installation of OpenMP. Here, the method of loop combination is used to match every two images. Increased accuracy, but very slow.

int loop1_top = Imgs.size ()-1, loop2_top = Imgs.size ();
int frame_num_i = 0; #pragma omp parallel for//parallel calculation for (frame_num_i = 0; frame_num_i < loop1_top; frame_num_i++) {for (int fr Ame_num_j = frame_num_i + 1; Frame_num_j < Loop2_top; frame_num_j++) {std::cout << "------------Match" << imgs_names[frame_num_i] << ", "< matches
            _tmp;
Feature_matcher->matchfeatures (FRAME_NUM_I,FRAME_NUM_J,&AMP;MATCHES_TMP);             STD::MAP&LT;STD::p air<int,int>, STD::VECTOR&LT;CV::D match> > Matches_matrix; This variable holds the two frames for matching and matching results.

            Matches_matrix[std::make_pair (frame_num_i,frame_num_j)] = matches_tmp;
                            STD::VECTOR&LT;CV::D match> matches_tmp_flip = flipmatches (matches_tmp);//swap position for two images to complete cross-match check filtering Matches_matrix[std::make_pair (frame_num_j,frame_num_i)] = MATCHes_tmp_flip; }
        }
 
Next, we analyze the most critical function in this feature_matcher->matchfeatures. This feature_matcher is the three methods described in the previous feature extraction. Each of these methods can be matched.

1: General rich feature matching.
The use of violent bf matching method plus Hamming distance to match, get a lot of matching groups, of course, there are many errors.

Enter the currently matched frame and get the match void richfeaturematcher::matchfeatures (int idx_i, int idx_j, vector<dmatch>* matches) {#ifdef _
    _sfm__debug__ const mat& img_1 = imgs[idx_i];
Const mat& img_2 = Imgs[idx_j];
    #endif//Gets the key point of the currently matched frame and describes the child const vector<keypoint>& IMGPTS1 = imgpts[idx_i];
    Const vector<keypoint>& IMGPTS2 = Imgpts[idx_j];
    Const mat& descriptors_1 = descriptors[idx_i];

    Const mat& descriptors_2 = Descriptors[idx_j];
    std::vector< dmatch > Good_matches_,very_good_matches_;
    Std::vector<keypoint> Keypoints_1, keypoints_2; Print the number of key points StringStream SS; SS << "Imgpts1 has" << imgpts1.size () << "points (descriptors" << descriptors_1.rows << "
    ) "<< Endl;
    cout << ss.str (); StringStream SS1; SS1 << "Imgpts2 has" << imgpts2.size () << "points (descriptors" << descriptors_2.rows <<
    ")" << Endl; cout << ss1.str ();

    Keypoints_1 = Imgpts1;

    Keypoints_2 = imgpts2;
    if (Descriptors_1.empty ()) {cv_error (0, "Descriptors_1 is empty");
    } if (Descriptors_2.empty ()) {cv_error (0, "descriptors_2 is empty"); }//using BF violence matching method for HAMMING matching, allowing cross check bfmatcher matcher (norm_hamming,true); Allow Cross-check.
    Use Hamming distance for binary descriptor (ORB) std::vector< dmatch > Matches_;
    if (matches = = NULL) {matches = &matches_;
    } if (matches->size () = = 0) {matcher.match (Descriptors_1, descriptors_2, *matches);

} assert (Matches->size () > 0); Double max_dist = 0;
Double min_dist = 1000.0; --Quick calculation of Max and min distances between keypoints//for (unsigned int i = 0; i < matches->s Ize (); i++)//{//Double dist = (*matches) [i].distance;//if (dist>1000.0) {dist = 1000.0;}//If
(Dist < min_dist) min_dist = dist; if (Dist > Max_dIST) max_dist = dist;
}////#ifdef __SFM__DEBUG__//printf ("--Max dist:%f \ n", max_dist);
printf ("-Min dist:%f \ n", min_dist);

#endif vector<keypoint> Imgpts1_good,imgpts2_good; if (min_dist <= 0) {//min_dist = 10.0;//}//Remove each repetition match of the training point, i.e. a training point with multiple query points eliminate any Re-matchi
    Ng of training points (multiple queries to one training)//double cutoff = 4.0*min_dist;
    Std::set<int> Existing_trainidx; for (unsigned int i = 0; i < matches->size (); i++) {//normalized match: Sometimes the subscript of the number of images is the subscript "normalize" of the training number MATCHING:SOMTI Mes Imgidx is the one holding the Trainidx if ((*matches) [i].trainidx <= 0) {(*matches) [I].trainid
        x = (*matches) [I].imgidx; }//Here Set::find means to find the key value in set, if found, return the position of the key value iterator, if not found, return to Set::end, this is to//prevent a training value corresponding to multiple query values.
        After each training value is detected, it is stored in set, and the next time the training value appears in the matching pair, it is judged as a multiple-to-one removal.
        Also the IF in here to judge the training value between 0 and the number of key points. if (Existing_trainidx.find (*matches) [I].trainIDX) = = Existing_trainidx.end () && (*matches) [i].trainidx >= 0 && (*matches) [I].trainidx &L T (int) (Keypoints_2.size ())/*&& (*matches) [I].distance > 0.0 && (*matches) [I].distance < cutoff */) {Good_matches_.push_back ((*matches) [i]);//Qualifying Matching group Imgpts1_good.push_back (KeyPoint s_1[(*matches) [i].queryidx]);//The query key of a qualifying matching group Imgpts2_good.push_back (keypoints_2[(*matches) [i].trainidx]);//Compliant
        The training key point of the matching group of the condition Existing_trainidx.insert ((*matches) [I].TRAINIDX];
}}//The first match here is done, and there are a lot of wrong matches.  #ifdef __sfm__debug__ cout << "keypoints_1.size ()" << keypoints_1.size () << "imgpts1_good.size ()"
    << imgpts1_good.size () << Endl; cout << "keypoints_2.size ()" << keypoints_2.size () << "imgpts2_good.size ()" << imgpts2_good.si

    Ze () << Endl; {//--Draw only "good" matches Mat img_matches; Drawmatches (img_1, Keypoints_1, Img_2, Keypoints_2, Good_matches_, Img_matches, Scalar::all ( -1), Scal        
        Ar::all ( -1), vector<char> (), drawmatchesflags::not_draw_single_points); --Show detected matches StringStream SS;
        SS << "Feature Matches" << idx_i << "-" << idx_j;
        Imshow (Ss.str (), img_matches);
        Waitkey (500);
    DestroyWindow (Ss.str ());
    #endif//The following will optimize the matching group again in the same way that the underlying matrix is computed.
    vector<uchar> status;

    Vector<keypoint> Imgpts2_very_good,imgpts1_very_good;
    ASSERT (Imgpts1_good.size () > 0);
    ASSERT (Imgpts2_good.size () > 0);
    ASSERT (Good_matches_.size () > 0);

    ASSERT (imgpts1_good.size () = = Imgpts2_good.size () && imgpts1_good.size () = = Good_matches_.size ());
    Select features this make epipolar sense//compute the underlying matrix of the matching group, refine the matching group again, and remove the error match. Getfundamentalmat (Keypoints_1,keypoints_2,imgpts1_very_good,imgpts2_very_gOod,good_matches_);
        Draw matches #ifdef __sfm__debug__ {//--Draw only "good" matches Mat img_matches; Drawmatches (img_1, Keypoints_1, Img_2, Keypoints_2, Good_matches_, Img_matches, Scalar::all ( -1), Scal        
        Ar::all ( -1), vector<char> (), drawmatchesflags::not_draw_single_points);
        --Show detected matches imshow ("Good matches", img_matches);
        Waitkey (100);
    DestroyWindow ("Good Matches");
 } #endif}
 
 2: Feature matching using GPU acceleration 
 matches using the KNN ratio test, where the best match distance/suboptimal is greater than 0.7, and the error match is removed. There will also be an error match. 

Also use brute force matching, except GPU acceleration.
    Matching descriptor vectors using Brute force matcher bruteforcematcher_gpu<l2<float> > Matcher;
    std::vector< dmatch > Matches_;
    if (matches = = NULL) {matches = &matches_; } if (matches->size () = = 0) {cout << match << descriptors_1.rows << "vs." <<

        Descriptors_2.rows << "...";
            if (use_ratio_test) {vector<vector<dmatch> > knn_matches;
            Gpumat trainidx,distance,alldist; Cv_profile ("Match", Matcher.knnmatchsingle (descriptors_1,descriptors_2,trainidx,distance,alldist,2);//Use
            KNN ratio test to match matcher.knnmatchdownload (trainidx,distance,knn_matches);

            ) (*matches). Clear (); Ratio test for (int i=0;i<knn_matches.size (); i++) {if (knn_matches[i][0].distance/knn_ma
           Tches[i][1].distance < 0.7) {//KNN ratio discriminant, eliminate bad matching groups         (*matches). push_back (Knn_matches[i][0]); }} cout << "kept" << (*matches). Size () << "features after ratio test" <<
        Endl
    } else {cv_profile ("Match", Matcher.match (Descriptors_1, descriptors_2, *matches);)} }
 
3: Optical flow method for matching. This is roughly how each of the left plots moves in the two picture, looking at each light flow point in the right image and matching it in a small area.

vector<keypoint>left_keypoints,right_keypoints;
Detect Keypoints in the left and right images
fastfeaturedetectorffd;
Ffd.detect (IMG1, left_keypoints);
Ffd.detect (Img2, right_keypoints);
vector<point2f>left_points;
Keypointstopoints (left_keypoints,left_points);//key point converted to point2f point
vector<point2f>right_points (Left_ Points.size ());
Making sure images is grayscale
Mat Prevgray,gray;
if (img1.channels () = = 3) {
cvtcolor (img1,prevgray,cv_rgb2gray);
Cvtcolor (Img2,gray,cv_rgb2gray);
} else {
Prevgray = IMG1;
Gray = Img2;
}
Calculate The Optical flow field:
//How each Left_point moved across the 2 images
Vector<uchar>vstatus ; vector<float>verror;
Calcopticalflowpyrlk (Prevgray, Gray, Left_points, right_points, 
vstatus, verror);
First, filter out the points with high error
vector<point2f>right_points_to_find;
Vector<int>right_points_to