multibandblend是目前圖像融和方面比較好的方法。原始論文為《a multivesolution spline with application to image mosaics 》，可以用百度學術找到。原始論文中采用的方法是直接對帶拼接的兩個圖檔進行拉普拉斯金字塔分解，而後一半對一半進行融合。國人也有許多改進，比如“首先采用拉普拉斯分辨率金字塔結構，将輸入的圖像分解成一系列不同頻率上的圖像，而後在每個頻段上面進行權重平均，把這些不同頻率上的圖像最後合并一個圖像”。這個說法來源于下面這篇論文，根據其他獲得的一些知識，應該是可行的。
在opencv内部已經實作了multibandblend，但是耦合度比較高，想拿出來單獨研究要對代碼有深入了解；網絡上面也有這樣的實作，這個下文提及[]。為了深入了解并運用，還是要自己實作，再複用别人的代碼
一、首先實作laplacian金字塔的分解和重建
#include "stdafx.h"
#include <iostream>
#include <vector>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/features2d/features2d.hpp>
#include <opencv2/calib3d/calib3d.hpp>
using namespace std;
using namespace cv;
int _tmain(int argc, _TCHAR* argv[])
{
Mat src = imread("Lena.jpg");
src.convertTo(src,CV_32F,1.0/255);
imshow("src",src);
Mat src2= src.clone();
Mat dst;
Mat lastmat;
vector<Mat> vecMats;
Mat tmp;
for (int i=0;i<4;i++)
{
pyrDown(src2,src2);
pyrUp(src2,tmp);
resize(tmp,tmp,src.size());
tmp = src - tmp;
vecMats.push_back(tmp);
src = src2;
}
lastmat = src;
//重建
for (int i=3;i>=0;i--)
pyrUp(lastmat,lastmat);
resize(lastmat,lastmat,vecMats[i].size());
lastmat = lastmat + vecMats[i];
imshow("dst",lastmat);
waitKey();
return 0;
}
二、實作每個金字塔層面的linearblend融合
Mat srcLeft = imread("apple.png");
Mat srcRight= imread("orange.png");
srcLeft.convertTo(srcLeft,CV_32F,1.0/255);
srcRight.convertTo(srcRight,CV_32F,1.0/255);
Mat rawLeft = srcLeft.clone();
Mat rawRight=srcRight.clone();
imshow("srcRight",srcRight);
imshow("srcLeft",srcLeft);
Mat srcLeft2= srcLeft.clone();
Mat srcRight2=srcRight.clone();
Mat lastmatLeft; Mat lastmatRight;
vector<Mat> vecMatsLeft; vector<Mat> vecMatsRight;
pyrDown(srcLeft2,srcLeft2);
pyrUp(srcLeft2,tmp);
resize(tmp,tmp,srcLeft.size());
tmp = srcLeft - tmp;
vecMatsLeft.push_back(tmp);
srcLeft = srcLeft2;
lastmatLeft = srcLeft;
pyrDown(srcRight2,srcRight2);
pyrUp(srcRight2,tmp);
resize(tmp,tmp,srcRight.size());
tmp = srcRight - tmp;
vecMatsRight.push_back(tmp);
srcRight = srcRight2;
lastmatRight = srcRight;
//每一層都要對準并融合
int ioffset =vecMatsLeft[0].cols - 100;//這裡-100 的操作是linearblend的小技巧
int istep = 1;
double dblend = 0.0;
vector<Mat> vecMatResult;//儲存結果
Mat tmpResult;
Mat roi;
//對準
tmpResult = Mat::zeros(vecMatsLeft[i].rows,vecMatsLeft[i].cols*2,vecMatsLeft[i].type());
roi = tmpResult(Rect(ioffset,0,vecMatsLeft[i].cols,vecMatsLeft[i].rows));
vecMatsRight[i].copyTo(roi);
roi = tmpResult(Rect(0,0,vecMatsLeft[i].cols,vecMatsLeft[i].rows));
vecMatsLeft[i].copyTo(roi);
//融合
for (int j = 0;j<(100/istep);j++)
{
tmpResult.col(ioffset + j)= tmpResult.col(ioffset+j)*(1-dblend) + vecMatsRight[i].col(j)*dblend;
dblend = dblend +0.01*istep;
}
//結尾
dblend = 0.0;
ioffset = ioffset/2;
istep = istep*2;
vecMatResult.push_back(tmpResult);
Mat latmatresult = Mat::zeros(lastmatLeft.rows,lastmatLeft.cols*2,lastmatLeft.type());
roi = latmatresult(Rect(0,0,lastmatLeft.cols,lastmatLeft.rows));
lastmatLeft.copyTo(roi);
roi = latmatresult(Rect(ioffset,0,lastmatLeft.cols,lastmatLeft.rows));
lastmatRight.copyTo(roi);
for (int j=0;j<(100/istep);j++)
latmatresult.col(ioffset + j)= latmatresult.col(ioffset+j)*(1-dblend) + lastmatRight.col(j)*dblend;
dblend = dblend +0.01*istep;
//重構
pyrUp(lastmatLeft,lastmatLeft);
resize(lastmatLeft,lastmatLeft,vecMatsLeft[i].size());
lastmatLeft = lastmatLeft + vecMatsLeft[i];
pyrUp(lastmatRight,lastmatRight);
resize(lastmatRight,lastmatRight,vecMatsRight[i].size());
lastmatRight = lastmatRight + vecMatsRight[i];
pyrUp(latmatresult,latmatresult);
resize(latmatresult,latmatresult,vecMatResult[i].size());
latmatresult = latmatresult + vecMatResult[i];
imshow("lastmatLeft",lastmatLeft);
lastmatLeft.convertTo(lastmatLeft,CV_8U,255);
imwrite("lastmatleft.png",lastmatLeft);
imshow("lastmatRight",lastmatRight);
imshow("multibandblend",latmatresult);
觀看細節，可以看到過渡的地方一點都不違和，蘋果上面的白色小斑點都過渡了過來。這個結果應該是非常接近論文上面的要求了。
三、改造現有的圖像拼接程式，并且運用于圖像拼接
//使用Multiblend進行圖像融合
#include "matcher.h"
int main()
{
// Read input images 這裡的命名最好為imageleft和imageright
cv::Mat image1= cv::imread("Univ4.jpg",1);
cv::Mat image2= cv::imread("Univ5.jpg",1);
if (!image1.data || !image2.data)
return 0;
// Prepare the matcher
RobustMatcher rmatcher;
rmatcher.setConfidenceLevel(0.98);
rmatcher.setMinDistanceToEpipolar(1.0);
rmatcher.setRatio(0.65f);
cv::Ptr<cv::FeatureDetector> pfd= new cv::SurfFeatureDetector(10);
rmatcher.setFeatureDetector(pfd);
// Match the two images
std::vector<cv::DMatch> matches;
std::vector<cv::KeyPoint> keypoints1, keypoints2;
cv::Mat fundemental= rmatcher.match(image1,image2,matches, keypoints1, keypoints2);
// draw the matches
cv::Mat imageMatches;
cv::drawMatches(image1,keypoints1, // 1st image and its keypoints
image2,keypoints2, // 2nd image and its keypoints
matches, // the matches
imageMatches, // the image produced
cv::Scalar(255,255,255)); // color of the lines
// Convert keypoints into Point2f
std::vector<cv::Point2f> points1, points2;
for (std::vector<cv::DMatch>::const_iterator it= matches.begin();
it!= matches.end(); ++it) {
// Get the position of left keypoints
float x= keypoints1[it->queryIdx].pt.x;
float y= keypoints1[it->queryIdx].pt.y;
points1.push_back(cv::Point2f(x,y));
// Get the position of right keypoints
x= keypoints2[it->trainIdx].pt.x;
y= keypoints2[it->trainIdx].pt.y;
points2.push_back(cv::Point2f(x,y));
std::cout << points1.size() << " " << points2.size() << std::endl;
// Find the homography between image 1 and image 2
std::vector<uchar> inliers(points1.size(),0);
cv::Mat homography= cv::findHomography(
cv::Mat(points1),cv::Mat(points2), // corresponding points
inliers, // outputed inliers matches
CV_RANSAC, // RANSAC method
1.); // max distance to reprojection point
// Warp image 1 to image 2
cv::Mat result;
cv::warpPerspective(image1, // input image
result, // output image
homography, // homography
cv::Size(2*image1.cols,image1.rows)); // size of output image
cv::Mat resultback;
result.copyTo(resultback);
// Copy image 1 on the first half of full image
cv::Mat half(result,cv::Rect(0,0,image2.cols,image2.rows));
image2.copyTo(half);
// Display the warp image
cv::namedWindow("After warping");
cv::imshow("After warping",result);
//需要注意的一點是，原始檔案的圖檔是按照從右至左邊進行移動的。
//進行Multiblend的融合,融合的輸入圖像為image2(左)和resultback(右)
Mat srcLeft = image2.clone();
Mat srcRight= resultback.clone();
roi = tmpResult(Rect(0,0,vecMatsRight[i].cols,vecMatsRight[i].rows));
for (int j = 0;j<(128/istep);j++)
tmpResult.col(ioffset + j)= tmpResult.col(ioffset+j)*(1-dblend) + vecMatsRight[i].col(ioffset+j)*dblend;
dblend = dblend +0.0078125*istep;
roi = latmatresult(Rect(0,0,lastmatRight.cols,lastmatRight.rows));
for (int j=0;j<(128/istep);j++)
latmatresult.col(ioffset+j)= latmatresult.col(ioffset+j)*(1-dblend) + lastmatRight.col(ioffset+j)*dblend;
dblend = dblend +0.00725*istep;
cv::waitKey();
可以看到紋理過渡不違和，但是光照變化劇烈
四、改造現有代碼
上文提及了已經實作的代碼，這個代碼寫的還是相當不錯的，其原理完全按照《a multivesolution spline with application to image mosaics 》來實作，為了應用于實際的圖像拼接，做了一些修改
//代碼重構
class LaplacianBlending {
private:
Mat_<Vec3f> top;
Mat_<Vec3f> down;
Mat_<float> blendMask;
vector<Mat_<Vec3f> > topLapPyr,downLapPyr,resultLapPyr;//Laplacian Pyramids
Mat topHighestLevel, downHighestLevel, resultHighestLevel;
vector<Mat_<Vec3f> > maskGaussianPyramid; //masks are 3-channels for easier multiplication with RGB
int levels;
//建立金字塔
void buildPyramids() {
//參數的解釋 top就是top ,topLapPyr就是top的laplacian的pyr,而topHighestLevel儲存的是最高端的高斯金字塔
buildLaplacianPyramid(top,topLapPyr,topHighestLevel);
buildLaplacianPyramid(down,downLapPyr,downHighestLevel);
buildGaussianPyramid();
}
//建立gauss金字塔
void buildGaussianPyramid() {//金字塔内容為每一層的掩模
assert(topLapPyr.size()>0);
maskGaussianPyramid.clear();
Mat currentImg;
//blendMask就是結果所在
cvtColor(blendMask, currentImg, CV_GRAY2BGR);//store color img of blend mask into maskGaussianPyramid
maskGaussianPyramid.push_back(currentImg); //0-level
currentImg = blendMask;
for (int l=1; l<levels+1; l++) {
Mat _down;
if (topLapPyr.size() > l)
pyrDown(currentImg, _down, topLapPyr[l].size());
else
pyrDown(currentImg, _down, topHighestLevel.size()); //lowest level
Mat down;
cvtColor(_down, down, CV_GRAY2BGR);
maskGaussianPyramid.push_back(down);//add color blend mask into mask Pyramid
currentImg = _down;
}
//建立laplacian金字塔
void buildLaplacianPyramid(const Mat& img, vector<Mat_<Vec3f> >& lapPyr, Mat& HighestLevel) {
lapPyr.clear();
Mat currentImg = img;
for (int l=0; l<levels; l++) {
Mat down,up;
pyrDown(currentImg, down);
pyrUp(down, up,currentImg.size());
Mat lap = currentImg - up; //存儲的就是殘差
lapPyr.push_back(lap);
currentImg = down;
currentImg.copyTo(HighestLevel);
Mat_<Vec3f> reconstructImgFromLapPyramid() {
//将左右laplacian圖像拼成的resultLapPyr金字塔中每一層
//從上到下插值放大并相加，即得blend圖像結果
Mat currentImg = resultHighestLevel;
for (int l=levels-1; l>=0; l--) {
Mat up;
pyrUp(currentImg, up, resultLapPyr[l].size());
currentImg = up + resultLapPyr[l];
return currentImg;
void blendLapPyrs() {
//獲得每層金字塔中直接用左右兩圖Laplacian變換拼成的圖像resultLapPyr
resultHighestLevel = topHighestLevel.mul(maskGaussianPyramid.back()) +
downHighestLevel.mul(Scalar(1.0,1.0,1.0) - maskGaussianPyramid.back());
Mat A = topLapPyr[l].mul(maskGaussianPyramid[l]);
Mat antiMask = Scalar(1.0,1.0,1.0) - maskGaussianPyramid[l];
Mat B = downLapPyr[l].mul(antiMask);
Mat_<Vec3f> blendedLevel = A + B;
resultLapPyr.push_back(blendedLevel);
public:
LaplacianBlending(const Mat_<Vec3f>& _top, const Mat_<Vec3f>& _down, const Mat_<float>& _blendMask, int _levels)://預設資料，使用 LaplacianBlending lb(l,r,m,4);
top(_top),down(_down),blendMask(_blendMask),levels(_levels)
{
assert(_top.size() == _down.size());
assert(_top.size() == _blendMask.size());
buildPyramids(); //建立laplacian金字塔和gauss金字塔
blendLapPyrs(); //将左右金字塔融合成為一個圖檔
};
Mat_<Vec3f> blend() {
return reconstructImgFromLapPyramid();//reconstruct Image from Laplacian Pyramid
};
Mat_<Vec3f> LaplacianBlend(const Mat_<Vec3f>& t, const Mat_<Vec3f>& d, const Mat_<float>& m) {
LaplacianBlending lb(t,d,m,4);
return lb.blend();
}
cv::Mat image1= cv::imread("Univ3.jpg",1);
cv::Mat image2= cv::imread("Univ4.jpg",1);
int ioffset = srcLeft.cols -100;
Mat imageL = srcLeft(Rect(ioffset,0,100,srcLeft.rows)).clone();
Mat imageR = srcRight(Rect(ioffset,0,100,srcLeft.rows)).clone();
Mat_<Vec3f> t; imageL.convertTo(t,CV_32F,1.0/255.0);//Vec3f表示有三個通道，即 l[row][column][depth]
Mat_<Vec3f> d; imageR.convertTo(d,CV_32F,1.0/255.0);
Mat_<float> m(t.rows,d.cols,0.0); //将m全部指派為0
m(Range::all(),Range(0,m.cols/2)) = 1.0; //取m全部行&[0,m.cols/2]列，指派為1.0
Mat_<Vec3f> matblend = LaplacianBlend(t,d, m);
Mat re;
matblend.convertTo(re,CV_8UC3,255);
Mat roi = srcLeft(Rect(ioffset,0,100,srcLeft.rows));
re.copyTo(roi);
roi = resultback(Rect(0,0,srcLeft.cols,srcLeft.rows));
srcLeft.copyTo(roi);
五、結果比較
我寫的multibandblend
六、小結
應該說我采用的算法，雖然也有一定程度上效果，但是“鬼影”并沒有減少。采用linearblend的方法，不管分不分金字塔，都是“鬼影”産生的原因。研究問題，還是應該讀原始論文，找到好的代碼，真正地解決問題。
這個論文背後的原理是什麼？
這個代碼是誰寫的，來自哪裡？
搞明白這些問題。
來自為知筆記(Wiz)附件清單
目前方向：圖像拼接融合、圖像識别
聯系方式：[email protected]