OpenCvSharp 通過特徵點匹配圖片

現在的手遊基本都是重複操作,一個動作要等好久,結束之後繼續另一個動作.很麻煩,所以動起了自己寫一個遊戲輔助的心思.

這個輔助本身沒什麼難度,就是通過不斷的截圖,然後從這個截圖中找出預先截好的能代表相應動作的按鈕或者觸發條件的小圖.

找到之後獲取該子區域的左上角坐標,然後通過windows API調用滑鼠或者鍵盤做操作就行了.

這裡面最難的也就是找圖了,因為要精準找圖,而且最好能適應不同的解析度下找圖,所以在模板匹配的基礎上,就有了SIFT和SURF的特徵點找圖方式.

在寫的過程中查找資料,大都是C++ 或者python的, 很少有原生的C#實現, 所以我就直接拿來翻譯過來了(稍作改動).

SIFT算法

public static Bitmap MatchPicBySift(Bitmap imgSrc, Bitmap imgSub)
{
using (Mat matSrc = imgSrc.ToMat())
using (Mat matTo = imgSub.ToMat())
using (Mat matSrcRet = new Mat())
using (Mat matToRet = new Mat())
{
KeyPoint[] keyPointsSrc, keyPointsTo;
using (var sift = OpenCvSharp.XFeatures2D.SIFT.Create())
{
sift.DetectAndCompute(matSrc, null, out keyPointsSrc, matSrcRet);
sift.DetectAndCompute(matTo, null, out keyPointsTo, matToRet);
}
using (var bfMatcher = new OpenCvSharp.BFMatcher())
{
var matches = bfMatcher.KnnMatch(matSrcRet, matToRet, k: 2);

var pointsSrc = new List<Point2f>();
var pointsDst = new List<Point2f>();
var goodMatches = new List<DMatch>();
foreach (DMatch[] items in matches.Where(x => x.Length > 1))
{
if (items[0].Distance < 0.5 * items[1].Distance)
{
pointsSrc.Add(keyPointsSrc[items[0].QueryIdx].Pt);
pointsDst.Add(keyPointsTo[items[0].TrainIdx].Pt);
goodMatches.Add(items[0]);
Console.WriteLine($"{keyPointsSrc[items[0].QueryIdx].Pt.X}, {keyPointsSrc[items[0].QueryIdx].Pt.Y}");
}
}

var outMat = new Mat();

// 算法RANSAC對匹配的結果做過濾
var pSrc = pointsSrc.ConvertAll(Point2fToPoint2d);
var pDst = pointsDst.ConvertAll(Point2fToPoint2d);
var outMask = new Mat();
// 如果原始的匹配結果為空, 則跳過過濾步驟
if (pSrc.Count > 0 && pDst.Count > 0)
Cv2.FindHomography(pSrc, pDst, HomographyMethods.Ransac, mask: outMask);
// 如果通過RANSAC處理後的匹配點大於10個,才應用過濾. 否則使用原始的匹配點結果(匹配點過少的時候通過RANSAC處理後,可能會得到0個匹配點的結果).
if (outMask.Rows > 10)
{
byte[] maskBytes = new byte[outMask.Rows * outMask.Cols];
outMask.GetArray(0, 0, maskBytes);
Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, matchesMask: maskBytes, flags: DrawMatchesFlags.NotDrawSinglePoints);
}
else
Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, flags: DrawMatchesFlags.NotDrawSinglePoints);
return OpenCvSharp.Extensions.BitmapConverter.ToBitmap(outMat);
}
}
}

SURF算法

public static Bitmap MatchPicBySurf(Bitmap imgSrc, Bitmap imgSub, double threshold = 400)
{
using (Mat matSrc = imgSrc.ToMat())
using (Mat matTo = imgSub.ToMat())
using (Mat matSrcRet = new Mat())
using (Mat matToRet = new Mat())
{
KeyPoint[] keyPointsSrc, keyPointsTo;
using (var surf = OpenCvSharp.XFeatures2D.SURF.Create(threshold,4,3,true,true))
{
surf.DetectAndCompute(matSrc, null, out keyPointsSrc, matSrcRet);
surf.DetectAndCompute(matTo, null, out keyPointsTo, matToRet);
}

using (var flnMatcher = new OpenCvSharp.FlannBasedMatcher())
{
var matches = flnMatcher.Match(matSrcRet, matToRet);
//求最小最大距離
double minDistance = 1000;//反向逼近
double maxDistance = 0;
for (int i = 0; i < matSrcRet.Rows; i++)
{
double distance = matches[i].Distance;
if (distance > maxDistance)
{
maxDistance = distance;
}
if (distance < minDistance)
{
minDistance = distance;
}
}
Console.WriteLine($"max distance : {maxDistance}");
Console.WriteLine($"min distance : {minDistance}");

var pointsSrc = new List<Point2f>();
var pointsDst = new List<Point2f>();
//篩選較好的匹配點
var goodMatches = new List<DMatch>();
for (int i = 0; i < matSrcRet.Rows; i++)
{
double distance = matches[i].Distance;
if (distance < Math.Max(minDistance * 2, 0.02))
{
pointsSrc.Add(keyPointsSrc[matches[i].QueryIdx].Pt);
pointsDst.Add(keyPointsTo[matches[i].TrainIdx].Pt);
//距離小於範圍的壓入新的DMatch
goodMatches.Add(matches[i]);
}
}

var outMat = new Mat();

// 算法RANSAC對匹配的結果做過濾
var pSrc = pointsSrc.ConvertAll(Point2fToPoint2d);
var pDst = pointsDst.ConvertAll(Point2fToPoint2d);
var outMask = new Mat();
// 如果原始的匹配結果為空, 則跳過過濾步驟
if (pSrc.Count > 0 && pDst.Count > 0)
Cv2.FindHomography(pSrc, pDst, HomographyMethods.Ransac, mask: outMask);
// 如果通過RANSAC處理後的匹配點大於10個,才應用過濾. 否則使用原始的匹配點結果(匹配點過少的時候通過RANSAC處理後,可能會得到0個匹配點的結果).
if (outMask.Rows > 10)
{
byte[] maskBytes = new byte[outMask.Rows * outMask.Cols];
outMask.GetArray(0, 0, maskBytes);
Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, matchesMask: maskBytes, flags: DrawMatchesFlags.NotDrawSinglePoints);
}
else
Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, flags: DrawMatchesFlags.NotDrawSinglePoints);
return OpenCvSharp.Extensions.BitmapConverter.ToBitmap(outMat);
}
}
}

模板匹配

public static System.Drawing.Point FindPicFromImage(Bitmap imgSrc, Bitmap imgSub, double threshold = 0.9)
{
OpenCvSharp.Mat srcMat = null;
OpenCvSharp.Mat dstMat = null;
OpenCvSharp.OutputArray outArray = null;
try
{
srcMat = imgSrc.ToMat();
dstMat = imgSub.ToMat();
outArray = OpenCvSharp.OutputArray.Create(srcMat);

OpenCvSharp.Cv2.MatchTemplate(srcMat, dstMat, outArray, Common.templateMatchModes);
double minValue, maxValue;
OpenCvSharp.Point location, point;
OpenCvSharp.Cv2.MinMaxLoc(OpenCvSharp.InputArray.Create(outArray.GetMat()), out minValue, out maxValue, out location, out point);
Console.WriteLine(maxValue);
if (maxValue >= threshold)
return new System.Drawing.Point(point.X, point.Y);
return System.Drawing.Point.Empty;
}
catch(Exception ex)
{
return System.Drawing.Point.Empty;
}
finally
{
if (srcMat != null)
srcMat.Dispose();
if (dstMat != null)
dstMat.Dispose();
if (outArray != null)
outArray.Dispose();
}
}