多谢支持
整体性能还比较好,识别一次基本上在几十 ms。
如果控件是一个图标的话,可以试一下基于图像识别的方法。
简单讲就是先把需要识别的图片截图,然后在需要寻找该图像的时候,调用封装好的方法
贴一个以前写的代码,楼主参考一下,是基于 opencv 的。
OpencvRecognition.match 方法接收 2 个参数,分别是需要寻找的图标文件路径,完整的屏幕截图文件路径,返回识别到的中心坐标。
并且会创建一个新的图片,标识出识别到的具体位置,用来 debug。
import org.opencv.calib3d.Calib3d;
import org.opencv.core.*;
import org.opencv.features2d.*;
import org.opencv.highgui.Highgui;
import java.io.File;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
/**
* @author weijianrao
*/
public class OpencvRecognition {
public static double[] match(String template, String source) {
double[] ret = new double[2];
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
System.out.println("开始匹配");
System.out.println("加载图片");
Mat templateImage = Highgui.imread(template, Highgui.CV_LOAD_IMAGE_COLOR);
Mat sourceImage = Highgui.imread(source, Highgui.CV_LOAD_IMAGE_COLOR);
MatOfKeyPoint templateKeyPoints = new MatOfKeyPoint();
FeatureDetector detector = FeatureDetector.create(FeatureDetector.SURF);
System.out.println("检测关键点");
detector.detect(templateImage, templateKeyPoints);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.SURF);
MatOfKeyPoint templateDescriptors = new MatOfKeyPoint();
System.out.println("计算描述符");
extractor.compute(templateImage, templateKeyPoints, templateDescriptors);
// Create the matrix for output image.
Mat outputImage = new Mat(templateImage.rows(), templateImage.cols(), Highgui.CV_LOAD_IMAGE_COLOR);
Scalar newKeypointColor = new Scalar(255, 0, 0);
System.out.println("Drawing key points on object image...");
Features2d.drawKeypoints(templateImage, templateKeyPoints, outputImage, newKeypointColor, 0);
// Match object image with the scene image
MatOfKeyPoint sourceKeyPoints = new MatOfKeyPoint();
MatOfKeyPoint sourceDescriptors = new MatOfKeyPoint();
System.out.println("Detecting key points in background image...");
detector.detect(sourceImage, sourceKeyPoints);
System.out.println("Computing descriptors in background image...");
extractor.compute(sourceImage, sourceKeyPoints, sourceDescriptors);
Mat matchoutput = new Mat(sourceImage.rows() * 2, sourceImage.cols() * 2, Highgui.CV_LOAD_IMAGE_COLOR);
Scalar matchestColor = new Scalar(0, 255, 0);
List<MatOfDMatch> matches = new LinkedList<MatOfDMatch>();
DescriptorMatcher descriptorMatcher = DescriptorMatcher.create(DescriptorMatcher.FLANNBASED);
System.out.println("Matching object and scene images...");
descriptorMatcher.knnMatch(templateDescriptors, sourceDescriptors, matches, 2);
System.out.println("Calculating good match list...");
LinkedList<DMatch> goodMatchesList = new LinkedList<DMatch>();
float nndrRatio = 0.7f;
for (int i = 0; i < matches.size(); i++) {
MatOfDMatch matofDMatch = matches.get(i);
DMatch[] dmatcharray = matofDMatch.toArray();
DMatch m1 = dmatcharray[0];
DMatch m2 = dmatcharray[1];
if (m1.distance <= m2.distance * nndrRatio) {
goodMatchesList.addLast(m1);
}
}
if (goodMatchesList.size() >= 7) {
System.out.println("找到了!!!");
List<KeyPoint> templateKeypointlist = templateKeyPoints.toList();
List<KeyPoint> sourceKeypointlist = sourceKeyPoints.toList();
LinkedList<Point> templatePoints = new LinkedList<Point>();
LinkedList<Point> sourcePoints = new LinkedList<Point>();
for (int i = 0; i < goodMatchesList.size(); i++) {
templatePoints.addLast(templateKeypointlist.get(goodMatchesList.get(i).queryIdx).pt);
sourcePoints.addLast(sourceKeypointlist.get(goodMatchesList.get(i).trainIdx).pt);
}
MatOfPoint2f templateMatOfPoint2f = new MatOfPoint2f();
templateMatOfPoint2f.fromList(templatePoints);
MatOfPoint2f scnMatOfPoint2f = new MatOfPoint2f();
scnMatOfPoint2f.fromList(sourcePoints);
Mat homography = Calib3d.findHomography(templateMatOfPoint2f, scnMatOfPoint2f, Calib3d.RANSAC, 3);
Mat templateCorners = new Mat(4, 1, CvType.CV_32FC2);
Mat sourceCorners = new Mat(4, 1, CvType.CV_32FC2);
templateCorners.put(0, 0, new double[]{0, 0});
templateCorners.put(1, 0, new double[]{templateImage.cols(), 0});
templateCorners.put(2, 0, new double[]{templateImage.cols(), templateImage.rows()});
templateCorners.put(3, 0, new double[]{0, templateImage.rows()});
System.out.println("Transforming object corners to scene corners...");
Core.perspectiveTransform(templateCorners, sourceCorners, homography);
Mat img = Highgui.imread(source, Highgui.CV_LOAD_IMAGE_COLOR);
Core.line(img, new Point(sourceCorners.get(0, 0)), new Point(sourceCorners.get(1, 0)), new Scalar(0, 255, 0), 4);
Core.line(img, new Point(sourceCorners.get(1, 0)), new Point(sourceCorners.get(2, 0)), new Scalar(0, 255, 0), 4);
Core.line(img, new Point(sourceCorners.get(2, 0)), new Point(sourceCorners.get(3, 0)), new Scalar(0, 255, 0), 4);
Core.line(img, new Point(sourceCorners.get(3, 0)), new Point(sourceCorners.get(0, 0)), new Scalar(0, 255, 0), 4);
ret[0] = (sourceCorners.get(0, 0)[0] + sourceCorners.get(1, 0)[0] + sourceCorners.get(2, 0)[0] + sourceCorners.get(3, 0)[0]) / 4;
ret[1] = (sourceCorners.get(0, 0)[1] + sourceCorners.get(1, 0)[1] + sourceCorners.get(2, 0)[1] + sourceCorners.get(3, 0)[1]) / 4;
Core.circle(img, new Point(ret), 10, new Scalar(0, 0, 255), 3);
System.out.println("Drawing matches image...");
MatOfDMatch goodMatches = new MatOfDMatch();
goodMatches.fromList(goodMatchesList);
Features2d.drawMatches(templateImage, templateKeyPoints, sourceImage, sourceKeyPoints, goodMatches, matchoutput, matchestColor, newKeypointColor, new MatOfByte(), 2);
//Highgui.imwrite("C:\\Users\\weijianrao.LEXINFINTECH\\Desktop\\outputImage.jpg", outputImage);
//Highgui.imwrite("C:\\Users\\weijianrao.LEXINFINTECH\\Desktop\\matchoutput.jpg", matchoutput);
String dirName = new File(source).getParent();
String baseName = new File(source).getName().replace(".png", "") + "_debug" + ".png";
String debugPic = Paths.get(dirName).resolve(baseName).toString();
Highgui.imwrite(debugPic, img);
} else {
System.out.println("Object Not Found");
}
return ret;
}
}
如果提示 command not found,可以确定那个可执行程序一定不在 PATH 里。
你可以把 PATH 变量打印出来看看。echo $PATH
jenkins 登录 slave 时,并不会 source /etc/bashrc, ~/.bashrc, ~/.bash_profile 等配置文件。
看来增量覆盖率是刚需