如何使 locateCenterOnScreen 更准确 -PYTHON-, -WINDOWS-

Question

你好，我的目标是能够让我的机器人在任何屏幕尺寸上点击我选择的东西，因为我认为这是主要问题。我试图降低置信度，但它最终只是点击了具有相同一般颜色的其他东西。我用精确的图像对其进行了测试，它点击了正确的位置，所以它不像坐标关闭或任何它只是图像识别。

这些是要经过的图像

（X1，下一课，箭头）

from pyautogui import *
import pyautogui
import time
import keyboard
import random

def NextLesson():
    keepGoing = True
    while keepGoing == True:
        counter = 0
        nl_coordinates = pyautogui.locateOnScreen('images/nextLesson.png', confidence=0.4)
        print(nl_coordinates)  # This will print out where it is
        if nl_coordinates:
            print(f"I can see it at {nl_coordinates}")
            pyautogui.click(nl_coordinates)
            keepGoing = False
        else:
            print("I cannot see it.")

def Arrow():
    keepGoing = True
    while keepGoing == True:
        counter = 0
        arrow_coordinates = pyautogui.locateOnScreen('images/arrow.png', confidence=0.4)
        print(arrow_coordinates)  # This will print out where it is
        if arrow_coordinates:
            print(f"I can see it at {arrow_coordinates}")
            pyautogui.click(arrow_coordinates)
            keepGoing = False
        else:
            print("I cannot see it.")

def X1():
    keepGoing = True
    while keepGoing == True:
        counter = 0
        x1_coordinates = pyautogui.locateOnScreen('images/x1.png', confidence=0.4)
        print(x1_coordinates)  # This will print out where it is
        if x1_coordinates:
            print(f"I can see it at {x1_coordinates}")
            pyautogui.click(x1_coordinates)
            keepGoing = False
        else:
            print("I cannot see it.")

while True:
    counter = 0
    counter2 = 0
    true = True

    time.sleep(2)
    X1()#
    time.sleep(8)
    NextLesson()#
    time.sleep(10)
    Arrow()#
    print("calibration complete ")


    time.sleep(5)
    cords = pyautogui.position()

    while counter != 1800:
        time.sleep(60)
        pyautogui.click(cords) #clicking where ouse is at
        print("clicked")
        counter += 60
        print(counter)
    if counter == 1800:
        time.sleep(5) #stops code for 5 secs
        X1() #clicks mouse to x button
        print("clicked x")

        time.sleep(5) #stops code for 5 secs
        NextLesson() #clicks mouse to the assignment button
        print("clicked assignemnt")

        time.sleep(15) #stops code for 2 secs
        Arrow() #clicks mouse to the second assignment button
        print("clicked 2nd assignment button ")

        time.sleep(5) #waits 5secs to put cursor at position
        cords = pyautogui.position() #grabs position
        print("grabbed position")

Answer 1

我们可以使用opencv-python进行多尺度模板匹配。这个想法是缩放模板图像并尝试在屏幕截图中找到调整大小的模板。下面的代码（改编自 here）在 [0.25,2] 范围内循环 50 个缩放参数并选择提供最佳匹配的那个。您可能希望降低缩放参数的范围或数量以提高效率。最好保存正确的缩放参数并将其重复用于多个图像。

import cv2
import pyscreeze
import numpy as np
import imutils
import pyautogui

def template_match_with_scaling(image,gs=True,confidence=0.8):
"""
Locate an image and return a pyscreeze box surrounding it. 
Template matching is done by default in grayscale (gs=True)
Detect image if normalized correlation coefficient is > confidence (0.8 is default)
"""
    templateim = pyscreeze._load_cv2(image,grayscale=gs) # template image
    (tH, tW)   = templateim.shape[:2]

    screenim_color = pyautogui.screenshot() # screenshot of image
    screenim_color = cv2.cvtColor(np.array(screenim_color),cv2.COLOR_RGB2BGR)
    if gs is True:
       screenim = cv2.cvtColor(np.array(screenim_color),cv2.COLOR_BGR2GRAY)
    else:
       screenim = screenim_color

    #try different scaling parameters and see which one matches best
    found = None #bookeeping variable for the maximum correlation coefficient, position and scale
    scalingrange = np.linspace(0.25,2,num=50)
    for scale in scalingrange:
        resizedtemplate = imutils.resize(templateim,  width = int(templateim.shape[1]*scale) ) # resizing with  imutils maintains the aspect ratio
        r = float(resizedtemplate.shape[1])/templateim.shape[1] # recompute scaling factor
        result = cv2.matchTemplate(screenim, resizedtemplate, cv2.TM_CCOEFF_NORMED) # template matching using the correlation coefficient
        (_, maxVal, _, maxLoc) = cv2.minMaxLoc(result) #returns a 4-tuple which includes the minimum correlation value, the maximum correlation value, the (x, y)-coordinate of the minimum value, and the (x, y)-coordinate of the maximum value
        if found is None or maxVal > found[0]:
           found = (maxVal, maxLoc, r)

    (maxVal, maxLoc, r) = found
    if maxVal > confidence:
       box = pyscreeze.Box(int(maxLoc[0]), int(maxLoc[1]), int(tW*r), int(tH*r) )
       return box
    else:
       return None

def locate_center_with_scaling(image,gs=True):
    loc = template_match_with_scaling(image,gs=gs) 
    if loc:
       return pyautogui.center(loc)
    else:
       raise Exception("Image not found")

#sample usage 
coords = locate_center_with_scaling('images/arrow.png')

Answer 2

我找到的答案是拍摄一张我需要的更准确的照片，比如尝试尽可能多地去除灰度 = true 背景并增加信心

如何使 locateCenterOnScreen 更准确 -PYTHON-, -WINDOWS-

how to make locateCenterOnScreen more accurate -PYTHON-, -WINDOWS-

python

windows

automation

image-recognition

pyautogui