折腾笔记[33]-使用uiautomation自动重复读取图片(被控程序为.net框架)
摘要
基于python使用uiautomation自动操作.net程序.读取目录中png以及查找与其对应的json数据输入软件和点击按钮.
实现
1. 简化版的被控程序(靶点窗口)
配置文件:
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>WinExe</OutputType>
<TargetFramework>net8.0-windows</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<UseWPF>true</UseWPF>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TZ.HMI.Common.UI" Version="1.0.0.1" />
</ItemGroup>
<ItemGroup>
<None Update="处理中.png">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
<ItemGroup>
<ApplicationDefinition Update="App.xaml">
<SubType>Designer</SubType>
</ApplicationDefinition>
</ItemGroup>
<ItemGroup>
<Compile Update="JsonImportWindow.xaml.cs">
<SubType>Code</SubType>
</Compile>
</ItemGroup>
<ItemGroup>
<Page Update="JsonImportWindow.xaml">
<SubType>Designer</SubType>
</Page>
<Page Update="MainWindow.xaml">
<SubType>Designer</SubType>
</Page>
</ItemGroup>
</Project>
界面:
<Window x:Class="MockWpf.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
Title="OCR自动核对" Height="450" Width="800">
<Grid>
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="*"/>
</Grid.RowDefinitions>
<!-- 顶部按钮区域 -->
<StackPanel Orientation="Horizontal" Grid.Row="0" Margin="10" HorizontalAlignment="Left">
<Button Name="btnLoadImage" Content="读取图片" Width="100" Height="30" Margin="5"/>
<Button Name="btnOpenFolder" Content="打开文件夹" Width="100" Height="30" Margin="5"/>
<Button Name="btnPrevImage" Content="上一张" Width="100" Height="30" Margin="5"/>
<Button Name="btnNextImage" Content="下一张" Width="100" Height="30" Margin="5"/>
<Button Name="btnDebug" Content="调试" Width="100" Height="30" Margin="5"/>
<Button Name="btnMesImport" Content="Mes导入" Width="100" Height="30" Margin="5"/>
</StackPanel>
<!-- 图片显示区域 -->
<Image Name="imgDisplay" Grid.Row="1" Margin="10" Stretch="Uniform"/>
</Grid>
</Window>
<Window x:Class="MockWpf.JsonImportWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
Title="Json导入" Height="300" Width="500"
WindowStartupLocation="CenterOwner">
<Grid Margin="10">
<Grid.RowDefinitions>
<RowDefinition Height="*"/>
<RowDefinition Height="Auto"/>
</Grid.RowDefinitions>
<TextBox x:Name="txtJson"
Grid.Row="0"
TextWrapping="Wrap"
AcceptsReturn="True"
ScrollViewer.HorizontalScrollBarVisibility="Auto"
ScrollViewer.VerticalScrollBarVisibility="Auto"
FontFamily="Consolas"/>
<Button x:Name="btnOk"
Grid.Row="1"
Content="确认"
Width="80" Height="28"
HorizontalAlignment="Right"
Margin="0,10,0,0"
Click="BtnOk_Click"/>
</Grid>
</Window>
代码:
using Microsoft.Win32;
using System;
using System.Collections.Generic;
using System.IO;
using System.Windows;
using System.Windows.Media.Imaging;
namespace MockWpf
{
public partial class MainWindow : Window
{
private List<string> imageFiles = new List<string>();
private int currentIndex = -1;
public MainWindow()
{
InitializeComponent();
btnOpenFolder.Click += BtnOpenFolder_Click;
btnLoadImage.Click += BtnLoadImage_Click;
btnPrevImage.Click += BtnPrevImage_Click;
btnNextImage.Click += BtnNextImage_Click;
btnDebug.Click += BtnDebug_Click;
btnMesImport.Click += BtnMesImport_Click;
}
private void BtnMesImport_Click(object sender, RoutedEventArgs e)
{
var w = new JsonImportWindow { Owner = this };
if (w.ShowDialog() == true) // 用户点了“确认”
{
string json = w.JsonText;
// TODO: 在这里写真正的 Json 解析/导入逻辑
// MessageBox.Show($"已获取到 Json 内容（长度 {json.Length}）\n{json}");
}
}
private void BtnDebug_Click(object sender, RoutedEventArgs e)
{
// 新窗口
var win = new Window
{
Owner = this,
Width = 600,
Height = 400,
WindowStartupLocation = WindowStartupLocation.CenterOwner,
WindowStyle = WindowStyle.ToolWindow,
ResizeMode = ResizeMode.NoResize,
Title = "提示"
};
// 图片控件
var img = new System.Windows.Controls.Image
{
Stretch = System.Windows.Media.Stretch.Uniform
};
try
{
img.Source = new BitmapImage(new Uri("./处理中.png", UriKind.Relative));
}
catch { /* 图片不存在则留空 */ }
win.Content = img;
win.Show();
// 3 秒后自动关闭
var timer = new System.Windows.Threading.DispatcherTimer
{
Interval = TimeSpan.FromSeconds(3)
};
timer.Tick += (_, __) =>
{
timer.Stop();
win.Close();
};
timer.Start();
}
private void BtnOpenFolder_Click(object sender, RoutedEventArgs e)
{
var dialog = new System.Windows.Forms.FolderBrowserDialog();
if (dialog.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
var folderPath = dialog.SelectedPath;
imageFiles = new List<string>(Directory.GetFiles(folderPath, "*.jpg"));
imageFiles.AddRange(Directory.GetFiles(folderPath, "*.png"));
imageFiles.AddRange(Directory.GetFiles(folderPath, "*.bmp"));
if (imageFiles.Count > 0)
{
currentIndex = 0;
LoadImage(imageFiles[currentIndex]);
}
else
{
MessageBox.Show("未找到图片文件");
}
}
}
private void BtnLoadImage_Click(object sender, RoutedEventArgs e)
{
OpenFileDialog openFileDialog = new OpenFileDialog
{
Filter = "图片文件|*.jpg;*.jpeg;*.png;*.bmp",
Title = "选择图片"
};
if (openFileDialog.ShowDialog() == true)
{
LoadImage(openFileDialog.FileName);
}
}
private void BtnPrevImage_Click(object sender, RoutedEventArgs e)
{
if (imageFiles.Count == 0) return;
currentIndex = (currentIndex - 1 + imageFiles.Count) % imageFiles.Count;
LoadImage(imageFiles[currentIndex]);
}
private void BtnNextImage_Click(object sender, RoutedEventArgs e)
{
if (imageFiles.Count == 0) return;
currentIndex = (currentIndex + 1) % imageFiles.Count;
LoadImage(imageFiles[currentIndex]);
}
private void LoadImage(string path)
{
try
{
var bitmap = new BitmapImage();
bitmap.BeginInit();
bitmap.UriSource = new Uri(path);
bitmap.CacheOption = BitmapCacheOption.OnLoad;
bitmap.EndInit();
imgDisplay.Source = bitmap;
}
catch (Exception ex)
{
MessageBox.Show($"加载图片失败: {ex.Message}");
}
}
}
}
using System;
using System.Windows;
namespace MockWpf
{
public partial class JsonImportWindow : Window
{
public string JsonText => txtJson.Text;
public JsonImportWindow()
{
InitializeComponent();
}
// 点击“确认”后把对话框结果设为 OK，主窗口就能取到 JsonText
private void BtnOk_Click(object sender, RoutedEventArgs e)
{
DialogResult = true;
// 关闭窗口
Close();
}
}
}
2. python脚本
pyproject.toml
[project]
name = "exp18-pyloid-pyautogui"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.13"
dependencies = [
"comtypes>=1.4.12",
"onnxruntime>=1.22.1",
"pillow>=11.3.0",
"psutil>=7.0.0",
"pyautogui>=0.9.54",
"pyinstaller>=6.15.0",
"pyloid>=0.24.8",
"pyperclip>=1.9.0",
"pyscreeze>=1.0.1",
"pywin32>=311",
"rapidocr>=3.4.0",
"uiautomation>=2.0.29",
]
脚本:
# -*- coding: utf-8 -*-
import uiautomation as auto
import win32api, win32con, win32gui, win32console, time, sys, os
import datetime
import pyperclip
CANDIDATE_TITLES = ['靶点窗口', 'OCR自动核对']
TASKS = ['Mes导入']
DEFAULT_IMG_DIR = r'D:\AppTest\测试工作空间'
# ---------------- 日志 ----------------
class _RealTimeTee:
def __init__(self, log_path: str):
# 行缓冲，保证实时落盘
self.file = open(log_path, 'a', encoding='utf-8', buffering=1)
# 防御：无控制台时退化为只写文件
self.console = sys.__stdout__ if sys.__stdout__ is not None else None
self._write_header(log_path)
def _write_header(self, log_path: str):
head = f'日志文件：{os.path.abspath(log_path)}\n'
self.file.write(head)
if self.console:
self.console.write(head)
def write(self, data: str):
self.file.write(data)
if self.console:
self.console.write(data)
self.flush()
def flush(self):
self.file.flush()
if self.console:
self.console.flush()
def close(self):
self.flush()
self.file.close()
# --------------------------------------------------
# 日志：仅提供 printLog 函数，不替换 sys.stdout
# --------------------------------------------------
_log_file = None # 全局日志句柄
def init_log(log_path: str):
"""初始化日志文件，写入头信息"""
global _log_file
_log_file = open(log_path, 'a', encoding='utf-8', buffering=1)
header = f'日志文件：{os.path.abspath(log_path)}\n'
printLog(header, end='') # 屏幕
_log_file.write(header) # 文件
def printLog(*args, **kwargs):
"""同时打印到屏幕并写入日志"""
if _log_file is None:
raise RuntimeError('请先调用 init_log() 初始化日志')
# 构造输出字符串
sep = kwargs.get('sep', ' ')
end = kwargs.get('end', '\n')
msg = sep.join(map(str, args)) + end
print(msg, end='') # 屏幕
_log_file.write(msg) # 文件
_log_file.flush()
def close_log():
"""程序结束前关闭日志句柄"""
global _log_file
if _log_file:
_log_file.close()
_log_file = None
# ---------------- 控制台工具 ----------------
def alloc_console():
if win32console.GetConsoleWindow() == 0:
win32console.AllocConsole()
sys.stdout = open('CONOUT$', 'w', encoding='utf-8')
sys.stdin = open('CONIN$', 'r', encoding='utf-8')
sys.stderr = sys.stdout
def show_console(visible=True, topmost=False):
hwnd = win32console.GetConsoleWindow()
if not hwnd:
return
win32gui.ShowWindow(hwnd,
win32con.SW_SHOW if visible else win32con.SW_MINIMIZE)
if topmost:
win32gui.SetWindowPos(hwnd, win32con.HWND_TOPMOST,
0, 0, 0, 0,
win32con.SWP_NOMOVE | win32con.SWP_NOSIZE)
# ---------------- 鼠标/高亮 ----------------
def highlight_rect(rect, sec=3):
left, top, right, bottom = map(int, (rect.left, rect.top,
rect.right, rect.bottom))
hdc = win32gui.GetDC(0)
pen = win32gui.CreatePen(win32con.PS_SOLID, 10, win32api.RGB(255, 0, 0))
old_pen = win32gui.SelectObject(hdc, pen)
win32gui.SelectObject(hdc, win32gui.GetStockObject(win32con.NULL_BRUSH))
for _ in range(int(sec * 5)):
win32gui.Rectangle(hdc, left, top, right, bottom)
time.sleep(0.1)
win32gui.SelectObject(hdc, old_pen)
win32gui.ReleaseDC(0, hdc)
def real_click(rect):
x = int((rect.left + rect.right) / 2)
y = int((rect.top + rect.bottom) / 2)
win32api.SetCursorPos((x, y))
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, 0, 0)
time.sleep(0.02)
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, 0, 0)
def deep_click(ctrl):
rect = ctrl.BoundingRectangle
highlight_rect(rect, 1)
sub = ctrl.GetFirstChildControl()
while sub:
if sub.ControlType in [auto.ButtonControl, auto.TextControl,
auto.CustomControl] \
and abs(sub.BoundingRectangle.left - rect.left) <= 5 \
and abs(sub.BoundingRectangle.top - rect.top) <= 5:
printLog(' 将点击子控件：', sub.Name)
rect = sub.BoundingRectangle
break
sub = sub.GetNextSiblingControl()
real_click(rect)
# ---------------- 业务逻辑 ----------------
def find_window():
for ttl in CANDIDATE_TITLES:
w = auto.WindowControl(searchDepth=1, Name=ttl)
if w.Exists(0.5):
printLog('已找到窗口：', ttl)
return w
return None
def click_button(win, name):
for tp in ['ButtonControl', 'CustomControl', 'TextControl']:
btn = getattr(win, tp)(Name=name)
if btn.Exists():
printLog('找到按钮：', name)
deep_click(btn)
return True
printLog('未找到按钮：', name)
return False
def wait_disappear(ctrl: auto.Control, timeout: int = 10):
for _ in range(int(timeout * 10)):
if not ctrl.Exists(0.05):
return
time.sleep(0.1)
raise RuntimeError(f'等待 {ctrl.Name} 消失超时')
# ---------------- 单次完整流程 ----------------
def single_flow(win: auto.WindowControl, png_path: str, json_path: str) -> float:
import win32gui as wg
import win32con as wc
import win32api as wa
t0 = time.perf_counter()
img_name = os.path.basename(png_path)
# 1. 读取图片（键盘版，原逻辑不变）
t1 = time.perf_counter()
if not click_button(win, '读取图片'):
raise RuntimeError('未找到【读取图片】按钮')
def _find_open_dialog():
lst = []
def enum_cb(hwnd, _):
if wg.GetClassName(hwnd) == '#32770' and wg.IsWindowVisible(hwnd):
lst.append(hwnd)
return True
wg.EnumWindows(enum_cb, None)
return lst[0] if lst else None
for _ in range(30):
h_dlg = _find_open_dialog()
if h_dlg:
break
time.sleep(0.1)
else:
raise RuntimeError('等待文件选择器超时')
if wg.IsIconic(h_dlg):
wg.ShowWindow(h_dlg, wc.SW_RESTORE)
wg.SetForegroundWindow(h_dlg)
# Alt+N
wa.keybd_event(wc.VK_MENU, 0, 0, 0); time.sleep(0.02)
wa.keybd_event(0x4E, 0, 0, 0); time.sleep(0.02)
wa.keybd_event(0x4E, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.02)
wa.keybd_event(wc.VK_MENU, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.2)
# Ctrl+A 并输入路径
wa.keybd_event(wc.VK_CONTROL, 0, 0, 0); time.sleep(0.02)
wa.keybd_event(0x41, 0, 0, 0); time.sleep(0.02)
wa.keybd_event(0x41, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.02)
wa.keybd_event(wc.VK_CONTROL, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.1)
auto.SendKeys(png_path); time.sleep(0.1)
# Alt+O
wa.keybd_event(wc.VK_MENU, 0, 0, 0); time.sleep(0.02)
wa.keybd_event(0x4F, 0, 0, 0); time.sleep(0.02)
wa.keybd_event(0x4F, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.02)
wa.keybd_event(wc.VK_MENU, 0, wc.KEYEVENTF_KEYUP, 0)
for _ in range(50):
if not wg.IsWindow(h_dlg):
break
time.sleep(0.1)
else:
raise RuntimeError('文件对话框仍未关闭')
t1 = time.perf_counter() - t1
# 2. Mes 导入 —— 采用“参考代码”剪贴板方案
t2 = time.perf_counter()
if not click_button(win, 'Mes导入'):
raise RuntimeError('未找到【Mes导入】按钮')
# 等待 Json 导入窗口
json_dlg = auto.WindowControl(searchDepth=2, Name='Json导入')
if not json_dlg.Exists(3):
raise RuntimeError('等待 Json导入 窗口超时')
# 读取 json 内容
try:
with open(json_path, 'r', encoding='utf-8') as f:
json_txt = f.read()
json_ok = True
except Exception as e:
json_txt = ''
json_ok = False
printLog('[警告] 读取 json 失败：', e)
# 找到编辑框并聚焦
edit_j = json_dlg.EditControl(foundIndex=1)
if not edit_j.Exists():
# 某些版本用 Custom 包了一层
edit_j = json_dlg.CustomControl(foundIndex=1).EditControl(foundIndex=1)
edit_j.SetFocus()
# 参考代码做法：全选 + 剪贴板写入 + Ctrl+V
auto.SendKeys('{Ctrl}A'); time.sleep(0.05)
pyperclip.copy(json_txt); time.sleep(0.05)
auto.SendKeys('{Ctrl}V'); time.sleep(0.1)
# 确定按钮
ok_btn = json_dlg.ButtonControl(Name='确认')
if ok_btn.Exists():
deep_click(ok_btn)
else:
json_dlg.SendKeys('{Enter}')
wait_disappear(json_dlg, 5)
t2 = time.perf_counter() - t2
# 3. 调试（原逻辑不变）
t3 = time.perf_counter()
if not click_button(win, '调试'):
raise RuntimeError('未找到【调试】按钮')
t3 = time.perf_counter() - t3
# 4. 等待提示窗（原逻辑不变）
t4 = time.perf_counter()
tip = auto.WindowControl(searchDepth=2, SubName='提示')
if tip.Exists(1):
wait_disappear(tip, 15)
t4 = time.perf_counter() - t4
total = time.perf_counter() - t0
printLog(f'[{img_name}] 读取图片:{t1:.2f}s '
f'Mes导入:{t2:.2f}s 调试:{t3:.2f}s 等待提示:{t4:.2f}s '
f'json成功:{json_ok} 总耗时:{total:.2f}s')
return total
# ---------------- 主流程 ----------------
def main():
alloc_console()
show_console(visible=True, topmost=True)
# ---- 生成唯一日志文件名 ----
log_name = datetime.datetime.now().strftime(r'自动测试_%Y%m%d_%H%M%S.log')
init_log(log_name) # 初始化日志
# 询问图片文件夹
img_dir = input(f'请输入图片文件夹路径(直接回车使用默认值)：\n{DEFAULT_IMG_DIR}\n> ').strip()
if not img_dir:
img_dir = DEFAULT_IMG_DIR
printLog('使用文件夹：', img_dir)
tasks = []
for fn in sorted(os.listdir(img_dir)):
if fn.lower().endswith('.png'):
json_fn = fn[:-4] + '.json'
json_path = os.path.join(img_dir, json_fn)
if os.path.isfile(json_path):
tasks.append((os.path.join(img_dir, fn), json_path))
else:
printLog(f'[跳过] {fn} 未找到同名 json')
if not tasks:
printLog('文件夹内没有可执行的 png/json 对！')
show_console(visible=True, topmost=True)
input('按回车键退出…')
return
total_cnt = len(tasks)
printLog(f'\n共发现 {total_cnt} 组待执行任务，即将开始自动化…\n')
show_console(visible=False)
win = find_window()
if not win:
printLog('未找到任何候选窗口：', CANDIDATE_TITLES)
show_console(visible=True, topmost=True)
input('按回车键退出…')
return
done = 0
sum_time = 0.0
try:
for png, json_ in tasks:
done += 1
cost = single_flow(win, png, json_)
sum_time += cost
printLog(f'[总进度] 已执行 {done}/{total_cnt} 张 累计耗时 {sum_time:.2f} s')
except Exception as e:
printLog('\n[异常] 流程中断：', e)
finally:
show_console(visible=True, topmost=True)
printLog(f'\n全部任务执行完毕！ 成功 {done}/{total_cnt} 张 总耗时 {sum_time:.2f} s')
input('按回车键退出…')
if __name__ == '__main__':
main()
打包:
uv run pyinstaller -F -w auto_test.py
浙公网安备 33010602011771号