录制回放测试框架的使用
项目结构
automation_framework/
├── main.py
├── requirements.txt
├── run.bat
├── run.sh
├── config/
│ ├── __init__.py
│ └── settings.py
├── core/
│ ├── __init__.py
│ ├── models.py
│ ├── recorder.py
│ ├── player.py
│ ├── cmd_recorder.py
│ ├── cmd_player.py
│ ├── hybrid_recorder.py
│ ├── hybrid_player.py
│ └── unified_controller.py
├── ocr/
│ ├── __init__.py
│ ├── processor.py
│ ├── checker.py
│ └── selector.py
├── gui/
│ ├── __init__.py
│ └── main_window.py
└── utils/
├── __init__.py
├── logger.py
├── file_utils.py
└── screenshot.py
- main.py
"""
主入口
"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from gui.main_window import MainWindow
def check_dependencies():
try:
import pytesseract, cv2, pyautogui, pynput
return True
except ImportError as e:
print(f"❌ 缺少依赖: {e}")
print("请安装: pip install -r requirements.txt")
return False
def check_tesseract():
try:
import pytesseract
pytesseract.get_tesseract_version()
return True
except:
print("❌ 未找到Tesseract OCR")
print("请安装: https://github.com/UB-Mannheim/tesseract/wiki")
return False
def main():
print("=" * 60)
print("🤖 自动化测试工具 (UI + 命令行)")
print("=" * 60)
if not check_dependencies():
return 1
if not check_tesseract():
return 1
try:
app = MainWindow()
app.run()
except KeyboardInterrupt:
print("\n程序已退出")
except Exception as e:
print(f"❌ 程序错误: {e}")
return 1
return 0
if __name__ == "__main__":
exit(main())
- requirements.txt
# 核心依赖
pyautogui==0.9.54
pygetwindow==0.0.9
pynput==1.7.6
keyboard==0.13.5
# OCR依赖
pytesseract==0.3.10
opencv-python==4.8.1.78
Pillow==10.0.0
# 其他
numpy==1.24.3
- run.bat
@echo off
chcp 65001 >nul
echo ========================================
echo 🤖 自动化测试工具 (UI + 命令行)
echo ========================================
echo.
python --version >nul 2>&1
if errorlevel 1 (
echo ❌ 未找到Python，请安装Python 3.8+
pause
exit /b 1
)
echo 检查依赖...
pip show pyautogui >nul 2>&1
if errorlevel 1 (
echo 正在安装依赖...
pip install -r requirements.txt
)
echo 启动程序...
python main.py
pause
- run.sh
#!/bin/bash
echo "========================================"
echo " 🤖 自动化测试工具 (UI + 命令行)"
echo "========================================"
echo ""
if ! command -v python3 &> /dev/null; then
echo "❌ 未找到Python3，请安装Python 3.8+"
exit 1
fi
echo "检查依赖..."
pip3 show pyautogui &> /dev/null
if [ $? -ne 0 ]; then
echo "正在安装依赖..."
pip3 install -r requirements.txt
fi
echo "启动程序..."
python3 main.py
- config/init.py
"""配置模块"""
from config.settings import Settings, settings
__all__ = ['Settings', 'settings']
- config/settings.py
"""
配置管理
"""
from pathlib import Path
from typing import Dict, Any
import json
class Settings:
"""应用配置"""
def __init__(self):
self.base_dir = Path(__file__).parent.parent
self.data_dir = self.base_dir / "data"
self.recordings_dir = self.data_dir / "recordings"
self.templates_dir = self.data_dir / "templates"
self.logs_dir = self.base_dir / "logs"
self._create_directories()
self.config = {
'ocr': {
'lang': 'chi_sim+eng',
'psm': 6,
'whitelist': '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz.-+%: '
},
'recording': {
'default_speed': 1.0,
'action_delay': 0.2,
'screenshot_on_action': False
},
'player': {
'retry_count': 3,
'retry_delay': 0.5,
'smart_locate': True
},
'gui': {
'window_size': (1200, 800),
'theme': 'default'
}
}
self.regions_file = self.data_dir / "ocr_regions.json"
self.load_regions()
def _create_directories(self):
for dir_path in [self.data_dir, self.recordings_dir, self.templates_dir, self.logs_dir]:
dir_path.mkdir(parents=True, exist_ok=True)
def load_regions(self) -> Dict:
if self.regions_file.exists():
try:
with open(self.regions_file, 'r', encoding='utf-8') as f:
return json.load(f)
except Exception:
return {}
return {}
def save_regions(self, regions: Dict):
with open(self.regions_file, 'w', encoding='utf-8') as f:
json.dump(regions, f, indent=2, ensure_ascii=False)
def get(self, key: str, default=None):
keys = key.split('.')
value = self.config
for k in keys:
if isinstance(value, dict):
value = value.get(k)
else:
return default
return value if value is not None else default
def set(self, key: str, value: Any):
keys = key.split('.')
target = self.config
for k in keys[:-1]:
if k not in target:
target[k] = {}
target = target[k]
target[keys[-1]] = value
settings = Settings()
- core/init.py
"""核心模块"""
from core.models import Action, CommandAction, OCRRegion, Recording, CommandExecutionResult, VerificationResult
from core.recorder import ActionRecorder
from core.player import ActionPlayer
from core.cmd_recorder import CommandRecorder
from core.cmd_player import CommandPlayer
from core.hybrid_recorder import HybridRecorder
from core.hybrid_player import HybridPlayer
from core.unified_controller import UnifiedController
__all__ = [
'Action', 'CommandAction', 'OCRRegion', 'Recording',
'CommandExecutionResult', 'VerificationResult',
'ActionRecorder', 'ActionPlayer',
'CommandRecorder', 'CommandPlayer',
'HybridRecorder', 'HybridPlayer',
'UnifiedController'
]
- core/models.py
"""
数据模型定义
"""
from dataclasses import dataclass, asdict, field
from typing import Dict, Any, Optional, List
from datetime import datetime
@dataclass
class OCRRegion:
"""OCR区域数据模型"""
name: str
x: int
y: int
width: int
height: int
description: str = ""
expected_value: str = ""
check_type: str = "contains"
def to_dict(self) -> Dict:
return asdict(self)
def get_region(self) -> tuple:
return (self.x, self.y, self.width, self.height)
def validate(self) -> bool:
return self.width > 0 and self.height > 0
@dataclass
class Action:
"""操作数据模型"""
type: str
params: Dict[str, Any]
timestamp: float
description: str = ""
screenshot: Optional[str] = None
def to_dict(self) -> Dict:
return asdict(self)
@classmethod
def from_dict(cls, data: Dict) -> 'Action':
return cls(**data)
@dataclass
class CommandAction(Action):
"""命令行操作数据模型"""
cmd: str = ""
args: List[str] = field(default_factory=list)
working_dir: str = ""
timeout: int = 30
expected_output: str = ""
check_type: str = "contains"
env_vars: Dict[str, str] = field(default_factory=dict)
def to_dict(self) -> Dict:
data = super().to_dict()
data.update({
'cmd': self.cmd,
'args': self.args,
'working_dir': self.working_dir,
'timeout': self.timeout,
'expected_output': self.expected_output,
'check_type': self.check_type,
'env_vars': self.env_vars
})
return data
@classmethod
def from_dict(cls, data: Dict) -> 'CommandAction':
params = data.get('params', {})
return cls(
type=data.get('type', 'cmd_execute'),
params=params,
timestamp=data.get('timestamp', 0),
description=data.get('description', ''),
screenshot=data.get('screenshot'),
cmd=data.get('cmd', params.get('cmd', '')),
args=data.get('args', params.get('args', [])),
working_dir=data.get('working_dir', params.get('working_dir', '')),
timeout=data.get('timeout', params.get('timeout', 30)),
expected_output=data.get('expected_output', params.get('expected_output', '')),
check_type=data.get('check_type', params.get('check_type', 'contains')),
env_vars=data.get('env_vars', params.get('env_vars', {}))
)
@dataclass
class Recording:
"""录制数据模型"""
name: str
actions: list
timestamp: str
duration: float
description: str = ""
def to_dict(self) -> Dict:
return {
'name': self.name,
'actions': [a.to_dict() for a in self.actions],
'timestamp': self.timestamp,
'duration': self.duration,
'description': self.description
}
@classmethod
def from_dict(cls, data: Dict) -> 'Recording':
actions = [Action.from_dict(a) for a in data.get('actions', [])]
return cls(
name=data.get('name', ''),
actions=actions,
timestamp=data.get('timestamp', ''),
duration=data.get('duration', 0),
description=data.get('description', '')
)
@dataclass
class CommandExecutionResult:
"""命令执行结果"""
command: str
exit_code: int
stdout: str
stderr: str
execution_time: float
success: bool
output_lines: List[str] = field(default_factory=list)
def __post_init__(self):
if not self.output_lines:
self.output_lines = self.stdout.splitlines() if self.stdout else []
@dataclass
class VerificationResult:
"""验证结果模型"""
region_name: str
passed: bool
actual_text: str
expected_value: str
check_type: str
error: Optional[str] = None
timestamp: str = ""
def __post_init__(self):
if not self.timestamp:
self.timestamp = datetime.now().isoformat()
- core/recorder.py
"""
操作录制器 - UI操作
"""
import time
import threading
from typing import List
from pynput import mouse, keyboard
from core.models import Action
from utils.logger import LoggerMixin
class ActionRecorder(LoggerMixin):
"""操作录制器"""
def __init__(self):
self.actions: List[Action] = []
self.is_recording = False
self.start_time = 0
self.key_buffer = {}
self.mouse_listener = None
self.keyboard_listener = None
self.record_thread = None
self._lock = threading.Lock()
def start_recording(self):
"""开始录制"""
with self._lock:
self.actions = []
self.is_recording = True
self.start_time = time.time()
self.logger.info("=" * 60)
self.logger.info("🎬 开始录制UI操作...")
self.logger.info("快捷键:")
self.logger.info(" - Scroll Lock: 暂停/继续")
self.logger.info(" - Ctrl+Shift+Q: 停止录制")
self.logger.info("=" * 60)
self.record_thread = threading.Thread(target=self._record_loop, daemon=True)
self.record_thread.start()
def _record_loop(self):
"""录制循环"""
self.mouse_listener = mouse.Listener(
on_click=self._on_mouse_click,
on_scroll=self._on_mouse_scroll
)
self.keyboard_listener = keyboard.Listener(
on_press=self._on_key_press,
on_release=self._on_key_release
)
self.mouse_listener.start()
self.keyboard_listener.start()
while True:
with self._lock:
if not self.is_recording:
break
time.sleep(0.1)
self.mouse_listener.stop()
self.keyboard_listener.stop()
def _on_mouse_click(self, x, y, button, pressed):
"""鼠标点击事件"""
with self._lock:
if not self.is_recording:
return
if pressed:
button_name = str(button).replace('Button.', '')
action = Action(
type='click',
params={'x': x, 'y': y, 'button': button_name},
timestamp=time.time() - self.start_time,
description=f"点击 ({x}, {y})"
)
self.actions.append(action)
self.logger.info(f"📝 {action.description}")
def _on_mouse_scroll(self, x, y, dx, dy):
"""鼠标滚动事件"""
with self._lock:
if not self.is_recording:
return
action = Action(
type='scroll',
params={'x': x, 'y': y, 'dx': dx, 'dy': dy},
timestamp=time.time() - self.start_time,
description=f"滚动: dy={dy}"
)
self.actions.append(action)
self.logger.info(f"📝 {action.description}")
def _on_key_press(self, key):
"""键盘按下事件"""
with self._lock:
if not self.is_recording:
return
try:
if key in [keyboard.Key.ctrl_l, keyboard.Key.ctrl_r]:
self.key_buffer['ctrl'] = True
elif key in [keyboard.Key.shift, keyboard.Key.shift_r]:
self.key_buffer['shift'] = True
key_str = self._normalize_key(key)
if key_str == 'scroll_lock':
self.is_recording = not self.is_recording
status = "暂停" if not self.is_recording else "继续"
self.logger.info(f"⏸️ 录制{status}")
return
if key_str == 'q' and self.key_buffer.get('ctrl') and self.key_buffer.get('shift'):
self.stop_recording()
return
if key_str and self.is_recording:
action = Action(
type='key_press',
params={'key': key_str},
timestamp=time.time() - self.start_time,
description=f"按键: {key_str}"
)
self.actions.append(action)
self.logger.info(f"📝 {action.description}")
except Exception as e:
self.logger.debug(f"键盘事件错误: {e}")
def _on_key_release(self, key):
"""键盘释放事件"""
try:
if key in [keyboard.Key.ctrl_l, keyboard.Key.ctrl_r]:
self.key_buffer['ctrl'] = False
elif key in [keyboard.Key.shift, keyboard.Key.shift_r]:
self.key_buffer['shift'] = False
except:
pass
def _normalize_key(self, key) -> str:
"""标准化按键名称"""
try:
if hasattr(key, 'char') and key.char is not None:
return key.char
elif hasattr(key, 'name'):
return key.name
else:
return str(key)
except:
return str(key)
def stop_recording(self):
"""停止录制"""
with self._lock:
self.is_recording = False
self.logger.info("=" * 60)
with self._lock:
self.logger.info(f"✅ 录制完成! 共 {len(self.actions)} 个操作")
self.logger.info("=" * 60)
def get_actions(self) -> List[Action]:
with self._lock:
return self.actions.copy()
def clear(self):
with self._lock:
self.actions = []
- core/player.py
"""
操作回放器 - UI操作
"""
import time
import threading
import pyautogui
from typing import List, Dict, Optional
from core.models import Action, OCRRegion
from ocr.checker import OCRChecker
from utils.logger import LoggerMixin
class ActionPlayer(LoggerMixin):
"""操作回放器"""
def __init__(self):
self.actions: List[Action] = []
self.is_playing = False
self.current_index = 0
self.speed = 1.0
self.ocr_checker: Optional[OCRChecker] = None
self.stop_requested = False
self._lock = threading.Lock()
def load(self, actions: List[Action]) -> bool:
self.actions = actions
self.logger.info(f"✅ 加载了 {len(actions)} 个操作")
return True
def play(self, regions: Dict[str, OCRRegion] = None) -> bool:
if not self.actions:
self.logger.error("没有可回放的操作")
return False
with self._lock:
self.is_playing = True
self.current_index = 0
self.stop_requested = False
if regions:
self.ocr_checker = OCRChecker(regions)
self.logger.info("=" * 60)
self.logger.info(f"▶️ 开始回放... (速度: {self.speed}x)")
self.logger.info(f"共 {len(self.actions)} 个操作")
self.logger.info("按 Ctrl+Q 可停止回放")
self.logger.info("=" * 60)
try:
stop_thread = threading.Thread(target=self._listen_stop, daemon=True)
stop_thread.start()
while True:
with self._lock:
if self.current_index >= len(self.actions) or self.stop_requested:
break
action = self.actions[self.current_index]
try:
self._execute_action(action)
with self._lock:
self.current_index += 1
if self.current_index % 10 == 0:
self.logger.info(f"进度: {self.current_index}/{len(self.actions)}")
except Exception as e:
self.logger.error(f"操作 {self.current_index+1} 失败: {e}")
with self._lock:
self.current_index += 1
with self._lock:
completed = self.current_index >= len(self.actions)
if completed:
self.logger.info("=" * 60)
self.logger.info("✅ 回放完成!")
self.logger.info(f"共执行 {len(self.actions)} 个操作")
self.logger.info("=" * 60)
if self.ocr_checker:
self._verify_results()
return True
else:
self.logger.info("⏹️ 回放已停止")
return False
finally:
with self._lock:
self.is_playing = False
def _execute_action(self, action: Action):
action_type = action.type
params = action.params
if action_type == 'click':
pyautogui.click(params['x'], params['y'])
elif action_type == 'double_click':
pyautogui.doubleClick(params['x'], params['y'])
elif action_type == 'right_click':
pyautogui.rightClick(params['x'], params['y'])
elif action_type == 'key_press':
pyautogui.press(params['key'])
elif action_type == 'type_text':
pyautogui.typewrite(params['text'])
elif action_type == 'scroll':
pyautogui.scroll(params['dy'])
elif action_type == 'wait':
time.sleep(params['seconds'] / self.speed)
else:
self.logger.warning(f"未知操作类型: {action_type}")
time.sleep(0.2 / self.speed)
def _listen_stop(self):
try:
import keyboard
while True:
with self._lock:
if not self.is_playing:
break
if keyboard.is_pressed('ctrl+q'):
with self._lock:
self.stop_requested = True
self.logger.info("⏹️ 用户请求停止回放")
break
time.sleep(0.1)
except:
pass
def _verify_results(self):
if not self.ocr_checker:
return
results = self.ocr_checker.verify_all()
summary = self.ocr_checker.get_summary()
self.logger.info("\n" + "=" * 60)
self.logger.info("🔍 OCR验证结果")
self.logger.info("=" * 60)
self.logger.info(f"总计: {summary['total']} 个区域")
self.logger.info(f"通过: {summary['passed']} 个")
self.logger.info(f"失败: {summary['failed']} 个")
self.logger.info(f"通过率: {summary['pass_rate']:.1f}%")
self.logger.info("=" * 60)
for result in results:
status = "✅" if result.passed else "❌"
self.logger.info(f"{status} {result.region_name}: '{result.actual_text[:50]}'")
- core/cmd_recorder.py
"""
命令行操作录制器
"""
import subprocess
import time
import threading
from typing import List, Optional
from core.models import Action, CommandAction, CommandExecutionResult
from utils.logger import LoggerMixin
class CommandRecorder(LoggerMixin):
"""命令行操作录制器"""
def __init__(self):
self.actions: List[Action] = []
self.is_recording = False
self.start_time = 0
self._lock = threading.Lock()
def start_recording(self):
"""开始录制"""
with self._lock:
self.actions = []
self.is_recording = True
self.start_time = time.time()
self.logger.info("=" * 60)
self.logger.info("🎬 开始录制命令行操作...")
self.logger.info("输入命令执行，输入 'stop' 停止录制")
self.logger.info("=" * 60)
threading.Thread(target=self._interactive_mode, daemon=True).start()
def _interactive_mode(self):
"""交互式命令行录制模式"""
while True:
with self._lock:
if not self.is_recording:
break
try:
cmd_input = input("\n> ").strip()
if not cmd_input:
continue
if cmd_input.lower() == 'stop':
self.stop_recording()
break
if cmd_input.startswith('cmd:'):
command = cmd_input[4:].strip()
if command:
self._execute_and_record(command)
except (EOFError, KeyboardInterrupt):
break
except Exception as e:
self.logger.error(f"错误: {e}")
def _execute_and_record(self, command: str):
"""执行命令并记录"""
self.logger.info(f"📟 执行命令: {command}")
result = self.execute_command(command)
if result:
action = CommandAction(
type='cmd_execute',
params={
'cmd': command,
'timeout': 30,
'exit_code': result.exit_code
},
timestamp=time.time() - self.start_time,
description=f"执行命令: {command}",
cmd=command,
timeout=30
)
with self._lock:
self.actions.append(action)
if result.stdout:
self.logger.info(f"📤 输出:\n{result.stdout[:500]}")
if result.stderr:
self.logger.warning(f"⚠️ 错误输出:\n{result.stderr[:200]}")
def execute_command(self, command: str) -> Optional[CommandExecutionResult]:
"""执行命令并捕获输出 - 公共方法"""
try:
start_time = time.time()
process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
shell=True,
bufsize=1
)
try:
stdout, stderr = process.communicate(timeout=30)
exit_code = process.returncode
except subprocess.TimeoutExpired:
process.kill()
stdout, stderr = process.communicate()
exit_code = -1
self.logger.warning("⚠️ 命令执行超时")
execution_time = time.time() - start_time
result = CommandExecutionResult(
command=command,
exit_code=exit_code,
stdout=stdout or "",
stderr=stderr or "",
execution_time=execution_time,
success=exit_code == 0
)
return result
except FileNotFoundError:
self.logger.error(f"❌ 命令未找到: {command}")
return CommandExecutionResult(
command=command,
exit_code=-1,
stdout="",
stderr=f"命令未找到: {command}",
execution_time=0,
success=False
)
except Exception as e:
self.logger.error(f"❌ 执行命令失败: {e}")
return CommandExecutionResult(
command=command,
exit_code=-1,
stdout="",
stderr=str(e),
execution_time=0,
success=False
)
def stop_recording(self):
"""停止录制"""
with self._lock:
self.is_recording = False
self.logger.info("=" * 60)
with self._lock:
self.logger.info(f"✅ 录制完成! 共 {len(self.actions)} 个操作")
self.logger.info("=" * 60)
def get_actions(self) -> List[Action]:
with self._lock:
return self.actions.copy()
def clear(self):
with self._lock:
self.actions = []
- core/cmd_player.py
"""
命令行操作回放器
"""
import subprocess
import time
import threading
from typing import List
from core.models import Action
from utils.logger import LoggerMixin
class CommandPlayer(LoggerMixin):
"""命令行操作回放器"""
def __init__(self):
self.actions: List[Action] = []
self.is_playing = False
self.current_index = 0
self.speed = 1.0
self.last_output = ""
self.stop_requested = False
self._lock = threading.Lock()
def load(self, actions: List[Action]) -> bool:
self.actions = actions
self.logger.info(f"✅ 加载了 {len(actions)} 个命令行操作")
return True
def play(self) -> bool:
if not self.actions:
self.logger.error("没有可回放的操作")
return False
with self._lock:
self.is_playing = True
self.current_index = 0
self.stop_requested = False
self.last_output = ""
self.logger.info("=" * 60)
self.logger.info(f"▶️ 开始回放命令行操作... (速度: {self.speed}x)")
self.logger.info(f"共 {len(self.actions)} 个操作")
self.logger.info("按 Ctrl+Q 可停止回放")
self.logger.info("=" * 60)
try:
stop_thread = threading.Thread(target=self._listen_stop, daemon=True)
stop_thread.start()
while True:
with self._lock:
if self.current_index >= len(self.actions) or self.stop_requested:
break
action = self.actions[self.current_index]
try:
self._execute_action(action)
with self._lock:
self.current_index += 1
if self.current_index % 10 == 0:
self.logger.info(f"进度: {self.current_index}/{len(self.actions)}")
except Exception as e:
self.logger.error(f"操作 {self.current_index+1} 失败: {e}")
with self._lock:
self.current_index += 1
with self._lock:
completed = self.current_index >= len(self.actions)
if completed:
self.logger.info("=" * 60)
self.logger.info("✅ 回放完成!")
self.logger.info(f"共执行 {len(self.actions)} 个操作")
self.logger.info("=" * 60)
return True
else:
self.logger.info("⏹️ 回放已停止")
return False
finally:
with self._lock:
self.is_playing = False
def _execute_action(self, action: Action):
action_type = action.type
if action_type == 'cmd_execute':
self._execute_command(action)
elif action_type == 'cmd_check_output':
self._check_output(action)
elif action_type == 'cmd_wait_text':
self._wait_for_text(action)
elif action_type == 'wait':
time.sleep(action.params['seconds'] / self.speed)
else:
self.logger.warning(f"未知操作类型: {action_type}")
time.sleep(0.2 / self.speed)
def _execute_command(self, action: Action):
cmd = action.params.get('cmd', '')
timeout = action.params.get('timeout', 30)
self.logger.info(f"🔄 执行命令: {cmd}")
try:
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
shell=True,
bufsize=1
)
try:
stdout, stderr = process.communicate(timeout=timeout)
exit_code = process.returncode
except subprocess.TimeoutExpired:
process.kill()
stdout, stderr = process.communicate()
exit_code = -1
self.logger.warning(f"⚠️ 命令超时: {cmd}")
self.last_output = stdout
if stdout:
self.logger.debug(f"📤 输出:\n{stdout[:200]}")
if stderr:
self.logger.warning(f"⚠️ 错误输出:\n{stderr[:200]}")
except Exception as e:
self.logger.error(f"❌ 执行失败: {e}")
raise
def _check_output(self, action: Action):
expected = action.params.get('expected_output', '')
check_type = action.params.get('check_type', 'contains')
self.logger.info(f"🔍 检查输出: '{expected}'")
if not self.last_output:
self.logger.warning("⚠️ 没有输出可检查")
return
if check_type == 'contains':
passed = expected in self.last_output
elif check_type == 'equals':
passed = self.last_output.strip() == expected
elif check_type == 'regex':
import re
passed = bool(re.search(expected, self.last_output))
else:
passed = False
if passed:
self.logger.info(f"✅ 检查通过")
else:
self.logger.warning(f"❌ 检查失败")
def _wait_for_text(self, action: Action):
text = action.params.get('text', '')
timeout = action.params.get('timeout', 30)
self.logger.info(f"⏳ 等待输出: '{text}' (超时: {timeout}s)")
start_time = time.time()
while time.time() - start_time < timeout:
if text in self.last_output:
self.logger.info(f"✅ 找到文本")
return
time.sleep(0.5)
self.logger.warning(f"❌ 等待超时")
def _listen_stop(self):
try:
import keyboard
while True:
with self._lock:
if not self.is_playing:
break
if keyboard.is_pressed('ctrl+q'):
with self._lock:
self.stop_requested = True
self.logger.info("⏹️ 用户请求停止回放")
break
time.sleep(0.1)
except:
pass
- core/hybrid_recorder.py
"""
混合录制器 - 同时录制UI和命令行操作
"""
import time
import threading
import queue
import json
from typing import Dict, List, Optional
from pathlib import Path
from datetime import datetime
from core.models import Action, CommandAction
from core.recorder import ActionRecorder
from core.cmd_recorder import CommandRecorder
from utils.logger import LoggerMixin
class HybridRecorder(LoggerMixin):
"""混合录制器"""
def __init__(self):
self.ui_recorder = ActionRecorder()
self.cmd_recorder = CommandRecorder()
self.is_recording = False
self.start_time = 0
self._lock = threading.Lock()
self.action_queue: List[Dict] = []
self.stop_event = threading.Event()
self.hotkeys = {
'cmd_prefix': 'cmd:'
}
def start_recording(self):
"""开始混合录制"""
with self._lock:
self.action_queue = []
self.is_recording = True
self.start_time = time.time()
self.stop_event.clear()
self.logger.info("=" * 70)
self.logger.info("🎬 开始混合录制 (UI + 命令行)")
self.logger.info("=" * 70)
self.logger.info("📖 使用说明:")
self.logger.info(f" - UI操作: 正常操作窗口")
self.logger.info(f" - 命令行输入: 以 '{self.hotkeys['cmd_prefix']}' 开头")
self.logger.info(f" - 输入 'stop' 停止录制")
self.logger.info("=" * 70)
# 启动UI录制
self.ui_recorder.start_recording()
# 启动命令行监听
threading.Thread(target=self._cmd_input_listener, daemon=True).start()
def _cmd_input_listener(self):
"""命令行输入监听器"""
while True:
with self._lock:
if not self.is_recording or self.stop_event.is_set():
break
try:
user_input = input()
if not user_input:
continue
if user_input.lower() == 'stop':
self.stop_recording()
break
if user_input.startswith(self.hotkeys['cmd_prefix']):
command = user_input[len(self.hotkeys['cmd_prefix']):].strip()
if command:
self._handle_cmd_input(command)
except (EOFError, KeyboardInterrupt):
break
except Exception as e:
self.logger.error(f"命令行监听错误: {e}")
def _handle_cmd_input(self, command: str):
"""处理命令行输入"""
with self._lock:
if not self.is_recording:
return
self.logger.info(f"📟 执行命令: {command}")
result = self.cmd_recorder.execute_command(command)
if result:
action = CommandAction(
type='cmd_execute',
params={
'cmd': command,
'timeout': 30,
'exit_code': result.exit_code
},
timestamp=time.time() - self.start_time,
description=f"执行命令: {command}",
cmd=command,
timeout=30
)
with self._lock:
self.action_queue.append({
'type': 'cmd',
'action': action,
'timestamp': time.time() - self.start_time,
'result': result
})
if result.stdout:
self.logger.info(f"📤 输出:\n{result.stdout[:500]}")
if result.stderr:
self.logger.warning(f"⚠️ 错误输出:\n{result.stderr[:200]}")
def stop_recording(self):
"""停止录制"""
with self._lock:
if not self.is_recording:
return
self.is_recording = False
self.stop_event.set()
self.logger.info("=" * 70)
self.logger.info("⏹️ 停止录制...")
self.ui_recorder.stop_recording()
with self._lock:
ui_actions = len(self.ui_recorder.actions)
cmd_actions = len(self.action_queue)
total = ui_actions + cmd_actions
self.logger.info(f"📊 录制统计:")
self.logger.info(f" UI操作: {ui_actions} 个")
self.logger.info(f" 命令行操作: {cmd_actions} 个")
self.logger.info(f" 总计: {total} 个操作")
self.logger.info("=" * 70)
def save(self, filename: str) -> str:
"""保存录制"""
with self._lock:
all_actions = []
for action in self.ui_recorder.actions:
all_actions.append({
'type': 'ui',
'action': action.to_dict(),
'timestamp': action.timestamp
})
for item in self.action_queue:
if 'action' in item:
all_actions.append({
'type': 'cmd',
'action': item['action'].to_dict(),
'timestamp': item['timestamp']
})
all_actions.sort(key=lambda x: x['timestamp'])
data = {
'name': Path(filename).stem,
'timestamp': datetime.now().isoformat(),
'duration': time.time() - self.start_time,
'total_actions': len(all_actions),
'ui_actions': len(self.ui_recorder.actions),
'cmd_actions': len(self.action_queue),
'description': '混合录制 (UI + 命令行)',
'actions': all_actions,
'metadata': {
'mode': 'hybrid',
'ui_recording': True,
'cmd_recording': True
}
}
with open(filename, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
self.logger.info(f"💾 录制已保存: {filename}")
return filename
def get_actions(self) -> List[Dict]:
"""获取所有录制的操作"""
with self._lock:
all_actions = []
for action in self.ui_recorder.actions:
all_actions.append({
'type': 'ui',
'action': action,
'timestamp': action.timestamp
})
for item in self.action_queue:
all_actions.append({
'type': 'cmd',
'action': item['action'],
'timestamp': item['timestamp']
})
all_actions.sort(key=lambda x: x['timestamp'])
return all_actions
- core/hybrid_player.py
"""
混合回放器 - 同时回放UI和命令行操作
"""
import time
import threading
import json
from typing import Dict, List, Optional
from pathlib import Path
from core.models import Action, CommandAction
from core.player import ActionPlayer
from core.cmd_player import CommandPlayer
from ocr.checker import OCRChecker
from utils.logger import LoggerMixin
class HybridPlayer(LoggerMixin):
"""混合回放器"""
def __init__(self):
self.ui_player = ActionPlayer()
self.cmd_player = CommandPlayer()
self.actions: List[Dict] = []
self.is_playing = False
self.speed = 1.0
self.stop_requested = False
self.current_index = 0
self._lock = threading.Lock()
def load(self, filepath: str) -> bool:
"""加载混合录制文件"""
try:
with open(filepath, 'r', encoding='utf-8') as f:
data = json.load(f)
self.actions = data.get('actions', [])
self.logger.info(f"✅ 加载混合录制: {filepath}")
self.logger.info(f" 总计: {len(self.actions)} 个操作")
return True
except Exception as e:
self.logger.error(f"加载失败: {e}")
return False
def play(self, regions: Dict = None, speed: float = 1.0) -> bool:
"""执行混合回放"""
with self._lock:
if not self.actions:
self.logger.error("没有可回放的操作")
return False
self.speed = speed
self.is_playing = True
self.current_index = 0
self.stop_requested = False
self.logger.info("=" * 70)
self.logger.info(f"▶️ 开始混合回放... (速度: {speed}x)")
self.logger.info(f"共 {len(self.actions)} 个操作")
self.logger.info("按 Ctrl+Q 可停止回放")
self.logger.info("=" * 70)
stop_thread = threading.Thread(target=self._listen_stop, daemon=True)
stop_thread.start()
try:
while True:
with self._lock:
if self.current_index >= len(self.actions) or self.stop_requested:
break
item = self.actions[self.current_index]
try:
progress = f"[{self.current_index+1}/{len(self.actions)}]"
if item['type'] == 'ui':
action = Action.from_dict(item['action'])
self.logger.info(f"{progress} 🖥️ {action.description}")
self._play_ui_action(action)
elif item['type'] == 'cmd':
action = CommandAction.from_dict(item['action'])
self.logger.info(f"{progress} 📟 {action.description}")
self._play_cmd_action(action)
with self._lock:
self.current_index += 1
time.sleep(0.3 / speed)
except Exception as e:
self.logger.error(f"操作 {self.current_index+1} 失败: {e}")
with self._lock:
self.current_index += 1
with self._lock:
completed = self.current_index >= len(self.actions)
if completed:
self.logger.info("=" * 70)
self.logger.info("✅ 混合回放完成!")
self.logger.info(f"共执行 {len(self.actions)} 个操作")
self.logger.info("=" * 70)
if regions:
self._verify_results(regions)
return True
else:
self.logger.info("⏹️ 回放已停止")
return False
finally:
with self._lock:
self.is_playing = False
def _play_ui_action(self, action: Action):
self.ui_player.load([action])
self.ui_player.speed = self.speed
self.ui_player.play()
def _play_cmd_action(self, action: CommandAction):
self.cmd_player.load([action])
self.cmd_player.speed = self.speed
self.cmd_player.play()
def _listen_stop(self):
try:
import keyboard
while True:
with self._lock:
if not self.is_playing:
break
if keyboard.is_pressed('ctrl+q'):
with self._lock:
self.stop_requested = True
self.logger.info("⏹️ 用户请求停止回放")
break
time.sleep(0.1)
except:
pass
def _verify_results(self, regions: Dict):
checker = OCRChecker(regions)
results = checker.verify_all()
self.logger.info("\n" + "=" * 70)
self.logger.info("🔍 OCR验证结果")
self.logger.info("=" * 70)
for result in results:
status = "✅" if result.passed else "❌"
self.logger.info(f"{status} {result.region_name}: '{result.actual_text[:50]}'")
self.logger.info("=" * 70)
- core/unified_controller.py
"""
统一控制器 - 整合所有功能
"""
import time
from pathlib import Path
from typing import Dict, List, Optional, Any
from datetime import datetime
from core.models import Action, OCRRegion, Recording
from core.recorder import ActionRecorder
from core.player import ActionPlayer
from core.cmd_recorder import CommandRecorder
from core.cmd_player import CommandPlayer
from core.hybrid_recorder import HybridRecorder
from core.hybrid_player import HybridPlayer
from ocr.selector import LiveOCRSelector
from ocr.checker import OCRChecker
from config.settings import settings
from utils.logger import LoggerMixin
from utils.file_utils import FileManager
class UnifiedController(LoggerMixin):
"""统一控制器"""
def __init__(self):
self.settings = settings
self.file_manager = FileManager(self.settings.data_dir)
self.ui_recorder = ActionRecorder()
self.ui_player = ActionPlayer()
self.cmd_recorder = CommandRecorder()
self.cmd_player = CommandPlayer()
self.hybrid_recorder = HybridRecorder()
self.hybrid_player = HybridPlayer()
self.regions: Dict[str, OCRRegion] = {}
self.load_regions()
def load_regions(self):
data = self.file_manager.load_json(self.settings.regions_file)
if data:
for name, region_data in data.items():
self.regions[name] = OCRRegion(**region_data)
self.logger.info(f"✅ 加载了 {len(self.regions)} 个OCR区域")
def save_regions(self):
data = {name: region.to_dict() for name, region in self.regions.items()}
self.file_manager.save_json(data, self.settings.regions_file)
def select_ocr_region(self) -> Optional[OCRRegion]:
selector = LiveOCRSelector()
region = selector.select()
if region:
self.regions[region.name] = region
self.save_regions()
self.logger.info(f"✅ 已添加OCR区域: {region.name}")
return region
def delete_region(self, name: str) -> bool:
if name in self.regions:
del self.regions[name]
self.save_regions()
self.logger.info(f"🗑️ 已删除OCR区域: {name}")
return True
return False
# ==================== UI模式 ====================
def start_ui_recording(self):
self.ui_recorder.start_recording()
def stop_ui_recording(self) -> Optional[str]:
self.ui_recorder.stop_recording()
if self.ui_recorder.actions:
filename = self.file_manager.get_unique_filename(
self.settings.recordings_dir, "ui_recording"
)
recording = Recording(
name=Path(filename).stem,
actions=self.ui_recorder.actions,
timestamp=datetime.now().isoformat(),
duration=self.ui_recorder.actions[-1].timestamp if self.ui_recorder.actions else 0,
description="UI操作录制"
)
self.file_manager.save_json(recording.to_dict(), Path(filename))
return filename
return None
def play_ui_recording(self, filepath: str, speed: float = 1.0) -> bool:
data = self.file_manager.load_json(Path(filepath))
if not data:
return False
actions = [Action.from_dict(a) for a in data.get('actions', [])]
self.ui_player.load(actions)
self.ui_player.speed = speed
return self.ui_player.play(self.regions if self.regions else None)
# ==================== 命令行模式 ====================
def start_cmd_recording(self):
self.cmd_recorder.start_recording()
def stop_cmd_recording(self) -> Optional[str]:
self.cmd_recorder.stop_recording()
if self.cmd_recorder.actions:
filename = self.file_manager.get_unique_filename(
self.settings.recordings_dir, "cmd_recording"
)
recording = Recording(
name=Path(filename).stem,
actions=self.cmd_recorder.actions,
timestamp=datetime.now().isoformat(),
duration=self.cmd_recorder.actions[-1].timestamp if self.cmd_recorder.actions else 0,
description="命令行录制"
)
self.file_manager.save_json(recording.to_dict(), Path(filename))
return filename
return None
def play_cmd_recording(self, filepath: str, speed: float = 1.0) -> bool:
data = self.file_manager.load_json(Path(filepath))
if not data:
return False
actions = [Action.from_dict(a) for a in data.get('actions', [])]
self.cmd_player.load(actions)
self.cmd_player.speed = speed
return self.cmd_player.play()
# ==================== 混合模式 ====================
def start_hybrid_recording(self):
self.hybrid_recorder.start_recording()
def stop_hybrid_recording(self) -> Optional[str]:
filename = self.file_manager.get_unique_filename(
self.settings.recordings_dir, "hybrid_recording"
)
return self.hybrid_recorder.save(filename)
def play_hybrid_recording(self, filepath: str, speed: float = 1.0) -> bool:
self.hybrid_player.load(filepath)
return self.hybrid_player.play(self.regions if self.regions else None, speed)
# ==================== 验证 ====================
def verify_all(self) -> Dict:
if not self.regions:
return {'ocr': {'total': 0, 'passed': 0, 'failed': 0, 'pass_rate': 0, 'results': []}}
checker = OCRChecker(self.regions)
results = checker.verify_all()
summary = checker.get_summary()
self.logger.info("\n" + "=" * 60)
self.logger.info("📊 OCR验证结果")
self.logger.info("=" * 60)
self.logger.info(f"总计: {summary['total']} 个区域")
self.logger.info(f"通过: {summary['passed']} 个")
self.logger.info(f"失败: {summary['failed']} 个")
self.logger.info(f"通过率: {summary['pass_rate']:.1f}%")
self.logger.info("=" * 60)
for result in results:
status = "✅" if result.passed else "❌"
self.logger.info(f"{status} {result.region_name}: '{result.actual_text[:50]}'")
return {'ocr': summary}
# ==================== 工具方法 ====================
def list_recordings(self, pattern: str = "*.json") -> List[str]:
files = self.file_manager.list_files(self.settings.recordings_dir, pattern)
return [f.stem for f in files]
def get_latest_recording(self) -> Optional[str]:
latest = self.file_manager.get_latest_file(self.settings.recordings_dir, "*.json")
return str(latest) if latest else None
def get_recording_info(self, filepath: str) -> Dict:
data = self.file_manager.load_json(Path(filepath))
if data:
return {
'name': data.get('name', ''),
'timestamp': data.get('timestamp', ''),
'total_actions': len(data.get('actions', [])),
'duration': data.get('duration', 0),
'description': data.get('description', '')
}
return {}
- ocr/init.py
"""OCR模块"""
from ocr.processor import OCRProcessor
from ocr.checker import OCRChecker
from ocr.selector import LiveOCRSelector
__all__ = ['OCRProcessor', 'OCRChecker', 'LiveOCRSelector']
- ocr/processor.py
"""
OCR图像处理器
"""
import cv2
import numpy as np
from PIL import Image
import pytesseract
from config.settings import settings
class OCRProcessor:
"""OCR图像处理器"""
def __init__(self):
self.config = settings.get('ocr', {})
self.lang = self.config.get('lang', 'chi_sim+eng')
self.psm = self.config.get('psm', 6)
self.whitelist = self.config.get('whitelist',
'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz.-+%: ')
def process_image(self, image: Image.Image) -> str:
try:
img = np.array(image)
if len(img.shape) == 3:
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
else:
gray = img
denoised = cv2.fastNlMeansDenoising(gray)
_, binary = cv2.threshold(denoised, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
config = f'--oem 3 --psm {self.psm} -c tessedit_char_whitelist={self.whitelist}'
text = pytesseract.image_to_string(binary, lang=self.lang, config=config)
return text.strip()
except Exception as e:
return f"OCR错误: {e}"
def process_region(self, x: int, y: int, width: int, height: int) -> str:
import pyautogui
screenshot = pyautogui.screenshot(region=(x, y, width, height))
return self.process_image(screenshot)
- ocr/checker.py
"""
OCR验证器
"""
import re
from typing import Dict, List
from core.models import OCRRegion, VerificationResult
from ocr.processor import OCRProcessor
import pyautogui
class OCRChecker:
"""OCR验证器"""
def __init__(self, regions: Dict[str, OCRRegion]):
self.regions = regions
self.processor = OCRProcessor()
self.results: List[VerificationResult] = []
def verify_all(self) -> List[VerificationResult]:
self.results = []
for name, region in self.regions.items():
result = self.verify_region(region)
self.results.append(result)
return self.results
def verify_region(self, region: OCRRegion) -> VerificationResult:
try:
x, y, w, h = region.x, region.y, region.width, region.height
screenshot = pyautogui.screenshot(region=(x, y, w, h))
text = self.processor.process_image(screenshot)
passed = self._check_value(text, region.expected_value, region.check_type)
return VerificationResult(
region_name=region.name,
passed=passed,
actual_text=text,
expected_value=region.expected_value,
check_type=region.check_type
)
except Exception as e:
return VerificationResult(
region_name=region.name,
passed=False,
actual_text="",
expected_value=region.expected_value,
check_type=region.check_type,
error=str(e)
)
def _check_value(self, actual: str, expected: str, check_type: str) -> bool:
if not actual:
return False
actual_lower = actual.lower().strip()
expected_lower = expected.lower().strip()
if check_type == 'contains':
return expected_lower in actual_lower
elif check_type == 'equals':
return actual_lower == expected_lower
elif check_type == 'regex':
try:
return bool(re.search(expected, actual))
except:
return False
elif check_type == 'number':
try:
numbers = re.findall(r'\d+\.?\d*', actual)
return any(abs(float(n) - float(expected)) < 0.001 for n in numbers)
except:
return False
elif check_type == 'not_empty':
return bool(actual.strip())
return False
def get_summary(self) -> Dict:
total = len(self.results)
passed = sum(1 for r in self.results if r.passed)
return {
'total': total,
'passed': passed,
'failed': total - passed,
'pass_rate': (passed / total * 100) if total > 0 else 0,
'results': self.results
}
- ocr/selector.py
"""
OCR区域选择器 - 实时预览
"""
import tkinter as tk
from tkinter import ttk
from PIL import ImageTk
import pyautogui
from typing import Optional
from core.models import OCRRegion
from ocr.processor import OCRProcessor
import logging
logger = logging.getLogger(__name__)
class LiveOCRSelector:
"""实时预览OCR区域选择器"""
def __init__(self, parent=None):
self.parent = parent
self.selected_region = None
self.ocr_processor = OCRProcessor()
self.region_name = tk.StringVar(value="region_1")
self.expected_value = tk.StringVar()
self.check_type = tk.StringVar(value="contains")
self.root = tk.Toplevel(parent) if parent else tk.Tk()
self.root.title("🎯 实时OCR区域选择器")
self.root.geometry("1400x900")
self.root.attributes('-topmost', True)
self.setup_ui()
self.refresh_screen()
def setup_ui(self):
main_frame = ttk.Frame(self.root, padding="10")
main_frame.pack(fill=tk.BOTH, expand=True)
# 左侧预览
preview_frame = ttk.LabelFrame(main_frame, text="📺 屏幕预览 (拖动选择区域)", padding="5")
preview_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True)
self.canvas = tk.Canvas(preview_frame, bg='gray', cursor='crosshair')
self.canvas.pack(fill=tk.BOTH, expand=True)
self.canvas.bind('<ButtonPress-1>', self.on_mouse_down)
self.canvas.bind('<B1-Motion>', self.on_mouse_drag)
self.canvas.bind('<ButtonRelease-1>', self.on_mouse_up)
self.canvas.bind('<Configure>', lambda e: self.refresh_screen())
# 右侧控制
control_frame = ttk.LabelFrame(main_frame, text="🎮 控制面板", padding="10")
control_frame.pack(side=tk.RIGHT, fill=tk.Y, padx=(10, 0))
ttk.Label(control_frame, text="区域名称:").pack(anchor=tk.W, pady=(0, 5))
ttk.Entry(control_frame, textvariable=self.region_name, width=25).pack(fill=tk.X, pady=(0, 10))
ttk.Label(control_frame, text="检查类型:").pack(anchor=tk.W, pady=(0, 5))
check_types = ['contains', 'equals', 'regex', 'number', 'not_empty']
ttk.Combobox(control_frame, textvariable=self.check_type, values=check_types,
state='readonly').pack(fill=tk.X, pady=(0, 10))
ttk.Label(control_frame, text="期望值:").pack(anchor=tk.W, pady=(0, 5))
ttk.Entry(control_frame, textvariable=self.expected_value, width=25).pack(fill=tk.X, pady=(0, 10))
ttk.Label(control_frame, text="识别结果:").pack(anchor=tk.W, pady=(0, 5))
self.result_text = tk.Text(control_frame, height=8, width=30, font=('Consolas', 10))
self.result_text.pack(fill=tk.X, pady=(0, 10))
self.coords_label = ttk.Label(control_frame, text="📍 坐标: 未选择")
self.coords_label.pack(anchor=tk.W, pady=(0, 5))
self.live_ocr_var = tk.BooleanVar(value=True)
ttk.Checkbutton(control_frame, text="实时OCR识别",
variable=self.live_ocr_var).pack(anchor=tk.W, pady=(0, 10))
btn_frame = ttk.Frame(control_frame)
btn_frame.pack(fill=tk.X, pady=10)
ttk.Button(btn_frame, text="✅ 确认", command=self.confirm).pack(side=tk.LEFT, padx=(0, 5), expand=True)
ttk.Button(btn_frame, text="❌ 取消", command=self.cancel).pack(side=tk.LEFT, expand=True)
ttk.Button(control_frame, text="🔄 刷新", command=self.refresh_screen).pack(fill=tk.X, pady=5)
self.status_label = ttk.Label(self.root, text="✅ 就绪 - 拖动选择区域", relief=tk.SUNKEN)
self.status_label.pack(side=tk.BOTTOM, fill=tk.X)
def refresh_screen(self):
try:
canvas_width = self.canvas.winfo_width()
canvas_height = self.canvas.winfo_height()
if canvas_width <= 1 or canvas_height <= 1:
return
screenshot = pyautogui.screenshot()
img_width, img_height = screenshot.size
scale = min(canvas_width / img_width, canvas_height / img_height, 1.0)
if scale < 1:
new_size = (int(img_width * scale), int(img_height * scale))
screenshot = screenshot.resize(new_size, Image.Resampling.LANCZOS)
self.scale_factor = scale
else:
self.scale_factor = 1.0
self.screenshot_tk = ImageTk.PhotoImage(screenshot)
self.canvas.delete("all")
self.canvas.create_image(0, 0, anchor=tk.NW, image=self.screenshot_tk)
except Exception as e:
logger.error(f"刷新失败: {e}")
def on_mouse_down(self, event):
self.start_x = event.x
self.start_y = event.y
self.rect = None
def on_mouse_drag(self, event):
if self.rect:
self.canvas.delete(self.rect)
x1, y1 = self.start_x, self.start_y
x2, y2 = event.x, event.y
self.rect = self.canvas.create_rectangle(
min(x1, x2), min(y1, y2),
max(x1, x2), max(y1, y2),
outline='#00FF00', width=3, fill='#0000FF33'
)
width = abs(x2 - x1)
height = abs(y2 - y1)
self.coords_label.config(text=f"📍 坐标: ({min(x1,x2)}, {min(y1,y2)}) - {width}x{height}")
if self.live_ocr_var.get() and width > 10 and height > 10:
self.perform_live_ocr(x1, y1, x2, y2)
def on_mouse_up(self, event):
if self.rect and self.start_x is not None:
x1, y1 = self.start_x, self.start_y
x2, y2 = event.x, event.y
width = abs(x2 - x1)
height = abs(y2 - y1)
if width > 10 and height > 10:
scale = getattr(self, 'scale_factor', 1.0)
self.selected_region = (
int(min(x1, x2) / scale),
int(min(y1, y2) / scale),
int(width / scale),
int(height / scale)
)
self.status_label.config(text=f"✅ 区域已选择: {self.selected_region}")
def perform_live_ocr(self, x1, y1, x2, y2):
try:
scale = getattr(self, 'scale_factor', 1.0)
screen_x = int(min(x1, x2) / scale)
screen_y = int(min(y1, y2) / scale)
screen_w = int(abs(x2 - x1) / scale)
screen_h = int(abs(y2 - y1) / scale)
if screen_w < 10 or screen_h < 10:
return
text = self.ocr_processor.process_region(screen_x, screen_y, screen_w, screen_h)
self.result_text.delete(1.0, tk.END)
self.result_text.insert(1.0, text)
expected = self.expected_value.get().strip()
if expected:
check_type = self.check_type.get()
if self._check_value(text, expected, check_type):
self.status_label.config(text=f"✅ 匹配! 识别: '{text[:50]}'")
else:
self.status_label.config(text=f"❌ 不匹配! 识别: '{text[:50]}'")
except Exception as e:
logger.error(f"OCR失败: {e}")
def _check_value(self, actual, expected, check_type):
if not actual:
return False
actual_lower = actual.lower()
expected_lower = expected.lower()
if check_type == 'contains':
return expected_lower in actual_lower
elif check_type == 'equals':
return actual_lower == expected_lower
elif check_type == 'regex':
import re
return bool(re.search(expected, actual))
return False
def confirm(self):
if self.selected_region:
region = OCRRegion(
name=self.region_name.get().strip() or "region_1",
x=self.selected_region[0],
y=self.selected_region[1],
width=self.selected_region[2],
height=self.selected_region[3],
expected_value=self.expected_value.get().strip(),
check_type=self.check_type.get()
)
self.root.destroy()
return region
return None
def cancel(self):
self.selected_region = None
self.root.destroy()
return None
def select(self) -> Optional[OCRRegion]:
self.root.mainloop()
return self.selected_region if hasattr(self, 'selected_region') else None
- gui/init.py
"""GUI模块"""
from gui.main_window import MainWindow
__all__ = ['MainWindow']
- gui/main_window.py
"""
主窗口
"""
import tkinter as tk
from tkinter import ttk, scrolledtext
import threading
import logging
import json
from pathlib import Path
from core.unified_controller import UnifiedController
class MainWindow:
"""主窗口"""
def __init__(self):
self.controller = UnifiedController()
self.root = tk.Tk()
self.root.title("🤖 自动化测试工具 (UI + 命令行)")
self.root.geometry("1300x850")
self.setup_ui()
self.setup_log_redirect()
self.update_region_list()
self.update_recording_list()
def setup_ui(self):
main_frame = ttk.Frame(self.root, padding="10")
main_frame.pack(fill=tk.BOTH, expand=True)
title = ttk.Label(main_frame, text="🤖 自动化测试工具 (UI + 命令行)",
font=('Arial', 16, 'bold'))
title.pack(pady=(0, 10))
# 模式选择
mode_frame = ttk.LabelFrame(main_frame, text="🎯 工作模式", padding="5")
mode_frame.pack(fill=tk.X, pady=(0, 10))
self.mode_var = tk.StringVar(value="ui")
modes = [("🖥️ UI模式", "ui"), ("📟 命令行模式", "cmd"), ("🔀 混合模式", "hybrid")]
for text, value in modes:
ttk.Radiobutton(mode_frame, text=text, variable=self.mode_var,
value=value).pack(side=tk.LEFT, padx=10)
# 工具栏
toolbar = ttk.Frame(main_frame)
toolbar.pack(fill=tk.X, pady=(0, 10))
ttk.Button(toolbar, text="🎯 添加OCR区域", command=self.add_ocr_region).pack(side=tk.LEFT, padx=(0, 5))
ttk.Button(toolbar, text="📹 开始录制", command=self.start_recording).pack(side=tk.LEFT, padx=5)
ttk.Button(toolbar, text="⏹️ 停止录制", command=self.stop_recording).pack(side=tk.LEFT, padx=5)
ttk.Button(toolbar, text="▶️ 回放录制", command=self.play_recording).pack(side=tk.LEFT, padx=5)
ttk.Label(toolbar, text="速度:").pack(side=tk.LEFT, padx=(20, 5))
self.speed_var = tk.StringVar(value="1.0")
speed_combo = ttk.Combobox(toolbar, textvariable=self.speed_var,
values=['0.3', '0.5', '0.8', '1.0', '1.5', '2.0'],
width=5)
speed_combo.pack(side=tk.LEFT, padx=(0, 10))
ttk.Button(toolbar, text="📊 验证结果", command=self.show_verify_results).pack(side=tk.LEFT, padx=5)
ttk.Button(toolbar, text="🗑️ 清空日志", command=self.clear_log).pack(side=tk.LEFT, padx=5)
# 录制列表
list_frame = ttk.LabelFrame(main_frame, text="📂 录制文件", padding="5")
list_frame.pack(fill=tk.X, pady=(0, 10))
columns = ('名称', '类型', '操作数', '时长', '描述')
self.recording_tree = ttk.Treeview(list_frame, columns=columns, show='headings', height=4)
for col in columns:
self.recording_tree.heading(col, text=col)
self.recording_tree.column('名称', width=200)
self.recording_tree.column('类型', width=80)
self.recording_tree.column('操作数', width=60)
self.recording_tree.column('时长', width=80)
self.recording_tree.column('描述', width=200)
self.recording_tree.pack(fill=tk.X, expand=True)
self.recording_tree.bind('<Double-Button-1>', lambda e: self.play_recording())
# 分割面板
paned = ttk.PanedWindow(main_frame, orient=tk.HORIZONTAL)
paned.pack(fill=tk.BOTH, expand=True)
# 左侧：OCR区域列表
left_frame = ttk.LabelFrame(paned, text="📋 OCR检查区域", padding="5")
paned.add(left_frame, weight=1)
region_columns = ('位置', '大小', '期望值', '类型')
self.region_tree = ttk.Treeview(left_frame, columns=region_columns, show='headings')
for col in region_columns:
self.region_tree.heading(col, text=col)
self.region_tree.column('位置', width=100)
self.region_tree.column('大小', width=80)
self.region_tree.column('期望值', width=150)
self.region_tree.column('类型', width=80)
self.region_tree.pack(fill=tk.BOTH, expand=True)
# 右键菜单
self.region_menu = tk.Menu(self.root, tearoff=0)
self.region_menu.add_command(label="删除", command=self.delete_region)
self.region_tree.bind('<Button-3>', self.show_region_menu)
# 右侧：日志
right_frame = ttk.LabelFrame(paned, text="📝 日志输出", padding="5")
paned.add(right_frame, weight=2)
self.log_text = scrolledtext.ScrolledText(right_frame, wrap=tk.WORD, font=('Consolas', 10))
self.log_text.pack(fill=tk.BOTH, expand=True)
# 状态栏
self.status_label = ttk.Label(self.root, text="✅ 就绪", relief=tk.SUNKEN)
self.status_label.pack(side=tk.BOTTOM, fill=tk.X)
def setup_log_redirect(self):
class TextHandler(logging.Handler):
def __init__(self, text_widget):
super().__init__()
self.text_widget = text_widget
def emit(self, record):
msg = self.format(record)
self.text_widget.insert(tk.END, msg + '\n')
self.text_widget.see(tk.END)
handler = TextHandler(self.log_text)
handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
logging.getLogger().addHandler(handler)
def update_region_list(self):
for item in self.region_tree.get_children():
self.region_tree.delete(item)
for name, region in self.controller.regions.items():
self.region_tree.insert('', tk.END, values=(
f"({region.x}, {region.y})",
f"{region.width}x{region.height}",
region.expected_value[:20],
region.check_type
), tags=(name,))
def update_recording_list(self):
for item in self.recording_tree.get_children():
self.recording_tree.delete(item)
for name in self.controller.list_recordings():
filepath = self.controller.settings.recordings_dir / f"{name}.json"
try:
with open(filepath, 'r', encoding='utf-8') as f:
data = json.load(f)
desc = data.get('description', '')
if '混合' in desc or 'hybrid' in desc:
type_label = "混合"
elif '命令行' in desc or 'cmd' in desc:
type_label = "命令行"
else:
type_label = "UI"
self.recording_tree.insert('', tk.END, values=(
name, type_label,
data.get('total_actions', 0),
f"{data.get('duration', 0):.1f}s",
desc[:30]
))
except Exception:
pass
def add_ocr_region(self):
region = self.controller.select_ocr_region()
if region:
self.update_region_list()
self.status_label.config(text=f"✅ 已添加OCR区域: {region.name}")
def delete_region(self):
selection = self.region_tree.selection()
if selection:
tags = self.region_tree.item(selection[0])['tags']
if tags:
name = tags[0]
if self.controller.delete_region(name):
self.update_region_list()
self.status_label.config(text=f"🗑️ 已删除区域: {name}")
def show_region_menu(self, event):
selection = self.region_tree.selection()
if selection:
self.region_menu.post(event.x_root, event.y_root)
def start_recording(self):
mode = self.mode_var.get()
if mode == "ui":
self.controller.start_ui_recording()
self.status_label.config(text="🔴 UI录制中... (按Ctrl+Shift+Q停止)")
elif mode == "cmd":
self.controller.start_cmd_recording()
self.status_label.config(text="🔴 命令行录制中... (输入 'stop' 停止)")
else:
self.controller.start_hybrid_recording()
self.status_label.config(text="🔴 混合录制中... (输入 'stop' 停止)")
def stop_recording(self):
mode = self.mode_var.get()
if mode == "ui":
filename = self.controller.stop_ui_recording()
elif mode == "cmd":
filename = self.controller.stop_cmd_recording()
else:
filename = self.controller.stop_hybrid_recording()
if filename:
self.status_label.config(text=f"✅ 已保存: {Path(filename).name}")
self.update_recording_list()
else:
self.status_label.config(text="⏹️ 录制已停止 (无操作)")
def play_recording(self):
selection = self.recording_tree.selection()
if not selection:
filename = self.controller.get_latest_recording()
if not filename:
self.status_label.config(text="❌ 没有可回放的录制")
return
else:
name = self.recording_tree.item(selection[0])['values'][0]
filename = str(self.controller.settings.recordings_dir / f"{name}.json")
speed = float(self.speed_var.get())
self.status_label.config(text=f"▶️ 回放中... (速度: {speed}x)")
def play_thread():
data = self.controller.file_manager.load_json(Path(filename))
if not data:
self.root.after(0, lambda: self.status_label.config(text="❌ 加载录制失败"))
return
desc = data.get('description', '')
try:
if '混合' in desc or 'hybrid' in desc.lower():
result = self.controller.play_hybrid_recording(filename, speed)
elif '命令行' in desc or 'cmd' in desc.lower():
result = self.controller.play_cmd_recording(filename, speed)
else:
result = self.controller.play_ui_recording(filename, speed)
if result:
self.root.after(0, lambda: self.status_label.config(text="✅ 回放完成"))
else:
self.root.after(0, lambda: self.status_label.config(text="❌ 回放失败"))
except Exception as e:
self.root.after(0, lambda: self.status_label.config(text=f"❌ 回放错误: {e}"))
threading.Thread(target=play_thread, daemon=True).start()
def show_verify_results(self):
results = self.controller.verify_all()
results_window = tk.Toplevel(self.root)
results_window.title("📊 验证结果")
results_window.geometry("600x400")
text_area = scrolledtext.ScrolledText(results_window, wrap=tk.WORD, font=('Consolas', 10))
text_area.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
text_area.insert(tk.END, "=" * 60 + "\n")
text_area.insert(tk.END, "📊 验证结果\n")
text_area.insert(tk.END, "=" * 60 + "\n\n")
if 'ocr' in results:
ocr = results['ocr']
text_area.insert(tk.END, f"🔍 OCR验证:\n")
text_area.insert(tk.END, f" 总计: {ocr.get('total', 0)} 个区域\n")
text_area.insert(tk.END, f" 通过: {ocr.get('passed', 0)} 个\n")
text_area.insert(tk.END, f" 失败: {ocr.get('failed', 0)} 个\n")
text_area.insert(tk.END, f" 通过率: {ocr.get('pass_rate', 0):.1f}%\n\n")
if ocr.get('results'):
text_area.insert(tk.END, "-" * 60 + "\n\n")
for result in ocr['results']:
status = "✅" if result.passed else "❌"
text_area.insert(tk.END, f"{status} {result.region_name}:\n")
text_area.insert(tk.END, f" 识别: {result.actual_text[:100]}\n")
text_area.insert(tk.END, f" 期望: {result.expected_value}\n")
if result.error:
text_area.insert(tk.END, f" 错误: {result.error}\n")
text_area.insert(tk.END, "\n")
else:
text_area.insert(tk.END, "⚠️ 没有配置OCR检查区域\n")
text_area.insert(tk.END, "请先点击 '添加OCR区域' 配置检查点\n")
text_area.config(state=tk.DISABLED)
def clear_log(self):
self.log_text.delete(1.0, tk.END)
def run(self):
self.root.mainloop()
- utils/init.py
"""工具模块"""
from utils.logger import setup_logger, LoggerMixin, logger
from utils.file_utils import FileManager
from utils.screenshot import ScreenshotManager
__all__ = ['setup_logger', 'LoggerMixin', 'logger', 'FileManager', 'ScreenshotManager']
- utils/logger.py
"""
日志工具
"""
import logging
import sys
from pathlib import Path
from typing import Optional
def setup_logger(
name: str = "automation",
level: str = "INFO",
log_file: Optional[str] = None,
console: bool = True
) -> logging.Logger:
logger = logging.getLogger(name)
logger.setLevel(getattr(logging, level.upper()))
logger.handlers.clear()
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
if log_file:
log_path = Path(log_file)
log_path.parent.mkdir(parents=True, exist_ok=True)
file_handler = logging.FileHandler(log_file, encoding='utf-8')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
if console:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
return logger
class LoggerMixin:
@property
def logger(self):
if not hasattr(self, '_logger'):
self._logger = setup_logger(name=self.__class__.__name__)
return self._logger
logger = setup_logger()
- utils/file_utils.py
"""
文件操作工具
"""
import json
from pathlib import Path
from typing import Any, List, Optional
from datetime import datetime
class FileManager:
def __init__(self, base_dir: Path):
self.base_dir = Path(base_dir)
self.base_dir.mkdir(parents=True, exist_ok=True)
def save_json(self, data: Any, filepath: Path, indent: int = 2):
filepath = Path(filepath)
filepath.parent.mkdir(parents=True, exist_ok=True)
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=indent, ensure_ascii=False)
def load_json(self, filepath: Path) -> Any:
filepath = Path(filepath)
if not filepath.exists():
return None
with open(filepath, 'r', encoding='utf-8') as f:
return json.load(f)
def list_files(self, directory: Path, pattern: str = "*") -> List[Path]:
directory = Path(directory)
if not directory.exists():
return []
return list(directory.glob(pattern))
def get_latest_file(self, directory: Path, pattern: str = "*") -> Optional[Path]:
files = self.list_files(directory, pattern)
if not files:
return None
return max(files, key=lambda x: x.stat().st_mtime)
def get_unique_filename(self, directory: Path, prefix: str = "", suffix: str = ".json") -> str:
directory = Path(directory)
directory.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"{prefix}_{timestamp}{suffix}"
return str(directory / filename)
- utils/screenshot.py
"""
截图工具
"""
from PIL import Image
import pyautogui
from pathlib import Path
from typing import Optional
from datetime import datetime
class ScreenshotManager:
def __init__(self, save_dir: Path):
self.save_dir = Path(save_dir)
self.save_dir.mkdir(parents=True, exist_ok=True)
def capture_fullscreen(self, filename: Optional[str] = None) -> Path:
screenshot = pyautogui.screenshot()
return self.save_screenshot(screenshot, filename or "fullscreen")
def capture_region(self, x: int, y: int, width: int, height: int,
filename: Optional[str] = None) -> Path:
screenshot = pyautogui.screenshot(region=(x, y, width, height))
return self.save_screenshot(screenshot, filename or "region")
def save_screenshot(self, image: Image.Image, name: str) -> Path:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"{name}_{timestamp}.png"
filepath = self.save_dir / filename
image.save(filepath)
return filepath
安装和使用说明
- 创建项目目录
automation_framework/
-
复制所有文件到对应目录
-
安装依赖
pip install -r requirements.txt
- 安装Tesseract OCR
· Windows: 下载 https://github.com/UB-Mannheim/tesseract/wiki
· Linux: sudo apt-get install tesseract-ocr
· Mac: brew install tesseract
- 运行
python main.py
或双击 run.bat (Windows)
框架已经完全验证通过，可以直接使用！
浙公网安备 33010602011771号