AI量化qlib学习笔记四：测试模型

这步就比较简单，把训练后保存的模型重新加载进来，把要测试的数据给它进行预测。

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
期货合约模型测试脚本 - 专门用于测试期货交易信号
"""

import qlib
import yaml
import joblib
import pandas as pd
import numpy as np
import os
import json
from datetime import datetime
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from matplotlib import rcParams

# 设置中文字体
rcParams['font.sans-serif'] = ['SimHei', 'Microsoft YaHei', 'DejaVu Sans']
rcParams['axes.unicode_minus'] = False

def test_data_loading(contract="EBL8.DCE", freq="15min"):
    """测试数据加载是否正常"""
    print("=== 测试期货数据加载 ===")
    
    # 1. 使用配置生成器生成配置
    from qlib_config_generator import QlibConfigGenerator
    generator = QlibConfigGenerator()
    config = generator.generate_config(contract, freq)
    print("✓ 配置生成完成")
    
    # 2. 初始化qlib
    qlib_config = config['qlib_init']
    qlib.init(
        provider_uri=qlib_config['provider_uri'],
        region=qlib_config['region']
    )
    print("✓ Qlib初始化完成")
    
    # 3. 测试自定义Handler
    try:
        from qlib.utils import init_instance_by_config
        dataset_config = config['task']['dataset']
        dataset = init_instance_by_config(dataset_config)
        print("✓ 自定义Handler创建成功")
        
        # 4. 获取合约信息
        handler = dataset.handler
        instruments = handler.instruments
        print(f"✓ 期货合约: {instruments}")
        print(f"Handler类型: {type(handler)}")
        
        # 5. 显示因子配置
        if hasattr(handler, 'get_feature_config'):
            features = handler.get_feature_config()
            print(f"✓ 因子配置获取成功，共 {len(features)} 个因子:")
            for i, feature in enumerate(features, 1):
                print(f"  {i}. {feature}")
        
        if hasattr(handler, 'get_label_config'):
            labels = handler.get_label_config()
            print(f"✓ 标签配置: {labels}")
        
        # 6. 测试数据形状
        try:
            train_data = dataset.prepare("train")
            valid_data = dataset.prepare("valid")
            test_data = dataset.prepare("test")
            
            # 安全地检查数据形状
            def get_data_shape(data):
                try:
                    if data is None:
                        return 'None'
                    if isinstance(data, (list, tuple)) and len(data) > 0:
                        return data[0].shape
                    elif hasattr(data, 'shape'):
                        return data.shape
                    else:
                        return f'Unknown type: {type(data)}'
                except Exception as e:
                    return f'Error: {e}'
            
            print(f"✓ 训练数据形状: {get_data_shape(train_data)}")
            print(f"✓ 验证数据形状: {get_data_shape(valid_data)}")
            print(f"✓ 测试数据形状: {get_data_shape(test_data)}")
            
            # 显示数据时间范围
            try:
                if train_data is not None and isinstance(train_data, (list, tuple)) and len(train_data) > 0:
                    train_df = train_data[0]
                    if hasattr(train_df, 'index') and hasattr(train_df.index, 'get_level_values'):
                        dates = train_df.index.get_level_values('datetime')
                        print(f"✓ 训练数据时间范围: {dates.min()} 到 {dates.max()}")
            except Exception as e:
                print(f"⚠ 无法获取时间范围: {e}")
            
        except Exception as e:
            print(f"⚠ 数据准备失败: {e}")
            return False
            
        return True
        
    except Exception as e:
        print(f"❌ 数据加载测试失败: {e}")
        import traceback
        traceback.print_exc()
        return False

def load_and_test_model(contract="EBL8.DCE", freq="15min"):
    """加载并测试期货模型"""
    print("=== 测试期货模型 ===")
    
    # 1. 使用配置生成器生成配置
    from qlib_config_generator import QlibConfigGenerator
    generator = QlibConfigGenerator()
    config = generator.generate_config(contract, freq)
    print("✓ 配置生成完成")
    
    # 2. 从配置获取合约信息
    instruments = config['task']['dataset']['kwargs']['handler']['kwargs']['instruments']
    qlib_freq = config['task']['dataset']['kwargs']['handler']['kwargs']['freq']
    
    if isinstance(instruments, list) and len(instruments) > 0:
        instrument = instruments[0]  # 取第一个合约
    else:
        instrument = instruments
    
    # 3. 构建模型文件路径
    model_file = os.path.join('models', instrument, freq, f'{instrument}_{freq}_latest.pkl')
    if not os.path.exists(model_file):
        print(f"❌ 模型文件不存在: {model_file}")
        print("请先运行 qlib_train_with_generator.py 训练模型")
        return None
    
    # 4. 初始化qlib
    qlib_config = config['qlib_init']
    qlib.init(
        provider_uri=qlib_config['provider_uri'],
        region=qlib_config['region']
    )
    print("✓ Qlib初始化完成")
    
    # 5. 加载模型
    model = joblib.load(model_file)
    print(f"✓ 模型加载成功: {model_file}")
    print(f"模型类型: {type(model)}")
    
    # 6. 加载模型信息
    try:
        info_file = os.path.join('models', instrument, freq, f'{instrument}_{freq}_latest_info.json')
        with open(info_file, 'r', encoding='utf-8') as f:
            model_info = json.load(f)
        print(f"✓ 模型训练时间: {model_info['timestamp']}")
        print(f"✓ 特征数量: {len(model_info['data_config']['features'])}")
        print(f"✓ 合约: {model_info['data_config']['instruments']}")
    except Exception as e:
        print(f"⚠ 无法加载模型信息: {e}")
    
    # 7. 重新创建dataset用于预测
    print("创建dataset用于预测...")
    from qlib.utils import init_instance_by_config
    dataset_config = config['task']['dataset']
    dataset = init_instance_by_config(dataset_config)
    
    # 8. 测试预测
    print("测试模型预测...")
    predictions = model.predict(dataset, segment="test")
    print(f"✓ 预测成功，结果形状: {predictions.shape}")
    
    # 9. 验证预测准确性 - 直接对比最高预测点
    validate_top_predictions_simple(predictions, dataset, qlib_freq)
    
    # 10. 分析预测结果
    # analyze_futures_predictions(predictions)
    
    # # 11. 生成交易信号和收益图表
    # generate_trading_signals(predictions)
    
    return model

def validate_top_predictions(predictions, dataset):
    """验证预测收益最高的10个时间点"""
    print("\n=== 验证预测准确性 ===")
    
    try:
        pred_df = predictions.reset_index()
        score_col = pred_df.columns[-1]
        
        # 确保有时间列
        if 'datetime' not in pred_df.columns:
            print("⚠ 预测数据中没有时间信息，无法进行时间点验证")
            return
        
        pred_df['datetime'] = pd.to_datetime(pred_df['datetime'])
        pred_df = pred_df.sort_values('datetime')
        
        # 找出预测收益最高和最低的10个时间点
        top_predictions = pred_df.nlargest(10, score_col)
        bottom_predictions = pred_df.nsmallest(10, score_col)
        
        print(f"预测收益最高的10个时间点验证 (做多信号):")
        print(f"{'序号':<4} {'时间':<20} {'预测收益':<12} {'验证结果'}")
        print("-" * 70)
        
        # 尝试获取原始价格数据进行验证
        try:
            from qlib.data import D
            
            # 获取合约信息
            handler = dataset.handler
            instruments = handler.instruments
            
            # 获取价格数据 (收盘价)
            start_time = pred_df['datetime'].min()
            end_time = pred_df['datetime'].max() + pd.Timedelta(days=5)  # 多取几天数据
            
            price_data = D.features(
                instruments, 
                ["$close"], 
                start_time=start_time, 
                end_time=end_time
            )
            
            if price_data is not None and not price_data.empty:
                price_df = price_data.reset_index()
                price_df['datetime'] = pd.to_datetime(price_df['datetime'])
                price_df = price_df.sort_values('datetime')
                
                correct_count = 0
                total_verified = 0
                
                # 验证每个高预测点
                for i, (idx, row) in enumerate(top_predictions.iterrows(), 1):
                    pred_time = row['datetime']
                    pred_score = row[score_col]
                    
                    # 找到预测时间点的价格
                    current_price_row = price_df[price_df['datetime'] == pred_time]
                    
                    if not current_price_row.empty:
                        current_price = current_price_row.iloc[0]['close']
                        
                        # 找到后续几个交易日的价格
                        future_prices = price_df[price_df['datetime'] > pred_time].head(3)
                        
                        if not future_prices.empty:
                            # 计算实际收益率
                            next_price = future_prices.iloc[0]['close']
                            actual_return = (next_price - current_price) / current_price
                            
                            # 判断预测是否正确
                            pred_direction = "涨" if pred_score > 0 else "跌"
                            actual_direction = "涨" if actual_return > 0 else "跌"
                            correct = "✓" if (pred_score > 0) == (actual_return > 0) else "✗"
                            
                            if (pred_score > 0) == (actual_return > 0):
                                correct_count += 1
                            total_verified += 1
                            
                            print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   {correct} 预测{pred_direction} 实际{actual_direction} ({actual_return:+.4f})")
                            
                            # 显示后续价格变化
                            print(f"     价格变化: {current_price:.2f} → ", end="")
                            for j, future_row in future_prices.iterrows():
                                future_price = future_row['close']
                                change = (future_price - current_price) / current_price * 100
                                print(f"{future_price:.2f}({change:+.2f}%) ", end="")
                            print()
                        else:
                            print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   ? 无后续价格数据")
                    else:
                        print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   ? 找不到当日价格")
                
                # 统计高预测点准确率
                if total_verified > 0:
                    accuracy = correct_count / total_verified * 100
                    print(f"\n高预测点验证结果: {correct_count}/{total_verified} = {accuracy:.1f}%")
                    
                    if accuracy >= 70:
                        print("🎉 做多预测质量: 优秀")
                    elif accuracy >= 60:
                        print("👍 做多预测质量: 良好") 
                    elif accuracy >= 50:
                        print("👌 做多预测质量: 一般")
                    else:
                        print("👎 做多预测质量: 较差")
                else:
                    print("⚠ 无法验证任何高预测点")
                
                # 验证最低预测点 (做空信号)
                print(f"\n" + "="*70)
                print(f"预测收益最低的10个时间点验证 (做空信号):")
                print(f"{'序号':<4} {'时间':<20} {'预测收益':<12} {'验证结果'}")
                print("-" * 70)
                
                correct_count_low = 0
                total_verified_low = 0
                
                # 验证每个低预测点
                for i, (idx, row) in enumerate(bottom_predictions.iterrows(), 1):
                    pred_time = row['datetime']
                    pred_score = row[score_col]
                    
                    # 找到预测时间点的价格
                    current_price_row = price_df[price_df['datetime'] == pred_time]
                    
                    if not current_price_row.empty:
                        current_price = current_price_row.iloc[0]['close']
                        
                        # 找到后续几个交易时间点的价格
                        future_prices = price_df[price_df['datetime'] > pred_time].head(3)
                        
                        if not future_prices.empty:
                            # 计算实际收益率
                            next_price = future_prices.iloc[0]['close']
                            actual_return = (next_price - current_price) / current_price
                            
                            # 判断预测是否正确 (负预测应该对应负收益)
                            pred_direction = "涨" if pred_score > 0 else "跌"
                            actual_direction = "涨" if actual_return > 0 else "跌"
                            correct = "✓" if (pred_score > 0) == (actual_return > 0) else "✗"
                            
                            if (pred_score > 0) == (actual_return > 0):
                                correct_count_low += 1
                            total_verified_low += 1
                            
                            print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   {correct} 预测{pred_direction} 实际{actual_direction} ({actual_return:+.4f})")
                            
                            # 显示后续价格变化
                            print(f"     价格变化: {current_price:.2f} → ", end="")
                            for j, future_row in future_prices.iterrows():
                                future_price = future_row['close']
                                change = (future_price - current_price) / current_price * 100
                                print(f"{future_price:.2f}({change:+.2f}%) ", end="")
                            print()
                        else:
                            print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   ? 无后续价格数据")
                    else:
                        print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   ? 找不到当日价格")
                
                # 统计低预测点准确率
                if total_verified_low > 0:
                    accuracy_low = correct_count_low / total_verified_low * 100
                    print(f"\n低预测点验证结果: {correct_count_low}/{total_verified_low} = {accuracy_low:.1f}%")
                    
                    if accuracy_low >= 70:
                        print("🎉 做空预测质量: 优秀")
                    elif accuracy_low >= 60:
                        print("👍 做空预测质量: 良好") 
                    elif accuracy_low >= 50:
                        print("👌 做空预测质量: 一般")
                    else:
                        print("👎 做空预测质量: 较差")
                else:
                    print("⚠ 无法验证任何低预测点")
                
                # 综合统计
                total_all_verified = total_verified + total_verified_low
                total_all_correct = correct_count + correct_count_low
                if total_all_verified > 0:
                    overall_accuracy = total_all_correct / total_all_verified * 100
                    print(f"\n综合验证结果: {total_all_correct}/{total_all_verified} = {overall_accuracy:.1f}%")
                    print(f"预测值与实际收益的数量级差异分析:")
                    print(f"  - 这可能表明模型预测的是相对强弱，而非绝对收益率")
                    print(f"  - 建议关注预测方向的准确性，而非绝对数值")
                    
            else:
                print("⚠ 无法获取价格数据进行验证")
                # 如果无法获取价格数据，至少显示预测结果
                for i, (idx, row) in enumerate(top_predictions.iterrows(), 1):
                    pred_time = row['datetime']
                    pred_score = row[score_col]
                    print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   (无法验证)")
                
                # 也显示最低预测点
                print(f"\n" + "="*70)
                print(f"预测收益最低的10个时间点 (做空信号):")
                print(f"{'序号':<4} {'时间':<20} {'预测收益':<12} {'验证结果'}")
                print("-" * 70)
                for i, (idx, row) in enumerate(bottom_predictions.iterrows(), 1):
                    pred_time = row['datetime']
                    pred_score = row[score_col]
                    print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   (无法验证)")
                
        except Exception as e:
            print(f"⚠ 获取价格数据失败: {e}")
            # 如果无法获取价格数据，至少显示预测结果
            for i, (idx, row) in enumerate(top_predictions.iterrows(), 1):
                pred_time = row['datetime']
                pred_score = row[score_col]
                print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   (无法验证)")
            
            # 也显示最低预测点
            print(f"\n" + "="*70)
            print(f"预测收益最低的10个时间点 (做空信号):")
            print(f"{'序号':<4} {'时间':<20} {'预测收益':<12} {'验证结果'}")
            print("-" * 70)
            for i, (idx, row) in enumerate(bottom_predictions.iterrows(), 1):
                pred_time = row['datetime']
                pred_score = row[score_col]
                print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   (无法验证)")
            
    except Exception as e:
        print(f"⚠ 预测验证失败: {e}")

 
 
def main():
    """主函数"""
    try:
        print("=== 期货合约模型测试 ===")
        
        # 配置测试参数
        contract = "EBL8.DCE"
        freq = "day"
        
        print(f"测试合约: {contract}")
        print(f"测试频率: {freq}")
        
        # 1. 先测试数据加载
        print("\n第一步：测试数据加载...")
        data_ok = test_data_loading(contract, freq)
        
        if not data_ok:
            print("❌ 数据加载测试失败，请检查配置和数据")
            return
        
        print("\n" + "="*60)
        
        # 2. 再测试模型加载（如果模型存在）
        print("第二步：测试模型加载和预测...")
        model = load_and_test_model(contract, freq)
        
        if model is not None:
            print(f"\n🎉 期货模型测试完成！")
            print(f"\n💡 实盘交易建议:")
            print(f"1. 根据预测信号调整仓位")
            print(f"2. 设置合理的止损止盈")
            print(f"3. 控制单次交易风险")
            print(f"4. 定期重新训练模型")
        else:
            print(f"\n⚠ 数据加载正常，但模型不存在")
            print(f"请先运行 qlib_train_with_generator.py 训练模型")
        
    except Exception as e:
        print(f"❌ 测试失败: {e}")
        import traceback
        traceback.print_exc()

def validate_top_predictions_simple(predictions, dataset, qlib_freq):
    """验证预测收益最高和最低的10个时间点 - 简化版"""
    print("\n=== 验证预测准确性 ===")
    
    try:
        pred_df = predictions.reset_index()
        score_col = pred_df.columns[-1]
        
        if 'datetime' not in pred_df.columns:
            print("⚠ 预测数据中没有时间信息")
            return
        
        pred_df['datetime'] = pd.to_datetime(pred_df['datetime'])
        pred_df = pred_df.sort_values('datetime')
        
        # 直接从qlib获取价格数据
        try:
            from qlib.data import D
            handler = dataset.handler
            instruments = handler.instruments
            
            start_time = pred_df['datetime'].min()
            end_time = pred_df['datetime'].max() + pd.Timedelta(hours=1)
            
            print(f"从qlib获取价格数据: {instruments}")
            print(f"使用频率: {qlib_freq}")
            print(f"时间范围: {start_time} 到 {end_time}")
            
            # 使用配置中的频率获取收盘价数据
            price_data = D.features(instruments, ["$close"], start_time=start_time, end_time=end_time, freq=qlib_freq)
            
            if price_data is None or price_data.empty:
                print("⚠ 无法获取价格数据")
                return
            
            price_df = price_data.reset_index()
            price_df['datetime'] = pd.to_datetime(price_df['datetime'])
            price_df = price_df.sort_values('datetime')
            price_col = '$close'  # qlib返回的列名是$close
            
            print(f"获取到 {len(price_df)} 条价格数据")
            print(f"实际时间范围: {price_df['datetime'].min()} 到 {price_df['datetime'].max()}")
            print(f"价格列名: {price_col}")
            
        except Exception as e:
            print(f"⚠ 获取价格数据失败: {e}")
            return
        
        # 验证最高预测点
        top_predictions = pred_df.nlargest(10, score_col)
        print(f"预测收益最高的10个时间点:")
        print(f"{'序号':<4} {'时间':<20} {'预测收益':<12} {'当前价格':<10} {'下期价格':<10} {'实际收益':<12} {'结果'}")
        print("-" * 90)
        
        close_index_pos=1
        correct_high = 0
        for i, (_, row) in enumerate(top_predictions.iterrows(), 1):
            pred_time = row['datetime']
            pred_score = row[score_col]
            
            # 找当前价格
            current_row = price_df[price_df['datetime'] == pred_time]
            if current_row.empty:
                print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   找不到价格")
                continue
            
            current_price = current_row.iloc[0][price_col]
            
            # 找下一根K线价格
            next_row = price_df[price_df['datetime'] > pred_time].head(close_index_pos)
            if next_row.empty:
                print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   {current_price:<10.2f} 无下期数据")
                continue
            
            next_price = next_row.iloc[close_index_pos-1][price_col]
            actual_return = (next_price - current_price) / current_price
            
            # 判断方向是否正确
            correct = "✓" if (pred_score > 0) == (actual_return > 0) else "✗"
            if (pred_score > 0) == (actual_return > 0):
                correct_high += 1
            
            print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   {current_price:<10.2f} {next_price:<10.2f} {actual_return:+.6f}   {correct}")
        
        # 验证最低预测点
        bottom_predictions = pred_df.nsmallest(10, score_col)
        print(f"\n预测收益最低的10个时间点:")
        print(f"{'序号':<4} {'时间':<20} {'预测收益':<12} {'当前价格':<10} {'下期价格':<10} {'实际收益':<12} {'结果'}")
        print("-" * 90)
        
        correct_low = 0
        for i, (_, row) in enumerate(bottom_predictions.iterrows(), 1):
            pred_time = row['datetime']
            pred_score = row[score_col]
            
            # 找当前价格
            current_row = price_df[price_df['datetime'] == pred_time]
            if current_row.empty:
                print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   找不到价格")
                continue
            
            current_price = current_row.iloc[0][price_col]
            
            # 找下一根K线价格
            next_row = price_df[price_df['datetime'] > pred_time].head(close_index_pos)
            if next_row.empty:
                print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   {current_price:<10.2f} 无下期数据")
                continue

            next_price = next_row.iloc[close_index_pos-1][price_col]
            actual_return = (next_price - current_price) / current_price
            
            # 判断方向是否正确
            correct = "✓" if (pred_score > 0) == (actual_return > 0) else "✗"
            if (pred_score > 0) == (actual_return > 0):
                correct_low += 1
            
            print(f"{i:<4} {pred_time.strftime('%Y-%m-%d %H:%M:%S'):<20} {pred_score:+.6f}   {current_price:<10.2f} {next_price:<10.2f} {actual_return:+.6f}   {correct}")
        
        # 统计结果
        total_correct = correct_high + correct_low
        total_tested = 20  # 最多20个点
        if total_tested > 0:
            accuracy = total_correct / total_tested * 100
            print(f"\n验证结果: {total_correct}/{total_tested} = {accuracy:.1f}%")
            print(f"做多准确率: {correct_high}/10 = {correct_high*10:.1f}%")
            print(f"做空准确率: {correct_low}/10 = {correct_low*10:.1f}%")
        
    except Exception as e:
        print(f"⚠ 验证失败: {e}")

if __name__ == "__main__":
    main()