AI量化qlib学习笔记三：训练模型

这里有几个重要注意点，一是模型，它提供了好几个模型，但对时间序列以及初入学者，基本都是选择LGBModel（LightGBM）

二是因子，它有一些因子库，最常见的就是Alpha158，可以自己去学习，因为我是想做自己的因子，所以都是自己定义的，事实证明花一两周的时间，只是个匹毛，这里一定要搞清楚模型根据因子学习的原理，举个例子，传统量化可能是找金叉再加其它条件，可能触发信号，AI里没有这个概念，因为你要在AI里也这样写，就没有用AI的意义了。你只要告诉它注意dif和dea这两个因子的数据，它会自己去学习。如果写得太具体，就容易过拟合，也就是定制信号，搞不好这种后面都不再出现。所以写因子会是整个AI量化的绝大分工作。另外要注意，不支持跨周期是我放弃的最大原因。

三是可以参考它的例子开启训练，qlib_src\examples\benchmarks\LightGBM下有好几个yaml配置文件 ，启动训练：qrun 配置.yaml ，这里yaml里的参数，够学习很长时间了。

它这个是一个整体工作流，从训练，到回测到分析都有，但实际对刚接触AI的人非常难，所以为了学习我在这里就把这些步骤分拆了训练就单训练。

四，用配置文件是个非常不方便的方法 ，所以我用代码来实现了配置文件qlib_config_generator.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
使用配置生成器的Qlib训练脚本 - 参考qlib_train_save.py的结构
"""

import qlib
import joblib
import os
import json
from qlib.utils import init_instance_by_config
from datetime import datetime, date
from pathlib import Path

# 导入我们的配置生成器
from qlib_config_generator import QlibConfigGenerator


def create_model_directory(contract, freq):
    """创建模型保存目录"""
    base_dir = 'models'
    model_dir = os.path.join(base_dir, contract, freq)
    os.makedirs(model_dir, exist_ok=True)
    return model_dir

def get_features_from_handler(dataset):
    """从dataset的handler中获取特征配置"""
    try:
        handler = dataset.handler
        if hasattr(handler, 'get_feature_config'):
            return handler.get_feature_config()
        elif hasattr(handler, 'data_loader') and hasattr(handler.data_loader, 'config'):
            return handler.data_loader.config.get('feature', [])
        else:
            return []
    except Exception as e:
        print(f"⚠ 获取特征配置失败: {e}")
        return []

def get_labels_from_handler(dataset):
    """从dataset的handler中获取标签配置"""
    try:
        handler = dataset.handler
        if hasattr(handler, 'get_label_config'):
            return handler.get_label_config()
        elif hasattr(handler, 'data_loader') and hasattr(handler.data_loader, 'config'):
            return handler.data_loader.config.get('label', [])
        else:
            return []
    except Exception as e:
        print(f"⚠ 获取标签配置失败: {e}")
        return []

def json_serializer(obj):
    """自定义JSON序列化器，处理date和datetime对象"""
    if isinstance(obj, (datetime, date)):
        return obj.isoformat()
    raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable")

def train_model(contract, freq, start_time=None, end_time=None):
    """
    训练模型
    
    Args:
        contract: 合约名称，如 "EBL8.DCE"
        freq: 频率，如 "1min", "5min", "15min", "day"
        start_time: 开始时间（可选，默认从数据中自动获取）
        end_time: 结束时间（可选，默认从数据中自动获取）
    """
    print(f"=== 按合约和周期训练并保存qlib模型 ===")
    print(f"合约: {contract}, 周期: {freq}")
    
    # 1. 生成配置
    generator = QlibConfigGenerator()
    config = generator.generate_config(contract, freq, start_time, end_time)
    print("✓ 配置生成完成")
    
    # 2. 创建模型保存目录
    model_dir = create_model_directory(contract, freq)
    print(f"✓ 模型保存目录: {model_dir}")
    
    # 3. 初始化qlib
    qlib_config = config['qlib_init']
    qlib.init(
        provider_uri=qlib_config['provider_uri'],
        region=qlib_config['region']
    )
    print("✓ Qlib初始化完成")
    print(f"  数据路径: {qlib_config['provider_uri']}")
    
    # 4. 创建数据集
    dataset_config = config['task']['dataset']
    dataset = init_instance_by_config(dataset_config)
    print("✓ 数据集创建完成")
    
    # 5. 训练模型
    model_config = config['task']['model']
    model = init_instance_by_config(model_config)
    
    print("开始训练模型...")
    model.fit(dataset)
    print("✓ 模型训练完成")
    
    # 6. 评估模型
    performance_stats = {}
    try:
        test_pred = model.predict(dataset, segment="test")
        pred_mean = test_pred.mean().iloc[0] if hasattr(test_pred.mean(), 'iloc') else test_pred.mean()
        pred_std = test_pred.std().iloc[0] if hasattr(test_pred.std(), 'iloc') else test_pred.std()
        
        performance_stats = {
            'pred_mean': float(pred_mean),
            'pred_std': float(pred_std),
            'pred_count': len(test_pred)
        }
        
        print(f"模型评估 - 预测均值: {pred_mean:.6f}, 标准差: {pred_std:.6f}")
    except Exception as e:
        print(f"⚠ 模型评估失败: {e}")
        performance_stats = {'error': str(e)}
    
    # 7. 保存模型（按目录结构）
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    
    # 模型文件路径
    model_file = os.path.join(model_dir, f'{contract}_{freq}_model_{timestamp}.pkl')
    latest_file = os.path.join(model_dir, f'{contract}_{freq}_latest.pkl')
    
    # 保存模型
    joblib.dump(model, model_file)
    joblib.dump(model, latest_file)
    
    print(f"✓ 模型已保存到: {model_file}")
    print(f"✓ 最新模型: {latest_file}")
    
    # 8. 保存详细的模型信息
    model_info = {
        'contract': contract,
        'frequency': freq,
        'timestamp': timestamp,
        'model_type': config['task']['model']['class'],
        'model_params': config['task']['model']['kwargs'],
        'training_period': {
            'train': config['task']['dataset']['kwargs']['segments']['train'],
            'valid': config['task']['dataset']['kwargs']['segments']['valid'],
            'test': config['task']['dataset']['kwargs']['segments']['test']
        },
        'data_config': {
            'instruments': config['task']['dataset']['kwargs']['handler']['kwargs']['instruments'],
            'features': get_features_from_handler(dataset),
            'label': get_labels_from_handler(dataset)
        },
        'performance': performance_stats,
        'files': {
            'model_file': model_file,
            'latest_file': latest_file
        }
    }
    
    # 保存模型信息
    info_file = os.path.join(model_dir, f'{contract}_{freq}_info_{timestamp}.json')
    latest_info_file = os.path.join(model_dir, f'{contract}_{freq}_latest_info.json')
    
    with open(info_file, 'w', encoding='utf-8') as f:
        json.dump(model_info, f, ensure_ascii=False, indent=2, default=json_serializer)
    
    with open(latest_info_file, 'w', encoding='utf-8') as f:
        json.dump(model_info, f, ensure_ascii=False, indent=2, default=json_serializer)
    
    print(f"✓ 模型信息已保存: {info_file}")
    
    # 9. 创建模型索引文件（便于管理多个模型）
    index_file = os.path.join('models', 'model_index.json')
    
    # 读取现有索引
    if os.path.exists(index_file):
        with open(index_file, 'r', encoding='utf-8') as f:
            index = json.load(f)
    else:
        index = {}
    
    # 更新索引
    key = f"{contract}_{freq}"
    index[key] = {
        'contract': contract,
        'frequency': freq,
        'latest_model': latest_file,
        'latest_info': latest_info_file,
        'last_updated': timestamp,
        'model_dir': model_dir
    }
    
    # 保存索引
    with open(index_file, 'w', encoding='utf-8') as f:
        json.dump(index, f, ensure_ascii=False, indent=2, default=json_serializer)
    
    print(f"✓ 模型索引已更新: {index_file}")
    
    print(f"\n🎉 模型训练和保存完成！")
    print(f"📁 模型目录结构:")
    print(f"   models/")
    print(f"   ├── {contract}/")
    print(f"   │   └── {freq}/")
    print(f"   │       ├── {contract}_{freq}_latest.pkl")
    print(f"   │       ├── {contract}_{freq}_latest_info.json")
    print(f"   │       └── {contract}_{freq}_model_{timestamp}.pkl")
    print(f"   └── model_index.json")
    print(f"\n💡 可以使用模型索引文件快速定位和加载模型")
    
    return model, model_info

def batch_train(contracts, frequencies, start_time=None, end_time=None):
    """
    批量训练多个合约和频率
    
    Args:
        contracts: 合约列表，如 ["EBL8.DCE", "RBL8.SHFE"]
        frequencies: 频率列表，如 ["1min", "15min", "day"]
        start_time: 开始时间（可选，默认从数据中自动获取）
        end_time: 结束时间（可选，默认从数据中自动获取）
    """
    print(f"=== 批量训练 ===")
    print(f"合约: {contracts}")
    print(f"频率: {frequencies}")
    if start_time and end_time:
        print(f"时间范围: {start_time} 到 {end_time}")
    else:
        print("时间范围: 自动从数据中获取")
    
    results = {}
    
    for contract in contracts:
        results[contract] = {}
        for freq in frequencies:
            print(f"\n{'='*80}")
            try:
                model, model_info = train_model(contract, freq, start_time, end_time)
                results[contract][freq] = "成功"
            except Exception as e:
                print(f"✗ {contract}-{freq} 训练失败: {e}")
                results[contract][freq] = f"失败: {e}"
                import traceback
                traceback.print_exc()
    
    # 打印汇总结果
    print(f"\n{'='*80}")
    print("=== 批量训练结果汇总 ===")
    for contract, freq_results in results.items():
        print(f"\n{contract}:")
        for freq, status in freq_results.items():
            print(f"  {freq}: {status}")
    
    return results

def main():
    """主函数"""
    print("=== Qlib训练脚本（使用配置生成器）===")
    
    try:
        # 单个训练示例
        contract = "EBL8.DCE"
        freq = "day"
        
        print(f"训练合约: {contract}")
        print(f"训练频率: {freq}")
        
        model, model_info = train_model(contract, freq)
        print("\n✓ 训练成功完成！")
        
        # 如果需要批量训练，取消下面的注释
        # print("\n" + "="*80)
        # batch_results = batch_train(
        #     contracts=["EBL8.DCE", "RBL8.SHFE"], 
        #     frequencies=["15min", "day"]
        # )
        
    except Exception as e:
        print(f"❌ 训练失败: {e}")
        import traceback
        traceback.print_exc()

if __name__ == "__main__":
    main()

qlib_config_generator.py

 

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Qlib配置生成器 - 集成自定义Handler和配置文件生成
"""

import pandas as pd
import numpy as np
import yaml
import os
from pathlib import Path
from qlib.data.dataset.handler import DataHandlerLP
from qlib.log import get_module_logger
import glob
from datetime import datetime
import qlib
from qlib.data import D

# ==================== 自定义Handler类 ====================

class CustomTrendHandler_m1(DataHandlerLP):
    """自定义因子处理器 - 1分钟"""
    
    def __init__(self, **kwargs):
        self.freq = kwargs.pop('freq', '1min')
        kwargs.pop('inst_processors', None)
        
        feature_config = self.get_feature_config()
        label_config = self.get_label_config()
        
        kwargs['data_loader'] = {
            'class': 'QlibDataLoader',
            'module_path': 'qlib.data.dataset.loader',
            'kwargs': {
                'freq': self.freq,
                'config': {
                    'feature': feature_config,
                    'label': label_config
                }
            }
        }
                
        super().__init__(**kwargs)
        self.logger = get_module_logger(self.__class__.__name__)
        self.logger.info(f"CustomTrendHandler初始化完成，使用 {len(feature_config)} 个因子")
        
    def get_feature_config(self):
        """1分钟级别的因子配置"""
        ma7 = "Mean(Mean($close, 7), 3)"
        ma20 = "Mean(Mean($close, 20), 3)" 
        ma60 = "Mean(Mean($close, 60), 3)"
        v5 = "Mean(Mean($volume, 5), 3)"
        v15= "Mean(Mean($volume, 15), 3)"
        v50= "Mean(Mean($volume, 50), 3)"
        
        volume_fields = [
            f"$volume / {v50}",
            f"Ref($volume,1) / Ref({v50},1)",
            f"Ref($volume,2) / Ref({v50},2)",
            f"Ref($volume,3) / Ref({v50},3)",
            f"({v5} - {v15})/(Ref({v5} - {v15},1))",
            f"({v5} - {v50})/(Ref({v5} - {v50},1))",
            f"({v15} - {v50})/(Ref({v15} - {v50},1))",
        ]
        
        ma_fields = [
            f"{ma7} / {ma20}",
            f"{ma7} / {ma60}",
            f"{ma20} / {ma60}",  
            f"(({ma7} - {ma20}) / Ref({ma7} - {ma20}, 1))",
            f"(({ma7} - {ma60}) / Ref({ma7} - {ma60}, 1))",
            f"(({ma20} - {ma60}) / Ref({ma20} - {ma60}, 1))",
        ]
        
        current_body = "Abs($close - $open)"
        prev_4_avg = "Mean(Ref(Abs($close - $open), 1), 4)"
        
        price_fields = [
            f"{current_body} / {prev_4_avg}",
            current_body,
            f"{current_body} / $close",
            "($close - $open) / $close",
            "($high - Greater($close, $open)) / $close",
            "(Less($close, $open) - $low) / $close",
            f"($close - Ref($close, 3)) / Ref($close, 3)",
        ]
        
        return volume_fields + ma_fields + price_fields
    
    def get_label_config(self):
        return ["Ref($close, -10) / $close - 1"]

class CustomTrendHandler_m5(DataHandlerLP):
    """自定义因子处理器 - 5分钟"""
    
    def __init__(self, **kwargs):
        self.freq = kwargs.pop('freq', '5min')
        kwargs.pop('inst_processors', None)
        
        feature_config = self.get_feature_config()
        label_config = self.get_label_config()
        
        kwargs['data_loader'] = {
            'class': 'QlibDataLoader',
            'module_path': 'qlib.data.dataset.loader',
            'kwargs': {
                'freq': self.freq,
                'config': {
                    'feature': feature_config,
                    'label': label_config
                }
            }
        }
                
        super().__init__(**kwargs)
        self.logger = get_module_logger(self.__class__.__name__)
        
    def get_feature_config(self):
        """5分钟级别的因子配置"""
        ma5 = "Mean($close, 5)"
        ma15 = "Mean($close, 15)" 
        ma30 = "Mean($close, 30)"
        
        volume_fields = [
            "$volume / Mean($volume, 20)",
            "($volume - Ref($volume, 1)) / Ref($volume, 1)",
            "$volume / Mean(Mean($volume, 30), 3)",
        ]
        
        ma_fields = [
            f"({ma5} - Ref({ma5}, 1)) / {ma5}",
            f"({ma15} - Ref({ma15}, 1)) / {ma15}",
            f"({ma30} - Ref({ma30}, 1)) / {ma30}",
            f"{ma5} / {ma15}",
            f"{ma15} / {ma30}",
        ]
        
        price_fields = [
            "Abs($close - $open) / $close",
            "($close - $open) / $close",
            "($high - Greater($close, $open)) / $close",
            "(Less($close, $open) - $low) / $close",
        ]
        
        return volume_fields + ma_fields + price_fields
    
    def get_label_config(self):
        return ["Ref($close, -5) / $close - 1"]

class CustomTrendHandler_m15(DataHandlerLP):
    """自定义因子处理器 - 15分钟"""
    
    def __init__(self, **kwargs):
        self.freq = kwargs.pop('freq', '15min')
        kwargs.pop('inst_processors', None)
        print("CustomTrendHandler_m15 初始化，freq =", self.freq)
        feature_config = self.get_feature_config()
        label_config = self.get_label_config()
        
        kwargs['data_loader'] = {
            'class': 'QlibDataLoader',
            'module_path': 'qlib.data.dataset.loader',
            'kwargs': {
                'freq': self.freq,
                'config': {
                    'feature': feature_config,
                    'label': label_config
                }
            }
        }
                
        super().__init__(**kwargs)
        self.logger = get_module_logger(self.__class__.__name__)
        
    def get_feature_config(self):
        """15分钟级别的因子配置"""
        ma4 = "Mean($close, 4)"
        ma12 = "Mean($close, 12)" 
        ma24 = "Mean($close, 24)"
        
        volume_fields = [
            "$volume / Mean($volume, 16)",
            "($volume - Ref($volume, 1)) / Ref($volume, 1)",
            "$volume / Mean(Mean($volume, 20), 2)",
        ]
        
        ma_fields = [
            f"({ma4} - Ref({ma4}, 1)) / {ma4}",
            f"({ma12} - Ref({ma12}, 1)) / {ma12}",
            f"({ma24} - Ref({ma24}, 1)) / {ma24}",
            f"{ma4} / {ma12}",
            f"{ma12} / {ma24}",
        ]
        
        price_fields = [
            "Abs($close - $open) / $close",
            "($close - $open) / $close",
            "($high - Greater($close, $open)) / $close",
            "(Less($close, $open) - $low) / $close",
            "($close - Ref($close, 2)) / Ref($close, 2)",
        ]
        
        return volume_fields + ma_fields + price_fields
    
    def get_label_config(self):
        return ["Ref($close, -3) / $close - 1"]

class CustomTrendHandler_day(DataHandlerLP):
    """自定义因子处理器 - 日线"""
    
    def __init__(self, **kwargs):
        self.freq = kwargs.pop('freq', 'day')
        kwargs.pop('inst_processors', None)
        print("CustomTrendHandler_day 初始化，freq =", self.freq)
        feature_config = self.get_feature_config()
        label_config = self.get_label_config()
        
        kwargs['data_loader'] = {
            'class': 'QlibDataLoader',
            'module_path': 'qlib.data.dataset.loader',
            'kwargs': {
                'freq': self.freq,
                'config': {
                    'feature': feature_config,
                    'label': label_config
                }
            }
        }
                
        super().__init__(**kwargs)
        self.logger = get_module_logger(self.__class__.__name__)
        
    def get_feature_config(self):
        """日线级别的因子配置 - 修复的大涨回调策略因子"""
        
        # 基础均线
        ma5 = "Mean($close, 5)"
        ma20 = "Mean($close, 20)" 
        ma60 = "Mean($close, 60)"
        
        # 原有的基础因子
        volume_fields = [
            "$volume / Mean($volume, 20)",
            "($volume - Ref($volume, 1)) / Ref($volume, 1)",
            "$volume / Mean($volume, 5)",
        ]
        
        ma_fields = [
            f"({ma5} - Ref({ma5}, 1)) / {ma5}",
            f"({ma20} - Ref({ma20}, 1)) / {ma20}",
            f"({ma60} - Ref({ma60}, 1)) / {ma60}",
            f"{ma5} / {ma20}",
            f"{ma20} / {ma60}",
        ]
        
        price_fields = [
            "Abs($close - $open) / $close",
            "($close - $open) / $close",
            "($high - Greater($close, $open)) / $close",
            "(Less($close, $open) - $low) / $close",
            "($close - Ref($close, 1)) / Ref($close, 1)",
        ]
        
        # 修复的大涨回调策略因子
        strategy_fields = [
            # 1. 检测各天的涨幅
            "(Ref($close, 1) - Ref($close, 2)) / Ref($close, 2)",  # 1天前涨幅
            "(Ref($close, 2) - Ref($close, 3)) / Ref($close, 3)",  # 2天前涨幅
            "(Ref($close, 3) - Ref($close, 4)) / Ref($close, 4)",  # 3天前涨幅
            "(Ref($close, 4) - Ref($close, 5)) / Ref($close, 5)",  # 4天前涨幅
            "(Ref($close, 5) - Ref($close, 6)) / Ref($close, 6)",  # 5天前涨幅
            
            # 2. 检测是否有大涨（>3%）
            "If((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03, 1, 0)",  # 1天前大涨
            "If((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03, 1, 0)",  # 2天前大涨
            "If((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03, 1, 0)",  # 3天前大涨
            "If((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03, 1, 0)",  # 4天前大涨
            "If((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03, 1, 0)",  # 5天前大涨
            
            # 3. 5天内是否有大涨
            "If(((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03) | " +
            "((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03) | " +
            "((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03) | " +
            "((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03) | " +
            "((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03), 1, 0)",  # 5天内有大涨
            
            # 4. 找到最近的大涨日收盘价（从近到远检查）
            "If((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03, Ref($close, 1), " +
            "If((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03, Ref($close, 2), " +
            "If((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03, Ref($close, 3), " +
            "If((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03, Ref($close, 4), " +
            "If((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03, Ref($close, 5), $close)))))",  # 最近大涨日收盘价
            
            # 5. 大涨后是否有回调（检查大涨日后的最低价是否低于大涨日收盘价）
            # 1天前大涨的情况
            "If((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03, " +
            "If($close < Ref($close, 1), 1, 0), 0)",  # 1天前大涨后今天回调
            
            # 2天前大涨的情况
            "If((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03, " +
            "If(Min($close, 2) < Ref($close, 2), 1, 0), 0)",  # 2天前大涨后有回调
            
            # 3天前大涨的情况
            "If((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03, " +
            "If(Min($close, 3) < Ref($close, 3), 1, 0), 0)",  # 3天前大涨后有回调
            
            # 4天前大涨的情况
            "If((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03, " +
            "If(Min($close, 4) < Ref($close, 4), 1, 0), 0)",  # 4天前大涨后有回调
            
            # 5天前大涨的情况
            "If((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03, " +
            "If(Min($close, 5) < Ref($close, 5), 1, 0), 0)",  # 5天前大涨后有回调
            
            # 6. 综合：是否有大涨后回调
            "If((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03, If($close < Ref($close, 1), 1, 0), " +
            "If((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03, If(Min($close, 2) < Ref($close, 2), 1, 0), " +
            "If((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03, If(Min($close, 3) < Ref($close, 3), 1, 0), " +
            "If((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03, If(Min($close, 4) < Ref($close, 4), 1, 0), " +
            "If((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03, If(Min($close, 5) < Ref($close, 5), 1, 0), 0)))))",
            
            # 7. 当前价格相对最近大涨日的比例（检查是否跌破80%）
            "If((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03, $close / Ref($close, 1), " +
            "If((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03, $close / Ref($close, 2), " +
            "If((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03, $close / Ref($close, 3), " +
            "If((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03, $close / Ref($close, 4), " +
            "If((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03, $close / Ref($close, 5), 1)))))",
            
            # 8. 今日放量上涨
            "If(($close > $open) & ($volume > Mean($volume, 5) * 1.2), 1, 0)",  # 今日放量上涨
            "($volume / Mean($volume, 5)) * If($close > $open, 1, 0)",  # 上涨放量倍数
            
            # 9. 完整策略信号
            "If((" +
            # 条件1：5天内有大涨且不是今天
            "((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03) | " +
            "((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03) | " +
            "((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03) | " +
            "((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03) | " +
            "((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03)" +
            ") & (" +
            # 条件2：大涨后有回调
            "If((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03, If($close < Ref($close, 1), 1, 0), " +
            "If((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03, If(Min($close, 2) < Ref($close, 2), 1, 0), " +
            "If((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03, If(Min($close, 3) < Ref($close, 3), 1, 0), " +
            "If((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03, If(Min($close, 4) < Ref($close, 4), 1, 0), " +
            "If((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03, If(Min($close, 5) < Ref($close, 5), 1, 0), 0))))) == 1" +
            ") & (" +
            # 条件3：未跌破80%
            "If((Ref($close, 1) - Ref($close, 2)) / Ref($close, 2) > 0.03, $close / Ref($close, 1), " +
            "If((Ref($close, 2) - Ref($close, 3)) / Ref($close, 3) > 0.03, $close / Ref($close, 2), " +
            "If((Ref($close, 3) - Ref($close, 4)) / Ref($close, 4) > 0.03, $close / Ref($close, 3), " +
            "If((Ref($close, 4) - Ref($close, 5)) / Ref($close, 5) > 0.03, $close / Ref($close, 4), " +
            "If((Ref($close, 5) - Ref($close, 6)) / Ref($close, 6) > 0.03, $close / Ref($close, 5), 1))))) >= 0.8" +
            ") & (" +
            # 条件4：今日放量上涨
            "($close > $open) & ($volume > Mean($volume, 5) * 1.2)" +
            "), 1, 0)",
            
            # 10. 辅助因子
            "Max(($close - Ref($close, 1)) / Ref($close, 1), 5)",  # 5天内最大涨幅
            "($close - Min($close, 5)) / Min($close, 5)",  # 相对5天最低点涨幅
            "(Max($close, 5) - $close) / Max($close, 5)",  # 相对5天最高点回调幅度
            
            # 11. 量价配合
            "($close - $open) / $close * ($volume / Mean($volume, 5))",  # 涨幅*放量倍数
            "If($volume > Mean($volume, 5) * 1.2, ($close - $open) / $close, 0)",  # 放量时的涨幅
        ]
        
        return volume_fields + ma_fields + price_fields + strategy_fields




    def get_label_config(self):
        return ["Ref($close, -1) / $close - 1"]

# ==================== 配置生成器 ====================

class QlibConfigGenerator:
    """Qlib配置生成器"""
    
    def __init__(self):
        self.base_data_path = "C:/ctp/q_qlib/qlib_data/cn_data"
        self.source_data_path = "C:/ctp/q_qlib/cleaned_data"
    
    def get_data_time_range(self, contract, freq):
        """
        从qlib数据源中获取时间范围
        
        Args:
            contract: 合约名称，如 "EBL8.DCE"
            freq: 频率，如 "1min", "5min", "15min", "day"
        
        Returns:
            tuple: (start_time, end_time, train_end, valid_start, valid_end)
        """
        try:
            # 频率映射
            freq_mapping = {
                "1min": "1min",
                "5min": "5min", 
                "15min": "15min",
                "day": "day"
            }
            
            qlib_freq = freq_mapping.get(freq, freq)
            contract_path = f"{self.base_data_path}/{contract}"
            
            print(f"  从qlib数据源读取: {contract_path}")
            
            # 临时初始化qlib以读取数据
            try:
                qlib.init(provider_uri=contract_path, region="cn")
            except:
                # 如果已经初始化过，忽略错误
                pass
            
            # 使用qlib的D.features接口读取数据
            # 读取一个简单的字段来获取时间索引
            data = D.features([contract], ["$close"], freq=qlib_freq)
            
            if data is None or len(data) == 0:
                raise ValueError(f"未找到qlib数据: {contract} - {qlib_freq}，请检查数据路径: {contract_path}")
            
            # 获取时间索引
            time_index = data.index.get_level_values('datetime')
            start_time = time_index.min()
            end_time = time_index.max()
            total_rows = len(time_index.unique())
            
            print(f"  数据时间范围: {start_time} 到 {end_time}")
            print(f"  总数据量: {total_rows} 行")
            
            # 计算分割点：90%用于训练，10%用于验证和测试
            train_rows = int(total_rows * 0.9)
            
            # 获取排序后的唯一时间点
            unique_times = sorted(time_index.unique())
            train_end_time = unique_times[train_rows - 1]
            
            # 格式化时间为字符串
            start_str = start_time.strftime('%Y-%m-%d')
            end_str = end_time.strftime('%Y-%m-%d')
            train_end_str = train_end_time.strftime('%Y-%m-%d')
            
            # valid和test都使用剩余的10%数据
            valid_start_str = train_end_str
            valid_end_str = end_str
            
            print(f"  时间分割:")
            print(f"    训练集: {start_str} 到 {train_end_str} ({train_rows} 行, {train_rows/total_rows*100:.1f}%)")
            print(f"    验证集: {valid_start_str} 到 {valid_end_str} ({total_rows-train_rows} 行, {(total_rows-train_rows)/total_rows*100:.1f}%)")
            print(f"    测试集: {valid_start_str} 到 {valid_end_str} ({total_rows-train_rows} 行, {(total_rows-train_rows)/total_rows*100:.1f}%)")
            
            return start_str, end_str, train_end_str, valid_start_str, valid_end_str
            
        except Exception as e:
            print(f"✗ 从qlib获取数据时间范围失败: {e}")
            import traceback
            traceback.print_exc()
            raise RuntimeError(f"无法获取合约 {contract} 频率 {freq} 的数据时间范围，请检查数据是否存在") from e
        
    def generate_config(self, contract, freq, start_time=None, end_time=None, handler_module=None):
        """
        生成配置字典
        
        Args:
            contract: 合约名称，如 "EBL8.DCE"
            freq: 频率，如 "1min", "5min", "15min", "day"
            start_time: 开始时间（可选，默认从数据中自动获取）
            end_time: 结束时间（可选，默认从数据中自动获取）
            handler_module: Handler模块路径（可选，默认自动检测）
        """
        # 如果没有提供时间范围，从数据中自动获取
        if start_time is None or end_time is None:
            print(f"自动获取 {contract} - {freq} 的数据时间范围...")
            auto_start, auto_end, train_end, valid_start, valid_end = self.get_data_time_range(contract, freq)
            start_time = start_time or auto_start
            end_time = end_time or auto_end
        else:
            # 使用提供的时间范围，按90%-10%分割
            print(f"使用指定时间范围: {start_time} 到 {end_time}")
            start_dt = datetime.strptime(start_time, '%Y-%m-%d')
            end_dt = datetime.strptime(end_time, '%Y-%m-%d')
            total_days = (end_dt - start_dt).days
            train_days = int(total_days * 0.9)
            train_end_dt = start_dt + pd.Timedelta(days=train_days)
            
            train_end = train_end_dt.strftime('%Y-%m-%d')
            valid_start = train_end
            valid_end = end_time
        
        # 频率映射
        freq_mapping = {
            "1min": {"handler": "CustomTrendHandler_m1", "qlib_freq": "1min"},
            "5min": {"handler": "CustomTrendHandler_m5", "qlib_freq": "5min"},
            "15min": {"handler": "CustomTrendHandler_m15", "qlib_freq": "15min"},
            "day": {"handler": "CustomTrendHandler_day", "qlib_freq": "day"}
        }
        
        if freq not in freq_mapping:
            raise ValueError(f"不支持的频率: {freq}")
        
        handler_info = freq_mapping[freq]
        contract_path = f"{self.base_data_path}/{contract}"
        
        # 自动检测Handler模块路径
        if handler_module is None:
            import inspect
            frame = inspect.currentframe()
            try:
                # 获取调用者的模块名
                caller_frame = frame.f_back.f_back  # 跳过generate_config和train_model
                caller_module = caller_frame.f_globals.get('__name__', 'qlib_config_generator')
                if caller_module == '__main__':
                    # 如果是主模块，尝试获取文件名
                    caller_file = caller_frame.f_globals.get('__file__', '')
                    if caller_file:
                        import os
                        module_name = os.path.splitext(os.path.basename(caller_file))[0]
                        handler_module = module_name
                    else:
                        handler_module = 'qlib_config_generator'
                else:
                    handler_module = caller_module
            finally:
                del frame
        
        # 简化方案：直接使用配置生成器模块
        handler_module = "qlib_config_generator"
        print(f"  使用Handler模块: {handler_module}")
        
        config = {
            "qlib_init": {
                "provider_uri": contract_path,
                "region": "cn"
            },
            "task": {
                "model": {
                    "class": "LGBModel",
                    "module_path": "qlib.contrib.model.gbdt",
                    "kwargs": {
                        "loss": "mse",
                        "colsample_bytree": 0.8879,
                        "learning_rate": 0.2,
                        "subsample": 0.8789,
                        "max_depth": 8,
                        "num_leaves": 210,
                        "num_threads": 20
                    }
                },
                "dataset": {
                    "class": "DatasetH",
                    "module_path": "qlib.data.dataset",
                    "kwargs": {
                        "handler": {
                            "class": handler_info["handler"],
                            "module_path": handler_module,  # 动态检测的模块路径
                            "kwargs": {
                                "start_time": start_time,
                                "end_time": end_time,
                                "instruments": [contract],
                                "freq": handler_info["qlib_freq"]
                            }
                        },
                        "segments": {
                            "train": [start_time, train_end],
                            "valid": [valid_start, valid_end],
                            "test": [valid_start, valid_end]
                        }
                    }
                },
                "record": [
                    {
                        "class": "SignalRecord",
                        "module_path": "qlib.workflow.record_temp",
                        "kwargs": {
                            "model": "<MODEL>",
                            "dataset": "<DATASET>"
                        }
                    },
                    {
                        "class": "SigAnaRecord",
                        "module_path": "qlib.workflow.record_temp",
                        "kwargs": {
                            "ana_long_short": False,
                            "ann_scaler": 252
                        }
                    }
                ]
            }
        }
        
        return config
    
    def save_config_yaml(self, config, filename):
        """保存配置为YAML文件"""
        with open(filename, 'w', encoding='utf-8') as f:
            yaml.dump(config, f, default_flow_style=False, allow_unicode=True, indent=2)
        print(f"✓ 配置文件已保存: {filename}")
    
    def generate_all_configs(self, contract):
        """为指定合约生成所有频率的配置文件"""
        frequencies = ["1min", "5min", "15min", "day"]
        
        for freq in frequencies:
            config = self.generate_config(contract, freq)
            filename = f"config_{contract.replace('.', '_')}_{freq}.yaml"
            self.save_config_yaml(config, filename)
        
        print(f"✓ 已为合约 {contract} 生成所有配置文件")

# ==================== 使用示例 ====================

def main():
    """主函数 - 演示如何使用"""
    generator = QlibConfigGenerator()
    
    # 生成单个配置（自动获取时间范围）
    print("=== 生成单个配置文件（自动时间范围）===")
    contract = "EBL8.DCE"
    freq = "15min"
    
    config = generator.generate_config(contract, freq)
    generator.save_config_yaml(config, f"my_config_{freq}.yaml")
    
    # 生成指定时间范围的配置
    print("\n=== 生成指定时间范围的配置文件 ===")
    config_custom = generator.generate_config(contract, freq, "2020-01-01", "2024-12-31")
    generator.save_config_yaml(config_custom, f"my_config_{freq}_custom.yaml")
    
    # 生成所有频率的配置
    print("\n=== 生成所有频率配置文件 ===")
    generator.generate_all_configs(contract)
    
    print("\n=== 配置生成完成 ===")
    print("✓ 自动时间范围：从数据中获取第一行到最后一行，90%训练，10%验证/测试")
    print("✓ 现在你可以直接使用这些配置文件进行训练，无需额外的factors目录！")

if __name__ == "__main__":
    main()