AI量化qlib学习笔记二：转换数据

这个步骤是把csv数据转成qlib的bin数据，坑巨多。

之前说到放弃，就是因为它对分时小周期数据的支持非常不友好，懒得说了，直接上源码。

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
简单的CSV转qlib工具 - 循环处理不同周期
"""

import os
import pandas as pd
from pathlib import Path
import subprocess
import sys
import tempfile
import shutil
from datetime import datetime, timedelta

def load_trading_calendar():
    """加载交易日历"""
    calendar_file = Path(__file__).parent / "day.txt"
    
    if not calendar_file.exists():
        print(f"⚠ 交易日历文件不存在: {calendar_file}")
        return None
    
    try:
        # 读取交易日历，假设格式为每行一个日期 YYYY-MM-DD
        with open(calendar_file, 'r', encoding='utf-8') as f:
            dates = []
            for line in f:
                line = line.strip()
                if line and not line.startswith('#'):  # 跳过空行和注释
                    try:
                        date_obj = datetime.strptime(line, '%Y-%m-%d').date()
                        dates.append(date_obj)
                    except ValueError:
                        continue
        
        dates.sort()  # 确保日期排序
        print(f"✓ 加载交易日历: {len(dates)} 个交易日")
        return dates
        
    except Exception as e:
        print(f"✗ 读取交易日历失败: {e}")
        return None

def get_previous_trading_date(current_date, trading_calendar):
    """获取指定日期的前一个交易日"""
    if not trading_calendar:
        return current_date - timedelta(days=1)  # 如果没有日历，简单减一天
    
    try:
        # 找到当前日期在交易日历中的位置
        if current_date in trading_calendar:
            current_index = trading_calendar.index(current_date)
            if current_index > 0:
                return trading_calendar[current_index - 1]
        
        # 如果当前日期不在交易日历中，找到最近的前一个交易日
        for i in range(len(trading_calendar) - 1, -1, -1):
            if trading_calendar[i] < current_date:
                return trading_calendar[i]
        
        # 如果找不到，返回简单减一天的结果
        return current_date - timedelta(days=1)
        
    except Exception as e:
        print(f"⚠ 获取前一交易日失败: {e}")
        return current_date - timedelta(days=1)

def adjust_night_session_time(df, trading_calendar):
    """调整夜盘时间：将21:00-23:59的数据日期改为前一个交易日"""
    if 'datetime' not in df.columns:
        return df
    
    print("  检查夜盘时间...")
    
    # 转换datetime列为pandas datetime类型
    df['datetime'] = pd.to_datetime(df['datetime'])
    
    adjusted_count = 0
    
    for idx, row in df.iterrows():
        dt = row['datetime']
        hour = dt.hour
        
        # 检查是否为夜盘时间 (21:00-23:59)
        if 21 <= hour <= 23:
            current_date = dt.date()
            previous_trading_date = get_previous_trading_date(current_date, trading_calendar)
            
            # 创建新的datetime，保持时间不变，只改日期
            new_datetime = datetime.combine(previous_trading_date, dt.time())
            df.at[idx, 'datetime'] = new_datetime
            adjusted_count += 1
    
    if adjusted_count > 0:
        print(f"  ✓ 调整了 {adjusted_count} 条夜盘数据的日期")
        # 重新按时间排序
        df = df.sort_values('datetime').reset_index(drop=True)
    else:
        print("  - 没有发现需要调整的夜盘数据")
    
    return df

def run_dump_bin_ordered(temp_csv_dir, target_path, freq, ordered_symbols):
    """按指定顺序运行qlib的dump_bin.py脚本，第一个用dump_all，后续用dump_update"""
    dump_script_path = Path(r'C:\ctp\q_qlib\qlib_src\scripts\dump_bin.py')
    
    if not dump_script_path.exists():
        print(f"✗ dump_bin.py脚本不存在: {dump_script_path}")
        return False
    
    print(f"使用dump_bin.py按顺序转换{freq}数据...")
    print(f"处理顺序: {ordered_symbols}")
    
    # 按顺序处理每个品种
    for i, symbol in enumerate(ordered_symbols):
        print(f"  处理 {i+1}/{len(ordered_symbols)}: {symbol}")
        
        # 查找该品种的CSV文件
        csv_files = list(Path(temp_csv_dir).glob(f"{symbol}.*.csv"))
        
        if not csv_files:
            print(f"    ⚠ 未找到 {symbol} 的文件")
            continue
        
        # 为该品种创建单独的临时目录
        symbol_temp_dir = Path(temp_csv_dir) / f"temp_{symbol}"
        symbol_temp_dir.mkdir(exist_ok=True)
        
        # 复制该品种的文件到单独目录
        for csv_file in csv_files:
            shutil.copy2(csv_file, symbol_temp_dir / csv_file.name)
        
        # 决定使用dump_all还是dump_update
        dump_mode = "dump_all" if  i==0   else "dump_update"
        
        cmd = [
            sys.executable, str(dump_script_path),
            dump_mode,
            "--csv_path", str(symbol_temp_dir),  # 传递该品种的目录
            "--qlib_dir", str(target_path),
            "--freq", freq,
            "--max_workers", "1",  # 单线程确保顺序
            "--include_fields", "open,close,high,low,volume",
            "--date_field_name", "datetime" 
        ]
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True)
            
            if result.returncode == 0:
                print(f"    ✓ {symbol} 转换成功 ({dump_mode})")
            else:
                print(f"    ✗ {symbol} 转换失败 ({dump_mode})")
                print(f"    错误: {result.stderr}")
                # 清理临时目录
                shutil.rmtree(symbol_temp_dir, ignore_errors=True)
                return False
                
        except Exception as e:
            print(f"    ✗ 处理 {symbol} 时出错: {e}")
            # 清理临时目录
            shutil.rmtree(symbol_temp_dir, ignore_errors=True)
            return False
        
        # 清理该品种的临时目录
        shutil.rmtree(symbol_temp_dir, ignore_errors=True)
    
    print(f"✓ 所有品种按顺序转换完成")
    return True

def run_dump_bin_ordered_update(temp_csv_dir, target_path, freq, ordered_symbols):
    """按指定顺序运行qlib的dump_bin.py脚本，第一个用dump_all，后续用dump_update（用于分钟周期）"""
    dump_script_path = Path(r'C:\ctp\q_qlib\qlib_src\scripts\dump_bin.py')
    
    if not dump_script_path.exists():
        print(f"✗ dump_bin.py脚本不存在: {dump_script_path}")
        return False
    
    print(f"使用dump_bin.py按顺序转换{freq}数据（第一个dump_all，后续dump_update）...")
    print(f"处理顺序: {ordered_symbols}")
    
    # 按顺序处理每个品种
    for i, symbol in enumerate(ordered_symbols):
        print(f"  处理 {i+1}/{len(ordered_symbols)}: {symbol}")
        
        # 查找该品种的CSV文件
        csv_files = list(Path(temp_csv_dir).glob(f"{symbol}.*.csv"))
        
        if not csv_files:
            print(f"    ⚠ 未找到 {symbol} 的文件")
            continue
        
        # 为该品种创建单独的临时目录
        symbol_temp_dir = Path(temp_csv_dir) / f"temp_{symbol}"
        symbol_temp_dir.mkdir(exist_ok=True)
        
        # 复制该品种的文件到单独目录
        for csv_file in csv_files:
            shutil.copy2(csv_file, symbol_temp_dir / csv_file.name)
        
        # 第一个品种使用dump_all，后续使用dump_update
        dump_mode = "dump_all" if i == 0 else "dump_update"
        
        cmd = [
            sys.executable, str(dump_script_path),
            dump_mode,
            "--csv_path", str(symbol_temp_dir),  # 传递该品种的目录
            "--qlib_dir", str(target_path),
            "--freq", freq,
            "--max_workers", "1",  # 单线程确保顺序
            "--include_fields", "open,close,high,low,volume",
            "--date_field_name", "datetime" 
        ]
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True)
            
            if result.returncode == 0:
                print(f"    ✓ {symbol} 转换成功 ({dump_mode})")
            else:
                print(f"    ✗ {symbol} 转换失败 ({dump_mode})")
                print(f"    错误: {result.stderr}")
                # 清理临时目录
                shutil.rmtree(symbol_temp_dir, ignore_errors=True)
                return False
                
        except Exception as e:
            print(f"    ✗ 处理 {symbol} 时出错: {e}")
            # 清理临时目录
            shutil.rmtree(symbol_temp_dir, ignore_errors=True)
            return False
        
        # 清理该品种的临时目录
        shutil.rmtree(symbol_temp_dir, ignore_errors=True)
    
    print(f"✓ 所有品种按顺序转换完成")
    return True

def run_dump_bin_group(temp_csv_dir, target_path, freq, dump_mode, symbols):
    """运行qlib的dump_bin.py脚本处理一组相同交易时间的合约"""
    dump_script_path = Path(r'C:\ctp\q_qlib\qlib_src\scripts\dump_bin.py')
    
    if not dump_script_path.exists():
        print(f"✗ dump_bin.py脚本不存在: {dump_script_path}")
        return False
    
    print(f"      使用{dump_mode}模式处理: {symbols}")
    
    # 检查临时目录中的文件
    temp_files = list(Path(temp_csv_dir).glob("*.csv"))
    print(f"      目录中有 {len(temp_files)} 个CSV文件")
    
    cmd = [
        sys.executable, str(dump_script_path),
        dump_mode,
        "--csv_path", str(temp_csv_dir),
        "--qlib_dir", str(target_path),
        "--freq", freq,
        "--max_workers", "1",
        "--include_fields", "open,close,high,low,volume",
        "--date_field_name", "datetime" 
    ]
    
    try:
        result = subprocess.run(cmd, capture_output=True, text=True)
        
        if result.returncode == 0:
            print(f"      ✓ 转换成功")
            return True
        else:
            print(f"      ✗ 转换失败")
            print(f"      错误: {result.stderr}")
            return False
            
    except Exception as e:
        print(f"      ✗ 运行dump_bin.py时出错: {e}")
        return False

def run_dump_bin(temp_csv_dir, target_path, freq):
    """运行qlib的dump_bin.py脚本"""
    dump_script_path = Path(r'C:\ctp\q_qlib\qlib_src\scripts\dump_bin.py')
    
    if not dump_script_path.exists():
        print(f"✗ dump_bin.py脚本不存在: {dump_script_path}")
        return False
    
    print(f"使用dump_bin.py转换{freq}数据...")
    
    # 检查临时目录中的文件
    temp_files = list(Path(temp_csv_dir).glob("*.csv"))
    print(f"  临时目录中有 {len(temp_files)} 个CSV文件")
    for f in temp_files[:3]:  # 只显示前3个
        print(f"    - {f.name}")
    if len(temp_files) > 3:
        print(f"    ... 还有 {len(temp_files) - 3} 个文件")
    
    cmd = [
        sys.executable, str(dump_script_path),
        "dump_all",
        "--csv_path", str(temp_csv_dir),
        "--qlib_dir", str(target_path),
        "--freq", freq,  # 使用传入的频率参数
        "--max_workers", "1",  # 改为单线程，便于调试
        "--include_fields", "open,close,high,low,volume",
        "--date_field_name", "datetime" 
    ]
    
    print(f"  执行命令: {' '.join(cmd)}")
    
    try:
        result = subprocess.run(cmd, capture_output=True, text=True)
        
        print(f"  返回码: {result.returncode}")
        
        if result.stdout:
            print(f"  标准输出:")
            for line in result.stdout.split('\n')[:10]:  # 只显示前10行
                if line.strip():
                    print(f"    {line}")
        
        if result.stderr:
            print(f"  错误输出:")
            for line in result.stderr.split('\n')[:10]:  # 只显示前10行
                if line.strip():
                    print(f"    {line}")
        
        if result.returncode == 0:
            print(f"  ✓ 数据转换成功")
            return True
        else:
            print(f"  ✗ 数据转换失败 (返回码: {result.returncode})")
            return False
            
    except Exception as e:
        print(f"✗ 运行dump_bin.py时出错: {e}")
        return False

def copy_symbol_period_files(source_dir, temp_dir, symbol, period, trading_calendar=None):
    """复制指定品种和周期的文件到临时目录"""
    source_path = Path(source_dir)
    temp_path = Path(temp_dir)
    temp_path.mkdir(parents=True, exist_ok=True)
    
    copied_count = 0
    
    # 查找指定品种和周期的文件
    pattern = f"{symbol}.*.{period}.csv"
    for csv_file in source_path.glob(pattern):
        try:
            print(f"    处理文件: {csv_file.name}")
            
            # 读取CSV文件
            df = pd.read_csv(csv_file, encoding='utf-8')
            
            # 解析文件名：RBL8.SHFE.day.csv
            filename = csv_file.stem  # 去掉.csv
            parts = filename.split('.')
            
            if len(parts) >= 3:
                file_symbol = parts[0].upper()
                exchange = parts[1]
                
                # 添加instrument列
                df['instrument'] = file_symbol
                
                # 调整夜盘时间（只对分钟级数据进行调整）
                if period in ['1m', '5m', '15m', '30m', '60m'] and trading_calendar:
                    df = adjust_night_session_time(df, trading_calendar)
                
                # 重新排列列顺序
                cols = ['instrument', 'datetime'] + [col for col in df.columns if col not in ['instrument', 'datetime']]
                df = df[cols]
                
                # 保存为标准格式：RBL8.SHFE.csv (去掉周期)
                new_filename = f"{file_symbol}.{exchange}.csv"
                output_file = temp_path / new_filename
                df.to_csv(output_file, index=False)
                
                print(f"    ✓ 完成: {csv_file.name} -> {new_filename}")
                copied_count += 1
                
        except Exception as e:
            print(f"    ✗ 处理文件 {csv_file} 时出错: {e}")
    
    return copied_count

def copy_period_files(source_dir, temp_dir, period, trading_calendar=None):
    """复制指定周期的文件到临时目录，并修正夜盘时间"""
    source_path = Path(source_dir)
    temp_path = Path(temp_dir)
    temp_path.mkdir(parents=True, exist_ok=True)
    
    copied_count = 0
    
    # 查找指定周期的文件
    pattern = f"*.{period}.csv"
    for csv_file in source_path.glob(pattern):
        try:
            print(f"  处理文件: {csv_file.name}")
            
            # 读取CSV文件
            df = pd.read_csv(csv_file, encoding='utf-8')
            
            # 解析文件名：RBL8.SHFE.day.csv
            filename = csv_file.stem  # 去掉.csv
            parts = filename.split('.')
            
            if len(parts) >= 3:
                symbol = parts[0].upper()
                exchange = parts[1]
                
                # 添加instrument列
                df['instrument'] = symbol
                
                # 调整夜盘时间（只对分钟级数据进行调整）
                if period in ['1m', '5m', '15m', '30m', '60m'] and trading_calendar:
                    df = adjust_night_session_time(df, trading_calendar)
                
                # 重新排列列顺序
                cols = ['instrument', 'datetime'] + [col for col in df.columns if col not in ['instrument', 'datetime']]
                df = df[cols]
                
                # 保存为标准格式：RBL8.SHFE.csv (去掉周期)
                new_filename = f"{symbol}.{exchange}.csv"
                output_file = temp_path / new_filename
                df.to_csv(output_file, index=False)
                
                print(f"  ✓ 完成: {csv_file.name} -> {new_filename}")
                copied_count += 1
                
        except Exception as e:
            print(f"  ✗ 处理文件 {csv_file} 时出错: {e}")
    
    return copied_count

def run_dump_bin_contract(temp_csv_dir, target_path, freq, contract, period):
    """为单个合约运行qlib的dump_bin.py脚本"""
    dump_script_path = Path(r'C:\ctp\q_qlib\qlib_src\scripts\dump_bin.py')
    
    if not dump_script_path.exists():
        print(f"      ✗ dump_bin.py脚本不存在: {dump_script_path}")
        return False
    
    print(f"      使用dump_bin.py转换{freq}数据...")
    
    cmd = [
        sys.executable, str(dump_script_path),
        "dump_all",  # 每个合约都用dump_all，独立处理
        "--csv_path", str(temp_csv_dir),
        "--qlib_dir", str(target_path),
        "--freq", freq,
        "--max_workers", "1",
        "--include_fields", "open,close,high,low,volume",
        "--date_field_name", "datetime" 
    ]
    
    try:
        result = subprocess.run(cmd, capture_output=True, text=True)
        
        if result.returncode == 0:
            print(f"      ✓ 转换成功")
            return True
        else:
            print(f"      ✗ 转换失败")
            if result.stderr:
                print(f"      错误: {result.stderr}")
            return False
            
    except Exception as e:
        print(f"      ✗ 运行dump_bin.py时出错: {e}")
        return False

def main():
    """主函数 - 按合约为单位处理，每个合约独立的数据目录"""
    # 配置
    source_dir = r"C:\ctp\q_qlib\cleaned_data"
    base_target_dir = r"C:\ctp\q_qlib\qlib_data\cn_data"
    
    # 定义要处理的周期和对应的qlib频率
    period_freq_mapping = {
        'day': 'day',
        '1m': '1min',
        '5m': '5min',
        '15m': '15min',
        '30m': '30min',
        '60m': '60min'
    }
    
    # 调整处理顺序：day放到最后，让all.txt时间周期更长
    periods = ['1m', '5m', '15m', '30m', '60m', 'day']
    
    print("=== 按合约为单位处理数据 ===")
    print(f"源目录: {source_dir}")
    print(f"目标目录: {base_target_dir}")
    print(f"处理周期: {periods} (day放在最后，让all.txt时间周期更长)")
    
    # 检查源目录
    if not os.path.exists(source_dir):
        print(f"✗ 源目录不存在: {source_dir}")
        return
    
    # 先检查源目录中实际存在的文件
    print(f"\n检查源目录中的文件...")
    source_path = Path(source_dir)
    all_files = list(source_path.glob("*.csv"))
    print(f"找到 {len(all_files)} 个CSV文件")
    
    # 加载交易日历
    print("\n加载交易日历...")
    trading_calendar = load_trading_calendar()
    if trading_calendar:
        print(f"交易日历范围: {trading_calendar[0]} 到 {trading_calendar[-1]}")
    else:
        print("⚠ 未加载交易日历，夜盘时间将使用简单日期减1的方式处理")
    
    # 创建目标目录
    Path(base_target_dir).mkdir(parents=True, exist_ok=True)
    
    # 获取所有合约（从实际存在的文件中）
    contract_files = {}  # {合约名: {周期: [文件列表]}}
    
    for csv_file in all_files:
        filename = csv_file.stem
        parts = filename.split('.')
        if len(parts) >= 3:
            contract = f"{parts[0].upper()}.{parts[1]}"  # 如 RBL8.SHFE
            period = parts[2]  # 如 day, 1m
            
            if contract not in contract_files:
                contract_files[contract] = {}
            if period not in contract_files[contract]:
                contract_files[contract][period] = []
            contract_files[contract][period].append(csv_file)
    
    print(f"\n发现 {len(contract_files)} 个合约:")
    for contract in sorted(contract_files.keys()):
        available_periods = list(contract_files[contract].keys())
        print(f"  {contract}: {available_periods}")
    
    # 按合约处理
    processed_contracts = 0
    
    for contract in sorted(contract_files.keys()):
        print(f"\n{'='*80}")
        print(f"处理合约: {contract}")
        print(f"{'='*80}")
        
        # 为该合约创建独立的目标目录
        contract_target_dir = Path(base_target_dir) / contract
        contract_target_dir.mkdir(parents=True, exist_ok=True)
        
        print(f"合约数据目录: {contract_target_dir}")
        
        # 处理该合约的所有周期
        contract_success = True
        
        for period in periods:
            if period not in contract_files[contract]:
                print(f"  ⚠ 跳过 {period} 周期：没有找到相关文件")
                continue
            
            qlib_freq = period_freq_mapping[period]
            print(f"\n  处理周期: {period} (qlib频率: {qlib_freq})")
            
            # 创建临时目录
            with tempfile.TemporaryDirectory() as temp_dir:
                print(f"    使用临时目录: {temp_dir}")
                
                # 复制该合约该周期的文件
                copied_count = 0
                for csv_file in contract_files[contract][period]:
                    try:
                        print(f"      处理文件: {csv_file.name}")
                        
                        # 读取CSV文件
                        df = pd.read_csv(csv_file, encoding='utf-8')
                        
                        # 解析文件名：RBL8.SHFE.day.csv
                        filename = csv_file.stem
                        parts = filename.split('.')
                        symbol = parts[0].upper()
                        exchange = parts[1]
                        
                        # 添加instrument列
                        df['instrument'] = symbol
                        
                        # 调整夜盘时间（只对分钟级数据进行调整）
                        if period in ['1m', '5m', '15m', '30m', '60m'] and trading_calendar:
                            df = adjust_night_session_time(df, trading_calendar)
                        
                        # 重新排列列顺序
                        cols = ['instrument', 'datetime'] + [col for col in df.columns if col not in ['instrument', 'datetime']]
                        df = df[cols]
                        
                        # 保存为标准格式：RBL8.SHFE.csv (去掉周期)
                        new_filename = f"{symbol}.{exchange}.csv"
                        output_file = Path(temp_dir) / new_filename
                        df.to_csv(output_file, index=False)
                        
                        print(f"      ✓ 完成: {csv_file.name} -> {new_filename}")
                        copied_count += 1
                        
                    except Exception as e:
                        print(f"      ✗ 处理文件 {csv_file} 时出错: {e}")
                        contract_success = False
                        break
                
                if copied_count == 0:
                    print(f"    ✗ 没有成功复制任何文件")
                    contract_success = False
                    continue
                
                print(f"    复制了 {copied_count} 个文件")
                
                # 使用dump_all处理该合约该周期的数据
                if run_dump_bin_contract(temp_dir, contract_target_dir, qlib_freq, contract, period):
                    print(f"    ✓ {period} 周期处理完成")
                else:
                    print(f"    ✗ {period} 周期处理失败")
                    contract_success = False
        
        if contract_success:
            print(f"✓ 合约 {contract} 处理完成")
            processed_contracts += 1
        else:
            print(f"✗ 合约 {contract} 处理失败")
    
    print(f"\n{'='*50}")
    print(f"处理完成！成功处理了 {processed_contracts}/{len(contract_files)} 个合约")
    print(f"数据路径: {base_target_dir}")
    print(f"每个合约都有独立的数据目录，避免了时间对齐问题")
    print(f"处理顺序: 分钟周期 → day周期，让all.txt时间周期更长")
    print(f"夜盘时间修正功能已启用 (21:00-23:59 → 前一交易日)")
    print(f"{'='*50}")



if __name__ == "__main__":
    main()
    #python qlib_convert_to_qlib.py