python Chapter 2:雨滴/雪花数据清洗
一、雨滴数据清洗
为了确保深度学习训练的有效性，数据清洗至关重要。因此，在将这些数据加入数据集之前，放弃了那些不合格的降水滴谱（DSD）数据，处理顺序（行为直径，列为速度）：
（1）实测粒子下落速度与经典值之间的差异大于5米每秒的数据
（2）进行粒子轴比订正，剔除前两个尺度档和订正后雨滴直径大于8 mm 的 观测数据；（删除前2个尺度档和后10个尺度档）
（3）某个直径区间内的粒子数少于2个
（4）每分钟内的粒子数量少于10个的数据；（这个得放最后一步，要不然会存在粒子总数少于10，u巨大的情况）
（因为我做过轴比订正的excel，发现倒数第11行原始直径6.5处对应轴比订正7.320611223 ，倒数第10行7.5处对应轴比订正8.446859103，所以直接剔除剔除前2个尺度档和后10档。）
#!/usr/bin/env python # -*- coding:utf-8 -*- """ @author: Suyue @file: process_raindrop_data_v9_fixed_modified.py @time: 2026/08/31 @desc: 雨滴数据清洗 1. 速度差>5m/s（用现成的掩膜） -> 剔除对应粒子 2. 剔除前2个直径档和后10个直径档（订正后直径） 3. 行粒子数<2 -> 该行全部置0（按直径区间统计） 4. 总粒子数<10 -> 剔除整分钟（最后一步） """ import numpy as np import re import os from datetime import datetime import pandas as pd # ==================== 常量定义 ==================== # 32个直径通道的中心直径 (mm) - 这是已经轴比订正后的直径 diameters_corrected = np.array([ 0.062, 0.187, 0.312, 0.437, 0.562, 0.687, 0.812, 0.937, 1.064, 1.193, 1.388, 1.651, 1.918, 2.188, 2.463, 2.882, 3.457, 4.050, 4.665, 5.303, 6.194, 7.321, 8.447, 9.573, 10.699, 12.389, 14.641, 16.894, 19.146, 21.399, 24.214, 27.593 ]) # 32个速度通道的中心速度 (m/s) measured_velocity = np.array([ 0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.1, 1.3, 1.5, 1.7, 1.9, 2.2, 2.6, 3, 3.4, 3.8, 4.4, 5.2, 6, 6.8, 7.6, 8.8, 10.4, 12, 13.6, 15.2, 17.6, 20.8 ]) # 需要剔除的直径档位索引（前2个和后10个） # 索引: 0,1 是前两个档位；22-31 是后10个档位（因为总32个，后10个是索引22到31） # 注意：现在是行索引，因为行代表直径 INVALID_DIAMETER_BINS = list(range(0, 2)) + list(range(22, 32)) def load_velocity_mask(mask_file): """ 加载速度掩膜Excel文件 注意：掩膜矩阵中，行代表直径，列代表速度 """ print(f"正在加载速度掩膜: {mask_file}") try: df = pd.read_excel(mask_file, header=None) mask = df.values if mask.shape != (32, 32): raise ValueError(f"掩膜形状错误: 期望(32,32)，实际{mask.shape}") unique_values = np.unique(mask) if not np.all(np.isin(unique_values, [0, 1])): print(f"警告: 掩膜中包含非0/1的值: {unique_values}") print(f"速度掩膜加载成功，形状: {mask.shape}") print(f"剔除位置数量: {np.sum(mask)} / 1024 ({np.sum(mask) / 1024 * 100:.1f}%)") print(f"掩膜矩阵说明: 行=直径通道({len(diameters_corrected)}个), 列=速度通道({len(measured_velocity)}个)") return mask except Exception as e: print(f"加载速度掩膜失败: {e}") raise def process_one_minute(data_32x32, velocity_mask, minute_number=None): """ 严格按照四步规则处理一分钟的32×32雨滴数据 数组结构：32行（直径）× 32列（速度） 规则顺序（调整后）： 1. 速度差>5m/s（用现成的掩膜） -> 剔除对应粒子 2. 剔除前2个直径档和后10个直径档（订正后直径） 3. 行粒子数<2 -> 该行全部置0（按直径区间统计） 4. 总粒子数<10 -> 剔除整分钟（最后一步） """ # 创建处理副本 processed = data_32x32.copy().astype(float) # ===== 规则1: 速度差异大于5m/s的数据（用现成的掩膜）===== # velocity_mask中标记为1的位置，将雨滴谱数值置为0 # 掩膜矩阵: 行=直径, 列=速度 processed = np.where(velocity_mask == 1, 0, processed) # ===== 规则2: 剔除前2个直径档和后10个直径档（订正后直径）===== # 现在行代表直径，所以需要剔除行 for row in INVALID_DIAMETER_BINS: processed[row, :] = 0 # ===== 规则3: 行粒子数<2 -> 该行全部置0（按直径区间统计）===== # 注意：行代表直径区间，按行求和 for row in range(32): row_sum = np.sum(processed[row, :]) if row_sum < 2: processed[row, :] = 0 # ===== 规则4: 每分钟内的粒子数量少于10个（最后一步）===== total_particles = np.sum(processed) if total_particles < 10: return None, 'particle_count_lt10' return processed, 'kept' def process_data_file(input_file, output_file, velocity_mask, log_file=None): """ 处理整个数据文件 """ stats = { 'total_minutes': 0, 'kept_minutes': 0, 'discarded_minutes': 0, 'discard_reasons': { 'total_lt10': 0, 'invalid_data': 0 } } timestamp_pattern = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') current_array = [] current_timestamp = None processed_data = [] minute_count = 0 if log_file: log_dir = os.path.dirname(log_file) if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) with open(log_file, 'w') as log: log.write(f"雨滴数据清洗日志\n") log.write(f"开始时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") log.write(f"输入文件: {input_file}\n") log.write(f"输出文件: {output_file}\n") log.write(f"速度掩膜形状: {velocity_mask.shape}\n") log.write(f"速度掩膜剔除比例: {np.sum(velocity_mask) / 1024 * 100:.1f}%\n\n") log.write("清洗规则顺序:\n") log.write(" 1. 速度差>5m/s -> 剔除对应粒子\n") log.write(" 2. 剔除前2个直径档和后10个直径档\n") log.write(" 3. 行粒子数<2 -> 该行全部置0\n") log.write(" 4. 总粒子数<10 -> 剔除整分钟\n\n") print(f"\n开始处理文件: {input_file}") print(f"数据矩阵说明: 行=直径通道({len(diameters_corrected)}个), 列=速度通道({len(measured_velocity)}个)") with open(input_file, 'r') as f: for line in f: line = line.strip() if timestamp_pattern.match(line): if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) != 32: if log_file: with open(log_file, 'a') as log: log.write(f"{current_timestamp}: 跳过 - 数据不是32行\n") current_timestamp = line current_array = [] continue try: array_32x32 = np.array(current_array, dtype=float) except Exception as e: if log_file: with open(log_file, 'a') as log: log.write(f"{current_timestamp}: 跳过 - 数据格式错误: {e}\n") current_timestamp = line current_array = [] continue processed, reason = process_one_minute(array_32x32, velocity_mask, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 if log_file: with open(log_file, 'a') as log: original_total = np.sum(array_32x32) log.write(f"{current_timestamp}: 删除 - 清洗后粒子数<10 (原始:{original_total:.0f}, 清洗后:<10)\n") else: stats['kept_minutes'] += 1 processed_data.append((current_timestamp, processed)) if stats['kept_minutes'] == 1: original_total = np.sum(array_32x32) final_total = np.sum(processed) print(f"\n第一个保留分钟:") print(f" 时间戳: {current_timestamp}") print(f" 原始粒子数: {original_total:.0f}") print(f" 清洗后粒子数: {final_total:.0f}") print(f" 数据矩阵形状: {processed.shape} (行=直径, 列=速度)") current_timestamp = line current_array = [] else: if line: try: row_data = list(map(float, line.split())) if len(row_data) == 32: current_array.append(row_data) except: pass # 处理最后一个分钟 if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) == 32: array_32x32 = np.array(current_array, dtype=float) processed, reason = process_one_minute(array_32x32, velocity_mask, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 else: stats['kept_minutes'] += 1 processed_data.append((current_timestamp, processed)) # 保存结果 print(f"\n保存处理结果到: {output_file}") with open(output_file, 'w') as out_f: for timestamp, data in processed_data: out_f.write(f"{timestamp}\n") for i in range(32): row_int = [int(round(x)) for x in data[i, :]] out_f.write(" ".join(map(str, row_int)) + "\n") out_f.write("\n") # 打印统计 print("\n" + "=" * 60) print("数据预处理统计报告") print("=" * 60) print(f"总处理分钟数: {stats['total_minutes']}") print(f"保留分钟数: {stats['kept_minutes']} ({stats['kept_minutes'] / max(stats['total_minutes'], 1) * 100:.1f}%)") print( f"删除分钟数: {stats['discarded_minutes']} ({stats['discarded_minutes'] / max(stats['total_minutes'], 1) * 100:.1f}%)") print(f"删除原因:") print(f" - 清洗后粒子数<10: {stats['discard_reasons']['total_lt10']}") if log_file: with open(log_file, 'a') as log: log.write(f"\n处理完成!\n") log.write(f"总处理分钟数: {stats['total_minutes']}\n") log.write(f"保留分钟数: {stats['kept_minutes']}\n") log.write(f"删除分钟数: {stats['discarded_minutes']}\n") return processed_data, stats if __name__ == "__main__": # ==================== 文件路径配置 ==================== input_file = 'D:/lianxi/50934-20210507000700-20210507050859-0.txt' output_file = 'D:/lianxi/50934-20210507000700-20210507050859-0-clean.txt' log_file = 'D:/lianxi/data_processing_log.txt' velocity_mask_file = 'D:/lianxi/02-速度差掩膜.xlsx' print("=" * 60) print("雨滴数据预处理程序") print("数据矩阵结构: 行=直径通道, 列=速度通道") print("清洗规则顺序（调整后）：") print(" 1. 速度差 > 5 m/s（用掩膜剔除）") print(" 2. 剔除前2个直径档和后10个直径档（行）") print(" 3. 行粒子数 < 2 -> 该行全部置0") print(" 4. 总粒子数 < 10 -> 剔除整分钟（最后一步）") print("=" * 60) print(f"输入文件: {input_file}") print(f"输出文件: {output_file}") print(f"日志文件: {log_file}") print(f"速度掩膜文件: {velocity_mask_file}") print("=" * 60) try: velocity_mask = load_velocity_mask(velocity_mask_file) processed_data, stats = process_data_file(input_file, output_file, velocity_mask, log_file) print(f"\n处理完成！结果已保存到: {output_file}") print(f"保留分钟数: {stats['kept_minutes']}") if processed_data: print(f"\n前5个保留的时间戳:") for i in range(min(5, len(processed_data))): timestamp, data = processed_data[i] total_particles = np.sum(data) print(f" {timestamp}: {total_particles:.0f} 个粒子") except FileNotFoundError as e: print(f"文件未找到: {e}") print("请检查文件路径是否正确") except Exception as e: print(f"处理过程中出现错误: {e}") import traceback traceback.print_exc()
批量处理
#!/usr/bin/env python # -*- coding:utf-8 -*- """ @author: Suyue @file: batch_process_raindrop_data.py @time: 2026/09/01 @desc: 雨滴数据批量清洗 规则顺序： 1. 速度差>5m/s（用现成的掩膜） -> 剔除对应粒子 2. 剔除前2个直径档和后10个直径档（订正后直径） 3. 行粒子数<2 -> 该行全部置0（按直径区间统计） 4. 总粒子数<10 -> 剔除整分钟（最后一步） """ import numpy as np import re import os from datetime import datetime import pandas as pd import glob # ==================== 常量定义 ==================== # 32个直径通道的中心直径 (mm) - 这是已经轴比订正后的直径 diameters_corrected = np.array([ 0.062, 0.187, 0.312, 0.437, 0.562, 0.687, 0.812, 0.937, 1.064, 1.193, 1.388, 1.651, 1.918, 2.188, 2.463, 2.882, 3.457, 4.050, 4.665, 5.303, 6.194, 7.321, 8.447, 9.573, 10.699, 12.389, 14.641, 16.894, 19.146, 21.399, 24.214, 27.593 ]) # 32个速度通道的中心速度 (m/s) measured_velocity = np.array([ 0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.1, 1.3, 1.5, 1.7, 1.9, 2.2, 2.6, 3, 3.4, 3.8, 4.4, 5.2, 6, 6.8, 7.6, 8.8, 10.4, 12, 13.6, 15.2, 17.6, 20.8 ]) # 需要剔除的直径档位索引（前2个和后10个） # 索引: 0,1 是前两个档位；22-31 是后10个档位（因为总32个，后10个是索引22到31） # 注意：行代表直径 INVALID_DIAMETER_BINS = list(range(0, 2)) + list(range(22, 32)) def load_velocity_mask(mask_file): """ 加载速度掩膜Excel文件 注意：掩膜矩阵中，行代表直径，列代表速度 """ print(f"正在加载速度掩膜: {mask_file}") try: df = pd.read_excel(mask_file, header=None) mask = df.values if mask.shape != (32, 32): raise ValueError(f"掩膜形状错误: 期望(32,32)，实际{mask.shape}") unique_values = np.unique(mask) if not np.all(np.isin(unique_values, [0, 1])): print(f"警告: 掩膜中包含非0/1的值: {unique_values}") print(f"速度掩膜加载成功，形状: {mask.shape}") print(f"剔除位置数量: {np.sum(mask)} / 1024 ({np.sum(mask) / 1024 * 100:.1f}%)") print(f"掩膜矩阵说明: 行=直径通道({len(diameters_corrected)}个), 列=速度通道({len(measured_velocity)}个)") return mask except Exception as e: print(f"加载速度掩膜失败: {e}") raise def process_one_minute(data_32x32, velocity_mask, minute_number=None): """ 严格按照四步规则处理一分钟的32×32雨滴数据 数组结构：32行（直径）× 32列（速度） 规则顺序： 1. 速度差>5m/s（用现成的掩膜） -> 剔除对应粒子 2. 剔除前2个直径档和后10个直径档（订正后直径） 3. 行粒子数<2 -> 该行全部置0（按直径区间统计） 4. 总粒子数<10 -> 剔除整分钟（最后一步） """ # 创建处理副本 processed = data_32x32.copy().astype(float) # ===== 规则1: 速度差异大于5m/s的数据（用现成的掩膜）===== # velocity_mask中标记为1的位置，将雨滴谱数值置为0 # 掩膜矩阵: 行=直径, 列=速度 processed = np.where(velocity_mask == 1, 0, processed) # ===== 规则2: 剔除前2个直径档和后10个直径档（订正后直径）===== # 现在行代表直径，所以需要剔除行 for row in INVALID_DIAMETER_BINS: processed[row, :] = 0 # ===== 规则3: 行粒子数<2 -> 该行全部置0（按直径区间统计）===== # 注意：行代表直径区间，按行求和 for row in range(32): row_sum = np.sum(processed[row, :]) if row_sum < 2: processed[row, :] = 0 # ===== 规则4: 每分钟内的粒子数量少于10个（最后一步）===== total_particles = np.sum(processed) if total_particles < 10: return None, 'particle_count_lt10' return processed, 'kept' def process_data_file_with_stats(input_file, output_file, velocity_mask): """ 处理整个数据文件，并返回统计信息（不生成单独的log文件） """ stats = { 'total_minutes': 0, 'kept_minutes': 0, 'discarded_minutes': 0, 'discard_reasons': { 'total_lt10': 0, 'invalid_data': 0 }, 'mask_removed_particles': 0, # 掩膜剔除的粒子数 'total_original_particles': 0, # 原始总粒子数 'mask_remove_rate': 0 # 掩膜剔除比例 } timestamp_pattern = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') current_array = [] current_timestamp = None processed_data = [] minute_count = 0 with open(input_file, 'r') as f: for line in f: line = line.strip() if timestamp_pattern.match(line): if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) != 32: current_timestamp = line current_array = [] continue try: array_32x32 = np.array(current_array, dtype=float) except Exception as e: current_timestamp = line current_array = [] continue # 统计原始粒子数 original_total = np.sum(array_32x32) stats['total_original_particles'] += original_total # 统计掩膜剔除的粒子数 mask_removed = np.sum(array_32x32[velocity_mask == 1]) stats['mask_removed_particles'] += mask_removed processed, reason = process_one_minute(array_32x32, velocity_mask, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 else: stats['kept_minutes'] += 1 processed_data.append((current_timestamp, processed)) current_timestamp = line current_array = [] else: if line: try: row_data = list(map(float, line.split())) if len(row_data) == 32: current_array.append(row_data) except: pass # 处理最后一个分钟 if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) == 32: array_32x32 = np.array(current_array, dtype=float) # 统计原始粒子数和掩膜剔除 original_total = np.sum(array_32x32) stats['total_original_particles'] += original_total mask_removed = np.sum(array_32x32[velocity_mask == 1]) stats['mask_removed_particles'] += mask_removed processed, reason = process_one_minute(array_32x32, velocity_mask, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 else: stats['kept_minutes'] += 1 processed_data.append((current_timestamp, processed)) # 计算掩膜剔除比例 if stats['total_original_particles'] > 0: stats['mask_remove_rate'] = stats['mask_removed_particles'] / stats['total_original_particles'] * 100 # 只有有数据时才保存文件 if processed_data: with open(output_file, 'w') as out_f: for timestamp, data in processed_data: out_f.write(f"{timestamp}\n") for i in range(32): row_int = [int(round(x)) for x in data[i, :]] out_f.write(" ".join(map(str, row_int)) + "\n") out_f.write("\n") else: # 如果没有数据，删除可能存在的空文件 if os.path.exists(output_file): os.remove(output_file) return processed_data, stats def get_file_time_range(input_file): """ 从文件中提取时间范围 """ timestamp_pattern = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') timestamps = [] try: with open(input_file, 'r') as f: for line in f: line = line.strip() if timestamp_pattern.match(line): timestamps.append(line) if len(timestamps) >= 2: break # 找最后一个时间戳 with open(input_file, 'r') as f: lines = f.readlines() for line in reversed(lines): line = line.strip() if timestamp_pattern.match(line): if len(timestamps) < 2 or line != timestamps[-1]: timestamps.append(line) break except: pass if len(timestamps) >= 2: start = timestamps[0] end = timestamps[-1] return f"{start} ~ {end}" elif len(timestamps) == 1: return timestamps[0] else: return "未知" def batch_process_files(input_dir, output_dir, velocity_mask_file, log_file=None): """ 批量处理目录下所有txt文件 参数: input_dir: 输入文件目录 output_dir: 输出文件目录 velocity_mask_file: 速度掩膜文件路径 log_file: 汇总日志文件路径（可选） """ # 加载掩膜 velocity_mask = load_velocity_mask(velocity_mask_file) mask_remove_rate_global = np.sum(velocity_mask) / 1024 * 100 # 创建输出目录 if not os.path.exists(output_dir): os.makedirs(output_dir) # 查找所有txt文件 txt_files = glob.glob(os.path.join(input_dir, "*.txt")) if not txt_files: print(f"在 {input_dir} 中没有找到txt文件") return print(f"\n找到 {len(txt_files)} 个txt文件") print("=" * 60) # 汇总数据 summary = [] total_kept = 0 total_all = 0 total_files = 0 total_original_particles = 0 total_mask_removed = 0 for idx, input_file in enumerate(txt_files, 1): # 生成输出文件名 base_name = os.path.basename(input_file) name_without_ext = os.path.splitext(base_name)[0] output_file = os.path.join(output_dir, f"{name_without_ext}-clean.txt") try: print(f"\n[{idx}/{len(txt_files)}] 处理: {base_name}") # 获取文件时间范围 time_range = get_file_time_range(input_file) # 处理文件（不生成单独的log） processed_data, stats = process_data_file_with_stats( input_file, output_file, velocity_mask ) # 记录汇总信息 summary.append({ 'filename': base_name, 'time_range': time_range, 'total_minutes': stats['total_minutes'], 'kept_minutes': stats['kept_minutes'], 'discarded_minutes': stats['discarded_minutes'], 'keep_rate': stats['kept_minutes'] / max(stats['total_minutes'], 1) * 100, 'mask_remove_rate': stats['mask_remove_rate'], 'total_particles': stats['total_original_particles'], 'mask_removed': stats['mask_removed_particles'] }) total_kept += stats['kept_minutes'] total_all += stats['total_minutes'] total_files += 1 total_original_particles += stats['total_original_particles'] total_mask_removed += stats['mask_removed_particles'] print(f" 总分钟: {stats['total_minutes']}") print(f" 保留: {stats['kept_minutes']} 分钟 ({summary[-1]['keep_rate']:.1f}%)") print(f" 删除: {stats['discarded_minutes']} 分钟") print(f" 速度掩膜剔除比例: {stats['mask_remove_rate']:.1f}%") except Exception as e: print(f" 处理 {base_name} 失败: {e}") summary.append({ 'filename': base_name, 'time_range': '处理失败', 'total_minutes': 0, 'kept_minutes': 0, 'discarded_minutes': 0, 'keep_rate': 0, 'mask_remove_rate': 0, 'total_particles': 0, 'mask_removed': 0, 'error': str(e) }) # 计算全局掩膜剔除比例 global_mask_rate = total_mask_removed / max(total_original_particles, 1) * 100 # 打印汇总 print("\n" + "=" * 60) print("批量处理完成！") print("=" * 60) print(f"处理文件数: {total_files}") print(f"总共处理: {total_all} 分钟") print(f"总共保留: {total_kept} 分钟") print(f"总保留率: {total_kept / max(total_all, 1) * 100:.1f}%") print(f"总原始粒子数: {total_original_particles:.0f}") print(f"掩膜剔除粒子数: {total_mask_removed:.0f}") print(f"掩膜剔除总比例: {global_mask_rate:.1f}%") print(f"输出目录: {output_dir}") # 生成汇总日志（如果有指定log_file） if log_file: log_dir = os.path.dirname(log_file) if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) with open(log_file, 'w', encoding='utf-8') as f: f.write("=" * 120 + "\n") f.write("雨滴数据批量处理汇总日志\n") f.write("=" * 120 + "\n") f.write(f"处理时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") f.write(f"输入目录: {input_dir}\n") f.write(f"输出目录: {output_dir}\n") f.write(f"速度掩膜文件: {velocity_mask_file}\n") f.write(f"速度掩膜剔除比例(全局固定): {mask_remove_rate_global:.1f}%\n") f.write(f"处理文件数: {total_files}\n") f.write(f"总共处理分钟数: {total_all}\n") f.write(f"总共保留分钟数: {total_kept}\n") f.write(f"总保留率: {total_kept / max(total_all, 1) * 100:.1f}%\n") f.write(f"总原始粒子数: {total_original_particles:.0f}\n") f.write(f"掩膜剔除总粒子数: {total_mask_removed:.0f}\n") f.write(f"掩膜剔除总比例: {global_mask_rate:.1f}%\n") f.write("\n" + "-" * 120 + "\n") f.write(f"{'序号':<6} {'文件名':<35} {'时间范围':<35} {'总分钟':<8} {'保留':<8} {'删除':<8} {'保留率':<10} {'掩膜剔除率':<10}\n") f.write("-" * 120 + "\n") for i, item in enumerate(summary, 1): error_info = item.get('error', '') if error_info: f.write( f"{i:<6} {item['filename']:<35} {item['time_range']:<35} {item['total_minutes']:<8} {item['kept_minutes']:<8} {item['discarded_minutes']:<8} {'':<10} 错误: {error_info}\n") else: f.write( f"{i:<6} {item['filename']:<35} {item['time_range']:<35} {item['total_minutes']:<8} {item['kept_minutes']:<8} {item['discarded_minutes']:<8} {item['keep_rate']:>6.1f}% {item['mask_remove_rate']:>6.1f}%\n") f.write("-" * 120 + "\n") f.write( f"{'总计':<6} {'':<35} {'':<35} {total_all:<8} {total_kept:<8} {total_all - total_kept:<8} {total_kept / max(total_all, 1) * 100:>6.1f}% {global_mask_rate:>6.1f}%\n") f.write("=" * 120 + "\n") if __name__ == "__main__": # ==================== 批量处理配置 ==================== input_dir = 'D:/lianxi/' # 原始txt文件所在目录 output_dir = 'D:/lianxi/cleaned/' # 清洗后文件输出目录 log_file = 'D:/lianxi/batch_processing_summary.txt' # 汇总日志文件（保留） velocity_mask_file = 'D:/lianxi/02-速度差掩膜.xlsx' print("=" * 60) print("雨滴数据批量预处理程序") print("数据矩阵结构: 行=直径通道, 列=速度通道") print("清洗规则顺序：") print(" 1. 速度差 > 5 m/s（用掩膜剔除）") print(" 2. 剔除前2个直径档和后10个直径档（行）") print(" 3. 行粒子数 < 2 -> 该行全部置0") print(" 4. 总粒子数 < 10 -> 剔除整分钟（最后一步）") print("=" * 60) print(f"输入目录: {input_dir}") print(f"输出目录: {output_dir}") print(f"汇总日志: {log_file}") print(f"速度掩膜: {velocity_mask_file}") print("=" * 60) try: batch_process_files(input_dir, output_dir, velocity_mask_file, log_file) print(f"\n汇总日志已保存到: {log_file}") except FileNotFoundError as e: print(f"文件未找到: {e}") print("请检查文件路径是否正确") except Exception as e: print(f"处理过程中出现错误: {e}") import traceback traceback.print_exc()
二、雪花数据清洗
- 剔除前两个直径通道（信噪比低，基于原始直径）
- 轴比订正（把原始直径换算成雪花最大直径）
- 某个直径区间内的粒子数少于2个（防止粒子数为1的时候，u是nan）
- 每分钟内的粒子数量少于10个的数据；（这个得放后面要不然u算出来巨大无比）
实测粒子下落速度与经典值之间的差异大于5米每秒的数据（论文：Atlas D, Srivastava R C, Sekhon R S. 1973. Doppler radar characteristics of precipitation at vertical incidence [J]. Rev. Geophys., 11 (1): 1–35, doi:10.1029/RG011i001p00001.）。这句根本不适用于雪花，这是针对雨滴的。也不需要看雪花的经典下落速度，因为雪花的速度量级完全不同，雪花下落速度极慢，“经典值”本身不“经典”：这是更核心的问题。雨滴有像 Gunn-Kinzer 那样被广泛接受的、稳定的“经典”下落速度公式。但雪花完全不同：形状极端多样、密度变化极大、公式不唯一，不存在一个“唯一正确”的雪花下落速度公式。
轴比订正采用 Battaglia et al. (2010) 的方法，但需注意：对雪花而言，订正后得到的是最大水平宽度（WHD），而非等效体积直径，后续计算质量、反射率等需另用质量-直径关系。
参考文献：
[1]Battaglia A, Rustemeier E, Tokay A, et al. 2010. PARSIVEL snow observations: A critical assessment [J]. J. Atmos. Ocean. Technol., 27 (2): 333-344. doi:10.1175/2009JTECHA1332.1
[2]李遥, 牛生杰, 吕晶晶, 等. 2019. 2018 年冬季南京三次暴雪过程微物理特征分析[J]. 大气科学, 43(5): 1095-1108.
#!/usr/bin/env python # -*- coding:utf-8 -*- """ @author: Suyue @file: process_snow_data.py @time: 2026/09/02 @desc: 雪花数据清洗 规则顺序： 1. 剔除前2个直径通道（信噪比低，基于原始直径） 2. 某个直径区间内的粒子数少于2个 -> 该列全部置0（防止粒子数为1时，u是nan） 3. 每分钟内的粒子数量少于10个 -> 剔除整分钟（防止u算出来巨大无比） 数据结构：行=直径（32行），列=速度（32列） """ import numpy as np import re import os from datetime import datetime # ==================== 常量定义 ==================== # 32个直径通道的中心直径 (mm) - 这是已经轴比订正后的直径 diameters_corrected = np.array([ 0.062, 0.187, 0.312, 0.437, 0.562, 0.687, 0.812, 0.937, 1.064, 1.193, 1.388, 1.651, 1.918, 2.188, 2.463, 2.882, 3.457, 4.050, 4.665, 5.303, 6.194, 7.321, 8.447, 9.573, 10.699, 12.389, 14.641, 16.894, 19.146, 21.399, 24.214, 27.593 ]) # 32个速度通道的中心速度 (m/s) measured_velocity = np.array([ 0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.1, 1.3, 1.5, 1.7, 1.9, 2.2, 2.6, 3, 3.4, 3.8, 4.4, 5.2, 6, 6.8, 7.6, 8.8, 10.4, 12, 13.6, 15.2, 17.6, 20.8 ]) # 需要剔除的直径档位索引（前2个） INVALID_DIAMETER_BINS = list(range(0, 2)) def process_one_minute(data_32x32, minute_number=None): """ 处理一分钟的32×32雪花数据 数据结构：行=直径（32行），列=速度（32列） 规则顺序： 1. 剔除前2个直径通道（信噪比低，基于原始直径） 2. 某个直径区间内的粒子数少于2个 -> 该行全部置0（防止粒子数为1时，u是nan） 3. 每分钟内的粒子数量少于10个 -> 剔除整分钟（防止u算出来巨大无比） """ # 创建处理副本 processed = data_32x32.copy().astype(float) # ===== 规则1: 剔除前2个直径档（信噪比低）===== # 行=直径，所以剔除前2行 for row in INVALID_DIAMETER_BINS: processed[row, :] = 0 # ===== 规则2: 行粒子数<2 -> 该行全部置0（防止粒子数为1时，u是nan）===== # 行=直径，所以检查每行（每个直径区间） for row in range(32): row_sum = np.sum(processed[row, :]) if row_sum < 2: processed[row, :] = 0 # ===== 规则3: 每分钟内的粒子数量少于10个 -> 剔除整分钟（防止u算出来巨大无比）===== total_particles = np.sum(processed) if total_particles < 10: return None, 'particle_count_lt10' return processed, 'kept' def process_data_file(input_file, output_file, log_file=None): """ 处理整个数据文件 """ stats = { 'total_minutes': 0, 'kept_minutes': 0, 'discarded_minutes': 0, 'discard_reasons': { 'total_lt10': 0 } } timestamp_pattern = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') current_array = [] current_timestamp = None processed_data = [] minute_count = 0 if log_file: log_dir = os.path.dirname(log_file) if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) with open(log_file, 'w') as log: log.write(f"雪花数据清洗日志\n") log.write(f"开始时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") log.write(f"输入文件: {input_file}\n") log.write(f"输出文件: {output_file}\n\n") log.write("数据格式: 行=直径(32行), 列=速度(32列)\n") log.write("清洗规则顺序:\n") log.write(" 1. 剔除前2个直径通道（信噪比低）\n") log.write(" 2. 行粒子数<2 -> 该行全部置0（防止粒子数为1时，u是nan）\n") log.write(" 3. 总粒子数<10 -> 剔除整分钟（防止u算出来巨大无比）\n\n") print(f"\n开始处理文件: {input_file}") print("数据格式: 行=直径(32行), 列=速度(32列)") with open(input_file, 'r') as f: for line in f: line = line.strip() if timestamp_pattern.match(line): if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) != 32: if log_file: with open(log_file, 'a') as log: log.write(f"{current_timestamp}: 跳过 - 数据不是32行（当前{len(current_array)}行）\n") current_timestamp = line current_array = [] continue try: # 原始数据：行=直径，列=速度 array_32x32 = np.array(current_array, dtype=float) except Exception as e: if log_file: with open(log_file, 'a') as log: log.write(f"{current_timestamp}: 跳过 - 数据格式错误: {e}\n") current_timestamp = line current_array = [] continue processed, reason = process_one_minute(array_32x32, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 if log_file: with open(log_file, 'a') as log: original_total = np.sum(array_32x32) log.write(f"{current_timestamp}: 删除 - 清洗后粒子数<10 (原始:{original_total:.0f}, 清洗后:<10)\n") else: stats['kept_minutes'] += 1 processed_data.append((current_timestamp, processed)) if stats['kept_minutes'] == 1: original_total = np.sum(array_32x32) final_total = np.sum(processed) print(f"\n第一个保留分钟:") print(f" 时间戳: {current_timestamp}") print(f" 原始粒子数: {original_total:.0f}") print(f" 清洗后粒子数: {final_total:.0f}") print(f" 清洗说明: 剔除前2个直径档(行) + 行粒子数<2置0 + 总粒子数<10剔除") current_timestamp = line current_array = [] else: if line: try: row_data = list(map(float, line.split())) if len(row_data) == 32: current_array.append(row_data) except: pass # 处理最后一个分钟 if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) == 32: array_32x32 = np.array(current_array, dtype=float) processed, reason = process_one_minute(array_32x32, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 else: stats['kept_minutes'] += 1 processed_data.append((current_timestamp, processed)) # 只有有数据时才保存文件 if processed_data: print(f"\n保存处理结果到: {output_file}") with open(output_file, 'w') as out_f: for timestamp, data in processed_data: out_f.write(f"{timestamp}\n") # data: 行=直径(32行), 列=速度(32列) for i in range(32): row_int = [int(round(x)) for x in data[i, :]] out_f.write(" ".join(map(str, row_int)) + "\n") out_f.write("\n") else: # 如果没有数据，删除可能存在的空文件 if os.path.exists(output_file): os.remove(output_file) print(f"\n没有保留的数据，已删除空文件: {output_file}") # 打印统计 print("\n" + "=" * 60) print("数据预处理统计报告") print("=" * 60) print(f"总处理分钟数: {stats['total_minutes']}") print(f"保留分钟数: {stats['kept_minutes']} ({stats['kept_minutes'] / max(stats['total_minutes'], 1) * 100:.1f}%)") print( f"删除分钟数: {stats['discarded_minutes']} ({stats['discarded_minutes'] / max(stats['total_minutes'], 1) * 100:.1f}%)") print(f"删除原因:") print(f" - 清洗后粒子数<10: {stats['discard_reasons']['total_lt10']}") print(f"清洗规则:") print(f" 1. 剔除前2个直径档（行索引0和1）") print(f" 2. 行粒子数<2 -> 该行全部置0") print(f" 3. 总粒子数<10 -> 剔除整分钟") if log_file: with open(log_file, 'a') as log: log.write(f"\n处理完成!\n") log.write(f"总处理分钟数: {stats['total_minutes']}\n") log.write(f"保留分钟数: {stats['kept_minutes']}\n") log.write(f"删除分钟数: {stats['discarded_minutes']}\n") log.write(f"删除原因 - 粒子数<10: {stats['discard_reasons']['total_lt10']}\n") return processed_data, stats if __name__ == "__main__": # ==================== 文件路径配置 ==================== input_file = 'D:/lianxi/50934-20210507000700-20210507050859-0.txt' output_file = 'D:/lianxi/50934-20210507000700-20210507050859-0-clean.txt' log_file = 'D:/lianxi/data_processing_log-snow.txt' print("=" * 60) print("雪花数据预处理程序") print("数据格式: 行=直径(32行), 列=速度(32列)") print("清洗规则顺序：") print(" 1. 剔除前2个直径通道（信噪比低）") print(" 2. 行粒子数 < 2 -> 该行全部置0（防止粒子数为1时，u是nan）") print(" 3. 总粒子数 < 10 -> 剔除整分钟（防止u算出来巨大无比）") print("=" * 60) print(f"输入文件: {input_file}") print(f"输出文件: {output_file}") print(f"日志文件: {log_file}") print("=" * 60) try: # 处理数据 processed_data, stats = process_data_file(input_file, output_file, log_file) print(f"\n处理完成！结果已保存到: {output_file}") print(f"保留分钟数: {stats['kept_minutes']}") if processed_data: print(f"\n前5个保留的时间戳:") for i in range(min(5, len(processed_data))): timestamp, data = processed_data[i] total_particles = np.sum(data) print(f" {timestamp}: {total_particles:.0f} 个粒子") except FileNotFoundError as e: print(f"文件未找到: {e}") print("请检查文件路径是否正确") except Exception as e: print(f"处理过程中出现错误: {e}") import traceback traceback.print_exc()
批量处理
#!/usr/bin/env python # -*- coding:utf-8 -*- """ @author: Suyue @file: batch_process_snow_data.py @time: 2026/07/02 @desc: 雪花数据批量清洗 规则顺序： 1. 剔除前2个直径通道（信噪比低，基于原始直径） 2. 某个直径区间内的粒子数少于2个 -> 该行全部置0（防止粒子数为1时，u是nan） 3. 每分钟内的粒子数量少于10个 -> 剔除整分钟（防止u算出来巨大无比） 数据结构：行=直径（32行），列=速度（32列） """ import numpy as np import re import os from datetime import datetime import glob # ==================== 常量定义 ==================== # 32个直径通道的中心直径 (mm) - 这是已经轴比订正后的直径 diameters_corrected = np.array([ 0.062, 0.187, 0.312, 0.437, 0.562, 0.687, 0.812, 0.937, 1.064, 1.193, 1.388, 1.651, 1.918, 2.188, 2.463, 2.882, 3.457, 4.050, 4.665, 5.303, 6.194, 7.321, 8.447, 9.573, 10.699, 12.389, 14.641, 16.894, 19.146, 21.399, 24.214, 27.593 ]) # 32个速度通道的中心速度 (m/s) measured_velocity = np.array([ 0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.1, 1.3, 1.5, 1.7, 1.9, 2.2, 2.6, 3, 3.4, 3.8, 4.4, 5.2, 6, 6.8, 7.6, 8.8, 10.4, 12, 13.6, 15.2, 17.6, 20.8 ]) # 需要剔除的直径档位索引（前2个） INVALID_DIAMETER_BINS = list(range(0, 2)) def process_one_minute(data_32x32, minute_number=None): """ 处理一分钟的32×32雪花数据 数据结构：行=直径（32行），列=速度（32列） 规则顺序： 1. 剔除前2个直径通道（信噪比低，基于原始直径） 2. 某个直径区间内的粒子数少于2个 -> 该行全部置0（防止粒子数为1时，u是nan） 3. 每分钟内的粒子数量少于10个 -> 剔除整分钟（防止u算出来巨大无比） """ # 创建处理副本 processed = data_32x32.copy().astype(float) # ===== 规则1: 剔除前2个直径档（信噪比低）===== # 行=直径，所以剔除前2行 for row in INVALID_DIAMETER_BINS: processed[row, :] = 0 # ===== 规则2: 行粒子数<2 -> 该行全部置0（防止粒子数为1时，u是nan）===== # 行=直径，所以检查每行（每个直径区间） for row in range(32): row_sum = np.sum(processed[row, :]) if row_sum < 2: processed[row, :] = 0 # ===== 规则3: 每分钟内的粒子数量少于10个 -> 剔除整分钟（防止u算出来巨大无比）===== total_particles = np.sum(processed) if total_particles < 10: return None, 'particle_count_lt10' return processed, 'kept' def process_data_file(input_file, output_file, log_file=None): """ 处理整个数据文件 """ stats = { 'total_minutes': 0, 'kept_minutes': 0, 'discarded_minutes': 0, 'discard_reasons': { 'total_lt10': 0 }, 'total_original_particles': 0, 'total_kept_particles': 0 } timestamp_pattern = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') current_array = [] current_timestamp = None processed_data = [] minute_count = 0 if log_file: log_dir = os.path.dirname(log_file) if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) with open(input_file, 'r') as f: for line in f: line = line.strip() if timestamp_pattern.match(line): if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) != 32: if log_file: with open(log_file, 'a') as log: log.write(f"{current_timestamp}: 跳过 - 数据不是32行\n") current_timestamp = line current_array = [] continue try: array_32x32 = np.array(current_array, dtype=float) except Exception as e: if log_file: with open(log_file, 'a') as log: log.write(f"{current_timestamp}: 跳过 - 数据格式错误: {e}\n") current_timestamp = line current_array = [] continue # 统计原始粒子数 original_total = np.sum(array_32x32) stats['total_original_particles'] += original_total processed, reason = process_one_minute(array_32x32, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 if log_file: with open(log_file, 'a') as log: log.write(f"{current_timestamp}: 删除 - 清洗后粒子数<10 (原始:{original_total:.0f}, 清洗后:<10)\n") else: stats['kept_minutes'] += 1 final_total = np.sum(processed) stats['total_kept_particles'] += final_total processed_data.append((current_timestamp, processed)) current_timestamp = line current_array = [] else: if line: try: row_data = list(map(float, line.split())) if len(row_data) == 32: current_array.append(row_data) except: pass # 处理最后一个分钟 if current_array and current_timestamp: stats['total_minutes'] += 1 minute_count += 1 if len(current_array) == 32: array_32x32 = np.array(current_array, dtype=float) original_total = np.sum(array_32x32) stats['total_original_particles'] += original_total processed, reason = process_one_minute(array_32x32, minute_count) if processed is None: stats['discarded_minutes'] += 1 if reason == 'particle_count_lt10': stats['discard_reasons']['total_lt10'] += 1 else: stats['kept_minutes'] += 1 final_total = np.sum(processed) stats['total_kept_particles'] += final_total processed_data.append((current_timestamp, processed)) # 只有有数据时才保存文件 if processed_data: with open(output_file, 'w') as out_f: for timestamp, data in processed_data: out_f.write(f"{timestamp}\n") # data: 行=直径(32行), 列=速度(32列) for i in range(32): row_int = [int(round(x)) for x in data[i, :]] out_f.write(" ".join(map(str, row_int)) + "\n") out_f.write("\n") else: # 如果没有数据，删除可能存在的空文件 if os.path.exists(output_file): os.remove(output_file) if log_file: with open(log_file, 'a') as log: log.write(f"\n处理完成!\n") log.write(f"总处理分钟数: {stats['total_minutes']}\n") log.write(f"保留分钟数: {stats['kept_minutes']}\n") log.write(f"删除分钟数: {stats['discarded_minutes']}\n") log.write(f"删除原因 - 粒子数<10: {stats['discard_reasons']['total_lt10']}\n") return processed_data, stats def batch_process_files(input_dir, output_dir, log_file=None): """ 批量处理目录下所有txt文件 参数: input_dir: 输入文件目录 output_dir: 输出文件目录 log_file: 汇总日志文件路径（可选） """ # 创建输出目录 if not os.path.exists(output_dir): os.makedirs(output_dir) # 查找所有txt文件 txt_files = glob.glob(os.path.join(input_dir, "*.txt")) if not txt_files: print(f"在 {input_dir} 中没有找到txt文件") return print(f"\n找到 {len(txt_files)} 个txt文件") print("数据格式: 行=直径(32行), 列=速度(32列)") print("=" * 60) # 汇总数据 summary = [] total_kept = 0 total_all = 0 total_files = 0 for input_file in txt_files: # 生成输出文件名 base_name = os.path.basename(input_file) name_without_ext = os.path.splitext(base_name)[0] output_file = os.path.join(output_dir, f"{name_without_ext}-clean.txt") try: # 获取文件时间范围 time_range = get_file_time_range(input_file) processed_data, stats = process_data_file(input_file, output_file, log_file=None) # 计算粒子保留率 particle_keep_rate = stats['total_kept_particles'] / max(stats['total_original_particles'], 1) * 100 # 记录汇总信息 summary.append({ 'filename': base_name, 'time_range': time_range, 'total_minutes': stats['total_minutes'], 'kept_minutes': stats['kept_minutes'], 'discarded_minutes': stats['discarded_minutes'], 'keep_rate': stats['kept_minutes'] / max(stats['total_minutes'], 1) * 100, 'particle_keep_rate': particle_keep_rate }) total_kept += stats['kept_minutes'] total_all += stats['total_minutes'] total_files += 1 except Exception as e: summary.append({ 'filename': base_name, 'time_range': '处理失败', 'total_minutes': 0, 'kept_minutes': 0, 'discarded_minutes': 0, 'keep_rate': 0, 'particle_keep_rate': 0, 'error': str(e) }) # 打印汇总 print("\n" + "=" * 60) print("批量处理完成！") print(f"处理文件数: {total_files}") print(f"总共处理: {total_all} 分钟") print(f"总共保留: {total_kept} 分钟") print(f"总保留率: {total_kept / max(total_all, 1) * 100:.1f}%") print(f"输出目录: {output_dir}") # 生成汇总日志 if log_file: log_dir = os.path.dirname(log_file) if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) with open(log_file, 'w', encoding='utf-8') as f: f.write("=" * 110 + "\n") f.write("雪花数据批量处理汇总日志\n") f.write("=" * 110 + "\n") f.write(f"处理时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") f.write(f"输入目录: {input_dir}\n") f.write(f"输出目录: {output_dir}\n") f.write(f"数据格式: 行=直径(32行), 列=速度(32列)\n") f.write(f"处理文件数: {total_files}\n") f.write(f"总共处理分钟数: {total_all}\n") f.write(f"总共保留分钟数: {total_kept}\n") f.write(f"总保留率: {total_kept / max(total_all, 1) * 100:.1f}%\n") f.write("\n" + "=" * 110 + "\n") f.write("清洗规则顺序:\n") f.write(" 1. 剔除前2个直径通道（信噪比低）- 剔除前2行\n") f.write(" 2. 行粒子数<2 -> 该行全部置0（防止粒子数为1时，u是nan）\n") f.write(" 3. 总粒子数<10 -> 剔除整分钟（防止u算出来巨大无比）\n") f.write("\n" + "-" * 110 + "\n") f.write(f"{'文件名':<35} {'时间范围':<30} {'总分钟':<8} {'保留':<8} {'删除':<8} {'分钟保留率':<10} {'粒子保留率':<10}\n") f.write("-" * 110 + "\n") for item in summary: error_info = item.get('error', '') if error_info: f.write( f"{item['filename']:<35} {item['time_range']:<30} {item['total_minutes']:<8} {item['kept_minutes']:<8} {item['discarded_minutes']:<8} {'':<10} 错误: {error_info}\n") else: f.write( f"{item['filename']:<35} {item['time_range']:<30} {item['total_minutes']:<8} {item['kept_minutes']:<8} {item['discarded_minutes']:<8} {item['keep_rate']:>6.1f}% {item['particle_keep_rate']:>6.1f}%\n") f.write("-" * 110 + "\n") f.write( f"总计{'':<33} {total_all:<8} {total_kept:<8} {total_all - total_kept:<8} {total_kept / max(total_all, 1) * 100:>6.1f}%\n") f.write("=" * 110 + "\n") def get_file_time_range(input_file): """ 从文件中提取时间范围 """ timestamp_pattern = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') timestamps = [] try: with open(input_file, 'r') as f: for line in f: line = line.strip() if timestamp_pattern.match(line): timestamps.append(line) if len(timestamps) >= 2: break # 找最后一个时间戳 with open(input_file, 'r') as f: lines = f.readlines() for line in reversed(lines): line = line.strip() if timestamp_pattern.match(line): if len(timestamps) < 2 or line != timestamps[-1]: timestamps.append(line) break except: pass if len(timestamps) >= 2: start = timestamps[0] end = timestamps[-1] return f"{start} ~ {end}" elif len(timestamps) == 1: return timestamps[0] else: return "未知" if __name__ == "__main__": # ==================== 批量处理配置 ==================== input_dir = 'D:/lianxi/' # 原始txt文件所在目录 output_dir = 'D:/lianxi/cleaned/' # 清洗后文件输出目录 log_file = 'D:/lianxi/batch_processing_summary_snow.txt' # 汇总日志文件 print("=" * 60) print("雪花数据批量预处理程序") print("数据格式: 行=直径(32行), 列=速度(32列)") print("清洗规则顺序：") print(" 1. 剔除前2个直径通道（信噪比低）- 剔除前2行") print(" 2. 行粒子数 < 2 -> 该行全部置0（防止粒子数为1时，u是nan）") print(" 3. 总粒子数 < 10 -> 剔除整分钟（防止u算出来巨大无比）") print("=" * 60) try: batch_process_files(input_dir, output_dir, log_file) print(f"\n汇总日志已保存到: {log_file}") except FileNotFoundError as e: print(f"文件未找到: {e}") print("请检查文件路径是否正确") except Exception as e: print(f"处理过程中出现错误: {e}") import traceback traceback.print_exc()
浙公网安备 33010602011771号