固定存储格式can数据分析软件 python

#!/usr/bin/python3.9
# -*- coding: utf-8 -*-  

import os
import subprocess,time
from subprocess import *
import threading
import PySimpleGUI as sg
from functools import partial


start_time = time.time()
# def readfile(inputfile, block_size = 1024*64):
    # # with open(inputfile, 'r') as f:
        # # while True:
            # # lines = f.read(block_size)
            # #print(lines)
    # for lines in iter(partial(inputfile.read, block_size), ''):
        # yield lines
class Get_file:
     
    def __init__(self, name):
        self.name = name
        self.file = ['101_1.asc', '222_2.asc', '444_3.asc', '331_4.asc', \
        '101_5.asc', '222_6.asc', '444_7.asc', '331_8.asc', '222_9.asc','101_10.asc']
        self.threads = []
        self.dataline = []
        self.canfd_filter = ['findstr /irc:"CANFD 1   Rx 101      0 0 d  32" ',
                        'findstr /irc:"CANFD 2   Rx 222      0 0 d  32" ',
                        'findstr /irc:"CANFD 3   Rx 444      0 0 d  32" ',
                        'findstr /irc:"CANFD 4   Rx 331      0 0 d  32" ',
                        'findstr /irc:"CANFD 5   Rx 101      0 0 d  32" ',
                        'findstr /irc:"CANFD 6   Rx 222      0 0 d  32" ',
                        'findstr /irc:"CANFD 7   Rx 444      0 0 d  32" ',
                        'findstr /irc:"CANFD 8   Rx 331      0 0 d  32" ',
                        'findstr /irc:"CANFD 9   Rx 222      0 0 d  32" ',
                        'findstr /irc:"CANFD 10  Rx 101      0 0 d  32" ',
                        ]
        self.can_filter = [
                    'findstr /irc:"1 101 Rx d 8" ',
                    'findstr /irc:"2 222 Rx d 8" ',
                    'findstr /irc:"3 444 Rx d 8" ',
                    'findstr /irc:"4 331 Rx d 8" ',
                    'findstr /irc:"5 101 Rx d 8" ',
                    'findstr /irc:"6 222 Rx d 8" ',
                    'findstr /irc:"7 444 Rx d 8" ',
                    'findstr /irc:"8 331 Rx d 8" ',
                    'findstr /irc:"9 222 Rx d 8" ',
                    'findstr /irc:"10 101 Rx d 8" ',
                    ]
        
    def __chunked_file_reader__(self, inputfile, block_size=1024 * 8):
        """生成器函数：分块读取文件内容，使用 iter 函数
        """
        # 首先使用 partial(fp.read, block_size) 构造一个新的无需参数的函数
        # 循环将不断返回 fp.read(block_size) 调用结果，直到其为 '' 时终止
        fp = open(inputfile, 'r')
        for chunk in iter(partial(fp.read, block_size), ''):
            yield chunk
            
    def __run_filter__(self, file, commandLine):
        with open(file,'w') as fp:
            subprocess.run(commandLine, stdout=fp)
            
    def file_split(self, inputfile):
        print("file path is %s" % inputfile)        
        data_lines_all = self.__chunked_file_reader__(inputfile) 
        data_lines_all = list(data_lines_all)
        self.data_line = data_lines_all[0].strip().split("\n")

        n= 10
        while True:
            split_line_is1 = self.data_line[n].strip().split(" ")
            if split_line_is1[2] != 'ErrorFrame':
                break
            n += 10
        while '' in split_line_is1:
            split_line_is1.remove('')
            
        if inputfile not in self.canfd_filter[0]:    
            for i in range(len(self.canfd_filter)):
                self.canfd_filter[i] = self.canfd_filter[i] + inputfile
            
            for i in range(len(self.can_filter)):
                self.can_filter[i] = self.can_filter[i] + inputfile
         
        
        
        if split_line_is1[1] == 'CANFD':
            for i in range(len(self.canfd_filter)):
                t = threading.Thread(target=self.__run_filter__, args=(self.file[i],self.canfd_filter[i],))
                t.start()
                self.threads.append(t)
                
            for t in self.threads:
                t.join()
        else:
            for i in range(len(self.can_filter )):
                t = threading.Thread(target=self.__run_filter__, args=(self.file[i],self.can_filter[i],))
                t.start()
                self.threads.append(t)
                
            for t in self.threads:
                t.join()


class SplitFile_processing:
    def __init__(self, name):
        self.name = name
        self.data_lines = [1,1,1,1,1,1,1,1,1,1]
        self.a = [1,1,1,1,1,1,1,1,1,1]
        self.sum1 = [1,1,1,1,1,1,1,1,1,1]
        self.All_lines = 0
        self.sum_all = 0
        self.file = ['101_1.asc', '222_2.asc', '444_3.asc', '331_4.asc', \
        '101_5.asc', '222_6.asc', '444_7.asc', '331_8.asc', '222_9.asc','101_10.asc']
         
    def __readsplitfile__(self, inputfile):
        with open(inputfile) as f:
            lines = f.readlines()
        return lines 
   
    def __datachange_can__(self, string_data):
        split_line = string_data.strip().split(" ")
        while '' in split_line:
            split_line.remove('')
        temp = split_line[12] + split_line[13]
        temp1 = int(temp,16)
        split_line.append(temp)
        split_line.append(temp1)
        
        return split_line
    
    def __datachange_canfd__(self, string_data):
        split_line = string_data.strip().split(" ")
        while '' in split_line:
            split_line.remove('')    
        temp = split_line[len(split_line)-8 - 2] + split_line[len(split_line)-8 - 1]
        temp1 = int(temp,16)
        split_line.append(temp)
        split_line.append(temp1)
        
        return split_line    
    

    def __change_svae__(self,data_lines1):
        sum1  = 0  
        for i in range(len(data_lines1)):
            split_line_is = data_lines1[1].strip().split(" ")
            while '' in split_line_is:
                split_line_is.remove('')
            if split_line_is[1] == 'CANFD':
                split_line = self.__datachange_canfd__(data_lines1[i])
            
                if i<1:
                    split_line.append('\n')
                    s2 = " ".join(['%s' %id for id in split_line])
                    data_lines1[i] = s2
                else:
                    split_line1 =self.__datachange_canfd__(data_lines1[i-1])                
                    temp = split_line[len(split_line)-1]- int(split_line1[50])         
                    split_line.append(temp)
                    split_line.append('\n')
                    #if (temp > 1) and (sum1>= 0):
                    if (temp != 1) and (temp!= -65535):
                        temp = temp - 1
                        sum1 = sum1+temp
                    s2 = " ".join(['%s' %id for id in split_line]) 
                    data_lines1[i] = s2

            else:            
                split_line = self.__datachange_can__(data_lines1[i])
            
                if i<1:
                    split_line.append('\n')
                    s2 = " ".join(['%s' %id for id in split_line])
                    data_lines1[i] = s2
                else:
                    split_line1 =self.__datachange_can__(data_lines1[i-1])
                    temp = split_line[len(split_line)-1]- int(split_line1[15])
                    split_line.append(temp)
                    split_line.append('\n')
                    if (temp != 1) and (temp!= -65535): 
                        temp = temp - 1
                        sum1 = sum1+temp
                    s2 = " ".join(['%s' %id for id in split_line])
                    data_lines1[i] = s2

        return data_lines1, sum1
    
    def print_function(self):
        for i in range(10):    
            self.All_lines = len(self.data_lines[i]) + self.All_lines
            
        for i in range(10):
            self.sum_all = self.sum1[i] + self.sum_all
        print('已读取完毕, channel文件总行数为：%d' %len(self.data_lines[0]))
        print('已读取完毕, channe2文件总行数为：%d' %len(self.data_lines[1]))
        print('已读取完毕, channe3文件总行数为：%d' %len(self.data_lines[2]))
        print('已读取完毕, channe4文件总行数为：%d' %len(self.data_lines[3]))
        print('已读取完毕, channe5文件总行数为：%d' %len(self.data_lines[4]))
        print('已读取完毕, channe6文件总行数为：%d' %len(self.data_lines[5]))
        print('已读取完毕, channe7文件总行数为：%d' %len(self.data_lines[6]))
        print('已读取完毕, channe8文件总行数为：%d' %len(self.data_lines[7]))
        print('已读取完毕, channe9文件总行数为：%d' %len(self.data_lines[8]))
        print('已读取完毕, channel0文件总行数为：%d' %len(self.data_lines[9]))
        print('已读取完毕, ALL channel文件总行数为：%d' %self.All_lines)
        print('\n')
        print("channel 1 lossframe %d" %self.sum1[0])
        print("channel 2 lossframe %d" %self.sum1[1])
        print("channel 3 lossframe %d" %self.sum1[2])
        print("channel 4 lossframe %d" %self.sum1[3])
        print("channel 5 lossframe %d" %self.sum1[4])
        print("channel 6 lossframe %d" %self.sum1[5])
        print("channel 7 lossframe %d" %self.sum1[6])
        print("channel 8 lossframe %d" %self.sum1[7])
        print("channel 9 lossframe %d" %self.sum1[8])
        print("channel 10 lossframe %d" %self.sum1[9])    
        print("ALL 10 channel total lossframe %d" %self.sum_all)
        print('\n')
        if((self.All_lines+self.sum_all) == 0):
            print("No response: wrong number fliter")
        else:
            print("Frame loss rate is %.6f" %(self.sum_all/(self.All_lines+self.sum_all)))
        print("数据分割转换成功")
        print("----程序运行时间%s---" %(time.time()- start_time))
        return self.sum_all
    
    def data_processing(self):
        for i in range(10):  
            self.data_lines[i] = self.__readsplitfile__(self.file[i])
        
        c = 'csv'
        for i in range(1,11):
            self.a[i-1] = str(i) + '.'+'csv'
            with open(self.a[i-1],'w') as fp:
                #if(data_lines[i-1] == )
                self.data_lines[i-1], self.sum1[i-1]=self.__change_svae__(self.data_lines[i-1])
                s1 = ''.join(self.data_lines[i-1])
                fp.write(s1)
                fp.close()
            
if __name__ == "__main__":
    Raw_data = Get_file("Combain_ALL")
    Split_data = SplitFile_processing("Split_file")
    sg.theme('Light Brown 1')  # please make your windows colorful
    layout = [[sg.Text('File of your choice  :',font=("宋体", 10)),sg.Text('',key='text1',size=(50,1),font=("宋体", 10))],
              [sg.Text('All lossframe is :', font=("宋体", 10)),sg.Text(size=(50,1), key='-OUTPUT-')],
              [sg.Text('Processing',justification='center')],
              [sg.Output(size=(80, 20),font=("宋体", 10))],              
              [sg.FileBrowse('Open file',key='folder',target='text1'), sg.Button('Data processing'), sg.Button('Exit')]
            ]    

    window = sg.Window('丢帧处理 : ', layout,font=("宋体", 15),default_element_size=(50,1))  

    while True:
        event, values = window.read()
        if event == 'Exit' or event == sg.WIN_CLOSED:   # 如果用户关闭窗口或点击`关闭`
            break
        if event == 'Data processing':
            if values['folder']:
                print('{0}正在分析原文件{0}'.format('*'*10))
                Raw_data.file_split(values['folder'])
                print('file split OK')
                Split_data.data_processing()
                values = Split_data.print_function()
                window['-OUTPUT-'].update(values)
                print(values)
                print('{0}数据处理完毕{0}'.format('*'*10))
            else:
                print('请先选择文件')
    window.close()   

 分为两个类一个是获取数据并过滤，利用windows的dos脚本自行过滤速度快，多线程并发过滤

数据处理模块，过滤的数据再次分解，得到所需的小量数据

处理级别，20G以内速度5~10min