txt保存为csv, csv筛选重复数据

# coding=utf-8
import pandas as pd
# 读取txt  保存到csv

f = open('right.txt', 'r', encoding='utf-8')
lines = f.readlines()#读取全部内容
l = []
for line in lines:
    res = [line.split(',')[0], line.split(',')[1], line.split(',')[2].split('>')[0], line.split(',')[2].split('>')[1], line.split(',')[2].split('>')[2].replace('"', '').replace('\n', '')]
    # res = [line.split(',')[0], line.split(',')[1], line.split(',')[2], line.split(',')[3], line.split(',')[4]]
    print(res)
    l.append(res)
df = pd.DataFrame(l)
df.to_csv('right_new.csv', index=False, header=['id', 'title', 'class1', 'class2', 'class3'])
#********************************************************************************************************************

# coding=utf-8
import pandas as pd

# 两个csv文件,源文件,新文件, 根据源文件筛选出 新文件中重复的id,

def read_y_csv():
    # 读取源文件
    read_y = pd.read_csv('right_new_new.csv')
    # read_y.set_index('id')
    # print(read_y)
    all_data = []
    rows = read_y.iterrows()
    for index, row in rows:
        # print(row)
        all_data.append(row)
    print('all_data = ', len(all_data))


def read_n_csv():
    # 读取新文件
    read_n = pd.read_csv('right_new_1.csv', header=1)
    # read_y.set_index('id')
    # print(read_y)
    id_list = []       # 不重复数据
    id_repeat_list = []     # 重复数据
    datas = []
    rows = read_n.iterrows()
    for index, row in rows:
        # print(row[0])
        if str(row[0]) not in id_list:
            id_list.append(str(row[0]))
            datas.append(row)
        else:
            id_repeat_list.append(str(row[0]))
    print('id_list = ', len(id_list))
    print('id_repeat_list = ', len(id_repeat_list))
    # 保存不重复的数据到csv
    df = pd.DataFrame(datas)
    df.to_csv('right_new_new.csv', index=False)



read_n_csv()
read_y_csv()