将DF逐行写入到csv或者txt中

一、背景

如果DataFrame(df)特别大，或者需要将多个表头一样的df写入多同一个文件中。

使用的场景是：

• df文件太大，无法一次性写入到同一个文件中
• 有循环操作，不想将循环的结果汇总成一个df,然后在导出到文件中，而是想在循环的时候，就写入到文件中

 

 

二、代码展示

import pandas as pd
import csv

data = {'state':['Ohio','Ohio','Ohio','Nevada','Nevada','Nevada'],
        'year':[2000,2001,2002,2001,2002,2003],
        'pop':['1.5','1.7','3.6','2.4','2.9','3.2']}
df = pd.DataFrame(data)

col = df.columns.tolist()
val = df.values.tolist()


def get_line_count(file_path):
    with open(file_path, 'r') as file:
        line_count = sum(1 for line in file)
    return line_count

def write_csv_line(file_path, data, mode='a'):
    with open(file_path, mode=mode, newline='') as file:
        writer = csv.writer(file)
        for row in data:
            writer.writerow(row)


def write_csv_chunk(file_path,data, mode='a'):
    with open(file_path, mode=mode, newline='') as file:
        writer = csv.writer(file)
        writer.writerows(data)                   # 批量写入





def write_txt_line(file_path, data,mode='a'):
    with open(file_path, mode=mode) as file:
        for row in data:
            file.write(', '.join(map(str, row)) + '\n')



def write_txt_chunk(file_path, data,mode='a'):
    with open(file_path, mode=mode) as file:
        content = '\n'.join([', '.join(map(str, row)) for row in data])
        file.write(content + '\n')  # 写入字符串，并在最后一行添加换行符




csv_file_path = 'output.csv'
txt_file_path = 'output.txt'

# csv文件
# 写入标题
write_csv_line(file_path=csv_file_path, data=[col], mode='w')

# 逐行写入内容
data1 = val[:3]
write_csv_line(file_path=csv_file_path, data=data1)

# 批量写入后续内容
line_count = get_line_count(csv_file_path)-1
data2 = val[line_count:]
write_csv_chunk(file_path=csv_file_path, data=data2)



# txt文件
# 写入标题
write_txt_line(file_path=txt_file_path, data=[col], mode='w')

# 逐行写入内容
data1 = val[:3]
write_txt_line(file_path=txt_file_path, data=data1)

# 批量写入后续内容
line_count = get_line_count(txt_file_path)-1
data2 = val[line_count:]
write_txt_chunk(file_path=txt_file_path, data=data2)