使用power shell 拆分 csv文件 将大文件拆分成小文件。

$sourceFile = "C:\path\to\your\large_file.csv"
$outputDir = "C:\path\to\output"
$chunkSize = 500MB
$bufferSize = 4MB

# 创建输出目录
New-Item -ItemType Directory -Force -Path $outputDir | Out-Null

$reader = [System.IO.StreamReader]::new($sourceFile)
$header = $reader.ReadLine()

$i = 1
$writer = $null
$bytesWritten = 0

while (!$reader.EndOfStream) {
    $outFile = "$outputDir\part_$i.csv"
    $writer = [System.IO.StreamWriter]::new($outFile)
    $writer.WriteLine($header)
    $bytesWritten = [Text.Encoding]::UTF8.GetByteCount($header + "`n")

    while (($line = $reader.ReadLine()) -ne $null) {
        $lineBytes = [Text.Encoding]::UTF8.GetByteCount($line + "`n")
        if ($bytesWritten + $lineBytes -ge $chunkSize) {
            break
        }
        $writer.WriteLine($line)
        $bytesWritten += $lineBytes
    }

    $writer.Close()
    $i++
}

$reader.Close()
Write-Output "拆分完成，共拆成 $($i - 1) 个文件。"