• 博客园logo
  • 会员
  • 众包
  • 新闻
  • 博问
  • 闪存
  • 赞助商
  • HarmonyOS
  • Chat2DB
    • 搜索
      所有博客
    • 搜索
      当前博客
  • 写随笔 我的博客 短消息 简洁模式
    用户头像
    我的博客 我的园子 账号设置 会员中心 简洁模式 ... 退出登录
    注册 登录

yxchun

  • 博客园
  • 联系
  • 订阅
  • 管理

公告

View Post

JAVA使用NIO拆分文件

比如一个jmeter结果文件有34GB那么大,将其拆分成多个文件,方便使用

package seperateFile;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

public class JtlFileSplitterNIO {

    public static void splitJtlFileNIO(String inputFile, int numParts) throws IOException {
        // 先读取header
        String header = getHeader(inputFile);
        boolean hasHeader = header != null;

        List<Long> splitPoints = findSplitPoints(inputFile, numParts);

        try (FileChannel inChannel = new FileInputStream(inputFile).getChannel()) {
            long startPos = 0;

            // 如果有header,跳过第一行
            if (hasHeader) {
                startPos = findNextLinePosition(inChannel, 0);
            }

            for (int i = 0; i < numParts; i++) {
                String outputFile = String.format("%s.part%d", inputFile, i + 1);
                long endPos = (i == numParts - 1) ? inChannel.size() : splitPoints.get(i);

                System.out.printf("生成文件 %s: 位置 %d - %d%n", outputFile, startPos, endPos);

                extractFileSegment(inChannel, startPos, endPos, outputFile, header, i == 0 && hasHeader);
                startPos = endPos;
            }
        }
    }

    private static List<Long> findSplitPoints(String filename, int numParts) throws IOException {
        List<Long> splitPoints = new ArrayList<>();
        try (FileChannel channel = new FileInputStream(filename).getChannel()) {
            long fileSize = channel.size();
            long partSize = fileSize / numParts;

            for (int i = 1; i < numParts; i++) {
                long splitPoint = i * partSize;
                // 找到下一个换行符位置
                long actualSplit = findNextLinePosition(channel, splitPoint);
                splitPoints.add(actualSplit);
            }
        }
        return splitPoints;
    }

    private static long findNextLinePosition(FileChannel channel, long start) throws IOException {
        ByteBuffer buffer = ByteBuffer.allocate(8192);
        long position = start;

        channel.position(position);
        int bytesRead = channel.read(buffer);

        while (bytesRead != -1) {
            buffer.flip();
            for (int i = 0; i < buffer.limit(); i++) {
                if (buffer.get() == '\n') {
                    long lineEnd = position + i + 1;
                    return lineEnd;
                }
            }
            position += bytesRead;
            buffer.clear();
            bytesRead = channel.read(buffer);
        }

        return channel.size();
    }

    private static void extractFileSegment(FileChannel inChannel, long start, long end,
                                           String outputFile, String header, boolean writeHeader)
            throws IOException {
        try (FileChannel outChannel = new FileOutputStream(outputFile).getChannel()) {
            long transferSize = end - start;
            long transferred = 0;

            // 写入header
            if (writeHeader && header != null) {
                ByteBuffer headerBuffer = ByteBuffer.wrap(
                        (header + System.lineSeparator()).getBytes(StandardCharsets.UTF_8));
                outChannel.write(headerBuffer);
            }

            // 传输数据
            while (transferred < transferSize) {
                long count = inChannel.transferTo(start + transferred,
                        Math.min(64 * 1024, transferSize - transferred),
                        outChannel);
                if (count == 0) break;
                transferred += count;
            }
        }
    }

    private static String getHeader(String filename) throws IOException {
        try (BufferedReader reader = new BufferedReader(
                new InputStreamReader(
                        new FileInputStream(filename), StandardCharsets.UTF_8))) {
            return reader.readLine();
        }
    }

    public static void main(String[] args) {
        try {
            String inputFile = "C:\\Users\\yxchun\\Desktop\\7日\\results_20250926_170540.12.jtl";
            int numParts = 4;

            System.out.println("开始使用NIO拆分文件: " + inputFile);
            splitJtlFileNIO(inputFile, numParts);
            System.out.println("文件拆分完成!");

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

 

posted on 2025-10-09 10:26  yxchun  阅读(3)  评论(0)    收藏  举报

刷新页面返回顶部
 
博客园  ©  2004-2025
浙公网安备 33010602011771号 浙ICP备2021040463号-3