触发器

 

public static boolean mergeFiles(String[] fpaths, String resultPath) {

     
    if (fpaths == null || fpaths.length < 1 || StringUtils.isEmpty(resultPath)) {
        return false;
    }
    if (fpaths.length == 1) {
        return new File(fpaths[0]).renameTo(new File(resultPath));
    }

    File[] files = new File[fpaths.length];
    for (int i = 0; i < fpaths.length; i ++) {
        files[i] = new File(fpaths[i]);
        if (StringUtils.isEmpty(fpaths[i]) || !files[i].exists() || !files[i].isFile()) {
            return false;
        }
    }

    File resultFile = new File(resultPath);

    try {
        int bufSize = 1024*5;
        BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(resultFile));
        byte[] buffer = new byte[bufSize];

        for (int i = 0; i < fpaths.length; i ++) {
            BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(files[i]));
            int readcount;
            while ((readcount = inputStream.read(buffer)) > 0) {
                outputStream.write(buffer, 0, readcount);
            }
            inputStream.close();
        }
        outputStream.close();
    } catch (FileNotFoundException e) {
        e.printStackTrace();
        return false;
    } catch (IOException e) {
        e.printStackTrace();
        return false;
    }

    for (int i = 0; i < fpaths.length; i ++) {
        files[i].delete();
    }

    return true;
}

mysql -uroot -p'Root1qaz!QAZ' -Dsdm

 @ShellMethod("写入测试CSV文件")
    public void writecsv(String filePath,int cols, int rows) {


        File file = new File(filePath);
        if(file.exists()){
            file.delete();
        }

        StopWatch stopWatch = new StopWatch();
        stopWatch.start();

        // 如果不能整除 写线程批次+1
        int countDownLatchSize = rows/batchsize;
        if(rows%batchsize>0){
            countDownLatchSize++;
        }


        LinkedBlockingQueue<BufferedWriter> blockingQueue = new LinkedBlockingQueue(poolsize);

        ExecutorService writePool =  Executors.newFixedThreadPool(poolsize);
        CountDownLatch countDownLatch = new CountDownLatch(countDownLatchSize);
        LongAdder num = new LongAdder();
        String[] fpaths = new String[poolsize];
        try {

            for(int i= 0 ;i< poolsize ;i++){
                File filex = new File(filePath+i);
                if(filex.exists()){
                    //
                    log.info("删除旧文件:{}",filex.getAbsolutePath());
                    filex.delete();
                }
                fpaths[i] = filePath+i;
                BufferedWriter bufferedWriter = new BufferedWriter(  new FileWriter(filex, true),8192);
                blockingQueue.put(bufferedWriter);
            }


            DataChunk<Object[]> dataChunk = new DataChunk<>();
            for(int i = 1 ;i<=rows;i++){
                Object[] args =  ColsFacker.genObjects(i,cols);
                dataChunk.addRecord(args);
                if(i%batchsize==0){
                    log.info("生成数据行：{}",i);
                    writePool.submit( new BufferedFileWriter(dataChunk,blockingQueue,countDownLatch,num));
                    dataChunk = new DataChunk<>();
                }
            }

            // 剩余未能整除的批次 再提交一个线程
            if(dataChunk.getDataList()!=null && dataChunk.getDataList().size()>0){
                log.info("生成数据行：{}",rows);
                writePool.submit( new BufferedFileWriter(dataChunk,blockingQueue,countDownLatch,num));
            }
            log.info("等待所有线程写入完成");
            countDownLatch.await();

            while(blockingQueue.size()>0){
                BufferedWriter bufferedWriter = blockingQueue.take();
                bufferedWriter.flush();
                bufferedWriter.close();

            }
            log.info("合并文件开始");
            mergeFiles(fpaths,filePath);
            log.info("合并文件完成 ");

        } catch (IOException | InterruptedException e) {
            e.printStackTrace();
        };



        // 默认8K
//        try (BufferedWriter bufferedWriter = new BufferedWriter(  new FileWriter(filePath, true),8192)) {
//
//            for(int i = 1 ;i<=rows;i++){
//                Object[] args =  ColsFacker.genObjects(i,cols);
//                bufferedWriter.write(ColsFacker.genObjectsLine(args));
//                bufferedWriter.newLine();
//            }
//        } catch (IOException e) {
//            e.printStackTrace();
//        }


        stopWatch.stop();


        long fileSize = FileUtils.sizeOf(file);

        log.info( "写入文件{} 共[{}]列,[{}] 行, 占用空间{},任务耗时:{}秒,效率:{}",
                filePath,cols,rows,SizeUtil.formetTabeSize(fileSize),(int)stopWatch.getTotalTimeSeconds() ,SizeUtil.formetTabeRate(fileSize,stopWatch.getTotalTimeSeconds()));
    }