easyExcel分批导入文件

 

原文地址 ： https://blog.csdn.net/qq_42060055/article/details/116139016

 

一些关于easyExcel导入文件操作

需求: 导入大数据量文件 其中数据达到万级、十万级, 错误文件进行错误单元格标红, 可导出修改完继续导入

由于数据量多大 一次行全部读到内存中可能会导致内存溢出问题

使用easyExcel poi的监听器进行操作

三步曲:

1、解析excel为inputStream流, 读取流,解析excel

2、判断excel中每条数据的格式, 正确和错误相对记录

3、通过监听器每解析150条数据, 进行入库操作, 错误数据存在内存中(考虑错误数据不多的情况)

// 这里用到ossfs 反正就是读取excel为input流,
涉及到两个系统之间流的传输, 这里直接把文件上传到oss
try {
    in = new FileInputStream(localFileName);
} catch (FileNotFoundException e) {
    in = HttpUtil.io(HttpUtil.Atom.builder().url(diseaseDto.getFileUrl()).build());
}

　　

// 这里解析excel其中
OltHosIcdDiseaseListener为自定义监听器
try {
    LoggerUtil.info(LOGGER, "开始解析IcdDisease");
    OltHosIcdDiseaseListener oltHosIcdDiseaseListener = new OltHosIcdDiseaseListener(isCfgPrd, icdCodeList, delIcdCodeList, diseaseDto, oltConfigService, exportTaskHandler);
    excelReader = EasyExcel.read(in, oltHosIcdDiseaseListener).build();
    ReadSheet readSheet = EasyExcel.readSheet(0).build();
    excelReader.read(readSheet);
} finally {
    try {
        if (in != null) {
            in.close();
        }
        if (excelReader != null) {
            // 这里千万别忘记关闭，读的时候会创建临时文件，到时磁盘会崩的
            excelReader.finish();
        }
    } catch (Exception e) {
        LoggerUtil.error(LOGGER, "{0},{1}", e.getMessage(), e);
    }
}

　　

// 通过构造函数, 初始化一些list与对象
// 这个是导入的核心方法, 所有的导入逻辑, 判断逻辑与入库都在这里操作
// 采用无对象方式
@Slf4j
public class OltHosIcdDiseaseListener extends AnalysisEventListener<Map<Integer, String>> {
    private OltConfigService oltConfigService;
    private ExportTaskHandler exportTaskHandler;
    private static final int batchCount = 150;
    private int countNum = 0;
    private boolean isCfgPrd;
    private int successCount = 0;
    private int errorCount = 0;
 
    private List<String> checkRepeatCode = new ArrayList<>();
    private List<String> icdCodeList;
    private List<String> delIcdCodeList;
    private OltHosIcdDiseaseDto diseaseDto;
    private List<OltHosIcdDiseaseDto> successList = new ArrayList<>();
    private List<OltHosIcdDiseaseDto> errorList = new ArrayList<>();
    private List<OltHosIcdDiseaseDto> tempErrorList = new ArrayList<>();
 
 
    public OltHosIcdDiseaseListener(boolean isCfgPrd, List<String> icdCodeList, List<String> delIcdCodeList, OltHosIcdDiseaseDto diseaseDto,
                                    OltConfigService oltConfigService, ExportTaskHandler exportTaskHandler) {
        this.isCfgPrd = isCfgPrd;
        this.icdCodeList = icdCodeList;
        this.delIcdCodeList = delIcdCodeList;
        this.diseaseDto = diseaseDto;
        this.oltConfigService = oltConfigService;
        this.exportTaskHandler = exportTaskHandler;
    }
 
/**
  * 这个每一条数据解析都会来调用
  * data --> 实体类
  * analysisContext excel信息
 */
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
    int rouNumber = context.readRowHolder().getRowIndex() + 1;
 
    // 这里是因为表头在第二行
    if (rouNumber == 2) {
        // 这里是校验表头
        checkExcelHead(data);
    } else if (rouNumber > 2) {
        // 这里是校验数据
        checkReadData(data);
    }
 
    // 超过150条就先入库
    if (countNum >= batchCount) {
        // 处理excel导出的正确数据
        batchOperateData();
    }
    countNum++;
}
 
/**
 * @author songhc
 * @create 
 * @desc 调用完成监听, 确保数据已全部处理完
 **/
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
    // 这里也要保存数据，确保最后遗留的数据也存储到数据库
    Map<String, Object> objMap = new HashMap<>();
    // 处理excel导出的正确数据
    batchOperateData();
 
    // 错误数据填充oss表格
    Object object = uploadErrorData(errorList, diseaseDto);
 
    objMap.put("errorInfo", object);
    objMap.put("successCount", successCount);
    objMap.put("errorCount", errorCount);
 
    // 错误数据记录redis, 下次使用
    RedisStringHandler.set(String.format(RedisKeyConstants.EXPORT_ERROR_RESULT, "disease" + diseaseDto.getUserId() + "_" + diseaseDto.getRgId() + "_" + diseaseDto.getHosId()), JSONObject.toJSONString(objMap));
}
 
// 这里是封装所有的错误数据
// 包括封装单元格
private Object uploadErrorData (List<OltHosIcdDiseaseDto> errorList, OltHosIcdDiseaseDto dto) {
    Map<Integer, List<Integer>> map = new HashMap<>();
    LinkedList<OltHosIcdDiseaseDto> newErrorList = new LinkedList<>();
    if (CollectionUtils.isNotEmpty(errorList)) {
        for (int i = 0; i < errorList.size(); i++) {
            OltHosIcdDiseaseDto e = errorList.get(i);
            List<Integer> integerList = new ArrayList<>();
            if (e.getErrorReasonMap() != null && !e.getErrorReasonMap().isEmpty()) {
                List<String> reasonList = new ArrayList<>();
                for (Integer key: e.getErrorReasonMap().keySet()) {
                    // 标红单元格
                    integerList.add(key);
                    reasonList.add(e.getErrorReasonMap().get(key));
                }
                map.put(i + 2, integerList);
                e.setErrorReason(String.join("、", reasonList));
            }
            newErrorList.add(e);
        }
    }
 
 
    // 封装导出服务入参
    String uuid = UUIDUtil.create();
    String errorFileName = dto.getHosName() + "(待处理诊断数据)" + dto.getStatDataStr() + ".xlsx";
    SysExportRecordDto sysExportRecordDto = SysExportRecordDto.builder().batchId(uuid).userId(dto.getUserId()).pfCode(dto.getPfCode())
            .source(dto.getSource()).fileName(errorFileName).creator(dto.getCreator()).operator(dto.getCreator()).build();
    // 创建导出记录
    QueueHandler.createTaskRecord(sysExportRecordDto);
    // 获取url
    // 伪代码
    String fileName = "aaa.xlsx";
    String BUCKET_NAME = "bbb";
    String fileUrl = String.format(OssClientConfig.OSS_REAL_PATH, BUCKET_NAME,
            UploadFileType.getFolderByType(UploadFileType.REPORT)).concat(fileName);
    // 加入异步线程任务
    this.exportTaskHandler.exportIcdErrorDiseaseData(OltErrorResult.builder().map(map).errorList(newErrorList)
                    .fileName(errorFileName).source(dto.getSource()).build(),
            uuid, errorFileName, fileUrl);
    // 构建返回队列信息
    return QueueHandler.buildQueueInfo(sysExportRecordDto);
}
 
private void batchOperateData() {
    checkErrorExcelList(tempErrorList, icdCodeList);
    checkSuccessExcelList(successList, tempErrorList, icdCodeList);
 
    // 将临时错误数据存储到所有错误数据列表
    this.errorList.addAll(tempErrorList);
    // 清理list
    this.successList.clear();
    this.tempErrorList.clear();
    this.countNum = 0;
}
 
private void checkExcelHead(Map<Integer, String> data) {
    boolean templateFlag = true;
    // 第二行  校验excel标题
    try {
        String diseaseCategoryStr = data.get(0);
        if (StringUtils.isBlank(diseaseCategoryStr) || !"诊eee（必填）".equals(diseaseCategoryStr)) {
            templateFlag = false;
        }
    } catch (Exception e) {
        templateFlag = false;
    }
    try {
        String icdNameStr = data.get(1);
        if (StringUtils.isBlank(icdNameStr) || !"医vv称（必填）".equals(icdNameStr)) {
            templateFlag = false;
        }
    } catch (Exception e) {
        templateFlag = false;
    }
    try {
        String icdCodeStr = data.get(2);
        if (StringUtils.isBlank(icdCodeStr) || !"医aa（必填）".equals(icdCodeStr)) {
            templateFlag = false;
        }
    } catch (Exception e) {
        templateFlag = false;
    }
    if (!templateFlag) {
        throw new PlatException("文件模版不匹配");
    }
}
 
private void checkReadData(Map<Integer, String> data) {
    // 循环cell
    OltHosIcdDiseaseDto temDisDto = OltHosIcdDiseaseDto.buildDefault();
    temDisDto.setHosId(diseaseDto.getHosId());
    // key为所在的列, value为错误原因
    Map<Integer, String> map = new HashMap<>();
    boolean flag = true;
    try {
        // 解析第二列
        String diseaseCategory = data.get(0);
        if (StringUtils.isBlank(diseaseCategory)) {
            temDisDto.setDiseaseCategoryStr(StringUtils.EMPTY);
            map.put(0, "aaa为空");
            flag = false;
        } else {
            temDisDto.setDiseaseCategoryStr(diseaseCategory);
        }
    } catch (Exception e) {
        temDisDto.setDiseaseCategoryStr(StringUtils.EMPTY);
        map.put(0, "bbb为空");
        flag = false;
    }
 
    try {
        String icdName = data.get(1);
        if (StringUtils.isBlank(icdName)) {
            temDisDto.setIcdName(StringUtils.EMPTY);
            map.put(1, "为空");
            flag = false;
        } else {
            temDisDto.setIcdName(icdName);
        }
    } catch (Exception e) {
        temDisDto.setIcdName(StringUtils.EMPTY);
        map.put(1, "ccc称为空");
        flag = false;
    }
 
    try {
        String icdCode = data.get(2);
        if (StringUtils.isBlank(icdCode)) {
            temDisDto.setIcdCode(StringUtils.EMPTY);
            map.put(2, "ddd为空");
            flag = false;
        } else {
            temDisDto.setIcdCode(icdCode);
        }
    } catch (Exception e) {
        temDisDto.setIcdCode(StringUtils.EMPTY);
        map.put(2, "ddd为空");
        flag = false;
    }
 
 
    try {
        if (!DiseaseCategory.TCM_SYNDROME.getDesc().equals(temDisDto.getDiseaseCategoryStr())) {
            String standardIcdName = data.get(3);
            if (isCfgPrd && StringUtils.isBlank(standardIcdName)) {
                temDisDto.setStandardIcdName(StringUtils.EMPTY);
                map.put(3, "vvv为空");
                flag = false;
            } else {
                temDisDto.setStandardIcdName(standardIcdName);
            }
        }
    } catch (Exception e) {
        temDisDto.setStandardIcdName(StringUtils.EMPTY);
        map.put(3, "vvv为空");
        flag = false;
    }
 
    try {
        if (!DiseaseCategory.TCM_SYNDROME.getDesc().equals(temDisDto.getDiseaseCategoryStr())) {
            String standardIcdCode = data.get(4);
            if (isCfgPrd && StringUtils.isBlank(standardIcdCode)) {
                temDisDto.setStandardIcdCode(StringUtils.EMPTY);
                map.put(4, "eee为空");
                flag = false;
            } else {
                temDisDto.setStandardIcdCode(standardIcdCode);
            }
        }
    } catch (Exception e) {
        temDisDto.setStandardIcdCode(StringUtils.EMPTY);
        map.put(4, "eee为空");
        flag = false;
    }
    temDisDto.setErrorReasonMap(map);
 
 
    // 如果flag为 false 说明数据有问题
    if (!flag) {
        tempErrorList.add(temDisDto);
    } else {
        successList.add(temDisDto);
    }
}
 
private void checkErrorExcelList(List<OltHosIcdDiseaseDto> errorList, List<String> icdCodeList) {
    if (CollectionUtils.isNotEmpty(errorList)) {
        // 错误就往里加, 正确重新定义列表
        errorList.forEach(e -> {
            Map<Integer, String> map = new HashMap<>();
            if (!DiseaseCategory.belongTo(e.getDiseaseCategoryStr())) {
                map.put(0, "aaa不正确");
            } else {
                e.setDiseaseCategory(DiseaseCategory.getCodeByDesc(e.getDiseaseCategoryStr()));
            }
            
            // excel是否存在重复数据
            if (checkRepeatCode.contains(e.getIcdCode())) {
                map.put(2, "bbb重复");
            }
 
            if (CollectionUtils.isNotEmpty(icdCodeList) && icdCodeList.contains(e.getIcdCode())) {
                map.put(2, "ttt重复");
            }
            if (e.getErrorReasonMap() != null && !e.getErrorReasonMap().isEmpty()) {
                Map<Integer, String> errorReasonMap = e.getErrorReasonMap();
                errorReasonMap.putAll(map);
                e.setErrorReasonMap(errorReasonMap);
            }
            errorCount++;
        });
    }
}
 
/**
 * 侵入式给errorList赋值
 * @param list
 * @param errorList
 * @param icdCodeList
 */
private void checkSuccessExcelList(List<OltHosIcdDiseaseDto> list, List<OltHosIcdDiseaseDto> errorList,
                                   List<String> icdCodeList) {
    List<OltHosIcdDiseaseDto> newList = new ArrayList<>();
 
    if (CollectionUtils.isNotEmpty(list)) {
        // 错误就往里加, 正确重新定义列表
        list.forEach(e -> {
            Map<Integer, String> map = new HashMap<>();
            boolean flag = false;
            // 判
            if (!DiseaseCategory.belongTo(e.getDiseaseCategoryStr())) {
                map.put(0, "不正确");
                flag = true;
            } else {
                e.setDiseaseCategory(DiseaseCategory.getCodeByDesc(e.getDiseaseCategoryStr()));
            }
 
            // excel是否存在重复数据
            if (checkRepeatCode.contains(e.getIcdCode())) {
                map.put(2, "重复");
                flag = true;
            } else {
                // 判断诊断编码
                if (CollectionUtils.isNotEmpty(icdCodeList) && icdCodeList.contains(e.getIcdCode())) {
                    map.put(2, "重复");
                    flag = true;
                }
            }
            e.setErrorReasonMap(map);
            if (flag) {
                errorCount++;
                errorList.add(e);
            } else {
                e.setIcdPinyin(HzUtils.getPinyinCap(e.getIcdName(), HzUtils.CaseType.UPPERCASE));
                e.setIcdWb(HzUtils.getWbCap(e.getIcdName(), HzUtils.CaseType.UPPERCASE));
                newList.add(e);
                checkRepeatCode.add(e.getIcdCode());
                successCount++;
            }
        });
    }
 
    // 正确数据入库
    if (CollectionUtils.isNotEmpty(newList)) {
        oltConfigService.batchAddHosIcdDisease(delIcdCodeList, newList);
    }
}

　　其中,导入错误数据用了easyExcel的模版填充方式, 模版存于oss上

/**
 * @author songhc
 * @create
 * @desc 导出错误数据
 **/
@Async
public void exportIcdErrorDiseaseData(OltErrorResult dto, String fileBatch, String fileName, String fileUrl) {
    Map<Integer, List> map = new HashMap<>();
    map.put(0, dto.getErrorList());
    Map<Integer, Map<Integer, List<Integer>>> styleMap = new HashMap<>();
    styleMap.put(0, dto.getMap());
    ExportExistHandler.exportExistTemplateData(map, styleMap, fileBatch, fileName, fileUrl);
}

　　接下来就是填充错误模版的实现

/**
 * @param errorMap key为sheetNo, value为填充的数据
 * @param styleMap key为sheetNo, value为错误数据坐标
 * @param fileBatch 批次号
 * @param fileName 文件名
 * @param fileUrl 文件路径
 * @description 导出服务封装方法(无需分页查询, 数据为动态传入)
 * @className exportNoModelData
 */
public static void exportExistTemplateData(Map<Integer, List> errorMap, Map<Integer, Map<Integer, List<Integer>>> styleMap, String fileBatch, String fileName, String fileUrl) {
    String ossFileName = fileName.substring(0, fileName.lastIndexOf('.'))
            .concat("-").concat(LocalDateTime.now()
                    .format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"))).concat(fileName.substring(fileName.lastIndexOf('.')));
    InputStream inputStream = HttpUtil.io(HttpUtil.Atom.builder().url(fileUrl).build());
    if (null == inputStream) {
        return;
    }
    String localFileName = String.format(TaskNoteHandler.staticExportConfig.getExportPath(), ossFileName);
    ExcelWriter excelWriter = null;
    int resultCount = 0;
    try {
        if (errorMap != null && !errorMap.isEmpty()) {
            excelWriter = EasyExcel.write(localFileName)
                        .withTemplate(inputStream)
                        .build();
            // for循环是一个excel可能有多个sheet的兼容写法
            for (Integer i: errorMap.keySet()) {
                // 这里使用easyExcel的 registerWriteHandler 方法, 自定义CellColorSheetWriteHandler实现, 给每一个单元格填充颜色
                WriteSheet writeSheet = EasyExcel.writerSheet(i).registerWriteHandler(new CellColorSheetWriteHandler(styleMap.get(i),
                        IndexedColors.RED1.getIndex())).build();
                excelWriter.fill(errorMap.get(i), writeSheet);
            }
        }
    } catch (Exception e){
        LoggerUtil.error(LOGGER, "文件写入异常，error{0}", e);
        // 文件导出失败
        TaskNoteHandler.doUploadFailed(fileBatch, resultCount);
        return;
    } finally {
        // 关闭流
        if (excelWriter != null) {
            excelWriter.finish();
        }
    }
    // 1、上传文件（多种方案）；2、更新记录
    TaskNoteHandler.doUploadAndNote(fileBatch, ossFileName, localFileName, resultCount);
}

　　

/**
 * @description 自定义单元格格式拦截器
 * @className CellColorSheetWriteHandler
 * @package 
 * @Author songhc
 */
public class CellColorSheetWriteHandler implements CellWriteHandler {
 
    /**
     * map
     * key：第i行
     * value：第i行中单元格索引集合
     */
    private Map<Integer, List<Integer>> map;
    /**
     * 颜色
     */
    private Short colorIndex;
 
    /**
     * 有参构造
     */
    public CellColorSheetWriteHandler(Map<Integer, List<Integer>> map, Short colorIndex) {
        this.map = map;
        this.colorIndex = colorIndex;
    }
 
    /**
     * 无参构造
     */
    public CellColorSheetWriteHandler() {
    }
 
    @Override
    public void beforeCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Row row, Head head, Integer integer, Integer integer1, Boolean aBoolean) {
    }
 
    /**
     * 在单元格创建后调用
     */
    @Override
    public void afterCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Cell cell, Head head, Integer integer, Boolean aBoolean) {
    }
 
    @Override
    public void afterCellDataConverted(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, CellData cellData, Cell cell, Head head, Integer integer, Boolean aBoolean) {
    }
 
    /**
     * 在单元上的所有操作完成后调用
     */
    @Override
    public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List<CellData> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) {
        //当前行的第i列
        int i = cell.getColumnIndex();
        // 根据单元格获取workbook
        Workbook workbook = cell.getSheet().getWorkbook();
        //不处理第一行
        if (0 != cell.getRowIndex()) {
            List<Integer> integerList = map.get(cell.getRowIndex());
            // 自定义单元格样式 
            if (CollectionUtils.isNotEmpty(integerList)) {
                if (integerList.contains(i)) {
                    // 单元格策略
                    WriteCellStyle contentWriteCellStyle = new WriteCellStyle();
                    // 设置背景颜色白色
                    contentWriteCellStyle.setFillForegroundColor(colorIndex);
                    // 设置垂直居中为居中对齐
                     contentWriteCellStyle.setVerticalAlignment(VerticalAlignment.CENTER);
                    // 设置左右对齐为中央对齐
                     contentWriteCellStyle.setHorizontalAlignment(HorizontalAlignment.RIGHT);
                    // 设置单元格上下左右边框为细边框
                      contentWriteCellStyle.setBorderBottom(BorderStyle.THIN);
                      contentWriteCellStyle.setBorderLeft(BorderStyle.THIN);
                     contentWriteCellStyle.setBorderRight(BorderStyle.THIN);
                     contentWriteCellStyle.setBorderTop(BorderStyle.THIN);
                    // 创建字体实例
                     WriteFont cellWriteFont = new WriteFont();
                    // 设置字体大小
                     cellWriteFont.setFontName("宋体");
                     cellWriteFont.setFontHeightInPoints((short) 10);
                    //设置字体颜色
                    // cellWriteFont.setColor(IndexedColors.BLACK1.getIndex());
                    //单元格颜色
                    //contentWriteCellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
                    contentWriteCellStyle.setWriteFont(cellWriteFont);
                    CellStyle cellStyle = StyleUtil.buildHeadCellStyle(workbook, contentWriteCellStyle);
                    //设置当前行第i列的样式
                    cell.getRow().getCell(i).setCellStyle(cellStyle);
                }
            }
        }
    }
}

　　对于一个excel多sheet, 操作也是一样

// 不同的是这里可以定义多个监听器
// readSheet(0)  —-> 这里的数据代表sheet的位置
OltDrugFrequencyListener oltDrugFrequencyListener = new OltDrugFrequencyListener(isCfgPrd, dfCodeList, frequencyDto, oltConfigService);
OltDrugUsageListener oltDrugUsageListener = new OltDrugUsageListener(isCfgPrd, dUCodeList, OltDrugUsageDto.builder().hosId(frequencyDto.getHosId()).build(), oltConfigService);
OltDrugDurationListener oltDrugDurationListener = new OltDrugDurationListener(durationCodeList, OltDrugDurationDefDto.builder().hosId(frequencyDto.getHosId()).build(), oltConfigService);
 
ReadSheet readSheet = EasyExcel.readSheet(0).registerReadListener(oltDrugFrequencyListener).build();
ReadSheet readSheet2 = EasyExcel.readSheet(2).registerReadListener(oltDrugUsageListener).build();
ReadSheet readSheet4 = EasyExcel.readSheet(4).registerReadListener(oltDrugDurationListener).build();
excelReader.read(readSheet, readSheet2, readSheet4);

　　经过测试, 该方法导入2W条数据差不多需要10秒, 也不会影响内存