使用Hutool读取大Excel
改造后可以在数秒内读取百万级别量的excel数据,注意:csv不行,先转成excel再读取
public class MyBigExcelUtil {
/**
* 储存表头
*/
private static List<Object> headLine;
/**
* 储存返回数据
*/
private static List<Map<String, Object>> datas = new ArrayList<>();
/**
* 储存表头行号,默认是第0行
*/
private static AtomicInteger headNo = new AtomicInteger(0);
/**
* 设置 表头 行号
*/
public static void setHeadNo(int headNum) {
headNo.set(headNum);
}
/**
* @param pathAndName 文件路径
* @param idOrRidOrSheetName Excel中的sheet id或者rid编号或sheet名称,rid必须加rId前缀,例如rId1,如果为-1处理所有编号的sheet
*/
public static List<Map<String, Object>> readBigExcel(String pathAndName, int idOrRidOrSheetName) {
datas.clear();
Excel07SaxReader reader = new Excel07SaxReader(new MyRowHandler());
reader.read(pathAndName, idOrRidOrSheetName);
Console.log("【{}】 读取完成 ... ", pathAndName);
return datas;
}
/**
* @param pathAndName 文件路径
* @param idOrRidOrSheetName Excel中的sheet id或者rid编号或sheet名称,rid必须加rId前缀,例如rId1,如果为-1处理所有编号的sheet
* @param beanType 类类型
* @param <T> 返回值为List<T>
* @return
*/
public static <T> List<T> read(String pathAndName, int idOrRidOrSheetName, Class<T> beanType) {
return read(pathAndName, idOrRidOrSheetName, beanType, Collections.EMPTY_MAP);
}
/**
* @param pathAndName 文件路径
* @param idOrRidOrSheetName Excel中的sheet id或者rid编号或sheet名称,rid必须加rId前缀,例如rId1,如果为-1处理所有编号的sheet, 一般传0
* @param beanType 类类型 映射成的实体类
* @param fieldMapping T类型对应的别名,即类类型对应的 excel 的关系 Map<excel列的名字, java中定义实体的字段名>
* @param <T> 返回值为List<T>
* @return
*/
public static <T> List<T> read(String pathAndName, int idOrRidOrSheetName, Class<T> beanType, Map<String, String> fieldMapping) {
CopyOptions copyOptions = CopyOptions.create();
if (CollUtil.isNotEmpty(fieldMapping)) {
copyOptions.setFieldMapping(fieldMapping);
}
readBigExcel(pathAndName, idOrRidOrSheetName);
List<T> datalist = new ArrayList<>();
for (Map<String, Object> data : datas) {
T t = ReflectUtil.newInstanceIfPossible(beanType);
datalist.add(BeanUtil.fillBeanWithMap(data, t, copyOptions));
}
return datalist;
}
/**
* 行数据处理
*/
private static class MyRowHandler implements RowHandler {
@Override
public void handle(int sheetIndex, long rowIndex, List<Object> rowList) {
//Console.log("[{}] [{}] {}", sheetIndex, rowIndex, rowList);
if (rowIndex < headNo.get()) {
return;
} else if (rowIndex == headNo.get()) {
headLine = rowList;
} else {
Map<String, Object> tMap = new HashMap<>(rowList.size());
for (int i = 0; i < headLine.size(); i++) {
Object key = headLine.get(i);
Object val = rowList.get(i);
tMap.put(key == null ? "" : key.toString(), val == null ? "" : val);
}
datas.add(tMap);
}
}
}
}