当前位置：首页 > article >正文

定时任务删除MongoDB历史数据

article 2024/11/30 16:58:27

前言

MongoDB数据过多，导致存储成本飙升，为了降低成本，需要将历史数据删除。

删除逻辑

添加配置文件控制删除逻辑

syncconfig:
  deleteMongoConfig:
    #同步状态，true同步，false不同步
    syncStatus: true
    #删除数据的时间，一个月前
    deleteDays: 31
    #分布式锁key
    lockRedisKey: order.sync.deleteMongo
    #每次删除数量
    limit: 2

配置类

@ConfigurationProperties(prefix = "syncconfig")
@Configuration
@RefreshScope
@Data
public class SyncConfig {
    private DeleteMongoDto deleteMongoConfig;

    @Data
    @NoArgsConstructor
    @AllArgsConstructor
    public static class DeleteMongoDto {
        /**
         * 同步状态，true同步，false不同步
         */
        Boolean syncStatus = false;
        /**
         * 删除的天数（createTime）
         */
        Integer deleteDays = 186;
        /**
         * locakRedisKey，用于做分布式锁，防止定时任务执行时间较长，两个任务重叠执行
         */
        String lockRedisKey;
        /**
         * 每次数量
         */
        Integer limit = 1000;
    }
}

核心代码

    @PostMapping("/deleteMongo")
    public BaseResponse<String> deleteMongo() {
        return orderService.deleteMongo();
    }

 @Override
    public BaseResponse<String> deleteMongo() {
        //获取配置信息
        SyncConfig.DeleteMongoDto deleteMongoDto = syncConfig.getDeleteMongoConfig();
        //判断同步是否开启
        if (deleteMongoDto == null || !deleteMongoDto.getSyncStatus()) {
            log.info("deleteMongosyncStatus=false,not sync");
            return ResponseUtils.success("deleteMongosyncStatus=false,not sync");
        }
        Boolean lockStatus = false;
        try {
            lockStatus = redisLockService.acquireLock(deleteMongoDto.getLockRedisKey());
            if (!lockStatus) {
                log.info("deleteMongo get lock fail,not sync");
                return ResponseUtils.fail("deleteMongo get lock fail,not sync");
            }
            //根据时间获取orderId
            Long orderId = getOrderId(deleteMongoDto.getDeleteDays());
            Query query = new Query(Criteria.where("orderId").lt(orderId));
            query.limit(deleteMongoDto.getLimit());
            // 使用投影来只选择orderId字段
            query.fields().include("orderId");
            // 这里不用分页是因为分页会查询total效率较低，数据量大的时候耗费资源
            /*Pageable pageable = PageRequest.of(0, deleteMongoDto.getLimit());
            Page<orderDao> page = orderDao.findByOrderIdLessThan(orderId, pageable);
            List<Order> list = page.getContent();*/
            List<Order> list=mongoTemplate.find(query,Order.class);
            for (Orderitem :
                    list) {
                orderDao.deleteById(item.getOrderId());
            }
            int count = 0;
            if (CollectionUtils.isNotEmpty(list)) {
                count = list.size();
            }
            return ResponseUtils.success("deleteMongo success,count:" + count);
        } catch (Exception ex) {
            log.error("deleteMongo Exception：" + ex.getMessage());
            return ResponseUtils.fail(ex.getMessage());
        } finally {
            //释放分布式锁
            if (lockStatus) {
                redisLockService.releaseLock(deleteMongoDto.getLockRedisKey());
            }
        }
    }
    //生成半年前的orderId
    private Long getOrderId(Integer deleteDays) {
        LocalDateTime deleteTime = LocalDateTime.now().minusDays(deleteDays);
        // 将LocalDateTime转换为毫秒值
        ZonedDateTime utcZonedDateTime = deleteTime.atZone(ZoneId.systemDefault());
        long milliseconds = utcZonedDateTime.toInstant().toEpochMilli();
        Long orderId = (milliseconds / 1000) << 32;
        return orderId;
    }