增加定时任务，一个月清除一次无用文件

chenhj

7 天以前 e7efe7784535a77a21347c0ca142056c16a94902

增加定时任务，一个月清除一次无用文件

 src/main/java/com/ruoyi/basic/mapper/StorageBlobMapper.java

@@ -3,6 +3,7 @@
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.ruoyi.basic.pojo.StorageBlob;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;

/**
 * <p>
@@ -15,4 +16,9 @@
@Mapper
public interface StorageBlobMapper extends BaseMapper<StorageBlob> {

    java.util.List<StorageBlob> selectOrphanBlobsByIdRange(@Param("lastId") long lastId, @Param("limit") int limit);

    int deleteByIdList(@Param("ids") java.util.List<Long> ids);

    java.util.List<String> selectExistingUidFilenames(@Param("fileNames") java.util.List<String> fileNames);
}

 src/main/java/com/ruoyi/basic/task/StorageBlobCleanupTask.java

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,184 @@
package com.ruoyi.basic.task;

import com.ruoyi.basic.mapper.StorageBlobMapper;
import com.ruoyi.basic.pojo.StorageBlob;
import com.ruoyi.common.config.FileProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;

import java.io.File;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * æ¸…理无效文件定时任务。
 */
@Slf4j
@Component
@RequiredArgsConstructor
public class StorageBlobCleanupTask {

    private static final int DB_BATCH_SIZE = 500;
    private static final int FILE_NAME_BATCH_SIZE = 1000;

    private final StorageBlobMapper storageBlobMapper;
    private final FileProperties fileProperties;

    private final AtomicBoolean running = new AtomicBoolean(false);

    /**
     * æ¯æœˆ 1 å·å‡Œæ™¨ 2 ç‚¹æ‰§è¡Œä¸€æ¬¡ï¼š
     * 1. åˆ é™¤ storage_blob ä¸­æœªè¢« storage_attachment å…³è”的记录及其文件
     * 2. åˆ é™¤ç£ç›˜ä¸Šä¸å­˜åœ¨äºŽ storage_blob.uid_filename çš„文件
     */
    @Scheduled(cron = "0 0 2 1 * ?")
    public void cleanupUnusedStorageFiles() {
        if (!running.compareAndSet(false, true)) {
            log.warn("文件清理任务正在执行，本次跳过");
            return;
        }

        long start = System.currentTimeMillis();
        log.info("文件清理任务开始执行，根目录：{}", fileProperties.getPath());
        try {
            int removedBlobCount = cleanupOrphanStorageBlobs();
            int removedDiskFileCount = cleanupOrphanDiskFiles();
            long cost = System.currentTimeMillis() - start;
            log.info("文件清理任务执行完成，删除孤儿 blob è®°å½•ï¼š{}，删除磁盘无效文件：{}，耗时：{} ms",
                    removedBlobCount, removedDiskFileCount, cost);
        } catch (Exception e) {
            log.error("文件清理任务执行失败", e);
        } finally {
            running.set(false);
        }
    }

    private int cleanupOrphanStorageBlobs() {
        long lastId = 0L;
        int removedCount = 0;

        while (true) {
            List<StorageBlob> orphanBlobs = storageBlobMapper.selectOrphanBlobsByIdRange(lastId, DB_BATCH_SIZE);
            if (CollectionUtils.isEmpty(orphanBlobs)) {
                break;
            }

            List<Long> ids = new ArrayList<>(orphanBlobs.size());
            for (StorageBlob storageBlob : orphanBlobs) {
                ids.add(storageBlob.getId());
                deleteBlobFiles(storageBlob);
            }
            storageBlobMapper.deleteByIdList(ids);
            removedCount += ids.size();
            lastId = orphanBlobs.get(orphanBlobs.size() - 1).getId();

            log.info("已删除一批孤儿 blob，batchSize={}，lastId={}", ids.size(), lastId);
        }

        return removedCount;
    }

    private int cleanupOrphanDiskFiles() {
        File rootDirectory = new File(fileProperties.getPath());
        if (!rootDirectory.exists() || !rootDirectory.isDirectory()) {
            log.warn("文件根目录不存在或不是目录，跳过磁盘清理：{}", fileProperties.getPath());
            return 0;
        }

        int deletedCount = 0;
        Deque<File> directories = new ArrayDeque<>();
        directories.push(rootDirectory);

        while (!directories.isEmpty()) {
            File currentDirectory = directories.pop();
            File[] children = currentDirectory.listFiles();
            if (children == null || children.length == 0) {
                continue;
            }

            List<File> filesInDirectory = new ArrayList<>();
            for (File child : children) {
                if (child.isDirectory()) {
                    directories.push(child);
                } else if (child.isFile()) {
                    filesInDirectory.add(child);
                }
            }

            deletedCount += cleanupFilesInDirectory(filesInDirectory);
        }

        return deletedCount;
    }

    private int cleanupFilesInDirectory(List<File> filesInDirectory) {
        if (CollectionUtils.isEmpty(filesInDirectory)) {
            return 0;
        }

        int deletedCount = 0;
        for (int start = 0; start < filesInDirectory.size(); start += FILE_NAME_BATCH_SIZE) {
            int end = Math.min(start + FILE_NAME_BATCH_SIZE, filesInDirectory.size());
            List<File> batchFiles = filesInDirectory.subList(start, end);
            List<String> fileNames = new ArrayList<>(batchFiles.size());
            for (File file : batchFiles) {
                fileNames.add(file.getName());
            }

            Set<String> existingFileNames = new HashSet<>(storageBlobMapper.selectExistingUidFilenames(fileNames));
            for (File file : batchFiles) {
                if (!existingFileNames.contains(file.getName()) && safeDelete(file)) {
                    deletedCount++;
                }
            }
        }
        return deletedCount;
    }

    private void deleteBlobFiles(StorageBlob storageBlob) {
        File originalFile = resolveBlobFile(storageBlob);
        safeDelete(originalFile);

        File compressedFile = resolveCompressedFile(originalFile);
        safeDelete(compressedFile);
    }

    private File resolveBlobFile(StorageBlob storageBlob) {
        String basePath = fileProperties.getPath();
        if (!StringUtils.hasText(storageBlob.getPath())) {
            return new File(basePath, storageBlob.getUidFilename());
        }
        return new File(new File(basePath, storageBlob.getPath()), storageBlob.getUidFilename());
    }

    private File resolveCompressedFile(File originalFile) {
        if (originalFile == null) {
            return null;
        }
        File parent = originalFile.getParentFile();
        if (parent == null) {
            return null;
        }
        return new File(parent, "thumb_" + originalFile.getName());
    }

    private boolean safeDelete(File file) {
        if (file == null || !file.exists() || !file.isFile()) {
            return false;
        }
        if (file.delete()) {
            return true;
        }
        log.warn("删除文件失败：{}", file.getAbsolutePath());
        return false;
    }
}

 src/main/resources/mapper/basic/StorageAttachmentMapper.xml

@@ -10,13 +10,9 @@
                    <result column="deleted" property="deleted" />
                    <result column="record_type" property="recordType" />
                    <result column="record_id" property="recordId" />
                    <result column="name" property="name" />
                    <result column="application" property="application" />
                    <result column="storage_blob_id" property="storageBlobId" />
        </resultMap>

        <!-- é€šç”¨æŸ¥è¯¢ç»“果列 -->
        <sql id="Base_Column_List">
            id, create_time, update_time, deleted, record_type, record_id, name, storage_blob_id
        </sql>

</mapper>
</mapper>

 src/main/resources/mapper/basic/StorageBlobMapper.xml

@@ -2,21 +2,50 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.basic.mapper.StorageBlobMapper">

        <!-- é€šç”¨æŸ¥è¯¢æ˜ å°„结果 -->
        <resultMap id="BaseResultMap" type="com.ruoyi.basic.pojo.StorageBlob">
                    <id column="id" property="id" />
                    <result column="create_time" property="createTime" />
                    <result column="key" property="key" />
                    <result column="content_type" property="contentType" />
                    <result column="original_filename" property="originalFilename" />
                    <result column="bucket_filename" property="bucketFilename" />
                    <result column="bucket_name" property="bucketName" />
                    <result column="byte_size" property="byteSize" />
        </resultMap>
    <!-- é€šç”¨æŸ¥è¯¢æ˜ å°„结果 -->
    <resultMap id="BaseResultMap" type="com.ruoyi.basic.pojo.StorageBlob">
        <id column="id" property="id"/>
        <result column="resource_key" property="resourceKey"/>
        <result column="content_type" property="contentType"/>
        <result column="original_filename" property="originalFilename"/>
        <result column="uid_filename" property="uidFilename"/>
        <result column="byte_size" property="byteSize"/>
        <result column="path" property="path"/>
    </resultMap>

        <!-- é€šç”¨æŸ¥è¯¢ç»“果列 -->
        <sql id="Base_Column_List">
            id, create_time, key, content_type, original_filename,bucket_filename,bucket_name,  byte_size
        </sql>
    <!-- é€šç”¨æŸ¥è¯¢ç»“果列 -->
    <sql id="Base_Column_List">
        id, resource_key, content_type, original_filename, uid_filename, byte_size, path
    </sql>

</mapper>
    <select id="selectOrphanBlobsByIdRange" resultMap="BaseResultMap">
        SELECT
        <include refid="Base_Column_List"/>
        FROM storage_blob sb
        LEFT JOIN storage_attachment sa
        ON sa.storage_blob_id = sb.id
        AND sa.deleted = 0
        WHERE sb.id <![CDATA[>]]> #{lastId}
        AND sa.id IS NULL
        ORDER BY sb.id ASC
        LIMIT #{limit}
    </select>

    <delete id="deleteByIdList">
        DELETE FROM storage_blob
        WHERE id IN
        <foreach collection="ids" item="id" open="(" separator="," close=")">
            #{id}
        </foreach>
    </delete>

    <select id="selectExistingUidFilenames" resultType="java.lang.String">
        SELECT uid_filename
        FROM storage_blob
        WHERE uid_filename IN
        <foreach collection="fileNames" item="fileName" open="(" separator="," close=")">
            #{fileName}
        </foreach>
    </select>

</mapper>