package cn.iocoder.yudao.module.system.service.storage;
|
|
import cn.iocoder.yudao.module.system.dal.dataobject.storage.SystemStorageBlobDO;
|
import cn.iocoder.yudao.module.system.dal.mysql.storage.SystemStorageBlobMapper;
|
import cn.iocoder.yudao.module.system.framework.storage.config.StorageProperties;
|
import lombok.RequiredArgsConstructor;
|
import lombok.extern.slf4j.Slf4j;
|
import org.springframework.scheduling.annotation.Scheduled;
|
import org.springframework.stereotype.Component;
|
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.StringUtils;
|
|
import java.io.File;
|
import java.util.ArrayDeque;
|
import java.util.ArrayList;
|
import java.util.Deque;
|
import java.util.HashSet;
|
import java.util.List;
|
import java.util.Set;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
|
/**
|
* 清理无效文件定时任务
|
* 每月1号凌晨2点执行一次
|
*
|
* 清理内容：
|
* 1. 删除 system_storage_blob 中未被 system_storage_attachment 关联的记录及其文件
|
* 2. 删除磁盘上不存在于 system_storage_blob.uid_filename 的文件
|
*/
|
@Slf4j
|
@Component
|
@RequiredArgsConstructor
|
public class StorageBlobCleanupTask {
|
|
private static final int DB_BATCH_SIZE = 500;
|
private static final int FILE_NAME_BATCH_SIZE = 1000;
|
|
private final SystemStorageBlobMapper storageBlobMapper;
|
private final StorageProperties properties;
|
|
private final AtomicBoolean running = new AtomicBoolean(false);
|
|
/**
|
* 每月 1 号凌晨 2 点执行一次
|
*/
|
@Scheduled(cron = "0 0 2 1 * ?")
|
public void cleanupUnusedStorageFiles() {
|
if (!running.compareAndSet(false, true)) {
|
log.warn("文件清理任务正在执行，本次跳过");
|
return;
|
}
|
|
long start = System.currentTimeMillis();
|
log.info("文件清理任务开始执行，根目录：{}", properties.getPath());
|
try {
|
int removedBlobCount = cleanupOrphanStorageBlobs();
|
int removedDiskFileCount = cleanupOrphanDiskFiles();
|
long cost = System.currentTimeMillis() - start;
|
log.info("文件清理任务执行完成，删除孤儿 blob 记录：{}，删除磁盘无效文件：{}，耗时：{} ms",
|
removedBlobCount, removedDiskFileCount, cost);
|
} catch (Exception e) {
|
log.error("文件清理任务执行失败", e);
|
} finally {
|
running.set(false);
|
}
|
}
|
|
private int cleanupOrphanStorageBlobs() {
|
long lastId = 0L;
|
int removedCount = 0;
|
|
while (true) {
|
List<SystemStorageBlobDO> orphanBlobs = storageBlobMapper.selectOrphanBlobsByIdRange(lastId, DB_BATCH_SIZE);
|
if (CollectionUtils.isEmpty(orphanBlobs)) {
|
break;
|
}
|
|
List<Long> ids = new ArrayList<>(orphanBlobs.size());
|
for (SystemStorageBlobDO blob : orphanBlobs) {
|
ids.add(blob.getId());
|
deleteBlobFiles(blob);
|
}
|
storageBlobMapper.deleteByIds(ids);
|
removedCount += ids.size();
|
lastId = orphanBlobs.get(orphanBlobs.size() - 1).getId();
|
|
log.info("已删除一批孤儿 blob，batchSize={}，lastId={}", ids.size(), lastId);
|
}
|
|
return removedCount;
|
}
|
|
private int cleanupOrphanDiskFiles() {
|
File rootDirectory = new File(properties.getPath());
|
if (!rootDirectory.exists() || !rootDirectory.isDirectory()) {
|
log.warn("文件根目录不存在或不是目录，跳过磁盘清理：{}", properties.getPath());
|
return 0;
|
}
|
|
int deletedCount = 0;
|
Deque<File> directories = new ArrayDeque<>();
|
directories.push(rootDirectory);
|
|
while (!directories.isEmpty()) {
|
File currentDirectory = directories.pop();
|
File[] children = currentDirectory.listFiles();
|
if (children == null || children.length == 0) {
|
continue;
|
}
|
|
List<File> filesInDirectory = new ArrayList<>();
|
for (File child : children) {
|
if (child.isDirectory()) {
|
directories.push(child);
|
} else if (child.isFile()) {
|
filesInDirectory.add(child);
|
}
|
}
|
|
deletedCount += cleanupFilesInDirectory(filesInDirectory);
|
}
|
|
return deletedCount;
|
}
|
|
private int cleanupFilesInDirectory(List<File> filesInDirectory) {
|
if (CollectionUtils.isEmpty(filesInDirectory)) {
|
return 0;
|
}
|
|
int deletedCount = 0;
|
for (int start = 0; start < filesInDirectory.size(); start += FILE_NAME_BATCH_SIZE) {
|
int end = Math.min(start + FILE_NAME_BATCH_SIZE, filesInDirectory.size());
|
List<File> batchFiles = filesInDirectory.subList(start, end);
|
List<String> fileNames = new ArrayList<>(batchFiles.size());
|
for (File file : batchFiles) {
|
fileNames.add(file.getName());
|
}
|
|
Set<String> existingFileNames = new HashSet<>(storageBlobMapper.selectExistingUidFilenames(fileNames));
|
for (File file : batchFiles) {
|
// 跳过压缩文件（thumb_ 前缀），它们随原文件一起管理
|
if (file.getName().startsWith("thumb_")) {
|
continue;
|
}
|
if (!existingFileNames.contains(file.getName()) && safeDelete(file)) {
|
deletedCount++;
|
}
|
}
|
}
|
return deletedCount;
|
}
|
|
private void deleteBlobFiles(SystemStorageBlobDO blob) {
|
File originalFile = resolveBlobFile(blob);
|
safeDelete(originalFile);
|
|
File compressedFile = resolveCompressedFile(originalFile);
|
safeDelete(compressedFile);
|
}
|
|
private File resolveBlobFile(SystemStorageBlobDO blob) {
|
String basePath = properties.getPath();
|
if (!StringUtils.hasText(blob.getPath())) {
|
return new File(basePath, blob.getUidFilename());
|
}
|
return new File(new File(basePath, blob.getPath()), blob.getUidFilename());
|
}
|
|
private File resolveCompressedFile(File originalFile) {
|
if (originalFile == null) {
|
return null;
|
}
|
File parent = originalFile.getParentFile();
|
if (parent == null) {
|
return null;
|
}
|
return new File(parent, "thumb_" + originalFile.getName());
|
}
|
|
private boolean safeDelete(File file) {
|
if (file == null || !file.exists() || !file.isFile()) {
|
return false;
|
}
|
if (file.delete()) {
|
return true;
|
}
|
log.warn("删除文件失败：{}", file.getAbsolutePath());
|
return false;
|
}
|
|
}
|
