feat(ai): 优化AI知识文档处理功能

云

2026-08-05 3ca1cefd9bed863131ee3f684ff127bd7ac877aa

 yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java

@@ -28,6 +28,10 @@
import org.apache.tika.parser.Parser;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.hwpf.HWPFDocument;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.xml.sax.ContentHandler;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
@@ -57,6 +61,8 @@
    private AiKnowledgeSegmentService segmentService;
    @Resource
    private AiModelService modelService;
    @Resource
    private cn.iocoder.yudao.module.system.service.storage.SystemStorageBlobService storageBlobService;

    @Override
    @Transactional(rollbackFor = Exception.class)
@@ -113,11 +119,28 @@
        }
    }

    /**
     * 从 storage-blob URL 解析出 fileName 和 token
     */
    private static java.util.regex.Pattern STORAGE_URL_PATTERN =
            java.util.regex.Pattern.compile("/storage-blob/(?:preview|download)/([^?]+)\\?.*token=([^&]+)");

    private byte[] downloadFile(String fileUrl) {
        // 优先通过 storage-blob 本地读取，避免 HTTP 认证问题
        var matcher = STORAGE_URL_PATTERN.matcher(fileUrl);
        if (matcher.find()) {
            String fileName = matcher.group(1);
            String token = matcher.group(2);
            try {
                java.io.File file = storageBlobService.getFileByToken(fileName, token);
                return java.nio.file.Files.readAllBytes(file.toPath());
            } catch (Exception e) {
                log.warn("本地读取存储文件失败，回退 HTTP 下载: {}", e.getMessage());
            }
        }
        // 回退 HTTP 下载
        try {
            // 手动编码 URL 路径中的非 ASCII 字符，避免 URI 构造函数报错
            String encodedUrl = encodeUrlPath(fileUrl);
            URL url = URL.of(new URI(encodedUrl), null);
            URL url = URL.of(new URI(encodeUrlPath(fileUrl)), null);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setConnectTimeout(30000);
            conn.setReadTimeout(60000);
@@ -184,7 +207,36 @@
            if (ArrayUtil.contains(new String[]{"txt", "md", "json", "xml", "csv", "yaml", "yml"}, ext)) {
                return new String(fileBytes, StandardCharsets.UTF_8);
            }
            // 使用 AutoDetectParser + EmbeddedDocumentExtractor 跳过嵌入图片，避免提取到二进制乱码
            // .docx 优先使用 POI 提取，中文支持更好
            if ("docx".equals(ext)) {
                try (XWPFDocument doc = new XWPFDocument(new java.io.ByteArrayInputStream(fileBytes));
                     XWPFWordExtractor extractor = new XWPFWordExtractor(doc)) {
                    String text = extractor.getText();
                    if (StrUtil.isNotEmpty(text)) return text.trim();
                } catch (Exception e) {
                    log.warn("POI 解析 docx 失败，回退 Tika: {}", e.getMessage());
                }
            }
            // .doc 优先使用 POI 提取
            if ("doc".equals(ext)) {
                try (HWPFDocument doc = new HWPFDocument(new java.io.ByteArrayInputStream(fileBytes));
                     WordExtractor extractor = new WordExtractor(doc)) {
                    String text = extractor.getText();
                    if (StrUtil.isNotEmpty(text)) return text.trim();
                } catch (Exception e) {
                    log.warn("POI 解析 doc 失败，回退 Tika: {}", e.getMessage());
                }
            }
            // 其他格式或 POI 失败时回退 Tika
            return extractTextWithTika(fileBytes);
        } catch (Exception e) {
            log.warn("文档内容提取失败，尝试 UTF-8 文本读取: {}", e.getMessage());
            return "";
        }
    }

    private String extractTextWithTika(byte[] fileBytes) {
        try {
            Parser parser = new AutoDetectParser();
            ParseContext context = new ParseContext();
            context.set(EmbeddedDocumentExtractor.class, new EmbeddedDocumentExtractor() {
@@ -203,7 +255,7 @@
            }
            return handler.toString().trim();
        } catch (Exception e) {
            log.warn("Tika 解析文档失败，尝试 UTF-8 文本读取: {}", e.getMessage());
            log.warn("Tika 解析失败: {}", e.getMessage());
            return "";
        }
    }