From fb5dcaeb2ab91d0f9ffea26fd15ddcbbe5d36bb9 Mon Sep 17 00:00:00 2001
From: 云 <2163098428@qq.com>
Date: 星期五, 31 七月 2026 17:57:27 +0800
Subject: [PATCH] feat(aftersales): 售后工单新增问题类型和严重程度字段
---
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java | 772 +++++++++++++++++++++++-----------------------------------
1 files changed, 308 insertions(+), 464 deletions(-)
diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
index 45ef125..544bca4 100644
--- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
+++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
@@ -1,509 +1,353 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.hutool.core.collection.CollUtil;
-import cn.hutool.core.collection.ListUtil;
-import cn.hutool.core.util.ObjUtil;
+import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
-
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
-import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
-import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
-import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentProcessRespVO;
-import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSaveReqVO;
-import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
+import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.*;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
-import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
+import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
-import cn.iocoder.yudao.module.ai.enums.AiDocumentSplitStrategyEnum;
import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchReqBO;
import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchRespBO;
-import cn.iocoder.yudao.module.ai.service.knowledge.splitter.MarkdownQaSplitter;
-import cn.iocoder.yudao.module.ai.service.knowledge.splitter.SemanticTextSplitter;
import cn.iocoder.yudao.module.ai.service.model.AiModelService;
-import com.alibaba.cloud.ai.dashscope.rerank.DashScopeRerankOptions;
-import com.alibaba.cloud.ai.model.RerankModel;
-import com.alibaba.cloud.ai.model.RerankRequest;
-import com.alibaba.cloud.ai.model.RerankResponse;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.document.Document;
-import org.springframework.ai.tokenizer.TokenCountEstimator;
-import org.springframework.ai.transformer.splitter.TextSplitter;
-import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
import java.util.*;
+import java.util.stream.Collectors;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
-import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.*;
-import static org.springframework.ai.vectorstore.SearchRequest.SIMILARITY_THRESHOLD_ACCEPT_ALL;
-/**
- * AI 鐭ヨ瘑搴撳垎鐗� Service 瀹炵幇绫�
- *
- * @author xiaoxin
- */
-@Service
@Slf4j
+@Service
public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService {
- private static final String VECTOR_STORE_METADATA_KNOWLEDGE_ID = "knowledgeId";
- private static final String VECTOR_STORE_METADATA_DOCUMENT_ID = "documentId";
- private static final String VECTOR_STORE_METADATA_SEGMENT_ID = "segmentId";
-
- private static final Map<String, Class<?>> VECTOR_STORE_METADATA_TYPES = Map.of(
- VECTOR_STORE_METADATA_KNOWLEDGE_ID, String.class,
- VECTOR_STORE_METADATA_DOCUMENT_ID, String.class,
- VECTOR_STORE_METADATA_SEGMENT_ID, String.class);
-
- /**
- * Rerank 鍦ㄥ悜閲忔绱㈡椂锛屾绱㈡暟閲� * 璇ョ郴鏁帮紝鐩殑鏄负浜嗘彁鍗� Rerank 鐨勬晥鏋�
- */
- private static final Integer RERANK_RETRIEVAL_FACTOR = 4;
+ private static final String METADATA_KNOWLEDGE_ID = "knowledgeId";
+ private static final String METADATA_DOCUMENT_ID = "documentId";
+ private static final String METADATA_SEGMENT_ID = "segmentId";
@Resource
private AiKnowledgeSegmentMapper segmentMapper;
-
+ @Resource
+ private AiKnowledgeDocumentMapper documentMapper;
@Resource
private AiKnowledgeService knowledgeService;
@Resource
- @Lazy // 寤惰繜鍔犺浇锛岄伩鍏嶅惊鐜緷璧�
- private AiKnowledgeDocumentService knowledgeDocumentService;
- @Resource
private AiModelService modelService;
- @Resource
- private TokenCountEstimator tokenCountEstimator;
-
- @Autowired(required = false) // 鐢变簬 spring.ai.model.rerank 閰嶇疆椤癸紝鍙互鍏抽棴 RerankModel 鐨勫姛鑳斤紝鎵�浠ヨ繖閲屽彧鑳戒笉寮哄埗娉ㄥ叆
- private RerankModel rerankModel;
-
@Override
- public PageResult<AiKnowledgeSegmentDO> getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) {
- return segmentMapper.selectPage(pageReqVO);
- }
-
- @Override
- public void createKnowledgeSegmentBySplitContent(Long documentId, String content) {
- // 1. 鏍￠獙
- AiKnowledgeDocumentDO documentDO = knowledgeDocumentService.validateKnowledgeDocumentExists(documentId);
- AiKnowledgeDO knowledgeDO = knowledgeService.validateKnowledgeExists(documentDO.getKnowledgeId());
- VectorStore vectorStore = getVectorStoreById(knowledgeDO);
-
- // 2. 鏂囨。鍒囩墖锛堜娇鐢ㄨ嚜鍔ㄦ娴嬬瓥鐣ワ級
- List<Document> documentSegments = splitContentByStrategy(content, documentDO.getSegmentMaxTokens(),
- AiDocumentSplitStrategyEnum.AUTO, documentDO.getUrl());
-
- // 3.1 瀛樺偍鍒囩墖
- List<AiKnowledgeSegmentDO> segmentDOs = convertList(documentSegments, segment -> {
- if (StrUtil.isEmpty(segment.getText())) {
- return null;
- }
- return new AiKnowledgeSegmentDO().setKnowledgeId(documentDO.getKnowledgeId()).setDocumentId(documentId)
- .setContent(segment.getText()).setContentLength(segment.getText().length())
- .setVectorId(AiKnowledgeSegmentDO.VECTOR_ID_EMPTY)
- .setTokens(tokenCountEstimator.estimate(segment.getText()))
- .setStatus(CommonStatusEnum.ENABLE.getStatus());
- });
- segmentMapper.insertBatch(segmentDOs);
- // 3.2 鍒囩墖鍚戦噺鍖�
- for (int i = 0; i < documentSegments.size(); i++) {
- Document segment = documentSegments.get(i);
- AiKnowledgeSegmentDO segmentDO = segmentDOs.get(i);
- writeVectorStore(vectorStore, segmentDO, segment);
- }
- }
-
- @Override
- public void updateKnowledgeSegment(AiKnowledgeSegmentSaveReqVO reqVO) {
- // 1. 鏍￠獙
- AiKnowledgeSegmentDO oldSegment = validateKnowledgeSegmentExists(reqVO.getId());
-
- // 2. 鍒犻櫎鍚戦噺
- VectorStore vectorStore = getVectorStoreById(oldSegment.getKnowledgeId());
- deleteVectorStore(vectorStore, oldSegment);
-
- // 3.1 鏇存柊鍒囩墖
- AiKnowledgeSegmentDO newSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
- segmentMapper.updateById(newSegment);
- // 3.2 閲嶆柊鍚戦噺鍖栵紝蹇呴』寮�鍚姸鎬�
- if (CommonStatusEnum.isEnable(oldSegment.getStatus())) {
- newSegment.setKnowledgeId(oldSegment.getKnowledgeId()).setDocumentId(oldSegment.getDocumentId());
- writeVectorStore(vectorStore, newSegment, new Document(newSegment.getContent()));
- }
- }
-
- @Override
- public void deleteKnowledgeSegment(Long id) {
- // 1. 鏍￠獙娈佃惤瀛樺湪
- AiKnowledgeSegmentDO segment = validateKnowledgeSegmentExists(id);
-
- // 2. 鍒犻櫎鍚戦噺
- VectorStore vectorStore = getVectorStoreById(segment.getKnowledgeId());
- deleteVectorStore(vectorStore, segment);
-
- // 3. 鍒犻櫎娈佃惤璁板綍
- segmentMapper.deleteById(id);
- }
-
- @Override
- public void deleteKnowledgeSegmentByDocumentId(Long documentId) {
- // 1. 鏌ヨ闇�瑕佸垹闄ょ殑娈佃惤
- List<AiKnowledgeSegmentDO> segments = segmentMapper.selectListByDocumentId(documentId);
- if (CollUtil.isEmpty(segments)) {
- return;
- }
-
- // 2. 鎵归噺鍒犻櫎娈佃惤璁板綍
- segmentMapper.deleteByIds(convertList(segments, AiKnowledgeSegmentDO::getId));
-
- // 3. 鍒犻櫎鍚戦噺瀛樺偍涓殑娈佃惤
- VectorStore vectorStore = getVectorStoreById(segments.getFirst().getKnowledgeId());
- vectorStore.delete(convertList(segments, AiKnowledgeSegmentDO::getVectorId));
- }
-
- @Override
- public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) {
- // 1. 鏍￠獙
- AiKnowledgeSegmentDO segment = validateKnowledgeSegmentExists(reqVO.getId());
-
- // 2. 鑾峰彇鐭ヨ瘑搴撳悜閲忓疄渚�
- VectorStore vectorStore = getVectorStoreById(segment.getKnowledgeId());
-
- // 3. 鏇存柊鐘舵��
- segmentMapper.updateById(new AiKnowledgeSegmentDO().setId(reqVO.getId()).setStatus(reqVO.getStatus()));
-
- // 4. 鏇存柊鍚戦噺
- if (CommonStatusEnum.isEnable(reqVO.getStatus())) {
- writeVectorStore(vectorStore, segment, new Document(segment.getContent()));
- } else {
- deleteVectorStore(vectorStore, segment);
- }
- }
-
- @Override
- public void reindexKnowledgeSegmentByKnowledgeId(Long knowledgeId) {
- // 1.1 鏍￠獙鐭ヨ瘑搴撳瓨鍦�
- AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(knowledgeId);
- // 1.2 鑾峰彇鐭ヨ瘑搴撳悜閲忓疄渚�
- VectorStore vectorStore = getVectorStoreById(knowledge);
-
- // 2.1 鏌ヨ鐭ヨ瘑搴撲笅鐨勬墍鏈夊惎鐢ㄧ姸鎬佺殑娈佃惤
- List<AiKnowledgeSegmentDO> segments = segmentMapper.selectListByKnowledgeIdAndStatus(
- knowledgeId, CommonStatusEnum.ENABLE.getStatus());
- if (CollUtil.isEmpty(segments)) {
- return;
- }
- // 2.2 閬嶅巻鎵�鏈夋钀斤紝閲嶆柊绱㈠紩
- for (AiKnowledgeSegmentDO segment : segments) {
- // 鍒犻櫎鏃х殑鍚戦噺
- deleteVectorStore(vectorStore, segment);
- // 閲嶆柊鍒涘缓鍚戦噺
- writeVectorStore(vectorStore, segment, new Document(segment.getContent()));
- }
- log.info("[reindexKnowledgeSegmentByKnowledgeId][鐭ヨ瘑搴�({}) 閲嶆柊绱㈠紩瀹屾垚锛屽叡澶勭悊 {} 涓钀絔",
- knowledgeId, segments.size());
- }
-
- private void writeVectorStore(VectorStore vectorStore, AiKnowledgeSegmentDO segmentDO, Document segment) {
- // 1. 鍚戦噺瀛樺偍
- // 涓轰粈涔堣 toString 鍛紵鍥犱负閮ㄥ垎 VectorStore 瀹炵幇锛屼笉鏀寔 Long 绫诲瀷锛屼緥濡傝 QdrantVectorStore
- segment.getMetadata().put(VECTOR_STORE_METADATA_KNOWLEDGE_ID, segmentDO.getKnowledgeId().toString());
- segment.getMetadata().put(VECTOR_STORE_METADATA_DOCUMENT_ID, segmentDO.getDocumentId().toString());
- segment.getMetadata().put(VECTOR_STORE_METADATA_SEGMENT_ID, segmentDO.getId().toString());
- vectorStore.add(List.of(segment));
-
- // 2. 鏇存柊鍚戦噺 ID
- segmentMapper.updateById(new AiKnowledgeSegmentDO().setId(segmentDO.getId()).setVectorId(segment.getId()));
- }
-
- private void deleteVectorStore(VectorStore vectorStore, AiKnowledgeSegmentDO segmentDO) {
- // 1. 鏇存柊鍚戦噺 ID
- if (StrUtil.isEmpty(segmentDO.getVectorId())) {
- return;
- }
- segmentMapper.updateById(new AiKnowledgeSegmentDO().setId(segmentDO.getId())
- .setVectorId(AiKnowledgeSegmentDO.VECTOR_ID_EMPTY));
-
- // 2. 鍒犻櫎鍚戦噺
- vectorStore.delete(List.of(segmentDO.getVectorId()));
- }
-
- @Override
- public List<AiKnowledgeSegmentSearchRespBO> searchKnowledgeSegment(AiKnowledgeSegmentSearchReqBO reqBO) {
- // 1. 鏍￠獙
- AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(reqBO.getKnowledgeId());
-
- // 2. 妫�绱�
- List<Document> documents = searchDocument(knowledge, reqBO);
- if (CollUtil.isEmpty(documents)) {
- return ListUtil.empty();
- }
-
- // 3.1 娈佃惤鍙洖
- List<AiKnowledgeSegmentDO> segments = segmentMapper
- .selectListByVectorIds(convertList(documents, Document::getId));
- if (CollUtil.isEmpty(segments)) {
- return ListUtil.empty();
- }
- // 3.2 澧炲姞鍙洖娆℃暟
- segmentMapper.updateRetrievalCountIncrByIds(convertList(segments, AiKnowledgeSegmentDO::getId));
-
- // 4. 鏋勫缓缁撴灉
- List<AiKnowledgeSegmentSearchRespBO> result = convertList(segments, segment -> {
- Document document = CollUtil.findOne(documents, // 鎵惧埌瀵瑰簲鐨勬枃妗�
- doc -> Objects.equals(doc.getId(), segment.getVectorId()));
- if (document == null) {
- return null;
- }
- return BeanUtils.toBean(segment, AiKnowledgeSegmentSearchRespBO.class)
- .setScore(document.getScore());
- });
- result.sort((o1, o2) -> Double.compare(o2.getScore(), o1.getScore())); // 鎸夌収鍒嗘暟闄嶅簭鎺掑簭
- return result;
- }
-
- /**
- * 鍩轰簬 Embedding + Rerank Model锛屾绱㈢煡璇嗗簱涓殑鏂囨。
- *
- * @param knowledge 鐭ヨ瘑搴�
- * @param reqBO 妫�绱㈣姹�
- * @return 鏂囨。鍒楄〃
- */
- private List<Document> searchDocument(AiKnowledgeDO knowledge, AiKnowledgeSegmentSearchReqBO reqBO) {
- VectorStore vectorStore = getVectorStoreById(knowledge);
- Integer topK = ObjUtil.defaultIfNull(reqBO.getTopK(), knowledge.getTopK());
- Double similarityThreshold = ObjUtil.defaultIfNull(reqBO.getSimilarityThreshold(), knowledge.getSimilarityThreshold());
-
- // 1. 鍚戦噺妫�绱�
- int searchTopK = rerankModel != null ? topK * RERANK_RETRIEVAL_FACTOR : topK;
- double searchSimilarityThreshold = rerankModel != null ? SIMILARITY_THRESHOLD_ACCEPT_ALL : similarityThreshold;
- SearchRequest.Builder searchRequestBuilder = SearchRequest.builder()
- .query(reqBO.getContent())
- .topK(searchTopK).similarityThreshold(searchSimilarityThreshold)
- .filterExpression(new FilterExpressionBuilder()
- .eq(VECTOR_STORE_METADATA_KNOWLEDGE_ID, reqBO.getKnowledgeId().toString()).build());
- List<Document> documents = vectorStore.similaritySearch(searchRequestBuilder.build());
- if (CollUtil.isEmpty(documents)) {
- return documents;
- }
-
- // 2. Rerank 閲嶆帓搴�
- if (rerankModel != null) {
- RerankResponse rerankResponse = rerankModel.call(new RerankRequest(reqBO.getContent(), documents,
- DashScopeRerankOptions.builder().topN(topK).build()));
- documents = convertList(rerankResponse.getResults(),
- documentWithScore -> documentWithScore.getScore() >= similarityThreshold
- ? documentWithScore.getOutput() : null);
- }
- return documents;
- }
-
- @Override
- public List<AiKnowledgeSegmentDO> splitContent(String url, Integer segmentMaxTokens) {
- // 1. 璇诲彇 URL 鍐呭
- String content = knowledgeDocumentService.readUrl(url);
-
- // 2.1 鑷姩妫�娴嬫枃妗ｇ被鍨嬪苟閫夋嫨绛栫暐
- AiDocumentSplitStrategyEnum strategy = detectDocumentStrategy(content, url);
- // 2.2 鏂囨。鍒囩墖
- List<Document> documentSegments = splitContentByStrategy(content, segmentMaxTokens, strategy, url);
-
- // 3. 杞崲涓烘钀藉璞�
- return convertList(documentSegments, segment -> {
- if (StrUtil.isEmpty(segment.getText())) {
- return null;
- }
- return new AiKnowledgeSegmentDO()
- .setContent(segment.getText())
- .setContentLength(segment.getText().length())
- .setTokens(tokenCountEstimator.estimate(segment.getText()));
- });
- }
-
- /**
- * 鏍￠獙娈佃惤鏄惁瀛樺湪
- *
- * @param id 鏂囨。缂栧彿
- * @return 娈佃惤淇℃伅
- */
- private AiKnowledgeSegmentDO validateKnowledgeSegmentExists(Long id) {
- AiKnowledgeSegmentDO knowledgeSegment = segmentMapper.selectById(id);
- if (knowledgeSegment == null) {
- throw exception(KNOWLEDGE_SEGMENT_NOT_EXISTS);
- }
- return knowledgeSegment;
- }
-
- private VectorStore getVectorStoreById(AiKnowledgeDO knowledge) {
- return modelService.getOrCreateVectorStore(knowledge.getEmbeddingModelId(), VECTOR_STORE_METADATA_TYPES);
- }
-
- private VectorStore getVectorStoreById(Long knowledgeId) {
- AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(knowledgeId);
- return getVectorStoreById(knowledge);
- }
-
- /**
- * 鏍规嵁绛栫暐鍒囧垎鍐呭
- *
- * @param content 鏂囨。鍐呭
- * @param segmentMaxTokens 鍒嗘鐨勬渶澶� Token 鏁�
- * @param strategy 鍒囩墖绛栫暐
- * @param url 鏂囨。 URL锛堢敤浜庤嚜鍔ㄦ娴嬫枃浠剁被鍨嬶級
- * @return 鍒囩墖鍚庣殑鏂囨。鍒楄〃
- */
- @SuppressWarnings("EnhancedSwitchMigration")
- private List<Document> splitContentByStrategy(String content, Integer segmentMaxTokens,
- AiDocumentSplitStrategyEnum strategy, String url) {
- // 鑷姩妫�娴嬬瓥鐣�
- if (strategy == AiDocumentSplitStrategyEnum.AUTO) {
- strategy = detectDocumentStrategy(content, url);
- log.info("[splitContentByStrategy][鑷姩妫�娴嬪埌鏂囨。绛栫暐: {}]", strategy.getName());
- }
- // 鏍规嵁绛栫暐鍒囧垎
- TextSplitter textSplitter;
- switch (strategy) {
- case MARKDOWN_QA:
- textSplitter = new MarkdownQaSplitter(segmentMaxTokens);
- break;
- case SEMANTIC:
- textSplitter = new SemanticTextSplitter(segmentMaxTokens);
- break;
- case PARAGRAPH:
- textSplitter = new SemanticTextSplitter(segmentMaxTokens, 0); // 娈佃惤鍒囧垎锛屾棤閲嶅彔
- break;
- case TOKEN:
- default:
- textSplitter = buildTokenTextSplitter(segmentMaxTokens);
- break;
- }
- // 鎵ц鍒囧垎
- return textSplitter.apply(Collections.singletonList(new Document(content)));
- }
-
- /**
- * 鑷姩妫�娴嬫枃妗ｇ被鍨嬪苟閫夋嫨鍒囩墖绛栫暐
- *
- * @param content 鏂囨。鍐呭
- * @param url 鏂囨。 URL
- * @return 鎺ㄨ崘鐨勫垏鐗囩瓥鐣�
- */
- private AiDocumentSplitStrategyEnum detectDocumentStrategy(String content, String url) {
- if (StrUtil.isEmpty(content)) {
- return AiDocumentSplitStrategyEnum.TOKEN;
- }
- // 1. 妫�娴� Markdown QA 鏍煎紡
- if (isMarkdownQaFormat(content, url)) {
- return AiDocumentSplitStrategyEnum.MARKDOWN_QA;
- }
- // 2. 妫�娴嬫櫘閫� Markdown 鏂囨。
- if (isMarkdownDocument(url)) {
- return AiDocumentSplitStrategyEnum.SEMANTIC;
- }
- // 3. 榛樿浣跨敤璇箟鍒囧垎锛堟瘮 Token 鍒囧垎鏇存櫤鑳斤級
- return AiDocumentSplitStrategyEnum.SEMANTIC;
- }
-
- /**
- * 妫�娴嬫槸鍚︿负 Markdown QA 鏍煎紡
- * 鐗瑰緛锛氬寘鍚涓簩绾ф爣棰橈紙## 锛変笖鏍囬鍚庣揣璺熺瓟妗堝唴瀹�
- */
- private boolean isMarkdownQaFormat(String content, String url) {
- // 鏂囦欢鎵╁睍鍚嶅垽鏂�
- if (StrUtil.isNotEmpty(url) && !url.toLowerCase().endsWith(".md")) {
- return false;
- }
-
- // 缁熻浜岀骇鏍囬鏁伴噺
- long h2Count = content.lines()
- .filter(line -> line.trim().startsWith("## "))
- .count();
-
- // 瑕佹眰涓�锛氳嚦灏戝寘鍚� 2 涓簩绾ф爣棰樻墠璁や负鏄� QA 鏍煎紡
- if (h2Count < 2) {
- return false;
- }
-
- // 瑕佹眰浜岋細妫�鏌ユ爣棰樺崰姣旓紙QA 鏂囨。鏍囬琛屾暟鐩稿杈冨锛夛紝濡傛灉浜岀骇鏍囬鍗犳瘮瓒呰繃 10%锛岃涓烘槸 QA 鏍煎紡
- long totalLines = content.lines().count();
- double h2Ratio = (double) h2Count / totalLines;
- return h2Ratio > 0.1;
- }
-
- /**
- * 妫�娴嬫槸鍚︿负 Markdown 鏂囨。
- */
- private boolean isMarkdownDocument(String url) {
- return StrUtil.endWithAnyIgnoreCase(url, ".md", ".markdown");
- }
-
- /**
- * 鏋勫缓鍩轰簬 Token 鐨勬枃鏈垏鐗囧櫒锛堝師鏈夐�昏緫淇濈暀锛�
- */
- private static TextSplitter buildTokenTextSplitter(Integer segmentMaxTokens) {
- return TokenTextSplitter.builder()
- .withChunkSize(segmentMaxTokens)
- .withMinChunkSizeChars(Integer.MAX_VALUE) // 蹇界暐瀛楃鐨勬埅鏂�
- .withMinChunkLengthToEmbed(1) // 鍏佽鐨勬渶灏忔湁鏁堝垎娈甸暱搴�
- .withMaxNumChunks(Integer.MAX_VALUE)
- .withKeepSeparator(true) // 淇濈暀鍒嗛殧绗�
- .build();
- }
-
- @Override
- public List<AiKnowledgeSegmentProcessRespVO> getKnowledgeSegmentProcessList(List<Long> documentIds) {
- if (CollUtil.isEmpty(documentIds)) {
- return Collections.emptyList();
- }
- return segmentMapper.selectProcessList(documentIds);
- }
-
- @Override
- public Long createKnowledgeSegment(AiKnowledgeSegmentSaveReqVO createReqVO) {
- // 1.1 鏍￠獙鏂囨。鏄惁瀛樺湪
- AiKnowledgeDocumentDO document = knowledgeDocumentService
- .validateKnowledgeDocumentExists(createReqVO.getDocumentId());
- // 1.2 鑾峰彇鐭ヨ瘑搴撲俊鎭�
- AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(document.getKnowledgeId());
- // 1.3 鏍￠獙 token 鐔熺粌
- Integer tokens = tokenCountEstimator.estimate(createReqVO.getContent());
- if (tokens > document.getSegmentMaxTokens()) {
- throw exception(KNOWLEDGE_SEGMENT_CONTENT_TOO_LONG, tokens, document.getSegmentMaxTokens());
- }
-
- // 2. 淇濆瓨娈佃惤
- AiKnowledgeSegmentDO segment = BeanUtils.toBean(createReqVO, AiKnowledgeSegmentDO.class)
- .setKnowledgeId(knowledge.getId()).setDocumentId(document.getId())
- .setContentLength(createReqVO.getContent().length()).setTokens(tokens)
- .setVectorId(AiKnowledgeSegmentDO.VECTOR_ID_EMPTY)
- .setRetrievalCount(0).setStatus(CommonStatusEnum.ENABLE.getStatus());
- segmentMapper.insert(segment);
-
- // 3. 鍚戦噺鍖�
- writeVectorStore(getVectorStoreById(knowledge), segment, new Document(segment.getContent()));
- return segment.getId();
- }
-
- @Override
- public AiKnowledgeSegmentDO getKnowledgeSegment(Long id) {
+ public AiKnowledgeSegmentDO getSegment(Long id) {
return segmentMapper.selectById(id);
}
@Override
- public List<AiKnowledgeSegmentDO> getKnowledgeSegmentList(Collection<Long> ids) {
- if (CollUtil.isEmpty(ids)) {
- return Collections.emptyList();
+ public AiKnowledgeSegmentDO validateSegmentExists(Long id) {
+ AiKnowledgeSegmentDO segment = segmentMapper.selectById(id);
+ if (segment == null) throw exception(KNOWLEDGE_SEGMENT_NOT_EXISTS);
+ return segment;
+ }
+
+ @Override
+ public PageResult<AiKnowledgeSegmentDO> getSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) {
+ return segmentMapper.selectPage(pageReqVO);
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public Long createSegment(AiKnowledgeSegmentSaveReqVO saveReqVO) {
+ if (saveReqVO.getKnowledgeId() == null) throw exception(KNOWLEDGE_NOT_EXISTS);
+ if (saveReqVO.getDocumentId() == null) throw exception(KNOWLEDGE_DOCUMENT_NOT_EXISTS);
+ AiKnowledgeDO knowledge = knowledgeService.validateKnowledge(saveReqVO.getKnowledgeId());
+ AiKnowledgeSegmentDO segment = new AiKnowledgeSegmentDO();
+ segment.setKnowledgeId(saveReqVO.getKnowledgeId());
+ segment.setDocumentId(saveReqVO.getDocumentId());
+ segment.setContent(saveReqVO.getContent());
+ segment.setContentLength(saveReqVO.getContent().length());
+ segment.setTokens(estimateTokens(saveReqVO.getContent()));
+ segment.setStatus(saveReqVO.getStatus() != null ? saveReqVO.getStatus()
+ : CommonStatusEnum.ENABLE.getStatus());
+ segment.setRetrievalCount(0);
+ segment.setVectorId(AiKnowledgeSegmentDO.VECTOR_ID_EMPTY);
+ segmentMapper.insert(segment);
+
+ // 鍚戦噺鍖�
+ VectorStore vectorStore = modelService.getOrCreateVectorStore(
+ knowledge.getEmbeddingModelId(), buildMetadataFields());
+ Map<String, Object> metadata = new HashMap<>();
+ metadata.put(METADATA_KNOWLEDGE_ID, knowledge.getId());
+ metadata.put(METADATA_DOCUMENT_ID, segment.getDocumentId());
+ metadata.put(METADATA_SEGMENT_ID, segment.getId());
+ Document doc = new Document(segment.getId().toString(), segment.getContent(), metadata);
+ try {
+ vectorStore.add(Collections.singletonList(doc));
+ segment.setVectorId(segment.getId().toString());
+ segmentMapper.updateById(segment);
+ } catch (Exception e) {
+ log.error("鎵嬪姩鍒涘缓鍒嗘鍚戦噺鍖栧け璐ワ紝segmentId={}", segment.getId(), e);
+ segment.setStatus(CommonStatusEnum.DISABLE.getStatus());
+ segmentMapper.updateById(segment);
}
- return segmentMapper.selectByIds(ids);
+ return segment.getId();
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void updateSegment(AiKnowledgeSegmentSaveReqVO saveReqVO) {
+ AiKnowledgeSegmentDO segment = validateSegmentExists(saveReqVO.getId());
+ AiKnowledgeDO knowledge = knowledgeService.validateKnowledge(segment.getKnowledgeId());
+ // 鍒犻櫎鏃у悜閲�
+ if (StrUtil.isNotEmpty(segment.getVectorId()) && !AiKnowledgeSegmentDO.VECTOR_ID_EMPTY.equals(segment.getVectorId())) {
+ VectorStore vectorStore = modelService.getOrCreateVectorStore(
+ knowledge.getEmbeddingModelId(), buildMetadataFields());
+ try {
+ vectorStore.delete(Collections.singletonList(segment.getVectorId()));
+ } catch (Exception e) {
+ log.warn("鍒犻櫎鏃у悜閲忓け璐�: {}", segment.getVectorId(), e);
+ }
+ }
+ segment.setContent(saveReqVO.getContent());
+ segment.setContentLength(saveReqVO.getContent().length());
+ segment.setTokens(estimateTokens(saveReqVO.getContent()));
+ if (saveReqVO.getStatus() != null) segment.setStatus(saveReqVO.getStatus());
+ segmentMapper.updateById(segment);
+
+ // 閲嶆柊鍚戦噺鍖�
+ VectorStore vectorStore = modelService.getOrCreateVectorStore(
+ knowledge.getEmbeddingModelId(), buildMetadataFields());
+ Map<String, Object> metadata = new HashMap<>();
+ metadata.put(METADATA_KNOWLEDGE_ID, knowledge.getId());
+ metadata.put(METADATA_DOCUMENT_ID, segment.getDocumentId());
+ metadata.put(METADATA_SEGMENT_ID, segment.getId());
+ Document doc = new Document(segment.getId().toString(), segment.getContent(), metadata);
+ try {
+ vectorStore.add(Collections.singletonList(doc));
+ segment.setVectorId(segment.getId().toString());
+ segmentMapper.updateById(segment);
+ } catch (Exception e) {
+ log.error("鏇存柊鍒嗘鍚戦噺鍖栧け璐ワ紝segmentId={}", segment.getId(), e);
+ segment.setStatus(CommonStatusEnum.DISABLE.getStatus());
+ segmentMapper.updateById(segment);
+ }
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void deleteSegment(Long id) {
+ AiKnowledgeSegmentDO segment = validateSegmentExists(id);
+ AiKnowledgeDO knowledge = knowledgeService.validateKnowledge(segment.getKnowledgeId());
+ if (StrUtil.isNotEmpty(segment.getVectorId()) && !AiKnowledgeSegmentDO.VECTOR_ID_EMPTY.equals(segment.getVectorId())) {
+ VectorStore vectorStore = modelService.getOrCreateVectorStore(
+ knowledge.getEmbeddingModelId(), buildMetadataFields());
+ try {
+ vectorStore.delete(Collections.singletonList(segment.getVectorId()));
+ } catch (Exception e) {
+ log.warn("鍒犻櫎鍚戦噺澶辫触: {}", segment.getVectorId(), e);
+ }
+ }
+ segmentMapper.deleteById(id);
+ }
+
+ @Override
+ public void updateSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO updateStatusReqVO) {
+ AiKnowledgeSegmentDO segment = validateSegmentExists(updateStatusReqVO.getId());
+ segment.setStatus(updateStatusReqVO.getStatus());
+ segmentMapper.updateById(segment);
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void deleteSegmentsByDocumentId(Long documentId) {
+ List<AiKnowledgeSegmentDO> segments = segmentMapper.selectListByDocumentId(documentId);
+ if (CollUtil.isEmpty(segments)) return;
+
+ AiKnowledgeDO knowledge = knowledgeService.getKnowledge(segments.get(0).getKnowledgeId());
+ VectorStore vectorStore = modelService.getOrCreateVectorStore(
+ knowledge.getEmbeddingModelId(), buildMetadataFields());
+
+ for (AiKnowledgeSegmentDO segment : segments) {
+ if (StrUtil.isNotEmpty(segment.getVectorId()) && !AiKnowledgeSegmentDO.VECTOR_ID_EMPTY.equals(segment.getVectorId())) {
+ try {
+ vectorStore.delete(Collections.singletonList(segment.getVectorId()));
+ } catch (Exception e) {
+ log.warn("鍒犻櫎鍚戦噺[{}]澶辫触: {}", segment.getVectorId(), e.getMessage());
+ }
+ }
+ }
+ segmentMapper.delete(new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper<AiKnowledgeSegmentDO>()
+ .eq(AiKnowledgeSegmentDO::getDocumentId, documentId));
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void saveSegments(List<AiKnowledgeSegmentDO> segments, Long knowledgeId, Long embeddingModelId) {
+ if (CollUtil.isEmpty(segments)) return;
+
+ AiKnowledgeDO knowledge = knowledgeService.validateKnowledge(knowledgeId);
+ VectorStore vectorStore = modelService.getOrCreateVectorStore(embeddingModelId, buildMetadataFields());
+
+ // 鎵归噺鎻掑叆鍒嗘璁板綍
+ for (AiKnowledgeSegmentDO segment : segments) {
+ segmentMapper.insert(segment);
+ }
+
+ // 杞崲涓� Spring AI Document 骞跺啓鍏ュ悜閲忓簱
+ List<Document> documents = new ArrayList<>();
+ for (AiKnowledgeSegmentDO segment : segments) {
+ Map<String, Object> metadata = new HashMap<>();
+ metadata.put(METADATA_KNOWLEDGE_ID, knowledgeId);
+ metadata.put(METADATA_DOCUMENT_ID, segment.getDocumentId());
+ metadata.put(METADATA_SEGMENT_ID, segment.getId());
+ Document doc = new Document(segment.getId().toString(), segment.getContent(), metadata);
+ documents.add(doc);
+ }
+
+ try {
+ vectorStore.add(documents);
+ // 鏇存柊 vectorId锛圡ilvus 杩斿洖鐨� ID 灏辨槸浼犲叆鐨� docId锛�
+ for (AiKnowledgeSegmentDO segment : segments) {
+ segment.setVectorId(segment.getId().toString());
+ segmentMapper.updateById(segment);
+ }
+ log.info("鎵归噺鍐欏叆鍚戦噺搴撴垚鍔燂紝鐭ヨ瘑搴�: {}, 鍒嗘鏁�: {}", knowledgeId, segments.size());
+ } catch (Exception e) {
+ log.error("鍐欏叆鍚戦噺搴撳け璐ワ紝鐭ヨ瘑搴�: {}", knowledgeId, e);
+ for (AiKnowledgeSegmentDO segment : segments) {
+ segment.setStatus(CommonStatusEnum.DISABLE.getStatus());
+ segmentMapper.updateById(segment);
+ }
+ throw new RuntimeException("鍚戦噺搴撳啓鍏ュけ璐�: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public List<AiKnowledgeSegmentSearchRespBO> searchSegments(AiKnowledgeSegmentSearchReqBO searchReqBO) {
+ AiKnowledgeDO knowledge = knowledgeService.validateKnowledge(searchReqBO.getKnowledgeId());
+ VectorStore vectorStore = modelService.getOrCreateVectorStore(
+ knowledge.getEmbeddingModelId(), buildMetadataFields());
+
+ int topK = searchReqBO.getTopK() != null ? searchReqBO.getTopK() : knowledge.getTopK();
+ double similarityThreshold = searchReqBO.getSimilarityThreshold() != null
+ ? searchReqBO.getSimilarityThreshold() : knowledge.getSimilarityThreshold();
+
+ Filter.Expression filterExpression = new FilterExpressionBuilder()
+ .eq(METADATA_KNOWLEDGE_ID, searchReqBO.getKnowledgeId())
+ .build();
+
+ SearchRequest request = SearchRequest.builder()
+ .query(searchReqBO.getContent())
+ .topK(topK)
+ .similarityThreshold(similarityThreshold)
+ .filterExpression(filterExpression)
+ .build();
+
+ log.info("鍚戦噺妫�绱�: query={}, topK={}, threshold={}, filter=kid={}",
+ searchReqBO.getContent(), topK, similarityThreshold, searchReqBO.getKnowledgeId());
+ List<Document> results = vectorStore.similaritySearch(request);
+ log.info("鍚戦噺妫�绱㈢粨鏋滄暟: {}", results != null ? results.size() : 0);
+ if (CollUtil.isEmpty(results)) return Collections.emptyList();
+
+ // 鏇存柊妫�绱㈡鏁�
+ List<String> vectorIds = results.stream()
+ .map(Document::getId)
+ .filter(StrUtil::isNotEmpty)
+ .collect(Collectors.toList());
+ if (CollUtil.isNotEmpty(vectorIds)) {
+ List<AiKnowledgeSegmentDO> hitSegments = segmentMapper.selectListByVectorIds(vectorIds);
+ if (CollUtil.isNotEmpty(hitSegments)) {
+ List<Long> hitIds = hitSegments.stream().map(AiKnowledgeSegmentDO::getId).collect(Collectors.toList());
+ segmentMapper.updateRetrievalCountIncrByIds(hitIds);
+ Set<Long> docIds = hitSegments.stream().map(AiKnowledgeSegmentDO::getDocumentId).collect(Collectors.toSet());
+ documentMapper.updateRetrievalCountIncr(docIds);
+ }
+ }
+
+ // 鏀堕泦鏂囨。 ID 骞舵壒閲忔煡璇㈡枃妗ｅ悕绉�
+ Set<Long> resultDocIds = results.stream()
+ .map(doc -> convertToLong(doc.getMetadata() != null
+ ? doc.getMetadata().get(METADATA_DOCUMENT_ID) : null))
+ .filter(id -> id != null)
+ .collect(Collectors.toSet());
+ final Map<Long, String> docNameMap;
+ if (CollUtil.isNotEmpty(resultDocIds)) {
+ docNameMap = documentMapper.selectBatchIds(resultDocIds).stream()
+ .collect(Collectors.toMap(
+ cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO::getId,
+ cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO::getName,
+ (a, b) -> a));
+ } else {
+ docNameMap = Collections.emptyMap();
+ }
+
+ return results.stream()
+ .map(doc -> {
+ AiKnowledgeSegmentSearchRespBO bo = new AiKnowledgeSegmentSearchRespBO();
+ bo.setContent(doc.getText());
+ bo.setScore(doc.getScore() != null ? doc.getScore() : 0.0);
+ if (doc.getMetadata() != null) {
+ bo.setId(convertToLong(doc.getMetadata().get(METADATA_SEGMENT_ID)));
+ bo.setDocumentId(convertToLong(doc.getMetadata().get(METADATA_DOCUMENT_ID)));
+ bo.setDocumentName(docNameMap.getOrDefault(bo.getDocumentId(), "鏈煡鏂囨。"));
+ bo.setKnowledgeId(convertToLong(doc.getMetadata().get(METADATA_KNOWLEDGE_ID)));
+ log.debug("妫�绱㈢粨鏋滃厓鏁版嵁: id={}, docId={}, kid={}, metadataKeys={}",
+ bo.getId(), bo.getDocumentId(), bo.getKnowledgeId(),
+ doc.getMetadata().keySet());
+ }
+ if (bo.getContent() != null) {
+ bo.setContentLength(bo.getContent().length());
+ bo.setTokens(estimateTokens(bo.getContent()));
+ }
+ return bo;
+ })
+ .collect(Collectors.toList());
+ }
+
+ private Map<String, Class<?>> buildMetadataFields() {
+ return MapUtil.<String, Class<?>>builder()
+ .put(METADATA_KNOWLEDGE_ID, Long.class)
+ .put(METADATA_DOCUMENT_ID, Long.class)
+ .put(METADATA_SEGMENT_ID, Long.class)
+ .build();
+ }
+
+ /**
+ * 灏� Milvus 杩斿洖鐨勫厓鏁版嵁鍊艰浆涓� Long銆�
+ * Gson 鍙嶅簭鍒楀寲 JSON 鏁板瓧榛樿涓� Double锛岄渶瑕佸吋瀹瑰绉嶇被鍨嬨��
+ */
+ private Long convertToLong(Object value) {
+ if (value == null) return null;
+ if (value instanceof Long v) return v;
+ if (value instanceof Integer v) return v.longValue();
+ if (value instanceof Double v) return v.longValue();
+ if (value instanceof Float v) return v.longValue();
+ if (value instanceof Number v) return v.longValue();
+ if (value instanceof String v) {
+ try { return Long.valueOf(v); } catch (NumberFormatException e) { return null; }
+ }
+ log.warn("鏃犳硶杞崲鍏冩暟鎹�肩被鍨�: {} = {}", value.getClass().getName(), value);
+ return null;
+ }
+
+ private Integer estimateTokens(String text) {
+ if (StrUtil.isEmpty(text)) return 0;
+ int chineseChars = 0, englishWords = 0;
+ for (char c : text.toCharArray()) {
+ if (c >= 0x4E00 && c <= 0x9FA5) chineseChars++;
+ }
+ for (String word : text.split("\\s+")) {
+ if (word.matches(".*[a-zA-Z].*")) englishWords++;
+ }
+ return chineseChars + (int) (englishWords * 1.3);
}
}
--
Gitblit v1.9.3