From 83e1b4d0e661f11a407fd6ea86e906b9b87b7180 Mon Sep 17 00:00:00 2001
From: 云 <2163098428@qq.com>
Date: 星期五, 31 七月 2026 17:57:09 +0800
Subject: [PATCH] feat(aftersales): 售后工单新增问题类型和严重程度字段
---
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java | 264 ++++++----------------------------------------------
1 files changed, 34 insertions(+), 230 deletions(-)
diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java
index 1fbf4f2..6239602 100644
--- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java
+++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java
@@ -5,7 +5,6 @@
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.transformer.splitter.TextSplitter;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -14,44 +13,16 @@
/**
* Markdown QA 鏍煎紡涓撶敤鍒囩墖鍣�
- *
- * <p>鍔熻兘鐗圭偣锛�
- * <ul>
- * <li>璇嗗埆浜岀骇鏍囬锛�## 锛変綔涓洪棶棰樻爣璁�</li>
- * <li>鐭� QA 瀵逛繚鎸佸畬鏁达紙涓嶈秴杩� Token 闄愬埗锛�</li>
- * <li>闀跨瓟妗堟櫤鑳藉垏鍒嗭紝姣忎釜鐗囨淇濈暀瀹屾暣闂浣滀负涓婁笅鏂�</li>
- * <li>鏀寔鑷畾涔� Token 浼扮畻鍣�</li>
- * </ul>
- *
- * @author runzhen
+ * 璇嗗埆浜岀骇鏍囬锛�## 锛変綔涓洪棶棰樻爣璁帮紝淇濇寔闂瓟瀵瑰畬鏁存��
*/
@Slf4j
-@SuppressWarnings("SizeReplaceableByIsEmpty")
public class MarkdownQaSplitter extends TextSplitter {
- /**
- * 浜岀骇鏍囬姝ｅ垯锛氬尮閰� "## " 寮�澶寸殑琛�
- */
private static final Pattern H2_PATTERN = Pattern.compile("^##\\s+(.+)$", Pattern.MULTILINE);
-
- /**
- * 娈佃惤鍒嗛殧绗︼細鍙屾崲琛�
- */
private static final String PARAGRAPH_SEPARATOR = "\n\n";
-
- /**
- * 鍙ュ瓙鍒嗛殧绗�
- */
private static final Pattern SENTENCE_PATTERN = Pattern.compile("[銆傦紒锛�.!?]\\s*");
- /**
- * 鍒嗘鐨勬渶澶� Token 鏁�
- */
private final int chunkSize;
-
- /**
- * Token 浼扮畻鍣紙绠�鍗曞疄鐜帮細涓枃鎸夊瓧绗︽暟锛岃嫳鏂囨寜鍗曡瘝鏁扮殑 1.3 鍊嶏級
- */
private final TokenEstimator tokenEstimator;
public MarkdownQaSplitter(int chunkSize) {
@@ -61,282 +32,115 @@
@Override
protected List<String> splitText(String text) {
- if (StrUtil.isEmpty(text)) {
- return Collections.emptyList();
- }
-
- // 瑙ｆ瀽 QA 瀵�
+ if (StrUtil.isEmpty(text)) return Collections.emptyList();
List<QaPair> qaPairs = parseQaPairs(text);
- if (CollUtil.isEmpty(qaPairs)) {
- // 濡傛灉娌℃湁璇嗗埆鍒� QA 鏍煎紡锛屾寜娈佃惤鍒囧垎
- return fallbackSplit(text);
- }
-
- // 澶勭悊姣忎釜 QA 瀵�
+ if (CollUtil.isEmpty(qaPairs)) return fallbackSplit(text);
List<String> result = new ArrayList<>();
- for (QaPair qaPair : qaPairs) {
- result.addAll(splitQaPair(qaPair));
- }
+ for (QaPair qaPair : qaPairs) result.addAll(splitQaPair(qaPair));
return result;
}
- /**
- * 瑙ｆ瀽 Markdown QA 瀵�
- *
- * @param content 鏂囨湰鍐呭
- * @return QA 瀵瑰垪琛�
- */
private List<QaPair> parseQaPairs(String content) {
- // 鎵惧埌鎵�鏈変簩绾ф爣棰樹綅缃�
List<QaPair> qaPairs = new ArrayList<>();
List<Integer> headingPositions = new ArrayList<>();
List<String> questions = new ArrayList<>();
Matcher matcher = H2_PATTERN.matcher(content);
- while (matcher.find()) {
- headingPositions.add(matcher.start());
- questions.add(matcher.group(1).trim());
- }
- if (CollUtil.isEmpty(headingPositions)) {
- return qaPairs;
- }
-
- // 鎻愬彇姣忎釜 QA 瀵�
+ while (matcher.find()) { headingPositions.add(matcher.start()); questions.add(matcher.group(1).trim()); }
+ if (CollUtil.isEmpty(headingPositions)) return qaPairs;
for (int i = 0; i < headingPositions.size(); i++) {
int start = headingPositions.get(i);
- int end = (i + 1 < headingPositions.size())
- ? headingPositions.get(i + 1)
- : content.length();
- String qaText = content.substring(start, end).trim();
+ int end = (i + 1 < headingPositions.size()) ? headingPositions.get(i + 1) : content.length();
+ String fullText = content.substring(start, end).trim();
String question = questions.get(i);
- // 鎻愬彇绛旀閮ㄥ垎锛堝幓鎺夐棶棰樻爣棰橈級
- String answer = qaText.substring(qaText.indexOf('\n') + 1).trim();
- qaPairs.add(new QaPair(question, answer, qaText));
+ String answer = fullText.substring(fullText.indexOf('\n') + 1).trim();
+ qaPairs.add(new QaPair(question, answer, fullText));
}
return qaPairs;
}
- /**
- * 鍒囧垎鍗曚釜 QA 瀵�
- *
- * @param qaPair QA 瀵�
- * @return 鍒囧垎鍚庣殑鏂囨湰鐗囨鍒楄〃
- */
private List<String> splitQaPair(QaPair qaPair) {
- // 濡傛灉鏁翠釜 QA 瀵逛笉瓒呰繃闄愬埗锛屼繚鎸佸畬鏁�
+ int qaTokens = tokenEstimator.estimate(qaPair.fullText);
+ if (qaTokens <= chunkSize) return Collections.singletonList(qaPair.fullText);
List<String> chunks = new ArrayList<>();
- String fullQa = qaPair.fullText;
- int qaTokens = tokenEstimator.estimate(fullQa);
- if (qaTokens <= chunkSize) {
- chunks.add(fullQa);
- return chunks;
- }
-
- // 闀跨瓟妗堥渶瑕佸垏鍒�
- log.debug("QA 瀵硅秴杩� Token 闄愬埗 ({} > {})锛屽紑濮嬫櫤鑳藉垏鍒�: {}", qaTokens, chunkSize, qaPair.question);
List<String> answerChunks = splitLongAnswer(qaPair.answer, qaPair.question);
for (String answerChunk : answerChunks) {
- // 姣忎釜鐗囨閮藉寘鍚畬鏁撮棶棰�
- String chunkText = "## " + qaPair.question + "\n" + answerChunk;
- chunks.add(chunkText);
+ chunks.add("## " + qaPair.question + "\n" + answerChunk);
}
return chunks;
}
- /**
- * 鍒囧垎闀跨瓟妗�
- *
- * @param answer 绛旀鏂囨湰
- * @param question 闂鏂囨湰
- * @return 鍒囧垎鍚庣殑绛旀鐗囨鍒楄〃
- */
private List<String> splitLongAnswer(String answer, String question) {
List<String> chunks = new ArrayList<>();
- // 棰勭暀闂鐨� Token 绌洪棿
String questionHeader = "## " + question + "\n";
int questionTokens = tokenEstimator.estimate(questionHeader);
- int availableTokens = chunkSize - questionTokens - 10; // 棰勭暀 10 涓� Token 鐨勭紦鍐�
-
- // 鍏堟寜娈佃惤鍒囧垎
+ int availableTokens = chunkSize - questionTokens - 10;
String[] paragraphs = answer.split(PARAGRAPH_SEPARATOR);
StringBuilder currentChunk = new StringBuilder();
int currentTokens = 0;
for (String paragraph : paragraphs) {
- if (StrUtil.isEmpty(paragraph)) {
- continue;
- }
+ if (StrUtil.isEmpty(paragraph)) continue;
int paragraphTokens = tokenEstimator.estimate(paragraph);
- // 濡傛灉鍗曚釜娈佃惤灏辫秴杩囬檺鍒讹紝闇�瑕佹寜鍙ュ瓙鍒囧垎
if (paragraphTokens > availableTokens) {
- // 鍏堜繚瀛樺綋鍓嶅潡
- if (currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- currentChunk = new StringBuilder();
- currentTokens = 0;
- }
- // 鎸夊彞瀛愬垏鍒嗛暱娈佃惤
+ if (currentChunk.length() > 0) { chunks.add(currentChunk.toString().trim()); currentChunk = new StringBuilder(); currentTokens = 0; }
chunks.addAll(splitLongParagraph(paragraph, availableTokens));
continue;
}
- // 濡傛灉鍔犱笂杩欎釜娈佃惤浼氳秴杩囬檺鍒�
- if (currentTokens + paragraphTokens > availableTokens && currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- currentChunk = new StringBuilder();
- currentTokens = 0;
- }
- if (currentChunk.length() > 0) {
- currentChunk.append("\n\n");
- }
- // 娣诲姞娈佃惤
+ if (currentTokens + paragraphTokens > availableTokens && currentChunk.length() > 0) { chunks.add(currentChunk.toString().trim()); currentChunk = new StringBuilder(); currentTokens = 0; }
+ if (currentChunk.length() > 0) currentChunk.append("\n\n");
currentChunk.append(paragraph);
currentTokens += paragraphTokens;
}
-
- // 娣诲姞鏈�鍚庝竴鍧�
- if (currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- }
+ if (currentChunk.length() > 0) chunks.add(currentChunk.toString().trim());
return CollUtil.isEmpty(chunks) ? Collections.singletonList(answer) : chunks;
}
- /**
- * 鍒囧垎闀挎钀斤紙鎸夊彞瀛愶級
- *
- * @param paragraph 娈佃惤鏂囨湰
- * @param availableTokens 鍙敤鐨� Token 鏁�
- * @return 鍒囧垎鍚庣殑鏂囨湰鐗囨鍒楄〃
- */
private List<String> splitLongParagraph(String paragraph, int availableTokens) {
- // 鎸夊彞瀛愬垏鍒�
List<String> chunks = new ArrayList<>();
String[] sentences = SENTENCE_PATTERN.split(paragraph);
-
- // 鎸夊彞瀛愮疮绉垏鍒�
StringBuilder currentChunk = new StringBuilder();
int currentTokens = 0;
for (String sentence : sentences) {
- if (StrUtil.isEmpty(sentence)) {
- continue;
- }
+ if (StrUtil.isEmpty(sentence)) continue;
int sentenceTokens = tokenEstimator.estimate(sentence);
- // 濡傛灉鍗曚釜鍙ュ瓙灏辫秴杩囬檺鍒讹紝寮哄埗鍒囧垎
- if (sentenceTokens > availableTokens) {
- if (currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- currentChunk = new StringBuilder();
- currentTokens = 0;
- }
- chunks.add(sentence.trim());
- continue;
- }
- // 濡傛灉鍔犱笂杩欎釜鍙ュ瓙浼氳秴杩囬檺鍒�
- if (currentTokens + sentenceTokens > availableTokens && currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- currentChunk = new StringBuilder();
- currentTokens = 0;
- }
- // 娣诲姞鍙ュ瓙
+ if (sentenceTokens > availableTokens) { if (currentChunk.length() > 0) { chunks.add(currentChunk.toString().trim()); currentChunk = new StringBuilder(); currentTokens = 0; } chunks.add(sentence.trim()); continue; }
+ if (currentTokens + sentenceTokens > availableTokens && currentChunk.length() > 0) { chunks.add(currentChunk.toString().trim()); currentChunk = new StringBuilder(); currentTokens = 0; }
currentChunk.append(sentence);
currentTokens += sentenceTokens;
}
-
- // 娣诲姞鏈�鍚庝竴鍧�
- if (currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- }
+ if (currentChunk.length() > 0) chunks.add(currentChunk.toString().trim());
return chunks.isEmpty() ? Collections.singletonList(paragraph) : chunks;
}
- /**
- * 闄嶇骇鍒囧垎绛栫暐锛堝綋鏈瘑鍒埌 QA 鏍煎紡鏃讹級
- *
- * @param content 鏂囨湰鍐呭
- * @return 鍒囧垎鍚庣殑鏂囨湰鐗囨鍒楄〃
- */
private List<String> fallbackSplit(String content) {
- // 鎸夋钀藉垏鍒�
List<String> chunks = new ArrayList<>();
String[] paragraphs = content.split(PARAGRAPH_SEPARATOR);
-
- // 鎸夋钀界疮绉垏鍒�
StringBuilder currentChunk = new StringBuilder();
int currentTokens = 0;
for (String paragraph : paragraphs) {
- if (StrUtil.isEmpty(paragraph)) {
- continue;
- }
+ if (StrUtil.isEmpty(paragraph)) continue;
int paragraphTokens = tokenEstimator.estimate(paragraph);
- // 濡傛灉鍔犱笂杩欎釜娈佃惤浼氳秴杩囬檺鍒�
- if (currentTokens + paragraphTokens > chunkSize && currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- currentChunk = new StringBuilder();
- currentTokens = 0;
- }
- // 娣诲姞娈佃惤
- if (currentChunk.length() > 0) {
- currentChunk.append("\n\n");
- }
+ if (currentTokens + paragraphTokens > chunkSize && currentChunk.length() > 0) { chunks.add(currentChunk.toString().trim()); currentChunk = new StringBuilder(); currentTokens = 0; }
+ if (currentChunk.length() > 0) currentChunk.append("\n\n");
currentChunk.append(paragraph);
currentTokens += paragraphTokens;
}
-
- // 娣诲姞鏈�鍚庝竴鍧�
- if (currentChunk.length() > 0) {
- chunks.add(currentChunk.toString().trim());
- }
+ if (currentChunk.length() > 0) chunks.add(currentChunk.toString().trim());
return chunks.isEmpty() ? Collections.singletonList(content) : chunks;
}
- /**
- * QA 瀵规暟鎹粨鏋�
- */
@AllArgsConstructor
- private static class QaPair {
+ private static class QaPair { String question; String answer; String fullText; }
- String question;
- String answer;
- String fullText;
+ public interface TokenEstimator { int estimate(String text); }
- }
-
- /**
- * Token 浼扮畻鍣ㄦ帴鍙�
- */
- public interface TokenEstimator {
-
- int estimate(String text);
-
- }
-
- /**
- * 绠�鍗曠殑 Token 浼扮畻鍣ㄥ疄鐜�
- * 涓枃锛�1 瀛楃 鈮� 1 Token
- * 鑻辨枃锛�1 鍗曡瘝 鈮� 1.3 Token
- */
private static class SimpleTokenEstimator implements TokenEstimator {
-
@Override
public int estimate(String text) {
- if (StrUtil.isEmpty(text)) {
- return 0;
- }
-
- int chineseChars = 0;
- int englishWords = 0;
- // 绠�鍗曠粺璁′腑鑻辨枃
- for (char c : text.toCharArray()) {
- if (c >= 0x4E00 && c <= 0x9FA5) {
- chineseChars++;
- }
- }
- // 鑻辨枃鍗曡瘝浼扮畻
- String[] words = text.split("\\s+");
- for (String word : words) {
- if (word.matches(".*[a-zA-Z].*")) {
- englishWords++;
- }
- }
- return chineseChars + (int) (englishWords * 1.3);
+ if (StrUtil.isEmpty(text)) return 0;
+ int chineseChars = 0, englishWords = 0;
+ for (char c : text.toCharArray()) { if (c >= 0x4E00 && c <= 0x9FA5) chineseChars++; }
+ for (String word : text.split("\\s+")) { if (word.matches(".*[a-zA-Z].*")) englishWords++; }
+ return chineseChars + (int)(englishWords * 1.3);
}
}
-
}
--
Gitblit v1.9.3