From 83e1b4d0e661f11a407fd6ea86e906b9b87b7180 Mon Sep 17 00:00:00 2001
From: 云 <2163098428@qq.com>
Date: 星期五, 31 七月 2026 17:57:09 +0800
Subject: [PATCH] feat(aftersales): 售后工单新增问题类型和严重程度字段
---
yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java | 227 ++++++--------------------------------------------------
1 files changed, 27 insertions(+), 200 deletions(-)
diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java
index 4c7112e..401dc64 100644
--- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java
+++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java
@@ -3,7 +3,6 @@
import cn.hutool.core.util.StrUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.transformer.splitter.TextSplitter;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -13,289 +12,117 @@
/**
* 璇箟鍖栨枃鏈垏鐗囧櫒
- *
- * <p>鍔熻兘鐗圭偣锛�
- * <ul>
- * <li>浼樺厛鍦ㄦ钀借竟鐣岋紙鍙屾崲琛岋級澶勫垏鍒�</li>
- * <li>鍏舵鍦ㄥ彞瀛愯竟鐣岋紙鍙ュ彿銆侀棶鍙枫�佹劅鍙瑰彿锛夊鍒囧垎</li>
- * <li>閬垮厤鍦ㄥ彞瀛愪腑闂存埅鏂紝淇濇寔璇箟瀹屾暣鎬�</li>
- * <li>鏀寔涓嫳鏂囨爣鐐圭鍙疯瘑鍒�</li>
- * </ul>
- *
- * @author runzhen
+ * 浼樺厛鍦ㄦ钀借竟鐣屽鍒囧垎锛屽叾娆″湪鍙ュ瓙杈圭晫澶勫垏鍒嗭紝淇濇寔璇箟瀹屾暣鎬�
*/
@Slf4j
public class SemanticTextSplitter extends TextSplitter {
- /**
- * 鍒嗘鐨勬渶澶� Token 鏁�
- */
private final int chunkSize;
-
- /**
- * 娈佃惤閲嶅彔澶у皬锛堢敤浜庝繚鎸佷笂涓嬫枃杩炶疮鎬э級
- */
private final int chunkOverlap;
-
- /**
- * 娈佃惤鍒嗛殧绗︼紙鎸変紭鍏堢骇鎺掑簭锛�
- */
- private static final List<String> PARAGRAPH_SEPARATORS = Arrays.asList(
- "\n\n\n", // 涓変釜鎹㈣
- "\n\n", // 鍙屾崲琛�
- "\n" // 鍗曟崲琛�
- );
-
- /**
- * 鍙ュ瓙缁撴潫鏍囪锛堜腑鑻辨枃鏍囩偣锛�
- */
- private static final Pattern SENTENCE_END_PATTERN = Pattern.compile(
- "[銆傦紒锛�.!?]+[\\s\"'锛�)銆慭\]]*"
- );
-
- /**
- * Token 浼扮畻鍣�
- */
+ private static final List<String> PARAGRAPH_SEPARATORS = Arrays.asList("\n\n\n", "\n\n", "\n");
+ private static final Pattern SENTENCE_END_PATTERN = Pattern.compile("[銆傦紒锛�.!?]+[\\s\"'锛�)銆慭\]]*");
private final MarkdownQaSplitter.TokenEstimator tokenEstimator;
public SemanticTextSplitter(int chunkSize, int chunkOverlap) {
this.chunkSize = chunkSize;
- this.chunkOverlap = Math.min(chunkOverlap, chunkSize / 2); // 閲嶅彔涓嶈秴杩囦竴鍗�
+ this.chunkOverlap = Math.min(chunkOverlap, chunkSize / 2);
this.tokenEstimator = new SimpleTokenEstimator();
}
- public SemanticTextSplitter(int chunkSize) {
- this(chunkSize, 50); // 榛樿閲嶅彔 50 涓� Token
- }
+ public SemanticTextSplitter(int chunkSize) { this(chunkSize, 50); }
@Override
protected List<String> splitText(String text) {
- if (StrUtil.isEmpty(text)) {
- return Collections.emptyList();
- }
+ if (StrUtil.isEmpty(text)) return Collections.emptyList();
return splitTextRecursive(text);
}
- /**
- * 鍒囧垎鏂囨湰锛堥�掑綊绛栫暐锛�
- *
- * @param text 寰呭垏鍒嗘枃鏈�
- * @return 鍒囧垎鍚庣殑鏂囨湰鍧楀垪琛�
- */
private List<String> splitTextRecursive(String text) {
- List<String> chunks = new ArrayList<>();
-
- // 濡傛灉鏂囨湰涓嶈秴杩囬檺鍒讹紝鐩存帴杩斿洖
int textTokens = tokenEstimator.estimate(text);
- if (textTokens <= chunkSize) {
- chunks.add(text.trim());
- return chunks;
- }
+ if (textTokens <= chunkSize) return Collections.singletonList(text.trim());
- // 灏濊瘯鎸変笉鍚屽垎闅旂鍒囧垎
List<String> splits = null;
String usedSeparator = null;
for (String separator : PARAGRAPH_SEPARATORS) {
- if (text.contains(separator)) {
- splits = Arrays.asList(text.split(Pattern.quote(separator)));
- usedSeparator = separator;
- break;
- }
+ if (text.contains(separator)) { splits = Arrays.asList(text.split(Pattern.quote(separator))); usedSeparator = separator; break; }
}
-
- // 濡傛灉娌℃湁鎵惧埌娈佃惤鍒嗛殧绗︼紝鎸夊彞瀛愬垏鍒�
- if (splits == null || splits.size() == 1) {
- splits = splitBySentences(text);
- usedSeparator = ""; // 鍙ュ瓙鍒囧垎涓嶉渶瑕佸垎闅旂
- }
-
- // 鍚堝苟灏忕墖娈�
- chunks = mergeSplits(splits, usedSeparator);
- return chunks;
+ if (splits == null || splits.size() == 1) { splits = splitBySentences(text); usedSeparator = ""; }
+ return mergeSplits(splits, usedSeparator);
}
- /**
- * 鎸夊彞瀛愬垏鍒�
- *
- * @param text 寰呭垏鍒嗘枃鏈�
- * @return 鍙ュ瓙鍒楄〃
- */
private List<String> splitBySentences(String text) {
- // 浣跨敤姝ｅ垯琛ㄨ揪寮忓尮閰嶅彞瀛愮粨鏉熶綅缃�
List<String> sentences = new ArrayList<>();
int lastEnd = 0;
Matcher matcher = SENTENCE_END_PATTERN.matcher(text);
while (matcher.find()) {
String sentence = text.substring(lastEnd, matcher.end()).trim();
- if (StrUtil.isNotEmpty(sentence)) {
- sentences.add(sentence);
- }
+ if (StrUtil.isNotEmpty(sentence)) sentences.add(sentence);
lastEnd = matcher.end();
}
-
- // 娣诲姞鍓╀綑閮ㄥ垎
- if (lastEnd < text.length()) {
- String remaining = text.substring(lastEnd).trim();
- if (StrUtil.isNotEmpty(remaining)) {
- sentences.add(remaining);
- }
- }
+ if (lastEnd < text.length()) { String remaining = text.substring(lastEnd).trim(); if (StrUtil.isNotEmpty(remaining)) sentences.add(remaining); }
return sentences.isEmpty() ? Collections.singletonList(text) : sentences;
}
- /**
- * 鍚堝苟鍒囧垎鍚庣殑灏忕墖娈�
- *
- * @param splits 鍒囧垎鍚庣殑鐗囨鍒楄〃
- * @param separator 鐗囨闂寸殑鍒嗛殧绗�
- * @return 鍚堝苟鍚庣殑鏂囨湰鍧楀垪琛�
- */
private List<String> mergeSplits(List<String> splits, String separator) {
List<String> chunks = new ArrayList<>();
List<String> currentChunks = new ArrayList<>();
int currentLength = 0;
-
for (String split : splits) {
- if (StrUtil.isEmpty(split)) {
- continue;
- }
+ if (StrUtil.isEmpty(split)) continue;
int splitTokens = tokenEstimator.estimate(split);
- // 濡傛灉鍗曚釜鐗囨灏辫秴杩囬檺鍒讹紝杩涗竴姝ラ�掑綊鍒囧垎
if (splitTokens > chunkSize) {
- // 鍏堜繚瀛樺綋鍓嶇疮绉殑鍧�
- if (!currentChunks.isEmpty()) {
- String chunkText = String.join(separator, currentChunks);
- chunks.add(chunkText.trim());
- currentChunks.clear();
- currentLength = 0;
- }
- // 閫掑綊鍒囧垎澶х墖娈�
- if (!separator.isEmpty()) {
- // 濡傛灉鏄钀藉垎闅旂锛屽皾璇曟寜鍙ュ瓙鍒囧垎
- chunks.addAll(splitTextRecursive(split));
- } else {
- // 濡傛灉宸茬粡鏄彞瀛愮骇鍒紝寮哄埗鎸夊瓧绗﹀垏鍒�
- chunks.addAll(forceSplitLongText(split));
- }
+ if (!currentChunks.isEmpty()) { chunks.add(String.join(separator, currentChunks).trim()); currentChunks.clear(); currentLength = 0; }
+ if (!separator.isEmpty()) { chunks.addAll(splitTextRecursive(split)); } else { chunks.addAll(forceSplitLongText(split)); }
continue;
}
- // 璁＄畻鍔犱笂鍒嗛殧绗︾殑 Token 鏁�
int separatorTokens = StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator);
- // 濡傛灉鍔犱笂杩欎釜鐗囨浼氳秴杩囬檺鍒�
if (!currentChunks.isEmpty() && currentLength + splitTokens + separatorTokens > chunkSize) {
- // 淇濆瓨褰撳墠鍧�
- String chunkText = String.join(separator, currentChunks);
- chunks.add(chunkText.trim());
-
- // 澶勭悊閲嶅彔锛氫繚鐣欐渶鍚庡嚑涓墖娈�
+ chunks.add(String.join(separator, currentChunks).trim());
currentChunks = getOverlappingChunks(currentChunks, separator);
currentLength = estimateTokens(currentChunks, separator);
}
- // 娣诲姞褰撳墠鐗囨
currentChunks.add(split);
currentLength += splitTokens + separatorTokens;
}
-
- // 娣诲姞鏈�鍚庝竴鍧�
- if (!currentChunks.isEmpty()) {
- String chunkText = String.join(separator, currentChunks);
- chunks.add(chunkText.trim());
- }
+ if (!currentChunks.isEmpty()) chunks.add(String.join(separator, currentChunks).trim());
return chunks;
}
- /**
- * 鑾峰彇閲嶅彔鐨勭墖娈碉紙鐢ㄤ簬淇濇寔涓婁笅鏂囷級
- *
- * @param chunks 褰撳墠鐗囨鍒楄〃
- * @param separator 鐗囨闂寸殑鍒嗛殧绗�
- * @return 閲嶅彔鐨勭墖娈靛垪琛�
- */
private List<String> getOverlappingChunks(List<String> chunks, String separator) {
- if (chunkOverlap == 0 || chunks.isEmpty()) {
- return new ArrayList<>();
- }
-
- // 浠庡悗寰�鍓嶅彇鐗囨锛岀洿鍒拌揪鍒伴噸鍙犲ぇ灏�
+ if (chunkOverlap == 0 || chunks.isEmpty()) return new ArrayList<>();
List<String> overlapping = new ArrayList<>();
int tokens = 0;
for (int i = chunks.size() - 1; i >= 0; i--) {
String chunk = chunks.get(i);
int chunkTokens = tokenEstimator.estimate(chunk);
- if (tokens + chunkTokens > chunkOverlap) {
- break;
- }
- // 娣诲姞鍒伴噸鍙犲垪琛ㄥ墠绔�
+ if (tokens + chunkTokens > chunkOverlap) break;
overlapping.add(0, chunk);
tokens += chunkTokens + (StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator));
}
return overlapping;
}
- /**
- * 浼扮畻鐗囨鍒楄〃鐨勬�� Token 鏁�
- *
- * @param chunks 鐗囨鍒楄〃
- * @param separator 鐗囨闂寸殑鍒嗛殧绗�
- * @return 鎬� Token 鏁�
- */
private int estimateTokens(List<String> chunks, String separator) {
int total = 0;
- for (int i = 0; i < chunks.size(); i++) {
- total += tokenEstimator.estimate(chunks.get(i));
- if (i < chunks.size() - 1 && StrUtil.isNotEmpty(separator)) {
- total += tokenEstimator.estimate(separator);
- }
- }
+ for (int i = 0; i < chunks.size(); i++) { total += tokenEstimator.estimate(chunks.get(i)); if (i < chunks.size() - 1 && StrUtil.isNotEmpty(separator)) total += tokenEstimator.estimate(separator); }
return total;
}
- /**
- * 寮哄埗鍒囧垎闀挎枃鏈紙褰撹涔夊垏鍒嗗け璐ユ椂锛�
- *
- * @param text 寰呭垏鍒嗘枃鏈�
- * @return 鍒囧垎鍚庣殑鏂囨湰鍧楀垪琛�
- */
private List<String> forceSplitLongText(String text) {
List<String> chunks = new ArrayList<>();
- int charsPerChunk = (int) (chunkSize * 0.8); // 淇濆畧浼拌
- for (int i = 0; i < text.length(); i += charsPerChunk) {
- int end = Math.min(i + charsPerChunk, text.length());
- String chunk = text.substring(i, end);
- chunks.add(chunk.trim());
- }
- log.warn("鏂囨湰杩囬暱锛屽凡寮哄埗鎸夊瓧绗﹀垏鍒嗭紝鍙兘褰卞搷璇箟瀹屾暣鎬�");
+ int charsPerChunk = (int)(chunkSize * 0.8);
+ for (int i = 0; i < text.length(); i += charsPerChunk) { int end = Math.min(i + charsPerChunk, text.length()); chunks.add(text.substring(i, end).trim()); }
return chunks;
}
- /**
- * 绠�鍗曠殑 Token 浼扮畻鍣ㄥ疄鐜�
- */
private static class SimpleTokenEstimator implements MarkdownQaSplitter.TokenEstimator {
-
@Override
public int estimate(String text) {
- if (StrUtil.isEmpty(text)) {
- return 0;
- }
-
- int chineseChars = 0;
- int englishWords = 0;
- // 绠�鍗曠粺璁′腑鑻辨枃
- for (char c : text.toCharArray()) {
- if (c >= 0x4E00 && c <= 0x9FA5) {
- chineseChars++;
- }
- }
- // 鑻辨枃鍗曡瘝浼扮畻
- String[] words = text.split("\\s+");
- for (String word : words) {
- if (word.matches(".*[a-zA-Z].*")) {
- englishWords++;
- }
- }
- return chineseChars + (int) (englishWords * 1.3);
+ if (StrUtil.isEmpty(text)) return 0;
+ int chineseChars = 0, englishWords = 0;
+ for (char c : text.toCharArray()) { if (c >= 0x4E00 && c <= 0x9FA5) chineseChars++; }
+ for (String word : text.split("\\s+")) { if (word.matches(".*[a-zA-Z].*")) englishWords++; }
+ return chineseChars + (int)(englishWords * 1.3);
}
}
-
}
--
Gitblit v1.9.3