Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

perf(search): reuse fixed query-expansion regexps (#4309)

钱晓缺36c40453fa6c
diff --git a/internal/application/service/chat_pipeline/query_expansion.go b/internal/application/service/chat_pipeline/query_expansion.go
index e6ca12712..3fed638a7 100644
--- a/internal/application/service/chat_pipeline/query_expansion.go
+++ b/internal/application/service/chat_pipeline/query_expansion.go
@@ -201,6 +201,11 @@ var stopwords = map[string]struct{}{
// Question words in Chinese
var questionWords = regexp.MustCompile(`^(什么是|什么|如何|怎么|怎样|为什么|为何|哪个|哪些|谁|何时|何地|请问|请告诉我|帮我|我想知道|我想了解)`)
+var (
+ quotedPhrasePattern = regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`)
+ delimiterPattern = regexp.MustCompile(`[,,;;、。!?!?\s]+`)
+)
+
func extractKeywords(text string) []string {
words := tokenize(text)
keywords := make([]string, 0, len(words))
@@ -216,8 +221,7 @@ func extractKeywords(text string) []string {
func extractPhrases(text string) []string {
// Extract quoted content
var phrases []string
- re := regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`)
- matches := re.FindAllStringSubmatch(text, -1)
+ matches := quotedPhrasePattern.FindAllStringSubmatch(text, -1)
for _, m := range matches {
if len(m) > 1 && len(m[1]) > 2 {
phrases = append(phrases, m[1])
@@ -228,8 +232,7 @@ func extractPhrases(text string) []string {
func splitByDelimiters(text string) []string {
// Split by common delimiters
- re := regexp.MustCompile(`[,,;;、。!?!?\s]+`)
- parts := re.Split(text, -1)
+ parts := delimiterPattern.Split(text, -1)
var result []string
for _, p := range parts {
p = strings.TrimSpace(p)
diff --git a/internal/application/service/chat_pipeline/query_expansion_test.go b/internal/application/service/chat_pipeline/query_expansion_test.go
index 4aeddf30b..f9722acca 100644
--- a/internal/application/service/chat_pipeline/query_expansion_test.go
+++ b/internal/application/service/chat_pipeline/query_expansion_test.go
@@ -66,6 +66,58 @@ func containsToken(values []string, want string) bool {
return false
}
+func TestExtractPhrases(t *testing.T) {
+ for _, tc := range []struct {
+ text string
+ want []string
+ }{
+ {"", nil},
+ {"plain text", nil},
+ {`"RAG search" and 'Go SDK'`, []string{"RAG search", "Go SDK"}},
+ {"「向量数据库」和『检索配置』", []string{"向量数据库", "检索配置"}},
+ {`"ab" "abc"`, []string{"abc"}},
+ {`"unterminated`, nil},
+ } {
+ t.Run(tc.text, func(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, tc.want, extractPhrases(tc.text))
+ })
+ }
+}
+
+func TestSplitByDelimiters(t *testing.T) {
+ for _, tc := range []struct {
+ text string
+ want []string
+ }{
+ {"", nil},
+ {" RAG search\tGo\nSDK ", []string{"RAG", "search", "Go", "SDK"}},
+ {"向量,数据库;检索、配置。测试!完成?", []string{"向量", "数据库", "检索", "配置", "测试", "完成"}},
+ {"one,two;three!four?five", []string{"one", "two", "three", "four", "five"}},
+ {",; \n!?", nil},
+ {"RAG-search", []string{"RAG-search"}},
+ } {
+ t.Run(tc.text, func(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, tc.want, splitByDelimiters(tc.text))
+ })
+ }
+}
+
+func BenchmarkExtractPhrases(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ extractPhrases(`如何配置「向量数据库」和 "RAG search"`)
+ }
+}
+
+func BenchmarkSplitByDelimiters(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ splitByDelimiters("向量数据库,RAG search;检索配置?")
+ }
+}
+
type recordingExpansionKBService struct {
interfaces.KnowledgeBaseService
mu sync.Mutex