Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
perf(search): reuse fixed query-expansion regexps (#4309)
钱晓缺36c40453fa6c
diff --git a/internal/application/service/chat_pipeline/query_expansion.go b/internal/application/service/chat_pipeline/query_expansion.goindex e6ca12712..3fed638a7 100644--- a/internal/application/service/chat_pipeline/query_expansion.go+++ b/internal/application/service/chat_pipeline/query_expansion.go@@ -201,6 +201,11 @@ var stopwords = map[string]struct{}{// Question words in Chinesevar questionWords = regexp.MustCompile(`^(什么是|什么|如何|怎么|怎样|为什么|为何|哪个|哪些|谁|何时|何地|请问|请告诉我|帮我|我想知道|我想了解)`)+var (+ quotedPhrasePattern = regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`)+ delimiterPattern = regexp.MustCompile(`[,,;;、。!?!?\s]+`)+)+func extractKeywords(text string) []string {words := tokenize(text)keywords := make([]string, 0, len(words))@@ -216,8 +221,7 @@ func extractKeywords(text string) []string {func extractPhrases(text string) []string {// Extract quoted contentvar phrases []string- re := regexp.MustCompile(`["'"'「」『』]([^"'"'「」『』]+)["'"'「」『』]`)- matches := re.FindAllStringSubmatch(text, -1)+ matches := quotedPhrasePattern.FindAllStringSubmatch(text, -1)for _, m := range matches {if len(m) > 1 && len(m[1]) > 2 {phrases = append(phrases, m[1])@@ -228,8 +232,7 @@ func extractPhrases(text string) []string {func splitByDelimiters(text string) []string {// Split by common delimiters- re := regexp.MustCompile(`[,,;;、。!?!?\s]+`)- parts := re.Split(text, -1)+ parts := delimiterPattern.Split(text, -1)var result []stringfor _, p := range parts {p = strings.TrimSpace(p)diff --git a/internal/application/service/chat_pipeline/query_expansion_test.go b/internal/application/service/chat_pipeline/query_expansion_test.goindex 4aeddf30b..f9722acca 100644--- a/internal/application/service/chat_pipeline/query_expansion_test.go+++ b/internal/application/service/chat_pipeline/query_expansion_test.go@@ -66,6 +66,58 @@ func containsToken(values []string, want string) bool {return false}+func TestExtractPhrases(t *testing.T) {+ for _, tc := range []struct {+ text string+ want []string+ }{+ {"", nil},+ {"plain text", nil},+ {`"RAG search" and 'Go SDK'`, []string{"RAG search", "Go SDK"}},+ {"「向量数据库」和『检索配置』", []string{"向量数据库", "检索配置"}},+ {`"ab" "abc"`, []string{"abc"}},+ {`"unterminated`, nil},+ } {+ t.Run(tc.text, func(t *testing.T) {+ t.Parallel()+ assert.Equal(t, tc.want, extractPhrases(tc.text))+ })+ }+}++func TestSplitByDelimiters(t *testing.T) {+ for _, tc := range []struct {+ text string+ want []string+ }{+ {"", nil},+ {" RAG search\tGo\nSDK ", []string{"RAG", "search", "Go", "SDK"}},+ {"向量,数据库;检索、配置。测试!完成?", []string{"向量", "数据库", "检索", "配置", "测试", "完成"}},+ {"one,two;three!four?five", []string{"one", "two", "three", "four", "five"}},+ {",; \n!?", nil},+ {"RAG-search", []string{"RAG-search"}},+ } {+ t.Run(tc.text, func(t *testing.T) {+ t.Parallel()+ assert.Equal(t, tc.want, splitByDelimiters(tc.text))+ })+ }+}++func BenchmarkExtractPhrases(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ extractPhrases(`如何配置「向量数据库」和 "RAG search"`)+ }+}++func BenchmarkSplitByDelimiters(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ splitByDelimiters("向量数据库,RAG search;检索配置?")+ }+}+type recordingExpansionKBService struct {interfaces.KnowledgeBaseServicemu sync.Mutex