RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(nlp): restore Infinity-safe escaping for synonyms and special chars (#20572)
li872b45c1cd1bcd0
diff --git a/internal/service/nlp/query_builder.go b/internal/service/nlp/query_builder.goindex f0bbe73462..3b9fb59812 100644--- a/internal/service/nlp/query_builder.go+++ b/internal/service/nlp/query_builder.go@@ -51,7 +51,7 @@ type QueryBuilder struct {// constants, so they are compiled once at package init.var (reEngAlpha = regexp.MustCompile(`^[a-zA-Z]+$`)- reSubSpecialChar = regexp.MustCompile(`([:{}/\[\]\-\*"\(\)\|\+~\^])`)+ reSubSpecialChar = regexp.MustCompile(`([:{}/\[\]\-\*\?"\(\)\|\+~\^])`)reStopWordsZH = regexp.MustCompile(`(?i)是*(怎么办|什么样的|哪家|一下|那家|请问|啥样|咋样了|什么时候|何时|何地|何人|是否|是不是|多少|哪里|怎么|哪儿|怎么样|如何|哪些|是啥|啥是|啊|吗|呢|吧|咋|什么|有没有|呀|谁|哪位|哪个)是*`)reStopWordsEN1 = regexp.MustCompile(`(?i)(^| )(what|who|how|which|where|why)('re|'s)? `)reStopWordsEN2 = regexp.MustCompile(`(?i)(^| )('s|'re|is|are|were|was|do|does|did|don't|doesn't|didn't|has|have|be|there|you|me|your|my|mine|just|please|may|i|should|would|wouldn't|will|won't|done|go|for|with|so|the|a|an|by|i'm|it's|he's|she's|they|they're|you're|as|by|on|in|at|up|out|down|of|to|or|and|if) `)@@ -142,8 +142,23 @@ func (qb *QueryBuilder) IsChinese(line string) bool {// SubSpecialChar escapes special characters for use in queries.func (qb *QueryBuilder) SubSpecialChar(line string) string {- // Regex matches : { } / [ ] - * " ( ) | + ~ ^ and prepends backslash- return reSubSpecialChar.ReplaceAllString(line, `\$1`)+ // Strip single quotes first so Infinity's lexer does not treat them as+ // string delimiters, then escape : { } / [ ] - * ? " ( ) | + ~ ^ and trim.+ // Reference: common/query_base.py sub_special_char+ line = strings.ReplaceAll(line, "'", "")+ return strings.TrimSpace(reSubSpecialChar.ReplaceAllString(line, `\$1`))+}++// cleanSynonym tokenizes a synonym and strips single quotes so it can be+// embedded in a full-text query. WordNet returns e.g. "cat-o'-nine-tails"+// for "cat", and a raw single quote breaks the Infinity query parser.+// Reference: rag/nlp/query.py question(), rag_tokenizer.tokenize(s).replace("'", "")+func cleanSynonym(s string) string {+ tokenized, err := tokenizer.Tokenize(s)+ if err != nil {+ tokenized = s+ }+ return strings.TrimSpace(strings.ReplaceAll(tokenized, "'", ""))}// RmWWW removes common stop words and question words from queries.@@ -327,14 +342,17 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ// Format synonyms with weight boost: term^weightvar synParts []stringfor _, syn := range tkSyns {- syn = strings.TrimSpace(syn)+ syn = cleanSynonym(syn)if syn != "" {+ // Do not escape here. Python's English branch embeds the+ // cleaned synonym as-is; SubSpecialChar would also escape '-',+ // which WordNet synonyms contain constantly.synParts = append(synParts, fmt.Sprintf(`"%s"^%.4f`, syn, tw.w/4.0))+ // Extend keywords with cleaned synonyms+ keywords = append(keywords, syn)}}syns[i] = strings.Join(synParts, " ")- // Extend keywords with synonyms- keywords = append(keywords, tkSyns...)} else {syns[i] = ""}diff --git a/internal/service/nlp/query_builder_test.go b/internal/service/nlp/query_builder_test.goindex 30ee061a9c..33233af3e1 100644--- a/internal/service/nlp/query_builder_test.go+++ b/internal/service/nlp/query_builder_test.go@@ -16,9 +16,11 @@ package nlpimport ("reflect"+ "strings""testing""ragflow/internal/engine/types"+ "ragflow/internal/tokenizer")func TestNewQueryBuilder(t *testing.T) {@@ -97,6 +99,10 @@ func TestQueryBuilder_SubSpecialChar(t *testing.T) {{"Tilde", "~tilde", `\~tilde`},{"Caret", "^caret", `\^caret`},{"Multiple", `:{}/[]-*"()|+~^`, `\:\{\}\/\[\]\-\*\"\(\)\|\+\~\^`},+ {"Single quote removed", "it's", "its"},+ {"Question mark", "a?b", `a\?b`},+ {"Trim surrounding spaces", " x ", "x"},+ {"Quote stripped then trimmed", " ' x ' ", "x"},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {@@ -443,3 +449,95 @@ func TestQueryBuilder_SetQueryFields(t *testing.T) {t.Errorf("Paragraph fields not updated after SetQueryFields, got %v, want %v", expr.Fields, newFields)}}++// TestQueryBuilder_Question_SynonymSingleQuote ensures English synonyms are+// cleaned of single quotes before being embedded in the full-text query.+// WordNet returns e.g. "cat-o'-nine-tails" for "cat", and a raw single quote+// breaks the Infinity query parser (see #13823).+//+// The Infinity engine's Tokenize returns the input unchanged, so quote+// stripping is done by cleanSynonym, not by the analyzer. Set the engine+// explicitly so the test does not pass via cleanSynonym's error fallback+// when the tokenizer pool is uninitialized.+func TestQueryBuilder_Question_SynonymSingleQuote(t *testing.T) {+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")+ tok, err := tokenizer.Tokenize("cat-o'-nine-tails")+ if err != nil {+ t.Fatalf("Tokenize: %v", err)+ }+ if tok != "cat-o'-nine-tails" {+ t.Fatalf("infinity Tokenize = %q, want input unchanged", tok)+ }++ qb := NewQueryBuilder()+ qb.synonym.dictionary = map[string][]string{+ "cat": {"cat-o'-nine-tails", "'"},+ }+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)+ if expr == nil {+ t.Fatal("Question returned nil expr")+ }+ if strings.Contains(expr.MatchingText, "'") {+ t.Errorf("MatchingText contains single quote: %q", expr.MatchingText)+ }+ if strings.Contains(expr.MatchingText, `""`) {+ t.Errorf("MatchingText contains empty synonym phrase: %q", expr.MatchingText)+ }+ if !strings.Contains(expr.MatchingText, `"cat-o-nine-tails"`) {+ t.Errorf("MatchingText missing cleaned synonym: %q", expr.MatchingText)+ }+ if strings.Contains(expr.MatchingText, `\`) {+ t.Errorf("MatchingText escaped a synonym character: %q", expr.MatchingText)+ }+ for _, kw := range keywords {+ if strings.Contains(kw, "'") || strings.TrimSpace(kw) == "" {+ t.Errorf("keyword not cleaned: %q (all: %q)", kw, keywords)+ }+ }+}++// TestQueryBuilder_Question_SynonymQuestionMark checks a synonym that contains+// "?" once the Infinity tokenizer is initialized. SetEngineType("infinity")+// makes Tokenize return the input unchanged, so "?" is preserved (the C+++// analyzer, used only when the engine is not "infinity", rewrites "what?cat"+// to "what cat"). Python's English branch does not escape "?", and neither+// does this path.+func TestQueryBuilder_Question_SynonymQuestionMark(t *testing.T) {+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")+ tok, err := tokenizer.Tokenize("what?cat")+ if err != nil {+ t.Fatalf("Tokenize: %v", err)+ }+ if tok != "what?cat" {+ t.Fatalf("infinity Tokenize = %q, want input unchanged", tok)+ }++ qb := NewQueryBuilder()+ qb.synonym.dictionary = map[string][]string{+ "cat": {"what?cat"},+ }+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)+ if expr == nil {+ t.Fatal("Question returned nil expr")+ }+ if !strings.Contains(expr.MatchingText, `"what?cat"`) {+ t.Errorf("MatchingText missing unescaped synonym: %q", expr.MatchingText)+ }+ if strings.Contains(expr.MatchingText, `\`) {+ t.Errorf("MatchingText should not escape ?: %q", expr.MatchingText)+ }+ found := false+ for _, kw := range keywords {+ if strings.Contains(kw, `\`) {+ t.Errorf("keyword should not be escaped: %q", kw)+ }+ if kw == "what?cat" {+ found = true+ }+ }+ if !found {+ t.Errorf("keywords missing cleaned synonym %q: %q", "what?cat", keywords)+ }+}