Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(nlp): restore Infinity-safe escaping for synonyms and special chars (#20572)

li872b45c1cd1bcd0
diff --git a/internal/service/nlp/query_builder.go b/internal/service/nlp/query_builder.go
index f0bbe73462..3b9fb59812 100644
--- a/internal/service/nlp/query_builder.go
+++ b/internal/service/nlp/query_builder.go
@@ -51,7 +51,7 @@ type QueryBuilder struct {
// constants, so they are compiled once at package init.
var (
reEngAlpha = regexp.MustCompile(`^[a-zA-Z]+$`)
- reSubSpecialChar = regexp.MustCompile(`([:{}/\[\]\-\*"\(\)\|\+~\^])`)
+ reSubSpecialChar = regexp.MustCompile(`([:{}/\[\]\-\*\?"\(\)\|\+~\^])`)
reStopWordsZH = regexp.MustCompile(`(?i)是*(怎么办|什么样的|哪家|一下|那家|请问|啥样|咋样了|什么时候|何时|何地|何人|是否|是不是|多少|哪里|怎么|哪儿|怎么样|如何|哪些|是啥|啥是|啊|吗|呢|吧|咋|什么|有没有|呀|谁|哪位|哪个)是*`)
reStopWordsEN1 = regexp.MustCompile(`(?i)(^| )(what|who|how|which|where|why)('re|'s)? `)
reStopWordsEN2 = regexp.MustCompile(`(?i)(^| )('s|'re|is|are|were|was|do|does|did|don't|doesn't|didn't|has|have|be|there|you|me|your|my|mine|just|please|may|i|should|would|wouldn't|will|won't|done|go|for|with|so|the|a|an|by|i'm|it's|he's|she's|they|they're|you're|as|by|on|in|at|up|out|down|of|to|or|and|if) `)
@@ -142,8 +142,23 @@ func (qb *QueryBuilder) IsChinese(line string) bool {
// SubSpecialChar escapes special characters for use in queries.
func (qb *QueryBuilder) SubSpecialChar(line string) string {
- // Regex matches : { } / [ ] - * " ( ) | + ~ ^ and prepends backslash
- return reSubSpecialChar.ReplaceAllString(line, `\$1`)
+ // Strip single quotes first so Infinity's lexer does not treat them as
+ // string delimiters, then escape : { } / [ ] - * ? " ( ) | + ~ ^ and trim.
+ // Reference: common/query_base.py sub_special_char
+ line = strings.ReplaceAll(line, "'", "")
+ return strings.TrimSpace(reSubSpecialChar.ReplaceAllString(line, `\$1`))
+}
+
+// cleanSynonym tokenizes a synonym and strips single quotes so it can be
+// embedded in a full-text query. WordNet returns e.g. "cat-o'-nine-tails"
+// for "cat", and a raw single quote breaks the Infinity query parser.
+// Reference: rag/nlp/query.py question(), rag_tokenizer.tokenize(s).replace("'", "")
+func cleanSynonym(s string) string {
+ tokenized, err := tokenizer.Tokenize(s)
+ if err != nil {
+ tokenized = s
+ }
+ return strings.TrimSpace(strings.ReplaceAll(tokenized, "'", ""))
}
// RmWWW removes common stop words and question words from queries.
@@ -327,14 +342,17 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
// Format synonyms with weight boost: term^weight
var synParts []string
for _, syn := range tkSyns {
- syn = strings.TrimSpace(syn)
+ syn = cleanSynonym(syn)
if syn != "" {
+ // Do not escape here. Python's English branch embeds the
+ // cleaned synonym as-is; SubSpecialChar would also escape '-',
+ // which WordNet synonyms contain constantly.
synParts = append(synParts, fmt.Sprintf(`"%s"^%.4f`, syn, tw.w/4.0))
+ // Extend keywords with cleaned synonyms
+ keywords = append(keywords, syn)
}
}
syns[i] = strings.Join(synParts, " ")
- // Extend keywords with synonyms
- keywords = append(keywords, tkSyns...)
} else {
syns[i] = ""
}
diff --git a/internal/service/nlp/query_builder_test.go b/internal/service/nlp/query_builder_test.go
index 30ee061a9c..33233af3e1 100644
--- a/internal/service/nlp/query_builder_test.go
+++ b/internal/service/nlp/query_builder_test.go
@@ -16,9 +16,11 @@ package nlp
import (
"reflect"
+ "strings"
"testing"
"ragflow/internal/engine/types"
+ "ragflow/internal/tokenizer"
)
func TestNewQueryBuilder(t *testing.T) {
@@ -97,6 +99,10 @@ func TestQueryBuilder_SubSpecialChar(t *testing.T) {
{"Tilde", "~tilde", `\~tilde`},
{"Caret", "^caret", `\^caret`},
{"Multiple", `:{}/[]-*"()|+~^`, `\:\{\}\/\[\]\-\*\"\(\)\|\+\~\^`},
+ {"Single quote removed", "it's", "its"},
+ {"Question mark", "a?b", `a\?b`},
+ {"Trim surrounding spaces", " x ", "x"},
+ {"Quote stripped then trimmed", " ' x ' ", "x"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -443,3 +449,95 @@ func TestQueryBuilder_SetQueryFields(t *testing.T) {
t.Errorf("Paragraph fields not updated after SetQueryFields, got %v, want %v", expr.Fields, newFields)
}
}
+
+// TestQueryBuilder_Question_SynonymSingleQuote ensures English synonyms are
+// cleaned of single quotes before being embedded in the full-text query.
+// WordNet returns e.g. "cat-o'-nine-tails" for "cat", and a raw single quote
+// breaks the Infinity query parser (see #13823).
+//
+// The Infinity engine's Tokenize returns the input unchanged, so quote
+// stripping is done by cleanSynonym, not by the analyzer. Set the engine
+// explicitly so the test does not pass via cleanSynonym's error fallback
+// when the tokenizer pool is uninitialized.
+func TestQueryBuilder_Question_SynonymSingleQuote(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ defer tokenizer.SetEngineType("")
+ tok, err := tokenizer.Tokenize("cat-o'-nine-tails")
+ if err != nil {
+ t.Fatalf("Tokenize: %v", err)
+ }
+ if tok != "cat-o'-nine-tails" {
+ t.Fatalf("infinity Tokenize = %q, want input unchanged", tok)
+ }
+
+ qb := NewQueryBuilder()
+ qb.synonym.dictionary = map[string][]string{
+ "cat": {"cat-o'-nine-tails", "'"},
+ }
+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)
+ if expr == nil {
+ t.Fatal("Question returned nil expr")
+ }
+ if strings.Contains(expr.MatchingText, "'") {
+ t.Errorf("MatchingText contains single quote: %q", expr.MatchingText)
+ }
+ if strings.Contains(expr.MatchingText, `""`) {
+ t.Errorf("MatchingText contains empty synonym phrase: %q", expr.MatchingText)
+ }
+ if !strings.Contains(expr.MatchingText, `"cat-o-nine-tails"`) {
+ t.Errorf("MatchingText missing cleaned synonym: %q", expr.MatchingText)
+ }
+ if strings.Contains(expr.MatchingText, `\`) {
+ t.Errorf("MatchingText escaped a synonym character: %q", expr.MatchingText)
+ }
+ for _, kw := range keywords {
+ if strings.Contains(kw, "'") || strings.TrimSpace(kw) == "" {
+ t.Errorf("keyword not cleaned: %q (all: %q)", kw, keywords)
+ }
+ }
+}
+
+// TestQueryBuilder_Question_SynonymQuestionMark checks a synonym that contains
+// "?" once the Infinity tokenizer is initialized. SetEngineType("infinity")
+// makes Tokenize return the input unchanged, so "?" is preserved (the C++
+// analyzer, used only when the engine is not "infinity", rewrites "what?cat"
+// to "what cat"). Python's English branch does not escape "?", and neither
+// does this path.
+func TestQueryBuilder_Question_SynonymQuestionMark(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ defer tokenizer.SetEngineType("")
+ tok, err := tokenizer.Tokenize("what?cat")
+ if err != nil {
+ t.Fatalf("Tokenize: %v", err)
+ }
+ if tok != "what?cat" {
+ t.Fatalf("infinity Tokenize = %q, want input unchanged", tok)
+ }
+
+ qb := NewQueryBuilder()
+ qb.synonym.dictionary = map[string][]string{
+ "cat": {"what?cat"},
+ }
+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)
+ if expr == nil {
+ t.Fatal("Question returned nil expr")
+ }
+ if !strings.Contains(expr.MatchingText, `"what?cat"`) {
+ t.Errorf("MatchingText missing unescaped synonym: %q", expr.MatchingText)
+ }
+ if strings.Contains(expr.MatchingText, `\`) {
+ t.Errorf("MatchingText should not escape ?: %q", expr.MatchingText)
+ }
+ found := false
+ for _, kw := range keywords {
+ if strings.Contains(kw, `\`) {
+ t.Errorf("keyword should not be escaped: %q", kw)
+ }
+ if kw == "what?cat" {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("keywords missing cleaned synonym %q: %q", "what?cat", keywords)
+ }
+}