Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(tokenizer): align important_kwd split to English comma (DSL parity, A2) (#17928)

Jackaddc5acdc0b5
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index db9c030eb6..f8001736f6 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -96,7 +96,6 @@ import (
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
- "ragflow/internal/utility"
)
const ComponentNameTokenizer = "Tokenizer"
@@ -698,7 +697,14 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)
}
}
if kw := ck.Keywords; kw != "" {
- if err = ck.SetExtraValue("important_kwd", utility.SplitKeywords(kw)); err != nil {
+ // A2: split on the ENGLISH COMMA only, matching the DSL tokenizer
+ // (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`) and
+ // the keyword_prompt contract ("delimited by ENGLISH COMMA"). CJK
+ // commas/semicolons and newlines stay part of the keyword so the
+ // Go index is byte-compatible with the Python-DSL-built index.
+ // strings.Split preserves empty elements, matching Python's
+ // "a,,b".split(",") == ["a","","b"].
+ if err = ck.SetExtraValue("important_kwd", strings.Split(kw, ",")); err != nil {
return fmt.Errorf("tokenizer: keyword list marshal: %w", err)
}
it, err := tok.Tokenize(kw)
diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.go
index 5ab725a6a1..1dbe579168 100644
--- a/internal/ingestion/component/tokenizer_test.go
+++ b/internal/ingestion/component/tokenizer_test.go
@@ -262,37 +262,58 @@ func TestTokenizerComponent_Invoke_FullTextOnly(t *testing.T) {
}
}
-// TestTokenizerComponent_Invoke_KeywordSplitCJK verifies important_kwd is
-// split by the full ASCII+CJK delimiter set, not just ASCII comma. A Chinese
-// LLM commonly emits CJK commas/semicolons even when asked for
-// "comma-separated"; ASCII-only splitting would leave keywords glued together.
-func TestTokenizerComponent_Invoke_KeywordSplitCJK(t *testing.T) {
+// TestTokenizerComponent_Invoke_KeywordSplitCommaOnly verifies important_kwd
+// is split on the ENGLISH COMMA ONLY, matching the DSL tokenizer
+// (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`) and the
+// keyword_prompt contract ("delimited by ENGLISH COMMA"). CJK commas,
+// semicolons, and newlines are NOT delimiters — they stay part of the
+// keyword. This keeps the Go index byte-compatible with the
+// Python-DSL-built index (A2 alignment).
+func TestTokenizerComponent_Invoke_KeywordSplitCommaOnly(t *testing.T) {
requireTokenizerPool(t)
_, stub := withStubEmbedder(t, 4)
c, _ := NewTokenizerComponent(map[string]any{
"search_method": []any{"full_text"},
})
- out, err := c.Invoke(context.Background(), nil, map[string]any{
- "output_format": "chunks",
- "chunks": []map[string]any{{"text": "alpha", "keywords": "kw1,kw2;kw3"}},
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
+
+ check := func(keywords string, want []string) {
+ t.Helper()
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "output_format": "chunks",
+ "chunks": []map[string]any{{"text": "alpha", "keywords": keywords}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke(%q): %v", keywords, err)
+ }
+ got, _ := out["chunks"].([]map[string]any)
+ if len(got) != 1 {
+ t.Fatalf("chunks len = %d, want 1", len(got))
+ }
+ kwd, ok := got[0]["important_kwd"].([]string)
+ if !ok {
+ t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])
+ }
+ if len(kwd) != len(want) {
+ t.Errorf("important_kwd(%q) = %v, want %v", keywords, kwd, want)
+ return
+ }
+ for i := range want {
+ if kwd[i] != want[i] {
+ t.Errorf("important_kwd(%q) = %v, want %v", keywords, kwd, want)
+ return
+ }
+ }
}
+
if stub.calls.Load() != 0 {
t.Errorf("embedder should not be called in full_text-only mode, got %d", stub.calls.Load())
}
- got, _ := out["chunks"].([]map[string]any)
- if len(got) != 1 {
- t.Fatalf("chunks len = %d, want 1", len(got))
- }
- kwd, ok := got[0]["important_kwd"].([]string)
- if !ok {
- t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])
- }
- if len(kwd) != 3 {
- t.Errorf("important_kwd must split CJK delimiters into 3 elements, got %d: %v", len(kwd), kwd)
- }
+ // Only the English comma splits.
+ check("kw1,kw2,kw3", []string{"kw1", "kw2", "kw3"})
+ // CJK commas and semicolons are NOT delimiters.
+ check("kwA,kwB;kwC", []string{"kwA,kwB;kwC"})
+ // Empty middle elements are preserved, matching Python "a,,b".split(",").
+ check("a,,b", []string{"a", "", "b"})
}
func TestTokenizerComponent_Invoke_FullTextAndEmbedding(t *testing.T) {
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 2bbd48c199..f82fc434f2 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -31,6 +31,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/tokenizer"
)
// stubEmbedder records every call and returns canned vectors.
@@ -535,3 +536,55 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {
t.Errorf("chunk 1 text = %q, want %q", chunks[1]["text"], "another valid")
}
}
+
+// TestTokenizerComponent_ImportantKwd_CommaOnly is the no-tag parity test for
+// A2: important_kwd must be split on the ENGLISH COMMA ONLY, matching the DSL
+// tokenizer (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`). It
+// runs without the C++ analyzer pool by switching the tokenizer engine to
+// "infinity" (identity: Tokenize returns its input unchanged), so it executes
+// in the default `go test ./...` CI tier and gives real regression protection.
+func TestTokenizerComponent_ImportantKwd_CommaOnly(t *testing.T) {
+ // Switch to identity tokenizer so tokenizeChunks needs no CGo pool, then
+ // restore the default engine type afterwards.
+ tokenizer.SetEngineType("infinity")
+ defer tokenizer.SetEngineType("")
+
+ c, err := NewTokenizerComponent(map[string]any{
+ "search_method": []any{"full_text"},
+ })
+ if err != nil {
+ t.Fatalf("NewTokenizerComponent: %v", err)
+ }
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ {"text": "doc body", "keywords": "kw1,kw2;kw3,kw4"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ got, ok := out["chunks"].([]map[string]any)
+ if !ok || len(got) != 1 {
+ t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])
+ }
+ kwd, ok := got[0]["important_kwd"].([]string)
+ if !ok {
+ t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])
+ }
+ // Only the English comma splits; CJK comma and semicolon stay attached.
+ want := []string{"kw1", "kw2;kw3,kw4"}
+ if len(kwd) != len(want) {
+ t.Fatalf("important_kwd = %v, want %v", kwd, want)
+ }
+ for i := range want {
+ if kwd[i] != want[i] {
+ t.Errorf("important_kwd = %v, want %v (only comma splits)", kwd, want)
+ }
+ }
+ // important_tks still tokenizes the full keyword string (identity mode
+ // returns it unchanged).
+ if tks, ok := got[0]["important_tks"].(string); !ok || tks != "kw1,kw2;kw3,kw4" {
+ t.Errorf("important_tks = %v, want full keyword string", got[0]["important_tks"])
+ }
+}