RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(tokenizer): align important_kwd split to English comma (DSL parity, A2) (#17928)
Jackaddc5acdc0b5
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex db9c030eb6..f8001736f6 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -96,7 +96,6 @@ import ("ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer"- "ragflow/internal/utility")const ComponentNameTokenizer = "Tokenizer"@@ -698,7 +697,14 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)}}if kw := ck.Keywords; kw != "" {- if err = ck.SetExtraValue("important_kwd", utility.SplitKeywords(kw)); err != nil {+ // A2: split on the ENGLISH COMMA only, matching the DSL tokenizer+ // (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`) and+ // the keyword_prompt contract ("delimited by ENGLISH COMMA"). CJK+ // commas/semicolons and newlines stay part of the keyword so the+ // Go index is byte-compatible with the Python-DSL-built index.+ // strings.Split preserves empty elements, matching Python's+ // "a,,b".split(",") == ["a","","b"].+ if err = ck.SetExtraValue("important_kwd", strings.Split(kw, ",")); err != nil {return fmt.Errorf("tokenizer: keyword list marshal: %w", err)}it, err := tok.Tokenize(kw)diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.goindex 5ab725a6a1..1dbe579168 100644--- a/internal/ingestion/component/tokenizer_test.go+++ b/internal/ingestion/component/tokenizer_test.go@@ -262,37 +262,58 @@ func TestTokenizerComponent_Invoke_FullTextOnly(t *testing.T) {}}-// TestTokenizerComponent_Invoke_KeywordSplitCJK verifies important_kwd is-// split by the full ASCII+CJK delimiter set, not just ASCII comma. A Chinese-// LLM commonly emits CJK commas/semicolons even when asked for-// "comma-separated"; ASCII-only splitting would leave keywords glued together.-func TestTokenizerComponent_Invoke_KeywordSplitCJK(t *testing.T) {+// TestTokenizerComponent_Invoke_KeywordSplitCommaOnly verifies important_kwd+// is split on the ENGLISH COMMA ONLY, matching the DSL tokenizer+// (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`) and the+// keyword_prompt contract ("delimited by ENGLISH COMMA"). CJK commas,+// semicolons, and newlines are NOT delimiters — they stay part of the+// keyword. This keeps the Go index byte-compatible with the+// Python-DSL-built index (A2 alignment).+func TestTokenizerComponent_Invoke_KeywordSplitCommaOnly(t *testing.T) {requireTokenizerPool(t)_, stub := withStubEmbedder(t, 4)c, _ := NewTokenizerComponent(map[string]any{"search_method": []any{"full_text"},})- out, err := c.Invoke(context.Background(), nil, map[string]any{- "output_format": "chunks",- "chunks": []map[string]any{{"text": "alpha", "keywords": "kw1,kw2;kw3"}},- })- if err != nil {- t.Fatalf("Invoke: %v", err)++ check := func(keywords string, want []string) {+ t.Helper()+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{{"text": "alpha", "keywords": keywords}},+ })+ if err != nil {+ t.Fatalf("Invoke(%q): %v", keywords, err)+ }+ got, _ := out["chunks"].([]map[string]any)+ if len(got) != 1 {+ t.Fatalf("chunks len = %d, want 1", len(got))+ }+ kwd, ok := got[0]["important_kwd"].([]string)+ if !ok {+ t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])+ }+ if len(kwd) != len(want) {+ t.Errorf("important_kwd(%q) = %v, want %v", keywords, kwd, want)+ return+ }+ for i := range want {+ if kwd[i] != want[i] {+ t.Errorf("important_kwd(%q) = %v, want %v", keywords, kwd, want)+ return+ }+ }}+if stub.calls.Load() != 0 {t.Errorf("embedder should not be called in full_text-only mode, got %d", stub.calls.Load())}- got, _ := out["chunks"].([]map[string]any)- if len(got) != 1 {- t.Fatalf("chunks len = %d, want 1", len(got))- }- kwd, ok := got[0]["important_kwd"].([]string)- if !ok {- t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])- }- if len(kwd) != 3 {- t.Errorf("important_kwd must split CJK delimiters into 3 elements, got %d: %v", len(kwd), kwd)- }+ // Only the English comma splits.+ check("kw1,kw2,kw3", []string{"kw1", "kw2", "kw3"})+ // CJK commas and semicolons are NOT delimiters.+ check("kwA,kwB;kwC", []string{"kwA,kwB;kwC"})+ // Empty middle elements are preserved, matching Python "a,,b".split(",").+ check("a,,b", []string{"a", "", "b"})}func TestTokenizerComponent_Invoke_FullTextAndEmbedding(t *testing.T) {diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 2bbd48c199..f82fc434f2 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -31,6 +31,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/tokenizer")// stubEmbedder records every call and returns canned vectors.@@ -535,3 +536,55 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {t.Errorf("chunk 1 text = %q, want %q", chunks[1]["text"], "another valid")}}++// TestTokenizerComponent_ImportantKwd_CommaOnly is the no-tag parity test for+// A2: important_kwd must be split on the ENGLISH COMMA ONLY, matching the DSL+// tokenizer (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`). It+// runs without the C++ analyzer pool by switching the tokenizer engine to+// "infinity" (identity: Tokenize returns its input unchanged), so it executes+// in the default `go test ./...` CI tier and gives real regression protection.+func TestTokenizerComponent_ImportantKwd_CommaOnly(t *testing.T) {+ // Switch to identity tokenizer so tokenizeChunks needs no CGo pool, then+ // restore the default engine type afterwards.+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")++ c, err := NewTokenizerComponent(map[string]any{+ "search_method": []any{"full_text"},+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponent: %v", err)+ }+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": "doc body", "keywords": "kw1,kw2;kw3,kw4"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ got, ok := out["chunks"].([]map[string]any)+ if !ok || len(got) != 1 {+ t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])+ }+ kwd, ok := got[0]["important_kwd"].([]string)+ if !ok {+ t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])+ }+ // Only the English comma splits; CJK comma and semicolon stay attached.+ want := []string{"kw1", "kw2;kw3,kw4"}+ if len(kwd) != len(want) {+ t.Fatalf("important_kwd = %v, want %v", kwd, want)+ }+ for i := range want {+ if kwd[i] != want[i] {+ t.Errorf("important_kwd = %v, want %v (only comma splits)", kwd, want)+ }+ }+ // important_tks still tokenizes the full keyword string (identity mode+ // returns it unchanged).+ if tks, ok := got[0]["important_tks"].(string); !ok || tks != "kw1,kw2;kw3,kw4" {+ t.Errorf("important_tks = %v, want full keyword string", got[0]["important_tks"])+ }+}