RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix(tokenizer): pin alignment guards (#18011)
Jackdc73163908ed
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 13b70e3159..dcdcc5bcf2 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -413,6 +413,13 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, emtokenCount inthasTitleVec bool)+ // go_intentional (A3): when the upstream name is empty we skip title+ // weighting entirely (hasTitleVec stays false, so the merged vector is the+ // content vector alone). From the end-user perspective an empty title must+ // not contribute the filename embedding weight; the Python DSL instead+ // computes 0.1*emb(""), injecting an undefined bias into every chunk. Go's+ // skip is the correct behavior (go_intentional). Do NOT "align" this to+ // the DSL.if trimmedName == "" {log.Printf("Tokenizer: empty name provided from upstream, embedding will skip title weighting")} else {@@ -680,13 +687,11 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)}}if kw := ck.Keywords; kw != "" {- // A2: split on the ENGLISH COMMA only, matching the DSL tokenizer- // (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`) and- // the keyword_prompt contract ("delimited by ENGLISH COMMA"). CJK- // commas/semicolons and newlines stay part of the keyword so the- // Go index is byte-compatible with the Python-DSL-built index.- // strings.Split preserves empty elements, matching Python's- // "a,,b".split(",") == ["a","","b"].+ // Split keywords on the ENGLISH COMMA ONLY. The keyword_prompt+ // contract specifies "delimited by ENGLISH COMMA", so CJK commas,+ // semicolons and newlines stay part of the keyword rather than+ // acting as separators. strings.Split also preserves empty+ // elements, matching Python's "a,,b".split(",") == ["a","","b"].if err = ck.SetExtraValue("important_kwd", strings.Split(kw, ",")); err != nil {return fmt.Errorf("tokenizer: keyword list marshal: %w", err)}@@ -772,6 +777,13 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {// shouldHaveEmbedding reports whether the tokenizer must attach embedding// vectors: the search method requests embedding AND a KB is present.+//+// go_intentional (A4): the kbID != "" guard is deliberate. Each dataset+// configures its own embedding model, so an empty kb_id (e.g. a canvas-debug+// dry run) must NOT fall back to the tenant's default embedding model — doing+// so would produce vectors a dataset cannot actually use at retrieval time.+// This is a deliberate, go_intentional divergence. Do NOT "align" this to a+// path that injects a default embedding.func shouldHaveEmbedding(searchMethods []string, kbID string) bool {return contains(searchMethods, "embedding") && kbID != ""}diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 2454ea79f8..985dd48981 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -702,3 +702,115 @@ func TestTokenizerComponent_ImportantKwd_CommaOnly(t *testing.T) {t.Errorf("important_tks = %v, want full keyword string", got[0]["important_tks"])}}++// TestTokenizerComponent_ImportantKwd_PreservesEmptyElements locks F3: the+// component path splits on the ENGLISH COMMA ONLY via strings.Split and+// PRESERVES empty elements, matching Python's "a,,b".split(",") ==+// ["a","","b"]. This is the contract asserted by the comment at+// tokenizer.go:688-689, and it is the intentional counterpart to the executor+// fallback (cleanupConsumedChunkFields -> utility.SplitKeywords) which DROPS+// empty parts. Runs without the C++ analyzer pool via the identity engine.+func TestTokenizerComponent_ImportantKwd_PreservesEmptyElements(t *testing.T) {+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")++ c, err := NewTokenizerComponent(map[string]any{+ "search_method": []any{"full_text"},+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponent: %v", err)+ }++ // Middle empty element must be preserved (["a","","b"]), not dropped.+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": "doc body", "keywords": "a,,b"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ got, ok := out["chunks"].([]map[string]any)+ if !ok || len(got) != 1 {+ t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])+ }+ kwd, ok := got[0]["important_kwd"].([]string)+ if !ok {+ t.Fatalf("important_kwd should be []string, got %T", got[0]["important_kwd"])+ }+ want := []string{"a", "", "b"}+ if len(kwd) != len(want) {+ t.Fatalf("important_kwd = %v, want %v (empty elements preserved)", kwd, want)+ }+ for i := range want {+ if kwd[i] != want[i] {+ t.Errorf("important_kwd[%d] = %q, want %q (empty elements preserved)", i, kwd[i], want[i])+ }+ }++ // Empty keyword string must not error. The component guards on a+ // non-empty keywords field (tokenizer.go:682), so important_kwd is left+ // unset (nil) rather than materialized as an empty array. Downstream+ // indexing treats a missing important_kwd as "no keywords", which is safe.+ outEmpty, err := c.Invoke(context.Background(), nil, map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": "doc body", "keywords": ""},+ },+ })+ if err != nil {+ t.Fatalf("Invoke(empty keywords): %v", err)+ }+ gotEmpty, ok := outEmpty["chunks"].([]map[string]any)+ if !ok || len(gotEmpty) != 1 {+ t.Fatalf("empty chunks = %v, want 1 chunk", outEmpty["chunks"])+ }+ if kwd, exists := gotEmpty[0]["important_kwd"]; exists && kwd != nil {+ t.Errorf("empty keywords should leave important_kwd unset, got %v", kwd)+ }+}++// TestTextPayloadToChunks_B1B2 pins the text-payload adapter used for+// markdown/text/html payloads (tokenizer.go:578). This is the Go-correct+// counterpart to a Python-DSL bug where turning full_text off dropped the+// entire document (B1), and where an empty payload failed to emit the chunks+// key at all (B2). The Go helper always returns a slice: non-empty payload ->+// one chunk carrying the raw text; nil/empty/whitespace payload -> a non-nil+// EMPTY slice so the downstream "chunks" key is present as [].+//+// No-tag unit test: textPayloadToChunks is a pure function, no CGo pool needed.+func TestTextPayloadToChunks_B1B2(t *testing.T) {+ // B1: a real payload yields exactly one chunk with the text preserved.+ payload := "plain payload body"+ got := textPayloadToChunks(&payload)+ if len(got) != 1 {+ t.Fatalf("non-empty payload: len(chunks) = %d, want 1", len(got))+ }+ if got[0].Text != payload {+ t.Errorf("chunk text = %q, want %q", got[0].Text, payload)+ }++ // B2: empty/whitespace/nil payloads must still return a non-nil EMPTY+ // slice (so the chunks key is present as [] downstream), never nil.+ for _, tc := range []struct {+ name string+ in *string+ }{+ {"empty string", ptr("")},+ {"whitespace only", ptr(" \n\t ")},+ {"nil pointer", nil},+ } {+ t.Run(tc.name, func(t *testing.T) {+ out := textPayloadToChunks(tc.in)+ if out == nil {+ t.Fatalf("textPayloadToChunks(%s) = nil, want non-nil empty slice", tc.name)+ }+ if len(out) != 0 {+ t.Errorf("textPayloadToChunks(%s) len = %d, want 0", tc.name, len(out))+ }+ })+ }+}++func ptr(s string) *string { return &s }diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex 168f040098..591f659444 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -345,3 +345,67 @@ func TestProcessChunkPositions_NoPositions(t *testing.T) {t.Error("_pdf_positions must be pruned even when positions is missing")}}++// TestCleanupConsumedChunkFields_ImportantKwdMultiDelimiter pins the executor+// fallback's important_kwd materialization. When the Tokenizer component did+// NOT pre-produce important_kwd, the executor falls back to+// utility.SplitKeywords, which splits on the full delimiter set+// (ASCII + CJK comma/semicolon/ideographic-comma/newline) and DROPS empty+// parts. This is intentionally different from the Tokenizer component path+// (internal/ingestion/component/tokenizer.go:690), which splits on the ENGLISH+// COMMA ONLY and PRESERVES empty elements to match the DSL+// (rag/flow/tokenizer/tokenizer.py:153 `keywords.split(",")`).+//+// The two layers deliberately diverge: the component aligns to the DSL keyword+// contract ("delimited by ENGLISH COMMA"); the executor fallback mirrors+// Python task_executor.run_dataflow:879 and tolerates mixed delimiters from+// older upstream producers. Neither side should be "unified" to the other —+// changing one without the other silently breaks the documented parity+// boundary. The component-side half of this contract is locked by+// TestTokenizerComponent_ImportantKwd_CommaOnly in the component package.+func TestCleanupConsumedChunkFields_ImportantKwdMultiDelimiter(t *testing.T) {+ ck := map[string]any{"text": "hello", "keywords": "kw1,kw2;kw3,kw4"}++ cleanupConsumedChunkFields(ck)++ kwd, ok := ck["important_kwd"].([]string)+ if !ok {+ t.Fatalf("important_kwd should be []string, got %T", ck["important_kwd"])+ }+ // Executor fallback splits on comma/semicolon/CJK-comma and drops empties:+ // "kw1,kw2;kw3,kw4" -> ["kw1","kw2","kw3","kw4"], NOT the component's+ // ["kw1","kw2;kw3,kw4"].+ want := []string{"kw1", "kw2", "kw3", "kw4"}+ if len(kwd) != len(want) {+ t.Fatalf("executor important_kwd = %v, want %v (multi-delimiter, empties dropped)", kwd, want)+ }+ for i := range want {+ if kwd[i] != want[i] {+ t.Errorf("executor important_kwd[%d] = %q, want %q", i, kwd[i], want[i])+ }+ }+ if _, exists := ck["keywords"]; exists {+ t.Error("keywords source field should be consumed/removed")+ }+}++// TestCleanupConsumedChunkFields_ImportantKwdDropsEmptyParts documents that the+// executor fallback drops empty parts (e.g. the middle empty token in+// "a,,b"), diverging from the component path which PRESERVES it as ["a","","b"].+// Together with the component CommaOnly test this locks the intentional+// divergence: same input, different important_kwd arrays per layer.+func TestCleanupConsumedChunkFields_ImportantKwdDropsEmptyParts(t *testing.T) {+ ck := map[string]any{"text": "hello", "keywords": "a,,b"}++ cleanupConsumedChunkFields(ck)++ kwd, ok := ck["important_kwd"].([]string)+ if !ok {+ t.Fatalf("important_kwd should be []string, got %T", ck["important_kwd"])+ }+ // Executor drops the empty middle part: ["a","b"], NOT ["a","","b"].+ want := []string{"a", "b"}+ if len(kwd) != len(want) || kwd[0] != "a" || kwd[1] != "b" {+ t.Fatalf("executor important_kwd = %v, want %v (empty parts dropped)", kwd, want)+ }+}diff --git a/internal/utility/split.go b/internal/utility/split.goindex 0334fa09d3..b9abc947af 100644--- a/internal/utility/split.go+++ b/internal/utility/split.go@@ -46,10 +46,17 @@ func nonEmpty(parts []string) []string {}// SplitKeywords splits a keywords string by common (ASCII + CJK) delimiters,-// dropping empty elements. Returns nil for an empty input. It is the single-// authority for materializing the important_kwd array from the keywords-// string, shared by the Tokenizer component (in-pipeline) and the executor's-// persist-schema mapping. Mirrors Python task_executor.run_dataflow:879.+// dropping empty elements. Returns nil for an empty input. It is the authority+// for materializing the important_kwd array in the executor's persist-schema+// mapping (rag/flow/task_executor.py run_dataflow:879), which tolerates mixed+// delimiters from older upstream producers.+//+// NOTE: the in-pipeline Tokenizer component does NOT use this function. For+// DSL byte-compatibility it splits keywords on the ENGLISH COMMA ONLY via+// strings.Split(kw, ",") and PRESERVES empty elements (mirroring Python+// rag/flow/tokenizer/tokenizer.py:153 "a,,b".split(",")), so the two paths+// intentionally diverge. See tokenizeChunks in+// internal/ingestion/component/tokenizer.go.func SplitKeywords(keywords string) []string {if keywords == "" {return nil