RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): align custom delimiter text metadata (#19005)
Loong25351386de02
diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.goindex 341a8428e1..bd480e39bc 100644--- a/internal/ingestion/component/chunker/custom_delim_test.go+++ b/internal/ingestion/component/chunker/custom_delim_test.go@@ -2,6 +2,7 @@ package chunkerimport ("context"+ "slices""testing")@@ -20,10 +21,10 @@ import (// "first segment line one" with the newline dropped. Go's chunkFromItem does// the same via splitDroppingDelim, matching the Python reference.//-// doc_type_kwd is intentionally asserted to remain present on every chunk.-// It is a load-bearing Go field (index column + media dispatch) and is NOT-// part of the divergence; it is classified go_intentional in known_diffs.json,-// not a bug to fix here.+// Plain text/markdown/html inputs must not gain doc_type_kwd merely because a+// custom delimiter is configured: Python emits only text for those paths.+// Structured JSON inputs keep their source doc_type_kwd, while ck_type remains+// on Go text chunks for downstream crop dispatch.const backtickNewline = "`\n`"@@ -54,6 +55,41 @@ func chunkTexts(chunks []map[string]any) []string {return out}+// assertPlainCustomDelimiterMetadata checks the metadata contract shared by+// plain text, Markdown, and HTML custom-delimiter paths.+func assertPlainCustomDelimiterMetadata(t *testing.T, chunks []map[string]any) {+ t.Helper()+ for i, chunk := range chunks {+ if _, ok := chunk["doc_type_kwd"]; ok {+ t.Errorf("chunk[%d] must not synthesize doc_type_kwd: %v", i, chunk)+ }+ if got := chunk["ck_type"]; got != "text" {+ t.Errorf("chunk[%d] ck_type: want %q got %v", i, "text", got)+ }+ }+}++// TestCustomDelimTextChildrenKeepCKType covers primary and child splitting in+// combination: child chunks remain unstructured but retain Go's live CKType.+func TestCustomDelimTextChildrenKeepCKType(t *testing.T) {+ params := map[string]any{+ "chunk_token_size": float64(128),+ "delimiters": []string{"`::`"},+ "children_delimiters": []string{"--"},+ }+ input := map[string]any{+ "name": "t", "output_format": "text",+ "text": "alpha--beta::gamma--delta",+ }+ chunks := invokeTokenChunks(t, params, input)++ want := []string{"alpha", "beta", "gamma", "delta"}+ if got := chunkTexts(chunks); !slices.Equal(got, want) {+ t.Fatalf("chunk texts: want %v got %v", want, got)+ }+ assertPlainCustomDelimiterMetadata(t, chunks)+}+// TestCustomDelimTextDropsDelimiter reproduces token__text_backtick.func TestCustomDelimTextDropsDelimiter(t *testing.T) {params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}@@ -72,10 +108,8 @@ func TestCustomDelimTextDropsDelimiter(t *testing.T) {if got != w {t.Errorf("chunk[%d] text: want %q got %q", i, w, got)}- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)- }}+ assertPlainCustomDelimiterMetadata(t, chunks)}// TestCustomDelimJSONDropsDelimiter reproduces token__json_backtick: the@@ -104,8 +138,10 @@ func TestCustomDelimJSONDropsDelimiter(t *testing.T) {if got != w {t.Errorf("chunk[%d] text: want %q got %q", i, w, got)}- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)+ }+ for i, chunk := range chunks {+ if kd, _ := chunk["doc_type_kwd"].(string); kd != "text" {+ t.Errorf("chunk[%d] doc_type_kwd: structured JSON must keep it, want %q got %q", i, "text", kd)}}}@@ -131,10 +167,8 @@ func TestCustomDelimMarkdownDropsDelimiter(t *testing.T) {if got != w {t.Errorf("chunk[%d] text: want %q got %q", i, w, got)}- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)- }}+ assertPlainCustomDelimiterMetadata(t, chunks)}// TestCustomDelimHTMLDropsDelimiter reproduces token__html_backtick.@@ -154,8 +188,6 @@ func TestCustomDelimHTMLDropsDelimiter(t *testing.T) {if got != w {t.Errorf("chunk[%d] text: want %q got %q", i, w, got)}- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)- }}+ assertPlainCustomDelimiterMetadata(t, chunks)}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.jsondeleted file mode 100644index 0dd944fff8..0000000000--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.json+++ /dev/null@@ -1,17 +0,0 @@-{- "case_id": "token__html_backtick",- "chunks": [- {- "doc_type_kwd": "text",- "text": "\u003cp\u003eone\u003c/p\u003e"- },- {- "doc_type_kwd": "text",- "text": "\u003cp\u003etwo\u003c/p\u003e"- },- {- "doc_type_kwd": "text",- "text": "\u003cp\u003ethree\u003c/p\u003e"- }- ]-}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.jsondeleted file mode 100644index ba22d3c31f..0000000000--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.json+++ /dev/null@@ -1,17 +0,0 @@-{- "case_id": "token__markdown_backtick",- "chunks": [- {- "doc_type_kwd": "text",- "text": "# Title"- },- {- "doc_type_kwd": "text",- "text": "Paragraph one."- },- {- "doc_type_kwd": "text",- "text": "Paragraph two."- }- ]-}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.jsondeleted file mode 100644index 0d30cc448f..0000000000--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.json+++ /dev/null@@ -1,17 +0,0 @@-{- "case_id": "token__text_backtick",- "chunks": [- {- "doc_type_kwd": "text",- "text": "first sentence here"- },- {- "doc_type_kwd": "text",- "text": "second sentence here"- },- {- "doc_type_kwd": "text",- "text": "third sentence here"- }- ]-}diff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.jsonindex 5284bf78be..45650fca4d 100644--- a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json@@ -19,15 +19,6 @@"permanent": true,"reason": "Python builds both fields internally (token_chunker.py:141,144) but _finalize_json_chunks (token_chunker.py:256-259) whitelists only text/doc_type_kwd/_pdf_positions/mom/img_id, dropping them. Go keeps them because they are still live downstream: needsCrop (pdfcrop_cgo.go:151) reads ck_type, and tokenizer.go:615 forwards tk_nums. The divergence follows from Go cropping after the chunker returns, whereas Python crops inside _invoke (token_chunker.py:370) before finalizing."},- {- "id": "token-custom-delim-newline-and-field",- "tag": "go_bug",- "kind": "chunk_text",- "applies_to": ["token__text_backtick", "token__markdown_backtick", "token__html_backtick"],- "owner_fix_side": "go",- "tracking": "TODO: file issue — Go emits an extra doc_type_kwd:'text' field on the text/markdown/html backtick path where Python emits only {'text': ...}",- "reason": "With a backtick-wrapped delimiter (e.g. `\\n`, _compile_delimiter_pattern, token_chunker.py:70-76) Python takes the custom-pattern branch. On that branch Python's _split_text_by_pattern (rag/flow/chunker/token_chunker.py:80-95) discards the delimiter, and the text path emits a list of {'text': ...} with no doc_type_kwd (token_chunker.py:341). Go no longer keeps the delimiter either: since #17868 it uses splitDroppingDelim, so chunk text matches Python exactly. The remaining Go-only behaviour is an extra doc_type_kwd:'text' field — invokeTextPayload (token.go:323, 338-341) defaults every chunk to DocType:'text', which Python's text-path output never includes."- },{"id": "token-hardcap-chunk-count","tag": "go_intentional",diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 040f874563..1cdefce641 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -355,7 +355,10 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child}textDocs := make([]schema.ChunkDoc, 0, len(cleaned))for _, s := range cleaned {- textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})+ // Python's unstructured text path emits only text here. Keep CKType for+ // Go's downstream crop dispatch, but do not synthesize structured-input+ // metadata such as doc_type_kwd.+ textDocs = append(textDocs, schema.ChunkDoc{Text: s, CKType: "text"})}docs := applyChildrenDelimText(textDocs, childrenPattern)return chunkOutputs(docs)@@ -1546,7 +1549,11 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []scif strings.TrimSpace(child) == "" {continue}- out = append(out, schema.ChunkDoc{Text: child, Mom: strings.TrimPrefix(t, "\n")})+ out = append(out, schema.ChunkDoc{+ Text: child,+ CKType: d.CKType,+ Mom: strings.TrimPrefix(t, "\n"),+ })}}return out