Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): align custom delimiter text metadata (#19005)

Loong25351386de02
diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.go
index 341a8428e1..bd480e39bc 100644
--- a/internal/ingestion/component/chunker/custom_delim_test.go
+++ b/internal/ingestion/component/chunker/custom_delim_test.go
@@ -2,6 +2,7 @@ package chunker
import (
"context"
+ "slices"
"testing"
)
@@ -20,10 +21,10 @@ import (
// "first segment line one" with the newline dropped. Go's chunkFromItem does
// the same via splitDroppingDelim, matching the Python reference.
//
-// doc_type_kwd is intentionally asserted to remain present on every chunk.
-// It is a load-bearing Go field (index column + media dispatch) and is NOT
-// part of the divergence; it is classified go_intentional in known_diffs.json,
-// not a bug to fix here.
+// Plain text/markdown/html inputs must not gain doc_type_kwd merely because a
+// custom delimiter is configured: Python emits only text for those paths.
+// Structured JSON inputs keep their source doc_type_kwd, while ck_type remains
+// on Go text chunks for downstream crop dispatch.
const backtickNewline = "`\n`"
@@ -54,6 +55,41 @@ func chunkTexts(chunks []map[string]any) []string {
return out
}
+// assertPlainCustomDelimiterMetadata checks the metadata contract shared by
+// plain text, Markdown, and HTML custom-delimiter paths.
+func assertPlainCustomDelimiterMetadata(t *testing.T, chunks []map[string]any) {
+ t.Helper()
+ for i, chunk := range chunks {
+ if _, ok := chunk["doc_type_kwd"]; ok {
+ t.Errorf("chunk[%d] must not synthesize doc_type_kwd: %v", i, chunk)
+ }
+ if got := chunk["ck_type"]; got != "text" {
+ t.Errorf("chunk[%d] ck_type: want %q got %v", i, "text", got)
+ }
+ }
+}
+
+// TestCustomDelimTextChildrenKeepCKType covers primary and child splitting in
+// combination: child chunks remain unstructured but retain Go's live CKType.
+func TestCustomDelimTextChildrenKeepCKType(t *testing.T) {
+ params := map[string]any{
+ "chunk_token_size": float64(128),
+ "delimiters": []string{"`::`"},
+ "children_delimiters": []string{"--"},
+ }
+ input := map[string]any{
+ "name": "t", "output_format": "text",
+ "text": "alpha--beta::gamma--delta",
+ }
+ chunks := invokeTokenChunks(t, params, input)
+
+ want := []string{"alpha", "beta", "gamma", "delta"}
+ if got := chunkTexts(chunks); !slices.Equal(got, want) {
+ t.Fatalf("chunk texts: want %v got %v", want, got)
+ }
+ assertPlainCustomDelimiterMetadata(t, chunks)
+}
+
// TestCustomDelimTextDropsDelimiter reproduces token__text_backtick.
func TestCustomDelimTextDropsDelimiter(t *testing.T) {
params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}
@@ -72,10 +108,8 @@ func TestCustomDelimTextDropsDelimiter(t *testing.T) {
if got != w {
t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
}
- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
- }
}
+ assertPlainCustomDelimiterMetadata(t, chunks)
}
// TestCustomDelimJSONDropsDelimiter reproduces token__json_backtick: the
@@ -104,8 +138,10 @@ func TestCustomDelimJSONDropsDelimiter(t *testing.T) {
if got != w {
t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
}
- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
+ }
+ for i, chunk := range chunks {
+ if kd, _ := chunk["doc_type_kwd"].(string); kd != "text" {
+ t.Errorf("chunk[%d] doc_type_kwd: structured JSON must keep it, want %q got %q", i, "text", kd)
}
}
}
@@ -131,10 +167,8 @@ func TestCustomDelimMarkdownDropsDelimiter(t *testing.T) {
if got != w {
t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
}
- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
- }
}
+ assertPlainCustomDelimiterMetadata(t, chunks)
}
// TestCustomDelimHTMLDropsDelimiter reproduces token__html_backtick.
@@ -154,8 +188,6 @@ func TestCustomDelimHTMLDropsDelimiter(t *testing.T) {
if got != w {
t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
}
- if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
- t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
- }
}
+ assertPlainCustomDelimiterMetadata(t, chunks)
}
diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.json
deleted file mode 100644
index 0dd944fff8..0000000000
--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "case_id": "token__html_backtick",
- "chunks": [
- {
- "doc_type_kwd": "text",
- "text": "\u003cp\u003eone\u003c/p\u003e"
- },
- {
- "doc_type_kwd": "text",
- "text": "\u003cp\u003etwo\u003c/p\u003e"
- },
- {
- "doc_type_kwd": "text",
- "text": "\u003cp\u003ethree\u003c/p\u003e"
- }
- ]
-}
diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.json
deleted file mode 100644
index ba22d3c31f..0000000000
--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "case_id": "token__markdown_backtick",
- "chunks": [
- {
- "doc_type_kwd": "text",
- "text": "# Title"
- },
- {
- "doc_type_kwd": "text",
- "text": "Paragraph one."
- },
- {
- "doc_type_kwd": "text",
- "text": "Paragraph two."
- }
- ]
-}
diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.json
deleted file mode 100644
index 0d30cc448f..0000000000
--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "case_id": "token__text_backtick",
- "chunks": [
- {
- "doc_type_kwd": "text",
- "text": "first sentence here"
- },
- {
- "doc_type_kwd": "text",
- "text": "second sentence here"
- },
- {
- "doc_type_kwd": "text",
- "text": "third sentence here"
- }
- ]
-}
diff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
index 5284bf78be..45650fca4d 100644
--- a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
@@ -19,15 +19,6 @@
"permanent": true,
"reason": "Python builds both fields internally (token_chunker.py:141,144) but _finalize_json_chunks (token_chunker.py:256-259) whitelists only text/doc_type_kwd/_pdf_positions/mom/img_id, dropping them. Go keeps them because they are still live downstream: needsCrop (pdfcrop_cgo.go:151) reads ck_type, and tokenizer.go:615 forwards tk_nums. The divergence follows from Go cropping after the chunker returns, whereas Python crops inside _invoke (token_chunker.py:370) before finalizing."
},
- {
- "id": "token-custom-delim-newline-and-field",
- "tag": "go_bug",
- "kind": "chunk_text",
- "applies_to": ["token__text_backtick", "token__markdown_backtick", "token__html_backtick"],
- "owner_fix_side": "go",
- "tracking": "TODO: file issue — Go emits an extra doc_type_kwd:'text' field on the text/markdown/html backtick path where Python emits only {'text': ...}",
- "reason": "With a backtick-wrapped delimiter (e.g. `\\n`, _compile_delimiter_pattern, token_chunker.py:70-76) Python takes the custom-pattern branch. On that branch Python's _split_text_by_pattern (rag/flow/chunker/token_chunker.py:80-95) discards the delimiter, and the text path emits a list of {'text': ...} with no doc_type_kwd (token_chunker.py:341). Go no longer keeps the delimiter either: since #17868 it uses splitDroppingDelim, so chunk text matches Python exactly. The remaining Go-only behaviour is an extra doc_type_kwd:'text' field — invokeTextPayload (token.go:323, 338-341) defaults every chunk to DocType:'text', which Python's text-path output never includes."
- },
{
"id": "token-hardcap-chunk-count",
"tag": "go_intentional",
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 040f874563..1cdefce641 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -355,7 +355,10 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child
}
textDocs := make([]schema.ChunkDoc, 0, len(cleaned))
for _, s := range cleaned {
- textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})
+ // Python's unstructured text path emits only text here. Keep CKType for
+ // Go's downstream crop dispatch, but do not synthesize structured-input
+ // metadata such as doc_type_kwd.
+ textDocs = append(textDocs, schema.ChunkDoc{Text: s, CKType: "text"})
}
docs := applyChildrenDelimText(textDocs, childrenPattern)
return chunkOutputs(docs)
@@ -1546,7 +1549,11 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc
if strings.TrimSpace(child) == "" {
continue
}
- out = append(out, schema.ChunkDoc{Text: child, Mom: strings.TrimPrefix(t, "\n")})
+ out = append(out, schema.ChunkDoc{
+ Text: child,
+ CKType: d.CKType,
+ Mom: strings.TrimPrefix(t, "\n"),
+ })
}
}
return out