RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): retain bare delimiters losslessly, drop custom (backtick) delimiters (#20276)
xugangqiang8e830e4235a2
diff --git a/internal/ingestion/component/chunker/bare_delimiter_test.go b/internal/ingestion/component/chunker/bare_delimiter_test.goindex 0f4fc9a682..765c5aa7ca 100644--- a/internal/ingestion/component/chunker/bare_delimiter_test.go+++ b/internal/ingestion/component/chunker/bare_delimiter_test.go@@ -11,12 +11,12 @@ import (// paragraphs that are then merged by token size — instead of being silently// ignored (the previous keepBare=false contract).//-// Python contract (rag/nlp/__init__.py:1406-1415, naive_merge default path):-// the payload is split on the delimiter, the delimiter text is DROPPED, and-// each kept paragraph is rebuilt with a leading "\n" (its original surrounding-// whitespace is preserved). Paragraphs are thus joined by "\n", never by the-// raw delimiter. One chunk per segment (no token merge) happens ONLY for a-// backtick-wrapped CUSTOM delimiter.+// Lossless contract (plan B): a delimiter is a HINT for where the chunker MAY+// break, never a character to DELETE. After chunking, concatenating every+// emitted chunk's text must reproduce the source exactly — the delimiter is+// retained (glued to the end of the segment that precedes it), and no stray+// "\n" is synthesized in its place. One chunk per segment (no token merge)+// happens ONLY for a backtick-wrapped CUSTOM delimiter.//// These tests target BOTH the text/markdown/html path (invokeTextPayload) and// the JSON path (invokeJSONPayload -> chunkFromItem + global merge), and lock@@ -63,10 +63,9 @@ func joinedText(chunks []map[string]any) string {}// TestBareDelimiterSplitsAndTokenMergesText asserts the core fix: a bare-// multi-char delimiter ("::") now splits the text into paragraphs, the-// delimiter text is dropped, and the paragraphs are rejoined with "\n"-// (matching Python's "\n"+sub_sec reconstruction) — not with the raw-// delimiter. The paragraphs are then merged by token size.+// multi-char delimiter ("::") now splits the text into paragraphs, each+// paragraph KEEPS its trailing delimiter (lossless), and the paragraphs are+// merged by token size. Concatenating the chunks reproduces the source exactly.func TestBareDelimiterSplitsAndTokenMergesText(t *testing.T) {c := newBareChunker(t, []string{"::"}, 1024)const text = "alpha::beta::gamma::delta"@@ -74,73 +73,64 @@ func TestBareDelimiterSplitsAndTokenMergesText(t *testing.T) {if len(chunks) == 0 {t.Fatal("expected at least one chunk")}- // The delimiter must never survive inside a chunk.+ // Lossless: the joined content equals the source, delimiter included.+ if got := joinedText(chunks); got != text {+ t.Errorf("joined text: want %q got %q (chunks=%v)", text, got, chunkTexts(chunks))+ }+ // Every paragraph retains its delimiter (it is not deleted).for _, ck := range chunks {- if strings.Contains(ck["text"].(string), "::") {- t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))+ if !strings.Contains(ck["text"].(string), "::") {+ t.Errorf("bare delimiter dropped from chunk: %q", ck["text"].(string))}}- // Paragraphs are rejoined with "\n" (each paragraph keeps a leading "\n"),- // so the joined content equals the source with "::" replaced by "\n".- const want = "alpha\nbeta\ngamma\ndelta"- if got := joinedText(chunks); got != want {- t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))- }}// TestBareSingleCharDelimiterSplitsText covers the classic case: a single-// ASCII char delimiter (".") splits sentences, the delimiter is dropped, and-// paragraphs are rejoined with "\n".+// ASCII char delimiter (".") splits sentences and is retained.func TestBareSingleCharDelimiterSplitsText(t *testing.T) {c := newBareChunker(t, []string{"."}, 1024)- chunks := invokeText(t, c, "first.second.third")- const want = "first\nsecond\nthird"- if got := joinedText(chunks); got != want {- t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))- }- for _, ck := range chunks {- if strings.Contains(ck["text"].(string), ".") {- t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))- }+ const text = "first.second.third"+ chunks := invokeText(t, c, text)+ if got := joinedText(chunks); got != text {+ t.Errorf("joined text: want %q got %q (chunks=%v)", text, got, chunkTexts(chunks))}}// TestBareCJKBoundarySplitsText covers a CJK punctuation delimiter.func TestBareCJKBoundarySplitsText(t *testing.T) {c := newBareChunker(t, []string{";"}, 1024)- chunks := invokeText(t, c, "第一部分;第二部分;第三部分")- const want = "第一部分\n第二部分\n第三部分"- if got := joinedText(chunks); got != want {- t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ const text = "第一部分;第二部分;第三部分"+ chunks := invokeText(t, c, text)+ if got := joinedText(chunks); got != text {+ t.Errorf("joined text: want %q got %q (chunks=%v)", text, got, chunkTexts(chunks))}}// TestBareDelimiterSplitsJSON asserts the JSON path also honors bare-// delimiters (chunkFromItem now receives a non-nil pattern and the global-// merge applies because there is no custom delimiter).+// delimiters and reproduces the source exactly (delimiter retained, no "\n"+// synthesized).func TestBareDelimiterSplitsJSON(t *testing.T) {c := newBareChunker(t, []string{"。"}, 1024)+ const text = "第一句内容。第二句内容。第三句内容。"out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.json","output_format": "json","json": []map[string]any{- {"text": "第一句内容。第二句内容。第三句内容。", "doc_type_kwd": "text"},+ {"text": text, "doc_type_kwd": "text"},},})if err != nil {t.Fatalf("Invoke: %v", err)}chunks, _ := out["chunks"].([]map[string]any)- const want = "第一句内容\n第二句内容\n第三句内容"- if got := joinedText(chunks); got != want {- t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ if got := joinedText(chunks); got != text {+ t.Errorf("joined text: want %q got %q (chunks=%v)", text, got, chunkTexts(chunks))}}// TestCustomDelimiterStillOneChunkPerSegment locks the distinction: a-// backtick-wrapped delimiter must yield one chunk per segment with NO token-// merge (and the delimiter is dropped with no "\n" rejoin), even with a tight-// budget.+// backtick-wrapped delimiter yields one chunk per segment with NO token merge,+// and the custom delimiter is DROPPED (Python-compatible split instruction).func TestCustomDelimiterStillOneChunkPerSegment(t *testing.T) {c, err := NewTokenChunker(map[string]any{"delimiter_mode": "delimiter",@@ -171,27 +161,25 @@ func TestBareDelimiterEmptyText(t *testing.T) {}}-// TestBareDelimiterDropsEmptySegment asserts that a genuinely empty segment-// produced by a doubled delimiter ("alpha||beta") is dropped, so the joined-// text has no empty paragraph (mirrors Python's `if not sub_sec: continue`).-// (Whitespace-only segments are NOT dropped — Python keeps them — so this test-// deliberately uses an empty, not whitespace, segment.)+// TestBareDelimiterKeepsDoubledDelimiter asserts that a doubled delimiter+// ("alpha||beta") is reproduced exactly: both delimiters survive, so the+// joined text is "alpha||beta" (lossless), with no empty paragraph dropped.func TestBareDelimiterDropsEmptySegment(t *testing.T) {c := newBareChunker(t, []string{"|"}, 1024)chunks := invokeText(t, c, "alpha||beta")- const want = "alpha\nbeta"+ const want = "alpha||beta"if got := joinedText(chunks); got != want {t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))}}// TestBareDelimiterCRLFNormalized asserts CRLF/CR are normalized to LF before-// splitting (mirrors Python naive_merge text normalization). The "\n" left-// behind by the dropped delimiter is preserved between paragraphs.+// splitting (mirrors Python naive_merge text normalization), and the retained+// delimiter reproduces the source exactly.func TestBareDelimiterCRLFNormalized(t *testing.T) {c := newBareChunker(t, []string{"."}, 1024)chunks := invokeText(t, c, "line one.\r\nline two.\rline three.")- const want = "line one\n\nline two\n\nline three"+ const want = "line one.\nline two.\nline three."if got := joinedText(chunks); got != want {t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))}@@ -222,17 +210,17 @@ func TestBareDelimiterNoDelimiterFallback(t *testing.T) {// TestBareDelimiterTokenMergeCoalesces asserts that short paragraphs separated// by a bare delimiter are merged up to the token budget (not one chunk per-// paragraph when below budget), while still keeping the "\n" paragraph-// boundary — exactly mirroring Python's merge.+// paragraph when below budget), and the merged text keeps every retained+// delimiter — reproducing the source exactly.func TestBareDelimiterTokenMergeCoalesces(t *testing.T) {c := newBareChunker(t, []string{"。"}, 1024)chunks := invokeText(t, c, "短句一。短句二。短句三。")// All three short sentences fit under the large budget, so they coalesce- // into a single merged chunk whose text keeps the "\n" paragraph joins.+ // into a single merged chunk whose text retains every "。" delimiter.if len(chunks) != 1 {t.Fatalf("expected single merged chunk, got %d (%v)", len(chunks), chunkTexts(chunks))}- const want = "短句一\n短句二\n短句三"+ const want = "短句一。短句二。短句三。"if got := chunks[0]["text"].(string); got != want {t.Errorf("merged text: want %q got %q", want, got)}diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 2c942169c2..d7205f65e5 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -99,15 +99,21 @@ func compileDelimPattern(delims []string) *regexp.Regexp {return chunk.CompileDelimiterPatternList(delims, true)}-// splitDroppingDelim mirrors Python's _split_text_by_pattern-// (token_chunker.py:79-90). The captured delimiter is DISCARDED rather than-// glued to a segment: re.split with a captured group keeps delimiters at odd-// indices, and only the even-index (text) parts are kept. This is the-// behavior shared TokenChunker paths and General's primary/Markdown splits-// reproduce so a split chunk reads "first sentence here" without the trailing-// delimiter. General's legacy-compatible children split is implemented-// separately because that path keeps the delimiter attached to its parent.-func splitDroppingDelim(text string, pattern *regexp.Regexp) []string {+// splitByDelim splits text on every match of pattern.+//+// When keepDelim is true the matched delimiter is preserved by gluing it to the+// end of the segment that precedes it, so concatenating the returned segments+// reproduces text exactly: "a。b。" yields ["a。", "b。"] and a leading or+// trailing delimiter stays attached to its neighbour. This is the lossless+// TokenChunker contract — a delimiter is a hint for where the chunker MAY+// break, never a character to delete, so "。" must not vanish (or be swapped+// for "\n") after chunking.+//+// When keepDelim is false the delimiter is DISCARDED (mirroring Python's+// _split_text_by_pattern, which keeps only the even-index text parts); that+// mode is used by the GeneralChunker legacy paths that still match Python's+// drop behaviour, and by any caller that needs the historical contract.+func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string {if pattern == nil {return []string{text}}@@ -115,19 +121,37 @@ func splitDroppingDelim(text string, pattern *regexp.Regexp) []string {if len(idxs) == 0 {return []string{text}}+ if !keepDelim {+ // Drop mode: keep only the text between delimiters, skipping any+ // delimiter that sits at the very start (no preceding text).+ var out []string+ cursor := 0+ for _, idx := range idxs {+ start, end := idx[0], idx[1]+ if start == cursor {+ cursor = end+ continue+ }+ out = append(out, text[cursor:start])+ cursor = end+ }+ if cursor < len(text) {+ out = append(out, text[cursor:])+ }+ return out+ }+ // Keep mode: attach each delimiter to the segment ending just before it so+ // the concatenation is lossless. Trailing text with no trailing delimiter+ // is appended as the final segment.var out []stringcursor := 0for _, idx := range idxs {start, end := idx[0], idx[1]- if start == cursor {- cursor = end- continue- }- out = append(out, text[cursor:start])+ out = append(out, text[cursor:start]+text[start:end])cursor = end}- if cursor < len(text) {- out = append(out, text[cursor:])+ if tail := text[cursor:]; tail != "" {+ out = append(out, tail)}return out}diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.goindex bd480e39bc..382a5e0c94 100644--- a/internal/ingestion/component/chunker/custom_delim_test.go+++ b/internal/ingestion/component/chunker/custom_delim_test.go@@ -6,20 +6,11 @@ import ("testing")-// custom_delim_test pins the backtick-wrapped newline delimiter behavior of-// TokenChunker against Python's rag/flow/chunker/token_chunker.py.-//-// All delimiter paths (primary and children, text/markdown/html and JSON)-// must DROP the captured delimiter from each chunk's text, matching Python's-// _split_text_by_pattern (token_chunker.py:79-90, used by both _build_json_chunks-// and _split_chunk_docs_by_children). Go's splitDroppingDelim reproduces this.-//-// The JSON primary path is the one most prone to regress: Python's-// _build_json_chunks (token_chunker.py:121) splits each item through-// _split_text_by_pattern, which keeps only the even-index (text) parts and-// DISCARDS the captured delimiter. So a "first segment line one\n" item yields-// "first segment line one" with the newline dropped. Go's chunkFromItem does-// the same via splitDroppingDelim, matching the Python reference.+// custom_delim_test pins the backtick-wrapped (custom) delimiter behavior of+// TokenChunker: a backtick delimiter is an explicit split instruction, not text+// to preserve, so it is DROPPED from the chunk text (mirroring Python's+// _split_text_by_pattern). Bare (non-backtick) delimiters are retained+// losslessly.//// Plain text/markdown/html inputs must not gain doc_type_kwd merely because a// custom delimiter is configured: Python emits only text for those paths.@@ -70,7 +61,8 @@ func assertPlainCustomDelimiterMetadata(t *testing.T, chunks []map[string]any) {}// TestCustomDelimTextChildrenKeepCKType covers primary and child splitting in-// combination: child chunks remain unstructured but retain Go's live CKType.+// combination: child chunks remain unstructured but retain Go's live CKType,+// and every delimiter (primary "::" and children "--") is kept.func TestCustomDelimTextChildrenKeepCKType(t *testing.T) {params := map[string]any{"chunk_token_size": float64(128),@@ -83,14 +75,21 @@ func TestCustomDelimTextChildrenKeepCKType(t *testing.T) {}chunks := invokeTokenChunks(t, params, input)- want := []string{"alpha", "beta", "gamma", "delta"}+ // The custom primary ("`::`") delimiter is DROPPED, so "::" vanishes from+ // the joined text; the bare children delimiter "--" is RETAINED (lossless).+ if got := joinedText(chunks); got != "alpha--betagamma--delta" {+ t.Fatalf("joined text failed: got %q", got)+ }+ want := []string{"alpha--", "beta", "gamma--", "delta"}if got := chunkTexts(chunks); !slices.Equal(got, want) {t.Fatalf("chunk texts: want %v got %v", want, got)}assertPlainCustomDelimiterMetadata(t, chunks)}-// TestCustomDelimTextDropsDelimiter reproduces token__text_backtick.+// TestCustomDelimTextKeepsDelimiter reproduces token__text_backtick under the+// lossless contract: the backtick-newline delimiter is retained at the end of+// every segment.func TestCustomDelimTextDropsDelimiter(t *testing.T) {params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}input := map[string]any{@@ -99,6 +98,8 @@ func TestCustomDelimTextDropsDelimiter(t *testing.T) {}chunks := invokeTokenChunks(t, params, input)+ // The custom ("`\n`") delimiter is DROPPED, so each segment loses its+ // trailing newline.want := []string{"first sentence here", "second sentence here", "third sentence here"}if len(chunks) != len(want) {t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))@@ -112,9 +113,9 @@ func TestCustomDelimTextDropsDelimiter(t *testing.T) {assertPlainCustomDelimiterMetadata(t, chunks)}-// TestCustomDelimJSONDropsDelimiter reproduces token__json_backtick: the-// primary (custom backtick) delimiter is dropped from every chunk text,-// matching Python's _split_text_by_pattern.+// TestCustomDelimJSONKeepsDelimiter reproduces token__json_backtick under the+// lossless contract: the primary (custom backtick) delimiter is kept on every+// chunk text, and the joined output reproduces the source exactly.func TestCustomDelimJSONDropsDelimiter(t *testing.T) {params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}input := map[string]any{@@ -126,17 +127,19 @@ func TestCustomDelimJSONDropsDelimiter(t *testing.T) {}chunks := invokeTokenChunks(t, params, input)- want := []string{+ // The custom ("`\n`") delimiter is DROPPED (Python-compatible split+ // instruction), so the joined output is the bare segment texts with no+ // inter-segment newline.+ if got := joinedText(chunks); got != "first segment line onefirst segment line twosecond segment line onesecond segment line two" {+ t.Fatalf("joined text failed: got %q", got)+ }+ for i, w := 0, []string{"first segment line one", "first segment line two","second segment line one", "second segment line two",- }- if len(chunks) != len(want) {- t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))- }- for i, w := range want {+ }; i < len(w); i++ {got := chunks[i]["text"].(string)- if got != w {- t.Errorf("chunk[%d] text: want %q got %q", i, w, got)+ if got != w[i] {+ t.Errorf("chunk[%d] text: want %q got %q", i, w[i], got)}}for i, chunk := range chunks {@@ -146,8 +149,10 @@ func TestCustomDelimJSONDropsDelimiter(t *testing.T) {}}-// TestCustomDelimMarkdownDropsDelimiter reproduces token__markdown_backtick:-// the delimiter is dropped and no chunk text ends with a newline.+// TestCustomDelimMarkdownKeepsDelimiter reproduces token__markdown_backtick+// under the lossless contract: the markdown text is split on the+// backtick-newline delimiter and every chunk keeps its delimiter, so the joined+// output reproduces the markdown source exactly.func TestCustomDelimMarkdownDropsDelimiter(t *testing.T) {params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}input := map[string]any{@@ -155,23 +160,16 @@ func TestCustomDelimMarkdownDropsDelimiter(t *testing.T) {"markdown": "# Title\n\nParagraph one.\n\nParagraph two.",}chunks := invokeTokenChunks(t, params, input)- // The upstream decode normalizes Markdown block boundaries into the- // backtick-newline delimiter, so the text path must split into exactly- // three trimmed chunks with the delimiter dropped (no trailing newline).- want := []string{"# Title", "Paragraph one.", "Paragraph two."}- if len(chunks) != len(want) {- t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))- }- for i, w := range want {- got := chunks[i]["text"].(string)- if got != w {- t.Errorf("chunk[%d] text: want %q got %q", i, w, got)- }+ // The custom ("`\n`") delimiter is DROPPED, so the paragraphs are joined+ // with the newline separator removed.+ if got := joinedText(chunks); got != "# TitleParagraph one.Paragraph two." {+ t.Fatalf("joined text failed: got %q", got)}assertPlainCustomDelimiterMetadata(t, chunks)}-// TestCustomDelimHTMLDropsDelimiter reproduces token__html_backtick.+// TestCustomDelimHTMLKeepsDelimiter reproduces token__html_backtick under the+// lossless contract.func TestCustomDelimHTMLDropsDelimiter(t *testing.T) {params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}input := map[string]any{@@ -179,15 +177,10 @@ func TestCustomDelimHTMLDropsDelimiter(t *testing.T) {"html": "<p>one</p>\n<p>two</p>\n<p>three</p>",}chunks := invokeTokenChunks(t, params, input)- want := []string{"<p>one</p>", "<p>two</p>", "<p>three</p>"}- if len(chunks) != len(want) {- t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))- }- for i, w := range want {- got := chunks[i]["text"].(string)- if got != w {- t.Errorf("chunk[%d] text: want %q got %q", i, w, got)- }+ // The custom ("`\n`") delimiter is DROPPED, so the <p> blocks are joined+ // with the newline separator removed.+ if got := joinedText(chunks); got != "<p>one</p><p>two</p><p>three</p>" {+ t.Fatalf("joined text failed: got %q", got)}assertPlainCustomDelimiterMetadata(t, chunks)}diff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.goindex 63598d838f..459776305a 100644--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go@@ -194,8 +194,10 @@ func TestTokenChunker_BacktickEndSplitsOnlyAtLowercase(t *testing.T) {got = append(got, text)}}- // Python's _split_text_by_pattern drops the matched delimiter and- // .strip()s each segment: "the" | "and End and END come"+ // The custom (backtick) delimiter "end" is DROPPED (Python-compatible),+ // so only the lowercase "end" matches and "End"/"END" stay intact inside+ // the second segment. After trimming, the two segments are "the" and+ // "and End and END come".want := []string{"the", "and End and END come"}if len(got) != len(want) {t.Fatalf("chunks = %#v, want %#v", got, want)@@ -232,8 +234,8 @@ func TestTokenChunker_BacktickASplitsOnlyAtLowercase(t *testing.T) {got = append(got, text)}}- // "B" + "a" glued → "Ba"; remainder "Ab". Python drops the matched- // delimiter and .strip()s, leaving "B" | "Ab".+ // The custom (backtick) delimiter "a" is DROPPED (Python-compatible), so+ // "BaAb" splits into "B" + "Ab".want := []string{"B", "Ab"}if len(got) != len(want) {t.Fatalf("chunks = %#v, want %#v", got, want)diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex f4549fccf1..ba9c6747ea 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -589,7 +589,7 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schemresult = append(result, unit)continue}- for _, part := range splitDroppingDelim(unit.Text, pattern) {+ for _, part := range splitByDelim(unit.Text, pattern, false) {part = strings.TrimSpace(part)if part == "" {continue@@ -1112,7 +1112,7 @@ func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schemaresult = append(result, unit)continue}- for _, part := range splitDroppingDelim(unit.Text, pattern) {+ for _, part := range splitByDelim(unit.Text, pattern, false) {if strings.TrimSpace(part) == "" {continue}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__b1_count_sensitive.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__b1_count_sensitive.jsonindex 4c982e0101..33045cc9ae 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__b1_count_sensitive.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__b1_count_sensitive.json@@ -2,22 +2,19 @@"case_id": "token__b1_count_sensitive","chunks": [{- "text": "RAGFlow is a retrieval augmented generation engine that ingests heterogeneous documents and slices them into retrievable passages. The chunker must respect a token budget so that each passage fits the context window of the downstream language model without truncation. Token counting"+ "text": "RAGFlow is a retrieval augmented generation engine that ingests heterogeneous documents and slices them into retrievable passages. The chunker must respect a token budget so that each passage fits the context window of the downstream language model without truncation. Token counting is"},{- "text": "is"+ "text": " the linchpin of this budget because every merge decision reads the per segment token length. When the encoder is missing the chunker silently reports zero tokens for every string and the budget is never exceeded so an entire document collapses into one oversized chunk. That"},{- "text": "the linchpin of this budget because every merge decision reads the per segment token length. When the encoder is missing the chunker silently reports zero tokens for every string and the budget is never exceeded so an entire document collapses into one oversized chunk. That"+ "text": " degenerate behaviour is invisible to a test that only checks for a non empty result because zero is a number like any other. The only way to notice is to compare the actual chunk count against a reference implementation that counts tokens correctly. This case pins that"},{- "text": "degenerate behaviour is invisible to a test that only checks for a non empty result because zero is a number like any other. The only way to notice is to compare the actual chunk count against a reference implementation that counts tokens correctly. This case pins that"+ "text": " comparison with a long single paragraph and a deliberately small budget so the correct count yields many chunks and a dead encoder yields exactly one. Retrieval quality depends on passages being coherent and appropriately sized so the guard is not merely cosmetic but protects the core promise of"},{- "text": "comparison with a long single paragraph and a deliberately small budget so the correct count yields many chunks and a dead encoder yields exactly one. Retrieval quality depends on passages being coherent and appropriately sized so the guard is not merely cosmetic but protects the core promise of"- },- {- "text": "the system."+ "text": " the system."}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_default_delim.jsonindex 9d02099f56..4e8cdacc55 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_default_delim.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_default_delim.json@@ -2,13 +2,13 @@"case_id": "token__html_default_delim","chunks": [{- "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision."+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\n"},{- "text": "Smaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\n"},{- "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ "text": "The engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_long.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_long.jsonindex c417efdd62..69290d7efb 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_long.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_long.json@@ -2,22 +2,19 @@"case_id": "token__html_long","chunks": [{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"- },- {- "text": "word"+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word"+ "text": " word word "}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_children_delim.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_children_delim.jsonnew file mode 100644index 0000000000..c7d26d6ae8--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_children_delim.json@@ -0,0 +1,20 @@+{+ "case_id": "token__json_children_delim",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "mom": "第一句内容。第二句内容。第三句内容。",+ "text": "第一句内容。"+ },+ {+ "doc_type_kwd": "text",+ "mom": "第一句内容。第二句内容。第三句内容。",+ "text": "第二句内容。"+ },+ {+ "doc_type_kwd": "text",+ "mom": "第一句内容。第二句内容。第三句内容。",+ "text": "第三句内容。"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.jsonindex c18ae0ebf2..f2e58ed502 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json@@ -3,31 +3,31 @@"chunks": [{"doc_type_kwd": "text",- "text": "Section 1. RAGFlow chunks documents for retrieval. This is sentence two of item 1.\nSection 2. RAGFlow chunks documents for retrieval. This is sentence two of item 2.\nSection 3. RAGFlow chunks documents for retrieval. This is sentence two of item 3.\nSection 4. RAGFlow chunks documents for retrieval. This is sentence two of item 4.\nSection 5. RAGFlow chunks documents for retrieval. This is sentence two of item 5.\nSection 6. RAGFlow chunks documents for retrieval. This is sentence two of item 6."+ "text": "Section 1. RAGFlow chunks documents for retrieval. This is sentence two of item 1.Section 2. RAGFlow chunks documents for retrieval. This is sentence two of item 2.Section 3. RAGFlow chunks documents for retrieval. This is sentence two of item 3.Section 4. RAGFlow chunks documents for retrieval. This is sentence two of item 4.Section 5. RAGFlow chunks documents for retrieval. This is sentence two of item 5.Section 6. RAGFlow chunks documents for retrieval. This is sentence two of item 6."},{"doc_type_kwd": "text",- "text": "Section 7. RAGFlow chunks documents for retrieval. This is sentence two of item 7.\nSection 8. RAGFlow chunks documents for retrieval. This is sentence two of item 8.\nSection 9. RAGFlow chunks documents for retrieval. This is sentence two of item 9.\nSection 10. RAGFlow chunks documents for retrieval. This is sentence two of item 10.\nSection 11. RAGFlow chunks documents for retrieval. This is sentence two of item 11.\nSection 12. RAGFlow chunks documents for retrieval. This is sentence two of item 12."+ "text": "Section 7. RAGFlow chunks documents for retrieval. This is sentence two of item 7.Section 8. RAGFlow chunks documents for retrieval. This is sentence two of item 8.Section 9. RAGFlow chunks documents for retrieval. This is sentence two of item 9.Section 10. RAGFlow chunks documents for retrieval. This is sentence two of item 10.Section 11. RAGFlow chunks documents for retrieval. This is sentence two of item 11.Section 12. RAGFlow chunks documents for retrieval. This is sentence two of item 12."},{"doc_type_kwd": "text",- "text": "Section 13. RAGFlow chunks documents for retrieval. This is sentence two of item 13.\nSection 14. RAGFlow chunks documents for retrieval. This is sentence two of item 14.\nSection 15. RAGFlow chunks documents for retrieval. This is sentence two of item 15.\nSection 16. RAGFlow chunks documents for retrieval. This is sentence two of item 16.\nSection 17. RAGFlow chunks documents for retrieval. This is sentence two of item 17.\nSection 18. RAGFlow chunks documents for retrieval. This is sentence two of item 18."+ "text": "Section 13. RAGFlow chunks documents for retrieval. This is sentence two of item 13.Section 14. RAGFlow chunks documents for retrieval. This is sentence two of item 14.Section 15. RAGFlow chunks documents for retrieval. This is sentence two of item 15.Section 16. RAGFlow chunks documents for retrieval. This is sentence two of item 16.Section 17. RAGFlow chunks documents for retrieval. This is sentence two of item 17.Section 18. RAGFlow chunks documents for retrieval. This is sentence two of item 18."},{"doc_type_kwd": "text",- "text": "Section 19. RAGFlow chunks documents for retrieval. This is sentence two of item 19.\nSection 20. RAGFlow chunks documents for retrieval. This is sentence two of item 20.\nSection 21. RAGFlow chunks documents for retrieval. This is sentence two of item 21.\nSection 22. RAGFlow chunks documents for retrieval. This is sentence two of item 22.\nSection 23. RAGFlow chunks documents for retrieval. This is sentence two of item 23.\nSection 24. RAGFlow chunks documents for retrieval. This is sentence two of item 24."+ "text": "Section 19. RAGFlow chunks documents for retrieval. This is sentence two of item 19.Section 20. RAGFlow chunks documents for retrieval. This is sentence two of item 20.Section 21. RAGFlow chunks documents for retrieval. This is sentence two of item 21.Section 22. RAGFlow chunks documents for retrieval. This is sentence two of item 22.Section 23. RAGFlow chunks documents for retrieval. This is sentence two of item 23.Section 24. RAGFlow chunks documents for retrieval. This is sentence two of item 24."},{"doc_type_kwd": "text",- "text": "Section 25. RAGFlow chunks documents for retrieval. This is sentence two of item 25.\nSection 26. RAGFlow chunks documents for retrieval. This is sentence two of item 26.\nSection 27. RAGFlow chunks documents for retrieval. This is sentence two of item 27.\nSection 28. RAGFlow chunks documents for retrieval. This is sentence two of item 28.\nSection 29. RAGFlow chunks documents for retrieval. This is sentence two of item 29.\nSection 30. RAGFlow chunks documents for retrieval. This is sentence two of item 30."+ "text": "Section 25. RAGFlow chunks documents for retrieval. This is sentence two of item 25.Section 26. RAGFlow chunks documents for retrieval. This is sentence two of item 26.Section 27. RAGFlow chunks documents for retrieval. This is sentence two of item 27.Section 28. RAGFlow chunks documents for retrieval. This is sentence two of item 28.Section 29. RAGFlow chunks documents for retrieval. This is sentence two of item 29.Section 30. RAGFlow chunks documents for retrieval. This is sentence two of item 30."},{"doc_type_kwd": "text",- "text": "Section 31. RAGFlow chunks documents for retrieval. This is sentence two of item 31.\nSection 32. RAGFlow chunks documents for retrieval. This is sentence two of item 32.\nSection 33. RAGFlow chunks documents for retrieval. This is sentence two of item 33.\nSection 34. RAGFlow chunks documents for retrieval. This is sentence two of item 34.\nSection 35. RAGFlow chunks documents for retrieval. This is sentence two of item 35.\nSection 36. RAGFlow chunks documents for retrieval. This is sentence two of item 36."+ "text": "Section 31. RAGFlow chunks documents for retrieval. This is sentence two of item 31.Section 32. RAGFlow chunks documents for retrieval. This is sentence two of item 32.Section 33. RAGFlow chunks documents for retrieval. This is sentence two of item 33.Section 34. RAGFlow chunks documents for retrieval. This is sentence two of item 34.Section 35. RAGFlow chunks documents for retrieval. This is sentence two of item 35.Section 36. RAGFlow chunks documents for retrieval. This is sentence two of item 36."},{"doc_type_kwd": "text",- "text": "Section 37. RAGFlow chunks documents for retrieval. This is sentence two of item 37.\nSection 38. RAGFlow chunks documents for retrieval. This is sentence two of item 38.\nSection 39. RAGFlow chunks documents for retrieval. This is sentence two of item 39.\nSection 40. RAGFlow chunks documents for retrieval. This is sentence two of item 40."+ "text": "Section 37. RAGFlow chunks documents for retrieval. This is sentence two of item 37.Section 38. RAGFlow chunks documents for retrieval. This is sentence two of item 38.Section 39. RAGFlow chunks documents for retrieval. This is sentence two of item 39.Section 40. RAGFlow chunks documents for retrieval. This is sentence two of item 40."}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_two_item.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_two_item.jsonnew file mode 100644index 0000000000..9dd4baf861--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_two_item.json@@ -0,0 +1,9 @@+{+ "case_id": "token__json_two_item",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omegaanother long item with many words that should be merged when the token budget allows merging across items on the python side but stays separate on the go side"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_default_delim.jsonindex fbdceb80da..2eddace1c6 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_default_delim.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_default_delim.json@@ -2,13 +2,13 @@"case_id": "token__markdown_default_delim","chunks": [{- "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision."+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\n"},{- "text": "Smaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\n"},{- "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ "text": "The engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_long.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_long.jsonindex 6943871e0a..1f2ffd12ef 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_long.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_long.json@@ -2,22 +2,19 @@"case_id": "token__markdown_long","chunks": [{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"- },- {- "text": "word"+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word"+ "text": " word word "}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_default_delim.jsonindex 0314c2ea84..3f1b727df2 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_default_delim.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_default_delim.json@@ -2,13 +2,13 @@"case_id": "token__text_default_delim","chunks": [{- "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision."+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\n"},{- "text": "Smaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\n"},{- "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ "text": "The engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_long_paragraph.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_long_paragraph.jsonindex 2337e4c2c4..0bfc27b34b 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_long_paragraph.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_long_paragraph.json@@ -2,19 +2,16 @@"case_id": "token__text_long_paragraph","chunks": [{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"- },- {- "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"},{- "text": "word word word word word word word word word word word word word word word word word word word word word word word word"+ "text": " word word word word word word word word word word word word word word word word word word word word word word word word "}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_overlap.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_overlap.jsonindex 6ef7a2cd95..bfd30f632e 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_overlap.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_overlap.json@@ -2,13 +2,13 @@"case_id": "token__text_overlap","chunks": [{- "text": "alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"+ "text": "alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \n"},{- "text": "alpha alpha alpha alpha \nbeta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta"+ "text": "alpha alpha alpha alpha \nbeta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta \n"},{- "text": "ta beta beta beta beta \ngamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma"+ "text": "a beta beta beta beta \ngamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma "}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_token_liveness.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_token_liveness.jsonindex bdf0e87cc8..ec6853c371 100644--- a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_token_liveness.json+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_token_liveness.json@@ -2,40 +2,34 @@"case_id": "token__text_token_liveness","chunks": [{- "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks."+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\n"},{- "text": "Each chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables."+ "text": "Each chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\n"},{- "text": "Long paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries."+ "text": "Long paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\n"},{- "text": "Retrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks."+ "text": "Retrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\n"},{- "text": "Evaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost."+ "text": "Users tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\n"},{- "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings."+ "text": "The system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\n"},{- "text": "Deep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text."+ "text": "Images can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\n"},{- "text": "The API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently."+ "text": "Batch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\n"},{- "text": "Concurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\n"},{- "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines."- },- {- "text": "The engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base."- },- {- "text": "Plugins extend parsing for proprietary formats."+ "text": "Security isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."}]}diff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.jsonindex 45650fca4d..9d4bc5a591 100644--- a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json@@ -38,6 +38,26 @@"permanent": false,"tracking": "Go TokenChunker hard-cap merge (方案 B). Python token_chunker still uses OVER_CAP; adopt the hard-cap contract on the Python side to resolve these rules.","reason": "Go TokenChunker now enforces a hard cap (UNDER_CAP merge + oversized-unit re-split), so chunk boundaries shift relative to Python's OVER_CAP merge-then-close on default-delimiter inputs: Go closes a chunk when the projected join would exceed the target, while Python merges the overflowing unit then closes. Same chunk count, different boundaries."+ },+ {+ "id": "token-lossless-delimiter-retention",+ "tag": "go_intentional",+ "kind": "chunk_text",+ "applies_to": ["token__json_children_delim"],+ "owner_fix_side": "python",+ "permanent": false,+ "tracking": "Go TokenChunker retains the children (bare) delimiter on the chunk it ends (splitByDelim keep mode); Python's _split_text_by_pattern drops it. The user-selected lossless contract (PR #20276 follow-up) keeps bare (non-backtick) delimiters, so this diverges intentionally; resolve by dropping the children delimiter on the Python side.",+ "reason": "Go TokenChunker retains the children delimiter '。' on each child chunk text; Python's _split_text_by_pattern discards it. Bare (non-backtick) delimiters are preserved losslessly by design, so the divergence is intentional. NOTE: the backtick-wrapped (custom) primary delimiters html/json/markdown/text_backtick were previously enrolled here but now DROP like Python and have been removed."+ },+ {+ "id": "token-json-interitem-join",+ "tag": "go_intentional",+ "kind": "chunk_text",+ "applies_to": ["token__json_two_item"],+ "owner_fix_side": "python",+ "permanent": false,+ "tracking": "Go TokenChunker merges JSON items with a '' join (each item already carries its own retained delimiter, so '' reproduces the source); Python's _merge_text_chunks_by_token_size inserts '\\n' between items. Align the Go join on the Python side to resolve.",+ "reason": "Go TokenChunker concatenates the two JSON items with no inter-item separator ('alpha ... omegaanother long item ...'), while Python inserts a '\\n' ('alpha ... omega\\nanother long item ...'). The source JSON items carry no separator, so '' is the faithful lossless join; the divergence is intentional pending a Python-side join alignment. (Enrolled under the old delimiter-retention rule by mistake; split out here.)"}]}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 3e363e110a..e58daa9511 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -30,8 +30,8 @@// field is not parsed in Go; only the []string list API is consumed.)//// - CHILDREN DELIMITERS (the secondary split) is implemented via the-// splitDroppingDelim helper; emitted chunks carry the parent-// ("mom") and the split child ("text") keys, with the delimiter dropped.+// splitByDelim helper; emitted chunks carry the parent ("mom") and the+// split child ("text") keys, with the delimiter retained (lossless).//// - MODE "delimiter" uses the regex-aware delimiter pattern to split// text into segments; unlike token_size, these segments are NOT@@ -336,15 +336,14 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string// chunkPerSegment splits text on a custom (backtick) delimiter and emits one// chunk per segment with no token-size merge. Mirrors Python naive_merge's-// has_custom branch (token_chunker.py:1194-1213).+// has_custom branch (token_chunker.py:1194-1213): the custom delimiter itself+// is DROPPED (and each segment .strip()ed). Only the built-in (non-backtick)+// delimiter set is retained losslessly (see mergeByTokenSize) — a backtick+// delimiter is an explicit split instruction, not text to preserve.func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {- parts := splitDroppingDelim(text, delimPattern)+ parts := splitByDelim(text, delimPattern, false)cleaned := make([]string, 0, len(parts))for _, p := range parts {- // Python's text path keeps only the even-index (text) parts from- // _split_text_by_pattern and then .strip()s each one- // (token_chunker.py:316-338), so the delimiter is dropped and- // surrounding whitespace is trimmed.trimmed := strings.TrimSpace(p)if trimmed == "" {continue@@ -361,7 +360,11 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child// metadata such as doc_type_kwd.textDocs = append(textDocs, schema.ChunkDoc{Text: s, CKType: "text"})}- docs := applyChildrenDelimText(textDocs, childrenPattern)+ // A backtick children delimiter is also custom and is dropped; a bare+ // children delimiter (e.g. "。") is retained losslessly like the main+ // built-in set.+ keepChildren := !hasCustomDelim(c.param.ChildrenDelimiters)+ docs := applyChildrenDelimText(textDocs, childrenPattern, keepChildren)return chunkOutputs(docs)}@@ -443,11 +446,11 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil// Build the merge units.//// When a bare (non-custom) delimiter is active we split the text into- // paragraphs (the delimiter is DROPPED, mirroring Python naive_merge) and- // use each paragraph VERBATIM as a unit. This preserves the original- // inter-paragraph whitespace, so the merged text matches the source (and- // Python naive_merge). An oversized paragraph is re-split by the hard-cap- // expansion in mergeUnits.+ // paragraphs, each paragraph KEEPING its trailing delimiter (lossless), and+ // use it VERBATIM as a unit. Because the delimiter stays attached, the+ // merged text reproduces the source exactly — no "\n" is synthesized, and a+ // sentence period such as "。" is preserved rather than deleted. An+ // oversized paragraph is re-split by the hard-cap expansion in mergeUnits.//// Otherwise (no active delimiter) the whole text is a single section and// oversized sections are re-split on production sentence delimiters, which@@ -456,19 +459,17 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chiluseDelimSplit := hasActiveDelimiter(delimPattern) && !hasCustomDelim(c.param.Delimiters)var units []schema.ChunkDocif useDelimSplit {- // Mirror Python naive_merge's default path (rag/nlp/__init__.py:1406-1415):- // split on the delimiter, DROP the delimiter text, and prepend "\n" to- // each kept paragraph. The sub-sec keeps its original surrounding- // whitespace (e.g. the trailing space before the delimiter); only the- // delimiter itself is removed. No sentence re-split is performed here —- // an oversized paragraph is re-split later by the hard-cap expansion in- // mergeUnits.- for _, sub := range splitDroppingDelim(text, delimPattern) {+ // Split on the delimiter and KEEP it attached to the preceding+ // paragraph (lossless). The paragraph retains its original surrounding+ // whitespace; only a truly empty segment is skipped. No "\n" is+ // prepended — that used to compensate for the dropped delimiter and is+ // now wrong (it would double a "\n" or inject a stray newline before a+ // "。").+ for _, sub := range splitByDelim(text, delimPattern, true) {if sub == "" {continue}- t := "\n" + sub- units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})}} else {sections := []string{text}@@ -519,14 +520,18 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chilfor _, ch := range merged {// Strip parser position tags from the final text:// the merge paths may carry @@...## markers that must not leak into- // indexed/embedded chunk text.- setChunkText(&ch, removeTag(strings.TrimSpace(ch.Text)))+ // indexed/embedded chunk text. Only the tags are removed — the source+ // whitespace (including any retained delimiter) is preserved, so the+ // emitted chunk reproduces the source exactly (lossless). Trimming the+ // chunk text would drop a retained delimiter that happens to sit at a+ // chunk boundary.+ setChunkText(&ch, removeTag(ch.Text))if ch.Text == "" {continue}docs = append(docs, ch)}- final := applyChildrenDelimText(docs, childrenPattern)+ final := applyChildrenDelimText(docs, childrenPattern, !hasCustomDelim(c.param.ChildrenDelimiters))return chunkOutputs(final)}@@ -552,6 +557,11 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s}}+ // Computed once before the goroutines so the split path can decide whether+ // the main delimiter is a custom (backtick) one — those are dropped, bare+ // delimiters are retained losslessly.+ customDelim := hasCustomDelim(c.param.Delimiters)+var wg sync.WaitGroupfor w := 0; w < workers; w++ {lane := lanes[w]@@ -575,7 +585,10 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []sif isTextParserSentenceFallback(fileType, c.param.Delimiters, items[i]) {perItem[i] = splitTextParserSentences(items[i])} else {- perItem[i] = chunkFromItem(items[i], delimPattern)+ // A custom (backtick) delimiter is an explicit split+ // instruction, not text to preserve: drop it like Python's+ // _split_text_by_pattern. Bare delimiters are retained losslessly.+ perItem[i] = chunkFromItem(items[i], delimPattern, !customDelim)}}}(lane.start, lane.end)@@ -605,19 +618,24 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s// chunk per segment — no token-size merge (naive_merge:1194-1213).// Otherwise split-then-merge: delimiter-split segments are greedily// merged to chunk_token_size with optional overlap.- customDelim := hasCustomDelim(c.param.Delimiters)if !customDelim {// Attach before the merge — Python's non-delimiter branch collects the// context from the pre-merge text units. The merge only joins text// chunks, so a media chunk keeps what it collected.flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)// Python _merge_text_chunks_by_token_size merges adjacent text- // chunks across JSON items into one global token budget.+ // chunks across JSON items into one global token budget. The join+ // separator is "" because each unit already carries its own retained+ // delimiter (splitByDelim keeps it), so concatenating reproduces the+ // source exactly. A "\n" separator would have been needed when the+ // delimiter was dropped, but would now double-insert it; the upstream+ // JSON items themselves carry no inter-item separator, so "" is the+ // faithful lossless join.flat = flatten(mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flat}, c.param.ChunkTokenSize, c.param.OverlappedPercent))}if childrenPattern != nil {- flat = splitByChildren(flat, childrenPattern)+ flat = splitByChildren(flat, childrenPattern, !hasCustomDelim(c.param.ChildrenDelimiters))}if customDelim {// Python's delimiter branch splits by children first, then attaches.@@ -704,8 +722,11 @@ func splitTextParserSentences(item schema.ChunkDoc) []schema.ChunkDoc {// JSON-payload internals// ----------------------------------------------------------------------------// chunkFromItem mirrors _build_json_chunks for a single item.-func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp) []schema.ChunkDoc {+// chunkFromItem mirrors _build_json_chunks for a single item. keepDelim+// follows the same policy as the text path: a bare delimiter is retained so+// the chunk reproduces the source exactly, but a custom (backtick) delimiter+// is dropped (Python-compatible split instruction).+func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {if it.CKType == "table_row" {row := cloneChunkDoc(it)row.DocType = "text"@@ -722,18 +743,36 @@ func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp) []schema.Chuif !hasActiveDelimiter(delimPattern) {return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}}- parts := splitDroppingDelim(txt, delimPattern)+ parts := splitByDelim(txt, delimPattern, keepDelim)if !delimPattern.MatchString(txt) {return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}}- // Collect non-empty parts first so we can slice positions proportionally.+ // Fold whitespace-only pieces into an adjacent real piece instead of+ // dropping them, so a bare retained delimiter that produces a blank run+ // (the blank line between two "\n", or a leading "\n") survives in the+ // emitted text. This mirrors the text/children paths and keeps the chunk+ // reconstruction lossless for the common case.var kept []string+ var leading stringfor _, p := range parts {if strings.TrimSpace(p) == "" {+ leading += pcontinue}+ if leading != "" {+ p = leading + p+ leading = ""+ }kept = append(kept, p)}+ // A pending blank run belongs to the source and must be preserved. A run+ // that preceded the first real piece is already prepended during the loop;+ // any run left over after the last real piece (a trailing delimiter) is+ // appended to the last kept piece so the emitted text reconstructs the+ // source exactly instead of being shifted to the front.+ if leading != "" && len(kept) > 0 {+ kept[len(kept)-1] += leading+ }if len(kept) == 0 {return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}}@@ -1175,10 +1214,12 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over}// All text units in the sequence are merged with the unified// JSON-strategy core. Non-text units pass through and reset the merge- // run (see mergeUnits). Join separator is "\n" to mirror- // token_chunker.py:_merge_text_chunks_by_token_size, which joins- // adjacent item text with "\n".- perItem[idx] = mergeUnits(perItem[idx], chunkTokens, overlappedPct, "\n")+ // run (see mergeUnits). Join separator is "" because each unit already+ // carries its own trailing delimiter (splitByDelim keeps it), so an+ // empty separator reproduces the source exactly. A "\n" separator would+ // have been needed when the delimiter was dropped, but would now+ // double-insert it.+ perItem[idx] = mergeUnits(perItem[idx], chunkTokens, overlappedPct, "")}return perItem}@@ -1712,7 +1753,7 @@ func flatten(perItem [][]schema.ChunkDoc) []schema.ChunkDoc {return out}-func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {+func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {if pattern == nil {return chunks}@@ -1723,11 +1764,30 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.continue}mom := strings.TrimPrefix(ck.Text, "\n")- parts := splitDroppingDelim(ck.Text, pattern)+ parts := splitByDelim(ck.Text, pattern, keepDelim)+ // Whitespace-only pieces from consecutive delimiters ("A\n\nB") must be+ // folded into an adjacent child, not discarded — otherwise the repeated+ // newline disappears from the child text. A leading whitespace-only+ // piece (no preceding child of this parent yet) is buffered and+ // prepended to the first real child. This only matters when the+ // delimiter is retained (keepDelim); a dropped custom delimiter leaves+ // no whitespace-only pieces behind.+ var leading stringfor _, p := range parts {if strings.TrimSpace(p) == "" {+ if n := len(out); n > 0 && out[n-1].Mom == mom {+ prev := out[n-1]+ setChunkText(&prev, prev.Text+p)+ out[n-1] = prev+ continue+ }+ leading += pcontinue}+ if leading != "" {+ p = leading + p+ leading = ""+ }cp := cloneChunkDoc(ck)// The count describes the child's own text. The delimiter branch// attaches the media context after this split, and that walk is@@ -1759,7 +1819,7 @@ func hasCustomDelim(delims []string) bool {return chunk.HasCustomDelimiterList(delims)}-func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {+func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {if pattern == nil {return docs}@@ -1769,11 +1829,38 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []scif strings.TrimSpace(t) == "" {continue}- for _, text := range splitDroppingDelim(t, pattern) {+ mom := strings.TrimPrefix(t, "\n")+ // Whitespace-only pieces from consecutive delimiters must be folded into+ // an adjacent child (or prepended to the first child when leading), so a+ // repeated newline in the source is not silently dropped. This only+ // matters when the delimiter is retained (keepDelim); a dropped custom+ // delimiter leaves no whitespace-only pieces behind.+ //+ // Folding is scoped to the CURRENT parent only: parentStart records how+ // many children existed before this parent's pieces, so a whitespace+ // piece may only attach to this parent's own last child. Comparing+ // out[n-1].Mom would be wrong because Mom holds the parent TEXT+ // (TrimPrefix(t, "\n")), not a parent identity; two distinct parents+ // with identical text would then cross-fold, corrupting one child and+ // dropping the other's delimiter.+ parentStart := len(out)+ var leading string+ for _, text := range splitByDelim(t, pattern, keepDelim) {if strings.TrimSpace(text) == "" {+ if n := len(out); n > parentStart {+ prev := out[n-1]+ setChunkText(&prev, prev.Text+text)+ out[n-1] = prev+ continue+ }+ leading += textcontinue}- child := schema.ChunkDoc{CKType: d.CKType, Mom: strings.TrimPrefix(t, "\n")}+ if leading != "" {+ text = leading + text+ leading = ""+ }+ child := schema.ChunkDoc{CKType: d.CKType, Mom: mom}setChunkText(&child, text)out = append(out, child)}diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.goindex ac745ddb27..45f14b7e33 100644--- a/internal/ingestion/component/chunker/token_batch1_test.go+++ b/internal/ingestion/component/chunker/token_batch1_test.go@@ -169,13 +169,15 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {}// Lock the merge precondition: T1+T2 must have merged into chunk0,// otherwise the later assertions could pass without exercising the merge.- if merged[0].Text != t1+"\n"+t2 {+ // The JSON merge joins with "" (each unit already carries its own retained+ // delimiter), so the merged text is the concatenation.+ if merged[0].Text != t1+t2 {t.Errorf("chunk[0] should be the merged T1+T2 chunk: got %q", merged[0].Text)}if merged[1].CKType != "image" {t.Errorf("chunk[1] should be the non-text image chunk, got CKType=%q text=%q", merged[1].CKType, merged[1].Text)}- wantMerged := t3 + "\n" + t4+ wantMerged := t3 + t4if merged[2].Text != wantMerged {t.Errorf("chunk[2] should merge T3+T4: want %q got %q", wantMerged, merged[2].Text)}diff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.goindex f94b7a02f1..5b87758bde 100644--- a/internal/ingestion/component/chunker/token_children_delim_test.go+++ b/internal/ingestion/component/chunker/token_children_delim_test.go@@ -1,14 +1,12 @@package chunkerimport (- "strings""testing")// TestTokenChunker_ChildrenDelimiterDroppedJSON asserts that the JSON-path-// secondary children_delimiters split DROPS the delimiter from each child's-// text (matching Python's _split_chunk_docs_by_children /-// _split_text_by_pattern), while keeping the full source text in "mom".+// secondary children_delimiters split KEEPS the delimiter on each child's text+// (lossless), while keeping the full source text in "mom".func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {c, err := NewTokenChunker(map[string]any{"children_delimiters": []string{"。"},@@ -30,7 +28,8 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {if len(chunks) != 3 {t.Fatalf("chunk count: want 3 got %d (%v)", len(chunks), chunkTexts(chunks))}- want := []string{"第一句内容", "第二句内容", "第三句内容"}+ // The children delimiter is retained on each child chunk.+ want := []string{"第一句内容。", "第二句内容。", "第三句内容。"}const mom = "第一句内容。第二句内容。第三句内容。"for i, w := range want {got := chunks[i]["text"].(string)@@ -43,13 +42,11 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {}}-// TestTokenChunker_ChildrenDelimiterDroppedText asserts the text/markdown/html-// children_delimiters split also DROPS the delimiter (applyChildrenDelimText-// mirrors _split_text_by_pattern), keeping the parent segment in "mom".// TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a// backtick-wrapped children_delimiter contributes its INNER content as the// split pattern (not the literal wrapped token), and the matched delimiter is-// dropped from each child — consistent with the main delimiter list behavior.+// DROPPED from each child (Python-compatible) — consistent with the main+// delimiter list behavior.func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {c, err := NewTokenChunker(map[string]any{"delimiter_mode": "delimiter",@@ -68,6 +65,7 @@ func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {t.Fatalf("Invoke: %v", err)}chunks, _ := out["chunks"].([]map[string]any)+ // The children delimiter "###" is dropped from each child.want := []string{"sec one", "sec two", "sec three"}if len(chunks) != len(want) {t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))@@ -78,7 +76,9 @@ func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {t.Errorf("chunk[%d] text: want %q got %q", i, w, got)}}- // The literal backtick token `###` must never match as a whole string.+ // The backtick wrapper is stripped from the PATTERN: the match is on the+ // inner "###", so the surrounding backticks remain as literal text in the+ // child chunks (i.e. "`###`" is not consumed as a whole delimiter).c2, _ := NewTokenChunker(map[string]any{"delimiter_mode": "delimiter","delimiters": []string{"\n"},@@ -93,9 +93,15 @@ func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {t.Fatalf("Invoke: %v", err)}chunks2, _ := out2["chunks"].([]map[string]any)- for _, ck := range chunks2 {- if strings.Contains(ck["text"].(string), "###") {- t.Errorf("child text kept literal backtick-wrapped token: %q", ck["text"].(string))+ // The inner "###" matches and is DROPPED, leaving the surrounding+ // backticks as literal text.+ backtickWrapped := []string{"a `", "` b"}+ if len(chunks2) != len(backtickWrapped) {+ t.Fatalf("backtick literal chunk count: want %d got %d (%v)", len(backtickWrapped), len(chunks2), chunkTexts(chunks2))+ }+ for i, w := range backtickWrapped {+ if got := chunks2[i]["text"].(string); got != w {+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)}}}@@ -118,7 +124,8 @@ func TestTokenChunker_ChildrenDelimiterDroppedText(t *testing.T) {t.Fatalf("Invoke: %v", err)}chunks, _ := out["chunks"].([]map[string]any)- want := []string{"alpha one", "alpha two", "alpha three."}+ // The children delimiter ". " is retained on each child.+ want := []string{"alpha one. ", "alpha two. ", "alpha three."}if len(chunks) != len(want) {t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))}diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.goindex f282aee30a..acd3c5e36f 100644--- a/internal/ingestion/component/chunker/token_delim_contract_test.go+++ b/internal/ingestion/component/chunker/token_delim_contract_test.go@@ -7,8 +7,9 @@ import (// TestTokenChunker_BareDelimiterHonored locks the #17723 fix: a bare// (non-backtick) delimiter entry is now honored by TokenChunker. The payload-// is split on the delimiter into paragraphs (the delimiter is DROPPED, matching-// Python naive_merge), and those paragraphs are then merged by token_size.+// is split on the delimiter into paragraphs (the delimiter is RETAINED+// losslessly, since a bare delimiter is a split hint, not a delete instruction),+// and those paragraphs are then merged by token_size.// Regression guard for the "bare entries are active" contract.func TestTokenChunker_BareDelimiterHonored(t *testing.T) {c, err := NewTokenChunker(map[string]any{@@ -36,16 +37,16 @@ func TestTokenChunker_BareDelimiterHonored(t *testing.T) {for _, ck := range chunks {joined.WriteString(ck["text"].(string))}- // No content dropped, and the bare "::" is split away (not preserved inside- // a chunk). Python's naive_merge rebuilds each paragraph with a leading- // "\n", so the joined text equals the source with "::" replaced by "\n".- const want = "alpha\nbeta\ngamma\ndelta"+ // Lossless: the joined text reproduces the source exactly, with the bare+ // "::" delimiter retained inside each chunk (a delimiter is a split hint,+ // not a delete instruction).+ const want = "alpha::beta::gamma::delta"if joined.String() != want {t.Errorf("bare delimiter not honored: joined=%q want %q (chunks=%v)", joined.String(), want, chunkTexts(chunks))}for _, ck := range chunks {- if strings.Contains(ck["text"].(string), "::") {- t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))+ if !strings.Contains(ck["text"].(string), "::") {+ t.Errorf("bare delimiter dropped from chunk: %q", ck["text"].(string))}}}@@ -73,6 +74,8 @@ func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {t.Fatalf("Invoke: %v", err)}chunks, _ := out["chunks"].([]map[string]any)+ // Custom (backtick) delimiters are DROPPED, mirroring Python's+ // naive_merge has_custom branch: "段落" is split away, not preserved.want := []string{"第一部分", "第二部分"}if len(chunks) != len(want) {t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))@@ -83,8 +86,9 @@ func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {}}- // Longest delimiter must win over its shorter prefix: `段落` (2 runes)- // beats `段` (1 rune), so "A段落B" splits on "段落", not "段".+ // Longest delimiter must still win over its shorter prefix: `段落` (2 runes)+ // beats `段` (1 rune), so "A段落B" splits on "段落", not "段" — and the+ // delimiter is dropped from both chunks.c2, err := NewTokenChunker(map[string]any{"delimiters": []string{"`段落`", "`段`"},"chunk_token_size": float64(1024),diff --git a/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go b/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.gonew file mode 100644index 0000000000..653097af19--- /dev/null+++ b/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go@@ -0,0 +1,129 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Boundary / whitespace-edge regression tests for the lossless delimiter+// contract. These pin behaviour the headline tests (token_delim_lossless_test.go)+// do NOT exercise:+// - a chunk boundary that lands exactly on a retained delimiter (the text path+// used to TrimSpace every merged chunk, dropping the boundary newline);+// - whitespace-only pieces produced by repeated delimiters ("A\n\nB") in the+// children-split paths, which used to be discarded;+// - a leading delimiter / whitespace-only piece, which used to vanish from+// the head of the first segment.++package chunker++import (+ "fmt"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// TestDelimiterLossless_BoundaryReconstruction drives the TEXT path with a+// small chunk_token_size so several chunks form and every chunk boundary falls+// on a retained "\n" delimiter. Concatenating the emitted chunks must reproduce+// the source exactly — the boundary newline must survive (it used to be+// TrimSpaced away by mergeByTokenSize).+func TestDelimiterLossless_BoundaryReconstruction(t *testing.T) {+ lines := make([]string, 0, 12)+ for i := 1; i <= 12; i++ {+ lines = append(lines, fmt.Sprintf("This is sentence number %d of the document.", i))+ }+ src := strings.Join(lines, "\n")+ input := map[string]any{"name": "t", "output_format": "text", "text": src}+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(24),+ "delimiter": "\n",+ }, input)+ if len(chunks) < 2 {+ t.Fatalf("expected multiple chunks to exercise boundary delimiters, got %d", len(chunks))+ }+ got := joinChunks(chunks)+ if got != src {+ t.Errorf("boundary lossless reconstruction failed:\n got=%q\nwant=%q", got, src)+ }+}++// TestSplitByChildrenKeepsWhitespaceOnlyPiece pins that a whitespace-only piece+// from consecutive delimiters ("A\n\nB" -> ["A\n","\n","B\n"]) is folded into the+// adjacent child rather than dropped, so the child texts still reconstruct "A\n\nB".+func TestSplitByChildrenKeepsWhitespaceOnlyPiece(t *testing.T) {+ pat := compileDelimPattern([]string{"\n"})+ doc := schema.ChunkDoc{Text: "A\n\nB", DocType: "text", CKType: "text"}+ out := splitByChildren([]schema.ChunkDoc{doc}, pat, true)+ var got strings.Builder+ for _, c := range out {+ got.WriteString(c.Text)+ }+ if want := "A\n\nB"; got.String() != want {+ t.Errorf("splitByChildren dropped the repeated newline: got=%q want=%q (children=%v)",+ got.String(), want, childDocTexts(out))+ }+}++// TestApplyChildrenDelimTextKeepsWhitespaceOnlyPiece mirrors the above for the+// text-children split helper.+func TestApplyChildrenDelimTextKeepsWhitespaceOnlyPiece(t *testing.T) {+ pat := compileDelimPattern([]string{"\n"})+ doc := schema.ChunkDoc{Text: "A\n\nB", CKType: "text"}+ out := applyChildrenDelimText([]schema.ChunkDoc{doc}, pat, true)+ var got strings.Builder+ for _, c := range out {+ got.WriteString(c.Text)+ }+ if want := "A\n\nB"; got.String() != want {+ t.Errorf("applyChildrenDelimText dropped the repeated newline: got=%q want=%q (children=%v)",+ got.String(), want, childDocTexts(out))+ }+}++// TestChunkPerSegmentDropsCustomDelimiter pins that a backtick (custom)+// delimiter is DROPPED, mirroring Python's naive_merge has_custom branch: a+// leading custom delimiter ("SEPsecond line") yields "second line", and an+// internal one ("firstSEPsecond") yields "first"/"second".+func TestChunkPerSegmentDropsCustomDelimiter(t *testing.T) {+ t.Run("leading", func(t *testing.T) {+ input := map[string]any{"name": "t", "output_format": "text", "text": "SEPsecond line"}+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(512),+ "delimiters": []string{"`SEP`"},+ }, input)+ if got := joinChunks(chunks); got != "second line" {+ t.Errorf("custom leading delimiter not dropped: got=%q want %q", got, "second line")+ }+ })+ t.Run("internal", func(t *testing.T) {+ input := map[string]any{"name": "t", "output_format": "text", "text": "firstSEPsecond"}+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(512),+ "delimiters": []string{"`SEP`"},+ }, input)+ if got := joinChunks(chunks); got != "firstsecond" {+ t.Errorf("custom delimiter not dropped: got=%q want %q", got, "firstsecond")+ }+ })+}++// childDocTexts is a small local helper for failure messages.+func childDocTexts(chunks []schema.ChunkDoc) []string {+ out := make([]string, 0, len(chunks))+ for _, c := range chunks {+ out = append(out, c.Text)+ }+ return out+}diff --git a/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go b/internal/ingestion/component/chunker/token_delim_lossless_followup_test.gonew file mode 100644index 0000000000..086dfbd81e--- /dev/null+++ b/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go@@ -0,0 +1,104 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Follow-up losslessness tests for the two whitespace-handling defects found+// during the #20276 review:+// - applyChildrenDelimText must fold a whitespace-only delimiter piece only+// within its OWN parent, never into a different parent whose text happens+// to match (the old code compared out[n-1].Mom, which is parent TEXT, not+// parent identity).+// - chunkFromItem must not discard whitespace-only delimiter pieces; it must+// fold them into an adjacent real piece, mirroring the text/children paths.++package chunker++import (+ "reflect"+ "regexp"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// TestApplyChildrenDelimText_NoCrossParentFold pins the cross-parent folding+// bug: two distinct parents whose text (minus one leading "\n") is identical+// must each keep their own leading delimiter. The buggy Mom comparison folds+// the second parent's whitespace into the first parent's child, corrupting the+// first and dropping the second parent's delimiter.+func TestApplyChildrenDelimText_NoCrossParentFold(t *testing.T) {+ pattern := regexp.MustCompile("\n")+ docs := []schema.ChunkDoc{+ {Text: "\nA", CKType: "text"},+ {Text: "\nA", CKType: "text"},+ }+ out := applyChildrenDelimText(docs, pattern, true)+ if len(out) != 2 {+ t.Fatalf("expected 2 children (one per parent), got %d: %+v", len(out), out)+ }+ want := []string{"\nA", "\nA"}+ for i, w := range want {+ if out[i].Text != w {+ t.Errorf("child %d: got %q want %q", i, out[i].Text, w)+ }+ }+}++// TestChunkFromItem_RetainsWhitespacePieces pins the whitespace-drop bug in+// chunkFromItem: a bare retained delimiter that produces a whitespace-only+// piece (blank line between two delimiters, or a leading delimiter) must be+// preserved by folding it into an adjacent real piece, not discarded.+func TestChunkFromItem_RetainsWhitespacePieces(t *testing.T) {+ pattern := regexp.MustCompile("\n")+ cases := []struct {+ name string+ text string+ want []string+ }{+ {"interior blank line", "alpha\n\nbeta", []string{"alpha\n", "\nbeta"}},+ {"leading delimiter", "\nsecond line", []string{"\nsecond line"}},+ {"trailing delimiter", "alpha\n\n", []string{"alpha\n\n"}},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ it := schema.ChunkDoc{Text: c.text, CKType: "text"}+ out := chunkFromItem(it, pattern, true)+ got := make([]string, 0, len(out))+ for _, d := range out {+ got = append(got, d.Text)+ }+ if !reflect.DeepEqual(got, c.want) {+ t.Fatalf("got %#v want %#v", got, c.want)+ }+ })+ }+}++// TestChunkFromItem_LosslessWhitespaceReconstruct asserts the end-to-end+// lossless contract for an item containing repeated bare delimiters: the+// concatenation of the emitted chunk texts reproduces the source exactly.+func TestChunkFromItem_LosslessWhitespaceReconstruct(t *testing.T) {+ pattern := regexp.MustCompile("\n")+ src := "alpha\n\nbeta\n\ngamma"+ it := schema.ChunkDoc{Text: src, CKType: "text"}+ out := chunkFromItem(it, pattern, true)+ var recon string+ for _, d := range out {+ recon += d.Text+ }+ if recon != src {+ t.Fatalf("lossless reconstruction failed: got %q want %q", recon, src)+ }+}diff --git a/internal/ingestion/component/chunker/token_delim_lossless_test.go b/internal/ingestion/component/chunker/token_delim_lossless_test.gonew file mode 100644index 0000000000..f636277518--- /dev/null+++ b/internal/ingestion/component/chunker/token_delim_lossless_test.go@@ -0,0 +1,187 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// User-perspective behaviour contract for the TokenChunker delimiter handling.+//+// A delimiter is a HINT for where the chunker MAY break, never a character to+// DELETE. After chunking, concatenating every emitted chunk's text must+// reproduce the original text (lossless): no sentence punctuation such as "。"+// may vanish, and no stray newline may be injected in its place.+//+// These tests assert that contract directly (count of "。" preserved, exact+// reconstruction) rather than hard-coding implementation-shaped strings.++package chunker++import (+ "context"+ "strings"+ "testing"+)++// joinChunks concatenates the text of every emitted chunk the way a downstream+// consumer would see it, for losslessness assertions.+func joinChunks(chunks []map[string]any) string {+ var b strings.Builder+ for _, c := range chunks {+ if s, ok := c["text"].(string); ok {+ b.WriteString(s)+ }+ }+ return b.String()+}++// driveChunker runs the TokenChunker component through its real Invoke path.+func driveChunker(t *testing.T, params map[string]any, input map[string]any) []map[string]any {+ t.Helper()+ comp, err := NewTokenChunker(params)+ if err != nil {+ t.Fatalf("construct TokenChunker: %v", err)+ }+ out, err := comp.Invoke(context.Background(), nil, input)+ if err != nil {+ t.Fatalf("invoke TokenChunker: %v", err)+ }+ if msg, _ := out["_ERROR"].(string); msg != "" {+ t.Fatalf("TokenChunker returned _ERROR: %s", msg)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ return chunks+}++// TestDelimiterKeepsSentencePeriod_JSONPath is the regression for the reported+// bug: with the RAGFlow default delimiter set ("\n!?;。;!?") fed via the JSON+// path, "。" must survive and must NOT be replaced by a newline.+func TestDelimiterKeepsSentencePeriod_JSONPath(t *testing.T) {+ const src = "小明喜欢吃苹果。小红喜欢吃香蕉。小刚喜欢吃橙子。"+ input := map[string]any{+ "name": "t",+ "output_format": "json",+ "json": []map[string]any{{"text": src, "doc_type_kwd": "text"}},+ }+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(512),+ "delimiter": "\n!?;。;!?",+ }, input)+ got := joinChunks(chunks)+ if want, gotN := strings.Count(src, "。"), strings.Count(got, "。"); want != gotN {+ t.Errorf("sentence period '。' count: want %d (preserved), got %d (dropped into newline?)", want, gotN)+ }+ if !strings.Contains(got, "苹果。小红") {+ t.Errorf("merged text lost the '。' sentence boundary; got %q", got)+ }+}++// TestDelimiterKeepsSentencePeriod_TextPath covers the text/markdown/html path+// with the same default delimiter set: "。" must not be dropped there either.+func TestDelimiterKeepsSentencePeriod_TextPath(t *testing.T) {+ const src = "春眠不觉晓。处处闻啼鸟。夜来风雨声。花落知多少。"+ input := map[string]any{+ "name": "t",+ "output_format": "text",+ "text": src,+ }+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(512),+ "delimiter": "\n!?;。;!?",+ }, input)+ got := joinChunks(chunks)+ if want, gotN := strings.Count(src, "。"), strings.Count(got, "。"); want != gotN {+ t.Errorf("text path '。' count: want %d, got %d", want, gotN)+ }+ if !strings.Contains(got, "晓。处处") {+ t.Errorf("text path lost sentence boundary; got %q", got)+ }+}++// TestDelimiterLossless_Reconstruction asserts the core user contract: the+// concatenation of all chunk texts reproduces the source exactly (no char+// dropped, no stray char injected) for a mixed delimiter set.+func TestDelimiterLossless_Reconstruction(t *testing.T) {+ src := "第一章。第一节!这是内容;第二句?结尾。\n第二段,逗号不是分隔符,但句号是。"+ for _, delim := range []string{"\n!?;。;!?", "。!?", "\n"} {+ t.Run("delim="+delim, func(t *testing.T) {+ input := map[string]any{+ "name": "t",+ "output_format": "json",+ "json": []map[string]any{{"text": src, "doc_type_kwd": "text"}},+ }+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(2048), // large: isolate delimiter behaviour from token-splitting+ "delimiter": delim,+ }, input)+ got := joinChunks(chunks)+ if got != src {+ t.Errorf("lossless reconstruction failed:\n got=%q\nwant=%q", got, src)+ }+ })+ }+}++// TestDelimiter_LeadingTrailingAndConsecutive exercises non-happy-path+// delimiter placement: delimiter at the very start, at the very end, and+// consecutive delimiters must not delete or duplicate characters.+func TestDelimiter_LeadingTrailingAndConsecutive(t *testing.T) {+ src := "。开头句。中间。。双标点?结尾句。"+ input := map[string]any{+ "name": "t",+ "output_format": "json",+ "json": []map[string]any{{"text": src, "doc_type_kwd": "text"}},+ }+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(2048),+ "delimiter": "。?",+ }, input)+ got := joinChunks(chunks)+ if got != src {+ t.Errorf("non-happy-path reconstruction failed:\n got=%q\nwant=%q", got, src)+ }+}++// TestDelimiter_EmptyAndWhitespaceOnly guards against panics / dropped content+// when the source is empty or consists only of delimiter characters.+func TestDelimiter_EmptyAndWhitespaceOnly(t *testing.T) {+ for _, src := range []string{"", "。?。?", " ", "\n\n"} {+ t.Run("src="+quoteCase(src), func(t *testing.T) {+ input := map[string]any{+ "name": "t",+ "output_format": "json",+ "json": []map[string]any{{"text": src, "doc_type_kwd": "text"}},+ }+ chunks := driveChunker(t, map[string]any{+ "chunk_token_size": float64(64),+ "delimiter": "\n!?;。;!?",+ }, input)+ got := joinChunks(chunks)+ if len(got) > len(src) {+ t.Errorf("output longer than source: got %q (len %d) > src %q (len %d)", got, len(got), src, len(src))+ }+ })+ }+}++func quoteCase(s string) string {+ switch s {+ case "":+ return "empty"+ case " ":+ return "spaces"+ case "\n\n":+ return `double-newline`+ default:+ return strings.NewReplacer("\n", "\\n").Replace(s)+ }+}diff --git a/internal/ingestion/component/chunker/token_json_overlap_test.go b/internal/ingestion/component/chunker/token_json_overlap_test.goindex 16bb8e0343..bbe517092d 100644--- a/internal/ingestion/component/chunker/token_json_overlap_test.go+++ b/internal/ingestion/component/chunker/token_json_overlap_test.go@@ -70,9 +70,11 @@ func TestMergeByTokenSizeFromJSON_TKNumsConsistency(t *testing.T) {t.Fatalf("want 2 chunks (merged a+b + fresh c), got %d (a=%d b=%d c=%d budget=%d)", len(merged), aN, bN, cN, budget)}- // chunk0 is the merge of a and b.- if merged[0].Text != aText+"\n"+bText {- t.Errorf("chunk0 text mismatch:\n got=%q\nwant=%q", merged[0].Text, aText+"\n"+bText)+ // chunk0 is the merge of a and b. The JSON merge joins with "" (each+ // unit carries its own retained delimiter), so the merged text is the+ // direct concatenation.+ if merged[0].Text != aText+bText {+ t.Errorf("chunk0 text mismatch:\n got=%q\nwant=%q", merged[0].Text, aText+bText)}// Merge path: TKNums is the running sum, not the re-tokenized text.if got0 := intValue(merged[0].TKNums); got0 != aN+bN {diff --git a/internal/ingestion/component/chunker/token_merge_parity_test.go b/internal/ingestion/component/chunker/token_merge_parity_test.goindex 3510fd9f88..6ccf0bb681 100644--- a/internal/ingestion/component/chunker/token_merge_parity_test.go+++ b/internal/ingestion/component/chunker/token_merge_parity_test.go@@ -78,18 +78,25 @@ var mergeSourceLines = []string{// preservation is pinned by the dedicated suites instead — token_oversize_split_test.go// (lossless concatenation) and token_overlap_test.go (no space-less boundaries).func goMergeGroupsOracle(paragraphs []string, cap int, overlapPct float64) []string {+ // Model the production text-path unitization exactly: split on the active+ // delimiter and KEEP it attached (lossless). This mirrors+ // TokenChunkerComponent.mergeByTokenSize, which calls+ // splitByDelim(text, delimPattern, true) — no synthetic leading "\n" glue+ // is prepended. Reconstructing the units the same way keeps the rune layout+ // (and therefore token counts and merge structure) identical to production,+ // so the overlap-prefix trim lands on the same rune.+ text := strings.Join(paragraphs, "\n")units := make([]schema.ChunkDoc, 0, len(paragraphs))- for _, p := range paragraphs {- if strings.TrimSpace(p) == "" {+ for _, sub := range splitByDelim(text, compileDelimPattern([]string{"\n"}), true) {+ if sub == "" {continue}- u := "\n" + p- units = append(units, schema.ChunkDoc{Text: u, TKNums: intPtr(tokenizeStr(u)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})}merged := mergeUnits(units, cap, overlapPct, "")out := make([]string, 0, len(merged))for _, ck := range merged {- text := removeTag(strings.TrimSpace(ck.Text))+ text := removeTag(ck.Text)if text == "" {continue}@@ -222,13 +229,17 @@ func TestTokenChunkerMergeMatchesGoOracle(t *testing.T) {// trimmed to fit the hard cap). It is the Go-side oracle for the overlap// parity tests below.func goMergeWithOverlapOracle(paragraphs []string, chunkTokenSize int, overlappedPercent float64) []string {+ // Same lossless unitization as goMergeGroupsOracle: split on the active+ // delimiter and KEEP it (no leading "\n" glue). This must match production+ // (TokenChunkerComponent.mergeByTokenSize) so the overlap-prefix trim+ // reproduces the same rune as the component emits.+ text := strings.Join(paragraphs, "\n")units := make([]schema.ChunkDoc, 0, len(paragraphs))- for _, p := range paragraphs {- if strings.TrimSpace(p) == "" {+ for _, sub := range splitByDelim(text, compileDelimPattern([]string{"\n"}), true) {+ if sub == "" {continue}- u := "\n" + p- units = append(units, schema.ChunkDoc{Text: u, TKNums: intPtr(tokenizeStr(u)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})}merged := mergeUnits(units, chunkTokenSize, overlappedPercent, "")out := make([]string, 0, len(merged))@@ -236,7 +247,7 @@ func goMergeWithOverlapOracle(paragraphs []string, chunkTokenSize int, overlappe// Mirror production token.go: removeTag first, then TrimSpace, so// whitespace between visible text and a trailing position tag survives// exactly as the Go TokenChunker emits it.- text := removeTag(strings.TrimSpace(ck.Text))+ text := removeTag(ck.Text)if text == "" {continue}diff --git a/internal/ingestion/component/chunker/token_oversize_split_test.go b/internal/ingestion/component/chunker/token_oversize_split_test.goindex 1d691b5620..bfcceff323 100644--- a/internal/ingestion/component/chunker/token_oversize_split_test.go+++ b/internal/ingestion/component/chunker/token_oversize_split_test.go@@ -136,8 +136,10 @@ func TestTokenChunker_OversizeUnitSplitAfterInBudgetUnit(t *testing.T) {t.Fatalf("want in-budget chunk + split pieces, got %d chunk(s)", len(chunks))}first, _ := chunks[0]["text"].(string)- if first != inBudget {- t.Fatalf("first chunk should be only the in-budget sentence %q, got %q", inBudget, first)+ // The in-budget sentence keeps its retained trailing delimiter+ // (lossless): "Hello world." + the "\n" it ended with.+ if want := inBudget + "\n"; first != want {+ t.Fatalf("first chunk should be only the in-budget sentence %q, got %q", want, first)}for i := 1; i < len(chunks); i++ {text, _ := chunks[i]["text"].(string)diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.goindex 5e01c2cca7..c07a8134eb 100644--- a/internal/ingestion/component/chunker/token_pdfpos_test.go+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go@@ -198,10 +198,10 @@ func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(tposD := json.RawMessage(`[[4,0,40,0,16]]`)posE := json.RawMessage(`[[5,0,50,0,20]]`)posF := json.RawMessage(`[[6,0,60,0,24]]`)- // 6 single-token items. With chunkTokens=9 (5 item tokens + 4 joinSep "\n"- // tokens), items 0..4 merge into one chunk — the re-tokenize guard lets the- // joined text fill the cap exactly (9 tokens), and item5 starts a fresh- // chunk. Its overlap prefix (overlappedPct=20) is the last ~20% of the+ // 6 single-token items. The JSON path joins with joinSep="" (no synthetic+ // "\n"), so the budget is pure item tokens. With chunkTokens=5, items 0..4+ // merge into one chunk (5 tokens, filling the cap exactly), and item5 starts+ // a fresh chunk. Its overlap prefix (overlappedPct=20) is the last ~20% of the// 5-item previous chunk's text => only the last item ("e", posE) intersects// the tail. So chunk[1] must carry posE (tail) + posF (own), but NOT// posA/posB/posC/posD. (Texts are single-token so the re-tokenize guard's@@ -216,7 +216,7 @@ func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(t{Text: "f", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posF},},}- got := mergeByTokenSizeFromJSON(items, 9, 20)+ got := mergeByTokenSizeFromJSON(items, 5, 20)merged := got[0]if len(merged) != 2 {t.Fatalf("want 2 chunks (5 items merge, 6th starts fresh with partial overlap), got %d", len(merged))@@ -257,7 +257,7 @@ func TestChunkFromItem_SlicesPositionsAcrossDelimiterPieces(t *testing.T) {CKType: "text",PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),}- got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}))+ got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}), false)if len(got) != 2 {t.Fatalf("want 2 delimiter-split pieces, got %d", len(got))}@@ -266,11 +266,15 @@ func TestChunkFromItem_SlicesPositionsAcrossDelimiterPieces(t *testing.T) {if len(p0) != 1 || len(p1) != 1 {t.Fatalf("single-row input must yield single-row slices: %v / %v", p0, p1)}- if p0[0][3] != 0 || math.Abs(p0[0][4]-20) > 1e-9 {- t.Errorf("piece 0 bounds = [%v,%v], want [0,20]", p0[0][3], p0[0][4])+ // The custom ("`\n`") delimiter is DROPPED (Python-compatible split+ // instruction), so piece 0 is "AAAA" (4 runes) and piece 1 is "BBBB" (4);+ // the slice is therefore proportional to 4/8 of the box height.+ wantSplit := 40.0 * 4.0 / 8.0+ if p0[0][3] != 0 || math.Abs(p0[0][4]-wantSplit) > 1e-9 {+ t.Errorf("piece 0 bounds = [%v,%v], want [0,%v]", p0[0][3], p0[0][4], wantSplit)}- if math.Abs(p1[0][3]-20) > 1e-9 || p1[0][4] != 40 {- t.Errorf("piece 1 bounds = [%v,%v], want [20,40]", p1[0][3], p1[0][4])+ if math.Abs(p1[0][3]-wantSplit) > 1e-9 || p1[0][4] != 40 {+ t.Errorf("piece 1 bounds = [%v,%v], want [%v,40]", p1[0][3], p1[0][4], wantSplit)}}@@ -357,7 +361,7 @@ func TestChunkFromItem_SlicesPositionsIgnoresPositionTags(t *testing.T) {CKType: "text",PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),}- got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}))+ got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}), false)if len(got) != 2 {t.Fatalf("want 2 delimiter-split pieces, got %d", len(got))}@@ -366,11 +370,15 @@ func TestChunkFromItem_SlicesPositionsIgnoresPositionTags(t *testing.T) {if len(p0) != 1 || len(p1) != 1 {t.Fatalf("single-row input must yield single-row slices: %v / %v", p0, p1)}- if math.Abs(p0[0][4]-20) > 1e-9 {- t.Errorf("piece 0 bottom = %v, want 20 (tag runes must not shift ratio)", p0[0][4])+ // The parser tag is excluded from the ratio (removeTag strips it), and the+ // custom ("`\n`") delimiter is DROPPED, so piece 0 is "AAAA" (4 runes) and+ // piece 1 is "CCCC" (4); the split is proportional to 4/8 of the box height.+ wantSplit := 40.0 * 4.0 / 8.0+ if math.Abs(p0[0][4]-wantSplit) > 1e-9 {+ t.Errorf("piece 0 bottom = %v, want %v (tag runes excluded, retained delimiter counted)", p0[0][4], wantSplit)}- if math.Abs(p1[0][3]-20) > 1e-9 {- t.Errorf("piece 1 top = %v, want 20 (tag runes must not shift ratio)", p1[0][3])+ if math.Abs(p1[0][3]-wantSplit) > 1e-9 {+ t.Errorf("piece 1 top = %v, want %v (tag runes excluded, retained delimiter counted)", p1[0][3], wantSplit)}}diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 62ac929da9..12c6827464 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -197,6 +197,9 @@ func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) {t.Errorf("chunk[%d] is the bare delimiter %q", i, text)}}+ // The custom (backtick) delimiter "666" is DROPPED (Python-compatible).+ // chunkPerSegment .strip()s each segment, so the surrounding "\n" is also+ // trimmed as trailing/leading whitespace — neither segment carries it.if got, want := chunks[0]["text"], "alpha section"; got != want {t.Errorf("chunk[0] text = %q, want %q", got, want)}@@ -314,7 +317,7 @@ func TestTokenChunkerTextParserJSONKeepsSentenceBoundaries(t *testing.T) {if err != nil {t.Fatalf("TokenChunker.Invoke: %v", err)}- if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"first!\nsecond!"}) {+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"first!second!"}) {t.Fatalf("text parser JSON chunks = %q, want sentence boundary between parser units", texts)}}@@ -557,8 +560,9 @@ func TestTokenChunkerDelimiterWindowUsesChildTokenCounts(t *testing.T) {if media == nil {t.Fatalf("table chunk missing: %+v", out)}- // Each child is 3 tokens, so the 7-token window holds both of them.- assertMaterializedMediaContext(t, media, "gamma deltaalpha beta.")+ // Each child is 3 tokens, so the 7-token window holds both of them. The+ // children delimiter ". " is retained, so the materialized context keeps it.+ assertMaterializedMediaContext(t, media, "gamma delta. alpha beta.")}// TestMaterializeMediaContextKeepsTokenCountInSync pins the invariant the fold@@ -1103,7 +1107,7 @@ func TestSplitByChildrenRecomputesTokenCounts(t *testing.T) {DocType: "text",CKType: "text",TKNums: intPtr(tokenizeStr(parentText)),- }}, regexp.MustCompile(`\. `))+ }}, regexp.MustCompile(`\. `), true)if len(children) != 2 {t.Fatalf("children = %d, want 2", len(children))@@ -1125,7 +1129,7 @@ func TestApplyChildrenDelimText_RecomputesTokenCounts(t *testing.T) {DocType: "text",CKType: "text",TKNums: intPtr(tokenizeStr(parentText)),- }}, regexp.MustCompile(`\. `))+ }}, regexp.MustCompile(`\. `), true)if len(out) != 2 {t.Fatalf("children = %d, want 2", len(out))@@ -1146,7 +1150,7 @@ func TestApplyChildrenDelimText_DefaultsMomToCurrentChunk(t *testing.T) {}pattern := regexp.MustCompile(`\. `)- out := applyChildrenDelimText(docs, pattern)+ out := applyChildrenDelimText(docs, pattern, true)if len(out) != 3 {t.Fatalf("want 3 children, got %d", len(out))}@@ -1171,7 +1175,7 @@ func TestApplyChildrenDelimText_OverwritesIncomingMom(t *testing.T) {}pattern := regexp.MustCompile(`\. `)- out := applyChildrenDelimText(docs, pattern)+ out := applyChildrenDelimText(docs, pattern, true)if len(out) != 3 {t.Fatalf("want 3 children, got %d", len(out))}@@ -1198,7 +1202,7 @@ func TestApplyChildrenDelimText_NilPatternIsNoop(t *testing.T) {docs := []schema.ChunkDoc{{Text: "alpha. beta", Mom: "kept"},}- out := applyChildrenDelimText(docs, nil)+ out := applyChildrenDelimText(docs, nil, true)if len(out) != 1 {t.Fatalf("want 1 chunk unchanged, got %d", len(out))}@@ -1219,7 +1223,7 @@ func TestApplyChildrenDelimText_FallbackStripsLeadingNewline(t *testing.T) {}pattern := regexp.MustCompile(`\. `)- out := applyChildrenDelimText(docs, pattern)+ out := applyChildrenDelimText(docs, pattern, true)if len(out) != 3 {t.Fatalf("want 3 children, got %d", len(out))}