RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): honor bare (non-backtick) delimiters in TokenChunker (#17723) (#18182)
Jackb505437db53e
diff --git a/internal/ingestion/component/chunker/bare_delimiter_test.go b/internal/ingestion/component/chunker/bare_delimiter_test.gonew file mode 100644index 0000000000..0f4fc9a682--- /dev/null+++ b/internal/ingestion/component/chunker/bare_delimiter_test.go@@ -0,0 +1,239 @@+package chunker++import (+ "context"+ "strings"+ "testing"+)++// bare_delimiter_test covers the fix for #17723: bare (non-backtick)+// delimiters must now be honored by TokenChunker — splitting the payload into+// paragraphs that are then merged by token size — instead of being silently+// ignored (the previous keepBare=false contract).+//+// Python contract (rag/nlp/__init__.py:1406-1415, naive_merge default path):+// the payload is split on the delimiter, the delimiter text is DROPPED, and+// each kept paragraph is rebuilt with a leading "\n" (its original surrounding+// whitespace is preserved). Paragraphs are thus joined by "\n", never by the+// raw delimiter. One chunk per segment (no token merge) happens ONLY for a+// backtick-wrapped CUSTOM delimiter.+//+// These tests target BOTH the text/markdown/html path (invokeTextPayload) and+// the JSON path (invokeJSONPayload -> chunkFromItem + global merge), and lock+// the custom(backtick) vs bare distinction.++// newBareChunker builds a TokenChunker for the given delimiter list and a small+// token budget so merges are observable.+func newBareChunker(t *testing.T, delims []string, tokenSize float64) *TokenChunkerComponent {+ t.Helper()+ c, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": delims,+ "chunk_token_size": tokenSize,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ return c.(*TokenChunkerComponent)+}++func invokeText(t *testing.T, c *TokenChunkerComponent, text string) []map[string]any {+ t.Helper()+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.txt",+ "output_format": "text",+ "text": text,+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if msg, ok := out["_ERROR"].(string); ok && msg != "" {+ t.Fatalf("Go returned _ERROR: %s", msg)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ return chunks+}++func joinedText(chunks []map[string]any) string {+ var b strings.Builder+ for _, ck := range chunks {+ b.WriteString(ck["text"].(string))+ }+ return b.String()+}++// TestBareDelimiterSplitsAndTokenMergesText asserts the core fix: a bare+// multi-char delimiter ("::") now splits the text into paragraphs, the+// delimiter text is dropped, and the paragraphs are rejoined with "\n"+// (matching Python's "\n"+sub_sec reconstruction) — not with the raw+// delimiter. The paragraphs are then merged by token size.+func TestBareDelimiterSplitsAndTokenMergesText(t *testing.T) {+ c := newBareChunker(t, []string{"::"}, 1024)+ const text = "alpha::beta::gamma::delta"+ chunks := invokeText(t, c, text)+ if len(chunks) == 0 {+ t.Fatal("expected at least one chunk")+ }+ // The delimiter must never survive inside a chunk.+ for _, ck := range chunks {+ if strings.Contains(ck["text"].(string), "::") {+ t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))+ }+ }+ // Paragraphs are rejoined with "\n" (each paragraph keeps a leading "\n"),+ // so the joined content equals the source with "::" replaced by "\n".+ const want = "alpha\nbeta\ngamma\ndelta"+ if got := joinedText(chunks); got != want {+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ }+}++// TestBareSingleCharDelimiterSplitsText covers the classic case: a single+// ASCII char delimiter (".") splits sentences, the delimiter is dropped, and+// paragraphs are rejoined with "\n".+func TestBareSingleCharDelimiterSplitsText(t *testing.T) {+ c := newBareChunker(t, []string{"."}, 1024)+ chunks := invokeText(t, c, "first.second.third")+ const want = "first\nsecond\nthird"+ if got := joinedText(chunks); got != want {+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ }+ for _, ck := range chunks {+ if strings.Contains(ck["text"].(string), ".") {+ t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))+ }+ }+}++// TestBareCJKBoundarySplitsText covers a CJK punctuation delimiter.+func TestBareCJKBoundarySplitsText(t *testing.T) {+ c := newBareChunker(t, []string{";"}, 1024)+ chunks := invokeText(t, c, "第一部分;第二部分;第三部分")+ const want = "第一部分\n第二部分\n第三部分"+ if got := joinedText(chunks); got != want {+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ }+}++// TestBareDelimiterSplitsJSON asserts the JSON path also honors bare+// delimiters (chunkFromItem now receives a non-nil pattern and the global+// merge applies because there is no custom delimiter).+func TestBareDelimiterSplitsJSON(t *testing.T) {+ c := newBareChunker(t, []string{"。"}, 1024)+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.json",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "第一句内容。第二句内容。第三句内容。", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ const want = "第一句内容\n第二句内容\n第三句内容"+ if got := joinedText(chunks); got != want {+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ }+}++// TestCustomDelimiterStillOneChunkPerSegment locks the distinction: a+// backtick-wrapped delimiter must yield one chunk per segment with NO token+// merge (and the delimiter is dropped with no "\n" rejoin), even with a tight+// budget.+func TestCustomDelimiterStillOneChunkPerSegment(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{"`::`"},+ "chunk_token_size": float64(4),+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha::beta::gamma::delta")+ want := []string{"alpha", "beta", "gamma", "delta"}+ if len(chunks) != len(want) {+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))+ }+ for i, w := range want {+ if got := chunks[i]["text"].(string); got != w {+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)+ }+ }+}++// TestBareDelimiterEmptyText asserts empty input yields no chunks (no panic).+func TestBareDelimiterEmptyText(t *testing.T) {+ c := newBareChunker(t, []string{"::"}, 8)+ chunks := invokeText(t, c, "")+ if len(chunks) != 0 {+ t.Errorf("empty text should produce no chunks, got %d", len(chunks))+ }+}++// TestBareDelimiterDropsEmptySegment asserts that a genuinely empty segment+// produced by a doubled delimiter ("alpha||beta") is dropped, so the joined+// text has no empty paragraph (mirrors Python's `if not sub_sec: continue`).+// (Whitespace-only segments are NOT dropped — Python keeps them — so this test+// deliberately uses an empty, not whitespace, segment.)+func TestBareDelimiterDropsEmptySegment(t *testing.T) {+ c := newBareChunker(t, []string{"|"}, 1024)+ chunks := invokeText(t, c, "alpha||beta")+ const want = "alpha\nbeta"+ if got := joinedText(chunks); got != want {+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ }+}++// TestBareDelimiterCRLFNormalized asserts CRLF/CR are normalized to LF before+// splitting (mirrors Python naive_merge text normalization). The "\n" left+// behind by the dropped delimiter is preserved between paragraphs.+func TestBareDelimiterCRLFNormalized(t *testing.T) {+ c := newBareChunker(t, []string{"."}, 1024)+ chunks := invokeText(t, c, "line one.\r\nline two.\rline three.")+ const want = "line one\n\nline two\n\nline three"+ if got := joinedText(chunks); got != want {+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))+ }+}++// TestBareDelimiterNoDelimiterFallback asserts that with no delimiter at all+// the classic single-section merge still applies (regression guard for the+// fallback path): the bare-delimiter machinery is bypassed entirely.+func TestBareDelimiterNoDelimiterFallback(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{},+ "chunk_token_size": float64(8),+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha beta gamma delta epsilon zeta eta theta")+ if len(chunks) == 0 {+ t.Fatalf("expected at least one chunk")+ }+ // No delimiter configured, so the whole text is one section merged by+ // token size; its content must be preserved in order.+ if got := joinedText(chunks); !strings.Contains(got, "alpha beta gamma") {+ t.Errorf("no-delimiter fallback lost content: %q", got)+ }+}++// TestBareDelimiterTokenMergeCoalesces asserts that short paragraphs separated+// by a bare delimiter are merged up to the token budget (not one chunk per+// paragraph when below budget), while still keeping the "\n" paragraph+// boundary — exactly mirroring Python's merge.+func TestBareDelimiterTokenMergeCoalesces(t *testing.T) {+ c := newBareChunker(t, []string{"。"}, 1024)+ chunks := invokeText(t, c, "短句一。短句二。短句三。")+ // All three short sentences fit under the large budget, so they coalesce+ // into a single merged chunk whose text keeps the "\n" paragraph joins.+ if len(chunks) != 1 {+ t.Fatalf("expected single merged chunk, got %d (%v)", len(chunks), chunkTexts(chunks))+ }+ const want = "短句一\n短句二\n短句三"+ if got := chunks[0]["text"].(string); got != want {+ t.Errorf("merged text: want %q got %q", want, got)+ }+}diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex da6699e6fc..92c32f5679 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -77,13 +77,15 @@ func stringListFromAny(in []any) []string {// ---------------------------------------------------------------------------// compileDelimPattern compiles a TokenChunker-style []string delimiter list.-// Only backtick-wrapped entries produce an active pattern (Python-// token_chunker / rag/nlp/delim list helper); plain (non-backtick) entries-// are ignored here. mergeByTokenSize uses its own hardcoded sentenceDelimiter,-// not this list. Canonical single-string parser_config.delimiter parsing-// lives in ragflow/internal/parser/chunk (ParseDelimiterField).+// Every non-empty entry is active, including bare (non-backtick) delimiters,+// mirroring Python naive_merge / rag/nlp/delim where bare single-character+// delimiters still split. Backtick-wrapped entries contribute their inner+// content. invokeTextPayload decides whether an active delimiter yields one+// chunk per segment (custom/backtick, no merge) or splits into paragraphs that+// are merged by token size (bare). Canonical single-string parser_config.delimiter+// parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField).func compileDelimPattern(delims []string) *regexp.Regexp {- return chunk.CompileDelimiterListPattern(delims)+ return chunk.CompileDelimiterPatternList(delims, true)}// splitDroppingDelim mirrors Python's _split_text_by_patterndiff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.goindex 613ab0e62b..63598d838f 100644--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go@@ -119,29 +119,42 @@ func TestCompileDelimPattern_BacktickEndIsCaseSensitive(t *testing.T) {}}-func TestCompileDelimPattern_BareCharIsNotActivePattern(t *testing.T) {- // Plain delimiters are not compiled — only backtick-wrapped tokens are.- if p := compileDelimPattern([]string{"a"}); p != nil {- t.Fatalf("compileDelimPattern([a]) = %v, want nil", p)+func TestCompileDelimPattern_BareCharIsActivePattern(t *testing.T) {+ // Plain (bare) delimiters ARE compiled into an active pattern (#17723+ // fix): they split the payload into paragraphs that are then merged by+ // token size. Only when no entry yields a pattern does compileDelimPattern+ // return nil.+ if p := compileDelimPattern([]string{"a"}); p == nil {+ t.Fatalf("compileDelimPattern([a]) = nil, want active pattern")+ }+ // Empty / all-empty entries still yield nil.+ if p := compileDelimPattern([]string{""}); p != nil {+ t.Fatalf("compileDelimPattern([\"\"]) = %v, want nil", p)}}func TestCompileDelimPattern_ExtractsPerEntryIndependently(t *testing.T) {- // Adjacent entries must not form a cross-boundary backtick pair.- // Concatenating "`aa" + "`bb`" would invent an "aa`bb" token; per-entry- // extraction only sees the complete "`bb`" pair in the second entry.+ // Adjacent entries must not form a cross-boundary backtick pair:+ // "`aa" + "`bb`" must NOT invent an "aa`bb" token. Per-entry extraction+ // compiles "`aa" as its literal content (keepBare only promotes entries+ // with NO backticks at all) and "`bb`" as the inner "bb"; the two are+ // alternated, never concatenated into a single "aa`bb" delimiter.p := compileDelimPattern([]string{"`aa", "`bb`"})if p == nil {- t.Fatal("expected pattern from second entry `bb`")+ t.Fatal("expected pattern from entries")}- if got := p.String(); got != "bb" {- t.Fatalf("pattern = %q, want %q (no cross-entry token)", got, "bb")+ // The complete backtick entry "`bb`" is active.+ if !p.MatchString("bb") {+ t.Fatalf("must match token from second entry; pattern=%q", p.String())}+ // An incomplete backtick entry "`aa" is compiled as its literal content,+ // not promoted to a bare "aa", so a bare "aa" must not match.if p.MatchString("aa") {- t.Fatalf("must not match incomplete first-entry token; pattern=%q", p.String())+ t.Fatalf("incomplete backtick entry must not match a bare token; pattern=%q", p.String())}- if !p.MatchString("bb") {- t.Fatalf("must match token from second entry; pattern=%q", p.String())+ // No cross-entry "aa`bb" delimiter was invented.+ if strings.Contains(p.String(), "aa`bb") {+ t.Fatalf("cross-entry backtick token invented; pattern=%q", p.String())}// Multiple well-formed entries still combine.diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex de85340dae..d54a105ed0 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -316,10 +316,29 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text stringreturn emptyOutputs()}+ // No active delimiter at all: single-section merge that only re-splits+ // oversized sections on sentence boundaries.if !hasActiveDelimiter(delimPattern) {- return c.mergeByTokenSize(text, childrenPattern)+ return c.mergeByTokenSize(text, nil, childrenPattern)}+ // Custom (backtick-wrapped) delimiter: one chunk per segment, no token+ // merge — mirrors Python naive_merge's has_custom branch+ // (token_chunker.py:1194-1213).+ if hasCustomDelim(c.param.Delimiters) {+ return c.chunkPerSegment(text, delimPattern, childrenPattern)+ }++ // Bare delimiter: split into paragraphs (delimiter dropped), then merge+ // by token size — mirrors Python naive_merge's default branch. This is+ // the #17723 fix: bare delimiters were previously ignored entirely.+ return c.mergeByTokenSize(text, delimPattern, childrenPattern)+}++// chunkPerSegment splits text on a custom (backtick) delimiter and emits one+// chunk per segment with no token-size merge. Mirrors Python naive_merge's+// has_custom branch (token_chunker.py:1194-1213).+func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {parts := splitDroppingDelim(text, delimPattern)cleaned := make([]string, 0, len(parts))for _, p := range parts {@@ -341,12 +360,6 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text stringtextDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})}docs := applyChildrenDelimText(textDocs, childrenPattern)-- // Python's naive_merge: a custom (backtick) delimiter yields one chunk- // per segment and no token-size merge (naive_merge:1194-1213). A- // non-custom active delimiter cannot reach here — delimPattern is- // non-nil only when a backtick delimiter exists, so the split-then-- // merge branch was unreachable and has been removed.return chunkOutputs(docs)}@@ -390,7 +403,7 @@ func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int)// unconditional overlap prefix), matching Python naive_merge / token_chunker.// When overlap>0 the previous chunk's tail is always prepended, so a chunk may// exceed chunk_token_size by up to the overlap amount.-func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *regexp.Regexp) map[string]any {+func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {target := c.param.ChunkTokenSizeoverlapPct := c.param.OverlappedPercent// Clamp to [0,100] so the merge math below never produces a@@ -408,48 +421,76 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r// naive_merge runs text.replace("\r\n", "\n").replace("\r", "\n"),// then treats the input string as one section.text = strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")- sections := []string{text}- if len(sections) == 0 {- return emptyOutputs()- }- // Build merge units from the (single) section. Each unit is a paragraph- // (split on sentence delimiters when the section exceeds target); its- // token count is precomputed so mergeUnits can keep a running sum.+ // Build the merge units.+ //+ // When a bare (non-custom) delimiter is active we split the text into+ // paragraphs (the delimiter is DROPPED, mirroring Python naive_merge) and+ // use each paragraph VERBATIM as a unit. This preserves the original+ // inter-paragraph whitespace, so the merged text matches the source (and+ // Python naive_merge); an oversized paragraph also stands alone (#17799)+ // instead of being re-split — both matching Python.+ //+ // Otherwise (no active delimiter) the whole text is a single section and+ // oversized sections are re-split on production sentence delimiters, which+ // injects "\n" to mirror Python's sentence-boundary handling. This is the+ // historical behavior and must stay byte-for-byte identical.+ useDelimSplit := hasActiveDelimiter(delimPattern) && !hasCustomDelim(c.param.Delimiters)var units []schema.ChunkDoc- for _, sec := range sections {- sec = strings.TrimSpace(sec)- if sec == "" {- continue- }- t := "\n" + sec- tk := tokenizeStr(t)- if tk <= target {- units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})- continue- }- // Oversized section: split on production sentence delimiters into- // units. An oversize unit (still exceeds the budget) is kept whole —- // no atom-split, matching Python naive_merge. Per the unified- // algorithm an over-budget unit STANDS ALONE (#17799): never merged- // into the previous chunk.- parts := sentenceDelimiter.Split(sec, -1)- hadPart := false- for _, part := range parts {- // Keep the raw split fragment, including any inter-line trailing- // whitespace (Python's naive_merge builds each unit from- // "\n" + sub_sec with its trailing space, never TrimSpaced). Only- // genuinely empty fragments are skipped.- if part == "" {+ if useDelimSplit {+ // Mirror Python naive_merge's default path (rag/nlp/__init__.py:1406-1415):+ // split on the delimiter, DROP the delimiter text, and prepend "\n" to+ // each kept paragraph. The sub-sec keeps its original surrounding+ // whitespace (e.g. the trailing space before the delimiter); only the+ // delimiter itself is removed. No sentence re-split is performed here —+ // an oversize paragraph stands alone as its own chunk, matching Python.+ for _, sub := range splitDroppingDelim(text, delimPattern) {+ if sub == "" {continue}- hadPart = true- seg := "\n" + part- units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})- }- if !hadPart {+ t := "\n" + subunits = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})}+ } else {+ sections := []string{text}+ for _, sec := range sections {+ sec = strings.TrimSpace(sec)+ if sec == "" {+ continue+ }+ t := "\n" + sec+ tk := tokenizeStr(t)+ if tk <= target {+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})+ continue+ }+ // Oversized section: split on production sentence delimiters into+ // units. An oversize unit (still exceeds the budget) is kept whole+ // — no atom-split, matching Python naive_merge. Per the unified+ // algorithm an over-budget unit STANDS ALONE (#17799): never merged+ // into the previous chunk.+ parts := sentenceDelimiter.Split(sec, -1)+ hadPart := false+ for _, part := range parts {+ // Keep the raw split fragment, including any inter-line+ // trailing whitespace (Python's naive_merge builds each unit+ // from "\n" + sub_sec with its trailing space, never TrimSpaced).+ // Only genuinely empty fragments are skipped.+ if part == "" {+ continue+ }+ hadPart = true+ seg := "\n" + part+ units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})+ }+ if !hadPart {+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})+ }+ }+ }++ if len(units) == 0 {+ return emptyOutputs()}// Merge with the unified core (scaled overlap threshold + unconditionaldiff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.goindex 5b55ddf326..859d969608 100644--- a/internal/ingestion/component/chunker/token_delim_contract_test.go+++ b/internal/ingestion/component/chunker/token_delim_contract_test.go@@ -6,13 +6,12 @@ import ("testing")-// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick)-// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting-// delimiter_mode + a bare delimiter is effectively a no-op — the text is-// merged by token_size and the bare token survives inside a chunk rather-// than acting as a split point. Regression guard for the "bare entries are-// ignored" contract.-func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {+// TestTokenChunker_BareDelimiterHonored locks the #17723 fix: a bare+// (non-backtick) delimiter entry is now honored by TokenChunker. The payload+// is split on the delimiter into paragraphs (the delimiter is DROPPED, matching+// Python naive_merge), and those paragraphs are then merged by token_size.+// Regression guard for the "bare entries are active" contract.+func TestTokenChunker_BareDelimiterHonored(t *testing.T) {c, err := NewTokenChunker(map[string]any{"delimiter_mode": "delimiter","delimiters": []string{"::"},@@ -38,10 +37,17 @@ func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {for _, ck := range chunks {joined.WriteString(ck["text"].(string))}- // No content dropped and the bare "::" is preserved (just chunked by- // token size, not split on "::").- if joined.String() != text {- t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks))+ // No content dropped, and the bare "::" is split away (not preserved inside+ // a chunk). Python's naive_merge rebuilds each paragraph with a leading+ // "\n", so the joined text equals the source with "::" replaced by "\n".+ const want = "alpha\nbeta\ngamma\ndelta"+ if joined.String() != want {+ t.Errorf("bare delimiter not honored: joined=%q want %q (chunks=%v)", joined.String(), want, chunkTexts(chunks))+ }+ for _, ck := range chunks {+ if strings.Contains(ck["text"].(string), "::") {+ t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))+ }}}diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.goindex f9926009d8..320a3c9f53 100644--- a/internal/ingestion/component/chunker/token_strict_cap_test.go+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go@@ -132,7 +132,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {t.Fatalf("NewTokenChunker: %v", err)}tc := comp.(*TokenChunkerComponent)- out := tc.mergeByTokenSize(b.String(), nil)+ out := tc.mergeByTokenSize(b.String(), nil, nil)chunks, _ := out["chunks"].([]map[string]any)if len(chunks) < 2 {t.Fatalf("want multiple chunks, got %d", len(chunks))@@ -165,7 +165,7 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) {if err != nil {t.Fatalf("NewTokenChunker: %v", err)}- out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil)+ out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil, nil)chunks, _ := out["chunks"].([]map[string]any)if len(chunks) != 1 {t.Fatalf("over-budget unit must stay whole, got %d chunk(s)", len(chunks))@@ -203,7 +203,7 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {if err != nil {t.Fatalf("NewTokenChunker: %v", err)}- out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil)+ out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil, nil)chunks, _ := out["chunks"].([]map[string]any)return chunks}diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 98ab935adb..4d2ac8e206 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -607,7 +607,7 @@ func TestMergeByTokenSize_CRLFNormalization(t *testing.T) {c := &TokenChunkerComponent{}c.param.ChunkTokenSize = 128c.param.OverlappedPercent = 0- out := c.mergeByTokenSize(text, nil)+ out := c.mergeByTokenSize(text, nil, nil)raw, ok := out["chunks"].([]map[string]any)if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)@@ -652,7 +652,7 @@ func TestMergeByTokenSize_PreservesBlankLines(t *testing.T) {c := &TokenChunkerComponent{}c.param.ChunkTokenSize = 128c.param.OverlappedPercent = 0- out := c.mergeByTokenSize("A\n\n\nB", nil)+ out := c.mergeByTokenSize("A\n\n\nB", nil, nil)raw, ok := out["chunks"].([]map[string]any)if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)@@ -681,7 +681,7 @@ func TestMergeByTokenSize_OversizeDropsDelimiters(t *testing.T) {c.param.ChunkTokenSize = 5c.param.OverlappedPercent = 0text := "第一句。第二句。第三句。第四句。第五句。"- out := c.mergeByTokenSize(text, nil)+ out := c.mergeByTokenSize(text, nil, nil)raw, ok := out["chunks"].([]map[string]any)if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)@@ -712,7 +712,7 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {// dropped, mirroring Python, so the blank line must not survive.block := strings.Repeat("知识库检索增强生成技术", 7) // 70 charstext := block + "\n\n" + block- out := c.mergeByTokenSize(text, nil)+ out := c.mergeByTokenSize(text, nil, nil)raw, ok := out["chunks"].([]map[string]any)if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)