Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): honor bare (non-backtick) delimiters in TokenChunker (#17723) (#18182)

Jackb505437db53e
diff --git a/internal/ingestion/component/chunker/bare_delimiter_test.go b/internal/ingestion/component/chunker/bare_delimiter_test.go
new file mode 100644
index 0000000000..0f4fc9a682
--- /dev/null
+++ b/internal/ingestion/component/chunker/bare_delimiter_test.go
@@ -0,0 +1,239 @@
+package chunker
+
+import (
+ "context"
+ "strings"
+ "testing"
+)
+
+// bare_delimiter_test covers the fix for #17723: bare (non-backtick)
+// delimiters must now be honored by TokenChunker — splitting the payload into
+// paragraphs that are then merged by token size — instead of being silently
+// ignored (the previous keepBare=false contract).
+//
+// Python contract (rag/nlp/__init__.py:1406-1415, naive_merge default path):
+// the payload is split on the delimiter, the delimiter text is DROPPED, and
+// each kept paragraph is rebuilt with a leading "\n" (its original surrounding
+// whitespace is preserved). Paragraphs are thus joined by "\n", never by the
+// raw delimiter. One chunk per segment (no token merge) happens ONLY for a
+// backtick-wrapped CUSTOM delimiter.
+//
+// These tests target BOTH the text/markdown/html path (invokeTextPayload) and
+// the JSON path (invokeJSONPayload -> chunkFromItem + global merge), and lock
+// the custom(backtick) vs bare distinction.
+
+// newBareChunker builds a TokenChunker for the given delimiter list and a small
+// token budget so merges are observable.
+func newBareChunker(t *testing.T, delims []string, tokenSize float64) *TokenChunkerComponent {
+ t.Helper()
+ c, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": delims,
+ "chunk_token_size": tokenSize,
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ return c.(*TokenChunkerComponent)
+}
+
+func invokeText(t *testing.T, c *TokenChunkerComponent, text string) []map[string]any {
+ t.Helper()
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.txt",
+ "output_format": "text",
+ "text": text,
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if msg, ok := out["_ERROR"].(string); ok && msg != "" {
+ t.Fatalf("Go returned _ERROR: %s", msg)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ return chunks
+}
+
+func joinedText(chunks []map[string]any) string {
+ var b strings.Builder
+ for _, ck := range chunks {
+ b.WriteString(ck["text"].(string))
+ }
+ return b.String()
+}
+
+// TestBareDelimiterSplitsAndTokenMergesText asserts the core fix: a bare
+// multi-char delimiter ("::") now splits the text into paragraphs, the
+// delimiter text is dropped, and the paragraphs are rejoined with "\n"
+// (matching Python's "\n"+sub_sec reconstruction) — not with the raw
+// delimiter. The paragraphs are then merged by token size.
+func TestBareDelimiterSplitsAndTokenMergesText(t *testing.T) {
+ c := newBareChunker(t, []string{"::"}, 1024)
+ const text = "alpha::beta::gamma::delta"
+ chunks := invokeText(t, c, text)
+ if len(chunks) == 0 {
+ t.Fatal("expected at least one chunk")
+ }
+ // The delimiter must never survive inside a chunk.
+ for _, ck := range chunks {
+ if strings.Contains(ck["text"].(string), "::") {
+ t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))
+ }
+ }
+ // Paragraphs are rejoined with "\n" (each paragraph keeps a leading "\n"),
+ // so the joined content equals the source with "::" replaced by "\n".
+ const want = "alpha\nbeta\ngamma\ndelta"
+ if got := joinedText(chunks); got != want {
+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+ }
+}
+
+// TestBareSingleCharDelimiterSplitsText covers the classic case: a single
+// ASCII char delimiter (".") splits sentences, the delimiter is dropped, and
+// paragraphs are rejoined with "\n".
+func TestBareSingleCharDelimiterSplitsText(t *testing.T) {
+ c := newBareChunker(t, []string{"."}, 1024)
+ chunks := invokeText(t, c, "first.second.third")
+ const want = "first\nsecond\nthird"
+ if got := joinedText(chunks); got != want {
+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+ }
+ for _, ck := range chunks {
+ if strings.Contains(ck["text"].(string), ".") {
+ t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))
+ }
+ }
+}
+
+// TestBareCJKBoundarySplitsText covers a CJK punctuation delimiter.
+func TestBareCJKBoundarySplitsText(t *testing.T) {
+ c := newBareChunker(t, []string{";"}, 1024)
+ chunks := invokeText(t, c, "第一部分;第二部分;第三部分")
+ const want = "第一部分\n第二部分\n第三部分"
+ if got := joinedText(chunks); got != want {
+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+ }
+}
+
+// TestBareDelimiterSplitsJSON asserts the JSON path also honors bare
+// delimiters (chunkFromItem now receives a non-nil pattern and the global
+// merge applies because there is no custom delimiter).
+func TestBareDelimiterSplitsJSON(t *testing.T) {
+ c := newBareChunker(t, []string{"。"}, 1024)
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.json",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "第一句内容。第二句内容。第三句内容。", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ const want = "第一句内容\n第二句内容\n第三句内容"
+ if got := joinedText(chunks); got != want {
+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+ }
+}
+
+// TestCustomDelimiterStillOneChunkPerSegment locks the distinction: a
+// backtick-wrapped delimiter must yield one chunk per segment with NO token
+// merge (and the delimiter is dropped with no "\n" rejoin), even with a tight
+// budget.
+func TestCustomDelimiterStillOneChunkPerSegment(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": []string{"`::`"},
+ "chunk_token_size": float64(4),
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha::beta::gamma::delta")
+ want := []string{"alpha", "beta", "gamma", "delta"}
+ if len(chunks) != len(want) {
+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+ }
+ for i, w := range want {
+ if got := chunks[i]["text"].(string); got != w {
+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+ }
+ }
+}
+
+// TestBareDelimiterEmptyText asserts empty input yields no chunks (no panic).
+func TestBareDelimiterEmptyText(t *testing.T) {
+ c := newBareChunker(t, []string{"::"}, 8)
+ chunks := invokeText(t, c, "")
+ if len(chunks) != 0 {
+ t.Errorf("empty text should produce no chunks, got %d", len(chunks))
+ }
+}
+
+// TestBareDelimiterDropsEmptySegment asserts that a genuinely empty segment
+// produced by a doubled delimiter ("alpha||beta") is dropped, so the joined
+// text has no empty paragraph (mirrors Python's `if not sub_sec: continue`).
+// (Whitespace-only segments are NOT dropped — Python keeps them — so this test
+// deliberately uses an empty, not whitespace, segment.)
+func TestBareDelimiterDropsEmptySegment(t *testing.T) {
+ c := newBareChunker(t, []string{"|"}, 1024)
+ chunks := invokeText(t, c, "alpha||beta")
+ const want = "alpha\nbeta"
+ if got := joinedText(chunks); got != want {
+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+ }
+}
+
+// TestBareDelimiterCRLFNormalized asserts CRLF/CR are normalized to LF before
+// splitting (mirrors Python naive_merge text normalization). The "\n" left
+// behind by the dropped delimiter is preserved between paragraphs.
+func TestBareDelimiterCRLFNormalized(t *testing.T) {
+ c := newBareChunker(t, []string{"."}, 1024)
+ chunks := invokeText(t, c, "line one.\r\nline two.\rline three.")
+ const want = "line one\n\nline two\n\nline three"
+ if got := joinedText(chunks); got != want {
+ t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+ }
+}
+
+// TestBareDelimiterNoDelimiterFallback asserts that with no delimiter at all
+// the classic single-section merge still applies (regression guard for the
+// fallback path): the bare-delimiter machinery is bypassed entirely.
+func TestBareDelimiterNoDelimiterFallback(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": []string{},
+ "chunk_token_size": float64(8),
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha beta gamma delta epsilon zeta eta theta")
+ if len(chunks) == 0 {
+ t.Fatalf("expected at least one chunk")
+ }
+ // No delimiter configured, so the whole text is one section merged by
+ // token size; its content must be preserved in order.
+ if got := joinedText(chunks); !strings.Contains(got, "alpha beta gamma") {
+ t.Errorf("no-delimiter fallback lost content: %q", got)
+ }
+}
+
+// TestBareDelimiterTokenMergeCoalesces asserts that short paragraphs separated
+// by a bare delimiter are merged up to the token budget (not one chunk per
+// paragraph when below budget), while still keeping the "\n" paragraph
+// boundary — exactly mirroring Python's merge.
+func TestBareDelimiterTokenMergeCoalesces(t *testing.T) {
+ c := newBareChunker(t, []string{"。"}, 1024)
+ chunks := invokeText(t, c, "短句一。短句二。短句三。")
+ // All three short sentences fit under the large budget, so they coalesce
+ // into a single merged chunk whose text keeps the "\n" paragraph joins.
+ if len(chunks) != 1 {
+ t.Fatalf("expected single merged chunk, got %d (%v)", len(chunks), chunkTexts(chunks))
+ }
+ const want = "短句一\n短句二\n短句三"
+ if got := chunks[0]["text"].(string); got != want {
+ t.Errorf("merged text: want %q got %q", want, got)
+ }
+}
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index da6699e6fc..92c32f5679 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -77,13 +77,15 @@ func stringListFromAny(in []any) []string {
// ---------------------------------------------------------------------------
// compileDelimPattern compiles a TokenChunker-style []string delimiter list.
-// Only backtick-wrapped entries produce an active pattern (Python
-// token_chunker / rag/nlp/delim list helper); plain (non-backtick) entries
-// are ignored here. mergeByTokenSize uses its own hardcoded sentenceDelimiter,
-// not this list. Canonical single-string parser_config.delimiter parsing
-// lives in ragflow/internal/parser/chunk (ParseDelimiterField).
+// Every non-empty entry is active, including bare (non-backtick) delimiters,
+// mirroring Python naive_merge / rag/nlp/delim where bare single-character
+// delimiters still split. Backtick-wrapped entries contribute their inner
+// content. invokeTextPayload decides whether an active delimiter yields one
+// chunk per segment (custom/backtick, no merge) or splits into paragraphs that
+// are merged by token size (bare). Canonical single-string parser_config.delimiter
+// parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField).
func compileDelimPattern(delims []string) *regexp.Regexp {
- return chunk.CompileDelimiterListPattern(delims)
+ return chunk.CompileDelimiterPatternList(delims, true)
}
// splitDroppingDelim mirrors Python's _split_text_by_pattern
diff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
index 613ab0e62b..63598d838f 100644
--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
@@ -119,29 +119,42 @@ func TestCompileDelimPattern_BacktickEndIsCaseSensitive(t *testing.T) {
}
}
-func TestCompileDelimPattern_BareCharIsNotActivePattern(t *testing.T) {
- // Plain delimiters are not compiled — only backtick-wrapped tokens are.
- if p := compileDelimPattern([]string{"a"}); p != nil {
- t.Fatalf("compileDelimPattern([a]) = %v, want nil", p)
+func TestCompileDelimPattern_BareCharIsActivePattern(t *testing.T) {
+ // Plain (bare) delimiters ARE compiled into an active pattern (#17723
+ // fix): they split the payload into paragraphs that are then merged by
+ // token size. Only when no entry yields a pattern does compileDelimPattern
+ // return nil.
+ if p := compileDelimPattern([]string{"a"}); p == nil {
+ t.Fatalf("compileDelimPattern([a]) = nil, want active pattern")
+ }
+ // Empty / all-empty entries still yield nil.
+ if p := compileDelimPattern([]string{""}); p != nil {
+ t.Fatalf("compileDelimPattern([\"\"]) = %v, want nil", p)
}
}
func TestCompileDelimPattern_ExtractsPerEntryIndependently(t *testing.T) {
- // Adjacent entries must not form a cross-boundary backtick pair.
- // Concatenating "`aa" + "`bb`" would invent an "aa`bb" token; per-entry
- // extraction only sees the complete "`bb`" pair in the second entry.
+ // Adjacent entries must not form a cross-boundary backtick pair:
+ // "`aa" + "`bb`" must NOT invent an "aa`bb" token. Per-entry extraction
+ // compiles "`aa" as its literal content (keepBare only promotes entries
+ // with NO backticks at all) and "`bb`" as the inner "bb"; the two are
+ // alternated, never concatenated into a single "aa`bb" delimiter.
p := compileDelimPattern([]string{"`aa", "`bb`"})
if p == nil {
- t.Fatal("expected pattern from second entry `bb`")
+ t.Fatal("expected pattern from entries")
}
- if got := p.String(); got != "bb" {
- t.Fatalf("pattern = %q, want %q (no cross-entry token)", got, "bb")
+ // The complete backtick entry "`bb`" is active.
+ if !p.MatchString("bb") {
+ t.Fatalf("must match token from second entry; pattern=%q", p.String())
}
+ // An incomplete backtick entry "`aa" is compiled as its literal content,
+ // not promoted to a bare "aa", so a bare "aa" must not match.
if p.MatchString("aa") {
- t.Fatalf("must not match incomplete first-entry token; pattern=%q", p.String())
+ t.Fatalf("incomplete backtick entry must not match a bare token; pattern=%q", p.String())
}
- if !p.MatchString("bb") {
- t.Fatalf("must match token from second entry; pattern=%q", p.String())
+ // No cross-entry "aa`bb" delimiter was invented.
+ if strings.Contains(p.String(), "aa`bb") {
+ t.Fatalf("cross-entry backtick token invented; pattern=%q", p.String())
}
// Multiple well-formed entries still combine.
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index de85340dae..d54a105ed0 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -316,10 +316,29 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
return emptyOutputs()
}
+ // No active delimiter at all: single-section merge that only re-splits
+ // oversized sections on sentence boundaries.
if !hasActiveDelimiter(delimPattern) {
- return c.mergeByTokenSize(text, childrenPattern)
+ return c.mergeByTokenSize(text, nil, childrenPattern)
}
+ // Custom (backtick-wrapped) delimiter: one chunk per segment, no token
+ // merge — mirrors Python naive_merge's has_custom branch
+ // (token_chunker.py:1194-1213).
+ if hasCustomDelim(c.param.Delimiters) {
+ return c.chunkPerSegment(text, delimPattern, childrenPattern)
+ }
+
+ // Bare delimiter: split into paragraphs (delimiter dropped), then merge
+ // by token size — mirrors Python naive_merge's default branch. This is
+ // the #17723 fix: bare delimiters were previously ignored entirely.
+ return c.mergeByTokenSize(text, delimPattern, childrenPattern)
+}
+
+// chunkPerSegment splits text on a custom (backtick) delimiter and emits one
+// chunk per segment with no token-size merge. Mirrors Python naive_merge's
+// has_custom branch (token_chunker.py:1194-1213).
+func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {
parts := splitDroppingDelim(text, delimPattern)
cleaned := make([]string, 0, len(parts))
for _, p := range parts {
@@ -341,12 +360,6 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})
}
docs := applyChildrenDelimText(textDocs, childrenPattern)
-
- // Python's naive_merge: a custom (backtick) delimiter yields one chunk
- // per segment and no token-size merge (naive_merge:1194-1213). A
- // non-custom active delimiter cannot reach here — delimPattern is
- // non-nil only when a backtick delimiter exists, so the split-then-
- // merge branch was unreachable and has been removed.
return chunkOutputs(docs)
}
@@ -390,7 +403,7 @@ func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int)
// unconditional overlap prefix), matching Python naive_merge / token_chunker.
// When overlap>0 the previous chunk's tail is always prepended, so a chunk may
// exceed chunk_token_size by up to the overlap amount.
-func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *regexp.Regexp) map[string]any {
+func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {
target := c.param.ChunkTokenSize
overlapPct := c.param.OverlappedPercent
// Clamp to [0,100] so the merge math below never produces a
@@ -408,48 +421,76 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r
// naive_merge runs text.replace("\r\n", "\n").replace("\r", "\n"),
// then treats the input string as one section.
text = strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")
- sections := []string{text}
- if len(sections) == 0 {
- return emptyOutputs()
- }
- // Build merge units from the (single) section. Each unit is a paragraph
- // (split on sentence delimiters when the section exceeds target); its
- // token count is precomputed so mergeUnits can keep a running sum.
+ // Build the merge units.
+ //
+ // When a bare (non-custom) delimiter is active we split the text into
+ // paragraphs (the delimiter is DROPPED, mirroring Python naive_merge) and
+ // use each paragraph VERBATIM as a unit. This preserves the original
+ // inter-paragraph whitespace, so the merged text matches the source (and
+ // Python naive_merge); an oversized paragraph also stands alone (#17799)
+ // instead of being re-split — both matching Python.
+ //
+ // Otherwise (no active delimiter) the whole text is a single section and
+ // oversized sections are re-split on production sentence delimiters, which
+ // injects "\n" to mirror Python's sentence-boundary handling. This is the
+ // historical behavior and must stay byte-for-byte identical.
+ useDelimSplit := hasActiveDelimiter(delimPattern) && !hasCustomDelim(c.param.Delimiters)
var units []schema.ChunkDoc
- for _, sec := range sections {
- sec = strings.TrimSpace(sec)
- if sec == "" {
- continue
- }
- t := "\n" + sec
- tk := tokenizeStr(t)
- if tk <= target {
- units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})
- continue
- }
- // Oversized section: split on production sentence delimiters into
- // units. An oversize unit (still exceeds the budget) is kept whole —
- // no atom-split, matching Python naive_merge. Per the unified
- // algorithm an over-budget unit STANDS ALONE (#17799): never merged
- // into the previous chunk.
- parts := sentenceDelimiter.Split(sec, -1)
- hadPart := false
- for _, part := range parts {
- // Keep the raw split fragment, including any inter-line trailing
- // whitespace (Python's naive_merge builds each unit from
- // "\n" + sub_sec with its trailing space, never TrimSpaced). Only
- // genuinely empty fragments are skipped.
- if part == "" {
+ if useDelimSplit {
+ // Mirror Python naive_merge's default path (rag/nlp/__init__.py:1406-1415):
+ // split on the delimiter, DROP the delimiter text, and prepend "\n" to
+ // each kept paragraph. The sub-sec keeps its original surrounding
+ // whitespace (e.g. the trailing space before the delimiter); only the
+ // delimiter itself is removed. No sentence re-split is performed here —
+ // an oversize paragraph stands alone as its own chunk, matching Python.
+ for _, sub := range splitDroppingDelim(text, delimPattern) {
+ if sub == "" {
continue
}
- hadPart = true
- seg := "\n" + part
- units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})
- }
- if !hadPart {
+ t := "\n" + sub
units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})
}
+ } else {
+ sections := []string{text}
+ for _, sec := range sections {
+ sec = strings.TrimSpace(sec)
+ if sec == "" {
+ continue
+ }
+ t := "\n" + sec
+ tk := tokenizeStr(t)
+ if tk <= target {
+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})
+ continue
+ }
+ // Oversized section: split on production sentence delimiters into
+ // units. An oversize unit (still exceeds the budget) is kept whole
+ // — no atom-split, matching Python naive_merge. Per the unified
+ // algorithm an over-budget unit STANDS ALONE (#17799): never merged
+ // into the previous chunk.
+ parts := sentenceDelimiter.Split(sec, -1)
+ hadPart := false
+ for _, part := range parts {
+ // Keep the raw split fragment, including any inter-line
+ // trailing whitespace (Python's naive_merge builds each unit
+ // from "\n" + sub_sec with its trailing space, never TrimSpaced).
+ // Only genuinely empty fragments are skipped.
+ if part == "" {
+ continue
+ }
+ hadPart = true
+ seg := "\n" + part
+ units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})
+ }
+ if !hadPart {
+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})
+ }
+ }
+ }
+
+ if len(units) == 0 {
+ return emptyOutputs()
}
// Merge with the unified core (scaled overlap threshold + unconditional
diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.go
index 5b55ddf326..859d969608 100644
--- a/internal/ingestion/component/chunker/token_delim_contract_test.go
+++ b/internal/ingestion/component/chunker/token_delim_contract_test.go
@@ -6,13 +6,12 @@ import (
"testing"
)
-// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick)
-// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting
-// delimiter_mode + a bare delimiter is effectively a no-op — the text is
-// merged by token_size and the bare token survives inside a chunk rather
-// than acting as a split point. Regression guard for the "bare entries are
-// ignored" contract.
-func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {
+// TestTokenChunker_BareDelimiterHonored locks the #17723 fix: a bare
+// (non-backtick) delimiter entry is now honored by TokenChunker. The payload
+// is split on the delimiter into paragraphs (the delimiter is DROPPED, matching
+// Python naive_merge), and those paragraphs are then merged by token_size.
+// Regression guard for the "bare entries are active" contract.
+func TestTokenChunker_BareDelimiterHonored(t *testing.T) {
c, err := NewTokenChunker(map[string]any{
"delimiter_mode": "delimiter",
"delimiters": []string{"::"},
@@ -38,10 +37,17 @@ func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {
for _, ck := range chunks {
joined.WriteString(ck["text"].(string))
}
- // No content dropped and the bare "::" is preserved (just chunked by
- // token size, not split on "::").
- if joined.String() != text {
- t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks))
+ // No content dropped, and the bare "::" is split away (not preserved inside
+ // a chunk). Python's naive_merge rebuilds each paragraph with a leading
+ // "\n", so the joined text equals the source with "::" replaced by "\n".
+ const want = "alpha\nbeta\ngamma\ndelta"
+ if joined.String() != want {
+ t.Errorf("bare delimiter not honored: joined=%q want %q (chunks=%v)", joined.String(), want, chunkTexts(chunks))
+ }
+ for _, ck := range chunks {
+ if strings.Contains(ck["text"].(string), "::") {
+ t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))
+ }
}
}
diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.go
index f9926009d8..320a3c9f53 100644
--- a/internal/ingestion/component/chunker/token_strict_cap_test.go
+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go
@@ -132,7 +132,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {
t.Fatalf("NewTokenChunker: %v", err)
}
tc := comp.(*TokenChunkerComponent)
- out := tc.mergeByTokenSize(b.String(), nil)
+ out := tc.mergeByTokenSize(b.String(), nil, nil)
chunks, _ := out["chunks"].([]map[string]any)
if len(chunks) < 2 {
t.Fatalf("want multiple chunks, got %d", len(chunks))
@@ -165,7 +165,7 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) {
if err != nil {
t.Fatalf("NewTokenChunker: %v", err)
}
- out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil)
+ out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil, nil)
chunks, _ := out["chunks"].([]map[string]any)
if len(chunks) != 1 {
t.Fatalf("over-budget unit must stay whole, got %d chunk(s)", len(chunks))
@@ -203,7 +203,7 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {
if err != nil {
t.Fatalf("NewTokenChunker: %v", err)
}
- out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil)
+ out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil, nil)
chunks, _ := out["chunks"].([]map[string]any)
return chunks
}
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index 98ab935adb..4d2ac8e206 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -607,7 +607,7 @@ func TestMergeByTokenSize_CRLFNormalization(t *testing.T) {
c := &TokenChunkerComponent{}
c.param.ChunkTokenSize = 128
c.param.OverlappedPercent = 0
- out := c.mergeByTokenSize(text, nil)
+ out := c.mergeByTokenSize(text, nil, nil)
raw, ok := out["chunks"].([]map[string]any)
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
@@ -652,7 +652,7 @@ func TestMergeByTokenSize_PreservesBlankLines(t *testing.T) {
c := &TokenChunkerComponent{}
c.param.ChunkTokenSize = 128
c.param.OverlappedPercent = 0
- out := c.mergeByTokenSize("A\n\n\nB", nil)
+ out := c.mergeByTokenSize("A\n\n\nB", nil, nil)
raw, ok := out["chunks"].([]map[string]any)
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
@@ -681,7 +681,7 @@ func TestMergeByTokenSize_OversizeDropsDelimiters(t *testing.T) {
c.param.ChunkTokenSize = 5
c.param.OverlappedPercent = 0
text := "第一句。第二句。第三句。第四句。第五句。"
- out := c.mergeByTokenSize(text, nil)
+ out := c.mergeByTokenSize(text, nil, nil)
raw, ok := out["chunks"].([]map[string]any)
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
@@ -712,7 +712,7 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {
// dropped, mirroring Python, so the blank line must not survive.
block := strings.Repeat("知识库检索增强生成技术", 7) // 70 chars
text := block + "\n\n" + block
- out := c.mergeByTokenSize(text, nil)
+ out := c.mergeByTokenSize(text, nil, nil)
raw, ok := out["chunks"].([]map[string]any)
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)