Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): align Go token merge with Python OVER_CAP and delimiter boundary (#17835)

Jack17bafb363f4c
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 0ee111d88a..1dfeadf237 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -118,6 +118,38 @@ func splitKeepingDelim(text string, pattern *regexp.Regexp) []string {
return out
}
+// splitDroppingDelim mirrors Python's _split_text_by_pattern
+// (token_chunker.py:79-90). Unlike splitKeepingDelim, the captured delimiter
+// is DISCARDED rather than glued to the preceding segment: re.split with a
+// captured group keeps delimiters at odd indices, and only the even-index
+// (text) parts are kept. This is the behaviour the text/markdown/html path
+// must reproduce so a split chunk reads "first sentence here" without the
+// trailing delimiter.
+func splitDroppingDelim(text string, pattern *regexp.Regexp) []string {
+ if pattern == nil {
+ return []string{text}
+ }
+ idxs := pattern.FindAllStringIndex(text, -1)
+ if len(idxs) == 0 {
+ return []string{text}
+ }
+ var out []string
+ cursor := 0
+ for _, idx := range idxs {
+ start, end := idx[0], idx[1]
+ if start == cursor {
+ cursor = end
+ continue
+ }
+ out = append(out, text[cursor:start])
+ cursor = end
+ }
+ if cursor < len(text) {
+ out = append(out, text[cursor:])
+ }
+ return out
+}
+
// ---------------------------------------------------------------------------
// chunk-doc helpers
// ---------------------------------------------------------------------------
diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.go
new file mode 100644
index 0000000000..58c18d6066
--- /dev/null
+++ b/internal/ingestion/component/chunker/custom_delim_test.go
@@ -0,0 +1,162 @@
+package chunker
+
+import (
+ "context"
+ "testing"
+)
+
+// custom_delim_test pins the backtick-wrapped newline delimiter behaviour of
+// TokenChunker against Python's rag/flow/chunker/token_chunker.py.
+//
+// There are two distinct, path-specific divergences that this file locks down:
+//
+// 1. text/markdown/html path: Python's _split_text_by_pattern (token_chunker.py:79-90)
+// uses re.split with a captured group and keeps only the even-index
+// (text) parts, so the delimiter is DROPPED. Go's splitKeepingDelim glues
+// the delimiter to the end of the preceding segment, so a chunk reads
+// "first sentence here\n" instead of "first sentence here". The fix makes
+// the text path drop the delimiter like Python.
+//
+// 2. json path: Python's _build_json_chunks + _finalize_json_chunks never
+// strip the chunk text, so the delimiter's trailing newline survives
+// ("first segment line one\n"). Go's invokeJSONPayload ran every chunk
+// through strings.TrimSpace, which deleted that newline. The fix stops
+// trimming, so the newline is kept like Python.
+//
+// doc_type_kwd is intentionally asserted to remain present on every chunk.
+// It is a load-bearing Go field (index column + media dispatch) and is NOT
+// part of the divergence; it is classified go_intentional in known_diffs.json,
+// not a bug to fix here.
+
+const backtickNewline = "`\n`"
+
+func invokeTokenChunks(t *testing.T, params, input map[string]any) []map[string]any {
+ t.Helper()
+ c, err := NewTokenChunker(params)
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(context.Background(), nil, input)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if msg, ok := out["_ERROR"].(string); ok && msg != "" {
+ t.Fatalf("Go returned _ERROR: %s", msg)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ return chunks
+}
+
+func chunkTexts(chunks []map[string]any) []string {
+ out := make([]string, 0, len(chunks))
+ for _, c := range chunks {
+ if s, ok := c["text"].(string); ok {
+ out = append(out, s)
+ }
+ }
+ return out
+}
+
+// TestCustomDelimTextDropsDelimiter reproduces token__text_backtick.
+func TestCustomDelimTextDropsDelimiter(t *testing.T) {
+ params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}
+ input := map[string]any{
+ "name": "t", "output_format": "text",
+ "text": "first sentence here\nsecond sentence here\nthird sentence here",
+ }
+ chunks := invokeTokenChunks(t, params, input)
+
+ want := []string{"first sentence here", "second sentence here", "third sentence here"}
+ if len(chunks) != len(want) {
+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+ }
+ for i, w := range want {
+ got := chunks[i]["text"].(string)
+ if got != w {
+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+ }
+ if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
+ t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
+ }
+ }
+}
+
+// TestCustomDelimJSONKeepsNewline reproduces token__json_backtick.
+func TestCustomDelimJSONKeepsNewline(t *testing.T) {
+ params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}
+ input := map[string]any{
+ "name": "t", "output_format": "json",
+ "json": []map[string]any{
+ {"text": "first segment line one\nfirst segment line two", "doc_type_kwd": "text"},
+ {"text": "second segment line one\nsecond segment line two", "doc_type_kwd": "text"},
+ },
+ }
+ chunks := invokeTokenChunks(t, params, input)
+
+ want := []string{
+ "first segment line one\n", "first segment line two",
+ "second segment line one\n", "second segment line two",
+ }
+ if len(chunks) != len(want) {
+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+ }
+ for i, w := range want {
+ got := chunks[i]["text"].(string)
+ if got != w {
+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+ }
+ if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
+ t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
+ }
+ }
+}
+
+// TestCustomDelimMarkdownDropsDelimiter reproduces token__markdown_backtick:
+// the delimiter is dropped and no chunk text ends with a newline.
+func TestCustomDelimMarkdownDropsDelimiter(t *testing.T) {
+ params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}
+ input := map[string]any{
+ "name": "t", "output_format": "markdown",
+ "markdown": "# Title\n\nParagraph one.\n\nParagraph two.",
+ }
+ chunks := invokeTokenChunks(t, params, input)
+ // The upstream decode normalizes markdown block boundaries into the
+ // backtick-newline delimiter, so the text path must split into exactly
+ // three trimmed chunks with the delimiter dropped (no trailing newline).
+ want := []string{"# Title", "Paragraph one.", "Paragraph two."}
+ if len(chunks) != len(want) {
+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+ }
+ for i, w := range want {
+ got := chunks[i]["text"].(string)
+ if got != w {
+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+ }
+ if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
+ t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
+ }
+ }
+}
+
+// TestCustomDelimHTMLDropsDelimiter reproduces token__html_backtick.
+func TestCustomDelimHTMLDropsDelimiter(t *testing.T) {
+ params := map[string]any{"chunk_token_size": float64(128), "delimiters": []string{backtickNewline}}
+ input := map[string]any{
+ "name": "t", "output_format": "html",
+ "html": "<p>one</p>\n<p>two</p>\n<p>three</p>",
+ }
+ chunks := invokeTokenChunks(t, params, input)
+ want := []string{"<p>one</p>", "<p>two</p>", "<p>three</p>"}
+ if len(chunks) != len(want) {
+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+ }
+ for i, w := range want {
+ got := chunks[i]["text"].(string)
+ if got != w {
+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+ }
+ if kd, _ := chunks[i]["doc_type_kwd"].(string); kd != "text" {
+ t.Errorf("chunk[%d] doc_type_kwd: Go must keep it, want %q got %q", i, "text", kd)
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
index fc097221c1..0be8a758c1 100644
--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
@@ -179,9 +179,9 @@ func TestTokenChunker_BacktickEndSplitsOnlyAtLowercase(t *testing.T) {
got = append(got, text)
}
}
- // splitKeepingDelim glues the matched delimiter onto the preceding
- // segment: "the end" | " and End and END come"
- want := []string{"the end", "and End and END come"}
+ // Python's _split_text_by_pattern drops the matched delimiter and
+ // .strip()s each segment: "the" | "and End and END come"
+ want := []string{"the", "and End and END come"}
if len(got) != len(want) {
t.Fatalf("chunks = %#v, want %#v", got, want)
}
@@ -217,8 +217,9 @@ func TestTokenChunker_BacktickASplitsOnlyAtLowercase(t *testing.T) {
got = append(got, text)
}
}
- // "B" + "a" glued → "Ba"; remainder "Ab"
- want := []string{"Ba", "Ab"}
+ // "B" + "a" glued → "Ba"; remainder "Ab". Python drops the matched
+ // delimiter and .strip()s, leaving "B" | "Ab".
+ want := []string{"B", "Ab"}
if len(got) != len(want) {
t.Fatalf("chunks = %#v, want %#v", got, want)
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 5a730f38da..060b647693 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -108,6 +108,9 @@ func (p *tokenChunkerParam) Update(conf map[string]any) {
if v, ok := schema.NumericFromAny(conf["image_context_size"]); ok {
p.TokenChunkerParam.ImageContextSize = int(v)
}
+ if v, ok := conf["under_cap"].(bool); ok {
+ p.TokenChunkerParam.UnderCap = v
+ }
}
func defaultsToken(p tokenChunkerParam) tokenChunkerParam {
@@ -304,13 +307,18 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
return c.mergeByTokenSize(text, childrenPattern)
}
- parts := splitKeepingDelim(text, delimPattern)
+ parts := splitDroppingDelim(text, delimPattern)
cleaned := make([]string, 0, len(parts))
for _, p := range parts {
- if strings.TrimSpace(p) == "" {
+ // Python's text path keeps only the even-index (text) parts from
+ // _split_text_by_pattern and then .strip()s each one
+ // (token_chunker.py:316-338), so the delimiter is dropped and
+ // surrounding whitespace is trimmed.
+ trimmed := strings.TrimSpace(p)
+ if trimmed == "" {
continue
}
- cleaned = append(cleaned, p)
+ cleaned = append(cleaned, trimmed)
}
if len(cleaned) == 0 {
return emptyOutputs()
@@ -326,7 +334,7 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
// Split-then-merge: split on delimiters, then greedily merge to
// chunk_token_size with optional overlap.
perItem := [][]schema.ChunkDoc{docs}
- merged := mergeByTokenSizeFromJSON(perItem, c.param.ChunkTokenSize, c.param.OverlappedPercent, true)
+ merged := mergeByTokenSizeFromJSON(perItem, c.param.ChunkTokenSize, c.param.OverlappedPercent, true, !c.param.UnderCap)
return chunkOutputs(flatten(merged))
}
@@ -468,6 +476,71 @@ func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int)
return overlap, tokenizeStr(overlap)
}
+// mergeAction is the decision returned by mergeDecision for one incoming unit.
+type mergeAction int
+
+const (
+ // mergeIntoPrev overrides the previous chunk's text with the joined text.
+ mergeIntoPrev mergeAction = iota
+ // startNewChunk appends a new chunk, optionally prefixed with an overlap
+ // slice carved from the previous chunk.
+ startNewChunk
+ // mergeThenClose overrides the previous chunk with the joined text and
+ // forces the NEXT incoming unit to start a brand-new chunk (OVER_CAP
+ // boundary overflow: a chunk may exceed target by at most one unit).
+ mergeThenClose
+)
+
+// mergeDecision computes the merge decision shared by the text and JSON merge
+// paths. prevText is the current chunk, incoming is the next unit, and joinSep
+// is the separator used to project the joined text ("" for the text path, "\n"
+// for the JSON path). target is the token cap.
+//
+// allowBoundaryOverflow=true selects OVER_CAP (Python's default, canonical):
+// when the joined text exceeds target but the incoming unit still fits target,
+// it is merged into the previous chunk and that chunk is then closed
+// (mergeThenClose), forcing the next unit to start a new chunk. An incoming
+// unit that already exceeds target is never merged — it stands alone as its own
+// chunk (Python OVER_CAP: an oversized paragraph is never combined with the
+// previous chunk). allowBoundaryOverflow=false selects UNDER_CAP (strict
+// no-overflow): an overflowing joined text starts a new chunk instead.
+//
+// JSON-only metadata (PDFPositions/Positions/TKNums) is the caller's
+// responsibility; this helper only returns the merged/new text and the action.
+func mergeDecision(prevText, incoming, joinSep string, target int, overlapPct float64, allowBoundaryOverflow bool) (string, mergeAction) {
+ incomingTokens := tokenizeStr(incoming)
+ // An incoming unit that already exceeds target can never be merged; it
+ // stands alone as its own chunk.
+ if incomingTokens > target {
+ return newChunkText(prevText, incoming, target, overlapPct, incomingTokens), startNewChunk
+ }
+ joined := prevText + joinSep + incoming
+ if tokenizeStr(joined) <= target {
+ return joined, mergeIntoPrev
+ }
+ if allowBoundaryOverflow {
+ // OVER_CAP: merge the overflowing unit but close the chunk so the
+ // next unit starts fresh.
+ return joined, mergeThenClose
+ }
+ return newChunkText(prevText, incoming, target, overlapPct, incomingTokens), startNewChunk
+}
+
+// newChunkText returns the text for a fresh chunk started after prevText,
+// prefixing an overlap slice from prevText when one fits within target. It is
+// used both by the UNDER_CAP overflow branch of mergeDecision and by the
+// caller when a previous chunk was closed by an OVER_CAP boundary overflow.
+func newChunkText(prevText, incoming string, target int, overlapPct float64, incomingTokens int) string {
+ if overlapPct > 0 {
+ if overlapText, overlapTokens := computeOverlapPrefix(prevText, overlapPct); overlapTokens > 0 {
+ if overlapTokens+incomingTokens <= target {
+ return overlapText + incoming
+ }
+ }
+ }
+ return incoming
+}
+
// mergeByTokenSize implements exact token-based chunk merging that mirrors
// Python's naive_merge (rag/nlp/__init__.py) after the strict chunk_token_num
// hard-cap fix. It uses tokenizeStr for precise token counting, treats the
@@ -503,6 +576,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r
// addChunk applies the projected-total merge and optional-overlap decision
// to one unit that already fits target.
+ var prevClosed bool
addChunk := func(segment string) {
tnum := tokenizeStr(segment)
if len(cks) == 0 {
@@ -510,27 +584,25 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r
tkns = append(tkns, tnum)
return
}
- merged := cks[len(cks)-1] + segment
- mergedN := tokenizeStr(merged)
- if mergedN <= target {
- cks[len(cks)-1] = merged
- tkns[len(tkns)-1] = mergedN
+ // Previous chunk was closed by an OVER_CAP boundary overflow: the
+ // next unit must start a fresh chunk (with overlap when it fits).
+ if prevClosed {
+ prevClosed = false
+ out := newChunkText(cks[len(cks)-1], segment, target, overlapPct, tnum)
+ cks = append(cks, out)
+ tkns = append(tkns, tokenizeStr(out))
return
}
- newText := segment
- newTokens := tnum
- if overlapPct > 0 {
- overlapText, _ := computeOverlapPrefix(cks[len(cks)-1], overlapPct)
- if overlapText != "" {
- candidate := overlapText + segment
- if candidateTokens := tokenizeStr(candidate); candidateTokens <= target {
- newText = candidate
- newTokens = candidateTokens
- }
- }
+ out, act := mergeDecision(cks[len(cks)-1], segment, "", target, overlapPct, !c.param.UnderCap)
+ switch act {
+ case mergeIntoPrev, mergeThenClose:
+ cks[len(cks)-1] = out
+ tkns[len(tkns)-1] = tokenizeStr(out)
+ prevClosed = act == mergeThenClose
+ case startNewChunk:
+ cks = append(cks, out)
+ tkns = append(tkns, tokenizeStr(out))
}
- cks = append(cks, newText)
- tkns = append(tkns, newTokens)
}
addUnit := func(unit string) {
@@ -560,7 +632,18 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r
parts := sentenceDelimiter.Split(sec, -1)
hadPart := false
for _, part := range parts {
- part = strings.TrimSpace(part)
+ // Keep the raw split fragment, including any inter-line trailing
+ // whitespace. Python's naive_merge builds each unit from
+ // "\n" + sub_sec (naive_merge:1357) where sub_sec retains its
+ // trailing space and is never TrimSpaced — the only post-processing
+ // is dropping the leading empty placeholder (naive_merge:1370-1375),
+ // never per-unit trimming. Trimming here drops that space, so the
+ // overlap prefix carved from the previous chunk (which runs over the
+ // untrimmed segment) loses a character and diverges from Python.
+ // Only genuinely empty fragments are skipped, mirroring naive_merge's
+ // `if not sub_sec` guard. (The final-output TrimSpace only strips a
+ // chunk's own leading/trailing whitespace, not the inter-line space
+ // preserved here.)
if part == "" {
continue
}
@@ -639,7 +722,7 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
// chunks across JSON items into one global token budget. Flatten the
// per-item structure into a single sequence first so the merge is
// global; non-text chunks still break the merge via their CKType.
- attached = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flatten(attached)}, c.param.ChunkTokenSize, c.param.OverlappedPercent, false)
+ attached = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flatten(attached)}, c.param.ChunkTokenSize, c.param.OverlappedPercent, false, !c.param.UnderCap)
}
flat := flatten(attached)
@@ -656,7 +739,7 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
// the merge paths may carry @@...## markers that must not leak into
// indexed/embedded chunk text. Crop above reads positions, not text,
// so the ordering is safe.
- m.Text = removeTag(strings.TrimSpace(m.Text))
+ m.Text = removeTag(m.Text)
if m.Text == "" {
continue
}
@@ -878,7 +961,12 @@ func takeFromStart(text string, tokens int) string {
// hard cap (rag/nlp/__init__.py after the strict chunk_token_num fix).
// Oversized text units are sub-split via splitOversizedUnit before merge;
// overlap is applied only when overlap+segment still fits the budget.
-func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overlappedPct float64, subSplitOversize bool) [][]schema.ChunkDoc {
+//
+// allowBoundaryOverflow selects the merge strategy: true = OVER_CAP (Python's
+// canonical default, a chunk may exceed the target by at most one incoming
+// unit), false = UNDER_CAP (never exceed the target; a projected overflow
+// starts a fresh chunk). The TokenChunker threads its UnderCap param here.
+func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overlappedPct float64, subSplitOversize bool, allowBoundaryOverflow bool) [][]schema.ChunkDoc {
// overlappedPct is a [0,100] percentage. Clamp defensively because this
// helper is also exercised directly by tests.
if overlappedPct < 0 {
@@ -895,6 +983,7 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over
// addTextChunk applies the projected-total merge / overlap-drop
// decision for one text unit that already fits chunkTokens.
+ var prevClosed bool
addTextChunk := func(ck schema.ChunkDoc) {
tk := intValue(ck.TKNums)
if tk <= 0 {
@@ -903,7 +992,11 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over
}
if len(merged) == 0 || merged[len(merged)-1].CKType != "text" {
// First text chunk, or first text after a non-text chunk:
- // no prior text to overlap with.
+ // no prior text to overlap with. A stale OVER_CAP boundary
+ // overflow (prevClosed) from a previous text chunk must be
+ // cleared here, otherwise the next text chunk would be wrongly
+ // forced into a fresh chunk instead of merging with this one.
+ prevClosed = false
merged = append(merged, cloneChunkDoc(ck))
return
}
@@ -911,31 +1004,48 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over
// Empty previous text: assign incoming text directly
// (diff Chunker-2.11 / token_chunker.py:236-239).
if prev.Text == "" {
+ // Invariant: every path that emits a chunk WITHOUT consuming
+ // the OVER_CAP boundary-overflow flag (prevClosed) must clear
+ // it, so a stale flag can never leak into a later chunk. The
+ // early-return above already does this for the first-text /
+ // after-non-text case; do the same here. (Today this branch is
+ // only reached with an empty prev, and mergeThenClose always
+ // leaves a non-empty prev, so prevClosed cannot actually be
+ // true here — the reset is defensive and keeps the invariant
+ // explicit against future refactors.)
+ prevClosed = false
prev.Text = ck.Text
prev.TKNums = intPtr(tk)
prev.PDFPositions = extendRawJSONArray(prev.PDFPositions, ck.PDFPositions)
prev.Positions = extendRawJSONArray(prev.Positions, ck.Positions)
return
}
+ // Previous chunk was closed by an OVER_CAP boundary overflow:
+ // the next unit must start a fresh chunk (with overlap when it
+ // fits). Coordinates stay on the new chunk only.
+ if prevClosed {
+ prevClosed = false
+ cp := cloneChunkDoc(ck)
+ cp.Text = newChunkText(prev.Text, ck.Text, chunkTokens, overlappedPct, tk)
+ cp.TKNums = intPtr(tokenizeStr(cp.Text))
+ merged = append(merged, cp)
+ return
+ }
// Proactive projected-total merge (joined with "\n").
- joined := prev.Text + "\n" + ck.Text
- joinedN := tokenizeStr(joined)
- if joinedN <= chunkTokens {
- prev.Text = joined
- prev.TKNums = intPtr(joinedN)
+ out, act := mergeDecision(prev.Text, ck.Text, "\n", chunkTokens, overlappedPct, allowBoundaryOverflow)
+ switch act {
+ case mergeIntoPrev, mergeThenClose:
+ prev.Text = out
+ prev.TKNums = intPtr(tokenizeStr(out))
prev.PDFPositions = extendRawJSONArray(prev.PDFPositions, ck.PDFPositions)
prev.Positions = extendRawJSONArray(prev.Positions, ck.Positions)
- return
- }
- // Start a new chunk; apply overlap only when it still fits.
- cp := cloneChunkDoc(ck)
- if overlappedPct > 0 {
- if overlapText, overlapTokens := computeOverlapPrefix(prev.Text, overlappedPct); overlapTokens > 0 && overlapTokens+tk <= chunkTokens {
- cp.Text = overlapText + cp.Text
- cp.TKNums = intPtr(tokenizeStr(cp.Text))
- }
+ prevClosed = act == mergeThenClose
+ case startNewChunk:
+ cp := cloneChunkDoc(ck)
+ cp.Text = out
+ cp.TKNums = intPtr(tokenizeStr(out))
+ merged = append(merged, cp)
}
- merged = append(merged, cp)
}
for _, ck := range chunks {
diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.go
index eb4d659601..3312b9acf9 100644
--- a/internal/ingestion/component/chunker/token_batch1_test.go
+++ b/internal/ingestion/component/chunker/token_batch1_test.go
@@ -59,44 +59,169 @@ func TestSentenceDelimiterMatchesBangAndQuestion(t *testing.T) {
// join exceeds the budget — so the first unit must already sit near the
// budget and the second unit must not fit alongside it.
func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {
- // Size a and b so:
- // - each unit alone fits the budget (no atom-split),
- // - the projected join exceeds the budget (forces a new chunk),
- // - overlap+b still fits (so the overlap path is exercised).
- aText := strings.Repeat("word ", 20) + "@@1\t2.3## tail"
- bText := "body"
- aN, bN := tokenizeStr(aText), tokenizeStr(bText)
- joinedN := tokenizeStr(aText + "\n" + bText)
- // Budget just below the join so a and b cannot merge, but each alone fits.
- budget := joinedN - 1
+ // OVER_CAP (Python's canonical default) merges an overflowing unit into
+ // the previous chunk and then closes it, so a chunk can exceed budget by
+ // at most one unit. To exercise the overlap path we use three units:
+ // - a carries a parser tag and fits the budget alone,
+ // - b makes a+b overflow, so a+b merge-then-close into chunk0,
+ // - c starts a fresh chunk (prevClosed) with an overlap prefix from
+ // chunk0, which is where the tag-stripping must hold.
+ aText := strings.Repeat("word ", 18) + "@@1\t2.3## tail"
+ bText := strings.Repeat("word ", 18)
+ cText := strings.Repeat("word ", 6)
+ aN, bN, cN := tokenizeStr(aText), tokenizeStr(bText), tokenizeStr(cText)
+ joinedAB := tokenizeStr(aText + "\n" + bText)
+ // Budget just below the a+b join so a and b cannot merge without
+ // overflowing, but a alone and c alone fit, and an overlap prefix carved
+ // from chunk0 fits ahead of c (overlap path is exercised on chunk 1).
+ budget := joinedAB - 1
if budget < aN {
budget = aN
}
- if budget < bN {
- budget = bN
+ if budget < cN {
+ budget = cN
}
- if joinedN <= budget {
- t.Fatalf("could not derive tight budget (a=%d b=%d joined=%d budget=%d)", aN, bN, joinedN, budget)
+ if joinedAB <= budget {
+ t.Fatalf("could not derive tight budget (a=%d b=%d joined=%d budget=%d)", aN, bN, joinedAB, budget)
}
items := [][]schema.ChunkDoc{
{
{Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},
{Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},
+ {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},
},
}
- got := mergeByTokenSizeFromJSON(items, budget, 30.0, true)
+ got := mergeByTokenSizeFromJSON(items, budget, 30.0, true, true)
merged := got[0]
if len(merged) != 2 {
- t.Fatalf("want 2 merged chunks (overlap path), got %d (a=%d b=%d budget=%d)", len(merged), aN, bN, budget)
+ t.Fatalf("want 2 chunks (overflow-closed + overlap chunk), got %d (a=%d b=%d c=%d budget=%d)", len(merged), aN, bN, cN, budget)
}
- // The overlap prefix is prepended to the SECOND chunk. The original
- // first chunk legitimately keeps its own parser tag; only the overlap
- // region (merged[1]) must be tag-free. .
+ // The overlap prefix must actually be prepended to chunk 1; otherwise the
+ // test would pass even if prevClosed started c without any overlap.
+ overlap, _ := computeOverlapPrefix(merged[0].Text, 30.0)
+ if overlap == "" {
+ t.Fatal("expected a non-empty overlap prefix")
+ }
+ if !strings.HasPrefix(merged[1].Text, overlap) {
+ t.Errorf("chunk 1 missing overlap prefix %q: %q", overlap, merged[1].Text)
+ }
+ // The overlap prefix is prepended to the SECOND chunk. The first chunk
+ // legitimately keeps its own parser tag; only the overlap region
+ // (merged[1]) must be tag-free.
if strings.Contains(merged[1].Text, "@@") || strings.Contains(merged[1].Text, "##") {
t.Errorf("overlap prefix leaked parser tag into chunk 1: %q", merged[1].Text)
}
if n := tokenizeStr(merged[1].Text); n > budget {
- t.Errorf("overlap pushed second chunk over budget: tokens=%d", n)
+ t.Errorf("overlap pushed second chunk over budget: tokens=%d (cap=%d)", n, budget)
+ }
+}
+
+// TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsPrevClosed is a TDD
+// regression test for the OVER_CAP boundary-overflow flag (prevClosed) leaking
+// across a non-text chunk. mergeByTokenSizeFromJSON must reset prevClosed when
+// the previous merged chunk is non-text; otherwise the first text chunk after a
+// non-text chunk carries the stale flag and forces the NEXT text chunk to start
+// a fresh chunk even though it should merge with its predecessor.
+//
+// Sequence: T1, T2 (overflow-merge-close into chunk0), N (non-text), T3, T4.
+// With a correct reset, T3 is the first text after N (new chunk) and T4 merges
+// back into T3 -> 3 chunks total. With the bug, prevClosed survives the N
+// boundary and T4 is wrongly forced into its own chunk -> 4 chunks.
+func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsPrevClosed(t *testing.T) {
+ t1 := strings.Repeat("word ", 18)
+ t2 := strings.Repeat("word ", 18)
+ t3 := strings.Repeat("word ", 9)
+ t4 := strings.Repeat("word ", 9)
+ t1N, t2N := tokenizeStr(t1), tokenizeStr(t2)
+ joined12 := tokenizeStr(t1 + "\n" + t2)
+ // Budget just below the T1+T2 join so T1 and T2 cannot merge without
+ // overflowing, forcing mergeThenClose on chunk0 (prevClosed=true).
+ budget := joined12 - 1
+ if budget < t1N {
+ budget = t1N
+ }
+ t34 := tokenizeStr(t3 + "\n" + t4)
+ if t34 > budget {
+ t.Fatalf("T3+T4 must fit budget to exercise the merge; t34=%d budget=%d", t34, budget)
+ }
+ if joined12 <= budget {
+ t.Fatalf("could not derive tight budget (t1=%d t2=%d joined=%d budget=%d)", t1N, t2N, joined12, budget)
+ }
+
+ items := [][]schema.ChunkDoc{
+ {
+ {Text: t1, DocType: "text", CKType: "text", TKNums: intPtr(t1N)},
+ {Text: t2, DocType: "text", CKType: "text", TKNums: intPtr(t2N)},
+ {Text: "[image]", DocType: "image", CKType: "image", TKNums: intPtr(1)},
+ {Text: t3, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(t3))},
+ {Text: t4, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(t4))},
+ },
+ }
+ got := mergeByTokenSizeFromJSON(items, budget, 0.0, true, true)
+ merged := got[0]
+ // Expect: chunk0 (T1+T2, overflow-closed), N (non-text), chunk1 (T3+T4 merged).
+ if len(merged) != 3 {
+ var texts []string
+ for _, c := range merged {
+ texts = append(texts, c.CKType+":"+c.Text)
+ }
+ t.Fatalf("want 3 chunks (overflow-closed + non-text + merged), got %d: %v", len(merged), texts)
+ }
+ // Lock the overflow-closed precondition: T1+T2 must have merged into
+ // chunk0 with prevClosed set, otherwise the later assertions could pass
+ // without exercising the boundary-overflow path at all.
+ if merged[0].Text != t1+"\n"+t2 {
+ t.Errorf("chunk[0] should be the overflow-closed T1+T2 chunk: got %q", merged[0].Text)
+ }
+ if merged[1].CKType != "image" {
+ t.Errorf("chunk[1] should be the non-text image chunk, got CKType=%q text=%q", merged[1].CKType, merged[1].Text)
+ }
+ wantMerged := t3 + "\n" + t4
+ if merged[2].Text != wantMerged {
+ t.Errorf("chunk[2] should merge T3+T4: want %q got %q", wantMerged, merged[2].Text)
+ }
+}
+
+// TestMergeByTokenSizeFromJSON_UnderCapNoOverflow exercises the UNDER_CAP
+// strategy (allowBoundaryOverflow=false): a projected join that would exceed
+// the target must start a fresh chunk instead of merging-then-closing. This is
+// the seam that lets Go follow Python's no-overflow (UNDER_CAP) strategy. Under
+// OVER_CAP the same input merges a+b and overflows chunk0; here a, b, c must
+// stay as three separate chunks, each within budget.
+func TestMergeByTokenSizeFromJSON_UnderCapNoOverflow(t *testing.T) {
+ aText := strings.Repeat("word ", 18)
+ bText := strings.Repeat("word ", 18)
+ cText := strings.Repeat("word ", 18)
+ aN, bN, cN := tokenizeStr(aText), tokenizeStr(bText), tokenizeStr(cText)
+ joinedAB := tokenizeStr(aText + "\n" + bText)
+ // Budget just below the a+b join so a and b cannot merge without
+ // overflowing; a alone and c alone fit.
+ budget := joinedAB - 1
+ if budget < aN {
+ budget = aN
+ }
+ if budget < cN {
+ budget = cN
+ }
+ if joinedAB <= budget {
+ t.Fatalf("could not derive tight budget (a=%d b=%d joined=%d budget=%d)", aN, bN, joinedAB, budget)
+ }
+ items := [][]schema.ChunkDoc{
+ {
+ {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},
+ {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},
+ {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},
+ },
+ }
+ got := mergeByTokenSizeFromJSON(items, budget, 0.0, true, false)
+ merged := got[0]
+ if len(merged) != 3 {
+ t.Fatalf("UNDER_CAP want 3 chunks (a, b, c separate), got %d", len(merged))
+ }
+ for i, ck := range merged {
+ if n := tokenizeStr(ck.Text); n > budget {
+ t.Errorf("UNDER_CAP chunk %d exceeds target: tokens=%d (cap=%d)", i, n, budget)
+ }
}
}
@@ -130,12 +255,12 @@ func clampOverlapFixture() [][]schema.ChunkDoc {
}
func TestMergeByTokenSizeFromJSON_ClampsOverlappedPct(t *testing.T) {
- at100 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 100, true)
+ at100 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 100, true, true)
if at100 == nil || len(at100) == 0 {
t.Fatalf("overlappedPct=100: nil/empty result")
}
- at150 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 150, true)
- atHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 1e300, true)
+ at150 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 150, true, true)
+ atHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 1e300, true, true)
if !reflect.DeepEqual(at100, at150) {
t.Errorf("overlappedPct=150 should clamp to 100; output differs from 100")
}
@@ -143,12 +268,12 @@ func TestMergeByTokenSizeFromJSON_ClampsOverlappedPct(t *testing.T) {
t.Errorf("overlappedPct=1e300 should clamp to 100; output differs from 100")
}
- at0 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 0, true)
+ at0 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 0, true, true)
if at0 == nil || len(at0) == 0 {
t.Fatalf("overlappedPct=0: nil/empty result")
}
- atNeg := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -5, true)
- atNegHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -1e300, true)
+ atNeg := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -5, true, true)
+ atNegHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -1e300, true, true)
if !reflect.DeepEqual(at0, atNeg) {
t.Errorf("overlappedPct=-5 should clamp to 0; output differs from 0")
}
@@ -169,7 +294,7 @@ func TestMergeByTokenSizeFromJSON_EmptyPrevKeepsChunk(t *testing.T) {
{Text: "keepme", DocType: "text", CKType: "text", TKNums: intPtr(5)},
},
}
- got := mergeByTokenSizeFromJSON(items, 128, 0, true)
+ got := mergeByTokenSizeFromJSON(items, 128, 0, true, true)
merged := got[0]
if len(merged) != 1 {
t.Fatalf("want 1 merged chunk, got %d", len(merged))
diff --git a/internal/ingestion/component/chunker/token_overlap_test.go b/internal/ingestion/component/chunker/token_overlap_test.go
new file mode 100644
index 0000000000..aa14254269
--- /dev/null
+++ b/internal/ingestion/component/chunker/token_overlap_test.go
@@ -0,0 +1,89 @@
+package chunker
+
+import (
+ "context"
+ "strings"
+ "testing"
+)
+
+// TestTokenChunker_TextOverlapPreservesInterLineSpace pins the fix for the
+// token-text-overlap-split divergence (parity rule token-text-overlap-split).
+//
+// The function under test is TokenChunkerComponent.mergeByTokenSize
+// (token.go), which the text path reaches ONLY when there is no active
+// delimiter: with empty delimiters, invokeTextPayload routes to
+// mergeByTokenSize (token.go:303-304); with a non-empty delimiter such as
+// ["\n"] it instead routes to mergeByTokenSizeFromJSON, a different code path
+// that is unaffected by this fix. This test therefore drives the component
+// through Invoke with delimiters: []string{} so it actually lands on the
+// patched function.
+//
+// mergeByTokenSize splits the payload on sentence delimiters and merges per
+// token budget, carrying an overlap prefix from the previous chunk.
+// Python's naive_merge builds each unit from "\n" + sub_sec where sub_sec
+// retains its trailing inter-line whitespace (naive_merge:1357 — it never
+// TrimSpaces a unit; the only post-processing is dropping the leading empty
+// placeholder at naive_merge:1370-1375). The Go merge must do the same: if it
+// TrimSpaces each split fragment, the trailing space of a line is dropped, the
+// overlap prefix carved from the previous (untrimmed) chunk loses that
+// character, and every following chunk head diverges from Python by one
+// character.
+//
+// This test locks the exact chunk texts against the Python reference
+// (rag/nlp.naive_merge with chunk_token_size=64, delimiters=[],
+// overlapped_percent=0.1).
+func TestTokenChunker_TextOverlapPreservesInterLineSpace(t *testing.T) {
+ const (
+ alpha = "alpha "
+ beta = "beta "
+ gamma = "gamma "
+ )
+ text := strings.Repeat(alpha, 40) + "\n" +
+ strings.Repeat(beta, 40) + "\n" +
+ strings.Repeat(gamma, 40)
+
+ c, err := NewTokenChunker(map[string]any{
+ "chunk_token_size": float64(64),
+ "delimiters": []string{}, // routes Invoke -> mergeByTokenSize (the patched function)
+ "overlapped_percent": 0.1,
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "t", "output_format": "text", "text": text,
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ // OVER_CAP (Python's canonical default) overflow-merges the first two
+ // groups (alpha+beta) into chunk0 and then starts a fresh, overlap-prefixed
+ // chunk1 for gamma.
+ if len(chunks) != 2 {
+ t.Fatalf("expected 2 chunks, got %d", len(chunks))
+ }
+ got := make([]string, len(chunks))
+ for i, ck := range chunks {
+ s, _ := ck["text"].(string)
+ got[i] = s
+ }
+
+ // The regression this test pins: the inter-line trailing space of every
+ // line must be preserved across a merge/overlap boundary (Python emits
+ // "alpha \nbeta", not "alpha\nbeta"). Under OVER_CAP the alpha→beta
+ // boundary is inside chunk0 and the beta→gamma boundary is inside the
+ // overlap-prefixed chunk1.
+ if !strings.Contains(got[0], "alpha \nbeta") {
+ t.Errorf("chunk[0] lost inter-line space before newline: %q", got[0])
+ }
+ if strings.Contains(got[0], "alpha\nbeta") {
+ t.Errorf("chunk[0] has space-less boundary (bug present): %q", got[0])
+ }
+ if !strings.Contains(got[1], "beta \ngamma") {
+ t.Errorf("chunk[1] lost inter-line space before newline: %q", got[1])
+ }
+ if strings.Contains(got[1], "beta\ngamma") {
+ t.Errorf("chunk[1] has space-less boundary (bug present): %q", got[1])
+ }
+}
diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.go
index d1eb951c7d..0f6b501a46 100644
--- a/internal/ingestion/component/chunker/token_pdfpos_test.go
+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go
@@ -38,7 +38,7 @@ func TestMergeByTokenSizeFromJSON_ExtendsPDFPositions(t *testing.T) {
{Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posB},
},
}
- got := mergeByTokenSizeFromJSON(items, 128, 0, true)
+ got := mergeByTokenSizeFromJSON(items, 128, 0, true, true)
merged := got[0]
if len(merged) != 1 {
t.Fatalf("want 1 merged chunk, got %d", len(merged))
@@ -63,7 +63,7 @@ func TestMergeByTokenSizeFromJSON_ExtendsPositions(t *testing.T) {
{Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},
},
}
- got := mergeByTokenSizeFromJSON(items, 128, 0, true)
+ got := mergeByTokenSizeFromJSON(items, 128, 0, true, true)
combined := string(got[0][0].Positions)
if !strings.Contains(combined, "1,2,3") || !strings.Contains(combined, "4,5,6") {
t.Errorf("merged chunk dropped/omitted `positions`: %s", combined)
@@ -103,7 +103,7 @@ func TestMergeByTokenSizeFromJSON_PositionsDecodeToMatrix(t *testing.T) {
{Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},
},
}
- got := mergeByTokenSizeFromJSON(items, 128, 0, true)
+ got := mergeByTokenSizeFromJSON(items, 128, 0, true, true)
m := got[0][0].ToMap()
raw, ok := m["positions"]
if !ok {
diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.go
index 0f99156df4..75eff6a4b5 100644
--- a/internal/ingestion/component/chunker/token_strict_cap_test.go
+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go
@@ -108,15 +108,19 @@ func TestMergeByTokenSizeFromJSON_StrictCapNoOvershoot(t *testing.T) {
Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),
})
}
- got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 0, true)
+ got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 0, true, true)
merged := got[0]
if len(merged) < 3 {
t.Fatalf("want >=3 chunks, got %d", len(merged))
}
+ // OVER_CAP (Python's canonical default) permits a chunk to exceed budget
+ // by at most one incoming unit: a chunk is closed right after the
+ // overflowing merge, so it can hold prev (<= budget) + one unit (<= budget).
+ unit := tokenizeStr(sections[0].Text)
for i, ck := range merged {
n := tokenizeStr(ck.Text)
- if n > budget {
- t.Errorf("chunk %d exceeds budget: tokens=%d text_len=%d", i, n, len(ck.Text))
+ if n > budget+unit {
+ t.Errorf("chunk %d exceeds budget by more than one unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)
}
}
}
@@ -131,10 +135,17 @@ func TestMergeByTokenSizeFromJSON_OverlapDroppedAtOverflow(t *testing.T) {
Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),
})
}
- got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 20, true)
+ got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 20, true, true)
+ unit := tokenizeStr(sections[0].Text)
for i, ck := range got[0] {
- if n := tokenizeStr(ck.Text); n > budget {
- t.Errorf("chunk %d exceeds budget with overlap: tokens=%d", i, n)
+ // OVER_CAP allows one boundary overflow (prev + one unit). The JSON
+ // path joins units with "\n" and cl100k is not additive across joins,
+ // so an overflow-closed chunk can land a little above budget+unit;
+ // allow one extra unit of slack for the separators + cl100k delta.
+ // Overlap is only ever prepended when it still fits budget, so the
+ // only chunks that exceed budget are the overflow-closed ones.
+ if n := tokenizeStr(ck.Text); n > budget+2*unit {
+ t.Errorf("chunk %d exceeds budget+two-units with overlap: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)
}
}
}
@@ -146,20 +157,23 @@ func TestMergeByTokenSizeFromJSON_OversizedUnitIsSubSplit(t *testing.T) {
items := [][]schema.ChunkDoc{{
{Text: long, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(long))},
}}
- got := mergeByTokenSizeFromJSON(items, budget, 0, true)
+ got := mergeByTokenSizeFromJSON(items, budget, 0, true, true)
if len(got[0]) < 2 {
t.Fatalf("oversized unit must yield multiple chunks, got %d", len(got[0]))
}
// cl100k is not additive across whitespace joins: token(a)+token(b) can be
// one less than token(a+b), so the running-sum flush used by both Python's
// _split_oversized_unit and the aligned Go port can leave a piece exactly
- // one token over the nominal budget. The invariant we defend here is that
- // the oversized unit is sub-split (not collapsed into one chunk), not a
- // byte-exact cap — matching the Python reference.
- const slack = 1
+ // one token over the nominal budget (each sub-split piece <= budget+1).
+ // OVER_CAP then merges at most two such pieces into one chunk before
+ // closing it, so the invariant we defend is that no chunk exceeds
+ // 2*(budget+1): the oversized unit is sub-split (not collapsed into one
+ // chunk) and at most one boundary overflow is allowed — matching the
+ // Python reference.
+ const slack = 2 * (budget + 1)
for i, ck := range got[0] {
- if n := tokenizeStr(ck.Text); n > budget+slack {
- t.Errorf("chunk %d exceeds budget by more than cl100k slack: tokens=%d (cap=%d)", i, n, budget)
+ if n := tokenizeStr(ck.Text); n > slack {
+ t.Errorf("chunk %d exceeds 2*(budget+1): tokens=%d (cap=%d)", i, n, budget)
}
}
}
@@ -167,6 +181,7 @@ func TestMergeByTokenSizeFromJSON_OversizedUnitIsSubSplit(t *testing.T) {
func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {
// End-to-end text path: long multi-paragraph input under a tight budget.
const budget = 40
+ unit := tokenizeStr(strings.TrimSpace(strings.Repeat("word ", 15)))
var b strings.Builder
for i := 0; i < 30; i++ {
b.WriteString(strings.TrimSpace(strings.Repeat("word ", 15)))
@@ -186,11 +201,71 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {
t.Fatalf("want multiple chunks, got %d", len(chunks))
}
for i, ck := range chunks {
+ text, _ := ck["text"].(string)
+ // OVER_CAP allows one boundary overflow (prev + one unit).
+ if n := tokenizeStr(text); n > budget+unit {
+ t.Errorf("chunk %d exceeds budget+one-unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)
+ }
+ }
+}
+
+func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {
+ // UNDER_CAP (under_cap=true) must never let a chunk exceed the token
+ // target: a projected join that would overflow starts a fresh chunk
+ // instead of merge-then-close (OVER_CAP). This exercises the seam that
+ // lets Go follow Python's no-overflow (UNDER_CAP) strategy without
+ // changing the default (OVER_CAP) behavior.
+ const sentence = "word word word word word word word word word word word word word " // 12 words
+ sentenceN := tokenizeStr(sentence)
+ budget := sentenceN*4 + 2 // four sentences fit, five overflow.
+ if budget < sentenceN*2 {
+ budget = sentenceN * 2
+ }
+ var b strings.Builder
+ for i := 0; i < 12; i++ {
+ b.WriteString(sentence)
+ b.WriteString("! ")
+ }
+
+ run := func(underCap bool) []map[string]any {
+ comp, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "token_size",
+ "chunk_token_size": budget,
+ "under_cap": underCap,
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil)
+ chunks, _ := out["chunks"].([]map[string]any)
+ return chunks
+ }
+
+ respect := run(true)
+ if len(respect) < 2 {
+ t.Fatalf("UNDER_CAP: want multiple chunks, got %d", len(respect))
+ }
+ for i, ck := range respect {
text, _ := ck["text"].(string)
if n := tokenizeStr(text); n > budget {
- t.Errorf("chunk %d exceeds budget: tokens=%d", i, n)
+ t.Errorf("UNDER_CAP chunk %d exceeds target: tokens=%d (cap=%d)", i, n, budget)
+ }
+ }
+
+ // Control: OVER_CAP (default) must overflow on the same input, proving the
+ // toggle changes behavior rather than being a no-op.
+ over := run(false)
+ overflowed := false
+ for _, ck := range over {
+ text, _ := ck["text"].(string)
+ if tokenizeStr(text) > budget {
+ overflowed = true
+ break
}
}
+ if !overflowed {
+ t.Errorf("OVER_CAP control produced no overflow on input that UNDER_CAP keeps within budget; toggle may be a no-op")
+ }
}
func TestMergeByTokenSize_UnbrokenAtomStrictCap(t *testing.T) {
@@ -224,8 +299,10 @@ func TestMergeByTokenSize_UnbrokenAtomStrictCap(t *testing.T) {
for i, ck := range chunks {
s, _ := ck["text"].(string)
joined.WriteString(s)
- if n := tokenizeStr(s); n > budget {
- t.Errorf("chunk %d exceeds budget: tokens=%d text=%q", i, n, s)
+ // Sub-split pieces are <= budget+1; OVER_CAP merges at most two before
+ // closing, so a chunk can reach 2*(budget+1).
+ if n := tokenizeStr(s); n > 2*(budget+1) {
+ t.Errorf("chunk %d exceeds 2*(budget+1): tokens=%d text=%q", i, n, s)
}
}
// mergeByTokenSize prefixes "\n" on sections; stripping newlines recovers
@@ -237,6 +314,7 @@ func TestMergeByTokenSize_UnbrokenAtomStrictCap(t *testing.T) {
func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {
const budget = 32
+ unit := tokenizeStr(strings.TrimSpace(strings.Repeat("alpha ", 12)))
var b strings.Builder
for i := 0; i < 20; i++ {
b.WriteString(strings.TrimSpace(strings.Repeat("alpha ", 12)))
@@ -266,8 +344,9 @@ func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {
}
for i, ck := range chunks {
text, _ := ck["text"].(string)
- if n := tokenizeStr(text); n > budget {
- t.Errorf("chunk %d exceeds budget: tokens=%d", i, n)
+ // OVER_CAP allows one boundary overflow (prev + one unit).
+ if n := tokenizeStr(text); n > budget+unit {
+ t.Errorf("chunk %d exceeds budget+one-unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)
}
}
}
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index c3cdff1e25..85588aa417 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -132,10 +132,10 @@ func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) {
t.Errorf("chunk[%d] is the bare delimiter %q", i, text)
}
}
- if got, want := chunks[0]["text"], "alpha section\n666"; got != want {
+ if got, want := chunks[0]["text"], "alpha section"; got != want {
t.Errorf("chunk[0] text = %q, want %q", got, want)
}
- if got, want := chunks[1]["text"], "\nbeta section"; got != want {
+ if got, want := chunks[1]["text"], "beta section"; got != want {
t.Errorf("chunk[1] text = %q, want %q", got, want)
}
}
diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go
index 826fdceabd..541a8c80c7 100644
--- a/internal/ingestion/component/schema/chunker.go
+++ b/internal/ingestion/component/schema/chunker.go
@@ -203,6 +203,17 @@ type TokenChunkerParam struct {
// ImageContextSize is the number of surrounding tokens to attach
// to image chunks. 0 disables.
ImageContextSize int `json:"image_context_size"`
+
+ // UnderCap selects the merge strategy when greedily accumulating
+ // adjacent units (token_chunker UNDER_CAP vs OVER_CAP).
+ // - false (default): OVER_CAP — mirrors Python's canonical default.
+ // A chunk may exceed the token target by at most one incoming unit
+ // (a boundary overflow then closes the chunk).
+ // - true: UNDER_CAP — strictly never exceed the target; when the
+ // projected join would overflow, start a fresh chunk instead.
+ // This is the seam that lets Go follow Python's no-overflow strategy
+ // without changing the default behavior.
+ UnderCap bool `json:"under_cap"`
}
// Defaults returns the Python default TokenChunkerParam.