RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): decide token merge on running sum, not re-tokenized join (#17948)
Jackcdef80455511
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 10fa83510a..2531ebf9eb 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -398,7 +398,10 @@ const (// mergeDecision computes the merge decision shared by the text and JSON merge// paths. prevText is the current chunk, incoming is the next unit, and joinSep// is the separator used to project the joined text ("" for the text path, "\n"-// for the JSON path). target is the token cap.+// for the JSON path). target is the token cap. prevTokens is the running sum of+// per-unit token counts already accumulated into prevText; incomingTokens is+// the token count of incoming, counted the same way the calling path counts a+// unit.//// strategy selects the merge strategy (schema.MergeStrategy), mirroring// Python's MergeStrategy:@@ -412,17 +415,40 @@ const (// - MergeUnderCap (Python UNDER_CAP, strict no-overflow): an overflowing// joined text starts a new chunk instead.//+// The merge decision is made on the RUNNING SUM of per-unit token counts+// (prevTokens + incomingTokens), NEVER on tokenizeStr(joined). BPE+// tokenization is non-additive across joinSep ("\n"), so re-tokenizing the+// joined string disagreed with Python's per-paragraph size() sum and shifted+// every downstream boundary by a line — and, once the budget is small enough,+// changed the chunk count. Both Python references accumulate a running sum of+// per-unit counts: rag/nlp/__init__.py:_merge_paragraph_groups+// (size("\n" + sub_sec)) and rag/flow/chunker/token_chunker.py:+// _merge_text_chunks_by_token_size (tk_nums += current["tk_nums"]). The joined+// text is still returned as the merged content.+//// JSON-only metadata (PDFPositions/Positions/TKNums) is the caller's// responsibility; this helper only returns the merged/new text and the action.-func mergeDecision(prevText, incoming, joinSep string, target int, overlapPct float64, strategy schema.MergeStrategy) (string, mergeAction) {- incomingTokens := tokenizeStr(incoming)+//+// Note on the JSON path: this helper decides on the running sum, but it still+// uses the OVER_CAP strategy (merge-then-close on overflow). The Python JSON+// reference (rag/flow/chunker/token_chunker.py:_merge_text_chunks_by_token_size)+// decides on prev_tk_nums > threshold instead, so JSON parity is only partially+// addressed here: the join-string re-tokenization is gone but the merge+// STRATEGY difference remains. The TKNums accounting contract is pinned by+// TestMergeByTokenSizeFromJSON_TKNumsConsistency.+func mergeDecision(prevText, incoming, joinSep string, target int, overlapPct float64, strategy schema.MergeStrategy, prevTokens, incomingTokens int) (string, mergeAction) {// An incoming unit that already exceeds target can never be merged; it// stands alone as its own chunk.if incomingTokens > target {return newChunkText(prevText, incoming, target, overlapPct, incomingTokens), startNewChunk}joined := prevText + joinSep + incoming- if tokenizeStr(joined) <= target {+ // Faithful to Python: decide on the running sum of per-unit token counts,+ // never on the BPE count of the re-joined string, which is non-additive+ // across joinSep ("\n") and therefore disagrees with Python's per-paragraph+ // size() sum, shifting every downstream boundary by a line (and, on a small+ // enough budget, changing the chunk count). See the function doc.+ if prevTokens+incomingTokens <= target {return joined, mergeIntoPrev}if strategy == schema.MergeOverCap {@@ -503,11 +529,13 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *rtkns = append(tkns, tokenizeStr(out))return}- out, act := mergeDecision(cks[len(cks)-1], segment, "", target, overlapPct, c.param.MergeStrategy())+ out, act := mergeDecision(cks[len(cks)-1], segment, "", target, overlapPct, c.param.MergeStrategy(), tkns[len(tkns)-1], tnum)switch act {case mergeIntoPrev, mergeThenClose:cks[len(cks)-1] = out- tkns[len(tkns)-1] = tokenizeStr(out)+ // Maintain the running sum of per-paragraph token counts so the+ // next merge decision matches Python's size() sum.+ tkns[len(tkns)-1] += tnumprevClosed = act == mergeThenClosecase startNewChunk:cks = append(cks, out)@@ -930,22 +958,33 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overprevClosed = falsecp := cloneChunkDoc(ck)cp.Text = newChunkText(prev.Text, ck.Text, chunkTokens, overlappedPct, tk)+ // Boundary path: TKNums is the RE-TOKENIZED new chunk text (which+ // may include an overlap prefix and the "\n" joins), NOT the+ // running sum used on the merge path above. With overlap > 0 this+ // mixes the actual text count into the otherwise-running-sum+ // accounting, so it can diverge from Python; pinned by+ // TestMergeByTokenSizeFromJSON_TKNumsConsistency.cp.TKNums = intPtr(tokenizeStr(cp.Text))merged = append(merged, cp)return}// Proactive projected-total merge (joined with "\n").- out, act := mergeDecision(prev.Text, ck.Text, "\n", chunkTokens, overlappedPct, strategy)+ out, act := mergeDecision(prev.Text, ck.Text, "\n", chunkTokens, overlappedPct, strategy, intValue(prev.TKNums), tk)switch act {case mergeIntoPrev, mergeThenClose:prev.Text = out- prev.TKNums = intPtr(tokenizeStr(out))+ // Maintain the running sum of upstream tk_nums so the next+ // merge decision matches Python's tk_nums += current["tk_nums"].+ prev.TKNums = intPtr(intValue(prev.TKNums) + tk)prev.PDFPositions = extendRawJSONArray(prev.PDFPositions, ck.PDFPositions)prev.Positions = extendRawJSONArray(prev.Positions, ck.Positions)prevClosed = act == mergeThenClosecase startNewChunk:cp := cloneChunkDoc(ck)cp.Text = out+ // Boundary path: TKNums is the RE-TOKENIZED new chunk text, not the+ // running sum (see the prevClosed branch above for why this is a+ // deliberate, divergence-prone accounting choice).cp.TKNums = intPtr(tokenizeStr(out))merged = append(merged, cp)}diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.goindex 8df8139fa5..3dabff167f 100644--- a/internal/ingestion/component/chunker/token_batch1_test.go+++ b/internal/ingestion/component/chunker/token_batch1_test.go@@ -70,19 +70,21 @@ func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {bText := strings.Repeat("word ", 18)cText := strings.Repeat("word ", 6)aN, bN, cN := tokenizeStr(aText), tokenizeStr(bText), tokenizeStr(cText)- joinedAB := tokenizeStr(aText + "\n" + bText)- // Budget just below the a+b join so a and b cannot merge without- // overflowing, but a alone and c alone fit, and an overlap prefix carved- // from chunk0 fits ahead of c (overlap path is exercised on chunk 1).- budget := joinedAB - 1+ // The merge decision is now the running sum of per-unit token counts+ // (aN + bN), faithful to Python's _merge_text_chunks_by_token_size, NOT the+ // re-tokenized a+b join. Budget just below the a+b running sum so a and b+ // cannot merge without overflowing (forcing mergeThenClose on chunk0), but+ // a alone and c alone fit, and an overlap prefix carved from chunk0 fits+ // ahead of c (overlap path is exercised on chunk 1).+ budget := aN + bN - 1if budget < aN {budget = aN}if budget < cN {budget = cN}- if joinedAB <= budget {- t.Fatalf("could not derive tight budget (a=%d b=%d joined=%d budget=%d)", aN, bN, joinedAB, budget)+ if aN+bN <= budget {+ t.Fatalf("could not derive tight budget (a=%d b=%d sum=%d budget=%d)", aN, bN, aN+bN, budget)}items := [][]schema.ChunkDoc{{diff --git a/internal/ingestion/component/chunker/token_json_overlap_test.go b/internal/ingestion/component/chunker/token_json_overlap_test.gonew file mode 100644index 0000000000..89f4c59c25--- /dev/null+++ b/internal/ingestion/component/chunker/token_json_overlap_test.go@@ -0,0 +1,110 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// TestMergeByTokenSizeFromJSON_TKNumsConsistency exercises the JSON merge path+// with overlap > 0, which previously had NO coverage, and pins the TKNums+// accounting so the running-sum merge decision cannot silently regress to the+// old re-tokenized-join count.+//+// Fixture: three text units a/b/c with a and b small enough to each fit the+// budget but a+b's running sum overflows it (so a+b merge-then-close into+// chunk0), and c starts a fresh chunk (prevClosed) carrying an overlap prefix+// from chunk0.+//+// TKNums accounting (see token.go mergeByTokenSizeFromJSON):+// - on the merge path (chunk0) TKNums is the RUNNING SUM of per-unit counts+// (aN+bN), never tokenizeStr(chunk0.Text);+// - on the boundary path (chunk1, via prevClosed) TKNums is reset to+// tokenizeStr(chunk1.Text) — i.e. the two paths use different caliber.+// This mixed accounting is documented in token.go; the assertions below+// lock the current behavior so any future unification is a deliberate+// change, not a silent drift.+func TestMergeByTokenSizeFromJSON_TKNumsConsistency(t *testing.T) {+ for _, overlapPct := range []float64{0, 30} {+ t.Run(overlapName(overlapPct), func(t *testing.T) {+ aText := strings.Repeat("word ", 18)+ bText := strings.Repeat("word ", 18)+ cText := strings.Repeat("word ", 6)+ aN, bN, cN := tokenizeStr(aText), tokenizeStr(bText), tokenizeStr(cText)+ // Budget just below the a+b running sum so a and b cannot merge+ // without overflowing (forcing mergeThenClose on chunk0), while a+ // alone and c alone fit.+ budget := aN + bN - 1+ if budget < aN {+ budget = aN+ }+ if budget < cN {+ budget = cN+ }+ if aN+bN <= budget {+ t.Fatalf("could not derive tight budget (a=%d b=%d sum=%d budget=%d)", aN, bN, aN+bN, budget)+ }+ items := [][]schema.ChunkDoc{+ {+ {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},+ {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},+ {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},+ },+ }+ got := mergeByTokenSizeFromJSON(items, budget, overlapPct, schema.MergeOverCap)+ merged := got[0]+ if len(merged) != 2 {+ t.Fatalf("want 2 chunks (overflow-closed + overlap/fresh chunk), got %d (a=%d b=%d c=%d budget=%d)", len(merged), aN, bN, cN, budget)+ }++ // chunk0 is the merge-then-close of a and b.+ if merged[0].Text != aText+"\n"+bText {+ t.Errorf("chunk0 text mismatch:\n got=%q\nwant=%q", merged[0].Text, aText+"\n"+bText)+ }+ // Merge path: TKNums is the running sum, not the re-tokenized text.+ if got0 := intValue(merged[0].TKNums); got0 != aN+bN {+ t.Errorf("chunk0 TKNums: running sum want %d, got %d (tokenizeStr(chunk0.Text)=%d)", aN+bN, got0, tokenizeStr(merged[0].Text))+ }++ // chunk1 is c (fresh, via prevClosed). With overlap>0 it carries a+ // prefix carved from chunk0; verify the overlap path actually ran.+ if overlapPct > 0 {+ overlap, _ := computeOverlapPrefix(merged[0].Text, overlapPct)+ if overlap == "" {+ t.Fatal("expected a non-empty overlap prefix from chunk0")+ }+ if !strings.HasPrefix(merged[1].Text, overlap) {+ t.Errorf("chunk1 should start with overlap prefix %q, got %q", overlap, merged[1].Text)+ }+ }+ // Boundary path: TKNums is the re-tokenized chunk1 text count.+ if got1 := intValue(merged[1].TKNums); got1 != tokenizeStr(merged[1].Text) {+ t.Errorf("chunk1 TKNums: want tokenizeStr(chunk1.Text)=%d, got %d", tokenizeStr(merged[1].Text), got1)+ }+ })+ }+}++func overlapName(p float64) string {+ if p == 0 {+ return "overlap0"+ }+ return "overlap30"+}diff --git a/internal/ingestion/component/chunker/token_merge_parity_test.go b/internal/ingestion/component/chunker/token_merge_parity_test.gonew file mode 100644index 0000000000..185414f154--- /dev/null+++ b/internal/ingestion/component/chunker/token_merge_parity_test.go@@ -0,0 +1,285 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/agent/runtime"+)++// mergeSourceLines is a fixed 29-line plain-text payload. The exact wording is+// the RAGFlow introduction paragraph used by the golden-parity fixtures; the+// chunk boundaries below were verified against Python's naive_merge+// (rag/nlp/__init__.py:_merge_paragraph_groups, OVER_CAP) on cl100k_base.+var mergeSourceLines = []string{+ "RAGFlow is an open-source retrieval-augmented generation engine.",+ "It ingests documents of many types and splits them into chunks.",+ "Each chunk is embedded and stored in a vector database for retrieval.",+ "The chunker honours document structure such as headings and tables.",+ "Long paragraphs are divided so each chunk fits a token budget.",+ "Overlap between chunks preserves context across boundaries.",+ "Retrieval returns the most relevant chunks for a user question.",+ "The generator then composes an answer from those chunks.",+ "Evaluation measures faithfulness and answer relevance.",+ "Users tune chunk size to balance recall and precision.",+ "Smaller chunks improve recall but increase storage cost.",+ "Larger chunks keep more context but may dilute relevance.",+ "The system supports many languages and encodings.",+ "Deep parsing extracts text from PDF images and scans.",+ "Tables are recognised and preserved as structured chunks.",+ "Images can be described and attached to neighbouring text.",+ "The API exposes dataset, document, and chat endpoints.",+ "A web UI lets users manage knowledge bases visually.",+ "Batch ingestion processes large corpora efficiently.",+ "Concurrency limits protect the embedding service.",+ "Caching avoids re-embedding unchanged content.",+ "Logging records ingestion progress and errors.",+ "Metrics help operators spot slow components.",+ "Configuration controls parsers and chunking strategies.",+ "Templates encode common ingestion pipelines.",+ "The engine scales horizontally behind a load balancer.",+ "Security isolates tenant data by document id.",+ "Auditing tracks who accessed which knowledge base.",+ "Plugins extend parsing for proprietary formats.",+}++// pythonMergeGroups is a faithful transcription of the Python reference+// chunker merge rag/nlp/__init__.py:_merge_paragraph_groups under+// MergeStrategy.OVER_CAP. It is used here as the oracle for+// TestTokenChunkerMergeMatchesPython. Because it is itself Go code, it only+// proves that the Go TokenChunker reproduces THIS transcription; its fidelity+// to the real Python algorithm is validated by hand against the source lines+// noted below. Keep those line references in sync if the Python side moves.+//+// Mapping to rag/nlp/__init__.py:_merge_paragraph_groups (OVER_CAP):+// - each paragraph is built as "\n" + sub_sec (naive_merge ~:1405), so the+// per-paragraph token count INCLUDES the leading "\n"; we mirror that with+// pt := tokenizeStr("\n" + p).+// - cur_t is the running sum of per-paragraph size("\n" + sub_sec); a+// paragraph merges when cur_t + pt <= cap.+// - an oversized paragraph (pt > cap) is emitted alone as its own chunk+// (Python: never combined with the previous chunk).+// - when cur_t + pt > cap, OVER_CAP merges the overflowing paragraph into the+// current group and then CLOSES the group (merge-then-close), so the next+// paragraph starts a fresh group.+// - empty/whitespace-only paragraphs are DROPPED before merging, mirroring+// the Python caller's `if not sub_sec` filter (rag/nlp/__init__.py:1403)+// and the Go chunker's TrimSpace skip (token.go:703). The oracle must model+// this so it stays a faithful reference for mergeByTokenSize.+func pythonMergeGroups(paragraphs []string, cap int) [][]string {+ groups := [][]string{}+ cur, curT := []string{}, 0+ for _, p := range paragraphs {+ // Empty fragments are dropped, exactly as the Python caller and the+ // Go chunker do. Skipping here keeps the oracle faithful.+ if strings.TrimSpace(p) == "" {+ continue+ }+ // Python's naive_merge builds each paragraph as "\n" + sub_sec+ // (rag/nlp/__init__.py:1405), so the per-paragraph token count+ // INCLUDES the leading "\n". Count it the same way so the running+ // sum matches Python's size("\n" + sub_sec).+ pt := tokenizeStr("\n" + p)+ if pt > cap {+ if len(cur) > 0 {+ groups = append(groups, cur)+ }+ groups = append(groups, []string{p})+ cur, curT = []string{}, 0+ continue+ }+ if len(cur) == 0 {+ cur, curT = []string{p}, pt+ continue+ }+ if curT+pt <= cap {+ cur = append(cur, p)+ curT += pt+ } else {+ // OVER_CAP: merge the overflowing paragraph, then close.+ cur = append(cur, p)+ curT += pt+ groups = append(groups, cur)+ cur, curT = []string{}, 0+ }+ }+ if len(cur) > 0 {+ groups = append(groups, cur)+ }+ return groups+}++// TestPythonMergeGroupsOracleSanity pins pythonMergeGroups itself, so the+// oracle cannot silently drift in lock-step with the chunker under test+// (which would make TestTokenChunkerMergeMatchesPython a no-op check). The+// assertions are tokenizer-independent INVARIANTS of the naive_merge+// (OVER_CAP) algorithm rather than hard-coded token magnitudes, so the test+// is stable across tokenizer revisions:+// 1. every input paragraph appears exactly once, in order (no loss/dup/reorder);+// 2. every group is VALID: either its running sum is within cap (a normal+// group), or it is a maximal merge-then-close overflow (running sum >+// cap but dropping its last paragraph is within cap), or it is a single+// oversized paragraph that stands alone. The group's position in the slice+// (trailing or not) is irrelevant: an overflow group may be last when the+// input ends right after an overflow.+func TestPythonMergeGroupsOracleSanity(t *testing.T) {+ const cap = 8+ inputs := [][]string{+ {"word word word word word word word word word word"}, // clearly oversized -> own group+ {"word", "word", "word", "word", "word", "word", "word", "word", "word", "word"}, // overflow path+ {"", "x", "y", "z"}, // empty + small+ {"alpha", "beta", "gamma", "delta"}, // plain small+ }+ for ci, in := range inputs {+ groups := pythonMergeGroups(in, cap)+ // Python (rag/nlp:1403) and the Go chunker (token.go:703) drop+ // empty/whitespace-only fragments, so the oracle does too. Compare+ // against the non-empty partition of the input rather than `in`.+ var nonEmpty []string+ for _, p := range in {+ if strings.TrimSpace(p) == "" {+ continue+ }+ nonEmpty = append(nonEmpty, p)+ }+ var flat []string+ for _, g := range groups {+ flat = append(flat, g...)+ }+ if len(flat) != len(nonEmpty) {+ t.Fatalf("case %d: paragraph count mismatch got=%d want=%d", ci, len(flat), len(nonEmpty))+ }+ for i := range nonEmpty {+ if nonEmpty[i] != flat[i] {+ t.Fatalf("case %d elem %d: order/loss mismatch got=%q want=%q", ci, i, flat[i], nonEmpty[i])+ }+ }+ // An oversized paragraph (its own token count > cap) must stand+ // alone: the oracle emits it as a single-element group.+ for gi, g := range groups {+ for _, p := range g {+ if tokenizeStr("\n"+p) > cap {+ if len(g) != 1 {+ t.Errorf("case %d group %d: oversized paragraph not isolated (len=%d)", ci, gi, len(g))+ }+ break+ }+ }+ }+ for gi, g := range groups {+ if len(g) == 0 {+ t.Fatalf("case %d group %d is empty", ci, gi)+ }+ sum := 0+ for _, p := range g {+ sum += tokenizeStr("\n" + p)+ }+ prefix := 0+ for _, p := range g[:len(g)-1] {+ prefix += tokenizeStr("\n" + p)+ }+ valid := sum <= cap || (sum > cap && prefix <= cap)+ if !valid {+ t.Errorf("case %d group %d invalid: sum=%d prefix=%d (cap=%d)", ci, gi, sum, prefix, cap)+ }+ }+ }+}++// expectedPythonChunks returns the chunk texts Python's naive_merge produces+// for mergeSourceLines at the given chunk_token_size.+func expectedPythonChunks(t *testing.T, chunkTokenSize int) []string {+ t.Helper()+ groups := pythonMergeGroups(mergeSourceLines, chunkTokenSize)+ want := make([]string, len(groups))+ for i, g := range groups {+ want[i] = strings.Join(g, "\n")+ }+ return want+}++// invokeTokenChunker runs the real TokenChunker component on plain text and+// returns the produced chunk texts, in order.+func invokeTokenChunker(t *testing.T, text string, chunkTokenSize int) []string {+ t.Helper()+ factory, _, _, ok := runtime.DefaultRegistry.Lookup("TokenChunker")+ if !ok || factory == nil {+ t.Fatalf("TokenChunker is not registered")+ }+ param := map[string]any{+ "chunk_token_size": float64(chunkTokenSize),+ "delimiters": []any{"\n"},+ }+ input := map[string]any{+ "output_format": "text",+ "name": "merge-parity",+ "text": text,+ }+ comp, err := factory("TokenChunker", param)+ if err != nil {+ t.Fatalf("construct TokenChunker: %v", err)+ }+ out, err := comp.Invoke(context.Background(), nil, input)+ if err != nil {+ t.Fatalf("invoke TokenChunker: %v", err)+ }+ if msg, ok := out["_ERROR"].(string); ok && msg != "" {+ t.Fatalf("TokenChunker returned _ERROR: %s", msg)+ }+ raw, _ := out["chunks"].([]map[string]any)+ texts := make([]string, 0, len(raw))+ for _, c := range raw {+ if s, ok := c["text"].(string); ok {+ texts = append(texts, s)+ }+ }+ return texts+}++// TestTokenChunkerMergeMatchesPython is the red test for the merge-boundary+// divergence (ragflow chunker parity):+//+// The Go merge decision used tokenizeStr(prev + "\n" + incoming) — the BPE+// token count of the JOINED string — while Python's naive_merge accumulates a+// running sum of per-paragraph token counts (cur_t + size(p)). Because BPE+// tokenization is not additive across the "\n" boundary, the two merge+// decisions disagree: Go can merge one paragraph more (or fewer) than Python,+// shifting every downstream boundary by a line and, once a budget is small+// enough, changing the chunk count outright.+//+// This test asserts the Go TokenChunker output equals Python's naive_merge+// output (verified chunk count AND per-chunk text) for two budgets:+// - chunk_token_size=32: Python emits 9 chunks (Go over-merged to 8 before+// the running-sum fix).+// - chunk_token_size=128: both emit 3 chunks, but Python closes chunk 0 one+// line earlier than Go (boundary offset before the fix).+func TestTokenChunkerMergeMatchesPython(t *testing.T) {+ text := strings.Join(mergeSourceLines, "\n")+ for _, cap := range []int{32, 128} {+ want := expectedPythonChunks(t, cap)+ got := invokeTokenChunker(t, text, cap)+ if len(got) != len(want) {+ t.Fatalf("chunk_token_size=%d: chunk count go=%d, python=%d", cap, len(got), len(want))+ }+ for i := range got {+ if got[i] != want[i] {+ t.Errorf("chunk_token_size=%d chunk[%d] mismatch:\n got=%q\nwant=%q", cap, i, got[i], want[i])+ }+ }+ }+}diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.goindex 16f2c33664..eaf68acb74 100644--- a/internal/ingestion/component/chunker/token_strict_cap_test.go+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go@@ -56,11 +56,18 @@ func TestMergeByTokenSizeFromJSON_StrictCapNoOvershoot(t *testing.T) {}// OVER_CAP (Python's canonical default) permits a chunk to exceed budget// by at most one incoming unit: a chunk is closed right after the- // overflowing merge, so it can hold prev (<= budget) + one unit (<= budget).+ // overflowing merge, so its running-sum token count is <= budget+unit.+ // The reconstructed chunk text carries the "\n" joins between units, and+ // cl100k is non-additive across joins, so the actual token count can land+ // a little above budget+unit; allow one extra unit of slack (matching the+ // sibling TestMergeByTokenSizeFromJSON_OverlapDroppedAtOverflow tolerance+ // for the same cl100k delta). Python's naive_merge produces the same 3+ // chunks here, so this pins the faithful boundary, not the old re-tokenized+ // (under-packed) one.unit := tokenizeStr(sections[0].Text)for i, ck := range merged {n := tokenizeStr(ck.Text)- if n > budget+unit {+ if n > budget+2*unit {t.Errorf("chunk %d exceeds budget by more than one unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)}}@@ -212,20 +219,26 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {}}- // Control: OVER_CAP (default) must overflow on the same input, proving the- // toggle changes behavior rather than being a no-op.+ // Control: OVER_CAP (default) must follow its one-boundary-overflow+ // contract on the same input, proving the toggle changes behavior rather+ // than being a no-op. With the running-sum merge decision (faithful to+ // Python's naive_merge), a chunk may hold prev (<= budget) + one unit, so+ // its running-sum token count is <= budget+unit; the reconstructed text+ // carries the "\n" joins and cl100k is non-additive across them, so allow+ // the same one-extra-unit slack. For this particular input Python's+ // OVER_CAP does not actually exceed budget, but it still permits the+ // one-unit overflow that UNDER_CAP forbids, so the two strategies produce+ // different chunk counts — proving the toggle is live.over := run(false)- overflowed := false- for _, ck := range over {+ if len(over) == len(respect) {+ t.Errorf("OVER_CAP produced the same chunk count as UNDER_CAP (%d); toggle may be a no-op", len(over))+ }+ for i, ck := range over {text, _ := ck["text"].(string)- if tokenizeStr(text) > budget {- overflowed = true- break+ if tokenizeStr(text) > budget+sentenceN {+ t.Errorf("OVER_CAP chunk %d exceeds one-unit overflow contract: tokens=%d (cap=%d unit=%d)", i, tokenizeStr(text), budget, sentenceN)}}- if !overflowed {- t.Errorf("OVER_CAP control produced no overflow on input that UNDER_CAP keeps within budget; toggle may be a no-op")- }}func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {@@ -339,3 +352,44 @@ func TestInvokeJSONPayload_UnderCapEndToEnd(t *testing.T) {}}}++// TestMergeDecisionOverCapVsUnderCapBoundary pins the semantic difference+// between the two merge strategies at the exact boundary+// (prevTokens+incomingTokens > target while incomingTokens <= target),+// independent of BPE text-token-count fluctuations. OVER_CAP must+// merge-then-close (a chunk may exceed target by at most one unit); UNDER_CAP+// must start a fresh chunk (strict no-overflow). This restores the strong+// constraint that the under_cap toggle is live — the integration test in+// TestMergeByTokenSize_UnderCapNoOverflow can only assert that the two+// strategies produce a different chunk COUNT, because on repetitive text the+// re-tokenized merged chunk can fall back under budget (cl100k is+// non-additive across the join). A regression that turns OVER_CAP into a+// no-op would not be caught there, but is caught here.+func TestMergeDecisionOverCapVsUnderCapBoundary(t *testing.T) {+ const prevT, incT = 10, 10+ target := prevT + incT - 1 // boundary: running sum > target, unit fits+ if incT > target {+ t.Fatalf("bad fixture: incoming unit must fit target (incT=%d target=%d)", incT, target)+ }++ _, overAct := mergeDecision("prev text", "incoming text", "\n", target, 0, schema.MergeOverCap, prevT, incT)+ if overAct != mergeThenClose {+ t.Errorf("OVER_CAP at boundary: want mergeThenClose, got %v", overAct)+ }++ _, underAct := mergeDecision("prev text", "incoming text", "\n", target, 0, schema.MergeUnderCap, prevT, incT)+ if underAct != startNewChunk {+ t.Errorf("UNDER_CAP at boundary: want startNewChunk, got %v", underAct)+ }++ // Both strategies must still refuse to merge an incoming unit that+ // already exceeds target — it stands alone as its own chunk.+ _, overBig := mergeDecision("prev text", "incoming text", "\n", target, 0, schema.MergeOverCap, prevT, target+5)+ if overBig != startNewChunk {+ t.Errorf("OVER_CAP with oversized incoming: want startNewChunk, got %v", overBig)+ }+ _, underBig := mergeDecision("prev text", "incoming text", "\n", target, 0, schema.MergeUnderCap, prevT, target+5)+ if underBig != startNewChunk {+ t.Errorf("UNDER_CAP with oversized incoming: want startNewChunk, got %v", underBig)+ }+}