RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(title-chunker): enforce chunk_token_cap on the Go title-family chunks (#18455) (#18551)
Jackd76c15e2a5cb
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.goindex 7a0572ec7e..fbfafaa7d3 100644--- a/internal/ingestion/component/chunker/group.go+++ b/internal/ingestion/component/chunker/group.go@@ -120,18 +120,24 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,if len(records) == 0 {return emptyOutputs(), nil}- return chunkFromRecords(parentCtx, db, inputs, p, records)+ return chunkFromRecords(parentCtx, db, inputs, p, records, p.ChunkTokenCap)}// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over// an already-extracted record list: resolve heading levels, split into-// sections, merge adjacent text records, build chunks, and perform on-demand-// PDF cropping. GroupTitleChunker feeds records in input order; ManualChunker-// feeds them after a (page, top, left) resort. Sharing this body guarantees-// both strategies emit byte-identical output for coordinate-free input (the-// docx manual branch) — the no-regression contract locked by-// TestManualChunker_NoPositionsEqualsGroupChunker.-func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord) (map[string]any, error) {+// sections, merge adjacent text records, build chunks, enforce the token cap+// (title family only), and perform on-demand PDF cropping. GroupTitleChunker+// feeds records in input order; ManualChunker feeds them after a (page, top,+// left) resort. Sharing this body keeps both strategies byte-identical for+// coordinate-free input as long as no built chunk exceeds the token cap+// (TestManualChunker_NoPositionsEqualsGroupChunker); once one does, only+// GroupTitleChunker re-splits it (ManualChunker is exempt).+//+// tokenCap is the title-family token ceiling (chunk_token_cap). ManualChunker+// must stay exempt from #18455's cap, so it passes 0; GroupTitleChunker passes+// p.ChunkTokenCap. The cap is applied right after build_chunks and BEFORE the+// on-demand crop, mirroring Python's invoke() order (build -> cap -> set_chunks).+func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord, tokenCap int) (map[string]any, error) {ctx := newLevelContext(records, outlineFromInputs(inputs), p)levels := ctx.Levels()// Count heading level distribution for debugging.@@ -169,6 +175,13 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]zap.Int("groups", len(groups)),)chunks := buildChunksFromRecordGroups(groups, p, isPlainTextFormat(inputs))+ // Enforce the title-family token ceiling (chunk_token_cap) right after+ // build_chunks and BEFORE the on-demand crop, mirroring Python's+ // invoke() order (build -> cap -> set_chunks). ManualChunker passes+ // tokenCap=0, so this is a no-op there.+ if tokenCap > 0 {+ chunks = enforceTitleTokenCap(chunks, tokenCap)+ }common.Debug("chunker stage",zap.String("component", "Chunker"),zap.String("variant", "group"),diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.goindex c8dfd0d35c..a7ca007f40 100644--- a/internal/ingestion/component/chunker/hierarchy.go+++ b/internal/ingestion/component/chunker/hierarchy.go@@ -241,6 +241,12 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a)chunks := buildChunksFromRecordGroups(recordGroups, p, isPlainTextFormat(inputs))+ // Enforce the title-family token ceiling (chunk_token_cap) right after+ // build_chunks and BEFORE the on-demand crop, mirroring Python's+ // invoke() order (build -> cap -> set_chunks).+ if p.ChunkTokenCap > 0 {+ chunks = enforceTitleTokenCap(chunks, p.ChunkTokenCap)+ }common.Debug("chunker stage",zap.String("component", "Chunker"),zap.String("variant", "hierarchy"),diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.goindex b690230e85..d783f1932e 100644--- a/internal/ingestion/component/chunker/manual.go+++ b/internal/ingestion/component/chunker/manual.go@@ -115,7 +115,9 @@ func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputsif hasPdfPositions(records) {sortRecordsByPosition(records)}- return chunkFromRecords(ctx, db, inputs, &c.param, records)+ // ManualChunker is exempt from the title-family token cap (#18455): it+ // does not inherit BaseTitleChunker, so it always passes tokenCap=0.+ return chunkFromRecords(ctx, db, inputs, &c.param, records, 0)}// hasPdfPositions reports whether any record carries a PDF coordinate matrixdiff --git a/internal/ingestion/component/chunker/sentence_boundary.go b/internal/ingestion/component/chunker/sentence_boundary.gonew file mode 100644index 0000000000..1d0e654027--- /dev/null+++ b/internal/ingestion/component/chunker/sentence_boundary.go@@ -0,0 +1,32 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except under the License.+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import "regexp"++// sentenceBoundaryRe is the single sentence-boundary definition shared by the+// title chunker family and the token chunker's JSON path. It mirrors Python's+// rag/flow/chunker/_sentence_boundary.py:SENTENCE_BOUNDARY_RE exactly: the+// Chinese/English period, exclamation mark, question mark, the Chinese+// variants, the newline, and the English ". " (period + space) boundary. The+// capturing group keeps the delimiter attached to the preceding sentence so+// re-merged text preserves original boundaries.+//+// The token chunker's TEXT path (naive_merge port) uses a different delimiter+// — sentenceDelimiter in token.go, mirroring Python naive_merge's production+// delimiter "\n!?。;!?" (no ". "). The two constants mirror two distinct+// Python constants and differ only in that English boundary, so they are+// intentionally NOT merged into one.+var sentenceBoundaryRe = regexp.MustCompile(`([。!??;!\n]|\. )`)diff --git a/internal/ingestion/component/chunker/sentence_boundary_test.go b/internal/ingestion/component/chunker/sentence_boundary_test.gonew file mode 100644index 0000000000..a8da922107--- /dev/null+++ b/internal/ingestion/component/chunker/sentence_boundary_test.go@@ -0,0 +1,58 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except under the License.+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import "testing"++// TestSentenceBoundary_SharedRegexMatchesPython pins sentenceBoundaryRe to the+// exact Python SENTENCE_BOUNDARY_RE (rag/flow/chunker/_sentence_boundary.py),+// including the English ". " boundary used by the title chunker and the token+// chunker's JSON path.+func TestSentenceBoundary_SharedRegexMatchesPython(t *testing.T) {+ const want = `([。!??;!\n]|\. )`+ if got := sentenceBoundaryRe.String(); got != want {+ t.Errorf("sentenceBoundaryRe = %q, want %q (Python SENTENCE_BOUNDARY_RE)", got, want)+ }+ // English ". " boundary must be recognized.+ if !sentenceBoundaryRe.MatchString("Hello. World") {+ t.Error("sentenceBoundaryRe does not match the English '. ' boundary")+ }+ // CJK boundaries still recognized.+ for _, p := range []string{"一。", "二!", "三?", "四;", "五\n"} {+ if !sentenceBoundaryRe.MatchString(p) {+ t.Errorf("sentenceBoundaryRe does not match boundary %q", p)+ }+ }+}++// TestSentenceBoundary_TokenTextPathDelimiterDistinct pins sentenceDelimiter+// (the token-chunker TEXT path / naive_merge port) to Python naive_merge's+// production delimiter "\n!?。;!?" — deliberately WITHOUT the English ". "+// boundary. The two constants mirror two distinct Python delimiters; asserting+// the exact difference documents that this is intentional, not a drift.+func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {+ const want = `(\n|[!?。;!?])`+ if got := sentenceDelimiter.String(); got != want {+ t.Errorf("sentenceDelimiter = %q, want %q (Python naive_merge delimiter)", got, want)+ }+ // The text-path delimiter must NOT split on the English ". " boundary+ // (matching Python naive_merge), while the shared regex does.+ if sentenceDelimiter.MatchString("Hello. World") {+ t.Error("sentenceDelimiter split on '. ' but naive_merge's delimiter has no '. ' boundary")+ }+ if !sentenceBoundaryRe.MatchString("Hello. World") {+ t.Error("sentenceBoundaryRe should split on '. '")+ }+}diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.goindex 7484925a5f..87f8f38278 100644--- a/internal/ingestion/component/chunker/title.go+++ b/internal/ingestion/component/chunker/title.go@@ -101,6 +101,9 @@ func (p *titleChunkerParam) Update(conf map[string]any) {if v, ok := conf["root_chunk_as_heading"].(bool); ok {p.TitleChunkerParam.RootChunkAsHeading = v}+ if v, ok := schema.NumericFromAny(conf["chunk_token_cap"]); ok {+ p.TitleChunkerParam.ChunkTokenCap = int(v)+ }}// parseLevels accepts a [[string]] representation — the naturaldiff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.gonew file mode 100644index 0000000000..17b7c6b8c8--- /dev/null+++ b/internal/ingestion/component/chunker/title_cap.go@@ -0,0 +1,266 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except under the License.+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++// Token-count cap for the title-chunker family (GroupTitle / HierarchyTitle),+// mirroring Python title_chunker/common.py:_enforce_token_cap (#18455).+//+// A built chunk that exceeds `chunk_token_cap` tokens is re-split on sentence+// boundaries into <= cap sub-chunks; a single boundary-less run that still+// exceeds the cap is hard-split so the ceiling always holds. Table/image+// chunks are atomic and never split. The first sub-chunk keeps the source+// chunk's merged PDF position matrix (Plan A); subsequent sub-chunks carry an+// empty array.++import (+ "encoding/json"+ "strings"+ "unicode/utf8"++ "ragflow/internal/tokenizer"+)++// realNumTokens / realTrimToTokenLimit are the production tokenizer functions.+// They are referenced through package-level vars (numTokens /+// trimToTokenLimit) so tests can swap in a deterministic stub.+var (+ realNumTokens = tokenizer.NumTokensFromString+ realTrimToTokenLimit = tokenizer.TrimContentToTokenLimit++ numTokens = realNumTokens+ trimToTokenLimit = realTrimToTokenLimit+)++// titleTokenCount counts tokens for text. numTokensFromString returns 0 when+// the encoder is unavailable; in that case fall back to the rune count so the+// cap is still enforced (mirrors Python #18455's character-count fallback).+func titleTokenCount(text string) int {+ if text == "" {+ return 0+ }+ n := numTokens(text)+ if n == 0 {+ return utf8.RuneCountInString(text)+ }+ return n+}++// titleSentenceSplit splits text on sentence boundaries, keeping each+// delimiter attached to the preceding sentence so the concatenation of the+// returned pieces reproduces the original text exactly. It uses the shared+// sentenceBoundaryRe (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE),+// which includes the English ". " boundary.+func titleSentenceSplit(text string) []string {+ if text == "" {+ return nil+ }+ idxs := sentenceBoundaryRe.FindAllStringIndex(text, -1)+ var out []string+ prev := 0+ for _, idx := range idxs {+ if idx[0] > prev {+ out = append(out, text[prev:idx[1]])+ } else {+ out = append(out, text[idx[0]:idx[1]])+ }+ prev = idx[1]+ }+ if prev < len(text) {+ out = append(out, text[prev:])+ }+ return out+}++// runePrefix returns the first n runes of s (a rune-safe prefix; never splits+// a multi-byte character). Used by the offline/hard-split fallback.+func runePrefix(s string, n int) string {+ if n <= 0 {+ return ""+ }+ count := 0+ for i := range s {+ if count >= n {+ return s[:i]+ }+ count+++ }+ return s+}++// hardSplitByTokens hard-splits text into pieces whose token count is <= cap.+// It mirrors Python's _hard_split_by_tokens (token_utils.truncate prefix).+// When the tokenizer is unavailable (numTokens returns 0) it degrades to a+// rune-prefix of length cap, keeping the ceiling intact (Python #18455+// character-count fallback).+func hardSplitByTokens(text string, cap int) []string {+ if cap <= 0 {+ if text == "" {+ return nil+ }+ return []string{text}+ }+ var out []string+ rest := text+ for utf8.RuneCountInString(rest) > 0 {+ // A remainder already within the token cap stays whole. This must be+ // checked on TOKENS: for English text runes far exceed tokens, so+ // comparing runes against the cap would over-fragment an in-cap tail.+ if titleTokenCount(rest) <= cap {+ out = append(out, rest)+ break+ }+ head := trimToTokenLimit(rest, cap)+ // No progress (tokenizer unavailable / cannot shrink): fall back to a+ // rune prefix so the loop always makes progress and the ceiling holds.+ if head == "" || head == rest || !strings.HasPrefix(rest, head) {+ head = runePrefix(rest, cap)+ if head == "" || head == rest {+ out = append(out, rest)+ break+ }+ }+ out = append(out, head)+ rest = rest[len(head):]+ }+ return out+}++// enforceTitleTokenCap applies the hard token ceiling to every text chunk+// after build_chunks. Mirrors Python BaseTitleChunker._enforce_token_cap:+// - cap <= 0 disables the ceiling (chunks returned unchanged);+// - non-text chunks (table/image) are atomic and skipped;+// - a text chunk already within cap is kept;+// - an over-cap text chunk is re-split via splitTitleChunkByCap.+func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {+ if cap <= 0 {+ return chunks+ }+ out := make([]map[string]any, 0, len(chunks))+ for _, chunk := range chunks {+ if toStringOrDefault(chunk["doc_type_kwd"], "text") != "text" {+ out = append(out, chunk)+ continue+ }+ text := toString(chunk["text"])+ if text == "" || titleTokenCount(text) <= cap {+ out = append(out, chunk)+ continue+ }+ out = append(out, splitTitleChunkByCap(chunk, cap)...)+ }+ return out+}++// splitTitleChunkByCap re-splits one oversized text chunk into <= cap+// sub-chunks. Sentences are tried first (greedy grouping); any remaining+// over-cap segment is hard-split. The first sub-chunk keeps the source+// chunk's merged PDF position matrix (Plan A); the rest carry an empty array.+func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {+ text := toString(chunk["text"])+ sentences := titleSentenceSplit(text)+ if len(sentences) == 0 {+ return []map[string]any{chunk}+ }+ groups := []string{}+ current := ""+ for _, s := range sentences {+ cand := s+ if current != "" {+ cand = current + s+ }+ if current != "" && titleTokenCount(cand) > cap {+ groups = append(groups, current)+ current = s+ } else {+ current = cand+ }+ }+ if current != "" {+ groups = append(groups, current)+ }+ finalGroups := []string{}+ for _, g := range groups {+ if titleTokenCount(g) <= cap {+ finalGroups = append(finalGroups, g)+ } else {+ finalGroups = append(finalGroups, hardSplitByTokens(g, cap)...)+ }+ }+ if len(finalGroups) == 0 {+ return []map[string]any{chunk}+ }++ out := make([]map[string]any, 0, len(finalGroups))+ for i, g := range finalGroups {+ sub := shallowCopyChunk(chunk)+ sub["text"] = g+ // Plan A: only the first sub-chunk keeps the position matrix; later+ // sub-chunks carry an empty matrix of the same type, or drop the key+ // when the source type is unknown.+ if i > 0 {+ if empty, ok := emptyPositionsLike(chunk["_pdf_positions"]); ok {+ sub["_pdf_positions"] = empty+ } else {+ delete(sub, "_pdf_positions")+ }+ if empty, ok := emptyPositionsLike(chunk["positions"]); ok {+ sub["positions"] = empty+ } else {+ delete(sub, "positions")+ }+ }+ out = append(out, sub)+ }+ return out+}++// shallowCopyChunk returns a shallow copy of c. Coordinate matrices+// ([][]float64) are shared by reference, but splitTitleChunkByCap replaces+// (never mutates) them for sub-chunks, so the source chunk is never aliased.+func shallowCopyChunk(c map[string]any) map[string]any {+ m := make(map[string]any, len(c))+ for k, v := range c {+ m[k] = v+ }+ return m+}++// emptyPositionsLike returns an empty coordinate array of the same Go type as+// the source value (mergePositionMatrix produces [][]float64; tests may feed+// json.RawMessage), so a Plan-A sub-chunk's position field stays type-consistent+// with the first sub-chunk's. The second result is false for unknown types;+// the caller then DELETES the key instead of storing a nil-valued one.+func emptyPositionsLike(v any) (any, bool) {+ switch v.(type) {+ case [][]float64:+ return [][]float64{}, true+ case json.RawMessage:+ return json.RawMessage("[]"), true+ default:+ return nil, false+ }+}++// toStringOrDefault returns the string value of v, or def when v is absent /+// nil / not a string. Used to treat a missing doc_type_kwd as "text".+func toStringOrDefault(v any, def string) string {+ if v == nil {+ return def+ }+ if s, ok := v.(string); ok {+ return s+ }+ return def+}diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.gonew file mode 100644index 0000000000..1c2b9a67e9--- /dev/null+++ b/internal/ingestion/component/chunker/title_token_cap_test.go@@ -0,0 +1,598 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except under the License.+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"+ "testing"+ "unicode/utf8"++ "ragflow/internal/ingestion/component/schema"+)++func testCtx() context.Context { return context.Background() }++// charTokenizer installs a deterministic "1 token == 1 rune" stub (mirroring+// the Python test_token_cap suite, which fakes num_tokens_from_string as+// len(text)). This makes the cap a hard rune ceiling so assertions are fully+// reproducible. Callers must defer restoreTokenizer().+func charTokenizer() {+ numTokens = func(s string) int {+ if s == "" {+ return 0+ }+ return utf8.RuneCountInString(s)+ }+ trimToTokenLimit = func(s string, limit int) string {+ if limit < 0 {+ limit = 0+ }+ if utf8.RuneCountInString(s) <= limit {+ return s+ }+ return runePrefix(s, limit)+ }+}++// restoreTokenizer resets the tokenizer seam to the real implementation.+func restoreTokenizer() {+ numTokens = realNumTokens+ trimToTokenLimit = realTrimToTokenLimit+}++// assertCapInvariants checks the post-_enforce_token_cap guarantees for a+// slice of built chunks: every text piece is <= cap (re-tokenized), non-text+// chunks are untouched, and the concatenated text reproduces the source+// (lossless). The optional trailing newline that build_chunks appends is+// stripped before comparison, matching Python's rstrip("\n").+func assertCapInvariants(t *testing.T, chunks []map[string]any, cap int, source string) {+ t.Helper()+ var got strings.Builder+ for _, ck := range chunks {+ text := toString(ck["text"])+ dt := toStringOrDefault(ck["doc_type_kwd"], "text")+ if dt != "text" {+ got.WriteString(text)+ continue+ }+ if n := titleTokenCount(text); n > cap {+ t.Errorf("chunk exceeds cap: tokens=%d (cap=%d) text=%q", n, cap, text)+ }+ got.WriteString(text)+ }+ if strings.TrimRight(got.String(), "\n") != strings.TrimRight(source, "\n") {+ t.Errorf("lossless check failed:\n got=%q\nwant=%q", got.String(), source)+ }+}++func TestTitleTokenCount_CharStub(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ if got := titleTokenCount("hello"); got != 5 {+ t.Errorf("titleTokenCount(char) = %d, want 5", got)+ }+}++func TestTitleTokenCount_OfflineFallback(t *testing.T) {+ // Simulate an unavailable tokenizer (num_tokens_from_string returns 0).+ saved := numTokens+ numTokens = func(string) int { return 0 }+ defer func() { numTokens = saved }()+ // With the offline fallback, non-empty text counts as its rune length.+ if got := titleTokenCount("hello世界"); got != 7 {+ t.Errorf("offline fallback token count = %d, want 7", got)+ }+ if got := titleTokenCount(""); got != 0 {+ t.Errorf("empty text token count = %d, want 0", got)+ }+}++func TestTitleSentenceSplit_Boundaries(t *testing.T) {+ // Chinese boundaries. The final fragment without a trailing delimiter is+ // kept as its own sentence (matches Python re.split reassembly).+ zh := "第一句。第二句!第三句?第四句;尾"+ got := titleSentenceSplit(zh)+ if len(got) != 5 {+ t.Fatalf("zh split = %d sentences, want 5: %v", len(got), got)+ }+ wantEnds := []string{"。", "!", "?", ";"}+ for i := 0; i < 4; i++ {+ if !strings.HasSuffix(got[i], wantEnds[i]) {+ t.Errorf("sentence %d = %q, want suffix %q", i, got[i], wantEnds[i])+ }+ }+ if got[4] != "尾" {+ t.Errorf("trailing sentence = %q, want \"尾\"", got[4])+ }+ // English ". " boundary (the Python #18455 regex includes `\. `).+ en := "Hello. World. Foo"+ eg := titleSentenceSplit(en)+ if len(eg) != 3 {+ t.Fatalf("en split = %d, want 3: %v", len(eg), eg)+ }+ if eg[0] != "Hello. " || eg[1] != "World. " || eg[2] != "Foo" {+ t.Errorf("en split = %v, want [Hello. World. Foo]", eg)+ }+}++func TestTitleSentenceSplit_Lossless(t *testing.T) {+ zh := "第一句。第二句!第三句?第四句;尾"+ if got := strings.Join(titleSentenceSplit(zh), ""); got != zh {+ t.Errorf("sentence split not lossless: %q", got)+ }+ en := "Hello. World. Foo"+ if got := strings.Join(titleSentenceSplit(en), ""); got != en {+ t.Errorf("en sentence split not lossless: %q", got)+ }+}++func TestEnforceTitleTokenCap_CapZeroNoop(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ chunks := []map[string]any{+ {"text": strings.Repeat("x", 100)},+ }+ got := enforceTitleTokenCap(chunks, 0)+ if len(got) != 1 {+ t.Fatalf("cap=0 must keep 1 chunk, got %d", len(got))+ }+ if toString(got[0]["text"]) != strings.Repeat("x", 100) {+ t.Errorf("cap=0 altered text: %q", toString(got[0]["text"]))+ }+}++func TestEnforceTitleTokenCap_WithinCapUnchanged(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ chunks := []map[string]any{{"text": "S00。S01。"}}+ got := enforceTitleTokenCap(chunks, 512)+ if len(got) != 1 {+ t.Fatalf("within-cap chunk split unexpectedly: %d chunks", len(got))+ }+ if toString(got[0]["text"]) != "S00。S01。" {+ t.Errorf("within-cap text altered: %q", toString(got[0]["text"]))+ }+}++func TestEnforceTitleTokenCap_OverCapResplits(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ chunks := []map[string]any{{"text": body}}+ got := enforceTitleTokenCap(chunks, 20)+ if len(got) <= 1 {+ t.Fatalf("expected oversized chunk to be split, got %d", len(got))+ }+ assertCapInvariants(t, got, 20, body)+ for _, ck := range got {+ if !strings.HasSuffix(strings.TrimRight(toString(ck["text"]), "\n"), "。") {+ t.Errorf("chunk cut mid-sentence: %q", toString(ck["text"]))+ }+ }+}++func TestEnforceTitleTokenCap_NonTextAtomic(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ big := strings.Repeat("x", 200)+ chunks := []map[string]any{{"text": big, "doc_type_kwd": "table"}}+ got := enforceTitleTokenCap(chunks, 20)+ if len(got) != 1 {+ t.Fatalf("table chunk must stay atomic, got %d", len(got))+ }+ if toString(got[0]["text"]) != big {+ t.Errorf("table text altered: len=%d", len(toString(got[0]["text"])))+ }+ // image too+ img := []map[string]any{{"text": big, "doc_type_kwd": "image"}}+ if got := enforceTitleTokenCap(img, 20); len(got) != 1 {+ t.Errorf("image chunk must stay atomic, got %d", len(got))+ }+}++func TestEnforceTitleTokenCap_BoundarylessHardSplit(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Repeat("x", 100)+ chunks := []map[string]any{{"text": body}}+ got := enforceTitleTokenCap(chunks, 20)+ if len(got) <= 1 {+ t.Fatalf("boundary-less run must be hard-split, got %d", len(got))+ }+ assertCapInvariants(t, got, 20, body)+}++func TestEnforceTitleTokenCap_TokenizerZeroFallback(t *testing.T) {+ // numTokens reports 0 everywhere -> offline char fallback must still cap.+ saved := numTokens+ numTokens = func(string) int { return 0 }+ defer func() { numTokens = saved }()+ body := strings.Repeat("x", 100)+ chunks := []map[string]any{{"text": body}}+ got := enforceTitleTokenCap(chunks, 20)+ if len(got) <= 1 {+ t.Fatalf("cap must apply even when tokenizer reports 0, got %d", len(got))+ }+ assertCapInvariants(t, got, 20, body)+}++// TestHardSplitByTokens_EnglishRemainderStaysWhole pins the over-fragmentation+// fix: a remainder that already satisfies the TOKEN cap must be kept whole even+// when its RUNE count exceeds the cap (English text is ~4 runes/token). The+// char stub (1 rune == 1 token) makes the two counts identical and masks this,+// so this test uses a 3-runes-per-token stub.+func TestHardSplitByTokens_EnglishRemainderStaysWhole(t *testing.T) {+ savedNum, savedTrim := numTokens, trimToTokenLimit+ // 3 runes == 1 token.+ numTokens = func(s string) int {+ if s == "" {+ return 0+ }+ return (utf8.RuneCountInString(s) + 2) / 3+ }+ trimToTokenLimit = func(s string, limit int) string {+ maxRunes := limit * 3+ if utf8.RuneCountInString(s) <= maxRunes {+ return s+ }+ return runePrefix(s, maxRunes)+ }+ defer func() { numTokens, trimToTokenLimit = savedNum, savedTrim }()++ const cap = 100+ // 600 runes == 200 tokens == exactly 2 cap units. The second unit's+ // remainder (300 runes == 100 tokens) is within the cap and must NOT be+ // re-cut on runes.+ body := strings.Repeat("ab", 300)+ got := hardSplitByTokens(body, cap)+ if len(got) != 2 {+ t.Fatalf("hardSplitByTokens produced %d pieces, want 2 (in-cap remainder must stay whole)", len(got))+ }+ if strings.Join(got, "") != body {+ t.Errorf("hard-split not lossless: %d runes vs %d", utf8.RuneCountInString(strings.Join(got, "")), utf8.RuneCountInString(body))+ }+ for i, p := range got {+ if n := numTokens(p); n > cap {+ t.Errorf("piece %d exceeds cap: tokens=%d (cap=%d)", i, n, cap)+ }+ }+}++func TestEnforceTitleTokenCap_PlanAPositions(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ pos := [][]float64{{1, 10, 200, 50, 80}}+ body := strings.Join(joinSentences(12), "")+ chunks := []map[string]any{+ {"text": body, "positions": pos},+ }+ got := enforceTitleTokenCap(chunks, 20)+ if len(got) <= 1 {+ t.Fatalf("expected split, got %d", len(got))+ }+ // First sub-chunk keeps the original position matrix.+ first, ok := got[0]["positions"].([][]float64)+ if !ok || len(first) == 0 || first[0][0] != 1 {+ t.Errorf("first sub-chunk positions = %#v, want [[1 10 200 50 80]]", got[0]["positions"])+ }+ // Remaining sub-chunks carry an empty matrix (Plan A).+ for i := 1; i < len(got); i++ {+ v, ok := got[i]["positions"].([][]float64)+ if !ok {+ t.Errorf("sub-chunk %d positions type = %T, want [][]float64", i, got[i]["positions"])+ continue+ }+ if len(v) != 0 {+ t.Errorf("sub-chunk %d positions = %#v, want empty", i, v)+ }+ }+}++func TestEnforceTitleTokenCap_PlanA_UnknownPositionsType(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ // An unknown position value type (not [][]float64 / json.RawMessage): the+ // key must be DROPPED from later sub-chunks, never stored as a nil value.+ chunks := []map[string]any{+ {"text": body, "positions": "not-a-matrix"},+ }+ got := enforceTitleTokenCap(chunks, 20)+ if len(got) <= 1 {+ t.Fatalf("expected split, got %d", len(got))+ }+ // First sub-chunk keeps the value via shallow copy.+ if _, ok := got[0]["positions"]; !ok {+ t.Error("first sub-chunk lost its positions key")+ }+ for i := 1; i < len(got); i++ {+ if _, ok := got[i]["positions"]; ok {+ t.Errorf("sub-chunk %d kept unknown-type positions key, want deleted", i)+ }+ }+}++func TestEnforceTitleTokenCap_GreedyGrouping(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ // 12 sentences, 4 runes each, cap 20 -> 5 sentences/chunk (20), not 1 each.+ body := strings.Join(joinSentences(12), "")+ chunks := []map[string]any{{"text": body}}+ got := enforceTitleTokenCap(chunks, 20)+ if len(got) != 3 { // 5+5+2+ t.Fatalf("greedy grouping produced %d chunks, want 3", len(got))+ }+ assertCapInvariants(t, got, 20, body)+}++// joinSentences builds 12 sentences "S00。".."S11。" mirroring the Python+// test_hierarchy_oversized_chunk_respects_cap body.+func joinSentences(n int) []string {+ out := make([]string, 0, n)+ for i := 0; i < n; i++ {+ out = append(out, sprintfSentence(i))+ }+ return out+}++func sprintfSentence(i int) string {+ return fmt.Sprintf("S%02d。", i)+}++// TestSprintfSentence_ThreeDigits pins the %02d formatting for 3-digit+// indexes: the old hand-rolled twoDigit derived each digit from a single rune+// addition and broke at i >= 100 (produced ":0").+func TestSprintfSentence_ThreeDigits(t *testing.T) {+ if got := sprintfSentence(100); got != "S100。" {+ t.Errorf("sprintfSentence(100) = %q, want \"S100。\"", got)+ }+}++func TestJoinSentences_Formatting(t *testing.T) {+ got := joinSentences(15)+ if len(got) != 15 {+ t.Fatalf("joinSentences(15) = %d, want 15", len(got))+ }+ if got[14] != "S14。" {+ t.Errorf("joinSentences(15)[14] = %q, want \"S14。\"", got[14])+ }+ if got[9] != "S09。" {+ t.Errorf("joinSentences(15)[9] = %q, want \"S09。\"", got[9])+ }+}++// ---------------------------------------------------------------------------+// Pipeline (integration) tests: cap applied through invokeGroup/invokeHierarchy+// ---------------------------------------------------------------------------++func newTitleParam(t *testing.T, method string, cap int, levels [][]string) titleChunkerParam {+ t.Helper()+ p := defaultsTitle()+ conf := map[string]any{"method": method, "chunk_token_cap": cap}+ if levels != nil {+ lv := make([]any, 0, len(levels))+ for _, g := range levels {+ inner := make([]any, 0, len(g))+ for _, s := range g {+ inner = append(inner, s)+ }+ lv = append(lv, inner)+ }+ conf["levels"] = lv+ }+ if method == "hierarchy" {+ conf["hierarchy"] = 1+ }+ p.Update(conf)+ // NOTE: validation is intentionally skipped here — the char-stub tests use+ // sub-128 caps (e.g. 20) that the production Validate() rejects, mirroring+ // the Python suite which stubs out check(). Range validation is covered+ // separately by TestTitleChunkerParam_ChunkTokenCapValidate.+ return p+}++func TestTitleCap_GroupPipeline_RespectsCap(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "group", 20, [][]string{{`^# `}})+ inputs := map[string]any{"output_format": "text", "text": body}+ got, err := invokeGroup(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeGroup: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected split, got %d", len(chunks))+ }+ assertCapInvariants(t, chunks, 20, body)+}++// TestTitleCap_GroupPipeline_ValidatedInRangeCap exercises the wired path with+// a cap inside the production-validated range (128..8000). The char-stub suite+// otherwise only uses sub-128 caps that production Validate() rejects, so this+// is the only end-to-end coverage of an accepted configuration.+func TestTitleCap_GroupPipeline_ValidatedInRangeCap(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Repeat("ab", 200) // 400 stub tokens > cap 128+ p := newTitleParam(t, "group", 128, [][]string{{`^# `}})+ if err := p.TitleChunkerParam.Validate(); err != nil {+ t.Fatalf("cap=128 must pass production Validate: %v", err)+ }+ inputs := map[string]any{"output_format": "text", "text": body}+ got, err := invokeGroup(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeGroup: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected cap=128 to re-split the oversized body, got %d", len(chunks))+ }+ assertCapInvariants(t, chunks, 128, body)+}++func TestTitleCap_HierarchyPipeline_RespectsCap(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})+ inputs := map[string]any{"output_format": "text", "text": body}+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeHierarchy: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected split, got %d", len(chunks))+ }+ assertCapInvariants(t, chunks, 20, body)+}++func TestTitleCap_GroupPipeline_CapZeroNoop(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "group", 0, [][]string{{`^# `}})+ inputs := map[string]any{"output_format": "text", "text": body}+ got, err := invokeGroup(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeGroup: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("cap=0 must keep 1 chunk, got %d", len(chunks))+ }+}++func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ // 6 records "S00。\n...\nS05。" -> built text is each on its own line.+ records := make([]string, 6)+ for i := 0; i < 6; i++ {+ records[i] = sprintfSentence(i)+ }+ body := strings.Join(records, "\n")+ p := newTitleParam(t, "group", 10, [][]string{{`^# `}})+ inputs := map[string]any{"output_format": "text", "text": body}+ got, err := invokeGroup(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeGroup: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected merged split, got %d", len(chunks))+ }+ // The built text joins each record with "\n" and appends a trailing "\n".+ var want strings.Builder+ for _, r := range records {+ want.WriteString(r)+ want.WriteString("\n")+ }+ assertCapInvariants(t, chunks, 10, want.String())+}++func TestTitleCap_HierarchyPipeline_PlanAPositions(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})+ inputs := map[string]any{+ "output_format": "chunks",+ "chunks": []schema.ChunkDoc{+ {Text: body, DocType: "text", Positions: json.RawMessage(`[[1,10,200,50,80]]`)},+ },+ }+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeHierarchy: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected split, got %d", len(chunks))+ }+ first := chunks[0]["positions"]+ if first == nil {+ t.Fatalf("first sub-chunk missing positions")+ }+ if fm, ok := first.([][]float64); !ok || len(fm) == 0 || fm[0][0] != 1 {+ t.Errorf("first positions = %#v, want [[1 10 200 50 80]]", first)+ }+ for i := 1; i < len(chunks); i++ {+ v, ok := chunks[i]["positions"]+ if !ok {+ t.Errorf("sub-chunk %d missing positions", i)+ continue+ }+ if vm, ok := v.([][]float64); !ok || len(vm) != 0 {+ t.Errorf("sub-chunk %d positions = %#v, want empty [][]float64", i, v)+ }+ }+}++// ---------------------------------------------------------------------------+// Schema validation+// ---------------------------------------------------------------------------++func TestTitleChunkerParam_ChunkTokenCapDefaults(t *testing.T) {+ if got := (schema.TitleChunkerParam{}).Defaults().ChunkTokenCap; got != 512 {+ t.Errorf("default ChunkTokenCap = %d, want 512", got)+ }+}++func TestTitleChunkerParam_ChunkTokenCapValidate(t *testing.T) {+ cases := []struct {+ cap int+ ok bool+ }{+ {0, true}, // disabled+ {50, false}, // below 128+ {127, false}, // below 128+ {128, true}, // lower bound+ {512, true}, // default+ {8000, true}, // upper bound+ {8001, false}, // above 8000+ {9000, false}, // above 8000+ }+ for _, c := range cases {+ p := schema.TitleChunkerParam{Method: "group", Levels: [][]string{{"^#"}}, ChunkTokenCap: c.cap}+ err := p.Validate()+ if c.ok && err != nil {+ t.Errorf("cap=%d: unexpected error %v", c.cap, err)+ }+ if !c.ok && err == nil {+ t.Errorf("cap=%d: expected error, got nil", c.cap)+ }+ }+}++// TestTitleChunkerParam_ChunkTokenCapValidate_EmptyMethodBypass pins the+// validation-order fix: the cap range check must run even when Method is ""+// (which otherwise early-returns nil), so an out-of-range cap cannot slip+// through as an active ceiling.+func TestTitleChunkerParam_ChunkTokenCapValidate_EmptyMethodBypass(t *testing.T) {+ p := schema.TitleChunkerParam{Method: "", Levels: [][]string{{"^#"}}, ChunkTokenCap: 1}+ if err := p.Validate(); err == nil {+ t.Error(`method="" with cap=1 must be rejected (out of 128..8000)`)+ }+ pOK := schema.TitleChunkerParam{Method: "", Levels: [][]string{{"^#"}}, ChunkTokenCap: 512}+ if err := pOK.Validate(); err != nil {+ t.Errorf(`method="" with cap=512 must pass: %v`, err)+ }+}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex fd6b5bb148..97b835e85f 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -364,13 +364,15 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, childreturn chunkOutputs(docs)}-// sentenceDelimiter is the sentence/clause-boundary regex used to split-// oversized sections. It mirrors the delimiter Python's chunker actually-// uses in production: rag/app/naive.py:1285 passes "\n!?。;!?" to-// naive_merge, which includes ASCII "!" and "?" as well as the CJK-// punctuation "。;!?". It deliberately does NOT include an English-// ". " fallback: Python's production delimiter has no "\.\s", so adding-// it would diverge from Python's chunk boundaries.+// sentenceDelimiter is the token-chunker TEXT-path sentence delimiter (the+// naive_merge port). It mirrors the delimiter Python's naive_merge receives in+// production: rag/app/naive.py passes "\n!?。;!?", which includes ASCII "!"+// and "?" plus the CJK punctuation "。;!?" but NOT an English ". " boundary.+//+// The Title chunker and the token chunker's JSON path use the shared+// sentenceBoundaryRe instead (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE,+// which adds ". "). The two constants mirror two distinct Python delimiters and+// differ only in that English boundary, exactly as Python does.var sentenceDelimiter = regexp.MustCompile(`(\n|[!?。;!?])`)// overlapCut returns the visible-text rune offset where the overlap prefixdiff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.goindex 0e1ee3e629..26c6be3884 100644--- a/internal/ingestion/component/schema/chunker.go+++ b/internal/ingestion/component/schema/chunker.go@@ -417,17 +417,26 @@ type TitleChunkerParam struct {// RootChunkAsHeading, when true, prepends the root chunk's text// to every emitted chunk (and drops the root chunk itself).RootChunkAsHeading bool `json:"root_chunk_as_heading"`++ // ChunkTokenCap is the hard ceiling on each emitted text chunk's token+ // count (mirrors Python title_chunker/common.py chunk_token_cap, added in+ // #18455). A built text chunk exceeding it is re-split on sentence+ // boundaries into <= cap sub-chunks. 0 disables the ceiling. The Python+ // default is 512; valid range when non-zero is 128..8000.+ ChunkTokenCap int `json:"chunk_token_cap"`}// Defaults returns the Python default TitleChunkerParam. `Method` is// not initialized in the Python `__init__` (it is set externally); the// default is left as the empty string and the component must supply it.+// `ChunkTokenCap` defaults to 512, matching Python #18455.func (TitleChunkerParam) Defaults() TitleChunkerParam {return TitleChunkerParam{Levels: [][]string{},Hierarchy: nil,IncludeHeadingContent: false,RootChunkAsHeading: false,+ ChunkTokenCap: 512,}}@@ -435,6 +444,14 @@ func (TitleChunkerParam) Defaults() TitleChunkerParam {// expressible in pure-data terms: when Method == "hierarchy" the// hierarchy depth and level config must be present.func (p *TitleChunkerParam) Validate() error {+ // chunk_token_cap: 0 disables the ceiling; when set it must be a positive+ // integer in [128, 8000] (mirrors Python title_chunker #18455). Checked+ // BEFORE the method switch so an unset method cannot bypass the range.+ if p.ChunkTokenCap != 0 {+ if p.ChunkTokenCap < 128 || p.ChunkTokenCap > 8000 {+ return errInvalidValue{Field: "chunk_token_cap", Value: fmt.Sprintf("%d", p.ChunkTokenCap)}+ }+ }switch p.Method {case "hierarchy", "group":case "":