RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): enforce strict chunk_token_num cap on .txt / PDF / email paths (#17203)
Sdeb3d0c20123
diff --git a/deepdoc/parser/txt_parser.py b/deepdoc/parser/txt_parser.pyindex a89e67846e..60b310cd46 100644--- a/deepdoc/parser/txt_parser.py+++ b/deepdoc/parser/txt_parser.py@@ -14,10 +14,12 @@# limitations under the License.#+import loggingimport re-from deepdoc.parser.utils import get_textfrom common.token_utils import num_tokens_from_string+from deepdoc.parser.utils import get_text+from rag.nlp import _split_oversized_unitclass RAGFlowTxtParser:@@ -36,23 +38,29 @@ class RAGFlowTxtParser:def add_chunk(t):nonlocal cks, tk_nums, delimitertnum = num_tokens_from_string(t)- if tk_nums[-1] > chunk_token_num:- cks.append(t)- tk_nums.append(tnum)- else:- if cks[-1]:- cks[-1] += "\n" + t- else:- cks[-1] += t- tk_nums[-1] += tnum++ if cks[-1] == "":+ cks[-1] = t+ tk_nums[-1] = tnum+ return++ merged = cks[-1] + "\n" + t+ merged_tnum = num_tokens_from_string(merged)+ if merged_tnum <= chunk_token_num:+ cks[-1] = merged+ tk_nums[-1] = merged_tnum+ return++ cks.append(t)+ tk_nums.append(tnum)dels = []s = 0for m in re.finditer(r"`([^`]+)`", delimiter):- f, t = m.span()+ f, m_t = m.span()dels.append(m.group(1))dels.extend(list(delimiter[s:f]))- s = t+ s = m_tif s < len(delimiter):dels.extend(list(delimiter[s:]))dels = [re.escape(d) for d in dels if d]@@ -62,6 +70,15 @@ class RAGFlowTxtParser:for sec in secs:if re.match(f"^{dels}$", sec):continue- add_chunk(sec)+ if not sec:+ continue+ if num_tokens_from_string(sec) <= chunk_token_num:+ add_chunk(sec)+ continue+ pieces = _split_oversized_unit(sec, chunk_token_num, token_count_fn=num_tokens_from_string)+ logging.debug("parser_txt: split oversized section (%d tokens) into %d pieces", num_tokens_from_string(sec), len(pieces))+ for piece in pieces:+ add_chunk(piece)+ logging.debug("parser_txt: %d sections -> %d chunks (chunk_token_num=%d)", len(secs), len(cks), chunk_token_num)return [[c, ""] for c in cks]diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex ca445867dd..07ca2e815b 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -337,13 +337,145 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string// it would diverge from Python's chunk boundaries.var sentenceDelimiter = regexp.MustCompile(`(\n|[!?。;!?])`)+// atomRE matches whitespace runs or non-whitespace runs. Mirrors Python+// `_split_oversized_unit`'s `re.findall(r"\s+|\S+", text)`.+var atomRE = regexp.MustCompile(`\s+|\S+`)++// splitAtomByTokenBudget splits a single non-whitespace atom into+// substrings that each have <= chunkTokenNum tokens. Mirrors Python+// rag/nlp._split_atom_by_token_budget (binary search on rune prefixes).+func splitAtomByTokenBudget(atom string, chunkTokenNum int, countFn func(string) int) []string {+ if atom == "" {+ return nil+ }+ if countFn == nil {+ countFn = tokenizeStr+ }+ if countFn(atom) <= chunkTokenNum {+ return []string{atom}+ }+ runes := []rune(atom)+ var pieces []string+ start := 0+ n := len(runes)+ for start < n {+ low := start + 1+ high := n+ bestEnd := start + 1+ for low <= high {+ mid := (low + high) / 2+ if countFn(string(runes[start:mid])) <= chunkTokenNum {+ bestEnd = mid+ low = mid + 1+ } else {+ high = mid - 1+ }+ }+ pieces = append(pieces, string(runes[start:bestEnd]))+ start = bestEnd+ }+ return pieces+}++// splitOversizedUnit splits a unit that exceeds chunkTokenNum tokens into+// pieces that each fit the budget. Whitespace is the primary break (mirrors+// Python rag/nlp._split_oversized_unit / HtmlParser._split_oversized_block);+// a single non-whitespace run longer than the budget falls back to+// token-budget-based character windows.+func splitOversizedUnit(text string, chunkTokenNum int) []string {+ return splitOversizedUnitWith(text, chunkTokenNum, tokenizeStr)+}++func splitOversizedUnitWith(text string, chunkTokenNum int, countFn func(string) int) []string {+ if countFn == nil {+ countFn = tokenizeStr+ }+ if countFn(text) <= chunkTokenNum {+ return []string{text}+ }+ var pieces []string+ current := ""+ tokenCache := map[string]int{}++ atomTokens := func(atom string) int {+ // Whitespace-only atoms contribute 0 in isolation (mirrors Python+ // atom.isspace()), matching the packing heuristic used by+ // rag/nlp._split_oversized_unit. Fit checks below still use an+ // exact projected countFn(current+atom) so cl100k space-join+ // effects cannot push a piece over the hard cap.+ if strings.TrimSpace(atom) == "" {+ return 0+ }+ if n, ok := tokenCache[atom]; ok {+ return n+ }+ n := countFn(atom)+ tokenCache[atom] = n+ return n+ }++ for _, atom := range atomRE.FindAllString(text, -1) {+ aTokens := atomTokens(atom)+ if aTokens > chunkTokenNum && strings.TrimSpace(atom) != "" {+ if current != "" {+ pieces = append(pieces, current)+ current = ""+ }+ pieces = append(pieces, splitAtomByTokenBudget(atom, chunkTokenNum, countFn)...)+ continue+ }+ // Exact projected-total check (not sum of atom counts): cl100k can+ // count a joined "word word" differently than token(word)+token(word).+ if current != "" && countFn(current+atom) > chunkTokenNum {+ pieces = append(pieces, current)+ current = ""+ // Leading whitespace after a flush has no content value; drop it+ // so the next piece does not start with a pure-space prefix that+ // would never fit usefully on its own.+ if strings.TrimSpace(atom) == "" {+ continue+ }+ // If the atom alone still exceeds (pathological), carve it.+ if atomTokens(atom) > chunkTokenNum {+ pieces = append(pieces, splitAtomByTokenBudget(atom, chunkTokenNum, countFn)...)+ continue+ }+ }+ current += atom+ }+ if current != "" {+ pieces = append(pieces, current)+ }+ return pieces+}++// computeOverlapPrefix returns (overlapText, overlapTokenCount) carved from+// the tail of prevText after stripping parser tags. overlappedPct is a+// percentage in [0, 100]. Mirrors Python rag/nlp._compute_overlap_prefix.+func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int) {+ visible := removeTag(prevText)+ if visible == "" {+ return "", 0+ }+ runes := []rune(visible)+ cut := int(float64(len(runes)) * (100 - overlappedPct) / 100.0)+ if cut < 0 {+ cut = 0+ }+ if cut >= len(runes) {+ return "", 0+ }+ overlap := string(runes[cut:])+ return overlap, tokenizeStr(overlap)+}+// mergeByTokenSize implements exact token-based chunk merging that mirrors-// Python's naive_merge (rag/nlp/__init__.py:1156). It uses-// tokenizeStr (= tokenizer.NumTokensFromString, cl100k_base BPE) for-// precise token counting, treats the payload as a single section, splits-// oversized sections on sentence delimiters (dropping the delimiter, as-// Python does), and greedily merges into chunks of approximately-// chunk_token_size tokens with optional overlap from the previous chunk.+// Python's naive_merge (rag/nlp/__init__.py) after the strict chunk_token_num+// hard-cap fix. It uses tokenizeStr for precise token counting, treats the+// payload as a single section, splits oversized sections on production sentence+// delimiters, hard-caps atomic oversize units via splitOversizedUnit, and merges+// only when the projected total stays within chunk_token_size. Overlap is+// applied only when the resulting chunk still fits the budget.func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *regexp.Regexp) map[string]any {target := c.param.ChunkTokenSizeoverlapPct := c.param.OverlappedPercent@@ -359,61 +491,58 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r}// Normalize line endings to LF before any splitting. Python's- // naive_merge (rag/nlp/__init__.py:1166) runs- // text = text.replace("\r\n", "\n").replace("\r", "\n")- // so CRLF/CR input must segment and split exactly like LF input.- // Without this, stray "\r" would survive inside chunks, diverging- // from Python.+ // naive_merge runs text.replace("\r\n", "\n").replace("\r", "\n"),+ // then treats the input string as one section.text = strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")-- // Treat the whole payload as a single section, mirroring Python's- // naive_merge (rag/nlp/__init__.py:1157) which wraps the input string- // as a one-element list. naive_merge does NOT pre-split on blank- // lines, and because "\n" is itself a delimiter it is dropped (blank- // lines collapse), exactly as Python does. CRLF/CR normalization- // already happened above.sections := []string{text}if len(sections) == 0 {return emptyOutputs()}- // Sentence/clause-boundary regex for splitting oversized sections.- // Mirrors Python's production delimiter (rag/app/naive.py:1285 passes- // "\n!?。;!?") — ASCII "!" and "?" plus the CJK punctuation, with no- // English ". " fallback.- sentenceDelim := sentenceDelimiter-- var cks []string // chunk texts- var tkns []int // token counts per chunk-- // mergeOrNew mirrors Python add_chunk in naive_merge:- // - If the current chunk is empty or would overflow the- // threshold → start a new chunk (with optional overlap prefix).- // - Otherwise → merge into the current chunk.- mergeOrNew := func(segment string, tokens int) {- threshold := float64(target) * (100 - overlapPct) / 100.0- if len(cks) == 0 || float64(tkns[len(tkns)-1]) > threshold {- seg := segment- segTokens := tokens- if overlapPct > 0 && len(cks) > 0 {- // Strip parser tags before computing the overlap suffix,- // matching Python nlp/__init__.py:1181- prev := removeTag(cks[len(cks)-1])- // Take the last overlapped_percent of the previous chunk- // (in runes, matching Python's len(overlapped) * ratio).- prevRunes := []rune(prev)- cut := int(float64(len(prevRunes)) * (100 - overlapPct) / 100.0)- if cut < len(prevRunes) {- suffix := string(prevRunes[cut:])- seg = suffix + segment- segTokens = tokenizeStr(seg)+ var cks []string+ var tkns []int++ // addChunk applies the projected-total merge and optional-overlap decision+ // to one unit that already fits target.+ addChunk := func(segment string) {+ tnum := tokenizeStr(segment)+ if len(cks) == 0 {+ cks = append(cks, segment)+ tkns = append(tkns, tnum)+ return+ }+ merged := cks[len(cks)-1] + segment+ mergedN := tokenizeStr(merged)+ if mergedN <= target {+ cks[len(cks)-1] = merged+ tkns[len(tkns)-1] = mergedN+ return+ }+ newText := segment+ newTokens := tnum+ if overlapPct > 0 {+ overlapText, _ := computeOverlapPrefix(cks[len(cks)-1], overlapPct)+ if overlapText != "" {+ candidate := overlapText + segment+ if candidateTokens := tokenizeStr(candidate); candidateTokens <= target {+ newText = candidate+ newTokens = candidateTokens}}- cks = append(cks, seg)- tkns = append(tkns, segTokens)- } else {- cks[len(cks)-1] += segment- tkns[len(tkns)-1] += tokens+ }+ cks = append(cks, newText)+ tkns = append(tkns, newTokens)+ }++ addUnit := func(unit string) {+ if tokenizeStr(unit) <= target {+ addChunk(unit)+ return+ }+ slog.Debug("TokenChunker: splitting oversized unit via splitOversizedUnit",+ "len", len(unit), "tokens", tokenizeStr(unit), "chunk_token_size", target)+ for _, piece := range splitOversizedUnit(unit, target) {+ addChunk(piece)}}@@ -423,46 +552,24 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *rcontinue}t := "\n" + sec- tn := tokenizeStr(t)-- if tn < 8 {- // Tiny section — always merge into the previous chunk.- if len(cks) > 0 {- cks[len(cks)-1] += t- tkns[len(tkns)-1] += tn- } else {- cks = append(cks, t)- tkns = append(tkns, tn)- }+ if tokenizeStr(t) <= target {+ addChunk(t)continue}-- if tn <= target {- mergeOrNew(t, tn)- continue- }-- // Oversized section: split on sentence delimiters. Python's- // naive_merge (rag/nlp/__init__.py:1216-1225) splits with a- // capturing-group re.split but then SKIPS any segment that is a- // pure delimiter (re.fullmatch(dels, sub_sec)), so the delimiter- // character (\n / 。 / ! / ?) is DROPPED from the chunk text- // rather than retained. We mirror that by using regexp.Split- // (which discards the delimiter) and prepending a single "\n" to- // each segment, matching Python's add_chunk("\n"+sub_sec).- parts := sentenceDelim.Split(sec, -1)+ // Oversized section: split on production sentence delimiters, then+ // hard-cap any unit that still exceeds the budget (unbroken atoms).+ parts := sentenceDelimiter.Split(sec, -1)+ hadPart := falsefor _, part := range parts {part = strings.TrimSpace(part)if part == "" {continue}- // Route every segment — including tiny <8-token fragments —- // through mergeOrNew so it honours the token threshold,- // mirroring Python's add_chunk. The old shortcut appended- // unconditionally, merging fragments into an already-overfull- // chunk (review #2).- p := "\n" + part- mergeOrNew(p, tokenizeStr(p))+ hadPart = true+ addUnit("\n" + part)+ }+ if !hadPart {+ addUnit(t)}}@@ -764,75 +871,99 @@ func takeFromStart(text string, tokens int) string {return best}-// mergeByTokenSizeFromJSON mirrors `naive_merge` at-// rag/nlp/__init__.py:1156.+// mergeByTokenSizeFromJSON mirrors Python naive_merge's projected-total+// hard cap (rag/nlp/__init__.py after the strict chunk_token_num fix).+// Oversized text units are sub-split via splitOversizedUnit before merge;+// overlap is applied only when overlap+segment still fits the budget.func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overlappedPct float64) [][]schema.ChunkDoc {- // overlappedPct is a [0,100] percentage. Clamp so the merge math below- // never yields a negative/inverted threshold for out-of-range input- // (review: yuzhichang, PR #17396).+ // overlappedPct is a [0,100] percentage. Clamp defensively because this+ // helper is also exercised directly by tests.if overlappedPct < 0 {overlappedPct = 0} else if overlappedPct > 100 {overlappedPct = 100}- threshold := float64(chunkTokens) * (100 - overlappedPct) / 100.0for idx := range perItem {chunks := perItem[idx]if len(chunks) == 0 {continue}var merged []schema.ChunkDoc++ // addTextChunk applies the projected-total merge / overlap-drop+ // decision for one text unit that already fits chunkTokens.+ addTextChunk := func(ck schema.ChunkDoc) {+ tk := intValue(ck.TKNums)+ if tk <= 0 {+ tk = tokenizeStr(ck.Text)+ ck.TKNums = intPtr(tk)+ }+ if len(merged) == 0 || merged[len(merged)-1].CKType != "text" {+ // First text chunk, or first text after a non-text chunk:+ // no prior text to overlap with.+ merged = append(merged, cloneChunkDoc(ck))+ return+ }+ prev := &merged[len(merged)-1]+ // Empty previous text: assign incoming text directly+ // (diff Chunker-2.11 / token_chunker.py:236-239).+ if prev.Text == "" {+ prev.Text = ck.Text+ prev.TKNums = intPtr(tk)+ prev.PDFPositions = extendRawJSONArray(prev.PDFPositions, ck.PDFPositions)+ prev.Positions = extendRawJSONArray(prev.Positions, ck.Positions)+ return+ }+ // Proactive projected-total merge (joined with "\n").+ joined := prev.Text + "\n" + ck.Text+ joinedN := tokenizeStr(joined)+ if joinedN <= chunkTokens {+ prev.Text = joined+ prev.TKNums = intPtr(joinedN)+ prev.PDFPositions = extendRawJSONArray(prev.PDFPositions, ck.PDFPositions)+ prev.Positions = extendRawJSONArray(prev.Positions, ck.Positions)+ return+ }+ // Start a new chunk; apply overlap only when it still fits.+ cp := cloneChunkDoc(ck)+ if overlappedPct > 0 {+ if overlapText, overlapTokens := computeOverlapPrefix(prev.Text, overlappedPct); overlapTokens > 0 && overlapTokens+tk <= chunkTokens {+ cp.Text = overlapText + cp.Text+ cp.TKNums = intPtr(tokenizeStr(cp.Text))+ }+ }+ merged = append(merged, cp)+ }+for _, ck := range chunks {- ckType := ck.CKType- if ckType != "text" {+ if ck.CKType != "text" {merged = append(merged, cloneChunkDoc(ck))continue}tk := intValue(ck.TKNums)- // Mirror Python's naive_merge.add_chunk: start a new chunk- // when either (a) this is the first text chunk, or- // (b) the currently accumulated chunk already exceeds the- // threshold (not the incoming segment).- if len(merged) == 0 || merged[len(merged)-1].CKType != "text" ||- float64(intValue(merged[len(merged)-1].TKNums)) > threshold {- cp := cloneChunkDoc(ck)- // Overlap: prepend tail of previous chunk onto the new- // chunk, matching Python's- // t = overlapped[overlap_cut:] + t- // tnum = num_tokens_from_string(t)- if len(merged) > 0 && merged[len(merged)-1].CKType == "text" && overlappedPct > 0 {- // Strip parser tags before computing the overlap- // suffix, matching Python nlp/__init__.py:1181- //- if prevText := removeTag(merged[len(merged)-1].Text); prevText != "" {- runes := []rune(prevText)- cut := int(float64(len(runes)) * (100 - overlappedPct) / 100.0)- if cut < len(runes) {- cp.Text = string(runes[cut:]) + cp.Text- cp.TKNums = intPtr(tokenizeStr(cp.Text))- }- }- }- merged = append(merged, cp)+ if tk <= 0 {+ tk = tokenizeStr(ck.Text)+ }+ if tk <= chunkTokens {+ addTextChunk(ck)continue}- // Merge into the accumulated text chunk.- prev := &merged[len(merged)-1]- // Mirror Python token_chunker.py:236-239: when the accumulated- // chunk has empty text, assign the incoming text directly instead- // of skipping it- if prev.Text == "" {- prev.Text = ck.Text- } else {- prev.Text = prev.Text + "\n" + ck.Text+ // Hard-cap atomic oversize units before merge.+ slog.Debug("TokenChunker: splitting oversized JSON unit via splitOversizedUnit",+ "len", len(ck.Text), "tokens", tk, "chunk_token_size", chunkTokens)+ for _, piece := range splitOversizedUnit(ck.Text, chunkTokens) {+ if strings.TrimSpace(piece) == "" {+ continue+ }+ cp := cloneChunkDoc(ck)+ cp.Text = piece+ cp.TKNums = intPtr(tokenizeStr(piece))+ // Coordinates stay on the first piece only to avoid duplicating+ // PDF bboxes across atom slices.+ addTextChunk(cp)+ ck.PDFPositions = nil+ ck.Positions = nil}- prev.TKNums = intPtr(intValue(prev.TKNums) + tk)- // Preserve PDF coordinates across the merge: extend the- // coordinate lists instead of dropping the incoming item's- // positions. Mirrors Python token_chunker.py:240- // `merged[prev][PDF_POSITIONS_KEY].extend(...)` (diffs 2.5 / 2.3).- prev.PDFPositions = extendRawJSONArray(prev.PDFPositions, ck.PDFPositions)- prev.Positions = extendRawJSONArray(prev.Positions, ck.Positions)}perItem[idx] = merged}diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.goindex 7e869e1ce4..0ec5fd41d4 100644--- a/internal/ingestion/component/chunker/token_batch1_test.go+++ b/internal/ingestion/component/chunker/token_batch1_test.go@@ -54,18 +54,40 @@ func TestSentenceDelimiterMatchesBangAndQuestion(t *testing.T) {// from the previous chunk AFTER remove_tag, otherwise parser tags (e.g.// "@@1\t2.3##") leak into the overlap region. Mirrors Python// nlp/__init__.py:1181 (remove_tag applied before overlap).+//+// After the strict-cap fix, a new chunk is started only when the projected+// join exceeds the budget — so the first unit must already sit near the+// budget and the second unit must not fit alongside it.func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {+ // Size a and b so:+ // - each unit alone fits the budget (no atom-split),+ // - the projected join exceeds the budget (forces a new chunk),+ // - overlap+b still fits (so the overlap path is exercised).aText := strings.Repeat("word ", 20) + "@@1\t2.3## tail"+ bText := "body"+ aN, bN := tokenizeStr(aText), tokenizeStr(bText)+ joinedN := tokenizeStr(aText + "\n" + bText)+ // Budget just below the join so a and b cannot merge, but each alone fits.+ budget := joinedN - 1+ if budget < aN {+ budget = aN+ }+ if budget < bN {+ budget = bN+ }+ if joinedN <= budget {+ t.Fatalf("could not derive tight budget (a=%d b=%d joined=%d budget=%d)", aN, bN, joinedN, budget)+ }items := [][]schema.ChunkDoc{{- {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(100)},- {Text: "body", DocType: "text", CKType: "text", TKNums: intPtr(5)},+ {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},+ {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},},}- got := mergeByTokenSizeFromJSON(items, 128, 30.0)+ got := mergeByTokenSizeFromJSON(items, budget, 30.0)merged := got[0]if len(merged) != 2 {- t.Fatalf("want 2 merged chunks (overlap path), got %d", len(merged))+ t.Fatalf("want 2 merged chunks (overlap path), got %d (a=%d b=%d budget=%d)", len(merged), aN, bN, budget)}// The overlap prefix is prepended to the SECOND chunk. The original// first chunk legitimately keeps its own parser tag; only the overlap@@ -73,6 +95,9 @@ func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {if strings.Contains(merged[1].Text, "@@") || strings.Contains(merged[1].Text, "##") {t.Errorf("overlap prefix leaked parser tag into chunk 1: %q", merged[1].Text)}+ if n := tokenizeStr(merged[1].Text); n > budget {+ t.Errorf("overlap pushed second chunk over budget: tokens=%d", n)+ }}// TestMergeByTokenSizeFromJSON_ClampsOverlappedPct locks the review findingdiff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.gonew file mode 100644index 0000000000..c00774a552--- /dev/null+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go@@ -0,0 +1,266 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "context"+ "strings"+ "testing"+ "unicode/utf8"++ "ragflow/internal/ingestion/component/schema"+)++// wordCount is a deterministic tokenizer stand-in used only via+// splitOversizedUnitWith in unit-level helper tests.+func wordCount(s string) int {+ s = strings.TrimSpace(s)+ if s == "" {+ return 0+ }+ return len(strings.Fields(s))+}++func charCount(s string) int { return utf8.RuneCountInString(s) }++func TestSplitOversizedUnit_WhitespacePacksToBudget(t *testing.T) {+ // 100 words, budget 30 → must yield multiple pieces, each ≤ 30 words.+ text := strings.TrimSpace(strings.Repeat("word ", 100))+ pieces := splitOversizedUnitWith(text, 30, wordCount)+ if len(pieces) < 2 {+ t.Fatalf("want multiple pieces, got %d: %#v", len(pieces), pieces)+ }+ total := 0+ for _, p := range pieces {+ n := wordCount(p)+ if n > 30 {+ t.Errorf("piece exceeds budget: tokens=%d text=%q", n, p)+ }+ total += n+ }+ if total != 100 {+ t.Errorf("word count not preserved: got %d want 100", total)+ }+}++func TestSplitOversizedUnit_UnbrokenAtomFallsBackToCharWindows(t *testing.T) {+ // Unbroken run with char-as-token counting — must sub-split on runes.+ atom := strings.Repeat("a", 80)+ pieces := splitOversizedUnitWith(atom, 50, charCount)+ if len(pieces) < 2 {+ t.Fatalf("want >=2 pieces for unbroken atom, got %d", len(pieces))+ }+ joined := strings.Join(pieces, "")+ if joined != atom {+ t.Errorf("content not preserved: got %q", joined)+ }+ for _, p := range pieces {+ if charCount(p) > 50 {+ t.Errorf("piece exceeds budget: %d runes in %q", charCount(p), p)+ }+ }+}++func TestSplitOversizedUnit_WithinBudgetUnchanged(t *testing.T) {+ text := "hello world"+ pieces := splitOversizedUnitWith(text, 100, wordCount)+ if len(pieces) != 1 || pieces[0] != text {+ t.Fatalf("within-budget text must be returned as-is, got %#v", pieces)+ }+}++func TestComputeOverlapPrefix_StripsTagsAndCounts(t *testing.T) {+ prev := strings.Repeat("word ", 20) + "@@1\t2.3## tail"+ overlap, n := computeOverlapPrefix(prev, 30)+ if strings.Contains(overlap, "@@") || strings.Contains(overlap, "##") {+ t.Errorf("overlap must strip parser tags, got %q", overlap)+ }+ if n <= 0 {+ t.Errorf("overlap token count must be >0, got %d", n)+ }+ if tokenizeStr(overlap) != n {+ t.Errorf("reported tokens %d != tokenizeStr(overlap) %d", n, tokenizeStr(overlap))+ }+}++func TestMergeByTokenSizeFromJSON_StrictCapNoOvershoot(t *testing.T) {+ // Eight 25-token-ish sections under a 50-token budget must pack without+ // any chunk exceeding the budget (Python test_strict_cap_no_overlap).+ const budget = 50+ sections := make([]schema.ChunkDoc, 0, 8)+ for i := 0; i < 8; i++ {+ text := strings.TrimSpace(strings.Repeat("w ", 25))+ sections = append(sections, schema.ChunkDoc{+ Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),+ })+ }+ got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 0)+ merged := got[0]+ if len(merged) < 3 {+ t.Fatalf("want >=3 chunks, got %d", len(merged))+ }+ for i, ck := range merged {+ n := tokenizeStr(ck.Text)+ if n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d text_len=%d", i, n, len(ck.Text))+ }+ }+}++func TestMergeByTokenSizeFromJSON_OverlapDroppedAtOverflow(t *testing.T) {+ // With a tight budget, overlap must never push a chunk over the cap.+ const budget = 25+ sections := make([]schema.ChunkDoc, 0, 20)+ for i := 0; i < 20; i++ {+ text := strings.TrimSpace(strings.Repeat("w ", 10))+ sections = append(sections, schema.ChunkDoc{+ Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),+ })+ }+ got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 20)+ for i, ck := range got[0] {+ if n := tokenizeStr(ck.Text); n > budget {+ t.Errorf("chunk %d exceeds budget with overlap: tokens=%d", i, n)+ }+ }+}++func TestMergeByTokenSizeFromJSON_OversizedUnitIsSubSplit(t *testing.T) {+ // A single unit larger than the budget must be atom-split before merge.+ const budget = 30+ long := strings.TrimSpace(strings.Repeat("word ", 100))+ items := [][]schema.ChunkDoc{{+ {Text: long, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(long))},+ }}+ got := mergeByTokenSizeFromJSON(items, budget, 0)+ if len(got[0]) < 2 {+ t.Fatalf("oversized unit must yield multiple chunks, got %d", len(got[0]))+ }+ for i, ck := range got[0] {+ if n := tokenizeStr(ck.Text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d", i, n)+ }+ }+}++func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {+ // End-to-end text path: long multi-paragraph input under a tight budget.+ const budget = 40+ var b strings.Builder+ for i := 0; i < 30; i++ {+ b.WriteString(strings.TrimSpace(strings.Repeat("word ", 15)))+ b.WriteString("\n\n")+ }+ comp, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "token_size",+ "chunk_token_size": budget,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ tc := comp.(*TokenChunkerComponent)+ out := tc.mergeByTokenSize(b.String(), nil)+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) < 2 {+ t.Fatalf("want multiple chunks, got %d", len(chunks))+ }+ for i, ck := range chunks {+ text, _ := ck["text"].(string)+ if n := tokenizeStr(text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d", i, n)+ }+ }+}++func TestMergeByTokenSize_UnbrokenAtomStrictCap(t *testing.T) {+ // Unbroken dense string (no whitespace / sentence delim) must still+ // hard-cap via the character-window fallback inside splitOversizedUnit.+ const budget = 20+ // Use many distinct ASCII letters so cl100k does not collapse the whole+ // run into a handful of tokens.+ var b strings.Builder+ for i := 0; i < 400; i++ {+ b.WriteByte(byte('a' + i%26))+ }+ text := b.String()+ if tokenizeStr(text) <= budget {+ t.Skipf("tokenizer collapsed unbroken atom to %d tokens (<= budget)", tokenizeStr(text))+ }+ comp, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "token_size",+ "chunk_token_size": budget,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ tc := comp.(*TokenChunkerComponent)+ out := tc.mergeByTokenSize(text, nil)+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) < 2 {+ t.Fatalf("want multiple chunks for unbroken atom, got %d (total_tokens=%d)", len(chunks), tokenizeStr(text))+ }+ var joined strings.Builder+ for i, ck := range chunks {+ s, _ := ck["text"].(string)+ joined.WriteString(s)+ if n := tokenizeStr(s); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d text=%q", i, n, s)+ }+ }+ // mergeByTokenSize prefixes "\n" on sections; stripping newlines recovers+ // the original unbroken atom.+ if strings.ReplaceAll(joined.String(), "\n", "") != text {+ t.Errorf("content not preserved after stripping newlines: got %q", joined.String())+ }+}++func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {+ const budget = 32+ var b strings.Builder+ for i := 0; i < 20; i++ {+ b.WriteString(strings.TrimSpace(strings.Repeat("alpha ", 12)))+ b.WriteByte('\n')+ }+ comp, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "token_size",+ "chunk_token_size": budget,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := comp.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.txt",+ "output_format": "text",+ "text": b.String(),+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if errMsg, _ := out["_ERROR"].(string); errMsg != "" {+ t.Fatalf("Invoke error payload: %s", errMsg)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) == 0 {+ t.Fatalf("expected chunks, got %#v", out)+ }+ for i, ck := range chunks {+ text, _ := ck["text"].(string)+ if n := tokenizeStr(text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d", i, n)+ }+ }+}diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.pyindex 793b972d13..a743637c31 100644--- a/rag/nlp/__init__.py+++ b/rag/nlp/__init__.py@@ -14,19 +14,19 @@# limitations under the License.#+import copyimport loggingimport random+import refrom collections import Counter, defaultdict-from common.token_utils import num_tokens_from_string-import re-import copy+import chardetimport roman_numbers as r-from word2number import w2nfrom cn2an import cn2anfrom PIL import Image+from word2number import w2n-import chardet+from common.token_utils import num_tokens_from_string__all__ = ["rag_tokenizer"]@@ -394,7 +394,7 @@ def tokenize_chunks(chunks, doc, eng, pdf_parser=None, child_delimiters_pattern=for ii, ck in enumerate(chunks):if len(ck.strip()) == 0:continue- logging.debug("-- {}".format(ck))+ logging.debug(f"-- {ck}")d = copy.deepcopy(doc)if pdf_parser:try:@@ -422,7 +422,7 @@ def doc_tokenize_chunks_with_images(chunks, doc, eng, child_delimiters_pattern=Ntext = ck.get("context_above", "") + ck.get("text") + ck.get("context_below", "")if len(text.strip()) == 0:continue- logging.debug("-- {}".format(ck))+ logging.debug(f"-- {ck}")d = copy.deepcopy(doc)if ck.get("image"):d["image"] = ck.get("image")@@ -448,7 +448,7 @@ def tokenize_chunks_with_images(chunks, doc, eng, images, child_delimiters_pattefor ii, (ck, image) in enumerate(zip(chunks, images)):if len(ck.strip()) == 0:continue- logging.debug("-- {}".format(ck))+ logging.debug(f"-- {ck}")d = copy.deepcopy(doc)d["image"] = imageadd_positions(d, [[ii] * 5])@@ -940,7 +940,7 @@ def remove_contents_table(sections, eng=False):def get(i):nonlocal sections- return (sections[i] if isinstance(sections[i], type("")) else sections[i][0]).strip()+ return (sections[i] if isinstance(sections[i], str) else sections[i][0]).strip()if not re.match(r"(contents|目录|目次|table of contents|致谢|acknowledge)$", re.sub(r"( | |\u3000)+", "", get(i).split("@@")[0], flags=re.IGNORECASE)):i += 1@@ -968,7 +968,7 @@ def remove_contents_table(sections, eng=False):def make_colon_as_title(sections):if not sections:return []- if isinstance(sections[0], type("")):+ if isinstance(sections[0], str):return sectionsi = 0while i < len(sections):@@ -1020,7 +1020,7 @@ def not_title(txt):def tree_merge(bull, sections, depth):if not sections or bull < 0:return sections- if isinstance(sections[0], type("")):+ if isinstance(sections[0], str):sections = [(s, "") for s in sections]# filter out position information in pdf sections@@ -1033,11 +1033,10 @@ def tree_merge(bull, sections, depth):for i, title in enumerate(BULLET_PATTERN[bull]):if re.match(title, text.strip()) and not not_bullet(text):return i + 1, text+ if re.search(r"(title|head)", layout) and not not_title(text):+ return len(BULLET_PATTERN[bull]) + 1, textelse:- if re.search(r"(title|head)", layout) and not not_title(text):- return len(BULLET_PATTERN[bull]) + 1, text- else:- return len(BULLET_PATTERN[bull]) + 2, text+ return len(BULLET_PATTERN[bull]) + 2, textlevel_set = set()lines = []@@ -1068,7 +1067,7 @@ def tree_merge(bull, sections, depth):def hierarchical_merge(bull, sections, depth):if not sections or bull < 0:return []- if isinstance(sections[0], type("")):+ if isinstance(sections[0], str):sections = [(s, "") for s in sections]sections = [(t, o) for t, o in sections if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip())]bullets_size = len(BULLET_PATTERN[bull])@@ -1154,9 +1153,136 @@ def hierarchical_merge(bull, sections, depth):return res-def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;!?", overlapped_percent=0):- from deepdoc.parser.pdf_parser import RAGFlowPdfParser+def _compute_overlap_prefix(prev_text, overlapped_percent):+ """Return (overlap_text, overlap_token_count) carved from the tail of ``prev_text``.++ ``prev_text`` is treated as if HTML/PDF markup has been stripped, so the carve+ index is computed against the visible characters, matching the existing+ behaviour of ``RAGFlowPdfParser.remove_tag`` callers above.+ """+ visible = re.sub(r"@@[\t0-9.-]+?##", "", prev_text or "")+ if not visible:+ return "", 0+ overlap_start = int(len(visible) * (100 - overlapped_percent) / 100.0)+ overlap_text = visible[overlap_start:]+ return overlap_text, num_tokens_from_string(overlap_text)+++def _split_atom_by_token_budget(atom, chunk_token_num, token_count_fn=None):+ """Split a single non-whitespace string `atom` into substrings that each+ have <= chunk_token_num tokens.+ """+ if token_count_fn is None:+ token_count_fn = num_tokens_from_string+ if not atom:+ return []+ if token_count_fn(atom) <= chunk_token_num:+ return [atom]+ pieces = []+ start = 0+ n = len(atom)+ while start < n:+ low = start + 1+ high = n+ best_end = start + 1+ while low <= high:+ mid = (low + high) // 2+ substring = atom[start:mid]+ if token_count_fn(substring) <= chunk_token_num:+ best_end = mid+ low = mid + 1+ else:+ high = mid - 1+ pieces.append(atom[start:best_end])+ start = best_end+ return pieces++def _split_oversized_unit(text, chunk_token_num, token_count_fn=None):+ """Split a single unit that exceeds ``chunk_token_num`` tokens into pieces+ that each fit the budget. Whitespace is used as the primary break (mirrors+ ``RAGFlowHtmlParser._split_oversized_block``); a single run of non-whitespace+ longer than the budget falls back to token-budget-based character windows.+ """+ if token_count_fn is None:+ token_count_fn = num_tokens_from_string+ if token_count_fn(text or "") <= chunk_token_num:+ return [text]+ pieces = []+ current = ""+ current_tokens = 0+ token_cache = {}++ def atom_tokens(atom):+ if atom.isspace():+ return 0+ if atom not in token_cache:+ token_cache[atom] = token_count_fn(atom)+ return token_cache[atom]++ # Match whitespace runs OR non-whitespace runs (i.e. individual words/tokens).+ for atom in re.findall(r"\s+|\S+", text or ""):+ a_tokens = atom_tokens(atom)+ if a_tokens > chunk_token_num and not atom.isspace():+ # An atom longer than the budget: flush current buffer, then carve+ # token-budget-based slices out of the atom itself.+ if current:+ pieces.append(current)+ current = ""+ current_tokens = 0+ for sub_piece in _split_atom_by_token_budget(atom, chunk_token_num, token_count_fn):+ pieces.append(sub_piece)+ continue+ if current and current_tokens + a_tokens > chunk_token_num:+ pieces.append(current)+ current = ""+ current_tokens = 0+ current += atom+ current_tokens += a_tokens+ if current:+ pieces.append(current)+ return pieces+++def _compute_chunk_update(last_ck: str, t: str, pos: str, chunk_token_num: int, overlapped_percent: float):+ tnum = num_tokens_from_string(t)+ if not pos or tnum < 8:+ pos = ""++ # First chunk ever — no previous content to overlap with.+ if last_ck == "":+ new_t = t + pos if t.find(pos) < 0 else t+ final_t = new_t if num_tokens_from_string(new_t) <= chunk_token_num else t+ return "first", final_t, num_tokens_from_string(final_t)++ # Proactive merge: append only if the *projected* total still fits.+ merged = last_ck + t+ merged_pos = merged + pos if last_ck.find(pos) < 0 else merged+ if num_tokens_from_string(merged_pos) <= chunk_token_num:+ return "merge", merged_pos, num_tokens_from_string(merged_pos)+ elif num_tokens_from_string(merged) <= chunk_token_num:+ return "merge", merged, num_tokens_from_string(merged)++ # Need a new chunk. Apply overlap prefix from the previous chunk —+ # but only when the projected size (overlap + t) fits — otherwise drop+ # the overlap for this boundary so the chunk stays within budget.+ new_t = t+ new_tnum = tnum+ if overlapped_percent > 0:+ overlap_text, overlap_tokens = _compute_overlap_prefix(last_ck, overlapped_percent)+ if overlap_tokens + new_tnum <= chunk_token_num:+ new_t = overlap_text + t+ new_tnum = num_tokens_from_string(new_t)+ if t.find(pos) < 0:+ new_t_with_pos = new_t + pos+ new_tnum_with_pos = num_tokens_from_string(new_t_with_pos)+ if new_tnum_with_pos <= chunk_token_num:+ new_t = new_t_with_pos+ new_tnum = new_tnum_with_pos+ return "append", new_t, new_tnum+++def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;!?", overlapped_percent=0):if not sections:return []if isinstance(sections, str):@@ -1169,28 +1295,14 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;tk_nums = [0]def add_chunk(t, pos):- nonlocal cks, tk_nums, delimiter- tnum = num_tokens_from_string(t)- if not pos:- pos = ""- if tnum < 8:- pos = ""- # Ensure that the length of the merged chunk does not exceed chunk_token_num- if cks[-1] == "" or tk_nums[-1] > chunk_token_num * (100 - overlapped_percent) / 100.0:- if cks:- overlapped = RAGFlowPdfParser.remove_tag(cks[-1])- t = overlapped[int(len(overlapped) * (100 - overlapped_percent) / 100.0) :] + t- # Recount with the overlap prefix included, else chunks overshoot chunk_token_num.- tnum = num_tokens_from_string(t)- if t.find(pos) < 0:- t += pos- cks.append(t)- tk_nums.append(tnum)+ nonlocal cks, tk_nums+ action, text, tk_num = _compute_chunk_update(cks[-1], t, pos, chunk_token_num, overlapped_percent)+ if action in ("first", "merge"):+ cks[-1] = text+ tk_nums[-1] = tk_numelse:- if cks[-1].find(pos) < 0:- t += pos- cks[-1] += t- tk_nums[-1] += tnum+ cks.append(text)+ tk_nums.append(tk_num)custom_delimiters = [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter)]has_custom = bool(custom_delimiters)@@ -1214,23 +1326,41 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;return cks# Split oversized sections at sentence delimiters; add_chunk re-merges to size.+ # Units that exceed the budget after the regex split (a single long line with+ # no delimiter, e.g. PDF / .txt runs of unbroken text) are sub-split on+ # whitespace atoms with a character-window fallback, mirroring the html path.dels = get_delimiters(delimiter)for sec, pos in sections:- if not dels or num_tokens_from_string(sec) < chunk_token_num:- add_chunk("\n" + sec, pos)+ sec_text = "\n" + sec+ if num_tokens_from_string(sec_text) <= chunk_token_num:+ add_chunk(sec_text, pos)continue- for sub_sec in re.split(r"(%s)" % dels, sec, flags=re.DOTALL):- if not sub_sec or re.fullmatch(dels, sub_sec):- continue- add_chunk("\n" + sub_sec, pos)+ if dels:+ for sub_sec in re.split(r"(%s)" % dels, sec, flags=re.DOTALL):+ if not sub_sec or re.fullmatch(dels, sub_sec):+ continue+ text = "\n" + sub_sec+ if num_tokens_from_string(text) <= chunk_token_num:+ add_chunk(text, pos)+ else:+ logging.debug("Splitting oversized unit (len=%d, tokens=%d) via _split_oversized_unit", len(text), num_tokens_from_string(text))+ for piece in _split_oversized_unit(text, chunk_token_num):+ add_chunk(piece, pos)+ else:+ logging.debug("Splitting oversized unit (len=%d, tokens=%d) via _split_oversized_unit (no delimiters)", len(sec_text), num_tokens_from_string(sec_text))+ for piece in _split_oversized_unit(sec_text, chunk_token_num):+ add_chunk(piece, pos)logging.debug("naive_merge: %d sections -> %d chunks (delimiter=%r)", len(sections), len(cks), delimiter)+ # Drop the leading empty placeholder that exists only so ``add_chunk`` could+ # detect "first chunk ever" without an extra flag.+ if cks and cks[0] == "":+ cks = cks[1:]+ tk_nums = tk_nums[1:]return cksdef naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。;!?", overlapped_percent=0):- from deepdoc.parser.pdf_parser import RAGFlowPdfParser-if not texts or len(texts) != len(images):return [], []cks = [""]@@ -1238,33 +1368,23 @@ def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。tk_nums = [0]def add_chunk(t, image, pos=""):- nonlocal cks, result_images, tk_nums, delimiter- tnum = num_tokens_from_string(t)- if not pos:- pos = ""- if tnum < 8:- pos = ""- # Ensure that the length of the merged chunk does not exceed chunk_token_num- if cks[-1] == "" or tk_nums[-1] > chunk_token_num * (100 - overlapped_percent) / 100.0:- if cks:- overlapped = RAGFlowPdfParser.remove_tag(cks[-1])- t = overlapped[int(len(overlapped) * (100 - overlapped_percent) / 100.0) :] + t- # Recount with the overlap prefix included, else chunks overshoot chunk_token_num.- tnum = num_tokens_from_string(t)- if t.find(pos) < 0:- t += pos- cks.append(t)- result_images.append(image)- tk_nums.append(tnum)- else:- if cks[-1].find(pos) < 0:- t += pos- cks[-1] += t+ nonlocal cks, result_images, tk_nums+ action, text, tk_num = _compute_chunk_update(cks[-1], t, pos, chunk_token_num, overlapped_percent)+ if action == "first":+ cks[-1] = text+ tk_nums[-1] = tk_num+ result_images[-1] = image+ elif action == "merge":+ cks[-1] = text+ tk_nums[-1] = tk_numif result_images[-1] is None:result_images[-1] = imageelse:result_images[-1] = concat_img(result_images[-1], image)- tk_nums[-1] += tnum+ else:+ cks.append(text)+ result_images.append(image)+ tk_nums.append(tk_num)custom_delimiters = [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter)]has_custom = bool(custom_delimiters)@@ -1294,6 +1414,8 @@ def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。# Split oversized sections at sentence delimiters; the section's image rides# along on every piece (concat_img dedupes when pieces re-merge into a chunk).+ # Units still exceeding the budget after the regex split are sub-split on+ # whitespace atoms so they cannot blow past the token cap.dels = get_delimiters(delimiter)for text, image in zip(texts, images):# if text is tuple, unpack it@@ -1303,15 +1425,32 @@ def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。else:text_str = text or ""text_pos = ""- if not dels or num_tokens_from_string(text_str) < chunk_token_num:- add_chunk("\n" + text_str, image, text_pos)++ text_seg = "\n" + text_str+ if num_tokens_from_string(text_seg) <= chunk_token_num:+ add_chunk(text_seg, image, text_pos)continue- for sub_sec in re.split(r"(%s)" % dels, text_str, flags=re.DOTALL):- if not sub_sec or re.fullmatch(dels, sub_sec):- continue- add_chunk("\n" + sub_sec, image, text_pos)+ if dels:+ for sub_sec in re.split(r"(%s)" % dels, text_str, flags=re.DOTALL):+ if not sub_sec or re.fullmatch(dels, sub_sec):+ continue+ sub_text = "\n" + sub_sec+ if num_tokens_from_string(sub_text) <= chunk_token_num:+ add_chunk(sub_text, image, text_pos)+ else:+ logging.debug("Splitting oversized unit (len=%d, tokens=%d) via _split_oversized_unit", len(sub_text), num_tokens_from_string(sub_text))+ for piece in _split_oversized_unit(sub_text, chunk_token_num):+ add_chunk(piece, image, text_pos)+ else:+ logging.debug("Splitting oversized unit (len=%d, tokens=%d) via _split_oversized_unit (no delimiters)", len(text_seg), num_tokens_from_string(text_seg))+ for piece in _split_oversized_unit(text_seg, chunk_token_num):+ add_chunk(piece, image, text_pos)logging.debug("naive_merge_with_images: %d texts -> %d chunks (delimiter=%r)", len(texts), len(cks), delimiter)+ if cks and cks[0] == "":+ cks = cks[1:]+ result_images = result_images[1:]+ tk_nums = tk_nums[1:]return cks, result_images@@ -1334,7 +1473,7 @@ def docx_question_level(p, bull=-1):def concat_img(img1, img2):- from rag.utils.lazy_image import ensure_pil_image, LazyImage+ from rag.utils.lazy_image import LazyImage, ensure_pil_image# Same image must not stack with itself (the LazyImage branch would otherwise# concatenate its blob list); mirrors the PIL branch's same-reference guard.@@ -1603,7 +1742,6 @@ def naive_merge_docx(table_context_size=0,image_context_size=0,):-if not sections:return [], []diff --git a/test/unit_test/deepdoc/parser/test_txt_parser.py b/test/unit_test/deepdoc/parser/test_txt_parser.pynew file mode 100644index 0000000000..289f77ac8e--- /dev/null+++ b/test/unit_test/deepdoc/parser/test_txt_parser.py@@ -0,0 +1,178 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Unit tests for ``RAGFlowTxtParser.parser_txt`` strict-cap behaviour.++The pre-fix ``add_chunk`` fired its size check *after* the append, so each+chunk could overshoot ``chunk_token_num`` by up to the size of one line. These+tests assert the proactive projected-total invariant: no produced chunk may+contain more than ``chunk_token_num`` tokens.+"""++import importlib.util+import os+import sys+from unittest import mock++_MOCK_MODULES = [+ "xgboost",+ "pdfplumber",+ "huggingface_hub",+ "PIL",+ "PIL.Image",+ "pypdf",+ "sklearn",+ "deepdoc.vision",+ "deepdoc",+ "deepdoc.parser",+ "deepdoc.parser.utils",+]+_orig_modules = {m: sys.modules.get(m) for m in _MOCK_MODULES}+_orig_get_text = getattr(sys.modules.get("deepdoc.parser.utils"), "get_text", None)++try:+ for _m in _MOCK_MODULES:+ if _m not in sys.modules:+ sys.modules[_m] = mock.MagicMock()++ # ``get_text`` is invoked by ``RAGFlowTxtParser.__call__`` only, not by+ # ``parser_txt``. Provide a permissive stub so the module loads.+ sys.modules["deepdoc.parser.utils"].get_text = lambda *a, **kw: ""++ def _find_project_root(marker="pyproject.toml"):+ d = os.path.dirname(os.path.abspath(__file__))+ while d != os.path.dirname(d):+ if os.path.exists(os.path.join(d, marker)):+ return d+ d = os.path.dirname(d)+ return None++ _PROJECT_ROOT = _find_project_root()++ _spec = importlib.util.spec_from_file_location(+ "deepdoc.parser._txt_parser_under_test",+ os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "txt_parser.py"),+ )+ _mod = importlib.util.module_from_spec(_spec)+ sys.modules["deepdoc.parser._txt_parser_under_test"] = _mod+ _spec.loader.exec_module(_mod)++ RAGFlowTxtParser = _mod.RAGFlowTxtParser+finally:+ for _m, _orig in _orig_modules.items():+ if _orig is None:+ sys.modules.pop(_m, None)+ else:+ sys.modules[_m] = _orig+ if _orig_modules.get("deepdoc.parser.utils") is not None and _orig_get_text is not None:+ _orig_modules["deepdoc.parser.utils"].get_text = _orig_get_text+ if _orig_modules.get("deepdoc.parser") is not None and _orig_modules.get("deepdoc.parser.utils") is not None:+ _orig_modules["deepdoc.parser"].utils = _orig_modules["deepdoc.parser.utils"]+++# A deterministic, tokenizer-free stand-in for ``num_tokens_from_string`` so+# the assertions below reason in plain words and are independent of tiktoken.+++def _patch_word_count(monkeypatch_module):+ def fake_num_tokens(s):+ return len((s or "").split())++ monkeypatch_module.setattr(_mod, "num_tokens_from_string", fake_num_tokens)+++def test_no_overshoot_when_packing_short_lines(monkeypatch):+ """Lines of 25 tokens, budget 100 — every chunk must be <= 100 tokens."""+ _patch_word_count(monkeypatch)+ txt = " ".join(["alpha"] * 25) + "\n" + " ".join(["beta"] * 25) + "\n" + " ".join(["gamma"] * 25)+ chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=100, delimiter="\n")+ sizes = [len(c[0].split()) for c in chunks if c[0].strip()]+ assert all(s <= 100 for s in sizes), sizes+ # 75 tokens of content, expected a single 75-token chunk.+ assert sum(sizes) == 75+++def test_no_overshoot_at_chunk_boundary(monkeypatch):+ """Lines of 30 tokens, budget 100. Pre-fix the boundary chunk was 130 tokens."""+ _patch_word_count(monkeypatch)+ lines = [" ".join([f"w{i}"] * 30) for i in range(10)] # 10 lines, 300 tokens+ chunks = RAGFlowTxtParser.parser_txt("\n".join(lines), chunk_token_num=100, delimiter="\n")+ sizes = [len(c[0].split()) for c in chunks if c[0].strip()]+ assert all(s <= 100 for s in sizes), sizes+++def test_atomic_oversized_line_is_sub_split_on_whitespace(monkeypatch):+ """A single line that exceeds the budget is split on whitespace atoms."""+ _patch_word_count(monkeypatch)+ huge_line = " ".join(["alpha"] * 80) # 80 tokens, no internal delimiter+ chunks = RAGFlowTxtParser.parser_txt(huge_line, chunk_token_num=50, delimiter="\n")+ sizes = [len(c[0].split()) for c in chunks if c[0].strip()]+ assert all(s <= 50 for s in sizes), sizes+ assert sum(sizes) == 80+ assert len(chunks) >= 2+++def test_empty_text_returns_empty(monkeypatch):+ _patch_word_count(monkeypatch)+ # Empty input produces a single empty chunk placeholder (existing+ # behaviour the callers rely on). The hard-cap guarantee is that any+ # chunk carrying content stays within the budget.+ result = RAGFlowTxtParser.parser_txt("", chunk_token_num=128, delimiter="\n")+ non_empty = [c for c in result if c[0].strip()]+ assert non_empty == []+ result2 = RAGFlowTxtParser.parser_txt(" \n\n ", chunk_token_num=128, delimiter="\n")+ non_empty2 = [c for c in result2 if c[0].strip()]+ assert non_empty2 == []+++def test_unbroken_token_exceeding_budget_fallback(monkeypatch):+ """A single unbroken non-whitespace string exceeding the budget is split+ via the character-window/token-slicing fallback.+ """++ def char_count_tokens(s):+ return len(s or "")++ monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)++ huge_word = "a" * 80 # 80 characters/tokens, no whitespace+ chunks = RAGFlowTxtParser.parser_txt(huge_word, chunk_token_num=30, delimiter="\n")++ non_empty = [c[0] for c in chunks if c[0].strip()]+ assert len(non_empty) >= 3+ assert all(char_count_tokens(c) <= 30 for c in non_empty)+ assert "".join(non_empty) == huge_word+++def test_newline_join_token_count_strict_cap(monkeypatch):+ """Verify that joining chunks with newline does not overshoot chunk_token_num+ even when individual token counts sum to <= budget but the newline pushes it over.+ """++ def char_count_tokens(s):+ return len(s or "")++ monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)++ # Two lines of 10 chars each. Budget = 20.+ # line1 + "\n" + line2 = 10 + 1 + 10 = 21 chars/tokens, exceeding budget of 20.+ line1 = "a" * 10+ line2 = "b" * 10+ txt = f"{line1}\n{line2}"+ chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=20, delimiter="\n")+ non_empty = [c[0] for c in chunks if c[0].strip()]+ assert all(char_count_tokens(c) <= 20 for c in non_empty)+ assert len(non_empty) == 2diff --git a/test/unit_test/rag/test_delimiter_case_sensitive.py b/test/unit_test/rag/test_delimiter_case_sensitive.pyindex 66c48553d7..6715293b94 100644--- a/test/unit_test/rag/test_delimiter_case_sensitive.py+++ b/test/unit_test/rag/test_delimiter_case_sensitive.py@@ -119,7 +119,7 @@ def force_every_section_above_budget(monkeypatch):chunk-size heuristics."""def fake(_s):- return 10**9+ return 9 if len(_s) >= 4 else 8monkeypatch.setattr(nlp, "num_tokens_from_string", fake)diff --git a/test/unit_test/rag/test_naive_merge.py b/test/unit_test/rag/test_naive_merge.pyindex 6ec4abb57f..c61e78024d 100644--- a/test/unit_test/rag/test_naive_merge.py+++ b/test/unit_test/rag/test_naive_merge.py@@ -16,17 +16,22 @@"""Regression tests for ``naive_merge`` / ``naive_merge_with_images``.-Guards against the regression introduced by commit db0f6840d (#11434) where the-default (non-custom-delimiter) path stopped splitting oversized sections at-sentence boundaries, and the overlap prefix was not counted toward a chunk's-token budget.+Guards against:++* the regression introduced by commit db0f6840d (#11434) where the default+ (non-custom-delimiter) path stopped splitting oversized sections at sentence+ boundaries, and the overlap prefix was not counted toward a chunk's token+ budget;+* the soft-cap bug where chunks systematically overshot ``chunk_token_num`` by+ up to one unit (sentence / line) because the size check fired *after* the+ append instead of using a projected-total check."""import reimport pytest-import rag.nlp as nlp+from rag import nlpfrom rag.nlp import naive_merge, naive_merge_with_imagesDEFAULT_DELIMITER = "\n!?。;!?"@@ -72,8 +77,10 @@ def test_oversized_section_is_split_at_sentence_boundaries():# Pre-regression behaviour: the section is broken into several chunks# instead of a single oversized one.assert len(chunks) > 1- # No chunk should greatly exceed the budget (allow one trailing sentence of slack).- assert all(_tok(c) <= 50 + 10 for c in chunks)+ # Hard cap: no chunk may exceed the budget. ``<=`` is exact; the slack+ # previously allowed (one trailing sentence) is no longer permitted because+ # the projected-total check fires before the append.+ assert all(_tok(c) <= 50 for c in chunks)# Content is preserved.assert "".join(chunks).count("word") == 200@@ -107,17 +114,19 @@ def test_empty_delimiter_falls_back_to_token_size_merge():@pytest.mark.p2def test_overlap_prefix_is_counted_in_token_budget():- # With overlap, each chunk = overlap-prefix + new content. The fix recomputes- # the chunk's token count after prepending the prefix, so chunks stay bounded.- # Pre-fix, the prefix tokens were not counted, so the per-chunk budget check- # fired late and chunks systematically overshot chunk_token_num.+ # With overlap, each chunk = overlap-prefix + new content. The proactive+ # projected-total check rejects a section that, even after prepending the+ # overlap prefix, would exceed chunk_token_num; the overlap is dropped at+ # that boundary instead of letting the chunk overshoot. Pre-fix, the prefix+ # tokens were not counted, so the per-chunk budget check fired late and+ # chunks systematically overshot chunk_token_num (observed up to 63).sentences = [" ".join(["w"] * 10) for _ in range(30)]chunks = _nonempty(naive_merge(sentences, chunk_token_num=50, delimiter=DEFAULT_DELIMITER, overlapped_percent=20))assert len(chunks) > 1- # Each 10-token sentence divides chunk_token_num evenly, so a correct- # accounting yields chunks of exactly the budget. The buggy version- # overshot (observed up to 63). A small tolerance guards tokenizer rounding.- assert all(_tok(c) <= 50 + 2 for c in chunks)+ # Each chunk stays within the budget. Sentences are 10 tokens, the budget+ # is 50, so even a 10-token overlap prefix (20% of 50) fits a 40-token+ # remainder and the projected-total guarantee holds exactly.+ assert all(_tok(c) <= 50 for c in chunks)# --------------------------------------------------------------------------- #@@ -159,7 +168,7 @@ def test_images_oversized_section_is_split():assert len(nonempty) > 1# Returned lists stay aligned.assert len(chunks) == len(imgs)- assert all(_tok(c) <= 50 + 10 for c in nonempty)+ assert all(_tok(c) <= 50 for c in nonempty)@pytest.mark.p2@@ -215,3 +224,101 @@ def test_images_distinct_lazyimages_are_concatenated():merged = nonempty_imgs[0]assert isinstance(merged, LazyImage)assert merged._blobs == [b"BLOB_A", b"BLOB_B"]+++# --------------------------------------------------------------------------- #+# Hard cap on chunk size (overshoot bug fix)+# --------------------------------------------------------------------------- #+++def test_strict_cap_no_overlap_packs_to_budget():+ sections = [" ".join(["w"] * 25) for _ in range(8)]+ chunks = _nonempty(naive_merge(sections, chunk_token_num=50, delimiter=DEFAULT_DELIMITER))+ assert len(chunks) >= 3+ assert all(_tok(c) <= 50 for c in chunks)+++def test_strict_cap_with_overlap_drops_overlap_at_overflow_boundary():+ sentences = [" ".join(["w"] * 10) for _ in range(20)]+ chunks = _nonempty(naive_merge(sentences, chunk_token_num=25, delimiter=DEFAULT_DELIMITER, overlapped_percent=20))+ assert all(_tok(c) <= 25 for c in chunks)+++def test_strict_cap_single_overlong_section_is_sub_split_on_whitespace(monkeypatch):+ # Override tokenizer in nlp to treat characters as tokens for testing character fallback+ def char_count_tokens(s):+ return len(s or "")++ monkeypatch.setattr(nlp, "num_tokens_from_string", char_count_tokens)++ big_section = "a" * 80 # unbroken, token-dense string+ chunks = _nonempty(naive_merge([big_section], chunk_token_num=50, delimiter=DEFAULT_DELIMITER))+ assert len(chunks) >= 2+ assert all(char_count_tokens(c) <= 50 for c in chunks)+ assert "".join(chunks) == big_section+++def test_strict_cap_overlap_chosen_when_it_fits():+ sentences = [" ".join(["w"] * 5) for _ in range(20)]+ chunks = _nonempty(naive_merge(sentences, chunk_token_num=20, delimiter=DEFAULT_DELIMITER, overlapped_percent=20))+ assert all(_tok(c) <= 20 for c in chunks)+ overlap_seen = False+ for a, b in zip(chunks, chunks[1:]):+ a_tokens = a.split()+ b_tokens = b.split()+ if a_tokens and b_tokens and any(t in b_tokens for t in a_tokens):+ overlap_seen = True+ break+ assert overlap_seen+++def test_images_strict_cap_packs_to_budget():+ sections = [" ".join(["w"] * 25) for _ in range(6)]+ images = [None] * len(sections)+ chunks, imgs = naive_merge_with_images(sections, images, chunk_token_num=50, delimiter=DEFAULT_DELIMITER)+ nonempty = _nonempty(chunks)+ assert all(_tok(c) <= 50 for c in nonempty)+ assert len(chunks) == len(imgs)+++def test_strict_cap_pos_text_does_not_overshoot_budget(monkeypatch):+ """Verify that pos text addition does not push chunk over chunk_token_num."""++ def char_count_tokens(s):+ return len(s or "")++ monkeypatch.setattr(nlp, "num_tokens_from_string", char_count_tokens)++ # section is 15 chars, pos is 10 chars. chunk_token_num is 20.+ # section + pos = 25 > 20, so pos should be omitted or chunk kept <= 20.+ pos_tag = "@@12345678"+ sections = [("\na" * 15, pos_tag)]+ chunks = _nonempty(naive_merge(sections, chunk_token_num=20, delimiter=DEFAULT_DELIMITER))+ assert all(char_count_tokens(c) <= 20 for c in chunks)+++def test_empty_delimiter_oversized_section_strictly_capped():+ # When delimiter="" and a section exceeds chunk_token_num, it must be sub-split+ # so no chunk exceeds chunk_token_num.+ long_section = "word " * 100 # ~100 tokens+ chunks = _nonempty(naive_merge([long_section], chunk_token_num=30, delimiter=""))+ assert len(chunks) > 1+ assert all(_tok(c) <= 30 for c in chunks)+++def test_images_empty_delimiter_oversized_section_strictly_capped():+ long_section = "word " * 100+ images = [None]+ chunks, imgs = naive_merge_with_images([long_section], images, chunk_token_num=30, delimiter="")+ nonempty = _nonempty(chunks)+ assert len(nonempty) > 1+ assert all(_tok(c) <= 30 for c in nonempty)+ assert len(chunks) == len(imgs)