RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(chunker): enforce a hard token cap in the TokenChunker merge (#18525)
Jack4eec5e4f3520
diff --git a/internal/ingestion/component/chunker/json_global_merge_test.go b/internal/ingestion/component/chunker/json_global_merge_test.goindex c42d438964..2e51662905 100644--- a/internal/ingestion/component/chunker/json_global_merge_test.go+++ b/internal/ingestion/component/chunker/json_global_merge_test.go@@ -85,11 +85,12 @@ func TestJSONGlobalMergeMatchesPython(t *testing.T) {}}-// TestJSONSingleItemNotSubSplitMatchesPython pins the TokenChunker json path's-// handling of a single item that exceeds chunk_token_size: the over-budget-// item is kept whole (no sub-split), yielding exactly one chunk whose text-// equals the input verbatim.-func TestJSONSingleItemNotSubSplitMatchesPython(t *testing.T) {+// TestJSONSingleItemSubSplitUnderHardCap pins the TokenChunker json path's+// hard-cap handling of a single item that exceeds chunk_token_size: the+// over-budget item is re-split into <= budget chunks (sentence boundaries+// first, hard token-split fallback) whose concatenated text reproduces the+// input verbatim. This replaces the former #17799 "kept whole" contract.+func TestJSONSingleItemSubSplitUnderHardCap(t *testing.T) {const budget = 128comp, err := NewTokenChunker(map[string]any{"chunk_token_size": float64(budget),@@ -116,10 +117,18 @@ func TestJSONSingleItemNotSubSplitMatchesPython(t *testing.T) {t.Fatalf("TokenChunker returned _ERROR: %s", msg)}chunks, _ := out["chunks"].([]map[string]any)- if len(chunks) != 1 {- t.Fatalf("want 1 chunk (over-budget single item kept whole), got %d", len(chunks))+ if len(chunks) < 2 {+ t.Fatalf("over-budget item must be split, got %d chunk(s)", len(chunks))+ }+ var joined string+ for i, ck := range chunks {+ text, _ := ck["text"].(string)+ if n := tokenizeStr(text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=%d)", i, n, budget)+ }+ joined += strings.TrimSpace(text)}- if got := strings.TrimSpace(chunks[0]["text"].(string)); got != strings.TrimSpace(long) {- t.Errorf("over-budget item not kept whole:\n got=%q\nwant=%q", got, strings.TrimSpace(long))+ if strings.ReplaceAll(joined, " ", "") != strings.ReplaceAll(strings.TrimSpace(long), " ", "") {+ t.Errorf("split not lossless:\n got=%q\nwant=%q", joined, strings.TrimSpace(long))}}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__b1_count_sensitive.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__b1_count_sensitive.jsonnew file mode 100644index 0000000000..4c982e0101--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__b1_count_sensitive.json@@ -0,0 +1,23 @@+{+ "case_id": "token__b1_count_sensitive",+ "chunks": [+ {+ "text": "RAGFlow is a retrieval augmented generation engine that ingests heterogeneous documents and slices them into retrievable passages. The chunker must respect a token budget so that each passage fits the context window of the downstream language model without truncation. Token counting"+ },+ {+ "text": "is"+ },+ {+ "text": "the linchpin of this budget because every merge decision reads the per segment token length. When the encoder is missing the chunker silently reports zero tokens for every string and the budget is never exceeded so an entire document collapses into one oversized chunk. That"+ },+ {+ "text": "degenerate behaviour is invisible to a test that only checks for a non empty result because zero is a number like any other. The only way to notice is to compare the actual chunk count against a reference implementation that counts tokens correctly. This case pins that"+ },+ {+ "text": "comparison with a long single paragraph and a deliberately small budget so the correct count yields many chunks and a dead encoder yields exactly one. Retrieval quality depends on passages being coherent and appropriately sized so the guard is not merely cosmetic but protects the core promise of"+ },+ {+ "text": "the system."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_default_delim.jsonnew file mode 100644index 0000000000..9d02099f56--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_default_delim.json@@ -0,0 +1,14 @@+{+ "case_id": "token__html_default_delim",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision."+ },+ {+ "text": "Smaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ },+ {+ "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_long.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_long.jsonnew file mode 100644index 0000000000..c417efdd62--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_long.json@@ -0,0 +1,23 @@+{+ "case_id": "token__html_long",+ "chunks": [+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.jsonnew file mode 100644index 0000000000..c18ae0ebf2--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_multi_item_40.json@@ -0,0 +1,33 @@+{+ "case_id": "token__json_multi_item_40",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "Section 1. RAGFlow chunks documents for retrieval. This is sentence two of item 1.\nSection 2. RAGFlow chunks documents for retrieval. This is sentence two of item 2.\nSection 3. RAGFlow chunks documents for retrieval. This is sentence two of item 3.\nSection 4. RAGFlow chunks documents for retrieval. This is sentence two of item 4.\nSection 5. RAGFlow chunks documents for retrieval. This is sentence two of item 5.\nSection 6. RAGFlow chunks documents for retrieval. This is sentence two of item 6."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 7. RAGFlow chunks documents for retrieval. This is sentence two of item 7.\nSection 8. RAGFlow chunks documents for retrieval. This is sentence two of item 8.\nSection 9. RAGFlow chunks documents for retrieval. This is sentence two of item 9.\nSection 10. RAGFlow chunks documents for retrieval. This is sentence two of item 10.\nSection 11. RAGFlow chunks documents for retrieval. This is sentence two of item 11.\nSection 12. RAGFlow chunks documents for retrieval. This is sentence two of item 12."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 13. RAGFlow chunks documents for retrieval. This is sentence two of item 13.\nSection 14. RAGFlow chunks documents for retrieval. This is sentence two of item 14.\nSection 15. RAGFlow chunks documents for retrieval. This is sentence two of item 15.\nSection 16. RAGFlow chunks documents for retrieval. This is sentence two of item 16.\nSection 17. RAGFlow chunks documents for retrieval. This is sentence two of item 17.\nSection 18. RAGFlow chunks documents for retrieval. This is sentence two of item 18."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 19. RAGFlow chunks documents for retrieval. This is sentence two of item 19.\nSection 20. RAGFlow chunks documents for retrieval. This is sentence two of item 20.\nSection 21. RAGFlow chunks documents for retrieval. This is sentence two of item 21.\nSection 22. RAGFlow chunks documents for retrieval. This is sentence two of item 22.\nSection 23. RAGFlow chunks documents for retrieval. This is sentence two of item 23.\nSection 24. RAGFlow chunks documents for retrieval. This is sentence two of item 24."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 25. RAGFlow chunks documents for retrieval. This is sentence two of item 25.\nSection 26. RAGFlow chunks documents for retrieval. This is sentence two of item 26.\nSection 27. RAGFlow chunks documents for retrieval. This is sentence two of item 27.\nSection 28. RAGFlow chunks documents for retrieval. This is sentence two of item 28.\nSection 29. RAGFlow chunks documents for retrieval. This is sentence two of item 29.\nSection 30. RAGFlow chunks documents for retrieval. This is sentence two of item 30."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 31. RAGFlow chunks documents for retrieval. This is sentence two of item 31.\nSection 32. RAGFlow chunks documents for retrieval. This is sentence two of item 32.\nSection 33. RAGFlow chunks documents for retrieval. This is sentence two of item 33.\nSection 34. RAGFlow chunks documents for retrieval. This is sentence two of item 34.\nSection 35. RAGFlow chunks documents for retrieval. This is sentence two of item 35.\nSection 36. RAGFlow chunks documents for retrieval. This is sentence two of item 36."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 37. RAGFlow chunks documents for retrieval. This is sentence two of item 37.\nSection 38. RAGFlow chunks documents for retrieval. This is sentence two of item 38.\nSection 39. RAGFlow chunks documents for retrieval. This is sentence two of item 39.\nSection 40. RAGFlow chunks documents for retrieval. This is sentence two of item 40."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_single_long.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_single_long.jsonnew file mode 100644index 0000000000..4b23169388--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__json_single_long.json@@ -0,0 +1,13 @@+{+ "case_id": "token__json_single_long",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega second alpha beta gamma"+ },+ {+ "doc_type_kwd": "text",+ "text": " delta epsilon zeta eta theta iota kappa"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_default_delim.jsonnew file mode 100644index 0000000000..fbdceb80da--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_default_delim.json@@ -0,0 +1,14 @@+{+ "case_id": "token__markdown_default_delim",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision."+ },+ {+ "text": "Smaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ },+ {+ "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_long.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_long.jsonnew file mode 100644index 0000000000..6943871e0a--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_long.json@@ -0,0 +1,23 @@+{+ "case_id": "token__markdown_long",+ "chunks": [+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_default_delim.jsonnew file mode 100644index 0000000000..0314c2ea84--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_default_delim.json@@ -0,0 +1,14 @@+{+ "case_id": "token__text_default_delim",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision."+ },+ {+ "text": "Smaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ },+ {+ "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_long_paragraph.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_long_paragraph.jsonnew file mode 100644index 0000000000..2337e4c2c4--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_long_paragraph.json@@ -0,0 +1,20 @@+{+ "case_id": "token__text_long_paragraph",+ "chunks": [+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ },+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_overlap.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_overlap.jsonnew file mode 100644index 0000000000..6ef7a2cd95--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_overlap.json@@ -0,0 +1,14 @@+{+ "case_id": "token__text_overlap",+ "chunks": [+ {+ "text": "alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"+ },+ {+ "text": "alpha alpha alpha alpha \nbeta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta"+ },+ {+ "text": "ta beta beta beta beta \ngamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_token_liveness.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_token_liveness.jsonnew file mode 100644index 0000000000..bdf0e87cc8--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_token_liveness.json@@ -0,0 +1,41 @@+{+ "case_id": "token__text_token_liveness",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks."+ },+ {+ "text": "Each chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables."+ },+ {+ "text": "Long paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries."+ },+ {+ "text": "Retrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks."+ },+ {+ "text": "Evaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost."+ },+ {+ "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings."+ },+ {+ "text": "Deep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text."+ },+ {+ "text": "The API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently."+ },+ {+ "text": "Concurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors."+ },+ {+ "text": "Metrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines."+ },+ {+ "text": "The engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base."+ },+ {+ "text": "Plugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.jsonindex 839edf6b14..5284bf78be 100644--- a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json@@ -27,6 +27,26 @@"owner_fix_side": "go","tracking": "TODO: file issue — Go emits an extra doc_type_kwd:'text' field on the text/markdown/html backtick path where Python emits only {'text': ...}","reason": "With a backtick-wrapped delimiter (e.g. `\\n`, _compile_delimiter_pattern, token_chunker.py:70-76) Python takes the custom-pattern branch. On that branch Python's _split_text_by_pattern (rag/flow/chunker/token_chunker.py:80-95) discards the delimiter, and the text path emits a list of {'text': ...} with no doc_type_kwd (token_chunker.py:341). Go no longer keeps the delimiter either: since #17868 it uses splitDroppingDelim, so chunk text matches Python exactly. The remaining Go-only behaviour is an extra doc_type_kwd:'text' field — invokeTextPayload (token.go:323, 338-341) defaults every chunk to DocType:'text', which Python's text-path output never includes."+ },+ {+ "id": "token-hardcap-chunk-count",+ "tag": "go_intentional",+ "kind": "chunk_count",+ "applies_to": ["token__b1_count_sensitive", "token__html_long", "token__json_multi_item_40", "token__json_single_long", "token__markdown_long", "token__text_long_paragraph", "token__text_overlap", "token__text_token_liveness"],+ "owner_fix_side": "python",+ "permanent": false,+ "tracking": "Go TokenChunker hard-cap merge (方案 B). Python token_chunker still uses OVER_CAP (#17799: a chunk may exceed the target by one unit and an oversized unit stands whole); adopt the hard-cap contract on the Python side to resolve these rules.",+ "reason": "Go TokenChunker now enforces a hard cap: no text chunk exceeds chunk_token_size. Oversized units are re-split on sentence boundaries with a hard token-split fallback, and the merge never lets a projected join exceed the target (UNDER_CAP). Python's token_chunker still uses OVER_CAP, so for over-budget inputs Go emits more, smaller chunks than Python."+ },+ {+ "id": "token-hardcap-chunk-text",+ "tag": "go_intentional",+ "kind": "chunk_text",+ "applies_to": ["token__html_default_delim", "token__markdown_default_delim", "token__text_default_delim"],+ "owner_fix_side": "python",+ "permanent": false,+ "tracking": "Go TokenChunker hard-cap merge (方案 B). Python token_chunker still uses OVER_CAP; adopt the hard-cap contract on the Python side to resolve these rules.",+ "reason": "Go TokenChunker now enforces a hard cap (UNDER_CAP merge + oversized-unit re-split), so chunk boundaries shift relative to Python's OVER_CAP merge-then-close on default-delimiter inputs: Go closes a chunk when the projected join would exceed the target, while Python merges the overflowing unit then closes. Same chunk count, different boundaries."}]}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 97b835e85f..e66f48d091 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -83,6 +83,7 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/tokenizer""ragflow/internal/parser/chunk")@@ -122,9 +123,6 @@ func (p *tokenChunkerParam) Update(conf map[string]any) {if v, ok := schema.NumericFromAny(conf["image_context_size"]); ok {p.TokenChunkerParam.ImageContextSize = int(v)}- if v, ok := conf["under_cap"].(bool); ok {- p.TokenChunkerParam.UnderCap = v- }}func defaultsToken(p tokenChunkerParam) tokenChunkerParam {@@ -410,16 +408,15 @@ func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int)}// mergeByTokenSize implements exact token-based chunk merging that mirrors-// Python's naive_merge (rag/nlp/__init__.py) after the strict chunk_token_num-// hard-cap fix. It uses tokenizeStr for precise token counting, treats the-// payload as a single section, and splits oversized sections on production-// sentence delimiters. An oversize unit (a single paragraph larger than the-// token budget) is kept whole as a standalone chunk — matching Python OVER_CAP,-// where the model layer truncates it later — instead of being atom-split.-// Sections are merged with the unified core (scaled overlap threshold +-// unconditional overlap prefix), matching Python naive_merge / token_chunker.-// When overlap>0 the previous chunk's tail is always prepended, so a chunk may-// exceed chunk_token_size by up to the overlap amount.+// Python naive_merge (rag/nlp/__init__.py) after the strict chunk_token_num+// hard-cap fix.+// It uses tokenizeStr for precise token counting, treats the payload as a+// single section, and splits oversized sections on production sentence+// delimiters. Sections are merged with the unified core (hard-cap expansion ++// UNDER_CAP merge): oversized units are re-split (sentence boundaries first,+// hard token-split fallback) so no text chunk exceeds chunk_token_size. When+// overlap>0 the previous chunk's tail is prepended and trimmed to fit, so the+// hard cap still holds.func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {target := c.param.ChunkTokenSizeoverlapPct := c.param.OverlappedPercent@@ -445,8 +442,8 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil// paragraphs (the delimiter is DROPPED, mirroring Python naive_merge) and// use each paragraph VERBATIM as a unit. This preserves the original// inter-paragraph whitespace, so the merged text matches the source (and- // Python naive_merge); an oversized paragraph also stands alone (#17799)- // instead of being re-split — both matching Python.+ // Python naive_merge). An oversized paragraph is re-split by the hard-cap+ // expansion in mergeUnits.//// Otherwise (no active delimiter) the whole text is a single section and// oversized sections are re-split on production sentence delimiters, which@@ -460,7 +457,8 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil// each kept paragraph. The sub-sec keeps its original surrounding// whitespace (e.g. the trailing space before the delimiter); only the// delimiter itself is removed. No sentence re-split is performed here —- // an oversize paragraph stands alone as its own chunk, matching Python.+ // an oversized paragraph is re-split later by the hard-cap expansion in+ // mergeUnits.for _, sub := range splitDroppingDelim(text, delimPattern) {if sub == "" {continue@@ -482,10 +480,9 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chilcontinue}// Oversized section: split on production sentence delimiters into- // units. An oversize unit (still exceeds the budget) is kept whole- // — no atom-split, matching Python naive_merge. Per the unified- // algorithm an over-budget unit STANDS ALONE (#17799): never merged- // into the previous chunk.+ // units. A unit that still exceeds the budget is re-split by the+ // hard-cap expansion in mergeUnits (sentence boundaries first,+ // hard token-split fallback), so no chunk ever exceeds the target.parts := sentenceDelimiter.Split(sec, -1)hadPart := falsefor _, part := range parts {@@ -510,12 +507,10 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chilreturn emptyOutputs()}- // Merge with the unified core (scaled overlap threshold + unconditional- // overlap prefix). joinSep is "" for the text path, mirroring Python- // naive_merge's concatenation of adjacent paragraphs. For overlap=0 this- // is exactly equivalent to the previous OVER_CAP running-sum merge, so- // existing overlap=0 output is unchanged.- merged := mergeUnits(units, target, overlapPct, c.param.MergeStrategy(), "")+ // Merge with the unified hard-cap core (oversized-unit expansion + UNDER_CAP+ // merge + overlap prefix trimmed to fit). joinSep is "" for the text path,+ // mirroring Python naive_merge's concatenation of adjacent paragraphs.+ merged := mergeUnits(units, target, overlapPct, "")docs := make([]schema.ChunkDoc, 0, len(merged))for _, ch := range merged {// Strip parser position tags from the final text:@@ -583,7 +578,7 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s// chunks across JSON items into one global token budget. Flatten the// per-item structure into a single sequence first so the merge is// global; non-text chunks still break the merge via their CKType.- attached = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flatten(attached)}, c.param.ChunkTokenSize, c.param.OverlappedPercent, c.param.MergeStrategy())+ attached = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flatten(attached)}, c.param.ChunkTokenSize, c.param.OverlappedPercent)}flat := flatten(attached)@@ -825,33 +820,30 @@ func takeFromStart(text string, tokens int) string {// the Python text path (rag/nlp naive_merge) is migrating to, so the two// languages and the two paths converge on ONE algorithm.//-// Unified contract (overlap>0):+// Hard-cap contract:+// - Every emitted text chunk is <= target tokens. Oversized units (a single+// unit whose token count exceeds target) are expanded FIRST: sentence+// boundaries are tried, then a hard token-split fallback guarantees each+// piece fits the target. The merge itself is UNDER_CAP: a projected join+// that would push the running sum over target starts a fresh chunk, so no+// chunk ever exceeds target.// - The merge threshold is SCALED to reserve room for overlap:// threshold = target * (100 - overlap) / 100. A chunk keeps receiving// units while its running token sum stays <= threshold; once it exceeds-// threshold the next unit starts a fresh chunk. For overlap=0 the-// threshold equals target and this is exactly equivalent to Python's-// OVER_CAP merge-then-close (verified 0/30000 mismatch), so overlap=0-// output is unchanged.+// threshold the next unit starts a fresh chunk.// - When a fresh chunk starts and overlap>0, the tail of the previous chunk// is UNCONDITIONALLY prepended (computeOverlapPrefix already strips parser-// tags) and the new chunk's token count is recomputed from the joined-// text. Overlap is never silently dropped, so every chunk boundary keeps-// its shared context — this is the user-visible reason the JSON strategy-// is preferred over the old fit-check overlap.-// - Over-budget units (a single unit whose token count exceeds target) STAND-// ALONE — they are never merged into the previous chunk. This matches-// Python naive_merge and Python token_chunker (both also stand the-// over-budget unit alone, #17799), so the Go TokenChunker, the Python text-// path, and the Python JSON path share one contract; the overlap prefix-// (when overlap>0) is kept like any other new-chunk boundary.-// - strategy == MergeUnderCap (Go-only strict mode; Python JSON has no such-// variant) additionally forbids a projected overflow: even when prev is-// below the scaled threshold, if prev+incoming would exceed target the-// incoming starts a fresh chunk instead.+// tags). If the overlap prefix would push the fresh chunk over target it is+// trimmed to fit, so the hard cap still holds with overlap enabled.//// Token counts use the RUNNING SUM of per-unit counts (never re-tokenizing the// joined string), matching Python's tk_nums += current["tk_nums"] (#17948).+// As a hard-cap safety net the merge ALSO re-checks the actual joined text via+// tokenizeStr before accepting a merge: joinSep (e.g. "\n" on the JSON path)+// can add tokens beyond the running sum, so a candidate whose joined text+// would exceed target starts a fresh chunk. The running-sum bookkeeping (and+// therefore chunk TKNums) stays as-is — it matches Python and can under-report+// the joinSep tokens by a few — while the emitted text never exceeds target.// Non-text units pass through unchanged and reset the merge run. joinSep is// "\n" for the JSON path and "" for the text path.// mergeItem records one source unit that contributed to a merged chunk: its@@ -877,17 +869,22 @@ func overlapTailPositions(prevItems []mergeItem, overlapStart int, joinSep strinreturn nil, nil}// Items are concatenated with joinSep (a single "\n" for the JSON path),- // matching the merge join at mergeUnits.+ // matching the merge join at mergeUnits. Offsets are measured on the+ // TAG-FREE visible text so they line up with overlapCut/overlapFitPrefix,+ // which carve the overlap from removeTag'd text; a coordinate tag in an+ // earlier item must not shift the boundaries of later items.+ visible := make([]string, len(prevItems))total := 0- for _, it := range prevItems {- total += utf8.RuneCountInString(it.Text) + utf8.RuneCountInString(joinSep)+ for i, it := range prevItems {+ visible[i] = removeTag(it.Text)+ total += utf8.RuneCountInString(visible[i]) + utf8.RuneCountInString(joinSep)}total -= utf8.RuneCountInString(joinSep)var pdfAcc, posAcc json.RawMessageoffset := 0- for _, it := range prevItems {+ for i, it := range prevItems {start := offset- end := offset + utf8.RuneCountInString(it.Text)+ end := offset + utf8.RuneCountInString(visible[i])if start < total && end > overlapStart {pdfAcc = extendRawJSONArray(pdfAcc, it.PDFPositions)posAcc = extendRawJSONArray(posAcc, it.Positions)@@ -897,12 +894,16 @@ func overlapTailPositions(prevItems []mergeItem, overlapStart int, joinSep strinreturn pdfAcc, posAcc}-func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, strategy schema.MergeStrategy, joinSep string) []schema.ChunkDoc {+func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {if overlapPct < 0 {overlapPct = 0} else if overlapPct > 100 {overlapPct = 100}+ // Expand oversized text units into <= target pieces BEFORE merging so the+ // UNDER_CAP merge can never exceed target (hard cap). Non-text units pass+ // through unchanged.+ units = expandOversizedUnits(units, target)// Scaled threshold reserves room for the unconditional overlap prefix.threshold := float64(target) * (100.0 - overlapPct) / 100.0@@ -937,38 +938,23 @@ func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, strategprevIdx = len(merged) - 1continue}- // #17799: an over-budget unit stands alone — it is never merged into- // the previous chunk. This matches Python naive_merge and Python- // token_chunker (both also stand the over-budget unit alone), so the- // Go TokenChunker, Python text path, and Python JSON path share one- // contract; the overlap prefix (when overlap>0) is kept like any- // other new-chunk boundary.- if tk > target {- cp := cloneChunkDoc(ck)- if overlapPct > 0 && merged[prevIdx].Text != "" {- overlap, _ := computeOverlapPrefix(merged[prevIdx].Text, overlapPct)- // Carry the previous chunk's tail coordinates so the overlap- // prefix is highlighted, not just the cur span (#18148).- pdfTail, posTail := overlapTailPositions(mergedItems[prevIdx], overlapCut(merged[prevIdx].Text, overlapPct), joinSep)- cp.Text = overlap + cp.Text- cp.PDFPositions = extendRawJSONArray(pdfTail, cp.PDFPositions)- cp.Positions = extendRawJSONArray(posTail, cp.Positions)- cp.TKNums = intPtr(tokenizeStr(cp.Text))- merged = append(merged, cp)- mergedItems = append(mergedItems, []mergeItem{{Text: cp.Text, PDFPositions: cp.PDFPositions, Positions: cp.Positions}})- prevIdx = len(merged) - 1- continue- }- cp.TKNums = intPtr(tk)- merged = append(merged, cp)- mergedItems = append(mergedItems, []mergeItem{cur})- prevIdx = len(merged) - 1- continue- }prev := &merged[prevIdx]- startNew := float64(intValue(prev.TKNums)) > threshold- if !startNew && strategy == schema.MergeUnderCap && intValue(prev.TKNums)+tk > target {- startNew = true+ // UNDER_CAP: a projected join that would exceed target starts a fresh+ // chunk. The scaled threshold reserves room for the overlap prefix.+ startNew := float64(intValue(prev.TKNums)) > threshold || intValue(prev.TKNums)+tk > target+ if !startNew {+ // The hard cap must hold on the ACTUAL joined text: joinSep (e.g.+ // "\n" on the JSON path) can add tokens beyond the running sum, so+ // re-check the candidate. The running-sum decision and bookkeeping+ // stay unchanged (Python parity #17948); this guard only splits+ // when the joined text itself would exceed target.+ candidate := prev.Text + ck.Text+ if prev.Text != "" && ck.Text != "" {+ candidate = prev.Text + joinSep + ck.Text+ }+ if tokenizeStr(candidate) > target {+ startNew = true+ }}if startNew {cp := cloneChunkDoc(ck)@@ -977,8 +963,11 @@ func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, strateg// prefix is a duplicate of prev's tail; carry prev's tail// coordinates so the overlap region is highlighted (#18148),// then keep only cur's coordinates for the non-overlap part.+ // If the prefix would push the fresh chunk over target it is+ // trimmed to fit so the hard cap still holds.overlap, _ := computeOverlapPrefix(prev.Text, overlapPct)- pdfTail, posTail := overlapTailPositions(mergedItems[prevIdx], overlapCut(prev.Text, overlapPct), joinSep)+ overlap, trimRunes := overlapFitPrefix(overlap, cp.Text, target)+ pdfTail, posTail := overlapTailPositions(mergedItems[prevIdx], overlapCut(prev.Text, overlapPct)+trimRunes, joinSep)cp.Text = overlap + cp.Textcp.PDFPositions = extendRawJSONArray(pdfTail, cp.PDFPositions)cp.Positions = extendRawJSONArray(posTail, cp.Positions)@@ -1009,15 +998,10 @@ func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, strateg}// mergeByTokenSizeFromJSON merges the text units of each upstream item using-// the unified mergeUnits core (scaled overlap threshold + unconditional-// overlap prefix), mirroring Python token_chunker.py:_merge_text_chunks_by-// _token_size via the unified mergeUnits core. Non-text units pass through and-// reset the merge run.-//-// strategy selects the merge strategy (schema.MergeStrategy): MergeOverCap =-// OVER_CAP (Python's canonical default), MergeUnderCap = UNDER_CAP (strict-// no-overflow, Go-only). The TokenChunker threads its MergeStrategy() here.-func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overlappedPct float64, strategy schema.MergeStrategy) [][]schema.ChunkDoc {+// the unified mergeUnits core (hard-cap expansion + UNDER_CAP merge),+// mirroring Python token_chunker.py:_merge_text_chunks_by_token_size. Non-text+// units pass through and reset the merge run.+func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overlappedPct float64) [][]schema.ChunkDoc {// overlappedPct is a [0,100] percentage. Clamp defensively because this// helper is also exercised directly by tests.if overlappedPct < 0 {@@ -1034,11 +1018,254 @@ func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, over// run (see mergeUnits). Join separator is "\n" to mirror// token_chunker.py:_merge_text_chunks_by_token_size, which joins// adjacent item text with "\n".- perItem[idx] = mergeUnits(perItem[idx], chunkTokens, overlappedPct, strategy, "\n")+ perItem[idx] = mergeUnits(perItem[idx], chunkTokens, overlappedPct, "\n")}return perItem}+// ---------------------------------------------------------------------------+// Hard-cap expansion+// ---------------------------------------------------------------------------++// expandOversizedUnits re-splits text units whose token count exceeds target+// so every returned text unit is <= target tokens. Sentence boundaries are+// tried first (delimiters preserved, lossless); a piece that still exceeds+// target is hard-split on token count (char-prefix via TrimContentToTokenLimit,+// also lossless). PDF positions follow Plan A: the original (coarse)+// coordinates attach to the first sub-piece only. Non-text units pass through+// unchanged.+func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {+ if target <= 0 {+ return units+ }+ out := make([]schema.ChunkDoc, 0, len(units))+ for _, ck := range units {+ if ck.CKType != "text" {+ out = append(out, ck)+ continue+ }+ tk := intValue(ck.TKNums)+ if tk <= 0 {+ tk = tokenizeStr(ck.Text)+ }+ if tk <= target {+ if ck.TKNums == nil {+ ck.TKNums = intPtr(tk)+ }+ out = append(out, ck)+ continue+ }+ out = append(out, splitOversizedText(ck, target)...)+ }+ return out+}++// splitOversizedText splits one oversized text unit into <= target pieces.+// Sentence boundaries are preferred; a boundary-less run that still exceeds+// target is hard token-split. Delimiters are preserved so the concatenated+// pieces reproduce the unit text exactly (lossless). Every piece keeps the+// source unit's DocType, and PDF positions follow Plan A (original coordinates+// on the first piece only).+func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {+ text := ck.Text+ // Preserve a leading "\n" glue (text-path units are built as "\n"+part).+ lead := ""+ if strings.HasPrefix(text, "\n") {+ lead = "\n"+ text = strings.TrimPrefix(text, "\n")+ }+ // Split into sentences (delimiters attached). A whitespace-only fragment+ // (a bare delimiter from a repeated run, e.g. "\n\n") is prepended to the+ // NEXT non-whitespace piece so the concatenated pieces reproduce the unit+ // text exactly; trailing whitespace attaches to the last piece. If+ // attaching whitespace pushes a piece over the target, that piece is+ // hard-split so the hard cap still holds.+ var pieces []schema.ChunkDoc+ var wsPending string+ for _, part := range splitSentencesLossless(text) {+ if strings.TrimSpace(part) == "" {+ wsPending += part+ continue+ }+ p := schema.ChunkDoc{Text: wsPending + part, DocType: ck.DocType, CKType: "text"}+ wsPending = ""+ if n := tokenizeStr(p.Text); n > target {+ pieces = append(pieces, hardSplitPiece(p.Text, ck.DocType, target)...)+ } else {+ p.TKNums = intPtr(n)+ pieces = append(pieces, p)+ }+ }+ if wsPending != "" {+ if len(pieces) > 0 {+ last := &pieces[len(pieces)-1]+ last.Text += wsPending+ if n := tokenizeStr(last.Text); n > target {+ replaced := hardSplitPiece(last.Text, ck.DocType, target)+ pieces = append(pieces[:len(pieces)-1], replaced...)+ } else {+ last.TKNums = intPtr(n)+ }+ } else {+ // All-whitespace unit: keep it lossless but still within the cap —+ // hard-split like any other oversized text so a whitespace-only run+ // whose token count exceeds the target cannot emit an over-cap chunk.+ if n := tokenizeStr(wsPending); n > target {+ pieces = append(pieces, hardSplitPiece(wsPending, ck.DocType, target)...)+ } else {+ pieces = append(pieces, schema.ChunkDoc{Text: wsPending, DocType: ck.DocType, TKNums: intPtr(n), CKType: "text"})+ }+ }+ }+ if len(pieces) == 0 {+ return nil+ }+ if lead != "" {+ pieces[0].Text = lead + pieces[0].Text+ // The leading "\n" adds a token; recompute so the running sum stays+ // faithful. Re-split if it pushes the piece over the target.+ if n := tokenizeStr(pieces[0].Text); n > target {+ replaced := hardSplitPiece(pieces[0].Text, ck.DocType, target)+ pieces = append(replaced, pieces[1:]...)+ } else {+ pieces[0].TKNums = intPtr(n)+ }+ }+ // Plan A: the first sub-piece inherits the source unit's metadata (coarse+ // positions + item attributes); later sub-pieces keep only the basics.+ // Build it from a clone so every ChunkDoc field survives the split.+ head := cloneChunkDoc(ck)+ head.Text = pieces[0].Text+ head.TKNums = pieces[0].TKNums+ head.CKType = "text"+ pieces[0] = head+ return pieces+}++// splitSentencesLossless splits text on sentenceDelimiter, keeping each+// delimiter attached to its preceding sentence, so the concatenated parts+// reproduce the input exactly (lossless). Mirrors Python's+// _split_text_by_sentences (rag/flow/chunker/title_chunker/common.py).+func splitSentencesLossless(text string) []string {+ if text == "" {+ return nil+ }+ idxs := sentenceDelimiter.FindAllStringIndex(text, -1)+ var out []string+ prev := 0+ for _, idx := range idxs {+ if idx[0] > prev {+ out = append(out, text[prev:idx[1]])+ } else {+ out = append(out, text[idx[0]:idx[1]])+ }+ prev = idx[1]+ }+ if prev < len(text) {+ out = append(out, text[prev:])+ }+ return out+}++// hardSplitPiece hard token-splits a boundary-less run into <= target pieces.+// Each piece is a true character prefix of the remaining text (U+FFFD tail+// trimmed), so the concatenated pieces reproduce the input exactly. A token+// cut never lands inside a @@...## coordinate tag: the cut is extended past+// the closing "##" when the budget allows, otherwise backed off to before the+// "@@" — so a tag is never split across two pieces.+func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {+ var out []schema.ChunkDoc+ rest := text+ for tokenizeStr(rest) > target {+ head := tokenizer.TrimContentToTokenLimit(rest, target)+ if head == "" || head == rest {+ break // cannot shrink further; avoid an infinite loop+ }+ // TrimContentToTokenLimit decodes a token prefix; if that lands on a+ // mid-multibyte boundary it can emit a trailing U+FFFD which is not a+ // true prefix of rest. Trim it so the rest[len(head):] advance stays+ // lossless.+ if !strings.HasPrefix(rest, head) {+ head = strings.TrimRight(head, "\uFFFD")+ if head == "" {+ break+ }+ }+ cut := adjustCutPastTag(rest, len(head), target)+ if cut <= 0 || cut >= len(rest) {+ // Safety: never stall. An empty cut or a whole-text cut means no+ // progress is possible; emit the remaining text as the last piece.+ if rest != "" {+ out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest)), CKType: "text"})+ }+ rest = ""+ break+ }+ head = rest[:cut]+ out = append(out, schema.ChunkDoc{Text: head, DocType: docType, TKNums: intPtr(tokenizeStr(head)), CKType: "text"})+ rest = rest[cut:]+ }+ if rest != "" {+ out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest)), CKType: "text"})+ }+ return out+}++// adjustCutPastTag shifts a character cut so it never lands inside a+// @@...## coordinate tag: extending past the closing "##" is preferred when+// the token budget allows; otherwise the cut backs off to just before the+// "@@". Tags with no closing "##" (unterminated) are ignored — they have no+// paired span to protect.+func adjustCutPastTag(text string, cut, target int) int {+ searchFrom := 0+ for {+ s := strings.Index(text[searchFrom:], "@@")+ if s < 0 {+ return cut+ }+ s += searchFrom+ eRel := strings.Index(text[s+2:], "##")+ if eRel < 0 {+ return cut // unterminated tag; nothing to protect+ }+ e := s + 2 + eRel + 2+ if cut > s && cut < e {+ // The cut lands inside a tag: prefer extending past the tag if it+ // stays within the token budget, else back off to before it. A+ // leading tag (s == 0) cannot back off to an empty piece, so it is+ // kept whole by extending past it.+ if e <= len(text) && tokenizeStr(text[:e]) <= target {+ return e+ }+ if s > 0 {+ return s+ }+ return e+ }+ searchFrom = e+ }+}++// overlapFitPrefix trims the overlap prefix so overlap+current fits within+// target tokens (hard cap with overlap enabled). It returns the trimmed+// overlap and the number of runes removed from its front, so callers can+// adjust the position-tail cut to the actual overlap region.+func overlapFitPrefix(overlap, current string, target int) (string, int) {+ if tokenizeStr(overlap+current) <= target {+ return overlap, 0+ }+ runes := []rune(overlap)+ for c := 1; c <= len(runes); c++ {+ suffix := string(runes[c:])+ if tokenizeStr(suffix+current) <= target {+ return suffix, c+ }+ }+ // current alone should never exceed target (units are pre-expanded); if it+ // somehow does, drop the overlap entirely.+ return "", len(runes)+}+func cloneChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {out := inif in.TKNums != nil {diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.goindex 3dabff167f..4515d63186 100644--- a/internal/ingestion/component/chunker/token_batch1_test.go+++ b/internal/ingestion/component/chunker/token_batch1_test.go@@ -59,23 +59,23 @@ func TestSentenceDelimiterMatchesBangAndQuestion(t *testing.T) {// join exceeds the budget — so the first unit must already sit near the// budget and the second unit must not fit alongside it.func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {- // OVER_CAP (Python's canonical default) merges an overflowing unit into- // the previous chunk and then closes it, so a chunk can exceed budget by- // at most one unit. To exercise the overlap path we use three units:+ // The hard-cap merge (UNDER_CAP) starts a fresh chunk when the projected+ // join would exceed the budget. Three units:// - a carries a parser tag and fits the budget alone,- // - b makes a+b overflow, so a+b merge-then-close into chunk0,- // - c starts a fresh chunk (prevClosed) with an overlap prefix from- // chunk0, which is where the tag-stripping must hold.+ // - b does not fit alongside a (a+b exceeds the budget), so b starts a+ // fresh chunk carrying an overlap prefix carved from a,+ // - c fits alongside b and merges into the same chunk; the tag-stripping+ // must hold in the overlap prefix on that chunk.aText := strings.Repeat("word ", 18) + "@@1\t2.3## tail"bText := strings.Repeat("word ", 18)cText := strings.Repeat("word ", 6)aN, bN, cN := tokenizeStr(aText), tokenizeStr(bText), tokenizeStr(cText)- // The merge decision is now the running sum of per-unit token counts- // (aN + bN), faithful to Python's _merge_text_chunks_by_token_size, NOT the+ // The merge decision is the running sum of per-unit token counts (aN+bN),+ // faithful to Python's _merge_text_chunks_by_token_size, NOT the// re-tokenized a+b join. Budget just below the a+b running sum so a and b- // cannot merge without overflowing (forcing mergeThenClose on chunk0), but- // a alone and c alone fit, and an overlap prefix carved from chunk0 fits- // ahead of c (overlap path is exercised on chunk 1).+ // cannot merge (b starts a fresh chunk), while a alone, c alone, and b+c+ // all fit, and an overlap prefix carved from a fits ahead of b (the overlap+ // path is exercised on chunk 1).budget := aN + bN - 1if budget < aN {budget = aN@@ -93,13 +93,13 @@ func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {{Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},},}- got := mergeByTokenSizeFromJSON(items, budget, 30.0, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, budget, 30.0)merged := got[0]if len(merged) != 2 {- t.Fatalf("want 2 chunks (overflow-closed + overlap chunk), got %d (a=%d b=%d c=%d budget=%d)", len(merged), aN, bN, cN, budget)+ t.Fatalf("want 2 chunks (a + overlap-prefixed b+c), got %d (a=%d b=%d c=%d budget=%d)", len(merged), aN, bN, cN, budget)}// The overlap prefix must actually be prepended to chunk 1; otherwise the- // test would pass even if prevClosed started c without any overlap.+ // test would pass even if b started without any overlap.overlap, _ := computeOverlapPrefix(merged[0].Text, 30.0)if overlap == "" {t.Fatal("expected a non-empty overlap prefix")@@ -118,36 +118,34 @@ func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {}}-// TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsPrevClosed is a TDD-// regression test for the OVER_CAP boundary-overflow flag (prevClosed) leaking-// across a non-text chunk. mergeByTokenSizeFromJSON must reset prevClosed when-// the previous merged chunk is non-text; otherwise the first text chunk after a-// non-text chunk carries the stale flag and forces the NEXT text chunk to start-// a fresh chunk even though it should merge with its predecessor.+// TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun is a regression+// test for the merge run leaking across a non-text chunk. mergeUnits must+// reset the running state when the previous merged chunk is non-text;+// otherwise the first text chunk after a non-text chunk carries the stale+// state and the next text chunk fails to merge with its predecessor.//-// Sequence: T1, T2 (overflow-merge-close into chunk0), N (non-text), T3, T4.-// With a correct reset, T3 is the first text after N (new chunk) and T4 merges-// back into T3 -> 3 chunks total. With the bug, prevClosed survives the N-// boundary and T4 is wrongly forced into its own chunk -> 4 chunks.-func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsPrevClosed(t *testing.T) {- t1 := strings.Repeat("word ", 18)- t2 := strings.Repeat("word ", 18)- t3 := strings.Repeat("word ", 9)- t4 := strings.Repeat("word ", 9)+// Sequence: T1, T2 (merged into chunk0), N (non-text), T3, T4. With a correct+// reset, T3 is the first text after N (new chunk) and T4 merges back into T3+// -> 3 chunks total. With the bug, T3 wrongly carries the stale run and T4 is+// forced into its own chunk -> 4 chunks.+func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {+ t1 := strings.Repeat("word ", 9)+ t2 := strings.Repeat("word ", 9)+ t3 := strings.Repeat("word ", 4)+ t4 := strings.Repeat("word ", 4)t1N, t2N := tokenizeStr(t1), tokenizeStr(t2)- joined12 := tokenizeStr(t1 + "\n" + t2)- // Budget just below the T1+T2 join so T1 and T2 cannot merge without- // overflowing, forcing mergeThenClose on chunk0 (prevClosed=true).- budget := joined12 - 1- if budget < t1N {- budget = t1N+ t3N, t4N := tokenizeStr(t3), tokenizeStr(t4)+ // Budget: T1+T2 fit exactly (merge into chunk0); T3+T4 must also fit so+ // they merge into chunk2 only if the run reset correctly.+ budget := t1N + t2N+ if budget < t3N {+ budget = t3N}- t34 := tokenizeStr(t3 + "\n" + t4)- if t34 > budget {- t.Fatalf("T3+T4 must fit budget to exercise the merge; t34=%d budget=%d", t34, budget)+ if t3N+t4N > budget {+ t.Fatalf("T3+T4 must fit budget to exercise the merge; t3=%d t4=%d sum=%d budget=%d", t3N, t4N, t3N+t4N, budget)}- if joined12 <= budget {- t.Fatalf("could not derive tight budget (t1=%d t2=%d joined=%d budget=%d)", t1N, t2N, joined12, budget)+ if t1N+t2N > budget {+ t.Fatalf("T1+T2 must fit budget; t1=%d t2=%d sum=%d budget=%d", t1N, t2N, t1N+t2N, budget)}items := [][]schema.ChunkDoc{@@ -155,25 +153,24 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsPrevClosed(t *testing.T){Text: t1, DocType: "text", CKType: "text", TKNums: intPtr(t1N)},{Text: t2, DocType: "text", CKType: "text", TKNums: intPtr(t2N)},{Text: "[image]", DocType: "image", CKType: "image", TKNums: intPtr(1)},- {Text: t3, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(t3))},- {Text: t4, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(t4))},+ {Text: t3, DocType: "text", CKType: "text", TKNums: intPtr(t3N)},+ {Text: t4, DocType: "text", CKType: "text", TKNums: intPtr(t4N)},},}- got := mergeByTokenSizeFromJSON(items, budget, 0.0, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, budget, 0.0)merged := got[0]- // Expect: chunk0 (T1+T2, overflow-closed), N (non-text), chunk1 (T3+T4 merged).+ // Expect: chunk0 (T1+T2 merged), N (non-text), chunk1 (T3+T4 merged).if len(merged) != 3 {var texts []stringfor _, c := range merged {texts = append(texts, c.CKType+":"+c.Text)}- t.Fatalf("want 3 chunks (overflow-closed + non-text + merged), got %d: %v", len(merged), texts)+ t.Fatalf("want 3 chunks (merged + non-text + merged), got %d: %v", len(merged), texts)}- // Lock the overflow-closed precondition: T1+T2 must have merged into- // chunk0 with prevClosed set, otherwise the later assertions could pass- // without exercising the boundary-overflow path at all.+ // Lock the merge precondition: T1+T2 must have merged into chunk0,+ // otherwise the later assertions could pass without exercising the merge.if merged[0].Text != t1+"\n"+t2 {- t.Errorf("chunk[0] should be the overflow-closed T1+T2 chunk: got %q", merged[0].Text)+ t.Errorf("chunk[0] should be the merged T1+T2 chunk: got %q", merged[0].Text)}if merged[1].CKType != "image" {t.Errorf("chunk[1] should be the non-text image chunk, got CKType=%q text=%q", merged[1].CKType, merged[1].Text)@@ -184,12 +181,10 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsPrevClosed(t *testing.T)}}-// TestMergeByTokenSizeFromJSON_UnderCapNoOverflow exercises the UNDER_CAP-// strategy (schema.MergeUnderCap): a projected join that would exceed-// the target must start a fresh chunk instead of merging-then-closing. This is-// the seam that lets Go follow Python's no-overflow (UNDER_CAP) strategy. Under-// OVER_CAP the same input merges a+b and overflows chunk0; here a, b, c must-// stay as three separate chunks, each within budget.+// TestMergeByTokenSizeFromJSON_UnderCapNoOverflow exercises the hard-cap merge+// contract (UNDER_CAP): a projected join that would exceed the target must+// start a fresh chunk instead of merging-then-closing. a, b, c must stay as+// three separate chunks, each within budget.func TestMergeByTokenSizeFromJSON_UnderCapNoOverflow(t *testing.T) {aText := strings.Repeat("word ", 18)bText := strings.Repeat("word ", 18)@@ -215,7 +210,7 @@ func TestMergeByTokenSizeFromJSON_UnderCapNoOverflow(t *testing.T) {{Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},},}- got := mergeByTokenSizeFromJSON(items, budget, 0.0, schema.MergeUnderCap)+ got := mergeByTokenSizeFromJSON(items, budget, 0.0)merged := got[0]if len(merged) != 3 {t.Fatalf("UNDER_CAP want 3 chunks (a, b, c separate), got %d", len(merged))@@ -257,12 +252,12 @@ func clampOverlapFixture() [][]schema.ChunkDoc {}func TestMergeByTokenSizeFromJSON_ClampsOverlappedPct(t *testing.T) {- at100 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 100, schema.MergeOverCap)- if at100 == nil || len(at100) == 0 {+ at100 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 100)+ if len(at100) == 0 {t.Fatalf("overlappedPct=100: nil/empty result")}- at150 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 150, schema.MergeOverCap)- atHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 1e300, schema.MergeOverCap)+ at150 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 150)+ atHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 1e300)if !reflect.DeepEqual(at100, at150) {t.Errorf("overlappedPct=150 should clamp to 100; output differs from 100")}@@ -270,12 +265,12 @@ func TestMergeByTokenSizeFromJSON_ClampsOverlappedPct(t *testing.T) {t.Errorf("overlappedPct=1e300 should clamp to 100; output differs from 100")}- at0 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 0, schema.MergeOverCap)- if at0 == nil || len(at0) == 0 {+ at0 := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, 0)+ if len(at0) == 0 {t.Fatalf("overlappedPct=0: nil/empty result")}- atNeg := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -5, schema.MergeOverCap)- atNegHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -1e300, schema.MergeOverCap)+ atNeg := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -5)+ atNegHuge := mergeByTokenSizeFromJSON(clampOverlapFixture(), 128, -1e300)if !reflect.DeepEqual(at0, atNeg) {t.Errorf("overlappedPct=-5 should clamp to 0; output differs from 0")}@@ -296,7 +291,7 @@ func TestMergeByTokenSizeFromJSON_EmptyPrevKeepsChunk(t *testing.T) {{Text: "keepme", DocType: "text", CKType: "text", TKNums: intPtr(5)},},}- got := mergeByTokenSizeFromJSON(items, 128, 0, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, 128, 0)merged := got[0]if len(merged) != 1 {t.Fatalf("want 1 merged chunk, got %d", len(merged))diff --git a/internal/ingestion/component/chunker/token_hardcap_test.go b/internal/ingestion/component/chunker/token_hardcap_test.gonew file mode 100644index 0000000000..c71e15d498--- /dev/null+++ b/internal/ingestion/component/chunker/token_hardcap_test.go@@ -0,0 +1,380 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// ---------------------------------------------------------------------------+// 方案 B: hard-cap merge contract.+//+// Every text chunk emitted by the TokenChunker merge must be <= chunk_token_size:+// 1. Oversized units are expanded first (sentence-boundary split, then a+// hard token-split fallback for boundary-less runs) so no unit exceeds the+// target.+// 2. The merge is UNDER_CAP: a projected join that would push the running sum+// over the target starts a fresh chunk instead of merge-then-close.+// 3. The overlap prefix is trimmed so it never pushes a fresh chunk over the+// target.+// 4. Non-text units pass through unchanged (atomic).+// 5. Expansion is lossless: sub-pieces reproduce the original text.+// ---------------------------------------------------------------------------++// TestMergeUnits_UnderCapRunningSum pins the UNDER_CAP decision rule with exact+// running sums (independent of BPE text-count noise): 25+25 fits 50 exactly,+// the next 25-unit must start a fresh chunk.+func TestMergeUnits_UnderCapRunningSum(t *testing.T) {+ const target = 50+ units := []schema.ChunkDoc{+ {Text: "u1", TKNums: intPtr(25), CKType: "text"},+ {Text: "u2", TKNums: intPtr(25), CKType: "text"},+ {Text: "u3", TKNums: intPtr(25), CKType: "text"},+ {Text: "u4", TKNums: intPtr(25), CKType: "text"},+ }+ got := mergeUnits(units, target, 0, "")+ if len(got) != 2 {+ t.Fatalf("want 2 chunks (25+25, 25+25), got %d", len(got))+ }+ for i, ck := range got {+ if sum := intValue(ck.TKNums); sum > target {+ t.Errorf("chunk %d running sum %d exceeds target %d", i, sum, target)+ }+ }+}++// TestMergeUnits_BoundaryExactFit pins the boundary: prev+incoming == target+// still merges (<= target), prev+incoming > target starts a fresh chunk.+func TestMergeUnits_BoundaryExactFit(t *testing.T) {+ fit := mergeUnits([]schema.ChunkDoc{+ {Text: "a", TKNums: intPtr(25), CKType: "text"},+ {Text: "b", TKNums: intPtr(25), CKType: "text"},+ }, 50, 0, "")+ if len(fit) != 1 {+ t.Fatalf("exact-fit join must merge, got %d chunks", len(fit))+ }+ over := mergeUnits([]schema.ChunkDoc{+ {Text: "a", TKNums: intPtr(30), CKType: "text"},+ {Text: "b", TKNums: intPtr(30), CKType: "text"},+ }, 50, 0, "")+ if len(over) != 2 {+ t.Fatalf("overflowing join must start a fresh chunk, got %d chunks", len(over))+ }+}++// TestMergeUnits_ExpandsOversizedUnit verifies a single boundary-less unit that+// exceeds the target is hard token-split into <= target pieces and the+// concatenated pieces reproduce the original text exactly (lossless).+func TestMergeUnits_ExpandsOversizedUnit(t *testing.T) {+ const target = 30+ body := strings.Repeat("word ", 100) // ~100 tokens, no sentence delimiter+ got := mergeUnits([]schema.ChunkDoc{+ {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ }, target, 0, "")+ if len(got) < 2 {+ t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))+ }+ var joined string+ for i, ck := range got {+ if n := tokenizeStr(ck.Text); n > target {+ t.Errorf("chunk %d exceeds target: tokens=%d (cap=%d)", i, n, target)+ }+ joined += ck.Text+ }+ if joined != body {+ t.Errorf("hard split not lossless:\n got %q\nwant %q", joined, body)+ }+}++// TestMergeUnits_ExpandsOversizedSentence verifies an oversized unit with+// sentence delimiters is split on sentence boundaries first (delimiters+// preserved, lossless), each piece <= target.+func TestMergeUnits_ExpandsOversizedSentence(t *testing.T) {+ sentence := "word word word word word word。"+ body := strings.Repeat(sentence, 4)+ target := tokenizeStr(sentence) // each sentence fits the target; the whole body does not+ if target <= 0 {+ t.Fatal("fixture: tokenize returned 0 for sentence")+ }+ got := mergeUnits([]schema.ChunkDoc{+ {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ }, target, 0, "")+ if len(got) < 2 {+ t.Fatalf("oversized unit must be sentence-split, got %d chunk(s)", len(got))+ }+ var joined string+ for i, ck := range got {+ if n := tokenizeStr(ck.Text); n > target {+ t.Errorf("chunk %d exceeds target: tokens=%d (cap=%d)", i, n, target)+ }+ joined += ck.Text+ }+ if joined != body {+ t.Errorf("sentence split not lossless:\n got %q\nwant %q", joined, body)+ }+}++// TestMergeUnits_PlanAPositions verifies Plan A: original (coarse) positions+// attach to the first sub-chunk of an expanded unit only; later sub-chunks+// carry none.+func TestMergeUnits_PlanAPositions(t *testing.T) {+ const target = 30+ body := strings.Repeat("word ", 100)+ pos := []byte(`[{"page":1}]`)+ got := mergeUnits([]schema.ChunkDoc{+ {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body)), PDFPositions: pos},+ }, target, 0, "")+ if len(got) < 2 {+ t.Fatalf("want >=2 chunks, got %d", len(got))+ }+ if len(got[0].PDFPositions) == 0 {+ t.Errorf("first sub-chunk must carry the original positions")+ }+ for i := 1; i < len(got); i++ {+ if len(got[i].PDFPositions) != 0 {+ t.Errorf("sub-chunk %d must carry no positions", i)+ }+ }+}++// TestMergeUnits_AtomicNonText verifies non-text chunks (tables/images) are+// never split even when their token count exceeds the target.+func TestMergeUnits_AtomicNonText(t *testing.T) {+ table := schema.ChunkDoc{Text: strings.Repeat("t", 500), CKType: "table", TKNums: intPtr(500)}+ got := mergeUnits([]schema.ChunkDoc{table}, 50, 0, "")+ if len(got) != 1 || got[0].Text != table.Text {+ t.Fatalf("table chunk must pass through unchanged, got %#v", got)+ }+}++// TestMergeUnits_ExpandedPiecesKeepDocType verifies that pieces produced by+// the oversized-unit expansion carry the source unit's DocType, so the emitted+// chunks keep doc_type_kwd="text" instead of silently dropping it.+func TestMergeUnits_ExpandedPiecesKeepDocType(t *testing.T) {+ const target = 30+ body := strings.Repeat("word ", 100)+ got := mergeUnits([]schema.ChunkDoc{+ {Text: body, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ }, target, 0, "")+ if len(got) < 2 {+ t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))+ }+ for i, ck := range got {+ if ck.DocType != "text" {+ t.Errorf("chunk %d lost DocType: got %q, want %q", i, ck.DocType, "text")+ }+ }+}++// TestMergeUnits_HardSplitNeverSplitsCoordTag verifies the hard token-split+// never cuts through a @@...## coordinate tag: a piece either contains the+// complete tag or no part of it.+func TestMergeUnits_HardSplitNeverSplitsCoordTag(t *testing.T) {+ const target = 8+ const tag = "@@1\t2\t3\t4##"+ body := strings.Repeat("word ", 30) + tag + strings.Repeat("pad ", 30)+ got := mergeUnits([]schema.ChunkDoc{+ {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ }, target, 0, "")+ if len(got) < 3 {+ t.Fatalf("want multiple pieces, got %d", len(got))+ }+ tagCount := 0+ for i, ck := range got {+ if strings.Contains(ck.Text, "@@") || strings.Contains(ck.Text, "##") {+ tagCount+++ if !strings.Contains(ck.Text, tag) {+ t.Errorf("piece %d contains a partial coord tag: %q", i, ck.Text)+ }+ }+ }+ if tagCount != 1 {+ t.Errorf("the coord tag must appear whole in exactly one piece, got %d", tagCount)+ }+}++// TestMergeUnits_OverlapStrictCap verifies the overlap prefix never pushes a+// fresh chunk over the target: overlap is trimmed to fit. Both the running sum+// (TKNums) and the re-tokenized emitted text are checked against the target.+func TestMergeUnits_OverlapStrictCap(t *testing.T) {+ const target = 40+ units := []schema.ChunkDoc{+ {Text: strings.Repeat("word ", 30), CKType: "text"}, // ~30 tokens+ {Text: strings.Repeat("pad ", 38), CKType: "text"}, // ~38 tokens, near target+ }+ got := mergeUnits(units, target, 20, "\n")+ if len(got) < 2 {+ t.Fatalf("want >=2 chunks, got %d", len(got))+ }+ for i, ck := range got {+ if n := intValue(ck.TKNums); n > target {+ t.Errorf("chunk %d exceeds target: tokens=%d (cap=%d)", i, n, target)+ }+ if n := tokenizeStr(ck.Text); n > target {+ t.Errorf("chunk %d emitted text exceeds target: tokens=%d (cap=%d)", i, n, target)+ }+ }+}++// TestMergeUnits_ExpansionPreservesWhitespaceFragments verifies that+// whitespace-only fragments from a repeated delimiter run (e.g. "\n\n") are+// carried into adjacent pieces, so the concatenated pieces reproduce the+// oversized unit text exactly (lossless).+func TestMergeUnits_ExpansionPreservesWhitespaceFragments(t *testing.T) {+ const target = 30+ body := strings.Repeat("word ", 60) + "\n\n" + strings.Repeat("pad ", 60)+ got := mergeUnits([]schema.ChunkDoc{+ {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ }, target, 0, "")+ if len(got) < 2 {+ t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))+ }+ var joined string+ for _, ck := range got {+ joined += ck.Text+ }+ if joined != body {+ t.Errorf("split dropped text (whitespace or content):\n got %q\nwant %q", joined, body)+ }+}++// TestMergeUnits_ExpandedFirstPieceCarriesMetadata verifies Plan A metadata+// inheritance: the first sub-piece of an expanded unit keeps every field the+// source unit carried (coarse positions plus item attributes), while later+// sub-pieces keep only the basics.+func TestMergeUnits_ExpandedFirstPieceCarriesMetadata(t *testing.T) {+ const target = 30+ body := strings.Repeat("word ", 100)+ unit := schema.ChunkDoc{+ Text: body,+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(tokenizeStr(body)),+ PDFPositions: []byte(`[{"page":1}]`),+ Mom: "parent-id",+ ImgID: "img-1",+ Layout: "text",+ Image: "raw-image",+ PageNumber: intPtr(3),+ TagKwd: []string{"t1"},+ ChunkOrderInt: intPtr(7),+ }+ got := mergeUnits([]schema.ChunkDoc{unit}, target, 0, "\n")+ if len(got) < 2 {+ t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))+ }+ head := got[0]+ if head.Mom != "parent-id" || head.ImgID != "img-1" || head.Layout != "text" || head.Image != "raw-image" {+ t.Errorf("first sub-piece lost item metadata: %#v", head)+ }+ if head.PageNumber == nil || *head.PageNumber != 3 {+ t.Errorf("first sub-piece lost PageNumber: %#v", head.PageNumber)+ }+ if len(head.TagKwd) != 1 || head.TagKwd[0] != "t1" {+ t.Errorf("first sub-piece lost TagKwd: %#v", head.TagKwd)+ }+ if head.ChunkOrderInt == nil || *head.ChunkOrderInt != 7 {+ t.Errorf("first sub-piece lost ChunkOrderInt: %#v", head.ChunkOrderInt)+ }+ if len(head.PDFPositions) == 0 {+ t.Errorf("first sub-piece must carry the original positions")+ }+ for i := 1; i < len(got); i++ {+ if len(got[i].PDFPositions) != 0 {+ t.Errorf("sub-piece %d must carry no positions", i)+ }+ if got[i].Mom != "" || got[i].ImgID != "" || got[i].Layout != "" {+ t.Errorf("sub-piece %d must not carry item metadata", i)+ }+ }+}++// TestMergeByTokenSizeFromJSON_HardCapStrict verifies the JSON path never emits+// a text chunk whose running sum exceeds the budget.+func TestMergeByTokenSizeFromJSON_HardCapStrict(t *testing.T) {+ const budget = 40+ var items []schema.ChunkDoc+ for i := 0; i < 12; i++ {+ text := strings.Repeat("w ", 12)+ items = append(items, schema.ChunkDoc{Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text))})+ }+ got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{items}, budget, 0)+ if len(got[0]) < 2 {+ t.Fatalf("want multiple chunks, got %d", len(got[0]))+ }+ for i, ck := range got[0] {+ if n := intValue(ck.TKNums); n > budget {+ t.Errorf("chunk %d running sum %d exceeds budget %d", i, n, budget)+ }+ }+}++// TestMergeUnits_JoinSepAddsTokens pins the re-tokenize guard: joinSep (the+// JSON path's "\n") can add tokens beyond the running sum, so the emitted text+// of a merge that the running sum deems safe can still exceed target. The+// merge must re-check the actual joined text and start a fresh chunk instead.+func TestMergeUnits_JoinSepAddsTokens(t *testing.T) {+ a := "word"+ b := "pad"+ na, nb := tokenizeStr(a), tokenizeStr(b)+ target := na + nb // running sum exactly fits+ if joinedTK := tokenizeStr(a + "\n" + b); joinedTK <= target {+ t.Skipf("fixture: joinSep newline adds no token for these texts (joined=%d target=%d)", joinedTK, target)+ }+ got := mergeUnits([]schema.ChunkDoc{+ {Text: a, TKNums: intPtr(na), CKType: "text"},+ {Text: b, TKNums: intPtr(nb), CKType: "text"},+ }, target, 0, "\n")+ if len(got) != 2 {+ t.Fatalf("joined text exceeds target via joinSep: want 2 chunks, got %d", len(got))+ }+ for i, ck := range got {+ if n := tokenizeStr(ck.Text); n > target {+ t.Errorf("chunk %d exceeds target: tokens=%d (cap=%d) text=%q", i, n, target, ck.Text)+ }+ }+}++// TestExpandOversizedUnits_AllWhitespaceOverCap pins the all-whitespace branch:+// a whitespace-only unit whose token count exceeds the target must be hard-split+// like any other oversized unit, not emitted as a single over-cap chunk. The+// fixture avoids a leading "\n" (which the lead-glue branch already re-splits)+// so it exercises the all-whitespace else branch in splitOversizedText.+func TestExpandOversizedUnits_AllWhitespaceOverCap(t *testing.T) {+ ws := strings.Repeat(" \n", 20) // 40 runes; each "\n" is one cl100k token+ const target = 8+ if n := tokenizeStr(ws); n <= target {+ t.Skipf("fixture: whitespace tokenizes to %d <= target %d", n, target)+ }+ got := expandOversizedUnits([]schema.ChunkDoc{{Text: ws, CKType: "text"}}, target)+ if len(got) < 2 {+ t.Fatalf("all-whitespace unit over cap must be hard-split, got %d chunk(s)", len(got))+ }+ var joined string+ for i, ck := range got {+ if n := tokenizeStr(ck.Text); n > target {+ t.Errorf("chunk %d exceeds target: tokens=%d (cap=%d)", i, n, target)+ }+ joined += ck.Text+ }+ if joined != ws {+ t.Errorf("split not lossless:\n got=%q\nwant=%q", joined, ws)+ }+}diff --git a/internal/ingestion/component/chunker/token_json_overlap_test.go b/internal/ingestion/component/chunker/token_json_overlap_test.goindex 89f4c59c25..16bb8e0343 100644--- a/internal/ingestion/component/chunker/token_json_overlap_test.go+++ b/internal/ingestion/component/chunker/token_json_overlap_test.go@@ -24,19 +24,17 @@ import ()// TestMergeByTokenSizeFromJSON_TKNumsConsistency exercises the JSON merge path-// with overlap > 0, which previously had NO coverage, and pins the TKNums-// accounting so the running-sum merge decision cannot silently regress to the-// old re-tokenized-join count.+// with overlap > 0 and pins the TKNums accounting so the running-sum merge+// decision cannot silently regress to the old re-tokenized-join count.//-// Fixture: three text units a/b/c with a and b small enough to each fit the-// budget but a+b's running sum overflows it (so a+b merge-then-close into-// chunk0), and c starts a fresh chunk (prevClosed) carrying an overlap prefix-// from chunk0.+// Fixture: three text units a/b/c with a+b fitting the budget (merge path into+// chunk0) and a+b+c overflowing it (so c starts a fresh chunk carrying an+// overlap prefix from chunk0).//// TKNums accounting (see token.go mergeByTokenSizeFromJSON):// - on the merge path (chunk0) TKNums is the RUNNING SUM of per-unit counts// (aN+bN), never tokenizeStr(chunk0.Text);-// - on the boundary path (chunk1, via prevClosed) TKNums is reset to+// - on the boundary path (chunk1, fresh) TKNums is reset to// tokenizeStr(chunk1.Text) — i.e. the two paths use different caliber.// This mixed accounting is documented in token.go; the assertions below// lock the current behavior so any future unification is a deliberate@@ -44,22 +42,20 @@ import (func TestMergeByTokenSizeFromJSON_TKNumsConsistency(t *testing.T) {for _, overlapPct := range []float64{0, 30} {t.Run(overlapName(overlapPct), func(t *testing.T) {- aText := strings.Repeat("word ", 18)- bText := strings.Repeat("word ", 18)+ aText := strings.Repeat("word ", 6)+ bText := strings.Repeat("word ", 6)cText := strings.Repeat("word ", 6)aN, bN, cN := tokenizeStr(aText), tokenizeStr(bText), tokenizeStr(cText)- // Budget just below the a+b running sum so a and b cannot merge- // without overflowing (forcing mergeThenClose on chunk0), while a- // alone and c alone fit.- budget := aN + bN - 1- if budget < aN {- budget = aN- }+ // Budget: a+b fits (merge), a+b+c overflows (fresh chunk), c alone+ // fits. For overlap>0 the overlap prefix is trimmed to fit, so the+ // exact prefix length varies; we only pin the accounting, not the+ // prefix.+ budget := aN + bNif budget < cN {budget = cN}- if aN+bN <= budget {- t.Fatalf("could not derive tight budget (a=%d b=%d sum=%d budget=%d)", aN, bN, aN+bN, budget)+ if aN+bN+cN <= budget {+ t.Fatalf("could not derive tight budget (a=%d b=%d c=%d sum=%d budget=%d)", aN, bN, cN, aN+bN+cN, budget)}items := [][]schema.ChunkDoc{{@@ -68,13 +64,13 @@ func TestMergeByTokenSizeFromJSON_TKNumsConsistency(t *testing.T) {{Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},},}- got := mergeByTokenSizeFromJSON(items, budget, overlapPct, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, budget, overlapPct)merged := got[0]if len(merged) != 2 {- t.Fatalf("want 2 chunks (overflow-closed + overlap/fresh chunk), got %d (a=%d b=%d c=%d budget=%d)", len(merged), aN, bN, cN, budget)+ t.Fatalf("want 2 chunks (merged a+b + fresh c), got %d (a=%d b=%d c=%d budget=%d)", len(merged), aN, bN, cN, budget)}- // chunk0 is the merge-then-close of a and b.+ // chunk0 is the merge of a and b.if merged[0].Text != aText+"\n"+bText {t.Errorf("chunk0 text mismatch:\n got=%q\nwant=%q", merged[0].Text, aText+"\n"+bText)}@@ -83,21 +79,21 @@ func TestMergeByTokenSizeFromJSON_TKNumsConsistency(t *testing.T) {t.Errorf("chunk0 TKNums: running sum want %d, got %d (tokenizeStr(chunk0.Text)=%d)", aN+bN, got0, tokenizeStr(merged[0].Text))}- // chunk1 is c (fresh, via prevClosed). With overlap>0 it carries a- // prefix carved from chunk0; verify the overlap path actually ran.+ // chunk1 is c (fresh). With overlap>0 it carries a prefix carved+ // from chunk0 (trimmed to fit the hard cap); c's text must still+ // be present as the suffix.if overlapPct > 0 {- overlap, _ := computeOverlapPrefix(merged[0].Text, overlapPct)- if overlap == "" {- t.Fatal("expected a non-empty overlap prefix from chunk0")- }- if !strings.HasPrefix(merged[1].Text, overlap) {- t.Errorf("chunk1 should start with overlap prefix %q, got %q", overlap, merged[1].Text)+ if !strings.HasSuffix(merged[1].Text, cText) {+ t.Errorf("chunk1 should end with c text %q, got %q", cText, merged[1].Text)}}// Boundary path: TKNums is the re-tokenized chunk1 text count.if got1 := intValue(merged[1].TKNums); got1 != tokenizeStr(merged[1].Text) {t.Errorf("chunk1 TKNums: want tokenizeStr(chunk1.Text)=%d, got %d", tokenizeStr(merged[1].Text), got1)}+ if n := intValue(merged[1].TKNums); n > budget {+ t.Errorf("chunk1 exceeds budget after overlap trim: tokens=%d (cap=%d)", n, budget)+ }})}}diff --git a/internal/ingestion/component/chunker/token_merge_parity_test.go b/internal/ingestion/component/chunker/token_merge_parity_test.goindex 8a0b52ef17..3510fd9f88 100644--- a/internal/ingestion/component/chunker/token_merge_parity_test.go+++ b/internal/ingestion/component/chunker/token_merge_parity_test.go@@ -20,6 +20,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema")// mergeSourceLines is a fixed 29-line plain-text payload. The exact wording is@@ -58,159 +59,96 @@ var mergeSourceLines = []string{"Plugins extend parsing for proprietary formats.",}-// pythonMergeGroups is a faithful transcription of the Python reference-// chunker merge rag/nlp/__init__.py:_merge_paragraph_groups under-// MergeStrategy.OVER_CAP. It is used here as the oracle for-// TestTokenChunkerMergeMatchesPython. Because it is itself Go code, it only-// proves that the Go TokenChunker reproduces THIS transcription; its fidelity-// to the real Python algorithm is validated by hand against the source lines-// noted below. Keep those line references in sync if the Python side moves.+// goMergeGroupsOracle is the Go-side reference for the hard-cap merge contract+// (方案 B): it builds text-path units ("\n" + paragraph) exactly like+// mergeByTokenSize and runs them through the mergeUnits core, so the+// component-vs-core comparison below validates the wiring (unitization,+// formatting) while the merge semantics themselves are pinned by the+// token_hardcap_test.go / token_strict_cap_test.go suites.//-// Mapping to rag/nlp/__init__.py:_merge_paragraph_groups (OVER_CAP):-// - each paragraph is built as "\n" + sub_sec (naive_merge ~:1405), so the-// per-paragraph token count INCLUDES the leading "\n"; we mirror that with-// pt := tokenizeStr("\n" + p).-// - cur_t is the running sum of per-paragraph size("\n" + sub_sec); a-// paragraph merges when cur_t + pt <= cap.-// - an oversized paragraph (pt > cap) is emitted alone as its own chunk-// (Python: never combined with the previous chunk).-// - when cur_t + pt > cap, OVER_CAP merges the overflowing paragraph into the-// current group and then CLOSES the group (merge-then-close), so the next-// paragraph starts a fresh group.-// - empty/whitespace-only paragraphs are DROPPED before merging, mirroring-// the Python caller's `if not sub_sec` filter (rag/nlp/__init__.py:1403)-// and the Go chunker's TrimSpace skip (token.go:703). The oracle must model-// this so it stays a faithful reference for mergeByTokenSize.-func pythonMergeGroups(paragraphs []string, cap int) [][]string {- groups := [][]string{}- cur, curT := []string{}, 0+// The Python-side OVER_CAP contract this file previously mirrored is now a+// deliberate Go divergence (go_intentional): Go never lets a text chunk exceed+// the token target, whereas Python's OVER_CAP closes a group right after an+// overflowing merge (a chunk may exceed the target by one unit) and keeps an+// oversized unit whole. The hard-cap behavior is documented in token.go.+//+// Whitespace exactness is OUTSIDE this oracle's contract: every emitted chunk+// is trimmed and the sanity check compares the normalized word stream, so a+// space/newline rewrite inside a chunk is not detected here. Whitespace+// preservation is pinned by the dedicated suites instead — token_oversize_split_test.go+// (lossless concatenation) and token_overlap_test.go (no space-less boundaries).+func goMergeGroupsOracle(paragraphs []string, cap int, overlapPct float64) []string {+ units := make([]schema.ChunkDoc, 0, len(paragraphs))for _, p := range paragraphs {- // Empty fragments are dropped, exactly as the Python caller and the- // Go chunker do. Skipping here keeps the oracle faithful.if strings.TrimSpace(p) == "" {continue}- // Python's naive_merge builds each paragraph as "\n" + sub_sec- // (rag/nlp/__init__.py:1405), so the per-paragraph token count- // INCLUDES the leading "\n". Count it the same way so the running- // sum matches Python's size("\n" + sub_sec).- pt := tokenizeStr("\n" + p)- if pt > cap {- if len(cur) > 0 {- groups = append(groups, cur)- }- groups = append(groups, []string{p})- cur, curT = []string{}, 0- continue- }- if len(cur) == 0 {- cur, curT = []string{p}, pt+ u := "\n" + p+ units = append(units, schema.ChunkDoc{Text: u, TKNums: intPtr(tokenizeStr(u)), CKType: "text"})+ }+ merged := mergeUnits(units, cap, overlapPct, "")+ out := make([]string, 0, len(merged))+ for _, ck := range merged {+ text := removeTag(strings.TrimSpace(ck.Text))+ if text == "" {continue}- if curT+pt <= cap {- cur = append(cur, p)- curT += pt- } else {- // OVER_CAP: merge the overflowing paragraph, then close.- cur = append(cur, p)- curT += pt- groups = append(groups, cur)- cur, curT = []string{}, 0- }- }- if len(cur) > 0 {- groups = append(groups, cur)+ out = append(out, text)}- return groups+ return out}-// TestPythonMergeGroupsOracleSanity pins pythonMergeGroups itself, so the-// oracle cannot silently drift in lock-step with the chunker under test-// (which would make TestTokenChunkerMergeMatchesPython a no-op check). The-// assertions are tokenizer-independent INVARIANTS of the naive_merge-// (OVER_CAP) algorithm rather than hard-coded token magnitudes, so the test-// is stable across tokenizer revisions:-// 1. every input paragraph appears exactly once, in order (no loss/dup/reorder);-// 2. every group is VALID: either its running sum is within cap (a normal-// group), or it is a maximal merge-then-close overflow (running sum >-// cap but dropping its last paragraph is within cap), or it is a single-// oversized paragraph that stands alone. The group's position in the slice-// (trailing or not) is irrelevant: an overflow group may be last when the-// input ends right after an overflow.-func TestPythonMergeGroupsOracleSanity(t *testing.T) {+// TestGoMergeGroupsOracleSanity pins the oracle output invariants that the+// hard-cap merge must satisfy, so the component-vs-core comparison cannot+// silently pass while the contract regresses: every emitted chunk is within the+// token cap, and the concatenated chunk texts reconstruct the source paragraph+// stream (order + no loss).+func TestGoMergeGroupsOracleSanity(t *testing.T) {const cap = 8inputs := [][]string{- {"word word word word word word word word word word"}, // clearly oversized -> own group- {"word", "word", "word", "word", "word", "word", "word", "word", "word", "word"}, // overflow path+ {"word word word word word word word word word word"}, // oversized -> hard split+ {"word", "word", "word", "word", "word", "word", "word", "word", "word", "word"}, // pack boundary{"", "x", "y", "z"}, // empty + small{"alpha", "beta", "gamma", "delta"}, // plain small}for ci, in := range inputs {- groups := pythonMergeGroups(in, cap)- // Python (rag/nlp:1403) and the Go chunker (token.go:703) drop- // empty/whitespace-only fragments, so the oracle does too. Compare- // against the non-empty partition of the input rather than `in`.- var nonEmpty []string- for _, p := range in {- if strings.TrimSpace(p) == "" {- continue- }- nonEmpty = append(nonEmpty, p)+ chunks := goMergeGroupsOracle(in, cap, 0)+ if len(chunks) == 0 {+ t.Fatalf("case %d: oracle produced no chunks", ci)}- var flat []string- for _, g := range groups {- flat = append(flat, g...)- }- if len(flat) != len(nonEmpty) {- t.Fatalf("case %d: paragraph count mismatch got=%d want=%d", ci, len(flat), len(nonEmpty))- }- for i := range nonEmpty {- if nonEmpty[i] != flat[i] {- t.Fatalf("case %d elem %d: order/loss mismatch got=%q want=%q", ci, i, flat[i], nonEmpty[i])+ for gi, text := range chunks {+ if n := tokenizeStr(text); n > cap {+ t.Errorf("case %d chunk %d exceeds cap: tokens=%d (cap=%d)", ci, gi, n, cap)}}- // An oversized paragraph (its own token count > cap) must stand- // alone: the oracle emits it as a single-element group.- for gi, g := range groups {- for _, p := range g {- if tokenizeStr("\n"+p) > cap {- if len(g) != 1 {- t.Errorf("case %d group %d: oversized paragraph not isolated (len=%d)", ci, gi, len(g))- }- break- }+ // Source preservation: the concatenated chunk texts must reproduce the+ // non-empty input paragraph stream (order + no loss). All whitespace is+ // normalized away because paragraph boundaries are joined with "\n" and+ // each emitted chunk is trimmed.+ var wantNormalized strings.Builder+ for _, p := range in {+ for _, w := range strings.Fields(p) {+ wantNormalized.WriteString(w)}}- for gi, g := range groups {- if len(g) == 0 {- t.Fatalf("case %d group %d is empty", ci, gi)- }- sum := 0- for _, p := range g {- sum += tokenizeStr("\n" + p)- }- prefix := 0- for _, p := range g[:len(g)-1] {- prefix += tokenizeStr("\n" + p)- }- valid := sum <= cap || (sum > cap && prefix <= cap)- if !valid {- t.Errorf("case %d group %d invalid: sum=%d prefix=%d (cap=%d)", ci, gi, sum, prefix, cap)+ var gotNormalized strings.Builder+ for _, text := range chunks {+ for _, w := range strings.Fields(text) {+ gotNormalized.WriteString(w)}}+ if gotNormalized.String() != wantNormalized.String() {+ t.Errorf("case %d: chunk stream dropped or reordered source text:\n got=%q\nwant=%q",+ ci, gotNormalized.String(), wantNormalized.String())+ }}}-// expectedPythonChunks returns the chunk texts Python's naive_merge produces-// for mergeSourceLines at the given chunk_token_size.-func expectedPythonChunks(t *testing.T, chunkTokenSize int) []string {+// expectedGoChunks returns the chunk texts the hard-cap merge produces for+// mergeSourceLines at the given chunk_token_size (overlap 0).+func expectedGoChunks(t *testing.T, chunkTokenSize int) []string {t.Helper()- groups := pythonMergeGroups(mergeSourceLines, chunkTokenSize)- want := make([]string, len(groups))- for i, g := range groups {- want[i] = strings.Join(g, "\n")- }- return want+ return goMergeGroupsOracle(mergeSourceLines, chunkTokenSize, 0)}// invokeTokenChunker runs the real TokenChunker component on plain text and@@ -252,30 +190,23 @@ func invokeTokenChunker(t *testing.T, text string, chunkTokenSize int, overlappereturn texts}-// TestTokenChunkerMergeMatchesPython is the red test for the merge-boundary-// divergence (ragflow chunker parity):-//-// The Go merge decision used tokenizeStr(prev + "\n" + incoming) — the BPE-// token count of the JOINED string — while Python's naive_merge accumulates a-// running sum of per-paragraph token counts (cur_t + size(p)). Because BPE-// tokenization is not additive across the "\n" boundary, the two merge-// decisions disagree: Go can merge one paragraph more (or fewer) than Python,-// shifting every downstream boundary by a line and, once a budget is small-// enough, changing the chunk count outright.+// TestTokenChunkerMergeMatchesGoOracle validates that the real TokenChunker+// component's text path reproduces the hard-cap merge contract chunk-for-chunk+// (count AND per-chunk text). The oracle goMergeGroupsOracle runs the same+// unitization as the component ("\n" + paragraph) through the mergeUnits core,+// so this test guards the component wiring; the merge semantics themselves are+// pinned by token_hardcap_test.go / token_strict_cap_test.go.//-// This test asserts the Go TokenChunker output equals Python's naive_merge-// output (verified chunk count AND per-chunk text) for two budgets:-// - chunk_token_size=32: Python emits 9 chunks (Go over-merged to 8 before-// the running-sum fix).-// - chunk_token_size=128: both emit 3 chunks, but Python closes chunk 0 one-// line earlier than Go (boundary offset before the fix).-func TestTokenChunkerMergeMatchesPython(t *testing.T) {+// Go intentionally diverges from Python's OVER_CAP here (go_intentional): Go+// never lets a text chunk exceed the target, while Python closes a group right+// after an overflowing merge and keeps oversized units whole.+func TestTokenChunkerMergeMatchesGoOracle(t *testing.T) {text := strings.Join(mergeSourceLines, "\n")for _, cap := range []int{32, 128} {- want := expectedPythonChunks(t, cap)+ want := expectedGoChunks(t, cap)got := invokeTokenChunker(t, text, cap, 0)if len(got) != len(want) {- t.Fatalf("chunk_token_size=%d: chunk count go=%d, python=%d", cap, len(got), len(want))+ t.Fatalf("chunk_token_size=%d: chunk count go=%d, oracle=%d", cap, len(got), len(want))}for i := range got {if got[i] != want[i] {@@ -285,64 +216,31 @@ func TestTokenChunkerMergeMatchesPython(t *testing.T) {}}-// pythonMergeWithOverlap mirrors rag/flow/chunker/token_chunker.py:-// _merge_text_chunks_by_token_size under the UNIFIED contract — scaled overlap-// threshold + unconditional char-based overlap prefix (no fit-check), with the-// #17799 over-budget unit standing alone. It is the Python-faithful oracle for-// TestTokenChunkerOverlapMatchesPython.-//-// Like the fixed Python JSON path (and Go's computeOverlapPrefix) it strips-// position tags before measuring the overlap cut, so the prefix is cut on the-// tag-free visible text. The 29-line parity payload carries no tags, so the-// strip is a no-op there; tag-bearing inputs are covered by the Python test-// rag/flow/tests/test_token_chunker_tag_overlap.py and the unit-level oracle in-// token_merge_units_test.go.-func pythonMergeWithOverlap(paragraphs []string, chunkTokenSize int, overlappedPercent float64) []string {- threshold := float64(chunkTokenSize) * (100.0 - overlappedPercent) / 100.0- type ck struct {- text string- tk int- }- merged := []ck{}- prev := -1+// goMergeWithOverlapOracle mirrors the text path of the hard-cap merge with+// overlap: it builds text-path units ("\n" + paragraph) and runs them through+// the mergeUnits core (scaled overlap threshold + unconditional overlap prefix+// trimmed to fit the hard cap). It is the Go-side oracle for the overlap+// parity tests below.+func goMergeWithOverlapOracle(paragraphs []string, chunkTokenSize int, overlappedPercent float64) []string {+ units := make([]schema.ChunkDoc, 0, len(paragraphs))for _, p := range paragraphs {- // Mirror Go's text-path unitization: each paragraph is built as- // "\n" + p (rag/nlp naive_merge:1405); the leading newline is part of- // the token count so the running sum matches the Go component.- unit := "\n" + p- pt := tokenizeStr(unit)- cur := unit- curTk := pt- // #17799: an over-budget unit stands alone.- startNew := prev < 0 || (prev >= 0 && float64(merged[prev].tk) > threshold)- if pt > chunkTokenSize {- startNew = true- }- if startNew {- if prev >= 0 && overlappedPercent > 0 && merged[prev].text != "" {- vis := []rune(removeTag(merged[prev].text))- cut := int(float64(len(vis)) * (100.0 - overlappedPercent) / 100.0)- if cut < 0 {- cut = 0- }- if cut < len(vis) {- cur = string(vis[cut:]) + cur- curTk = tokenizeStr(cur)- }- }- merged = append(merged, ck{cur, curTk})- prev = len(merged) - 1+ if strings.TrimSpace(p) == "" {continue}- merged[prev].text += cur- merged[prev].tk += curTk+ u := "\n" + p+ units = append(units, schema.ChunkDoc{Text: u, TKNums: intPtr(tokenizeStr(u)), CKType: "text"})}- out := make([]string, len(merged))- for i := range merged {- // Mirror production token.go:465 (removeTag first, then TrimSpace) so+ merged := mergeUnits(units, chunkTokenSize, overlappedPercent, "")+ out := make([]string, 0, len(merged))+ for _, ck := range merged {+ // Mirror production token.go: removeTag first, then TrimSpace, so// whitespace between visible text and a trailing position tag survives// exactly as the Go TokenChunker emits it.- out[i] = removeTag(strings.TrimSpace(merged[i].text))+ text := removeTag(strings.TrimSpace(ck.Text))+ if text == "" {+ continue+ }+ out = append(out, text)}return out}@@ -360,11 +258,11 @@ func pythonMergeWithOverlap(paragraphs []string, chunkTokenSize int, overlappedP// prefix) against Go/Python drift at overlap>0, which #17948 (overlap=0) does// not exercise.// TestTokenChunkerOverlapStripsTagsTrimOrder guards the final-strip ORDER for-// tag-bearing text: production (token.go:465) does removeTag(TrimSpace(text)),-// so whitespace that sits BETWEEN visible text and a trailing position tag is-// PRESERVED (TrimSpace cannot see past the tag). The oracle-// pythonMergeWithOverlap must use the SAME order, otherwise the parity test-// would assert a text that the production code never emits.+// tag-bearing text: production (token.go) does removeTag(TrimSpace(text)), so+// whitespace that sits BETWEEN visible text and a trailing position tag is+// PRESERVED (TrimSpace cannot see past the tag). The oracle goMergeWithOverlap+// must use the SAME order, otherwise the parity test would assert a text that+// the production code never emits.//// Input: every paragraph ends with "<space><space>@@page...##", so each merged// chunk's tail has two spaces before its closing tag. With the production@@ -378,10 +276,10 @@ func TestTokenChunkerOverlapStripsTagsTrimOrder(t *testing.T) {text := strings.Join(tagged, "\n")const overlap = 20.0for _, cap := range []int{8, 32} {- want := pythonMergeWithOverlap(tagged, cap, overlap)+ want := goMergeWithOverlapOracle(tagged, cap, overlap)got := invokeTokenChunker(t, text, cap, overlap)if len(got) != len(want) {- t.Fatalf("chunk_token_size=%d overlap=%v: chunk count go=%d, python=%d", cap, overlap, len(got), len(want))+ t.Fatalf("chunk_token_size=%d overlap=%v: chunk count go=%d, oracle=%d", cap, overlap, len(got), len(want))}for i := range got {if got[i] != want[i] {@@ -391,14 +289,14 @@ func TestTokenChunkerOverlapStripsTagsTrimOrder(t *testing.T) {}}-func TestTokenChunkerOverlapMatchesPython(t *testing.T) {+func TestTokenChunkerOverlapMatchesGoOracle(t *testing.T) {text := strings.Join(mergeSourceLines, "\n")const overlap = 20.0for _, cap := range []int{32, 64, 128} {- want := pythonMergeWithOverlap(mergeSourceLines, cap, overlap)+ want := goMergeWithOverlapOracle(mergeSourceLines, cap, overlap)got := invokeTokenChunker(t, text, cap, overlap)if len(got) != len(want) {- t.Fatalf("chunk_token_size=%d overlap=%v: chunk count go=%d, python=%d", cap, overlap, len(got), len(want))+ t.Fatalf("chunk_token_size=%d overlap=%v: chunk count go=%d, oracle=%d", cap, overlap, len(got), len(want))}for i := range got {if got[i] != want[i] {diff --git a/internal/ingestion/component/chunker/token_merge_units_test.go b/internal/ingestion/component/chunker/token_merge_units_test.goindex f0e1877377..632956deda 100644--- a/internal/ingestion/component/chunker/token_merge_units_test.go+++ b/internal/ingestion/component/chunker/token_merge_units_test.go@@ -28,28 +28,33 @@ type mergeUnitsOracleRow struct {tk int}-// pythonMergeUnitsOracle is the Go port of the UNIFIED merge contract shared by-// the Go TokenChunker, Python naive_merge, and Python token_chunker's JSON-// path. It is the oracle that mergeUnits (the single, unified merge core) must-// match chunk-for-chunk (text AND count).+// hardCapMergeUnitsOracle is the Go reference for the hard-cap merge contract+// (方案 B) that mergeUnits (the single, unified merge core) must match+// chunk-for-chunk (text AND count). It mirrors the same algorithm as the Go+// TokenChunker, so the comparison pins the merge behavior, not Python's+// OVER_CAP contract (which this code no longer follows).//-// Contract mirrored here (the unified "hybrid" algorithm):+// Contract mirrored here:// - token counts are the RUNNING SUM of per-unit counts (never re-tokenizing-// the joined text);+// the joined text), but the merge ALSO re-checks the actual joined text+// against the target — joinSep (e.g. "\n" on the JSON path) can add tokens+// beyond the running sum, so a candidate whose joined text exceeds target+// starts a fresh chunk (hard-cap safety net, matching mergeUnits);// - overlap>0 uses a SCALED threshold target*(100-overlap)/100 and prepends-// the previous chunk's tail UNCONDITIONALLY (no fit-check);+// the previous chunk's tail, trimmed to fit the target (computeOverlapPrefix+// - overlapFitPrefix, matching mergeUnits);// - a non-text unit passes through and resets the merge run;-// - an over-budget unit (tk > target) STANDS ALONE — never merged into the-// previous chunk (retains the #17799 contract; diverges from Python JSON's-// native merge-into-prev on purpose, so the product-wide behaviour change-// is avoided);-// - MergeUnderCap additionally forbids a projected overflow.+// - the merge is UNDER_CAP: a projected join that would push the running sum+// over target starts a fresh chunk (never merge-then-close);+// - oversized units are expanded by mergeUnits BEFORE merging (sentence split+// - hard token-split fallback); this oracle takes pre-expanded units, so+// every tkNums it sees is already <= target.//// Token counts: the initial chunk and merged running sum use the EXPLICIT// tkNums (so merge decisions are deterministic and offline-independent), but// an overlap-prefixed chunk recomputes its count via tokenizeStr — exactly as// mergeUnits does — so the two stay in lock-step.-func pythonMergeUnitsOracle(texts []string, tkNums []int, kinds []string, target int, overlap float64, joinSep string, strat schema.MergeStrategy) []mergeUnitsOracleRow {+func hardCapMergeUnitsOracle(texts []string, tkNums []int, kinds []string, target int, overlap float64, joinSep string) []mergeUnitsOracleRow {threshold := float64(target) * (100.0 - overlap) / 100.0merged := []mergeUnitsOracleRow{}prev := -1@@ -65,44 +70,29 @@ func pythonMergeUnitsOracle(texts []string, tkNums []int, kinds []string, targetprev = len(merged) - 1continue}- // #17799: an over-budget unit stands alone — never merged into prev.- if tk > target {- text := texts[i]- cpTk := tk- if overlap > 0 && merged[prev].text != "" {- vis := []rune(removeTag(merged[prev].text))- cut := int(float64(len(vis)) * (100.0 - overlap) / 100.0)- if cut < 0 {- cut = 0- }- if cut < len(vis) {- text = string(vis[cut:]) + texts[i]- }- cpTk = tokenizeStr(text)+ // UNDER_CAP: a projected join that would exceed target starts a fresh+ // chunk. The scaled threshold reserves room for the overlap prefix.+ startNew := float64(merged[prev].tk) > threshold || merged[prev].tk+tk > target+ if !startNew {+ // Hard-cap safety net: re-check the actual joined text (joinSep can+ // add tokens beyond the running sum), mirroring mergeUnits.+ cand := merged[prev].text + texts[i]+ if merged[prev].text != "" && texts[i] != "" {+ cand = merged[prev].text + joinSep + texts[i]+ }+ if tokenizeStr(cand) > target {+ startNew = true}- merged = append(merged, mergeUnitsOracleRow{text, cpTk})- prev = len(merged) - 1- continue- }- startNew := float64(merged[prev].tk) > threshold- if !startNew && strat == schema.MergeUnderCap && merged[prev].tk+tk > target {- startNew = true}if startNew {text := texts[i]cpTk := tkif overlap > 0 && merged[prev].text != "" {- vis := []rune(removeTag(merged[prev].text))- cut := int(float64(len(vis)) * (100.0 - overlap) / 100.0)- if cut < 0 {- cut = 0- }- if cut < len(vis) {- text = string(vis[cut:]) + texts[i]- }- // Mirror mergeUnits: recompute the overlap chunk's token- // count from its (prefix+cur) text. Only when a prefix was- // actually prepended; otherwise keep the explicit count.+ // Mirror mergeUnits: carve the overlap prefix from the previous+ // chunk's tag-free tail, then trim it to fit the target.+ prefix, _ := computeOverlapPrefix(merged[prev].text, overlap)+ prefix, _ = overlapFitPrefix(prefix, texts[i], target)+ text = prefix + texts[i]cpTk = tokenizeStr(text)}merged = append(merged, mergeUnitsOracleRow{text, cpTk})@@ -119,7 +109,7 @@ func pythonMergeUnitsOracle(texts []string, tkNums []int, kinds []string, targetreturn merged}-func TestMergeUnitsMatchesPythonOracle(t *testing.T) {+func TestMergeUnitsMatchesHardCapOracle(t *testing.T) {cases := []struct {name stringtexts []string@@ -128,63 +118,55 @@ func TestMergeUnitsMatchesPythonOracle(t *testing.T) {target intoverlap float64joinSep string- strat schema.MergeStrategy}{{name: "json overlap0",texts: []string{"a", "b", "c", "d"},tkNums: []int{5, 5, 5, 5},kinds: []string{"text", "text", "text", "text"},- target: 8, overlap: 0, joinSep: "\n", strat: schema.MergeOverCap,+ target: 8, overlap: 0, joinSep: "\n",},{name: "json overlap20 unconditional prefix",texts: []string{"a", "b", "c", "d"},tkNums: []int{5, 5, 5, 5},kinds: []string{"text", "text", "text", "text"},- target: 8, overlap: 20, joinSep: "\n", strat: schema.MergeOverCap,+ target: 8, overlap: 20, joinSep: "\n",},{name: "text overlap0 joinsep empty",texts: []string{"a", "b", "c"},tkNums: []int{5, 5, 5},kinds: []string{"text", "text", "text"},- target: 8, overlap: 0, joinSep: "", strat: schema.MergeOverCap,+ target: 8, overlap: 0, joinSep: "",},{name: "text overlap20 unconditional prefix",texts: []string{"a", "b", "c"},tkNums: []int{5, 5, 5},kinds: []string{"text", "text", "text"},- target: 8, overlap: 20, joinSep: "", strat: schema.MergeOverCap,+ target: 8, overlap: 20, joinSep: "",},{name: "nontext breaks merge run",texts: []string{"a", "IMG", "b"},tkNums: []int{5, 0, 5},kinds: []string{"text", "image", "text"},- target: 8, overlap: 20, joinSep: "\n", strat: schema.MergeOverCap,- },- {- name: "oversized stands alone (#17799, not merged into prev)",- texts: []string{"small", "verylongunitthat exceedsbudgetbyalot", "tiny"},- tkNums: []int{3, 50, 3},- kinds: []string{"text", "text", "text"},- target: 8, overlap: 0, joinSep: "\n", strat: schema.MergeOverCap,+ target: 8, overlap: 20, joinSep: "\n",},{- name: "under_cap no overflow",+ name: "three units under budget no overflow",texts: []string{"a", "b", "c"},tkNums: []int{5, 5, 5},kinds: []string{"text", "text", "text"},- target: 8, overlap: 0, joinSep: "\n", strat: schema.MergeUnderCap,+ target: 8, overlap: 0, joinSep: "\n",},{- name: "under_cap overlap20 still no overflow but carries overlap",+ name: "overlap20 still no overflow but carries overlap",texts: []string{"a", "b", "c", "d"},tkNums: []int{5, 5, 5, 5},kinds: []string{"text", "text", "text", "text"},- target: 8, overlap: 20, joinSep: "\n", strat: schema.MergeUnderCap,+ target: 8, overlap: 20, joinSep: "\n",},{// Tag-bearing overlap source: the previous chunk's text carries a@@ -195,7 +177,7 @@ func TestMergeUnitsMatchesPythonOracle(t *testing.T) {texts: []string{"aa@@1\t2\t3\t4##bb", "cc"},tkNums: []int{5, 5},kinds: []string{"text", "text"},- target: 4, overlap: 20, joinSep: "\n", strat: schema.MergeOverCap,+ target: 6, overlap: 20, joinSep: "\n",},{// Longer tag so the RAW-text cut would land INSIDE the tag. The@@ -206,7 +188,28 @@ func TestMergeUnitsMatchesPythonOracle(t *testing.T) {texts: []string{"abcd@@100\t200\t300\t400##", "wxyz"},tkNums: []int{10, 4},kinds: []string{"text", "text"},- target: 5, overlap: 20, joinSep: "\n", strat: schema.MergeOverCap,+ target: 12, overlap: 20, joinSep: "\n",+ },+ {+ // Current unit near the cap: the overlap prefix carved from the+ // previous chunk would push the fresh chunk over the target, so+ // overlapFitPrefix trims it. The oracle must mirror the trim.+ name: "overlap prefix trimmed to fit hard cap",+ texts: []string{strings.Repeat("word ", 10), strings.Repeat("pad ", 10)},+ tkNums: []int{tokenizeStr(strings.Repeat("word ", 10)), tokenizeStr(strings.Repeat("pad ", 10))},+ kinds: []string{"text", "text"},+ target: 22, overlap: 20, joinSep: "\n",+ },+ {+ // The re-tokenize guard (hard-cap safety net): the joined text+ // (with joinSep "\n") exceeds target even though the running sum+ // fits, so the merge must start a fresh chunk. Both mergeUnits and+ // the oracle must fire the guard identically.+ name: "joinsep guard fires on actual joined text",+ texts: []string{"word", "pad"},+ tkNums: []int{1, 1},+ kinds: []string{"text", "text"},+ target: 2, overlap: 0, joinSep: "\n",},}for _, c := range cases {@@ -215,8 +218,8 @@ func TestMergeUnitsMatchesPythonOracle(t *testing.T) {for i, tx := range c.texts {units[i] = schema.ChunkDoc{Text: tx, TKNums: intPtr(c.tkNums[i]), CKType: c.kinds[i]}}- want := pythonMergeUnitsOracle(c.texts, c.tkNums, c.kinds, c.target, c.overlap, c.joinSep, c.strat)- got := mergeUnits(units, c.target, c.overlap, c.strat, c.joinSep)+ want := hardCapMergeUnitsOracle(c.texts, c.tkNums, c.kinds, c.target, c.overlap, c.joinSep)+ got := mergeUnits(units, c.target, c.overlap, c.joinSep)if len(got) != len(want) {t.Fatalf("chunk count go=%d want=%d\n got=%v\nwant=%v", len(got), len(want), got, want)}@@ -245,9 +248,9 @@ func TestMergeUnitsOverlapPrefixIsTagFree(t *testing.T) {target intoverlap float64}{- {"cut inside tag", "abcd@@100\t200\t300\t400##", "wxyz", 5, 20},- {"tag at end", "hello world@@1\t2\t3\t4##", "next", 5, 20},- {"tag at start", "@@1\t2\t3\t4##leading text", "next", 5, 20},+ {"cut inside tag", "abcd@@100\t200\t300\t400##", "wxyz", 10, 20},+ {"tag at end", "hello world@@1\t2\t3\t4##", "next", 10, 20},+ {"tag at start", "@@1\t2\t3\t4##leading text", "next", 10, 20},}for _, c := range cases {t.Run(c.name, func(t *testing.T) {@@ -255,7 +258,7 @@ func TestMergeUnitsOverlapPrefixIsTagFree(t *testing.T) {{Text: c.prev, TKNums: intPtr(10), CKType: "text"},{Text: c.cur, TKNums: intPtr(4), CKType: "text"},}- got := mergeUnits(units, c.target, c.overlap, schema.MergeOverCap, "\n")+ got := mergeUnits(units, c.target, c.overlap, "\n")if len(got) < 2 {t.Fatalf("expected >=2 chunks, got %d: %v", len(got), got)}diff --git a/internal/ingestion/component/chunker/token_overlap_test.go b/internal/ingestion/component/chunker/token_overlap_test.goindex aa14254269..c050eab68d 100644--- a/internal/ingestion/component/chunker/token_overlap_test.go+++ b/internal/ingestion/component/chunker/token_overlap_test.go@@ -2,6 +2,7 @@ package chunkerimport ("context"+ "encoding/json""strings""testing")@@ -57,33 +58,61 @@ func TestTokenChunker_TextOverlapPreservesInterLineSpace(t *testing.T) {t.Fatalf("Invoke: %v", err)}chunks, _ := out["chunks"].([]map[string]any)- // OVER_CAP (Python's canonical default) overflow-merges the first two- // groups (alpha+beta) into chunk0 and then starts a fresh, overlap-prefixed- // chunk1 for gamma.- if len(chunks) != 2 {- t.Fatalf("expected 2 chunks, got %d", len(chunks))+ if len(chunks) == 0 {+ t.Fatalf("expected chunks, got none")}got := make([]string, len(chunks))for i, ck := range chunks {s, _ := ck["text"].(string)got[i] = s+ // Hard cap: no chunk may exceed the token budget.+ if n := tokenizeStr(s); n > 64 {+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=64)", i, n)+ }}// The regression this test pins: the inter-line trailing space of every// line must be preserved across a merge/overlap boundary (Python emits- // "alpha \nbeta", not "alpha\nbeta"). Under OVER_CAP the alpha→beta- // boundary is inside chunk0 and the beta→gamma boundary is inside the- // overlap-prefixed chunk1.- if !strings.Contains(got[0], "alpha \nbeta") {- t.Errorf("chunk[0] lost inter-line space before newline: %q", got[0])+ // "alpha \nbeta", not "alpha\nbeta"). No emitted chunk may contain a+ // space-less "\n" boundary, and at least one space-preserving boundary must+ // actually appear (otherwise the fixture no longer exercises the path).+ spacePreserved := 0+ for i, s := range got {+ if strings.Contains(s, "alpha\nbeta") {+ t.Errorf("chunk[%d] has space-less alpha/beta boundary (bug present): %q", i, s)+ }+ if strings.Contains(s, "beta\ngamma") {+ t.Errorf("chunk[%d] has space-less beta/gamma boundary (bug present): %q", i, s)+ }+ if strings.Contains(s, "alpha \nbeta") || strings.Contains(s, "beta \ngamma") {+ spacePreserved+++ }}- if strings.Contains(got[0], "alpha\nbeta") {- t.Errorf("chunk[0] has space-less boundary (bug present): %q", got[0])+ if spacePreserved == 0 {+ t.Errorf("no chunk preserved an inter-line space across a boundary; the regression fixture is not exercised")}- if !strings.Contains(got[1], "beta \ngamma") {- t.Errorf("chunk[1] lost inter-line space before newline: %q", got[1])+}++// TestOverlapTailPositions_VisibleTextOffsets pins the visible-text offset+// contract: overlapCut/overlapFitPrefix measure the cut on the tag-free visible+// text, so overlapTailPositions must map item spans using the same visible+// lengths. A coordinate tag in an earlier item must not shift the boundaries of+// later items (which would pull head coordinates into the overlap tail).+func TestOverlapTailPositions_VisibleTextOffsets(t *testing.T) {+ items := []mergeItem{+ // visible text "hello world" (11 runes); the tag inflates the RAW length+ // to 23 runes.+ {Text: "hello@@1\t2\t3\t4## world", PDFPositions: json.RawMessage(`[["p0"]]`), Positions: json.RawMessage(`[["q0"]]`)},+ {Text: "tail", PDFPositions: json.RawMessage(`[["p1"]]`), Positions: json.RawMessage(`[["q1"]]`)},+ }+ // Visible layout: item0 [0,11), joinSep "\n" at 11, item1 [12,16). An+ // overlapStart of 12 lands in item1 only, so item0's head coordinates must+ // NOT be included.+ pdf, pos := overlapTailPositions(items, 12, "\n")+ if string(pdf) != `[["p1"]]` {+ t.Errorf("pdf positions = %s, want only item1 [\"p1\"] (item0 head must be excluded)", pdf)}- if strings.Contains(got[1], "beta\ngamma") {- t.Errorf("chunk[1] has space-less boundary (bug present): %q", got[1])+ if string(pos) != `[["q1"]]` {+ t.Errorf("positions = %s, want only item1 [\"q1\"] (item0 head must be excluded)", pos)}}diff --git a/internal/ingestion/component/chunker/token_oversize_whole_test.go b/internal/ingestion/component/chunker/token_oversize_split_test.gosimilarity index 62%rename from internal/ingestion/component/chunker/token_oversize_whole_test.gorename to internal/ingestion/component/chunker/token_oversize_split_test.goindex c286931cb1..1d691b5620 100644--- a/internal/ingestion/component/chunker/token_oversize_whole_test.go+++ b/internal/ingestion/component/chunker/token_oversize_split_test.go@@ -20,13 +20,12 @@ import ("testing")-// TestTokenChunker_OversizeUnitKeptWhole pins the Python OVER_CAP contract for+// TestTokenChunker_OversizeUnitSplit pins the hard-cap contract (方案 B) for// the text/markdown path: a single paragraph that exceeds chunk_token_size is-// kept as one standalone chunk. Python's naive_merge (_merge_paragraph_groups,-// rag/nlp/__init__.py) never atom-splits an oversize unit; it is kept whole and-// the model layer truncates it later. An unbroken input line with no delimiter-// forces the oversize path while isolating it from the delimiter-splitting logic.-func TestTokenChunker_OversizeUnitKeptWhole(t *testing.T) {+// re-split into <= budget pieces (sentence boundaries first, hard token-split+// fallback), never kept whole. An unbroken input line with no delimiter forces+// the hard-split path while isolating it from the delimiter-splitting logic.+func TestTokenChunker_OversizeUnitSplit(t *testing.T) {var longLine = strings.Repeat("word ", 400) // ~400 tokens, far above the 32 budgetcases := []struct {@@ -64,25 +63,35 @@ func TestTokenChunker_OversizeUnitKeptWhole(t *testing.T) {if !ok {t.Fatalf("chunks missing or wrong type: %T", out["chunks"])}- if len(chunks) != 1 {- t.Fatalf("oversize unit: want 1 standalone chunk, got %d", len(chunks))+ if len(chunks) < 2 {+ t.Fatalf("oversize unit must be split, got %d chunk(s)", len(chunks))}- if got, _ := chunks[0]["text"].(string); strings.TrimSpace(got) != strings.TrimSpace(longLine) {- t.Fatalf("oversize unit content not preserved: got %q", got)+ for i, ck := range chunks {+ text, _ := ck["text"].(string)+ if n := tokenizeStr(text); n > 32 {+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=32)", i, n)+ }+ }+ // Lossless modulo whitespace normalization: each emitted chunk is+ // trimmed, and hard-split pieces cut at token/word boundaries, so+ // the concatenated words reproduce the original word sequence.+ var joined string+ for _, ck := range chunks {+ joined += strings.TrimSpace(ck["text"].(string))+ }+ if strings.ReplaceAll(joined, " ", "") != strings.ReplaceAll(longLine, " ", "") {+ t.Errorf("split content lost words:\n got %q\nwant %q", joined, longLine)}})}}-// TestTokenChunker_OversizeUnitStandsAloneAfterInBudgetUnit pins the #17799-// contract under the unified algorithm: an oversize unit (single paragraph >-// chunk_token_size) STANDS ALONE as its own chunk, even when an in-budget-// sentence precedes it. The unified merge decision depends only on-// prev.tk_nums > threshold for merging, but an over-budget unit never merges-// into the previous chunk (#17799 retained on purpose to avoid a product-wide-// behaviour change). The lone oversize case is pinned by-// TestTokenChunker_OversizeUnitKeptWhole.-func TestTokenChunker_OversizeUnitStandsAloneAfterInBudgetUnit(t *testing.T) {+// TestTokenChunker_OversizeUnitSplitAfterInBudgetUnit pins the hard-cap+// contract under the merge: an in-budget unit is followed by an oversized+// unit, which is re-split into <= budget pieces. The in-budget unit stays a+// chunk of its own; the oversized unit becomes multiple chunks that together+// reproduce its text.+func TestTokenChunker_OversizeUnitSplitAfterInBudgetUnit(t *testing.T) {var longLine = strings.Repeat("word ", 400) // ~400 tokens, far above the 32 budgetinBudget := "Hello world." // ASCII period is not a sentence delimiter; fits 32@@ -123,16 +132,18 @@ func TestTokenChunker_OversizeUnitStandsAloneAfterInBudgetUnit(t *testing.T) {if !ok {t.Fatalf("chunks missing or wrong type: %T", out["chunks"])}- if len(chunks) != 2 {- t.Fatalf("oversize after in-budget: want 2 chunks (in-budget + standalone oversize), got %d", len(chunks))+ if len(chunks) < 3 {+ t.Fatalf("want in-budget chunk + split pieces, got %d chunk(s)", len(chunks))}first, _ := chunks[0]["text"].(string)if first != inBudget {t.Fatalf("first chunk should be only the in-budget sentence %q, got %q", inBudget, first)}- second, _ := chunks[1]["text"].(string)- if second != strings.TrimSpace(longLine) {- t.Fatalf("oversize chunk should be the whole long line kept whole, got %q", second)+ for i := 1; i < len(chunks); i++ {+ text, _ := chunks[i]["text"].(string)+ if n := tokenizeStr(text); n > 32 {+ t.Errorf("split chunk %d exceeds budget: tokens=%d (cap=32)", i, n)+ }}})}diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.goindex 56274d05a8..2d6dc1e5e9 100644--- a/internal/ingestion/component/chunker/token_pdfpos_test.go+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go@@ -38,7 +38,7 @@ func TestMergeByTokenSizeFromJSON_ExtendsPDFPositions(t *testing.T) {{Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posB},},}- got := mergeByTokenSizeFromJSON(items, 128, 0, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, 128, 0)merged := got[0]if len(merged) != 1 {t.Fatalf("want 1 merged chunk, got %d", len(merged))@@ -63,7 +63,7 @@ func TestMergeByTokenSizeFromJSON_ExtendsPositions(t *testing.T) {{Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},},}- got := mergeByTokenSizeFromJSON(items, 128, 0, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, 128, 0)combined := string(got[0][0].Positions)if !strings.Contains(combined, "1,2,3") || !strings.Contains(combined, "4,5,6") {t.Errorf("merged chunk dropped/omitted `positions`: %s", combined)@@ -103,7 +103,7 @@ func TestMergeByTokenSizeFromJSON_PositionsDecodeToMatrix(t *testing.T) {{Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},},}- got := mergeByTokenSizeFromJSON(items, 128, 0, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, 128, 0)m := got[0][0].ToMap()raw, ok := m["positions"]if !ok {@@ -146,7 +146,7 @@ func TestMergeByTokenSizeFromJSON_OverlapPrefixCarriesPrevPositions(t *testing.T{Text: "gamma", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posC},},}- got := mergeByTokenSizeFromJSON(items, 20, 100, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, 20, 100)merged := got[0]if len(merged) != 3 {t.Fatalf("want 3 chunks (each unit starts fresh at overlapPct=100), got %d", len(merged))@@ -195,30 +195,32 @@ func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(tposD := json.RawMessage(`[[4,0,40,0,16]]`)posE := json.RawMessage(`[[5,0,50,0,20]]`)posF := json.RawMessage(`[[6,0,60,0,24]]`)- // 6 equal-length items (5 runes each). With chunkTokens=5 and each item- // TKNums=1, items 0..4 merge into one chunk (tk reaches 5); item5 starts a- // fresh chunk. Its overlap prefix (overlappedPct=20) is the last ~20% of- // the 5-item previous chunk's text => only the last item ("eeeee", posE)- // intersects the tail. So chunk[1] must carry posE (tail) + posF (own), but- // NOT posA/posB/posC/posD.+ // 6 single-token items. With chunkTokens=9 (5 item tokens + 4 joinSep "\n"+ // tokens), items 0..4 merge into one chunk — the re-tokenize guard lets the+ // joined text fill the cap exactly (9 tokens), and item5 starts a fresh+ // chunk. Its overlap prefix (overlappedPct=20) is the last ~20% of the+ // 5-item previous chunk's text => only the last item ("e", posE) intersects+ // the tail. So chunk[1] must carry posE (tail) + posF (own), but NOT+ // posA/posB/posC/posD. (Texts are single-token so the re-tokenize guard's+ // actual-count check agrees with the declared TKNums.)items := [][]schema.ChunkDoc{{- {Text: "aaaaa", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posA},- {Text: "bbbbb", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posB},- {Text: "ccccc", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posC},- {Text: "ddddd", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posD},- {Text: "eeeee", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posE},- {Text: "fffff", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posF},+ {Text: "a", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posA},+ {Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posB},+ {Text: "c", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posC},+ {Text: "d", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posD},+ {Text: "e", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posE},+ {Text: "f", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posF},},}- got := mergeByTokenSizeFromJSON(items, 5, 20, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON(items, 9, 20)merged := got[0]if len(merged) != 2 {t.Fatalf("want 2 chunks (5 items merge, 6th starts fresh with partial overlap), got %d", len(merged))}// The new chunk's overlap text is the tail of the previous chunk.- if !strings.Contains(merged[1].Text, "eeeee") {+ if !strings.Contains(merged[1].Text, "e") {t.Errorf("chunk[1] missing overlap tail text from prev chunk: text=%q", merged[1].Text)}// The tail item's coordinates MUST be carried.diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.goindex 320a3c9f53..106b458155 100644--- a/internal/ingestion/component/chunker/token_strict_cap_test.go+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go@@ -24,23 +24,14 @@ import ("ragflow/internal/ingestion/component/schema")-func TestComputeOverlapPrefix_StripsTagsAndCounts(t *testing.T) {- prev := strings.Repeat("word ", 20) + "@@1\t2.3## tail"- overlap, n := computeOverlapPrefix(prev, 30)- if strings.Contains(overlap, "@@") || strings.Contains(overlap, "##") {- t.Errorf("overlap must strip parser tags, got %q", overlap)- }- if n <= 0 {- t.Errorf("overlap token count must be >0, got %d", n)- }- if tokenizeStr(overlap) != n {- t.Errorf("reported tokens %d != tokenizeStr(overlap) %d", n, tokenizeStr(overlap))- }-}+// Hard-cap contract tests: no text chunk may exceed the token target. The+// merge decision uses the RUNNING SUM of per-unit counts (#17948), so the+// assertions below pin that sum; the end-to-end tests additionally assert the+// re-tokenized emitted text stays within the budget for their fixtures.-func TestMergeByTokenSizeFromJSON_StrictCapNoOvershoot(t *testing.T) {+func TestMergeByTokenSizeFromJSON_HardCapNoOvershoot(t *testing.T) {// Eight 25-token-ish sections under a 50-token budget must pack without- // any chunk exceeding the budget (Python test_strict_cap_no_overlap).+ // any chunk's running sum exceeding the budget (no one-unit overflow).const budget = 50sections := make([]schema.ChunkDoc, 0, 8)for i := 0; i < 8; i++ {@@ -49,32 +40,21 @@ func TestMergeByTokenSizeFromJSON_StrictCapNoOvershoot(t *testing.T) {Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),})}- got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 0, schema.MergeOverCap)+ got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 0)merged := got[0]if len(merged) < 3 {t.Fatalf("want >=3 chunks, got %d", len(merged))}- // OVER_CAP (Python's canonical default) permits a chunk to exceed budget- // by at most one incoming unit: a chunk is closed right after the- // overflowing merge, so its running-sum token count is <= budget+unit.- // The reconstructed chunk text carries the "\n" joins between units, and- // cl100k is non-additive across joins, so the actual token count can land- // a little above budget+unit; allow one extra unit of slack (matching the- // sibling TestMergeByTokenSizeFromJSON_OverlapDroppedAtOverflow tolerance- // for the same cl100k delta). Python's naive_merge produces the same 3- // chunks here, so this pins the faithful boundary, not the old re-tokenized- // (under-packed) one.- unit := tokenizeStr(sections[0].Text)for i, ck := range merged {- n := tokenizeStr(ck.Text)- if n > budget+2*unit {- t.Errorf("chunk %d exceeds budget by more than one unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)+ if n := intValue(ck.TKNums); n > budget {+ t.Errorf("chunk %d running sum %d exceeds budget %d", i, n, budget)}}}-func TestMergeByTokenSizeFromJSON_OverlapDroppedAtOverflow(t *testing.T) {- // With a tight budget, overlap must never push a chunk over the cap.+func TestMergeByTokenSizeFromJSON_OverlapStrictCap(t *testing.T) {+ // With overlap the overlap prefix is trimmed so it never pushes a chunk+ // over the budget.const budget = 25sections := make([]schema.ChunkDoc, 0, 20)for i := 0; i < 20; i++ {@@ -83,42 +63,43 @@ func TestMergeByTokenSizeFromJSON_OverlapDroppedAtOverflow(t *testing.T) {Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),})}- got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 20, schema.MergeOverCap)- unit := tokenizeStr(sections[0].Text)+ got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 20)for i, ck := range got[0] {- // OVER_CAP allows one boundary overflow (prev + one unit). The JSON- // path joins units with "\n" and cl100k is not additive across joins,- // so an overflow-closed chunk can land a little above budget+unit;- // allow one extra unit of slack for the separators + cl100k delta.- // Overlap is only ever prepended when it still fits budget, so the- // only chunks that exceed budget are the overflow-closed ones.- if n := tokenizeStr(ck.Text); n > budget+2*unit {- t.Errorf("chunk %d exceeds budget+two-units with overlap: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)+ if n := intValue(ck.TKNums); n > budget {+ t.Errorf("chunk %d exceeds budget with overlap: tokens=%d (cap=%d)", i, n, budget)}}}-func TestMergeByTokenSizeFromJSON_OversizedUnitStaysWhole(t *testing.T) {- // Per the #17808 contract an over-budget unit is never atom-split: it- // stands alone as one chunk and the model layer truncates it later.+func TestMergeByTokenSizeFromJSON_OversizedUnitSplit(t *testing.T) {+ // A single over-budget unit is no longer kept whole (#17799 replaced by the+ // hard cap): it is re-split into <= budget pieces and the concatenated+ // pieces reproduce the original text exactly (lossless).const budget = 30long := strings.TrimSpace(strings.Repeat("word ", 100))items := [][]schema.ChunkDoc{{{Text: long, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(long))},}}- got := mergeByTokenSizeFromJSON(items, budget, 0, schema.MergeOverCap)- if len(got) != 1 || len(got[0]) != 1 {- t.Fatalf("over-budget unit must stay whole, got %d chunk(s)", len(got[0]))+ got := mergeByTokenSizeFromJSON(items, budget, 0)+ merged := got[0]+ if len(merged) < 2 {+ t.Fatalf("oversized unit must be split, got %d chunk(s)", len(merged))+ }+ var joined string+ for i, ck := range merged {+ if n := tokenizeStr(ck.Text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=%d)", i, n, budget)+ }+ joined += ck.Text}- if got[0][0].Text != long {- t.Errorf("over-budget chunk text changed: got %q", got[0][0].Text)+ if joined != long {+ t.Errorf("split not lossless:\n got %q\nwant %q", joined, long)}}func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {// End-to-end text path: long multi-paragraph input under a tight budget.const budget = 40- unit := tokenizeStr(strings.TrimSpace(strings.Repeat("word ", 15)))var b strings.Builderfor i := 0; i < 30; i++ {b.WriteString(strings.TrimSpace(strings.Repeat("word ", 15)))@@ -139,24 +120,16 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {}for i, ck := range chunks {text, _ := ck["text"].(string)- // OVER_CAP allows one boundary overflow (prev + one unit).- if n := tokenizeStr(text); n > budget+unit {- t.Errorf("chunk %d exceeds budget+one-unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)+ if n := tokenizeStr(text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=%d)", i, n, budget)}}}-// TestMergeByTokenSize_OversizedUnitStaysWhole mirrors-// TestMergeByTokenSizeFromJSON_OversizedUnitStaysWhole on the text path: a-// single block of text that exceeds chunk_token_size and contains no-// sentence delimiter must be emitted as ONE whole chunk — never atom-split.-// This pins the #17799 contract invariant on the text path (the JSON path-// is already covered by TestMergeByTokenSizeFromJSON_OversizedUnitStaysWhole).-func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) {+func TestMergeByTokenSize_OversizedBoundarylessRunSplit(t *testing.T) {+ // A single boundary-less run longer than chunk_token_size must be hard+ // token-split into <= budget pieces (never kept whole).const budget = 30- // One long run with no '\n' / '!?' / '。;!?' delimiter: the text path- // splits oversized sections only on sentenceDelimiter, so this whole- // run is one unit that still exceeds the budget and must stay whole.long := strings.TrimSpace(strings.Repeat("word ", 100))comp, err := NewTokenChunker(map[string]any{"delimiter_mode": "delimiter",@@ -167,83 +140,19 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) {}out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil, nil)chunks, _ := out["chunks"].([]map[string]any)- if len(chunks) != 1 {- t.Fatalf("over-budget unit must stay whole, got %d chunk(s)", len(chunks))- }- text, _ := chunks[0]["text"].(string)- if text != long {- t.Errorf("over-budget chunk text changed: got %q", text)- }-}--func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {- // UNDER_CAP (under_cap=true) must never let a chunk exceed the token- // target: a projected join that would overflow starts a fresh chunk- // instead of merge-then-close (OVER_CAP). This exercises the seam that- // lets Go follow Python's no-overflow (UNDER_CAP) strategy without- // changing the default (OVER_CAP) behavior.- const sentence = "word word word word word word word word word word word word word " // 12 words- sentenceN := tokenizeStr(sentence)- budget := sentenceN*4 + 2 // four sentences fit, five overflow.- if budget < sentenceN*2 {- budget = sentenceN * 2- }- var b strings.Builder- for i := 0; i < 12; i++ {- b.WriteString(sentence)- b.WriteString("! ")- }-- run := func(underCap bool) []map[string]any {- comp, err := NewTokenChunker(map[string]any{- "delimiter_mode": "delimiter",- "chunk_token_size": budget,- "under_cap": underCap,- })- if err != nil {- t.Fatalf("NewTokenChunker: %v", err)- }- out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil, nil)- chunks, _ := out["chunks"].([]map[string]any)- return chunks- }-- respect := run(true)- if len(respect) < 2 {- t.Fatalf("UNDER_CAP: want multiple chunks, got %d", len(respect))+ if len(chunks) < 2 {+ t.Fatalf("oversized run must be split, got %d chunk(s)", len(chunks))}- for i, ck := range respect {+ for i, ck := range chunks {text, _ := ck["text"].(string)if n := tokenizeStr(text); n > budget {- t.Errorf("UNDER_CAP chunk %d exceeds target: tokens=%d (cap=%d)", i, n, budget)- }- }-- // Control: OVER_CAP (default) must follow its one-boundary-overflow- // contract on the same input, proving the toggle changes behavior rather- // than being a no-op. With the running-sum merge decision (faithful to- // Python's naive_merge), a chunk may hold prev (<= budget) + one unit, so- // its running-sum token count is <= budget+unit; the reconstructed text- // carries the "\n" joins and cl100k is non-additive across them, so allow- // the same one-extra-unit slack. For this particular input Python's- // OVER_CAP does not actually exceed budget, but it still permits the- // one-unit overflow that UNDER_CAP forbids, so the two strategies produce- // different chunk counts — proving the toggle is live.- over := run(false)- if len(over) == len(respect) {- t.Errorf("OVER_CAP produced the same chunk count as UNDER_CAP (%d); toggle may be a no-op", len(over))- }- for i, ck := range over {- text, _ := ck["text"].(string)- if tokenizeStr(text) > budget+sentenceN {- t.Errorf("OVER_CAP chunk %d exceeds one-unit overflow contract: tokens=%d (cap=%d unit=%d)", i, tokenizeStr(text), budget, sentenceN)+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=%d)", i, n, budget)}}}-func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {+func TestInvokeTextPayload_HardCapEndToEnd(t *testing.T) {const budget = 32- unit := tokenizeStr(strings.TrimSpace(strings.Repeat("alpha ", 12)))var b strings.Builderfor i := 0; i < 20; i++ {b.WriteString(strings.TrimSpace(strings.Repeat("alpha ", 12)))@@ -273,25 +182,14 @@ func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {}for i, ck := range chunks {text, _ := ck["text"].(string)- // OVER_CAP allows one boundary overflow (prev + one unit).- if n := tokenizeStr(text); n > budget+unit {- t.Errorf("chunk %d exceeds budget+one-unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)+ if n := tokenizeStr(text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=%d)", i, n, budget)}}}-// TestInvokeJSONPayload_UnderCapEndToEnd exercises the UNDER_CAP strategy on-// the JSON path end-to-end (invokeJSONPayload). Many single-line JSON items-// are flattened into one global merge sequence, and under_cap=true must keep-// every chunk within chunk_token_size. The OVER_CAP control is asserted to-// pack fewer chunks than UNDER_CAP, proving the toggle is live on the JSON-// path — not just the text path covered by TestInvokeTextPayload_StrictCapEndToEnd.-func TestInvokeJSONPayload_UnderCapEndToEnd(t *testing.T) {+func TestInvokeJSONPayload_HardCapEndToEnd(t *testing.T) {const budget = 32- unit := tokenizeStr(strings.TrimSpace(strings.Repeat("alpha ", 12)))-- // Each item is one ~unit-sized token block; 24 items give the global- // merge plenty to accumulate.var items []map[string]anyfor i := 0; i < 24; i++ {items = append(items, map[string]any{@@ -299,113 +197,74 @@ func TestInvokeJSONPayload_UnderCapEndToEnd(t *testing.T) {"doc_type_kwd": "text",})}-- run := func(underCap bool) []map[string]any {- comp, err := NewTokenChunker(map[string]any{- "delimiter_mode": "delimiter",- "delimiters": []string{"\n"},- "chunk_token_size": budget,- "under_cap": underCap,- })- if err != nil {- t.Fatalf("NewTokenChunker: %v", err)- }- out, err := comp.Invoke(context.Background(), nil, map[string]any{- "name": "doc.json",- "output_format": "json",- "json": items,- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }- if errMsg, _ := out["_ERROR"].(string); errMsg != "" {- t.Fatalf("Invoke error payload: %s", errMsg)- }- chunks, _ := out["chunks"].([]map[string]any)- return chunks+ comp, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{"\n"},+ "chunk_token_size": budget,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)}-- // UNDER_CAP: no chunk may exceed the token target.- under := run(true)- if len(under) < 2 {- t.Fatalf("UNDER_CAP: want multiple chunks, got %d", len(under))+ out, err := comp.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.json",+ "output_format": "json",+ "json": items,+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)}- for i, ck := range under {- text, _ := ck["text"].(string)- if n := tokenizeStr(text); n > budget {- t.Errorf("UNDER_CAP chunk %d exceeds target: tokens=%d (cap=%d)", i, n, budget)- }+ if errMsg, _ := out["_ERROR"].(string); errMsg != "" {+ t.Fatalf("Invoke error payload: %s", errMsg)}-- // OVER_CAP control: the same input packs fewer chunks (it allows one- // boundary overflow per chunk), proving the toggle is live on the JSON- // path. Equal lengths would mean under_cap is a no-op here.- over := run(false)- if len(over) >= len(under) {- t.Errorf("OVER_CAP should pack fewer chunks than UNDER_CAP (over=%d under=%d); toggle may be a no-op on the JSON path", len(over), len(under))- }- for i, ck := range over {+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) < 2 {+ t.Fatalf("want multiple chunks, got %d", len(chunks))+ }+ for i, ck := range chunks {text, _ := ck["text"].(string)- // OVER_CAP may overflow by at most one unit.- if n := tokenizeStr(text); n > budget+unit {- t.Errorf("OVER_CAP chunk %d overflows by more than one unit: tokens=%d (cap=%d unit=%d)", i, n, budget, unit)+ if n := tokenizeStr(text); n > budget {+ t.Errorf("chunk %d exceeds budget: tokens=%d (cap=%d)", i, n, budget)}}}-// TestMergeDecisionOverCapVsUnderCapBoundary pins the semantic difference-// between the two merge strategies at the exact boundary-// (prevTokens+incomingTokens > target while incomingTokens <= target),-// independent of BPE text-token-count fluctuations. OVER_CAP must-// merge-then-close (a chunk may exceed target by at most one unit); UNDER_CAP-// must start a fresh chunk (strict no-overflow). This restores the strong-// constraint that the under_cap toggle is live — the integration test in-// TestMergeByTokenSize_UnderCapNoOverflow can only assert that the two-// strategies produce a different chunk COUNT, because on repetitive text the-// re-tokenized merged chunk can fall back under budget (cl100k is-// non-additive across the join). A regression that turns OVER_CAP into a-// no-op would not be caught there, but is caught here.-func TestMergeDecisionOverCapVsUnderCapBoundary(t *testing.T) {- const prevT, incT = 10, 10- target := prevT + incT - 1 // boundary: running sum > target, unit fits- if incT > target {- t.Fatalf("bad fixture: incoming unit must fit target (incT=%d target=%d)", incT, target)- }-+// TestMergeUnits_UnderCapBoundaryDecision pins the boundary semantics of the+// hard-cap merge independent of BPE text-token-count fluctuations: an exact-fit+// join (prev+incoming == target) merges; an overflowing join starts a fresh+// chunk; an oversized incoming unit is expanded into <= target pieces instead of+// standing alone.+func TestMergeUnits_UnderCapBoundaryDecision(t *testing.T) {units := []schema.ChunkDoc{- {Text: "prev text", TKNums: intPtr(prevT), CKType: "text"},- {Text: "incoming text", TKNums: intPtr(incT), CKType: "text"},+ {Text: "prev text", TKNums: intPtr(10), CKType: "text"},+ {Text: "incoming text", TKNums: intPtr(10), CKType: "text"},}-- // OVER_CAP at the boundary: the overflowing incoming is merged into the- // previous chunk (merge-then-close) — a single merged chunk carrying the- // running sum prevT+incT.- over := mergeUnits(units, target, 0, schema.MergeOverCap, "\n")- if len(over) != 1 {- t.Fatalf("OVER_CAP at boundary: want 1 merged chunk, got %d", len(over))+ // prev+incoming == target: merge into one chunk.+ got := mergeUnits(units, 20, 0, "\n")+ if len(got) != 1 {+ t.Fatalf("exact-fit join: want 1 merged chunk, got %d", len(got))}- if got := intValue(over[0].TKNums); got != prevT+incT {- t.Errorf("OVER_CAP merged chunk running sum: want %d, got %d", prevT+incT, got)+ if sum := intValue(got[0].TKNums); sum != 20 {+ t.Errorf("merged chunk running sum: want 20, got %d", sum)}- // UNDER_CAP at the boundary: the overflowing incoming starts a fresh chunk.- under := mergeUnits(units, target, 0, schema.MergeUnderCap, "\n")- if len(under) != 2 {- t.Fatalf("UNDER_CAP at boundary: want 2 chunks, got %d", len(under))+ // prev+incoming > target: fresh chunk, no overflow.+ over := mergeUnits(units, 19, 0, "\n")+ if len(over) != 2 {+ t.Fatalf("overflowing join: want 2 chunks, got %d", len(over))}- // Both strategies must still refuse to merge an incoming unit that- // already exceeds target — it stands alone as its own chunk.+ // An oversized incoming unit is expanded (split) instead of standing whole.big := []schema.ChunkDoc{- {Text: "prev text", TKNums: intPtr(prevT), CKType: "text"},- {Text: "incoming text", TKNums: intPtr(target + 5), CKType: "text"},+ {Text: "prev text", TKNums: intPtr(10), CKType: "text"},+ {Text: strings.Repeat("word ", 100), TKNums: intPtr(100), CKType: "text"},}- overBig := mergeUnits(big, target, 0, schema.MergeOverCap, "\n")- if len(overBig) != 2 {- t.Errorf("OVER_CAP with oversized incoming: want 2 chunks (stands alone), got %d", len(overBig))+ bigMerged := mergeUnits(big, 30, 0, "\n")+ if len(bigMerged) < 2 {+ t.Fatalf("oversized incoming unit must be split, got %d chunk(s)", len(bigMerged))}- underBig := mergeUnits(big, target, 0, schema.MergeUnderCap, "\n")- if len(underBig) != 2 {- t.Errorf("UNDER_CAP with oversized incoming: want 2 chunks (stands alone), got %d", len(underBig))+ for i, ck := range bigMerged {+ if n := intValue(ck.TKNums); n > 30 {+ t.Errorf("chunk %d exceeds target: tokens=%d", i, n)+ }}}diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.goindex 26c6be3884..3cb5987527 100644--- a/internal/ingestion/component/schema/chunker.go+++ b/internal/ingestion/component/schema/chunker.go@@ -204,44 +204,6 @@ type TokenChunkerParam struct {// ImageContextSize is the number of surrounding tokens to attach// to image chunks. 0 disables.ImageContextSize int `json:"image_context_size"`-- // UnderCap selects the merge strategy when greedily accumulating- // adjacent units (token_chunker UNDER_CAP vs OVER_CAP).- // - false (default): OVER_CAP — mirrors Python's canonical default.- // A chunk may exceed the token target by at most one incoming unit- // (a boundary overflow then closes the chunk).- // - true: UNDER_CAP — strictly never exceed the target; when the- // projected join would overflow, start a fresh chunk instead.- // This is the wire-facing bool; the active strategy is exposed as- // MergeStrategy so callers never have to invert it.- UnderCap bool `json:"under_cap"`-}--// MergeStrategy selects how the TokenChunker greedily accumulates adjacent-// units into a chunk. It mirrors Python's rag/nlp/__init__.py MergeStrategy so-// the Go and Python implementations stay on the same vocabulary (OVER_CAP /-// UNDER_CAP) instead of a bare inlined bool.-type MergeStrategy int--const (- // MergeOverCap is the canonical default (Python OVER_CAP): a chunk may- // exceed the token target by at most one incoming unit (a boundary- // overflow then closes the chunk).- MergeOverCap MergeStrategy = iota- // MergeUnderCap never exceeds the target; when the projected join would- // overflow, a fresh chunk is started instead (Python UNDER_CAP).- MergeUnderCap-)--// MergeStrategy reports the active merge strategy for this param. It is derived-// from UnderCap so existing configs that set "under_cap" keep working without a-// schema break, and callers read the strategy directly instead of inverting a-// bool at every call site.-func (p TokenChunkerParam) MergeStrategy() MergeStrategy {- if p.UnderCap {- return MergeUnderCap- }- return MergeOverCap}// Defaults returns the Python default TokenChunkerParam.diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex c27ae5976d..609b496c1c 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -567,15 +567,3 @@ func TestExtractorOutputsJSONRoundTrip(t *testing.T) {// ---------------------------------------------------------------------------func ptrString(s string) *string { return &s }--// TestTokenChunkerParamMergeStrategy locks the UnderCap bool -> MergeStrategy-// mapping so a future refactor cannot silently flip OVER_CAP/UNDER_CAP without-// a failing test.-func TestTokenChunkerParamMergeStrategy(t *testing.T) {- if got := (TokenChunkerParam{}).MergeStrategy(); got != MergeOverCap {- t.Errorf("default (UnderCap=false) want MergeOverCap, got %v", got)- }- if got := (TokenChunkerParam{UnderCap: true}).MergeStrategy(); got != MergeUnderCap {- t.Errorf("UnderCap=true want MergeUnderCap, got %v", got)- }-}