RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(chunker): converge delimiter_mode to {delimiter, one}, drop token_size (#17979)
Jack1aa4e3c1f318
diff --git a/agent/templates/advanced_ingestion_pipeline.json b/agent/templates/advanced_ingestion_pipeline.jsonindex ecb8fd30fa..b709b741e6 100644--- a/agent/templates/advanced_ingestion_pipeline.json+++ b/agent/templates/advanced_ingestion_pipeline.json@@ -336,7 +336,7 @@"params": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [],"image_context_size": 0,"outputs": {@@ -576,7 +576,7 @@"form": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [{"value": "\n"diff --git a/agent/templates/chunk_summary.json b/agent/templates/chunk_summary.jsonindex 980ba62b5b..2f76945db8 100644--- a/agent/templates/chunk_summary.json+++ b/agent/templates/chunk_summary.json@@ -219,7 +219,7 @@"params": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [],"image_context_size": 0,"outputs": {@@ -429,7 +429,7 @@"data": {"form": {"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [{"value": "\n"diff --git a/agent/templates/compiler.json b/agent/templates/compiler.jsonindex 437524b766..3895f903eb 100644--- a/agent/templates/compiler.json+++ b/agent/templates/compiler.json@@ -192,7 +192,7 @@"params": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [],"image_context_size": 0,"outputs": {@@ -374,7 +374,7 @@"form": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [{"value": "\n"diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 93781081a1..8e5d14fa17 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -16,7 +16,7 @@// SCOPE (honest) for token.go://-// - WHITELIST: delimiter_mode ∈ {"token_size","delimiter"} (the+// - WHITELIST: delimiter_mode ∈ {"delimiter"} (the// single-chunk "one" behaviour moved to OneChunker in one.go).// chunk_token_size > 0, overlapped_percent accepts a [0,1) fraction or a// [0,90] percentage (normalized to [0,90] by normalizeOverlappedPercent,diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.goindex eaf68acb74..9e2a339c03 100644--- a/internal/ingestion/component/chunker/token_strict_cap_test.go+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go@@ -125,7 +125,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {b.WriteString("\n\n")}comp, err := NewTokenChunker(map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": budget,})if err != nil {@@ -159,7 +159,7 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) {// run is one unit that still exceeds the budget and must stay whole.long := strings.TrimSpace(strings.Repeat("word ", 100))comp, err := NewTokenChunker(map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": budget,})if err != nil {@@ -196,7 +196,7 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {run := func(underCap bool) []map[string]any {comp, err := NewTokenChunker(map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": budget,"under_cap": underCap,})@@ -250,7 +250,7 @@ func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {b.WriteByte('\n')}comp, err := NewTokenChunker(map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": budget,})if err != nil {diff --git a/internal/ingestion/component/chunker/token_tag_test.go b/internal/ingestion/component/chunker/token_tag_test.goindex 97ba086a21..bca95db810 100644--- a/internal/ingestion/component/chunker/token_tag_test.go+++ b/internal/ingestion/component/chunker/token_tag_test.go@@ -25,7 +25,7 @@ import (// the body text still carries coordinate markers.func TestTokenChunker_TextPath_StripsParserTags(t *testing.T) {c, err := NewTokenChunker(map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": 1000,"delimiters": []string{"\n"},})diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 8da9c5d474..98ab935adb 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -145,7 +145,7 @@ func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) {// token-size merge and emit >=1 chunk.func TestTokenChunker_InvokeTokenSize_FallbackToMerge(t *testing.T) {c, err := NewTokenChunker(map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": 50,"delimiters": []string{"`\n\n`"},})@@ -365,9 +365,9 @@ func TestTokenChunker_NewRejectsBadParam(t *testing.T) {}{{"bad delimiter_mode", map[string]any{"delimiter_mode": "nope"}},{"one delimiter_mode (use OneChunker)", map[string]any{"delimiter_mode": "one"}},- {"zero chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 0}},- {"negative chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": -5}},- {"negative table_context_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 50, "table_context_size": -1}},+ {"zero chunk_token_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 0}},+ {"negative chunk_token_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": -5}},+ {"negative table_context_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 50, "table_context_size": -1}},}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {@@ -380,14 +380,14 @@ func TestTokenChunker_NewRejectsBadParam(t *testing.T) {// TestTokenChunker_NewAcceptsDefaults ensures the no-config// constructor returns a usable component with a working default-// delimiter_mode = "token_size".+// delimiter_mode = "delimiter".func TestTokenChunker_NewAcceptsDefaults(t *testing.T) {c, err := NewTokenChunker(nil)if err != nil {t.Fatalf("NewTokenChunker(nil): %v", err)}- if got := c.(*TokenChunkerComponent).param.DelimiterMode; got != "token_size" {- t.Errorf("default delimiter_mode = %q, want token_size", got)+ if got := c.(*TokenChunkerComponent).param.DelimiterMode; got != "delimiter" {+ t.Errorf("default delimiter_mode = %q, want delimiter", got)}}@@ -439,7 +439,7 @@ func TestTokenChunker_NewAcceptsPythonOverlappedRange(t *testing.T) {// percentages, including out-of-range inputs that Python clamps).for _, pct := range []float64{0, 0.1, 0.5, 15, 30, 50, 90, 95, -5} {conf := map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": 100,"overlapped_percent": pct,}@@ -535,7 +535,7 @@ func TestTokenChunker_NormalizesOverlappedPercent(t *testing.T) {for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {c, err := NewTokenChunker(map[string]any{- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","chunk_token_size": 100,"overlapped_percent": tc.in,})@@ -575,7 +575,7 @@ func TestTokenChunkerParam_ValidateOverlappedRange(t *testing.T) {for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {p := schema.TokenChunkerParam{- DelimiterMode: "token_size",+ DelimiterMode: "delimiter",ChunkTokenSize: 100,OverlappedPercent: tc.in,}diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.goindex ad9b9a421c..e920950de1 100644--- a/internal/ingestion/component/schema/chunker.go+++ b/internal/ingestion/component/schema/chunker.go@@ -173,7 +173,7 @@ type ChunkerOutputs struct {type TokenChunkerParam struct {// DelimiterMode selects the chunking strategy.- // Allowed values: "token_size", "delimiter".+ // Allowed value: "delimiter".// The single-chunk "one" behavior is provided by the separate// OneChunker component.DelimiterMode string `json:"delimiter_mode"`@@ -246,7 +246,7 @@ func (p TokenChunkerParam) MergeStrategy() MergeStrategy {// Defaults returns the Python default TokenChunkerParam.func (TokenChunkerParam) Defaults() TokenChunkerParam {return TokenChunkerParam{- DelimiterMode: "token_size",+ DelimiterMode: "delimiter",ChunkTokenSize: 512,Delimiters: []string{"\n"},OverlappedPercent: 0,@@ -357,7 +357,7 @@ func (p *TokenChunkerParam) Validate() error {p.OverlappedPercent = math.Round(v * 100)}switch p.DelimiterMode {- case "token_size", "delimiter":+ case "delimiter":default:return errInvalidValue{Field: "delimiter_mode", Value: p.DelimiterMode}}diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex 517513f951..d9559b0e66 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -278,8 +278,8 @@ func TestChunkerOutputsJSONRoundTrip(t *testing.T) {func TestTokenChunkerParamDefaults(t *testing.T) {p := TokenChunkerParam{}.Defaults()- if p.DelimiterMode != "token_size" {- t.Errorf("default delimiter_mode = %q, want token_size", p.DelimiterMode)+ if p.DelimiterMode != "delimiter" {+ t.Errorf("default delimiter_mode = %q, want delimiter", p.DelimiterMode)}if p.ChunkTokenSize != 512 {t.Errorf("default chunk_token_size = %d, want 512", p.ChunkTokenSize)diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json b/internal/ingestion/pipeline/template/ingestion_pipeline_email.jsonindex 805a5df922..09c58c291e 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json@@ -83,7 +83,7 @@"params": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": ["\n","!",@@ -265,7 +265,7 @@"form": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [{"value": "\n"diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.jsonindex ff4a3165ba..22d0d6fe7e 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json@@ -161,7 +161,7 @@"params": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": ["\n","!",@@ -391,7 +391,7 @@"form": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": [{"value": "\n"diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.jsonindex 364bc66b09..db0ab19d38 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json@@ -157,7 +157,7 @@"params": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "token_size",+ "delimiter_mode": "delimiter","delimiters": ["\n","!",diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex b1eca5024f..fc483af648 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -40,8 +40,8 @@ import (var builtinComponentParamsGolden = map[string]string{"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",+ "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",@@ -51,7 +51,7 @@ var builtinComponentParamsGolden = map[string]string{"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}","resume": "{\"Extractor:ThreeDrinksAct\": {\"field_name\": \"metadata\", \"frequencyPenaltyEnabled\": true, \"frequency_penalty\": 0.7, \"llm_id\": \"THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW\", \"maxTokensEnabled\": false, \"max_tokens\": 256, \"presencePenaltyEnabled\": true, \"presence_penalty\": 0.4, \"prompts\": [{\"content\": \"Content: {TitleChunker:FlatMiceFix@chunks}\", \"role\": \"user\"}], \"sys_prompt\": \"Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\\n\\nRules:\\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\\n3. Use only these field names:\\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\\n5. Keep values in the same language as the source text whenever possible.\\n6. Remove duplicates and keep only concise, high-value metadata.\\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\\n\\nField guidance:\\n- highest_degree: highest explicit degree level mentioned\\n- degree_levels: all explicit degree levels mentioned\\n- school_names: explicit school, college, or university names\\n- majors: explicit fields of study\\n- graduation_years: explicit graduation years only\\n- work_experience_years: only if explicitly stated\\n- current_job_title: only if explicitly current or most recent\\n- job_titles: explicit role titles\\n- company_names: explicit employer names\\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\\n- industries: explicit industry names only\\n- target_job_titles: explicit desired roles only\\n- target_locations: explicit desired work locations only\\n- skills: concise, core, search-useful skills explicitly mentioned\\n- certificates: explicit certificate names only\\n- awards: explicit award names only\\n- summary_tags: short, high-value tags strictly supported by the content\\n\\nReturn only the extracted metadata. Do not output explanatory text.\", \"temperature\": 0.1, \"temperatureEnabled\": true, \"tenant_llm_id\": 29, \"topPEnabled\": true, \"top_p\": 0.3, \"auto_keywords\": 0, \"auto_questions\": 0, \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}, \"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:FlatMiceFix\": {\"hierarchy\": 1, \"include_heading_content\": false, \"levels\": [[\"^\\\\s*(?i:(?:\\\\d+[\\\\.\\\\)]\\\\s*)?(?:EDUCATION|ACADEMIC\\\\s*BACKGROUND|ACADEMIC\\\\s*HISTORY|EDUCATIONAL\\\\s*BACKGROUND|RELEVANT\\\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\\\s*EXPERIENCE|PROFESSIONAL\\\\s*EXPERIENCE|RELEVANT\\\\s*EXPERIENCE|EMPLOYMENT\\\\s*HISTORY|CAREER\\\\s*HISTORY|INTERNSHIP\\\\s*EXPERIENCE|PROJECTS|PROJECT\\\\s*EXPERIENCE|ACADEMIC\\\\s*PROJECTS|PROFESSIONAL\\\\s*PROJECTS|SKILLS|TECHNICAL\\\\s*SKILLS|CORE\\\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\\\s*OF\\\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\\\s*ACTIVITIES|ACTIVITIES\\\\s*(?:&|AND)\\\\s*SKILLS|INVOLVEMENT|CAMPUS\\\\s*INVOLVEMENT|VOLUNTEER\\\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\\\s*PROFILE|SUMMARY|PROFESSIONAL\\\\s*SUMMARY|CAREER\\\\s*SUMMARY|OBJECTIVE|CAREER\\\\s*OBJECTIVE|PERSONAL\\\\s*INFORMATION|CONTACT\\\\s*INFORMATION|ADDITIONAL\\\\s*INFORMATION|TRAINING))\\\\s*[::]?\\\\s*$\"], [\"^\\\\s*(?:\\\\d+[\\\\.、\\\\)]\\\\s*)?(?:教育背景|教育经历|学历背景|学术背景|技术背景|工作经历|工作经验|实习经历|项目经历|项目经验|科研经历|研究经历|校园经历|实践经历|专业经历|职业经历|技能|专业技能|技能特长|核心技能|技术栈|个人技能|工作技能|职业技能|技能与评价|技能与自我评价|工作技能与自我评价|职业技能与自我评价|证书|资格证书|职业资格|资质证书|获奖情况|获奖经历|荣誉|荣誉奖项|奖项|科研成果|论文发表|发表论文|领导经历|学生工作|校园活动|社团经历|活动经历|志愿经历|志愿服务|社会实践|语言能力|语言|自我评价|个人评价|自我总结|个人总结|个人优势|个人简介|个人信息|基本信息|联系方式|求职意向|应聘意向|职业目标|求职目标|兴趣爱好|兴趣特长|培训经历|其他信息|附加信息)\\\\s*[::]?\\\\s*$\"]], \"method\": \"hierarchy\"}, \"Tokenizer:KindHandsWin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}","table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",- "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",+ "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",}// Per-template test methods. Each resolves default component params from adiff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.pyindex a9a9ff7a95..0fce2c18c8 100644--- a/rag/flow/chunker/token_chunker.py+++ b/rag/flow/chunker/token_chunker.py@@ -32,7 +32,7 @@ from rag.nlp import naive_mergeclass TokenChunkerParam(ProcessParamBase):def __init__(self):super().__init__()- self.delimiter_mode = "token_size"+ self.delimiter_mode = "delimiter"self.chunk_token_size = 512self.delimiters = ["\n"]self.overlapped_percent = 0@@ -41,7 +41,13 @@ class TokenChunkerParam(ProcessParamBase):self.image_context_size = 0def check(self):- self.check_valid_value(self.delimiter_mode, "Delimiter mode abnormal.", ["token_size", "delimiter", "one"])+ # Backward-compat: "token_size" was removed but is behaviorally identical+ # to "delimiter" at runtime (both route through the same code path), so+ # accept and coerce it instead of rejecting legacy configs / pre-fix+ # frontends. Only genuinely unknown values are rejected.+ if self.delimiter_mode == "token_size":+ self.delimiter_mode = "delimiter"+ self.check_valid_value(self.delimiter_mode, "Delimiter mode abnormal.", ["delimiter", "one"])if self.delimiters is None:self.delimiters = []elif isinstance(self.delimiters, str):@@ -316,9 +322,7 @@ class TokenChunker(ProcessBase):self.set_output("chunks", [{"text": payload}] if payload.strip() else [])self.callback(1, "Done.")return- if self._param.delimiter_mode == "delimiter":- cks = _split_text_by_pattern(payload, delimiter_pattern)- elif delimiter_pattern:+ if delimiter_pattern:cks = _split_text_by_pattern(payload, delimiter_pattern)else:cks = naive_merge(@@ -358,8 +362,14 @@ class TokenChunker(ProcessBase):self.callback(1, "Done.")return- if self._param.delimiter_mode == "delimiter":- text_chunks = _build_json_chunks(json_result, "")+ # Both branches start from per-item chunks (no pre-split by the+ # delimiter pattern). The delimiter branch splits the buffered text+ # stream while preserving per-segment PDF positions; the no-delimiter+ # branch merges adjacent text items to chunk_token_size (the removed+ # "token_size" behaviour, and a parity match with the Go JSON path).+ text_chunks = _build_json_chunks(json_result, "")++ if delimiter_pattern:chunks = []text_buffer = []text_buffer_pos = []@@ -372,9 +382,9 @@ class TokenChunker(ProcessBase):# The delimiter is then applied to the combined text; a segment may# span across item boundaries (the "\n" glue is not itself a# delimiter), so each segment carries only the PDF positions of the- # buffered item(s) whose text contributed to it -- never the union of- # every item (which previously leaked page-N coordinates into- # page-M chunks and made all segments share one preview image).+ # item(s) that contributed to it -- never the union of every item+ # (which previously leaked page-N coordinates into page-M chunks and+ # made all segments share one preview image).parts = []item_ranges = [] # (start, end) of each buffered item in combined_textoffset = 0@@ -387,27 +397,24 @@ class TokenChunker(ProcessBase):offset += 1combined_text = "".join(parts[:-1]) # drop the trailing glue- if delimiter_pattern:- raw = re.split(r"(%s)" % delimiter_pattern, combined_text, flags=re.DOTALL)- segments = [] # (text, start, end) within combined_text- pos = 0- for i in range(0, len(raw), 2):- seg = raw[i]- seg_start = pos- seg_end = pos + len(seg)- if seg:- segments.append((seg, seg_start, seg_end))- pos = seg_end- if i + 1 < len(raw):- pos += len(raw[i + 1])- else:- segments = [(combined_text, 0, len(combined_text))]+ raw = re.split(r"(%s)" % delimiter_pattern, combined_text, flags=re.DOTALL)+ segments = [] # (text, start, end) within combined_text+ pos = 0+ for i in range(0, len(raw), 2):+ seg = raw[i]+ seg_start = pos+ seg_end = pos + len(seg)+ if seg:+ segments.append((seg, seg_start, seg_end))+ pos = seg_end+ if i + 1 < len(raw):+ pos += len(raw[i + 1])for text, seg_start, seg_end in segments:if not text.strip():continueseg_pos = []- for (istart, iend), item_pos in zip(item_ranges, text_buffer_pos):+ for (istart, iend), item_pos in zip(item_ranges, text_buffer_pos, strict=True):# A segment overlaps an item when their character ranges# intersect; collect that item's coordinates.if seg_start < iend and istart < seg_end:@@ -436,22 +443,19 @@ class TokenChunker(ProcessBase):if custom_pattern:chunks = _split_chunk_docs_by_children(chunks, custom_pattern)_attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)- await restore_pdf_text_previews(chunks, from_upstream, self._canvas)- self.set_output("chunks", _finalize_json_chunks(chunks))- self.callback(1, "Done.")- return-- # Structured JSON input is normalized first, then optionally enriched with- # media context, and finally merged only when delimiter splitting is inactive.- chunks = _build_json_chunks(json_result, delimiter_pattern)- _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)- if self._param.delimiter_mode == "token_size" and not delimiter_pattern:- chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent)-- if custom_pattern:- chunks = _split_chunk_docs_by_children(chunks, custom_pattern)+ else:+ # No active delimiter: merge adjacent text items to chunk_token_size.+ # This runs on the per-item chunks (NOT a single concatenated chunk),+ # so the token cap is actually enforced -- matching the previous+ # "token_size" mode and the Go JSON path. Media chunks break the merge.+ # Media context is attached on the per-item chunks before merging, as+ # the removed "token_size" branch did, to preserve context windows.+ _attach_context_to_media_chunks(text_chunks, self._param.table_context_size, self._param.image_context_size)+ chunks = _merge_text_chunks_by_token_size(text_chunks, self._param.chunk_token_size, overlapped_percent)+ if custom_pattern:+ chunks = _split_chunk_docs_by_children(chunks, custom_pattern)await restore_pdf_text_previews(chunks, from_upstream, self._canvas)- cks = _finalize_json_chunks(chunks)- self.set_output("chunks", cks)+ self.set_output("chunks", _finalize_json_chunks(chunks))self.callback(1, "Done.")+ returndiff --git a/rag/flow/tests/test_token_chunker.py b/rag/flow/tests/test_token_chunker.pyindex 8866b53de2..a33732a9c0 100644--- a/rag/flow/tests/test_token_chunker.py+++ b/rag/flow/tests/test_token_chunker.py@@ -55,6 +55,19 @@ def _load_token_chunker_with_stubs():def __init__(self):pass+ def check_valid_value(self, value, msg, allowed):+ if value not in allowed:+ raise ValueError(msg)++ def check_positive_integer(self, value, msg):+ pass++ def check_decimal_float(self, value, msg):+ pass++ def check_nonnegative_number(self, value, msg):+ pass+class ProcessBase:def __init__(self, _pipeline, _id, param):self._pipeline = _pipeline@@ -277,6 +290,73 @@ def test_json_delimiter_mode_pdf_positions_retained():assert chunks[0].get("pdf_positions") == [[1, 0, 10, 0, 5], [2, 0, 20, 0, 8]]+def test_token_size_mode_normalized_to_delimiter():+ # Backward-compat: the removed "token_size" value must still be accepted by+ # check() and coerced to "delimiter" (runtime behavior is identical), so+ # legacy configs / pre-fix frontends don't get rejected. Unknown values are+ # still rejected.+ with _load_token_chunker_with_stubs() as token_chunker_module:+ param = token_chunker_module.TokenChunkerParam()+ param.delimiter_mode = "token_size"+ param.check()+ assert param.delimiter_mode == "delimiter"++ bad = token_chunker_module.TokenChunkerParam()+ bad.delimiter_mode = "nope"+ try:+ bad.check()+ raise AssertionError("expected check() to reject unknown delimiter_mode")+ except Exception:+ pass+++def test_json_no_delimiter_mode_merges_to_token_cap():+ # Regression for #17979: with no active (backtick) delimiter, the JSON path+ # must merge per-item text chunks up to chunk_token_size -- mirroring the old+ # "token_size" mode and the Go JSON path. Concatenating every item into a+ # single chunk before the merge would defeat the cap and emit one oversized+ # chunk. The per-token stub (num_tokens_from_string -> 1) makes the cap easy+ # to exceed: 12 one-token items under a cap of 5 must yield several chunks.+ for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5}):+ kwargs = {+ "name": "token_chunker",+ "output_format": "json",+ "json_result": [{"text": f"item{i}"} for i in range(12)],+ }+ asyncio.run(chunker._invoke(**kwargs))+ chunks = chunker._outputs["chunks"]+ # 12 one-token items under a cap of 5 must NOT collapse into one chunk.+ assert len(chunks) > 1, f"cap not enforced: 12 items collapsed to {len(chunks)} chunk(s)"+ # Each merged chunk holds at most one overflow unit past the cap (<= 6+ # items); the final output drops tk_nums, so count item markers instead.+ for c in chunks:+ assert c["text"].count("item") <= 6, f"chunk exceeds cap: {c['text'].count('item')} items"+ # No text lost: all 12 items must survive, joined by the "\n" glue.+ joined = "\n".join(c["text"] for c in chunks)+ for i in range(12):+ assert f"item{i}" in joined, f"item{i} dropped from output"+++def test_json_no_delimiter_mode_media_breaks_merge():+ # A non-text (media) chunk interleaved between text items must stay as its+ # own chunk and reset the merge, so text before/after it are sized separately.+ for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5}):+ kwargs = {+ "name": "token_chunker",+ "output_format": "json",+ "json_result": [{"text": f"t{i}", "doc_type_kwd": "text"} for i in range(6)]+ + [{"text": "IMG", "doc_type_kwd": "image", "img_id": "im1"}]+ + [{"text": f"u{i}", "doc_type_kwd": "text"} for i in range(12)],+ }+ asyncio.run(chunker._invoke(**kwargs))+ chunks = chunker._outputs["chunks"]+ doc_types = [c["doc_type_kwd"] for c in chunks]+ # The media chunk is preserved and breaks the text merge.+ assert doc_types.count("image") == 1, doc_types+ # Several text chunks on each side of the media boundary.+ assert doc_types.count("text") > 2, doc_types++def test_json_delimiter_mode_pdf_positions_per_segment_not_broadcast():# Regression for #3 (PDF coordinate leak): when consecutive text items from# different pages are buffered and then split by a custom delimiter, each@@ -346,11 +426,11 @@ def test_json_delimiter_mode_consecutive_delimiter_keeps_boundary():assert all("##" not in t for t in texts)-def test_text_delimiter_mode_token_size_zero_or_one_no_atom_split():- # token_size=0/1 must not atom-split delimiter segments into 1-token chunks;- # the delimiter path produces delimiter-boundary chunks regardless of cap.- for module, chunker in _build_json_chunker({"delimiter_mode": "token_size", "delimiters": ["`|`"]}):- # text path: delimiter_mode is token_size but a custom delimiter is+def test_text_delimiter_mode_zero_or_one_no_atom_split():+ # chunk_token_size=0/1 must not atom-split delimiter segments into 1-token+ # chunks; the delimiter path produces delimiter-boundary chunks regardless of cap.+ for module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):+ # text path: delimiter_mode is delimiter but a custom delimiter is# present, so the delimiter branch (_split_text_by_pattern) is used.kwargs = {"name": "token_chunker",@@ -362,4 +442,4 @@ def test_text_delimiter_mode_token_size_zero_or_one_no_atom_split():asyncio.run(chunker._invoke(**kwargs))chunks = chunker._outputs["chunks"]texts = [c["text"] for c in chunks]- assert texts == ["aaa", "bbb", "ccc"], f"token_size={chunk_token_size} atom-split a delimiter segment: {texts}"+ assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"