Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(chunker): converge delimiter_mode to {delimiter, one}, drop token_size (#17979)

Jack1aa4e3c1f318
diff --git a/agent/templates/advanced_ingestion_pipeline.json b/agent/templates/advanced_ingestion_pipeline.json
index ecb8fd30fa..b709b741e6 100644
--- a/agent/templates/advanced_ingestion_pipeline.json
+++ b/agent/templates/advanced_ingestion_pipeline.json
@@ -336,7 +336,7 @@
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [],
"image_context_size": 0,
"outputs": {
@@ -576,7 +576,7 @@
"form": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
{
"value": "\n"
diff --git a/agent/templates/chunk_summary.json b/agent/templates/chunk_summary.json
index 980ba62b5b..2f76945db8 100644
--- a/agent/templates/chunk_summary.json
+++ b/agent/templates/chunk_summary.json
@@ -219,7 +219,7 @@
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [],
"image_context_size": 0,
"outputs": {
@@ -429,7 +429,7 @@
"data": {
"form": {
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
{
"value": "\n"
diff --git a/agent/templates/compiler.json b/agent/templates/compiler.json
index 437524b766..3895f903eb 100644
--- a/agent/templates/compiler.json
+++ b/agent/templates/compiler.json
@@ -192,7 +192,7 @@
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [],
"image_context_size": 0,
"outputs": {
@@ -374,7 +374,7 @@
"form": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
{
"value": "\n"
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 93781081a1..8e5d14fa17 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -16,7 +16,7 @@
// SCOPE (honest) for token.go:
//
-// - WHITELIST: delimiter_mode ∈ {"token_size","delimiter"} (the
+// - WHITELIST: delimiter_mode ∈ {"delimiter"} (the
// single-chunk "one" behaviour moved to OneChunker in one.go).
// chunk_token_size > 0, overlapped_percent accepts a [0,1) fraction or a
// [0,90] percentage (normalized to [0,90] by normalizeOverlappedPercent,
diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.go
index eaf68acb74..9e2a339c03 100644
--- a/internal/ingestion/component/chunker/token_strict_cap_test.go
+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go
@@ -125,7 +125,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {
b.WriteString("\n\n")
}
comp, err := NewTokenChunker(map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": budget,
})
if err != nil {
@@ -159,7 +159,7 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) {
// run is one unit that still exceeds the budget and must stay whole.
long := strings.TrimSpace(strings.Repeat("word ", 100))
comp, err := NewTokenChunker(map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": budget,
})
if err != nil {
@@ -196,7 +196,7 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {
run := func(underCap bool) []map[string]any {
comp, err := NewTokenChunker(map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": budget,
"under_cap": underCap,
})
@@ -250,7 +250,7 @@ func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) {
b.WriteByte('\n')
}
comp, err := NewTokenChunker(map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": budget,
})
if err != nil {
diff --git a/internal/ingestion/component/chunker/token_tag_test.go b/internal/ingestion/component/chunker/token_tag_test.go
index 97ba086a21..bca95db810 100644
--- a/internal/ingestion/component/chunker/token_tag_test.go
+++ b/internal/ingestion/component/chunker/token_tag_test.go
@@ -25,7 +25,7 @@ import (
// the body text still carries coordinate markers.
func TestTokenChunker_TextPath_StripsParserTags(t *testing.T) {
c, err := NewTokenChunker(map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": 1000,
"delimiters": []string{"\n"},
})
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index 8da9c5d474..98ab935adb 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -145,7 +145,7 @@ func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) {
// token-size merge and emit >=1 chunk.
func TestTokenChunker_InvokeTokenSize_FallbackToMerge(t *testing.T) {
c, err := NewTokenChunker(map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": 50,
"delimiters": []string{"`\n\n`"},
})
@@ -365,9 +365,9 @@ func TestTokenChunker_NewRejectsBadParam(t *testing.T) {
}{
{"bad delimiter_mode", map[string]any{"delimiter_mode": "nope"}},
{"one delimiter_mode (use OneChunker)", map[string]any{"delimiter_mode": "one"}},
- {"zero chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 0}},
- {"negative chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": -5}},
- {"negative table_context_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 50, "table_context_size": -1}},
+ {"zero chunk_token_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 0}},
+ {"negative chunk_token_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": -5}},
+ {"negative table_context_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 50, "table_context_size": -1}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -380,14 +380,14 @@ func TestTokenChunker_NewRejectsBadParam(t *testing.T) {
// TestTokenChunker_NewAcceptsDefaults ensures the no-config
// constructor returns a usable component with a working default
-// delimiter_mode = "token_size".
+// delimiter_mode = "delimiter".
func TestTokenChunker_NewAcceptsDefaults(t *testing.T) {
c, err := NewTokenChunker(nil)
if err != nil {
t.Fatalf("NewTokenChunker(nil): %v", err)
}
- if got := c.(*TokenChunkerComponent).param.DelimiterMode; got != "token_size" {
- t.Errorf("default delimiter_mode = %q, want token_size", got)
+ if got := c.(*TokenChunkerComponent).param.DelimiterMode; got != "delimiter" {
+ t.Errorf("default delimiter_mode = %q, want delimiter", got)
}
}
@@ -439,7 +439,7 @@ func TestTokenChunker_NewAcceptsPythonOverlappedRange(t *testing.T) {
// percentages, including out-of-range inputs that Python clamps).
for _, pct := range []float64{0, 0.1, 0.5, 15, 30, 50, 90, 95, -5} {
conf := map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": 100,
"overlapped_percent": pct,
}
@@ -535,7 +535,7 @@ func TestTokenChunker_NormalizesOverlappedPercent(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
c, err := NewTokenChunker(map[string]any{
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"chunk_token_size": 100,
"overlapped_percent": tc.in,
})
@@ -575,7 +575,7 @@ func TestTokenChunkerParam_ValidateOverlappedRange(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
p := schema.TokenChunkerParam{
- DelimiterMode: "token_size",
+ DelimiterMode: "delimiter",
ChunkTokenSize: 100,
OverlappedPercent: tc.in,
}
diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go
index ad9b9a421c..e920950de1 100644
--- a/internal/ingestion/component/schema/chunker.go
+++ b/internal/ingestion/component/schema/chunker.go
@@ -173,7 +173,7 @@ type ChunkerOutputs struct {
type TokenChunkerParam struct {
// DelimiterMode selects the chunking strategy.
- // Allowed values: "token_size", "delimiter".
+ // Allowed value: "delimiter".
// The single-chunk "one" behavior is provided by the separate
// OneChunker component.
DelimiterMode string `json:"delimiter_mode"`
@@ -246,7 +246,7 @@ func (p TokenChunkerParam) MergeStrategy() MergeStrategy {
// Defaults returns the Python default TokenChunkerParam.
func (TokenChunkerParam) Defaults() TokenChunkerParam {
return TokenChunkerParam{
- DelimiterMode: "token_size",
+ DelimiterMode: "delimiter",
ChunkTokenSize: 512,
Delimiters: []string{"\n"},
OverlappedPercent: 0,
@@ -357,7 +357,7 @@ func (p *TokenChunkerParam) Validate() error {
p.OverlappedPercent = math.Round(v * 100)
}
switch p.DelimiterMode {
- case "token_size", "delimiter":
+ case "delimiter":
default:
return errInvalidValue{Field: "delimiter_mode", Value: p.DelimiterMode}
}
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index 517513f951..d9559b0e66 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -278,8 +278,8 @@ func TestChunkerOutputsJSONRoundTrip(t *testing.T) {
func TestTokenChunkerParamDefaults(t *testing.T) {
p := TokenChunkerParam{}.Defaults()
- if p.DelimiterMode != "token_size" {
- t.Errorf("default delimiter_mode = %q, want token_size", p.DelimiterMode)
+ if p.DelimiterMode != "delimiter" {
+ t.Errorf("default delimiter_mode = %q, want delimiter", p.DelimiterMode)
}
if p.ChunkTokenSize != 512 {
t.Errorf("default chunk_token_size = %d, want 512", p.ChunkTokenSize)
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json
index 805a5df922..09c58c291e 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json
@@ -83,7 +83,7 @@
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
"\n",
"!",
@@ -265,7 +265,7 @@
"form": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
{
"value": "\n"
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
index ff4a3165ba..22d0d6fe7e 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
@@ -161,7 +161,7 @@
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
"\n",
"!",
@@ -391,7 +391,7 @@
"form": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
{
"value": "\n"
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json
index 364bc66b09..db0ab19d38 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json
@@ -157,7 +157,7 @@
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "token_size",
+ "delimiter_mode": "delimiter",
"delimiters": [
"\n",
"!",
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index b1eca5024f..fc483af648 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -40,8 +40,8 @@ import (
var builtinComponentParamsGolden = map[string]string{
"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
+ "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
@@ -51,7 +51,7 @@ var builtinComponentParamsGolden = map[string]string{
"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
"resume": "{\"Extractor:ThreeDrinksAct\": {\"field_name\": \"metadata\", \"frequencyPenaltyEnabled\": true, \"frequency_penalty\": 0.7, \"llm_id\": \"THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW\", \"maxTokensEnabled\": false, \"max_tokens\": 256, \"presencePenaltyEnabled\": true, \"presence_penalty\": 0.4, \"prompts\": [{\"content\": \"Content: {TitleChunker:FlatMiceFix@chunks}\", \"role\": \"user\"}], \"sys_prompt\": \"Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\\n\\nRules:\\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\\n3. Use only these field names:\\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\\n5. Keep values in the same language as the source text whenever possible.\\n6. Remove duplicates and keep only concise, high-value metadata.\\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\\n\\nField guidance:\\n- highest_degree: highest explicit degree level mentioned\\n- degree_levels: all explicit degree levels mentioned\\n- school_names: explicit school, college, or university names\\n- majors: explicit fields of study\\n- graduation_years: explicit graduation years only\\n- work_experience_years: only if explicitly stated\\n- current_job_title: only if explicitly current or most recent\\n- job_titles: explicit role titles\\n- company_names: explicit employer names\\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\\n- industries: explicit industry names only\\n- target_job_titles: explicit desired roles only\\n- target_locations: explicit desired work locations only\\n- skills: concise, core, search-useful skills explicitly mentioned\\n- certificates: explicit certificate names only\\n- awards: explicit award names only\\n- summary_tags: short, high-value tags strictly supported by the content\\n\\nReturn only the extracted metadata. Do not output explanatory text.\", \"temperature\": 0.1, \"temperatureEnabled\": true, \"tenant_llm_id\": 29, \"topPEnabled\": true, \"top_p\": 0.3, \"auto_keywords\": 0, \"auto_questions\": 0, \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}, \"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:FlatMiceFix\": {\"hierarchy\": 1, \"include_heading_content\": false, \"levels\": [[\"^\\\\s*(?i:(?:\\\\d+[\\\\.\\\\)]\\\\s*)?(?:EDUCATION|ACADEMIC\\\\s*BACKGROUND|ACADEMIC\\\\s*HISTORY|EDUCATIONAL\\\\s*BACKGROUND|RELEVANT\\\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\\\s*EXPERIENCE|PROFESSIONAL\\\\s*EXPERIENCE|RELEVANT\\\\s*EXPERIENCE|EMPLOYMENT\\\\s*HISTORY|CAREER\\\\s*HISTORY|INTERNSHIP\\\\s*EXPERIENCE|PROJECTS|PROJECT\\\\s*EXPERIENCE|ACADEMIC\\\\s*PROJECTS|PROFESSIONAL\\\\s*PROJECTS|SKILLS|TECHNICAL\\\\s*SKILLS|CORE\\\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\\\s*OF\\\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\\\s*ACTIVITIES|ACTIVITIES\\\\s*(?:&|AND)\\\\s*SKILLS|INVOLVEMENT|CAMPUS\\\\s*INVOLVEMENT|VOLUNTEER\\\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\\\s*PROFILE|SUMMARY|PROFESSIONAL\\\\s*SUMMARY|CAREER\\\\s*SUMMARY|OBJECTIVE|CAREER\\\\s*OBJECTIVE|PERSONAL\\\\s*INFORMATION|CONTACT\\\\s*INFORMATION|ADDITIONAL\\\\s*INFORMATION|TRAINING))\\\\s*[::]?\\\\s*$\"], [\"^\\\\s*(?:\\\\d+[\\\\.、\\\\)]\\\\s*)?(?:教育背景|教育经历|学历背景|学术背景|技术背景|工作经历|工作经验|实习经历|项目经历|项目经验|科研经历|研究经历|校园经历|实践经历|专业经历|职业经历|技能|专业技能|技能特长|核心技能|技术栈|个人技能|工作技能|职业技能|技能与评价|技能与自我评价|工作技能与自我评价|职业技能与自我评价|证书|资格证书|职业资格|资质证书|获奖情况|获奖经历|荣誉|荣誉奖项|奖项|科研成果|论文发表|发表论文|领导经历|学生工作|校园活动|社团经历|活动经历|志愿经历|志愿服务|社会实践|语言能力|语言|自我评价|个人评价|自我总结|个人总结|个人优势|个人简介|个人信息|基本信息|联系方式|求职意向|应聘意向|职业目标|求职目标|兴趣爱好|兴趣特长|培训经历|其他信息|附加信息)\\\\s*[::]?\\\\s*$\"]], \"method\": \"hierarchy\"}, \"Tokenizer:KindHandsWin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
"table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
- "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",
+ "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",
}
// Per-template test methods. Each resolves default component params from a
diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.py
index a9a9ff7a95..0fce2c18c8 100644
--- a/rag/flow/chunker/token_chunker.py
+++ b/rag/flow/chunker/token_chunker.py
@@ -32,7 +32,7 @@ from rag.nlp import naive_merge
class TokenChunkerParam(ProcessParamBase):
def __init__(self):
super().__init__()
- self.delimiter_mode = "token_size"
+ self.delimiter_mode = "delimiter"
self.chunk_token_size = 512
self.delimiters = ["\n"]
self.overlapped_percent = 0
@@ -41,7 +41,13 @@ class TokenChunkerParam(ProcessParamBase):
self.image_context_size = 0
def check(self):
- self.check_valid_value(self.delimiter_mode, "Delimiter mode abnormal.", ["token_size", "delimiter", "one"])
+ # Backward-compat: "token_size" was removed but is behaviorally identical
+ # to "delimiter" at runtime (both route through the same code path), so
+ # accept and coerce it instead of rejecting legacy configs / pre-fix
+ # frontends. Only genuinely unknown values are rejected.
+ if self.delimiter_mode == "token_size":
+ self.delimiter_mode = "delimiter"
+ self.check_valid_value(self.delimiter_mode, "Delimiter mode abnormal.", ["delimiter", "one"])
if self.delimiters is None:
self.delimiters = []
elif isinstance(self.delimiters, str):
@@ -316,9 +322,7 @@ class TokenChunker(ProcessBase):
self.set_output("chunks", [{"text": payload}] if payload.strip() else [])
self.callback(1, "Done.")
return
- if self._param.delimiter_mode == "delimiter":
- cks = _split_text_by_pattern(payload, delimiter_pattern)
- elif delimiter_pattern:
+ if delimiter_pattern:
cks = _split_text_by_pattern(payload, delimiter_pattern)
else:
cks = naive_merge(
@@ -358,8 +362,14 @@ class TokenChunker(ProcessBase):
self.callback(1, "Done.")
return
- if self._param.delimiter_mode == "delimiter":
- text_chunks = _build_json_chunks(json_result, "")
+ # Both branches start from per-item chunks (no pre-split by the
+ # delimiter pattern). The delimiter branch splits the buffered text
+ # stream while preserving per-segment PDF positions; the no-delimiter
+ # branch merges adjacent text items to chunk_token_size (the removed
+ # "token_size" behaviour, and a parity match with the Go JSON path).
+ text_chunks = _build_json_chunks(json_result, "")
+
+ if delimiter_pattern:
chunks = []
text_buffer = []
text_buffer_pos = []
@@ -372,9 +382,9 @@ class TokenChunker(ProcessBase):
# The delimiter is then applied to the combined text; a segment may
# span across item boundaries (the "\n" glue is not itself a
# delimiter), so each segment carries only the PDF positions of the
- # buffered item(s) whose text contributed to it -- never the union of
- # every item (which previously leaked page-N coordinates into
- # page-M chunks and made all segments share one preview image).
+ # item(s) that contributed to it -- never the union of every item
+ # (which previously leaked page-N coordinates into page-M chunks and
+ # made all segments share one preview image).
parts = []
item_ranges = [] # (start, end) of each buffered item in combined_text
offset = 0
@@ -387,27 +397,24 @@ class TokenChunker(ProcessBase):
offset += 1
combined_text = "".join(parts[:-1]) # drop the trailing glue
- if delimiter_pattern:
- raw = re.split(r"(%s)" % delimiter_pattern, combined_text, flags=re.DOTALL)
- segments = [] # (text, start, end) within combined_text
- pos = 0
- for i in range(0, len(raw), 2):
- seg = raw[i]
- seg_start = pos
- seg_end = pos + len(seg)
- if seg:
- segments.append((seg, seg_start, seg_end))
- pos = seg_end
- if i + 1 < len(raw):
- pos += len(raw[i + 1])
- else:
- segments = [(combined_text, 0, len(combined_text))]
+ raw = re.split(r"(%s)" % delimiter_pattern, combined_text, flags=re.DOTALL)
+ segments = [] # (text, start, end) within combined_text
+ pos = 0
+ for i in range(0, len(raw), 2):
+ seg = raw[i]
+ seg_start = pos
+ seg_end = pos + len(seg)
+ if seg:
+ segments.append((seg, seg_start, seg_end))
+ pos = seg_end
+ if i + 1 < len(raw):
+ pos += len(raw[i + 1])
for text, seg_start, seg_end in segments:
if not text.strip():
continue
seg_pos = []
- for (istart, iend), item_pos in zip(item_ranges, text_buffer_pos):
+ for (istart, iend), item_pos in zip(item_ranges, text_buffer_pos, strict=True):
# A segment overlaps an item when their character ranges
# intersect; collect that item's coordinates.
if seg_start < iend and istart < seg_end:
@@ -436,22 +443,19 @@ class TokenChunker(ProcessBase):
if custom_pattern:
chunks = _split_chunk_docs_by_children(chunks, custom_pattern)
_attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
- await restore_pdf_text_previews(chunks, from_upstream, self._canvas)
- self.set_output("chunks", _finalize_json_chunks(chunks))
- self.callback(1, "Done.")
- return
-
- # Structured JSON input is normalized first, then optionally enriched with
- # media context, and finally merged only when delimiter splitting is inactive.
- chunks = _build_json_chunks(json_result, delimiter_pattern)
- _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
- if self._param.delimiter_mode == "token_size" and not delimiter_pattern:
- chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent)
-
- if custom_pattern:
- chunks = _split_chunk_docs_by_children(chunks, custom_pattern)
+ else:
+ # No active delimiter: merge adjacent text items to chunk_token_size.
+ # This runs on the per-item chunks (NOT a single concatenated chunk),
+ # so the token cap is actually enforced -- matching the previous
+ # "token_size" mode and the Go JSON path. Media chunks break the merge.
+ # Media context is attached on the per-item chunks before merging, as
+ # the removed "token_size" branch did, to preserve context windows.
+ _attach_context_to_media_chunks(text_chunks, self._param.table_context_size, self._param.image_context_size)
+ chunks = _merge_text_chunks_by_token_size(text_chunks, self._param.chunk_token_size, overlapped_percent)
+ if custom_pattern:
+ chunks = _split_chunk_docs_by_children(chunks, custom_pattern)
await restore_pdf_text_previews(chunks, from_upstream, self._canvas)
- cks = _finalize_json_chunks(chunks)
- self.set_output("chunks", cks)
+ self.set_output("chunks", _finalize_json_chunks(chunks))
self.callback(1, "Done.")
+ return
diff --git a/rag/flow/tests/test_token_chunker.py b/rag/flow/tests/test_token_chunker.py
index 8866b53de2..a33732a9c0 100644
--- a/rag/flow/tests/test_token_chunker.py
+++ b/rag/flow/tests/test_token_chunker.py
@@ -55,6 +55,19 @@ def _load_token_chunker_with_stubs():
def __init__(self):
pass
+ def check_valid_value(self, value, msg, allowed):
+ if value not in allowed:
+ raise ValueError(msg)
+
+ def check_positive_integer(self, value, msg):
+ pass
+
+ def check_decimal_float(self, value, msg):
+ pass
+
+ def check_nonnegative_number(self, value, msg):
+ pass
+
class ProcessBase:
def __init__(self, _pipeline, _id, param):
self._pipeline = _pipeline
@@ -277,6 +290,73 @@ def test_json_delimiter_mode_pdf_positions_retained():
assert chunks[0].get("pdf_positions") == [[1, 0, 10, 0, 5], [2, 0, 20, 0, 8]]
+def test_token_size_mode_normalized_to_delimiter():
+ # Backward-compat: the removed "token_size" value must still be accepted by
+ # check() and coerced to "delimiter" (runtime behavior is identical), so
+ # legacy configs / pre-fix frontends don't get rejected. Unknown values are
+ # still rejected.
+ with _load_token_chunker_with_stubs() as token_chunker_module:
+ param = token_chunker_module.TokenChunkerParam()
+ param.delimiter_mode = "token_size"
+ param.check()
+ assert param.delimiter_mode == "delimiter"
+
+ bad = token_chunker_module.TokenChunkerParam()
+ bad.delimiter_mode = "nope"
+ try:
+ bad.check()
+ raise AssertionError("expected check() to reject unknown delimiter_mode")
+ except Exception:
+ pass
+
+
+def test_json_no_delimiter_mode_merges_to_token_cap():
+ # Regression for #17979: with no active (backtick) delimiter, the JSON path
+ # must merge per-item text chunks up to chunk_token_size -- mirroring the old
+ # "token_size" mode and the Go JSON path. Concatenating every item into a
+ # single chunk before the merge would defeat the cap and emit one oversized
+ # chunk. The per-token stub (num_tokens_from_string -> 1) makes the cap easy
+ # to exceed: 12 one-token items under a cap of 5 must yield several chunks.
+ for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5}):
+ kwargs = {
+ "name": "token_chunker",
+ "output_format": "json",
+ "json_result": [{"text": f"item{i}"} for i in range(12)],
+ }
+ asyncio.run(chunker._invoke(**kwargs))
+ chunks = chunker._outputs["chunks"]
+ # 12 one-token items under a cap of 5 must NOT collapse into one chunk.
+ assert len(chunks) > 1, f"cap not enforced: 12 items collapsed to {len(chunks)} chunk(s)"
+ # Each merged chunk holds at most one overflow unit past the cap (<= 6
+ # items); the final output drops tk_nums, so count item markers instead.
+ for c in chunks:
+ assert c["text"].count("item") <= 6, f"chunk exceeds cap: {c['text'].count('item')} items"
+ # No text lost: all 12 items must survive, joined by the "\n" glue.
+ joined = "\n".join(c["text"] for c in chunks)
+ for i in range(12):
+ assert f"item{i}" in joined, f"item{i} dropped from output"
+
+
+def test_json_no_delimiter_mode_media_breaks_merge():
+ # A non-text (media) chunk interleaved between text items must stay as its
+ # own chunk and reset the merge, so text before/after it are sized separately.
+ for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5}):
+ kwargs = {
+ "name": "token_chunker",
+ "output_format": "json",
+ "json_result": [{"text": f"t{i}", "doc_type_kwd": "text"} for i in range(6)]
+ + [{"text": "IMG", "doc_type_kwd": "image", "img_id": "im1"}]
+ + [{"text": f"u{i}", "doc_type_kwd": "text"} for i in range(12)],
+ }
+ asyncio.run(chunker._invoke(**kwargs))
+ chunks = chunker._outputs["chunks"]
+ doc_types = [c["doc_type_kwd"] for c in chunks]
+ # The media chunk is preserved and breaks the text merge.
+ assert doc_types.count("image") == 1, doc_types
+ # Several text chunks on each side of the media boundary.
+ assert doc_types.count("text") > 2, doc_types
+
+
def test_json_delimiter_mode_pdf_positions_per_segment_not_broadcast():
# Regression for #3 (PDF coordinate leak): when consecutive text items from
# different pages are buffered and then split by a custom delimiter, each
@@ -346,11 +426,11 @@ def test_json_delimiter_mode_consecutive_delimiter_keeps_boundary():
assert all("##" not in t for t in texts)
-def test_text_delimiter_mode_token_size_zero_or_one_no_atom_split():
- # token_size=0/1 must not atom-split delimiter segments into 1-token chunks;
- # the delimiter path produces delimiter-boundary chunks regardless of cap.
- for module, chunker in _build_json_chunker({"delimiter_mode": "token_size", "delimiters": ["`|`"]}):
- # text path: delimiter_mode is token_size but a custom delimiter is
+def test_text_delimiter_mode_zero_or_one_no_atom_split():
+ # chunk_token_size=0/1 must not atom-split delimiter segments into 1-token
+ # chunks; the delimiter path produces delimiter-boundary chunks regardless of cap.
+ for module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):
+ # text path: delimiter_mode is delimiter but a custom delimiter is
# present, so the delimiter branch (_split_text_by_pattern) is used.
kwargs = {
"name": "token_chunker",
@@ -362,4 +442,4 @@ def test_text_delimiter_mode_token_size_zero_or_one_no_atom_split():
asyncio.run(chunker._invoke(**kwargs))
chunks = chunker._outputs["chunks"]
texts = [c["text"] for c in chunks]
- assert texts == ["aaa", "bbb", "ccc"], f"token_size={chunk_token_size} atom-split a delimiter segment: {texts}"
+ assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"