Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: Separate the `hierarchy` and `group` configurations for the `title` chunker into distinct fields. (#21013)

balibabu80b277c3aee7
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 2f55d93e0c..9d5d55a517 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -50,13 +50,46 @@ var extractorValidParamKeys = func() map[string]struct{} {
// families) or whose param struct is not part of the schema package
// (GeneralChunker, QAChunker) stay out of this table; Extractor and Compiler
// keep their cpnID-prefixed dynamic whitelists.
+// titleChunkerPersistenceKeys are per-method config keys the frontend
+// persists alongside the runtime params (see transformTitleChunkerParams in
+// web/src/pages/agent/utils.ts). The runtime components only read
+// levels/hierarchy/chunk_token_cap for the active method and never touch
+// these; they are whitelisted so the stored config keeps the Hierarchy and
+// Group configurations independent of each other.
+var titleChunkerPersistenceKeys = map[string]struct{}{
+ "hierarchyRules": {},
+ "groupRules": {},
+ "hierarchyHierarchy": {},
+ "hierarchyGroup": {},
+ "hierarchyChunkTokenCap": {},
+ "groupChunkTokenCap": {},
+}
+
+// unionParamKeys merges key sets into one for componentParamSchemaKeys entries.
+func unionParamKeys(sets ...map[string]struct{}) map[string]struct{} {
+ merged := make(map[string]struct{})
+ for _, set := range sets {
+ for k := range set {
+ merged[k] = struct{}{}
+ }
+ }
+ return merged
+}
+
+// titleFamilyParamKeys is the accepted-key set shared by TitleChunker and
+// ManualChunker: the component schema plus the per-method persistence keys.
+var titleFamilyParamKeys = unionParamKeys(
+ extractJSONTags(schema.TitleChunkerParam{}),
+ titleChunkerPersistenceKeys,
+)
+
var componentParamSchemaKeys = map[string]map[string]struct{}{
- "titlechunker": extractJSONTags(schema.TitleChunkerParam{}),
+ "titlechunker": titleFamilyParamKeys,
// ManualChunker reuses TitleChunkerParam. It pins method=group and is exempt
// from the token cap, so method, chunk_token_cap and include_heading_content
// (hierarchy-only) are accepted here but ignored by the component — the
// operator form omits them for the same reason.
- "manualchunker": extractJSONTags(schema.TitleChunkerParam{}),
+ "manualchunker": titleFamilyParamKeys,
"tokenchunker": extractJSONTags(schema.TokenChunkerParam{}),
"tokenizer": extractJSONTags(schema.TokenizerParam{}),
}
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 1cbedfe29d..b828df7bd8 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -1022,6 +1022,40 @@ func TestCleanComponentParams_TitleFamilyKeepsSchemaParamsWithoutBakedKeys(t *te
}
}
+// TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys locks the persistence
+// contract for the title-chunker family: the frontend stores each method's
+// rules, hierarchy value and token cap under its own key so the dataset
+// config keeps Hierarchy and Group independent; CleanComponentParams must not
+// drop them.
+func TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys(t *testing.T) {
+ dslJSON := titleFamilyDSL(t)
+ overrides := map[string]any{
+ "TitleChunker:StaleSnapshot": map[string]any{
+ "method": "group",
+ "levels": []any{[]any{"第[0-9]+章"}},
+ "hierarchy": 0,
+ "hierarchyRules": []any{map[string]any{"levels": []any{map[string]any{"expression": "^#[^#]"}}}},
+ "groupRules": []any{map[string]any{"levels": []any{map[string]any{"expression": "第[0-9]+章"}}}},
+ "hierarchyHierarchy": "3",
+ "hierarchyGroup": "2",
+ "hierarchyChunkTokenCap": 1024,
+ "groupChunkTokenCap": 2048,
+ },
+ }
+ params, ok := BuildParserConfig(dslJSON, overrides)["TitleChunker:StaleSnapshot"].(map[string]any)
+ if !ok {
+ t.Fatalf("expected TitleChunker params to survive, got %#v", params)
+ }
+ for _, key := range []string{
+ "hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",
+ "hierarchyChunkTokenCap", "groupChunkTokenCap",
+ } {
+ if _, ok := params[key]; !ok {
+ t.Errorf("per-method persistence key %q dropped by BuildParserConfig: %#v", key, params)
+ }
+ }
+}
+
// TestBuildParserConfig_TitleFamilyParamsSurviveBuiltinTemplates locks the
// dataset/document save path for the title-chunker family: both params must
// survive BuildParserConfig for every builtin template that carries a
@@ -1083,8 +1117,17 @@ var operatorFormParamKeys = map[string][]string{
"TitleChunker": {
"method", "levels", "hierarchy", "include_heading_content",
"root_chunk_as_heading", "chunk_token_cap",
+ // Per-method persistence keys: each method's rules, hierarchy value
+ // and token cap are stored under their own key so the dataset config
+ // keeps them independent of the active method (the runtime reads only
+ // levels/hierarchy/chunk_token_cap).
+ "hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",
+ "hierarchyChunkTokenCap", "groupChunkTokenCap",
+ },
+ "ManualChunker": {
+ "method", "levels", "hierarchy", "root_chunk_as_heading",
+ "hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",
},
- "ManualChunker": {"method", "levels", "hierarchy", "root_chunk_as_heading"},
"TokenChunker": {
"chunk_token_size", "overlapped_percent", "delimiters", "delimiter_mode",
"children_delimiters", "enable_children", "table_context_size", "image_context_size",
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json
index 844e039c79..4018bb62c3 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json
@@ -155,6 +155,190 @@
"Article [0-9]+"
]
],
+ "hierarchyRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "groupRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "hierarchyHierarchy": "5",
+ "hierarchyGroup": "0",
"method": "hierarchy"
}
},
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json
index 9854cb6d18..a4d5078b22 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json
@@ -168,6 +168,190 @@
"Article [0-9]+"
]
],
+ "hierarchyRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "groupRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "hierarchyHierarchy": "2",
+ "hierarchyGroup": "0",
"method": "hierarchy"
}
},
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
index 0dbd509e15..cd16bf9d3a 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
@@ -135,6 +135,190 @@
"Article [0-9]+"
]
],
+ "hierarchyRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "groupRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "hierarchyHierarchy": "0",
+ "hierarchyGroup": "0",
"method": "group"
}
},
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json
index 43e15deb1f..30fb3eb720 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json
@@ -116,6 +116,190 @@
"Article [0-9]+"
]
],
+ "hierarchyRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "groupRules": [
+ {
+ "levels": [
+ {
+ "expression": "^#[^#]"
+ },
+ {
+ "expression": "^##[^#]"
+ },
+ {
+ "expression": "^###[^#]"
+ },
+ {
+ "expression": "^####[^#]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+条"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[0-9]+章"
+ },
+ {
+ "expression": "第[0-9]+节"
+ },
+ {
+ "expression": "[0-9]{1,2}[\\. 、]"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"
+ },
+ {
+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+章"
+ },
+ {
+ "expression": "第[零一二三四五六七八九十百0-9]+节"
+ },
+ {
+ "expression": "[零一二三四五六七八九十百]+[ 、]"
+ },
+ {
+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"
+ },
+ {
+ "expression": "[\\((][0-9]{,2}[\\))]"
+ }
+ ]
+ },
+ {
+ "levels": [
+ {
+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"
+ },
+ {
+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"
+ },
+ {
+ "expression": "Section [0-9]+"
+ },
+ {
+ "expression": "Article [0-9]+"
+ }
+ ]
+ }
+ ],
+ "hierarchyHierarchy": "0",
+ "hierarchyGroup": "0",
"method": "group"
}
},
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index 034abe563a..92a326e57f 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -39,13 +39,13 @@ import (
// serialization order inside each entry is irrelevant.
var builtinComponentParamsGolden = map[string]string{
"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
+ "manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
"picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 2367222ec4..a317b7abee 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -360,7 +360,8 @@ export const initialTitleChunkerValues = {
hierarchyGroup: '0',
include_heading_content: false,
root_chunk_as_heading: false,
- chunk_token_cap: 512,
+ hierarchyChunkTokenCap: 512,
+ groupChunkTokenCap: 512,
hierarchyRules: cloneDeep(originalRules),
groupRules: cloneDeep(originalRules),
};
diff --git a/web/src/pages/agent/form/title-chunker-form/hook.ts b/web/src/pages/agent/form/title-chunker-form/hook.ts
index f1becfade5..1d8f2fc6c4 100644
--- a/web/src/pages/agent/form/title-chunker-form/hook.ts
+++ b/web/src/pages/agent/form/title-chunker-form/hook.ts
@@ -1,4 +1,4 @@
-import { cloneDeep } from 'lodash';
+import { cloneDeep, isNil } from 'lodash';
import { useMemo } from 'react';
import { UseFormReturn, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
@@ -55,6 +55,14 @@ function filterEmptyRules(rules: any[]) {
.filter((rule) => rule !== null);
}
+function parseChunkTokenCap(value: unknown): number {
+ if (isNil(value) || value === '') {
+ return 512;
+ }
+ const cap = Number(value);
+ return Number.isFinite(cap) ? cap : 512;
+}
+
function transformApiResponseToForm(
apiData: Record<string, any>,
): TitleChunkerFormSchemaType {
@@ -122,19 +130,18 @@ function transformApiResponseToForm(
}
}
- let chunk_token_cap = apiData.chunk_token_cap;
- if (
- chunk_token_cap === undefined ||
- chunk_token_cap === null ||
- chunk_token_cap === ''
- ) {
- chunk_token_cap = 512;
- } else {
- chunk_token_cap = Number(chunk_token_cap);
- if (!Number.isFinite(chunk_token_cap)) {
- chunk_token_cap = 512;
- }
- }
+ // chunk_token_cap is one backend-facing key read by both methods' runtime
+ // components, but the form keeps each method's cap under its own key
+ // (mirroring the hierarchyRules/groupRules split) so editing one mode never
+ // changes the other. Configs stored before the split carry only the shared
+ // key — seed both methods from it so they behave as before until edited.
+ const sharedChunkTokenCap = parseChunkTokenCap(apiData.chunk_token_cap);
+ const hierarchyChunkTokenCap = parseChunkTokenCap(
+ apiData.hierarchyChunkTokenCap ?? sharedChunkTokenCap,
+ );
+ const groupChunkTokenCap = parseChunkTokenCap(
+ apiData.groupChunkTokenCap ?? sharedChunkTokenCap,
+ );
return {
method,
@@ -142,7 +149,8 @@ function transformApiResponseToForm(
hierarchyGroup,
include_heading_content: Boolean(apiData.include_heading_content),
root_chunk_as_heading: Boolean(apiData.root_chunk_as_heading),
- chunk_token_cap,
+ hierarchyChunkTokenCap,
+ groupChunkTokenCap,
hierarchyRules,
groupRules,
};
diff --git a/web/src/pages/agent/form/title-chunker-form/index.tsx b/web/src/pages/agent/form/title-chunker-form/index.tsx
index a7c8dcd583..8398d80f00 100644
--- a/web/src/pages/agent/form/title-chunker-form/index.tsx
+++ b/web/src/pages/agent/form/title-chunker-form/index.tsx
@@ -66,7 +66,8 @@ export const FormSchema = z.object({
root_chunk_as_heading: z.boolean().optional(),
hierarchyRules: rulesSchema,
groupRules: rulesSchema,
- chunk_token_cap: z.coerce.number().int().min(128).max(8000).optional(),
+ hierarchyChunkTokenCap: z.coerce.number().int().min(128).max(8000).optional(),
+ groupChunkTokenCap: z.coerce.number().int().min(128).max(8000).optional(),
});
export enum TitleChunkerRulesField {
@@ -75,8 +76,15 @@ export enum TitleChunkerRulesField {
}
export type TitleChunkerFormSchemaType = z.infer<typeof FormSchema> & {
- rules: z.infer<typeof rulesSchema>;
- hierarchy: string;
+ // Pre-split keys carried by stored configs; the transforms migrate them
+ // into the split fields above, so they are absent from fresh form values.
+ rules?: z.infer<typeof rulesSchema>;
+ hierarchy?: string;
+ // Backend-facing cap key shared by both methods. The form edits the
+ // per-method split keys; transformTitleChunkerParams resolves the active
+ // method's value back into this one and falls back to it for canvas
+ // snapshots stored before the split existed.
+ chunk_token_cap?: number;
};
type LevelItemProps = {
@@ -286,7 +294,11 @@ const TitleChunkerForm = ({
return;
}
onValuesChange?.(
- omit(values, ['include_heading_content', 'chunk_token_cap']),
+ omit(values, [
+ 'include_heading_content',
+ 'hierarchyChunkTokenCap',
+ 'groupChunkTokenCap',
+ ]),
);
},
[pinMethodGroup, onValuesChange],
@@ -345,8 +357,15 @@ const TitleChunkerForm = ({
</div>
</div>
{!pinMethodGroup && (
+ // Each method edits its own cap field; key={method} remounts the
+ // slider so it displays the active method's value.
<SliderInputFormField
- name="chunk_token_cap"
+ key={method}
+ name={
+ method === TitleChunkerMethod.Group
+ ? 'groupChunkTokenCap'
+ : 'hierarchyChunkTokenCap'
+ }
max={8000}
min={128}
label={t('flow.chunkTokenCap', 'Chunk token cap')}
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index eeacf6f461..067cafdb4a 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -439,21 +439,36 @@ export function transformTitleChunkerParams(
? params.hierarchyGroup
: params.hierarchyHierarchy) ?? params.hierarchy;
+ // The token cap is split per method like rules/hierarchy above; the runtime
+ // components read the shared chunk_token_cap, so resolve the active
+ // method's value into it. Snapshots stored before the split carry only the
+ // shared key — fall back to it. When no cap key exists at all (the manual
+ // chunker strips them), write none.
+ const chunkTokenCap =
+ (params.method === TitleChunkerMethod.Group
+ ? params.groupChunkTokenCap
+ : params.hierarchyChunkTokenCap) ?? params.chunk_token_cap;
+
+ // Persist each method's rules, hierarchy value and token cap under its own
+ // key so the stored config keeps them independent (the shared
+ // `levels`/`hierarchy`/`chunk_token_cap` keys carry only the active
+ // method's values for the backend components). The load side
+ // (`transformTitleChunkerConfigToForm` and `transformApiResponseToForm`)
+ // reads these split keys first and only falls back to migrating the shared
+ // keys when they are absent.
return {
...omit(params, [
- 'hierarchyRules',
- 'groupRules',
- 'hierarchyHierarchy',
- 'hierarchyGroup',
// Legacy pre-split rules field. The active rules have already been
- // resolved (with `rules` as fallback) into `levels` above; keep the
- // serialized params identical to the template's obj.params shape.
+ // resolved (with `rules` as fallback) into `levels` above.
'rules',
]),
method: params.method,
hierarchy: Number(hierarchyValue || 0),
include_heading_content: Boolean(params.include_heading_content),
root_chunk_as_heading: Boolean(params.root_chunk_as_heading),
+ ...(chunkTokenCap === undefined || chunkTokenCap === null
+ ? {}
+ : { chunk_token_cap: Number(chunkTokenCap) }),
levels,
};
}
@@ -725,7 +740,12 @@ export const buildDslComponentsByGraph = (
// by the backend ManualChunker component (manual.go pins method=group,
// ignores the token cap).
params = transformTitleChunkerParams(
- omit(params, ['include_heading_content', 'chunk_token_cap']) as any,
+ omit(params, [
+ 'include_heading_content',
+ 'chunk_token_cap',
+ 'hierarchyChunkTokenCap',
+ 'groupChunkTokenCap',
+ ]) as any,
);
break;
case Operator.Extractor:
diff --git a/web/src/pages/agent/utils/__tests__/build-dsl-components.test.ts b/web/src/pages/agent/utils/__tests__/build-dsl-components.test.ts
index 4c69050d59..58ceb4bbdf 100644
--- a/web/src/pages/agent/utils/__tests__/build-dsl-components.test.ts
+++ b/web/src/pages/agent/utils/__tests__/build-dsl-components.test.ts
@@ -2,7 +2,11 @@
// transforms. Focus: ManualChunker must serialize the form (rules /
// hierarchyGroup) into the backend params shape (levels / numeric hierarchy)
// — the same contract the Manual template's obj.params bakes in and
-// manual.go consumes.
+// manual.go consumes. Each method's rules/hierarchy value/token cap are
+// additionally kept under their own split keys (hierarchyRules/groupRules/
+// hierarchyHierarchy/hierarchyGroup/hierarchyChunkTokenCap/
+// groupChunkTokenCap) so the stored config stays independent of the active
+// method; the backend reads only levels/hierarchy and ignores the rest.
import { Operator } from '../../constant';
import { buildDslComponentsByGraph } from '../../utils';
@@ -29,7 +33,11 @@ describe('buildDslComponentsByGraph ManualChunker params', () => {
rules: [{ levels: [{ expression: '^#[^#]' }] }],
include_heading_content: true,
root_chunk_as_heading: true,
+ // The manual chunker ignores the token cap, so neither the legacy
+ // shared key nor the per-method split keys may leak into the params.
chunk_token_cap: 512,
+ hierarchyChunkTokenCap: 512,
+ groupChunkTokenCap: 512,
outputs: { chunks: { type: 'Array<Object>', value: [] } },
},
},
@@ -46,13 +54,22 @@ describe('buildDslComponentsByGraph ManualChunker params', () => {
expect(params.hierarchy).toBe(3);
expect(params.levels).toEqual([['第[0-9]+章'], ['第[0-9]+节']]);
- // UI-only fields must not leak into the backend params.
- expect(params.groupRules).toBeUndefined();
- expect(params.hierarchyRules).toBeUndefined();
+ // The per-method split fields persist alongside the shared
+ // levels/hierarchy keys so Hierarchy and Group configs stay independent.
+ expect(params.hierarchyRules).toEqual([
+ { levels: [{ expression: '^#[^#]' }] },
+ ]);
+ expect(params.groupRules).toEqual([
+ { levels: [{ expression: '第[0-9]+章' }] },
+ { levels: [{ expression: '第[0-9]+节' }] },
+ ]);
+ expect(params.hierarchyHierarchy).toBe('0');
+ expect(params.hierarchyGroup).toBe('3');
+ // The legacy pre-split rules field must not leak (levels carries it).
expect(params.rules).toBeUndefined();
- expect(params.hierarchyHierarchy).toBeUndefined();
- expect(params.hierarchyGroup).toBeUndefined();
expect(params.chunk_token_cap).toBeUndefined();
+ expect(params.hierarchyChunkTokenCap).toBeUndefined();
+ expect(params.groupChunkTokenCap).toBeUndefined();
// include_heading_content is stripped before the transform but the
// transform unconditionally re-adds it as false — the backend
// (manual.go) pins method=group and ignores it either way.
diff --git a/web/src/utils/__tests__/pipeline-operator.test.ts b/web/src/utils/__tests__/pipeline-operator.test.ts
index ba2cb810b1..e6f14e948b 100644
--- a/web/src/utils/__tests__/pipeline-operator.test.ts
+++ b/web/src/utils/__tests__/pipeline-operator.test.ts
@@ -263,3 +263,107 @@ describe('buildOperatorNode dataset-level metadata precedence', () => {
expect(form).not.toHaveProperty('metadata');
});
});
+
+describe('TitleChunker per-method config round trip', () => {
+ it('keeps hierarchy and group rules independent across load and save', () => {
+ const form = transformApiConfigToForm('TitleChunker', {
+ method: 'hierarchy',
+ hierarchy: 3,
+ levels: [['^#[^#]']],
+ chunk_token_cap: 1024,
+ hierarchyHierarchy: '3',
+ hierarchyGroup: '0',
+ hierarchyRules: [{ levels: [{ expression: '^#[^#]' }] }],
+ groupRules: [{ levels: [{ expression: '第[0-9]+章' }] }],
+ });
+ expect(form.hierarchyRules).toEqual([
+ { levels: [{ expression: '^#[^#]' }] },
+ ]);
+ expect(form.groupRules).toEqual([
+ { levels: [{ expression: '第[0-9]+章' }] },
+ ]);
+ expect(form.hierarchyHierarchy).toBe('3');
+ expect(form.hierarchyGroup).toBe('0');
+ // The shared backend-facing cap seeds both per-method fields on load.
+ expect(form.hierarchyChunkTokenCap).toBe(1024);
+ expect(form.groupChunkTokenCap).toBe(1024);
+ expect(form).not.toHaveProperty('levels');
+ expect(form).not.toHaveProperty('hierarchy');
+ expect(form).not.toHaveProperty('chunk_token_cap');
+
+ // Switch to group mode, edit only the group rules and the group cap, and
+ // save: the shared levels/hierarchy/chunk_token_cap keys carry the active
+ // method's values while the hierarchy config survives untouched under its
+ // own keys.
+ const api = transformFormConfigToApi('TitleChunker', {
+ ...form,
+ method: 'group',
+ hierarchyGroup: '2',
+ groupRules: [
+ { levels: [{ expression: '第[0-9]+章' }] },
+ { levels: [{ expression: '第[0-9]+节' }] },
+ ],
+ groupChunkTokenCap: 2048,
+ });
+ expect(api.method).toBe('group');
+ expect(api.levels).toEqual([['第[0-9]+章'], ['第[0-9]+节']]);
+ expect(api.hierarchy).toBe(2);
+ expect(api.chunk_token_cap).toBe(2048);
+ expect(api.hierarchyRules).toEqual([
+ { levels: [{ expression: '^#[^#]' }] },
+ ]);
+ expect(api.groupRules).toEqual([
+ { levels: [{ expression: '第[0-9]+章' }] },
+ { levels: [{ expression: '第[0-9]+节' }] },
+ ]);
+ expect(api.hierarchyHierarchy).toBe('3');
+ expect(api.hierarchyGroup).toBe('2');
+ expect(api.hierarchyChunkTokenCap).toBe(1024);
+ expect(api.groupChunkTokenCap).toBe(2048);
+ expect(api).not.toHaveProperty('rules');
+ });
+
+ it('resolves the shared chunk_token_cap from the active method on save', () => {
+ const base = {
+ hierarchyHierarchy: '3',
+ hierarchyGroup: '0',
+ hierarchyRules: [{ levels: [{ expression: '^#[^#]' }] }],
+ groupRules: [{ levels: [{ expression: '第[0-9]+章' }] }],
+ hierarchyChunkTokenCap: 1024,
+ groupChunkTokenCap: 2048,
+ };
+ expect(
+ transformFormConfigToApi('TitleChunker', {
+ ...base,
+ method: 'hierarchy',
+ }).chunk_token_cap,
+ ).toBe(1024);
+ expect(
+ transformFormConfigToApi('TitleChunker', { ...base, method: 'group' })
+ .chunk_token_cap,
+ ).toBe(2048);
+ });
+
+ it('falls back to a stored shared chunk_token_cap when split keys are absent', () => {
+ // Canvas snapshots saved before the split carry only the shared key; the
+ // save path must preserve it instead of dropping the configured cap.
+ const api = transformFormConfigToApi('TitleChunker', {
+ method: 'hierarchy',
+ hierarchyHierarchy: '3',
+ hierarchyRules: [{ levels: [{ expression: '^#[^#]' }] }],
+ groupRules: [{ levels: [{ expression: '第[0-9]+章' }] }],
+ chunk_token_cap: 777,
+ });
+ expect(api.chunk_token_cap).toBe(777);
+ });
+
+ it('migrates a legacy levels-only config into both method rule sets', () => {
+ const form = transformApiConfigToForm('TitleChunker', {
+ method: 'hierarchy',
+ hierarchy: 2,
+ levels: [['^##']],
+ });
+ expect(form.hierarchyRules).toEqual([{ levels: [{ expression: '^##' }] }]);
+ expect(form.groupRules).toEqual([{ levels: [{ expression: '^##' }] }]);
+ });
+});
diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.ts
index 6d88b7ed40..758893b9ec 100644
--- a/web/src/utils/pipeline-operator.ts
+++ b/web/src/utils/pipeline-operator.ts
@@ -286,8 +286,8 @@ function transformGeneralChunkerConfigToForm(
/**
* Converts TitleChunker config from API/DSL format to form format.
- * DSL: { method: "hierarchy", hierarchy: "3", levels: [...], include_heading_content, root_chunk_as_heading }
- * Form: { method, hierarchyHierarchy, hierarchyGroup, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules }
+ * DSL: { method: "hierarchy", hierarchy: "3", levels: [...], chunk_token_cap, include_heading_content, root_chunk_as_heading }
+ * Form: { method, hierarchyHierarchy, hierarchyGroup, hierarchyChunkTokenCap, groupChunkTokenCap, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules }
*/
function transformTitleChunkerConfigToForm(
config: Record<string, any> | undefined,
@@ -319,10 +319,22 @@ function transformTitleChunkerConfigToForm(
result.groupRules = transformLevelsToRules(config.levels);
}
+ // Split the backend-facing shared `chunk_token_cap` into the per-method
+ // fields the form edits (mirroring the rules/hierarchy split above).
+ if (config.chunk_token_cap !== undefined) {
+ if (result.hierarchyChunkTokenCap === undefined) {
+ result.hierarchyChunkTokenCap = config.chunk_token_cap;
+ }
+ if (result.groupChunkTokenCap === undefined) {
+ result.groupChunkTokenCap = config.chunk_token_cap;
+ }
+ }
+
// Clean up DSL-only fields
delete result.hierarchy;
delete result.levels;
delete result.rules;
+ delete result.chunk_token_cap;
return result;
}
@@ -439,7 +451,12 @@ export function transformFormConfigToApi(
// the backend ManualChunker component (manual.go pins method=group,
// ignores the token cap).
return transformTitleChunkerParams(
- omit(config, ['include_heading_content', 'chunk_token_cap']) as any,
+ omit(config, [
+ 'include_heading_content',
+ 'chunk_token_cap',
+ 'hierarchyChunkTokenCap',
+ 'groupChunkTokenCap',
+ ]) as any,
);
default:
return config;