RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: Separate the `hierarchy` and `group` configurations for the `title` chunker into distinct fields. (#21013)
balibabu80b277c3aee7
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 2f55d93e0c..9d5d55a517 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -50,13 +50,46 @@ var extractorValidParamKeys = func() map[string]struct{} {// families) or whose param struct is not part of the schema package// (GeneralChunker, QAChunker) stay out of this table; Extractor and Compiler// keep their cpnID-prefixed dynamic whitelists.+// titleChunkerPersistenceKeys are per-method config keys the frontend+// persists alongside the runtime params (see transformTitleChunkerParams in+// web/src/pages/agent/utils.ts). The runtime components only read+// levels/hierarchy/chunk_token_cap for the active method and never touch+// these; they are whitelisted so the stored config keeps the Hierarchy and+// Group configurations independent of each other.+var titleChunkerPersistenceKeys = map[string]struct{}{+ "hierarchyRules": {},+ "groupRules": {},+ "hierarchyHierarchy": {},+ "hierarchyGroup": {},+ "hierarchyChunkTokenCap": {},+ "groupChunkTokenCap": {},+}++// unionParamKeys merges key sets into one for componentParamSchemaKeys entries.+func unionParamKeys(sets ...map[string]struct{}) map[string]struct{} {+ merged := make(map[string]struct{})+ for _, set := range sets {+ for k := range set {+ merged[k] = struct{}{}+ }+ }+ return merged+}++// titleFamilyParamKeys is the accepted-key set shared by TitleChunker and+// ManualChunker: the component schema plus the per-method persistence keys.+var titleFamilyParamKeys = unionParamKeys(+ extractJSONTags(schema.TitleChunkerParam{}),+ titleChunkerPersistenceKeys,+)+var componentParamSchemaKeys = map[string]map[string]struct{}{- "titlechunker": extractJSONTags(schema.TitleChunkerParam{}),+ "titlechunker": titleFamilyParamKeys,// ManualChunker reuses TitleChunkerParam. It pins method=group and is exempt// from the token cap, so method, chunk_token_cap and include_heading_content// (hierarchy-only) are accepted here but ignored by the component — the// operator form omits them for the same reason.- "manualchunker": extractJSONTags(schema.TitleChunkerParam{}),+ "manualchunker": titleFamilyParamKeys,"tokenchunker": extractJSONTags(schema.TokenChunkerParam{}),"tokenizer": extractJSONTags(schema.TokenizerParam{}),}diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex 1cbedfe29d..b828df7bd8 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -1022,6 +1022,40 @@ func TestCleanComponentParams_TitleFamilyKeepsSchemaParamsWithoutBakedKeys(t *te}}+// TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys locks the persistence+// contract for the title-chunker family: the frontend stores each method's+// rules, hierarchy value and token cap under its own key so the dataset+// config keeps Hierarchy and Group independent; CleanComponentParams must not+// drop them.+func TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys(t *testing.T) {+ dslJSON := titleFamilyDSL(t)+ overrides := map[string]any{+ "TitleChunker:StaleSnapshot": map[string]any{+ "method": "group",+ "levels": []any{[]any{"第[0-9]+章"}},+ "hierarchy": 0,+ "hierarchyRules": []any{map[string]any{"levels": []any{map[string]any{"expression": "^#[^#]"}}}},+ "groupRules": []any{map[string]any{"levels": []any{map[string]any{"expression": "第[0-9]+章"}}}},+ "hierarchyHierarchy": "3",+ "hierarchyGroup": "2",+ "hierarchyChunkTokenCap": 1024,+ "groupChunkTokenCap": 2048,+ },+ }+ params, ok := BuildParserConfig(dslJSON, overrides)["TitleChunker:StaleSnapshot"].(map[string]any)+ if !ok {+ t.Fatalf("expected TitleChunker params to survive, got %#v", params)+ }+ for _, key := range []string{+ "hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",+ "hierarchyChunkTokenCap", "groupChunkTokenCap",+ } {+ if _, ok := params[key]; !ok {+ t.Errorf("per-method persistence key %q dropped by BuildParserConfig: %#v", key, params)+ }+ }+}+// TestBuildParserConfig_TitleFamilyParamsSurviveBuiltinTemplates locks the// dataset/document save path for the title-chunker family: both params must// survive BuildParserConfig for every builtin template that carries a@@ -1083,8 +1117,17 @@ var operatorFormParamKeys = map[string][]string{"TitleChunker": {"method", "levels", "hierarchy", "include_heading_content","root_chunk_as_heading", "chunk_token_cap",+ // Per-method persistence keys: each method's rules, hierarchy value+ // and token cap are stored under their own key so the dataset config+ // keeps them independent of the active method (the runtime reads only+ // levels/hierarchy/chunk_token_cap).+ "hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",+ "hierarchyChunkTokenCap", "groupChunkTokenCap",+ },+ "ManualChunker": {+ "method", "levels", "hierarchy", "root_chunk_as_heading",+ "hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",},- "ManualChunker": {"method", "levels", "hierarchy", "root_chunk_as_heading"},"TokenChunker": {"chunk_token_size", "overlapped_percent", "delimiters", "delimiter_mode","children_delimiters", "enable_children", "table_context_size", "image_context_size",diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json b/internal/ingestion/pipeline/template/ingestion_pipeline_book.jsonindex 844e039c79..4018bb62c3 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_book.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_book.json@@ -155,6 +155,190 @@"Article [0-9]+"]],+ "hierarchyRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "groupRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "hierarchyHierarchy": "5",+ "hierarchyGroup": "0","method": "hierarchy"}},diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.jsonindex 9854cb6d18..a4d5078b22 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_laws.json@@ -168,6 +168,190 @@"Article [0-9]+"]],+ "hierarchyRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "groupRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "hierarchyHierarchy": "2",+ "hierarchyGroup": "0","method": "hierarchy"}},diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.jsonindex 0dbd509e15..cd16bf9d3a 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json@@ -135,6 +135,190 @@"Article [0-9]+"]],+ "hierarchyRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "groupRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "hierarchyHierarchy": "0",+ "hierarchyGroup": "0","method": "group"}},diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.jsonindex 43e15deb1f..30fb3eb720 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_paper.json@@ -116,6 +116,190 @@"Article [0-9]+"]],+ "hierarchyRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "groupRules": [+ {+ "levels": [+ {+ "expression": "^#[^#]"+ },+ {+ "expression": "^##[^#]"+ },+ {+ "expression": "^###[^#]"+ },+ {+ "expression": "^####[^#]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+(分?编|部分)"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+条"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[0-9]+章"+ },+ {+ "expression": "第[0-9]+节"+ },+ {+ "expression": "[0-9]{1,2}[\\. 、]"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])"+ },+ {+ "expression": "[0-9]{1,2}\\.[0-9]{1,2}\\.[0-9]{1,2}"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "第[零一二三四五六七八九十百0-9]+章"+ },+ {+ "expression": "第[零一二三四五六七八九十百0-9]+节"+ },+ {+ "expression": "[零一二三四五六七八九十百]+[ 、]"+ },+ {+ "expression": "[\\((][零一二三四五六七八九十百]+[\\))]"+ },+ {+ "expression": "[\\((][0-9]{,2}[\\))]"+ }+ ]+ },+ {+ "levels": [+ {+ "expression": "PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)"+ },+ {+ "expression": "Chapter (I+V?|VI*|XI|IX|X)"+ },+ {+ "expression": "Section [0-9]+"+ },+ {+ "expression": "Article [0-9]+"+ }+ ]+ }+ ],+ "hierarchyHierarchy": "0",+ "hierarchyGroup": "0","method": "group"}},diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex 034abe563a..92a326e57f 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -39,13 +39,13 @@ import (// serialization order inside each entry is irrelevant.var builtinComponentParamsGolden = map[string]string{"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",- "book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}","email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",- "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",- "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",+ "manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}","one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",- "paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "paper": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"pdf\":{\"enable_multi_column\":true,\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"TitleChunker:SparklySchoolsTravel\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:GreatCarsWash\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}","picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex 2367222ec4..a317b7abee 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -360,7 +360,8 @@ export const initialTitleChunkerValues = {hierarchyGroup: '0',include_heading_content: false,root_chunk_as_heading: false,- chunk_token_cap: 512,+ hierarchyChunkTokenCap: 512,+ groupChunkTokenCap: 512,hierarchyRules: cloneDeep(originalRules),groupRules: cloneDeep(originalRules),};diff --git a/web/src/pages/agent/form/title-chunker-form/hook.ts b/web/src/pages/agent/form/title-chunker-form/hook.tsindex f1becfade5..1d8f2fc6c4 100644--- a/web/src/pages/agent/form/title-chunker-form/hook.ts+++ b/web/src/pages/agent/form/title-chunker-form/hook.ts@@ -1,4 +1,4 @@-import { cloneDeep } from 'lodash';+import { cloneDeep, isNil } from 'lodash';import { useMemo } from 'react';import { UseFormReturn, useWatch } from 'react-hook-form';import { useTranslation } from 'react-i18next';@@ -55,6 +55,14 @@ function filterEmptyRules(rules: any[]) {.filter((rule) => rule !== null);}+function parseChunkTokenCap(value: unknown): number {+ if (isNil(value) || value === '') {+ return 512;+ }+ const cap = Number(value);+ return Number.isFinite(cap) ? cap : 512;+}+function transformApiResponseToForm(apiData: Record<string, any>,): TitleChunkerFormSchemaType {@@ -122,19 +130,18 @@ function transformApiResponseToForm(}}- let chunk_token_cap = apiData.chunk_token_cap;- if (- chunk_token_cap === undefined ||- chunk_token_cap === null ||- chunk_token_cap === ''- ) {- chunk_token_cap = 512;- } else {- chunk_token_cap = Number(chunk_token_cap);- if (!Number.isFinite(chunk_token_cap)) {- chunk_token_cap = 512;- }- }+ // chunk_token_cap is one backend-facing key read by both methods' runtime+ // components, but the form keeps each method's cap under its own key+ // (mirroring the hierarchyRules/groupRules split) so editing one mode never+ // changes the other. Configs stored before the split carry only the shared+ // key — seed both methods from it so they behave as before until edited.+ const sharedChunkTokenCap = parseChunkTokenCap(apiData.chunk_token_cap);+ const hierarchyChunkTokenCap = parseChunkTokenCap(+ apiData.hierarchyChunkTokenCap ?? sharedChunkTokenCap,+ );+ const groupChunkTokenCap = parseChunkTokenCap(+ apiData.groupChunkTokenCap ?? sharedChunkTokenCap,+ );return {method,@@ -142,7 +149,8 @@ function transformApiResponseToForm(hierarchyGroup,include_heading_content: Boolean(apiData.include_heading_content),root_chunk_as_heading: Boolean(apiData.root_chunk_as_heading),- chunk_token_cap,+ hierarchyChunkTokenCap,+ groupChunkTokenCap,hierarchyRules,groupRules,};diff --git a/web/src/pages/agent/form/title-chunker-form/index.tsx b/web/src/pages/agent/form/title-chunker-form/index.tsxindex a7c8dcd583..8398d80f00 100644--- a/web/src/pages/agent/form/title-chunker-form/index.tsx+++ b/web/src/pages/agent/form/title-chunker-form/index.tsx@@ -66,7 +66,8 @@ export const FormSchema = z.object({root_chunk_as_heading: z.boolean().optional(),hierarchyRules: rulesSchema,groupRules: rulesSchema,- chunk_token_cap: z.coerce.number().int().min(128).max(8000).optional(),+ hierarchyChunkTokenCap: z.coerce.number().int().min(128).max(8000).optional(),+ groupChunkTokenCap: z.coerce.number().int().min(128).max(8000).optional(),});export enum TitleChunkerRulesField {@@ -75,8 +76,15 @@ export enum TitleChunkerRulesField {}export type TitleChunkerFormSchemaType = z.infer<typeof FormSchema> & {- rules: z.infer<typeof rulesSchema>;- hierarchy: string;+ // Pre-split keys carried by stored configs; the transforms migrate them+ // into the split fields above, so they are absent from fresh form values.+ rules?: z.infer<typeof rulesSchema>;+ hierarchy?: string;+ // Backend-facing cap key shared by both methods. The form edits the+ // per-method split keys; transformTitleChunkerParams resolves the active+ // method's value back into this one and falls back to it for canvas+ // snapshots stored before the split existed.+ chunk_token_cap?: number;};type LevelItemProps = {@@ -286,7 +294,11 @@ const TitleChunkerForm = ({return;}onValuesChange?.(- omit(values, ['include_heading_content', 'chunk_token_cap']),+ omit(values, [+ 'include_heading_content',+ 'hierarchyChunkTokenCap',+ 'groupChunkTokenCap',+ ]),);},[pinMethodGroup, onValuesChange],@@ -345,8 +357,15 @@ const TitleChunkerForm = ({</div></div>{!pinMethodGroup && (+ // Each method edits its own cap field; key={method} remounts the+ // slider so it displays the active method's value.<SliderInputFormField- name="chunk_token_cap"+ key={method}+ name={+ method === TitleChunkerMethod.Group+ ? 'groupChunkTokenCap'+ : 'hierarchyChunkTokenCap'+ }max={8000}min={128}label={t('flow.chunkTokenCap', 'Chunk token cap')}diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex eeacf6f461..067cafdb4a 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -439,21 +439,36 @@ export function transformTitleChunkerParams(? params.hierarchyGroup: params.hierarchyHierarchy) ?? params.hierarchy;+ // The token cap is split per method like rules/hierarchy above; the runtime+ // components read the shared chunk_token_cap, so resolve the active+ // method's value into it. Snapshots stored before the split carry only the+ // shared key — fall back to it. When no cap key exists at all (the manual+ // chunker strips them), write none.+ const chunkTokenCap =+ (params.method === TitleChunkerMethod.Group+ ? params.groupChunkTokenCap+ : params.hierarchyChunkTokenCap) ?? params.chunk_token_cap;++ // Persist each method's rules, hierarchy value and token cap under its own+ // key so the stored config keeps them independent (the shared+ // `levels`/`hierarchy`/`chunk_token_cap` keys carry only the active+ // method's values for the backend components). The load side+ // (`transformTitleChunkerConfigToForm` and `transformApiResponseToForm`)+ // reads these split keys first and only falls back to migrating the shared+ // keys when they are absent.return {...omit(params, [- 'hierarchyRules',- 'groupRules',- 'hierarchyHierarchy',- 'hierarchyGroup',// Legacy pre-split rules field. The active rules have already been- // resolved (with `rules` as fallback) into `levels` above; keep the- // serialized params identical to the template's obj.params shape.+ // resolved (with `rules` as fallback) into `levels` above.'rules',]),method: params.method,hierarchy: Number(hierarchyValue || 0),include_heading_content: Boolean(params.include_heading_content),root_chunk_as_heading: Boolean(params.root_chunk_as_heading),+ ...(chunkTokenCap === undefined || chunkTokenCap === null+ ? {}+ : { chunk_token_cap: Number(chunkTokenCap) }),levels,};}@@ -725,7 +740,12 @@ export const buildDslComponentsByGraph = (// by the backend ManualChunker component (manual.go pins method=group,// ignores the token cap).params = transformTitleChunkerParams(- omit(params, ['include_heading_content', 'chunk_token_cap']) as any,+ omit(params, [+ 'include_heading_content',+ 'chunk_token_cap',+ 'hierarchyChunkTokenCap',+ 'groupChunkTokenCap',+ ]) as any,);break;case Operator.Extractor:diff --git a/web/src/pages/agent/utils/__tests__/build-dsl-components.test.ts b/web/src/pages/agent/utils/__tests__/build-dsl-components.test.tsindex 4c69050d59..58ceb4bbdf 100644--- a/web/src/pages/agent/utils/__tests__/build-dsl-components.test.ts+++ b/web/src/pages/agent/utils/__tests__/build-dsl-components.test.ts@@ -2,7 +2,11 @@// transforms. Focus: ManualChunker must serialize the form (rules /// hierarchyGroup) into the backend params shape (levels / numeric hierarchy)// — the same contract the Manual template's obj.params bakes in and-// manual.go consumes.+// manual.go consumes. Each method's rules/hierarchy value/token cap are+// additionally kept under their own split keys (hierarchyRules/groupRules/+// hierarchyHierarchy/hierarchyGroup/hierarchyChunkTokenCap/+// groupChunkTokenCap) so the stored config stays independent of the active+// method; the backend reads only levels/hierarchy and ignores the rest.import { Operator } from '../../constant';import { buildDslComponentsByGraph } from '../../utils';@@ -29,7 +33,11 @@ describe('buildDslComponentsByGraph ManualChunker params', () => {rules: [{ levels: [{ expression: '^#[^#]' }] }],include_heading_content: true,root_chunk_as_heading: true,+ // The manual chunker ignores the token cap, so neither the legacy+ // shared key nor the per-method split keys may leak into the params.chunk_token_cap: 512,+ hierarchyChunkTokenCap: 512,+ groupChunkTokenCap: 512,outputs: { chunks: { type: 'Array<Object>', value: [] } },},},@@ -46,13 +54,22 @@ describe('buildDslComponentsByGraph ManualChunker params', () => {expect(params.hierarchy).toBe(3);expect(params.levels).toEqual([['第[0-9]+章'], ['第[0-9]+节']]);- // UI-only fields must not leak into the backend params.- expect(params.groupRules).toBeUndefined();- expect(params.hierarchyRules).toBeUndefined();+ // The per-method split fields persist alongside the shared+ // levels/hierarchy keys so Hierarchy and Group configs stay independent.+ expect(params.hierarchyRules).toEqual([+ { levels: [{ expression: '^#[^#]' }] },+ ]);+ expect(params.groupRules).toEqual([+ { levels: [{ expression: '第[0-9]+章' }] },+ { levels: [{ expression: '第[0-9]+节' }] },+ ]);+ expect(params.hierarchyHierarchy).toBe('0');+ expect(params.hierarchyGroup).toBe('3');+ // The legacy pre-split rules field must not leak (levels carries it).expect(params.rules).toBeUndefined();- expect(params.hierarchyHierarchy).toBeUndefined();- expect(params.hierarchyGroup).toBeUndefined();expect(params.chunk_token_cap).toBeUndefined();+ expect(params.hierarchyChunkTokenCap).toBeUndefined();+ expect(params.groupChunkTokenCap).toBeUndefined();// include_heading_content is stripped before the transform but the// transform unconditionally re-adds it as false — the backend// (manual.go) pins method=group and ignores it either way.diff --git a/web/src/utils/__tests__/pipeline-operator.test.ts b/web/src/utils/__tests__/pipeline-operator.test.tsindex ba2cb810b1..e6f14e948b 100644--- a/web/src/utils/__tests__/pipeline-operator.test.ts+++ b/web/src/utils/__tests__/pipeline-operator.test.ts@@ -263,3 +263,107 @@ describe('buildOperatorNode dataset-level metadata precedence', () => {expect(form).not.toHaveProperty('metadata');});});++describe('TitleChunker per-method config round trip', () => {+ it('keeps hierarchy and group rules independent across load and save', () => {+ const form = transformApiConfigToForm('TitleChunker', {+ method: 'hierarchy',+ hierarchy: 3,+ levels: [['^#[^#]']],+ chunk_token_cap: 1024,+ hierarchyHierarchy: '3',+ hierarchyGroup: '0',+ hierarchyRules: [{ levels: [{ expression: '^#[^#]' }] }],+ groupRules: [{ levels: [{ expression: '第[0-9]+章' }] }],+ });+ expect(form.hierarchyRules).toEqual([+ { levels: [{ expression: '^#[^#]' }] },+ ]);+ expect(form.groupRules).toEqual([+ { levels: [{ expression: '第[0-9]+章' }] },+ ]);+ expect(form.hierarchyHierarchy).toBe('3');+ expect(form.hierarchyGroup).toBe('0');+ // The shared backend-facing cap seeds both per-method fields on load.+ expect(form.hierarchyChunkTokenCap).toBe(1024);+ expect(form.groupChunkTokenCap).toBe(1024);+ expect(form).not.toHaveProperty('levels');+ expect(form).not.toHaveProperty('hierarchy');+ expect(form).not.toHaveProperty('chunk_token_cap');++ // Switch to group mode, edit only the group rules and the group cap, and+ // save: the shared levels/hierarchy/chunk_token_cap keys carry the active+ // method's values while the hierarchy config survives untouched under its+ // own keys.+ const api = transformFormConfigToApi('TitleChunker', {+ ...form,+ method: 'group',+ hierarchyGroup: '2',+ groupRules: [+ { levels: [{ expression: '第[0-9]+章' }] },+ { levels: [{ expression: '第[0-9]+节' }] },+ ],+ groupChunkTokenCap: 2048,+ });+ expect(api.method).toBe('group');+ expect(api.levels).toEqual([['第[0-9]+章'], ['第[0-9]+节']]);+ expect(api.hierarchy).toBe(2);+ expect(api.chunk_token_cap).toBe(2048);+ expect(api.hierarchyRules).toEqual([+ { levels: [{ expression: '^#[^#]' }] },+ ]);+ expect(api.groupRules).toEqual([+ { levels: [{ expression: '第[0-9]+章' }] },+ { levels: [{ expression: '第[0-9]+节' }] },+ ]);+ expect(api.hierarchyHierarchy).toBe('3');+ expect(api.hierarchyGroup).toBe('2');+ expect(api.hierarchyChunkTokenCap).toBe(1024);+ expect(api.groupChunkTokenCap).toBe(2048);+ expect(api).not.toHaveProperty('rules');+ });++ it('resolves the shared chunk_token_cap from the active method on save', () => {+ const base = {+ hierarchyHierarchy: '3',+ hierarchyGroup: '0',+ hierarchyRules: [{ levels: [{ expression: '^#[^#]' }] }],+ groupRules: [{ levels: [{ expression: '第[0-9]+章' }] }],+ hierarchyChunkTokenCap: 1024,+ groupChunkTokenCap: 2048,+ };+ expect(+ transformFormConfigToApi('TitleChunker', {+ ...base,+ method: 'hierarchy',+ }).chunk_token_cap,+ ).toBe(1024);+ expect(+ transformFormConfigToApi('TitleChunker', { ...base, method: 'group' })+ .chunk_token_cap,+ ).toBe(2048);+ });++ it('falls back to a stored shared chunk_token_cap when split keys are absent', () => {+ // Canvas snapshots saved before the split carry only the shared key; the+ // save path must preserve it instead of dropping the configured cap.+ const api = transformFormConfigToApi('TitleChunker', {+ method: 'hierarchy',+ hierarchyHierarchy: '3',+ hierarchyRules: [{ levels: [{ expression: '^#[^#]' }] }],+ groupRules: [{ levels: [{ expression: '第[0-9]+章' }] }],+ chunk_token_cap: 777,+ });+ expect(api.chunk_token_cap).toBe(777);+ });++ it('migrates a legacy levels-only config into both method rule sets', () => {+ const form = transformApiConfigToForm('TitleChunker', {+ method: 'hierarchy',+ hierarchy: 2,+ levels: [['^##']],+ });+ expect(form.hierarchyRules).toEqual([{ levels: [{ expression: '^##' }] }]);+ expect(form.groupRules).toEqual([{ levels: [{ expression: '^##' }] }]);+ });+});diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.tsindex 6d88b7ed40..758893b9ec 100644--- a/web/src/utils/pipeline-operator.ts+++ b/web/src/utils/pipeline-operator.ts@@ -286,8 +286,8 @@ function transformGeneralChunkerConfigToForm(/*** Converts TitleChunker config from API/DSL format to form format.- * DSL: { method: "hierarchy", hierarchy: "3", levels: [...], include_heading_content, root_chunk_as_heading }- * Form: { method, hierarchyHierarchy, hierarchyGroup, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules }+ * DSL: { method: "hierarchy", hierarchy: "3", levels: [...], chunk_token_cap, include_heading_content, root_chunk_as_heading }+ * Form: { method, hierarchyHierarchy, hierarchyGroup, hierarchyChunkTokenCap, groupChunkTokenCap, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules }*/function transformTitleChunkerConfigToForm(config: Record<string, any> | undefined,@@ -319,10 +319,22 @@ function transformTitleChunkerConfigToForm(result.groupRules = transformLevelsToRules(config.levels);}+ // Split the backend-facing shared `chunk_token_cap` into the per-method+ // fields the form edits (mirroring the rules/hierarchy split above).+ if (config.chunk_token_cap !== undefined) {+ if (result.hierarchyChunkTokenCap === undefined) {+ result.hierarchyChunkTokenCap = config.chunk_token_cap;+ }+ if (result.groupChunkTokenCap === undefined) {+ result.groupChunkTokenCap = config.chunk_token_cap;+ }+ }+// Clean up DSL-only fieldsdelete result.hierarchy;delete result.levels;delete result.rules;+ delete result.chunk_token_cap;return result;}@@ -439,7 +451,12 @@ export function transformFormConfigToApi(// the backend ManualChunker component (manual.go pins method=group,// ignores the token cap).return transformTitleChunkerParams(- omit(config, ['include_heading_content', 'chunk_token_cap']) as any,+ omit(config, [+ 'include_heading_content',+ 'chunk_token_cap',+ 'hierarchyChunkTokenCap',+ 'groupChunkTokenCap',+ ]) as any,);default:return config;