RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(pipeline): support image and email in general template (#21072)
jay77721cfd41e569f0c
diff --git a/internal/ingestion/pipeline/general_template_contract_test.go b/internal/ingestion/pipeline/general_template_contract_test.goindex c10bb074cf..fc4efa8afb 100644--- a/internal/ingestion/pipeline/general_template_contract_test.go+++ b/internal/ingestion/pipeline/general_template_contract_test.go@@ -72,6 +72,23 @@ func TestGeneralTemplateUsesGeneralChunker(t *testing.T) {if got := spreadsheet["output_format"]; got != "json" {t.Fatalf("general spreadsheet output_format = %v, want json", got)}+ image, ok := parserParams["image"].(map[string]interface{})+ if !ok {+ t.Fatal("general parser has no image setup")+ }+ if got := image["ocr_enabled"]; got != true {+ t.Fatalf("general image ocr_enabled = %v, want true", got)+ }+ if got := image["output_format"]; got != "json" {+ t.Fatalf("general image output_format = %v, want json", got)+ }+ email, ok := parserParams["email"].(map[string]interface{})+ if !ok {+ t.Fatal("general parser has no email setup")+ }+ if _, ok := email["fields"]; !ok {+ t.Fatal("general email setup has no fields")+ }}func TestGeneralTemplateParserGraphUsesJSONContract(t *testing.T) {@@ -111,17 +128,26 @@ func TestGeneralTemplateParserGraphUsesJSONContract(t *testing.T) {if !ok {t.Fatal("general parser graph form has no setups")}+ seen := map[string]bool{}for _, rawSetup := range setups {setup, ok := rawSetup.(map[string]interface{})- if !ok || setup["fileFormat"] != "spreadsheet" {+ if !ok {continue}- if got := setup["output_format"]; got != "json" {- t.Fatalf("general graph spreadsheet output_format = %v, want json", got)+ format, _ := setup["fileFormat"].(string)+ seen[format] = true+ if format == "spreadsheet" {+ if got := setup["output_format"]; got != "json" {+ t.Fatalf("general graph spreadsheet output_format = %v, want json", got)+ }+ }+ }+ for _, want := range []string{"spreadsheet", "image", "email"} {+ if !seen[want] {+ t.Fatalf("general parser graph form has no %s setup", want)}- return}- t.Fatal("general parser graph form has no spreadsheet setup")+ return}t.Fatal("general graph has no Parser:HipSignsRhyme node")}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.jsonindex bc62e676f4..001f29a0f8 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json@@ -131,6 +131,44 @@"kt","sql"]+ },+ "image": {+ "ocr_enabled": true,+ "output_format": "json",+ "preprocess": [+ "main_content"+ ],+ "suffix": [+ "bmp",+ "gif",+ "jpeg",+ "jpg",+ "png",+ "svg",+ "tif",+ "tiff",+ "webp"+ ]+ },+ "email": {+ "fields": [+ "from",+ "to",+ "cc",+ "bcc",+ "date",+ "subject",+ "body",+ "attachments"+ ],+ "output_format": "text",+ "preprocess": [+ "main_content"+ ],+ "suffix": [+ "eml",+ "msg"+ ]}}},@@ -342,6 +380,31 @@"main_content"],"vlm": {}+ },+ {+ "fileFormat": "image",+ "ocr_enabled": true,+ "output_format": "json",+ "preprocess": [+ "main_content"+ ]+ },+ {+ "fileFormat": "email",+ "fields": [+ "from",+ "to",+ "cc",+ "bcc",+ "date",+ "subject",+ "body",+ "attachments"+ ],+ "output_format": "text",+ "preprocess": [+ "main_content"+ ]}]},diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex 92a326e57f..61c2ed43ac 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -41,7 +41,7 @@ var builtinComponentParamsGolden = map[string]string{"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}","email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}","manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}","one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.goindex baba45f2f3..ba1d48c22d 100644--- a/internal/service/document/document_parser_selection_test.go+++ b/internal/service/document/document_parser_selection_test.go@@ -120,6 +120,8 @@ func generalParserConfig() entity.JSONMap {"pdf": map[string]any{"suffix": []any{"pdf"}},"text&code": map[string]any{},"docx": map[string]any{},+ "image": map[string]any{"suffix": []any{"jpg"}},+ "email": map[string]any{"suffix": []any{"eml", "msg"}},},}}@@ -242,6 +244,16 @@ func TestResolveDocumentParser_Priority(t *testing.T) {wantFamilyIn: "",wantFamilyOut: "",},+ {+ name: "legacy flat config still routes audio",+ kbParser: "naive",+ kbConfig: legacyConfig,+ filename: "talk.mp3",+ filetype: utility.FileTypeAURAL,+ wantParserID: "audio",+ wantFamilyIn: "audio",+ wantFamilyOut: "",+ },{name: "legacy flat config still routes an owned family",kbParser: "naive",@@ -252,6 +264,16 @@ func TestResolveDocumentParser_Priority(t *testing.T) {wantFamilyIn: "slides",wantFamilyOut: "",},+ {+ name: "legacy flat config keeps an image in general",+ kbParser: "naive",+ kbConfig: legacyConfig,+ filename: "photo.jpg",+ filetype: utility.FileTypeVISUAL,+ wantParserID: "naive",+ wantFamilyIn: "",+ wantFamilyOut: "",+ },{name: "canvas keeps a family it declares",kbParser: "presentation",@@ -401,10 +423,10 @@ func TestResolveDocumentParser_Priority(t *testing.T) {}// TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers pins the Python-// FileService.get_parser parity: an uploaded image or presentation is re-pointed-// at the pipeline that owns its file type and carries that pipeline's document-// config (the config the parser-gap validation and the runtime read), while a-// plain text upload keeps the dataset parser and config untouched.+// FileService.get_parser parity: an uploaded presentation or audio file is+// re-pointed at the pipeline that owns its file type and carries that+// pipeline's document config, while image/email/text uploads stay on the+// dataset parser now that the general template declares those families.func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -422,6 +444,12 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {}kbConfig := entity.JSONMap{+ "Parser:HipSignsRhyme": map[string]any{+ "pdf": map[string]any{"suffix": []any{"pdf"}},+ "text&code": map[string]any{},+ "image": map[string]any{"ocr_enabled": true, "suffix": []any{"jpg"}},+ "email": map[string]any{"suffix": []any{"eml", "msg"}},+ },"GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": 512},"metadata": map[string]any{"enabled": true,@@ -446,6 +474,8 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {files := []*multipart.FileHeader{makeTestFileHeader(t, "file", "photo.jpg", []byte("jpeg-bytes")),makeTestFileHeader(t, "file", "deck.pptx", []byte("pptx-bytes")),+ makeTestFileHeader(t, "file", "note.eml", []byte("eml-bytes")),+ makeTestFileHeader(t, "file", "talk.mp3", []byte("mp3-bytes")),makeTestFileHeader(t, "file", "notes.txt", []byte("plain text")),}got, errs := svc.UploadLocalDocuments(ctx, kb, "user-1", files, "", nil)@@ -456,12 +486,12 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {t.Fatalf("uploaded docs = %d, want %d", len(got), len(files))}- if parserID, _ := got[0]["parser_id"].(string); parserID != "picture" {- t.Fatalf("jpg parser_id = %q, want picture", parserID)+ if parserID, _ := got[0]["parser_id"].(string); parserID != "naive" {+ t.Fatalf("jpg parser_id = %q, want the dataset parser naive", parserID)}imageSetup := parserFamilySetup(t, got[0], "image")if imageSetup["ocr_enabled"] != true {- t.Fatalf("jpg image setup = %#v, want the picture pipeline defaults", imageSetup)+ t.Fatalf("jpg image setup = %#v, want the general pipeline defaults", imageSetup)}if parserID, _ := got[1]["parser_id"].(string); parserID != "presentation" {@@ -472,10 +502,19 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {t.Fatalf("pptx slides setup = %#v, want the presentation pipeline defaults", slidesSetup)}- // The rebuilt config goes through the same assembly as a parser switch, so- // the tenant LLM id and the dataset metadata group reach the target+ if parserID, _ := got[2]["parser_id"].(string); parserID != "naive" {+ t.Fatalf("eml parser_id = %q, want the dataset parser naive", parserID)+ }+ parserFamilySetup(t, got[2], "email")++ if parserID, _ := got[3]["parser_id"].(string); parserID != "audio" {+ t.Fatalf("mp3 parser_id = %q, want audio", parserID)+ }++ // The rebuilt configs go through the same assembly as a parser switch, so+ // the tenant LLM id and the dataset metadata group reach each target// pipeline's Extractor node.- for _, doc := range got[:2] {+ for _, doc := range []map[string]interface{}{got[1], got[3]} {cfg := doc["parser_config"].(map[string]interface{})extractor, _ := cfg["Extractor:AutoExtractDefault"].(map[string]interface{})if extractor["llm_id"] != "llm-default" {@@ -487,10 +526,10 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {}}- if parserID, _ := got[2]["parser_id"].(string); parserID != "naive" {+ if parserID, _ := got[4]["parser_id"].(string); parserID != "naive" {t.Fatalf("txt parser_id = %q, want the dataset parser naive", parserID)}- txtCfg, _ := got[2]["parser_config"].(map[string]interface{})+ txtCfg, _ := got[4]["parser_config"].(map[string]interface{})if _, ok := txtCfg["GeneralChunker:SixApplesFall"]; !ok {t.Fatalf("txt parser_config = %#v, want the dataset config untouched", txtCfg)}diff --git a/web/src/pages/dataset/dataset/use-upload-document.test.ts b/web/src/pages/dataset/dataset/use-upload-document.test.tsindex 8a39c85fae..555e9258c7 100644--- a/web/src/pages/dataset/dataset/use-upload-document.test.ts+++ b/web/src/pages/dataset/dataset/use-upload-document.test.ts@@ -67,20 +67,21 @@ beforeEach(() => {});// The regression case for the auto-parse bug: an unsupported file type (here-// an image) is reseeded by the backend with its dedicated pipeline, and the-// uploaded response carries the converted parser_config. The pre-upload gap+// a presentation) is reseeded by the backend with its dedicated pipeline, and+// the uploaded response carries the converted parser_config. The pre-upload gap// prediction must not exclude the document from auto-parse anymore —// runnability follows the document's actual config.it('auto-parses a document the backend switched to a dedicated parser', async () => {mockUploadGaps = [- { reason: 'unsupportedType', name: 'photo.jpg', fileType: 'Image' },+ { reason: 'unsupportedType', name: 'deck.pptx', fileType: 'PowerPoint' },];mockUpload.mockResolvedValue(uploadOkResponse([makeDoc({- id: 'doc-img',+ id: 'doc-deck',+ name: 'deck.pptx',parser_config: {- 'Parser:parse': { setups: [{ fileFormat: 'Image' }] },+ 'Parser:parse': { setups: [{ fileFormat: 'slides' }] },},}),]),@@ -89,7 +90,7 @@ it('auto-parses a document the backend switched to a dedicated parser', async ()const { result } = renderHook(() => useHandleUploadDocument());await act(async () => {await result.current.onDocumentUploadOk({- fileList: [new File(['x'], 'photo.jpg')],+ fileList: [new File(['x'], 'deck.pptx')],parseOnCreation: true,});});@@ -97,7 +98,7 @@ it('auto-parses a document the backend switched to a dedicated parser', async ()expect(mockWarning).toHaveBeenCalledTimes(1);await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));expect(mockRun).toHaveBeenCalledWith({- documentIds: ['doc-img'],+ documentIds: ['doc-deck'],run: 1,});});diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.tsindex 4aa50509f5..26f6b2ddec 100644--- a/web/src/pages/dataset/dataset/utils.test.ts+++ b/web/src/pages/dataset/dataset/utils.test.ts@@ -143,6 +143,7 @@ const SavedConfig: SavedParserConfig = {setups: [{ fileFormat: FileType.PDF, parse_method: 'DeepDOC' },{ fileFormat: FileType.Image, ocr_enabled: true },+ { fileFormat: FileType.Email, fields: ['subject', 'body'] },{ fileFormat: FileType.Audio, vlm: { llm_id: '' } },{ fileFormat: FileType.Video },],@@ -218,6 +219,11 @@ describe('findParserGap', () => {});});+ it('passes image/email with a general-like setup — no switch needed', () => {+ expect(findParserGap(FileType.Image, SavedConfig)).toBeNull();+ expect(findParserGap(FileType.Email, SavedConfig)).toBeNull();+ });+it('passes image with an ocr-only setup — no model required', () => {expect(findParserGap(FileType.Image, SavedConfig)).toBeNull();});@@ -231,7 +237,14 @@ describe('findParserGap', () => {describe('findFilesParserGaps', () => {it('returns a gap per failing file, mixing both reasons', () => {const gaps = findFilesParserGaps(- ['song.mp3', 'notes.docx', 'photo.jpg', 'movie.mp4'],+ [+ 'song.mp3',+ 'notes.docx',+ 'photo.jpg',+ 'movie.mp4',+ 'deck.pptx',+ 'mail.eml',+ ],SavedConfig,);expect(gaps).toEqual([@@ -246,6 +259,11 @@ describe('findFilesParserGaps', () => {reason: ParserGapReason.UnsupportedType,fileType: FileType.Docx,},+ {+ name: 'deck.pptx',+ reason: ParserGapReason.UnsupportedType,+ fileType: FileType.PowerPoint,+ },]);});});