Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(pipeline): support image and email in general template (#21072)

jay77721cfd41e569f0c
diff --git a/internal/ingestion/pipeline/general_template_contract_test.go b/internal/ingestion/pipeline/general_template_contract_test.go
index c10bb074cf..fc4efa8afb 100644
--- a/internal/ingestion/pipeline/general_template_contract_test.go
+++ b/internal/ingestion/pipeline/general_template_contract_test.go
@@ -72,6 +72,23 @@ func TestGeneralTemplateUsesGeneralChunker(t *testing.T) {
if got := spreadsheet["output_format"]; got != "json" {
t.Fatalf("general spreadsheet output_format = %v, want json", got)
}
+ image, ok := parserParams["image"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general parser has no image setup")
+ }
+ if got := image["ocr_enabled"]; got != true {
+ t.Fatalf("general image ocr_enabled = %v, want true", got)
+ }
+ if got := image["output_format"]; got != "json" {
+ t.Fatalf("general image output_format = %v, want json", got)
+ }
+ email, ok := parserParams["email"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general parser has no email setup")
+ }
+ if _, ok := email["fields"]; !ok {
+ t.Fatal("general email setup has no fields")
+ }
}
func TestGeneralTemplateParserGraphUsesJSONContract(t *testing.T) {
@@ -111,17 +128,26 @@ func TestGeneralTemplateParserGraphUsesJSONContract(t *testing.T) {
if !ok {
t.Fatal("general parser graph form has no setups")
}
+ seen := map[string]bool{}
for _, rawSetup := range setups {
setup, ok := rawSetup.(map[string]interface{})
- if !ok || setup["fileFormat"] != "spreadsheet" {
+ if !ok {
continue
}
- if got := setup["output_format"]; got != "json" {
- t.Fatalf("general graph spreadsheet output_format = %v, want json", got)
+ format, _ := setup["fileFormat"].(string)
+ seen[format] = true
+ if format == "spreadsheet" {
+ if got := setup["output_format"]; got != "json" {
+ t.Fatalf("general graph spreadsheet output_format = %v, want json", got)
+ }
+ }
+ }
+ for _, want := range []string{"spreadsheet", "image", "email"} {
+ if !seen[want] {
+ t.Fatalf("general parser graph form has no %s setup", want)
}
- return
}
- t.Fatal("general parser graph form has no spreadsheet setup")
+ return
}
t.Fatal("general graph has no Parser:HipSignsRhyme node")
}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
index bc62e676f4..001f29a0f8 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
@@ -131,6 +131,44 @@
"kt",
"sql"
]
+ },
+ "image": {
+ "ocr_enabled": true,
+ "output_format": "json",
+ "preprocess": [
+ "main_content"
+ ],
+ "suffix": [
+ "bmp",
+ "gif",
+ "jpeg",
+ "jpg",
+ "png",
+ "svg",
+ "tif",
+ "tiff",
+ "webp"
+ ]
+ },
+ "email": {
+ "fields": [
+ "from",
+ "to",
+ "cc",
+ "bcc",
+ "date",
+ "subject",
+ "body",
+ "attachments"
+ ],
+ "output_format": "text",
+ "preprocess": [
+ "main_content"
+ ],
+ "suffix": [
+ "eml",
+ "msg"
+ ]
}
}
},
@@ -342,6 +380,31 @@
"main_content"
],
"vlm": {}
+ },
+ {
+ "fileFormat": "image",
+ "ocr_enabled": true,
+ "output_format": "json",
+ "preprocess": [
+ "main_content"
+ ]
+ },
+ {
+ "fileFormat": "email",
+ "fields": [
+ "from",
+ "to",
+ "cc",
+ "bcc",
+ "date",
+ "subject",
+ "body",
+ "attachments"
+ ],
+ "output_format": "text",
+ "preprocess": [
+ "main_content"
+ ]
}
]
},
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index 92a326e57f..61c2ed43ac 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -41,7 +41,7 @@ var builtinComponentParamsGolden = map[string]string{
"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"OneChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"book": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:GrumpyGarlicsBake\":{\"hierarchy\":5,\"include_heading_content\":true,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"5\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:HotDonutsRing\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}, \"image\": {\"ocr_enabled\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\", \"msg\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"laws": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"html\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"htm\",\"html\"]},\"markdown\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"md\",\"markdown\",\"mdx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"remove_toc\":true,\"suffix\":[\"pdf\"],\"vlm\":{}},\"text&code\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"txt\"]}},\"TitleChunker:SpicyKeysKick\":{\"hierarchy\":2,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"2\",\"hierarchyGroup\":\"0\",\"method\":\"hierarchy\"},\"Tokenizer:PublicJobsTake\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
"manual": "{\"File\":{},\"Parser:HipSignsRhyme\":{\"enable_vision_enhancement\":false,\"vlm\":{\"llm_id\":\"\"},\"doc\":{\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"doc\"]},\"docx\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"preprocess\":[\"main_content\"],\"suffix\":[\"docx\"],\"vlm\":{}},\"pdf\":{\"flatten_media_to_text\":false,\"output_format\":\"json\",\"parse_method\":\"DeepDOC\",\"preprocess\":[\"main_content\"],\"suffix\":[\"pdf\"],\"vlm\":{}}},\"ManualChunker:NineInsectsFind\":{\"hierarchy\":0,\"include_heading_content\":false,\"levels\":[[\"^#[^#]\",\"^##[^#]\",\"^###[^#]\",\"^####[^#]\"],[\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\",\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"第[零一二三四五六七八九十百0-9]+条\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"],[\"第[0-9]+章\",\"第[0-9]+节\",\"[0-9]{1,2}[\\\\. 、]\",\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\",\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"],[\"第[零一二三四五六七八九十百0-9]+章\",\"第[零一二三四五六七八九十百0-9]+节\",\"[零一二三四五六七八九十百]+[ 、]\",\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\",\"[\\\\((][0-9]{,2}[\\\\))]\"],[\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\",\"Chapter (I+V?|VI*|XI|IX|X)\",\"Section [0-9]+\",\"Article [0-9]+\"]],\"hierarchyRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"groupRules\":[{\"levels\":[{\"expression\":\"^#[^#]\"},{\"expression\":\"^##[^#]\"},{\"expression\":\"^###[^#]\"},{\"expression\":\"^####[^#]\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+条\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"}]},{\"levels\":[{\"expression\":\"第[0-9]+章\"},{\"expression\":\"第[0-9]+节\"},{\"expression\":\"[0-9]{1,2}[\\\\. 、]\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\"},{\"expression\":\"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"}]},{\"levels\":[{\"expression\":\"第[零一二三四五六七八九十百0-9]+章\"},{\"expression\":\"第[零一二三四五六七八九十百0-9]+节\"},{\"expression\":\"[零一二三四五六七八九十百]+[ 、]\"},{\"expression\":\"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"},{\"expression\":\"[\\\\((][0-9]{,2}[\\\\))]\"}]},{\"levels\":[{\"expression\":\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\"},{\"expression\":\"Chapter (I+V?|VI*|XI|IX|X)\"},{\"expression\":\"Section [0-9]+\"},{\"expression\":\"Article [0-9]+\"}]}],\"hierarchyHierarchy\":\"0\",\"hierarchyGroup\":\"0\",\"method\":\"group\"},\"Tokenizer:FunnyBalloonsGrin\":{\"fields\":\"text\",\"filename_embd_weight\":0.1,\"search_method\":[\"embedding\",\"full_text\"]},\"Extractor:AutoExtractDefault\":{\"llm_id\":\"\",\"keywords\":{\"top_n\":0},\"questions\":{\"top_n\":0},\"tags\":{\"top_n\":0,\"tag_file_id\":\"\"},\"summary\":{\"enabled\":false},\"metadata\":{\"enabled\":false,\"metadata\":[],\"built_in_metadata\":[]}}}",
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.go
index baba45f2f3..ba1d48c22d 100644
--- a/internal/service/document/document_parser_selection_test.go
+++ b/internal/service/document/document_parser_selection_test.go
@@ -120,6 +120,8 @@ func generalParserConfig() entity.JSONMap {
"pdf": map[string]any{"suffix": []any{"pdf"}},
"text&code": map[string]any{},
"docx": map[string]any{},
+ "image": map[string]any{"suffix": []any{"jpg"}},
+ "email": map[string]any{"suffix": []any{"eml", "msg"}},
},
}
}
@@ -242,6 +244,16 @@ func TestResolveDocumentParser_Priority(t *testing.T) {
wantFamilyIn: "",
wantFamilyOut: "",
},
+ {
+ name: "legacy flat config still routes audio",
+ kbParser: "naive",
+ kbConfig: legacyConfig,
+ filename: "talk.mp3",
+ filetype: utility.FileTypeAURAL,
+ wantParserID: "audio",
+ wantFamilyIn: "audio",
+ wantFamilyOut: "",
+ },
{
name: "legacy flat config still routes an owned family",
kbParser: "naive",
@@ -252,6 +264,16 @@ func TestResolveDocumentParser_Priority(t *testing.T) {
wantFamilyIn: "slides",
wantFamilyOut: "",
},
+ {
+ name: "legacy flat config keeps an image in general",
+ kbParser: "naive",
+ kbConfig: legacyConfig,
+ filename: "photo.jpg",
+ filetype: utility.FileTypeVISUAL,
+ wantParserID: "naive",
+ wantFamilyIn: "",
+ wantFamilyOut: "",
+ },
{
name: "canvas keeps a family it declares",
kbParser: "presentation",
@@ -401,10 +423,10 @@ func TestResolveDocumentParser_Priority(t *testing.T) {
}
// TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers pins the Python
-// FileService.get_parser parity: an uploaded image or presentation is re-pointed
-// at the pipeline that owns its file type and carries that pipeline's document
-// config (the config the parser-gap validation and the runtime read), while a
-// plain text upload keeps the dataset parser and config untouched.
+// FileService.get_parser parity: an uploaded presentation or audio file is
+// re-pointed at the pipeline that owns its file type and carries that
+// pipeline's document config, while image/email/text uploads stay on the
+// dataset parser now that the general template declares those families.
func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -422,6 +444,12 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
}
kbConfig := entity.JSONMap{
+ "Parser:HipSignsRhyme": map[string]any{
+ "pdf": map[string]any{"suffix": []any{"pdf"}},
+ "text&code": map[string]any{},
+ "image": map[string]any{"ocr_enabled": true, "suffix": []any{"jpg"}},
+ "email": map[string]any{"suffix": []any{"eml", "msg"}},
+ },
"GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": 512},
"metadata": map[string]any{
"enabled": true,
@@ -446,6 +474,8 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
files := []*multipart.FileHeader{
makeTestFileHeader(t, "file", "photo.jpg", []byte("jpeg-bytes")),
makeTestFileHeader(t, "file", "deck.pptx", []byte("pptx-bytes")),
+ makeTestFileHeader(t, "file", "note.eml", []byte("eml-bytes")),
+ makeTestFileHeader(t, "file", "talk.mp3", []byte("mp3-bytes")),
makeTestFileHeader(t, "file", "notes.txt", []byte("plain text")),
}
got, errs := svc.UploadLocalDocuments(ctx, kb, "user-1", files, "", nil)
@@ -456,12 +486,12 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
t.Fatalf("uploaded docs = %d, want %d", len(got), len(files))
}
- if parserID, _ := got[0]["parser_id"].(string); parserID != "picture" {
- t.Fatalf("jpg parser_id = %q, want picture", parserID)
+ if parserID, _ := got[0]["parser_id"].(string); parserID != "naive" {
+ t.Fatalf("jpg parser_id = %q, want the dataset parser naive", parserID)
}
imageSetup := parserFamilySetup(t, got[0], "image")
if imageSetup["ocr_enabled"] != true {
- t.Fatalf("jpg image setup = %#v, want the picture pipeline defaults", imageSetup)
+ t.Fatalf("jpg image setup = %#v, want the general pipeline defaults", imageSetup)
}
if parserID, _ := got[1]["parser_id"].(string); parserID != "presentation" {
@@ -472,10 +502,19 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
t.Fatalf("pptx slides setup = %#v, want the presentation pipeline defaults", slidesSetup)
}
- // The rebuilt config goes through the same assembly as a parser switch, so
- // the tenant LLM id and the dataset metadata group reach the target
+ if parserID, _ := got[2]["parser_id"].(string); parserID != "naive" {
+ t.Fatalf("eml parser_id = %q, want the dataset parser naive", parserID)
+ }
+ parserFamilySetup(t, got[2], "email")
+
+ if parserID, _ := got[3]["parser_id"].(string); parserID != "audio" {
+ t.Fatalf("mp3 parser_id = %q, want audio", parserID)
+ }
+
+ // The rebuilt configs go through the same assembly as a parser switch, so
+ // the tenant LLM id and the dataset metadata group reach each target
// pipeline's Extractor node.
- for _, doc := range got[:2] {
+ for _, doc := range []map[string]interface{}{got[1], got[3]} {
cfg := doc["parser_config"].(map[string]interface{})
extractor, _ := cfg["Extractor:AutoExtractDefault"].(map[string]interface{})
if extractor["llm_id"] != "llm-default" {
@@ -487,10 +526,10 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
}
}
- if parserID, _ := got[2]["parser_id"].(string); parserID != "naive" {
+ if parserID, _ := got[4]["parser_id"].(string); parserID != "naive" {
t.Fatalf("txt parser_id = %q, want the dataset parser naive", parserID)
}
- txtCfg, _ := got[2]["parser_config"].(map[string]interface{})
+ txtCfg, _ := got[4]["parser_config"].(map[string]interface{})
if _, ok := txtCfg["GeneralChunker:SixApplesFall"]; !ok {
t.Fatalf("txt parser_config = %#v, want the dataset config untouched", txtCfg)
}
diff --git a/web/src/pages/dataset/dataset/use-upload-document.test.ts b/web/src/pages/dataset/dataset/use-upload-document.test.ts
index 8a39c85fae..555e9258c7 100644
--- a/web/src/pages/dataset/dataset/use-upload-document.test.ts
+++ b/web/src/pages/dataset/dataset/use-upload-document.test.ts
@@ -67,20 +67,21 @@ beforeEach(() => {
});
// The regression case for the auto-parse bug: an unsupported file type (here
-// an image) is reseeded by the backend with its dedicated pipeline, and the
-// uploaded response carries the converted parser_config. The pre-upload gap
+// a presentation) is reseeded by the backend with its dedicated pipeline, and
+// the uploaded response carries the converted parser_config. The pre-upload gap
// prediction must not exclude the document from auto-parse anymore —
// runnability follows the document's actual config.
it('auto-parses a document the backend switched to a dedicated parser', async () => {
mockUploadGaps = [
- { reason: 'unsupportedType', name: 'photo.jpg', fileType: 'Image' },
+ { reason: 'unsupportedType', name: 'deck.pptx', fileType: 'PowerPoint' },
];
mockUpload.mockResolvedValue(
uploadOkResponse([
makeDoc({
- id: 'doc-img',
+ id: 'doc-deck',
+ name: 'deck.pptx',
parser_config: {
- 'Parser:parse': { setups: [{ fileFormat: 'Image' }] },
+ 'Parser:parse': { setups: [{ fileFormat: 'slides' }] },
},
}),
]),
@@ -89,7 +90,7 @@ it('auto-parses a document the backend switched to a dedicated parser', async ()
const { result } = renderHook(() => useHandleUploadDocument());
await act(async () => {
await result.current.onDocumentUploadOk({
- fileList: [new File(['x'], 'photo.jpg')],
+ fileList: [new File(['x'], 'deck.pptx')],
parseOnCreation: true,
});
});
@@ -97,7 +98,7 @@ it('auto-parses a document the backend switched to a dedicated parser', async ()
expect(mockWarning).toHaveBeenCalledTimes(1);
await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));
expect(mockRun).toHaveBeenCalledWith({
- documentIds: ['doc-img'],
+ documentIds: ['doc-deck'],
run: 1,
});
});
diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.ts
index 4aa50509f5..26f6b2ddec 100644
--- a/web/src/pages/dataset/dataset/utils.test.ts
+++ b/web/src/pages/dataset/dataset/utils.test.ts
@@ -143,6 +143,7 @@ const SavedConfig: SavedParserConfig = {
setups: [
{ fileFormat: FileType.PDF, parse_method: 'DeepDOC' },
{ fileFormat: FileType.Image, ocr_enabled: true },
+ { fileFormat: FileType.Email, fields: ['subject', 'body'] },
{ fileFormat: FileType.Audio, vlm: { llm_id: '' } },
{ fileFormat: FileType.Video },
],
@@ -218,6 +219,11 @@ describe('findParserGap', () => {
});
});
+ it('passes image/email with a general-like setup — no switch needed', () => {
+ expect(findParserGap(FileType.Image, SavedConfig)).toBeNull();
+ expect(findParserGap(FileType.Email, SavedConfig)).toBeNull();
+ });
+
it('passes image with an ocr-only setup — no model required', () => {
expect(findParserGap(FileType.Image, SavedConfig)).toBeNull();
});
@@ -231,7 +237,14 @@ describe('findParserGap', () => {
describe('findFilesParserGaps', () => {
it('returns a gap per failing file, mixing both reasons', () => {
const gaps = findFilesParserGaps(
- ['song.mp3', 'notes.docx', 'photo.jpg', 'movie.mp4'],
+ [
+ 'song.mp3',
+ 'notes.docx',
+ 'photo.jpg',
+ 'movie.mp4',
+ 'deck.pptx',
+ 'mail.eml',
+ ],
SavedConfig,
);
expect(gaps).toEqual([
@@ -246,6 +259,11 @@ describe('findFilesParserGaps', () => {
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Docx,
},
+ {
+ name: 'deck.pptx',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.PowerPoint,
+ },
]);
});
});