RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): remove dead output_format readers (#21336)
jay7772117f173c288a8
diff --git a/docs/guides/agent/ingestion_pipeline/configure_parser_component.md b/docs/guides/agent/ingestion_pipeline/configure_parser_component.mdindex eb5f332d14..4958883b9d 100644--- a/docs/guides/agent/ingestion_pipeline/configure_parser_component.md+++ b/docs/guides/agent/ingestion_pipeline/configure_parser_component.md@@ -65,8 +65,7 @@ by tenant-model ID or by `model@instance@provider` reference, for example:```json{"pdf": {- "parse_method": "SoMark-model@production@SoMark",- "output_format": "json"+ "parse_method": "SoMark-model@production@SoMark"}}```diff --git a/internal/agent/canvas/compile_override_params_test.go b/internal/agent/canvas/compile_override_params_test.goindex 1b70f48b2e..c1b4315e1d 100644--- a/internal/agent/canvas/compile_override_params_test.go+++ b/internal/agent/canvas/compile_override_params_test.go@@ -24,11 +24,11 @@ func TestCompile_OverrideParams(t *testing.T) {ComponentName: "Parser",Params: map[string]any{"pdf": map[string]any{- "output_format": "one",- "parse_method": "naive",+ "lang": "one",+ "parse_method": "naive",},"doc": map[string]any{- "output_format": "one",+ "lang": "one",},},},@@ -50,10 +50,10 @@ func TestCompile_OverrideParams(t *testing.T) {override := map[string]any{"parser_0": map[string]any{"pdf": map[string]any{- "output_format": "detailed",+ "lang": "detailed",},"docx": map[string]any{- "output_format": "one",+ "lang": "one",},},}@@ -68,11 +68,11 @@ func TestCompile_OverrideParams(t *testing.T) {t.Fatalf("Parser factory not called")}// doc preserved from base.- if v, _ := got["doc"].(map[string]any); v == nil || v["output_format"] != "one" {+ if v, _ := got["doc"].(map[string]any); v == nil || v["lang"] != "one" {t.Errorf("doc should be preserved from base: %#v", got["doc"])}// docx injected by the override (was absent in base).- if v, _ := got["docx"].(map[string]any); v == nil || v["output_format"] != "one" {+ if v, _ := got["docx"].(map[string]any); v == nil || v["lang"] != "one" {t.Errorf("docx injection missing: %#v", got["docx"])}// pdf: override fully replaces the base entry (shallow merge).@@ -80,8 +80,8 @@ func TestCompile_OverrideParams(t *testing.T) {if pdf == nil {t.Fatalf("pdf setup missing: %#v", got)}- if pdf["output_format"] != "detailed" {- t.Errorf("pdf.output_format not overridden: %#v", pdf)+ if pdf["lang"] != "detailed" {+ t.Errorf("pdf.lang not overridden: %#v", pdf)}if _, ok := pdf["parse_method"]; ok {t.Errorf("pdf.parse_method should be dropped by shallow merge: %#v", pdf)diff --git a/internal/agent/canvas/node_body_override_params_test.go b/internal/agent/canvas/node_body_override_params_test.goindex 7d5c37ddb4..6f375b95a8 100644--- a/internal/agent/canvas/node_body_override_params_test.go+++ b/internal/agent/canvas/node_body_override_params_test.go@@ -36,11 +36,11 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) {baseParams := map[string]any{"pdf": map[string]any{- "output_format": "one",- "parse_method": "naive",+ "lang": "one",+ "parse_method": "naive",},"doc": map[string]any{- "output_format": "one",+ "lang": "one",},}// Run-level override is keyed by cpnID. For "cpn-parser": the whole@@ -50,15 +50,15 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) {override := map[string]any{"cpn-parser": map[string]any{"pdf": map[string]any{- "output_format": "detailed",+ "lang": "detailed",},"docx": map[string]any{- "output_format": "one",+ "lang": "one",},},"cpn-other": map[string]any{"pdf": map[string]any{- "output_format": "should-not-apply",+ "lang": "should-not-apply",},},}@@ -75,21 +75,21 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) {}// doc is untouched (absent from the override).- if v, _ := captured["doc"].(map[string]any); v == nil || v["output_format"] != "one" {+ if v, _ := captured["doc"].(map[string]any); v == nil || v["lang"] != "one" {t.Errorf("doc should be preserved from base: %#v", captured["doc"])}// docx injected by the override (was absent in base).- if v, _ := captured["docx"].(map[string]any); v == nil || v["output_format"] != "one" {+ if v, _ := captured["docx"].(map[string]any); v == nil || v["lang"] != "one" {t.Errorf("docx injection missing: %#v", captured["docx"])}- // pdf: the override fully replaces the base entry, so output_format is+ // pdf: the override fully replaces the base entry, so lang is// overridden and the base-only parse_method is dropped (shallow merge).pdf, _ := captured["pdf"].(map[string]any)if pdf == nil {t.Fatalf("pdf setup missing: %#v", captured)}- if pdf["output_format"] != "detailed" {- t.Errorf("pdf.output_format not overridden: %#v", pdf)+ if pdf["lang"] != "detailed" {+ t.Errorf("pdf.lang not overridden: %#v", pdf)}if _, ok := pdf["parse_method"]; ok {t.Errorf("pdf.parse_method should be dropped by shallow merge: %#v", pdf)diff --git a/internal/agent/templates/advanced_ingestion_pipeline.json b/internal/agent/templates/advanced_ingestion_pipeline.jsonindex f4215787c2..26b2f85d6d 100644--- a/internal/agent/templates/advanced_ingestion_pipeline.json+++ b/internal/agent/templates/advanced_ingestion_pipeline.json@@ -127,7 +127,6 @@}},"doc": {- "output_format": "json","preprocess": "main_content","suffix": ["doc"@@ -135,7 +134,6 @@},"docx": {"flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content","suffix": ["docx"@@ -153,7 +151,6 @@"body","attachments"],- "output_format": "text","preprocess": "main_content","suffix": ["eml",@@ -161,7 +158,6 @@]},"html": {- "output_format": "json","preprocess": "main_content","suffix": ["htm",@@ -169,7 +165,6 @@]},"image": {- "output_format": "json","parse_method": "ocr","preprocess": "main_content","suffix": [@@ -182,7 +177,6 @@},"markdown": {"flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content","suffix": ["md",@@ -193,7 +187,6 @@},"pdf": {"flatten_media_to_text": false,- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -202,7 +195,6 @@"vlm": {}},"slides": {- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -212,7 +204,6 @@},"spreadsheet": {"flatten_media_to_text": false,- "output_format": "html","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -223,7 +214,6 @@"vlm": {}},"text&code": {- "output_format": "json","preprocess": "main_content","suffix": ["txt",@@ -374,20 +364,17 @@{"fileFormat": "pdf","flatten_media_to_text": false,- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content"},{"fileFormat": "spreadsheet","flatten_media_to_text": false,- "output_format": "html","parse_method": "DeepDOC","preprocess": "main_content"},{"fileFormat": "image",- "output_format": "json","parse_method": "ocr","preprocess": "main_content","system_prompt": ""@@ -404,39 +391,32 @@"attachments"],"fileFormat": "email",- "output_format": "text","preprocess": "main_content"},{"fileFormat": "markdown","flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content"},{"fileFormat": "text&code",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "html",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "doc",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "docx","flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content"},{"fileFormat": "slides",- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content"}diff --git a/internal/agent/templates/chunk_summary.json b/internal/agent/templates/chunk_summary.jsonindex f862f68c1c..24a936e6c0 100644--- a/internal/agent/templates/chunk_summary.json+++ b/internal/agent/templates/chunk_summary.json@@ -84,7 +84,6 @@}},"doc": {- "output_format": "json","preprocess": "main_content","suffix": ["doc"@@ -92,7 +91,6 @@},"docx": {"flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content","suffix": ["docx"@@ -110,7 +108,6 @@"body","attachments"],- "output_format": "text","preprocess": "main_content","suffix": ["eml",@@ -118,7 +115,6 @@]},"html": {- "output_format": "json","preprocess": "main_content","suffix": ["htm",@@ -126,7 +122,6 @@]},"image": {- "output_format": "json","parse_method": "ocr","preprocess": "main_content","suffix": [@@ -139,7 +134,6 @@},"markdown": {"flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content","suffix": ["md",@@ -150,7 +144,6 @@},"pdf": {"flatten_media_to_text": false,- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -159,7 +152,6 @@"vlm": {}},"slides": {- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -169,7 +161,6 @@},"spreadsheet": {"flatten_media_to_text": false,- "output_format": "html","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -180,7 +171,6 @@"vlm": {}},"text&code": {- "output_format": "json","preprocess": "main_content","suffix": ["txt",@@ -334,20 +324,17 @@{"fileFormat": "pdf","flatten_media_to_text": false,- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content"},{"fileFormat": "spreadsheet","flatten_media_to_text": false,- "output_format": "html","parse_method": "DeepDOC","preprocess": "main_content"},{"fileFormat": "image",- "output_format": "json","parse_method": "ocr","preprocess": "main_content","system_prompt": ""@@ -364,39 +351,32 @@"attachments"],"fileFormat": "email",- "output_format": "text","preprocess": "main_content"},{"fileFormat": "markdown","flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content"},{"fileFormat": "text&code",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "html",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "doc",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "docx","flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content"},{"fileFormat": "slides",- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content"}diff --git a/internal/agent/templates/compiler.json b/internal/agent/templates/compiler.jsonindex 52fce12251..7e2c54b4cc 100644--- a/internal/agent/templates/compiler.json+++ b/internal/agent/templates/compiler.json@@ -59,7 +59,6 @@"doc": {"flatten_media_to_text": false,"order_index": 7,- "output_format": "json","remove_header_footer": false,"suffix": ["doc"@@ -69,7 +68,6 @@"docx": {"flatten_media_to_text": false,"order_index": 8,- "output_format": "json","remove_header_footer": false,"suffix": ["docx"@@ -88,7 +86,6 @@"attachments"],"order_index": 3,- "output_format": "text","suffix": ["eml","msg"@@ -96,7 +93,6 @@},"html": {"order_index": 6,- "output_format": "json","remove_header_footer": false,"suffix": ["htm",@@ -105,7 +101,6 @@},"image": {"order_index": 2,- "output_format": "json","parse_method": "ocr","suffix": ["jpg",@@ -117,7 +112,6 @@"markdown": {"flatten_media_to_text": false,"order_index": 4,- "output_format": "json","suffix": ["md","markdown",@@ -128,7 +122,6 @@"pdf": {"flatten_media_to_text": false,"order_index": 0,- "output_format": "json","parse_method": "DeepDOC","remove_header_footer": false,"suffix": [@@ -138,7 +131,6 @@},"slides": {"order_index": 9,- "output_format": "json","parse_method": "DeepDOC","suffix": ["pptx",@@ -148,7 +140,6 @@"spreadsheet": {"flatten_media_to_text": false,"order_index": 1,- "output_format": "html","parse_method": "DeepDOC","suffix": ["xls",@@ -159,7 +150,6 @@},"text&code": {"order_index": 5,- "output_format": "json","suffix": ["txt","py",@@ -285,7 +275,6 @@{"fileFormat": "pdf","flatten_media_to_text": false,- "output_format": "json","parse_method": "DeepDOC","remove_header_footer": false,"vlm": {}@@ -293,13 +282,11 @@{"fileFormat": "spreadsheet","flatten_media_to_text": false,- "output_format": "html","parse_method": "DeepDOC","vlm": {}},{"fileFormat": "image",- "output_format": "json","parse_method": "ocr"},{@@ -313,41 +300,34 @@"body","attachments"],- "fileFormat": "email",- "output_format": "text"+ "fileFormat": "email"},{"fileFormat": "markdown","flatten_media_to_text": false,- "output_format": "json","vlm": {}},{- "fileFormat": "text&code",- "output_format": "json"+ "fileFormat": "text&code"},{"fileFormat": "html",- "output_format": "json","remove_header_footer": false},{"fileFormat": "doc","flatten_media_to_text": false,- "output_format": "json","remove_header_footer": false,"vlm": {}},{"fileFormat": "docx","flatten_media_to_text": false,- "output_format": "json","remove_header_footer": false,"vlm": {}},{"fileFormat": "slides",- "output_format": "json","parse_method": "DeepDOC"}]diff --git a/internal/agent/templates/title_chunker.json b/internal/agent/templates/title_chunker.jsonindex 294cd87276..ad845fbb88 100644--- a/internal/agent/templates/title_chunker.json+++ b/internal/agent/templates/title_chunker.json@@ -84,7 +84,6 @@}},"doc": {- "output_format": "json","preprocess": "main_content","suffix": ["doc"@@ -92,7 +91,6 @@},"docx": {"flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content","suffix": ["docx"@@ -110,7 +108,6 @@"body","attachments"],- "output_format": "text","preprocess": "main_content","suffix": ["eml",@@ -118,7 +115,6 @@]},"html": {- "output_format": "json","preprocess": "main_content","suffix": ["htm",@@ -126,7 +122,6 @@]},"image": {- "output_format": "json","parse_method": "ocr","preprocess": "main_content","suffix": [@@ -139,7 +134,6 @@},"markdown": {"flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content","suffix": ["md",@@ -150,7 +144,6 @@},"pdf": {"flatten_media_to_text": false,- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -159,7 +152,6 @@"vlm": {}},"slides": {- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -169,7 +161,6 @@},"spreadsheet": {"flatten_media_to_text": false,- "output_format": "html","parse_method": "DeepDOC","preprocess": "main_content","suffix": [@@ -180,7 +171,6 @@"vlm": {}},"text&code": {- "output_format": "json","preprocess": "main_content","suffix": ["txt",@@ -353,20 +343,17 @@{"fileFormat": "pdf","flatten_media_to_text": false,- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content"},{"fileFormat": "spreadsheet","flatten_media_to_text": false,- "output_format": "html","parse_method": "DeepDOC","preprocess": "main_content"},{"fileFormat": "image",- "output_format": "json","parse_method": "ocr","preprocess": "main_content","system_prompt": ""@@ -383,39 +370,32 @@"attachments"],"fileFormat": "email",- "output_format": "text","preprocess": "main_content"},{"fileFormat": "markdown","flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content"},{"fileFormat": "text&code",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "html",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "doc",- "output_format": "json","preprocess": "main_content"},{"fileFormat": "docx","flatten_media_to_text": false,- "output_format": "json","preprocess": "main_content"},{"fileFormat": "slides",- "output_format": "json","parse_method": "DeepDOC","preprocess": "main_content"}diff --git a/internal/dao/ingestion_parser_config_runtime_test.go b/internal/dao/ingestion_parser_config_runtime_test.goindex 3f8a8e9027..4b529fcada 100644--- a/internal/dao/ingestion_parser_config_runtime_test.go+++ b/internal/dao/ingestion_parser_config_runtime_test.go@@ -625,7 +625,6 @@ var overriddenFamilyKeys = map[string]bool{// questions (this one, "is the template's value safe to lose"; that one, "may// the value the migration writes differ from the template's").var inertFamilyKeys = map[string]string{- "output_format": "normalizeParserOutputFormats forces JSON on every family","preprocess": "the value has no Go reader; the preprocessing stage is selected by the chunker's own options","suffix": "no Go reader; file routing uses the document's own suffix column, not the family list","vlm": "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all",diff --git a/internal/deepdoc/parser/pdf/layout/boxes_sections.go b/internal/deepdoc/parser/pdf/layout/boxes_sections.goindex 8ea419c066..bfc5dc3082 100644--- a/internal/deepdoc/parser/pdf/layout/boxes_sections.go+++ b/internal/deepdoc/parser/pdf/layout/boxes_sections.go@@ -146,25 +146,6 @@ func InlinePNGDataURL(raw string) string {return "data:image/png;base64," + cleaned}-func SectionsToMarkdown(sections []pdf.Section) string {- var b strings.Builder- for _, s := range sections {- if s.LayoutType == pdf.LayoutTypeTitle {- b.WriteString("\n## ")- }- imgURL := InlinePNGDataURL(s.Image)- if (s.LayoutType == pdf.LayoutTypeFigure || s.LayoutType == "image" || s.DocTypeKwd == "image" || (s.LayoutType == pdf.LayoutTypeTable && strings.TrimSpace(s.Text) == "")) && imgURL != "" {- b.WriteString("\n- b.WriteString(imgURL)- b.WriteString(")")- continue- }- b.WriteString(s.Text)- b.WriteString("\n")- }- return b.String()-}-// SectionsToJSON converts Sections to a Python-compatible JSON dict format.//// Each dict has keys: text, layout_type, doc_type_kwd, _pdf_positions, image.diff --git a/internal/deepdoc/parser/pdf/layout/boxes_sections_test.go b/internal/deepdoc/parser/pdf/layout/boxes_sections_test.goindex 3d6e36e6ad..28d55b08c9 100644--- a/internal/deepdoc/parser/pdf/layout/boxes_sections_test.go+++ b/internal/deepdoc/parser/pdf/layout/boxes_sections_test.go@@ -231,166 +231,6 @@ func TestNormalizeSectionPositions_EmptySlice(t *testing.T) {}}-func TestSectionsToMarkdown_Title(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeTitle, Text: "标题", Image: ""},- }- got := SectionsToMarkdown(sections)- want := "\n## 标题\n"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--func TestSectionsToMarkdown_FigureWithImage(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: "abc123"},- }- got := SectionsToMarkdown(sections)- want := "\n"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--func TestSectionsToMarkdown_FigureWithDataURLImage(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: "data:image/png;base64,abc123"},- }- got := SectionsToMarkdown(sections)- want := "\n"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--func TestSectionsToMarkdown_FigureWithoutImage(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: ""},- }- got := SectionsToMarkdown(sections)- want := "图1\n"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--func TestSectionsToMarkdown_FigureWithWhitespaceImage(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: " \t\n"},- }- got := SectionsToMarkdown(sections)- want := "图1\n"- if got != want {- t.Errorf("got %q, want %q (whitespace image should not emit empty image tag or drop text)", got, want)- }-}--func TestSectionsToMarkdown_Text(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeText, Text: "普通内容", Image: ""},- }- got := SectionsToMarkdown(sections)- want := "普通内容\n"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--func TestSectionsToMarkdown_Table(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeTable, Text: "表格内容", Image: ""},- }- got := SectionsToMarkdown(sections)- want := "表格内容\n"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--func TestSectionsToMarkdown_TableWithImageAndText(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeTable, Text: "<table><tr><td>1</td></tr></table>", Image: "dGFibGVpbWc="},- }- got := SectionsToMarkdown(sections)- want := "<table><tr><td>1</td></tr></table>\n"- if got != want {- t.Errorf("got %q, want %q (table with text should render table text)", got, want)- }-}--func TestSectionsToMarkdown_TableFallbackWhenEmptyText(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeTable, Text: "", Image: "dGFibGVpbWc="},- }- got := SectionsToMarkdown(sections)- want := "\n"- if got != want {- t.Errorf("got %q, want %q (table with empty text should fallback to image)", got, want)- }-}--func TestSectionsToMarkdown_TableFallbackWhitespaceImage(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeTable, Text: "", Image: " \t\n"},- }- got := SectionsToMarkdown(sections)- want := "\n"- if got != want {- t.Errorf("got %q, want %q (table with empty text and whitespace image should render empty text newline, no empty tag)", got, want)- }-}--func TestSectionsToMarkdown_DocTypeKwdImage(t *testing.T) {- sections := []pdf.Section{- {LayoutType: "custom_block", DocTypeKwd: "image", Text: "caption", Image: "aGVsbG8="},- }- got := SectionsToMarkdown(sections)- want := "\n"- if got != want {- t.Errorf("got %q, want %q (DocTypeKwd == 'image' should render image)", got, want)- }-}--func TestSectionsToMarkdown_DocTypeKwdImage_Whitespace(t *testing.T) {- sections := []pdf.Section{- {LayoutType: "custom_block", DocTypeKwd: "image", Text: "caption", Image: " "},- }- got := SectionsToMarkdown(sections)- want := "caption\n"- if got != want {- t.Errorf("got %q, want %q (DocTypeKwd == 'image' with whitespace image should keep text)", got, want)- }-}--func TestSectionsToMarkdown_Mixed(t *testing.T) {- sections := []pdf.Section{- {LayoutType: pdf.LayoutTypeTitle, Text: "标题", Image: ""},- {LayoutType: pdf.LayoutTypeText, Text: "内容", Image: ""},- {LayoutType: pdf.LayoutTypeFigure, Text: "图", Image: "img"},- {LayoutType: pdf.LayoutTypeTable, Text: "表格", Image: ""},- }- got := SectionsToMarkdown(sections)- want := "\n## 标题\n内容\n\n表格\n"- if got != want {- t.Errorf("got %q, want %q", got, want)- }-}--func TestSectionsToMarkdown_EmptySlice(t *testing.T) {- got := SectionsToMarkdown([]pdf.Section{})- if got != "" {- t.Errorf("got %q, want empty string", got)- }-}--func TestSectionsToMarkdown_NilSlice(t *testing.T) {- got := SectionsToMarkdown(nil)- if got != "" {- t.Errorf("got %q, want empty string", got)- }-}-func TestSectionsToJSON_SingleSection(t *testing.T) {sections := []pdf.Section{{diff --git a/internal/deepdoc/parser/pdf/parse_cgo.go b/internal/deepdoc/parser/pdf/parse_cgo.goindex d48e9e4e9b..9112f3ccb5 100644--- a/internal/deepdoc/parser/pdf/parse_cgo.go+++ b/internal/deepdoc/parser/pdf/parse_cgo.go@@ -13,16 +13,13 @@ import (//// Engine lifetime / ownership: Parse creates the native PDF engine and// transfers ownership of it to the returned *pdf.ParseResult (result.Engine is-// populated inside ParseRaw/processPages). The engine is intentionally kept-// alive past Parse so the serialization step can still use it — most-// importantly the Markdown path (cropMarkdownFigures) renders and crops figure-// images on demand. Parse must therefore NOT close the engine itself.+// populated inside ParseRaw/processPages). Parse must therefore NOT close the+// engine itself.//-// The caller (the adapter layer: pdfParseResultToJSONWithOptions /-// pdfParseResultToMarkdownWithOptions) releases the engine via result.Close()-// once serialization is complete. The only place Parse closes the engine is on-// its own error return, since in that case no ParseResult is handed back for-// the caller to release.+// The caller (the adapter layer: pdfParseResultToJSONWithOptions) releases the+// engine via result.Close() once serialization is complete. The only place Parse+// closes the engine is on its own error return, since in that case no+// ParseResult is handed back for the caller to release.func (p *Parser) Parse(ctx context.Context, data []byte, docAnalyzer pdf.DocAnalyzer) (*pdf.ParseResult, error) {engine, err := NewEngine(data)if err != nil {diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex e0d50afd8a..e54adcfdf9 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -520,10 +520,8 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {}// TestMaybeDispatchImage_HardcodesJSONOutput verifies the image family-// always emits json regardless of setup["output_format"]. Python-// rag/app/picture.py:chunk() has no output_format concept — it always-// returns a structured doc. Honoring a "text" override produced a bare-// Text payload that lost the image attachment and set doc_type to "text".+// always emits structured JSON items. Python rag/app/picture.py:chunk()+// has no output format concept — it always returns a structured doc.func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {origResolver := resolveTenantModelByTypedefer func() { resolveTenantModelByType = origResolver }()@@ -534,7 +532,6 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {}setups := defaultSetups()- setups["image"]["output_format"] = "text" // legacy/override; must be ignoredctx := t.Context()res, _, err := maybeDispatchImage(ctx,@@ -550,10 +547,10 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {t.Fatalf("maybeDispatchImage: %v", err)}if res.OutputFormat != "json" {- t.Fatalf("OutputFormat = %q, want json (image family must ignore output_format override)", res.OutputFormat)+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)}if len(res.JSON) != 1 {- t.Fatalf("JSON len = %d, want 1 even when setup says text", len(res.JSON))+ t.Fatalf("JSON len = %d, want 1", len(res.JSON))}}@@ -569,11 +566,11 @@ func (d *audioTranscribeDriver) TranscribeAudio(ctx context.Context, _ *string,return &modelModule.ASRResponse{Text: d.transcription}, nil}-// TestMaybeDispatchAudio_JSONCarriesTranscription pins diff 2.11: when the-// audio family's output_format is "json", the ASR transcription must be-// carried in the JSON items (not only in the Text field). Before the fix the-// branch returned Text only with an empty JSON slice, and the Invoke switch-// silently dropped the transcription because it has no "json" branch.+// TestMaybeDispatchAudio_JSONCarriesTranscription pins that the ASR+// transcription is carried in the JSON items (not only in the Text field).+// Before the fix the branch returned Text only with an empty JSON slice, and+// the Invoke switch silently dropped the transcription because it has no+// "json" branch.func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {origResolver := resolveTenantModelByTypedefer func() { resolveTenantModelByType = origResolver }()@@ -585,7 +582,6 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {}setups := defaultSetups()- setups["audio"]["output_format"] = "json"ctx := t.Context()res, dispatched, err := maybeDispatchAudio(@@ -617,49 +613,8 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {}}-func TestMaybeDispatchAudio_IgnoresLegacyOutputFormat(t *testing.T) {- origResolver := resolveTenantModelByType- defer func() { resolveTenantModelByType = origResolver }()-- const want = "hello world"- drv := &audioTranscribeDriver{transcription: want}- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {- return drv, "asr-model", &modelModule.APIConfig{}, 0, nil- }-- setups := defaultSetups()- setups["audio"]["output_format"] = "text"-- ctx := t.Context()- res, dispatched, err := maybeDispatchAudio(- ctx,- dao.DB,- utility.FileTypeAURAL,- "test.mp3",- []byte("fake-audio"),- map[string]any{"tenant_id": "t1"},- setups,- )- if err != nil {- t.Fatalf("maybeDispatchAudio: %v", err)- }- if !dispatched {- t.Fatalf("expected dispatched=true for AURAL file")- }- if res.OutputFormat != "json" {- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)- }- if len(res.JSON) != 1 {- t.Fatalf("JSON len = %d, want 1", len(res.JSON))- }- if got, _ := res.JSON[0].ToMap()["text"].(string); got != want {- t.Fatalf("JSON[0].text = %q, want %q", got, want)- }-}--// TestMaybeDispatchAudio_DefaultOutputFormatJson covers the-// maybeDispatchAudio always returns a JSON transcription item, including when-// the setup has no output_format key.+// TestMaybeDispatchAudio_DefaultOutputFormatJson covers that+// maybeDispatchAudio always returns a JSON transcription item.func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {const want = "hello world"drv := &audioTranscribeDriver{transcription: want}@@ -668,8 +623,6 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {return drv, "asr-model", &modelModule.APIConfig{}, 0, nil}- // No output_format key — exercise the default path inside- // maybeDispatchAudio.setups := map[string]schema.ParserSetup{"audio": {}}res, dispatched, err := maybeDispatchAudio(context.Background(),@@ -691,27 +644,6 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {}}-// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family output-// format must be "json" (matching Python parser.py:212), not "text".-// With "text" the structured email fields (from/to/subject/attachments/...) are-// flattened into a blob and lost downstream. The value is owned by-// normalizeParserOutputFormats, so the assertion goes through the constructed-// component rather than the raw defaults.-func TestDefaultEmailOutputFormatIsJSON(t *testing.T) {- comp, err := NewParserComponent(nil)- if err != nil {- t.Fatalf("NewParserComponent: %v", err)- }- pc, ok := comp.(*ParserComponent)- if !ok {- t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)- }- got, _ := pc.setups["email"]["output_format"].(string)- if got != "json" {- t.Fatalf("email output_format = %q, want json", got)- }-}-func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {origTenantResolver := resolveTenantModelByTypeorigModelResolver := resolveModelConfigdiff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 279db84f74..561e9e7fb4 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -128,11 +128,10 @@ type visionSettings struct {}// NewParserComponent constructs a Parser from a DSL param map.-// The default setups are overlaid with the supplied values. Historical-// output_format values are accepted but normalized to JSON so downstream-// components consume one parser output protocol. This applies to every-// family, including PDF and office documents: Markdown is an internal-// backend representation only and is never a public Parser output.+// The default setups are overlaid with the supplied values. The Parser+// emits one output protocol: structured JSON items for every family,+// including PDF and office documents. Markdown remains an internal backend+// representation only and is never a public Parser output.//// Param map shape (all keys optional)://@@ -149,7 +148,6 @@ type visionSettings struct {func NewParserComponent(params map[string]any) (runtime.Component, error) {s := defaultSetups()if params == nil {- normalizeParserOutputFormats(s)return &ParserComponent{setups: s}, nil}// Canvases saved by the Python-era frontend nest the per-family setups@@ -197,7 +195,6 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {s[k][fk] = cloneParserSetupValue(fv)}}- normalizeParserOutputFormats(s)pc := &ParserComponent{setups: s, enableVisionEnhancement: enableVisionEnhancement, vision: vision}if err := pc.Check(); err != nil {return nil, fmt.Errorf("parser: %w", err)@@ -205,15 +202,6 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {return pc, nil}-func normalizeParserOutputFormats(setups map[string]schema.ParserSetup) {- // The Go Parser component intentionally exposes JSON only. Keep this- // normalization unconditional so PDF/office legacy Markdown settings- // cannot silently select a second public output path.- for _, setup := range setups {- setup["output_format"] = "json"- }-}-func cloneParserSetupValue(value any) any {switch v := value.(type) {case map[string]any:@@ -309,11 +297,6 @@ func imageSetupRunsOCR(setup schema.ParserSetup) bool {}func defaultSetups() map[string]schema.ParserSetup {- // Normalize up front so any caller (including tests that build a- // ParserComponent directly from these defaults) gets the single JSON- // output protocol. NewParserComponent re-normalizes idempotently; the- // per-family literals were removed because normalizeParserOutputFormats- // owns this key unconditionally.s := map[string]schema.ParserSetup{"pdf": {"parse_method": "deepdoc",@@ -395,7 +378,6 @@ func defaultSetups() map[string]schema.ParserSetup {"suffix": []string{"json", "jsonl", "ldjson"},},}- normalizeParserOutputFormats(s)return s}diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.goindex bc3539055f..99bc3127b4 100644--- a/internal/ingestion/component/parser_check_test.go+++ b/internal/ingestion/component/parser_check_test.go@@ -140,11 +140,11 @@ func TestParserComponent_Check(t *testing.T) {// --- audio/video: vlm.llm_id not validated (matches Python check()) ---{name: "audio: no vlm field → pass (model falls back to tenant default)",- setups: map[string]schema.ParserSetup{"audio": {"output_format": "text"}},+ setups: map[string]schema.ParserSetup{"audio": {}},},{name: "video: no vlm field → pass",- setups: map[string]schema.ParserSetup{"video": {"output_format": "text"}},+ setups: map[string]schema.ParserSetup{"video": {}},},{name: "audio: vlm.llm_id empty → pass (empty falls back to tenant default)",@@ -162,7 +162,7 @@ func TestParserComponent_Check(t *testing.T) {},{name: "only unrelated family → pass",- setups: map[string]schema.ParserSetup{"markdown": {"output_format": "json"}},+ setups: map[string]schema.ParserSetup{"markdown": {}},},}diff --git a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.goindex 90697b8fb4..83dbb30818 100644--- a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go+++ b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go@@ -63,7 +63,6 @@ func TestDispatch_PDFVisionJSON_RealPDFFixture(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"- setups["pdf"]["output_format"] = "json"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 62070b38ab..a09fe23228 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -302,7 +302,6 @@ func TestDispatch_JSONOutput(t *testing.T) {func TestDispatchNormalizesConfiguredOutputFormatToJSON(t *testing.T) {setups := defaultSetups()- setups["email"]["output_format"] = "text"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -433,7 +432,6 @@ func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {}setups := defaultSetups()- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -470,7 +468,6 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {t.Run(method, func(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = method- setups["pdf"]["output_format"] = "json"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -565,7 +562,6 @@ func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"- setups["pdf"]["output_format"] = "json"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -631,7 +627,6 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"- setups["pdf"]["output_format"] = "json"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -688,7 +683,6 @@ func TestDispatch_PDFVisionJSON_HonorsPageRanges(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"- setups["pdf"]["output_format"] = "json"setups["pdf"]["pages"] = []any{[]any{float64(2), float64(3)}}c := &ParserComponent{setups: setups}@@ -728,7 +722,6 @@ func TestDispatch_PDFVisionJSON_RejectsInvalidPages(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"- setups["pdf"]["output_format"] = "json"setups["pdf"]["pages"] = "abc"c := &ParserComponent{setups: setups}@@ -774,7 +767,6 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -822,7 +814,6 @@ func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"- setups["pdf"]["output_format"] = "json"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -880,7 +871,6 @@ func TestDispatch_PDFMinerUMarkdown_SendsServerURLFromProviderConfig(t *testing.setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -952,7 +942,6 @@ func TestDispatch_PDFMinerUV1_WiresParseAndExtract(t *testing.T) {}setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"- setups["pdf"]["output_format"] = "markdown"setups["pdf"]["mineru_backend"] = "standard"return invokeMinerUDispatch(t, nil, setups)}@@ -1106,7 +1095,6 @@ func TestDispatch_PDFMinerUV1_WiresParseAndExtract(t *testing.T) {}setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"- setups["pdf"]["output_format"] = "markdown"setups["pdf"]["mineru_backend"] = "standard"c := &ParserComponent{setups: setups}_, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1162,7 +1150,7 @@ func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {}component, err := NewParserComponent(map[string]any{- "pdf": map[string]any{"parse_method": "monkeyocrv2", "output_format": "markdown"},+ "pdf": map[string]any{"parse_method": "monkeyocrv2"},})if err != nil {t.Fatalf("NewParserComponent: %v", err)@@ -1265,7 +1253,6 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesTenantModel(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "PaddleOCR"- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1327,7 +1314,6 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "PaddleOCR"- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1352,7 +1338,6 @@ func TestDispatch_PDFPaddleOCR_NoTenantModel_HardErrors(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "paddleocr"- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}_, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1415,7 +1400,6 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel(t *testing.T) {setups := defaultSetups()setups["pdf"]["layout_recognizer"] = modelID- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1479,7 +1463,6 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = modelID- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1554,7 +1537,6 @@ func TestDispatch_PDFMinerU_BareModelUUID_InParseMethod(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = modelID- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1618,7 +1600,6 @@ func TestDispatch_PDFMinerU_LanguageChain(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"- setups["pdf"]["output_format"] = "markdown"for k, v := range tc.overrides {setups["pdf"][k] = v}@@ -1692,7 +1673,6 @@ func TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "deepdoc"setups["pdf"]["layout_recognizer"] = modelID- setups["pdf"]["output_format"] = "markdown"c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -1808,7 +1788,6 @@ func TestDispatch_PDFMinerU_HonorsMinerUParseMethodOption(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"setups["pdf"]["mineru_parse_method"] = "ocr"- setups["pdf"]["output_format"] = "markdown"out := invokeMinerUDispatch(t, nil, setups)requireJSONText(t, out, "Real Resolver Title")@@ -1837,7 +1816,6 @@ func TestDispatch_PDFMinerU_CompositeSelectorInParseMethod(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = selector- setups["pdf"]["output_format"] = "markdown"out := invokeMinerUDispatch(t, nil, setups)requireJSONText(t, out, "Real Resolver Title")@@ -1855,7 +1833,6 @@ func TestDispatch_PDFMinerU_NamedSelector_ResolvesMinerUProviderModel(t *testingsetups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"- setups["pdf"]["output_format"] = "markdown"out := invokeMinerUDispatch(t, db, setups)requireJSONText(t, out, "Real Resolver Title")@@ -1873,7 +1850,6 @@ func TestDispatch_PDFMinerU_BareModelUUID_ResolvesThroughRealProbeAndResolver(tsetups := defaultSetups()setups["pdf"]["parse_method"] = modelID- setups["pdf"]["output_format"] = "markdown"out := invokeMinerUDispatch(t, db, setups)requireJSONText(t, out, "Real Resolver Title")@@ -1892,7 +1868,6 @@ func TestDispatch_PDFMinerU_CompositeSelector_ResolvesThroughRealResolver(t *tessetups := defaultSetups()setups["pdf"]["parse_method"] = "deepdoc"setups["pdf"]["layout_recognizer"] = "MinerU-model@default@MinerU"- setups["pdf"]["output_format"] = "markdown"out := invokeMinerUDispatch(t, db, setups)requireJSONText(t, out, "Real Resolver Title")@@ -1975,7 +1950,6 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "Docling"- setups["pdf"]["output_format"] = "markdown"setups["pdf"]["docling_server_url"] = server.URLsetups["pdf"]["docling_api_key"] = "doc-secret"c := &ParserComponent{setups: setups}@@ -2011,7 +1985,6 @@ func TestDispatch_PDFOpenDataLoaderLegacyMarkdownEmitsJSON(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "OpenDataLoader"- setups["pdf"]["output_format"] = "markdown"setups["pdf"]["opendataloader_apiserver"] = server.URLc := &ParserComponent{setups: setups}@@ -2042,7 +2015,6 @@ func TestDispatch_PDFSoMarkLegacyMarkdownEmitsJSON(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "SoMark"- setups["pdf"]["output_format"] = "markdown"setups["pdf"]["somark_base_url"] = server.URLc := &ParserComponent{setups: setups}@@ -2075,7 +2047,6 @@ func TestDispatch_PDFTCADPLegacyMarkdownEmitsJSON(t *testing.T) {setups := defaultSetups()setups["pdf"]["parse_method"] = "TCADP parser"- setups["pdf"]["output_format"] = "markdown"setups["pdf"]["tcadp_apiserver"] = server.URLc := &ParserComponent{setups: setups}@@ -2147,16 +2118,13 @@ func TestPythonFamilyName_FileTypeConstants(t *testing.T) {}}-// TestConfigureParserFromSetups_VisualAural pins that image and audio-// files pick up their setup (parse_method / lang / vlm) via the family-// mapping. Before the fix, configureParserFromSetups silently skipped-// configuration because resolveParserFamily returned "picture" / "aural",-// neither of which existed in defaultSetups.-func TestConfigureParserFromSetups_VisualAural(t *testing.T) {+// TestConfigureParserFromSetups_Visual pins that image files pick up their+// setup (parse_method / lang / vlm) via the family mapping. Before the fix,+// configureParserFromSetups silently skipped configuration because+// resolveParserFamily returned "picture", which did not exist in+// defaultSetups.+func TestConfigureParserFromSetups_Visual(t *testing.T) {setups := defaultSetups()- // The component normalizes output_format onto every family before the- // dispatch layer sees the setups, so mirror that here.- normalizeParserOutputFormats(setups)t.Run("visual resolves to image setup", func(t *testing.T) {got := &captureSetupConfigurer{}@@ -2170,15 +2138,4 @@ func TestConfigureParserFromSetups_VisualAural(t *testing.T) {}})- t.Run("aural resolves to audio setup", func(t *testing.T) {- got := &captureSetupConfigurer{}- configureParserFromSetups(got, utility.FileTypeAURAL, setups)- want := map[string]any(setups["audio"])- if got.setup == nil {- t.Fatal("ConfigureFromSetup not called for FileTypeAURAL")- }- if v, _ := got.setup["output_format"].(string); v != want["output_format"] {- t.Errorf("FileTypeAURAL output_format = %v, want %v", v, want["output_format"])- }- })}diff --git a/internal/ingestion/component/parser_legacy_output_format_test.go b/internal/ingestion/component/parser_legacy_output_format_test.gonew file mode 100644index 0000000000..3796fd3617--- /dev/null+++ b/internal/ingestion/component/parser_legacy_output_format_test.go@@ -0,0 +1,76 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "testing"++ "ragflow/internal/parser/parser"+)++// TestLegacyOutputFormatKeyIsIgnored pins the one contract that survives the+// key's removal: a persisted canvas may still carry output_format in a Parser+// family setup, and no reader consumes it — the component builds, parses, and+// emits the single JSON protocol regardless of the baked value.+func TestLegacyOutputFormatKeyIsIgnored(t *testing.T) {+ const eml = "From: [email protected]\r\n" ++ "To: [email protected]\r\n" ++ "Subject: Hi\r\n" ++ "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" ++ "\r\n" ++ "Hello body.\r\n"++ cases := []struct {+ name string+ params map[string]any+ filename string+ binary []byte+ }{+ {+ name: "markdown legacy text",+ params: map[string]any{"markdown": map[string]any{"output_format": "text"}},+ filename: "note.md",+ binary: []byte("# Heading\n\nBody paragraph.\n"),+ },+ {+ name: "email legacy text",+ params: map[string]any{"email": map[string]any{"output_format": "text"}},+ filename: "mail.eml",+ binary: []byte(eml),+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ comp, err := NewParserComponent(tc.params)+ if err != nil {+ t.Fatalf("NewParserComponent: %v", err)+ }+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"binary": tc.binary, "name": tc.filename})+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)+ }+ items, ok := out["json"].([]parser.DocItem)+ if !ok || len(items) == 0 {+ t.Fatalf("json items = %#v, want non-empty", out["json"])+ }+ })+ }+}diff --git a/internal/ingestion/component/parser_output_format_test.go b/internal/ingestion/component/parser_output_format_test.godeleted file mode 100644index d891751d02..0000000000--- a/internal/ingestion/component/parser_output_format_test.go+++ /dev/null@@ -1,71 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package component--import (- "testing"-)--// TestDefaultSetupsNormalizedToJSON pins that the raw defaults already carry-// the single JSON output protocol. defaultSetups() must not ship a per-family-// output_format literal (normalizeParserOutputFormats owns that key), yet-// every family it returns must end up on JSON — otherwise a caller building a-// ParserComponent straight from defaultSetups() would silently get an-// un-normalized setup (the fragility flagged in the PR #21084 review).-func TestDefaultSetupsNormalizedToJSON(t *testing.T) {- for family, setup := range defaultSetups() {- if got := setup["output_format"]; got != "json" {- t.Errorf("defaultSetups()[%q].output_format = %v, want json", family, got)- }- }-}--// TestParserOutputFormatAlwaysJSON pins the observable contract that survives-// the literal cleanup: however the component is constructed — defaults, an-// empty param map, or a legacy per-family override — every family ends up on-// the single JSON output protocol.-func TestParserOutputFormatAlwaysJSON(t *testing.T) {- cases := []struct {- name string- params map[string]any- }{- {"defaults", nil},- {"empty params", map[string]any{}},- {"legacy markdown override", map[string]any{"pdf": map[string]any{"output_format": "markdown"}}},- {"legacy text override", map[string]any{"text&code": map[string]any{"output_format": "text"}}},- }- for _, c := range cases {- t.Run(c.name, func(t *testing.T) {- comp, err := NewParserComponent(c.params)- if err != nil {- t.Fatalf("NewParserComponent: %v", err)- }- pc, ok := comp.(*ParserComponent)- if !ok {- t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)- }- if len(pc.setups) == 0 {- t.Fatal("expected non-empty setups")- }- for family, setup := range pc.setups {- if got := setup["output_format"]; got != "json" {- t.Errorf("family %q output_format = %v, want json", family, got)- }- }- })- }-}diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.goindex 58b216e5b6..820f61f9f9 100644--- a/internal/ingestion/component/parser_test.go+++ b/internal/ingestion/component/parser_test.go@@ -181,11 +181,9 @@ func TestParserComponentInvoke_LanguageFromGlobals(t *testing.T) {}}-func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {+func TestNewParserComponentSkipsNonFamilyParams(t *testing.T) {component, err := NewParserComponent(map[string]any{- "pdf": map[string]any{"output_format": "markdown"},- "spreadsheet": map[string]any{"output_format": "html"},- "email": map[string]any{"output_format": "text"},+ "pdf": map[string]any{"parse_method": "deepdoc"},"allowed_output_format": map[string]any{"pdf": []any{"json", "markdown"},},@@ -195,11 +193,6 @@ func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {}parserComponent := component.(*ParserComponent)- for _, family := range []string{"pdf", "spreadsheet", "email"} {- if got := parserComponent.setups[family]["output_format"]; got != "json" {- t.Errorf("%s output_format = %v, want json", family, got)- }- }if _, ok := parserComponent.setups["allowed_output_format"]; ok {t.Error("allowed_output_format must not be treated as a parser setup")}@@ -381,24 +374,6 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {}}-// TestParserComponent_New_Defaults constructs a Parser from a nil param map-// and verifies every parser family uses the canonical JSON output format.-func TestParserComponent_New_Defaults(t *testing.T) {- c, err := NewParserComponent(nil)- if err != nil {- t.Fatalf("NewParserComponent(nil): %v", err)- }- pc, ok := c.(*ParserComponent)- if !ok {- t.Fatalf("NewParserComponent returned %T, want *ParserComponent", c)- }- for family, setup := range pc.setups {- if got := setup["output_format"]; got != "json" {- t.Errorf("%s output_format = %v, want json", family, got)- }- }-}-// TestParserComponent_New_Overrides verifies that a non-nil// param map with a file-type entry is layered on top of the// defaults.diff --git a/internal/ingestion/component/pdf_parser_model_test.go b/internal/ingestion/component/pdf_parser_model_test.goindex 5a36076c48..0fa2cc7e45 100644--- a/internal/ingestion/component/pdf_parser_model_test.go+++ b/internal/ingestion/component/pdf_parser_model_test.go@@ -142,7 +142,7 @@ func TestTenantPDFParserModelSelection(t *testing.T) {if form == "id" {ref = "model-b"}- setup := schema.ParserSetup{"parse_method": "deepdoc", "output_format": "json", field: ref}+ setup := schema.ParserSetup{"parse_method": "deepdoc", field: ref}before, _ := json.Marshal(setup)res, handled, err := maybeDispatchPDFVision(t.Context(), db, utility.FileTypePDF,"sample.pdf", []byte("%PDF local protocol fixture"), map[string]any{"tenant_id": "tenant-1"},@@ -257,7 +257,7 @@ func TestTenantPDFParserMissingModelIDIsHandled(t *testing.T) {for _, field := range []string{"parse_method", "layout_recognizer"} {t.Run(provider+"/"+field, func(t *testing.T) {db, _, requests := tenantPDFParserFixture(t, provider)- setup := schema.ParserSetup{"parse_method": "deepdoc", "output_format": "json", field: missingModelID}+ setup := schema.ParserSetup{"parse_method": "deepdoc", field: missingModelID}before, err := json.Marshal(setup)if err != nil {t.Fatal(err)@@ -347,7 +347,7 @@ func TestTenantPDFParserMissingModelIDInvokeDoesNotFallback(t *testing.T) {})setup := map[string]any{- "parse_method": tc.method, "output_format": "json", tc.field: missingModelID,+ "parse_method": tc.method, tc.field: missingModelID,"somark_base_url": fallback.URL + "/setup/somark", "somark_api_key": "setup-key-must-not-be-used","opendataloader_apiserver": fallback.URL + "/setup/opendataloader","opendataloader_api_key": "setup-key-must-not-be-used",diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex 9cf69b4d3b..8ffdeaf770 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -111,7 +111,7 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {t.Fatalf("result=%+v", result)}- resultMD, err := dispatchMonkeyOCRv2PDF(context.Background(), nil, "sample.pdf", []byte("pdf"), "", schema.ParserSetup{"monkeyocrv2_server_url": server.URL, "output_format": "markdown"}, "")+ resultMD, err := dispatchMonkeyOCRv2PDF(context.Background(), nil, "sample.pdf", []byte("pdf"), "", schema.ParserSetup{"monkeyocrv2_server_url": server.URL}, "")if err != nil {t.Fatal(err)}@@ -175,14 +175,12 @@ func TestDispatchPaddleOCRPDFUsesStructuredOutput(t *testing.T) {resolvePaddleOCRModelForDispatch = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, error) {return &paddleOCRFakeDriver{pages: []modelModule.OCRPage{{Index: 0, Blocks: []modelModule.OCRBlock{{Text: "# Heading\n\nBody"}}}}}, "ocr-model", &modelModule.APIConfig{}, nil}- for _, format := range []string{"json", "markdown", ""} {- result, err := dispatchPaddleOCRPdf(t.Context(), nil, "sample.pdf", nil, "tenant", schema.ParserSetup{"output_format": format}, "model")- if err != nil {- t.Fatal(err)- }- if result.OutputFormat != "json" || len(result.JSON) == 0 {- t.Fatalf("result=%+v", result)- }+ result, err := dispatchPaddleOCRPdf(t.Context(), nil, "sample.pdf", nil, "tenant", schema.ParserSetup{}, "model")+ if err != nil {+ t.Fatal(err)+ }+ if result.OutputFormat != "json" || len(result.JSON) == 0 {+ t.Fatalf("result=%+v", result)}}diff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.goindex 32a4c9508f..8ca4abe79d 100644--- a/internal/ingestion/component/schema/parser.go+++ b/internal/ingestion/component/schema/parser.go@@ -17,7 +17,7 @@package schema// ParserSetup is a per-filetype parser configuration block. The keys are heterogeneous (e.g.,-// `parse_method`, `lang`, `output_format`, `suffix`, `fields`, `vlm`),+// `parse_method`, `lang`, `suffix`, `fields`, `vlm`),// so a free-form map best mirrors the Python dict literal.type ParserSetup map[string]anydiff --git a/internal/ingestion/pipeline/image_ocr_test.go b/internal/ingestion/pipeline/image_ocr_test.goindex 42856485a1..0d039510d9 100644--- a/internal/ingestion/pipeline/image_ocr_test.go+++ b/internal/ingestion/pipeline/image_ocr_test.go@@ -115,7 +115,7 @@ func TestImageOCRSavedConfigToSearchableChunk(t *testing.T) {}const parserID = "Parser:ViewsCaptureLight"config := BuildParserConfig(dsl, map[string]any{parserID: map[string]any{- "image": map[string]any{"ocr_enabled": true, "ocr_model_id": "ocr-model", "output_format": "json"},+ "image": map[string]any{"ocr_enabled": true, "ocr_model_id": "ocr-model"},"enable_vision_enhancement": false,}})for _, row := range []any{diff --git a/internal/ingestion/pipeline/pdf_pages_test.go b/internal/ingestion/pipeline/pdf_pages_test.goindex 2256e23492..35224beee1 100644--- a/internal/ingestion/pipeline/pdf_pages_test.go+++ b/internal/ingestion/pipeline/pdf_pages_test.go@@ -76,7 +76,7 @@ func TestNormalizeParserConfigPages(t *testing.T) {t.Run("no pages key -> unchanged", func(t *testing.T) {cfg := map[string]any{"Parser:X": map[string]any{- "pdf": map[string]any{"output_format": "json"},+ "pdf": map[string]any{"parse_method": "DeepDOC"},},}if err := NormalizeParserConfigPages(cfg); err != nil {@@ -86,7 +86,7 @@ func TestNormalizeParserConfigPages(t *testing.T) {if _, ok := pdf["pages"]; ok {t.Error("pages key should not exist")}- if pdf["output_format"] != "json" {+ if pdf["parse_method"] != "DeepDOC" {t.Error("other fields should not be touched")}})diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex 4f2f6f4471..c56b3d2208 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -21,7 +21,7 @@ func generalDSL(t *testing.T) []byte {"params": map[string]any{"outputs": map[string]any{},"pdf": map[string]any{"parse_method": "DeepDOC", "lang": "en"},- "docx": map[string]any{"output_format": "json"},+ "docx": map[string]any{"lang": "en"},},},},diff --git a/internal/ingestion/pipeline/template_output_format_test.go b/internal/ingestion/pipeline/template_output_format_test.goindex c8cc2cc45d..fe14309cfb 100644--- a/internal/ingestion/pipeline/template_output_format_test.go+++ b/internal/ingestion/pipeline/template_output_format_test.go@@ -17,16 +17,17 @@ package pipelineimport ("encoding/json"+ "os"+ "path/filepath"+ "strings""testing")// parserSetupParamsKeys are the Parser params keys that are NOT per-family-// setups: the wire-format declaration, the global vision settings and the-// legacy allow-list. Every other object-valued params key is a file family-// whose setup map may carry output_format.+// setups: the wire-format declaration and the global vision settings. Every+// other object-valued params key is a file family.var parserSetupParamsKeys = map[string]bool{"outputs": true,- "allowed_output_format": true,"enable_vision_enhancement": true,"vlm": true,}@@ -34,12 +35,9 @@ var parserSetupParamsKeys = map[string]bool{// TestBuiltinTemplatesCarryNoParserOutputFormat pins that no builtin// ingestion template declares a Parser family's output_format.//-// The Go Parser component exposes JSON only: the Parser runtime overwrites-// every family's output_format with "json" when the component is built, so a-// template-baked value never reaches the parser that would read it. The key-// is therefore dead configuration in the templates: it states an intent the-// runtime discards, and leaving it in place invites a reader to believe the-// value is honoured.+// The Go Parser component exposes JSON only: no parser reads the key, so a+// template-baked value states an intent that is never honoured. Leaving it in+// place invites a reader to believe the value matters.//// Both copies are checked. dsl.components holds the params the runtime// builds the Parser component from; dsl.graph.nodes holds the per-family@@ -93,7 +91,7 @@ func assertNoParserOutputFormatInParams(t *testing.T, ref string, root map[strincontinue}if value, found := setup["output_format"]; found {- t.Errorf("%s: Parser %s family %q declares output_format = %v; the Parser runtime overwrites it with \"json\", so the key is dead configuration", ref, cpnID, family, value)+ t.Errorf("%s: Parser %s family %q declares output_format = %v; no parser reads the key", ref, cpnID, family, value)}}}@@ -120,8 +118,50 @@ func assertNoParserOutputFormatInGraph(t *testing.T, ref string, root map[stringsetup, _ := rawSetup.(map[string]any)fileFormat, _ := setup["fileFormat"].(string)if value, found := setup["output_format"]; found {- t.Errorf("%s: Parser node %s family %q declares output_format = %v; the Parser runtime overwrites it with \"json\", so the key is dead configuration", ref, id, fileFormat, value)+ t.Errorf("%s: Parser node %s family %q declares output_format = %v; no parser reads the key", ref, id, fileFormat, value)}}}}++// TestAgentTemplatesCarryNoParserOutputFormat extends the pin to the seeded+// canvas templates under internal/agent/templates. The ingestion-pipeline+// canvases there (title_chunker, chunk_summary, advanced_ingestion_pipeline)+// carry Parser nodes too, and the dao seeder copies their DSL verbatim into+// the canvas_template table, so a baked key there reaches live canvases.+// compiler.json is a bare DSL fixture without the metadata wrapper, so the+// "dsl" key is optional.+func TestAgentTemplatesCarryNoParserOutputFormat(t *testing.T) {+ dir := filepath.Join("..", "..", "agent", "templates")+ entries, err := os.ReadDir(dir)+ if err != nil {+ t.Fatalf("read agent templates: %v", err)+ }+ checked := 0+ for _, entry := range entries {+ name := entry.Name()+ if entry.IsDir() || !strings.HasSuffix(name, ".json") {+ continue+ }+ raw, err := os.ReadFile(filepath.Join(dir, name))+ if err != nil {+ t.Fatalf("read %s: %v", name, err)+ }+ var wrapper map[string]any+ if err := json.Unmarshal(raw, &wrapper); err != nil {+ t.Fatalf("decode %s: %v", name, err)+ }+ root, ok := wrapper["dsl"].(map[string]any)+ if !ok {+ root = wrapper+ }+ checked+++ t.Run(name, func(t *testing.T) {+ assertNoParserOutputFormatInParams(t, name, root)+ assertNoParserOutputFormatInGraph(t, name, root)+ })+ }+ if checked == 0 {+ t.Fatal("no agent templates found")+ }+}diff --git a/internal/parser/parser/audio_parser.go b/internal/parser/parser/audio_parser.goindex 8bdf5e0188..bbcd22a1c3 100644--- a/internal/parser/parser/audio_parser.go+++ b/internal/parser/parser/audio_parser.go@@ -39,26 +39,13 @@ var audioExtensions = map[string]bool{}// AudioParser handles audio files for transcription.-type AudioParser struct {- OutputFormat string-}+type AudioParser struct{}// NewAudioParser constructs an AudioParser.func NewAudioParser() *AudioParser {return &AudioParser{}}-// ConfigureFromSetup reads audio-specific configuration from the-// parser setup map.-func (p *AudioParser) ConfigureFromSetup(setup map[string]any) {- if p == nil || setup == nil {- return- }- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }-}-// ParseWithResult implements ParseResultProducer. It validates the// file extension against the audio extension whitelist. The actual// speech-to-text transcription happens via maybeDispatchAudio at the@@ -74,13 +61,8 @@ func (p *AudioParser) ParseWithResult(ctx context.Context, filename string, data}}- outFmt := p.OutputFormat- if outFmt == "" {- outFmt = "text"- }-return ParseResult{- OutputFormat: outFmt,+ OutputFormat: "json",File: map[string]any{"name": filename,"size": len(data),diff --git a/internal/parser/parser/audio_parser_test.go b/internal/parser/parser/audio_parser_test.goindex 12e03393fe..e47d1b8d27 100644--- a/internal/parser/parser/audio_parser_test.go+++ b/internal/parser/parser/audio_parser_test.go@@ -24,16 +24,13 @@ import (func TestAudioParser_ValidExtension(t *testing.T) {ctx := t.Context()p := NewAudioParser()- p.ConfigureFromSetup(map[string]any{- "output_format": "text",- })result := p.ParseWithResult(ctx, "test.mp3", []byte{1, 2, 3, 4})if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- if result.OutputFormat != "text" {- t.Errorf("output_format = %q, want text", result.OutputFormat)+ if result.OutputFormat != "json" {+ t.Errorf("output_format = %q, want json", result.OutputFormat)}if name, ok := result.File["name"].(string); !ok || name != "test.mp3" {t.Errorf("name = %q, want test.mp3", name)@@ -63,18 +60,6 @@ func TestAudioParser_InvalidExtension(t *testing.T) {}}-func TestAudioParser_ConfigureNilSafe(t *testing.T) {- ctx := t.Context()- p := NewAudioParser()- p.ConfigureFromSetup(nil)- p.ConfigureFromSetup(map[string]any{})- // Should not panic.- result := p.ParseWithResult(ctx, "test.wav", []byte{})- if result.Err != nil {- t.Errorf("unexpected error: %v", result.Err)- }-}-func TestAudioParser_DefaultOutputFormat(t *testing.T) {ctx := t.Context()p := NewAudioParser()@@ -82,7 +67,7 @@ func TestAudioParser_DefaultOutputFormat(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- if result.OutputFormat != "text" {- t.Errorf("output_format = %q, want text (default)", result.OutputFormat)+ if result.OutputFormat != "json" {+ t.Errorf("output_format = %q, want json", result.OutputFormat)}}diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 88274d3e75..499463e230 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -49,7 +49,6 @@ const (// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.type CSVParser struct {ParseMethod string- OutputFormat stringTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType string@@ -74,9 +73,6 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["parse_method"].(string); ok && v != "" {p.ParseMethod = v}- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }deprecatedHTML4Excel(setup, p.String())deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {@@ -105,7 +101,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [ctx, filename, data, "CSV",p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,- p.OutputFormat,)case "", "csv":// Continue with the local CSV parser.diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex 33fd994dec..55ad19f27c 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -102,9 +102,6 @@ func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {csvData := []byte("col1,col2\nval1,val2\n")p := NewCSVParser()- p.ConfigureFromSetup(map[string]any{- "output_format": "json",- })res := p.ParseWithResult(context.Background(), "test.csv", csvData)if res.Err != nil {@@ -127,9 +124,6 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {func TestCSVParser_EmptyContent(t *testing.T) {p := NewCSVParser()- p.ConfigureFromSetup(map[string]any{- "output_format": "json",- })res := p.ParseWithResult(context.Background(), "empty.csv", []byte(" \n\n "))if res.Err != nil {diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 1aff00bfac..5aaff1c8d6 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -37,14 +37,7 @@ func (p *DOCParser) String() string {return "DOCParser"}-func (p *DOCParser) ConfigureFromSetup(setup map[string]any) {- // DOC always emits JSON (the DSL hardcodes output_format to "json" via- // normalizeParserOutputFormats); the legacy text/markdown output paths were- // removed with the dead ParseResult.Text field.-}--// ParseWithResult extracts text for the DOC family and converts it to the-// requested output format (defaulting to "json" to match Python's default).+// ParseWithResult extracts text for the DOC family as structured JSON items.func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {irJSON, mdText, plainText, _, err := docExtract(data, "doc")if err != nil {@@ -59,12 +52,13 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}}- // The DSL hardcodes output_format to "json" (normalizeParserOutputFormats),- // so the DOC parser always emits structured JSON items. Prefer the IR- // sections, which recover table/list structure for the chunker's HTML-table- // rendering (#20918); fall back to the flattened text only when the IR yields- // no usable sections, so a document whose IR is empty/unusable still keeps- // its content rather than silently producing 0 chunks.+ // Prefer the structured IR, which office_oxide recovers table+ // (and list) structure for. Emitting structured sections lets the chunker+ // render tables as HTML (#20918) instead of losing them to plain text — the+ // legacy path flattened the whole document (tables included) into a single+ // text item. Fall back to the flattened text only when the IR yields no+ // usable sections, so a document whose IR is empty/unusable still keeps its+ // content (PlainText / ToMarkdown) rather than silently producing 0 chunks.if strings.TrimSpace(irJSON) != "" {budget := newEmbeddedMediaBudget()if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {diff --git a/internal/parser/parser/docx_caption_cgo_test.go b/internal/parser/parser/docx_caption_cgo_test.goindex 12c944dbc0..1987dd1e94 100644--- a/internal/parser/parser/docx_caption_cgo_test.go+++ b/internal/parser/parser/docx_caption_cgo_test.go@@ -111,7 +111,6 @@ func docxCaptionFindTable(t *testing.T, res ParseResult) *entity.TableData {func docxCaptionParse(t *testing.T, bodyInner string) ParseResult {p := NewDOCXParser()- p.outputFormat = "json"res := p.ParseWithResult(t.Context(), "caption.docx", docxCaptionTestDOCX(t, bodyInner))if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.goindex d0c3aa982c..13521f6f8b 100644--- a/internal/parser/parser/docx_parser.go+++ b/internal/parser/parser/docx_parser.go@@ -19,7 +19,6 @@ import ("context""errors""fmt"- "strings"deepdocoffice "ragflow/internal/deepdoc/parser/office")@@ -37,7 +36,6 @@ var docxExtract = deepdocoffice.OpenAndExtract// so they compile and test without native libraries.type DOCXParser struct {libType string- outputFormat string // from DSL config; "json" or "markdown"RemoveTOC boolRemoveHeaderFooter bool}@@ -47,15 +45,11 @@ func NewDOCXParser() *DOCXParser {}// ConfigureFromSetup implements parserSetupConfigurer, receiving the-// DSL "docx" family setup map. The output_format key drives whether-// ParseWithResult produces JSON items (structured) or Markdown.+// DSL "docx" family setup map.func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {if p == nil || setup == nil {return}- if v, ok := setup["output_format"].(string); ok && v != "" {- p.outputFormat = v- }if v, ok := setup["remove_toc"].(bool); ok {p.RemoveTOC = v}@@ -64,13 +58,9 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {}}-// ParseWithResult produces structured JSON items (when-// p.outputFormat == "json") or markdown (default) from a-// docx document. The JSON path emits bounded image items; the-// Markdown path attaches bounded figure metadata to the file result.-//-// JSON path mirrors python parser.py:_docx() output_format == "json".-// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().+// ParseWithResult produces structured JSON items from a docx document.+// It emits bounded image items and attaches figure metadata to the file+// result.//// File["format"] in the returned ParseResult reflects the effective// container format after magic-byte sniffing (e.g. "doc" for an OLE@@ -92,7 +82,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data// opens the payload as ZIP and fails closed, so RemoveHeaderFooter becomes// a no-op for legacy OLE inputs. This is acceptable — legacy DOC has no// OOXML header/footer parts to strip.- irJSON, md, _, mdErr, err := docxExtract(data, format)+ irJSON, _, _, _, err := docxExtract(data, format)if err != nil {if errors.Is(err, deepdocoffice.ErrOfficeCGORequired) {return ParseResult{Err: fmt.Errorf("%w: %s", ErrOfficeCGORequired, filename)}@@ -105,9 +95,9 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data"format": format,}- // Extract IR JSON for section building (JSON path) and- // embedded-image extraction (both paths). The budget bounds image- // payloads so the downstream VLM enhancement stays within limits.+ // Extract IR JSON for section building and embedded-image extraction.+ // The budget bounds image payloads so the downstream VLM enhancement+ // stays within limits.budget := newEmbeddedMediaBudget()if irJSON != "" {figures := extractDOCXFiguresFromIR(irJSON, budget)@@ -116,98 +106,46 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data}}- if p.outputFormat == "json" {- // The JSON path is built entirely from the structured IR, so an- // empty IR is a fatal parse error for this output format — the- // Markdown path below still has the plain-text fallback. The- // OpenAndExtract contract swallows per-view errors, so we cannot- // distinguish "IR failed" from "IR legitimately empty"; either way- // the JSON path has nothing to build from, so we fail loud- // (Finding B) rather than emit empty/garbage JSON sections.- if irJSON == "" {- return ParseResult{Err: fmt.Errorf("docx extract: empty IR, cannot build JSON output")}- }- // TEMPORARY Caption-styled-paragraph detection (office_oxide does- // not yet expose paragraph styles; yfedoseev/office_oxide#455).- // Remove once office_oxide is upgraded to emit them.- tableCaptions, capErr := extractDOCXTableCaptions(data)- if capErr != nil {- tableCaptions = nil- }- sections := buildDOCXJSONSectionsWithCaptions(irJSON, budget, tableCaptions)- if err := ctx.Err(); err != nil {- return ParseResult{Err: err}- }- // remove_header_footer: drop sections whose normalized text- // matches a docx header/footer entry (mirrors Python- // parser.py:889-891 extract_docx_header_footer_texts +- // remove_header_footer_docx_sections).- if p.RemoveHeaderFooter {- hfTexts := extractDOCXHeaderFooterTexts(data)- sections = removeDOCXHeaderFooterSections(sections, hfTexts)- }- // remove_toc: filter TOC entries using heading outlines- // (mirrors Python parser.py:892-893 remove_toc_word).- if p.RemoveTOC {- outlines := extractDOCXOutlines(irJSON)- sections = removeTOCWord(sections, outlines, isEnglishItems(sections))- }- if len(sections) == 0 {- sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}- }- return ParseResult{- OutputFormat: "json",- File: fileMeta,- JSON: MapsToItems(sections),- Warnings: budget.warnings(),- }+ // The result is built entirely from the structured IR, so an empty IR+ // is a fatal parse error. The OpenAndExtract contract swallows per-view+ // errors, so we cannot distinguish "IR failed" from "IR legitimately+ // empty"; either way there is nothing to build from, so fail loud+ // rather than emit empty/garbage JSON sections.+ if irJSON == "" {+ return ParseResult{Err: fmt.Errorf("docx extract: empty IR, cannot build JSON output")}}-- // Default / Markdown path.- if mdErr != nil {- // The Markdown view failed but other views survived. Surface the- // Markdown render error rather than silently emitting empty markdown- // (PR review Finding 3). The JSON path above is unaffected because it- // is built from the IR view.- return ParseResult{Err: fmt.Errorf("docx extract: markdown render failed: %w", mdErr)}+ // TEMPORARY Caption-styled-paragraph detection (office_oxide does+ // not yet expose paragraph styles; yfedoseev/office_oxide#455).+ // Remove once office_oxide is upgraded to emit them.+ tableCaptions, capErr := extractDOCXTableCaptions(data)+ if capErr != nil {+ tableCaptions = nil}- markdownPayload := md- // remove_header_footer on Markdown: filter lines by exact match- // (mirrors Python parser.py:923-926 split lines → filter → rejoin).+ sections := buildDOCXJSONSectionsWithCaptions(irJSON, budget, tableCaptions)+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }+ // remove_header_footer: drop sections whose normalized text+ // matches a docx header/footer entry (mirrors Python+ // parser.py:889-891 extract_docx_header_footer_texts ++ // remove_header_footer_docx_sections).if p.RemoveHeaderFooter {hfTexts := extractDOCXHeaderFooterTexts(data)- lines := strings.Split(markdownPayload, "\n")- lineItems := make([]map[string]any, 0, len(lines))- for _, ln := range lines {- lineItems = append(lineItems, map[string]any{"text": ln})- }- lineItems = removeDOCXHeaderFooterSections(lineItems, hfTexts)- rebuilt := make([]string, 0, len(lineItems))- for _, item := range lineItems {- rebuilt = append(rebuilt, itemText(item))- }- markdownPayload = strings.Join(rebuilt, "\n")+ sections = removeDOCXHeaderFooterSections(sections, hfTexts)}- // remove_toc on Markdown: split lines, filter, rejoin- // (mirrors Python parser.py:927-928 remove_toc_word on Markdown).- if p.RemoveTOC && irJSON != "" {+ // remove_toc: filter TOC entries using heading outlines+ // (mirrors Python parser.py:892-893 remove_toc_word).+ if p.RemoveTOC {outlines := extractDOCXOutlines(irJSON)- lines := strings.Split(markdownPayload, "\n")- lineItems := make([]map[string]any, 0, len(lines))- for _, ln := range lines {- lineItems = append(lineItems, map[string]any{"text": ln})- }- filtered := removeTOCWord(lineItems, outlines, isEnglishItems(lineItems))- rebuilt := make([]string, 0, len(filtered))- for _, item := range filtered {- rebuilt = append(rebuilt, itemText(item))- }- markdownPayload = strings.Join(rebuilt, "\n")+ sections = removeTOCWord(sections, outlines, isEnglishItems(sections))+ }+ if len(sections) == 0 {+ sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}return ParseResult{- OutputFormat: "markdown",+ OutputFormat: "json",File: fileMeta,- Markdown: markdownPayload,+ JSON: MapsToItems(sections),Warnings: budget.warnings(),}}diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.goindex 1141a0ded0..1a147dc860 100644--- a/internal/parser/parser/docx_parser_cgo_test.go+++ b/internal/parser/parser/docx_parser_cgo_test.go@@ -12,7 +12,6 @@ import (func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {ctx := t.Context()p := NewDOCXParser()- p.outputFormat = "json"data := minimalDOCX(t, "Hello from JSON path")res := p.ParseWithResult(ctx, "sample.docx", data)if res.Err != nil {@@ -34,48 +33,18 @@ func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {}}-func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) {- ctx := t.Context()+func TestDOCXParser_ConfigureFromSetup_Options(t *testing.T) {p := NewDOCXParser()- p.ConfigureFromSetup(map[string]any{"output_format": "json"})- if p.outputFormat != "json" {- t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "json")+ p.ConfigureFromSetup(map[string]any{"remove_toc": true, "remove_header_footer": true})+ if !p.RemoveTOC {+ t.Error("RemoveTOC not configured")+ }+ if !p.RemoveHeaderFooter {+ t.Error("RemoveHeaderFooter not configured")}if p.libType != "" {t.Errorf("libType unexpectedly = %q", p.libType)}- // Full round-trip: json config → json output- data := minimalDOCX(t, "Config test")- res := p.ParseWithResult(ctx, "sample.docx", data)- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- if res.OutputFormat != "json" {- t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "json")- }- if len(res.JSON) == 0 {- t.Error("JSON items is empty")- }-}--func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) {- ctx := t.Context()- p := NewDOCXParser()- p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})- if p.outputFormat != "markdown" {- t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "markdown")- }- data := minimalDOCX(t, "Config md test")- res := p.ParseWithResult(ctx, "sample.docx", data)- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- if res.OutputFormat != "markdown" {- t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "markdown")- }- if res.Markdown == "" {- t.Error("Markdown is empty")- }}func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {@@ -86,11 +55,11 @@ func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if got, want := res.OutputFormat, "markdown"; got != want {+ if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if res.Markdown == "" {- t.Fatal("Markdown is empty; want parsed content")+ if len(res.JSON) == 0 {+ t.Fatal("JSON items is empty; want parsed content")}}@@ -139,41 +108,29 @@ func writeZipFile(t *testing.T, zw *zip.Writer, name, body string) {// that fixed this defect class; v0.1.12 is the pinned version.func TestDOCXParser_PreservesXMLCharacterReferences(t *testing.T) {cases := []struct {- name string- run string // run text as it appears inside <w:t>, entities unescaped- want string- wantMarkdown string+ name string+ run string // run text as it appears inside <w:t>, entities unescaped+ want string}{- {"named", "<SEP>", "<SEP>", `\<SEP\>`},- {"decimal", "<SEP>", "<SEP>", `\<SEP\>`},- {"hex", "<SEP>", "<SEP>", `\<SEP\>`},- {"ampersand", "A&B", "A&B", "A&B"},- {"apostrophe", "it's", "it's", "it's"},- {"emdash", "a—b", "a—b", "a—b"},+ {"named", "<SEP>", "<SEP>"},+ {"decimal", "<SEP>", "<SEP>"},+ {"hex", "<SEP>", "<SEP>"},+ {"ampersand", "A&B", "A&B"},+ {"apostrophe", "it's", "it's"},+ {"emdash", "a—b", "a—b"},}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {data := minimalDOCX(t, tc.run)- jsonParser := NewDOCXParser()- jsonParser.ConfigureFromSetup(map[string]any{"output_format": "json"})- jsonRes := jsonParser.ParseWithResult(t.Context(), "entities.docx", data)- if jsonRes.Err != nil {- t.Fatalf("ParseWithResult(json): %v", jsonRes.Err)+ p := NewDOCXParser()+ res := p.ParseWithResult(t.Context(), "entities.docx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)}- if got := joinItemTexts(ItemsToMaps(jsonRes.JSON)); !strings.Contains(got, tc.want) {+ if got := joinItemTexts(ItemsToMaps(res.JSON)); !strings.Contains(got, tc.want) {t.Errorf("json text = %q, want it to contain %q", got, tc.want)}-- mdParser := NewDOCXParser()- mdParser.ConfigureFromSetup(map[string]any{"output_format": "markdown"})- mdRes := mdParser.ParseWithResult(t.Context(), "entities.docx", data)- if mdRes.Err != nil {- t.Fatalf("ParseWithResult(markdown): %v", mdRes.Err)- }- if !strings.Contains(mdRes.Markdown, tc.wantMarkdown) {- t.Errorf("markdown = %q, want it to contain %q", mdRes.Markdown, tc.wantMarkdown)- }})}}diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex a14cc293a0..c55baf39ae 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -39,23 +39,22 @@ import ()// EmailParser parses .eml (RFC 5322 email) and .msg (Outlook OLE2) files-// into structured JSON or plain-text output. Mirrors Python's _email()-// method in rag/flow/parser/parser.py, whose .msg branch uses extract_msg.+// into structured JSON output. Mirrors Python's _email() method in+// rag/flow/parser/parser.py, whose .msg branch uses extract_msg.type EmailParser struct {fields []string}+var defaultEmailFields = []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"}+func NewEmailParser() *EmailParser {- return &EmailParser{}+ return &EmailParser{fields: append([]string(nil), defaultEmailFields...)}}func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {if p == nil || setup == nil {return}- // Email always emits JSON (the DSL hardcodes output_format to "json" via- // normalizeParserOutputFormats); the legacy text output path was removed with- // the dead ParseResult.Text field.if raw, ok := setup["fields"]; ok {switch list := raw.(type) {case []string:@@ -70,7 +69,7 @@ func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {}}if len(p.fields) == 0 {- p.fields = []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"}+ p.fields = append([]string(nil), defaultEmailFields...)}}@@ -406,8 +405,6 @@ func attachmentFilename(part *multipart.Part) string {return "attachment.bin"}-// -- HTML body → visible text (text output only) ---// decodeMailPayload attempts charset decoding using the unified DecodeToUTF8 helper.func decodeMailPayload(payload []byte, charset string) string {if len(payload) == 0 {@@ -617,10 +614,8 @@ func recoverParse(fn func() ParseResult) (res ParseResult, panicked bool) {// without failing the whole email (mirrors email.py's per-attachment// try/except).//-// The function returns both forms the caller needs:-// - extraItems: structured JSON items (one per non-empty text segment),-// each carrying only "text" and "doc_type_kwd", for the JSON output path.-// - text: the concatenated attachment text, for the text output path.+// The function returns structured JSON items (one per non-empty text+// segment), each carrying only "text" and "doc_type_kwd".//// Re-chunking stops at one level of nesting: a nested email is parsed (so its// body becomes retrievable) but its own attachments are not re-chunked, to@@ -677,13 +672,9 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s// Reuse the top-level field configuration (including "body") so a// nested .eml/.msg is parsed for its body instead of with a fresh,// unconfigured parser that would only emit metadata and index- // garbage. We always request JSON output for the nested parse so- // the extracted "text" field is returned clean (the text output- // path would otherwise also flatten metadata into the result).+ // garbage.ep := NewEmailParser()- ep.ConfigureFromSetup(map[string]any{- "fields": p.fields,- })+ ep.ConfigureFromSetup(map[string]any{"fields": p.fields})res, panicked = recoverParse(func() ParseResult {return ep.parseEmail(ctx, fn, []byte(payload), depth+1)})diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex 2299d23cee..de78bacf8c 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -54,8 +54,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "to", "cc", "date", "subject", "body", "metadata"},+ "fields": []string{"from", "to", "cc", "date", "subject", "body", "metadata"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -223,8 +222,7 @@ func TestEmailParser_MsgSupported(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},+ "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},})result := p.ParseWithResult(ctx, "sample.msg", data)@@ -314,8 +312,7 @@ func TestEmailParser_MsgMetadataAlwaysPresent(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "subject"}, // "metadata" intentionally absent+ "fields": []string{"from", "subject"}, // "metadata" intentionally absent})result := p.ParseWithResult(ctx, "sample.msg", data)@@ -418,8 +415,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "body", "attachments"},+ "fields": []string{"from", "body", "attachments"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -496,8 +492,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "body", "attachments"},+ "fields": []string{"from", "body", "attachments"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -553,8 +548,7 @@ func TestEmailParser_Multipart(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "body"},+ "fields": []string{"from", "body"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -590,7 +584,6 @@ func TestEmailParser_MetadataAlwaysPresent(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",// Note: "metadata" is intentionally NOT in fields."fields": []string{"from", "subject", "body"},})@@ -651,8 +644,7 @@ func TestEmailParser_TextHTMLAlwaysPresent(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "body"},+ "fields": []string{"from", "body"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -713,8 +705,7 @@ func TestEmailParser_AttachmentSearchableJSON(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "body", "attachments"},+ "fields": []string{"from", "body", "attachments"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -775,8 +766,7 @@ func TestEmailParser_AttachmentEmptyPayloadSkipped(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "body", "attachments"},+ "fields": []string{"from", "body", "attachments"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -831,8 +821,7 @@ func TestEmailParser_AttachmentUnknownExtSkipped(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "body", "attachments"},+ "fields": []string{"from", "body", "attachments"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))@@ -877,7 +866,6 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "json",// NOTE: "body" is intentionally NOT in fields."fields": []string{"from", "attachments"},})@@ -1301,7 +1289,6 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {"Subject: Q3\r\n" + tt.headers + "\r\n\r\n" + tt.body + "\r\n"p := NewEmailParser()- p.ConfigureFromSetup(map[string]any{"output_format": "json"})result := p.ParseWithResult(t.Context(), "test.eml", []byte(raw))if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 1edba36d67..ce033a5835 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -49,7 +49,6 @@ var ssrfAllowLoopback booltype MarkdownParser struct {libType stringParseMethod string- OutputFormat stringVLM map[string]anyFlattenMediaToText bool// FetchRemoteImages controls whether HTTP(S) Markdown images are@@ -77,9 +76,6 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["parse_method"].(string); ok && v != "" {p.ParseMethod = v}- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }if v, ok := setup["vlm"].(map[string]any); ok {p.VLM = v}@@ -89,8 +85,7 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {}// ParseWithResult implements ParseResultProducer (plan §6.5) and-// returns a structured Markdown payload that mirrors the Python-// parser's `output_format == "json"` shape. Each top-level block+// returns a structured JSON payload. Each top-level block// emits one item with `text` + `doc_type_kwd: "text"`. When the// block contains a Markdown image reference (), the image// data is resolved and the item carries `doc_type_kwd: "image"` withdiff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 3218b3e246..aee919a9f2 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -394,16 +394,12 @@ func TestMarkdownParser_ConfigureFromSetup(t *testing.T) {p, _ := NewMarkdownParser(GoMarkdown)p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc",- "output_format": "json","vlm": map[string]any{"llm_id": "gpt-4-vision"},"flatten_media_to_text": false,})if p.ParseMethod != "deepdoc" {t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc")}- if p.OutputFormat != "json" {- t.Fatalf("OutputFormat = %q, want %q", p.OutputFormat, "json")- }if p.VLM == nil {t.Fatal("VLM is nil; want map")}diff --git a/internal/parser/parser/office_detect_test.go b/internal/parser/parser/office_detect_test.goindex ad2ea035ee..52db5a87fb 100644--- a/internal/parser/parser/office_detect_test.go+++ b/internal/parser/parser/office_detect_test.go@@ -73,7 +73,6 @@ func TestDOCXParser_FallsBackToDOCForOLEHeader(t *testing.T) {func TestDOCXParser_JSONEmptyIRReturnsError(t *testing.T) {ctx := t.Context()p := NewDOCXParser()- p.ConfigureFromSetup(map[string]any{"output_format": "json"})orig := docxExtractdefer func() { docxExtract = orig }()@@ -88,40 +87,16 @@ func TestDOCXParser_JSONEmptyIRReturnsError(t *testing.T) {}}-// TestDOCXParser_MarkdownViewErrorPropagated locks Finding 3: when the-// native backend's Markdown view fails but the other views survive,-// OpenAndExtract surfaces the Markdown error via mdErr and the Markdown output-// path must fail loud instead of silently emitting empty markdown. The JSON-// path is unaffected because it is built from the IR view.-func TestDOCXParser_MarkdownViewErrorPropagated(t *testing.T) {+// TestDOCXParser_AllViewsEmptyReturnsError locks the engine-level contract+// at the parser boundary: when the native backend opens the document but+// every view (IR / Markdown / PlainText) is empty, OpenAndExtract reports a+// joined error, and the parser must fail loud rather than silently emit+// empty JSON. (office_oxide always returns a structured IR for an openable+// doc, so this all-empty error is simulated via the seam rather than a real+// fixture.)+func TestDOCXParser_AllViewsEmptyReturnsError(t *testing.T) {ctx := t.Context()p := NewDOCXParser()- p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})-- orig := docxExtract- defer func() { docxExtract = orig }()- docxExtract = func(data []byte, format string) (string, string, string, error, error) {- // IR and plain text survive; only the Markdown view fails.- return "<ir/>", "", "plain text fallback", fmt.Errorf("markdown boom"), nil- }-- res := p.ParseWithResult(ctx, "broken.docx", []byte("PK\x03\x04"))- if res.Err == nil {- t.Fatal("expected error when only the Markdown view fails, got nil")- }-}--// TestDOCXParser_AllViewsEmptyMarkdownReturnsError locks the engine-level-// contract from Finding 1 at the parser boundary: when the native backend-// opens the document but every view (IR / Markdown / PlainText) is empty,-// OpenAndExtract reports a joined error, and the default Markdown output path-// must fail loud rather than silently emit empty markdown. (office_oxide-// always returns a structured IR for an openable doc, so this all-empty error-// is simulated via the seam rather than a real fixture.)-func TestDOCXParser_AllViewsEmptyMarkdownReturnsError(t *testing.T) {- ctx := t.Context()- p := NewDOCXParser()- p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})orig := docxExtractdefer func() { docxExtract = orig }()diff --git a/internal/parser/parser/office_image_fixture_cgo_test.go b/internal/parser/parser/office_image_fixture_cgo_test.goindex 2241e27f08..dedececab0 100644--- a/internal/parser/parser/office_image_fixture_cgo_test.go+++ b/internal/parser/parser/office_image_fixture_cgo_test.go@@ -34,7 +34,6 @@ func TestDOCXParser_RealTableImage(t *testing.T) {t.Fatal(err)}p := NewDOCXParser()- p.ConfigureFromSetup(map[string]any{"output_format": "json"})result := p.ParseWithResult(t.Context(), "docx_table_image.docx", data)if result.Err != nil {t.Fatal(result.Err)diff --git a/internal/parser/parser/pdf_analyzer_dispatch_test.go b/internal/parser/parser/pdf_analyzer_dispatch_test.goindex 0752f7f616..4517cd01e6 100644--- a/internal/parser/parser/pdf_analyzer_dispatch_test.go+++ b/internal/parser/parser/pdf_analyzer_dispatch_test.go@@ -254,7 +254,7 @@ func TestParsePDFWithDeepDocOptions_RoutesRegisteredMethodToFactory(t *testing.Treturn &deepdoctype.ParseResult{}, nil}res := parsePDFWithDeepDocOptions(context.Background(), "f.pdf", []byte("dummy"),- pdfPostProcessOptions{outputFormat: "json", parseMethod: "ext_wire"}, parseFn)+ pdfPostProcessOptions{parseMethod: "ext_wire"}, parseFn)if res.Err != nil {t.Fatalf("unexpected error: %v", res.Err)}@@ -281,7 +281,7 @@ func TestParsePDFWithDeepDocOptions_EmptyMethodUsesInProcess(t *testing.T) {return &deepdoctype.ParseResult{}, nil}res := parsePDFWithDeepDocOptions(context.Background(), "f.pdf", []byte("dummy"),- pdfPostProcessOptions{outputFormat: "json"}, parseFn)+ pdfPostProcessOptions{}, parseFn)if res.Err != nil {t.Fatalf("unexpected error: %v", res.Err)}diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex aa57203814..7ab8d5c55e 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -20,7 +20,6 @@ import ("context""errors""fmt"- "image""os""sort""strings"@@ -31,7 +30,6 @@ import ("ragflow/internal/common"deepdocpdf "ragflow/internal/deepdoc/parser/pdf"pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"- "ragflow/internal/deepdoc/parser/pdf/util"deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/utility")@@ -65,7 +63,6 @@ type PDFParser struct {RemoveTOC boolRemoveHeaderFooter boolEnableMultiColumn bool- OutputFormat stringParseMethod string// UseTenantModelConfig disables global endpoint/credential/option fallback// for explicitly resolved SoMark/OpenDataLoader instances. Runtime-only;@@ -121,7 +118,6 @@ func NewPDFParser() *PDFParser {Model: "DeepDoc@buildin@ragflow",LibType: "pdf_oxide",ParseMethod: "deepdoc",- OutputFormat: "json",MinerUBackend: "",MinerUPollTimeout: minerUPollTimeout,OpenDataLoaderTimeout: 600,@@ -179,9 +175,6 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["mineru_timeout_seconds"].(float64); ok && v > 0 {p.MinerUPollTimeout = time.Duration(v * float64(time.Second))}- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }if v, ok := setup["docling_server_url"].(string); ok && v != "" {p.DoclingServerURL = v}@@ -373,7 +366,7 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [if err != nil {return ParseResult{Err: fmt.Errorf("parser: plain_text: %w", err)}}- return pdfItemsToResult(filename, items, p.OutputFormat, pageCount)+ return pdfItemsToResult(filename, items, pageCount)case "mineru":return parsePDFWithMinerU(ctx, filename, data, p)case "docling":@@ -391,7 +384,6 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [cfg.RemoveHeaderFooter = p.RemoveHeaderFootercfg.OnPageDone = p.OnPageDoneres := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{- outputFormat: p.OutputFormat,zoom: cfg.Zoom,enableMultiColumn: p.EnableMultiColumn,flattenMediaToText: p.FlattenMediaToText,@@ -553,8 +545,7 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR// the parser-phase memory peak. Image/table sections keep only their PDF// positions; the chunker re-acquires the source PDF and crops on demand at// index time, and the VLM path crops on demand when it needs to describe a- // figure/table. The Markdown path below still inlines, because markdown- // output embeds images directly and has no downstream on-demand consumer.+ // figure/table.items := pdflayout.SectionsToJSON(processed.Sections)if len(items) == 0 {@@ -586,31 +577,6 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR}}-func pdfParseResultToMarkdownWithOptions(filename string, parsed *deepdoctype.ParseResult, opts pdfPostProcessOptions) ParseResult {- if parsed == nil {- return ParseResult{Err: fmt.Errorf("parser: nil DeepDOC PDF result for %s", filename)}- }- processed := *parsed- processed.Sections = append([]deepdoctype.Section(nil), parsed.Sections...)- processed.Outlines = append([]deepdoctype.Outline(nil), parsed.Outlines...)- if opts.enableMultiColumn && opts.pageWidth <= 0 {- opts.pageWidth = firstPDFPageWidth(processed.PageWidth)- }- applyPDFPostProcess(&processed, opts)- defer processed.Close()- cropMediaSections(&processed)-- return ParseResult{- OutputFormat: "markdown",- File: map[string]any{- "name": filename,- "page_count": len(processed.PageHeight),- "outline": outlinesToFileMeta(processed.Outlines),- },- Markdown: sectionsToMarkdown(processed.Sections),- }-}-func outlinesToFileMeta(outlines []deepdoctype.Outline) []map[string]any {if len(outlines) == 0 {return []map[string]any{}@@ -643,247 +609,6 @@ func firstPageNumber(raw any) int {}}-func sectionsToMarkdown(sections []deepdoctype.Section) string {- return pdflayout.SectionsToMarkdown(sections)-}--// cropMediaSections crops figure and table sections from rendered PDF page-// images while the engine is still alive, populating sec.Image.-// It is used by both the JSON and Markdown serialization paths.-func cropMediaSections(result *deepdoctype.ParseResult) {- engine := result.Engine- if engine == nil || len(result.PageHeight) == 0 {- return- }- // Render each page at most once across all media sections. A PDF can have many- // figures/tables on the same page, so a cache shared across the section loop- // avoids re-rendering the page for every section.- //- // Sections are sorted by firstSectionPage, but a cross-page merge can leave an- // earlier page in a later position, so the TRUE minimum page is not monotonic.- // cropMediaSections therefore keeps a sliding window of page images bounded by- // the minimum page any still-unprocessed renderable section references- // (computeFuturePageWindow): a page strictly below that future window can never- // be referenced again, so it is evicted. This bounds memory to the current- // section's page span instead of caching the whole PDF, and stays bounded even- // after a page-order break — the old code permanently disabled eviction on the- // first break, which OOM'd large PDFs.- pageCache := make(map[int]image.Image)- var lastMinPage = -1- sectionsOrdered := true- renderPage := func(pn int) image.Image {- if img, ok := pageCache[pn]; ok {- return img- }- img, err := deepdocpdf.RenderPageToImage(engine, pn)- if err != nil || img == nil {- common.Warn("cropMediaSections: render failed, skipping section",- zap.Int("page", pn), zap.Error(err))- pageCache[pn] = nil- return nil- }- pageCache[pn] = img- return img- }-- // Precompute, for each section index i, the minimum page number any- // section that renders from pageCache references from i..end- // (computeFuturePageWindow: suffix-min over renderable sections only).- // A page strictly below that window can never be referenced by the current- // or a later section, so it is safe to evict even after a page-order break.- // Skipped sections (text, pre-cropped, or with no positions) do NOT- // contribute: a late skipped section with a low merged page must not keep- // the window low, or pageCache would grow unbounded again (residual of- // #19938). A -1 entry means "no renderable section with known pages is- // still ahead" and eviction is paused for safety.- minFuturePage := computeFuturePageWindow(result.Sections)-- // Page-order break detection, computed once and reused to arm the- // warn-once latch below. sectionOrderBreaksEviction is also tested- // directly. It uses the TRUE minimum page (not firstSectionPage), so a- // cross-page merge that leaves an earlier page in a later position is- // still detected.- orderBroken, breakAt := sectionOrderBreaksEviction(result.Sections)-- for i := range result.Sections {- sec := &result.Sections[i]- // TRUE minimum page this section touches, tracked for every section so- // the page-order-break diagnostic below matches sectionOrderBreaksEviction- // exactly. A section with no page info (minPage < 0) cannot establish- // order and is skipped rather than treated as a (false) decrease.- minPage := sectionMinPage(*sec)- if minPage >= 0 {- if lastMinPage >= 0 && minPage < lastMinPage {- if orderBroken && sectionsOrdered && i == breakAt {- common.Warn("cropMediaSections: sections out of page order; eviction now uses future-page window",- zap.Int("section_index", i),- zap.Int("total_sections", len(result.Sections)),- zap.Int("min_page", minPage),- zap.Int("last_min_page", lastMinPage),- zap.Int("min_future_page", minFuturePage[i]),- zap.Int("page_cache_size", len(pageCache)))- sectionsOrdered = false- }- }- lastMinPage = minPage- }- if !sectionRendersFromCache(*sec) {- continue- }- // Collect every distinct page this section spans.- pages := make(map[int]struct{})- for _, pos := range sec.Positions {- for _, pn := range pos.PageNumbers {- pages[pn] = struct{}{}- }- }- // Evict every page image no current or future section can reference.- //- // minFuturePage[i] is the minimum page number referenced by sections- // i..end (precomputed above). When sections are in page order this- // reduces to pn < minPage, the original sliding-window bound. When a- // cross-page merge breaks the order, minPage would under-state the- // window and stalling eviction would let pageCache grow without bound- // on large PDFs — so we fall back to the TRUE future minimum page and- // keep evicting. A -1 window means a section with no page info is- // still ahead, so we retain the cache rather than risk dropping a page- // a later section still needs.- if minFuturePage[i] >= 0 {- for pn := range pageCache {- if pn < minFuturePage[i] {- delete(pageCache, pn)- }- }- } else if i%64 == 0 {- // Future-page window unknown (a section with no page info is- // still ahead): eviction is paused so we never drop a page a- // later section may still reference. Log the cache size so a- // regression is observable instead of silently OOMing.- common.Warn("cropMediaSections: future-page window unknown; page cache retained",- zap.Int("section_index", i),- zap.Int("total_sections", len(result.Sections)),- zap.Int("page_cache_size", len(pageCache)),- zap.Int("min_page", minPage))- }- // Collect every distinct page this section spans so CropSectionByDLA- // can crop and vertically concatenate each page (mirroring Python's- // cropout multi-page branch). Single-page sections still render exactly- // one page, and the pageCache above guarantees a page is rendered at- // most once even when several sections share it.- single := make(map[int]image.Image, len(pages))- for pn := range pages {- if img := renderPage(pn); img != nil {- single[pn] = img- }- }- if len(single) == 0 {- continue- }- if sec.LayoutType == deepdoctype.LayoutTypeFigure {- if dla := util.CropSectionByDLA(*sec, result.DLARegions, single); dla != "" {- sec.Image = dla- continue- }- }- if len(sec.Positions) > 0 {- if img := util.CropSectionPositions(sec.Positions, single, deepdoctype.DlaScale); img != "" {- sec.Image = img- continue- }- }- sec.Image = util.CropSectionImage(sec.PositionTag, single, deepdoctype.DlaScale)- }-}--// sectionMinPage returns the minimum page number touched by any position of s.-// It is the key cropMediaSections uses to bound its page-image sliding window.-// It deliberately differs from firstSectionPage (Positions[0].PageNumbers[0]),-// which sortSectionsByPosition orders sections by: a section sorted later by-// reading order can still contain an earlier page in one of its (merged)-// positions, so the two orderings are not equivalent.-func sectionMinPage(s deepdoctype.Section) int {- minPage := -1- for _, pos := range s.Positions {- for _, pn := range pos.PageNumbers {- if pn < minPage || minPage < 0 {- minPage = pn- }- }- }- return minPage-}--// sectionRendersFromCache reports whether sec is cropped from a rendered page-// image by cropMediaSections and therefore participates in the page-image-// sliding window. It mirrors the render loop's skip predicate exactly, so the-// precomputed future-page window and the loop agree on which sections consume-// pageCache.-func sectionRendersFromCache(sec deepdoctype.Section) bool {- if strings.TrimSpace(sec.Image) != "" {- return false- }- if sec.LayoutType != deepdoctype.LayoutTypeFigure &&- sec.LayoutType != deepdoctype.DLALabelFigureCaption &&- sec.LayoutType != deepdoctype.LayoutTypeTable &&- strings.TrimSpace(sec.LayoutType) != "image" &&- sec.DocTypeKwd != "image" && sec.DocTypeKwd != "table" {- return false- }- return len(sec.Positions) > 0-}--// computeFuturePageWindow returns, for each section index i, the minimum page-// number referenced by any section that renders from pageCache in the suffix-// i..end. It feeds cropMediaSections' eviction: a page strictly below the-// window can never be referenced by the current or a later section, so it is-// safe to evict. Only sections that actually render (sectionRendersFromCache)-// contribute — a skipped section with a low merged page must not keep the-// window low, or the cache would grow unbounded again (residual of #19938).-// A -1 entry means no renderable section with known pages is still ahead, so-// eviction is paused for safety.-func computeFuturePageWindow(sections []deepdoctype.Section) []int {- minFuturePage := make([]int, len(sections))- cur := -1- for j := len(sections) - 1; j >= 0; j-- {- if sectionRendersFromCache(sections[j]) {- if mp := sectionMinPage(sections[j]); mp >= 0 {- if cur < 0 || mp < cur {- cur = mp- }- }- }- minFuturePage[j] = cur- }- return minFuturePage-}--// sectionOrderBreaksEviction walks sections in reading order (as produced by-// sortSectionsByPosition) and reports whether the TRUE minimum page ever-// decreases relative to the previous section with known pages. cropMediaSections-// reuses the result to arm its warn-once latch. A decrease is exactly what a-// cross-page merge produces when it leaves an earlier page in a later position;-// because the sort only guarantees monotonicity in firstSectionPage (not the-// TRUE min page), the break is detected here rather than assumed away. A-// section with no page info (minPage < 0) is skipped instead of being treated-// as a false decrease. The returned index is the first offending section, or-// -1 when order holds.-func sectionOrderBreaksEviction(sections []deepdoctype.Section) (bool, int) {- lastMinPage := -1- for i := range sections {- minPage := sectionMinPage(sections[i])- if minPage < 0 {- // Unknown page: cannot establish order, skip rather than treat- // it as a (false) decrease.- continue- }- if lastMinPage >= 0 && minPage < lastMinPage {- return true, i- }- lastMinPage = minPage- }- return false, -1-}-// firstPDFPageWidth returns the first page's width from a map of// per-page PDF-point widths. The map is guaranteed to store values// already converted to PDF-point space by processPage.@@ -1005,14 +730,13 @@ func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []bytif err != nil {return ParseResult{Err: err}}- var res ParseResult- switch strings.ToLower(strings.TrimSpace(opts.outputFormat)) {- case "", "json":- res = pdfParseResultToJSONWithOptions(filename, parsed, opts)- case "markdown":- res = pdfParseResultToMarkdownWithOptions(filename, parsed, opts)- default:- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", opts.outputFormat)}+ res := pdfParseResultToJSONWithOptions(filename, parsed, opts)+ for i := range res.JSON {+ m := res.JSON[i].ToMap()+ if img, _ := m["image"].(string); img != "" {+ m["image"] = pdflayout.InlinePNGDataURL(img)+ res.JSON[i] = DocItemFromMap(m)+ }}return res}diff --git a/internal/parser/parser/pdf_parser_common_eviction_test.go b/internal/parser/parser/pdf_parser_common_eviction_test.godeleted file mode 100644index c5e91f9266..0000000000--- a/internal/parser/parser/pdf_parser_common_eviction_test.go+++ /dev/null@@ -1,187 +0,0 @@-package parser--import (- "testing"-- deepdoctype "ragflow/internal/deepdoc/parser/type"-)--// TestSectionOrderBreaksEvictionCleanOrder is the sanity case: sections whose-// positions are page-monotonic keep eviction enabled. This documents the-// intended (bounded) behavior cropMediaSections relies on.-func TestSectionOrderBreaksEvictionCleanOrder(t *testing.T) {- sections := []deepdoctype.Section{- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{2, 3}}}},- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{4}}}},- }- broken, idx := sectionOrderBreaksEviction(sections)- if broken {- t.Fatalf("clean in-order sections must not break eviction, got break at index %d", idx)- }-}--// TestSectionOrderBreaksEvictionCrossPageMerge reproduces the defect: a figure-// whose first position is on a LATER page (25) but which also carries a-// caption merged onto an EARLIER page (18). sortSectionsByPosition orders by-// firstSectionPage, so this section sorts after a table on page 20, yet its-// true minimum page (18) is below that table's page. The mismatch flips-// sectionsOrdered off and permanently disables eviction.-func TestSectionOrderBreaksEvictionCrossPageMerge(t *testing.T) {- sections := []deepdoctype.Section{- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{20}}}},- {LayoutType: deepdoctype.LayoutTypeFigure, Positions: []deepdoctype.Position{- {PageNumbers: []int{25}},- {PageNumbers: []int{18}},- }},- }- broken, idx := sectionOrderBreaksEviction(sections)- if !broken {- t.Fatalf("expected eviction break from cross-page-merge section, got none")- }- if idx != 1 {- t.Errorf("expected break at index 1, got %d", idx)- }-}--// TestCropMediaSectionsEvictionContinuesAfterPageOrderBreak replays the exact-// eviction loop from cropMediaSections (future-page window) with a fake render-// that records which pages are kept. It proves the fix for the defect above:-// after a page-order break, eviction keeps running (using the TRUE future-// minimum page) instead of stalling forever — so the cache stays bounded to a-// sliding window of the current section's pages rather than growing without-// bound and OOMing large PDFs.-func TestCropMediaSectionsEvictionContinuesAfterPageOrderBreak(t *testing.T) {- sections := []deepdoctype.Section{- // table on page 20- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{20}}}},- // figure with cross-page-merge artifact: first position page 25, caption page 18- {LayoutType: deepdoctype.LayoutTypeFigure, Positions: []deepdoctype.Position{- {PageNumbers: []int{25}},- {PageNumbers: []int{18}},- }},- // two sections far down the document- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{300}}}},- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{301}}}},- }-- // Mirror cropMediaSections' corrected eviction decision: a suffix-min- // future-page window keeps eviction running even after a page-order break.- n := len(sections)- minFuturePage := make([]int, n)- cur := -1- for j := n - 1; j >= 0; j-- {- if mp := sectionMinPage(sections[j]); mp >= 0 {- if cur < 0 || mp < cur {- cur = mp- }- }- minFuturePage[j] = cur- }- pageCache := map[int]bool{} // page -> rendered and kept- for i := range sections {- if minFuturePage[i] >= 0 {- for pn := range pageCache {- if pn < minFuturePage[i] {- delete(pageCache, pn)- }- }- }- for _, pos := range sections[i].Positions {- for _, pn := range pos.PageNumbers {- pageCache[pn] = true- }- }- }-- // The cross-page-merge break must NOT disable eviction: once we advance- // past pages 20/25 to 300/301, the early pages are pruned by the- // future-page window, so the cache holds only the current section's page- // instead of all four (the old unbounded behaviour).- if pageCache[20] {- t.Error("expected page 20 to be evicted by the future-page window, but it was retained")- }- if pageCache[25] {- t.Error("expected page 25 to be evicted by the future-page window, but it was retained")- }- if !pageCache[301] {- t.Error("expected page 301 to remain cached (current section), but it was evicted")- }- if len(pageCache) != 1 {- t.Errorf("expected bounded cache to hold only the last section's page {301}, got %d: %v", len(pageCache), pageCache)- }-}--// TestSectionRendersFromCache pins the eligibility predicate cropMediaSections-// and computeFuturePageWindow share: only media sections that actually render-// from pageCache (no pre-existing Image, a media layout/type, and at least one-// Position) participate in the eviction window. Skipped sections must not-// contribute to the window, or a late skipped section with a low merged page-// would keep the window low and let pageCache grow (the residual of #19938).-func TestSectionRendersFromCache(t *testing.T) {- renderable := []deepdoctype.Section{- {LayoutType: deepdoctype.LayoutTypeFigure, Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{2}}}},- {LayoutType: deepdoctype.DLALabelFigureCaption, Positions: []deepdoctype.Position{{PageNumbers: []int{3}}}},- {LayoutType: "image", Positions: []deepdoctype.Position{{PageNumbers: []int{4}}}},- {DocTypeKwd: "image", Positions: []deepdoctype.Position{{PageNumbers: []int{5}}}},- {DocTypeKwd: "table", Positions: []deepdoctype.Position{{PageNumbers: []int{6}}}},- }- for i, sec := range renderable {- if !sectionRendersFromCache(sec) {- t.Errorf("case %d: expected media section to render from cache", i)- }- }- skipped := []deepdoctype.Section{- {LayoutType: deepdoctype.LayoutTypeText, Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},- {LayoutType: deepdoctype.LayoutTypeFigure, Image: "already-cropped", Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},- {LayoutType: deepdoctype.LayoutTypeTable}, // no Positions- }- for i, sec := range skipped {- if sectionRendersFromCache(sec) {- t.Errorf("case %d: expected skipped section NOT to render from cache", i)- }- }-}--// TestComputeFuturePageWindowIgnoresSkippedSections is the regression for-// Finding A of the #19938 review: a section that cropMediaSections skips (here-// a text section) but which carries a low merged page late in the document-// must NOT pull the future-page window down. With the bug, the window at the-// first section would equal that low page (5) and early rendered pages could-// never be evicted; after the fix the window stays at the first renderable-// section's page (100).-func TestComputeFuturePageWindowIgnoresSkippedSections(t *testing.T) {- sections := []deepdoctype.Section{- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{100}}}},- {LayoutType: deepdoctype.LayoutTypeText, Positions: []deepdoctype.Position{{PageNumbers: []int{5}}}},- }- win := computeFuturePageWindow(sections)- if win[0] != 100 {- t.Errorf("skipped low-page section must not lower the future window; got win[0]=%d, want 100", win[0])- }- if win[1] != -1 {- t.Errorf("skipped section has no renderable future window; got win[1]=%d, want -1", win[1])- }-}--// TestSectionOrderBreaksEvictionSkipsUnknownPages is the regression for the-// -1 handling in sectionOrderBreaksEviction (Finding C): a section with no-// page info (sectionMinPage == -1) must not be treated as a page-order break.-// Before the fix, the -1 section was reported as the offending break; after-// the fix the break is correctly attributed to the next known lower page (18-// following 20), so the offending index is 2, not 1.-func TestSectionOrderBreaksEvictionSkipsUnknownPages(t *testing.T) {- sections := []deepdoctype.Section{- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{20}}}},- {LayoutType: deepdoctype.LayoutTypeFigure}, // no Positions -> minPage -1- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{18}}}},- }- broken, idx := sectionOrderBreaksEviction(sections)- if !broken {- t.Fatalf("expected a page-order break from 20 -> 18, got none")- }- if idx != 2 {- t.Errorf("expected break at index 2 (the -1 section must be skipped), got %d", idx)- }-}diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.goindex 703ebd8f4d..9dccb609dc 100644--- a/internal/parser/parser/pdf_parser_common_test.go+++ b/internal/parser/parser/pdf_parser_common_test.go@@ -139,9 +139,9 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {// A figure caption with an inlined image but NO positions is NOT promoted// to image: positions are the sole classifier now. (In the PDF JSON path- // an inlined image without positions does not occur — cropMediaSections was- // removed — so this documents that the presence of a stray image alone is- // insufficient.)+ // media are not cropped — the chunker crops on demand from positions — so+ // an inlined image without positions does not occur; this documents that+ // the presence of a stray image alone is insufficient.)withImageNoPos := &deepdoctype.ParseResult{Sections: []deepdoctype.Section{{Text: "小灰灰",@@ -281,15 +281,11 @@ func TestPDFParser_ConfigureFromSetup(t *testing.T) {p := NewPDFParser()p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc",- "output_format": "markdown","enable_multi_column": true,"flatten_media_to_text": true,"remove_toc": true,"remove_header_footer": true,})- if got, want := p.OutputFormat, "markdown"; got != want {- t.Fatalf("OutputFormat = %q, want %q", got, want)- }if !p.EnableMultiColumn {t.Fatal("EnableMultiColumn = false, want true")}@@ -307,145 +303,6 @@ func TestPDFParser_ConfigureFromSetup(t *testing.T) {}}-func TestPDFParseResultToMarkdownWithOptions_RendersLikePython(t *testing.T) {- parsed := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "Title", LayoutType: deepdoctype.LayoutTypeTitle},- {Text: "Figure", LayoutType: deepdoctype.LayoutTypeFigure, Image: "aGVsbG8="},- {Text: "WhitespaceFigureText", LayoutType: deepdoctype.LayoutTypeFigure, Image: " \t\n"},- {Text: "<table><tr><td>cell</td></tr></table>", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVpbWc="},- {Text: "", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVvbmx5"},- {Text: "ImageCaption", LayoutType: "image", Image: "aW1hZ2Vvbmx5"},- {Text: "Body", LayoutType: deepdoctype.LayoutTypeText},- },- }-- res := pdfParseResultToMarkdownWithOptions("sample.pdf", parsed, pdfPostProcessOptions{})- if res.Err != nil {- t.Fatalf("pdfParseResultToMarkdownWithOptions: %v", res.Err)- }- if got, want := res.OutputFormat, "markdown"; got != want {- t.Fatalf("OutputFormat = %q, want %q", got, want)- }- if res.Markdown == "" {- t.Fatal("Markdown is empty; want rendered content")- }- if !strings.Contains(res.Markdown, "## Title") {- t.Fatalf("Markdown = %q, want title heading", res.Markdown)- }- if !strings.Contains(res.Markdown, "") {- t.Fatalf("Markdown = %q, want inline figure image", res.Markdown)- }- if !strings.Contains(res.Markdown, "WhitespaceFigureText") {- t.Fatalf("Markdown = %q, want whitespace figure text preserved", res.Markdown)- }- if strings.Contains(res.Markdown, "![Image]()") {- t.Fatalf("Markdown = %q, unexpected empty image tag", res.Markdown)- }- if !strings.Contains(res.Markdown, "<table><tr><td>cell</td></tr></table>") {- t.Fatalf("Markdown = %q, want table text", res.Markdown)- }- if !strings.Contains(res.Markdown, "") {- t.Fatalf("Markdown = %q, want fallback table image when text is empty", res.Markdown)- }- if !strings.Contains(res.Markdown, "") {- t.Fatalf("Markdown = %q, want image section", res.Markdown)- }- if !strings.Contains(res.Markdown, "Body") {- t.Fatalf("Markdown = %q, want body text", res.Markdown)- }- if len(res.JSON) != 0 {- t.Fatalf("JSON len = %d, want 0 for markdown output", len(res.JSON))- }-}--func TestPDFParseResultToMarkdownWithOptions_TableFallback(t *testing.T) {- // 1. Table with text + image -> renders table text- withText := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "<table><tr><td>content</td></tr></table>", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVpbWc="},- },- }- resText := pdfParseResultToMarkdownWithOptions("table.pdf", withText, pdfPostProcessOptions{})- if !strings.Contains(resText.Markdown, "<table><tr><td>content</td></tr></table>") {- t.Fatalf("Markdown = %q, want table text", resText.Markdown)- }- if strings.Contains(resText.Markdown, "![Image]") {- t.Fatalf("Markdown = %q, unexpected image tag when table text is present", resText.Markdown)- }-- // 2. Table with empty text + image -> falls back to image tag- emptyText := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVvbmx5"},- },- }- resFallback := pdfParseResultToMarkdownWithOptions("table.pdf", emptyText, pdfPostProcessOptions{})- if !strings.Contains(resFallback.Markdown, "") {- t.Fatalf("Markdown = %q, want fallback table image tag", resFallback.Markdown)- }-- // 3. Table with empty text + whitespace-only image -> empty string, no broken tags- wsImg := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "", LayoutType: deepdoctype.LayoutTypeTable, Image: " \t\n"},- },- }- resWS := pdfParseResultToMarkdownWithOptions("table.pdf", wsImg, pdfPostProcessOptions{})- if strings.TrimSpace(resWS.Markdown) != "" {- t.Fatalf("Markdown = %q, want empty output for empty text + whitespace image", resWS.Markdown)- }-}--func TestSectionsToMarkdown_DocTypeKwdImage(t *testing.T) {- // LayoutType is not figure/image, but DocTypeKwd == "image"- sections := []deepdoctype.Section{- {Text: "Caption", LayoutType: "custom_block", DocTypeKwd: "image", Image: "aW1hZ2Vvbmx5"},- }- got := sectionsToMarkdown(sections)- want := "\n"- if got != want {- t.Fatalf("got %q, want %q", got, want)- }-- // DocTypeKwd == "image" with whitespace image preserves text and drops empty tag- sectionsWS := []deepdoctype.Section{- {Text: "Caption", LayoutType: "custom_block", DocTypeKwd: "image", Image: " \t\n"},- }- gotWS := sectionsToMarkdown(sectionsWS)- wantWS := "Caption\n"- if gotWS != wantWS {- t.Fatalf("got %q, want %q", gotWS, wantWS)- }-}--func TestPDFParseResultToMarkdownWithOptions_ImageSection(t *testing.T) {- parsed := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "Caption", LayoutType: "image", Image: "aW1hZ2Vvbmx5"},- },- }- res := pdfParseResultToMarkdownWithOptions("doc.pdf", parsed, pdfPostProcessOptions{})- if !strings.Contains(res.Markdown, "") {- t.Fatalf("Markdown = %q, want image embed for LayoutType == 'image'", res.Markdown)- }-}--func TestPDFParseResultToMarkdownWithOptions_WhitespaceOnlyImage(t *testing.T) {- parsed := &deepdoctype.ParseResult{- Sections: []deepdoctype.Section{- {Text: "Figure Caption", LayoutType: deepdoctype.LayoutTypeFigure, Image: " \r\n\t "},- },- }- res := pdfParseResultToMarkdownWithOptions("doc.pdf", parsed, pdfPostProcessOptions{})- if !strings.Contains(res.Markdown, "Figure Caption") {- t.Fatalf("Markdown = %q, want figure caption preserved", res.Markdown)- }- if strings.Contains(res.Markdown, "![Image]") {- t.Fatalf("Markdown = %q, unexpected image tag for whitespace-only image", res.Markdown)- }-}-func TestPDFParser_ValidateParseMethod(t *testing.T) {p := NewPDFParser()if err := p.validateParseMethod(); err != nil {diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.goindex b99a84e107..f320e83234 100644--- a/internal/parser/parser/pdf_parser_docling.go+++ b/internal/parser/parser/pdf_parser_docling.go@@ -113,13 +113,13 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, parscontinue}if candidate.chunked {- if res, ok := parseDoclingChunkedResult(filename, body, parser.OutputFormat); ok {+ if res, ok := parseDoclingChunkedResult(filename, body); ok {return res}lastErr = fmt.Errorf("%s: chunked response contained no usable text", candidate.endpoint)continue}- if res, ok := parseDoclingStandardResult(ctx, filename, body, parser.OutputFormat); ok {+ if res, ok := parseDoclingStandardResult(ctx, filename, body); ok {return res}lastErr = fmt.Errorf("%s: standard response contained no parsed document", candidate.endpoint)@@ -184,7 +184,7 @@ func doclingChunkedPayload(filename string, encoded string, alpha bool, convertOreturn payload}-func parseDoclingChunkedResult(filename string, body []byte, outputFormat string) (ParseResult, bool) {+func parseDoclingChunkedResult(filename string, body []byte) (ParseResult, bool) {var chunks []doclingChunkif err := json.Unmarshal(body, &chunks); err != nil {var wrapped struct {@@ -212,10 +212,10 @@ func parseDoclingChunkedResult(filename string, body []byte, outputFormat stringif len(texts) > 0 {pageCount = len(texts)}- return doclingTextsToResult(filename, texts, outputFormat, pageCount), true+ return doclingTextsToResult(filename, texts, pageCount), true}-func parseDoclingStandardResult(ctx context.Context, filename string, body []byte, outputFormat string) (ParseResult, bool) {+func parseDoclingStandardResult(ctx context.Context, filename string, body []byte) (ParseResult, bool) {var payload doclingResponseif err := json.Unmarshal(body, &payload); err != nil {return ParseResult{}, false@@ -235,41 +235,29 @@ func parseDoclingStandardResult(ctx context.Context, filename string, body []byt}for _, doc := range docs {if md := strings.TrimSpace(doc.MDContent); md != "" {- return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true+ return pdfMarkdownToResult(ctx, filename, md, len(docs)), true}if txt := strings.TrimSpace(doc.TextContent); txt != "" {- return doclingTextsToResult(filename, []string{txt}, outputFormat, len(docs)), true+ return doclingTextsToResult(filename, []string{txt}, len(docs)), true}if md, _ := doc.JSONContent["md_content"].(string); strings.TrimSpace(md) != "" {- return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true+ return pdfMarkdownToResult(ctx, filename, md, len(docs)), true}}return ParseResult{}, false}-func doclingTextsToResult(filename string, texts []string, outputFormat string, pageCount int) ParseResult {- fileMeta := pdfFileMeta(filename, pageCount)- switch strings.ToLower(strings.TrimSpace(outputFormat)) {- case "", "json":- items := make([]map[string]any, 0, len(texts))- for _, text := range texts {- items = append(items, map[string]any{- "text": text,- "doc_type_kwd": "text",- })- }- return ParseResult{- OutputFormat: "json",- File: fileMeta,- JSON: MapsToItems(items),- }- case "markdown":- return ParseResult{- OutputFormat: "markdown",- File: fileMeta,- Markdown: strings.Join(texts, "\n\n"),- }- default:- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}+func doclingTextsToResult(filename string, texts []string, pageCount int) ParseResult {+ items := make([]map[string]any, 0, len(texts))+ for _, text := range texts {+ items = append(items, map[string]any{+ "text": text,+ "doc_type_kwd": "text",+ })+ }+ return ParseResult{+ OutputFormat: "json",+ File: pdfFileMeta(filename, pageCount),+ JSON: MapsToItems(items),}}diff --git a/internal/parser/parser/pdf_parser_docling_test.go b/internal/parser/parser/pdf_parser_docling_test.goindex 0115fe2a4d..130fcd6925 100644--- a/internal/parser/parser/pdf_parser_docling_test.go+++ b/internal/parser/parser/pdf_parser_docling_test.go@@ -11,7 +11,7 @@ import ("testing")-func TestPDFParser_ParseWithResult_DoclingChunkedMarkdownIntegration(t *testing.T) {+func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {withSSRFBypass(t)var requestCount atomic.Int32@@ -67,7 +67,6 @@ func TestPDFParser_ParseWithResult_DoclingChunkedMarkdownIntegration(t *testing.pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "Docling",- "output_format": "markdown","docling_server_url": server.URL,"docling_api_key": "doc-secret",})@@ -77,11 +76,17 @@ func TestPDFParser_ParseWithResult_DoclingChunkedMarkdownIntegration(t *testing.if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if got, want := res.OutputFormat, "markdown"; got != want {+ if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if got, want := res.Markdown, "Chunk A\n\nChunk B"; got != want {- t.Fatalf("Markdown = %q, want %q", got, want)+ var got strings.Builder+ for _, item := range ItemsToMaps(res.JSON) {+ text, _ := item["text"].(string)+ got.WriteString(text)+ got.WriteString("\n")+ }+ if !strings.Contains(got.String(), "Chunk A") || !strings.Contains(got.String(), "Chunk B") {+ t.Fatalf("JSON text = %q, want both chunks", got.String())}}@@ -132,7 +137,6 @@ func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *tpdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "Docling",- "output_format": "json","docling_server_url": server.URL,})@@ -235,7 +239,6 @@ func TestPDFParser_ParseWithResult_DoclingConvertOptions(t *testing.T) {setup := map[string]any{"parse_method": "Docling",- "output_format": "markdown","docling_server_url": server.URL,}maps.Copy(setup, tt.setup)diff --git a/internal/parser/parser/pdf_parser_fixture_cgo_test.go b/internal/parser/parser/pdf_parser_fixture_cgo_test.goindex 83219849a5..40f273671a 100644--- a/internal/parser/parser/pdf_parser_fixture_cgo_test.go+++ b/internal/parser/parser/pdf_parser_fixture_cgo_test.go@@ -55,33 +55,6 @@ func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) {}}-func TestPDFParser_ParseWithResult_CGOFixtureMarkdown(t *testing.T) {- useMockDocAnalyzer(t)-- path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")- data, err := os.ReadFile(path)- if err != nil {- t.Fatalf("ReadFile(%s): %v", path, err)- }- pdf := NewPDFParser()- pdf.ConfigureFromSetup(map[string]any{"output_format": "markdown"})-- ctx := t.Context()- res := pdf.ParseWithResult(ctx, "Doc1.pdf", data)- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- if got, want := res.OutputFormat, "markdown"; got != want {- t.Fatalf("OutputFormat = %q, want %q", got, want)- }- if res.Markdown == "" {- t.Fatal("Markdown is empty; want rendered content")- }- if len(res.JSON) != 0 {- t.Fatalf("JSON len = %d, want 0 for markdown output", len(res.JSON))- }-}-func TestPDFParser_ParseWithResult_CGOFixturePlainText(t *testing.T) {useMockDocAnalyzer(t)diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.goindex 333ded8f3d..bef0466224 100644--- a/internal/parser/parser/pdf_parser_mineru.go+++ b/internal/parser/parser/pdf_parser_mineru.go@@ -72,7 +72,7 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parseif strings.TrimSpace(content) != "" {pageCount = 1}- return pdfMarkdownToResult(ctx, filename, content, parser.OutputFormat, pageCount)+ return pdfMarkdownToResult(ctx, filename, content, pageCount)}func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskID string, apiConfig *models.APIConfig, timeout time.Duration) (string, error) {diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.goindex adaefaf5a6..66f6ae03f7 100644--- a/internal/parser/parser/pdf_parser_mineru_test.go+++ b/internal/parser/parser/pdf_parser_mineru_test.go@@ -11,7 +11,7 @@ import ("testing")-func TestPDFParser_ParseWithResult_MinerUMarkdownIntegration(t *testing.T) {+func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {withSSRFBypass(t)var submitCalled atomic.Boolvar resultCalled atomic.Bool@@ -76,7 +76,6 @@ func TestPDFParser_ParseWithResult_MinerUMarkdownIntegration(t *testing.T) {pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "MinerU",- "output_format": "markdown","mineru_apiserver": server.URL,"mineru_api_key": "secret","mineru_backend": "pipeline",@@ -90,56 +89,23 @@ func TestPDFParser_ParseWithResult_MinerUMarkdownIntegration(t *testing.T) {if !submitCalled.Load() || !resultCalled.Load() {t.Fatalf("submit/result called = %v/%v, want true/true", submitCalled.Load(), resultCalled.Load())}- if got, want := res.OutputFormat, "markdown"; got != want {+ if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if got, want := res.Markdown, "# Title\n\nBody paragraph.\n"; got != want {- t.Fatalf("Markdown = %q, want %q", got, want)+ var text strings.Builder+ for _, item := range ItemsToMaps(res.JSON) {+ value, _ := item["text"].(string)+ text.WriteString(value)+ text.WriteString("\n")+ }+ if !strings.Contains(text.String(), "Title") || !strings.Contains(text.String(), "Body paragraph.") {+ t.Fatalf("JSON text = %q, want title and body", text.String())}if got, want := res.File["name"], "sample.pdf"; got != want {t.Fatalf("File.name = %v, want %v", got, want)}}-func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {- withSSRFBypass(t)- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- switch {- case r.Method == http.MethodPost && r.URL.Path == "/file_parse":- w.Header().Set("Content-Type", "application/json")- _, _ = w.Write([]byte(`{"data":{"task_id":"task-2"}}`))- case r.Method == http.MethodGet && r.URL.Path == "/tasks/task-2/result":- w.Header().Set("Content-Type", "application/json")- _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# Title\n\nBody paragraph.\n"}}}`))- default:- http.NotFound(w, r)- }- }))- defer server.Close()-- pdf := NewPDFParser()- pdf.ConfigureFromSetup(map[string]any{- "parse_method": "MinerU",- "output_format": "json",- "mineru_apiserver": server.URL,- })-- ctx := t.Context()- res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- if got, want := res.OutputFormat, "json"; got != want {- t.Fatalf("OutputFormat = %q, want %q", got, want)- }- if len(res.JSON) == 0 {- t.Fatal("JSON is empty; want markdown-normalized items")- }- if got := res.JSON[0].ToMap()["text"]; got == nil {- t.Fatal("JSON[0].text missing")- }-}-func TestPDFParser_ParseWithResult_MinerURequiresAPIServer(t *testing.T) {pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "MinerU"})@@ -206,7 +172,6 @@ func TestPDFParser_ParseWithResult_MinerUSendsServerURLForHTTPClientBackend(t *tpdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "MinerU",- "output_format": "markdown","mineru_apiserver": server.URL,"mineru_backend": "vlm-http-client","mineru_server_url": "http://vllm-host:30000",@@ -267,7 +232,6 @@ func TestPDFParser_ParseWithResult_MinerUReadsBackendFromProviderJSONApiKey(t *tpdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "MinerU",- "output_format": "markdown","mineru_api_key": apiKeyJSON,})res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.goindex 46e0b2b495..cdb448cff9 100644--- a/internal/parser/parser/pdf_parser_opendataloader.go+++ b/internal/parser/parser/pdf_parser_opendataloader.go@@ -64,11 +64,11 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []bytif payload.JSONDoc != nil {items := openDataLoaderItems(payload.JSONDoc)if len(items) > 0 {- return pdfItemsToResult(filename, items, parser.OutputFormat, openDataLoaderPageCount(payload.JSONDoc))+ return pdfItemsToResult(filename, items, openDataLoaderPageCount(payload.JSONDoc))}}if strings.TrimSpace(payload.MDText) != "" {- return pdfMarkdownToResult(ctx, filename, payload.MDText, parser.OutputFormat, 1)+ return pdfMarkdownToResult(ctx, filename, payload.MDText, 1)}return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader returned no parsed content")}}diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.goindex ea84fe3b68..5888c876df 100644--- a/internal/parser/parser/pdf_parser_opendataloader_test.go+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go@@ -66,7 +66,6 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "OpenDataLoader",- "output_format": "json","opendataloader_apiserver": server.URL,"opendataloader_api_key": "odl-secret","hybrid": "docling-fast",@@ -86,7 +85,7 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {}}-func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T) {+func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallbackToJSON(t *testing.T) {withSSRFBypass(t)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.URL.Path != "/file_parse" {@@ -101,7 +100,6 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T)pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "OpenDataLoader",- "output_format": "markdown","opendataloader_apiserver": server.URL,})ctx := t.Context()@@ -109,8 +107,14 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(res.Markdown, "ODL Title") {- t.Fatalf("Markdown = %q, want ODL Title", res.Markdown)+ var text strings.Builder+ for _, item := range ItemsToMaps(res.JSON) {+ value, _ := item["text"].(string)+ text.WriteString(value)+ text.WriteString("\n")+ }+ if !strings.Contains(text.String(), "ODL Title") {+ t.Fatalf("JSON text = %q, want ODL Title", text.String())}}diff --git a/internal/parser/parser/pdf_parser_paddleocr_test.go b/internal/parser/parser/pdf_parser_paddleocr_test.goindex caf21ab35c..9d4286414d 100644--- a/internal/parser/parser/pdf_parser_paddleocr_test.go+++ b/internal/parser/parser/pdf_parser_paddleocr_test.go@@ -117,10 +117,6 @@ func TestPaddleOCRTitleLabels(t *testing.T) {if item["layout"] != tc.layout || item["text"] != tc.want {t.Fatalf("title semantics: %v", item)}- formatted := pdfItemsToResult("title.pdf", ItemsToMaps(result.JSON), "markdown", 1)- if formatted.Markdown != tc.want {- t.Fatalf("markdown=%q; want %q", formatted.Markdown, tc.want)- }})}}diff --git a/internal/parser/parser/pdf_parser_pages_test.go b/internal/parser/parser/pdf_parser_pages_test.goindex d12b0e01ef..3a1ca88a42 100644--- a/internal/parser/parser/pdf_parser_pages_test.go+++ b/internal/parser/parser/pdf_parser_pages_test.go@@ -67,7 +67,6 @@ func TestConfigureFromSetup_Pages(t *testing.T) {setup := map[string]any{"flatten_media_to_text": true,"parse_method": "DeepDOC",- "output_format": "json","pages": []any{[]any{float64(1), float64(100)},},@@ -79,9 +78,6 @@ func TestConfigureFromSetup_Pages(t *testing.T) {if p.ParseMethod != "DeepDOC" {t.Errorf("ParseMethod = %q, want DeepDOC", p.ParseMethod)}- if p.OutputFormat != "json" {- t.Errorf("OutputFormat = %q, want json", p.OutputFormat)- }if want := [][]int{{1, 100}}; !reflect.DeepEqual(p.Pages, want) {t.Errorf("Pages = %v, want %v", p.Pages, want)}diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.goindex 1073df5ab9..e39189e610 100644--- a/internal/parser/parser/pdf_parser_remote_common.go+++ b/internal/parser/parser/pdf_parser_remote_common.go@@ -2,11 +2,7 @@ package parserimport ("context"- "fmt""strings"-- "ragflow/internal/entity"- "ragflow/internal/parser/tableutil")func pdfFileMeta(filename string, pageCount int) map[string]any {@@ -20,60 +16,15 @@ func pdfFileMeta(filename string, pageCount int) map[string]any {}}-// markdownItemText renders a single parse item to markdown. A table uses the-// structured TableData contract directly (rendered via RenderTableHTML); every-// other item uses its text. Producers emit `table` for tables and never the-// legacy HTML `text`, so a table-typed item without structured data is skipped-// rather than falling back to a raw HTML string — the contract is "tables are-// structured, not markup".-func markdownItemText(item map[string]any) string {- if dt, _ := item["doc_type_kwd"].(string); dt == "table" {- if td, ok := item["table"].(*entity.TableData); ok && td != nil {- markup := tableutil.RenderTableHTML(td)- if notes, _ := item["text"].(string); strings.TrimSpace(notes) != "" {- return strings.TrimRight(markup, "\n") + "\n\n" + notes- }- return markup- }- return ""- }- text, _ := item["text"].(string)- return text-}--func pdfItemsToResult(filename string, items []map[string]any, outputFormat string, pageCount int) ParseResult {+func pdfItemsToResult(filename string, items []map[string]any, pageCount int) ParseResult {if len(items) == 0 {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}pageCount = normalizePDFPageCount(pageCount, items)- switch strings.ToLower(strings.TrimSpace(outputFormat)) {- case "", "json":- return ParseResult{- OutputFormat: "json",- File: pdfFileMeta(filename, pageCount),- JSON: MapsToItems(items),- }- case "markdown":- var b strings.Builder- for _, item := range items {- text := markdownItemText(item)- layout, _ := item["layout"].(string)- if strings.TrimSpace(text) == "" {- continue- }- if layout == "title" && !strings.HasPrefix(strings.TrimSpace(text), "#") {- b.WriteString("## ")- }- b.WriteString(text)- b.WriteString("\n\n")- }- return ParseResult{- OutputFormat: "markdown",- File: pdfFileMeta(filename, pageCount),- Markdown: strings.TrimRight(b.String(), "\n"),- }- default:- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}+ return ParseResult{+ OutputFormat: "json",+ File: pdfFileMeta(filename, pageCount),+ JSON: MapsToItems(items),}}@@ -156,34 +107,23 @@ func rowsHaveText(rows [][]string) bool {return false}-func pdfMarkdownToResult(ctx context.Context, filename, markdown, outputFormat string, pageCount int) ParseResult {+func pdfMarkdownToResult(ctx context.Context, filename, markdown string, pageCount int) ParseResult {fileMeta := pdfFileMeta(filename, pageCount)- switch strings.ToLower(strings.TrimSpace(outputFormat)) {- case "", "json":- res := parsePDFMarkdown(ctx, filename, markdown)- if res.Err != nil {- if ctx.Err() != nil {- res.File = fileMeta- return res- }- // Same text fallback as NormalizeResult: keep the backend- // markdown as raw text instead of failing the task.- return ParseResult{- OutputFormat: "json",- File: fileMeta,- Warnings: append(res.Warnings, "markdown normalization fell back to text: "+res.Err.Error()),- JSON: []DocItem{NewTextJSONItem(markdown)},- }+ res := parsePDFMarkdown(ctx, filename, markdown)+ if res.Err != nil {+ if ctx.Err() != nil {+ res.File = fileMeta+ return res}- res.File = fileMeta- return res- case "markdown":+ // Same text fallback as NormalizeResult: keep the backend+ // markdown as raw text instead of failing the task.return ParseResult{- OutputFormat: "markdown",+ OutputFormat: "json",File: fileMeta,- Markdown: markdown,+ Warnings: append(res.Warnings, "markdown normalization fell back to text: "+res.Err.Error()),+ JSON: []DocItem{NewTextJSONItem(markdown)},}- default:- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}}+ res.File = fileMeta+ return res}diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.goindex c78c0755f8..f5af8d8382 100644--- a/internal/parser/parser/pdf_parser_somark.go+++ b/internal/parser/parser/pdf_parser_somark.go@@ -42,7 +42,7 @@ func parsePDFWithSoMark(filename string, data []byte, parser *PDFParser) ParseReif len(items) == 0 {return ParseResult{Err: fmt.Errorf("parser: SoMark returned no usable blocks")}}- return pdfItemsToResult(filename, items, parser.OutputFormat, pageCount)+ return pdfItemsToResult(filename, items, pageCount)}// soMarkSubmit uploads a PDF with the supplied credentials and parser options,diff --git a/internal/parser/parser/pdf_parser_somark_test.go b/internal/parser/parser/pdf_parser_somark_test.goindex d94d326720..db33a1aa09 100644--- a/internal/parser/parser/pdf_parser_somark_test.go+++ b/internal/parser/parser/pdf_parser_somark_test.go@@ -61,7 +61,6 @@ func TestPDFParser_ParseWithResult_SoMarkJSONIntegration(t *testing.T) {pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "SoMark",- "output_format": "json","somark_base_url": server.URL,"somark_api_key": "somark-secret",})@@ -78,35 +77,6 @@ func TestPDFParser_ParseWithResult_SoMarkJSONIntegration(t *testing.T) {}}-func TestPDFParser_ParseWithResult_SoMarkMarkdownIntegration(t *testing.T) {- withSSRFBypass(t)- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- switch r.URL.Path {- case "/parse/async":- _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-2"}}`))- case "/parse/async_check":- _, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"title","content":"SoMark Title","title_level":1},{"type":"text","content":"Body"}]}]}}}}}`))- default:- http.NotFound(w, r)- }- }))- defer server.Close()- pdf := NewPDFParser()- pdf.ConfigureFromSetup(map[string]any{- "parse_method": "SoMark",- "output_format": "markdown",- "somark_base_url": server.URL,- })- ctx := t.Context()- res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- if !strings.Contains(res.Markdown, "SoMark Title") {- t.Fatalf("Markdown = %q, want title", res.Markdown)- }-}-func TestPDFParser_ParseWithResult_SoMarkRequiresBaseURL(t *testing.T) {pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "SoMark"})diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.goindex e52f173647..3072ef4da4 100644--- a/internal/parser/parser/pdf_parser_tcadp.go+++ b/internal/parser/parser/pdf_parser_tcadp.go@@ -17,8 +17,7 @@ import (func parsePDFWithTCADP(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {return parseWithTCADP(ctx, filename, data, "PDF",parser.TCADPAPIServer, parser.TCADPAPIKey,- parser.TCADPTableResultType, parser.TCADPMarkdownImageResponseType,- parser.OutputFormat)+ parser.TCADPTableResultType, parser.TCADPMarkdownImageResponseType)}func tcadpItemsFromZip(zipBytes []byte) ([]map[string]any, int, error) {diff --git a/internal/parser/parser/pdf_parser_tcadp_test.go b/internal/parser/parser/pdf_parser_tcadp_test.goindex dbaff9b772..94a042a34c 100644--- a/internal/parser/parser/pdf_parser_tcadp_test.go+++ b/internal/parser/parser/pdf_parser_tcadp_test.go@@ -35,7 +35,6 @@ func TestPDFParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "json","tcadp_apiserver": server.URL,"tcadp_api_key": "tcadp-secret",})@@ -49,7 +48,7 @@ func TestPDFParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {}}-func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {+func TestPDFParser_ParseWithResult_TCADPIntegration(t *testing.T) {withSSRFBypass(t)zipPayload := tcadpZipFixture(t)var server *httptest.Server@@ -67,7 +66,6 @@ func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "markdown","tcadp_apiserver": server.URL,})ctx := t.Context()@@ -75,8 +73,14 @@ func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(res.Markdown, "Hello TCADP") {- t.Fatalf("Markdown = %q, want fixture text", res.Markdown)+ var text strings.Builder+ for _, item := range ItemsToMaps(res.JSON) {+ value, _ := item["text"].(string)+ text.WriteString(value)+ text.WriteString("\n")+ }+ if !strings.Contains(text.String(), "Hello TCADP") {+ t.Fatalf("JSON text = %q, want fixture text", text.String())}}@@ -106,7 +110,6 @@ func TestPDFParser_ParseWithResult_TCADPCancellationAbortsDownload(t *testing.T)pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "json","tcadp_apiserver": submit.URL,"tcadp_api_key": "tcadp-secret",})diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.goindex 2fe6d81daf..bce6649e49 100644--- a/internal/parser/parser/pdf_postprocess.go+++ b/internal/parser/parser/pdf_postprocess.go@@ -14,7 +14,6 @@ import (// per-box geometry signals those detectors rely on are still intact; they are// intentionally not repeated here.type pdfPostProcessOptions struct {- outputFormat stringpageWidth float64zoom float64enableMultiColumn booldiff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.goindex 903517c0fc..445e547e67 100644--- a/internal/parser/parser/pdf_result_test.go+++ b/internal/parser/parser/pdf_result_test.go@@ -84,7 +84,7 @@ func TestPDFMarkdownBackendDoesNotFetchRemoteImages(t *testing.T) {var requests atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(http.StatusNotFound) }))defer server.Close()- result := pdfMarkdownToResult(t.Context(), "remote.pdf", "", "json", 1)+ result := pdfMarkdownToResult(t.Context(), "remote.pdf", "", 1)if result.Err != nil {t.Fatal(result.Err)}@@ -264,22 +264,16 @@ func TestProjectPDFTablePreservesExistingStructureAndNotes(t *testing.T) {}}-func TestPDFTableDescriptionUsesTextItemForBothOutputs(t *testing.T) {+func TestPDFTableDescriptionUsesTextItem(t *testing.T) {table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}item := map[string]any{"table": table, "doc_type_kwd": "table", "text": "Footnote", "page_number": 3}- for _, format := range []string{"json", "markdown"} {- result := pdfItemsToResult("sample.pdf", []map[string]any{item}, format, 3)- if result.Err != nil {- t.Fatal(result.Err)- }- if format == "json" {- result = NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", result)- if len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != nil || result.JSON[1].ToMap()["text"] != "Footnote" || result.JSON[1].ToMap()["page_number"] != 3 {- t.Fatalf("table description not separated: %#v", result.JSON)- }- } else if !strings.Contains(result.Markdown, "Cell value") || !strings.Contains(result.Markdown, "Footnote") {- t.Fatalf("table description lost: %s", result.Markdown)- }+ result := pdfItemsToResult("sample.pdf", []map[string]any{item}, 3)+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ result = NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", result)+ if len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != nil || result.JSON[1].ToMap()["text"] != "Footnote" || result.JSON[1].ToMap()["page_number"] != 3 {+ t.Fatalf("table description not separated: %#v", result.JSON)}if item["text"] != "Footnote" {t.Fatal("source item mutated")@@ -289,7 +283,7 @@ func TestPDFTableDescriptionUsesTextItemForBothOutputs(t *testing.T) {func TestPDFWhitespaceTableDescriptionCannotHideCells(t *testing.T) {item := map[string]any{"table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, "doc_type_kwd": "table", "text": " \n "}results := []ParseResult{- NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 1)),+ NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 1)),NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{item})}),}for _, result := range results {@@ -318,7 +312,7 @@ func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) {originalText := item["text"]p := NewPDFParser()p.FlattenMediaToText = flatten- result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))+ result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3))if result.Err != nil {t.Fatal(result.Err)}@@ -373,7 +367,7 @@ func TestPDFNormalizeResultIsIdempotent(t *testing.T) {"doc_type_kwd": "table", "text": markup, "page_number": 3,"positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,}- once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))+ once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3))if once.Err != nil {t.Fatal(once.Err)}@@ -504,7 +498,7 @@ func TestPDFNormalizeResultCancellationPreservesFile(t *testing.T) {func TestPDFMarkdownToResultPreservesFileOnCancel(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())cancel()- result := pdfMarkdownToResult(ctx, "cancel.pdf", "body", "json", 3)+ result := pdfMarkdownToResult(ctx, "cancel.pdf", "body", 3)if !errors.Is(result.Err, context.Canceled) {t.Fatalf("err = %v", result.Err)}diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.goindex 305965af88..c5f29ae81b 100644--- a/internal/parser/parser/pdf_table_no_html_text_test.go+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -1,11 +1,9 @@package parserimport (- "strings""testing""ragflow/internal/entity"- "ragflow/internal/parser/tableutil")// TestPDFTableItemsOmitHTMLText pins the structured TableData contract: a table@@ -68,49 +66,6 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {}}-// TestPDFItemsToMarkdownRendersTableFromTableField verifies the markdown-// renderer derives table markup from `table` when the legacy HTML `text` is-// absent. This is the consumer that previously forced producers to keep `text`;-// once producers drop `text`, markdown must still embed the table. The output-// must be byte-equivalent to rendering `table` directly (stable across the-// ParseTableHTML/RenderTableHTML round-trip).-func TestPDFItemsToMarkdownRendersTableFromTableField(t *testing.T) {- td := &entity.TableData{- Grid: [][]entity.TableCell{{{Text: "What is RAGFlow?"}, {Text: "A RAG engine."}}, {{Text: "Is it open source?"}, {Text: "Yes."}}},- HeaderRows: 1,- }- // Producer-after-cleanup shape: `table` present, `text` absent.- item := map[string]any{"doc_type_kwd": "table", "layout": "table", "table": td}- res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)- if res.Err != nil {- t.Fatalf("pdfItemsToResult markdown: %v", res.Err)- }- want := strings.TrimSpace(tableutil.RenderTableHTML(td))- if got := strings.TrimSpace(res.Markdown); got != want {- t.Fatalf("markdown table mismatch:\n got: %q\nwant: %q", got, want)- }- if !strings.Contains(res.Markdown, "What is RAGFlow?") {- t.Fatalf("markdown lost table cell text: %q", res.Markdown)- }-}--// TestPDFItemsToMarkdownSkipsTableWithoutStructuredData guards the-// structured-first contract: a table-typed item that carries no `table`-// (e.g. a mislabeled table region) must be skipped from markdown rather than-// falling back to a raw HTML `text`. This pins "tables are structured, not-// markup" so a future change cannot reintroduce an HTML fallback for tables.-func TestPDFItemsToMarkdownSkipsTableWithoutStructuredData(t *testing.T) {- // doc_type_kwd "table" but no `table` field and no `text`.- item := map[string]any{"doc_type_kwd": "table", "layout": "table"}- res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)- if res.Err != nil {- t.Fatalf("pdfItemsToResult markdown: %v", res.Err)- }- if strings.TrimSpace(res.Markdown) != "" {- t.Fatalf("markdown must skip table-typed item without structured data, got %q", res.Markdown)- }-}-// TestProjectPDFTableKeepsTextWhenNotProjected guards the safety net: items// that are not tables, or whose text is not <table> markup, must keep their// `text` untouched and receive no `table` — we must never delete text fordiff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.goindex c5f7bc1e0a..8948a0b771 100644--- a/internal/parser/parser/picture_parser.go+++ b/internal/parser/parser/picture_parser.go@@ -42,26 +42,13 @@ var imageExtensions = map[string]bool{}// PictureParser emits image files for optional vision enhancement.-type PictureParser struct {- OutputFormat string-}+type PictureParser struct{}// NewPictureParser constructs a PictureParser.func NewPictureParser() *PictureParser {return &PictureParser{}}-// ConfigureFromSetup reads picture-specific configuration from the-// parser setup map.-func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {- if p == nil || setup == nil {- return- }- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }-}-// ParseWithResult validates the image extension and returns its image payload.func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {ext := strings.ToLower(filepath.Ext(filename))@@ -86,13 +73,6 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, dareturn ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), MaxImagePayloadBytes)}}- // Parser output is normalized to JSON at the component boundary, so an- // absent backend format defaults to JSON here as well.- outFmt := p.OutputFormat- if outFmt == "" {- outFmt = "json"- }-release, err := AcquireImageMedia(ctx)if err != nil {return ParseResult{Err: err}@@ -108,7 +88,7 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, dareturn ParseResult{Err: err}}return ParseResult{- OutputFormat: outFmt,+ OutputFormat: "json",File: map[string]any{"name": filename,"size": len(data),diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.goindex 5dc809790c..8022158945 100644--- a/internal/parser/parser/picture_parser_test.go+++ b/internal/parser/parser/picture_parser_test.go@@ -45,25 +45,11 @@ func TestNewPictureParser(t *testing.T) {if p == nil {t.Fatal("NewPictureParser returned nil")}- if p.OutputFormat != "" {- t.Errorf("OutputFormat = %q, want empty", p.OutputFormat)- }-}--func TestPictureParser_ConfigureFromSetup(t *testing.T) {- p := NewPictureParser()- p.ConfigureFromSetup(map[string]any{- "output_format": "text",- })- if p.OutputFormat != "text" {- t.Errorf("OutputFormat = %q, want text", p.OutputFormat)- }}func TestPictureParser_ParseWithResult_NilSetup(t *testing.T) {ctx := t.Context()p := NewPictureParser()- p.ConfigureFromSetup(nil)res := p.ParseWithResult(ctx, "photo.png", []byte("\x89PNG"))if res.Err != nil {t.Errorf("unexpected error for nil setup: %v", res.Err)@@ -120,9 +106,6 @@ func TestPictureParser_ParseWithResult_VideoExtension(t *testing.T) {func TestPictureParser_ParseWithResult_OutputFormat(t *testing.T) {ctx := t.Context()p := NewPictureParser()- p.ConfigureFromSetup(map[string]any{- "output_format": "json",- })res := p.ParseWithResult(ctx, "photo.jpg", []byte{1, 2, 3})if res.Err != nil {t.Fatalf("unexpected error: %v", res.Err)diff --git a/internal/parser/parser/ppt_parser_cgo_test.go b/internal/parser/parser/ppt_parser_cgo_test.goindex 5aad089300..bdf43ca3e1 100644--- a/internal/parser/parser/ppt_parser_cgo_test.go+++ b/internal/parser/parser/ppt_parser_cgo_test.go@@ -159,8 +159,7 @@ func TestPPTParser_TCADPFileType(t *testing.T) {// PPTParser delegates to PPTXParser{format:"ppt"}.p := NewPPTParser()setup := map[string]any{- "parse_method": "tcadp",- "output_format": "json",+ "parse_method": "tcadp",}p.ConfigureFromSetup(setup)@@ -172,9 +171,6 @@ func TestPPTParser_TCADPFileType(t *testing.T) {if p.pptx.ParseMethod != "tcadp" {t.Errorf("PPTParser.pptx.ParseMethod = %q, want %q", p.pptx.ParseMethod, "tcadp")}- if p.pptx.OutputFormat != "json" {- t.Errorf("PPTParser.pptx.OutputFormat = %q, want %q", p.pptx.OutputFormat, "json")- }}// TestPPTXParser_TCADPFileType verifies that a PPTXParser@@ -185,8 +181,7 @@ func TestPPTXParser_TCADPFileType(t *testing.T) {t.Fatalf("NewPPTXParser().format = %q, want pptx", p.format)}setup := map[string]any{- "parse_method": "tcadp",- "output_format": "json",+ "parse_method": "tcadp",}p.ConfigureFromSetup(setup)if p.ParseMethod != "tcadp" {@@ -248,7 +243,6 @@ func testPresentationTCADPIntegration(t *testing.T, p tcadpPresentationParser, wp.ConfigureFromSetup(map[string]any{"parse_method": "tcadp",- "output_format": "json","tcadp_apiserver": server.URL,})ctx := t.Context()@@ -288,7 +282,6 @@ func TestPPTXParser_TCADPDownloadHTTPError(t *testing.T) {p := NewPPTXParser()p.ConfigureFromSetup(map[string]any{"parse_method": "tcadp",- "output_format": "json","tcadp_apiserver": server.URL,})ctx := t.Context()diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex 61cc0a4a77..6ebfaf41c4 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -43,7 +43,6 @@ type PPTXParser struct {TCADPAPIKey stringTCADPTableResultType stringTCADPMarkdownImageResponseType string- OutputFormat string}func NewPPTXParser() *PPTXParser {@@ -75,17 +74,10 @@ func (p *PPTXParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["markdown_image_response_type"].(string); ok {p.TCADPMarkdownImageResponseType = v}- if v, ok := setup["output_format"].(string); ok {- p.OutputFormat = v- }- if p.OutputFormat == "" {- p.OutputFormat = "json"- }}// ParseWithResult emits one JSON item per slide with the slide's-// plain text. Forces OutputFormat to "json" and emits one JSON item-// per slide section.+// plain text.func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {// p == nil guard: the struct is embedded by value in PPTParser and// always created via NewPPTXParser or the "ppt"-format constructor in@@ -108,7 +100,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datafilename, data, strings.ToUpper(p.format),p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,- p.OutputFormat,)case "", "deepdoc":// Continue with the local office_oxide parser.diff --git a/internal/parser/parser/tcadp_common.go b/internal/parser/parser/tcadp_common.goindex 28770b3cad..b56a54466d 100644--- a/internal/parser/parser/tcadp_common.go+++ b/internal/parser/parser/tcadp_common.go@@ -49,7 +49,6 @@ func parseWithTCADP(fileType string,tcadpAPIServer, tcadpAPIKey string,tableResultType, markdownImageResponseType string,- outputFormat string,) ParseResult {if len(data) == 0 {return emptyPDFResult(filename)@@ -118,7 +117,7 @@ func parseWithTCADP(if err != nil {return ParseResult{Err: err}}- return pdfItemsToResult(filename, items, outputFormat, pageCount)+ return pdfItemsToResult(filename, items, pageCount)}// authHeaderForDownload returns the bearer Authorization header value todiff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex d22e1758c6..01e6a5ffaf 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -30,7 +30,6 @@ var compoundDocumentSignature = []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a,type XLSParser struct {ParseMethod string- OutputFormat stringTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType string@@ -55,9 +54,6 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["parse_method"].(string); ok && v != "" {p.ParseMethod = v}- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }deprecatedHTML4Excel(setup, p.String())deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {@@ -82,7 +78,6 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [ctx, filename, data, "XLS",p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,- p.OutputFormat,)case "", "excelize":// Continue with the local Excelize parser.diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex 0a9a2e0bbd..5eb1b21919 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -237,10 +237,6 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])}- // Configure output format to "json"- p.ConfigureFromSetup(map[string]any{- "output_format": "json",- })resJSON := p.ParseWithResult(context.Background(), "test.xls", data)if resJSON.Err != nil {t.Fatalf("ParseWithResult failed: %v", resJSON.Err)@@ -274,21 +270,6 @@ func TestXLSXParser_JSONOutput(t *testing.T) {if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {t.Errorf("item doc_type_kwd = %v, want table", res.JSON[0].ToMap()["doc_type_kwd"])}-- // Even if configured to legacy "html", format is unified to "json"- p.ConfigureFromSetup(map[string]any{- "output_format": "html",- })- resHTML := p.ParseWithResult(context.Background(), "test.xlsx", data)- if resHTML.Err != nil {- t.Fatalf("ParseWithResult failed: %v", resHTML.Err)- }- if resHTML.OutputFormat != "json" {- t.Errorf("resHTML.OutputFormat = %q, want 'json'", resHTML.OutputFormat)- }- if len(resHTML.JSON) == 0 {- t.Errorf("resHTML.JSON should have items")- }}func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.goindex 84a6fe4973..6de76217e5 100644--- a/internal/parser/parser/xls_tcadp_test.go+++ b/internal/parser/parser/xls_tcadp_test.go@@ -35,7 +35,6 @@ func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {}p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "json","chunk_rows": 50,"tcadp_apiserver": server.URL,"tcadp_api_key": "tcadp-secret",@@ -74,7 +73,6 @@ func TestXLSParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {}p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "json","tcadp_apiserver": server.URL,"tcadp_api_key": "tcadp-secret",})@@ -107,7 +105,6 @@ func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {p := NewCSVParser()p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "json","tcadp_apiserver": server.URL,"tcadp_api_key": "tcadp-secret",})@@ -120,7 +117,7 @@ func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {}}-func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {+func TestXLSXParser_ParseWithResult_TCADPIntegration(t *testing.T) {withSSRFBypass(t)ctx := t.Context()zipPayload := tcadpZipFixture(t)@@ -143,15 +140,14 @@ func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {}p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "markdown","tcadp_apiserver": server.URL,})res := p.ParseWithResult(ctx, "sample.xlsx", []byte("mock xlsx content"))if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if res.Markdown == "" {- t.Fatal("Markdown is empty; want rendered content")+ if len(res.JSON) == 0 {+ t.Fatal("JSON is empty; want rendered content")}}@@ -240,7 +236,6 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {}p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser",- "output_format": "json","chunk_rows": 50,"tcadp_apiserver": "https://tcadp.example.com","tcadp_api_key": "secret",@@ -250,9 +245,6 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {if got, want := p.ParseMethod, "TCADP parser"; got != want {t.Fatalf("ParseMethod = %q, want %q", got, want)}- if got, want := p.OutputFormat, "json"; got != want {- t.Fatalf("OutputFormat = %q, want %q", got, want)- }if got, want := p.TCADPAPIServer, "https://tcadp.example.com"; got != want {t.Fatalf("TCADPAPIServer = %q, want %q", got, want)}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex b0dc873283..ed397ff5c7 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -27,7 +27,6 @@ import (type XLSXParser struct {ParseMethod string- OutputFormat stringTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType string@@ -52,9 +51,6 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["parse_method"].(string); ok && v != "" {p.ParseMethod = v}- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }deprecatedHTML4Excel(setup, p.String())deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {@@ -96,7 +92,6 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, datactx, filename, data, "XLSX",p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,- p.OutputFormat,)case "", "excelize":// Continue with the local Excelize parser.diff --git a/internal/service/file/file_content_cgo_test.go b/internal/service/file/file_content_cgo_test.gonew file mode 100644index 0000000000..60a3957823--- /dev/null+++ b/internal/service/file/file_content_cgo_test.go@@ -0,0 +1,81 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++//go:build cgo++package file++import (+ "archive/zip"+ "bytes"+ "strings"+ "testing"+)++// minimalDOCX builds a one-paragraph .docx so the upload path exercises the+// office_oxide-backed DOCX parser without a checked-in binary fixture.+func minimalDOCX(t *testing.T, text string) []byte {+ t.Helper()+ var buf bytes.Buffer+ zw := zip.NewWriter(&buf)+ write := func(name, body string) {+ t.Helper()+ w, err := zw.Create(name)+ if err != nil {+ t.Fatalf("create zip entry %s: %v", name, err)+ }+ if _, err := w.Write([]byte(body)); err != nil {+ t.Fatalf("write zip entry %s: %v", name, err)+ }+ }+ write("[Content_Types].xml", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">+ <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>+ <Default Extension="xml" ContentType="application/xml"/>+ <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>+</Types>`)+ write("_rels/.rels", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">+ <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>+</Relationships>`)+ write("word/document.xml", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">+ <w:body>+ <w:p><w:r><w:t>`+text+`</w:t></w:r></w:p>+ </w:body>+</w:document>`)+ if err := zw.Close(); err != nil {+ t.Fatalf("zip close: %v", err)+ }+ return buf.Bytes()+}++// TestParseFileContent_DOCXReadsJSONItems pins the upload path contract after+// the parser became JSON-only: a .docx must be flattened from its structured+// JSON items into readable text. The DOCX parser's former Markdown default is+// gone, so a raw-bytes fallback (the parseFileContent failure path) would mean+// the JSON branch regressed.+func TestParseFileContent_DOCXReadsJSONItems(t *testing.T) {+ data := minimalDOCX(t, "Hello DOCX content")++ content := parseFileContent(t.Context(), "note.docx", data)+ if !strings.Contains(content, "Hello DOCX content") {+ t.Fatalf("docx content missing item text: %q", content)+ }+ if content == string(data) {+ t.Fatal("content fell back to raw docx bytes instead of parsed items")+ }+}diff --git a/tools/mdprobe/main.go b/tools/mdprobe/main.goindex 3703173a78..e2cc6c3d3d 100644--- a/tools/mdprobe/main.go+++ b/tools/mdprobe/main.go@@ -40,8 +40,9 @@ func main() {fmt.Fprintln(os.Stderr, err)os.Exit(1)}- // The pipeline's setups for the markdown family: output_format=json.- p.ConfigureFromSetup(map[string]any{"output_format": "json"})+ // The pipeline's setups for the markdown family carry no output key:+ // the parser emits JSON unconditionally.+ p.ConfigureFromSetup(map[string]any{})res := p.ParseWithResult(context.Background(), filepath.Base(path), data)fmt.Printf("file=%s bytes=%d output_format=%s items=%d\n", filepath.Base(path), len(data), res.OutputFormat, len(res.JSON))