Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): remove dead output_format readers (#21336)

jay7772117f173c288a8
diff --git a/docs/guides/agent/ingestion_pipeline/configure_parser_component.md b/docs/guides/agent/ingestion_pipeline/configure_parser_component.md
index eb5f332d14..4958883b9d 100644
--- a/docs/guides/agent/ingestion_pipeline/configure_parser_component.md
+++ b/docs/guides/agent/ingestion_pipeline/configure_parser_component.md
@@ -65,8 +65,7 @@ by tenant-model ID or by `model@instance@provider` reference, for example:
```json
{
"pdf": {
- "parse_method": "SoMark-model@production@SoMark",
- "output_format": "json"
+ "parse_method": "SoMark-model@production@SoMark"
}
}
```
diff --git a/internal/agent/canvas/compile_override_params_test.go b/internal/agent/canvas/compile_override_params_test.go
index 1b70f48b2e..c1b4315e1d 100644
--- a/internal/agent/canvas/compile_override_params_test.go
+++ b/internal/agent/canvas/compile_override_params_test.go
@@ -24,11 +24,11 @@ func TestCompile_OverrideParams(t *testing.T) {
ComponentName: "Parser",
Params: map[string]any{
"pdf": map[string]any{
- "output_format": "one",
- "parse_method": "naive",
+ "lang": "one",
+ "parse_method": "naive",
},
"doc": map[string]any{
- "output_format": "one",
+ "lang": "one",
},
},
},
@@ -50,10 +50,10 @@ func TestCompile_OverrideParams(t *testing.T) {
override := map[string]any{
"parser_0": map[string]any{
"pdf": map[string]any{
- "output_format": "detailed",
+ "lang": "detailed",
},
"docx": map[string]any{
- "output_format": "one",
+ "lang": "one",
},
},
}
@@ -68,11 +68,11 @@ func TestCompile_OverrideParams(t *testing.T) {
t.Fatalf("Parser factory not called")
}
// doc preserved from base.
- if v, _ := got["doc"].(map[string]any); v == nil || v["output_format"] != "one" {
+ if v, _ := got["doc"].(map[string]any); v == nil || v["lang"] != "one" {
t.Errorf("doc should be preserved from base: %#v", got["doc"])
}
// docx injected by the override (was absent in base).
- if v, _ := got["docx"].(map[string]any); v == nil || v["output_format"] != "one" {
+ if v, _ := got["docx"].(map[string]any); v == nil || v["lang"] != "one" {
t.Errorf("docx injection missing: %#v", got["docx"])
}
// pdf: override fully replaces the base entry (shallow merge).
@@ -80,8 +80,8 @@ func TestCompile_OverrideParams(t *testing.T) {
if pdf == nil {
t.Fatalf("pdf setup missing: %#v", got)
}
- if pdf["output_format"] != "detailed" {
- t.Errorf("pdf.output_format not overridden: %#v", pdf)
+ if pdf["lang"] != "detailed" {
+ t.Errorf("pdf.lang not overridden: %#v", pdf)
}
if _, ok := pdf["parse_method"]; ok {
t.Errorf("pdf.parse_method should be dropped by shallow merge: %#v", pdf)
diff --git a/internal/agent/canvas/node_body_override_params_test.go b/internal/agent/canvas/node_body_override_params_test.go
index 7d5c37ddb4..6f375b95a8 100644
--- a/internal/agent/canvas/node_body_override_params_test.go
+++ b/internal/agent/canvas/node_body_override_params_test.go
@@ -36,11 +36,11 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) {
baseParams := map[string]any{
"pdf": map[string]any{
- "output_format": "one",
- "parse_method": "naive",
+ "lang": "one",
+ "parse_method": "naive",
},
"doc": map[string]any{
- "output_format": "one",
+ "lang": "one",
},
}
// Run-level override is keyed by cpnID. For "cpn-parser": the whole
@@ -50,15 +50,15 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) {
override := map[string]any{
"cpn-parser": map[string]any{
"pdf": map[string]any{
- "output_format": "detailed",
+ "lang": "detailed",
},
"docx": map[string]any{
- "output_format": "one",
+ "lang": "one",
},
},
"cpn-other": map[string]any{
"pdf": map[string]any{
- "output_format": "should-not-apply",
+ "lang": "should-not-apply",
},
},
}
@@ -75,21 +75,21 @@ func TestBuildNodeBody_OverrideParams(t *testing.T) {
}
// doc is untouched (absent from the override).
- if v, _ := captured["doc"].(map[string]any); v == nil || v["output_format"] != "one" {
+ if v, _ := captured["doc"].(map[string]any); v == nil || v["lang"] != "one" {
t.Errorf("doc should be preserved from base: %#v", captured["doc"])
}
// docx injected by the override (was absent in base).
- if v, _ := captured["docx"].(map[string]any); v == nil || v["output_format"] != "one" {
+ if v, _ := captured["docx"].(map[string]any); v == nil || v["lang"] != "one" {
t.Errorf("docx injection missing: %#v", captured["docx"])
}
- // pdf: the override fully replaces the base entry, so output_format is
+ // pdf: the override fully replaces the base entry, so lang is
// overridden and the base-only parse_method is dropped (shallow merge).
pdf, _ := captured["pdf"].(map[string]any)
if pdf == nil {
t.Fatalf("pdf setup missing: %#v", captured)
}
- if pdf["output_format"] != "detailed" {
- t.Errorf("pdf.output_format not overridden: %#v", pdf)
+ if pdf["lang"] != "detailed" {
+ t.Errorf("pdf.lang not overridden: %#v", pdf)
}
if _, ok := pdf["parse_method"]; ok {
t.Errorf("pdf.parse_method should be dropped by shallow merge: %#v", pdf)
diff --git a/internal/agent/templates/advanced_ingestion_pipeline.json b/internal/agent/templates/advanced_ingestion_pipeline.json
index f4215787c2..26b2f85d6d 100644
--- a/internal/agent/templates/advanced_ingestion_pipeline.json
+++ b/internal/agent/templates/advanced_ingestion_pipeline.json
@@ -127,7 +127,6 @@
}
},
"doc": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"doc"
@@ -135,7 +134,6 @@
},
"docx": {
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"docx"
@@ -153,7 +151,6 @@
"body",
"attachments"
],
- "output_format": "text",
"preprocess": "main_content",
"suffix": [
"eml",
@@ -161,7 +158,6 @@
]
},
"html": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"htm",
@@ -169,7 +165,6 @@
]
},
"image": {
- "output_format": "json",
"parse_method": "ocr",
"preprocess": "main_content",
"suffix": [
@@ -182,7 +177,6 @@
},
"markdown": {
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"md",
@@ -193,7 +187,6 @@
},
"pdf": {
"flatten_media_to_text": false,
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -202,7 +195,6 @@
"vlm": {}
},
"slides": {
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -212,7 +204,6 @@
},
"spreadsheet": {
"flatten_media_to_text": false,
- "output_format": "html",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -223,7 +214,6 @@
"vlm": {}
},
"text&code": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"txt",
@@ -374,20 +364,17 @@
{
"fileFormat": "pdf",
"flatten_media_to_text": false,
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content"
},
{
"fileFormat": "spreadsheet",
"flatten_media_to_text": false,
- "output_format": "html",
"parse_method": "DeepDOC",
"preprocess": "main_content"
},
{
"fileFormat": "image",
- "output_format": "json",
"parse_method": "ocr",
"preprocess": "main_content",
"system_prompt": ""
@@ -404,39 +391,32 @@
"attachments"
],
"fileFormat": "email",
- "output_format": "text",
"preprocess": "main_content"
},
{
"fileFormat": "markdown",
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "text&code",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "html",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "doc",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "docx",
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "slides",
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content"
}
diff --git a/internal/agent/templates/chunk_summary.json b/internal/agent/templates/chunk_summary.json
index f862f68c1c..24a936e6c0 100644
--- a/internal/agent/templates/chunk_summary.json
+++ b/internal/agent/templates/chunk_summary.json
@@ -84,7 +84,6 @@
}
},
"doc": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"doc"
@@ -92,7 +91,6 @@
},
"docx": {
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"docx"
@@ -110,7 +108,6 @@
"body",
"attachments"
],
- "output_format": "text",
"preprocess": "main_content",
"suffix": [
"eml",
@@ -118,7 +115,6 @@
]
},
"html": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"htm",
@@ -126,7 +122,6 @@
]
},
"image": {
- "output_format": "json",
"parse_method": "ocr",
"preprocess": "main_content",
"suffix": [
@@ -139,7 +134,6 @@
},
"markdown": {
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"md",
@@ -150,7 +144,6 @@
},
"pdf": {
"flatten_media_to_text": false,
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -159,7 +152,6 @@
"vlm": {}
},
"slides": {
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -169,7 +161,6 @@
},
"spreadsheet": {
"flatten_media_to_text": false,
- "output_format": "html",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -180,7 +171,6 @@
"vlm": {}
},
"text&code": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"txt",
@@ -334,20 +324,17 @@
{
"fileFormat": "pdf",
"flatten_media_to_text": false,
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content"
},
{
"fileFormat": "spreadsheet",
"flatten_media_to_text": false,
- "output_format": "html",
"parse_method": "DeepDOC",
"preprocess": "main_content"
},
{
"fileFormat": "image",
- "output_format": "json",
"parse_method": "ocr",
"preprocess": "main_content",
"system_prompt": ""
@@ -364,39 +351,32 @@
"attachments"
],
"fileFormat": "email",
- "output_format": "text",
"preprocess": "main_content"
},
{
"fileFormat": "markdown",
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "text&code",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "html",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "doc",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "docx",
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "slides",
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content"
}
diff --git a/internal/agent/templates/compiler.json b/internal/agent/templates/compiler.json
index 52fce12251..7e2c54b4cc 100644
--- a/internal/agent/templates/compiler.json
+++ b/internal/agent/templates/compiler.json
@@ -59,7 +59,6 @@
"doc": {
"flatten_media_to_text": false,
"order_index": 7,
- "output_format": "json",
"remove_header_footer": false,
"suffix": [
"doc"
@@ -69,7 +68,6 @@
"docx": {
"flatten_media_to_text": false,
"order_index": 8,
- "output_format": "json",
"remove_header_footer": false,
"suffix": [
"docx"
@@ -88,7 +86,6 @@
"attachments"
],
"order_index": 3,
- "output_format": "text",
"suffix": [
"eml",
"msg"
@@ -96,7 +93,6 @@
},
"html": {
"order_index": 6,
- "output_format": "json",
"remove_header_footer": false,
"suffix": [
"htm",
@@ -105,7 +101,6 @@
},
"image": {
"order_index": 2,
- "output_format": "json",
"parse_method": "ocr",
"suffix": [
"jpg",
@@ -117,7 +112,6 @@
"markdown": {
"flatten_media_to_text": false,
"order_index": 4,
- "output_format": "json",
"suffix": [
"md",
"markdown",
@@ -128,7 +122,6 @@
"pdf": {
"flatten_media_to_text": false,
"order_index": 0,
- "output_format": "json",
"parse_method": "DeepDOC",
"remove_header_footer": false,
"suffix": [
@@ -138,7 +131,6 @@
},
"slides": {
"order_index": 9,
- "output_format": "json",
"parse_method": "DeepDOC",
"suffix": [
"pptx",
@@ -148,7 +140,6 @@
"spreadsheet": {
"flatten_media_to_text": false,
"order_index": 1,
- "output_format": "html",
"parse_method": "DeepDOC",
"suffix": [
"xls",
@@ -159,7 +150,6 @@
},
"text&code": {
"order_index": 5,
- "output_format": "json",
"suffix": [
"txt",
"py",
@@ -285,7 +275,6 @@
{
"fileFormat": "pdf",
"flatten_media_to_text": false,
- "output_format": "json",
"parse_method": "DeepDOC",
"remove_header_footer": false,
"vlm": {}
@@ -293,13 +282,11 @@
{
"fileFormat": "spreadsheet",
"flatten_media_to_text": false,
- "output_format": "html",
"parse_method": "DeepDOC",
"vlm": {}
},
{
"fileFormat": "image",
- "output_format": "json",
"parse_method": "ocr"
},
{
@@ -313,41 +300,34 @@
"body",
"attachments"
],
- "fileFormat": "email",
- "output_format": "text"
+ "fileFormat": "email"
},
{
"fileFormat": "markdown",
"flatten_media_to_text": false,
- "output_format": "json",
"vlm": {}
},
{
- "fileFormat": "text&code",
- "output_format": "json"
+ "fileFormat": "text&code"
},
{
"fileFormat": "html",
- "output_format": "json",
"remove_header_footer": false
},
{
"fileFormat": "doc",
"flatten_media_to_text": false,
- "output_format": "json",
"remove_header_footer": false,
"vlm": {}
},
{
"fileFormat": "docx",
"flatten_media_to_text": false,
- "output_format": "json",
"remove_header_footer": false,
"vlm": {}
},
{
"fileFormat": "slides",
- "output_format": "json",
"parse_method": "DeepDOC"
}
]
diff --git a/internal/agent/templates/title_chunker.json b/internal/agent/templates/title_chunker.json
index 294cd87276..ad845fbb88 100644
--- a/internal/agent/templates/title_chunker.json
+++ b/internal/agent/templates/title_chunker.json
@@ -84,7 +84,6 @@
}
},
"doc": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"doc"
@@ -92,7 +91,6 @@
},
"docx": {
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"docx"
@@ -110,7 +108,6 @@
"body",
"attachments"
],
- "output_format": "text",
"preprocess": "main_content",
"suffix": [
"eml",
@@ -118,7 +115,6 @@
]
},
"html": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"htm",
@@ -126,7 +122,6 @@
]
},
"image": {
- "output_format": "json",
"parse_method": "ocr",
"preprocess": "main_content",
"suffix": [
@@ -139,7 +134,6 @@
},
"markdown": {
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"md",
@@ -150,7 +144,6 @@
},
"pdf": {
"flatten_media_to_text": false,
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -159,7 +152,6 @@
"vlm": {}
},
"slides": {
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -169,7 +161,6 @@
},
"spreadsheet": {
"flatten_media_to_text": false,
- "output_format": "html",
"parse_method": "DeepDOC",
"preprocess": "main_content",
"suffix": [
@@ -180,7 +171,6 @@
"vlm": {}
},
"text&code": {
- "output_format": "json",
"preprocess": "main_content",
"suffix": [
"txt",
@@ -353,20 +343,17 @@
{
"fileFormat": "pdf",
"flatten_media_to_text": false,
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content"
},
{
"fileFormat": "spreadsheet",
"flatten_media_to_text": false,
- "output_format": "html",
"parse_method": "DeepDOC",
"preprocess": "main_content"
},
{
"fileFormat": "image",
- "output_format": "json",
"parse_method": "ocr",
"preprocess": "main_content",
"system_prompt": ""
@@ -383,39 +370,32 @@
"attachments"
],
"fileFormat": "email",
- "output_format": "text",
"preprocess": "main_content"
},
{
"fileFormat": "markdown",
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "text&code",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "html",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "doc",
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "docx",
"flatten_media_to_text": false,
- "output_format": "json",
"preprocess": "main_content"
},
{
"fileFormat": "slides",
- "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": "main_content"
}
diff --git a/internal/dao/ingestion_parser_config_runtime_test.go b/internal/dao/ingestion_parser_config_runtime_test.go
index 3f8a8e9027..4b529fcada 100644
--- a/internal/dao/ingestion_parser_config_runtime_test.go
+++ b/internal/dao/ingestion_parser_config_runtime_test.go
@@ -625,7 +625,6 @@ var overriddenFamilyKeys = map[string]bool{
// questions (this one, "is the template's value safe to lose"; that one, "may
// the value the migration writes differ from the template's").
var inertFamilyKeys = map[string]string{
- "output_format": "normalizeParserOutputFormats forces JSON on every family",
"preprocess": "the value has no Go reader; the preprocessing stage is selected by the chunker's own options",
"suffix": "no Go reader; file routing uses the document's own suffix column, not the family list",
"vlm": "empty in every template; absent falls back to the family llm_id and then the tenant model, and only Markdown and audio read a family vlm at all",
diff --git a/internal/deepdoc/parser/pdf/layout/boxes_sections.go b/internal/deepdoc/parser/pdf/layout/boxes_sections.go
index 8ea419c066..bfc5dc3082 100644
--- a/internal/deepdoc/parser/pdf/layout/boxes_sections.go
+++ b/internal/deepdoc/parser/pdf/layout/boxes_sections.go
@@ -146,25 +146,6 @@ func InlinePNGDataURL(raw string) string {
return "data:image/png;base64," + cleaned
}
-func SectionsToMarkdown(sections []pdf.Section) string {
- var b strings.Builder
- for _, s := range sections {
- if s.LayoutType == pdf.LayoutTypeTitle {
- b.WriteString("\n## ")
- }
- imgURL := InlinePNGDataURL(s.Image)
- if (s.LayoutType == pdf.LayoutTypeFigure || s.LayoutType == "image" || s.DocTypeKwd == "image" || (s.LayoutType == pdf.LayoutTypeTable && strings.TrimSpace(s.Text) == "")) && imgURL != "" {
- b.WriteString("\n![Image](")
- b.WriteString(imgURL)
- b.WriteString(")")
- continue
- }
- b.WriteString(s.Text)
- b.WriteString("\n")
- }
- return b.String()
-}
-
// SectionsToJSON converts Sections to a Python-compatible JSON dict format.
//
// Each dict has keys: text, layout_type, doc_type_kwd, _pdf_positions, image.
diff --git a/internal/deepdoc/parser/pdf/layout/boxes_sections_test.go b/internal/deepdoc/parser/pdf/layout/boxes_sections_test.go
index 3d6e36e6ad..28d55b08c9 100644
--- a/internal/deepdoc/parser/pdf/layout/boxes_sections_test.go
+++ b/internal/deepdoc/parser/pdf/layout/boxes_sections_test.go
@@ -231,166 +231,6 @@ func TestNormalizeSectionPositions_EmptySlice(t *testing.T) {
}
}
-func TestSectionsToMarkdown_Title(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeTitle, Text: "标题", Image: ""},
- }
- got := SectionsToMarkdown(sections)
- want := "\n## 标题\n"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func TestSectionsToMarkdown_FigureWithImage(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: "abc123"},
- }
- got := SectionsToMarkdown(sections)
- want := "\n![Image](data:image/png;base64,abc123)"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func TestSectionsToMarkdown_FigureWithDataURLImage(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: "data:image/png;base64,abc123"},
- }
- got := SectionsToMarkdown(sections)
- want := "\n![Image](data:image/png;base64,abc123)"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func TestSectionsToMarkdown_FigureWithoutImage(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: ""},
- }
- got := SectionsToMarkdown(sections)
- want := "图1\n"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func TestSectionsToMarkdown_FigureWithWhitespaceImage(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeFigure, Text: "图1", Image: " \t\n"},
- }
- got := SectionsToMarkdown(sections)
- want := "图1\n"
- if got != want {
- t.Errorf("got %q, want %q (whitespace image should not emit empty image tag or drop text)", got, want)
- }
-}
-
-func TestSectionsToMarkdown_Text(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeText, Text: "普通内容", Image: ""},
- }
- got := SectionsToMarkdown(sections)
- want := "普通内容\n"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func TestSectionsToMarkdown_Table(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeTable, Text: "表格内容", Image: ""},
- }
- got := SectionsToMarkdown(sections)
- want := "表格内容\n"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func TestSectionsToMarkdown_TableWithImageAndText(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeTable, Text: "<table><tr><td>1</td></tr></table>", Image: "dGFibGVpbWc="},
- }
- got := SectionsToMarkdown(sections)
- want := "<table><tr><td>1</td></tr></table>\n"
- if got != want {
- t.Errorf("got %q, want %q (table with text should render table text)", got, want)
- }
-}
-
-func TestSectionsToMarkdown_TableFallbackWhenEmptyText(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeTable, Text: "", Image: "dGFibGVpbWc="},
- }
- got := SectionsToMarkdown(sections)
- want := "\n![Image](data:image/png;base64,dGFibGVpbWc=)"
- if got != want {
- t.Errorf("got %q, want %q (table with empty text should fallback to image)", got, want)
- }
-}
-
-func TestSectionsToMarkdown_TableFallbackWhitespaceImage(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeTable, Text: "", Image: " \t\n"},
- }
- got := SectionsToMarkdown(sections)
- want := "\n"
- if got != want {
- t.Errorf("got %q, want %q (table with empty text and whitespace image should render empty text newline, no empty tag)", got, want)
- }
-}
-
-func TestSectionsToMarkdown_DocTypeKwdImage(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: "custom_block", DocTypeKwd: "image", Text: "caption", Image: "aGVsbG8="},
- }
- got := SectionsToMarkdown(sections)
- want := "\n![Image](data:image/png;base64,aGVsbG8=)"
- if got != want {
- t.Errorf("got %q, want %q (DocTypeKwd == 'image' should render image)", got, want)
- }
-}
-
-func TestSectionsToMarkdown_DocTypeKwdImage_Whitespace(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: "custom_block", DocTypeKwd: "image", Text: "caption", Image: " "},
- }
- got := SectionsToMarkdown(sections)
- want := "caption\n"
- if got != want {
- t.Errorf("got %q, want %q (DocTypeKwd == 'image' with whitespace image should keep text)", got, want)
- }
-}
-
-func TestSectionsToMarkdown_Mixed(t *testing.T) {
- sections := []pdf.Section{
- {LayoutType: pdf.LayoutTypeTitle, Text: "标题", Image: ""},
- {LayoutType: pdf.LayoutTypeText, Text: "内容", Image: ""},
- {LayoutType: pdf.LayoutTypeFigure, Text: "图", Image: "img"},
- {LayoutType: pdf.LayoutTypeTable, Text: "表格", Image: ""},
- }
- got := SectionsToMarkdown(sections)
- want := "\n## 标题\n内容\n\n![Image](data:image/png;base64,img)表格\n"
- if got != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func TestSectionsToMarkdown_EmptySlice(t *testing.T) {
- got := SectionsToMarkdown([]pdf.Section{})
- if got != "" {
- t.Errorf("got %q, want empty string", got)
- }
-}
-
-func TestSectionsToMarkdown_NilSlice(t *testing.T) {
- got := SectionsToMarkdown(nil)
- if got != "" {
- t.Errorf("got %q, want empty string", got)
- }
-}
-
func TestSectionsToJSON_SingleSection(t *testing.T) {
sections := []pdf.Section{
{
diff --git a/internal/deepdoc/parser/pdf/parse_cgo.go b/internal/deepdoc/parser/pdf/parse_cgo.go
index d48e9e4e9b..9112f3ccb5 100644
--- a/internal/deepdoc/parser/pdf/parse_cgo.go
+++ b/internal/deepdoc/parser/pdf/parse_cgo.go
@@ -13,16 +13,13 @@ import (
//
// Engine lifetime / ownership: Parse creates the native PDF engine and
// transfers ownership of it to the returned *pdf.ParseResult (result.Engine is
-// populated inside ParseRaw/processPages). The engine is intentionally kept
-// alive past Parse so the serialization step can still use it — most
-// importantly the Markdown path (cropMarkdownFigures) renders and crops figure
-// images on demand. Parse must therefore NOT close the engine itself.
+// populated inside ParseRaw/processPages). Parse must therefore NOT close the
+// engine itself.
//
-// The caller (the adapter layer: pdfParseResultToJSONWithOptions /
-// pdfParseResultToMarkdownWithOptions) releases the engine via result.Close()
-// once serialization is complete. The only place Parse closes the engine is on
-// its own error return, since in that case no ParseResult is handed back for
-// the caller to release.
+// The caller (the adapter layer: pdfParseResultToJSONWithOptions) releases the
+// engine via result.Close() once serialization is complete. The only place Parse
+// closes the engine is on its own error return, since in that case no
+// ParseResult is handed back for the caller to release.
func (p *Parser) Parse(ctx context.Context, data []byte, docAnalyzer pdf.DocAnalyzer) (*pdf.ParseResult, error) {
engine, err := NewEngine(data)
if err != nil {
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index e0d50afd8a..e54adcfdf9 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -520,10 +520,8 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
}
// TestMaybeDispatchImage_HardcodesJSONOutput verifies the image family
-// always emits json regardless of setup["output_format"]. Python
-// rag/app/picture.py:chunk() has no output_format concept — it always
-// returns a structured doc. Honoring a "text" override produced a bare
-// Text payload that lost the image attachment and set doc_type to "text".
+// always emits structured JSON items. Python rag/app/picture.py:chunk()
+// has no output format concept — it always returns a structured doc.
func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
origResolver := resolveTenantModelByType
defer func() { resolveTenantModelByType = origResolver }()
@@ -534,7 +532,6 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
}
setups := defaultSetups()
- setups["image"]["output_format"] = "text" // legacy/override; must be ignored
ctx := t.Context()
res, _, err := maybeDispatchImage(
ctx,
@@ -550,10 +547,10 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
t.Fatalf("maybeDispatchImage: %v", err)
}
if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json (image family must ignore output_format override)", res.OutputFormat)
+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
}
if len(res.JSON) != 1 {
- t.Fatalf("JSON len = %d, want 1 even when setup says text", len(res.JSON))
+ t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
}
@@ -569,11 +566,11 @@ func (d *audioTranscribeDriver) TranscribeAudio(ctx context.Context, _ *string,
return &modelModule.ASRResponse{Text: d.transcription}, nil
}
-// TestMaybeDispatchAudio_JSONCarriesTranscription pins diff 2.11: when the
-// audio family's output_format is "json", the ASR transcription must be
-// carried in the JSON items (not only in the Text field). Before the fix the
-// branch returned Text only with an empty JSON slice, and the Invoke switch
-// silently dropped the transcription because it has no "json" branch.
+// TestMaybeDispatchAudio_JSONCarriesTranscription pins that the ASR
+// transcription is carried in the JSON items (not only in the Text field).
+// Before the fix the branch returned Text only with an empty JSON slice, and
+// the Invoke switch silently dropped the transcription because it has no
+// "json" branch.
func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {
origResolver := resolveTenantModelByType
defer func() { resolveTenantModelByType = origResolver }()
@@ -585,7 +582,6 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {
}
setups := defaultSetups()
- setups["audio"]["output_format"] = "json"
ctx := t.Context()
res, dispatched, err := maybeDispatchAudio(
@@ -617,49 +613,8 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {
}
}
-func TestMaybeDispatchAudio_IgnoresLegacyOutputFormat(t *testing.T) {
- origResolver := resolveTenantModelByType
- defer func() { resolveTenantModelByType = origResolver }()
-
- const want = "hello world"
- drv := &audioTranscribeDriver{transcription: want}
- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- return drv, "asr-model", &modelModule.APIConfig{}, 0, nil
- }
-
- setups := defaultSetups()
- setups["audio"]["output_format"] = "text"
-
- ctx := t.Context()
- res, dispatched, err := maybeDispatchAudio(
- ctx,
- dao.DB,
- utility.FileTypeAURAL,
- "test.mp3",
- []byte("fake-audio"),
- map[string]any{"tenant_id": "t1"},
- setups,
- )
- if err != nil {
- t.Fatalf("maybeDispatchAudio: %v", err)
- }
- if !dispatched {
- t.Fatalf("expected dispatched=true for AURAL file")
- }
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
- }
- if len(res.JSON) != 1 {
- t.Fatalf("JSON len = %d, want 1", len(res.JSON))
- }
- if got, _ := res.JSON[0].ToMap()["text"].(string); got != want {
- t.Fatalf("JSON[0].text = %q, want %q", got, want)
- }
-}
-
-// TestMaybeDispatchAudio_DefaultOutputFormatJson covers the
-// maybeDispatchAudio always returns a JSON transcription item, including when
-// the setup has no output_format key.
+// TestMaybeDispatchAudio_DefaultOutputFormatJson covers that
+// maybeDispatchAudio always returns a JSON transcription item.
func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
const want = "hello world"
drv := &audioTranscribeDriver{transcription: want}
@@ -668,8 +623,6 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
return drv, "asr-model", &modelModule.APIConfig{}, 0, nil
}
- // No output_format key — exercise the default path inside
- // maybeDispatchAudio.
setups := map[string]schema.ParserSetup{"audio": {}}
res, dispatched, err := maybeDispatchAudio(
context.Background(),
@@ -691,27 +644,6 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
}
}
-// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family output
-// format must be "json" (matching Python parser.py:212), not "text".
-// With "text" the structured email fields (from/to/subject/attachments/...) are
-// flattened into a blob and lost downstream. The value is owned by
-// normalizeParserOutputFormats, so the assertion goes through the constructed
-// component rather than the raw defaults.
-func TestDefaultEmailOutputFormatIsJSON(t *testing.T) {
- comp, err := NewParserComponent(nil)
- if err != nil {
- t.Fatalf("NewParserComponent: %v", err)
- }
- pc, ok := comp.(*ParserComponent)
- if !ok {
- t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)
- }
- got, _ := pc.setups["email"]["output_format"].(string)
- if got != "json" {
- t.Fatalf("email output_format = %q, want json", got)
- }
-}
-
func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {
origTenantResolver := resolveTenantModelByType
origModelResolver := resolveModelConfig
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 279db84f74..561e9e7fb4 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -128,11 +128,10 @@ type visionSettings struct {
}
// NewParserComponent constructs a Parser from a DSL param map.
-// The default setups are overlaid with the supplied values. Historical
-// output_format values are accepted but normalized to JSON so downstream
-// components consume one parser output protocol. This applies to every
-// family, including PDF and office documents: Markdown is an internal
-// backend representation only and is never a public Parser output.
+// The default setups are overlaid with the supplied values. The Parser
+// emits one output protocol: structured JSON items for every family,
+// including PDF and office documents. Markdown remains an internal backend
+// representation only and is never a public Parser output.
//
// Param map shape (all keys optional):
//
@@ -149,7 +148,6 @@ type visionSettings struct {
func NewParserComponent(params map[string]any) (runtime.Component, error) {
s := defaultSetups()
if params == nil {
- normalizeParserOutputFormats(s)
return &ParserComponent{setups: s}, nil
}
// Canvases saved by the Python-era frontend nest the per-family setups
@@ -197,7 +195,6 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
s[k][fk] = cloneParserSetupValue(fv)
}
}
- normalizeParserOutputFormats(s)
pc := &ParserComponent{setups: s, enableVisionEnhancement: enableVisionEnhancement, vision: vision}
if err := pc.Check(); err != nil {
return nil, fmt.Errorf("parser: %w", err)
@@ -205,15 +202,6 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
return pc, nil
}
-func normalizeParserOutputFormats(setups map[string]schema.ParserSetup) {
- // The Go Parser component intentionally exposes JSON only. Keep this
- // normalization unconditional so PDF/office legacy Markdown settings
- // cannot silently select a second public output path.
- for _, setup := range setups {
- setup["output_format"] = "json"
- }
-}
-
func cloneParserSetupValue(value any) any {
switch v := value.(type) {
case map[string]any:
@@ -309,11 +297,6 @@ func imageSetupRunsOCR(setup schema.ParserSetup) bool {
}
func defaultSetups() map[string]schema.ParserSetup {
- // Normalize up front so any caller (including tests that build a
- // ParserComponent directly from these defaults) gets the single JSON
- // output protocol. NewParserComponent re-normalizes idempotently; the
- // per-family literals were removed because normalizeParserOutputFormats
- // owns this key unconditionally.
s := map[string]schema.ParserSetup{
"pdf": {
"parse_method": "deepdoc",
@@ -395,7 +378,6 @@ func defaultSetups() map[string]schema.ParserSetup {
"suffix": []string{"json", "jsonl", "ldjson"},
},
}
- normalizeParserOutputFormats(s)
return s
}
diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.go
index bc3539055f..99bc3127b4 100644
--- a/internal/ingestion/component/parser_check_test.go
+++ b/internal/ingestion/component/parser_check_test.go
@@ -140,11 +140,11 @@ func TestParserComponent_Check(t *testing.T) {
// --- audio/video: vlm.llm_id not validated (matches Python check()) ---
{
name: "audio: no vlm field → pass (model falls back to tenant default)",
- setups: map[string]schema.ParserSetup{"audio": {"output_format": "text"}},
+ setups: map[string]schema.ParserSetup{"audio": {}},
},
{
name: "video: no vlm field → pass",
- setups: map[string]schema.ParserSetup{"video": {"output_format": "text"}},
+ setups: map[string]schema.ParserSetup{"video": {}},
},
{
name: "audio: vlm.llm_id empty → pass (empty falls back to tenant default)",
@@ -162,7 +162,7 @@ func TestParserComponent_Check(t *testing.T) {
},
{
name: "only unrelated family → pass",
- setups: map[string]schema.ParserSetup{"markdown": {"output_format": "json"}},
+ setups: map[string]schema.ParserSetup{"markdown": {}},
},
}
diff --git a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
index 90697b8fb4..83dbb30818 100644
--- a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
+++ b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
@@ -63,7 +63,6 @@ func TestDispatch_PDFVisionJSON_RealPDFFixture(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
- setups["pdf"]["output_format"] = "json"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 62070b38ab..a09fe23228 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -302,7 +302,6 @@ func TestDispatch_JSONOutput(t *testing.T) {
func TestDispatchNormalizesConfiguredOutputFormatToJSON(t *testing.T) {
setups := defaultSetups()
- setups["email"]["output_format"] = "text"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -433,7 +432,6 @@ func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {
}
setups := defaultSetups()
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -470,7 +468,6 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {
t.Run(method, func(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = method
- setups["pdf"]["output_format"] = "json"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -565,7 +562,6 @@ func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
- setups["pdf"]["output_format"] = "json"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -631,7 +627,6 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
- setups["pdf"]["output_format"] = "json"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -688,7 +683,6 @@ func TestDispatch_PDFVisionJSON_HonorsPageRanges(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
- setups["pdf"]["output_format"] = "json"
setups["pdf"]["pages"] = []any{[]any{float64(2), float64(3)}}
c := &ParserComponent{setups: setups}
@@ -728,7 +722,6 @@ func TestDispatch_PDFVisionJSON_RejectsInvalidPages(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
- setups["pdf"]["output_format"] = "json"
setups["pdf"]["pages"] = "abc"
c := &ParserComponent{setups: setups}
@@ -774,7 +767,6 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -822,7 +814,6 @@ func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
- setups["pdf"]["output_format"] = "json"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -880,7 +871,6 @@ func TestDispatch_PDFMinerUMarkdown_SendsServerURLFromProviderConfig(t *testing.
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -952,7 +942,6 @@ func TestDispatch_PDFMinerUV1_WiresParseAndExtract(t *testing.T) {
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
- setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["mineru_backend"] = "standard"
return invokeMinerUDispatch(t, nil, setups)
}
@@ -1106,7 +1095,6 @@ func TestDispatch_PDFMinerUV1_WiresParseAndExtract(t *testing.T) {
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
- setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["mineru_backend"] = "standard"
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1162,7 +1150,7 @@ func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {
}
component, err := NewParserComponent(map[string]any{
- "pdf": map[string]any{"parse_method": "monkeyocrv2", "output_format": "markdown"},
+ "pdf": map[string]any{"parse_method": "monkeyocrv2"},
})
if err != nil {
t.Fatalf("NewParserComponent: %v", err)
@@ -1265,7 +1253,6 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesTenantModel(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "PaddleOCR"
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1327,7 +1314,6 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "PaddleOCR"
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1352,7 +1338,6 @@ func TestDispatch_PDFPaddleOCR_NoTenantModel_HardErrors(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "paddleocr"
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1415,7 +1400,6 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["layout_recognizer"] = modelID
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1479,7 +1463,6 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = modelID
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1554,7 +1537,6 @@ func TestDispatch_PDFMinerU_BareModelUUID_InParseMethod(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = modelID
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1618,7 +1600,6 @@ func TestDispatch_PDFMinerU_LanguageChain(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
- setups["pdf"]["output_format"] = "markdown"
for k, v := range tc.overrides {
setups["pdf"][k] = v
}
@@ -1692,7 +1673,6 @@ func TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "deepdoc"
setups["pdf"]["layout_recognizer"] = modelID
- setups["pdf"]["output_format"] = "markdown"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -1808,7 +1788,6 @@ func TestDispatch_PDFMinerU_HonorsMinerUParseMethodOption(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
setups["pdf"]["mineru_parse_method"] = "ocr"
- setups["pdf"]["output_format"] = "markdown"
out := invokeMinerUDispatch(t, nil, setups)
requireJSONText(t, out, "Real Resolver Title")
@@ -1837,7 +1816,6 @@ func TestDispatch_PDFMinerU_CompositeSelectorInParseMethod(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = selector
- setups["pdf"]["output_format"] = "markdown"
out := invokeMinerUDispatch(t, nil, setups)
requireJSONText(t, out, "Real Resolver Title")
@@ -1855,7 +1833,6 @@ func TestDispatch_PDFMinerU_NamedSelector_ResolvesMinerUProviderModel(t *testing
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
- setups["pdf"]["output_format"] = "markdown"
out := invokeMinerUDispatch(t, db, setups)
requireJSONText(t, out, "Real Resolver Title")
@@ -1873,7 +1850,6 @@ func TestDispatch_PDFMinerU_BareModelUUID_ResolvesThroughRealProbeAndResolver(t
setups := defaultSetups()
setups["pdf"]["parse_method"] = modelID
- setups["pdf"]["output_format"] = "markdown"
out := invokeMinerUDispatch(t, db, setups)
requireJSONText(t, out, "Real Resolver Title")
@@ -1892,7 +1868,6 @@ func TestDispatch_PDFMinerU_CompositeSelector_ResolvesThroughRealResolver(t *tes
setups := defaultSetups()
setups["pdf"]["parse_method"] = "deepdoc"
setups["pdf"]["layout_recognizer"] = "MinerU-model@default@MinerU"
- setups["pdf"]["output_format"] = "markdown"
out := invokeMinerUDispatch(t, db, setups)
requireJSONText(t, out, "Real Resolver Title")
@@ -1975,7 +1950,6 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "Docling"
- setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["docling_server_url"] = server.URL
setups["pdf"]["docling_api_key"] = "doc-secret"
c := &ParserComponent{setups: setups}
@@ -2011,7 +1985,6 @@ func TestDispatch_PDFOpenDataLoaderLegacyMarkdownEmitsJSON(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "OpenDataLoader"
- setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["opendataloader_apiserver"] = server.URL
c := &ParserComponent{setups: setups}
@@ -2042,7 +2015,6 @@ func TestDispatch_PDFSoMarkLegacyMarkdownEmitsJSON(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "SoMark"
- setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["somark_base_url"] = server.URL
c := &ParserComponent{setups: setups}
@@ -2075,7 +2047,6 @@ func TestDispatch_PDFTCADPLegacyMarkdownEmitsJSON(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "TCADP parser"
- setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["tcadp_apiserver"] = server.URL
c := &ParserComponent{setups: setups}
@@ -2147,16 +2118,13 @@ func TestPythonFamilyName_FileTypeConstants(t *testing.T) {
}
}
-// TestConfigureParserFromSetups_VisualAural pins that image and audio
-// files pick up their setup (parse_method / lang / vlm) via the family
-// mapping. Before the fix, configureParserFromSetups silently skipped
-// configuration because resolveParserFamily returned "picture" / "aural",
-// neither of which existed in defaultSetups.
-func TestConfigureParserFromSetups_VisualAural(t *testing.T) {
+// TestConfigureParserFromSetups_Visual pins that image files pick up their
+// setup (parse_method / lang / vlm) via the family mapping. Before the fix,
+// configureParserFromSetups silently skipped configuration because
+// resolveParserFamily returned "picture", which did not exist in
+// defaultSetups.
+func TestConfigureParserFromSetups_Visual(t *testing.T) {
setups := defaultSetups()
- // The component normalizes output_format onto every family before the
- // dispatch layer sees the setups, so mirror that here.
- normalizeParserOutputFormats(setups)
t.Run("visual resolves to image setup", func(t *testing.T) {
got := &captureSetupConfigurer{}
@@ -2170,15 +2138,4 @@ func TestConfigureParserFromSetups_VisualAural(t *testing.T) {
}
})
- t.Run("aural resolves to audio setup", func(t *testing.T) {
- got := &captureSetupConfigurer{}
- configureParserFromSetups(got, utility.FileTypeAURAL, setups)
- want := map[string]any(setups["audio"])
- if got.setup == nil {
- t.Fatal("ConfigureFromSetup not called for FileTypeAURAL")
- }
- if v, _ := got.setup["output_format"].(string); v != want["output_format"] {
- t.Errorf("FileTypeAURAL output_format = %v, want %v", v, want["output_format"])
- }
- })
}
diff --git a/internal/ingestion/component/parser_legacy_output_format_test.go b/internal/ingestion/component/parser_legacy_output_format_test.go
new file mode 100644
index 0000000000..3796fd3617
--- /dev/null
+++ b/internal/ingestion/component/parser_legacy_output_format_test.go
@@ -0,0 +1,76 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "testing"
+
+ "ragflow/internal/parser/parser"
+)
+
+// TestLegacyOutputFormatKeyIsIgnored pins the one contract that survives the
+// key's removal: a persisted canvas may still carry output_format in a Parser
+// family setup, and no reader consumes it — the component builds, parses, and
+// emits the single JSON protocol regardless of the baked value.
+func TestLegacyOutputFormatKeyIsIgnored(t *testing.T) {
+ const eml = "From: [email protected]\r\n" +
+ "To: [email protected]\r\n" +
+ "Subject: Hi\r\n" +
+ "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" +
+ "\r\n" +
+ "Hello body.\r\n"
+
+ cases := []struct {
+ name string
+ params map[string]any
+ filename string
+ binary []byte
+ }{
+ {
+ name: "markdown legacy text",
+ params: map[string]any{"markdown": map[string]any{"output_format": "text"}},
+ filename: "note.md",
+ binary: []byte("# Heading\n\nBody paragraph.\n"),
+ },
+ {
+ name: "email legacy text",
+ params: map[string]any{"email": map[string]any{"output_format": "text"}},
+ filename: "mail.eml",
+ binary: []byte(eml),
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ comp, err := NewParserComponent(tc.params)
+ if err != nil {
+ t.Fatalf("NewParserComponent: %v", err)
+ }
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"binary": tc.binary, "name": tc.filename})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
+ }
+ items, ok := out["json"].([]parser.DocItem)
+ if !ok || len(items) == 0 {
+ t.Fatalf("json items = %#v, want non-empty", out["json"])
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/parser_output_format_test.go b/internal/ingestion/component/parser_output_format_test.go
deleted file mode 100644
index d891751d02..0000000000
--- a/internal/ingestion/component/parser_output_format_test.go
+++ /dev/null
@@ -1,71 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package component
-
-import (
- "testing"
-)
-
-// TestDefaultSetupsNormalizedToJSON pins that the raw defaults already carry
-// the single JSON output protocol. defaultSetups() must not ship a per-family
-// output_format literal (normalizeParserOutputFormats owns that key), yet
-// every family it returns must end up on JSON — otherwise a caller building a
-// ParserComponent straight from defaultSetups() would silently get an
-// un-normalized setup (the fragility flagged in the PR #21084 review).
-func TestDefaultSetupsNormalizedToJSON(t *testing.T) {
- for family, setup := range defaultSetups() {
- if got := setup["output_format"]; got != "json" {
- t.Errorf("defaultSetups()[%q].output_format = %v, want json", family, got)
- }
- }
-}
-
-// TestParserOutputFormatAlwaysJSON pins the observable contract that survives
-// the literal cleanup: however the component is constructed — defaults, an
-// empty param map, or a legacy per-family override — every family ends up on
-// the single JSON output protocol.
-func TestParserOutputFormatAlwaysJSON(t *testing.T) {
- cases := []struct {
- name string
- params map[string]any
- }{
- {"defaults", nil},
- {"empty params", map[string]any{}},
- {"legacy markdown override", map[string]any{"pdf": map[string]any{"output_format": "markdown"}}},
- {"legacy text override", map[string]any{"text&code": map[string]any{"output_format": "text"}}},
- }
- for _, c := range cases {
- t.Run(c.name, func(t *testing.T) {
- comp, err := NewParserComponent(c.params)
- if err != nil {
- t.Fatalf("NewParserComponent: %v", err)
- }
- pc, ok := comp.(*ParserComponent)
- if !ok {
- t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)
- }
- if len(pc.setups) == 0 {
- t.Fatal("expected non-empty setups")
- }
- for family, setup := range pc.setups {
- if got := setup["output_format"]; got != "json" {
- t.Errorf("family %q output_format = %v, want json", family, got)
- }
- }
- })
- }
-}
diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go
index 58b216e5b6..820f61f9f9 100644
--- a/internal/ingestion/component/parser_test.go
+++ b/internal/ingestion/component/parser_test.go
@@ -181,11 +181,9 @@ func TestParserComponentInvoke_LanguageFromGlobals(t *testing.T) {
}
}
-func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {
+func TestNewParserComponentSkipsNonFamilyParams(t *testing.T) {
component, err := NewParserComponent(map[string]any{
- "pdf": map[string]any{"output_format": "markdown"},
- "spreadsheet": map[string]any{"output_format": "html"},
- "email": map[string]any{"output_format": "text"},
+ "pdf": map[string]any{"parse_method": "deepdoc"},
"allowed_output_format": map[string]any{
"pdf": []any{"json", "markdown"},
},
@@ -195,11 +193,6 @@ func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {
}
parserComponent := component.(*ParserComponent)
- for _, family := range []string{"pdf", "spreadsheet", "email"} {
- if got := parserComponent.setups[family]["output_format"]; got != "json" {
- t.Errorf("%s output_format = %v, want json", family, got)
- }
- }
if _, ok := parserComponent.setups["allowed_output_format"]; ok {
t.Error("allowed_output_format must not be treated as a parser setup")
}
@@ -381,24 +374,6 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {
}
}
-// TestParserComponent_New_Defaults constructs a Parser from a nil param map
-// and verifies every parser family uses the canonical JSON output format.
-func TestParserComponent_New_Defaults(t *testing.T) {
- c, err := NewParserComponent(nil)
- if err != nil {
- t.Fatalf("NewParserComponent(nil): %v", err)
- }
- pc, ok := c.(*ParserComponent)
- if !ok {
- t.Fatalf("NewParserComponent returned %T, want *ParserComponent", c)
- }
- for family, setup := range pc.setups {
- if got := setup["output_format"]; got != "json" {
- t.Errorf("%s output_format = %v, want json", family, got)
- }
- }
-}
-
// TestParserComponent_New_Overrides verifies that a non-nil
// param map with a file-type entry is layered on top of the
// defaults.
diff --git a/internal/ingestion/component/pdf_parser_model_test.go b/internal/ingestion/component/pdf_parser_model_test.go
index 5a36076c48..0fa2cc7e45 100644
--- a/internal/ingestion/component/pdf_parser_model_test.go
+++ b/internal/ingestion/component/pdf_parser_model_test.go
@@ -142,7 +142,7 @@ func TestTenantPDFParserModelSelection(t *testing.T) {
if form == "id" {
ref = "model-b"
}
- setup := schema.ParserSetup{"parse_method": "deepdoc", "output_format": "json", field: ref}
+ setup := schema.ParserSetup{"parse_method": "deepdoc", field: ref}
before, _ := json.Marshal(setup)
res, handled, err := maybeDispatchPDFVision(t.Context(), db, utility.FileTypePDF,
"sample.pdf", []byte("%PDF local protocol fixture"), map[string]any{"tenant_id": "tenant-1"},
@@ -257,7 +257,7 @@ func TestTenantPDFParserMissingModelIDIsHandled(t *testing.T) {
for _, field := range []string{"parse_method", "layout_recognizer"} {
t.Run(provider+"/"+field, func(t *testing.T) {
db, _, requests := tenantPDFParserFixture(t, provider)
- setup := schema.ParserSetup{"parse_method": "deepdoc", "output_format": "json", field: missingModelID}
+ setup := schema.ParserSetup{"parse_method": "deepdoc", field: missingModelID}
before, err := json.Marshal(setup)
if err != nil {
t.Fatal(err)
@@ -347,7 +347,7 @@ func TestTenantPDFParserMissingModelIDInvokeDoesNotFallback(t *testing.T) {
})
setup := map[string]any{
- "parse_method": tc.method, "output_format": "json", tc.field: missingModelID,
+ "parse_method": tc.method, tc.field: missingModelID,
"somark_base_url": fallback.URL + "/setup/somark", "somark_api_key": "setup-key-must-not-be-used",
"opendataloader_apiserver": fallback.URL + "/setup/opendataloader",
"opendataloader_api_key": "setup-key-must-not-be-used",
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 9cf69b4d3b..8ffdeaf770 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -111,7 +111,7 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {
t.Fatalf("result=%+v", result)
}
- resultMD, err := dispatchMonkeyOCRv2PDF(context.Background(), nil, "sample.pdf", []byte("pdf"), "", schema.ParserSetup{"monkeyocrv2_server_url": server.URL, "output_format": "markdown"}, "")
+ resultMD, err := dispatchMonkeyOCRv2PDF(context.Background(), nil, "sample.pdf", []byte("pdf"), "", schema.ParserSetup{"monkeyocrv2_server_url": server.URL}, "")
if err != nil {
t.Fatal(err)
}
@@ -175,14 +175,12 @@ func TestDispatchPaddleOCRPDFUsesStructuredOutput(t *testing.T) {
resolvePaddleOCRModelForDispatch = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, error) {
return &paddleOCRFakeDriver{pages: []modelModule.OCRPage{{Index: 0, Blocks: []modelModule.OCRBlock{{Text: "# Heading\n\nBody"}}}}}, "ocr-model", &modelModule.APIConfig{}, nil
}
- for _, format := range []string{"json", "markdown", ""} {
- result, err := dispatchPaddleOCRPdf(t.Context(), nil, "sample.pdf", nil, "tenant", schema.ParserSetup{"output_format": format}, "model")
- if err != nil {
- t.Fatal(err)
- }
- if result.OutputFormat != "json" || len(result.JSON) == 0 {
- t.Fatalf("result=%+v", result)
- }
+ result, err := dispatchPaddleOCRPdf(t.Context(), nil, "sample.pdf", nil, "tenant", schema.ParserSetup{}, "model")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.OutputFormat != "json" || len(result.JSON) == 0 {
+ t.Fatalf("result=%+v", result)
}
}
diff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.go
index 32a4c9508f..8ca4abe79d 100644
--- a/internal/ingestion/component/schema/parser.go
+++ b/internal/ingestion/component/schema/parser.go
@@ -17,7 +17,7 @@
package schema
// ParserSetup is a per-filetype parser configuration block. The keys are heterogeneous (e.g.,
-// `parse_method`, `lang`, `output_format`, `suffix`, `fields`, `vlm`),
+// `parse_method`, `lang`, `suffix`, `fields`, `vlm`),
// so a free-form map best mirrors the Python dict literal.
type ParserSetup map[string]any
diff --git a/internal/ingestion/pipeline/image_ocr_test.go b/internal/ingestion/pipeline/image_ocr_test.go
index 42856485a1..0d039510d9 100644
--- a/internal/ingestion/pipeline/image_ocr_test.go
+++ b/internal/ingestion/pipeline/image_ocr_test.go
@@ -115,7 +115,7 @@ func TestImageOCRSavedConfigToSearchableChunk(t *testing.T) {
}
const parserID = "Parser:ViewsCaptureLight"
config := BuildParserConfig(dsl, map[string]any{parserID: map[string]any{
- "image": map[string]any{"ocr_enabled": true, "ocr_model_id": "ocr-model", "output_format": "json"},
+ "image": map[string]any{"ocr_enabled": true, "ocr_model_id": "ocr-model"},
"enable_vision_enhancement": false,
}})
for _, row := range []any{
diff --git a/internal/ingestion/pipeline/pdf_pages_test.go b/internal/ingestion/pipeline/pdf_pages_test.go
index 2256e23492..35224beee1 100644
--- a/internal/ingestion/pipeline/pdf_pages_test.go
+++ b/internal/ingestion/pipeline/pdf_pages_test.go
@@ -76,7 +76,7 @@ func TestNormalizeParserConfigPages(t *testing.T) {
t.Run("no pages key -> unchanged", func(t *testing.T) {
cfg := map[string]any{
"Parser:X": map[string]any{
- "pdf": map[string]any{"output_format": "json"},
+ "pdf": map[string]any{"parse_method": "DeepDOC"},
},
}
if err := NormalizeParserConfigPages(cfg); err != nil {
@@ -86,7 +86,7 @@ func TestNormalizeParserConfigPages(t *testing.T) {
if _, ok := pdf["pages"]; ok {
t.Error("pages key should not exist")
}
- if pdf["output_format"] != "json" {
+ if pdf["parse_method"] != "DeepDOC" {
t.Error("other fields should not be touched")
}
})
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 4f2f6f4471..c56b3d2208 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -21,7 +21,7 @@ func generalDSL(t *testing.T) []byte {
"params": map[string]any{
"outputs": map[string]any{},
"pdf": map[string]any{"parse_method": "DeepDOC", "lang": "en"},
- "docx": map[string]any{"output_format": "json"},
+ "docx": map[string]any{"lang": "en"},
},
},
},
diff --git a/internal/ingestion/pipeline/template_output_format_test.go b/internal/ingestion/pipeline/template_output_format_test.go
index c8cc2cc45d..fe14309cfb 100644
--- a/internal/ingestion/pipeline/template_output_format_test.go
+++ b/internal/ingestion/pipeline/template_output_format_test.go
@@ -17,16 +17,17 @@ package pipeline
import (
"encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
"testing"
)
// parserSetupParamsKeys are the Parser params keys that are NOT per-family
-// setups: the wire-format declaration, the global vision settings and the
-// legacy allow-list. Every other object-valued params key is a file family
-// whose setup map may carry output_format.
+// setups: the wire-format declaration and the global vision settings. Every
+// other object-valued params key is a file family.
var parserSetupParamsKeys = map[string]bool{
"outputs": true,
- "allowed_output_format": true,
"enable_vision_enhancement": true,
"vlm": true,
}
@@ -34,12 +35,9 @@ var parserSetupParamsKeys = map[string]bool{
// TestBuiltinTemplatesCarryNoParserOutputFormat pins that no builtin
// ingestion template declares a Parser family's output_format.
//
-// The Go Parser component exposes JSON only: the Parser runtime overwrites
-// every family's output_format with "json" when the component is built, so a
-// template-baked value never reaches the parser that would read it. The key
-// is therefore dead configuration in the templates: it states an intent the
-// runtime discards, and leaving it in place invites a reader to believe the
-// value is honoured.
+// The Go Parser component exposes JSON only: no parser reads the key, so a
+// template-baked value states an intent that is never honoured. Leaving it in
+// place invites a reader to believe the value matters.
//
// Both copies are checked. dsl.components holds the params the runtime
// builds the Parser component from; dsl.graph.nodes holds the per-family
@@ -93,7 +91,7 @@ func assertNoParserOutputFormatInParams(t *testing.T, ref string, root map[strin
continue
}
if value, found := setup["output_format"]; found {
- t.Errorf("%s: Parser %s family %q declares output_format = %v; the Parser runtime overwrites it with \"json\", so the key is dead configuration", ref, cpnID, family, value)
+ t.Errorf("%s: Parser %s family %q declares output_format = %v; no parser reads the key", ref, cpnID, family, value)
}
}
}
@@ -120,8 +118,50 @@ func assertNoParserOutputFormatInGraph(t *testing.T, ref string, root map[string
setup, _ := rawSetup.(map[string]any)
fileFormat, _ := setup["fileFormat"].(string)
if value, found := setup["output_format"]; found {
- t.Errorf("%s: Parser node %s family %q declares output_format = %v; the Parser runtime overwrites it with \"json\", so the key is dead configuration", ref, id, fileFormat, value)
+ t.Errorf("%s: Parser node %s family %q declares output_format = %v; no parser reads the key", ref, id, fileFormat, value)
}
}
}
}
+
+// TestAgentTemplatesCarryNoParserOutputFormat extends the pin to the seeded
+// canvas templates under internal/agent/templates. The ingestion-pipeline
+// canvases there (title_chunker, chunk_summary, advanced_ingestion_pipeline)
+// carry Parser nodes too, and the dao seeder copies their DSL verbatim into
+// the canvas_template table, so a baked key there reaches live canvases.
+// compiler.json is a bare DSL fixture without the metadata wrapper, so the
+// "dsl" key is optional.
+func TestAgentTemplatesCarryNoParserOutputFormat(t *testing.T) {
+ dir := filepath.Join("..", "..", "agent", "templates")
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatalf("read agent templates: %v", err)
+ }
+ checked := 0
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.IsDir() || !strings.HasSuffix(name, ".json") {
+ continue
+ }
+ raw, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil {
+ t.Fatalf("read %s: %v", name, err)
+ }
+ var wrapper map[string]any
+ if err := json.Unmarshal(raw, &wrapper); err != nil {
+ t.Fatalf("decode %s: %v", name, err)
+ }
+ root, ok := wrapper["dsl"].(map[string]any)
+ if !ok {
+ root = wrapper
+ }
+ checked++
+ t.Run(name, func(t *testing.T) {
+ assertNoParserOutputFormatInParams(t, name, root)
+ assertNoParserOutputFormatInGraph(t, name, root)
+ })
+ }
+ if checked == 0 {
+ t.Fatal("no agent templates found")
+ }
+}
diff --git a/internal/parser/parser/audio_parser.go b/internal/parser/parser/audio_parser.go
index 8bdf5e0188..bbcd22a1c3 100644
--- a/internal/parser/parser/audio_parser.go
+++ b/internal/parser/parser/audio_parser.go
@@ -39,26 +39,13 @@ var audioExtensions = map[string]bool{
}
// AudioParser handles audio files for transcription.
-type AudioParser struct {
- OutputFormat string
-}
+type AudioParser struct{}
// NewAudioParser constructs an AudioParser.
func NewAudioParser() *AudioParser {
return &AudioParser{}
}
-// ConfigureFromSetup reads audio-specific configuration from the
-// parser setup map.
-func (p *AudioParser) ConfigureFromSetup(setup map[string]any) {
- if p == nil || setup == nil {
- return
- }
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
-}
-
// ParseWithResult implements ParseResultProducer. It validates the
// file extension against the audio extension whitelist. The actual
// speech-to-text transcription happens via maybeDispatchAudio at the
@@ -74,13 +61,8 @@ func (p *AudioParser) ParseWithResult(ctx context.Context, filename string, data
}
}
- outFmt := p.OutputFormat
- if outFmt == "" {
- outFmt = "text"
- }
-
return ParseResult{
- OutputFormat: outFmt,
+ OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/audio_parser_test.go b/internal/parser/parser/audio_parser_test.go
index 12e03393fe..e47d1b8d27 100644
--- a/internal/parser/parser/audio_parser_test.go
+++ b/internal/parser/parser/audio_parser_test.go
@@ -24,16 +24,13 @@ import (
func TestAudioParser_ValidExtension(t *testing.T) {
ctx := t.Context()
p := NewAudioParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "text",
- })
result := p.ParseWithResult(ctx, "test.mp3", []byte{1, 2, 3, 4})
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "text" {
- t.Errorf("output_format = %q, want text", result.OutputFormat)
+ if result.OutputFormat != "json" {
+ t.Errorf("output_format = %q, want json", result.OutputFormat)
}
if name, ok := result.File["name"].(string); !ok || name != "test.mp3" {
t.Errorf("name = %q, want test.mp3", name)
@@ -63,18 +60,6 @@ func TestAudioParser_InvalidExtension(t *testing.T) {
}
}
-func TestAudioParser_ConfigureNilSafe(t *testing.T) {
- ctx := t.Context()
- p := NewAudioParser()
- p.ConfigureFromSetup(nil)
- p.ConfigureFromSetup(map[string]any{})
- // Should not panic.
- result := p.ParseWithResult(ctx, "test.wav", []byte{})
- if result.Err != nil {
- t.Errorf("unexpected error: %v", result.Err)
- }
-}
-
func TestAudioParser_DefaultOutputFormat(t *testing.T) {
ctx := t.Context()
p := NewAudioParser()
@@ -82,7 +67,7 @@ func TestAudioParser_DefaultOutputFormat(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "text" {
- t.Errorf("output_format = %q, want text (default)", result.OutputFormat)
+ if result.OutputFormat != "json" {
+ t.Errorf("output_format = %q, want json", result.OutputFormat)
}
}
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 88274d3e75..499463e230 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -49,7 +49,6 @@ const (
// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.
type CSVParser struct {
ParseMethod string
- OutputFormat string
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -74,9 +73,6 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["parse_method"].(string); ok && v != "" {
p.ParseMethod = v
}
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
deprecatedHTML4Excel(setup, p.String())
deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
@@ -105,7 +101,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
ctx, filename, data, "CSV",
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
- p.OutputFormat,
)
case "", "csv":
// Continue with the local CSV parser.
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index 33fd994dec..55ad19f27c 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -102,9 +102,6 @@ func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {
func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
csvData := []byte("col1,col2\nval1,val2\n")
p := NewCSVParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- })
res := p.ParseWithResult(context.Background(), "test.csv", csvData)
if res.Err != nil {
@@ -127,9 +124,6 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
func TestCSVParser_EmptyContent(t *testing.T) {
p := NewCSVParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- })
res := p.ParseWithResult(context.Background(), "empty.csv", []byte(" \n\n "))
if res.Err != nil {
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 1aff00bfac..5aaff1c8d6 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -37,14 +37,7 @@ func (p *DOCParser) String() string {
return "DOCParser"
}
-func (p *DOCParser) ConfigureFromSetup(setup map[string]any) {
- // DOC always emits JSON (the DSL hardcodes output_format to "json" via
- // normalizeParserOutputFormats); the legacy text/markdown output paths were
- // removed with the dead ParseResult.Text field.
-}
-
-// ParseWithResult extracts text for the DOC family and converts it to the
-// requested output format (defaulting to "json" to match Python's default).
+// ParseWithResult extracts text for the DOC family as structured JSON items.
func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
irJSON, mdText, plainText, _, err := docExtract(data, "doc")
if err != nil {
@@ -59,12 +52,13 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}
}
- // The DSL hardcodes output_format to "json" (normalizeParserOutputFormats),
- // so the DOC parser always emits structured JSON items. Prefer the IR
- // sections, which recover table/list structure for the chunker's HTML-table
- // rendering (#20918); fall back to the flattened text only when the IR yields
- // no usable sections, so a document whose IR is empty/unusable still keeps
- // its content rather than silently producing 0 chunks.
+ // Prefer the structured IR, which office_oxide recovers table
+ // (and list) structure for. Emitting structured sections lets the chunker
+ // render tables as HTML (#20918) instead of losing them to plain text — the
+ // legacy path flattened the whole document (tables included) into a single
+ // text item. Fall back to the flattened text only when the IR yields no
+ // usable sections, so a document whose IR is empty/unusable still keeps its
+ // content (PlainText / ToMarkdown) rather than silently producing 0 chunks.
if strings.TrimSpace(irJSON) != "" {
budget := newEmbeddedMediaBudget()
if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {
diff --git a/internal/parser/parser/docx_caption_cgo_test.go b/internal/parser/parser/docx_caption_cgo_test.go
index 12c944dbc0..1987dd1e94 100644
--- a/internal/parser/parser/docx_caption_cgo_test.go
+++ b/internal/parser/parser/docx_caption_cgo_test.go
@@ -111,7 +111,6 @@ func docxCaptionFindTable(t *testing.T, res ParseResult) *entity.TableData {
func docxCaptionParse(t *testing.T, bodyInner string) ParseResult {
p := NewDOCXParser()
- p.outputFormat = "json"
res := p.ParseWithResult(t.Context(), "caption.docx", docxCaptionTestDOCX(t, bodyInner))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go
index d0c3aa982c..13521f6f8b 100644
--- a/internal/parser/parser/docx_parser.go
+++ b/internal/parser/parser/docx_parser.go
@@ -19,7 +19,6 @@ import (
"context"
"errors"
"fmt"
- "strings"
deepdocoffice "ragflow/internal/deepdoc/parser/office"
)
@@ -37,7 +36,6 @@ var docxExtract = deepdocoffice.OpenAndExtract
// so they compile and test without native libraries.
type DOCXParser struct {
libType string
- outputFormat string // from DSL config; "json" or "markdown"
RemoveTOC bool
RemoveHeaderFooter bool
}
@@ -47,15 +45,11 @@ func NewDOCXParser() *DOCXParser {
}
// ConfigureFromSetup implements parserSetupConfigurer, receiving the
-// DSL "docx" family setup map. The output_format key drives whether
-// ParseWithResult produces JSON items (structured) or Markdown.
+// DSL "docx" family setup map.
func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
if p == nil || setup == nil {
return
}
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.outputFormat = v
- }
if v, ok := setup["remove_toc"].(bool); ok {
p.RemoveTOC = v
}
@@ -64,13 +58,9 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
}
}
-// ParseWithResult produces structured JSON items (when
-// p.outputFormat == "json") or markdown (default) from a
-// docx document. The JSON path emits bounded image items; the
-// Markdown path attaches bounded figure metadata to the file result.
-//
-// JSON path mirrors python parser.py:_docx() output_format == "json".
-// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().
+// ParseWithResult produces structured JSON items from a docx document.
+// It emits bounded image items and attaches figure metadata to the file
+// result.
//
// File["format"] in the returned ParseResult reflects the effective
// container format after magic-byte sniffing (e.g. "doc" for an OLE
@@ -92,7 +82,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
// opens the payload as ZIP and fails closed, so RemoveHeaderFooter becomes
// a no-op for legacy OLE inputs. This is acceptable — legacy DOC has no
// OOXML header/footer parts to strip.
- irJSON, md, _, mdErr, err := docxExtract(data, format)
+ irJSON, _, _, _, err := docxExtract(data, format)
if err != nil {
if errors.Is(err, deepdocoffice.ErrOfficeCGORequired) {
return ParseResult{Err: fmt.Errorf("%w: %s", ErrOfficeCGORequired, filename)}
@@ -105,9 +95,9 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
"format": format,
}
- // Extract IR JSON for section building (JSON path) and
- // embedded-image extraction (both paths). The budget bounds image
- // payloads so the downstream VLM enhancement stays within limits.
+ // Extract IR JSON for section building and embedded-image extraction.
+ // The budget bounds image payloads so the downstream VLM enhancement
+ // stays within limits.
budget := newEmbeddedMediaBudget()
if irJSON != "" {
figures := extractDOCXFiguresFromIR(irJSON, budget)
@@ -116,98 +106,46 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
}
}
- if p.outputFormat == "json" {
- // The JSON path is built entirely from the structured IR, so an
- // empty IR is a fatal parse error for this output format — the
- // Markdown path below still has the plain-text fallback. The
- // OpenAndExtract contract swallows per-view errors, so we cannot
- // distinguish "IR failed" from "IR legitimately empty"; either way
- // the JSON path has nothing to build from, so we fail loud
- // (Finding B) rather than emit empty/garbage JSON sections.
- if irJSON == "" {
- return ParseResult{Err: fmt.Errorf("docx extract: empty IR, cannot build JSON output")}
- }
- // TEMPORARY Caption-styled-paragraph detection (office_oxide does
- // not yet expose paragraph styles; yfedoseev/office_oxide#455).
- // Remove once office_oxide is upgraded to emit them.
- tableCaptions, capErr := extractDOCXTableCaptions(data)
- if capErr != nil {
- tableCaptions = nil
- }
- sections := buildDOCXJSONSectionsWithCaptions(irJSON, budget, tableCaptions)
- if err := ctx.Err(); err != nil {
- return ParseResult{Err: err}
- }
- // remove_header_footer: drop sections whose normalized text
- // matches a docx header/footer entry (mirrors Python
- // parser.py:889-891 extract_docx_header_footer_texts +
- // remove_header_footer_docx_sections).
- if p.RemoveHeaderFooter {
- hfTexts := extractDOCXHeaderFooterTexts(data)
- sections = removeDOCXHeaderFooterSections(sections, hfTexts)
- }
- // remove_toc: filter TOC entries using heading outlines
- // (mirrors Python parser.py:892-893 remove_toc_word).
- if p.RemoveTOC {
- outlines := extractDOCXOutlines(irJSON)
- sections = removeTOCWord(sections, outlines, isEnglishItems(sections))
- }
- if len(sections) == 0 {
- sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
- }
- return ParseResult{
- OutputFormat: "json",
- File: fileMeta,
- JSON: MapsToItems(sections),
- Warnings: budget.warnings(),
- }
+ // The result is built entirely from the structured IR, so an empty IR
+ // is a fatal parse error. The OpenAndExtract contract swallows per-view
+ // errors, so we cannot distinguish "IR failed" from "IR legitimately
+ // empty"; either way there is nothing to build from, so fail loud
+ // rather than emit empty/garbage JSON sections.
+ if irJSON == "" {
+ return ParseResult{Err: fmt.Errorf("docx extract: empty IR, cannot build JSON output")}
}
-
- // Default / Markdown path.
- if mdErr != nil {
- // The Markdown view failed but other views survived. Surface the
- // Markdown render error rather than silently emitting empty markdown
- // (PR review Finding 3). The JSON path above is unaffected because it
- // is built from the IR view.
- return ParseResult{Err: fmt.Errorf("docx extract: markdown render failed: %w", mdErr)}
+ // TEMPORARY Caption-styled-paragraph detection (office_oxide does
+ // not yet expose paragraph styles; yfedoseev/office_oxide#455).
+ // Remove once office_oxide is upgraded to emit them.
+ tableCaptions, capErr := extractDOCXTableCaptions(data)
+ if capErr != nil {
+ tableCaptions = nil
}
- markdownPayload := md
- // remove_header_footer on Markdown: filter lines by exact match
- // (mirrors Python parser.py:923-926 split lines → filter → rejoin).
+ sections := buildDOCXJSONSectionsWithCaptions(irJSON, budget, tableCaptions)
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err}
+ }
+ // remove_header_footer: drop sections whose normalized text
+ // matches a docx header/footer entry (mirrors Python
+ // parser.py:889-891 extract_docx_header_footer_texts +
+ // remove_header_footer_docx_sections).
if p.RemoveHeaderFooter {
hfTexts := extractDOCXHeaderFooterTexts(data)
- lines := strings.Split(markdownPayload, "\n")
- lineItems := make([]map[string]any, 0, len(lines))
- for _, ln := range lines {
- lineItems = append(lineItems, map[string]any{"text": ln})
- }
- lineItems = removeDOCXHeaderFooterSections(lineItems, hfTexts)
- rebuilt := make([]string, 0, len(lineItems))
- for _, item := range lineItems {
- rebuilt = append(rebuilt, itemText(item))
- }
- markdownPayload = strings.Join(rebuilt, "\n")
+ sections = removeDOCXHeaderFooterSections(sections, hfTexts)
}
- // remove_toc on Markdown: split lines, filter, rejoin
- // (mirrors Python parser.py:927-928 remove_toc_word on Markdown).
- if p.RemoveTOC && irJSON != "" {
+ // remove_toc: filter TOC entries using heading outlines
+ // (mirrors Python parser.py:892-893 remove_toc_word).
+ if p.RemoveTOC {
outlines := extractDOCXOutlines(irJSON)
- lines := strings.Split(markdownPayload, "\n")
- lineItems := make([]map[string]any, 0, len(lines))
- for _, ln := range lines {
- lineItems = append(lineItems, map[string]any{"text": ln})
- }
- filtered := removeTOCWord(lineItems, outlines, isEnglishItems(lineItems))
- rebuilt := make([]string, 0, len(filtered))
- for _, item := range filtered {
- rebuilt = append(rebuilt, itemText(item))
- }
- markdownPayload = strings.Join(rebuilt, "\n")
+ sections = removeTOCWord(sections, outlines, isEnglishItems(sections))
+ }
+ if len(sections) == 0 {
+ sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
return ParseResult{
- OutputFormat: "markdown",
+ OutputFormat: "json",
File: fileMeta,
- Markdown: markdownPayload,
+ JSON: MapsToItems(sections),
Warnings: budget.warnings(),
}
}
diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.go
index 1141a0ded0..1a147dc860 100644
--- a/internal/parser/parser/docx_parser_cgo_test.go
+++ b/internal/parser/parser/docx_parser_cgo_test.go
@@ -12,7 +12,6 @@ import (
func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
- p.outputFormat = "json"
data := minimalDOCX(t, "Hello from JSON path")
res := p.ParseWithResult(ctx, "sample.docx", data)
if res.Err != nil {
@@ -34,48 +33,18 @@ func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
}
}
-func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) {
- ctx := t.Context()
+func TestDOCXParser_ConfigureFromSetup_Options(t *testing.T) {
p := NewDOCXParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "json"})
- if p.outputFormat != "json" {
- t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "json")
+ p.ConfigureFromSetup(map[string]any{"remove_toc": true, "remove_header_footer": true})
+ if !p.RemoveTOC {
+ t.Error("RemoveTOC not configured")
+ }
+ if !p.RemoveHeaderFooter {
+ t.Error("RemoveHeaderFooter not configured")
}
if p.libType != "" {
t.Errorf("libType unexpectedly = %q", p.libType)
}
- // Full round-trip: json config → json output
- data := minimalDOCX(t, "Config test")
- res := p.ParseWithResult(ctx, "sample.docx", data)
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "json")
- }
- if len(res.JSON) == 0 {
- t.Error("JSON items is empty")
- }
-}
-
-func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) {
- ctx := t.Context()
- p := NewDOCXParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
- if p.outputFormat != "markdown" {
- t.Fatalf("After ConfigureFromSetup, outputFormat = %q, want %q", p.outputFormat, "markdown")
- }
- data := minimalDOCX(t, "Config md test")
- res := p.ParseWithResult(ctx, "sample.docx", data)
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- if res.OutputFormat != "markdown" {
- t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "markdown")
- }
- if res.Markdown == "" {
- t.Error("Markdown is empty")
- }
}
func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {
@@ -86,11 +55,11 @@ func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "markdown"; got != want {
+ if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if res.Markdown == "" {
- t.Fatal("Markdown is empty; want parsed content")
+ if len(res.JSON) == 0 {
+ t.Fatal("JSON items is empty; want parsed content")
}
}
@@ -139,41 +108,29 @@ func writeZipFile(t *testing.T, zw *zip.Writer, name, body string) {
// that fixed this defect class; v0.1.12 is the pinned version.
func TestDOCXParser_PreservesXMLCharacterReferences(t *testing.T) {
cases := []struct {
- name string
- run string // run text as it appears inside <w:t>, entities unescaped
- want string
- wantMarkdown string
+ name string
+ run string // run text as it appears inside <w:t>, entities unescaped
+ want string
}{
- {"named", "&lt;SEP&gt;", "<SEP>", `\<SEP\>`},
- {"decimal", "&#60;SEP&#62;", "<SEP>", `\<SEP\>`},
- {"hex", "&#x3C;SEP&#x3E;", "<SEP>", `\<SEP\>`},
- {"ampersand", "A&amp;B", "A&B", "A&B"},
- {"apostrophe", "it&apos;s", "it's", "it's"},
- {"emdash", "a&#8212;b", "a—b", "a—b"},
+ {"named", "&lt;SEP&gt;", "<SEP>"},
+ {"decimal", "&#60;SEP&#62;", "<SEP>"},
+ {"hex", "&#x3C;SEP&#x3E;", "<SEP>"},
+ {"ampersand", "A&amp;B", "A&B"},
+ {"apostrophe", "it&apos;s", "it's"},
+ {"emdash", "a&#8212;b", "a—b"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
data := minimalDOCX(t, tc.run)
- jsonParser := NewDOCXParser()
- jsonParser.ConfigureFromSetup(map[string]any{"output_format": "json"})
- jsonRes := jsonParser.ParseWithResult(t.Context(), "entities.docx", data)
- if jsonRes.Err != nil {
- t.Fatalf("ParseWithResult(json): %v", jsonRes.Err)
+ p := NewDOCXParser()
+ res := p.ParseWithResult(t.Context(), "entities.docx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got := joinItemTexts(ItemsToMaps(jsonRes.JSON)); !strings.Contains(got, tc.want) {
+ if got := joinItemTexts(ItemsToMaps(res.JSON)); !strings.Contains(got, tc.want) {
t.Errorf("json text = %q, want it to contain %q", got, tc.want)
}
-
- mdParser := NewDOCXParser()
- mdParser.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
- mdRes := mdParser.ParseWithResult(t.Context(), "entities.docx", data)
- if mdRes.Err != nil {
- t.Fatalf("ParseWithResult(markdown): %v", mdRes.Err)
- }
- if !strings.Contains(mdRes.Markdown, tc.wantMarkdown) {
- t.Errorf("markdown = %q, want it to contain %q", mdRes.Markdown, tc.wantMarkdown)
- }
})
}
}
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index a14cc293a0..c55baf39ae 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -39,23 +39,22 @@ import (
)
// EmailParser parses .eml (RFC 5322 email) and .msg (Outlook OLE2) files
-// into structured JSON or plain-text output. Mirrors Python's _email()
-// method in rag/flow/parser/parser.py, whose .msg branch uses extract_msg.
+// into structured JSON output. Mirrors Python's _email() method in
+// rag/flow/parser/parser.py, whose .msg branch uses extract_msg.
type EmailParser struct {
fields []string
}
+var defaultEmailFields = []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"}
+
func NewEmailParser() *EmailParser {
- return &EmailParser{}
+ return &EmailParser{fields: append([]string(nil), defaultEmailFields...)}
}
func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {
if p == nil || setup == nil {
return
}
- // Email always emits JSON (the DSL hardcodes output_format to "json" via
- // normalizeParserOutputFormats); the legacy text output path was removed with
- // the dead ParseResult.Text field.
if raw, ok := setup["fields"]; ok {
switch list := raw.(type) {
case []string:
@@ -70,7 +69,7 @@ func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {
}
}
if len(p.fields) == 0 {
- p.fields = []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"}
+ p.fields = append([]string(nil), defaultEmailFields...)
}
}
@@ -406,8 +405,6 @@ func attachmentFilename(part *multipart.Part) string {
return "attachment.bin"
}
-// -- HTML body → visible text (text output only) --
-
// decodeMailPayload attempts charset decoding using the unified DecodeToUTF8 helper.
func decodeMailPayload(payload []byte, charset string) string {
if len(payload) == 0 {
@@ -617,10 +614,8 @@ func recoverParse(fn func() ParseResult) (res ParseResult, panicked bool) {
// without failing the whole email (mirrors email.py's per-attachment
// try/except).
//
-// The function returns both forms the caller needs:
-// - extraItems: structured JSON items (one per non-empty text segment),
-// each carrying only "text" and "doc_type_kwd", for the JSON output path.
-// - text: the concatenated attachment text, for the text output path.
+// The function returns structured JSON items (one per non-empty text
+// segment), each carrying only "text" and "doc_type_kwd".
//
// Re-chunking stops at one level of nesting: a nested email is parsed (so its
// body becomes retrievable) but its own attachments are not re-chunked, to
@@ -677,13 +672,9 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s
// Reuse the top-level field configuration (including "body") so a
// nested .eml/.msg is parsed for its body instead of with a fresh,
// unconfigured parser that would only emit metadata and index
- // garbage. We always request JSON output for the nested parse so
- // the extracted "text" field is returned clean (the text output
- // path would otherwise also flatten metadata into the result).
+ // garbage.
ep := NewEmailParser()
- ep.ConfigureFromSetup(map[string]any{
- "fields": p.fields,
- })
+ ep.ConfigureFromSetup(map[string]any{"fields": p.fields})
res, panicked = recoverParse(func() ParseResult {
return ep.parseEmail(ctx, fn, []byte(payload), depth+1)
})
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index 2299d23cee..de78bacf8c 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -54,8 +54,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "to", "cc", "date", "subject", "body", "metadata"},
+ "fields": []string{"from", "to", "cc", "date", "subject", "body", "metadata"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -223,8 +222,7 @@ func TestEmailParser_MsgSupported(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},
+ "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},
})
result := p.ParseWithResult(ctx, "sample.msg", data)
@@ -314,8 +312,7 @@ func TestEmailParser_MsgMetadataAlwaysPresent(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "subject"}, // "metadata" intentionally absent
+ "fields": []string{"from", "subject"}, // "metadata" intentionally absent
})
result := p.ParseWithResult(ctx, "sample.msg", data)
@@ -418,8 +415,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "body", "attachments"},
+ "fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -496,8 +492,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "body", "attachments"},
+ "fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -553,8 +548,7 @@ func TestEmailParser_Multipart(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "body"},
+ "fields": []string{"from", "body"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -590,7 +584,6 @@ func TestEmailParser_MetadataAlwaysPresent(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
// Note: "metadata" is intentionally NOT in fields.
"fields": []string{"from", "subject", "body"},
})
@@ -651,8 +644,7 @@ func TestEmailParser_TextHTMLAlwaysPresent(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "body"},
+ "fields": []string{"from", "body"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -713,8 +705,7 @@ func TestEmailParser_AttachmentSearchableJSON(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "body", "attachments"},
+ "fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -775,8 +766,7 @@ func TestEmailParser_AttachmentEmptyPayloadSkipped(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "body", "attachments"},
+ "fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -831,8 +821,7 @@ func TestEmailParser_AttachmentUnknownExtSkipped(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "body", "attachments"},
+ "fields": []string{"from", "body", "attachments"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
@@ -877,7 +866,6 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
// NOTE: "body" is intentionally NOT in fields.
"fields": []string{"from", "attachments"},
})
@@ -1301,7 +1289,6 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {
"Subject: Q3\r\n" + tt.headers + "\r\n\r\n" + tt.body + "\r\n"
p := NewEmailParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "json"})
result := p.ParseWithResult(t.Context(), "test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 1edba36d67..ce033a5835 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -49,7 +49,6 @@ var ssrfAllowLoopback bool
type MarkdownParser struct {
libType string
ParseMethod string
- OutputFormat string
VLM map[string]any
FlattenMediaToText bool
// FetchRemoteImages controls whether HTTP(S) Markdown images are
@@ -77,9 +76,6 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["parse_method"].(string); ok && v != "" {
p.ParseMethod = v
}
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
if v, ok := setup["vlm"].(map[string]any); ok {
p.VLM = v
}
@@ -89,8 +85,7 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
}
// ParseWithResult implements ParseResultProducer (plan §6.5) and
-// returns a structured Markdown payload that mirrors the Python
-// parser's `output_format == "json"` shape. Each top-level block
+// returns a structured JSON payload. Each top-level block
// emits one item with `text` + `doc_type_kwd: "text"`. When the
// block contains a Markdown image reference (![alt](src)), the image
// data is resolved and the item carries `doc_type_kwd: "image"` with
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 3218b3e246..aee919a9f2 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -394,16 +394,12 @@ func TestMarkdownParser_ConfigureFromSetup(t *testing.T) {
p, _ := NewMarkdownParser(GoMarkdown)
p.ConfigureFromSetup(map[string]any{
"parse_method": "deepdoc",
- "output_format": "json",
"vlm": map[string]any{"llm_id": "gpt-4-vision"},
"flatten_media_to_text": false,
})
if p.ParseMethod != "deepdoc" {
t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc")
}
- if p.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want %q", p.OutputFormat, "json")
- }
if p.VLM == nil {
t.Fatal("VLM is nil; want map")
}
diff --git a/internal/parser/parser/office_detect_test.go b/internal/parser/parser/office_detect_test.go
index ad2ea035ee..52db5a87fb 100644
--- a/internal/parser/parser/office_detect_test.go
+++ b/internal/parser/parser/office_detect_test.go
@@ -73,7 +73,6 @@ func TestDOCXParser_FallsBackToDOCForOLEHeader(t *testing.T) {
func TestDOCXParser_JSONEmptyIRReturnsError(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "json"})
orig := docxExtract
defer func() { docxExtract = orig }()
@@ -88,40 +87,16 @@ func TestDOCXParser_JSONEmptyIRReturnsError(t *testing.T) {
}
}
-// TestDOCXParser_MarkdownViewErrorPropagated locks Finding 3: when the
-// native backend's Markdown view fails but the other views survive,
-// OpenAndExtract surfaces the Markdown error via mdErr and the Markdown output
-// path must fail loud instead of silently emitting empty markdown. The JSON
-// path is unaffected because it is built from the IR view.
-func TestDOCXParser_MarkdownViewErrorPropagated(t *testing.T) {
+// TestDOCXParser_AllViewsEmptyReturnsError locks the engine-level contract
+// at the parser boundary: when the native backend opens the document but
+// every view (IR / Markdown / PlainText) is empty, OpenAndExtract reports a
+// joined error, and the parser must fail loud rather than silently emit
+// empty JSON. (office_oxide always returns a structured IR for an openable
+// doc, so this all-empty error is simulated via the seam rather than a real
+// fixture.)
+func TestDOCXParser_AllViewsEmptyReturnsError(t *testing.T) {
ctx := t.Context()
p := NewDOCXParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
-
- orig := docxExtract
- defer func() { docxExtract = orig }()
- docxExtract = func(data []byte, format string) (string, string, string, error, error) {
- // IR and plain text survive; only the Markdown view fails.
- return "<ir/>", "", "plain text fallback", fmt.Errorf("markdown boom"), nil
- }
-
- res := p.ParseWithResult(ctx, "broken.docx", []byte("PK\x03\x04"))
- if res.Err == nil {
- t.Fatal("expected error when only the Markdown view fails, got nil")
- }
-}
-
-// TestDOCXParser_AllViewsEmptyMarkdownReturnsError locks the engine-level
-// contract from Finding 1 at the parser boundary: when the native backend
-// opens the document but every view (IR / Markdown / PlainText) is empty,
-// OpenAndExtract reports a joined error, and the default Markdown output path
-// must fail loud rather than silently emit empty markdown. (office_oxide
-// always returns a structured IR for an openable doc, so this all-empty error
-// is simulated via the seam rather than a real fixture.)
-func TestDOCXParser_AllViewsEmptyMarkdownReturnsError(t *testing.T) {
- ctx := t.Context()
- p := NewDOCXParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
orig := docxExtract
defer func() { docxExtract = orig }()
diff --git a/internal/parser/parser/office_image_fixture_cgo_test.go b/internal/parser/parser/office_image_fixture_cgo_test.go
index 2241e27f08..dedececab0 100644
--- a/internal/parser/parser/office_image_fixture_cgo_test.go
+++ b/internal/parser/parser/office_image_fixture_cgo_test.go
@@ -34,7 +34,6 @@ func TestDOCXParser_RealTableImage(t *testing.T) {
t.Fatal(err)
}
p := NewDOCXParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "json"})
result := p.ParseWithResult(t.Context(), "docx_table_image.docx", data)
if result.Err != nil {
t.Fatal(result.Err)
diff --git a/internal/parser/parser/pdf_analyzer_dispatch_test.go b/internal/parser/parser/pdf_analyzer_dispatch_test.go
index 0752f7f616..4517cd01e6 100644
--- a/internal/parser/parser/pdf_analyzer_dispatch_test.go
+++ b/internal/parser/parser/pdf_analyzer_dispatch_test.go
@@ -254,7 +254,7 @@ func TestParsePDFWithDeepDocOptions_RoutesRegisteredMethodToFactory(t *testing.T
return &deepdoctype.ParseResult{}, nil
}
res := parsePDFWithDeepDocOptions(context.Background(), "f.pdf", []byte("dummy"),
- pdfPostProcessOptions{outputFormat: "json", parseMethod: "ext_wire"}, parseFn)
+ pdfPostProcessOptions{parseMethod: "ext_wire"}, parseFn)
if res.Err != nil {
t.Fatalf("unexpected error: %v", res.Err)
}
@@ -281,7 +281,7 @@ func TestParsePDFWithDeepDocOptions_EmptyMethodUsesInProcess(t *testing.T) {
return &deepdoctype.ParseResult{}, nil
}
res := parsePDFWithDeepDocOptions(context.Background(), "f.pdf", []byte("dummy"),
- pdfPostProcessOptions{outputFormat: "json"}, parseFn)
+ pdfPostProcessOptions{}, parseFn)
if res.Err != nil {
t.Fatalf("unexpected error: %v", res.Err)
}
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index aa57203814..7ab8d5c55e 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -20,7 +20,6 @@ import (
"context"
"errors"
"fmt"
- "image"
"os"
"sort"
"strings"
@@ -31,7 +30,6 @@ import (
"ragflow/internal/common"
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
- "ragflow/internal/deepdoc/parser/pdf/util"
deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/utility"
)
@@ -65,7 +63,6 @@ type PDFParser struct {
RemoveTOC bool
RemoveHeaderFooter bool
EnableMultiColumn bool
- OutputFormat string
ParseMethod string
// UseTenantModelConfig disables global endpoint/credential/option fallback
// for explicitly resolved SoMark/OpenDataLoader instances. Runtime-only;
@@ -121,7 +118,6 @@ func NewPDFParser() *PDFParser {
Model: "DeepDoc@buildin@ragflow",
LibType: "pdf_oxide",
ParseMethod: "deepdoc",
- OutputFormat: "json",
MinerUBackend: "",
MinerUPollTimeout: minerUPollTimeout,
OpenDataLoaderTimeout: 600,
@@ -179,9 +175,6 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["mineru_timeout_seconds"].(float64); ok && v > 0 {
p.MinerUPollTimeout = time.Duration(v * float64(time.Second))
}
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
if v, ok := setup["docling_server_url"].(string); ok && v != "" {
p.DoclingServerURL = v
}
@@ -373,7 +366,7 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: plain_text: %w", err)}
}
- return pdfItemsToResult(filename, items, p.OutputFormat, pageCount)
+ return pdfItemsToResult(filename, items, pageCount)
case "mineru":
return parsePDFWithMinerU(ctx, filename, data, p)
case "docling":
@@ -391,7 +384,6 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [
cfg.RemoveHeaderFooter = p.RemoveHeaderFooter
cfg.OnPageDone = p.OnPageDone
res := parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{
- outputFormat: p.OutputFormat,
zoom: cfg.Zoom,
enableMultiColumn: p.EnableMultiColumn,
flattenMediaToText: p.FlattenMediaToText,
@@ -553,8 +545,7 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
// the parser-phase memory peak. Image/table sections keep only their PDF
// positions; the chunker re-acquires the source PDF and crops on demand at
// index time, and the VLM path crops on demand when it needs to describe a
- // figure/table. The Markdown path below still inlines, because markdown
- // output embeds images directly and has no downstream on-demand consumer.
+ // figure/table.
items := pdflayout.SectionsToJSON(processed.Sections)
if len(items) == 0 {
@@ -586,31 +577,6 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
}
}
-func pdfParseResultToMarkdownWithOptions(filename string, parsed *deepdoctype.ParseResult, opts pdfPostProcessOptions) ParseResult {
- if parsed == nil {
- return ParseResult{Err: fmt.Errorf("parser: nil DeepDOC PDF result for %s", filename)}
- }
- processed := *parsed
- processed.Sections = append([]deepdoctype.Section(nil), parsed.Sections...)
- processed.Outlines = append([]deepdoctype.Outline(nil), parsed.Outlines...)
- if opts.enableMultiColumn && opts.pageWidth <= 0 {
- opts.pageWidth = firstPDFPageWidth(processed.PageWidth)
- }
- applyPDFPostProcess(&processed, opts)
- defer processed.Close()
- cropMediaSections(&processed)
-
- return ParseResult{
- OutputFormat: "markdown",
- File: map[string]any{
- "name": filename,
- "page_count": len(processed.PageHeight),
- "outline": outlinesToFileMeta(processed.Outlines),
- },
- Markdown: sectionsToMarkdown(processed.Sections),
- }
-}
-
func outlinesToFileMeta(outlines []deepdoctype.Outline) []map[string]any {
if len(outlines) == 0 {
return []map[string]any{}
@@ -643,247 +609,6 @@ func firstPageNumber(raw any) int {
}
}
-func sectionsToMarkdown(sections []deepdoctype.Section) string {
- return pdflayout.SectionsToMarkdown(sections)
-}
-
-// cropMediaSections crops figure and table sections from rendered PDF page
-// images while the engine is still alive, populating sec.Image.
-// It is used by both the JSON and Markdown serialization paths.
-func cropMediaSections(result *deepdoctype.ParseResult) {
- engine := result.Engine
- if engine == nil || len(result.PageHeight) == 0 {
- return
- }
- // Render each page at most once across all media sections. A PDF can have many
- // figures/tables on the same page, so a cache shared across the section loop
- // avoids re-rendering the page for every section.
- //
- // Sections are sorted by firstSectionPage, but a cross-page merge can leave an
- // earlier page in a later position, so the TRUE minimum page is not monotonic.
- // cropMediaSections therefore keeps a sliding window of page images bounded by
- // the minimum page any still-unprocessed renderable section references
- // (computeFuturePageWindow): a page strictly below that future window can never
- // be referenced again, so it is evicted. This bounds memory to the current
- // section's page span instead of caching the whole PDF, and stays bounded even
- // after a page-order break — the old code permanently disabled eviction on the
- // first break, which OOM'd large PDFs.
- pageCache := make(map[int]image.Image)
- var lastMinPage = -1
- sectionsOrdered := true
- renderPage := func(pn int) image.Image {
- if img, ok := pageCache[pn]; ok {
- return img
- }
- img, err := deepdocpdf.RenderPageToImage(engine, pn)
- if err != nil || img == nil {
- common.Warn("cropMediaSections: render failed, skipping section",
- zap.Int("page", pn), zap.Error(err))
- pageCache[pn] = nil
- return nil
- }
- pageCache[pn] = img
- return img
- }
-
- // Precompute, for each section index i, the minimum page number any
- // section that renders from pageCache references from i..end
- // (computeFuturePageWindow: suffix-min over renderable sections only).
- // A page strictly below that window can never be referenced by the current
- // or a later section, so it is safe to evict even after a page-order break.
- // Skipped sections (text, pre-cropped, or with no positions) do NOT
- // contribute: a late skipped section with a low merged page must not keep
- // the window low, or pageCache would grow unbounded again (residual of
- // #19938). A -1 entry means "no renderable section with known pages is
- // still ahead" and eviction is paused for safety.
- minFuturePage := computeFuturePageWindow(result.Sections)
-
- // Page-order break detection, computed once and reused to arm the
- // warn-once latch below. sectionOrderBreaksEviction is also tested
- // directly. It uses the TRUE minimum page (not firstSectionPage), so a
- // cross-page merge that leaves an earlier page in a later position is
- // still detected.
- orderBroken, breakAt := sectionOrderBreaksEviction(result.Sections)
-
- for i := range result.Sections {
- sec := &result.Sections[i]
- // TRUE minimum page this section touches, tracked for every section so
- // the page-order-break diagnostic below matches sectionOrderBreaksEviction
- // exactly. A section with no page info (minPage < 0) cannot establish
- // order and is skipped rather than treated as a (false) decrease.
- minPage := sectionMinPage(*sec)
- if minPage >= 0 {
- if lastMinPage >= 0 && minPage < lastMinPage {
- if orderBroken && sectionsOrdered && i == breakAt {
- common.Warn("cropMediaSections: sections out of page order; eviction now uses future-page window",
- zap.Int("section_index", i),
- zap.Int("total_sections", len(result.Sections)),
- zap.Int("min_page", minPage),
- zap.Int("last_min_page", lastMinPage),
- zap.Int("min_future_page", minFuturePage[i]),
- zap.Int("page_cache_size", len(pageCache)))
- sectionsOrdered = false
- }
- }
- lastMinPage = minPage
- }
- if !sectionRendersFromCache(*sec) {
- continue
- }
- // Collect every distinct page this section spans.
- pages := make(map[int]struct{})
- for _, pos := range sec.Positions {
- for _, pn := range pos.PageNumbers {
- pages[pn] = struct{}{}
- }
- }
- // Evict every page image no current or future section can reference.
- //
- // minFuturePage[i] is the minimum page number referenced by sections
- // i..end (precomputed above). When sections are in page order this
- // reduces to pn < minPage, the original sliding-window bound. When a
- // cross-page merge breaks the order, minPage would under-state the
- // window and stalling eviction would let pageCache grow without bound
- // on large PDFs — so we fall back to the TRUE future minimum page and
- // keep evicting. A -1 window means a section with no page info is
- // still ahead, so we retain the cache rather than risk dropping a page
- // a later section still needs.
- if minFuturePage[i] >= 0 {
- for pn := range pageCache {
- if pn < minFuturePage[i] {
- delete(pageCache, pn)
- }
- }
- } else if i%64 == 0 {
- // Future-page window unknown (a section with no page info is
- // still ahead): eviction is paused so we never drop a page a
- // later section may still reference. Log the cache size so a
- // regression is observable instead of silently OOMing.
- common.Warn("cropMediaSections: future-page window unknown; page cache retained",
- zap.Int("section_index", i),
- zap.Int("total_sections", len(result.Sections)),
- zap.Int("page_cache_size", len(pageCache)),
- zap.Int("min_page", minPage))
- }
- // Collect every distinct page this section spans so CropSectionByDLA
- // can crop and vertically concatenate each page (mirroring Python's
- // cropout multi-page branch). Single-page sections still render exactly
- // one page, and the pageCache above guarantees a page is rendered at
- // most once even when several sections share it.
- single := make(map[int]image.Image, len(pages))
- for pn := range pages {
- if img := renderPage(pn); img != nil {
- single[pn] = img
- }
- }
- if len(single) == 0 {
- continue
- }
- if sec.LayoutType == deepdoctype.LayoutTypeFigure {
- if dla := util.CropSectionByDLA(*sec, result.DLARegions, single); dla != "" {
- sec.Image = dla
- continue
- }
- }
- if len(sec.Positions) > 0 {
- if img := util.CropSectionPositions(sec.Positions, single, deepdoctype.DlaScale); img != "" {
- sec.Image = img
- continue
- }
- }
- sec.Image = util.CropSectionImage(sec.PositionTag, single, deepdoctype.DlaScale)
- }
-}
-
-// sectionMinPage returns the minimum page number touched by any position of s.
-// It is the key cropMediaSections uses to bound its page-image sliding window.
-// It deliberately differs from firstSectionPage (Positions[0].PageNumbers[0]),
-// which sortSectionsByPosition orders sections by: a section sorted later by
-// reading order can still contain an earlier page in one of its (merged)
-// positions, so the two orderings are not equivalent.
-func sectionMinPage(s deepdoctype.Section) int {
- minPage := -1
- for _, pos := range s.Positions {
- for _, pn := range pos.PageNumbers {
- if pn < minPage || minPage < 0 {
- minPage = pn
- }
- }
- }
- return minPage
-}
-
-// sectionRendersFromCache reports whether sec is cropped from a rendered page
-// image by cropMediaSections and therefore participates in the page-image
-// sliding window. It mirrors the render loop's skip predicate exactly, so the
-// precomputed future-page window and the loop agree on which sections consume
-// pageCache.
-func sectionRendersFromCache(sec deepdoctype.Section) bool {
- if strings.TrimSpace(sec.Image) != "" {
- return false
- }
- if sec.LayoutType != deepdoctype.LayoutTypeFigure &&
- sec.LayoutType != deepdoctype.DLALabelFigureCaption &&
- sec.LayoutType != deepdoctype.LayoutTypeTable &&
- strings.TrimSpace(sec.LayoutType) != "image" &&
- sec.DocTypeKwd != "image" && sec.DocTypeKwd != "table" {
- return false
- }
- return len(sec.Positions) > 0
-}
-
-// computeFuturePageWindow returns, for each section index i, the minimum page
-// number referenced by any section that renders from pageCache in the suffix
-// i..end. It feeds cropMediaSections' eviction: a page strictly below the
-// window can never be referenced by the current or a later section, so it is
-// safe to evict. Only sections that actually render (sectionRendersFromCache)
-// contribute — a skipped section with a low merged page must not keep the
-// window low, or the cache would grow unbounded again (residual of #19938).
-// A -1 entry means no renderable section with known pages is still ahead, so
-// eviction is paused for safety.
-func computeFuturePageWindow(sections []deepdoctype.Section) []int {
- minFuturePage := make([]int, len(sections))
- cur := -1
- for j := len(sections) - 1; j >= 0; j-- {
- if sectionRendersFromCache(sections[j]) {
- if mp := sectionMinPage(sections[j]); mp >= 0 {
- if cur < 0 || mp < cur {
- cur = mp
- }
- }
- }
- minFuturePage[j] = cur
- }
- return minFuturePage
-}
-
-// sectionOrderBreaksEviction walks sections in reading order (as produced by
-// sortSectionsByPosition) and reports whether the TRUE minimum page ever
-// decreases relative to the previous section with known pages. cropMediaSections
-// reuses the result to arm its warn-once latch. A decrease is exactly what a
-// cross-page merge produces when it leaves an earlier page in a later position;
-// because the sort only guarantees monotonicity in firstSectionPage (not the
-// TRUE min page), the break is detected here rather than assumed away. A
-// section with no page info (minPage < 0) is skipped instead of being treated
-// as a false decrease. The returned index is the first offending section, or
-// -1 when order holds.
-func sectionOrderBreaksEviction(sections []deepdoctype.Section) (bool, int) {
- lastMinPage := -1
- for i := range sections {
- minPage := sectionMinPage(sections[i])
- if minPage < 0 {
- // Unknown page: cannot establish order, skip rather than treat
- // it as a (false) decrease.
- continue
- }
- if lastMinPage >= 0 && minPage < lastMinPage {
- return true, i
- }
- lastMinPage = minPage
- }
- return false, -1
-}
-
// firstPDFPageWidth returns the first page's width from a map of
// per-page PDF-point widths. The map is guaranteed to store values
// already converted to PDF-point space by processPage.
@@ -1005,14 +730,13 @@ func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []byt
if err != nil {
return ParseResult{Err: err}
}
- var res ParseResult
- switch strings.ToLower(strings.TrimSpace(opts.outputFormat)) {
- case "", "json":
- res = pdfParseResultToJSONWithOptions(filename, parsed, opts)
- case "markdown":
- res = pdfParseResultToMarkdownWithOptions(filename, parsed, opts)
- default:
- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", opts.outputFormat)}
+ res := pdfParseResultToJSONWithOptions(filename, parsed, opts)
+ for i := range res.JSON {
+ m := res.JSON[i].ToMap()
+ if img, _ := m["image"].(string); img != "" {
+ m["image"] = pdflayout.InlinePNGDataURL(img)
+ res.JSON[i] = DocItemFromMap(m)
+ }
}
return res
}
diff --git a/internal/parser/parser/pdf_parser_common_eviction_test.go b/internal/parser/parser/pdf_parser_common_eviction_test.go
deleted file mode 100644
index c5e91f9266..0000000000
--- a/internal/parser/parser/pdf_parser_common_eviction_test.go
+++ /dev/null
@@ -1,187 +0,0 @@
-package parser
-
-import (
- "testing"
-
- deepdoctype "ragflow/internal/deepdoc/parser/type"
-)
-
-// TestSectionOrderBreaksEvictionCleanOrder is the sanity case: sections whose
-// positions are page-monotonic keep eviction enabled. This documents the
-// intended (bounded) behavior cropMediaSections relies on.
-func TestSectionOrderBreaksEvictionCleanOrder(t *testing.T) {
- sections := []deepdoctype.Section{
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{2, 3}}}},
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{4}}}},
- }
- broken, idx := sectionOrderBreaksEviction(sections)
- if broken {
- t.Fatalf("clean in-order sections must not break eviction, got break at index %d", idx)
- }
-}
-
-// TestSectionOrderBreaksEvictionCrossPageMerge reproduces the defect: a figure
-// whose first position is on a LATER page (25) but which also carries a
-// caption merged onto an EARLIER page (18). sortSectionsByPosition orders by
-// firstSectionPage, so this section sorts after a table on page 20, yet its
-// true minimum page (18) is below that table's page. The mismatch flips
-// sectionsOrdered off and permanently disables eviction.
-func TestSectionOrderBreaksEvictionCrossPageMerge(t *testing.T) {
- sections := []deepdoctype.Section{
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{20}}}},
- {LayoutType: deepdoctype.LayoutTypeFigure, Positions: []deepdoctype.Position{
- {PageNumbers: []int{25}},
- {PageNumbers: []int{18}},
- }},
- }
- broken, idx := sectionOrderBreaksEviction(sections)
- if !broken {
- t.Fatalf("expected eviction break from cross-page-merge section, got none")
- }
- if idx != 1 {
- t.Errorf("expected break at index 1, got %d", idx)
- }
-}
-
-// TestCropMediaSectionsEvictionContinuesAfterPageOrderBreak replays the exact
-// eviction loop from cropMediaSections (future-page window) with a fake render
-// that records which pages are kept. It proves the fix for the defect above:
-// after a page-order break, eviction keeps running (using the TRUE future
-// minimum page) instead of stalling forever — so the cache stays bounded to a
-// sliding window of the current section's pages rather than growing without
-// bound and OOMing large PDFs.
-func TestCropMediaSectionsEvictionContinuesAfterPageOrderBreak(t *testing.T) {
- sections := []deepdoctype.Section{
- // table on page 20
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{20}}}},
- // figure with cross-page-merge artifact: first position page 25, caption page 18
- {LayoutType: deepdoctype.LayoutTypeFigure, Positions: []deepdoctype.Position{
- {PageNumbers: []int{25}},
- {PageNumbers: []int{18}},
- }},
- // two sections far down the document
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{300}}}},
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{301}}}},
- }
-
- // Mirror cropMediaSections' corrected eviction decision: a suffix-min
- // future-page window keeps eviction running even after a page-order break.
- n := len(sections)
- minFuturePage := make([]int, n)
- cur := -1
- for j := n - 1; j >= 0; j-- {
- if mp := sectionMinPage(sections[j]); mp >= 0 {
- if cur < 0 || mp < cur {
- cur = mp
- }
- }
- minFuturePage[j] = cur
- }
- pageCache := map[int]bool{} // page -> rendered and kept
- for i := range sections {
- if minFuturePage[i] >= 0 {
- for pn := range pageCache {
- if pn < minFuturePage[i] {
- delete(pageCache, pn)
- }
- }
- }
- for _, pos := range sections[i].Positions {
- for _, pn := range pos.PageNumbers {
- pageCache[pn] = true
- }
- }
- }
-
- // The cross-page-merge break must NOT disable eviction: once we advance
- // past pages 20/25 to 300/301, the early pages are pruned by the
- // future-page window, so the cache holds only the current section's page
- // instead of all four (the old unbounded behaviour).
- if pageCache[20] {
- t.Error("expected page 20 to be evicted by the future-page window, but it was retained")
- }
- if pageCache[25] {
- t.Error("expected page 25 to be evicted by the future-page window, but it was retained")
- }
- if !pageCache[301] {
- t.Error("expected page 301 to remain cached (current section), but it was evicted")
- }
- if len(pageCache) != 1 {
- t.Errorf("expected bounded cache to hold only the last section's page {301}, got %d: %v", len(pageCache), pageCache)
- }
-}
-
-// TestSectionRendersFromCache pins the eligibility predicate cropMediaSections
-// and computeFuturePageWindow share: only media sections that actually render
-// from pageCache (no pre-existing Image, a media layout/type, and at least one
-// Position) participate in the eviction window. Skipped sections must not
-// contribute to the window, or a late skipped section with a low merged page
-// would keep the window low and let pageCache grow (the residual of #19938).
-func TestSectionRendersFromCache(t *testing.T) {
- renderable := []deepdoctype.Section{
- {LayoutType: deepdoctype.LayoutTypeFigure, Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{2}}}},
- {LayoutType: deepdoctype.DLALabelFigureCaption, Positions: []deepdoctype.Position{{PageNumbers: []int{3}}}},
- {LayoutType: "image", Positions: []deepdoctype.Position{{PageNumbers: []int{4}}}},
- {DocTypeKwd: "image", Positions: []deepdoctype.Position{{PageNumbers: []int{5}}}},
- {DocTypeKwd: "table", Positions: []deepdoctype.Position{{PageNumbers: []int{6}}}},
- }
- for i, sec := range renderable {
- if !sectionRendersFromCache(sec) {
- t.Errorf("case %d: expected media section to render from cache", i)
- }
- }
- skipped := []deepdoctype.Section{
- {LayoutType: deepdoctype.LayoutTypeText, Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},
- {LayoutType: deepdoctype.LayoutTypeFigure, Image: "already-cropped", Positions: []deepdoctype.Position{{PageNumbers: []int{1}}}},
- {LayoutType: deepdoctype.LayoutTypeTable}, // no Positions
- }
- for i, sec := range skipped {
- if sectionRendersFromCache(sec) {
- t.Errorf("case %d: expected skipped section NOT to render from cache", i)
- }
- }
-}
-
-// TestComputeFuturePageWindowIgnoresSkippedSections is the regression for
-// Finding A of the #19938 review: a section that cropMediaSections skips (here
-// a text section) but which carries a low merged page late in the document
-// must NOT pull the future-page window down. With the bug, the window at the
-// first section would equal that low page (5) and early rendered pages could
-// never be evicted; after the fix the window stays at the first renderable
-// section's page (100).
-func TestComputeFuturePageWindowIgnoresSkippedSections(t *testing.T) {
- sections := []deepdoctype.Section{
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{100}}}},
- {LayoutType: deepdoctype.LayoutTypeText, Positions: []deepdoctype.Position{{PageNumbers: []int{5}}}},
- }
- win := computeFuturePageWindow(sections)
- if win[0] != 100 {
- t.Errorf("skipped low-page section must not lower the future window; got win[0]=%d, want 100", win[0])
- }
- if win[1] != -1 {
- t.Errorf("skipped section has no renderable future window; got win[1]=%d, want -1", win[1])
- }
-}
-
-// TestSectionOrderBreaksEvictionSkipsUnknownPages is the regression for the
-// -1 handling in sectionOrderBreaksEviction (Finding C): a section with no
-// page info (sectionMinPage == -1) must not be treated as a page-order break.
-// Before the fix, the -1 section was reported as the offending break; after
-// the fix the break is correctly attributed to the next known lower page (18
-// following 20), so the offending index is 2, not 1.
-func TestSectionOrderBreaksEvictionSkipsUnknownPages(t *testing.T) {
- sections := []deepdoctype.Section{
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{20}}}},
- {LayoutType: deepdoctype.LayoutTypeFigure}, // no Positions -> minPage -1
- {LayoutType: deepdoctype.LayoutTypeTable, Positions: []deepdoctype.Position{{PageNumbers: []int{18}}}},
- }
- broken, idx := sectionOrderBreaksEviction(sections)
- if !broken {
- t.Fatalf("expected a page-order break from 20 -> 18, got none")
- }
- if idx != 2 {
- t.Errorf("expected break at index 2 (the -1 section must be skipped), got %d", idx)
- }
-}
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index 703ebd8f4d..9dccb609dc 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -139,9 +139,9 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {
// A figure caption with an inlined image but NO positions is NOT promoted
// to image: positions are the sole classifier now. (In the PDF JSON path
- // an inlined image without positions does not occur — cropMediaSections was
- // removed — so this documents that the presence of a stray image alone is
- // insufficient.)
+ // media are not cropped — the chunker crops on demand from positions — so
+ // an inlined image without positions does not occur; this documents that
+ // the presence of a stray image alone is insufficient.)
withImageNoPos := &deepdoctype.ParseResult{
Sections: []deepdoctype.Section{{
Text: "小灰灰",
@@ -281,15 +281,11 @@ func TestPDFParser_ConfigureFromSetup(t *testing.T) {
p := NewPDFParser()
p.ConfigureFromSetup(map[string]any{
"parse_method": "deepdoc",
- "output_format": "markdown",
"enable_multi_column": true,
"flatten_media_to_text": true,
"remove_toc": true,
"remove_header_footer": true,
})
- if got, want := p.OutputFormat, "markdown"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
- }
if !p.EnableMultiColumn {
t.Fatal("EnableMultiColumn = false, want true")
}
@@ -307,145 +303,6 @@ func TestPDFParser_ConfigureFromSetup(t *testing.T) {
}
}
-func TestPDFParseResultToMarkdownWithOptions_RendersLikePython(t *testing.T) {
- parsed := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "Title", LayoutType: deepdoctype.LayoutTypeTitle},
- {Text: "Figure", LayoutType: deepdoctype.LayoutTypeFigure, Image: "aGVsbG8="},
- {Text: "WhitespaceFigureText", LayoutType: deepdoctype.LayoutTypeFigure, Image: " \t\n"},
- {Text: "<table><tr><td>cell</td></tr></table>", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVpbWc="},
- {Text: "", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVvbmx5"},
- {Text: "ImageCaption", LayoutType: "image", Image: "aW1hZ2Vvbmx5"},
- {Text: "Body", LayoutType: deepdoctype.LayoutTypeText},
- },
- }
-
- res := pdfParseResultToMarkdownWithOptions("sample.pdf", parsed, pdfPostProcessOptions{})
- if res.Err != nil {
- t.Fatalf("pdfParseResultToMarkdownWithOptions: %v", res.Err)
- }
- if got, want := res.OutputFormat, "markdown"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
- }
- if res.Markdown == "" {
- t.Fatal("Markdown is empty; want rendered content")
- }
- if !strings.Contains(res.Markdown, "## Title") {
- t.Fatalf("Markdown = %q, want title heading", res.Markdown)
- }
- if !strings.Contains(res.Markdown, "![Image](data:image/png;base64,aGVsbG8=)") {
- t.Fatalf("Markdown = %q, want inline figure image", res.Markdown)
- }
- if !strings.Contains(res.Markdown, "WhitespaceFigureText") {
- t.Fatalf("Markdown = %q, want whitespace figure text preserved", res.Markdown)
- }
- if strings.Contains(res.Markdown, "![Image]()") {
- t.Fatalf("Markdown = %q, unexpected empty image tag", res.Markdown)
- }
- if !strings.Contains(res.Markdown, "<table><tr><td>cell</td></tr></table>") {
- t.Fatalf("Markdown = %q, want table text", res.Markdown)
- }
- if !strings.Contains(res.Markdown, "![Image](data:image/png;base64,dGFibGVvbmx5)") {
- t.Fatalf("Markdown = %q, want fallback table image when text is empty", res.Markdown)
- }
- if !strings.Contains(res.Markdown, "![Image](data:image/png;base64,aW1hZ2Vvbmx5)") {
- t.Fatalf("Markdown = %q, want image section", res.Markdown)
- }
- if !strings.Contains(res.Markdown, "Body") {
- t.Fatalf("Markdown = %q, want body text", res.Markdown)
- }
- if len(res.JSON) != 0 {
- t.Fatalf("JSON len = %d, want 0 for markdown output", len(res.JSON))
- }
-}
-
-func TestPDFParseResultToMarkdownWithOptions_TableFallback(t *testing.T) {
- // 1. Table with text + image -> renders table text
- withText := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "<table><tr><td>content</td></tr></table>", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVpbWc="},
- },
- }
- resText := pdfParseResultToMarkdownWithOptions("table.pdf", withText, pdfPostProcessOptions{})
- if !strings.Contains(resText.Markdown, "<table><tr><td>content</td></tr></table>") {
- t.Fatalf("Markdown = %q, want table text", resText.Markdown)
- }
- if strings.Contains(resText.Markdown, "![Image]") {
- t.Fatalf("Markdown = %q, unexpected image tag when table text is present", resText.Markdown)
- }
-
- // 2. Table with empty text + image -> falls back to image tag
- emptyText := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "", LayoutType: deepdoctype.LayoutTypeTable, Image: "dGFibGVvbmx5"},
- },
- }
- resFallback := pdfParseResultToMarkdownWithOptions("table.pdf", emptyText, pdfPostProcessOptions{})
- if !strings.Contains(resFallback.Markdown, "![Image](data:image/png;base64,dGFibGVvbmx5)") {
- t.Fatalf("Markdown = %q, want fallback table image tag", resFallback.Markdown)
- }
-
- // 3. Table with empty text + whitespace-only image -> empty string, no broken tags
- wsImg := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "", LayoutType: deepdoctype.LayoutTypeTable, Image: " \t\n"},
- },
- }
- resWS := pdfParseResultToMarkdownWithOptions("table.pdf", wsImg, pdfPostProcessOptions{})
- if strings.TrimSpace(resWS.Markdown) != "" {
- t.Fatalf("Markdown = %q, want empty output for empty text + whitespace image", resWS.Markdown)
- }
-}
-
-func TestSectionsToMarkdown_DocTypeKwdImage(t *testing.T) {
- // LayoutType is not figure/image, but DocTypeKwd == "image"
- sections := []deepdoctype.Section{
- {Text: "Caption", LayoutType: "custom_block", DocTypeKwd: "image", Image: "aW1hZ2Vvbmx5"},
- }
- got := sectionsToMarkdown(sections)
- want := "\n![Image](data:image/png;base64,aW1hZ2Vvbmx5)"
- if got != want {
- t.Fatalf("got %q, want %q", got, want)
- }
-
- // DocTypeKwd == "image" with whitespace image preserves text and drops empty tag
- sectionsWS := []deepdoctype.Section{
- {Text: "Caption", LayoutType: "custom_block", DocTypeKwd: "image", Image: " \t\n"},
- }
- gotWS := sectionsToMarkdown(sectionsWS)
- wantWS := "Caption\n"
- if gotWS != wantWS {
- t.Fatalf("got %q, want %q", gotWS, wantWS)
- }
-}
-
-func TestPDFParseResultToMarkdownWithOptions_ImageSection(t *testing.T) {
- parsed := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "Caption", LayoutType: "image", Image: "aW1hZ2Vvbmx5"},
- },
- }
- res := pdfParseResultToMarkdownWithOptions("doc.pdf", parsed, pdfPostProcessOptions{})
- if !strings.Contains(res.Markdown, "![Image](data:image/png;base64,aW1hZ2Vvbmx5)") {
- t.Fatalf("Markdown = %q, want image embed for LayoutType == 'image'", res.Markdown)
- }
-}
-
-func TestPDFParseResultToMarkdownWithOptions_WhitespaceOnlyImage(t *testing.T) {
- parsed := &deepdoctype.ParseResult{
- Sections: []deepdoctype.Section{
- {Text: "Figure Caption", LayoutType: deepdoctype.LayoutTypeFigure, Image: " \r\n\t "},
- },
- }
- res := pdfParseResultToMarkdownWithOptions("doc.pdf", parsed, pdfPostProcessOptions{})
- if !strings.Contains(res.Markdown, "Figure Caption") {
- t.Fatalf("Markdown = %q, want figure caption preserved", res.Markdown)
- }
- if strings.Contains(res.Markdown, "![Image]") {
- t.Fatalf("Markdown = %q, unexpected image tag for whitespace-only image", res.Markdown)
- }
-}
-
func TestPDFParser_ValidateParseMethod(t *testing.T) {
p := NewPDFParser()
if err := p.validateParseMethod(); err != nil {
diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.go
index b99a84e107..f320e83234 100644
--- a/internal/parser/parser/pdf_parser_docling.go
+++ b/internal/parser/parser/pdf_parser_docling.go
@@ -113,13 +113,13 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, pars
continue
}
if candidate.chunked {
- if res, ok := parseDoclingChunkedResult(filename, body, parser.OutputFormat); ok {
+ if res, ok := parseDoclingChunkedResult(filename, body); ok {
return res
}
lastErr = fmt.Errorf("%s: chunked response contained no usable text", candidate.endpoint)
continue
}
- if res, ok := parseDoclingStandardResult(ctx, filename, body, parser.OutputFormat); ok {
+ if res, ok := parseDoclingStandardResult(ctx, filename, body); ok {
return res
}
lastErr = fmt.Errorf("%s: standard response contained no parsed document", candidate.endpoint)
@@ -184,7 +184,7 @@ func doclingChunkedPayload(filename string, encoded string, alpha bool, convertO
return payload
}
-func parseDoclingChunkedResult(filename string, body []byte, outputFormat string) (ParseResult, bool) {
+func parseDoclingChunkedResult(filename string, body []byte) (ParseResult, bool) {
var chunks []doclingChunk
if err := json.Unmarshal(body, &chunks); err != nil {
var wrapped struct {
@@ -212,10 +212,10 @@ func parseDoclingChunkedResult(filename string, body []byte, outputFormat string
if len(texts) > 0 {
pageCount = len(texts)
}
- return doclingTextsToResult(filename, texts, outputFormat, pageCount), true
+ return doclingTextsToResult(filename, texts, pageCount), true
}
-func parseDoclingStandardResult(ctx context.Context, filename string, body []byte, outputFormat string) (ParseResult, bool) {
+func parseDoclingStandardResult(ctx context.Context, filename string, body []byte) (ParseResult, bool) {
var payload doclingResponse
if err := json.Unmarshal(body, &payload); err != nil {
return ParseResult{}, false
@@ -235,41 +235,29 @@ func parseDoclingStandardResult(ctx context.Context, filename string, body []byt
}
for _, doc := range docs {
if md := strings.TrimSpace(doc.MDContent); md != "" {
- return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true
+ return pdfMarkdownToResult(ctx, filename, md, len(docs)), true
}
if txt := strings.TrimSpace(doc.TextContent); txt != "" {
- return doclingTextsToResult(filename, []string{txt}, outputFormat, len(docs)), true
+ return doclingTextsToResult(filename, []string{txt}, len(docs)), true
}
if md, _ := doc.JSONContent["md_content"].(string); strings.TrimSpace(md) != "" {
- return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true
+ return pdfMarkdownToResult(ctx, filename, md, len(docs)), true
}
}
return ParseResult{}, false
}
-func doclingTextsToResult(filename string, texts []string, outputFormat string, pageCount int) ParseResult {
- fileMeta := pdfFileMeta(filename, pageCount)
- switch strings.ToLower(strings.TrimSpace(outputFormat)) {
- case "", "json":
- items := make([]map[string]any, 0, len(texts))
- for _, text := range texts {
- items = append(items, map[string]any{
- "text": text,
- "doc_type_kwd": "text",
- })
- }
- return ParseResult{
- OutputFormat: "json",
- File: fileMeta,
- JSON: MapsToItems(items),
- }
- case "markdown":
- return ParseResult{
- OutputFormat: "markdown",
- File: fileMeta,
- Markdown: strings.Join(texts, "\n\n"),
- }
- default:
- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}
+func doclingTextsToResult(filename string, texts []string, pageCount int) ParseResult {
+ items := make([]map[string]any, 0, len(texts))
+ for _, text := range texts {
+ items = append(items, map[string]any{
+ "text": text,
+ "doc_type_kwd": "text",
+ })
+ }
+ return ParseResult{
+ OutputFormat: "json",
+ File: pdfFileMeta(filename, pageCount),
+ JSON: MapsToItems(items),
}
}
diff --git a/internal/parser/parser/pdf_parser_docling_test.go b/internal/parser/parser/pdf_parser_docling_test.go
index 0115fe2a4d..130fcd6925 100644
--- a/internal/parser/parser/pdf_parser_docling_test.go
+++ b/internal/parser/parser/pdf_parser_docling_test.go
@@ -11,7 +11,7 @@ import (
"testing"
)
-func TestPDFParser_ParseWithResult_DoclingChunkedMarkdownIntegration(t *testing.T) {
+func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {
withSSRFBypass(t)
var requestCount atomic.Int32
@@ -67,7 +67,6 @@ func TestPDFParser_ParseWithResult_DoclingChunkedMarkdownIntegration(t *testing.
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "Docling",
- "output_format": "markdown",
"docling_server_url": server.URL,
"docling_api_key": "doc-secret",
})
@@ -77,11 +76,17 @@ func TestPDFParser_ParseWithResult_DoclingChunkedMarkdownIntegration(t *testing.
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "markdown"; got != want {
+ if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if got, want := res.Markdown, "Chunk A\n\nChunk B"; got != want {
- t.Fatalf("Markdown = %q, want %q", got, want)
+ var got strings.Builder
+ for _, item := range ItemsToMaps(res.JSON) {
+ text, _ := item["text"].(string)
+ got.WriteString(text)
+ got.WriteString("\n")
+ }
+ if !strings.Contains(got.String(), "Chunk A") || !strings.Contains(got.String(), "Chunk B") {
+ t.Fatalf("JSON text = %q, want both chunks", got.String())
}
}
@@ -132,7 +137,6 @@ func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *t
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "Docling",
- "output_format": "json",
"docling_server_url": server.URL,
})
@@ -235,7 +239,6 @@ func TestPDFParser_ParseWithResult_DoclingConvertOptions(t *testing.T) {
setup := map[string]any{
"parse_method": "Docling",
- "output_format": "markdown",
"docling_server_url": server.URL,
}
maps.Copy(setup, tt.setup)
diff --git a/internal/parser/parser/pdf_parser_fixture_cgo_test.go b/internal/parser/parser/pdf_parser_fixture_cgo_test.go
index 83219849a5..40f273671a 100644
--- a/internal/parser/parser/pdf_parser_fixture_cgo_test.go
+++ b/internal/parser/parser/pdf_parser_fixture_cgo_test.go
@@ -55,33 +55,6 @@ func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) {
}
}
-func TestPDFParser_ParseWithResult_CGOFixtureMarkdown(t *testing.T) {
- useMockDocAnalyzer(t)
-
- path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")
- data, err := os.ReadFile(path)
- if err != nil {
- t.Fatalf("ReadFile(%s): %v", path, err)
- }
- pdf := NewPDFParser()
- pdf.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
-
- ctx := t.Context()
- res := pdf.ParseWithResult(ctx, "Doc1.pdf", data)
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- if got, want := res.OutputFormat, "markdown"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
- }
- if res.Markdown == "" {
- t.Fatal("Markdown is empty; want rendered content")
- }
- if len(res.JSON) != 0 {
- t.Fatalf("JSON len = %d, want 0 for markdown output", len(res.JSON))
- }
-}
-
func TestPDFParser_ParseWithResult_CGOFixturePlainText(t *testing.T) {
useMockDocAnalyzer(t)
diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.go
index 333ded8f3d..bef0466224 100644
--- a/internal/parser/parser/pdf_parser_mineru.go
+++ b/internal/parser/parser/pdf_parser_mineru.go
@@ -72,7 +72,7 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parse
if strings.TrimSpace(content) != "" {
pageCount = 1
}
- return pdfMarkdownToResult(ctx, filename, content, parser.OutputFormat, pageCount)
+ return pdfMarkdownToResult(ctx, filename, content, pageCount)
}
func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskID string, apiConfig *models.APIConfig, timeout time.Duration) (string, error) {
diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.go
index adaefaf5a6..66f6ae03f7 100644
--- a/internal/parser/parser/pdf_parser_mineru_test.go
+++ b/internal/parser/parser/pdf_parser_mineru_test.go
@@ -11,7 +11,7 @@ import (
"testing"
)
-func TestPDFParser_ParseWithResult_MinerUMarkdownIntegration(t *testing.T) {
+func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {
withSSRFBypass(t)
var submitCalled atomic.Bool
var resultCalled atomic.Bool
@@ -76,7 +76,6 @@ func TestPDFParser_ParseWithResult_MinerUMarkdownIntegration(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "MinerU",
- "output_format": "markdown",
"mineru_apiserver": server.URL,
"mineru_api_key": "secret",
"mineru_backend": "pipeline",
@@ -90,56 +89,23 @@ func TestPDFParser_ParseWithResult_MinerUMarkdownIntegration(t *testing.T) {
if !submitCalled.Load() || !resultCalled.Load() {
t.Fatalf("submit/result called = %v/%v, want true/true", submitCalled.Load(), resultCalled.Load())
}
- if got, want := res.OutputFormat, "markdown"; got != want {
+ if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if got, want := res.Markdown, "# Title\n\nBody paragraph.\n"; got != want {
- t.Fatalf("Markdown = %q, want %q", got, want)
+ var text strings.Builder
+ for _, item := range ItemsToMaps(res.JSON) {
+ value, _ := item["text"].(string)
+ text.WriteString(value)
+ text.WriteString("\n")
+ }
+ if !strings.Contains(text.String(), "Title") || !strings.Contains(text.String(), "Body paragraph.") {
+ t.Fatalf("JSON text = %q, want title and body", text.String())
}
if got, want := res.File["name"], "sample.pdf"; got != want {
t.Fatalf("File.name = %v, want %v", got, want)
}
}
-func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {
- withSSRFBypass(t)
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch {
- case r.Method == http.MethodPost && r.URL.Path == "/file_parse":
- w.Header().Set("Content-Type", "application/json")
- _, _ = w.Write([]byte(`{"data":{"task_id":"task-2"}}`))
- case r.Method == http.MethodGet && r.URL.Path == "/tasks/task-2/result":
- w.Header().Set("Content-Type", "application/json")
- _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# Title\n\nBody paragraph.\n"}}}`))
- default:
- http.NotFound(w, r)
- }
- }))
- defer server.Close()
-
- pdf := NewPDFParser()
- pdf.ConfigureFromSetup(map[string]any{
- "parse_method": "MinerU",
- "output_format": "json",
- "mineru_apiserver": server.URL,
- })
-
- ctx := t.Context()
- res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
- }
- if len(res.JSON) == 0 {
- t.Fatal("JSON is empty; want markdown-normalized items")
- }
- if got := res.JSON[0].ToMap()["text"]; got == nil {
- t.Fatal("JSON[0].text missing")
- }
-}
-
func TestPDFParser_ParseWithResult_MinerURequiresAPIServer(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "MinerU"})
@@ -206,7 +172,6 @@ func TestPDFParser_ParseWithResult_MinerUSendsServerURLForHTTPClientBackend(t *t
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "MinerU",
- "output_format": "markdown",
"mineru_apiserver": server.URL,
"mineru_backend": "vlm-http-client",
"mineru_server_url": "http://vllm-host:30000",
@@ -267,7 +232,6 @@ func TestPDFParser_ParseWithResult_MinerUReadsBackendFromProviderJSONApiKey(t *t
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "MinerU",
- "output_format": "markdown",
"mineru_api_key": apiKeyJSON,
})
res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.go
index 46e0b2b495..cdb448cff9 100644
--- a/internal/parser/parser/pdf_parser_opendataloader.go
+++ b/internal/parser/parser/pdf_parser_opendataloader.go
@@ -64,11 +64,11 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byt
if payload.JSONDoc != nil {
items := openDataLoaderItems(payload.JSONDoc)
if len(items) > 0 {
- return pdfItemsToResult(filename, items, parser.OutputFormat, openDataLoaderPageCount(payload.JSONDoc))
+ return pdfItemsToResult(filename, items, openDataLoaderPageCount(payload.JSONDoc))
}
}
if strings.TrimSpace(payload.MDText) != "" {
- return pdfMarkdownToResult(ctx, filename, payload.MDText, parser.OutputFormat, 1)
+ return pdfMarkdownToResult(ctx, filename, payload.MDText, 1)
}
return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader returned no parsed content")}
}
diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.go
index ea84fe3b68..5888c876df 100644
--- a/internal/parser/parser/pdf_parser_opendataloader_test.go
+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go
@@ -66,7 +66,6 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "OpenDataLoader",
- "output_format": "json",
"opendataloader_apiserver": server.URL,
"opendataloader_api_key": "odl-secret",
"hybrid": "docling-fast",
@@ -86,7 +85,7 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {
}
}
-func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T) {
+func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallbackToJSON(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/file_parse" {
@@ -101,7 +100,6 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T)
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "OpenDataLoader",
- "output_format": "markdown",
"opendataloader_apiserver": server.URL,
})
ctx := t.Context()
@@ -109,8 +107,14 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderMarkdownFallback(t *testing.T)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(res.Markdown, "ODL Title") {
- t.Fatalf("Markdown = %q, want ODL Title", res.Markdown)
+ var text strings.Builder
+ for _, item := range ItemsToMaps(res.JSON) {
+ value, _ := item["text"].(string)
+ text.WriteString(value)
+ text.WriteString("\n")
+ }
+ if !strings.Contains(text.String(), "ODL Title") {
+ t.Fatalf("JSON text = %q, want ODL Title", text.String())
}
}
diff --git a/internal/parser/parser/pdf_parser_paddleocr_test.go b/internal/parser/parser/pdf_parser_paddleocr_test.go
index caf21ab35c..9d4286414d 100644
--- a/internal/parser/parser/pdf_parser_paddleocr_test.go
+++ b/internal/parser/parser/pdf_parser_paddleocr_test.go
@@ -117,10 +117,6 @@ func TestPaddleOCRTitleLabels(t *testing.T) {
if item["layout"] != tc.layout || item["text"] != tc.want {
t.Fatalf("title semantics: %v", item)
}
- formatted := pdfItemsToResult("title.pdf", ItemsToMaps(result.JSON), "markdown", 1)
- if formatted.Markdown != tc.want {
- t.Fatalf("markdown=%q; want %q", formatted.Markdown, tc.want)
- }
})
}
}
diff --git a/internal/parser/parser/pdf_parser_pages_test.go b/internal/parser/parser/pdf_parser_pages_test.go
index d12b0e01ef..3a1ca88a42 100644
--- a/internal/parser/parser/pdf_parser_pages_test.go
+++ b/internal/parser/parser/pdf_parser_pages_test.go
@@ -67,7 +67,6 @@ func TestConfigureFromSetup_Pages(t *testing.T) {
setup := map[string]any{
"flatten_media_to_text": true,
"parse_method": "DeepDOC",
- "output_format": "json",
"pages": []any{
[]any{float64(1), float64(100)},
},
@@ -79,9 +78,6 @@ func TestConfigureFromSetup_Pages(t *testing.T) {
if p.ParseMethod != "DeepDOC" {
t.Errorf("ParseMethod = %q, want DeepDOC", p.ParseMethod)
}
- if p.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want json", p.OutputFormat)
- }
if want := [][]int{{1, 100}}; !reflect.DeepEqual(p.Pages, want) {
t.Errorf("Pages = %v, want %v", p.Pages, want)
}
diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.go
index 1073df5ab9..e39189e610 100644
--- a/internal/parser/parser/pdf_parser_remote_common.go
+++ b/internal/parser/parser/pdf_parser_remote_common.go
@@ -2,11 +2,7 @@ package parser
import (
"context"
- "fmt"
"strings"
-
- "ragflow/internal/entity"
- "ragflow/internal/parser/tableutil"
)
func pdfFileMeta(filename string, pageCount int) map[string]any {
@@ -20,60 +16,15 @@ func pdfFileMeta(filename string, pageCount int) map[string]any {
}
}
-// markdownItemText renders a single parse item to markdown. A table uses the
-// structured TableData contract directly (rendered via RenderTableHTML); every
-// other item uses its text. Producers emit `table` for tables and never the
-// legacy HTML `text`, so a table-typed item without structured data is skipped
-// rather than falling back to a raw HTML string — the contract is "tables are
-// structured, not markup".
-func markdownItemText(item map[string]any) string {
- if dt, _ := item["doc_type_kwd"].(string); dt == "table" {
- if td, ok := item["table"].(*entity.TableData); ok && td != nil {
- markup := tableutil.RenderTableHTML(td)
- if notes, _ := item["text"].(string); strings.TrimSpace(notes) != "" {
- return strings.TrimRight(markup, "\n") + "\n\n" + notes
- }
- return markup
- }
- return ""
- }
- text, _ := item["text"].(string)
- return text
-}
-
-func pdfItemsToResult(filename string, items []map[string]any, outputFormat string, pageCount int) ParseResult {
+func pdfItemsToResult(filename string, items []map[string]any, pageCount int) ParseResult {
if len(items) == 0 {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
pageCount = normalizePDFPageCount(pageCount, items)
- switch strings.ToLower(strings.TrimSpace(outputFormat)) {
- case "", "json":
- return ParseResult{
- OutputFormat: "json",
- File: pdfFileMeta(filename, pageCount),
- JSON: MapsToItems(items),
- }
- case "markdown":
- var b strings.Builder
- for _, item := range items {
- text := markdownItemText(item)
- layout, _ := item["layout"].(string)
- if strings.TrimSpace(text) == "" {
- continue
- }
- if layout == "title" && !strings.HasPrefix(strings.TrimSpace(text), "#") {
- b.WriteString("## ")
- }
- b.WriteString(text)
- b.WriteString("\n\n")
- }
- return ParseResult{
- OutputFormat: "markdown",
- File: pdfFileMeta(filename, pageCount),
- Markdown: strings.TrimRight(b.String(), "\n"),
- }
- default:
- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}
+ return ParseResult{
+ OutputFormat: "json",
+ File: pdfFileMeta(filename, pageCount),
+ JSON: MapsToItems(items),
}
}
@@ -156,34 +107,23 @@ func rowsHaveText(rows [][]string) bool {
return false
}
-func pdfMarkdownToResult(ctx context.Context, filename, markdown, outputFormat string, pageCount int) ParseResult {
+func pdfMarkdownToResult(ctx context.Context, filename, markdown string, pageCount int) ParseResult {
fileMeta := pdfFileMeta(filename, pageCount)
- switch strings.ToLower(strings.TrimSpace(outputFormat)) {
- case "", "json":
- res := parsePDFMarkdown(ctx, filename, markdown)
- if res.Err != nil {
- if ctx.Err() != nil {
- res.File = fileMeta
- return res
- }
- // Same text fallback as NormalizeResult: keep the backend
- // markdown as raw text instead of failing the task.
- return ParseResult{
- OutputFormat: "json",
- File: fileMeta,
- Warnings: append(res.Warnings, "markdown normalization fell back to text: "+res.Err.Error()),
- JSON: []DocItem{NewTextJSONItem(markdown)},
- }
+ res := parsePDFMarkdown(ctx, filename, markdown)
+ if res.Err != nil {
+ if ctx.Err() != nil {
+ res.File = fileMeta
+ return res
}
- res.File = fileMeta
- return res
- case "markdown":
+ // Same text fallback as NormalizeResult: keep the backend
+ // markdown as raw text instead of failing the task.
return ParseResult{
- OutputFormat: "markdown",
+ OutputFormat: "json",
File: fileMeta,
- Markdown: markdown,
+ Warnings: append(res.Warnings, "markdown normalization fell back to text: "+res.Err.Error()),
+ JSON: []DocItem{NewTextJSONItem(markdown)},
}
- default:
- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}
}
+ res.File = fileMeta
+ return res
}
diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.go
index c78c0755f8..f5af8d8382 100644
--- a/internal/parser/parser/pdf_parser_somark.go
+++ b/internal/parser/parser/pdf_parser_somark.go
@@ -42,7 +42,7 @@ func parsePDFWithSoMark(filename string, data []byte, parser *PDFParser) ParseRe
if len(items) == 0 {
return ParseResult{Err: fmt.Errorf("parser: SoMark returned no usable blocks")}
}
- return pdfItemsToResult(filename, items, parser.OutputFormat, pageCount)
+ return pdfItemsToResult(filename, items, pageCount)
}
// soMarkSubmit uploads a PDF with the supplied credentials and parser options,
diff --git a/internal/parser/parser/pdf_parser_somark_test.go b/internal/parser/parser/pdf_parser_somark_test.go
index d94d326720..db33a1aa09 100644
--- a/internal/parser/parser/pdf_parser_somark_test.go
+++ b/internal/parser/parser/pdf_parser_somark_test.go
@@ -61,7 +61,6 @@ func TestPDFParser_ParseWithResult_SoMarkJSONIntegration(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "SoMark",
- "output_format": "json",
"somark_base_url": server.URL,
"somark_api_key": "somark-secret",
})
@@ -78,35 +77,6 @@ func TestPDFParser_ParseWithResult_SoMarkJSONIntegration(t *testing.T) {
}
}
-func TestPDFParser_ParseWithResult_SoMarkMarkdownIntegration(t *testing.T) {
- withSSRFBypass(t)
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch r.URL.Path {
- case "/parse/async":
- _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-2"}}`))
- case "/parse/async_check":
- _, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"title","content":"SoMark Title","title_level":1},{"type":"text","content":"Body"}]}]}}}}}`))
- default:
- http.NotFound(w, r)
- }
- }))
- defer server.Close()
- pdf := NewPDFParser()
- pdf.ConfigureFromSetup(map[string]any{
- "parse_method": "SoMark",
- "output_format": "markdown",
- "somark_base_url": server.URL,
- })
- ctx := t.Context()
- res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- if !strings.Contains(res.Markdown, "SoMark Title") {
- t.Fatalf("Markdown = %q, want title", res.Markdown)
- }
-}
-
func TestPDFParser_ParseWithResult_SoMarkRequiresBaseURL(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{"parse_method": "SoMark"})
diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.go
index e52f173647..3072ef4da4 100644
--- a/internal/parser/parser/pdf_parser_tcadp.go
+++ b/internal/parser/parser/pdf_parser_tcadp.go
@@ -17,8 +17,7 @@ import (
func parsePDFWithTCADP(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {
return parseWithTCADP(ctx, filename, data, "PDF",
parser.TCADPAPIServer, parser.TCADPAPIKey,
- parser.TCADPTableResultType, parser.TCADPMarkdownImageResponseType,
- parser.OutputFormat)
+ parser.TCADPTableResultType, parser.TCADPMarkdownImageResponseType)
}
func tcadpItemsFromZip(zipBytes []byte) ([]map[string]any, int, error) {
diff --git a/internal/parser/parser/pdf_parser_tcadp_test.go b/internal/parser/parser/pdf_parser_tcadp_test.go
index dbaff9b772..94a042a34c 100644
--- a/internal/parser/parser/pdf_parser_tcadp_test.go
+++ b/internal/parser/parser/pdf_parser_tcadp_test.go
@@ -35,7 +35,6 @@ func TestPDFParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "json",
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
})
@@ -49,7 +48,7 @@ func TestPDFParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
}
}
-func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
+func TestPDFParser_ParseWithResult_TCADPIntegration(t *testing.T) {
withSSRFBypass(t)
zipPayload := tcadpZipFixture(t)
var server *httptest.Server
@@ -67,7 +66,6 @@ func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "markdown",
"tcadp_apiserver": server.URL,
})
ctx := t.Context()
@@ -75,8 +73,14 @@ func TestPDFParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(res.Markdown, "Hello TCADP") {
- t.Fatalf("Markdown = %q, want fixture text", res.Markdown)
+ var text strings.Builder
+ for _, item := range ItemsToMaps(res.JSON) {
+ value, _ := item["text"].(string)
+ text.WriteString(value)
+ text.WriteString("\n")
+ }
+ if !strings.Contains(text.String(), "Hello TCADP") {
+ t.Fatalf("JSON text = %q, want fixture text", text.String())
}
}
@@ -106,7 +110,6 @@ func TestPDFParser_ParseWithResult_TCADPCancellationAbortsDownload(t *testing.T)
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "json",
"tcadp_apiserver": submit.URL,
"tcadp_api_key": "tcadp-secret",
})
diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.go
index 2fe6d81daf..bce6649e49 100644
--- a/internal/parser/parser/pdf_postprocess.go
+++ b/internal/parser/parser/pdf_postprocess.go
@@ -14,7 +14,6 @@ import (
// per-box geometry signals those detectors rely on are still intact; they are
// intentionally not repeated here.
type pdfPostProcessOptions struct {
- outputFormat string
pageWidth float64
zoom float64
enableMultiColumn bool
diff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.go
index 903517c0fc..445e547e67 100644
--- a/internal/parser/parser/pdf_result_test.go
+++ b/internal/parser/parser/pdf_result_test.go
@@ -84,7 +84,7 @@ func TestPDFMarkdownBackendDoesNotFetchRemoteImages(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(http.StatusNotFound) }))
defer server.Close()
- result := pdfMarkdownToResult(t.Context(), "remote.pdf", "![remote]("+server.URL+"/image.png)", "json", 1)
+ result := pdfMarkdownToResult(t.Context(), "remote.pdf", "![remote]("+server.URL+"/image.png)", 1)
if result.Err != nil {
t.Fatal(result.Err)
}
@@ -264,22 +264,16 @@ func TestProjectPDFTablePreservesExistingStructureAndNotes(t *testing.T) {
}
}
-func TestPDFTableDescriptionUsesTextItemForBothOutputs(t *testing.T) {
+func TestPDFTableDescriptionUsesTextItem(t *testing.T) {
table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}
item := map[string]any{"table": table, "doc_type_kwd": "table", "text": "Footnote", "page_number": 3}
- for _, format := range []string{"json", "markdown"} {
- result := pdfItemsToResult("sample.pdf", []map[string]any{item}, format, 3)
- if result.Err != nil {
- t.Fatal(result.Err)
- }
- if format == "json" {
- result = NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", result)
- if len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != nil || result.JSON[1].ToMap()["text"] != "Footnote" || result.JSON[1].ToMap()["page_number"] != 3 {
- t.Fatalf("table description not separated: %#v", result.JSON)
- }
- } else if !strings.Contains(result.Markdown, "Cell value") || !strings.Contains(result.Markdown, "Footnote") {
- t.Fatalf("table description lost: %s", result.Markdown)
- }
+ result := pdfItemsToResult("sample.pdf", []map[string]any{item}, 3)
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ result = NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", result)
+ if len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != nil || result.JSON[1].ToMap()["text"] != "Footnote" || result.JSON[1].ToMap()["page_number"] != 3 {
+ t.Fatalf("table description not separated: %#v", result.JSON)
}
if item["text"] != "Footnote" {
t.Fatal("source item mutated")
@@ -289,7 +283,7 @@ func TestPDFTableDescriptionUsesTextItemForBothOutputs(t *testing.T) {
func TestPDFWhitespaceTableDescriptionCannotHideCells(t *testing.T) {
item := map[string]any{"table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, "doc_type_kwd": "table", "text": " \n "}
results := []ParseResult{
- NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 1)),
+ NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 1)),
NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{item})}),
}
for _, result := range results {
@@ -318,7 +312,7 @@ func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) {
originalText := item["text"]
p := NewPDFParser()
p.FlattenMediaToText = flatten
- result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))
+ result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3))
if result.Err != nil {
t.Fatal(result.Err)
}
@@ -373,7 +367,7 @@ func TestPDFNormalizeResultIsIdempotent(t *testing.T) {
"doc_type_kwd": "table", "text": markup, "page_number": 3,
"positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,
}
- once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))
+ once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 3))
if once.Err != nil {
t.Fatal(once.Err)
}
@@ -504,7 +498,7 @@ func TestPDFNormalizeResultCancellationPreservesFile(t *testing.T) {
func TestPDFMarkdownToResultPreservesFileOnCancel(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
- result := pdfMarkdownToResult(ctx, "cancel.pdf", "body", "json", 3)
+ result := pdfMarkdownToResult(ctx, "cancel.pdf", "body", 3)
if !errors.Is(result.Err, context.Canceled) {
t.Fatalf("err = %v", result.Err)
}
diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.go
index 305965af88..c5f29ae81b 100644
--- a/internal/parser/parser/pdf_table_no_html_text_test.go
+++ b/internal/parser/parser/pdf_table_no_html_text_test.go
@@ -1,11 +1,9 @@
package parser
import (
- "strings"
"testing"
"ragflow/internal/entity"
- "ragflow/internal/parser/tableutil"
)
// TestPDFTableItemsOmitHTMLText pins the structured TableData contract: a table
@@ -68,49 +66,6 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {
}
}
-// TestPDFItemsToMarkdownRendersTableFromTableField verifies the markdown
-// renderer derives table markup from `table` when the legacy HTML `text` is
-// absent. This is the consumer that previously forced producers to keep `text`;
-// once producers drop `text`, markdown must still embed the table. The output
-// must be byte-equivalent to rendering `table` directly (stable across the
-// ParseTableHTML/RenderTableHTML round-trip).
-func TestPDFItemsToMarkdownRendersTableFromTableField(t *testing.T) {
- td := &entity.TableData{
- Grid: [][]entity.TableCell{{{Text: "What is RAGFlow?"}, {Text: "A RAG engine."}}, {{Text: "Is it open source?"}, {Text: "Yes."}}},
- HeaderRows: 1,
- }
- // Producer-after-cleanup shape: `table` present, `text` absent.
- item := map[string]any{"doc_type_kwd": "table", "layout": "table", "table": td}
- res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)
- if res.Err != nil {
- t.Fatalf("pdfItemsToResult markdown: %v", res.Err)
- }
- want := strings.TrimSpace(tableutil.RenderTableHTML(td))
- if got := strings.TrimSpace(res.Markdown); got != want {
- t.Fatalf("markdown table mismatch:\n got: %q\nwant: %q", got, want)
- }
- if !strings.Contains(res.Markdown, "What is RAGFlow?") {
- t.Fatalf("markdown lost table cell text: %q", res.Markdown)
- }
-}
-
-// TestPDFItemsToMarkdownSkipsTableWithoutStructuredData guards the
-// structured-first contract: a table-typed item that carries no `table`
-// (e.g. a mislabeled table region) must be skipped from markdown rather than
-// falling back to a raw HTML `text`. This pins "tables are structured, not
-// markup" so a future change cannot reintroduce an HTML fallback for tables.
-func TestPDFItemsToMarkdownSkipsTableWithoutStructuredData(t *testing.T) {
- // doc_type_kwd "table" but no `table` field and no `text`.
- item := map[string]any{"doc_type_kwd": "table", "layout": "table"}
- res := pdfItemsToResult("doc.pdf", []map[string]any{item}, "markdown", 1)
- if res.Err != nil {
- t.Fatalf("pdfItemsToResult markdown: %v", res.Err)
- }
- if strings.TrimSpace(res.Markdown) != "" {
- t.Fatalf("markdown must skip table-typed item without structured data, got %q", res.Markdown)
- }
-}
-
// TestProjectPDFTableKeepsTextWhenNotProjected guards the safety net: items
// that are not tables, or whose text is not <table> markup, must keep their
// `text` untouched and receive no `table` — we must never delete text for
diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go
index c5f7bc1e0a..8948a0b771 100644
--- a/internal/parser/parser/picture_parser.go
+++ b/internal/parser/parser/picture_parser.go
@@ -42,26 +42,13 @@ var imageExtensions = map[string]bool{
}
// PictureParser emits image files for optional vision enhancement.
-type PictureParser struct {
- OutputFormat string
-}
+type PictureParser struct{}
// NewPictureParser constructs a PictureParser.
func NewPictureParser() *PictureParser {
return &PictureParser{}
}
-// ConfigureFromSetup reads picture-specific configuration from the
-// parser setup map.
-func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {
- if p == nil || setup == nil {
- return
- }
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
-}
-
// ParseWithResult validates the image extension and returns its image payload.
func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
ext := strings.ToLower(filepath.Ext(filename))
@@ -86,13 +73,6 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), MaxImagePayloadBytes)}
}
- // Parser output is normalized to JSON at the component boundary, so an
- // absent backend format defaults to JSON here as well.
- outFmt := p.OutputFormat
- if outFmt == "" {
- outFmt = "json"
- }
-
release, err := AcquireImageMedia(ctx)
if err != nil {
return ParseResult{Err: err}
@@ -108,7 +88,7 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
return ParseResult{Err: err}
}
return ParseResult{
- OutputFormat: outFmt,
+ OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.go
index 5dc809790c..8022158945 100644
--- a/internal/parser/parser/picture_parser_test.go
+++ b/internal/parser/parser/picture_parser_test.go
@@ -45,25 +45,11 @@ func TestNewPictureParser(t *testing.T) {
if p == nil {
t.Fatal("NewPictureParser returned nil")
}
- if p.OutputFormat != "" {
- t.Errorf("OutputFormat = %q, want empty", p.OutputFormat)
- }
-}
-
-func TestPictureParser_ConfigureFromSetup(t *testing.T) {
- p := NewPictureParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "text",
- })
- if p.OutputFormat != "text" {
- t.Errorf("OutputFormat = %q, want text", p.OutputFormat)
- }
}
func TestPictureParser_ParseWithResult_NilSetup(t *testing.T) {
ctx := t.Context()
p := NewPictureParser()
- p.ConfigureFromSetup(nil)
res := p.ParseWithResult(ctx, "photo.png", []byte("\x89PNG"))
if res.Err != nil {
t.Errorf("unexpected error for nil setup: %v", res.Err)
@@ -120,9 +106,6 @@ func TestPictureParser_ParseWithResult_VideoExtension(t *testing.T) {
func TestPictureParser_ParseWithResult_OutputFormat(t *testing.T) {
ctx := t.Context()
p := NewPictureParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- })
res := p.ParseWithResult(ctx, "photo.jpg", []byte{1, 2, 3})
if res.Err != nil {
t.Fatalf("unexpected error: %v", res.Err)
diff --git a/internal/parser/parser/ppt_parser_cgo_test.go b/internal/parser/parser/ppt_parser_cgo_test.go
index 5aad089300..bdf43ca3e1 100644
--- a/internal/parser/parser/ppt_parser_cgo_test.go
+++ b/internal/parser/parser/ppt_parser_cgo_test.go
@@ -159,8 +159,7 @@ func TestPPTParser_TCADPFileType(t *testing.T) {
// PPTParser delegates to PPTXParser{format:"ppt"}.
p := NewPPTParser()
setup := map[string]any{
- "parse_method": "tcadp",
- "output_format": "json",
+ "parse_method": "tcadp",
}
p.ConfigureFromSetup(setup)
@@ -172,9 +171,6 @@ func TestPPTParser_TCADPFileType(t *testing.T) {
if p.pptx.ParseMethod != "tcadp" {
t.Errorf("PPTParser.pptx.ParseMethod = %q, want %q", p.pptx.ParseMethod, "tcadp")
}
- if p.pptx.OutputFormat != "json" {
- t.Errorf("PPTParser.pptx.OutputFormat = %q, want %q", p.pptx.OutputFormat, "json")
- }
}
// TestPPTXParser_TCADPFileType verifies that a PPTXParser
@@ -185,8 +181,7 @@ func TestPPTXParser_TCADPFileType(t *testing.T) {
t.Fatalf("NewPPTXParser().format = %q, want pptx", p.format)
}
setup := map[string]any{
- "parse_method": "tcadp",
- "output_format": "json",
+ "parse_method": "tcadp",
}
p.ConfigureFromSetup(setup)
if p.ParseMethod != "tcadp" {
@@ -248,7 +243,6 @@ func testPresentationTCADPIntegration(t *testing.T, p tcadpPresentationParser, w
p.ConfigureFromSetup(map[string]any{
"parse_method": "tcadp",
- "output_format": "json",
"tcadp_apiserver": server.URL,
})
ctx := t.Context()
@@ -288,7 +282,6 @@ func TestPPTXParser_TCADPDownloadHTTPError(t *testing.T) {
p := NewPPTXParser()
p.ConfigureFromSetup(map[string]any{
"parse_method": "tcadp",
- "output_format": "json",
"tcadp_apiserver": server.URL,
})
ctx := t.Context()
diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.go
index 61cc0a4a77..6ebfaf41c4 100644
--- a/internal/parser/parser/pptx_parser.go
+++ b/internal/parser/parser/pptx_parser.go
@@ -43,7 +43,6 @@ type PPTXParser struct {
TCADPAPIKey string
TCADPTableResultType string
TCADPMarkdownImageResponseType string
- OutputFormat string
}
func NewPPTXParser() *PPTXParser {
@@ -75,17 +74,10 @@ func (p *PPTXParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["markdown_image_response_type"].(string); ok {
p.TCADPMarkdownImageResponseType = v
}
- if v, ok := setup["output_format"].(string); ok {
- p.OutputFormat = v
- }
- if p.OutputFormat == "" {
- p.OutputFormat = "json"
- }
}
// ParseWithResult emits one JSON item per slide with the slide's
-// plain text. Forces OutputFormat to "json" and emits one JSON item
-// per slide section.
+// plain text.
func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
// p == nil guard: the struct is embedded by value in PPTParser and
// always created via NewPPTXParser or the "ppt"-format constructor in
@@ -108,7 +100,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
filename, data, strings.ToUpper(p.format),
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
- p.OutputFormat,
)
case "", "deepdoc":
// Continue with the local office_oxide parser.
diff --git a/internal/parser/parser/tcadp_common.go b/internal/parser/parser/tcadp_common.go
index 28770b3cad..b56a54466d 100644
--- a/internal/parser/parser/tcadp_common.go
+++ b/internal/parser/parser/tcadp_common.go
@@ -49,7 +49,6 @@ func parseWithTCADP(
fileType string,
tcadpAPIServer, tcadpAPIKey string,
tableResultType, markdownImageResponseType string,
- outputFormat string,
) ParseResult {
if len(data) == 0 {
return emptyPDFResult(filename)
@@ -118,7 +117,7 @@ func parseWithTCADP(
if err != nil {
return ParseResult{Err: err}
}
- return pdfItemsToResult(filename, items, outputFormat, pageCount)
+ return pdfItemsToResult(filename, items, pageCount)
}
// authHeaderForDownload returns the bearer Authorization header value to
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index d22e1758c6..01e6a5ffaf 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -30,7 +30,6 @@ var compoundDocumentSignature = []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a,
type XLSParser struct {
ParseMethod string
- OutputFormat string
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -55,9 +54,6 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["parse_method"].(string); ok && v != "" {
p.ParseMethod = v
}
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
deprecatedHTML4Excel(setup, p.String())
deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
@@ -82,7 +78,6 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
ctx, filename, data, "XLS",
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
- p.OutputFormat,
)
case "", "excelize":
// Continue with the local Excelize parser.
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index 0a9a2e0bbd..5eb1b21919 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -237,10 +237,6 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
}
- // Configure output format to "json"
- p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- })
resJSON := p.ParseWithResult(context.Background(), "test.xls", data)
if resJSON.Err != nil {
t.Fatalf("ParseWithResult failed: %v", resJSON.Err)
@@ -274,21 +270,6 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {
t.Errorf("item doc_type_kwd = %v, want table", res.JSON[0].ToMap()["doc_type_kwd"])
}
-
- // Even if configured to legacy "html", format is unified to "json"
- p.ConfigureFromSetup(map[string]any{
- "output_format": "html",
- })
- resHTML := p.ParseWithResult(context.Background(), "test.xlsx", data)
- if resHTML.Err != nil {
- t.Fatalf("ParseWithResult failed: %v", resHTML.Err)
- }
- if resHTML.OutputFormat != "json" {
- t.Errorf("resHTML.OutputFormat = %q, want 'json'", resHTML.OutputFormat)
- }
- if len(resHTML.JSON) == 0 {
- t.Errorf("resHTML.JSON should have items")
- }
}
func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go
index 84a6fe4973..6de76217e5 100644
--- a/internal/parser/parser/xls_tcadp_test.go
+++ b/internal/parser/parser/xls_tcadp_test.go
@@ -35,7 +35,6 @@ func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
}
p.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "json",
"chunk_rows": 50,
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
@@ -74,7 +73,6 @@ func TestXLSParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
}
p.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "json",
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
})
@@ -107,7 +105,6 @@ func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
p := NewCSVParser()
p.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "json",
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
})
@@ -120,7 +117,7 @@ func TestCSVParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
}
}
-func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
+func TestXLSXParser_ParseWithResult_TCADPIntegration(t *testing.T) {
withSSRFBypass(t)
ctx := t.Context()
zipPayload := tcadpZipFixture(t)
@@ -143,15 +140,14 @@ func TestXLSXParser_ParseWithResult_TCADPMarkdownIntegration(t *testing.T) {
}
p.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "markdown",
"tcadp_apiserver": server.URL,
})
res := p.ParseWithResult(ctx, "sample.xlsx", []byte("mock xlsx content"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if res.Markdown == "" {
- t.Fatal("Markdown is empty; want rendered content")
+ if len(res.JSON) == 0 {
+ t.Fatal("JSON is empty; want rendered content")
}
}
@@ -240,7 +236,6 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {
}
p.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
- "output_format": "json",
"chunk_rows": 50,
"tcadp_apiserver": "https://tcadp.example.com",
"tcadp_api_key": "secret",
@@ -250,9 +245,6 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {
if got, want := p.ParseMethod, "TCADP parser"; got != want {
t.Fatalf("ParseMethod = %q, want %q", got, want)
}
- if got, want := p.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
- }
if got, want := p.TCADPAPIServer, "https://tcadp.example.com"; got != want {
t.Fatalf("TCADPAPIServer = %q, want %q", got, want)
}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index b0dc873283..ed397ff5c7 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -27,7 +27,6 @@ import (
type XLSXParser struct {
ParseMethod string
- OutputFormat string
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -52,9 +51,6 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["parse_method"].(string); ok && v != "" {
p.ParseMethod = v
}
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
deprecatedHTML4Excel(setup, p.String())
deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
@@ -96,7 +92,6 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
ctx, filename, data, "XLSX",
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
- p.OutputFormat,
)
case "", "excelize":
// Continue with the local Excelize parser.
diff --git a/internal/service/file/file_content_cgo_test.go b/internal/service/file/file_content_cgo_test.go
new file mode 100644
index 0000000000..60a3957823
--- /dev/null
+++ b/internal/service/file/file_content_cgo_test.go
@@ -0,0 +1,81 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+//go:build cgo
+
+package file
+
+import (
+ "archive/zip"
+ "bytes"
+ "strings"
+ "testing"
+)
+
+// minimalDOCX builds a one-paragraph .docx so the upload path exercises the
+// office_oxide-backed DOCX parser without a checked-in binary fixture.
+func minimalDOCX(t *testing.T, text string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ write := func(name, body string) {
+ t.Helper()
+ w, err := zw.Create(name)
+ if err != nil {
+ t.Fatalf("create zip entry %s: %v", name, err)
+ }
+ if _, err := w.Write([]byte(body)); err != nil {
+ t.Fatalf("write zip entry %s: %v", name, err)
+ }
+ }
+ write("[Content_Types].xml", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+ <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+ <Default Extension="xml" ContentType="application/xml"/>
+ <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+</Types>`)
+ write("_rels/.rels", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+ <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+</Relationships>`)
+ write("word/document.xml", `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+ <w:body>
+ <w:p><w:r><w:t>`+text+`</w:t></w:r></w:p>
+ </w:body>
+</w:document>`)
+ if err := zw.Close(); err != nil {
+ t.Fatalf("zip close: %v", err)
+ }
+ return buf.Bytes()
+}
+
+// TestParseFileContent_DOCXReadsJSONItems pins the upload path contract after
+// the parser became JSON-only: a .docx must be flattened from its structured
+// JSON items into readable text. The DOCX parser's former Markdown default is
+// gone, so a raw-bytes fallback (the parseFileContent failure path) would mean
+// the JSON branch regressed.
+func TestParseFileContent_DOCXReadsJSONItems(t *testing.T) {
+ data := minimalDOCX(t, "Hello DOCX content")
+
+ content := parseFileContent(t.Context(), "note.docx", data)
+ if !strings.Contains(content, "Hello DOCX content") {
+ t.Fatalf("docx content missing item text: %q", content)
+ }
+ if content == string(data) {
+ t.Fatal("content fell back to raw docx bytes instead of parsed items")
+ }
+}
diff --git a/tools/mdprobe/main.go b/tools/mdprobe/main.go
index 3703173a78..e2cc6c3d3d 100644
--- a/tools/mdprobe/main.go
+++ b/tools/mdprobe/main.go
@@ -40,8 +40,9 @@ func main() {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
- // The pipeline's setups for the markdown family: output_format=json.
- p.ConfigureFromSetup(map[string]any{"output_format": "json"})
+ // The pipeline's setups for the markdown family carry no output key:
+ // the parser emits JSON unconditionally.
+ p.ConfigureFromSetup(map[string]any{})
res := p.ParseWithResult(context.Background(), filepath.Base(path), data)
fmt.Printf("file=%s bytes=%d output_format=%s items=%d\n", filepath.Base(path), len(data), res.OutputFormat, len(res.JSON))