Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): drop ParseResult.OutputFormat/Markdown, gate on Produced() (#21414)

xugangqiang0410c4a2667b
diff --git a/internal/ingestion/component/chunker/pdf_postprocess_test.go b/internal/ingestion/component/chunker/pdf_postprocess_test.go
index 8b993c3116..c1fc5d104b 100644
--- a/internal/ingestion/component/chunker/pdf_postprocess_test.go
+++ b/internal/ingestion/component/chunker/pdf_postprocess_test.go
@@ -9,7 +9,7 @@ import (
)
func TestPDFTableNotesDoNotHideCellsDuringChunking(t *testing.T) {
- result := parser.NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{
+ result := parser.NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", parser.ParseResult{JSON: []parser.DocItem{
parser.TableItem{Table: &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, Extra: map[string]any{"text": "Footnote"}},
}})
if result.Err != nil {
diff --git a/internal/ingestion/component/chunker/pdfcrop_manual_test.go b/internal/ingestion/component/chunker/pdfcrop_manual_test.go
index a141905a32..e19e069b74 100644
--- a/internal/ingestion/component/chunker/pdfcrop_manual_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_manual_test.go
@@ -46,7 +46,7 @@ func TestHongLouMengCropPreview(t *testing.T) {
if result.Err != nil {
t.Fatal(result.Err)
}
- inputs := map[string]any{"name": filepath.Base(input), "file_type": "pdf", "output_format": result.OutputFormat, "json": result.JSON, "file": result.File, "bucket": "crop-test", "path": "source.pdf"}
+ inputs := map[string]any{"name": filepath.Base(input), "file_type": "pdf", "output_format": "json", "json": result.JSON, "file": result.File, "bucket": "crop-test", "path": "source.pdf"}
encoded, err := json.Marshal(inputs)
if err != nil {
t.Fatal(err)
diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
index 583ee33d5b..25a3114912 100644
--- a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
@@ -23,7 +23,7 @@ func TestCSVQAFirstRowIsData(t *testing.T) {
out, err := comp.Invoke(t.Context(), nil, map[string]any{
"name": "qa.csv",
"file_type": "csv",
- "output_format": res.OutputFormat,
+ "output_format": "json",
"json": res.JSON,
})
if err != nil {
diff --git a/internal/ingestion/component/chunker/sheet_identity_test.go b/internal/ingestion/component/chunker/sheet_identity_test.go
index 0580d496ef..d3643d85df 100644
--- a/internal/ingestion/component/chunker/sheet_identity_test.go
+++ b/internal/ingestion/component/chunker/sheet_identity_test.go
@@ -73,7 +73,7 @@ func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": source.name, "output_format": source.result.OutputFormat, "json": items})
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": source.name, "output_format": "json", "json": items})
if err != nil {
t.Fatal(err)
}
@@ -165,7 +165,7 @@ func TestQAChunkerSheetNameOnlyPreservesSourceRows(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": "rows.xlsx", "output_format": parsed.OutputFormat, "json": parsed.JSON})
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": "rows.xlsx", "output_format": "json", "json": parsed.JSON})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
index 4c85241678..215dfff48d 100644
--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
@@ -46,11 +46,8 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {
t.Fatal(res.Err)
}
- inputs := map[string]any{"name": "qa.xlsx", "file_type": "xlsx", "output_format": res.OutputFormat}
- switch res.OutputFormat {
- case "json":
- inputs["json"] = res.JSON
- }
+ inputs := map[string]any{"name": "qa.xlsx", "file_type": "xlsx", "output_format": "json"}
+ inputs["json"] = res.JSON
comp, err := NewQAChunker(nil)
if err != nil {
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index c411d4bce9..ccca6ff1ea 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -146,7 +146,6 @@ func maybeDispatchImage(
if len(parsed.JSON) == 0 {
return parsed, true, fmt.Errorf("parser: image parser returned no image item")
}
- parsed.OutputFormat = "json"
imageData, _ := parsed.JSON[0].ToMap()["image"].(string)
m := parsed.JSON[0].ToMap()
m["text"] = text
@@ -390,8 +389,7 @@ func maybeDispatchAudio(
}
return parser.ParseResult{
- OutputFormat: "json",
- JSON: []parser.DocItem{parser.NewTextJSONItem(transcription)},
+ JSON: []parser.DocItem{parser.NewTextJSONItem(transcription)},
}, true, nil
}
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index e54adcfdf9..56befd96d5 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -345,8 +345,8 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
// (OutputFormat=="json"), not a bare Text field. The combined text
// now lives in JSON[0]["text"]; the legacy res.Text is no longer
// populated for the image family.
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json (image family is always structured)", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want json (image family is always structured)", res.Produced())
}
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1 (image result must be a single JSON item)", len(res.JSON))
@@ -500,8 +500,8 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
if !dispatched {
t.Fatalf("expected dispatched=true")
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want json", res.Produced())
}
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
@@ -546,8 +546,8 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want json", res.Produced())
}
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
@@ -599,8 +599,8 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {
if !dispatched {
t.Fatalf("expected dispatched=true for AURAL file")
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want json", res.Produced())
}
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1 (transcription must be carried as a JSON item)", len(res.JSON))
@@ -639,8 +639,8 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
if !dispatched {
t.Fatal("expected dispatched=true for AURAL file")
}
- if res.OutputFormat != "json" {
- t.Fatalf("default OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("default produced = %t, want true", res.Produced())
}
}
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 561e9e7fb4..8b5c220e32 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -598,9 +598,6 @@ func resolveParserNormalizationSource(dispatched parser.ParseResult) string {
if len(dispatched.JSON) > 0 {
return "json"
}
- if dispatched.Markdown != "" {
- return "markdown"
- }
return "raw"
}
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index 85019f7498..0d5370cb1d 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -28,13 +28,10 @@ import (
"fmt"
"strings"
- "ragflow/internal/common"
"ragflow/internal/ingestion/component/progress"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
"ragflow/internal/utility"
-
- "go.uber.org/zap"
)
type parserSetupConfigurer interface {
@@ -65,8 +62,8 @@ func configureParserFromSetups(p any, fileType utility.FileType, setups map[stri
// its structured ParseWithResult contract.
//
// The function NEVER returns a partial result. On error the result
-// is the zero value (OutputFormat == "" + Err != nil). Callers can
-// detect the success/failure boundary on the OutputFormat alone.
+// is the zero value (Err != nil, no JSON and no File). Callers detect the
+// success/failure boundary with ParseResult.Produced().
//
// fileType may be utility.FileTypeOTHER when the upstream did not
// supply a filename; the dispatch then takes text-page mode
@@ -318,7 +315,7 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name
if lang != "" {
out["lang"] = lang
}
- if dispatched.Err == nil && dispatched.OutputFormat != "" {
+ if dispatched.Produced() {
out["output_format"] = "json"
// Emit the parser's typed []DocItem directly. The chunker consumes
// it via schema.ChunkDocsFromDocItem (no ToMap round-trip); other
@@ -354,20 +351,9 @@ func normalizeParserJSON(ctx context.Context, filename string, dispatched parser
if len(dispatched.JSON) > 0 {
return dispatched.JSON
}
- if dispatched.Markdown != "" {
- return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)
- }
return []parser.DocItem{}
}
-func warnParserNormalizationFallback(filename, source string, err error) {
- common.Warn("parser normalization fell back to text",
- zap.String("filename", filename),
- zap.String("normalized_from", source),
- zap.Error(err),
- )
-}
-
func parserInputName(inputs map[string]any, docID string) string {
if inputs != nil {
if name, ok := inputs["name"].(string); ok && name != "" {
diff --git a/internal/ingestion/component/parser_dispatch_gate_test.go b/internal/ingestion/component/parser_dispatch_gate_test.go
new file mode 100644
index 0000000000..f2d0a86039
--- /dev/null
+++ b/internal/ingestion/component/parser_dispatch_gate_test.go
@@ -0,0 +1,47 @@
+package component
+
+import (
+ "errors"
+ "testing"
+
+ "ragflow/internal/parser/parser"
+)
+
+// TestBuildParserOutputsUsesProduced pins that buildParserOutputs emits the
+// structured json output only for a Produced() dispatch result, and otherwise
+// takes the raw-text fallback path. This replaces the old
+// `Err == nil && OutputFormat != ""` gate, so the boundary behaviour must stay
+// identical for the production producers (which all set JSON/File on success).
+func TestBuildParserOutputsUsesProduced(t *testing.T) {
+ // Produced result (JSON present) => structured output emitted.
+ produced := parser.ParseResult{JSON: []parser.DocItem{parser.NewTextJSONItem("hi")}}
+ out := buildParserOutputs(t.Context(), produced, "f.pdf", nil, "")
+ if out["output_format"] != "json" {
+ t.Errorf("produced result must set output_format=json, got %v", out["output_format"])
+ }
+ if _, ok := out["json"]; !ok {
+ t.Error("produced result must emit json")
+ }
+
+ // Not produced: success with neither JSON nor File => raw fallback path.
+ notProduced := parser.ParseResult{}
+ outFallback := buildParserOutputs(t.Context(), notProduced, "f.pdf", []byte("raw page"), "")
+ if _, ok := outFallback["json"]; !ok {
+ t.Fatal("fallback must still emit json items")
+ }
+ items, _ := outFallback["json"].([]parser.DocItem)
+ if len(items) != 1 {
+ t.Fatalf("fallback must emit one raw text item, got %d", len(items))
+ }
+ if text, _ := items[0].ToMap()["text"].(string); text != "raw page" {
+ t.Errorf("fallback text item mismatch: %q", text)
+ }
+
+ // Error result => also takes the fallback path (never emits structured JSON).
+ errored := parser.ParseResult{Err: errors.New("boom")}
+ outErr := buildParserOutputs(t.Context(), errored, "f.pdf", []byte("raw"), "")
+ itemsErr, _ := outErr["json"].([]parser.DocItem)
+ if len(itemsErr) != 1 {
+ t.Fatalf("errored result must fall back to one raw text item, got %d", len(itemsErr))
+ }
+}
diff --git a/internal/ingestion/component/parser_dispatch_output_format_test.go b/internal/ingestion/component/parser_dispatch_output_format_test.go
new file mode 100644
index 0000000000..1ed8f8b232
--- /dev/null
+++ b/internal/ingestion/component/parser_dispatch_output_format_test.go
@@ -0,0 +1,66 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "testing"
+
+ "ragflow/internal/parser/parser"
+)
+
+// TestBuildParserOutputs_FileTypeOTHER_NotProduced pins that a zero (not-produced)
+// ParseResult — the shape dispatchParse returns for an unknown FileTypeOTHER —
+// still emits the wire output_format="json" key but is routed to the raw-text
+// fallback (never treated as a structured empty payload). This is the exact
+// equivalence the Produced() gate must preserve for the FileTypeOTHER case.
+func TestBuildParserOutputs_FileTypeOTHER_NotProduced(t *testing.T) {
+ out := buildParserOutputs(context.Background(), parser.ParseResult{}, "f.bin", nil, "")
+ if out["output_format"] != "json" {
+ t.Errorf("output_format = %v, want json", out["output_format"])
+ }
+ items, ok := out["json"].([]parser.DocItem)
+ if !ok {
+ t.Fatalf("json has unexpected type %T", out["json"])
+ }
+ if len(items) == 0 {
+ t.Error("expected raw-text fallback items for a zero result")
+ }
+}
+
+// TestBuildParserOutputs_MarkdownProducerBecomesJSON pins that a producer that
+// already returns normalized JSON (the post-refactor contract — markdown is
+// converted at the producer boundary via NormalizeMarkdown, never carried in a
+// ParseResult.Markdown field) is emitted verbatim by buildParserOutputs.
+func TestBuildParserOutputs_MarkdownProducerBecomesJSON(t *testing.T) {
+ dispatched := parser.ParseResult{
+ JSON: parser.MapsToItems([]map[string]any{
+ {"text": "x", "doc_type_kwd": "text"},
+ }),
+ }
+ out := buildParserOutputs(context.Background(), dispatched, "f.md", nil, "")
+ if out["output_format"] != "json" {
+ t.Errorf("output_format = %v, want json", out["output_format"])
+ }
+ items, ok := out["json"].([]parser.DocItem)
+ if !ok {
+ t.Fatalf("json has unexpected type %T", out["json"])
+ }
+ if len(items) != 1 {
+ t.Errorf("len(items) = %d, want 1", len(items))
+ }
+}
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index a09fe23228..0ef41e37a9 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -30,7 +30,6 @@ import (
"context"
"encoding/json"
"fmt"
- "net"
"net/http"
"net/http/httptest"
"os"
@@ -95,55 +94,6 @@ func requireJSONText(t *testing.T, out map[string]any, want string) {
t.Fatalf("json payload does not contain %q: %#v", want, items)
}
-func TestBuildParserOutputsNormalizesMarkdownToJSON(t *testing.T) {
- tests := []struct {
- name string
- dispatched parser.ParseResult
- wantText string
- }{
- {
- name: "markdown",
- dispatched: parser.ParseResult{
- OutputFormat: "markdown",
- Markdown: "# Title\n\nBody",
- },
- wantText: "Title",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- out := buildParserOutputs(t.Context(), tt.dispatched, "sample."+tt.name, nil, "")
-
- requireJSONText(t, out, tt.wantText)
- if tt.name == "markdown" {
- jsonDocs, ok := out["json"].([]parser.DocItem)
- items := parser.ItemsToMaps(jsonDocs)
- if !ok {
- t.Fatalf("json = %T, want []map[string]any", out["json"])
- }
- if len(items) != 2 {
- t.Fatalf("markdown json item count = %d, want 2: %#v", len(items), items)
- }
- if got := items[0]["layout"]; got != "heading" {
- t.Errorf("markdown item[0].layout = %v, want heading", got)
- }
- if got := items[0]["text"]; got != "# Title" {
- t.Errorf("markdown item[0].text = %v, want # Title", got)
- }
- if got := items[1]["text"]; got != "Body" {
- t.Errorf("markdown item[1].text = %v, want Body", got)
- }
- }
- for _, key := range []string{"markdown", "html", "text"} {
- if _, ok := out[key]; ok {
- t.Errorf("output contains obsolete %q payload: %#v", key, out[key])
- }
- }
- })
- }
-}
-
// TestNormalizeParserJSON_EmptyPayloadYieldsEmpty pins that a result with no
// populated payload family normalizes to an empty JSON slice rather than
// injecting a bogus item. HTML is not a supported backend output format
@@ -151,29 +101,16 @@ func TestBuildParserOutputsNormalizesMarkdownToJSON(t *testing.T) {
// HTML payload), so there is no HTML normalization branch to fall through
// to — this guards that regression.
func TestNormalizeParserJSON_EmptyPayloadYieldsEmpty(t *testing.T) {
- r := parser.ParseResult{
- OutputFormat: "json",
- }
+ r := parser.ParseResult{}
items := normalizeParserJSON(t.Context(), "doc.txt", r)
if len(items) != 0 {
t.Fatalf("normalizeParserJSON(empty) = %d items, want 0 (no payload → empty)", len(items))
}
}
-func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {
- out := buildParserOutputs(t.Context(), parser.ParseResult{
- OutputFormat: "json",
- JSON: parser.MapsToItems([]map[string]any{}),
- Markdown: "# Recovered title",
- }, "sample.md", nil, "")
-
- requireJSONText(t, out, "Recovered title")
-}
-
func TestBuildParserOutputsPreservesPDFOutlineMetadata(t *testing.T) {
wantOutline := []map[string]any{{"title": "Overview", "level": 0, "page_number": 1}}
out := buildParserOutputs(t.Context(), parser.ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": "report.pdf",
"outline": wantOutline,
@@ -193,33 +130,6 @@ func TestBuildParserOutputsPreservesPDFOutlineMetadata(t *testing.T) {
}
}
-func TestParseMarkdownToJSONItemsDoesNotResolveRemoteImages(t *testing.T) {
- originalResolver := net.DefaultResolver
- dnsLookup := make(chan struct{}, 1)
- net.DefaultResolver = &net.Resolver{
- PreferGo: true,
- StrictErrors: true,
- Dial: func(context.Context, string, string) (net.Conn, error) {
- select {
- case dnsLookup <- struct{}{}:
- default:
- }
- return nil, fmt.Errorf("unexpected DNS lookup")
- },
- }
- t.Cleanup(func() { net.DefaultResolver = originalResolver })
-
- items := parseMarkdownToJSONItems(t.Context(), "ocr.md", "![remote](https://images.example.invalid/figure.png)")
- if len(items) != 1 {
- t.Fatalf("items = %#v, want one Markdown image block", items)
- }
- select {
- case <-dnsLookup:
- t.Fatal("Markdown-to-JSON normalization performed a remote image DNS lookup")
- default:
- }
-}
-
func TestLogParserOutputReportsNormalizationSource(t *testing.T) {
core, logs := observer.New(zap.DebugLevel)
originalLogger := common.Logger
@@ -227,8 +137,7 @@ func TestLogParserOutputReportsNormalizationSource(t *testing.T) {
t.Cleanup(func() { common.Logger = originalLogger })
logParserOutput(parser.ParseResult{
- OutputFormat: "markdown",
- Markdown: "# Title\n\nBody",
+ JSON: []parser.DocItem{parser.NewTextJSONItem("Title"), parser.NewTextJSONItem("Body")},
}, []parser.DocItem{parser.NewTextJSONItem("Title"), parser.NewTextJSONItem("Body")})
entries := logs.FilterMessage("parser stage output").All()
@@ -239,38 +148,14 @@ func TestLogParserOutputReportsNormalizationSource(t *testing.T) {
if _, ok := fields["output_format"]; ok {
t.Errorf("output_format must be omitted because it is always json: %v", fields)
}
- if got := fields["normalized_from"]; got != "markdown" {
- t.Errorf("normalized_from = %v, want markdown", got)
+ if got := fields["normalized_from"]; got != "json" {
+ t.Errorf("normalized_from = %v, want json", got)
}
if got := fields["json_items"]; got != int64(2) {
t.Errorf("json_items = %v, want 2", got)
}
}
-func TestWarnParserNormalizationFallbackReportsCause(t *testing.T) {
- core, logs := observer.New(zap.WarnLevel)
- originalLogger := common.Logger
- common.Logger = zap.New(core)
- t.Cleanup(func() { common.Logger = originalLogger })
-
- warnParserNormalizationFallback("broken.md", "markdown", fmt.Errorf("parse failed"))
-
- entries := logs.FilterMessage("parser normalization fell back to text").All()
- if len(entries) != 1 {
- t.Fatalf("normalization fallback log count = %d, want 1", len(entries))
- }
- fields := entries[0].ContextMap()
- if got := fields["filename"]; got != "broken.md" {
- t.Errorf("filename = %v, want broken.md", got)
- }
- if got := fields["normalized_from"]; got != "markdown" {
- t.Errorf("normalized_from = %v, want markdown", got)
- }
- if got := fields["error"]; got != "parse failed" {
- t.Errorf("error = %v, want parse failed", got)
- }
-}
-
func TestDispatch_JSONOutput(t *testing.T) {
setups := defaultSetups()
c := &ParserComponent{setups: setups}
diff --git a/internal/ingestion/component/pdf_postprocess_test.go b/internal/ingestion/component/pdf_postprocess_test.go
index 5416951833..6a1f705bb3 100644
--- a/internal/ingestion/component/pdf_postprocess_test.go
+++ b/internal/ingestion/component/pdf_postprocess_test.go
@@ -38,7 +38,7 @@ func TestPDFDispatchNormalizesBeforeOutputsAndEnhancement(t *testing.T) {
}
setup := schema.ParserSetup{"parse_method": "monkeyocrv2", "monkeyocrv2_server_url": server.URL}
result, handled, err := maybeDispatchPDFVision(t.Context(), nil, utility.FileTypePDF, "sample.pdf", []byte("pdf"), nil, map[string]schema.ParserSetup{"pdf": setup})
- if err != nil || !handled || result.OutputFormat != "json" || len(result.JSON) == 0 {
+ if err != nil || !handled || !result.Produced() || len(result.JSON) == 0 {
t.Fatalf("dispatch did not emit structured JSON: %+v, %v", result, err)
}
for _, enabled := range []bool{false, true} {
@@ -92,7 +92,7 @@ func TestMinerUTablePreservesEmbeddedCaption(t *testing.T) {
func TestPDFTableEnhancementKeepsDescriptionOutsideTableBody(t *testing.T) {
invoker := &visionEnhanceCaptureInvoker{}
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
- result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{
+ result := parser.ParseResult{JSON: []parser.DocItem{
parser.NewTextJSONItem("Before"),
parser.TableItem{Table: &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, Extra: map[string]any{"image": visionTestPNGBase64(t), "page_number": 3}},
parser.NewTextJSONItem("After"),
@@ -146,7 +146,7 @@ func TestVisionEnhancementSkipsBackendDescribedImages(t *testing.T) {
"doc_type_kwd": "image", "layout": "figure",
"text": "Caption", "image": png,
})
- result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{marked, plain}}
+ result := parser.ParseResult{JSON: []parser.DocItem{marked, plain}}
enhanced, handled, err := maybeDispatchVisionEnhancement(t.Context(), nil, utility.FileTypePDF, result, map[string]any{"tenant_id": "tenant-1"}, visionSettings{}, progress.Window{})
if err != nil || !handled {
t.Fatalf("enhancement failed: %v", err)
@@ -174,7 +174,7 @@ func TestVisionEnhancementSkipsBackendDescribedImages(t *testing.T) {
}
func TestStripBackendDescribedResultClearsMarker(t *testing.T) {
- result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{
+ result := parser.ParseResult{JSON: []parser.DocItem{
parser.DocItemFromMap(map[string]any{"doc_type_kwd": "image", "text": "t", backendDescribedKey: true}),
parser.NewTextJSONItem("plain"),
}}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index b65f90f476..d9e7610021 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -83,10 +83,10 @@ func maybeDispatchPDFVision(
tenantID := getStringOr(inputs, "tenant_id", "")
// The tenant-selected parser and the built-in backends share one
// normalization boundary so a future normalization fix cannot miss one
- // of the two paths. ParseWithResult already normalizes JSON internally,
- // but markdown stays unconverted there; running it through the same
- // NormalizeResult keeps markdown→JSON conversion in the parser package
- // and makes the call idempotent for already-normalized JSON.
+ // of the two paths. PDF markdown engines (MonkeyOCRv2, MinerU V1
+ // fallback) are converted to JSON by NormalizeMarkdown before dispatch
+ // returns, so the NormalizeResult closure below only ever applies item
+ // policies to already-normalized JSON and stays idempotent.
flatten, _ := setup["flatten_media_to_text"].(bool)
normalize := func(result parser.ParseResult) parser.ParseResult {
pdf := parser.PDFParser{FlattenMediaToText: flatten}
@@ -273,41 +273,17 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
return parser.ParseResult{}, err
}
md := strings.Join(sections, "\n\n")
- return buildMarkdownOCRDispatchResult(md), nil
+ return normalizePDFMarkdown(ctx, filename, md, setup), nil
}
-func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []parser.DocItem {
- // Fallback markdown→JSON converter for non-PDF families via
- // buildParserOutputs/normalizeParserJSON. PDF dispatch results are
- // normalized earlier by PDFParser.NormalizeResult (parser.parsePDFMarkdown),
- // so this stays as the final safety net rather than a second PDF boundary.
- mp, err := parser.NewMarkdownParser(parser.GoMarkdown)
- if err != nil {
- warnParserNormalizationFallback(filename, "markdown", err)
- return []parser.DocItem{parser.NewTextJSONItem(mdText)}
- }
- // OCR Markdown is an untrusted backend response. Keep normalization
- // deterministic and local: embedded data URIs remain available, while
- // remote image URLs do not introduce network I/O at the Parser boundary.
- mp.FetchRemoteImages = false
- res := mp.ParseWithResult(ctx, filename, []byte(mdText))
- if res.Err != nil {
- warnParserNormalizationFallback(filename, "markdown", res.Err)
- return []parser.DocItem{parser.NewTextJSONItem(mdText)}
- }
- if len(res.JSON) == 0 {
- warnParserNormalizationFallback(filename, "markdown", fmt.Errorf("parser returned no JSON items"))
- return []parser.DocItem{parser.NewTextJSONItem(mdText)}
- }
- return res.JSON
-}
-
-func buildMarkdownOCRDispatchResult(md string) parser.ParseResult {
- // The PDF dispatch boundary converts backend Markdown before enhancement.
- return parser.ParseResult{
- OutputFormat: "markdown",
- Markdown: md,
- }
+// normalizePDFMarkdown converts a PDF backend Markdown response (MonkeyOCRv2,
+// MinerU V1 fallback) into the structured JSON contract via the parser
+// package's NormalizeMarkdown. This keeps the markdown→JSON boundary in the
+// parser package, matching the NormalizeResult path used for JSON backends, and
+// lets the downstream NormalizeResult closure stay idempotent for JSON.
+func normalizePDFMarkdown(ctx context.Context, filename, md string, setup schema.ParserSetup) parser.ParseResult {
+ flatten, _ := setup["flatten_media_to_text"].(bool)
+ return (&parser.PDFParser{FlattenMediaToText: flatten}).NormalizeMarkdown(ctx, filename, md)
}
var isMonkeyOCRv2LayoutModelID = defaultIsMonkeyOCRv2LayoutModelID
@@ -617,7 +593,7 @@ func dispatchMinerUPDF(
return mineruJSONParseResult(filename, items), nil
}
if v1Markdown != "" {
- return buildMarkdownOCRDispatchResult(result.Markdown), nil
+ return normalizePDFMarkdown(ctx, filename, result.Markdown, setup), nil
}
if extractErr != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", extractErr)
@@ -1143,7 +1119,6 @@ func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {
func mineruJSONParseResult(filename string, items []map[string]any) parser.ParseResult {
return parser.ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"page_count": mineruItemPageCount(items),
@@ -1246,9 +1221,8 @@ func dispatchPDFVision(
"parse_method": modelID,
}
return parser.ParseResult{
- OutputFormat: "json",
- File: fileMeta,
- JSON: parser.MapsToItems(items),
+ File: fileMeta,
+ JSON: parser.MapsToItems(items),
}, nil
}
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 8ffdeaf770..e119dec540 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -107,7 +107,7 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if result.OutputFormat != "markdown" || result.Markdown != "hello" || result.JSON != nil {
+ if !result.Produced() || len(result.JSON) == 0 {
t.Fatalf("result=%+v", result)
}
@@ -115,7 +115,7 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if resultMD.OutputFormat != "markdown" || resultMD.Markdown != "hello" {
+ if !resultMD.Produced() || len(resultMD.JSON) == 0 {
t.Fatalf("resultMD=%+v", resultMD)
}
}
@@ -151,7 +151,7 @@ func TestDispatchMonkeyOCRv2PDFUsesSelectedCompositeModel(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if result.Markdown != "selected" {
+ if !result.Produced() || len(result.JSON) == 0 {
t.Fatalf("result=%+v", result)
}
}
@@ -179,7 +179,7 @@ func TestDispatchPaddleOCRPDFUsesStructuredOutput(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if result.OutputFormat != "json" || len(result.JSON) == 0 {
+ if !result.Produced() || len(result.JSON) == 0 {
t.Fatalf("result=%+v", result)
}
}
@@ -197,7 +197,7 @@ func TestDispatchPaddleOCRPDFRejectsMissingBlocks(t *testing.T) {
if err == nil {
t.Fatalf("dispatchPaddleOCRPdf without page blocks: expected error, got result %+v", res)
}
- if res.OutputFormat != "" || res.Markdown != "" {
+ if res.Produced() {
t.Errorf("expected zero-value result on error, got %+v", res)
}
}
@@ -337,7 +337,7 @@ func TestDispatchPaddleOCRPDFPreservesLayout(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if result.OutputFormat != "json" || len(result.JSON) != 7 {
+ if !result.Produced() || len(result.JSON) != 7 {
t.Fatalf("result=%+v", result)
}
if result.JSON[0].ToMap()["positions"] == nil || result.JSON[2].ToMap()["table"] == nil {
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index 2856eceb7a..305be7bd4c 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -189,8 +189,9 @@ func maybeDispatchVisionEnhancement(
) (parser.ParseResult, bool, error) {
w := window
- // Only enhance successful JSON output format containing items.
- if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
+ // Only enhance successful results that carry JSON items. Produced()
+ // replaces the historical OutputFormat != "json" check.
+ if dispatched.Err != nil || len(dispatched.JSON) == 0 {
return stripBackendDescribedResult(dispatched), false, nil
}
diff --git a/internal/ingestion/component/vision_enhancement_output_format_test.go b/internal/ingestion/component/vision_enhancement_output_format_test.go
new file mode 100644
index 0000000000..96adb0c0f2
--- /dev/null
+++ b/internal/ingestion/component/vision_enhancement_output_format_test.go
@@ -0,0 +1,44 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "testing"
+
+ "ragflow/internal/ingestion/component/progress"
+ "ragflow/internal/parser/parser"
+ "ragflow/internal/utility"
+)
+
+// TestVisionEnhancement_SkipsEmptyJSON pins the gate equivalence: a not-produced
+// (zero) ParseResult is skipped by the vision enhancement pass and returned
+// unchanged after the backend-description marker strip. Previously gated on
+// OutputFormat != "json"; now gated on Err != nil || len(JSON) == 0.
+func TestVisionEnhancement_SkipsEmptyJSON(t *testing.T) {
+ res, handled, err := maybeDispatchVisionEnhancement(
+ context.Background(), nil, utility.FileTypePDF,
+ parser.ParseResult{}, map[string]any{}, visionSettings{}, progress.Window{},
+ )
+ if err != nil {
+ t.Fatalf("err: %v", err)
+ }
+ if handled {
+ t.Error("expected not handled for a zero (not-produced) result")
+ }
+ _ = res
+}
diff --git a/internal/ingestion/component/vision_enhancement_progress_test.go b/internal/ingestion/component/vision_enhancement_progress_test.go
index efaaeed9f1..cce297db22 100644
--- a/internal/ingestion/component/vision_enhancement_progress_test.go
+++ b/internal/ingestion/component/vision_enhancement_progress_test.go
@@ -23,7 +23,6 @@ func TestVisionEnhancement_ReportsPerFigureFractions(t *testing.T) {
png := visionTestPNGBase64(t)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "1", "image": png, "doc_type_kwd": "image"},
{"text": "2", "image": png, "doc_type_kwd": "image"},
@@ -78,7 +77,6 @@ func TestVisionEnhancement_SettledFractionsCountFailedFigures(t *testing.T) {
ctx := rec.context(t, "Parser:vision")
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "valid", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"},
{"text": "invalid", "image": "not-valid-base64", "doc_type_kwd": "image"},
@@ -125,7 +123,6 @@ func TestVisionEnhancement_ZeroWindowReportsNothing(t *testing.T) {
ctx := rec.context(t, "Parser:vision")
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "1", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"},
}),
@@ -160,7 +157,6 @@ func TestVisionEnhancement_CancelledContextReportsNothing(t *testing.T) {
png := visionTestPNGBase64(t)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "1", "image": png, "doc_type_kwd": "image"},
{"text": "2", "image": png, "doc_type_kwd": "image"},
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index 367e19bdba..d94b2eed9e 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -220,7 +220,6 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{{
"text": "Existing caption",
"image": visionTestPNGBase64(t),
@@ -258,7 +257,6 @@ func TestVisionEnhancementPreparesPDFSourceBeforeImageAdmission(t *testing.T) {
t.Cleanup(func() { visionImageCropperFactory = originalFactory })
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{{
"image": cropper.payload,
"doc_type_kwd": "image",
@@ -281,7 +279,6 @@ func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {
}, fakePrompt)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{{
"image": visionTestPNGBase64(t),
"doc_type_kwd": "image",
@@ -309,7 +306,6 @@ func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {
t.Cleanup(func() { visionImageCropperFactory = originalFactory })
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{{
"image": visionTestPNGBase64(t),
"doc_type_kwd": "image",
@@ -342,7 +338,6 @@ func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.T
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{{
"text": "Existing table",
"image": visionTestPNGBase64(t),
@@ -371,7 +366,6 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
imagePayload := visionTestPNGBase64(t)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "<table><tr><td>figure</td></tr></table>", "image": imagePayload, "doc_type_kwd": "table"},
{
@@ -411,8 +405,7 @@ func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {
invoker := &visionEnhanceCaptureInvoker{}
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
dispatched := parser.ParseResult{
- OutputFormat: "json",
- JSON: parser.MapsToItems([]map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}}),
+ JSON: parser.MapsToItems([]map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}}),
}
result, handled, err := maybeDispatchVisionEnhancement(
@@ -453,7 +446,6 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
})
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},
{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
@@ -540,7 +532,6 @@ func TestVisionEnhancement_LanguagePriority(t *testing.T) {
})
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
}),
@@ -562,53 +553,10 @@ func TestVisionEnhancement_LanguagePriority(t *testing.T) {
}
}
-func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
- called := false
- swapVisionGlobals(t,
- func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- called = true
- return &visionEnhanceFakeDriver{}, "m", &modelModule.APIConfig{}, 0, nil
- },
- func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
- called = true
- ans := "x"
- return &modelModule.ChatResponse{Answer: &ans}, nil
- },
- nil,
- )
-
- dispatched := parser.ParseResult{
- OutputFormat: "markdown",
- Markdown: "![Image](data:image/png;base64,abc)",
- File: map[string]any{"figures": []map[string]any{{"image": "abc", "marker": "x"}}},
- }
-
- res, handled, err := maybeDispatchVisionEnhancement(
- t.Context(),
- dao.DB,
- utility.FileTypeDOCX,
- dispatched,
- map[string]any{"tenant_id": "t1"},
- visionSettings{}, progress.Window{})
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if handled {
- t.Error("handled = true, want false (markdown path must not be enhanced)")
- }
- if called {
- t.Error("vision model was resolved/invoked on the markdown path")
- }
- if res.Markdown != "![Image](data:image/png;base64,abc)" {
- t.Errorf("markdown mutated: %q", res.Markdown)
- }
-}
-
func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
invoker := &visionEnhanceCaptureInvoker{}
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "caption", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -637,7 +585,6 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -664,8 +611,7 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {
parseErr := errors.New("parse failed")
dispatched := parser.ParseResult{
- Err: parseErr,
- OutputFormat: "json",
+ Err: parseErr,
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -703,7 +649,6 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {
cancel() // pre-cancel
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -742,7 +687,6 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {
)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
// non-string image field — filtered by target collector
{"text": "para", "image": 12345, "doc_type_kwd": "image"},
@@ -790,8 +734,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
}
}
dispatched := parser.ParseResult{
- OutputFormat: "json",
- JSON: parser.MapsToItems(items),
+ JSON: parser.MapsToItems(items),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -829,7 +772,6 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -868,7 +810,6 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {
)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -907,7 +848,6 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {
)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -958,8 +898,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T
}
}
dispatched := parser.ParseResult{
- OutputFormat: "json",
- JSON: parser.MapsToItems(items),
+ JSON: parser.MapsToItems(items),
}
ctx, cancel := context.WithCancel(t.Context())
@@ -1182,7 +1121,6 @@ func TestVisionEnhancement_GlobalModelPreferredOverPDFParser(t *testing.T) {
figureVisionPromptBuilder = fakePrompt
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
}),
@@ -1224,7 +1162,6 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {
)
dispatched := parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "keep", "image": "!!!not-base64!!!", "doc_type_kwd": "image"},
}),
diff --git a/internal/ingestion/component/vision_source_admission_test.go b/internal/ingestion/component/vision_source_admission_test.go
index 23eda6c731..20de2598de 100644
--- a/internal/ingestion/component/vision_source_admission_test.go
+++ b/internal/ingestion/component/vision_source_admission_test.go
@@ -76,7 +76,7 @@ func TestVisionSourceFetchDoesNotBlockImageAdmission(t *testing.T) {
}
}
run := func(inputs map[string]any) []string {
- dispatched := parser.ParseResult{OutputFormat: "json", JSON: parser.MapsToItems([]map[string]any{newItem()})}
+ dispatched := parser.ParseResult{JSON: parser.MapsToItems([]map[string]any{newItem()})}
res, _, err := maybeDispatchVisionEnhancement(
context.Background(), dao.DB, utility.FileTypePDF, dispatched,
inputs, visionSettings{}, progress.Window{},
diff --git a/internal/parser/parser/audio_parser.go b/internal/parser/parser/audio_parser.go
index bbcd22a1c3..7702827426 100644
--- a/internal/parser/parser/audio_parser.go
+++ b/internal/parser/parser/audio_parser.go
@@ -62,7 +62,6 @@ func (p *AudioParser) ParseWithResult(ctx context.Context, filename string, data
}
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/audio_parser_test.go b/internal/parser/parser/audio_parser_test.go
index e47d1b8d27..c8b498cff7 100644
--- a/internal/parser/parser/audio_parser_test.go
+++ b/internal/parser/parser/audio_parser_test.go
@@ -29,8 +29,8 @@ func TestAudioParser_ValidExtension(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "json" {
- t.Errorf("output_format = %q, want json", result.OutputFormat)
+ if !result.Produced() {
+ t.Errorf("produced = %t, want true", result.Produced())
}
if name, ok := result.File["name"].(string); !ok || name != "test.mp3" {
t.Errorf("name = %q, want test.mp3", name)
@@ -60,14 +60,14 @@ func TestAudioParser_InvalidExtension(t *testing.T) {
}
}
-func TestAudioParser_DefaultOutputFormat(t *testing.T) {
+func TestAudioParser_Defaultproduced(t *testing.T) {
ctx := t.Context()
p := NewAudioParser()
result := p.ParseWithResult(ctx, "test.flac", []byte{})
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "json" {
- t.Errorf("output_format = %q, want json", result.OutputFormat)
+ if !result.Produced() {
+ t.Errorf("produced = %t, want true", result.Produced())
}
}
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 499463e230..fbdc50ab4a 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -115,7 +115,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
text := string(decoded)
if strings.TrimSpace(text) == "" {
return ParseResult{
- OutputFormat: spreadsheetOutputFormat,
File: map[string]any{
"name": filename,
"size": len(data),
@@ -142,7 +141,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
// row-aligned positions (csv has no in-cell image anchors).
items := buildSheetItems(records, csvSheetName, 1, 1, dataRows, nil)
return ParseResult{
- OutputFormat: spreadsheetOutputFormat,
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index 55ad19f27c..ab7ee95cdf 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -18,8 +18,8 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {
}
// Default output format is "json"
- if res.OutputFormat != "json" {
- t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, "json")
+ if !res.Produced() {
+ t.Fatalf("res.produced = %t, want true", res.Produced())
}
// One wire shape: the whole sheet is one segmented HTML table item.
@@ -99,7 +99,7 @@ func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {
}
}
-func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
+func TestCSVParser_ConfigureproducedJSON(t *testing.T) {
csvData := []byte("col1,col2\nval1,val2\n")
p := NewCSVParser()
@@ -108,8 +108,8 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("res.produced = %t, want true", res.Produced())
}
if len(res.JSON) != 1 {
t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
@@ -130,8 +130,8 @@ func TestCSVParser_EmptyContent(t *testing.T) {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("res.produced = %t, want true", res.Produced())
}
if len(res.JSON) != 0 {
t.Fatalf("len(res.JSON) = %d, want 0", len(res.JSON))
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 5aaff1c8d6..1daeb0394b 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -63,10 +63,9 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
budget := newEmbeddedMediaBudget()
if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {
return ParseResult{
- OutputFormat: "json",
- File: map[string]any{"name": filename, "format": "doc"},
- JSON: MapsToItems(sections),
- Warnings: budget.warnings(),
+ File: map[string]any{"name": filename, "format": "doc"},
+ JSON: MapsToItems(sections),
+ Warnings: budget.warnings(),
}
}
}
@@ -74,9 +73,8 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
return ParseResult{Err: fmt.Errorf("doc extract: no text content")}
}
return ParseResult{
- OutputFormat: "json",
- File: map[string]any{"name": filename, "format": "doc"},
- JSON: []DocItem{NewTextJSONItem(text)},
+ File: map[string]any{"name": filename, "format": "doc"},
+ JSON: []DocItem{NewTextJSONItem(text)},
}
}
diff --git a/internal/parser/parser/doc_parser_doc_table_test.go b/internal/parser/parser/doc_parser_doc_table_test.go
index 77359b58f4..e9c7ab270e 100644
--- a/internal/parser/parser/doc_parser_doc_table_test.go
+++ b/internal/parser/parser/doc_parser_doc_table_test.go
@@ -34,8 +34,8 @@ func TestDOCParser_TableEmittedAsStructuredItem(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult error: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want true", res.Produced())
}
if len(res.JSON) == 0 {
t.Fatalf("JSON result is empty: %#v", res)
@@ -102,8 +102,8 @@ func TestDOCParser_NoTableDocStaysTextItem(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult error: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want true", res.Produced())
}
if len(res.JSON) == 0 {
t.Fatalf("doc with no structured sections must still emit a text item, got empty JSON")
@@ -120,7 +120,7 @@ func TestDOCParser_NoTableDocStaysTextItem(t *testing.T) {
}
// TestDOCParserTextOutputRemoved verifies the DOC parser has no "text" output
-// path. The parser only emits JSON regardless of any output_format request.
+// path. The parser only emits JSON regardless of any produced request.
func TestDOCParserTextOutputRemoved(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))
if err != nil {
@@ -132,8 +132,8 @@ func TestDOCParserTextOutputRemoved(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult error: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("DOC must ignore text output_format and emit json, got OutputFormat=%q", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("DOC must ignore text produced and emit json, got produced=%t", res.Produced())
}
if len(res.JSON) == 0 {
t.Fatalf("DOC json output must contain items, got empty")
diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.go
index cd0fa55904..fcd8a9214a 100644
--- a/internal/parser/parser/doc_parser_synth_test.go
+++ b/internal/parser/parser/doc_parser_synth_test.go
@@ -128,8 +128,8 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
if pr.Err != nil {
t.Fatalf("ParseWithResult error: %v", pr.Err)
}
- if pr.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "json")
+ if !pr.Produced() {
+ t.Fatalf("produced = %t, want true", pr.Produced())
}
if len(pr.JSON) == 0 {
t.Fatalf("ParseWithResult returned 0 JSON items, want at least 1")
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go
index 13521f6f8b..0642327787 100644
--- a/internal/parser/parser/docx_parser.go
+++ b/internal/parser/parser/docx_parser.go
@@ -143,10 +143,9 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
return ParseResult{
- OutputFormat: "json",
- File: fileMeta,
- JSON: MapsToItems(sections),
- Warnings: budget.warnings(),
+ File: fileMeta,
+ JSON: MapsToItems(sections),
+ Warnings: budget.warnings(),
}
}
diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.go
index 1a147dc860..c47bb21550 100644
--- a/internal/parser/parser/docx_parser_cgo_test.go
+++ b/internal/parser/parser/docx_parser_cgo_test.go
@@ -17,8 +17,8 @@ func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) == 0 {
t.Fatal("JSON items is empty; expected parsed content")
@@ -55,8 +55,8 @@ func TestDOCXParser_ParseWithResult_CGOMinimalDocument(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) == 0 {
t.Fatal("JSON items is empty; want parsed content")
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index c55baf39ae..7397509b13 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -134,9 +134,8 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
}
items = append(items, extraItems...)
return ParseResult{
- OutputFormat: "json",
- File: map[string]any{"name": filename},
- JSON: MapsToItems(items),
+ File: map[string]any{"name": filename},
+ JSON: MapsToItems(items),
}
}
@@ -699,7 +698,7 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s
}
var texts []string
- if res.OutputFormat == "json" && len(res.JSON) > 0 {
+ if len(res.JSON) > 0 {
for _, it := range res.JSON {
if t, ok := it.ToMap()["text"].(string); ok {
if t = strings.TrimSpace(t); t != "" {
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index de78bacf8c..bc68110952 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -61,8 +61,8 @@ func TestEmailParser_EmlJSON(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "json" {
- t.Fatalf("expected output_format json, got %q", result.OutputFormat)
+ if !result.Produced() {
+ t.Fatalf("expected json output, got %t", result.Produced())
}
if len(result.JSON) != 2 {
t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))
@@ -161,8 +161,8 @@ func TestEmailParser_EmlTextChunkText(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "json" {
- t.Fatalf("expected output_format json, got %q", result.OutputFormat)
+ if !result.Produced() {
+ t.Fatalf("expected json output, got %t", result.Produced())
}
item := result.JSON[0].ToMap()
@@ -352,8 +352,8 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "json" {
- t.Fatalf("output format = %q, want json", result.OutputFormat)
+ if !result.Produced() {
+ t.Fatalf("output = %t, want true", result.Produced())
}
if len(result.JSON) == 0 {
t.Fatalf("expected non-empty JSON items")
@@ -1069,7 +1069,7 @@ func TestRecoverParse_IsolatesPanic(t *testing.T) {
// TestRecoverParse_PassesThrough verifies a normal parser result is returned
// unchanged with panicked=false.
func TestRecoverParse_PassesThrough(t *testing.T) {
- want := ParseResult{OutputFormat: "json", JSON: []DocItem{NewTextJSONItem("hello")}}
+ want := ParseResult{JSON: []DocItem{NewTextJSONItem("hello")}}
res, panicked := recoverParse(func() ParseResult {
return want
})
@@ -1326,8 +1326,8 @@ func TestEmailParserJSONOmitsDuplicateHeaderText(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want true", res.Produced())
}
if len(res.JSON) == 0 {
t.Fatal("JSON: want non-empty items")
@@ -1339,8 +1339,10 @@ func TestEmailParserJSONOmitsDuplicateHeaderText(t *testing.T) {
}
// TestEmailParserTextOutputRemoved verifies the Email parser has no "text"
-// output path. A setup requesting output_format "text" must be ignored and the
-// result must be JSON, never a Text payload.
+// output path. A setup requesting produced "text" must be ignored and the
+// result must be JSON, identical to a default-config parse. This pins that the
+// removed text path cannot silently reappear: the "text" produced value must
+// not change the emitted JSON.
func TestEmailParserTextOutputRemoved(t *testing.T) {
ctx := context.Background()
raw := strings.Join([]string{
@@ -1352,20 +1354,46 @@ func TestEmailParserTextOutputRemoved(t *testing.T) {
"Hello, world!",
}, "\r\n")
- p := NewEmailParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "text",
- "fields": []string{"from", "to", "subject", "body"},
+ fields := []string{"from", "to", "subject", "body"}
+
+ withText := NewEmailParser()
+ withText.ConfigureFromSetup(map[string]any{
+ "produced": "text",
+ "fields": fields,
})
+ textResult := withText.ParseWithResult(ctx, "test.eml", []byte(raw))
- result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
- if result.Err != nil {
- t.Fatalf("unexpected error: %v", result.Err)
+ defaultCfg := NewEmailParser()
+ defaultCfg.ConfigureFromSetup(map[string]any{
+ "fields": fields,
+ })
+ defaultResult := defaultCfg.ParseWithResult(ctx, "test.eml", []byte(raw))
+
+ for _, result := range []ParseResult{textResult, defaultResult} {
+ if result.Err != nil {
+ t.Fatalf("unexpected error: %v", result.Err)
+ }
+ if !result.Produced() {
+ t.Fatalf("email must emit json, got produced=%t", result.Produced())
+ }
+ if len(result.JSON) == 0 {
+ t.Fatalf("email json output must contain items, got empty")
+ }
}
- if result.OutputFormat != "json" {
- t.Fatalf("email must ignore text output_format and emit json, got OutputFormat=%q", result.OutputFormat)
+
+ // The "text" produced config must be ignored: the JSON payload is
+ // identical to the default-config parse.
+ if len(textResult.JSON) != len(defaultResult.JSON) {
+ t.Fatalf("email json item count changed by produced=text: %d vs %d",
+ len(textResult.JSON), len(defaultResult.JSON))
}
- if len(result.JSON) == 0 {
- t.Fatalf("email json output must contain items, got empty")
+ for i := range textResult.JSON {
+ a, b := textResult.JSON[i].ToMap(), defaultResult.JSON[i].ToMap()
+ if a["text"] != b["text"] {
+ t.Errorf("email item %d text differs under produced=text: %q vs %q", i, a["text"], b["text"])
+ }
+ if a["doc_type_kwd"] != b["doc_type_kwd"] {
+ t.Errorf("email item %d doc_type_kwd differs under produced=text: %v vs %v", i, a["doc_type_kwd"], b["doc_type_kwd"])
+ }
}
}
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go
index acf6d1d0b9..7e0381f54d 100644
--- a/internal/parser/parser/epub_parser.go
+++ b/internal/parser/parser/epub_parser.go
@@ -59,7 +59,6 @@ func (p *EPUBParser) String() string {
func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
if len(data) == 0 {
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": 0,
@@ -92,7 +91,6 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index 686a3b46b4..ac68c4b095 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -91,7 +91,6 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"encoding": encName,
diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.go
index e49862aa80..07b8491ba8 100644
--- a/internal/parser/parser/json_parser.go
+++ b/internal/parser/parser/json_parser.go
@@ -59,7 +59,6 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data
text := string(bytes.TrimSpace(decoded))
if text == "" {
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
@@ -74,7 +73,6 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index ce033a5835..9f835465ad 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -123,7 +123,6 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, d
warnings = append(warnings, fmt.Sprintf("Markdown parser could not resolve %d image source(s)", unresolvedImages))
}
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
},
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index aee919a9f2..2fd89dbea5 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -24,8 +24,8 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want at least one item")
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 5799c4d24c..e311ae9823 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -15,19 +15,22 @@
//
// ParseResult is the structured output contract for the Go parser
-// library. Port-rag-flow-pipeline-to-go.md §6.5 mandates that
-// parsers surface enough data to reconstruct a Python-compatible
-// stage-boundary payload:
+// library. Parsers surface a Python-compatible stage-boundary payload:
//
-// output_format
// file (enriched metadata)
-// backend payload to be normalized by the owning component
+// JSON (structured backend output, typed as DocItems)
// err
+// warnings
//
-// Go parser callers now consume only the structured ParseResult
-// contract. The legacy `Parse(filename, []byte) error` interface has
-// been removed so parser dispatch, ingestion, and service paths all
-// share the same typed payload contract.
+// The owning Parser component hardcodes the wire `output_format` to "json"
+// for every family; that value is a property of the component boundary, not
+// of the parser result, so it is no longer carried on ParseResult. Backend
+// markdown responses are normalized to JSON inside the parser package before
+// they reach the component, so no raw markdown travels on this struct either.
+//
+// Parser callers consume the structured ParseResult payload. Use Produced to
+// check whether a result carries a usable payload: Err is nil and JSON or File
+// is non-nil.
package parser
@@ -38,14 +41,9 @@ import (
)
// ParseResult is the structured return value of a parse operation. JSON holds
-// structured backend output when available; the rendered fields carry
-// non-JSON backend responses to the Parser component's normalization boundary.
-// On failure, Err is non-nil and OutputFormat is empty.
+// structured backend output when available; File carries parser-enriched
+// metadata. On failure, Err is non-nil and the payload fields are zero values.
type ParseResult struct {
- // OutputFormat identifies the backend payload representation. The Parser
- // component normalizes successful results to JSON. Empty when Err is non-nil.
- OutputFormat string
-
// File is metadata produced by the parser backend (for example
// `outline` on the PDF path or `page_count` for paginated formats).
// Nil when the backend does not produce metadata. It is not an augmented
@@ -71,11 +69,6 @@ type ParseResult struct {
// and, after ingestion enhancement, its VLM description.
JSON []DocItem
- // Markdown is a backend response awaiting normalization (PDF markdown
- // engines only; the DSL hardcodes output_format to "json" for all other
- // families via normalizeParserOutputFormats).
- Markdown string
-
// Err is the failure reason. On non-nil Err, all payload
// fields are zero values.
Err error
@@ -84,6 +77,15 @@ type ParseResult struct {
Warnings []string
}
+// Produced reports whether the parse succeeded and yielded a payload the
+// Parser component can forward: structured JSON items and/or enriched File
+// metadata. It replaces the historical `OutputFormat != ""` success check. An
+// error result is never produced, and a result with neither JSON nor File is
+// treated as empty (the component falls back to raw-text indexing).
+func (r ParseResult) Produced() bool {
+ return r.Err == nil && (r.JSON != nil || r.File != nil)
+}
+
// ParseResultProducer is the parser package's single structured-output
// contract. Every parser returned by GetParser must implement it.
type ParseResultProducer interface {
@@ -97,8 +99,6 @@ const (
DocTypeText = "text"
DocTypeTable = "table"
DocTypeImage = "image"
-
- spreadsheetOutputFormat = "json"
)
// NewTextJSONItem constructs a canonical text DocItem for parser output.
diff --git a/internal/parser/parser/parse_result_produced_test.go b/internal/parser/parser/parse_result_produced_test.go
new file mode 100644
index 0000000000..5c2349322e
--- /dev/null
+++ b/internal/parser/parser/parse_result_produced_test.go
@@ -0,0 +1,36 @@
+package parser
+
+import (
+ "errors"
+ "testing"
+)
+
+// TestParseResultProduced pins the success/failure boundary that replaced the
+// old produced != "" check. A result is "produced" only when it carries a
+// structured payload (JSON items or enriched File metadata) and has no error.
+func TestParseResultProduced(t *testing.T) {
+ someErr := errors.New("boom")
+
+ // Error set => never produced, even when a payload is present.
+ if (ParseResult{Err: someErr}).Produced() {
+ t.Error("result with Err must not be Produced")
+ }
+ if (ParseResult{Err: someErr, JSON: []DocItem{NewTextJSONItem("x")}}).Produced() {
+ t.Error("result with Err and JSON must not be Produced")
+ }
+
+ // Success with JSON items.
+ if !(ParseResult{JSON: []DocItem{NewTextJSONItem("x")}}).Produced() {
+ t.Error("success with JSON must be Produced")
+ }
+
+ // Success with enriched File but no JSON (e.g. emptyPDFResult / metadata-only).
+ if !(ParseResult{File: map[string]any{"name": "x"}}).Produced() {
+ t.Error("success with File must be Produced")
+ }
+
+ // Success with neither JSON nor File => raw-text fallback path.
+ if (ParseResult{}).Produced() {
+ t.Error("empty success must not be Produced (raw fallback expected)")
+ }
+}
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index 96f920a9eb..7444ac6cbf 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -47,79 +47,48 @@ func TestNewTableDataJSONItem_IncludesDocType(t *testing.T) {
}
}
-// TestParseResult_Contract pins the wire-shape guarantees
-// port-rag-flow-pipeline-to-go.md §6.5 requires:
+// TestParseResult_Contract pins the public-key invariants the component-level
+// caller (component/parser.go) relies on after the produced/Markdown fields
+// were removed:
//
-// - exactly one payload family populated on success
-// - empty payload fields on failure (Err != nil)
-// - OutputFormat is "" on failure
-//
-// These are the public-key invariants a component-level caller
-// (component/parser.go) relies on. The test does not exercise any
-// specific parser implementation — it pins the contract that the
-// ParseResult type itself enforces, so future parser additions
-// inherit the same guarantees.
+// - a successful result carries a populated payload (JSON and/or File)
+// - a failed result has Err set and no payload
+// - Produced() is the single success/failure boundary
func TestParseResult_Contract(t *testing.T) {
cases := []struct {
- name string
- in ParseResult
- wantErr bool
- wantFmt string // expected OutputFormat on success
+ name string
+ in ParseResult
+ wantErr bool
+ wantProduced bool
}{
{
- name: "json family only",
- in: ParseResult{
- OutputFormat: "json",
- JSON: MapsToItems([]map[string]any{
- {"text": "hello", "doc_type_kwd": "text"},
- }),
- },
- wantFmt: "json",
+ name: "json payload",
+ in: ParseResult{JSON: MapsToItems([]map[string]any{{"text": "hello", "doc_type_kwd": "text"}})},
+ wantProduced: true,
},
{
- name: "markdown family only",
- in: ParseResult{
- OutputFormat: "markdown",
- Markdown: "# Title\n\nbody",
- },
- wantFmt: "markdown",
+ name: "file metadata only",
+ in: ParseResult{File: map[string]any{"name": "x"}},
+ wantProduced: true,
},
{
- name: "failure clears payload",
+ name: "failure carries no payload",
in: ParseResult{Err: errSentinel},
wantErr: true,
},
+ {
+ name: "empty success carries no payload",
+ in: ParseResult{},
+ wantProduced: false,
+ },
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- if tc.wantErr {
- if tc.in.Err == nil {
- t.Fatal("Err: want non-nil, got nil")
- }
- if tc.in.OutputFormat != "" {
- t.Errorf("OutputFormat on failure = %q, want empty", tc.in.OutputFormat)
- }
- if tc.in.JSON != nil || tc.in.Markdown != "" {
- t.Errorf("payload fields must be zero on failure; got %+v", tc.in)
- }
- return
- }
- if tc.in.Err != nil {
- t.Errorf("Err on success: want nil, got %v", tc.in.Err)
- }
- if tc.in.OutputFormat != tc.wantFmt {
- t.Errorf("OutputFormat = %q, want %q", tc.in.OutputFormat, tc.wantFmt)
+ if tc.wantErr && tc.in.Err == nil {
+ t.Errorf("Err: want non-nil, got nil")
}
- // Exactly-one-payload-family: pick the family declared
- // by OutputFormat; every other field must be zero.
- active := payloadFamily(tc.in)
- for _, other := range allPayloadFamilies {
- if other == active {
- continue
- }
- if !isPayloadZero(tc.in, other) {
- t.Errorf("OutputFormat=%s: secondary family %s should be zero", tc.in.OutputFormat, other)
- }
+ if tc.in.Produced() != tc.wantProduced {
+ t.Errorf("Produced() = %v, want %v (Err=%v)", tc.in.Produced(), tc.wantProduced, tc.in.Err)
}
})
}
@@ -136,7 +105,7 @@ func (s sentinelErr) Error() string { return string(s) }
// TestMarkdownParser_ParseWithResult pins the migration exemplar
// — port-rag-flow-pipeline-to-go.md §6.5 marks MarkdownParser as
// the first ported format whose ParseWithResult surface emulates
-// the Python "json" output_format. The fixture is intentionally
+// the Python "json" produced. The fixture is intentionally
// tiny: 1 heading, 1 paragraph, 1 unordered list item, no nested
// formatting.
func TestMarkdownParser_ParseWithResult(t *testing.T) {
@@ -147,8 +116,8 @@ func TestMarkdownParser_ParseWithResult(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Errorf("Produced() = false, want true (markdown parser emits JSON)")
}
if res.File == nil || res.File["name"] != "doc.md" {
t.Errorf("File.name = %v, want doc.md", res.File)
@@ -185,28 +154,3 @@ func TestParseResultProducer_PDFIsProducer(t *testing.T) {
"dispatch seam routes PDFs through ParseWithResult")
}
}
-
-// payloadFamily returns the field name that should be populated
-// given the declared OutputFormat. "" when OutputFormat is
-// unrecognized.
-func payloadFamily(r ParseResult) string {
- switch r.OutputFormat {
- case "json":
- return "json"
- case "markdown":
- return "markdown"
- }
- return ""
-}
-
-var allPayloadFamilies = []string{"json", "markdown"}
-
-func isPayloadZero(r ParseResult, family string) bool {
- switch family {
- case "json":
- return r.JSON == nil
- case "markdown":
- return r.Markdown == ""
- }
- return false
-}
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index 74b88cce62..4dcdea6760 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -52,8 +52,8 @@ func TestTextParser_ParseWithResult_PreservesOneTextUnit(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want true", res.Produced())
}
if got, want := res.File["name"], "doc.txt"; got != want {
t.Fatalf("File.name = %v, want %v", got, want)
@@ -148,8 +148,8 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Errorf("produced = %t, want true", res.Produced())
}
if len(res.JSON) != 3 {
t.Fatalf("JSON len = %d, want 3 (h1, p, ul)", len(res.JSON))
diff --git a/internal/parser/parser/pdf_markdown_to_result_test.go b/internal/parser/parser/pdf_markdown_to_result_test.go
new file mode 100644
index 0000000000..3acc607605
--- /dev/null
+++ b/internal/parser/parser/pdf_markdown_to_result_test.go
@@ -0,0 +1,69 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package parser
+
+import (
+ "context"
+ "testing"
+)
+
+// TestPDFMarkdownToResultMatchesNormalizeMarkdown pins that the two PDF
+// markdown entry points — pdfMarkdownToResult (remote-backed producers:
+// mineru/docling/opendataloader) and (*PDFParser).NormalizeMarkdown — produce
+// the same structured JSON for identical markdown. They must share one
+// conversion + item-policy path so a markdown producer can never silently
+// diverge from the local Markdown path. File metadata (page_count) is not
+// compared because only the remote producers carry it.
+func TestPDFMarkdownToResultMatchesNormalizeMarkdown(t *testing.T) {
+ cases := []struct {
+ name string
+ markdown string
+ }{
+ {"plain", "# Title\n\nSome body text.\n"},
+ {"table", "# T\n\n| A | B |\n| --- | --- |\n| 1 | 2 |\n\nbody\n"},
+ {"remote_image_ignored", "![img](https://example.com/x.png)\n"},
+ {"data_image", "![img](data:image/png;base64,abc)\n"},
+ {"empty", ""},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ remote := pdfMarkdownToResult(context.Background(), "doc.md", tc.markdown, 7)
+ local := NewPDFParser().NormalizeMarkdown(context.Background(), "doc.md", tc.markdown)
+ if remote.Err != nil {
+ t.Fatalf("pdfMarkdownToResult err: %v", remote.Err)
+ }
+ if local.Err != nil {
+ t.Fatalf("NormalizeMarkdown err: %v", local.Err)
+ }
+ if len(remote.JSON) != len(local.JSON) {
+ t.Fatalf("item count mismatch: remote %d vs local %d", len(remote.JSON), len(local.JSON))
+ }
+ for i := range remote.JSON {
+ ra, la := remote.JSON[i].ToMap(), local.JSON[i].ToMap()
+ if ra["doc_type_kwd"] != la["doc_type_kwd"] {
+ t.Errorf("item %d doc_type_kwd: remote %v vs local %v", i, ra["doc_type_kwd"], la["doc_type_kwd"])
+ }
+ if ra["text"] != la["text"] {
+ t.Errorf("item %d text: remote %q vs local %q", i, ra["text"], la["text"])
+ }
+ if (ra["table"] != nil) != (la["table"] != nil) {
+ t.Errorf("item %d table presence mismatch: remote %v vs local %v", i, ra["table"] != nil, la["table"] != nil)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/parser/parser/pdf_parser_cgo_test.go b/internal/parser/parser/pdf_parser_cgo_test.go
index b0e3b9a475..31cc801e55 100644
--- a/internal/parser/parser/pdf_parser_cgo_test.go
+++ b/internal/parser/parser/pdf_parser_cgo_test.go
@@ -24,8 +24,8 @@ func TestPDFParser_ParseWithResult_CGOEmpty(t *testing.T) {
if res.Err != nil {
t.Fatalf("empty input: want nil err, got %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want true", res.Produced())
}
if got, want := res.File["name"], "empty.pdf"; got != want {
t.Fatalf("File.name = %v, want %v", got, want)
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index bc7b775582..59d0fbe68d 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -363,7 +363,7 @@ func (p *PDFParser) validateParseMethod() error {
// ErrPDFEngineUnavailable while plain_text falls back to the pure-Go engine.
func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
result := p.parseWithResult(ctx, filename, data)
- if result.Err == nil && result.OutputFormat == "json" {
+ if result.Err == nil {
return p.NormalizeResult(ctx, filename, result)
}
return result
@@ -418,7 +418,6 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [
func emptyPDFResult(filename string) ParseResult {
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"page_count": 0,
@@ -584,7 +583,6 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
}
}
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"page_count": len(processed.PageHeight),
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index 66cf76e611..22f6c0dfd9 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -53,8 +53,8 @@ func TestPDFParseResultToJSON_NormalizesCoreFields(t *testing.T) {
if res.Err != nil {
t.Fatalf("pdfParseResultToJSON: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Fatalf("produced = %t, want true", res.Produced())
}
if got, want := res.File["name"], "sample.pdf"; got != want {
t.Fatalf("File.name = %v, want %v", got, want)
diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.go
index f320e83234..ab9d414111 100644
--- a/internal/parser/parser/pdf_parser_docling.go
+++ b/internal/parser/parser/pdf_parser_docling.go
@@ -256,8 +256,7 @@ func doclingTextsToResult(filename string, texts []string, pageCount int) ParseR
})
}
return ParseResult{
- OutputFormat: "json",
- File: pdfFileMeta(filename, pageCount),
- JSON: MapsToItems(items),
+ File: pdfFileMeta(filename, pageCount),
+ JSON: MapsToItems(items),
}
}
diff --git a/internal/parser/parser/pdf_parser_docling_test.go b/internal/parser/parser/pdf_parser_docling_test.go
index 130fcd6925..dac14b7b93 100644
--- a/internal/parser/parser/pdf_parser_docling_test.go
+++ b/internal/parser/parser/pdf_parser_docling_test.go
@@ -76,8 +76,8 @@ func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
var got strings.Builder
for _, item := range ItemsToMaps(res.JSON) {
@@ -145,8 +145,8 @@ func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *t
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want markdown-normalized items")
diff --git a/internal/parser/parser/pdf_parser_fixture_cgo_test.go b/internal/parser/parser/pdf_parser_fixture_cgo_test.go
index 40f273671a..f6f532dc45 100644
--- a/internal/parser/parser/pdf_parser_fixture_cgo_test.go
+++ b/internal/parser/parser/pdf_parser_fixture_cgo_test.go
@@ -41,8 +41,8 @@ func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want at least 1 parsed item")
@@ -70,8 +70,8 @@ func TestPDFParser_ParseWithResult_CGOFixturePlainText(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want page text items")
diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.go
index 66f6ae03f7..69fcbdb1b2 100644
--- a/internal/parser/parser/pdf_parser_mineru_test.go
+++ b/internal/parser/parser/pdf_parser_mineru_test.go
@@ -89,8 +89,8 @@ func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {
if !submitCalled.Load() || !resultCalled.Load() {
t.Fatalf("submit/result called = %v/%v, want true/true", submitCalled.Load(), resultCalled.Load())
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
var text strings.Builder
for _, item := range ItemsToMaps(res.JSON) {
diff --git a/internal/parser/parser/pdf_parser_nocgo_test.go b/internal/parser/parser/pdf_parser_nocgo_test.go
index 9e0350849f..c8dd12ec95 100644
--- a/internal/parser/parser/pdf_parser_nocgo_test.go
+++ b/internal/parser/parser/pdf_parser_nocgo_test.go
@@ -14,8 +14,8 @@ func TestPDFParser_ParseWithResult_NoCGO(t *testing.T) {
if empty.Err != nil {
t.Fatalf("empty input: want nil err, got %v", empty.Err)
}
- if empty.OutputFormat != "json" {
- t.Fatalf("empty input OutputFormat = %q, want json", empty.OutputFormat)
+ if !empty.Produced() {
+ t.Fatalf("empty input produced = %t, want true", empty.Produced())
}
if len(empty.JSON) != 1 {
t.Fatalf("empty input JSON len = %d, want 1", len(empty.JSON))
diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.go
index 5888c876df..481f57906c 100644
--- a/internal/parser/parser/pdf_parser_opendataloader_test.go
+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go
@@ -77,8 +77,8 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderJSONIntegration(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) < 2 {
t.Fatalf("JSON len = %d, want >=2", len(res.JSON))
diff --git a/internal/parser/parser/pdf_parser_paddleocr.go b/internal/parser/parser/pdf_parser_paddleocr.go
index c9d7821527..c169f471db 100644
--- a/internal/parser/parser/pdf_parser_paddleocr.go
+++ b/internal/parser/parser/pdf_parser_paddleocr.go
@@ -36,7 +36,7 @@ func PaddleOCRPDFResult(ctx context.Context, filename string, data []byte, respo
if err := ctx.Err(); err != nil {
return ParseResult{Err: err}
}
- result := ParseResult{OutputFormat: "json", Warnings: append([]string(nil), response.Warnings...)}
+ result := ParseResult{Warnings: append([]string(nil), response.Warnings...)}
hasPageCount := pageCount > 0
if geometryErr != nil {
result.Warnings = append(result.Warnings, "PaddleOCR PDF geometry unavailable: "+geometryErr.Error())
diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.go
index e39189e610..5b39ffadac 100644
--- a/internal/parser/parser/pdf_parser_remote_common.go
+++ b/internal/parser/parser/pdf_parser_remote_common.go
@@ -22,9 +22,8 @@ func pdfItemsToResult(filename string, items []map[string]any, pageCount int) Pa
}
pageCount = normalizePDFPageCount(pageCount, items)
return ParseResult{
- OutputFormat: "json",
- File: pdfFileMeta(filename, pageCount),
- JSON: MapsToItems(items),
+ File: pdfFileMeta(filename, pageCount),
+ JSON: MapsToItems(items),
}
}
@@ -108,22 +107,5 @@ func rowsHaveText(rows [][]string) bool {
}
func pdfMarkdownToResult(ctx context.Context, filename, markdown string, pageCount int) ParseResult {
- fileMeta := pdfFileMeta(filename, pageCount)
- res := parsePDFMarkdown(ctx, filename, markdown)
- if res.Err != nil {
- if ctx.Err() != nil {
- res.File = fileMeta
- return res
- }
- // Same text fallback as NormalizeResult: keep the backend
- // markdown as raw text instead of failing the task.
- return ParseResult{
- OutputFormat: "json",
- File: fileMeta,
- Warnings: append(res.Warnings, "markdown normalization fell back to text: "+res.Err.Error()),
- JSON: []DocItem{NewTextJSONItem(markdown)},
- }
- }
- res.File = fileMeta
- return res
+ return (&PDFParser{}).normalizePDFMarkdown(ctx, filename, markdown, pdfFileMeta(filename, pageCount))
}
diff --git a/internal/parser/parser/pdf_parser_tcadp_test.go b/internal/parser/parser/pdf_parser_tcadp_test.go
index 94a042a34c..006792d25a 100644
--- a/internal/parser/parser/pdf_parser_tcadp_test.go
+++ b/internal/parser/parser/pdf_parser_tcadp_test.go
@@ -221,7 +221,7 @@ func TestTCADPItemsKeepPageIdentityThroughNormalization(t *testing.T) {
if len(items) == 0 {
t.Fatalf("tcadpAnyToItems(%v) returned no items", raw)
}
- result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems(items)})
if result.Err != nil {
t.Fatal(result.Err)
}
diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.go
index 5b91414fc0..3e8f740272 100644
--- a/internal/parser/parser/pdf_result.go
+++ b/internal/parser/parser/pdf_result.go
@@ -27,36 +27,6 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result
result.File = ensurePDFResultFile(result.File, filename)
return result
}
- if len(result.JSON) == 0 {
- switch {
- case result.Markdown != "":
- parsed := parsePDFMarkdown(ctx, filename, result.Markdown)
- if parsed.Err != nil {
- if ctx.Err() != nil {
- // Cancellation is fatal; preserve file metadata for callers.
- parsed.File = ensurePDFResultFile(result.File, filename)
- return parsed
- }
- // Mirror the historical markdown→JSON fallback: a backend
- // markdown payload that cannot be structured is still indexed
- // as raw text instead of failing the task.
- common.Warn("parser normalization fell back to text",
- zap.String("filename", filename),
- zap.String("normalized_from", "markdown"),
- zap.Error(parsed.Err))
- result.Warnings = append(result.Warnings, parsed.Warnings...)
- result.Warnings = append(result.Warnings, "markdown normalization fell back to text: "+parsed.Err.Error())
- result.JSON = []DocItem{NewTextJSONItem(result.Markdown)}
- } else {
- result.Warnings = append(result.Warnings, parsed.Warnings...)
- if len(parsed.JSON) == 0 {
- result.JSON = []DocItem{NewTextJSONItem(result.Markdown)}
- } else {
- result.JSON = parsed.JSON
- }
- }
- }
- }
items := make([]DocItem, 0, len(result.JSON))
droppedPositions := 0
for _, item := range result.JSON {
@@ -137,11 +107,61 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result
zap.Int("dropped_rows", droppedPositions))
}
result.JSON = items
- result.OutputFormat, result.Markdown = "json", ""
result.File = ensurePDFResultFile(result.File, filename)
return result
}
+// NormalizeMarkdown converts a PDF backend Markdown response into the
+// structured JSON contract and applies the same item policies as
+// NormalizeResult. PDF markdown engines (MonkeyOCRv2, MinerU V1 fallback)
+// emit Markdown rather than JSON; the owning component routes them through
+// this method before vision enhancement so markdown→JSON conversion stays
+// entirely inside the parser package. On parse failure (or a result that
+// yields no structured items) the raw Markdown is indexed as a single text
+// item, mirroring the historical markdown→JSON fallback. A cancelled context
+// is fatal and preserves the file metadata for callers.
+func (p *PDFParser) NormalizeMarkdown(ctx context.Context, filename, markdown string) ParseResult {
+ return p.normalizePDFMarkdown(ctx, filename, markdown, ensurePDFResultFile(nil, filename))
+}
+
+// normalizePDFMarkdown converts a PDF backend Markdown response into the
+// structured JSON contract and applies the same item policies as
+// NormalizeResult. fileMeta seeds the result File: remote-backed producers
+// (mineru/docling/opendataloader) carry page_count, while the local Markdown
+// path only needs the filename. The cancellation and empty/error fallback
+// paths preserve fileMeta so callers can still report which document failed.
+// NormalizeMarkdown and pdfMarkdownToResult share this implementation so the
+// two markdown entry points can never diverge in conversion or item policy.
+func (p *PDFParser) normalizePDFMarkdown(ctx context.Context, filename, markdown string, fileMeta map[string]any) ParseResult {
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err, File: fileMeta}
+ }
+ converted := parsePDFMarkdown(ctx, filename, markdown)
+ if converted.Err != nil {
+ if ctx.Err() != nil {
+ // Cancellation is fatal; preserve file metadata for callers.
+ converted.File = fileMeta
+ return converted
+ }
+ // Mirror the historical markdown→JSON fallback: a backend
+ // markdown payload that cannot be structured is still indexed
+ // as raw text instead of failing the task.
+ common.Warn("parser normalization fell back to text",
+ zap.String("filename", filename),
+ zap.String("normalized_from", "markdown"),
+ zap.Error(converted.Err))
+ converted.Warnings = append(converted.Warnings, "markdown normalization fell back to text: "+converted.Err.Error())
+ converted.JSON = []DocItem{NewTextJSONItem(markdown)}
+ converted.Err = nil
+ } else if len(converted.JSON) == 0 {
+ converted.JSON = []DocItem{NewTextJSONItem(markdown)}
+ }
+ if converted.File == nil {
+ converted.File = fileMeta
+ }
+ return p.NormalizeResult(ctx, filename, converted)
+}
+
// ensurePDFResultFile preserves backend file metadata while guaranteeing the
// filename is present, including on error paths where callers still need to
// report which document failed.
diff --git a/internal/parser/parser/pdf_result_normalize_markdown_test.go b/internal/parser/parser/pdf_result_normalize_markdown_test.go
new file mode 100644
index 0000000000..f2fec16302
--- /dev/null
+++ b/internal/parser/parser/pdf_result_normalize_markdown_test.go
@@ -0,0 +1,149 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package parser
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+// TestPDFParser_NormalizeMarkdown_ConvertsToJSON pins that a backend markdown
+// payload is converted into normalized JSON items with doc_type_kwd set.
+func TestPDFParser_NormalizeMarkdown_ConvertsToJSON(t *testing.T) {
+ p := NewPDFParser()
+ res := p.NormalizeMarkdown(context.Background(), "doc.md", "# Title\n\nSome body text.\n")
+ if res.Err != nil {
+ t.Fatalf("err: %v", res.Err)
+ }
+ if !res.Produced() {
+ t.Fatal("NormalizeMarkdown did not produce a payload")
+ }
+ if len(res.JSON) == 0 {
+ t.Fatal("empty JSON after markdown normalization")
+ }
+ for _, it := range res.JSON {
+ if _, ok := it.ToMap()["doc_type_kwd"]; !ok {
+ t.Errorf("item missing doc_type_kwd: %#v", it.ToMap())
+ }
+ }
+}
+
+// TestPDFParser_NormalizeMarkdown_DoesNotFetchRemoteImages pins that backend
+// markdown normalization never pulls remote image URLs (untrusted OCR response,
+// deterministic local normalization only). ssrfAllowLoopback is enabled so the
+// SSRF guard does not reject the loopback request before the handler runs; that
+// would make requests stay 0 regardless and mask a regression where NormalizeMarkdown
+// started fetching remote images.
+func TestPDFParser_NormalizeMarkdown_DoesNotFetchRemoteImages(t *testing.T) {
+ prev := ssrfAllowLoopback
+ ssrfAllowLoopback = true
+ t.Cleanup(func() { ssrfAllowLoopback = prev })
+
+ var requests int
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests++
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer srv.Close()
+
+ p := NewPDFParser()
+ res := p.NormalizeMarkdown(context.Background(), "doc.md", "![remote]("+srv.URL+"/img.png)\n")
+ if res.Err != nil {
+ t.Fatalf("err: %v", res.Err)
+ }
+ if requests != 0 {
+ t.Errorf("NormalizeMarkdown fetched %d remote image(s); want 0", requests)
+ }
+}
+
+// TestPDFParser_NormalizeMarkdown_FlattenMediaToText pins that, with flatten
+// enabled, image markdown is flattened to a text carrier (doc_type_kwd == text).
+func TestPDFParser_NormalizeMarkdown_FlattenMediaToText(t *testing.T) {
+ p := NewPDFParser()
+ p.FlattenMediaToText = true
+ res := p.NormalizeMarkdown(context.Background(), "doc.md", "![img](data:image/png;base64,abc)\n")
+ if res.Err != nil {
+ t.Fatalf("err: %v", res.Err)
+ }
+ for _, it := range res.JSON {
+ if dt, _ := it.ToMap()["doc_type_kwd"].(string); dt != "text" {
+ t.Errorf("doc_type_kwd = %q, want text", dt)
+ }
+ }
+}
+
+// TestPDFParser_NormalizeMarkdown_PreservesFileOnCancel pins that a cancelled
+// context returns the error (not silently swallowed) and still carries the
+// filename in File so callers can report which document failed.
+func TestPDFParser_NormalizeMarkdown_PreservesFileOnCancel(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ p := NewPDFParser()
+ res := p.NormalizeMarkdown(ctx, "doc.md", "# Title\n\nbody\n")
+ if res.Err == nil {
+ t.Fatal("expected cancellation error")
+ }
+ if _, ok := res.File["name"]; !ok {
+ t.Error("File.name not preserved on cancellation")
+ }
+}
+
+// TestPDFParser_NormalizeMarkdown_IdempotentAfterSecondNormalize pins that the
+// result of NormalizeMarkdown can be safely passed through NormalizeResult again
+// (the PDF vision dispatch path re-normalizes already-normalized JSON), with no
+// change to item shape.
+func TestPDFParser_NormalizeMarkdown_IdempotentAfterSecondNormalize(t *testing.T) {
+ p := NewPDFParser()
+ res1 := p.NormalizeMarkdown(context.Background(), "doc.md",
+ "# Title\n\n| A | B |\n| --- | --- |\n| 1 | 2 |\n\nbody\n")
+ if res1.Err != nil {
+ t.Fatalf("err: %v", res1.Err)
+ }
+ res2 := p.NormalizeResult(context.Background(), "doc.md", res1)
+ if res2.Err != nil {
+ t.Fatalf("second normalize err: %v", res2.Err)
+ }
+ if len(res1.JSON) != len(res2.JSON) {
+ t.Fatalf("item count changed: %d -> %d", len(res1.JSON), len(res2.JSON))
+ }
+ for i := range res1.JSON {
+ a, b := res1.JSON[i].ToMap(), res2.JSON[i].ToMap()
+ if a["doc_type_kwd"] != b["doc_type_kwd"] {
+ t.Errorf("doc_type_kwd mismatch at item %d: %v -> %v", i, a["doc_type_kwd"], b["doc_type_kwd"])
+ }
+ }
+}
+
+// TestPDFParser_NormalizeMarkdown_EmptyFallsBackToText pins the historical text
+// fallback: a backend markdown that yields no structured items is still indexed
+// as a single raw-text item rather than failing the task.
+func TestPDFParser_NormalizeMarkdown_EmptyFallsBackToText(t *testing.T) {
+ p := NewPDFParser()
+ res := p.NormalizeMarkdown(context.Background(), "ocr.pdf", "")
+ if res.Err != nil {
+ t.Fatalf("NormalizeMarkdown returned error: %v", res.Err)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("empty markdown must fall back to a single text item, got %d", len(res.JSON))
+ }
+ if text, _ := res.JSON[0].ToMap()["text"].(string); text != "" {
+ t.Errorf("fallback text item should carry the raw markdown, got %q", text)
+ }
+}
diff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.go
index 445e547e67..10b4a542d6 100644
--- a/internal/parser/parser/pdf_result_test.go
+++ b/internal/parser/parser/pdf_result_test.go
@@ -18,7 +18,7 @@ func TestPDFNormalizeResultPositionsAndMetadata(t *testing.T) {
file := map[string]any{"outline": []string{"Chapter"}, "page_count": 20, "parse_method": "selected-model"}
input := map[string]any{"text": "caption", "layout": "caption", "doc_type_kwd": "text", "page_number": 2,
"positions": []any{[]any{2.0, 10.0, 30.0, 20.0, 40.0}, []any{3.0, math.NaN(), 30.0, 20.0, 40.0}, []any{4, 0, 0, 0, 0}}}
- result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", File: file, Warnings: []string{"source warning"}, JSON: MapsToItems([]map[string]any{input, {"text": "", "page_number": 4, "positions": []float64{4, 0, 0, 0, 0}}})})
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{File: file, Warnings: []string{"source warning"}, JSON: MapsToItems([]map[string]any{input, {"text": "", "page_number": 4, "positions": []float64{4, 0, 0, 0, 0}}})})
if result.Err != nil {
t.Fatal(result.Err)
}
@@ -52,8 +52,8 @@ func TestPDFNormalizeResultMarkdownAndCancellation(t *testing.T) {
requests := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests++ }))
defer server.Close()
- result := NewPDFParser().NormalizeResult(t.Context(), "ocr.pdf", ParseResult{OutputFormat: "markdown", Markdown: "| A | B |\n| --- | --- |\n| one | two |\n\n![remote](" + server.URL + "/image.png)"})
- if result.Err != nil || result.OutputFormat != "json" || result.Markdown != "" || requests != 0 {
+ result := NewPDFParser().NormalizeMarkdown(t.Context(), "ocr.pdf", "| A | B |\n| --- | --- |\n| one | two |\n\n![remote]("+server.URL+"/image.png)")
+ if result.Err != nil || len(result.JSON) == 0 || requests != 0 {
t.Fatalf("result=%+v requests=%d", result, requests)
}
var table *entity.TableData
@@ -67,7 +67,7 @@ func TestPDFNormalizeResultMarkdownAndCancellation(t *testing.T) {
}
ctx, cancel := context.WithCancel(t.Context())
cancel()
- result = NewPDFParser().NormalizeResult(ctx, "ocr.pdf", ParseResult{OutputFormat: "markdown", Markdown: "body"})
+ result = NewPDFParser().NormalizeMarkdown(ctx, "ocr.pdf", "body")
if !errors.Is(result.Err, context.Canceled) {
t.Fatalf("cancellation = %+v", result)
}
@@ -95,7 +95,7 @@ func TestPDFMarkdownBackendDoesNotFetchRemoteImages(t *testing.T) {
func TestPDFNormalizeResultKeepsTCADPPageWithoutCrop(t *testing.T) {
items := tcadpAnyToItems(map[string]any{"type": "text", "content": "page text", "page_num": 7})
- result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems(items)})
if result.Err != nil {
t.Fatal(result.Err)
}
@@ -115,7 +115,7 @@ func TestPDFNormalizeResultKeepsTCADPPageWithoutCrop(t *testing.T) {
func TestPDFNormalizeResultInfersStructuredTable(t *testing.T) {
table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}}
- result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{"table": table, "page_number": 3}})})
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{"table": table, "page_number": 3}})})
item := result.JSON[0].ToMap()
if item["doc_type_kwd"] != "table" {
t.Fatalf("table classified incorrectly: %#v", item)
@@ -129,7 +129,7 @@ func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testi
p := NewPDFParser()
p.FlattenMediaToText = true
table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}, Caption: "Caption"}
- result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: []DocItem{
+ result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: []DocItem{
TableItem{Table: table, Extra: map[string]any{"text": "Footnote"}},
ImageItem{Text: "Figure", Image: "image-data", Positions: Positions{AbsoluteBox{PageNumber: 2, Left: 1, Right: 5, Top: 2, Bottom: 6}}},
TextItem{Text: "Title", Extra: map[string]any{"layout": "heading"}},
@@ -161,7 +161,7 @@ func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testi
}
func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {
- result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{
"doc_type_kwd": "table", "layout": "table", "text": "Recognized text without cells",
}})})
if result.Err != nil {
@@ -177,7 +177,7 @@ func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {
for _, flatten := range []bool{false, true} {
p := NewPDFParser()
p.FlattenMediaToText = flatten
- result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
+ result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{
"doc_type_kwd": "table", "text": "Recognized table without cells",
"positions": [][]any{{2, 10, 30, 20, 40}},
}})})
@@ -203,7 +203,7 @@ func TestPDFNormalizeResultRecoversUnclosedTableCells(t *testing.T) {
"<table><tr><td>Cell value</td></tr>",
"<table><tr><td>Cell value",
} {
- result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{{
"doc_type_kwd": "table", "text": markup,
}})})
if result.Err != nil || len(result.JSON) != 1 {
@@ -284,7 +284,7 @@ func TestPDFWhitespaceTableDescriptionCannotHideCells(t *testing.T) {
item := map[string]any{"table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, "doc_type_kwd": "table", "text": " \n "}
results := []ParseResult{
NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, 1)),
- NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{item})}),
+ NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems([]map[string]any{item})}),
}
for _, result := range results {
if result.Err != nil || len(result.JSON) != 1 {
@@ -401,7 +401,7 @@ func TestPDFNormalizeResultKeepsTCADPPageOnlyPosition(t *testing.T) {
if len(items) != 1 {
t.Fatalf("items = %#v", items)
}
- result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{JSON: MapsToItems(items)})
if result.Err != nil {
t.Fatal(result.Err)
}
@@ -483,9 +483,9 @@ func TestPDFNormalizeResultCancellationPreservesFile(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
result := NewPDFParser().NormalizeResult(ctx, "cancel.pdf", ParseResult{
- OutputFormat: "json",
- File: map[string]any{"parse_method": "mineru"},
- JSON: MapsToItems([]map[string]any{{"text": "hi", "doc_type_kwd": "text"}}),
+
+ File: map[string]any{"parse_method": "mineru"},
+ JSON: MapsToItems([]map[string]any{{"text": "hi", "doc_type_kwd": "text"}}),
})
if !errors.Is(result.Err, context.Canceled) {
t.Fatalf("err = %v", result.Err)
diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go
index 8948a0b771..162613cf13 100644
--- a/internal/parser/parser/picture_parser.go
+++ b/internal/parser/parser/picture_parser.go
@@ -88,7 +88,6 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
return ParseResult{Err: err}
}
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.go
index 8022158945..566a0a118f 100644
--- a/internal/parser/parser/picture_parser_test.go
+++ b/internal/parser/parser/picture_parser_test.go
@@ -54,8 +54,8 @@ func TestPictureParser_ParseWithResult_NilSetup(t *testing.T) {
if res.Err != nil {
t.Errorf("unexpected error for nil setup: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want json (image only allows json)", res.OutputFormat)
+ if !res.Produced() {
+ t.Errorf("produced = %t, want true (image only allows json)", res.Produced())
}
file, ok := res.File["doc_type_kwd"].(string)
if !ok || file != "image" {
@@ -73,8 +73,8 @@ func TestPictureParser_ParseWithResult_ValidExtensions(t *testing.T) {
if res.Err != nil {
t.Errorf("unexpected error for .%s: %v", ext, res.Err)
}
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q for .%s, want json (image only allows json)", res.OutputFormat, ext)
+ if !res.Produced() {
+ t.Errorf("produced = %t for .%s, want true (image only allows json)", res.Produced(), ext)
}
}
}
@@ -103,15 +103,15 @@ func TestPictureParser_ParseWithResult_VideoExtension(t *testing.T) {
}
}
-func TestPictureParser_ParseWithResult_OutputFormat(t *testing.T) {
+func TestPictureParser_ParseWithResult_produced(t *testing.T) {
ctx := t.Context()
p := NewPictureParser()
res := p.ParseWithResult(ctx, "photo.jpg", []byte{1, 2, 3})
if res.Err != nil {
t.Fatalf("unexpected error: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Errorf("produced = %t, want true", res.Produced())
}
}
diff --git a/internal/parser/parser/ppt_parser_cgo_test.go b/internal/parser/parser/ppt_parser_cgo_test.go
index bdf43ca3e1..10dd7585f4 100644
--- a/internal/parser/parser/ppt_parser_cgo_test.go
+++ b/internal/parser/parser/ppt_parser_cgo_test.go
@@ -39,8 +39,8 @@ func TestPPTXParser_ParseWithResult_CGO(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want %q", res.OutputFormat, "json")
+ if !res.Produced() {
+ t.Errorf("produced = %t, want true", res.Produced())
}
if got := res.File["format"]; got != "pptx" {
t.Errorf("File[format] = %v, want %q", got, "pptx")
@@ -253,8 +253,8 @@ func testPresentationTCADPIntegration(t *testing.T, p tcadpPresentationParser, w
if gotFileType != wantFileType {
t.Errorf("TCADP request file_type = %q, want %q", gotFileType, wantFileType)
}
- if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want json", res.OutputFormat)
+ if !res.Produced() {
+ t.Errorf("produced = %t, want true", res.Produced())
}
if len(res.JSON) == 0 {
t.Fatalf("JSON items = 0, want the fixture's parsed content")
diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.go
index 6ebfaf41c4..b050d87e36 100644
--- a/internal/parser/parser/pptx_parser.go
+++ b/internal/parser/parser/pptx_parser.go
@@ -147,9 +147,8 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{Err: fmt.Errorf("presentation ir-json: empty result")}
}
return ParseResult{
- OutputFormat: "json",
- File: map[string]any{"name": filename, "format": effFormat},
- JSON: MapsToItems(itemsFromPlainText(plainText)),
+ File: map[string]any{"name": filename, "format": effFormat},
+ JSON: MapsToItems(itemsFromPlainText(plainText)),
}
}
budget := newEmbeddedMediaBudget()
@@ -173,9 +172,8 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
}
return ParseResult{
- OutputFormat: "json",
- File: map[string]any{"name": filename, "format": effFormat},
- JSON: MapsToItems(items),
- Warnings: budget.warnings(),
+ File: map[string]any{"name": filename, "format": effFormat},
+ JSON: MapsToItems(items),
+ Warnings: budget.warnings(),
}
}
diff --git a/internal/parser/parser/producer_audit_test.go b/internal/parser/parser/producer_audit_test.go
new file mode 100644
index 0000000000..71540b60c4
--- /dev/null
+++ b/internal/parser/parser/producer_audit_test.go
@@ -0,0 +1,52 @@
+package parser
+
+import (
+ "context"
+ "testing"
+)
+
+// TestAllPureGoProducersAreProduced audits that every pure-Go parser (the
+// families that do not require native cgo libraries) returns a Produced()
+// result on a successful parse. This guards the gate change in the Parser
+// component: buildParserOutputs now keys off Produced() instead of the removed
+// produced field, so a successful parser that neglected to set JSON/File
+// would silently fall through to the raw-text fallback.
+//
+// PDF and office families require cgo native libraries and are covered by the
+// native-tier tests; they are intentionally excluded here so the default unit
+// tier stays free of external dependencies.
+func TestAllPureGoProducersAreProduced(t *testing.T) {
+ type producer struct {
+ name string
+ p ParseResultProducer
+ file string
+ data []byte
+ }
+
+ producers := []producer{
+ {"text", NewTextParser(), "doc.txt", []byte("hello world")},
+ }
+
+ if mp, err := NewMarkdownParser(GoMarkdown); err == nil {
+ producers = append(producers, producer{"markdown", mp, "doc.md", []byte("# Title\n\nbody")})
+ }
+
+ producers = append(producers,
+ producer{"json", NewJSONParser(), "doc.json", []byte(`{"k":"v"}`)},
+ producer{"html", NewHTMLParser(), "doc.html", []byte("<p>hello</p>")},
+ producer{"csv", NewCSVParser(), "doc.csv", []byte("a,b\n1,2")},
+ )
+
+ for _, pr := range producers {
+ res := pr.p.ParseWithResult(context.Background(), pr.file, pr.data)
+ if res.Err != nil {
+ // A parser that cannot handle the sample input is out of scope; only
+ // audit the success path.
+ t.Logf("%s parser returned err on sample input (skipped): %v", pr.name, res.Err)
+ continue
+ }
+ if !res.Produced() {
+ t.Errorf("%s parser succeeded but result is not Produced (no JSON and no File)", pr.name)
+ }
+ }
+}
diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go
index e1f00eff0e..4cca42cc6e 100644
--- a/internal/parser/parser/text_parser.go
+++ b/internal/parser/parser/text_parser.go
@@ -47,7 +47,6 @@ func NewTextParser() *TextParser {
func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
decoded, encName := DecodeToUTF8(data, "text/plain")
return ParseResult{
- OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index 01e6a5ffaf..1e424ee517 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -108,10 +108,9 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
return ParseResult{
- OutputFormat: spreadsheetOutputFormat,
- File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},
- JSON: MapsToItems(items),
- Warnings: warnings,
+ File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},
+ JSON: MapsToItems(items),
+ Warnings: warnings,
}
}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index 5eb1b21919..a4c952f47e 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -160,8 +160,8 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if res.OutputFormat != spreadsheetOutputFormat {
- t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, spreadsheetOutputFormat)
+ if !res.Produced() {
+ t.Fatalf("res.produced = %t, want true", res.Produced())
}
if got := res.File["format"]; got != "xls" {
t.Fatalf("res.File[format] = %v, want xls", got)
@@ -218,8 +218,8 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
}
// Default output format for XLSParser is "json"
- if res.OutputFormat != "json" {
- t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ if !res.Produced() {
+ t.Errorf("res.produced = %t, want true", res.Produced())
}
if len(res.JSON) == 0 {
t.Fatalf("res.JSON should have structured table items")
@@ -241,8 +241,8 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
if resJSON.Err != nil {
t.Fatalf("ParseWithResult failed: %v", resJSON.Err)
}
- if resJSON.OutputFormat != "json" {
- t.Errorf("resJSON.OutputFormat = %q, want 'json'", resJSON.OutputFormat)
+ if !resJSON.Produced() {
+ t.Errorf("resJSON.produced = %t, want true", resJSON.Produced())
}
if len(resJSON.JSON) == 0 {
t.Errorf("resJSON.JSON should not be empty")
@@ -261,8 +261,8 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if res.OutputFormat != "json" {
- t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ if !res.Produced() {
+ t.Errorf("res.produced = %t, want true", res.Produced())
}
if len(res.JSON) != 1 {
t.Fatalf("res.JSON = %#v, want one segmented HTML table item", res.JSON)
diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go
index 6de76217e5..c14542d6d8 100644
--- a/internal/parser/parser/xls_tcadp_test.go
+++ b/internal/parser/parser/xls_tcadp_test.go
@@ -218,8 +218,8 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) != 1 {
t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.go
index 643728a757..3b037d389e 100644
--- a/internal/parser/parser/xlsx_parse_method_test.go
+++ b/internal/parser/parser/xlsx_parse_method_test.go
@@ -74,8 +74,8 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult(%s): %v", tc.method, res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) != 1 {
t.Fatalf("JSON item count = %d, want header-only result", len(res.JSON))
@@ -171,8 +171,8 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult(deepdoc): %v", res.Err)
}
- if got, want := res.OutputFormat, "json"; got != want {
- t.Fatalf("OutputFormat = %q, want %q", got, want)
+ if got, want := res.Produced(), true; got != want {
+ t.Fatalf("produced = %t, want %t", got, want)
}
if len(res.JSON) != 1 {
t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index ed397ff5c7..8e86ffd7d7 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -170,9 +170,8 @@ func parseXLSXBytes(data []byte, mediaBudget *embeddedMediaBudget) ([]map[string
func xlsxParseResult(filename string, items []map[string]any, warnings []string, sheets int) ParseResult {
return ParseResult{
- OutputFormat: spreadsheetOutputFormat,
- File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},
- JSON: MapsToItems(items),
- Warnings: warnings,
+ File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},
+ JSON: MapsToItems(items),
+ Warnings: warnings,
}
}
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 9bdf9ac38f..37bb7ebdcc 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -238,43 +238,28 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,
}
// parseResultText converts a parser result into the readable text expected by
-// sys.files. JSON results are flattened in item order, preferring each item's
-// text field and serializing items without one as a final fallback.
+// sys.files. Every parser result is structured JSON, so the items are flattened
+// in order, preferring each item's text field and rendering tables; items
+// without either are serialized as a final fallback.
func parseResultText(res parser.ParseResult) (string, error) {
- switch strings.ToLower(strings.TrimSpace(res.OutputFormat)) {
- case "markdown":
- return res.Markdown, nil
- case "json":
- if len(res.JSON) > 0 {
- parts := make([]string, 0, len(res.JSON))
- for _, item := range res.JSON {
- m := item.ToMap()
- if text, ok := m["text"].(string); ok {
- parts = append(parts, text)
- continue
- }
- if td, ok := m["table"].(*entity.TableData); ok && td != nil {
- parts = append(parts, tableutil.RenderTableHTML(td))
- continue
- }
- raw, err := json.Marshal(m)
- if err != nil {
- return "", err
- }
- parts = append(parts, string(raw))
- }
- return strings.Join(parts, "\n"), nil
+ parts := make([]string, 0, len(res.JSON))
+ for _, item := range res.JSON {
+ m := item.ToMap()
+ if text, ok := m["text"].(string); ok {
+ parts = append(parts, text)
+ continue
+ }
+ if td, ok := m["table"].(*entity.TableData); ok && td != nil {
+ parts = append(parts, tableutil.RenderTableHTML(td))
+ continue
}
- // Some legacy parsers mark the result as JSON while only populating a
- // rendered companion field. Preserve that content instead of returning
- // an empty sys.files value.
- if res.Markdown != "" {
- return res.Markdown, nil
+ raw, err := json.Marshal(m)
+ if err != nil {
+ return "", err
}
- return "", nil
- default:
- return "", fmt.Errorf("unsupported parser output format %q", res.OutputFormat)
+ parts = append(parts, string(raw))
}
+ return strings.Join(parts, "\n"), nil
}
// parseFileContent tries to parse a file's contents using the appropriate parser.
diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.go
index e72bf1d1dc..b445eabd8a 100644
--- a/internal/service/file/file_content_test.go
+++ b/internal/service/file/file_content_test.go
@@ -59,7 +59,6 @@ func TestParseFileContent_JSONItems(t *testing.T) {
func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T) {
result, err := parseResultText(parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": "first"},
{"text": "second"},
@@ -81,7 +80,6 @@ func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T)
func TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup(t *testing.T) {
segment := "<table><caption>Data</caption>\n<tr><th>a</th><th>b</th></tr>\n<tr><td>1</td><td>2</td></tr>\n</table>\n"
result, err := parseResultText(parser.ParseResult{
- OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
{"text": segment, "doc_type_kwd": "table", "sheet": "Data", "sheet_index": 1},
}),
@@ -113,21 +111,8 @@ func TestParseResultText_EmailJSONKeepsSearchableHeaders(t *testing.T) {
}
}
-func TestParseResultText_EmptyJSONUsesRenderedFallback(t *testing.T) {
- result, err := parseResultText(parser.ParseResult{
- OutputFormat: "json",
- Markdown: "legacy markdown",
- })
- if err != nil {
- t.Fatalf("parseResultText: %v", err)
- }
- if result != "legacy markdown" {
- t.Fatalf("parseResultText = %q, want rendered fallback", result)
- }
-}
-
func TestParseResultText_EmptyJSONWithoutFallbackReturnsEmpty(t *testing.T) {
- result, err := parseResultText(parser.ParseResult{OutputFormat: "json"})
+ result, err := parseResultText(parser.ParseResult{})
if err != nil {
t.Fatalf("parseResultText: %v", err)
}
diff --git a/tools/mdprobe/main.go b/tools/mdprobe/main.go
index e2cc6c3d3d..50b6c490c2 100644
--- a/tools/mdprobe/main.go
+++ b/tools/mdprobe/main.go
@@ -45,7 +45,7 @@ func main() {
p.ConfigureFromSetup(map[string]any{})
res := p.ParseWithResult(context.Background(), filepath.Base(path), data)
- fmt.Printf("file=%s bytes=%d output_format=%s items=%d\n", filepath.Base(path), len(data), res.OutputFormat, len(res.JSON))
+ fmt.Printf("file=%s bytes=%d produced=%t items=%d\n", filepath.Base(path), len(data), res.Produced(), len(res.JSON))
var total int
for i, item := range res.JSON {
m := item.ToMap()