Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): unify parser outputs to structured JSON items (#19467)

jay77721fbd374e92f39
diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.go
index f9d8d0621a..a59e2aeadb 100644
--- a/internal/ingestion/component/chunker/page.go
+++ b/internal/ingestion/component/chunker/page.go
@@ -87,10 +87,9 @@ func (c *PageChunkerComponent) invoke(_ context.Context, inputs map[string]any)
}, nil
}
- // The presentation template only configures pdf and slides
- // parser setups, matching Python's restriction to .pptx/.ppt/.pdf.
- // The upstream parser therefore always emits per-slide JSON, never
- // flat text/markdown/html, so every payload keeps one-chunk-per-slide.
+ // The Page chunker preserves one incoming JSON item per output chunk.
+ // It has no file-family restriction: item granularity is determined by
+ // the upstream parser backend.
items := slideItems(upstream.JSONResult, upstream.Chunks)
if len(items) == 0 {
return emptyOutputs(), nil
diff --git a/internal/ingestion/component/chunker/qa_id_test.go b/internal/ingestion/component/chunker/qa_id_test.go
index 22f728bc13..b175c3a87b 100644
--- a/internal/ingestion/component/chunker/qa_id_test.go
+++ b/internal/ingestion/component/chunker/qa_id_test.go
@@ -9,7 +9,7 @@ import (
)
// xlsxQATableHTML is the HTML the spreadsheet parser emits for a
-// headerless two-column Q&A workbook (renderSheetTables repeats the
+// headerless two-column Q&A workbook (the table renderer repeats the
// first row as the <th> header; extractQATable still reads it as a pair).
const xlsxQATableHTML = `<table><caption>Sheet1</caption>
<tr><th>门打不开怎么办</th><th>请家人回来开门</th></tr>
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index 93095b954f..4fe546146e 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -228,6 +228,41 @@ func TestTokenChunker_InvokeJSONPayload(t *testing.T) {
}
}
+func TestTokenChunker_InvokeJSONPayload_IndexesHeaderOnlyEmail(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": []string{"\n"},
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ // Email's structured body item can have no text for a header-only
+ // message. Parser emits the second text item specifically so headers are
+ // still indexed through the JSON path.
+ items := []map[string]any{
+ {"from": "[email protected]", "subject": "Status", "doc_type_kwd": "text"},
+ {"text": "from:[email protected]\nsubject:Status\n", "doc_type_kwd": "text"},
+ }
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "message.eml",
+ "output_format": "json",
+ "json": items,
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) == 0 {
+ t.Fatalf("chunks missing: %#v", out["chunks"])
+ }
+ for _, chunk := range chunks {
+ if text, _ := chunk["text"].(string); strings.Contains(text, "subject:Status") {
+ return
+ }
+ }
+ t.Fatalf("header item was not indexed: %#v", chunks)
+}
+
// TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone is the
// regression lock for #17889: when merging adjacent segments, only
// "text" segments may be merged; "table"/"image" (any non-text type)
@@ -281,6 +316,10 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {
if got != wantTypes[i] {
t.Errorf("chunk %d: doc_type_kwd = %q, want %q (full chunk: %+v)", i, got, wantTypes[i], ch)
}
+ ckType, _ := ch["ck_type"].(string)
+ if ckType != wantTypes[i] {
+ t.Errorf("chunk %d: ck_type = %q, want %q (derived from doc_type_kwd)", i, ckType, wantTypes[i])
+ }
}
// The two text segments on either side of the table/image must remain
// distinct — they must NOT be merged across the non-text segments.
@@ -357,6 +396,41 @@ func TestTokenChunker_InputsOutputs_NonEmpty(t *testing.T) {
}
}
+// TestTokenChunker_SpreadsheetTablePreservesSelectionRange guards the Parser
+// JSON path: spreadsheet positions describe a sheet selection, not a PDF
+// bounding box, and a table item must pass through without proportional text
+// splitting or coordinate rewriting.
+func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{"delimiter": "\n"})
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ wantPositions := [][]float64{{1, 2, 10, 1, 5}}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "book.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{{
+ "text": "<table><tr><td>A</td></tr>\n<tr><td>B</td></tr></table>",
+ "doc_type_kwd": "table",
+ "positions": wantPositions,
+ "sheet": "Sheet1",
+ }},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %d, want one unsplit table item", len(chunks))
+ }
+ if got := chunks[0]["ck_type"]; got != "table" {
+ t.Errorf("ck_type = %v, want table", got)
+ }
+ if got := chunks[0]["positions"]; !reflect.DeepEqual(got, wantPositions) {
+ t.Errorf("positions = %#v, want unchanged %#v", got, wantPositions)
+ }
+}
+
// TestTokenChunker_NewRejectsBadParam enforces the param validation
// at construction time (mirrors python `check()`).
func TestTokenChunker_NewRejectsBadParam(t *testing.T) {
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 08eeb38c74..8fdd9cd1d6 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -21,7 +21,7 @@
//
// These follow the maybeDispatchPDFVision pattern: they bypass
// dispatchParse and call the model directly from the component
-// layer, returning a parserDispatchResult.
+// layer, returning a parser.ParseResult.
package component
@@ -65,12 +65,12 @@ func maybeDispatchVideo(
binary []byte,
inputs map[string]any,
setups map[string]schema.ParserSetup,
-) (parserDispatchResult, bool, error) {
+) (parser.ParseResult, bool, error) {
if fileType != utility.FileTypeVIDEO {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
if _, ok := setups["video"]; !ok {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
// Video parsing is intentionally not implemented yet: the underlying
@@ -81,8 +81,10 @@ func maybeDispatchVideo(
// video_url. Returning an explicit error is safer than silently producing
// a description from the prompt text alone. The real implementation must
// be provider-specific (OpenAI-compatible: frame extraction -> image_url;
- // Gemini: raw-bytes inline_data; Qwen: file://)
- return parserDispatchResult{}, true,
+ // Gemini: raw-bytes inline_data; Qwen: file://).
+ // When video analysis is implemented, it emits standard Text JSON items:
+ // [{"text": transcript, "doc_type_kwd": "text"}] with output_format "json".
+ return parser.ParseResult{}, true,
fmt.Errorf("Parser: video parsing is not yet supported; underlying video analysis capability is pending")
}
@@ -102,17 +104,17 @@ func maybeDispatchImage(
binary []byte,
inputs map[string]any,
setups map[string]schema.ParserSetup,
-) (parserDispatchResult, bool, error) {
+) (parser.ParseResult, bool, error) {
if fileType != utility.FileTypeVISUAL {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
setup, ok := setups["image"]
if !ok {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
tenantID := getStringOr(inputs, "tenant_id", "")
if tenantID == "" {
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: image requires tenant_id")
}
@@ -181,7 +183,7 @@ func maybeDispatchImage(
if ocrText != "" {
return imageDispatchResult(ocrText, dataURI), true, nil
}
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: picture image2text model: %w", err)
}
@@ -205,7 +207,7 @@ func maybeDispatchImage(
if ocrText != "" {
return imageDispatchResult(ocrText, dataURI), true, nil
}
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: picture describe: %w", err)
}
vlmText := ""
@@ -230,10 +232,9 @@ func maybeDispatchImage(
// family: a single item carrying the combined text, the image attachment
// (data URI), and doc_type_kwd "image". Mirrors Python
// rag/app/picture.py:71-72.
-func imageDispatchResult(text, dataURI string) parserDispatchResult {
- return parserDispatchResult{
+func imageDispatchResult(text, dataURI string) parser.ParseResult {
+ return parser.ParseResult{
OutputFormat: "json",
- DocType: "image",
JSON: []map[string]any{{
"text": text,
"image": dataURI,
@@ -256,17 +257,17 @@ func maybeDispatchAudio(
binary []byte,
inputs map[string]any,
setups map[string]schema.ParserSetup,
-) (parserDispatchResult, bool, error) {
+) (parser.ParseResult, bool, error) {
if fileType != utility.FileTypeAURAL {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
setup, ok := setups["audio"]
if !ok {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
tenantID := getStringOr(inputs, "tenant_id", "")
if tenantID == "" {
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: audio requires tenant_id")
}
@@ -286,20 +287,20 @@ func maybeDispatchAudio(
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)
}
if err != nil {
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: audio speech2text model: %w", err)
}
tmpFile, err := writeTempAudioFile(filename, binary)
if err != nil {
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: audio temp file: %w", err)
}
defer os.Remove(tmpFile)
resp, err := driver.TranscribeAudio(ctx, &modelName, &tmpFile, apiConfig, nil, nil)
if err != nil {
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("Parser: audio transcription: %w", err)
}
@@ -308,29 +309,12 @@ func maybeDispatchAudio(
transcription = resp.Text
}
- outputFormat, _ := setup["output_format"].(string)
- if outputFormat == "" {
- outputFormat = "json"
- }
- // Diff 2.11: when output_format is "json" the transcription must be
- // carried as a JSON item. Returning it only in Text made the Invoke
- // switch silently drop it (the switch has no "json" branch and the
- // JSON slice was empty). Mirror the JSON-item shape used by the
- // other parser branches.
- if outputFormat == "json" {
- return parserDispatchResult{
- OutputFormat: "json",
- DocType: "audio",
- JSON: []map[string]any{{
- "text": transcription,
- "doc_type_kwd": "audio",
- }},
- }, true, nil
- }
- return parserDispatchResult{
- OutputFormat: outputFormat,
- DocType: "audio",
- Text: transcription,
+ return parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{{
+ "text": transcription,
+ "doc_type_kwd": "text",
+ }},
}, true, nil
}
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index 89cfe82488..f07e675ce7 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -318,15 +318,12 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {
if got, _ := res.JSON[0]["text"].(string); got != want {
t.Fatalf("JSON[0].text = %q, want %q", got, want)
}
- if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "audio" {
- t.Fatalf("JSON[0].doc_type_kwd = %q, want audio", got)
+ if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "text" {
+ t.Fatalf("JSON[0].doc_type_kwd = %q, want text", got)
}
}
-// TestMaybeDispatchAudio_TextCarriesTranscription guards the text path: with
-// output_format "text" the transcription stays in the Text field and JSON is
-// empty (current default after aligning with Python parser.py:232).
-func TestMaybeDispatchAudio_TextCarriesTranscription(t *testing.T) {
+func TestMaybeDispatchAudio_IgnoresLegacyOutputFormat(t *testing.T) {
origResolver := resolveTenantModelByType
defer func() { resolveTenantModelByType = origResolver }()
@@ -355,24 +352,20 @@ func TestMaybeDispatchAudio_TextCarriesTranscription(t *testing.T) {
if !dispatched {
t.Fatalf("expected dispatched=true for AURAL file")
}
- if res.OutputFormat != "text" {
- t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)
+ if res.OutputFormat != "json" {
+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
}
- if res.Text != want {
- t.Fatalf("Text = %q, want %q", res.Text, want)
+ if len(res.JSON) != 1 {
+ t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
- if len(res.JSON) != 0 {
- t.Fatalf("JSON len = %d, want 0 for text output", len(res.JSON))
+ if got, _ := res.JSON[0]["text"].(string); got != want {
+ t.Fatalf("JSON[0].text = %q, want %q", got, want)
}
}
// TestMaybeDispatchAudio_DefaultOutputFormatJson covers the
-// maybeDispatchAudio fallback: when an audio setup omits
-// output_format entirely, the dispatch defaults to "json" and wraps
-// the transcription as a JSON item. (The defaultSetups value is
-// "text" to mirror the Python audio setup in parser.py; this test
-// deliberately supplies an empty setup to exercise the fallback
-// inside the dispatch itself.)
+// maybeDispatchAudio always returns a JSON transcription item, including when
+// the setup has no output_format key.
func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
const want = "hello world"
drv := &audioTranscribeDriver{transcription: want}
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 946d1da930..d264ac9513 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -28,10 +28,8 @@
// - Per-page parallelism is delegated to the parser backends
// (e.g. internal/deepdoc/parser/pdf fans out one worker per
// page and assembles the results in page order). This
-// component only reshapes the parser output into the
-// schema.Page layout and keeps the deterministic, page-number
-// sorted merge contract (plan §8 R8) that the downstream
-// chunker / tokenizer rely on for stable chunk IDs.
+// component normalizes the parser output into structured JSON
+// items while preserving the backend's deterministic item order.
//
// - Progress (start/done callback) and elapsed-time stamping
// (_created_time / _elapsed_time) are owned by the canvas
@@ -74,15 +72,14 @@
// tenant default models (resolveTenantModelByType) rather than
// setup["vlm"]["llm_id"], so that check is intentionally omitted.
//
-// - NO PERSISTENCE: parsed pages live only in the per-run
-// output map, exactly as the schema.Page type is intended.
+// - NO PERSISTENCE: structured parser items live only in the per-run
+// output map.
package component
import (
"context"
"errors"
"fmt"
- "sort"
"strings"
"unicode/utf8"
@@ -93,6 +90,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
)
@@ -105,43 +103,41 @@ const ComponentNameParser = "Parser"
const pageFormFeed = '\f'
// ParserComponent runs the configured parser branch against the
-// upstream "binary" payload and returns a deterministic, page-
-// sorted slice of schema.Page values.
+// upstream "binary" payload and returns structured parser outputs.
//
// The instance is safe for concurrent invocation: each Invoke call
// builds its own per-batch goroutine tree and merges results in
// the goroutine that returned from Invoke. The static Param is
// read-only after construction.
type ParserComponent struct {
- Setups map[string]schema.ParserSetup
- Param schema.ParserParam
+ setups map[string]schema.ParserSetup
}
// NewParserComponent constructs a Parser from a DSL param map.
-// The map is decoded into schema.ParserParam.Defaults() and then
-// overlaid with the supplied values. This matches the Python
-// "default + override" pattern in parser.py:ParserParam.__init__.
+// The default setups are overlaid with the supplied values. Historical
+// output_format values are accepted but normalized to JSON so downstream
+// components consume one parser output protocol. This applies to every
+// family, including PDF and office documents: Markdown is an internal
+// backend representation only and is never a public Parser output.
//
-// Param map shape (all keys optional; missing keys fall back to
-// schema.ParserParam.Defaults() values):
+// Param map shape (all keys optional):
//
// {
// "pdf": map[string]any,
// "docx": map[string]any,
// ...
-// "allowed_output_format": map[string][]string,
// }
//
// Errors here surface as canvas compile failures so a malformed
// param is caught at build time rather than mid-run.
func NewParserComponent(params map[string]any) (runtime.Component, error) {
- p := schema.ParserParam{}.Defaults()
s := defaultSetups()
if params == nil {
- return &ParserComponent{Setups: s, Param: p}, nil
+ normalizeParserOutputFormats(s)
+ return &ParserComponent{setups: s}, nil
}
for k, raw := range params {
- if k == "outputs" {
+ if k == "outputs" || k == "allowed_output_format" {
continue
}
ftCfg, ok := raw.(map[string]any)
@@ -152,33 +148,61 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
s[k] = schema.ParserSetup{}
}
for fk, fv := range ftCfg {
- s[k][fk] = fv
+ s[k][fk] = cloneParserSetupValue(fv)
}
}
- if rawAllowed, ok := params["allowed_output_format"].(map[string]any); ok {
- allowed := make(map[string][]string, len(rawAllowed))
- for fileType, raw := range rawAllowed {
- list, ok := raw.([]any)
- if !ok {
- continue
- }
- formats := make([]string, 0, len(list))
- for _, item := range list {
- if s, ok := item.(string); ok {
- formats = append(formats, s)
- }
- }
- allowed[fileType] = formats
- }
- p.AllowedOutputFormat = allowed
- }
- pc := &ParserComponent{Setups: s, Param: p}
+ normalizeParserOutputFormats(s)
+ pc := &ParserComponent{setups: s}
if err := pc.Check(); err != nil {
return nil, fmt.Errorf("parser: %w", err)
}
return pc, nil
}
+func normalizeParserOutputFormats(setups map[string]schema.ParserSetup) {
+ // The Go Parser component intentionally exposes JSON only. Keep this
+ // normalization unconditional so PDF/office legacy Markdown settings
+ // cannot silently select a second public output path.
+ for _, setup := range setups {
+ setup["output_format"] = "json"
+ }
+}
+
+func cloneParserSetupValue(value any) any {
+ switch v := value.(type) {
+ case map[string]any:
+ cloned := make(map[string]any, len(v))
+ for key, nested := range v {
+ cloned[key] = cloneParserSetupValue(nested)
+ }
+ return cloned
+ case schema.ParserSetup:
+ cloned := make(schema.ParserSetup, len(v))
+ for key, nested := range v {
+ cloned[key] = cloneParserSetupValue(nested)
+ }
+ return cloned
+ case []any:
+ cloned := make([]any, len(v))
+ for i, nested := range v {
+ cloned[i] = cloneParserSetupValue(nested)
+ }
+ return cloned
+ case []string:
+ return append([]string(nil), v...)
+ case []int:
+ return append([]int(nil), v...)
+ case [][]int:
+ cloned := make([][]int, len(v))
+ for i, nested := range v {
+ cloned[i] = append([]int(nil), nested...)
+ }
+ return cloned
+ default:
+ return value
+ }
+}
+
// Check mirrors the applicable subset of Python ParserParam.check()
// (rag/flow/parser/parser.py:251-321). Runs at construction time so
// a malformed DSL surfaces as a canvas compile failure rather than a
@@ -186,8 +210,6 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
// (Python raises ValueError on the first failure).
//
// NOT covered here (intentional):
-// - output_format whitelist: already enforced at Invoke time by
-// resolveOutputFormat (parser_dispatch.go:100-122).
// - audio/video vlm.llm_id: Go media_dispatch uses tenant default
// models (resolveTenantModelByType), not setup["vlm"]["llm_id"].
// The Python flow check() for vlm.llm_id does not apply — Go
@@ -195,7 +217,7 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
// valid audio/video pipeline (see ingestion_pipeline_audio.json).
func (c *ParserComponent) Check() error {
// PDF family (parser.py:252-261).
- if pdf, ok := c.Setups["pdf"]; ok {
+ if pdf, ok := c.setups["pdf"]; ok {
pm, _ := pdf["parse_method"].(string)
if pm == "" {
return errors.New("parse method abnormal. does not support empty value")
@@ -214,7 +236,7 @@ func (c *ParserComponent) Check() error {
}
}
// image family (parser.py:283-287).
- if img, ok := c.Setups["image"]; ok {
+ if img, ok := c.setups["image"]; ok {
pm, _ := img["parse_method"].(string)
// OCR mode does not need a VLM language; any other value does.
if pm != "ocr" {
@@ -252,7 +274,7 @@ func defaultSetups() map[string]schema.ParserSetup {
"spreadsheet": {
"parse_method": "deepdoc",
"flatten_media_to_text": false,
- "output_format": "html",
+ "output_format": "json",
"suffix": []string{"xls", "xlsx", "csv"},
},
"doc": {
@@ -314,11 +336,11 @@ func defaultSetups() map[string]schema.ParserSetup {
"aiff", "au", "midi", "wma", "realaudio", "vqf",
"oggvorbis", "ape",
},
- "output_format": "text",
+ "output_format": "json",
},
"video": {
"suffix": []string{"mp4", "avi", "mkv"},
- "output_format": "text",
+ "output_format": "json",
"prompt": "",
},
"epub": {
@@ -336,39 +358,54 @@ func defaultSetups() map[string]schema.ParserSetup {
// reads the following from the inputs map at Invoke time:
//
// binary ([]byte, optional) — file bytes from upstream File.
-// When absent, Parser resolves
-// them from bucket/path or doc_id.
+// When absent, Parser resolves them from
+// bucket/path or doc_id.
+// name (string, optional) — resolved source filename.
+// file (map[string]any, optional) — source descriptor; its name is
+// used when name is absent.
+// file_type (string, optional) — explicit parser routing hint.
+// lang (string, optional) — language forwarded to downstream stages.
// doc_id (string, optional) — document ID used for naming and,
// when binary is absent, storage lookup.
func (c *ParserComponent) Inputs() map[string]string {
return map[string]string{
- "binary": "Optional file bytes ([]byte). When absent, Parser resolves them from bucket/path or doc_id.",
- "doc_id": "Optional document ID (string). Used for downstream correlation and doc_id-driven storage lookup.",
- "bucket": "Optional storage bucket override. Used when binary is absent.",
- "path": "Optional storage object key override. Used when binary is absent.",
+ "binary": "Optional file bytes ([]byte). When absent, Parser resolves them from bucket/path or doc_id.",
+ "name": "Optional resolved source filename. Takes precedence over file.name.",
+ "file": "Optional source file descriptor (map[string]any). file.name is used when name is absent.",
+ "file_type": "Optional explicit parser routing hint (string).",
+ "lang": "Optional language for downstream tokenization (string).",
+ "doc_id": "Optional document ID (string). Used for downstream correlation and doc_id-driven storage lookup.",
+ "bucket": "Optional storage bucket override. Used when binary is absent.",
+ "path": "Optional storage object key override. Used when binary is absent.",
}
}
// Outputs returns the public surface that downstream ingestion
// components (Chunker, Tokenizer, Extractor) can wire into.
//
-// pages []schema.Page — sorted by PageNumber. Deterministic
-// merge per plan §8 R8.
-// name string — carried over from the upstream file/document
+// name string — carried over from the upstream file/document
// name (or doc_id when no name is available).
-// output_format string — "text" when emitting text pages,
-// otherwise the parser-selected wire
-// format.
-// _ERROR string — populated when the component short-
-// circuits with an error message
-// (mirrors Python set_output("_ERROR", ...)).
+// output_format string — always "json".
+// json []map[string]any — canonical structured parser items.
+// lang string — language for tokenization.
+// file map[string]any — backend-produced file metadata, when present.
+// doc_id string — source document ID, when present.
+// bucket string — source storage bucket, when present.
+// path string — source storage path, when present.
+//
+// Parser failures are returned as Go errors. The canvas execution wrapper
+// preserves that error path and does not convert failures into an _ERROR
+// output field.
func (c *ParserComponent) Outputs() map[string]string {
return map[string]string{
- "pages": "[]schema.Page: parsed pages sorted by PageNumber.",
"name": "string: the upstream file/document name (or doc_id when no name is available).",
- "output_format": "string: the active output format (\"text\" when emitting text pages).",
+ "output_format": "string: always \"json\".",
+ "json": "[]map[string]any: canonical structured parser items.",
"lang": "string: the language for tokenization (e.g. English, Dutch, Chinese).",
- "_ERROR": "string: set on short-circuit errors.",
+ "file": "map[string]any: backend-produced file metadata, when present.",
+ "doc_id": "string: source document ID, when present.",
+ "bucket": "string: source storage bucket, when present.",
+ "path": "string: source storage object path, when present.",
}
}
@@ -377,9 +414,9 @@ func (c *ParserComponent) Outputs() map[string]string {
// Returns:
//
// {
-// "pages": []schema.Page (sorted by PageNumber),
-// "name": string (from inputs["doc_id"]),
-// "output_format": "text",
+// "name": string (from inputs["name"], file.name, or doc_id),
+// "output_format": "json",
+// "json": []map[string]any,
// "lang": string (from inputs["lang"]; e.g. English, Dutch),
// "_created_time": RFC3339Nano (via TrackElapsed),
// "_elapsed_time": float64 seconds (via TrackElapsed),
@@ -389,13 +426,6 @@ func (c *ParserComponent) Outputs() map[string]string {
// backends (e.g. internal/deepdoc/parser/pdf fans out one worker
// per page and assembles the results in page order), so this
// component does no goroutine fan-out of its own.
-//
-// DETERMINISTIC MERGE (plan §8 R8): after the page slice is built,
-// it is sorted by PageNumber. This guarantees the same input
-// produces byte-identical output across runs and is the contract
-// that downstream Chunker / Tokenizer rely on for stable chunk
-// IDs (chunks that span pages must reference adjacent PageNumbers
-// in input order).
func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
// 1. Decode the binary input.
binary, err := readParserBinary(ctx, db, inputs)
@@ -404,6 +434,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
docID, _ := inputs["doc_id"].(string)
filename := parserInputName(inputs, docID)
+ setups := c.setups
// Inject run-level metadata from Globals into inputs so media
// dispatch branches (audio/image/video) can resolve tenant_id.
@@ -424,29 +455,9 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
// "docx", ...). Used by parser.GetParser, whose switch
// arms are keyed off the utility constants.
//
- // - fileTypeFam — the python-side family name ("markdown",
- // "docx", ...). Used by setups[fileType] and
- // allowed_output_format[fileType] lookups, which are keyed
- // off the python family identifiers in schema.ParserParam.
- //
- // For most families the two forms coincide; the divergence
- // exists for Markdown ("md" vs "markdown") and slides
- // ("ppt"/"pptx" vs "slides") and is intentional — the python
- // ParserParam collapses the slide family into a single key.
fileTypeExt := fileTypeFromInputs(inputs)
- fileTypeFam := pythonFamilyName(string(fileTypeExt))
- // 2a. Validate the requested output_format against the
- // family-specific allowed_output_format whitelist. We do
- // this even when no setups entry exists so a misconfigured
- // DSL surfaces as _ERROR instead of a silent fallback.
- if _, hasSetup := c.Setups[fileTypeFam]; hasSetup {
- if _, verr := resolveOutputFormat(fileTypeFam, c.Setups, c.Param.AllowedOutputFormat); verr != nil {
- return nil, verr
- }
- }
-
- dispatched, handledVision, visionErr := maybeDispatchPDFVision(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)
+ dispatched, handledVision, visionErr := maybeDispatchPDFVision(ctx, db, fileTypeExt, filename, binary, inputs, setups)
if visionErr != nil {
return nil, visionErr
}
@@ -455,7 +466,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
if !handledVision {
// Video dispatch: IMAGE2TEXT vision chat.
// Mirrors Python's _video().
- dispatched, handledMedia, visionErr = maybeDispatchVideo(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)
+ dispatched, handledMedia, visionErr = maybeDispatchVideo(ctx, db, fileTypeExt, filename, binary, inputs, setups)
if visionErr != nil {
return nil, visionErr
}
@@ -464,7 +475,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
if !handledVision && !handledMedia {
// Image/Picture dispatch: OCR + IMAGE2TEXT vision describe.
// Mirrors Python's rag/app/picture.py:chunk() image branch.
- dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)
+ dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups)
if visionErr != nil {
return nil, visionErr
}
@@ -473,13 +484,13 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
if !handledVision && !handledMedia && !handledImage {
// Audio dispatch: SPEECH2TEXT transcription.
// Mirrors Python's rag/app/audio.py:chunk().
- dispatched, handledAudio, visionErr = maybeDispatchAudio(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)
+ dispatched, handledAudio, visionErr = maybeDispatchAudio(ctx, db, fileTypeExt, filename, binary, inputs, setups)
if visionErr != nil {
return nil, visionErr
}
}
if !handledVision && !handledMedia && !handledImage && !handledAudio {
- dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, c.Setups)
+ dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups)
// Vision figure enhancement: on the JSON output path,
// append vision-model descriptions to embedded image and
@@ -488,7 +499,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
// Errors (including context cancellation) are intentionally
// discarded — enhancement is best-effort, matching Python's
// try/except pass pattern.
- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.Setups)
+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)
}
// Known/supported families must fail loudly when dispatch or
// parsing breaks. Only unknown families keep the raw-text fallback.
@@ -497,56 +508,11 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
reportParserWarnings(ctx, dispatched.Warnings)
- // 3. Build the legacy `pages` slice. When the dispatch path
- // produced a JSON payload, we re-shape it into the page
- // layout the chunker side consumes (`{text, doc_type_kwd,
- // page_number?}`); when the dispatch produced a string
- // payload we emit a single page carrying the rendered text;
- // otherwise we slice the binary on ASCII form-feed and
- // treat the input as text pages.
- var pages [][]byte
- var dispatchedPages []schema.Page
- switch {
- case dispatched.Err == nil && dispatched.OutputFormat == "json":
- dispatchedPages = jsonItemsToPages(dispatched.JSON)
- pages = pagesFromDispatch(dispatchedPages)
- case dispatched.Err == nil && dispatched.OutputFormat != "":
- var text string
- switch dispatched.OutputFormat {
- case "markdown":
- text = dispatched.Markdown
- case "html":
- text = dispatched.HTML
- case "text":
- text = dispatched.Text
- }
- pages = [][]byte{[]byte(text)}
- default:
- pages = splitIntoPages(binary)
- if len(pages) == 0 {
- pages = [][]byte{nil}
- }
- }
-
- // 3b. (pages-based page selection is handled upstream via the
- // component setup: ParserConfig[cpnID][filetype]["pages"] is a
- // list of page ranges delivered through override_params and
- // consumed by the deepdoc/pdf parser. No inputs-level handling
- // here.)
-
- // 4. Build the page slice sequentially. Per-page parallelism now
- // lives in the parser backends (e.g. internal/deepdoc/parser/pdf
- // fans out one worker per page and assembles in page order), so
- // this component only reshapes the parser output. The DETERMINISTIC
- // MERGE (plan §8 R8) keeps pages sorted by PageNumber so the
- // downstream chunker / tokenizer get stable chunk IDs.
- parsed, err := buildPagesFromBytes(ctx, pages, dispatched.DocType)
- if err != nil {
+ if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("parser: %w", err)
}
- sortPagesByNumber(parsed)
lang, _ := getString(inputs, "lang")
- out := buildParserOutputs(parsed, dispatched, filename, fileTypeExt, lang)
+ out := buildParserOutputs(ctx, dispatched, filename, binary, lang)
// Forward the storage references so a downstream chunker can
// re-acquire the source PDF and crop section images on demand,
// instead of carrying the binary across the component boundary.
@@ -565,58 +531,42 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
// forwards only this explicit output to the next node, so shared
// fields must live in Globals.
globals.PublishGlobals(ctx, out)
- // Debug log: summarize parser output for pipeline debugging.
- if dispatched.OutputFormat == "json" {
- common.Debug("parser stage output",
- zap.String("component", "Parser"),
- zap.String("output_format", "json"),
- zap.Int("json_items", len(dispatched.JSON)),
- )
- } else if dispatched.OutputFormat != "" {
- common.Debug("parser stage output",
- zap.String("component", "Parser"),
- zap.String("output_format", dispatched.OutputFormat),
- )
- }
+ items, _ := out["json"].([]map[string]any)
+ logParserOutput(dispatched, items)
// Progress (_created_time / _elapsed_time stamping, start/done
// callbacks) is owned by the canvas framework (realComponentBody),
// not by this component, so we return the work result directly.
return out, nil
}
-func reportParserWarnings(ctx context.Context, warnings []string) {
- for _, warning := range warnings {
- runtime.ReportProgressMessage(ctx, "Parser", "WARNING: "+warning)
+func logParserOutput(dispatched parser.ParseResult, items []map[string]any) {
+ common.Debug("parser stage output",
+ zap.String("component", "Parser"),
+ zap.String("normalized_from", resolveParserNormalizationSource(dispatched)),
+ zap.Int("json_items", len(items)),
+ )
+}
+
+func resolveParserNormalizationSource(dispatched parser.ParseResult) string {
+ if len(dispatched.JSON) > 0 {
+ return "json"
+ }
+ if dispatched.Markdown != "" {
+ return "markdown"
+ }
+ if dispatched.HTML != "" {
+ return "html"
+ }
+ if dispatched.Text != "" {
+ return "text"
}
+ return "raw"
}
-// buildPagesFromBytes reshapes already-prepared page bytes into the
-// schema.Page layout the downstream chunker consumes. The per-page
-// parse (including any parallelism) now lives in the parser backends
-// (internal/parser/parser and internal/deepdoc/parser/pdf); this
-// component only wraps the bytes into pages and honors context
-// cancellation so an abandoned run does not keep reshaping pages.
-//
-// The function is format-agnostic: it does not resolve parsers or
-// inspect file families — it only carries the raw bytes under the
-// "text" key with the given doc_type_kwd (defaults to "text" when
-// empty), matching the shape downstream readers expect from the
-// raw-text fallback and dispatch paths.
-func buildPagesFromBytes(ctx context.Context, pages [][]byte, docType string) ([]schema.Page, error) {
- if docType == "" {
- docType = "text"
- }
- out := make([]schema.Page, 0, len(pages))
- for _, raw := range pages {
- if err := ctx.Err(); err != nil {
- return nil, err
- }
- out = append(out, schema.Page{
- "text": string(raw),
- "doc_type_kwd": docType,
- })
+func reportParserWarnings(ctx context.Context, warnings []string) {
+ for _, warning := range warnings {
+ runtime.ReportProgressMessage(ctx, "Parser", "WARNING: "+warning)
}
- return out, nil
}
// --- input helpers ---
@@ -695,57 +645,6 @@ func containsFormFeed(b []byte) bool {
return false
}
-// sortPagesByNumber orders pages by their PageNumber key
-// ascending. Pages without a PageNumber key (or with a non-int
-// value) sort to the END so the deterministic contract is
-// "numbered pages first, then unnumbered" — this matches the
-// Python component's loop order (it processes pages in input
-// order, not in PageNumber order, but the Go merge is
-// intentionally stricter so the test can assert exact byte
-// equality across runs).
-func sortPagesByNumber(pages []schema.Page) {
- sort.SliceStable(pages, func(i, j int) bool {
- pi, oki := numericPageNumber(pages[i])
- pj, okj := numericPageNumber(pages[j])
- switch {
- case oki && okj:
- return pi < pj
- case oki:
- return true // i is numbered, j is not
- case okj:
- return false
- default:
- return false // stable
- }
- })
-}
-
-func numericPageNumber(p schema.Page) (int, bool) {
- if p == nil {
- return 0, false
- }
- v, ok := p["page_number"]
- if !ok {
- return 0, false
- }
- switch n := v.(type) {
- case int:
- return n, true
- case int64:
- return int(n), true
- case float64:
- return int(n), true
- }
- return 0, false
-}
-
-// toAnyPages is a tiny adapter that hands the page slice to
-// the output map as `any`. We use it instead of a direct cast
-// so the type stays `[]schema.Page` in the Go source and the
-// output map value type is `any` — matching the runtime.Component
-// contract.
-func toAnyPages(pages []schema.Page) any { return pages }
-
// init registers Parser under CategoryIngestion per plan §4
// Phase 2.2. The factory is a thin closure that decodes the
// DSL param map; the static Metadata is derived from
diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.go
index 109a59818b..7213e50f90 100644
--- a/internal/ingestion/component/parser_check_test.go
+++ b/internal/ingestion/component/parser_check_test.go
@@ -24,8 +24,7 @@ import (
)
// TestParserComponent_Check covers the construction-time business
-// validation that mirrors the applicable subset of Python
-// ParserParam.check() (rag/flow/parser/parser.py:251-321).
+// validation for parser methods that require language configuration.
//
// Go does NOT validate audio/video vlm.llm_id because media_dispatch
// resolves tenant default models via resolveTenantModelByType, not
@@ -131,7 +130,7 @@ func TestParserComponent_Check(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- c := &ParserComponent{Setups: tc.setups, Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{setups: tc.setups}
err := c.Check()
if tc.wantErr != "" {
if err == nil {
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index 69fb1c8510..24daa44820 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -14,8 +14,8 @@
// limitations under the License.
//
-// Parser dispatch validates the requested output format, resolves the
-// parser backend, and returns the structured ParseWithResult payload.
+// Parser dispatch resolves the parser backend and returns the structured
+// ParseWithResult payload for component-boundary normalization.
//
// `parse_method` is carried through file metadata for downstream
// consumers, while the actual backend work stays in
@@ -26,37 +26,15 @@ package component
import (
"context"
"fmt"
- "maps"
"strings"
+ "ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
"ragflow/internal/utility"
-)
-// parserDispatchResult is the typed outcome of dispatchParse. The
-// component's Invoke translates it into the runtime output map.
-//
-// OutputFormat is the wire format the parser actually emitted. It
-// always matches setups[fileType].output_format on success; on a
-// format-mismatch failure (the whitelist rejects it) OutputFormat is
-// empty and Err is non-nil.
-//
-// File is the per-parser file metadata and may be nil.
-//
-// Payload holds exactly one populated field per the ParseResult
-// contract (see internal/parser/parser/parse_result.go).
-type parserDispatchResult struct {
- OutputFormat string
- DocType string // doc_type_kwd for media dispatch ("image", "video", "audio")
- File map[string]any
- JSON []map[string]any
- Markdown string
- Text string
- HTML string
- Warnings []string
- Err error
-}
+ "go.uber.org/zap"
+)
type parserSetupConfigurer interface {
ConfigureFromSetup(setup map[string]any)
@@ -82,90 +60,6 @@ func configureParserFromSetups(p any, fileType utility.FileType, setups map[stri
cfg.ConfigureFromSetup(map[string]any(setup))
}
-// resolveOutputFormat picks the wire format for this run. The
-// Python side asks the setup, then checks the value is in
-// allowed_output_format[fileType]. We mirror that exact sequence:
-//
-// 1. setups[fileType].output_format (or "" when absent),
-// 2. if absent and the family has an allowed_output_format entry,
-// use the Python per-family default (mirrors ParserParam.__init__
-// setups, e.g. "image" → "json", "pdf" → "json", "spreadsheet" →
-// "html"), falling back to allowed[0] only when no explicit default
-// is known,
-// 3. if absent and the family has no allowed_output_format entry,
-// return "" (the component falls back to text-page mode
-// without validating).
-//
-// The whitelist check returns "" + Err when the requested format is
-// not in the allowed set; this is the validation Python raises
-// via check_empty / check_valid_value, surfaced as an error so the
-// component short-circuits with _ERROR rather than emitting a
-// payload the downstream chunker cannot consume.
-//
-// Strict mode: explicit image:text is rejected by the whitelist.
-func resolveOutputFormat(family string, setups map[string]schema.ParserSetup, allowed map[string][]string) (string, error) {
- setup, ok := setups[family]
- if !ok {
- // Family not configured — text-page mode; no validation.
- return "", nil
- }
- format, _ := setup["output_format"].(string)
- allowedList, ok := allowed[family]
- if !ok || len(allowedList) == 0 {
- // No whitelist entry — accept what the setup asked for, or
- // fall back to "text" for families without a whitelist.
- if format == "" {
- format = "text"
- }
- return format, nil
- }
- if format == "" {
- // No explicit format: use the Python per-family default declared
- // in ParserParam.__init__ / defaultSetups(). This is not always
- // allowed[0] (e.g. spreadsheet default is "html" but allowed[0]
- // is "json"; markdown default is "json" but allowed[0] is "text").
- if def, ok := defaultOutputFormatForFamily(family); ok {
- return def, nil
- }
- return allowedList[0], nil
- }
- for _, candidate := range allowedList {
- if strings.EqualFold(candidate, format) {
- return format, nil
- }
- }
- return "", fmt.Errorf(
- "parser: output_format %q for %q is not in allowed_output_format %v",
- format, family, allowedList,
- )
-}
-
-// defaultOutputFormatForFamily returns the per-family default
-// output_format used when setups[family] exists but output_format is
-// empty. For most families it is the value in defaultSetups()
-// (which mirrors rag/flow/parser/parser.py ParserParam.setups).
-// Two families diverge intentionally and are overridden here:
-// - email: defaultSetups is "json" (Python), but dispatch defaults
-// to "text" to match frontend InitialOutputFormatMap, EmailParser
-// text fallback, and ingestion_pipeline_email.json.
-// - audio: defaultSetups is "text" (Python), but dispatch defaults
-// to "json" to match media_dispatch json fallback and Python
-// whitelist [json] (audio:text would be rejected if validated).
-func defaultOutputFormatForFamily(family string) (string, bool) {
- if family == "email" {
- return "text", true
- }
- if family == "audio" {
- return "json", true
- }
- if s, ok := defaultSetups()[family]; ok {
- if v, ok := s["output_format"].(string); ok && v != "" {
- return v, true
- }
- }
- return "", false
-}
-
// dispatchParse resolves the parser for the given fileType and invokes
// its structured ParseWithResult contract.
//
@@ -182,12 +76,12 @@ func defaultOutputFormatForFamily(family string) (string, bool) {
// re-reading setups. lib_type is no longer threaded through: the
// Python dispatcher picks a single backend per family and the Go
// constructors mirror that.
-func dispatchParse(ctx context.Context, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup) parserDispatchResult {
+func dispatchParse(ctx context.Context, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup) parser.ParseResult {
if fileType == utility.FileTypeOTHER {
// Unknown / unset family. The component treats the bytes
// as text pages; splitIntoPages handles it. We return no
// result here so the caller routes to that path.
- return parserDispatchResult{}
+ return parser.ParseResult{}
}
var parseMethod string
@@ -199,13 +93,13 @@ func dispatchParse(ctx context.Context, fileType utility.FileType, filename stri
p, err := parser.GetParser(fileType)
if err != nil {
- return parserDispatchResult{Err: fmt.Errorf("parser: resolve %q: %w", fileType, err)}
+ return parser.ParseResult{Err: fmt.Errorf("parser: resolve %q: %w", fileType, err)}
}
configureParserFromSetups(p, fileType, setups)
res := p.ParseWithResult(ctx, filename, data)
if res.Err != nil {
- return parserDispatchResult{Err: fmt.Errorf("parser: %q: %w", fileType, res.Err)}
+ return parser.ParseResult{Err: fmt.Errorf("parser: %q: %w", fileType, res.Err)}
}
// Carry the configured parse_method on the file metadata so
// downstream consumers can read which provider ran.
@@ -215,15 +109,7 @@ func dispatchParse(ctx context.Context, fileType utility.FileType, filename stri
}
res.File["parse_method"] = parseMethod
}
- return parserDispatchResult{
- OutputFormat: res.OutputFormat,
- File: res.File,
- JSON: res.JSON,
- Markdown: res.Markdown,
- Text: res.Text,
- HTML: res.HTML,
- Warnings: res.Warnings,
- }
+ return res
}
// fileTypeFromInputs derives the parser-library extension form
@@ -238,9 +124,9 @@ func dispatchParse(ctx context.Context, fileType utility.FileType, filename stri
// or the python family name ("markdown"); both are normalized
// to the extension form via the pythonFamilyName / familyToExt
// lookup tables below.
-// 2. inputs["file"].name — fall back to the filename so a caller
-// that only supplies the path is still routed correctly.
-// 3. inputs["name"] — last-resort filename.
+// 2. inputs["name"] — the resolved source filename.
+// 3. inputs["file"].name — fallback for callers that only supply a
+// file descriptor.
// 4. utility.FileTypeOTHER — text-page mode.
//
// The function never errors; unknown / absent filenames degrade to
@@ -270,14 +156,14 @@ func fileTypeFromInputs(inputs map[string]any) utility.FileType {
return ft
}
}
+ if name, ok := inputs["name"].(string); ok && name != "" {
+ return utility.GetFileType(name)
+ }
if m, ok := inputs["file"].(map[string]any); ok {
if name, ok := m["name"].(string); ok && name != "" {
return utility.GetFileType(name)
}
}
- if name, ok := inputs["name"].(string); ok && name != "" {
- return utility.GetFileType(name)
- }
return utility.FileTypeOTHER
}
@@ -321,7 +207,7 @@ func familyToExt(family string) utility.FileType {
}
// pythonFamilyName normalises a free-form file-type hint to the
-// python family identifier used by schema.ParserParam.Setups.
+// Python family identifier used by ParserComponent setups.
// Returns "" when the hint is unknown.
func pythonFamilyName(raw string) string {
switch raw {
@@ -374,101 +260,79 @@ func ParserFileFamily(ext string) string {
return pythonFamilyName(ext)
}
-// jsonItemsToPages reshapes a parsed JSON payload into the
-// schema.Page layout the chunker side consumes. Each JSON item
-// becomes one page carrying `text`, `doc_type_kwd`, and any
-// ck_type / image / positions fields the parser attached. The page
-// number is assigned sequentially so the deterministic-merge
-// contract in plan §8 R8 holds.
-func jsonItemsToPages(items []map[string]any) []schema.Page {
- out := make([]schema.Page, 0, len(items))
- for i, it := range items {
- page := schema.Page{}
- maps.Copy(page, it)
- // page_number anchors the deterministic-merge sort; the
- // parser-supplied number (if any) takes precedence so
- // PDF / DOCX outputs that already carry `page_number`
- // survive the round-trip.
- if _, ok := page["page_number"]; !ok {
- page["page_number"] = i
- }
- if _, ok := page["doc_type_kwd"]; !ok {
- page["doc_type_kwd"] = "text"
- }
- out = append(out, page)
- }
- return out
-}
-
-// pagesFromDispatch extracts the per-page bytes from a parsed
-// schema.Page slice so the page builder can reshape them into the
-// schema.Page layout the chunker consumes. Pages without a `text`
-// field emit an empty buffer (treated as a zero-length page).
-func pagesFromDispatch(pages []schema.Page) [][]byte {
- out := make([][]byte, 0, len(pages))
- for _, p := range pages {
- var buf []byte
- if s, ok := p["text"].(string); ok {
- buf = []byte(s)
- }
- out = append(out, buf)
- }
- return out
-}
-
// buildParserOutputs assembles the runtime output map from the
-// merged pages slice AND the dispatch result (when the dispatch
-// succeeded). The output shape:
+// dispatch result (when the dispatch succeeded) or raw binary fallback.
+// The output shape:
//
-// - pages []schema.Page — sorted by PageNumber
// - name string — from the upstream file/document name
// (or doc_id when no filename is available)
-// - output_format string — the dispatch's OutputFormat,
-// or "text" for the raw-text
-// fallback
-// - json | Markdown | text | html — the dispatched payload on
-// the matching family key (only
-// populated on a structured
-// dispatch)
+// - output_format string — always "json"
+// - json []map[string]any — normalized parser items
// - file map[string]any — the parser-enriched file
// metadata, when present
-//
-// This mirrors the Python Parser component's `set_output()` calls
-// at rag/flow/parser/parser.py:_invoke — the downstream chunker
-// / tokenizer / extractor components read the matching family
-// key, with "pages" as the universal fallback shape.
-func buildParserOutputs(parsed []schema.Page, dispatched parserDispatchResult, name string, fileType utility.FileType, lang string) map[string]any {
+func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name string, rawBinary []byte, lang string) map[string]any {
out := map[string]any{
- "pages": toAnyPages(parsed),
- "name": name,
+ "name": name,
}
if lang != "" {
out["lang"] = lang
}
if dispatched.Err == nil && dispatched.OutputFormat != "" {
- out["output_format"] = dispatched.OutputFormat
- switch dispatched.OutputFormat {
- case "json":
- out["json"] = dispatched.JSON
- case "markdown":
- out["markdown"] = dispatched.Markdown
- case "html":
- out["html"] = dispatched.HTML
- case "text":
- out["text"] = dispatched.Text
- }
+ out["output_format"] = "json"
+ out["json"] = normalizeParserJSON(ctx, name, dispatched)
if dispatched.File != nil {
out["file"] = dispatched.File
}
return out
}
- // Raw-text fallback path: emit output_format = "text" so a
- // chunker branching on the format key still sees a sane value.
- out["output_format"] = "text"
- _ = fileType // reserved for a future "raw_text per-family" extension
+ // Raw-text fallback path: emit one JSON item per page.
+ rawPages := splitIntoPages(rawBinary)
+ if len(rawPages) == 0 {
+ rawPages = [][]byte{nil}
+ }
+ fallbackItems := make([]map[string]any, 0, len(rawPages))
+ for _, pageBytes := range rawPages {
+ txt := string(pageBytes)
+ fallbackItems = append(fallbackItems, parser.NewTextJSONItem(txt))
+ }
+ out["output_format"] = "json"
+ out["json"] = fallbackItems
return out
}
+func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []map[string]any {
+ if len(dispatched.JSON) > 0 {
+ return dispatched.JSON
+ }
+ if dispatched.Markdown != "" {
+ return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)
+ }
+ if dispatched.HTML != "" {
+ res := parser.NewHTMLParser().ParseWithResult(ctx, filename, []byte(dispatched.HTML))
+ if res.Err == nil && len(res.JSON) > 0 {
+ return res.JSON
+ }
+ if res.Err != nil {
+ warnParserNormalizationFallback(filename, "html", res.Err)
+ } else {
+ warnParserNormalizationFallback(filename, "html", fmt.Errorf("parser returned no JSON items"))
+ }
+ return []map[string]any{parser.NewTextJSONItem(dispatched.HTML)}
+ }
+ if dispatched.Text != "" {
+ return []map[string]any{parser.NewTextJSONItem(dispatched.Text)}
+ }
+ return []map[string]any{}
+}
+
+func warnParserNormalizationFallback(filename, source string, err error) {
+ common.Warn("parser normalization fell back to text",
+ zap.String("filename", filename),
+ zap.String("normalized_from", source),
+ zap.Error(err),
+ )
+}
+
func parserInputName(inputs map[string]any, docID string) string {
if inputs != nil {
if name, ok := inputs["name"].(string); ok && name != "" {
diff --git a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
index 043a8a88c6..d6bf465d11 100644
--- a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
+++ b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
@@ -11,7 +11,6 @@ import (
"testing"
modelModule "ragflow/internal/entity/models"
- "ragflow/internal/ingestion/component/schema"
"gorm.io/gorm"
)
@@ -61,11 +60,10 @@ func TestDispatch_PDFVisionJSON_RealPDFFixture(t *testing.T) {
t.Fatalf("ReadFile(%s): %v", path, err)
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
setups["pdf"]["output_format"] = "json"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": data,
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 16375ae71b..ef20c6a6c4 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -18,10 +18,9 @@
//
// - FileTypeOTHER + missing setups → text-page mode.
// - FileTypeMarkdown → JSON payload family on the matching output
-// key, with the pages slice preserved.
-// - FileTypePDF + setups["pdf"].output_format set to a value not
-// in allowed_output_format["pdf"] → component errors with the
-// format-mismatch message (matches the Python check() behavior).
+// key.
+// - Historical output_format values, including PDF Markdown, are accepted
+// and normalized to the single JSON payload.
package component
@@ -33,6 +32,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "net"
"net/http"
"net/http/httptest"
"os"
@@ -47,9 +47,11 @@ import (
doctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
- "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zaptest/observer"
"gorm.io/gorm"
)
@@ -75,14 +77,178 @@ func (c *captureSetupConfigurer) ConfigureFromSetup(setup map[string]any) {
c.setup = setup
}
-// TestDispatch_OutputFormatValidation_Allowed is the happy-path
-// pin: a Markdown file with output_format=json passes the
-// allowed_output_format check and runs the structured dispatch.
-func TestDispatch_OutputFormatValidation_Allowed(t *testing.T) {
- param := schema.ParserParam{}.Defaults()
+func requireJSONText(t *testing.T, out map[string]any, want string) {
+ t.Helper()
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
+ }
+ items, ok := out["json"].([]map[string]any)
+ if !ok || len(items) == 0 {
+ t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
+ }
+ for _, item := range items {
+ if text, _ := item["text"].(string); strings.Contains(text, want) {
+ return
+ }
+ }
+ t.Fatalf("json payload does not contain %q: %#v", want, items)
+}
+
+func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ dispatched parser.ParseResult
+ wantText string
+ }{
+ {
+ name: "markdown",
+ dispatched: parser.ParseResult{
+ OutputFormat: "markdown",
+ Markdown: "# Title\n\nBody",
+ },
+ wantText: "Title",
+ },
+ {
+ name: "html",
+ dispatched: parser.ParseResult{
+ OutputFormat: "html",
+ HTML: "<h1>Title</h1><p>Body</p>",
+ },
+ wantText: "Title",
+ },
+ {
+ name: "text",
+ dispatched: parser.ParseResult{
+ OutputFormat: "text",
+ Text: "plain body",
+ },
+ wantText: "plain body",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ out := buildParserOutputs(t.Context(), tt.dispatched, "sample."+tt.name, nil, "")
+
+ requireJSONText(t, out, tt.wantText)
+ if tt.name == "markdown" {
+ items, ok := out["json"].([]map[string]any)
+ if !ok {
+ t.Fatalf("json = %T, want []map[string]any", out["json"])
+ }
+ if len(items) != 2 {
+ t.Fatalf("markdown json item count = %d, want 2: %#v", len(items), items)
+ }
+ if got := items[0]["ck_type"]; got != "heading" {
+ t.Errorf("markdown item[0].ck_type = %v, want heading", got)
+ }
+ if got := items[0]["text"]; got != "Title" {
+ t.Errorf("markdown item[0].text = %v, want Title", got)
+ }
+ if got := items[1]["text"]; got != "Body" {
+ t.Errorf("markdown item[1].text = %v, want Body", got)
+ }
+ }
+ for _, key := range []string{"markdown", "html", "text"} {
+ if _, ok := out[key]; ok {
+ t.Errorf("output contains obsolete %q payload: %#v", key, out[key])
+ }
+ }
+ })
+ }
+}
+
+func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {
+ out := buildParserOutputs(t.Context(), parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{},
+ Markdown: "# Recovered title",
+ }, "sample.md", nil, "")
+
+ requireJSONText(t, out, "Recovered title")
+}
+
+func TestParseMarkdownToJSONItemsDoesNotResolveRemoteImages(t *testing.T) {
+ originalResolver := net.DefaultResolver
+ dnsLookup := make(chan struct{}, 1)
+ net.DefaultResolver = &net.Resolver{
+ PreferGo: true,
+ StrictErrors: true,
+ Dial: func(context.Context, string, string) (net.Conn, error) {
+ select {
+ case dnsLookup <- struct{}{}:
+ default:
+ }
+ return nil, fmt.Errorf("unexpected DNS lookup")
+ },
+ }
+ t.Cleanup(func() { net.DefaultResolver = originalResolver })
+
+ items := parseMarkdownToJSONItems(t.Context(), "ocr.md", "![remote](https://images.example.invalid/figure.png)")
+ if len(items) != 1 {
+ t.Fatalf("items = %#v, want one Markdown image block", items)
+ }
+ select {
+ case <-dnsLookup:
+ t.Fatal("Markdown-to-JSON normalization performed a remote image DNS lookup")
+ default:
+ }
+}
+
+func TestLogParserOutputReportsNormalizationSource(t *testing.T) {
+ core, logs := observer.New(zap.DebugLevel)
+ originalLogger := common.Logger
+ common.Logger = zap.New(core)
+ t.Cleanup(func() { common.Logger = originalLogger })
+
+ logParserOutput(parser.ParseResult{
+ OutputFormat: "markdown",
+ Markdown: "# Title\n\nBody",
+ }, []map[string]any{{"text": "Title"}, {"text": "Body"}})
+
+ entries := logs.FilterMessage("parser stage output").All()
+ if len(entries) != 1 {
+ t.Fatalf("parser output log count = %d, want 1", len(entries))
+ }
+ fields := entries[0].ContextMap()
+ if _, ok := fields["output_format"]; ok {
+ t.Errorf("output_format must be omitted because it is always json: %v", fields)
+ }
+ if got := fields["normalized_from"]; got != "markdown" {
+ t.Errorf("normalized_from = %v, want markdown", got)
+ }
+ if got := fields["json_items"]; got != int64(2) {
+ t.Errorf("json_items = %v, want 2", got)
+ }
+}
+
+func TestWarnParserNormalizationFallbackReportsCause(t *testing.T) {
+ core, logs := observer.New(zap.WarnLevel)
+ originalLogger := common.Logger
+ common.Logger = zap.New(core)
+ t.Cleanup(func() { common.Logger = originalLogger })
+
+ warnParserNormalizationFallback("broken.md", "markdown", fmt.Errorf("parse failed"))
+
+ entries := logs.FilterMessage("parser normalization fell back to text").All()
+ if len(entries) != 1 {
+ t.Fatalf("normalization fallback log count = %d, want 1", len(entries))
+ }
+ fields := entries[0].ContextMap()
+ if got := fields["filename"]; got != "broken.md" {
+ t.Errorf("filename = %v, want broken.md", got)
+ }
+ if got := fields["normalized_from"]; got != "markdown" {
+ t.Errorf("normalized_from = %v, want markdown", got)
+ }
+ if got := fields["error"]; got != "parse failed" {
+ t.Errorf("error = %v, want parse failed", got)
+ }
+}
+
+func TestDispatch_JSONOutput(t *testing.T) {
setups := defaultSetups()
- // Defaults already include Markdown → {text, json}.
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("# Title\n\nbody\n"),
@@ -102,67 +268,42 @@ func TestDispatch_OutputFormatValidation_Allowed(t *testing.T) {
if len(jsonItems) == 0 {
t.Errorf("json payload empty; want at least 1 item")
}
- // Pages must still exist for chunker-side consumers.
- pages, ok := out["pages"].([]schema.Page)
- if !ok || len(pages) == 0 {
- t.Errorf("pages slice missing or empty: %T", out["pages"])
- }
- if ok && len(pages) > 0 {
- if got, _ := pages[0]["text"].(string); !strings.Contains(got, "Title") {
- t.Errorf("pages[0].text = %q, want content containing Title", got)
- }
- }
// File metadata is carried through dispatch.
if fm, ok := out["file"].(map[string]any); !ok || fm["name"] != "doc.md" {
t.Errorf("file metadata missing or wrong: %+v", out["file"])
}
}
-// TestDispatch_OutputFormatValidation_Rejection pins the
-// whitelist enforcement: a request for output_format=html on the
-// Markdown family is rejected because Markdown's allowed list is
-// {text, json}. The component must surface this as a hard error
-// before any fallback so a misconfigured template cannot silently
-// degrade.
-func TestDispatch_OutputFormatValidation_Rejection(t *testing.T) {
- param := schema.ParserParam{}.Defaults()
+func TestDispatchNormalizesConfiguredOutputFormatToJSON(t *testing.T) {
setups := defaultSetups()
- // Override the Markdown setup to ask for an unsupported format.
- // The key is "markdown" (the python-side family identifier),
- // NOT "md" — utility.FileTypeMarkdown happens to be the string
- // "md" but the setup key is the family name. resolveOutputFormat
- // looks up setups[string(fileType)], so the fileType passed in
- // here must match the setup key.
- setups["markdown"] = schema.ParserSetup{"output_format": "html"}
- // inputs["file_type"] must also be "markdown" so fileTypeFromInputs
- // returns a FileType whose string form matches the setup key.
- c := &ParserComponent{Param: param, Setups: setups}
+ setups["email"]["output_format"] = "text"
+ c := &ParserComponent{setups: setups}
- _, err := c.Invoke(t.Context(), nil, map[string]any{
- "binary": []byte("# Title\n"),
- "file_type": "md",
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: Hello\r\n\r\nBody\r\n"),
+ "name": "mail.eml",
+ "file_type": "eml",
})
- if err == nil {
- t.Fatal("Invoke: want error for unsupported output_format, got nil")
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
}
- if !strings.Contains(err.Error(), "output_format") {
- t.Errorf("error %q must mention output_format", err.Error())
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
}
- if !strings.Contains(err.Error(), "markdown") && !strings.Contains(err.Error(), "md") {
- t.Errorf("error %q must mention the family", err.Error())
+ if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {
+ t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
}
// TestDispatch_TextPageMode_NoFileType pins the no-dispatch
// path. When the upstream inputs supply neither file_type nor
// file.name, the component degrades to text-page mode and
-// emits output_format=text. This is the documented behavior for
+// emits structured JSON items. This is the documented behavior for
// canvas-bound invocations that wire the binary directly without
// a family hint.
func TestDispatch_TextPageMode_NoFileType(t *testing.T) {
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("plain content\n"),
@@ -171,12 +312,15 @@ func TestDispatch_TextPageMode_NoFileType(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if got, want := out["output_format"], "text"; got != want {
- t.Errorf("output_format = %v, want %v (text-page mode)", got, want)
+ if got, want := out["output_format"], "json"; got != want {
+ t.Errorf("output_format = %v, want %v (raw-text mode)", got, want)
}
- pages, ok := out["pages"].([]schema.Page)
- if !ok || len(pages) == 0 {
- t.Fatalf("pages slice missing or empty: %T", out["pages"])
+ jsonItems, ok := out["json"].([]map[string]any)
+ if !ok || len(jsonItems) == 0 {
+ t.Fatalf("json items missing or empty: %T", out["json"])
+ }
+ if _, ok := out["text"]; ok {
+ t.Fatalf("output contains obsolete text payload: %#v", out["text"])
}
}
@@ -185,9 +329,8 @@ func TestDispatch_TextPageMode_NoFileType(t *testing.T) {
// resolution/execution failures must surface as errors instead of
// silently degrading to text-page mode.
func TestDispatch_SupportedFamilyFailure_HardErrors(t *testing.T) {
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("PDF payload as bytes (not a real PDF — stub test)\n"),
@@ -205,8 +348,8 @@ func TestDispatch_SupportedFamilyFailure_HardErrors(t *testing.T) {
// rules documented on parser_dispatch.go:fileTypeFromInputs:
//
// 1. inputs["file_type"] (explicit family hint)
-// 2. inputs["file"].name (filename in the file descriptor)
-// 3. inputs["name"] (last-resort filename)
+// 2. inputs["name"] (the resolved source filename)
+// 3. inputs["file"].name (filename in the file descriptor)
// 4. FileTypeOTHER (text-page mode)
func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {
cases := []struct {
@@ -222,6 +365,7 @@ func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {
{"explicit slides (family name)", map[string]any{"file_type": "slides"}, "pptx"},
{"explicit spreadsheet (family name)", map[string]any{"file_type": "spreadsheet"}, "xlsx"},
{"explicit markdown (family form)", map[string]any{"file_type": "markdown"}, "md"},
+ {"name wins over conflicting file.name", map[string]any{"name": "report.txt", "file": map[string]any{"name": "report.pdf"}}, "txt"},
{"file.name docx", map[string]any{"file": map[string]any{"name": "report.docx"}}, "docx"},
{"name fallback md", map[string]any{"name": "notes.md"}, "md"},
{"unrelated inputs", map[string]any{"binary": []byte("x"), "doc_id": "abc"}, "other"},
@@ -238,132 +382,7 @@ func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {
}
}
-// TestResolveOutputFormat_DefaultsAndWhitelist pins the two-layer
-// behavior of resolveOutputFormat: it returns the setup's
-// output_format when present (or the per-family default when absent,
-// e.g. markdown→json, spreadsheet→html), and rejects values not in
-// the allowed_output_format list. Explicit image:text is rejected.
-func TestResolveOutputFormat_DefaultsAndWhitelist(t *testing.T) {
- allowed := map[string][]string{
- "pdf": {"json", "markdown"},
- "markdown": {"text", "json"},
- "image": {"json"},
- "spreadsheet": {"json", "markdown", "html"},
- "email": {"text", "json"},
- "audio": {"text", "json"},
- }
- cases := []struct {
- name string
- setups map[string]schema.ParserSetup
- family string
- want string
- wantErr bool
- }{
- {
- name: "no setup → empty (text-page mode)",
- setups: nil,
- family: "pdf",
- want: "",
- },
- {
- name: "setup with output_format=json → json",
- setups: map[string]schema.ParserSetup{"pdf": {"output_format": "json"}},
- family: "pdf",
- want: "json",
- },
- {
- name: "setup with output_format=markdown → markdown",
- setups: map[string]schema.ParserSetup{"pdf": {"output_format": "markdown"}},
- family: "pdf",
- want: "markdown",
- },
- {
- name: "setup without output_format → per-family default (markdown→json)",
- setups: map[string]schema.ParserSetup{"markdown": {}},
- family: "markdown",
- want: "json",
- },
- {
- name: "setup without output_format → per-family default (spreadsheet→html)",
- setups: map[string]schema.ParserSetup{"spreadsheet": {}},
- family: "spreadsheet",
- want: "html",
- },
- {
- name: "setup without output_format → per-family default (image→json)",
- setups: map[string]schema.ParserSetup{"image": {}},
- family: "image",
- want: "json",
- },
- {
- name: "setup without output_format → per-family default (email→text)",
- setups: map[string]schema.ParserSetup{"email": {}},
- family: "email",
- want: "text",
- },
- {
- name: "image explicit text (legacy) → strict reject",
- setups: map[string]schema.ParserSetup{"image": {"output_format": "text"}},
- family: "image",
- wantErr: true,
- },
- {
- name: "image explicit TEXT uppercase → strict reject",
- setups: map[string]schema.ParserSetup{"image": {"output_format": "TEXT"}},
- family: "image",
- wantErr: true,
- },
- {
- name: "pdf asking for html (not allowed) → reject",
- setups: map[string]schema.ParserSetup{"pdf": {"output_format": "html"}},
- family: "pdf",
- wantErr: true,
- },
- {
- name: "setup without output_format → per-family default (audio→json)",
- setups: map[string]schema.ParserSetup{"audio": {}},
- family: "audio",
- want: "json",
- },
- {
- name: "setup without output_format → per-family default (pdf→json)",
- setups: map[string]schema.ParserSetup{"pdf": {}},
- family: "pdf",
- want: "json",
- },
- {
- name: "family with no whitelist → accept setup value",
- setups: map[string]schema.ParserSetup{"video": {"output_format": "json"}},
- family: "video",
- want: "json",
- },
- {
- name: "family with no whitelist empty → default text",
- setups: map[string]schema.ParserSetup{"video": {}},
- family: "video",
- want: "text",
- },
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- got, err := resolveOutputFormat(tc.family, tc.setups, allowed)
- if tc.wantErr {
- if err == nil {
- t.Fatalf("want error, got nil (value=%q)", got)
- }
- return
- }
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if got != tc.want {
- t.Errorf("got %q, want %q", got, tc.want)
- }
- })
- }
-}
-
-func TestDefaultSetups_DOCX_OutputFormatMarkdown(t *testing.T) {
+func TestDefaultSetups_DOCX_OutputFormatJSON(t *testing.T) {
setups := defaultSetups()
docx, ok := setups["docx"]
if !ok {
@@ -377,130 +396,7 @@ func TestDefaultSetups_DOCX_OutputFormatMarkdown(t *testing.T) {
t.Errorf("docx.output_format = %q, want %q", got, "json")
}
}
-
-// TestDefaultOutputFormatForFamily_Sync verifies the dispatch default
-// stays in sync with the allowed whitelist and, except for the two
-// intentional overrides (email:text, audio:json), with defaultSetups.
-func TestDefaultOutputFormatForFamily_Sync(t *testing.T) {
- allowed := schema.ParserParam{}.Defaults().AllowedOutputFormat
- overrides := map[string]string{"email": "text", "audio": "json"}
- for family, def := range map[string]string{
- "pdf": "json",
- "spreadsheet": "html",
- "doc": "json",
- "docx": "json",
- "slides": "json",
- "image": "json",
- "markdown": "json",
- "text&code": "json",
- "html": "json",
- "epub": "json",
- "json": "json",
- "email": "text",
- "audio": "json",
- "video": "text",
- } {
- got, ok := defaultOutputFormatForFamily(family)
- if !ok {
- t.Errorf("defaultOutputFormatForFamily(%q) missing", family)
- continue
- }
- if got != def {
- t.Errorf("defaultOutputFormatForFamily(%q)=%q, want %q", family, got, def)
- }
- if list, hasWL := allowed[family]; hasWL && len(list) > 0 {
- found := false
- for _, c := range list {
- if strings.EqualFold(c, got) {
- found = true
- break
- }
- }
- if !found {
- t.Errorf("default %q for %q not in allowed %v", got, family, list)
- }
- }
- if ov, isOverride := overrides[family]; isOverride {
- if got != ov {
- t.Errorf("override %q got %q want %q", family, got, ov)
- }
- continue
- }
- if ds, ok := defaultSetups()[family]; ok {
- if want, ok := ds["output_format"].(string); ok && want != got {
- t.Errorf("family %q dispatch default %q != defaultSetups %q (should be synced or listed as override)", family, got, want)
- }
- }
- }
- if _, ok := defaultOutputFormatForFamily("unknown"); ok {
- t.Errorf("unknown family should return !ok")
- }
-}
-
-// TestResolveOutputFormat_AudioOutputFormats pins the audio-family
-// whitelist against the builtin audio template. The template
-// (ingestion_pipeline_audio.json) and the Python default audio setup
-// (rag/flow/parser/parser.py) both use output_format="text" — an audio
-// transcription is inherently plain text. "text" must therefore pass
-// the whitelist, "json" must stay accepted, and a format outside the
-// whitelist must still be rejected.
-func TestResolveOutputFormat_AudioOutputFormats(t *testing.T) {
- allowed := schema.ParserParam{}.Defaults().AllowedOutputFormat
- audioAllowed, ok := allowed["audio"]
- if !ok {
- t.Fatal("allowed_output_format: audio key missing")
- }
- has := func(want string) bool {
- for _, v := range audioAllowed {
- if strings.EqualFold(v, want) {
- return true
- }
- }
- return false
- }
- if !has("text") {
- t.Errorf("allowed_output_format[audio] = %v, want it to include %q (builtin audio template and Python default use it)", audioAllowed, "text")
- }
- if !has("json") {
- t.Errorf("allowed_output_format[audio] = %v, want it to include %q", audioAllowed, "json")
- }
-
- cases := []struct {
- name string
- format string
- wantErr bool
- }{
- {name: "text accepted", format: "text"},
- {name: "json accepted", format: "json"},
- {name: "html rejected", format: "html", wantErr: true},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- setups := map[string]schema.ParserSetup{"audio": {"output_format": tc.format}}
- got, err := resolveOutputFormat("audio", setups, allowed)
- if tc.wantErr {
- if err == nil {
- t.Fatalf("want error for audio output_format=%q, got %q", tc.format, got)
- }
- return
- }
- if err != nil {
- t.Fatalf("audio output_format=%q: unexpected error: %v", tc.format, err)
- }
- if got != tc.format {
- t.Errorf("got %q, want %q", got, tc.format)
- }
- })
- }
-}
-
-// TestDefaultSetups_Audio_OutputFormatText pins the audio default to
-// "text", matching the Python default setup
-// (rag/flow/parser/parser.py audio block) and the builtin audio
-// template. The default feeds both the whitelist gate and the ASR
-// dispatch, so it must not drift to a format the audio pipeline does
-// not produce.
-func TestDefaultSetups_Audio_OutputFormatText(t *testing.T) {
+func TestDefaultSetups_Audio_OutputFormatJSON(t *testing.T) {
setups := defaultSetups()
audio, ok := setups["audio"]
if !ok {
@@ -510,8 +406,8 @@ func TestDefaultSetups_Audio_OutputFormatText(t *testing.T) {
if !ok {
t.Fatal("defaultSetups: audio.output_format missing or not a string")
}
- if got != "text" {
- t.Errorf("audio.output_format = %q, want %q", got, "text")
+ if got != "json" {
+ t.Errorf("audio.output_format = %q, want %q", got, "json")
}
}
@@ -527,7 +423,7 @@ func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) {
}
}
-func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) {
+func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {
useMockDocAnalyzer(t)
path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")
@@ -536,10 +432,9 @@ func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) {
t.Fatalf("ReadFile(%s): %v", path, err)
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["output_format"] = "markdown"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": data,
@@ -549,15 +444,11 @@ func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if got, want := out["output_format"], "markdown"; got != want {
- t.Fatalf("output_format = %v, want %v", got, want)
- }
- md, ok := out["markdown"].(string)
- if !ok || md == "" {
- t.Fatalf("markdown payload missing or empty: %T", out["markdown"])
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
}
- if _, ok = out["json"]; ok {
- t.Fatalf("json payload must be absent for markdown output: %+v", out["json"])
+ if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {
+ t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
}
@@ -568,11 +459,10 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {
t.Fatalf("ReadFile(%s): %v", path, err)
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "plain_text"
setups["pdf"]["output_format"] = "json"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": data,
@@ -592,10 +482,9 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {
}
func TestDispatch_PDFUnsupportedParseMethod_HardErrors(t *testing.T) {
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -662,11 +551,10 @@ func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) {
return &models.ChatResponse{Answer: &answer}, nil
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
setups["pdf"]["output_format"] = "json"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -728,11 +616,10 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) {
return &models.ChatResponse{Answer: &answer}, nil
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
setups["pdf"]["output_format"] = "json"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -778,11 +665,10 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
setups["pdf"]["output_format"] = "markdown"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -793,12 +679,57 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if got, want := out["output_format"], "markdown"; got != want {
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
+ }
+ requireJSONText(t, out, "Title")
+}
+
+func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodPost && r.URL.Path == "/file_parse" {
+ buf := new(bytes.Buffer)
+ zw := zip.NewWriter(buf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody paragraph\n"}]`))
+ _ = zw.Close()
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(buf.Bytes())
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+
+ origResolver := resolveTenantModelByType
+ defer func() { resolveTenantModelByType = origResolver }()
+ baseURL := server.URL
+ apiKey := ""
+ resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ setups["pdf"]["output_format"] = "json"
+ c := &ParserComponent{setups: setups}
+
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "sample.pdf",
+ "tenant_id": "test-tenant",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if got, want := out["output_format"], "json"; got != want {
t.Fatalf("output_format = %v, want %v", got, want)
}
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "Title") {
- t.Fatalf("markdown payload = %#v, want Title content", out["markdown"])
+ items, ok := out["json"].([]map[string]any)
+ if !ok || len(items) == 0 {
+ t.Fatalf("json payload = %#v, want non-empty structured items", out["json"])
}
}
@@ -854,9 +785,7 @@ func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if out["output_format"] != "markdown" || out["markdown"] != "MonkeyOCRv2 title" {
- t.Fatalf("output=%#v", out)
- }
+ requireJSONText(t, out, "MonkeyOCRv2 title")
}
// mineruTestDriver is a minimal ModelDriver mock whose Name() returns "mineru".
@@ -940,11 +869,10 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesTenantModel(t *testing.T) {
return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "PaddleOCR"
setups["pdf"]["output_format"] = "markdown"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -955,13 +883,10 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesTenantModel(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if got, want := out["output_format"], "markdown"; got != want {
- t.Fatalf("output_format = %v, want %v", got, want)
- }
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "Paddle Title") {
- t.Fatalf("markdown payload = %#v, want Paddle Title content", out["markdown"])
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
}
+ requireJSONText(t, out, "Paddle Title")
}
// TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload pins the cloud
@@ -1008,11 +933,10 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload(t *testing.T) {
return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &emptyBaseURL}, 0, nil
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "PaddleOCR"
setups["pdf"]["output_format"] = "markdown"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1023,10 +947,7 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "Unwrapped Title") {
- t.Fatalf("markdown payload = %#v, want Unwrapped Title content", out["markdown"])
- }
+ requireJSONText(t, out, "Unwrapped Title")
}
func TestDispatch_PDFPaddleOCR_NoTenantModel_HardErrors(t *testing.T) {
@@ -1037,11 +958,10 @@ func TestDispatch_PDFPaddleOCR_NoTenantModel_HardErrors(t *testing.T) {
return nil, "", nil, 0, fmt.Errorf("no active PaddleOCR OCR model")
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "paddleocr"
setups["pdf"]["output_format"] = "markdown"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1101,11 +1021,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel(t *testing.T) {
return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["layout_recognizer"] = modelID
setups["pdf"]["output_format"] = "markdown"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1116,13 +1035,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if got, want := out["output_format"], "markdown"; got != want {
- t.Fatalf("output_format = %v, want %v", got, want)
- }
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "Cloud Paddle Title") {
- t.Fatalf("markdown payload = %#v, want Cloud Paddle Title content", out["markdown"])
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
}
+ requireJSONText(t, out, "Cloud Paddle Title")
}
// TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod pins the routing of a
@@ -1169,11 +1085,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {
return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = modelID
setups["pdf"]["output_format"] = "markdown"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1184,13 +1099,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if got, want := out["output_format"], "markdown"; got != want {
- t.Fatalf("output_format = %v, want %v", got, want)
- }
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "Cloud Paddle Title") {
- t.Fatalf("markdown payload = %#v, want Cloud Paddle Title content", out["markdown"])
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
}
+ requireJSONText(t, out, "Cloud Paddle Title")
}
// paddleocrTestDriver is a minimal ModelDriver mock whose Name() returns "paddleocr".
@@ -1345,13 +1257,12 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {
}))
defer server.Close()
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "Docling"
setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["docling_server_url"] = server.URL
setups["pdf"]["docling_api_key"] = "doc-secret"
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1361,19 +1272,16 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if got, want := out["output_format"], "markdown"; got != want {
- t.Fatalf("output_format = %v, want %v", got, want)
- }
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "Docling Title") {
- t.Fatalf("markdown payload = %#v, want Docling Title content", out["markdown"])
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
}
+ requireJSONText(t, out, "Docling Title")
if got, want := requestCount, 3; got != want {
t.Fatalf("requestCount = %d, want %d", got, want)
}
}
-func TestDispatch_PDFOpenDataLoaderMarkdown_UsesConfiguredBackend(t *testing.T) {
+func TestDispatch_PDFOpenDataLoaderLegacyMarkdownEmitsJSON(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
@@ -1385,12 +1293,11 @@ func TestDispatch_PDFOpenDataLoaderMarkdown_UsesConfiguredBackend(t *testing.T)
}))
defer server.Close()
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "OpenDataLoader"
setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["opendataloader_apiserver"] = server.URL
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1400,13 +1307,10 @@ func TestDispatch_PDFOpenDataLoaderMarkdown_UsesConfiguredBackend(t *testing.T)
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "ODL Title") {
- t.Fatalf("markdown payload = %#v, want ODL Title", out["markdown"])
- }
+ requireJSONText(t, out, "ODL Title")
}
-func TestDispatch_PDFSoMarkMarkdown_UsesConfiguredBackend(t *testing.T) {
+func TestDispatch_PDFSoMarkLegacyMarkdownEmitsJSON(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
@@ -1420,12 +1324,11 @@ func TestDispatch_PDFSoMarkMarkdown_UsesConfiguredBackend(t *testing.T) {
}))
defer server.Close()
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "SoMark"
setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["somark_base_url"] = server.URL
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1435,13 +1338,10 @@ func TestDispatch_PDFSoMarkMarkdown_UsesConfiguredBackend(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "SoMark Title") {
- t.Fatalf("markdown payload = %#v, want SoMark Title", out["markdown"])
- }
+ requireJSONText(t, out, "SoMark Title")
}
-func TestDispatch_PDFTCADPMarkdown_UsesConfiguredBackend(t *testing.T) {
+func TestDispatch_PDFTCADPLegacyMarkdownEmitsJSON(t *testing.T) {
withSSRFBypass(t)
zipPayload := tcadpZipFixtureForComponent(t)
var server *httptest.Server
@@ -1457,12 +1357,11 @@ func TestDispatch_PDFTCADPMarkdown_UsesConfiguredBackend(t *testing.T) {
}))
defer server.Close()
- param := schema.ParserParam{}.Defaults()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "TCADP parser"
setups["pdf"]["output_format"] = "markdown"
setups["pdf"]["tcadp_apiserver"] = server.URL
- c := &ParserComponent{Param: param, Setups: setups}
+ c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
@@ -1472,10 +1371,7 @@ func TestDispatch_PDFTCADPMarkdown_UsesConfiguredBackend(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- md, ok := out["markdown"].(string)
- if !ok || !strings.Contains(md, "Hello TCADP") {
- t.Fatalf("markdown payload = %#v, want Hello TCADP", out["markdown"])
- }
+ requireJSONText(t, out, "Hello TCADP")
}
func tcadpZipFixtureForComponent(t *testing.T) []byte {
@@ -1495,7 +1391,7 @@ func tcadpZipFixtureForComponent(t *testing.T) []byte {
// TestPythonFamilyName_FileTypeConstants pins the contract that every
// utility.FileType semantic constant resolves to the setups key used by
-// defaultSetups / AllowedOutputFormat. Before the fix, FileTypeVISUAL mapped
+// defaultSetups. Before the fix, FileTypeVISUAL mapped
// to "picture" (mismatching the "image" setups key) and FileTypeAURAL matched
// no case at all (returning ""), so output_format validation and
// configureParserFromSetups were silently skipped for image/audio files.
@@ -1527,16 +1423,11 @@ func TestPythonFamilyName_FileTypeConstants(t *testing.T) {
if got != c.want {
t.Errorf("pythonFamilyName(%q) = %q, want %q", c.ft, got, c.want)
}
- // Every mapped family must have a matching setups key and
- // allowed_output_format entry, otherwise output_format validation
- // is silently skipped.
+ // Every mapped family must have a matching setup so its parser receives
+ // the canonical output format and family-specific configuration.
if _, ok := defaultSetups()[got]; !ok {
t.Errorf("pythonFamilyName(%q) → %q has no defaultSetups entry", c.ft, got)
}
- allowed := schema.ParserParam{}.Defaults().AllowedOutputFormat
- if _, ok := allowed[got]; !ok {
- t.Errorf("pythonFamilyName(%q) → %q has no allowed_output_format entry", c.ft, got)
- }
}
}
diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go
index 0d2900cc10..aecbd754e0 100644
--- a/internal/ingestion/component/parser_test.go
+++ b/internal/ingestion/component/parser_test.go
@@ -18,14 +18,12 @@ package component
import (
"encoding/json"
- "reflect"
"strings"
"testing"
"github.com/xuri/excelize/v2"
"ragflow/internal/agent/runtime"
"ragflow/internal/entity"
- "ragflow/internal/ingestion/component/schema"
)
func TestReportParserWarningsAsProgressMessages(t *testing.T) {
@@ -79,13 +77,50 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {
if len(out) == 0 {
t.Errorf("Outputs() returned empty map")
}
- // The contract from the file header: at least "binary" in,
- // "pages" out. Anything else is informational.
- if _, ok := in["binary"]; !ok {
- t.Errorf("Inputs() missing key %q", "binary")
+ // The component catalog must expose every caller-provided input that
+ // Invoke reads before dispatching.
+ for _, key := range []string{"binary", "name", "file", "file_type", "lang", "doc_id", "bucket", "path"} {
+ if _, ok := in[key]; !ok {
+ t.Errorf("Inputs() missing runtime input key %q", key)
+ }
+ }
+ if _, ok := out["output_format"]; !ok {
+ t.Errorf("Outputs() missing key %q", "output_format")
}
- if _, ok := out["pages"]; !ok {
- t.Errorf("Outputs() missing key %q", "pages")
+ if _, ok := out["json"]; !ok {
+ t.Errorf("Outputs() missing key %q", "json")
+ }
+ for _, key := range []string{"name", "lang", "file", "doc_id", "bucket", "path"} {
+ if _, ok := out[key]; !ok {
+ t.Errorf("Outputs() missing runtime output key %q", key)
+ }
+ }
+ if _, ok := out["_ERROR"]; ok {
+ t.Error("Outputs() must not advertise _ERROR; Parser failures return Go errors")
+ }
+}
+
+func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {
+ component, err := NewParserComponent(map[string]any{
+ "pdf": map[string]any{"output_format": "markdown"},
+ "spreadsheet": map[string]any{"output_format": "html"},
+ "email": map[string]any{"output_format": "text"},
+ "allowed_output_format": map[string]any{
+ "pdf": []any{"json", "markdown"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewParserComponent: %v", err)
+ }
+
+ parserComponent := component.(*ParserComponent)
+ for _, family := range []string{"pdf", "spreadsheet", "email"} {
+ if got := parserComponent.setups[family]["output_format"]; got != "json" {
+ t.Errorf("%s output_format = %v, want json", family, got)
+ }
+ }
+ if _, ok := parserComponent.setups["allowed_output_format"]; ok {
+ t.Error("allowed_output_format must not be treated as a parser setup")
}
}
@@ -96,22 +131,25 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {
// owned by the canvas framework, not by this component, so they are
// not asserted here.
func TestParserComponent_Invoke_TextInput(t *testing.T) {
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": "hello world",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- pages, ok := out["pages"].([]schema.Page)
- if !ok {
- t.Fatalf("pages: got %T, want []schema.Page", out["pages"])
+ if got, want := out["output_format"], "json"; got != want {
+ t.Errorf("output_format = %v, want %v", got, want)
}
- if len(pages) != 1 {
- t.Fatalf("pages len = %d, want 1", len(pages))
+ jsonItems, ok := out["json"].([]map[string]any)
+ if !ok || len(jsonItems) != 1 {
+ t.Fatalf("json: got %T (len %d), want 1 item", out["json"], len(jsonItems))
}
- if got := pages[0]["text"]; got != "hello world" {
- t.Errorf("pages[0][text] = %q, want %q", got, "hello world")
+ if got := jsonItems[0]["text"]; got != "hello world" {
+ t.Errorf("json[0][text] = %q, want %q", got, "hello world")
+ }
+ if got := jsonItems[0]["doc_type_kwd"]; got != "text" {
+ t.Errorf("json[0][doc_type_kwd] = %q, want %q", got, "text")
}
}
@@ -125,7 +163,7 @@ func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {
t.Fatalf("Close: %v", err)
}
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults(), Setups: defaultSetups()}
+ c := &ParserComponent{setups: defaultSetups()}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": data.Bytes(),
"file_type": "xlsx",
@@ -141,55 +179,45 @@ func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {
if len(items) != 0 {
t.Fatalf("empty XLSX JSON = %v, want no raw-binary items", items)
}
- pages, ok := out["pages"].([]schema.Page)
- if !ok {
- t.Fatalf("pages = %T, want []schema.Page", out["pages"])
- }
- if len(pages) != 0 {
- t.Fatalf("empty XLSX pages = %v, want none", pages)
- }
}
// TestParserComponent_Invoke_PageRangeFilter asserts that
// form-feed boundaries are honored: "A\fB\fC" yields three
-// pages, in input order, with text intact.
+// items, in input order, with text intact.
func TestParserComponent_Invoke_PageRangeFilter(t *testing.T) {
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": "pageA\fpageB\fpageC",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- pages, ok := out["pages"].([]schema.Page)
+ jsonItems, ok := out["json"].([]map[string]any)
if !ok {
- t.Fatalf("pages: got %T, want []schema.Page", out["pages"])
+ t.Fatalf("json: got %T, want []map[string]any", out["json"])
}
- if len(pages) != 3 {
- t.Fatalf("pages len = %d, want 3", len(pages))
+ if len(jsonItems) != 3 {
+ t.Fatalf("json len = %d, want 3", len(jsonItems))
}
want := []string{"pageA", "pageB", "pageC"}
- for i, p := range pages {
- if got := p["text"]; got != want[i] {
- t.Errorf("pages[%d][text] = %q, want %q", i, got, want[i])
+ for i, it := range jsonItems {
+ if got := it["text"]; got != want[i] {
+ t.Errorf("json[%d][text] = %q, want %q", i, got, want[i])
}
}
}
-// TestParserComponent_Invoke_DeterministicMerge is the
-// golden-file test for plan §8 R8 (DETERMINISTIC MERGE).
+// TestParserComponent_Invoke_DeterministicMerge verifies stable output order.
//
// We invoke the component 5 times with identical input and
// assert byte-for-byte equality of the JSON-encoded output.
// The test is expected to pass under `go test -count=10 -race`
// — that flag is run separately in the verification block.
//
-// Text-page mode has no page_number key, so the input order is
-// the output order; the deterministic sort keeps that order
-// stable across runs, which is the contract the downstream
-// chunker relies on for stable chunk IDs.
+// Text-page mode preserves input order, which downstream chunkers rely on for
+// stable chunk IDs.
func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
// 8 form-feed-separated pages.
input := "p1\fp2\fp3\fp4\fp5\fp6\fp7\fp8"
@@ -200,7 +228,7 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {
if err != nil {
t.Fatalf("Invoke (first): %v", err)
}
- canonical, err := json.Marshal(first["pages"])
+ canonical, err := json.Marshal(first["json"])
if err != nil {
t.Fatalf("Marshal canonical: %v", err)
}
@@ -213,7 +241,7 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {
if err != nil {
t.Fatalf("Invoke (run %d): %v", i, err)
}
- encoded, err := json.Marshal(got["pages"])
+ encoded, err := json.Marshal(got["json"])
if err != nil {
t.Fatalf("Marshal run %d: %v", i, err)
}
@@ -224,10 +252,8 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {
}
}
-// TestParserComponent_New_Defaults constructs a Parser from a
-// nil param map and verifies the static Param is the
-// Defaults() value (i.e., NewParserComponent does not mutate
-// the schema default).
+// TestParserComponent_New_Defaults constructs a Parser from a nil param map
+// and verifies every parser family uses the canonical JSON output format.
func TestParserComponent_New_Defaults(t *testing.T) {
c, err := NewParserComponent(nil)
if err != nil {
@@ -237,9 +263,10 @@ func TestParserComponent_New_Defaults(t *testing.T) {
if !ok {
t.Fatalf("NewParserComponent returned %T, want *ParserComponent", c)
}
- defaults := schema.ParserParam{}.Defaults()
- if !reflect.DeepEqual(pc.Param, defaults) {
- t.Errorf("Param differs from Defaults:\n got=%+v\nwant=%+v", pc.Param, defaults)
+ for family, setup := range pc.setups {
+ if got := setup["output_format"]; got != "json" {
+ t.Errorf("%s output_format = %v, want json", family, got)
+ }
}
}
@@ -259,7 +286,7 @@ func TestParserComponent_New_Overrides(t *testing.T) {
if !ok {
t.Fatalf("NewParserComponent returned %T", c)
}
- setup, ok := pc.Setups["text&code"]
+ setup, ok := pc.setups["text&code"]
if !ok {
t.Fatalf("Setups[text&code] missing after override")
}
@@ -267,15 +294,38 @@ func TestParserComponent_New_Overrides(t *testing.T) {
t.Errorf("Setups[text&code][chunk_token_size] = %v, want 256", setup["chunk_token_size"])
}
// Defaults must still be present for other file types.
- if _, ok := pc.Setups["pdf"]; !ok {
+ if _, ok := pc.setups["pdf"]; !ok {
t.Errorf("Setups[pdf] missing; override should not erase defaults")
}
}
+func TestParserComponent_NewOwnsNestedSetup(t *testing.T) {
+ vlm := map[string]any{"llm_id": "original"}
+ c, err := NewParserComponent(map[string]any{
+ "pdf": map[string]any{"vlm": vlm},
+ })
+ if err != nil {
+ t.Fatalf("NewParserComponent: %v", err)
+ }
+ pc, ok := c.(*ParserComponent)
+ if !ok {
+ t.Fatalf("NewParserComponent returned %T, want *ParserComponent", c)
+ }
+
+ vlm["llm_id"] = "mutated"
+ gotVLM, ok := pc.setups["pdf"]["vlm"].(map[string]any)
+ if !ok {
+ t.Fatalf("pdf.vlm = %T, want map[string]any", pc.setups["pdf"]["vlm"])
+ }
+ if got := gotVLM["llm_id"]; got != "original" {
+ t.Errorf("pdf.vlm.llm_id = %v after caller mutation, want original", got)
+ }
+}
+
// TestParserComponent_Invoke_DocIDCarried asserts the optional
// doc_id input flows through to the "name" output.
func TestParserComponent_Invoke_DocIDCarried(t *testing.T) {
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": "x",
"doc_id": "doc-123",
@@ -311,17 +361,17 @@ func TestParserComponent_Invoke_ResolvesBinaryFromDocID(t *testing.T) {
t.Fatalf("seed doc: %v", err)
}
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
out, err := c.Invoke(ctx, db, map[string]any{"doc_id": "doc-parser"})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- pages, ok := out["pages"].([]schema.Page)
- if !ok || len(pages) != 2 {
- t.Fatalf("pages = %T/%v, want 2 schema.Page entries", out["pages"], out["pages"])
+ jsonItems, ok := out["json"].([]map[string]any)
+ if !ok || len(jsonItems) != 2 {
+ t.Fatalf("json = %T/%v, want 2 items", out["json"], out["json"])
}
- if pages[0]["text"] != "alpha" || pages[1]["text"] != "beta" {
- t.Fatalf("pages = %+v, want [alpha beta]", pages)
+ if jsonItems[0]["text"] != "alpha" || jsonItems[1]["text"] != "beta" {
+ t.Fatalf("json = %+v, want [alpha beta]", jsonItems)
}
if got, _ := out["name"].(string); got != "doc-parser" {
t.Fatalf("name = %q, want %q", got, "doc-parser")
@@ -335,7 +385,7 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {
t.Fatalf("seed storage: %v", err)
}
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
out, err := c.Invoke(ctx, nil, map[string]any{
"bucket": "bucket-1",
"path": "docs/explicit.txt",
@@ -343,12 +393,12 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- pages, ok := out["pages"].([]schema.Page)
- if !ok || len(pages) != 1 {
- t.Fatalf("pages = %T/%v, want 1 schema.Page entry", out["pages"], out["pages"])
+ jsonItems, ok := out["json"].([]map[string]any)
+ if !ok || len(jsonItems) != 1 {
+ t.Fatalf("json = %T/%v, want 1 item", out["json"], out["json"])
}
- if got := pages[0]["text"]; got != "bucket content" {
- t.Fatalf("pages[0][text] = %q, want %q", got, "bucket content")
+ if got := jsonItems[0]["text"]; got != "bucket content" {
+ t.Fatalf("json[0][text] = %q, want %q", got, "bucket content")
}
}
@@ -358,7 +408,7 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {
// look like this if a caller mistakenly handed a base64 string
// without decoding it).
func TestParserComponent_Invoke_RejectsInvalidUTF8(t *testing.T) {
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
_, err := c.Invoke(t.Context(), nil, map[string]any{
// 0xFF alone is not valid UTF-8 start byte.
"binary": string([]byte{0xFF, 0xFE, 0xFD}),
@@ -375,49 +425,21 @@ func TestParserComponent_Invoke_RejectsInvalidUTF8(t *testing.T) {
// caller's normal form ([]byte) — the alternative to a UTF-8
// string.
func TestParserComponent_Invoke_AcceptsBytes(t *testing.T) {
- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}
+ c := &ParserComponent{}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("alpha\fbeta"),
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- pages, ok := out["pages"].([]schema.Page)
+ jsonItems, ok := out["json"].([]map[string]any)
if !ok {
- t.Fatalf("pages: got %T", out["pages"])
+ t.Fatalf("json: got %T", out["json"])
}
- if len(pages) != 2 {
- t.Fatalf("pages len = %d, want 2", len(pages))
+ if len(jsonItems) != 2 {
+ t.Fatalf("json len = %d, want 2", len(jsonItems))
}
- if pages[0]["text"] != "alpha" || pages[1]["text"] != "beta" {
- t.Errorf("pages = %+v, want [alpha beta]", pages)
- }
-}
-
-// TestBuildPagesFromBytes_FormatAgnostic pins the page-builder
-// contract: buildPagesFromBytes does not resolve parsers or inspect
-// file families. It only wraps already-prepared page bytes into
-// schema.Page items.
-func TestBuildPagesFromBytes_FormatAgnostic(t *testing.T) {
- got, err := buildPagesFromBytes(t.Context(), [][]byte{
- []byte("first page from dispatch"),
- []byte("<table>second page from html dispatch</table>"),
- }, "")
- if err != nil {
- t.Fatalf("buildPagesFromBytes: %v", err)
- }
- if len(got) != 2 {
- t.Fatalf("len(got) = %d, want 2", len(got))
- }
- if got[0]["text"] != "first page from dispatch" {
- t.Fatalf("got[0][text] = %v, want first page from dispatch", got[0]["text"])
- }
- if got[1]["text"] != "<table>second page from html dispatch</table>" {
- t.Fatalf("got[1][text] = %v, want HTML payload preserved verbatim", got[1]["text"])
- }
- for i := range got {
- if got[i]["doc_type_kwd"] != "text" {
- t.Fatalf("got[%d][doc_type_kwd] = %v, want text", i, got[i]["doc_type_kwd"])
- }
+ if jsonItems[0]["text"] != "alpha" || jsonItems[1]["text"] != "beta" {
+ t.Errorf("json = %+v, want [alpha beta]", jsonItems)
}
}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 40265d9451..b18a5e2d15 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -24,6 +24,7 @@ import (
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
"go.uber.org/zap"
@@ -68,13 +69,13 @@ func maybeDispatchPDFVision(
binary []byte,
inputs map[string]any,
setups map[string]schema.ParserSetup,
-) (parserDispatchResult, bool, error) {
+) (parser.ParseResult, bool, error) {
if fileType != utility.FileTypePDF {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
setup, ok := setups["pdf"]
if !ok {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
method := getStringOr(setup, "parse_method", "")
@@ -113,12 +114,12 @@ func maybeDispatchPDFVision(
zap.String("layout_recognizer", layout),
zap.String("tenant_id", tenantID))
if tenantID == "" {
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: MinerU requires tenant_id")
}
res, err := dispatchMinerUPDF(ctx, db, filename, binary, tenantID, setup)
if err != nil {
- return parserDispatchResult{}, true, err
+ return parser.ParseResult{}, true, err
}
return res, true, nil
}
@@ -150,7 +151,7 @@ func maybeDispatchPDFVision(
}
if isPaddleOCRMatch || isPaddleOCRByUUID {
if tenantID == "" {
- return parserDispatchResult{}, true,
+ return parser.ParseResult{}, true,
fmt.Errorf("parser: PaddleOCR requires tenant_id")
}
// Only a resolved UUID may be passed to the model resolver; a string
@@ -162,27 +163,27 @@ func maybeDispatchPDFVision(
}
res, err := dispatchPaddleOCRPdf(ctx, db, filename, binary, tenantID, setup, dispatchModelID)
if err != nil {
- return parserDispatchResult{}, true, err
+ return parser.ParseResult{}, true, err
}
return res, true, nil
}
modelID, useVision := resolvePDFVisionModelID(setup)
if !useVision {
- return parserDispatchResult{}, false, nil
+ return parser.ParseResult{}, false, nil
}
if tenantID == "" {
- return parserDispatchResult{}, true, fmt.Errorf(
+ return parser.ParseResult{}, true, fmt.Errorf(
`parser: pdf parse_method %q requires tenant_id to resolve VLM model`, modelID)
}
- res, err := dispatchPDFVision(ctx, db, filename, binary, tenantID, modelID, setup)
+ res, err := dispatchPDFVision(ctx, db, filename, binary, tenantID, modelID)
if err != nil {
- return parserDispatchResult{}, true, err
+ return parser.ParseResult{}, true, err
}
return res, true, nil
}
-func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, binary []byte, tenantID string, setup schema.ParserSetup, modelRef string) (parserDispatchResult, error) {
+func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, binary []byte, tenantID string, setup schema.ParserSetup, modelRef string) (parser.ParseResult, error) {
baseURL := strings.TrimRight(getStringOr(setup, "monkeyocrv2_server_url", os.Getenv(common.EnvMonkeyOCRv2ServerURL)), "/")
timeout := monkeyOCRv2RequestTimeout(setup, nil)
if tenantID != "" {
@@ -196,7 +197,7 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
}
if err == nil {
if !strings.EqualFold(driver.Name(), "monkeyocrv2") {
- return parserDispatchResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires a MonkeyOCRv2 OCR model; found %q", driver.Name())
+ return parser.ParseResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires a MonkeyOCRv2 OCR model; found %q", driver.Name())
}
if apiConfig != nil && apiConfig.BaseURL != nil && strings.TrimSpace(*apiConfig.BaseURL) != "" {
baseURL = strings.TrimRight(*apiConfig.BaseURL, "/")
@@ -205,21 +206,53 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
}
timeout = monkeyOCRv2RequestTimeout(setup, apiConfig)
} else if baseURL == "" {
- return parserDispatchResult{}, fmt.Errorf("parser: MonkeyOCRv2 model: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: MonkeyOCRv2 model: %w", err)
}
}
if baseURL == "" {
- return parserDispatchResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires monkeyocrv2_server_url or MONKEYOCRV2_SERVER_URL")
+ return parser.ParseResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires monkeyocrv2_server_url or MONKEYOCRV2_SERVER_URL")
}
zipBytes, err := monkeyOCRv2ParseRequest(ctx, baseURL+"/parse", filename, binary, timeout)
if err != nil {
- return parserDispatchResult{}, err
+ return parser.ParseResult{}, err
}
sections, err := monkeyOCRv2ExtractSections(zipBytes)
if err != nil {
- return parserDispatchResult{}, err
+ return parser.ParseResult{}, err
+ }
+ md := strings.Join(sections, "\n\n")
+ return buildMarkdownOCRDispatchResult(md), nil
+}
+
+func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []map[string]any {
+ mp, err := parser.NewMarkdownParser(parser.GoMarkdown)
+ if err != nil {
+ warnParserNormalizationFallback(filename, "markdown", err)
+ return []map[string]any{parser.NewTextJSONItem(mdText)}
+ }
+ // OCR Markdown is an untrusted backend response. Keep normalization
+ // deterministic and local: embedded data URIs remain available, while
+ // remote image URLs do not introduce network I/O at the Parser boundary.
+ mp.FetchRemoteImages = false
+ res := mp.ParseWithResult(ctx, filename, []byte(mdText))
+ if res.Err != nil {
+ warnParserNormalizationFallback(filename, "markdown", res.Err)
+ return []map[string]any{parser.NewTextJSONItem(mdText)}
+ }
+ if len(res.JSON) == 0 {
+ warnParserNormalizationFallback(filename, "markdown", fmt.Errorf("parser returned no JSON items"))
+ return []map[string]any{parser.NewTextJSONItem(mdText)}
+ }
+ return res.JSON
+}
+
+func buildMarkdownOCRDispatchResult(md string) parser.ParseResult {
+ // Keep the backend's Markdown intact here. buildParserOutputs is the
+ // single normalization boundary for all non-JSON parser responses.
+ return parser.ParseResult{
+ OutputFormat: "markdown",
+ Markdown: md,
}
- return parserDispatchResult{OutputFormat: "markdown", Markdown: strings.Join(sections, "\n\n")}, nil
}
var isMonkeyOCRv2LayoutModelID = defaultIsMonkeyOCRv2LayoutModelID
@@ -447,17 +480,17 @@ func getAnyString(object map[string]any, keys ...string) string {
func dispatchMinerUPDF(
ctx context.Context,
db *gorm.DB,
- _ string,
+ filename string,
binary []byte,
tenantID string,
setup schema.ParserSetup,
-) (parserDispatchResult, error) {
+) (parser.ParseResult, error) {
driver, _, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeOCR)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: MinerU model: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU model: %w", err)
}
if !isMinerUDriver(driver) {
- return parserDispatchResult{}, fmt.Errorf(
+ return parser.ParseResult{}, fmt.Errorf(
"parser: MinerU requires a MinerU OCR model; found %q. Please add a MinerU OCR model to your tenant", driver.Name())
}
@@ -478,12 +511,12 @@ func dispatchMinerUPDF(
zipBytes, err := mineruStreamParse(apiURL, apiConfig.ApiKey, binary, parseMethod, mineruLang, backend)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: MinerU stream: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)
}
sections, err := mineruExtractSections(zipBytes)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: MinerU extract: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", err)
}
var parts []string
@@ -494,17 +527,7 @@ func dispatchMinerUPDF(
}
md := strings.Join(parts, "\n")
- // MinerU always returns rendered markdown text (md), regardless of
- // the requested output_format; label the payload as markdown so the
- // downstream chunker consumes it instead of a nil JSONResult.
- if format := strings.TrimSpace(getStringOr(setup, "output_format", "markdown")); !strings.EqualFold(format, "markdown") {
- common.Warn("mineru parser: output_format %q requested but backend only returns markdown; treating result as markdown",
- zap.String("output_format", format))
- }
- return parserDispatchResult{
- OutputFormat: "markdown",
- Markdown: md,
- }, nil
+ return buildMarkdownOCRDispatchResult(md), nil
}
// resolvePaddleOCRModelForDispatch resolves the OCR model used by the
@@ -546,13 +569,13 @@ func dispatchPaddleOCRPdf(
tenantID string,
setup schema.ParserSetup,
modelID string,
-) (parserDispatchResult, error) {
+) (parser.ParseResult, error) {
driver, modelName, apiConfig, err := resolvePaddleOCRModelForDispatch(ctx, db, tenantID, modelID)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR model: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR model: %w", err)
}
if !isPaddleOCRDriver(driver) {
- return parserDispatchResult{}, fmt.Errorf(
+ return parser.ParseResult{}, fmt.Errorf(
"parser: PaddleOCR requires a PaddleOCR OCR model; found %q. Please add a PaddleOCR OCR model to your tenant", driver.Name())
}
@@ -581,7 +604,7 @@ func dispatchPaddleOCRPdf(
baseURL = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRAPIURL))
}
if baseURL == "" {
- return parserDispatchResult{}, fmt.Errorf(
+ return parser.ParseResult{}, fmt.Errorf(
"parser: PaddleOCR requires a base url from the tenant PaddleOCR OCR model or PADDLEOCR_BASE_URL")
}
@@ -611,28 +634,16 @@ func dispatchPaddleOCRPdf(
Algorithm: algorithm,
}, nil)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR OCRFile: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR OCRFile: %w", err)
}
if resp == nil || resp.Text == nil {
- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")
+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")
}
if strings.TrimSpace(*resp.Text) == "" {
- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")
+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")
}
- // PaddleOCR backends always return rendered markdown text
- // (OCRFile.Text), regardless of the requested output_format. The
- // payload MUST be labelled markdown so the downstream chunker
- // consumes the text; a non-markdown setup value only means this
- // backend cannot produce the requested layout format.
- if format := strings.TrimSpace(getStringOr(setup, "output_format", "markdown")); !strings.EqualFold(format, "markdown") {
- common.Warn("paddleocr parser: output_format %q requested but backend only returns markdown; treating result as markdown",
- zap.String("output_format", format))
- }
- return parserDispatchResult{
- OutputFormat: "markdown",
- Markdown: *resp.Text,
- }, nil
+ return buildMarkdownOCRDispatchResult(*resp.Text), nil
}
// resolveMinerUBaseURL extracts the resolved base URL from a model driver.
@@ -873,28 +884,26 @@ func dispatchPDFVision(
binary []byte,
tenantID string,
modelID string,
- setup schema.ParserSetup,
-) (parserDispatchResult, error) {
+) (parser.ParseResult, error) {
renderedPages, err := pdfVisionPageRenderer(binary)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: pdf vision render: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: pdf vision render: %w", err)
}
driver, resolvedModelName, apiConfig, err := pdfVisionModelResolver(ctx, db, tenantID, modelID)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: pdf vision model %q: %w", modelID, err)
+ return parser.ParseResult{}, fmt.Errorf("parser: pdf vision model %q: %w", modelID, err)
}
promptTemplate, err := pdfVisionPromptLoader("vision_llm_describe_prompt")
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: load vision prompt: %w", err)
+ return parser.ParseResult{}, fmt.Errorf("parser: load vision prompt: %w", err)
}
items := make([]map[string]any, 0, len(renderedPages))
- markdownParts := make([]string, 0, len(renderedPages))
for _, page := range renderedPages {
prompt := renderPDFVisionPrompt(promptTemplate, page.PageNumber)
resp, err := pdfVisionChatInvoker(ctx, driver, resolvedModelName, buildPDFVisionMessages(prompt, page.ImageURL), apiConfig)
if err != nil {
- return parserDispatchResult{}, fmt.Errorf("parser: pdf vision page %d: %w", page.PageNumber, err)
+ return parser.ParseResult{}, fmt.Errorf("parser: pdf vision page %d: %w", page.PageNumber, err)
}
text := extractVisionAnswer(resp)
positions := [][]any{{page.PageNumber, 0.0, page.WidthPts, 0.0, page.HeightPts}}
@@ -905,37 +914,19 @@ func dispatchPDFVision(
"_pdf_positions": positions,
"positions": positions,
})
- if text != "" {
- markdownParts = append(markdownParts, text)
- }
}
- outputFormat := "json"
- if v, ok := setup["output_format"].(string); ok && strings.TrimSpace(v) != "" {
- outputFormat = strings.ToLower(strings.TrimSpace(v))
- }
fileMeta := map[string]any{
"name": filename,
"page_count": len(renderedPages),
"outline": []map[string]any{},
"parse_method": modelID,
}
- switch outputFormat {
- case "json":
- return parserDispatchResult{
- OutputFormat: "json",
- File: fileMeta,
- JSON: items,
- }, nil
- case "markdown":
- return parserDispatchResult{
- OutputFormat: "markdown",
- File: fileMeta,
- Markdown: strings.TrimSpace(strings.Join(markdownParts, "\n\n")),
- }, nil
- default:
- return parserDispatchResult{}, fmt.Errorf("parser: unsupported PDF output_format %q for vision parse_method %q", outputFormat, modelID)
- }
+ return parser.ParseResult{
+ OutputFormat: "json",
+ File: fileMeta,
+ JSON: items,
+ }, nil
}
func buildPDFVisionMessages(prompt string, imageURL string) []modelModule.Message {
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 78d2cfacd8..899b3c166b 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -106,9 +106,30 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if result.OutputFormat != "markdown" || result.Markdown != "hello" {
+ if result.OutputFormat != "markdown" || result.Markdown != "hello" || result.JSON != nil {
t.Fatalf("result=%+v", result)
}
+
+ resultMD, err := dispatchMonkeyOCRv2PDF(context.Background(), nil, "sample.pdf", []byte("pdf"), "", schema.ParserSetup{"monkeyocrv2_server_url": server.URL, "output_format": "markdown"}, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resultMD.OutputFormat != "markdown" || resultMD.Markdown != "hello" {
+ t.Fatalf("resultMD=%+v", resultMD)
+ }
+}
+
+func TestBuildMarkdownOCRDispatchResultDefersJSONNormalization(t *testing.T) {
+ result := buildMarkdownOCRDispatchResult("# Heading")
+ if result.OutputFormat != "markdown" {
+ t.Fatalf("OutputFormat = %q, want markdown", result.OutputFormat)
+ }
+ if result.Markdown != "# Heading" {
+ t.Fatalf("Markdown = %q, want original OCR markdown", result.Markdown)
+ }
+ if result.JSON != nil {
+ t.Fatalf("JSON = %#v, want nil until buildParserOutputs normalization", result.JSON)
+ }
}
type monkeyOCRv2FakeDriver struct {
@@ -160,14 +181,10 @@ func (d *paddleOCRFakeDriver) OCRFile(_ context.Context, _ *string, _ []byte, _
return &modelModule.OCRFileResponse{Text: &d.text}, nil
}
-// TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown guards against the
-// format-mismatch bug: PaddleOCR backends always return markdown text via
-// OCRFile.Text, so the dispatch result MUST be labelled OutputFormat
-// "markdown" regardless of what setup["output_format"] says (the pdf default
-// is "json"). If the setup value leaked into OutputFormat, buildParserOutputs
-// would emit a nil "json" payload and the downstream TokenChunker would
-// consume an empty JSONResult -> "completed with 0 chunks".
-func TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown(t *testing.T) {
+// TestDispatchPaddleOCRPdfPreservesMarkdownForCentralNormalization verifies
+// that PaddleOCR dispatch returns backend Markdown for the shared Parser
+// output normalizer, regardless of the normalized component setup format.
+func TestDispatchPaddleOCRPdfPreservesMarkdownForCentralNormalization(t *testing.T) {
orig := resolvePaddleOCRModelForDispatch
t.Cleanup(func() { resolvePaddleOCRModelForDispatch = orig })
@@ -177,8 +194,7 @@ func TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown(t *testing.T) {
return &paddleOCRFakeDriver{text: md}, "ocr-model", &modelModule.APIConfig{BaseURL: &baseURL}, nil
}
- // The real run had output_format=json in the pdf setup; the payload must
- // still be labelled markdown because that is what the backend produced.
+ // 1. output_format="json": preserve Markdown for central normalization.
res, err := dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", schema.ParserSetup{"output_format": "json"}, "some-uuid")
if err != nil {
t.Fatalf("dispatchPaddleOCRPdf: %v", err)
@@ -186,20 +202,41 @@ func TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown(t *testing.T) {
if res.OutputFormat != "markdown" {
t.Errorf("OutputFormat = %q, want markdown", res.OutputFormat)
}
+ if res.JSON != nil {
+ t.Fatalf("res.JSON = %#v, want nil before central normalization", res.JSON)
+ }
if res.Markdown != md {
t.Errorf("Markdown = %q, want %q", res.Markdown, md)
}
- // Default setup (no output_format key) must behave identically.
- res, err = dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", nil, "some-uuid")
+ // 2. output_format="markdown": the backend payload is unchanged.
+ resMD, err := dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", schema.ParserSetup{"output_format": "markdown"}, "some-uuid")
+ if err != nil {
+ t.Fatalf("dispatchPaddleOCRPdf (markdown): %v", err)
+ }
+ if resMD.OutputFormat != "markdown" {
+ t.Errorf("OutputFormat = %q, want markdown", resMD.OutputFormat)
+ }
+ if resMD.Markdown != md {
+ t.Errorf("Markdown = %q, want %q", resMD.Markdown, md)
+ }
+ if resMD.JSON != nil {
+ t.Errorf("resMD.JSON = %#v, want nil before central normalization", resMD.JSON)
+ }
+
+ // 3. Default setup (no output_format key) follows the same central path.
+ resDef, err := dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", nil, "some-uuid")
if err != nil {
t.Fatalf("dispatchPaddleOCRPdf (default setup): %v", err)
}
- if res.OutputFormat != "markdown" {
- t.Errorf("OutputFormat = %q, want markdown", res.OutputFormat)
+ if resDef.OutputFormat != "markdown" {
+ t.Errorf("OutputFormat = %q, want markdown", resDef.OutputFormat)
}
- if res.Markdown != md {
- t.Errorf("Markdown = %q, want %q", res.Markdown, md)
+ if resDef.JSON != nil {
+ t.Fatalf("resDef.JSON = %#v, want nil before central normalization", resDef.JSON)
+ }
+ if resDef.Markdown != md {
+ t.Errorf("Markdown = %q, want %q", resDef.Markdown, md)
}
}
diff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.go
index 5f6e91d1fc..da429fa717 100644
--- a/internal/ingestion/component/schema/parser.go
+++ b/internal/ingestion/component/schema/parser.go
@@ -16,121 +16,34 @@
package schema
-// ParserFromUpstream is the upstream payload consumed by the Parser
-// component. It mirrors rag/flow/parser/schema.py:ParserFromUpstream
-// (Pydantic BaseModel with populate_by_name, extra="forbid").
-//
-// created_time: float | None (alias _created_time)
-// elapsed_time: float | None (alias _elapsed_time)
-// name: str (required)
-// file: dict | None
-// abstract: bool = False
-// author: bool = False
-type ParserFromUpstream struct {
- CreatedTime *float64 `json:"_created_time,omitempty"`
- ElapsedTime *float64 `json:"_elapsed_time,omitempty"`
-
- Name string `json:"name"`
-
- // File is the optional upstream file descriptor. Python allows None
- // when the parser is invoked via a canvas-bound doc_id path.
- File map[string]any `json:"file,omitempty"`
-
- Abstract bool `json:"abstract,omitempty"`
- Author bool `json:"author,omitempty"`
-}
-
-// Validate enforces the only required field in ParserFromUpstream: Name.
-// Returns nil when Name is non-empty.
-func (p *ParserFromUpstream) Validate() error {
- if p.Name == "" {
- return errRequiredField{Field: "name"}
- }
- return nil
-}
-
-// Page is one parsed document section. The Parser component does not emit
-// a typed page model — Python code passes around `dict` literals with
-// shape `{text, layout_type, doc_type_kwd, positions?, image?, ...}`. To
-// keep the wire schema typed without overcommitting to a parser-specific
-// shape, Page is left as a generic map and provided for forward
-// documentation; downstream chunker code operates on the same dict shape.
-type Page map[string]any
-
-// ParserSetup is the per-filetype configuration block stored on
-// ParserParam.setups[fileType]. The keys are heterogeneous (e.g.,
+// ParserSetup is a per-filetype parser configuration block. The keys are heterogeneous (e.g.,
// `parse_method`, `lang`, `output_format`, `suffix`, `fields`, `vlm`),
// so a free-form map best mirrors the Python dict literal.
type ParserSetup map[string]any
-// ParserParam is the static configuration for the Parser component.
-type ParserParam struct {
- AllowedOutputFormat map[string][]string `json:"allowed_output_format"`
-}
-
-// Defaults returns a ParserParam populated with the Python defaults —
-// the full setups table copied verbatim from
-// rag/flow/parser/parser.py:ParserParam.__init__ and the corresponding
-// allowed_output_format map.
-func (ParserParam) Defaults() ParserParam {
- return ParserParam{
- AllowedOutputFormat: map[string][]string{
- "pdf": {"json", "markdown"},
- "spreadsheet": {"json", "markdown", "html"},
- "doc": {"json", "markdown"},
- "docx": {"json", "markdown"},
- "slides": {"json"},
- "image": {"json"},
- "email": {"text", "json"},
- "markdown": {"text", "json"},
- "text&code": {"text", "json"},
- "html": {"text", "json"},
- "audio": {"text", "json"},
- "video": {},
- "epub": {"text", "json"},
- "json": {"json"},
- },
- }
-}
-
-// Validate returns nil. ParserParam's field set is fully defaulted by
-// Defaults(); the component's own `check()` method performs business
-// validation (e.g., "parse_method" must be one of the allowed set), and
-// that runs in the component implementation.
-func (ParserParam) Validate() error { return nil }
-
// ParserOutputs is the result of invoking the Parser component. The
-// Python component calls `self.set_output(...)` with a mix of
-// string-typed, list-typed, and dict-typed values. The wire schema
-// below is the typed surface consumed by downstream components.
-//
-// Mirrors what Parser sets at rag/flow/parser/parser.py:_invoke. The
-// parser writes to EITHER ("json" | "markdown" | "text" | "html") and
-// always sets "output_format" + "file" + "_ERROR".
+// wire format is "json", with structured JSON items as the only payload.
type ParserOutputs struct {
- // OutputFormat is the active output format for this run
- // (one of "json", "markdown", "text", "html"). The downstream
- // Tokenizer branches on this field.
- OutputFormat string `json:"output_format,omitempty"`
-
- // JSON holds the list of structured sections when output_format == "json".
- JSON []map[string]any `json:"json,omitempty"`
+ // Name is the resolved source filename.
+ Name string `json:"name"`
- // Markdown holds the rendered Markdown when output_format == "markdown".
- Markdown string `json:"markdown,omitempty"`
+ // OutputFormat is always "json". Downstream components consume structured JSON items.
+ OutputFormat string `json:"output_format,omitempty"`
- // Text holds the rendered plain text when output_format == "text".
- Text string `json:"text,omitempty"`
+ // JSON holds the list of structured sections (primary payload).
+ JSON []map[string]any `json:"json"`
- // HTML holds the rendered HTML when output_format == "html".
- HTML string `json:"html,omitempty"`
+ // Lang is the language used by downstream tokenization and model calls.
+ Lang string `json:"lang,omitempty"`
- // File is the upstream file descriptor with parser-derived metadata
- // (e.g., outlines) merged in. Mirrors the Python `set_output("file", ...)`
- // at parser.py:609, 791, 828.
+ // File is backend-produced file metadata (for example page_count,
+ // outline, format, or sheets). When present, it replaces rather than
+ // merges with the upstream file descriptor.
File map[string]any `json:"file,omitempty"`
- // Error is set when the component short-circuits with an error
- // message (Python: set_output("_ERROR", ...)).
- Error string `json:"_ERROR,omitempty"`
+ // DocID, Bucket, and Path let downstream chunkers reacquire the source PDF
+ // for preview cropping without carrying its binary in the payload.
+ DocID string `json:"doc_id,omitempty"`
+ Bucket string `json:"bucket,omitempty"`
+ Path string `json:"path,omitempty"`
}
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index 609b496c1c..0943c23a73 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -120,98 +120,64 @@ func TestFileOutputsJSONRoundTrip(t *testing.T) {
// Parser
// ---------------------------------------------------------------------------
-func TestParserFromUpstreamValidate(t *testing.T) {
- // Name is required.
- if err := (&ParserFromUpstream{}).Validate(); err == nil {
- t.Fatal("expected Validate to fail when Name is empty")
- }
- if err := (&ParserFromUpstream{Name: "doc.pdf"}).Validate(); err != nil {
- t.Fatalf("Validate with Name unexpectedly failed: %v", err)
- }
-}
-
-func TestParserParamDefaults(t *testing.T) {
- p := ParserParam{}.Defaults()
- if err := p.Validate(); err != nil {
- t.Fatalf("default ParserParam failed Validate: %v", err)
- }
- if got := p.AllowedOutputFormat["pdf"]; len(got) != 2 || got[0] != "json" || got[1] != "markdown" {
- t.Errorf("default pdf allowed_output_format = %v, want [json markdown]", got)
+func TestParserOutputsJSONRoundTrip(t *testing.T) {
+ original := ParserOutputs{
+ Name: "input.pdf",
+ OutputFormat: "json",
+ JSON: []map[string]any{{"text": "hello", "doc_type_kwd": "text"}},
+ Lang: "English",
+ File: map[string]any{"name": "input.pdf", "page_count": float64(1)},
+ DocID: "doc-1",
+ Bucket: "bucket-1",
+ Path: "tenant/doc-1.pdf",
}
-}
-
-func TestParserParamJSONRoundTrip(t *testing.T) {
- original := ParserParam{}.Defaults()
data, err := json.Marshal(original)
if err != nil {
t.Fatalf("marshal: %v", err)
}
- if !strings.Contains(string(data), `"allowed_output_format"`) {
- t.Errorf("expected allowed_output_format in JSON, got %s", data)
+ if !strings.Contains(string(data), `"output_format":"json"`) {
+ t.Errorf("expected output_format in JSON, got %s", data)
}
- var decoded ParserParam
- if err = json.Unmarshal(data, &decoded); err != nil {
+ var decoded ParserOutputs
+ if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
- if got := decoded.AllowedOutputFormat["pdf"]; len(got) != 2 || got[0] != "json" || got[1] != "markdown" {
- t.Errorf("round-trip lost pdf allowed_output_format: got %v", got)
- }
-}
-
-func TestParserFromUpstreamJSONRoundTrip(t *testing.T) {
- original := ParserFromUpstream{
- Name: "input.pdf",
- Abstract: true,
- Author: false,
- }
- data, err := json.Marshal(original)
- if err != nil {
- t.Fatalf("marshal: %v", err)
+ if decoded.OutputFormat != "json" {
+ t.Errorf("OutputFormat round-trip mismatch: got %q", decoded.OutputFormat)
}
- // abstract=true should be emitted; author=false has omitempty so it's
- // dropped (zero-value bool with omitempty). We test the
- // non-zero path.
- if !strings.Contains(string(data), `"abstract":true`) {
- t.Errorf("expected abstract=true in JSON, got %s", data)
+ if len(decoded.JSON) != 1 {
+ t.Errorf("JSON round-trip mismatch: got %d", len(decoded.JSON))
}
- var decoded ParserFromUpstream
- if err := json.Unmarshal(data, &decoded); err != nil {
- t.Fatalf("unmarshal: %v", err)
+ if decoded.Name != original.Name || decoded.Lang != original.Lang {
+ t.Errorf("parser identity round-trip mismatch: got name=%q lang=%q", decoded.Name, decoded.Lang)
}
- if decoded.Name != original.Name {
- t.Errorf("Name round-trip mismatch: got %q", decoded.Name)
+ if decoded.DocID != original.DocID || decoded.Bucket != original.Bucket || decoded.Path != original.Path {
+ t.Errorf("parser storage round-trip mismatch: got doc_id=%q bucket=%q path=%q", decoded.DocID, decoded.Bucket, decoded.Path)
}
- if !decoded.Abstract {
- t.Errorf("Abstract round-trip mismatch: got %v", decoded.Abstract)
- }
- // author field is omitempty, so JSON round-trip should leave it false
- // (default value) on both sides.
- if decoded.Author {
- t.Errorf("Author should be false, got true")
+ if decoded.File["name"] != "input.pdf" {
+ t.Errorf("parser file metadata round-trip mismatch: got %#v", decoded.File)
}
}
-func TestParserOutputsJSONRoundTrip(t *testing.T) {
+func TestParserOutputsJSONRoundTripPreservesEmptyItems(t *testing.T) {
original := ParserOutputs{
+ Name: "empty.txt",
OutputFormat: "json",
- JSON: []map[string]any{{"text": "hello", "doc_type_kwd": "text"}},
+ JSON: []map[string]any{},
}
data, err := json.Marshal(original)
if err != nil {
t.Fatalf("marshal: %v", err)
}
- if !strings.Contains(string(data), `"output_format":"json"`) {
- t.Errorf("expected output_format in JSON, got %s", data)
+ if !strings.Contains(string(data), `"json":[]`) {
+ t.Fatalf("empty JSON payload omitted: %s", data)
}
var decoded ParserOutputs
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
- if decoded.OutputFormat != "json" {
- t.Errorf("OutputFormat round-trip mismatch: got %q", decoded.OutputFormat)
- }
- if len(decoded.JSON) != 1 {
- t.Errorf("JSON round-trip mismatch: got %d", len(decoded.JSON))
+ if decoded.JSON == nil || len(decoded.JSON) != 0 {
+ t.Fatalf("empty JSON payload round-trip mismatch: %#v", decoded.JSON)
}
}
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index dc2f81308c..4823c97bb8 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -38,6 +38,7 @@ import (
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
"gorm.io/gorm"
@@ -162,10 +163,10 @@ func maybeDispatchVisionEnhancement(
ctx context.Context,
db *gorm.DB,
fileType utility.FileType,
- dispatched parserDispatchResult,
+ dispatched parser.ParseResult,
inputs map[string]any,
setups map[string]schema.ParserSetup,
-) (parserDispatchResult, bool, error) {
+) (parser.ParseResult, bool, error) {
// 0. FileType allowlist guard.
if !isVisionEnhancementAllowed(fileType) {
return dispatched, false, nil
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index 84b107f0cd..8d18b536ba 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -31,6 +31,7 @@ import (
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
"gorm.io/gorm"
@@ -124,9 +125,8 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
return "describe the figure in " + language, nil
})
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
- DocType: string(tc.fileType),
JSON: []map[string]any{
{"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},
{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
@@ -186,9 +186,8 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
nil,
)
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "markdown",
- DocType: "docx",
Markdown: "![Image](data:image/png;base64,abc)",
File: map[string]any{"figures": []map[string]any{{"image": "abc", "marker": "x"}}},
}
@@ -214,9 +213,8 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
}
func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
- DocType: "other",
JSON: []map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
},
@@ -240,9 +238,8 @@ func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
}
func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
- DocType: "docx",
JSON: []map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
},
@@ -267,7 +264,7 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
// returned unchanged — enhancement must not touch items when dispatched.Err != nil.
func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {
parseErr := errors.New("parse failed")
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
Err: parseErr,
OutputFormat: "json",
JSON: []map[string]any{
@@ -305,7 +302,7 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel() // pre-cancel
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
@@ -343,7 +340,7 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {
fakePrompt,
)
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
// non-string image field — filtered by target collector
@@ -390,7 +387,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
"doc_type_kwd": "image",
}
}
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: items,
}
@@ -427,7 +424,7 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
fakePrompt,
)
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
@@ -465,7 +462,7 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {
},
)
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
@@ -503,7 +500,7 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {
fakePrompt,
)
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
@@ -553,7 +550,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T
"doc_type_kwd": "image",
}
}
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: items,
}
@@ -765,7 +762,7 @@ func TestVisionEnhancement_PerCallModelPreferred(t *testing.T) {
setups := map[string]schema.ParserSetup{
"pdf": {"vlm": map[string]any{"llm_id": "custom-vlm@provider"}},
}
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
@@ -808,7 +805,7 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {
fakePrompt,
)
- dispatched := parserDispatchResult{
+ dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
{"text": "keep", "image": "!!!not-base64!!!", "doc_type_kwd": "image"},
diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go
index cacd43c796..7f50d792e4 100644
--- a/internal/parser/parser/charset_test.go
+++ b/internal/parser/parser/charset_test.go
@@ -162,12 +162,16 @@ func TestCSVParser_GBK(t *testing.T) {
if res.Err != nil {
t.Fatalf("CSVParser.ParseWithResult failed: %v", res.Err)
}
- if strings.ContainsRune(res.HTML, '\ufffd') {
- t.Errorf("CSV HTML output contains replacement runes: %s", res.HTML)
+ if len(res.JSON) == 0 {
+ t.Fatalf("CSVParser returned no JSON items")
+ }
+ tableHTML, _ := res.JSON[0]["text"].(string)
+ if strings.ContainsRune(tableHTML, '\ufffd') {
+ t.Errorf("CSV HTML output contains replacement runes: %s", tableHTML)
}
for _, want := range []string{"笔记本电脑", "电子", "特价销售,送鼠标", "无线耳机"} {
- if !strings.Contains(res.HTML, want) {
- t.Errorf("CSV HTML missing expected substring %q; got: %s", want, res.HTML)
+ if !strings.Contains(tableHTML, want) {
+ t.Errorf("CSV HTML missing expected substring %q; got: %s", want, tableHTML)
}
}
if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" && enc != "gb2312" {
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 57d6208731..64a3d206d6 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -14,8 +14,7 @@
// limitations under the License.
//
-// CSVParser renders CSV data as HTML tables, matching the spreadsheet
-// family output_format == "html" convention from ParserParam.Defaults().
+// CSVParser renders CSV data as HTML table text in structured JSON table items.
//
// Mirrors Python's deepdoc/parser/excel_parser.py:RAGFlowExcelParser.html():
// - CSV data is rendered as an HTML <table> with <caption> "Data".
@@ -40,7 +39,7 @@ import (
const csvDefaultChunkRows = defaultTableChunkRows
const csvSheetName = "Data"
-// CSVParser reads RFC-4180 CSV data and emits HTML <table> payloads.
+// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.
type CSVParser struct {
ParseMethod string
OutputFormat string
@@ -128,14 +127,17 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
decoded, encName := DecodeToUTF8(data, "text/csv")
text := string(decoded)
if strings.TrimSpace(text) == "" {
+ emptyHTML := "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>"
return ParseResult{
- OutputFormat: "html",
+ OutputFormat: spreadsheetOutputFormat,
File: map[string]any{
"name": filename,
"size": len(data),
"encoding": encName,
+ "format": "csv",
+ "sheets": 1,
},
- HTML: "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>",
+ JSON: []map[string]any{NewTableJSONItem(emptyHTML, csvSheetName, [][]float64{{1, 1, 1, 1, 1}})},
}
}
@@ -157,13 +159,26 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
chunkRows = csvDefaultChunkRows
}
+ chunks := recordsToHTMLTableChunkList(records, chunkRows, csvSheetName, 1)
+ items := make([]map[string]any, 0, len(chunks))
+ for _, ch := range chunks {
+ items = append(items, NewTableJSONItem(ch.HTML, csvSheetName, [][]float64{{
+ 1,
+ float64(ch.RowStart),
+ float64(ch.RowEnd),
+ float64(ch.ColStart),
+ float64(ch.ColEnd),
+ }}))
+ }
return ParseResult{
- OutputFormat: "html",
+ OutputFormat: spreadsheetOutputFormat,
File: map[string]any{
"name": filename,
"size": len(data),
"encoding": encName,
+ "format": "csv",
+ "sheets": 1,
},
- HTML: recordsToHTMLTableChunks(records, chunkRows, csvSheetName),
+ JSON: items,
}
}
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
new file mode 100644
index 0000000000..a8504ba47b
--- /dev/null
+++ b/internal/parser/parser/csv_parser_test.go
@@ -0,0 +1,124 @@
+package parser
+
+import (
+ "context"
+ "strings"
+ "testing"
+)
+
+func TestCSVParser_BasicHTMLAndJSON(t *testing.T) {
+ csvData := []byte("Name,Age,City\nAlice,30,New York\nBob,25,San Francisco\n")
+ p := NewCSVParser()
+
+ res := p.ParseWithResult(context.Background(), "test.csv", csvData)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+
+ // Default output format is "json"
+ if res.OutputFormat != "json" {
+ t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, "json")
+ }
+
+ if len(res.JSON) != 1 {
+ t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))
+ }
+
+ item := res.JSON[0]
+ if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {
+ t.Fatal("item text should contain rendered <table>")
+ }
+ if item["doc_type_kwd"] != "table" {
+ t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
+ }
+ if item["ck_type"] != "table" {
+ t.Errorf("item ck_type = %v, want 'table'", item["ck_type"])
+ }
+ if item["sheet"] != "Data" {
+ t.Errorf("item sheet = %v, want 'Data'", item["sheet"])
+ }
+ positions, ok := item["positions"].([][]float64)
+ if !ok || len(positions) == 0 {
+ t.Fatalf("item positions invalid: %v", item["positions"])
+ }
+ pos := positions[0]
+ // [sheet, rowStart, rowEnd, colStart, colEnd]
+ if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 3 || pos[3] != 1 || pos[4] != 3 {
+ t.Errorf("unexpected positions: %v", pos)
+ }
+}
+
+func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
+ csvData := []byte("col1,col2\nval1,val2\n")
+ p := NewCSVParser()
+ p.ConfigureFromSetup(map[string]any{
+ "output_format": "json",
+ })
+
+ res := p.ParseWithResult(context.Background(), "test.csv", csvData)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+
+ if res.OutputFormat != "json" {
+ t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))
+ }
+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
+ t.Fatal("item text should contain rendered <table>")
+ }
+}
+
+func TestCSVParser_EmptyContent(t *testing.T) {
+ p := NewCSVParser()
+ p.ConfigureFromSetup(map[string]any{
+ "output_format": "json",
+ })
+
+ res := p.ParseWithResult(context.Background(), "empty.csv", []byte(" \n\n "))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+
+ if res.OutputFormat != "json" {
+ t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))
+ }
+ item := res.JSON[0]
+ if item["doc_type_kwd"] != "table" {
+ t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
+ }
+}
+
+func TestCSVParser_Chunking(t *testing.T) {
+ // Header + 4 data rows, chunk size = 2
+ csvData := []byte("Header\nr1\nr2\nr3\nr4\n")
+ p := NewCSVParser()
+ p.ChunkRows = 2
+
+ res := p.ParseWithResult(context.Background(), "chunked.csv", csvData)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+
+ // 4 data rows with chunk_rows=2 => 2 chunks
+ if len(res.JSON) != 2 {
+ t.Fatalf("len(res.JSON) = %d, want 2", len(res.JSON))
+ }
+
+ pos1 := res.JSON[0]["positions"].([][]float64)[0]
+ pos2 := res.JSON[1]["positions"].([][]float64)[0]
+
+ // chunk 1: rows 2..3
+ if pos1[1] != 2 || pos1[2] != 3 {
+ t.Errorf("chunk 1 row range: [%v, %v], want [2, 3]", pos1[1], pos1[2])
+ }
+ // chunk 2: rows 4..5
+ if pos2[1] != 4 || pos2[2] != 5 {
+ t.Errorf("chunk 2 row range: [%v, %v], want [4, 5]", pos2[1], pos2[2])
+ }
+}
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index d4df6fa880..8bc8c4cdaa 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -26,27 +26,31 @@ import (
officeOxide "github.com/yfedoseev/office_oxide/go"
)
-type DOCParser struct{}
+type DOCParser struct {
+ OutputFormat string
+}
func NewDOCParser() *DOCParser {
- return &DOCParser{}
+ return &DOCParser{
+ OutputFormat: "json",
+ }
}
func (p *DOCParser) String() string {
return "DOCParser"
}
-// ParseWithResult extracts text for the DOC family. The Go side uses
-// office_oxide, which supports legacy .doc. We prefer the structured IR
-// (flattened to plain text) and fall back to ToMarkdown, then to PlainText.
-//
-// office_oxide recovers table structure from legacy .doc through the IR
-// (office_oxide/go v0.1.9, #116); list structure depends on whether the .doc
-// reader surfaces list elements. Paragraph and heading (heuristically
-// detected) lines are always present. When the IR carries table/list
-// structure it is preferred so that content unique to the IR view is never
-// shadowed by a longer but flatter view. OutputFormat stays "text" to keep
-// the downstream contract unchanged.
+func (p *DOCParser) ConfigureFromSetup(setup map[string]any) {
+ if p == nil || setup == nil {
+ return
+ }
+ if v, ok := setup["output_format"].(string); ok && v != "" {
+ p.OutputFormat = v
+ }
+}
+
+// ParseWithResult extracts text for the DOC family and converts it to the
+// requested output format (defaulting to "json" to match Python's default).
func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc, err := officeOxide.OpenFromBytes(data, "doc")
if err != nil {
@@ -54,16 +58,38 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
}
defer doc.Close()
- text, err := extractDocText(doc)
+ text, mdText, err := extractDocText(doc)
if err != nil {
return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}
}
- return ParseResult{
- OutputFormat: "text",
+ outFmt := p.OutputFormat
+ if outFmt == "" {
+ outFmt = "json"
+ }
+
+ markdownPayload := text
+ if strings.TrimSpace(mdText) != "" {
+ markdownPayload = mdText
+ }
+
+ res := ParseResult{
+ OutputFormat: outFmt,
File: map[string]any{"name": filename, "format": "doc"},
Text: text,
+ Markdown: markdownPayload,
}
+
+ if strings.EqualFold(outFmt, "markdown") {
+ return res
+ }
+ if strings.EqualFold(outFmt, "text") {
+ return res
+ }
+
+ res.OutputFormat = "json"
+ res.JSON = []map[string]any{NewTextJSONItem(text)}
+ return res
}
// extractDocText returns the best-effort plain text for a legacy .doc
@@ -81,7 +107,7 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
// is chosen so a sparser view never shadows a more complete one. A failure at
// every stage degrades to the PlainText error, preserving the original "no
// text at all" failure semantics.
-func extractDocText(doc *officeOxide.Document) (string, error) {
+func extractDocText(doc *officeOxide.Document) (string, string, error) {
var irJSON, irText, mdText, plainText string
if j, err := doc.ToIRJSON(); err == nil {
irJSON = j
@@ -95,9 +121,9 @@ func extractDocText(doc *officeOxide.Document) (string, error) {
} else if strings.TrimSpace(irText) == "" && strings.TrimSpace(mdText) == "" {
// Every view failed (or produced nothing): keep the original
// "no text at all" failure semantics.
- return "", err
+ return "", "", err
}
- return selectDocTextView(irJSON, irText, mdText, plainText), nil
+ return selectDocTextView(irJSON, irText, mdText, plainText), mdText, nil
}
// selectDocTextView chooses the best plain-text rendering from office_oxide's
diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.go
index 65c6056c01..8c7d1bafed 100644
--- a/internal/parser/parser/doc_parser_synth_test.go
+++ b/internal/parser/parser/doc_parser_synth_test.go
@@ -146,8 +146,11 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
if pr.Err != nil {
t.Fatalf("ParseWithResult error: %v", pr.Err)
}
- if pr.OutputFormat != "text" {
- t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "text")
+ if pr.OutputFormat != "json" {
+ t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "json")
+ }
+ if len(pr.JSON) != 1 {
+ t.Fatalf("ParseWithResult returned %d JSON items, want 1", len(pr.JSON))
}
if pr.Text == "" {
t.Fatal("ParseWithResult returned empty text — synthetic .doc was not parsed")
@@ -157,4 +160,15 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)
}
}
+ if got, _ := pr.JSON[0]["text"].(string); got != pr.Text {
+ t.Errorf("JSON text = %q, want complete parser text %q", got, pr.Text)
+ }
+
+ // Also verify that explicit output_format="text" in setup is respected
+ pText := NewDOCParser()
+ pText.ConfigureFromSetup(map[string]any{"output_format": "text"})
+ prText := pText.ParseWithResult(context.Background(), "synthetic.doc", data)
+ if prText.OutputFormat != "text" {
+ t.Errorf("configured text OutputFormat = %q, want text", prText.OutputFormat)
+ }
}
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index 99829273ef..825b6be234 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -47,7 +47,9 @@ type EmailParser struct {
}
func NewEmailParser() *EmailParser {
- return &EmailParser{}
+ return &EmailParser{
+ outputFormat: "json",
+ }
}
func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {
@@ -110,12 +112,16 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
}
content = msg
} else {
- content = parseEML(bytes.NewReader(data), p.fields)
+ var err error
+ content, err = parseEMLWithError(bytes.NewReader(data), p.fields)
+ if err != nil {
+ return ParseResult{Err: fmt.Errorf("email: .eml: %w", err)}
+ }
}
outputFormat := p.outputFormat
if outputFormat == "" {
- outputFormat = "text"
+ outputFormat = "json"
}
// Re-chunk attachments so their content becomes retrievable within the
@@ -125,27 +131,13 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
// never breaks the whole email.
extraItems, attachmentText := p.rechunkEmailAttachments(ctx, content, depth)
- // attachments has been consumed by rechunkEmailAttachments (which
- // re-parses each attachment by extension to make its content
- // retrievable). It is otherwise dead weight: jsonItemsToPages copies
- // every key into a schema.Page, but buildPagesFromBytes keeps only
- // text+doc_type_kwd, so carrying the full attachment payloads through to
- // the chunker would only bloat the intermediate pages before being
- // discarded. Drop it from the result content.
+ // Attachments have been consumed by rechunkEmailAttachments, which
+ // re-parses each one by extension into a searchable item. Keeping their
+ // raw payloads in the parent item would only bloat the parser result.
delete(content, "attachments")
- if outputFormat == "json" {
- content["doc_type_kwd"] = "text"
- items := []map[string]any{content}
- items = append(items, extraItems...)
- return ParseResult{
- OutputFormat: "json",
- File: map[string]any{"name": filename},
- JSON: items,
- }
- }
-
- // Text output: flatten fields into a single string.
+ // Text representation: flatten fields into a single string for explicit
+ // text output and callers that need a display representation.
var sb strings.Builder
for k, v := range content {
// The metadata map (every non-basic header: Received chains,
@@ -193,11 +185,46 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
sb.WriteString(attachmentText)
sb.WriteString("\n")
}
+ text := sb.String()
+
+ if outputFormat == "text" {
+ return ParseResult{
+ OutputFormat: "text",
+ File: map[string]any{"name": filename},
+ Text: text,
+ }
+ }
+
+ content["doc_type_kwd"] = "text"
+ items := []map[string]any{content}
+ if headerText := searchableEmailHeaders(content); headerText != "" {
+ items = append(items, NewTextJSONItem(headerText))
+ }
+ items = append(items, extraItems...)
return ParseResult{
- OutputFormat: "text",
+ OutputFormat: "json",
File: map[string]any{"name": filename},
- Text: sb.String(),
+ JSON: items,
+ Text: text,
+ }
+}
+
+// searchableEmailHeaders returns the user-facing headers as a separate JSON
+// text item. The main email item keeps its structured fields and body; the
+// chunker indexes only item text, so headers need their own text item.
+func searchableEmailHeaders(content map[string]any) string {
+ var sb strings.Builder
+ for _, key := range []string{"from", "to", "cc", "bcc", "date", "subject"} {
+ value, ok := content[key].(string)
+ if !ok || value == "" {
+ continue
+ }
+ sb.WriteString(key)
+ sb.WriteString(":")
+ sb.WriteString(value)
+ sb.WriteString("\n")
}
+ return sb.String()
}
// -- field set helpers --
@@ -236,14 +263,13 @@ func decodeHeaderWord(val string) string {
// -- .eml parsing (RFC 5322 with multipart support) --
-func parseEML(r io.Reader, fields []string) map[string]any {
+func parseEMLWithError(r io.Reader, fields []string) (map[string]any, error) {
target := targetFieldsSet(fields)
content := map[string]any{}
msg, err := mail.ReadMessage(r)
if err != nil {
- content["error"] = fmt.Sprintf("email: parse error: %v", err)
- return content
+ return nil, err
}
// Headers. net/mail does not decode RFC 2047 encoded-words, so decode
@@ -299,7 +325,7 @@ func parseEML(r io.Reader, fields []string) map[string]any {
}
}
- return content
+ return content, nil
}
// readMailBody reads the body of an email message, handling
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index 3aa93711aa..acc1d07e4c 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -20,6 +20,7 @@ import (
"bytes"
"context"
"encoding/base64"
+ "io"
"mime/multipart"
"net/textproto"
"os"
@@ -28,6 +29,15 @@ import (
"time"
)
+func mustParseEML(t *testing.T, reader io.Reader, fields []string) map[string]any {
+ t.Helper()
+ content, err := parseEMLWithError(reader, fields)
+ if err != nil {
+ t.Fatalf("parseEMLWithError: %v", err)
+ }
+ return content
+}
+
func TestEmailParser_EmlJSON(t *testing.T) {
ctx := t.Context()
raw := strings.Join([]string{
@@ -55,8 +65,8 @@ func TestEmailParser_EmlJSON(t *testing.T) {
if result.OutputFormat != "json" {
t.Fatalf("expected output_format json, got %q", result.OutputFormat)
}
- if len(result.JSON) != 1 {
- t.Fatalf("expected 1 JSON item, got %d", len(result.JSON))
+ if len(result.JSON) != 2 {
+ t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))
}
item := result.JSON[0]
@@ -82,6 +92,31 @@ func TestEmailParser_EmlJSON(t *testing.T) {
if v, ok := item["doc_type_kwd"].(string); !ok || v != "text" {
t.Errorf("doc_type_kwd: got %q", v)
}
+
+ header := result.JSON[1]
+ headerText, ok := header["text"].(string)
+ if !ok {
+ t.Fatalf("header item text missing: %#v", header)
+ }
+ for _, want := range []string{"from:[email protected]", "to:[email protected]", "subject:Test Email"} {
+ if !strings.Contains(headerText, want) {
+ t.Errorf("header item missing %q: %q", want, headerText)
+ }
+ }
+ if strings.Contains(headerText, "This is the body of the test email.") {
+ t.Errorf("header item must not duplicate body text: %q", headerText)
+ }
+ if v, ok := header["doc_type_kwd"].(string); !ok || v != "text" {
+ t.Errorf("header doc_type_kwd: got %q", v)
+ }
+}
+
+func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {
+ p := NewEmailParser()
+ result := p.ParseWithResult(t.Context(), "broken.eml", []byte("not a valid RFC 5322 header"))
+ if result.Err == nil {
+ t.Fatalf("expected malformed EML error, got result %#v", result)
+ }
}
func TestEmailParser_EmlText(t *testing.T) {
@@ -233,8 +268,8 @@ func TestEmailParser_MsgSupported(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if len(result.JSON) != 1 {
- t.Fatalf("expected 1 JSON item, got %d", len(result.JSON))
+ if len(result.JSON) != 2 {
+ t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))
}
item := result.JSON[0]
@@ -349,7 +384,7 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {
t.Fatalf("read fixture: %v", err)
}
- p := NewEmailParser() // text output; default fields
+ p := NewEmailParser() // json output; default fields
p.ConfigureFromSetup(map[string]any{
"fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},
})
@@ -357,8 +392,11 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- if result.OutputFormat != "text" {
- t.Fatalf("output format = %q, want text", result.OutputFormat)
+ if result.OutputFormat != "json" {
+ t.Fatalf("output format = %q, want json", result.OutputFormat)
+ }
+ if len(result.JSON) == 0 {
+ t.Fatalf("expected non-empty JSON items")
}
if !strings.Contains(result.Text, "subject:asdf") {
@@ -421,7 +459,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {
t.Error("attachments must be dropped from the final ParseResult")
}
// ...and verify the .eml branch still decodes the base64 attachment.
- eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})
+ eml := mustParseEML(t, bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})
atts, ok := eml["attachments"].([]map[string]any)
if !ok {
t.Fatalf("attachments missing or wrong type: %T", eml["attachments"])
@@ -507,7 +545,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {
t.Error("attachments must be dropped from the final ParseResult")
}
// ...and verify the .eml branch still decodes the base64 attachment.
- eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})
+ eml := mustParseEML(t, bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})
atts, ok := eml["attachments"].([]map[string]any)
if !ok || len(atts) != 1 {
t.Fatalf("expected 1 attachment, got %d", len(atts))
@@ -933,7 +971,7 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {
t.Error("attachments must be dropped from the final ParseResult")
}
// ...and verify the .eml branch still extracts them even without body.
- eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "attachments"})
+ eml := mustParseEML(t, bytes.NewReader([]byte(raw)), []string{"from", "attachments"})
atts, ok := eml["attachments"].([]map[string]any)
if !ok {
t.Fatalf("attachments missing or wrong type: %T", eml["attachments"])
@@ -1254,7 +1292,7 @@ func TestParseEML_GB2312EncodedSubject(t *testing.T) {
"body",
}, "\r\n")
- content := parseEML(strings.NewReader(raw), []string{"subject"})
+ content := mustParseEML(t, strings.NewReader(raw), []string{"subject"})
if content["subject"] != "中文" {
t.Errorf("subject = %q, want 中文", content["subject"])
}
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 680db0a5c1..bb3dce85be 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -51,13 +51,18 @@ type MarkdownParser struct {
OutputFormat string
VLM map[string]any
FlattenMediaToText bool
+ // FetchRemoteImages controls whether HTTP(S) Markdown images are
+ // downloaded. NewMarkdownParser enables it for direct Markdown parsing;
+ // callers normalizing untrusted backend responses may disable it.
+ FetchRemoteImages bool
}
func NewMarkdownParser(libType string) (*MarkdownParser, error) {
switch libType {
case GoMarkdown:
return &MarkdownParser{
- libType: GoMarkdown,
+ libType: GoMarkdown,
+ FetchRemoteImages: true,
}, nil
default:
return nil, fmt.Errorf("unsupported Markdown library type: %s", libType)
@@ -103,7 +108,7 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, d
doc := markdownNew().Parse([]byte(rendered))
var items []map[string]any
- walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText)
+ walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText, p.FetchRemoteImages)
if items == nil {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
@@ -275,7 +280,7 @@ func markdownTableCells(line string) []string {
// doc_type_kwd:"table" to keep the table whole and attach table context
// to neighbouring chunks (chunker/token.go). Non-table HTML blocks
// (<div>, <style>, …) are emitted as ordinary text with no ck_type.
-func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten bool) {
+func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool) {
for _, child := range doc.GetChildren() {
var ckType string
var docTypeKwd string
@@ -342,10 +347,13 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten b
// flatten is true, keep doc_type_kwd="text" (Python
// parser.py:1034: flatten_media_to_text overrides image).
if imgURL, ok := findBlockImage(child); ok {
- if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {
- item["image"] = imgData
- if !flatten {
- item["doc_type_kwd"] = "image"
+ isRemote := strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")
+ if fetchRemoteImages || !isRemote {
+ if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {
+ item["image"] = imgData
+ if !flatten {
+ item["doc_type_kwd"] = "image"
+ }
}
}
}
diff --git a/internal/parser/parser/office_parsers_no_cgo.go b/internal/parser/parser/office_parsers_no_cgo.go
index 8f1443fde2..b00d0c6f60 100644
--- a/internal/parser/parser/office_parsers_no_cgo.go
+++ b/internal/parser/parser/office_parsers_no_cgo.go
@@ -62,11 +62,6 @@ func NewDOCXParser() *DOCXParser {
return &DOCXParser{}
}
-func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
- // No-op in the no-CGO stub: the real implementation in
- // docx_parser.go reads output_format from setup.
-}
-
func (p *DOCXParser) String() string {
return "DOCXParser(no-cgo)"
}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index dc50ba0c54..fd4b81f356 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -161,25 +161,7 @@ type htmlTableChunk struct {
ColEnd int
}
-// recordsToHTMLTableChunks renders records as one or more self-contained HTML
-// <table> chunks. The first row is always the header (<th>). Data rows are
-// split into chunks of chunkRows, each chunk being a complete <table> with
-// <caption> and a repeated header row. Chunks are joined with newlines.
-//
-// The tag schema is <table><caption>{caption}</caption><tr><th>…</th></tr>
-// <tr><td>…</td></tr>…</table>. Rows are intentionally NOT wrapped in
-// <thead>/<tbody>, so every <table> is one atomic chunk that downstream
-// chunkers can consume independently.
-func recordsToHTMLTableChunks(records [][]string, chunkRows int, caption string) string {
- chunks := recordsToHTMLTableChunkList(records, chunkRows, caption, 1)
- parts := make([]string, len(chunks))
- for i, ch := range chunks {
- parts[i] = ch.HTML
- }
- return strings.Join(parts, "")
-}
-
-// recordsToHTMLTableChunkList is the structured form of recordsToHTMLTableChunks.
+// recordsToHTMLTableChunkList renders records as structured htmlTableChunk items.
// headerRowAbs is the 1-based workbook row number of records[0] (normally 1).
func recordsToHTMLTableChunkList(records [][]string, chunkRows int, caption string, headerRowAbs int) []htmlTableChunk {
if headerRowAbs <= 0 {
@@ -400,7 +382,7 @@ func mergeExtentCol(ranges []mergeRange) int {
}
// padRowToWidth grows a single row to at least maxCol, padding with empty
-// strings. Only the header row is padded (see renderSheetTables): merged-master
+// strings. Only the header row is padded: merged-master
// text is inherited into the header alone, so data rows must not be widened —
// widening them would emit a sea of empty <td> cells for every far merge in the
// sheet and is the memory blow-up flagged in review.
@@ -634,22 +616,6 @@ func decodeChunkRows(setup map[string]any) int {
return defaultTableChunkRows
}
-// renderSheetTables renders a single workbook sheet into one or more
-// self-contained <table> chunks using the shared spreadsheet-HTML contract:
-// detect the header row, inherit merged-master text into the header, and split
-// data into chunkRows-sized atomic tables each repeating the header.
-func renderSheetTables(f *excelize.File, sheet string, chunkRows int) (string, []string, error) {
- chunks, warnings, err := renderSheetTableChunks(f, sheet, chunkRows)
- if err != nil {
- return "", warnings, err
- }
- parts := make([]string, len(chunks))
- for i, ch := range chunks {
- parts[i] = ch.HTML
- }
- return strings.Join(parts, ""), warnings, nil
-}
-
func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]htmlTableChunk, []string, error) {
rows, err := f.GetRows(sheet)
if err != nil {
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index 956afb5e62..1407301720 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -103,12 +103,16 @@ func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.
}
}
-// TestRecordsToHTMLTableChunks_Alignment asserts the chunked output uses the
+// TestRecordsToHTMLTableChunkList_Alignment asserts the chunked output uses the
// shared schema: <caption>, first row as <th>, data as <td>, repeated header
// per 256-row chunk, and NO <thead>/<tbody> wrapper.
-func TestRecordsToHTMLTableChunks_Alignment(t *testing.T) {
+func TestRecordsToHTMLTableChunkList_Alignment(t *testing.T) {
records := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}
- out := recordsToHTMLTableChunks(records, 256, "Sheet1")
+ chunks := recordsToHTMLTableChunkList(records, 256, "Sheet1", 1)
+ if len(chunks) != 1 {
+ t.Fatalf("want 1 chunk, got %d", len(chunks))
+ }
+ out := chunks[0].HTML
if !strings.Contains(out, `<caption>Sheet1</caption>`) {
t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", out)
@@ -122,28 +126,26 @@ func TestRecordsToHTMLTableChunks_Alignment(t *testing.T) {
if strings.Contains(out, "<thead>") || strings.Contains(out, "<tbody>") {
t.Fatalf("must not emit <thead>/<tbody> to stay byte-compatible with Python/CSV, got:\n%s", out)
}
- // Exactly one <table> for <=256 data rows.
- if n := strings.Count(out, "<table>"); n != 1 {
- t.Fatalf("want 1 <table>, got %d:\n%s", n, out)
- }
}
-// TestRecordsToHTMLTableChunks_Chunking asserts 256-row chunking with a repeated
+// TestRecordsToHTMLTableChunkList_Chunking asserts 256-row chunking with a repeated
// header (ceil(n_data / chunk_rows) chunks).
-func TestRecordsToHTMLTableChunks_Chunking(t *testing.T) {
+func TestRecordsToHTMLTableChunkList_Chunking(t *testing.T) {
const dataRows = 300
records := make([][]string, 0, dataRows+1)
records = append(records, []string{"C1", "C2"})
for i := 0; i < dataRows; i++ {
records = append(records, []string{"x", "y"})
}
- out := recordsToHTMLTableChunks(records, 256, "S")
+ chunks := recordsToHTMLTableChunkList(records, 256, "S", 1)
// 300 data rows → ceil(300/256) = 2 chunks, each repeating the header.
- if n := strings.Count(out, "<table>"); n != 2 {
- t.Fatalf("want 2 <table> chunks, got %d", n)
+ if len(chunks) != 2 {
+ t.Fatalf("want 2 chunks, got %d", len(chunks))
}
- if n := strings.Count(out, "<tr><th>C1</th><th>C2</th></tr>"); n != 2 {
- t.Fatalf("want header repeated in both chunks, got %d", n)
+ for i, ch := range chunks {
+ if !strings.Contains(ch.HTML, "<tr><th>C1</th><th>C2</th></tr>") {
+ t.Fatalf("want header repeated in chunk %d, got %s", i, ch.HTML)
+ }
}
}
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 339e976d52..4904151a1c 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -19,9 +19,9 @@
// parsers surface enough data to reconstruct a Python-compatible
// stage-boundary payload:
//
-// output_format ∈ {"json","markdown","text","html"}
-// file (enriched metadata)
-// exactly one payload family populated (matching output_format)
+// output_format
+// file (enriched metadata)
+// backend payload to be normalized by the owning component
// err
//
// Go parser callers now consume only the structured ParseResult
@@ -33,28 +33,19 @@ package parser
import "context"
-// ParseResult is the structured return value of a successful parse.
-// Exactly one of the payload fields (JSON / Markdown / Text / HTML)
-// is populated on success, matching the Python contract — see
-// port-rag-flow-pipeline-to-go.md §4.2:
-//
-// - OutputFormat = "json" → JSON populated
-// - OutputFormat = "markdown" → Markdown populated
-// - OutputFormat = "text" → Text populated
-// - OutputFormat = "html" → HTML populated
-//
-// On failure (Err != nil), all payload fields are zero values and
-// OutputFormat is empty.
+// ParseResult is the structured return value of a parse operation. JSON holds
+// structured backend output when available; the rendered fields carry
+// non-JSON backend responses to the Parser component's normalization boundary.
+// On failure, Err is non-nil and OutputFormat is empty.
type ParseResult struct {
- // OutputFormat is the wire-compatible format the parser
- // chose. Empty when Err is non-nil.
+ // OutputFormat identifies the backend payload representation. The Parser
+ // component normalizes successful results to JSON. Empty when Err is non-nil.
OutputFormat string
- // File is the enriched file metadata the parser emits. In
- // Python this is the dict form of the original `file`
- // descriptor, augmented with format-specific keys (e.g.
- // `outline` on the PDF path, `page_count` for paginated
- // formats). Nil when the parser did not enrich.
+ // File is metadata produced by the parser backend (for example
+ // `outline` on the PDF path or `page_count` for paginated formats).
+ // Nil when the backend does not produce metadata. It is not an augmented
+ // copy of the upstream file descriptor.
//
// For the office family (docx/doc, pptx/ppt), File["format"]
// reflects the real container format detected via magic-byte
@@ -66,25 +57,22 @@ type ParseResult struct {
// echoed the requested extension.
File map[string]any
- // JSON is the structured payload when OutputFormat == "json".
+ // JSON is the structured payload when available.
// Shape depends on the parser family: PDF emits
// `[]map[string]any` with `text` + `doc_type_kwd` keys (and
// optional `image` / `layout` / `positions` fields);
// Markdown / HTML / text emit normalized
// `{text, doc_type_kwd}` items; image emits OCR/VLM result
- // items. Exactly one payload family is populated on success.
+ // items.
JSON []map[string]any
- // Markdown is the string payload when OutputFormat ==
- // "markdown". Empty otherwise.
+ // Markdown is a backend response awaiting normalization.
Markdown string
- // Text is the string payload when OutputFormat == "text".
- // Empty otherwise.
+ // Text is a backend response awaiting normalization.
Text string
- // HTML is the string payload when OutputFormat == "html".
- // Empty otherwise.
+ // HTML is a backend response awaiting normalization.
HTML string
// Err is the failure reason. On non-nil Err, all payload
@@ -100,3 +88,38 @@ type ParseResult struct {
type ParseResultProducer interface {
ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult
}
+
+// Canonical document type identifiers used in structured JSON items.
+// These align directly with Python's doc_type_kwd contract.
+const (
+ DocTypeKey = "doc_type_kwd"
+ DocTypeText = "text"
+ DocTypeTable = "table"
+ DocTypeImage = "image"
+
+ spreadsheetOutputFormat = "json"
+)
+
+// NewTextJSONItem constructs a canonical text JSON item for parser output.
+func NewTextJSONItem(text string) map[string]any {
+ return map[string]any{
+ "text": text,
+ DocTypeKey: DocTypeText,
+ }
+}
+
+// NewTableJSONItem constructs a canonical table JSON item for parser output.
+func NewTableJSONItem(html string, sheet string, positions [][]float64) map[string]any {
+ item := map[string]any{
+ "text": html,
+ DocTypeKey: DocTypeTable,
+ "ck_type": DocTypeTable,
+ }
+ if sheet != "" {
+ item["sheet"] = sheet
+ }
+ if len(positions) > 0 {
+ item["positions"] = positions
+ }
+ return item
+}
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index ffb3707755..031e378dad 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -21,6 +21,17 @@ import (
"testing"
)
+func TestNewTableJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
+ item := NewTableJSONItem("<table><tr><td>value</td></tr></table>", "Sheet1", [][]float64{{1, 2, 2, 1, 1}})
+
+ if got, want := item[DocTypeKey], DocTypeTable; got != want {
+ t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)
+ }
+ if got, want := item["ck_type"], DocTypeTable; got != want {
+ t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)
+ }
+}
+
// TestParseResult_Contract pins the wire-shape guarantees
// port-rag-flow-pipeline-to-go.md §6.5 requires:
//
diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go
index 47406db6f6..558ea70e6c 100644
--- a/internal/parser/parser/picture_parser.go
+++ b/internal/parser/parser/picture_parser.go
@@ -129,9 +129,8 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
// OutputFormat, VLMModelID, ImageContextSize, and LayoutRecognize
// are consumed by maybeDispatchImage at the component layer.
- // Image family only allows json (schema/parser.go, Python parser.py),
- // so default empty to json. Strict: explicit text is kept as-is
- // and will be rejected by the dispatch whitelist.
+ // Parser output is normalized to JSON at the component boundary, so an
+ // absent backend format defaults to JSON here as well.
outFmt := p.OutputFormat
if outFmt == "" {
outFmt = "json"
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index e542d8c3ec..3dc510333b 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -17,12 +17,8 @@
package parser
import (
- "bytes"
"context"
"fmt"
- "strings"
-
- "github.com/xuri/excelize/v2"
)
type XLSParser struct {
@@ -95,33 +91,20 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
}
- f, err := excelize.OpenReader(bytes.NewReader(data))
- if err != nil {
- return ParseResult{Err: fmt.Errorf("xls open: %w", err)}
- }
- defer f.Close()
-
- sheets := f.GetSheetList()
chunkRows := p.ChunkRows
if chunkRows <= 0 {
chunkRows = defaultTableChunkRows
}
- var html strings.Builder
- var warnings []string
- for _, sheet := range sheets {
- rendered, sheetWarnings, err := renderSheetTables(f, sheet, chunkRows)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}
- }
- html.WriteString(rendered)
- warnings = append(warnings, sheetWarnings...)
+ items, warnings, sheetsCount, err := parseXLSXBytes(data, chunkRows)
+ if err != nil {
+ return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}
}
return ParseResult{
- OutputFormat: "html",
- File: map[string]any{"name": filename, "format": "xls"},
- HTML: html.String(),
+ OutputFormat: spreadsheetOutputFormat,
+ File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},
+ JSON: items,
Warnings: warnings,
}
}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
new file mode 100644
index 0000000000..6aef5e68cd
--- /dev/null
+++ b/internal/parser/parser/xls_parser_test.go
@@ -0,0 +1,112 @@
+package parser
+
+import (
+ "bytes"
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/xuri/excelize/v2"
+)
+
+func createTestExcelBytes(t *testing.T) []byte {
+ t.Helper()
+ f := excelize.NewFile()
+ defer f.Close()
+
+ sheet := "Sheet1"
+ _ = f.SetCellValue(sheet, "A1", "Header1")
+ _ = f.SetCellValue(sheet, "B1", "Header2")
+ _ = f.SetCellValue(sheet, "A2", "Val1")
+ _ = f.SetCellValue(sheet, "B2", "Val2")
+
+ var buf bytes.Buffer
+ if err := f.Write(&buf); err != nil {
+ t.Fatalf("Write excel file failed: %v", err)
+ }
+ return buf.Bytes()
+}
+
+func TestXLSParser_HTMLAndJSONOutput(t *testing.T) {
+ data := createTestExcelBytes(t)
+ p, err := NewXLSParser("excelize")
+ if err != nil {
+ t.Fatalf("NewXLSParser failed: %v", err)
+ }
+
+ res := p.ParseWithResult(context.Background(), "test.xls", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+
+ // Default output format for XLSParser is "json"
+ if res.OutputFormat != "json" {
+ t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatalf("res.JSON should have structured table items")
+ }
+ item := res.JSON[0]
+ if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {
+ t.Errorf("table item text should contain rendered <table>, got %v", item["text"])
+ }
+ if item["doc_type_kwd"] != "table" {
+ t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
+ }
+ if item["ck_type"] != "table" {
+ t.Errorf("ck_type = %v, want 'table'", item["ck_type"])
+ }
+
+ // Configure output format to "json"
+ p.ConfigureFromSetup(map[string]any{
+ "output_format": "json",
+ })
+ resJSON := p.ParseWithResult(context.Background(), "test.xls", data)
+ if resJSON.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", resJSON.Err)
+ }
+ if resJSON.OutputFormat != "json" {
+ t.Errorf("resJSON.OutputFormat = %q, want 'json'", resJSON.OutputFormat)
+ }
+ if len(resJSON.JSON) == 0 {
+ t.Errorf("resJSON.JSON should not be empty")
+ }
+}
+
+func TestXLSXParser_JSONOutput(t *testing.T) {
+ data := createTestExcelBytes(t)
+ p, err := NewXLSXParser("excelize")
+ if err != nil {
+ t.Fatalf("NewXLSXParser failed: %v", err)
+ }
+
+ // Default output format is "json"
+ res := p.ParseWithResult(context.Background(), "test.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+ if res.OutputFormat != "json" {
+ t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatalf("res.JSON should have items")
+ }
+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
+ t.Errorf("item text should contain HTML table, got %v", res.JSON[0]["text"])
+ }
+
+ // Even if configured to legacy "html", format is unified to "json"
+ p.ConfigureFromSetup(map[string]any{
+ "output_format": "html",
+ })
+ resHTML := p.ParseWithResult(context.Background(), "test.xlsx", data)
+ if resHTML.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", resHTML.Err)
+ }
+ if resHTML.OutputFormat != "json" {
+ t.Errorf("resHTML.OutputFormat = %q, want 'json'", resHTML.OutputFormat)
+ }
+ if len(resHTML.JSON) == 0 {
+ t.Errorf("resHTML.JSON should have items")
+ }
+}
diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go
index 96b5a8bc73..bf3fe936d7 100644
--- a/internal/parser/parser/xls_tcadp_test.go
+++ b/internal/parser/parser/xls_tcadp_test.go
@@ -3,6 +3,7 @@ package parser
import (
"net/http"
"net/http/httptest"
+ "strings"
"testing"
)
@@ -216,11 +217,14 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.OutputFormat, "html"; got != want {
+ if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if res.HTML == "" {
- t.Fatal("HTML is empty; want rendered table")
+ if len(res.JSON) == 0 {
+ t.Fatal("JSON items is empty; want structured table items")
+ }
+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
+ t.Fatalf("JSON item text = %v; want rendered table", res.JSON[0]["text"])
}
}
diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.go
index 2d69f8cf34..e91c5c97eb 100644
--- a/internal/parser/parser/xlsx_parse_method_test.go
+++ b/internal/parser/parser/xlsx_parse_method_test.go
@@ -167,11 +167,14 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult(deepdoc): %v", res.Err)
}
- if got, want := res.OutputFormat, "html"; got != want {
+ if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if !strings.Contains(res.HTML, "<table>") {
- t.Fatalf("HTML = %q, want a rendered <table>", res.HTML)
+ if len(res.JSON) == 0 {
+ t.Fatal("JSON items is empty; want structured table items")
+ }
+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
+ t.Fatalf("JSON item text = %v, want a rendered <table>", res.JSON[0]["text"])
}
}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index e370623a91..eea92c925d 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -155,19 +155,13 @@ func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int
}
warnings = append(warnings, sheetWarnings...)
for _, table := range tables {
- items = append(items, map[string]any{
- "text": table.HTML,
- "doc_type_kwd": "table",
- "ck_type": "table",
- "sheet": sheet,
- "positions": [][]float64{{
- float64(sheetIdx + 1),
- float64(table.RowStart),
- float64(table.RowEnd),
- float64(table.ColStart),
- float64(table.ColEnd),
- }},
- })
+ items = append(items, NewTableJSONItem(table.HTML, sheet, [][]float64{{
+ float64(sheetIdx + 1),
+ float64(table.RowStart),
+ float64(table.RowEnd),
+ float64(table.ColStart),
+ float64(table.ColEnd),
+ }}))
}
images, imageWarnings := extractXLSXImages(f, sheet)
items = append(items, images...)
@@ -178,7 +172,7 @@ func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int
func xlsxParseResult(filename string, items []map[string]any, warnings []string, sheets int) ParseResult {
return ParseResult{
- OutputFormat: "json",
+ OutputFormat: spreadsheetOutputFormat,
File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},
JSON: items,
Warnings: warnings,
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index e7cd72a19d..5e0d8b9c02 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -210,14 +210,28 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,
if res.Err != nil {
return "", res.Err
}
- switch res.OutputFormat {
- case "text":
- content = res.Text
- case "markdown":
- content = res.Markdown
- case "html":
- content = res.HTML
- case "json":
+ parsed, err := parseResultText(res)
+ if err != nil {
+ return "", err
+ }
+ content = parsed
+ }
+ return fmt.Sprintf("\n -----------------\nFile: %s\nContent as following: \n%s", filename, content), nil
+}
+
+// parseResultText converts a parser result into the readable text expected by
+// sys.files. JSON results are flattened in item order, preferring each item's
+// text field and serializing items without one as a final fallback.
+func parseResultText(res parser.ParseResult) (string, error) {
+ switch strings.ToLower(strings.TrimSpace(res.OutputFormat)) {
+ case "text":
+ return res.Text, nil
+ case "markdown":
+ return res.Markdown, nil
+ case "html":
+ return res.HTML, nil
+ case "json":
+ if len(res.JSON) > 0 {
parts := make([]string, 0, len(res.JSON))
for _, item := range res.JSON {
if text, ok := item["text"].(string); ok {
@@ -230,10 +244,20 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,
}
parts = append(parts, string(raw))
}
- content = strings.Join(parts, "\n")
+ return strings.Join(parts, "\n"), nil
}
+ // Some legacy parsers mark the result as JSON while only populating a
+ // rendered companion field. Preserve that content instead of returning
+ // an empty sys.files value.
+ for _, fallback := range []string{res.Markdown, res.HTML, res.Text} {
+ if fallback != "" {
+ return fallback, nil
+ }
+ }
+ return "", nil
+ default:
+ return "", fmt.Errorf("unsupported parser output format %q", res.OutputFormat)
}
- return fmt.Sprintf("\n -----------------\nFile: %s\nContent as following: \n%s", filename, content), nil
}
// parseFileContent tries to parse a file's contents using the appropriate parser.
@@ -251,16 +275,9 @@ func parseFileContent(ctx context.Context, filename string, data []byte) string
if res.Err != nil {
return string(data)
}
- switch res.OutputFormat {
- case "text":
- return res.Text
- case "markdown":
- return res.Markdown
- case "html":
- return res.HTML
- case "json":
- return string(data)
- default:
+ content, err := parseResultText(res)
+ if err != nil {
return string(data)
}
+ return content
}
diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.go
index 9fc4ca7f40..795de53997 100644
--- a/internal/service/file/file_content_test.go
+++ b/internal/service/file/file_content_test.go
@@ -1,6 +1,9 @@
package file
import (
+ "strings"
+
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
"testing"
)
@@ -38,16 +41,92 @@ func TestLooksLikeHTML(t *testing.T) {
}
}
-func TestParseFileContent_HTMLOutputFormat(t *testing.T) {
+func TestParseFileContent_JSONItems(t *testing.T) {
ctx := t.Context()
- // CSV parser produces OutputFormat "html".
+ // CSV parser now produces JSON table items. sys.files still needs the
+ // readable item text, not the original CSV bytes.
result := parseFileContent(ctx, "data.csv", []byte("a,b,c\n1,2,3\n"))
- if result == "" || result == string([]byte("a,b,c\n1,2,3\n")) {
- t.Skip("CSV parser not available or returned raw text; integration-only test")
+ if result == "" {
+ t.Fatal("CSV parser returned empty content")
+ }
+ if strings.Contains(result, "a,b,c\n1,2,3\n") {
+ t.Errorf("CSV content used original bytes instead of parsed item text: %q", result)
+ }
+ if !strings.Contains(result, "<table") || !strings.Contains(result, "1") {
+ t.Errorf("CSV content lost readable table text: %q", result)
+ }
+}
+
+func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T) {
+ result, err := parseResultText(parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "first"},
+ {"text": "second"},
+ {"value": 3},
+ },
+ })
+ if err != nil {
+ t.Fatalf("parseResultText: %v", err)
+ }
+ want := "first\nsecond\n{\"value\":3}"
+ if result != want {
+ t.Fatalf("parseResultText = %q, want %q", result, want)
+ }
+}
+
+func TestParseResultText_EmailJSONKeepsSearchableHeaders(t *testing.T) {
+ raw := []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: Parser contract\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEmail body")
+ p := parser.NewEmailParser()
+ p.ConfigureFromSetup(map[string]any{"output_format": "json", "fields": []string{"from", "to", "subject", "body"}})
+ parsed := p.ParseWithResult(t.Context(), "message.eml", raw)
+ if parsed.Err != nil {
+ t.Fatalf("ParseWithResult: %v", parsed.Err)
+ }
+ content, err := parseResultText(parsed)
+ if err != nil {
+ t.Fatalf("parseResultText: %v", err)
+ }
+ for _, want := range []string{"from:[email protected]", "to:[email protected]", "subject:Parser contract", "Email body"} {
+ if !strings.Contains(content, want) {
+ t.Errorf("sys.files text missing %q: %q", want, content)
+ }
+ }
+}
+
+func TestParseResultText_EmptyJSONUsesRenderedFallback(t *testing.T) {
+ result, err := parseResultText(parser.ParseResult{
+ OutputFormat: "json",
+ Markdown: "legacy markdown",
+ })
+ if err != nil {
+ t.Fatalf("parseResultText: %v", err)
+ }
+ if result != "legacy markdown" {
+ t.Fatalf("parseResultText = %q, want rendered fallback", result)
+ }
+}
+
+func TestParseResultText_EmptyJSONWithoutFallbackReturnsEmpty(t *testing.T) {
+ result, err := parseResultText(parser.ParseResult{OutputFormat: "json"})
+ if err != nil {
+ t.Fatalf("parseResultText: %v", err)
+ }
+ if result != "" {
+ t.Fatalf("parseResultText = %q, want empty text", result)
+ }
+}
+
+func TestParseAgentUploadContent_JSONUsesParsedItemText(t *testing.T) {
+ content, err := parseAgentUploadContent(t.Context(), "data.csv", []byte("a,b\n1,2\n"), "")
+ if err != nil {
+ t.Fatalf("parseAgentUploadContent: %v", err)
+ }
+ if strings.Contains(content, "a,b\n1,2\n") {
+ t.Fatalf("upload content used original bytes instead of parsed item text: %q", content)
}
- // CSV parser emits an HTML table; must not contain raw CSV comma-separated rows.
- if result == "a,b,c\n1,2,3\n" {
- t.Errorf("CSV should produce HTML output, got raw CSV: %q", result)
+ if !strings.Contains(content, "<table") || !strings.Contains(content, "1") {
+ t.Fatalf("upload content lost readable table text: %q", content)
}
}