RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): unify parser outputs to structured JSON items (#19467)
jay77721fbd374e92f39
diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.goindex f9d8d0621a..a59e2aeadb 100644--- a/internal/ingestion/component/chunker/page.go+++ b/internal/ingestion/component/chunker/page.go@@ -87,10 +87,9 @@ func (c *PageChunkerComponent) invoke(_ context.Context, inputs map[string]any)}, nil}- // The presentation template only configures pdf and slides- // parser setups, matching Python's restriction to .pptx/.ppt/.pdf.- // The upstream parser therefore always emits per-slide JSON, never- // flat text/markdown/html, so every payload keeps one-chunk-per-slide.+ // The Page chunker preserves one incoming JSON item per output chunk.+ // It has no file-family restriction: item granularity is determined by+ // the upstream parser backend.items := slideItems(upstream.JSONResult, upstream.Chunks)if len(items) == 0 {return emptyOutputs(), nildiff --git a/internal/ingestion/component/chunker/qa_id_test.go b/internal/ingestion/component/chunker/qa_id_test.goindex 22f728bc13..b175c3a87b 100644--- a/internal/ingestion/component/chunker/qa_id_test.go+++ b/internal/ingestion/component/chunker/qa_id_test.go@@ -9,7 +9,7 @@ import ()// xlsxQATableHTML is the HTML the spreadsheet parser emits for a-// headerless two-column Q&A workbook (renderSheetTables repeats the+// headerless two-column Q&A workbook (the table renderer repeats the// first row as the <th> header; extractQATable still reads it as a pair).const xlsxQATableHTML = `<table><caption>Sheet1</caption><tr><th>门打不开怎么办</th><th>请家人回来开门</th></tr>diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 93095b954f..4fe546146e 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -228,6 +228,41 @@ func TestTokenChunker_InvokeJSONPayload(t *testing.T) {}}+func TestTokenChunker_InvokeJSONPayload_IndexesHeaderOnlyEmail(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{"\n"},+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ // Email's structured body item can have no text for a header-only+ // message. Parser emits the second text item specifically so headers are+ // still indexed through the JSON path.+ items := []map[string]any{+ {"from": "[email protected]", "subject": "Status", "doc_type_kwd": "text"},+ {"text": "from:[email protected]\nsubject:Status\n", "doc_type_kwd": "text"},+ }+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "message.eml",+ "output_format": "json",+ "json": items,+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) == 0 {+ t.Fatalf("chunks missing: %#v", out["chunks"])+ }+ for _, chunk := range chunks {+ if text, _ := chunk["text"].(string); strings.Contains(text, "subject:Status") {+ return+ }+ }+ t.Fatalf("header item was not indexed: %#v", chunks)+}+// TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone is the// regression lock for #17889: when merging adjacent segments, only// "text" segments may be merged; "table"/"image" (any non-text type)@@ -281,6 +316,10 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {if got != wantTypes[i] {t.Errorf("chunk %d: doc_type_kwd = %q, want %q (full chunk: %+v)", i, got, wantTypes[i], ch)}+ ckType, _ := ch["ck_type"].(string)+ if ckType != wantTypes[i] {+ t.Errorf("chunk %d: ck_type = %q, want %q (derived from doc_type_kwd)", i, ckType, wantTypes[i])+ }}// The two text segments on either side of the table/image must remain// distinct — they must NOT be merged across the non-text segments.@@ -357,6 +396,41 @@ func TestTokenChunker_InputsOutputs_NonEmpty(t *testing.T) {}}+// TestTokenChunker_SpreadsheetTablePreservesSelectionRange guards the Parser+// JSON path: spreadsheet positions describe a sheet selection, not a PDF+// bounding box, and a table item must pass through without proportional text+// splitting or coordinate rewriting.+func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{"delimiter": "\n"})+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ wantPositions := [][]float64{{1, 2, 10, 1, 5}}+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "book.xlsx",+ "output_format": "json",+ "json": []map[string]any{{+ "text": "<table><tr><td>A</td></tr>\n<tr><td>B</td></tr></table>",+ "doc_type_kwd": "table",+ "positions": wantPositions,+ "sheet": "Sheet1",+ }},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want one unsplit table item", len(chunks))+ }+ if got := chunks[0]["ck_type"]; got != "table" {+ t.Errorf("ck_type = %v, want table", got)+ }+ if got := chunks[0]["positions"]; !reflect.DeepEqual(got, wantPositions) {+ t.Errorf("positions = %#v, want unchanged %#v", got, wantPositions)+ }+}+// TestTokenChunker_NewRejectsBadParam enforces the param validation// at construction time (mirrors python `check()`).func TestTokenChunker_NewRejectsBadParam(t *testing.T) {diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex 08eeb38c74..8fdd9cd1d6 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -21,7 +21,7 @@//// These follow the maybeDispatchPDFVision pattern: they bypass// dispatchParse and call the model directly from the component-// layer, returning a parserDispatchResult.+// layer, returning a parser.ParseResult.package component@@ -65,12 +65,12 @@ func maybeDispatchVideo(binary []byte,inputs map[string]any,setups map[string]schema.ParserSetup,-) (parserDispatchResult, bool, error) {+) (parser.ParseResult, bool, error) {if fileType != utility.FileTypeVIDEO {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}if _, ok := setups["video"]; !ok {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}// Video parsing is intentionally not implemented yet: the underlying@@ -81,8 +81,10 @@ func maybeDispatchVideo(// video_url. Returning an explicit error is safer than silently producing// a description from the prompt text alone. The real implementation must// be provider-specific (OpenAI-compatible: frame extraction -> image_url;- // Gemini: raw-bytes inline_data; Qwen: file://)- return parserDispatchResult{}, true,+ // Gemini: raw-bytes inline_data; Qwen: file://).+ // When video analysis is implemented, it emits standard Text JSON items:+ // [{"text": transcript, "doc_type_kwd": "text"}] with output_format "json".+ return parser.ParseResult{}, true,fmt.Errorf("Parser: video parsing is not yet supported; underlying video analysis capability is pending")}@@ -102,17 +104,17 @@ func maybeDispatchImage(binary []byte,inputs map[string]any,setups map[string]schema.ParserSetup,-) (parserDispatchResult, bool, error) {+) (parser.ParseResult, bool, error) {if fileType != utility.FileTypeVISUAL {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}setup, ok := setups["image"]if !ok {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}tenantID := getStringOr(inputs, "tenant_id", "")if tenantID == "" {- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: image requires tenant_id")}@@ -181,7 +183,7 @@ func maybeDispatchImage(if ocrText != "" {return imageDispatchResult(ocrText, dataURI), true, nil}- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: picture image2text model: %w", err)}@@ -205,7 +207,7 @@ func maybeDispatchImage(if ocrText != "" {return imageDispatchResult(ocrText, dataURI), true, nil}- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: picture describe: %w", err)}vlmText := ""@@ -230,10 +232,9 @@ func maybeDispatchImage(// family: a single item carrying the combined text, the image attachment// (data URI), and doc_type_kwd "image". Mirrors Python// rag/app/picture.py:71-72.-func imageDispatchResult(text, dataURI string) parserDispatchResult {- return parserDispatchResult{+func imageDispatchResult(text, dataURI string) parser.ParseResult {+ return parser.ParseResult{OutputFormat: "json",- DocType: "image",JSON: []map[string]any{{"text": text,"image": dataURI,@@ -256,17 +257,17 @@ func maybeDispatchAudio(binary []byte,inputs map[string]any,setups map[string]schema.ParserSetup,-) (parserDispatchResult, bool, error) {+) (parser.ParseResult, bool, error) {if fileType != utility.FileTypeAURAL {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}setup, ok := setups["audio"]if !ok {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}tenantID := getStringOr(inputs, "tenant_id", "")if tenantID == "" {- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: audio requires tenant_id")}@@ -286,20 +287,20 @@ func maybeDispatchAudio(driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)}if err != nil {- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: audio speech2text model: %w", err)}tmpFile, err := writeTempAudioFile(filename, binary)if err != nil {- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: audio temp file: %w", err)}defer os.Remove(tmpFile)resp, err := driver.TranscribeAudio(ctx, &modelName, &tmpFile, apiConfig, nil, nil)if err != nil {- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("Parser: audio transcription: %w", err)}@@ -308,29 +309,12 @@ func maybeDispatchAudio(transcription = resp.Text}- outputFormat, _ := setup["output_format"].(string)- if outputFormat == "" {- outputFormat = "json"- }- // Diff 2.11: when output_format is "json" the transcription must be- // carried as a JSON item. Returning it only in Text made the Invoke- // switch silently drop it (the switch has no "json" branch and the- // JSON slice was empty). Mirror the JSON-item shape used by the- // other parser branches.- if outputFormat == "json" {- return parserDispatchResult{- OutputFormat: "json",- DocType: "audio",- JSON: []map[string]any{{- "text": transcription,- "doc_type_kwd": "audio",- }},- }, true, nil- }- return parserDispatchResult{- OutputFormat: outputFormat,- DocType: "audio",- Text: transcription,+ return parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{{+ "text": transcription,+ "doc_type_kwd": "text",+ }},}, true, nil}diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex 89cfe82488..f07e675ce7 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -318,15 +318,12 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {if got, _ := res.JSON[0]["text"].(string); got != want {t.Fatalf("JSON[0].text = %q, want %q", got, want)}- if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "audio" {- t.Fatalf("JSON[0].doc_type_kwd = %q, want audio", got)+ if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "text" {+ t.Fatalf("JSON[0].doc_type_kwd = %q, want text", got)}}-// TestMaybeDispatchAudio_TextCarriesTranscription guards the text path: with-// output_format "text" the transcription stays in the Text field and JSON is-// empty (current default after aligning with Python parser.py:232).-func TestMaybeDispatchAudio_TextCarriesTranscription(t *testing.T) {+func TestMaybeDispatchAudio_IgnoresLegacyOutputFormat(t *testing.T) {origResolver := resolveTenantModelByTypedefer func() { resolveTenantModelByType = origResolver }()@@ -355,24 +352,20 @@ func TestMaybeDispatchAudio_TextCarriesTranscription(t *testing.T) {if !dispatched {t.Fatalf("expected dispatched=true for AURAL file")}- if res.OutputFormat != "text" {- t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)+ if res.OutputFormat != "json" {+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)}- if res.Text != want {- t.Fatalf("Text = %q, want %q", res.Text, want)+ if len(res.JSON) != 1 {+ t.Fatalf("JSON len = %d, want 1", len(res.JSON))}- if len(res.JSON) != 0 {- t.Fatalf("JSON len = %d, want 0 for text output", len(res.JSON))+ if got, _ := res.JSON[0]["text"].(string); got != want {+ t.Fatalf("JSON[0].text = %q, want %q", got, want)}}// TestMaybeDispatchAudio_DefaultOutputFormatJson covers the-// maybeDispatchAudio fallback: when an audio setup omits-// output_format entirely, the dispatch defaults to "json" and wraps-// the transcription as a JSON item. (The defaultSetups value is-// "text" to mirror the Python audio setup in parser.py; this test-// deliberately supplies an empty setup to exercise the fallback-// inside the dispatch itself.)+// maybeDispatchAudio always returns a JSON transcription item, including when+// the setup has no output_format key.func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {const want = "hello world"drv := &audioTranscribeDriver{transcription: want}diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 946d1da930..d264ac9513 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -28,10 +28,8 @@// - Per-page parallelism is delegated to the parser backends// (e.g. internal/deepdoc/parser/pdf fans out one worker per// page and assembles the results in page order). This-// component only reshapes the parser output into the-// schema.Page layout and keeps the deterministic, page-number-// sorted merge contract (plan §8 R8) that the downstream-// chunker / tokenizer rely on for stable chunk IDs.+// component normalizes the parser output into structured JSON+// items while preserving the backend's deterministic item order.//// - Progress (start/done callback) and elapsed-time stamping// (_created_time / _elapsed_time) are owned by the canvas@@ -74,15 +72,14 @@// tenant default models (resolveTenantModelByType) rather than// setup["vlm"]["llm_id"], so that check is intentionally omitted.//-// - NO PERSISTENCE: parsed pages live only in the per-run-// output map, exactly as the schema.Page type is intended.+// - NO PERSISTENCE: structured parser items live only in the per-run+// output map.package componentimport ("context""errors""fmt"- "sort""strings""unicode/utf8"@@ -93,6 +90,7 @@ import ("ragflow/internal/common""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser""ragflow/internal/utility")@@ -105,43 +103,41 @@ const ComponentNameParser = "Parser"const pageFormFeed = '\f'// ParserComponent runs the configured parser branch against the-// upstream "binary" payload and returns a deterministic, page--// sorted slice of schema.Page values.+// upstream "binary" payload and returns structured parser outputs.//// The instance is safe for concurrent invocation: each Invoke call// builds its own per-batch goroutine tree and merges results in// the goroutine that returned from Invoke. The static Param is// read-only after construction.type ParserComponent struct {- Setups map[string]schema.ParserSetup- Param schema.ParserParam+ setups map[string]schema.ParserSetup}// NewParserComponent constructs a Parser from a DSL param map.-// The map is decoded into schema.ParserParam.Defaults() and then-// overlaid with the supplied values. This matches the Python-// "default + override" pattern in parser.py:ParserParam.__init__.+// The default setups are overlaid with the supplied values. Historical+// output_format values are accepted but normalized to JSON so downstream+// components consume one parser output protocol. This applies to every+// family, including PDF and office documents: Markdown is an internal+// backend representation only and is never a public Parser output.//-// Param map shape (all keys optional; missing keys fall back to-// schema.ParserParam.Defaults() values):+// Param map shape (all keys optional)://// {// "pdf": map[string]any,// "docx": map[string]any,// ...-// "allowed_output_format": map[string][]string,// }//// Errors here surface as canvas compile failures so a malformed// param is caught at build time rather than mid-run.func NewParserComponent(params map[string]any) (runtime.Component, error) {- p := schema.ParserParam{}.Defaults()s := defaultSetups()if params == nil {- return &ParserComponent{Setups: s, Param: p}, nil+ normalizeParserOutputFormats(s)+ return &ParserComponent{setups: s}, nil}for k, raw := range params {- if k == "outputs" {+ if k == "outputs" || k == "allowed_output_format" {continue}ftCfg, ok := raw.(map[string]any)@@ -152,33 +148,61 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {s[k] = schema.ParserSetup{}}for fk, fv := range ftCfg {- s[k][fk] = fv+ s[k][fk] = cloneParserSetupValue(fv)}}- if rawAllowed, ok := params["allowed_output_format"].(map[string]any); ok {- allowed := make(map[string][]string, len(rawAllowed))- for fileType, raw := range rawAllowed {- list, ok := raw.([]any)- if !ok {- continue- }- formats := make([]string, 0, len(list))- for _, item := range list {- if s, ok := item.(string); ok {- formats = append(formats, s)- }- }- allowed[fileType] = formats- }- p.AllowedOutputFormat = allowed- }- pc := &ParserComponent{Setups: s, Param: p}+ normalizeParserOutputFormats(s)+ pc := &ParserComponent{setups: s}if err := pc.Check(); err != nil {return nil, fmt.Errorf("parser: %w", err)}return pc, nil}+func normalizeParserOutputFormats(setups map[string]schema.ParserSetup) {+ // The Go Parser component intentionally exposes JSON only. Keep this+ // normalization unconditional so PDF/office legacy Markdown settings+ // cannot silently select a second public output path.+ for _, setup := range setups {+ setup["output_format"] = "json"+ }+}++func cloneParserSetupValue(value any) any {+ switch v := value.(type) {+ case map[string]any:+ cloned := make(map[string]any, len(v))+ for key, nested := range v {+ cloned[key] = cloneParserSetupValue(nested)+ }+ return cloned+ case schema.ParserSetup:+ cloned := make(schema.ParserSetup, len(v))+ for key, nested := range v {+ cloned[key] = cloneParserSetupValue(nested)+ }+ return cloned+ case []any:+ cloned := make([]any, len(v))+ for i, nested := range v {+ cloned[i] = cloneParserSetupValue(nested)+ }+ return cloned+ case []string:+ return append([]string(nil), v...)+ case []int:+ return append([]int(nil), v...)+ case [][]int:+ cloned := make([][]int, len(v))+ for i, nested := range v {+ cloned[i] = append([]int(nil), nested...)+ }+ return cloned+ default:+ return value+ }+}+// Check mirrors the applicable subset of Python ParserParam.check()// (rag/flow/parser/parser.py:251-321). Runs at construction time so// a malformed DSL surfaces as a canvas compile failure rather than a@@ -186,8 +210,6 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {// (Python raises ValueError on the first failure).//// NOT covered here (intentional):-// - output_format whitelist: already enforced at Invoke time by-// resolveOutputFormat (parser_dispatch.go:100-122).// - audio/video vlm.llm_id: Go media_dispatch uses tenant default// models (resolveTenantModelByType), not setup["vlm"]["llm_id"].// The Python flow check() for vlm.llm_id does not apply — Go@@ -195,7 +217,7 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {// valid audio/video pipeline (see ingestion_pipeline_audio.json).func (c *ParserComponent) Check() error {// PDF family (parser.py:252-261).- if pdf, ok := c.Setups["pdf"]; ok {+ if pdf, ok := c.setups["pdf"]; ok {pm, _ := pdf["parse_method"].(string)if pm == "" {return errors.New("parse method abnormal. does not support empty value")@@ -214,7 +236,7 @@ func (c *ParserComponent) Check() error {}}// image family (parser.py:283-287).- if img, ok := c.Setups["image"]; ok {+ if img, ok := c.setups["image"]; ok {pm, _ := img["parse_method"].(string)// OCR mode does not need a VLM language; any other value does.if pm != "ocr" {@@ -252,7 +274,7 @@ func defaultSetups() map[string]schema.ParserSetup {"spreadsheet": {"parse_method": "deepdoc","flatten_media_to_text": false,- "output_format": "html",+ "output_format": "json","suffix": []string{"xls", "xlsx", "csv"},},"doc": {@@ -314,11 +336,11 @@ func defaultSetups() map[string]schema.ParserSetup {"aiff", "au", "midi", "wma", "realaudio", "vqf","oggvorbis", "ape",},- "output_format": "text",+ "output_format": "json",},"video": {"suffix": []string{"mp4", "avi", "mkv"},- "output_format": "text",+ "output_format": "json","prompt": "",},"epub": {@@ -336,39 +358,54 @@ func defaultSetups() map[string]schema.ParserSetup {// reads the following from the inputs map at Invoke time://// binary ([]byte, optional) — file bytes from upstream File.-// When absent, Parser resolves-// them from bucket/path or doc_id.+// When absent, Parser resolves them from+// bucket/path or doc_id.+// name (string, optional) — resolved source filename.+// file (map[string]any, optional) — source descriptor; its name is+// used when name is absent.+// file_type (string, optional) — explicit parser routing hint.+// lang (string, optional) — language forwarded to downstream stages.// doc_id (string, optional) — document ID used for naming and,// when binary is absent, storage lookup.func (c *ParserComponent) Inputs() map[string]string {return map[string]string{- "binary": "Optional file bytes ([]byte). When absent, Parser resolves them from bucket/path or doc_id.",- "doc_id": "Optional document ID (string). Used for downstream correlation and doc_id-driven storage lookup.",- "bucket": "Optional storage bucket override. Used when binary is absent.",- "path": "Optional storage object key override. Used when binary is absent.",+ "binary": "Optional file bytes ([]byte). When absent, Parser resolves them from bucket/path or doc_id.",+ "name": "Optional resolved source filename. Takes precedence over file.name.",+ "file": "Optional source file descriptor (map[string]any). file.name is used when name is absent.",+ "file_type": "Optional explicit parser routing hint (string).",+ "lang": "Optional language for downstream tokenization (string).",+ "doc_id": "Optional document ID (string). Used for downstream correlation and doc_id-driven storage lookup.",+ "bucket": "Optional storage bucket override. Used when binary is absent.",+ "path": "Optional storage object key override. Used when binary is absent.",}}// Outputs returns the public surface that downstream ingestion// components (Chunker, Tokenizer, Extractor) can wire into.//-// pages []schema.Page — sorted by PageNumber. Deterministic-// merge per plan §8 R8.-// name string — carried over from the upstream file/document+// name string — carried over from the upstream file/document// name (or doc_id when no name is available).-// output_format string — "text" when emitting text pages,-// otherwise the parser-selected wire-// format.-// _ERROR string — populated when the component short--// circuits with an error message-// (mirrors Python set_output("_ERROR", ...)).+// output_format string — always "json".+// json []map[string]any — canonical structured parser items.+// lang string — language for tokenization.+// file map[string]any — backend-produced file metadata, when present.+// doc_id string — source document ID, when present.+// bucket string — source storage bucket, when present.+// path string — source storage path, when present.+//+// Parser failures are returned as Go errors. The canvas execution wrapper+// preserves that error path and does not convert failures into an _ERROR+// output field.func (c *ParserComponent) Outputs() map[string]string {return map[string]string{- "pages": "[]schema.Page: parsed pages sorted by PageNumber.","name": "string: the upstream file/document name (or doc_id when no name is available).",- "output_format": "string: the active output format (\"text\" when emitting text pages).",+ "output_format": "string: always \"json\".",+ "json": "[]map[string]any: canonical structured parser items.","lang": "string: the language for tokenization (e.g. English, Dutch, Chinese).",- "_ERROR": "string: set on short-circuit errors.",+ "file": "map[string]any: backend-produced file metadata, when present.",+ "doc_id": "string: source document ID, when present.",+ "bucket": "string: source storage bucket, when present.",+ "path": "string: source storage object path, when present.",}}@@ -377,9 +414,9 @@ func (c *ParserComponent) Outputs() map[string]string {// Returns://// {-// "pages": []schema.Page (sorted by PageNumber),-// "name": string (from inputs["doc_id"]),-// "output_format": "text",+// "name": string (from inputs["name"], file.name, or doc_id),+// "output_format": "json",+// "json": []map[string]any,// "lang": string (from inputs["lang"]; e.g. English, Dutch),// "_created_time": RFC3339Nano (via TrackElapsed),// "_elapsed_time": float64 seconds (via TrackElapsed),@@ -389,13 +426,6 @@ func (c *ParserComponent) Outputs() map[string]string {// backends (e.g. internal/deepdoc/parser/pdf fans out one worker// per page and assembles the results in page order), so this// component does no goroutine fan-out of its own.-//-// DETERMINISTIC MERGE (plan §8 R8): after the page slice is built,-// it is sorted by PageNumber. This guarantees the same input-// produces byte-identical output across runs and is the contract-// that downstream Chunker / Tokenizer rely on for stable chunk-// IDs (chunks that span pages must reference adjacent PageNumbers-// in input order).func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {// 1. Decode the binary input.binary, err := readParserBinary(ctx, db, inputs)@@ -404,6 +434,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st}docID, _ := inputs["doc_id"].(string)filename := parserInputName(inputs, docID)+ setups := c.setups// Inject run-level metadata from Globals into inputs so media// dispatch branches (audio/image/video) can resolve tenant_id.@@ -424,29 +455,9 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// "docx", ...). Used by parser.GetParser, whose switch// arms are keyed off the utility constants.//- // - fileTypeFam — the python-side family name ("markdown",- // "docx", ...). Used by setups[fileType] and- // allowed_output_format[fileType] lookups, which are keyed- // off the python family identifiers in schema.ParserParam.- //- // For most families the two forms coincide; the divergence- // exists for Markdown ("md" vs "markdown") and slides- // ("ppt"/"pptx" vs "slides") and is intentional — the python- // ParserParam collapses the slide family into a single key.fileTypeExt := fileTypeFromInputs(inputs)- fileTypeFam := pythonFamilyName(string(fileTypeExt))- // 2a. Validate the requested output_format against the- // family-specific allowed_output_format whitelist. We do- // this even when no setups entry exists so a misconfigured- // DSL surfaces as _ERROR instead of a silent fallback.- if _, hasSetup := c.Setups[fileTypeFam]; hasSetup {- if _, verr := resolveOutputFormat(fileTypeFam, c.Setups, c.Param.AllowedOutputFormat); verr != nil {- return nil, verr- }- }-- dispatched, handledVision, visionErr := maybeDispatchPDFVision(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)+ dispatched, handledVision, visionErr := maybeDispatchPDFVision(ctx, db, fileTypeExt, filename, binary, inputs, setups)if visionErr != nil {return nil, visionErr}@@ -455,7 +466,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stif !handledVision {// Video dispatch: IMAGE2TEXT vision chat.// Mirrors Python's _video().- dispatched, handledMedia, visionErr = maybeDispatchVideo(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)+ dispatched, handledMedia, visionErr = maybeDispatchVideo(ctx, db, fileTypeExt, filename, binary, inputs, setups)if visionErr != nil {return nil, visionErr}@@ -464,7 +475,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stif !handledVision && !handledMedia {// Image/Picture dispatch: OCR + IMAGE2TEXT vision describe.// Mirrors Python's rag/app/picture.py:chunk() image branch.- dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)+ dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups)if visionErr != nil {return nil, visionErr}@@ -473,13 +484,13 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stif !handledVision && !handledMedia && !handledImage {// Audio dispatch: SPEECH2TEXT transcription.// Mirrors Python's rag/app/audio.py:chunk().- dispatched, handledAudio, visionErr = maybeDispatchAudio(ctx, db, fileTypeExt, filename, binary, inputs, c.Setups)+ dispatched, handledAudio, visionErr = maybeDispatchAudio(ctx, db, fileTypeExt, filename, binary, inputs, setups)if visionErr != nil {return nil, visionErr}}if !handledVision && !handledMedia && !handledImage && !handledAudio {- dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, c.Setups)+ dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups)// Vision figure enhancement: on the JSON output path,// append vision-model descriptions to embedded image and@@ -488,7 +499,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// Errors (including context cancellation) are intentionally// discarded — enhancement is best-effort, matching Python's// try/except pass pattern.- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.Setups)+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)}// Known/supported families must fail loudly when dispatch or// parsing breaks. Only unknown families keep the raw-text fallback.@@ -497,56 +508,11 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st}reportParserWarnings(ctx, dispatched.Warnings)- // 3. Build the legacy `pages` slice. When the dispatch path- // produced a JSON payload, we re-shape it into the page- // layout the chunker side consumes (`{text, doc_type_kwd,- // page_number?}`); when the dispatch produced a string- // payload we emit a single page carrying the rendered text;- // otherwise we slice the binary on ASCII form-feed and- // treat the input as text pages.- var pages [][]byte- var dispatchedPages []schema.Page- switch {- case dispatched.Err == nil && dispatched.OutputFormat == "json":- dispatchedPages = jsonItemsToPages(dispatched.JSON)- pages = pagesFromDispatch(dispatchedPages)- case dispatched.Err == nil && dispatched.OutputFormat != "":- var text string- switch dispatched.OutputFormat {- case "markdown":- text = dispatched.Markdown- case "html":- text = dispatched.HTML- case "text":- text = dispatched.Text- }- pages = [][]byte{[]byte(text)}- default:- pages = splitIntoPages(binary)- if len(pages) == 0 {- pages = [][]byte{nil}- }- }-- // 3b. (pages-based page selection is handled upstream via the- // component setup: ParserConfig[cpnID][filetype]["pages"] is a- // list of page ranges delivered through override_params and- // consumed by the deepdoc/pdf parser. No inputs-level handling- // here.)-- // 4. Build the page slice sequentially. Per-page parallelism now- // lives in the parser backends (e.g. internal/deepdoc/parser/pdf- // fans out one worker per page and assembles in page order), so- // this component only reshapes the parser output. The DETERMINISTIC- // MERGE (plan §8 R8) keeps pages sorted by PageNumber so the- // downstream chunker / tokenizer get stable chunk IDs.- parsed, err := buildPagesFromBytes(ctx, pages, dispatched.DocType)- if err != nil {+ if err := ctx.Err(); err != nil {return nil, fmt.Errorf("parser: %w", err)}- sortPagesByNumber(parsed)lang, _ := getString(inputs, "lang")- out := buildParserOutputs(parsed, dispatched, filename, fileTypeExt, lang)+ out := buildParserOutputs(ctx, dispatched, filename, binary, lang)// Forward the storage references so a downstream chunker can// re-acquire the source PDF and crop section images on demand,// instead of carrying the binary across the component boundary.@@ -565,58 +531,42 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// forwards only this explicit output to the next node, so shared// fields must live in Globals.globals.PublishGlobals(ctx, out)- // Debug log: summarize parser output for pipeline debugging.- if dispatched.OutputFormat == "json" {- common.Debug("parser stage output",- zap.String("component", "Parser"),- zap.String("output_format", "json"),- zap.Int("json_items", len(dispatched.JSON)),- )- } else if dispatched.OutputFormat != "" {- common.Debug("parser stage output",- zap.String("component", "Parser"),- zap.String("output_format", dispatched.OutputFormat),- )- }+ items, _ := out["json"].([]map[string]any)+ logParserOutput(dispatched, items)// Progress (_created_time / _elapsed_time stamping, start/done// callbacks) is owned by the canvas framework (realComponentBody),// not by this component, so we return the work result directly.return out, nil}-func reportParserWarnings(ctx context.Context, warnings []string) {- for _, warning := range warnings {- runtime.ReportProgressMessage(ctx, "Parser", "WARNING: "+warning)+func logParserOutput(dispatched parser.ParseResult, items []map[string]any) {+ common.Debug("parser stage output",+ zap.String("component", "Parser"),+ zap.String("normalized_from", resolveParserNormalizationSource(dispatched)),+ zap.Int("json_items", len(items)),+ )+}++func resolveParserNormalizationSource(dispatched parser.ParseResult) string {+ if len(dispatched.JSON) > 0 {+ return "json"+ }+ if dispatched.Markdown != "" {+ return "markdown"+ }+ if dispatched.HTML != "" {+ return "html"+ }+ if dispatched.Text != "" {+ return "text"}+ return "raw"}-// buildPagesFromBytes reshapes already-prepared page bytes into the-// schema.Page layout the downstream chunker consumes. The per-page-// parse (including any parallelism) now lives in the parser backends-// (internal/parser/parser and internal/deepdoc/parser/pdf); this-// component only wraps the bytes into pages and honors context-// cancellation so an abandoned run does not keep reshaping pages.-//-// The function is format-agnostic: it does not resolve parsers or-// inspect file families — it only carries the raw bytes under the-// "text" key with the given doc_type_kwd (defaults to "text" when-// empty), matching the shape downstream readers expect from the-// raw-text fallback and dispatch paths.-func buildPagesFromBytes(ctx context.Context, pages [][]byte, docType string) ([]schema.Page, error) {- if docType == "" {- docType = "text"- }- out := make([]schema.Page, 0, len(pages))- for _, raw := range pages {- if err := ctx.Err(); err != nil {- return nil, err- }- out = append(out, schema.Page{- "text": string(raw),- "doc_type_kwd": docType,- })+func reportParserWarnings(ctx context.Context, warnings []string) {+ for _, warning := range warnings {+ runtime.ReportProgressMessage(ctx, "Parser", "WARNING: "+warning)}- return out, nil}// --- input helpers ---@@ -695,57 +645,6 @@ func containsFormFeed(b []byte) bool {return false}-// sortPagesByNumber orders pages by their PageNumber key-// ascending. Pages without a PageNumber key (or with a non-int-// value) sort to the END so the deterministic contract is-// "numbered pages first, then unnumbered" — this matches the-// Python component's loop order (it processes pages in input-// order, not in PageNumber order, but the Go merge is-// intentionally stricter so the test can assert exact byte-// equality across runs).-func sortPagesByNumber(pages []schema.Page) {- sort.SliceStable(pages, func(i, j int) bool {- pi, oki := numericPageNumber(pages[i])- pj, okj := numericPageNumber(pages[j])- switch {- case oki && okj:- return pi < pj- case oki:- return true // i is numbered, j is not- case okj:- return false- default:- return false // stable- }- })-}--func numericPageNumber(p schema.Page) (int, bool) {- if p == nil {- return 0, false- }- v, ok := p["page_number"]- if !ok {- return 0, false- }- switch n := v.(type) {- case int:- return n, true- case int64:- return int(n), true- case float64:- return int(n), true- }- return 0, false-}--// toAnyPages is a tiny adapter that hands the page slice to-// the output map as `any`. We use it instead of a direct cast-// so the type stays `[]schema.Page` in the Go source and the-// output map value type is `any` — matching the runtime.Component-// contract.-func toAnyPages(pages []schema.Page) any { return pages }-// init registers Parser under CategoryIngestion per plan §4// Phase 2.2. The factory is a thin closure that decodes the// DSL param map; the static Metadata is derived fromdiff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.goindex 109a59818b..7213e50f90 100644--- a/internal/ingestion/component/parser_check_test.go+++ b/internal/ingestion/component/parser_check_test.go@@ -24,8 +24,7 @@ import ()// TestParserComponent_Check covers the construction-time business-// validation that mirrors the applicable subset of Python-// ParserParam.check() (rag/flow/parser/parser.py:251-321).+// validation for parser methods that require language configuration.//// Go does NOT validate audio/video vlm.llm_id because media_dispatch// resolves tenant default models via resolveTenantModelByType, not@@ -131,7 +130,7 @@ func TestParserComponent_Check(t *testing.T) {for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- c := &ParserComponent{Setups: tc.setups, Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{setups: tc.setups}err := c.Check()if tc.wantErr != "" {if err == nil {diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex 69fb1c8510..24daa44820 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -14,8 +14,8 @@// limitations under the License.//-// Parser dispatch validates the requested output format, resolves the-// parser backend, and returns the structured ParseWithResult payload.+// Parser dispatch resolves the parser backend and returns the structured+// ParseWithResult payload for component-boundary normalization.//// `parse_method` is carried through file metadata for downstream// consumers, while the actual backend work stays in@@ -26,37 +26,15 @@ package componentimport ("context""fmt"- "maps""strings"+ "ragflow/internal/common""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser""ragflow/internal/utility"-)-// parserDispatchResult is the typed outcome of dispatchParse. The-// component's Invoke translates it into the runtime output map.-//-// OutputFormat is the wire format the parser actually emitted. It-// always matches setups[fileType].output_format on success; on a-// format-mismatch failure (the whitelist rejects it) OutputFormat is-// empty and Err is non-nil.-//-// File is the per-parser file metadata and may be nil.-//-// Payload holds exactly one populated field per the ParseResult-// contract (see internal/parser/parser/parse_result.go).-type parserDispatchResult struct {- OutputFormat string- DocType string // doc_type_kwd for media dispatch ("image", "video", "audio")- File map[string]any- JSON []map[string]any- Markdown string- Text string- HTML string- Warnings []string- Err error-}+ "go.uber.org/zap"+)type parserSetupConfigurer interface {ConfigureFromSetup(setup map[string]any)@@ -82,90 +60,6 @@ func configureParserFromSetups(p any, fileType utility.FileType, setups map[stricfg.ConfigureFromSetup(map[string]any(setup))}-// resolveOutputFormat picks the wire format for this run. The-// Python side asks the setup, then checks the value is in-// allowed_output_format[fileType]. We mirror that exact sequence:-//-// 1. setups[fileType].output_format (or "" when absent),-// 2. if absent and the family has an allowed_output_format entry,-// use the Python per-family default (mirrors ParserParam.__init__-// setups, e.g. "image" → "json", "pdf" → "json", "spreadsheet" →-// "html"), falling back to allowed[0] only when no explicit default-// is known,-// 3. if absent and the family has no allowed_output_format entry,-// return "" (the component falls back to text-page mode-// without validating).-//-// The whitelist check returns "" + Err when the requested format is-// not in the allowed set; this is the validation Python raises-// via check_empty / check_valid_value, surfaced as an error so the-// component short-circuits with _ERROR rather than emitting a-// payload the downstream chunker cannot consume.-//-// Strict mode: explicit image:text is rejected by the whitelist.-func resolveOutputFormat(family string, setups map[string]schema.ParserSetup, allowed map[string][]string) (string, error) {- setup, ok := setups[family]- if !ok {- // Family not configured — text-page mode; no validation.- return "", nil- }- format, _ := setup["output_format"].(string)- allowedList, ok := allowed[family]- if !ok || len(allowedList) == 0 {- // No whitelist entry — accept what the setup asked for, or- // fall back to "text" for families without a whitelist.- if format == "" {- format = "text"- }- return format, nil- }- if format == "" {- // No explicit format: use the Python per-family default declared- // in ParserParam.__init__ / defaultSetups(). This is not always- // allowed[0] (e.g. spreadsheet default is "html" but allowed[0]- // is "json"; markdown default is "json" but allowed[0] is "text").- if def, ok := defaultOutputFormatForFamily(family); ok {- return def, nil- }- return allowedList[0], nil- }- for _, candidate := range allowedList {- if strings.EqualFold(candidate, format) {- return format, nil- }- }- return "", fmt.Errorf(- "parser: output_format %q for %q is not in allowed_output_format %v",- format, family, allowedList,- )-}--// defaultOutputFormatForFamily returns the per-family default-// output_format used when setups[family] exists but output_format is-// empty. For most families it is the value in defaultSetups()-// (which mirrors rag/flow/parser/parser.py ParserParam.setups).-// Two families diverge intentionally and are overridden here:-// - email: defaultSetups is "json" (Python), but dispatch defaults-// to "text" to match frontend InitialOutputFormatMap, EmailParser-// text fallback, and ingestion_pipeline_email.json.-// - audio: defaultSetups is "text" (Python), but dispatch defaults-// to "json" to match media_dispatch json fallback and Python-// whitelist [json] (audio:text would be rejected if validated).-func defaultOutputFormatForFamily(family string) (string, bool) {- if family == "email" {- return "text", true- }- if family == "audio" {- return "json", true- }- if s, ok := defaultSetups()[family]; ok {- if v, ok := s["output_format"].(string); ok && v != "" {- return v, true- }- }- return "", false-}-// dispatchParse resolves the parser for the given fileType and invokes// its structured ParseWithResult contract.//@@ -182,12 +76,12 @@ func defaultOutputFormatForFamily(family string) (string, bool) {// re-reading setups. lib_type is no longer threaded through: the// Python dispatcher picks a single backend per family and the Go// constructors mirror that.-func dispatchParse(ctx context.Context, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup) parserDispatchResult {+func dispatchParse(ctx context.Context, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup) parser.ParseResult {if fileType == utility.FileTypeOTHER {// Unknown / unset family. The component treats the bytes// as text pages; splitIntoPages handles it. We return no// result here so the caller routes to that path.- return parserDispatchResult{}+ return parser.ParseResult{}}var parseMethod string@@ -199,13 +93,13 @@ func dispatchParse(ctx context.Context, fileType utility.FileType, filename strip, err := parser.GetParser(fileType)if err != nil {- return parserDispatchResult{Err: fmt.Errorf("parser: resolve %q: %w", fileType, err)}+ return parser.ParseResult{Err: fmt.Errorf("parser: resolve %q: %w", fileType, err)}}configureParserFromSetups(p, fileType, setups)res := p.ParseWithResult(ctx, filename, data)if res.Err != nil {- return parserDispatchResult{Err: fmt.Errorf("parser: %q: %w", fileType, res.Err)}+ return parser.ParseResult{Err: fmt.Errorf("parser: %q: %w", fileType, res.Err)}}// Carry the configured parse_method on the file metadata so// downstream consumers can read which provider ran.@@ -215,15 +109,7 @@ func dispatchParse(ctx context.Context, fileType utility.FileType, filename stri}res.File["parse_method"] = parseMethod}- return parserDispatchResult{- OutputFormat: res.OutputFormat,- File: res.File,- JSON: res.JSON,- Markdown: res.Markdown,- Text: res.Text,- HTML: res.HTML,- Warnings: res.Warnings,- }+ return res}// fileTypeFromInputs derives the parser-library extension form@@ -238,9 +124,9 @@ func dispatchParse(ctx context.Context, fileType utility.FileType, filename stri// or the python family name ("markdown"); both are normalized// to the extension form via the pythonFamilyName / familyToExt// lookup tables below.-// 2. inputs["file"].name — fall back to the filename so a caller-// that only supplies the path is still routed correctly.-// 3. inputs["name"] — last-resort filename.+// 2. inputs["name"] — the resolved source filename.+// 3. inputs["file"].name — fallback for callers that only supply a+// file descriptor.// 4. utility.FileTypeOTHER — text-page mode.//// The function never errors; unknown / absent filenames degrade to@@ -270,14 +156,14 @@ func fileTypeFromInputs(inputs map[string]any) utility.FileType {return ft}}+ if name, ok := inputs["name"].(string); ok && name != "" {+ return utility.GetFileType(name)+ }if m, ok := inputs["file"].(map[string]any); ok {if name, ok := m["name"].(string); ok && name != "" {return utility.GetFileType(name)}}- if name, ok := inputs["name"].(string); ok && name != "" {- return utility.GetFileType(name)- }return utility.FileTypeOTHER}@@ -321,7 +207,7 @@ func familyToExt(family string) utility.FileType {}// pythonFamilyName normalises a free-form file-type hint to the-// python family identifier used by schema.ParserParam.Setups.+// Python family identifier used by ParserComponent setups.// Returns "" when the hint is unknown.func pythonFamilyName(raw string) string {switch raw {@@ -374,101 +260,79 @@ func ParserFileFamily(ext string) string {return pythonFamilyName(ext)}-// jsonItemsToPages reshapes a parsed JSON payload into the-// schema.Page layout the chunker side consumes. Each JSON item-// becomes one page carrying `text`, `doc_type_kwd`, and any-// ck_type / image / positions fields the parser attached. The page-// number is assigned sequentially so the deterministic-merge-// contract in plan §8 R8 holds.-func jsonItemsToPages(items []map[string]any) []schema.Page {- out := make([]schema.Page, 0, len(items))- for i, it := range items {- page := schema.Page{}- maps.Copy(page, it)- // page_number anchors the deterministic-merge sort; the- // parser-supplied number (if any) takes precedence so- // PDF / DOCX outputs that already carry `page_number`- // survive the round-trip.- if _, ok := page["page_number"]; !ok {- page["page_number"] = i- }- if _, ok := page["doc_type_kwd"]; !ok {- page["doc_type_kwd"] = "text"- }- out = append(out, page)- }- return out-}--// pagesFromDispatch extracts the per-page bytes from a parsed-// schema.Page slice so the page builder can reshape them into the-// schema.Page layout the chunker consumes. Pages without a `text`-// field emit an empty buffer (treated as a zero-length page).-func pagesFromDispatch(pages []schema.Page) [][]byte {- out := make([][]byte, 0, len(pages))- for _, p := range pages {- var buf []byte- if s, ok := p["text"].(string); ok {- buf = []byte(s)- }- out = append(out, buf)- }- return out-}-// buildParserOutputs assembles the runtime output map from the-// merged pages slice AND the dispatch result (when the dispatch-// succeeded). The output shape:+// dispatch result (when the dispatch succeeded) or raw binary fallback.+// The output shape://-// - pages []schema.Page — sorted by PageNumber// - name string — from the upstream file/document name// (or doc_id when no filename is available)-// - output_format string — the dispatch's OutputFormat,-// or "text" for the raw-text-// fallback-// - json | Markdown | text | html — the dispatched payload on-// the matching family key (only-// populated on a structured-// dispatch)+// - output_format string — always "json"+// - json []map[string]any — normalized parser items// - file map[string]any — the parser-enriched file// metadata, when present-//-// This mirrors the Python Parser component's `set_output()` calls-// at rag/flow/parser/parser.py:_invoke — the downstream chunker-// / tokenizer / extractor components read the matching family-// key, with "pages" as the universal fallback shape.-func buildParserOutputs(parsed []schema.Page, dispatched parserDispatchResult, name string, fileType utility.FileType, lang string) map[string]any {+func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name string, rawBinary []byte, lang string) map[string]any {out := map[string]any{- "pages": toAnyPages(parsed),- "name": name,+ "name": name,}if lang != "" {out["lang"] = lang}if dispatched.Err == nil && dispatched.OutputFormat != "" {- out["output_format"] = dispatched.OutputFormat- switch dispatched.OutputFormat {- case "json":- out["json"] = dispatched.JSON- case "markdown":- out["markdown"] = dispatched.Markdown- case "html":- out["html"] = dispatched.HTML- case "text":- out["text"] = dispatched.Text- }+ out["output_format"] = "json"+ out["json"] = normalizeParserJSON(ctx, name, dispatched)if dispatched.File != nil {out["file"] = dispatched.File}return out}- // Raw-text fallback path: emit output_format = "text" so a- // chunker branching on the format key still sees a sane value.- out["output_format"] = "text"- _ = fileType // reserved for a future "raw_text per-family" extension+ // Raw-text fallback path: emit one JSON item per page.+ rawPages := splitIntoPages(rawBinary)+ if len(rawPages) == 0 {+ rawPages = [][]byte{nil}+ }+ fallbackItems := make([]map[string]any, 0, len(rawPages))+ for _, pageBytes := range rawPages {+ txt := string(pageBytes)+ fallbackItems = append(fallbackItems, parser.NewTextJSONItem(txt))+ }+ out["output_format"] = "json"+ out["json"] = fallbackItemsreturn out}+func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []map[string]any {+ if len(dispatched.JSON) > 0 {+ return dispatched.JSON+ }+ if dispatched.Markdown != "" {+ return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)+ }+ if dispatched.HTML != "" {+ res := parser.NewHTMLParser().ParseWithResult(ctx, filename, []byte(dispatched.HTML))+ if res.Err == nil && len(res.JSON) > 0 {+ return res.JSON+ }+ if res.Err != nil {+ warnParserNormalizationFallback(filename, "html", res.Err)+ } else {+ warnParserNormalizationFallback(filename, "html", fmt.Errorf("parser returned no JSON items"))+ }+ return []map[string]any{parser.NewTextJSONItem(dispatched.HTML)}+ }+ if dispatched.Text != "" {+ return []map[string]any{parser.NewTextJSONItem(dispatched.Text)}+ }+ return []map[string]any{}+}++func warnParserNormalizationFallback(filename, source string, err error) {+ common.Warn("parser normalization fell back to text",+ zap.String("filename", filename),+ zap.String("normalized_from", source),+ zap.Error(err),+ )+}+func parserInputName(inputs map[string]any, docID string) string {if inputs != nil {if name, ok := inputs["name"].(string); ok && name != "" {diff --git a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.goindex 043a8a88c6..d6bf465d11 100644--- a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go+++ b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go@@ -11,7 +11,6 @@ import ("testing"modelModule "ragflow/internal/entity/models"- "ragflow/internal/ingestion/component/schema""gorm.io/gorm")@@ -61,11 +60,10 @@ func TestDispatch_PDFVisionJSON_RealPDFFixture(t *testing.T) {t.Fatalf("ReadFile(%s): %v", path, err)}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"setups["pdf"]["output_format"] = "json"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": data,diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 16375ae71b..ef20c6a6c4 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -18,10 +18,9 @@//// - FileTypeOTHER + missing setups → text-page mode.// - FileTypeMarkdown → JSON payload family on the matching output-// key, with the pages slice preserved.-// - FileTypePDF + setups["pdf"].output_format set to a value not-// in allowed_output_format["pdf"] → component errors with the-// format-mismatch message (matches the Python check() behavior).+// key.+// - Historical output_format values, including PDF Markdown, are accepted+// and normalized to the single JSON payload.package component@@ -33,6 +32,7 @@ import ("encoding/json""fmt""io"+ "net""net/http""net/http/httptest""os"@@ -47,9 +47,11 @@ import (doctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/entity""ragflow/internal/entity/models"- "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser""ragflow/internal/utility"+ "go.uber.org/zap"+ "go.uber.org/zap/zaptest/observer""gorm.io/gorm")@@ -75,14 +77,178 @@ func (c *captureSetupConfigurer) ConfigureFromSetup(setup map[string]any) {c.setup = setup}-// TestDispatch_OutputFormatValidation_Allowed is the happy-path-// pin: a Markdown file with output_format=json passes the-// allowed_output_format check and runs the structured dispatch.-func TestDispatch_OutputFormatValidation_Allowed(t *testing.T) {- param := schema.ParserParam{}.Defaults()+func requireJSONText(t *testing.T, out map[string]any, want string) {+ t.Helper()+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)+ }+ items, ok := out["json"].([]map[string]any)+ if !ok || len(items) == 0 {+ t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])+ }+ for _, item := range items {+ if text, _ := item["text"].(string); strings.Contains(text, want) {+ return+ }+ }+ t.Fatalf("json payload does not contain %q: %#v", want, items)+}++func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {+ tests := []struct {+ name string+ dispatched parser.ParseResult+ wantText string+ }{+ {+ name: "markdown",+ dispatched: parser.ParseResult{+ OutputFormat: "markdown",+ Markdown: "# Title\n\nBody",+ },+ wantText: "Title",+ },+ {+ name: "html",+ dispatched: parser.ParseResult{+ OutputFormat: "html",+ HTML: "<h1>Title</h1><p>Body</p>",+ },+ wantText: "Title",+ },+ {+ name: "text",+ dispatched: parser.ParseResult{+ OutputFormat: "text",+ Text: "plain body",+ },+ wantText: "plain body",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ out := buildParserOutputs(t.Context(), tt.dispatched, "sample."+tt.name, nil, "")++ requireJSONText(t, out, tt.wantText)+ if tt.name == "markdown" {+ items, ok := out["json"].([]map[string]any)+ if !ok {+ t.Fatalf("json = %T, want []map[string]any", out["json"])+ }+ if len(items) != 2 {+ t.Fatalf("markdown json item count = %d, want 2: %#v", len(items), items)+ }+ if got := items[0]["ck_type"]; got != "heading" {+ t.Errorf("markdown item[0].ck_type = %v, want heading", got)+ }+ if got := items[0]["text"]; got != "Title" {+ t.Errorf("markdown item[0].text = %v, want Title", got)+ }+ if got := items[1]["text"]; got != "Body" {+ t.Errorf("markdown item[1].text = %v, want Body", got)+ }+ }+ for _, key := range []string{"markdown", "html", "text"} {+ if _, ok := out[key]; ok {+ t.Errorf("output contains obsolete %q payload: %#v", key, out[key])+ }+ }+ })+ }+}++func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {+ out := buildParserOutputs(t.Context(), parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{},+ Markdown: "# Recovered title",+ }, "sample.md", nil, "")++ requireJSONText(t, out, "Recovered title")+}++func TestParseMarkdownToJSONItemsDoesNotResolveRemoteImages(t *testing.T) {+ originalResolver := net.DefaultResolver+ dnsLookup := make(chan struct{}, 1)+ net.DefaultResolver = &net.Resolver{+ PreferGo: true,+ StrictErrors: true,+ Dial: func(context.Context, string, string) (net.Conn, error) {+ select {+ case dnsLookup <- struct{}{}:+ default:+ }+ return nil, fmt.Errorf("unexpected DNS lookup")+ },+ }+ t.Cleanup(func() { net.DefaultResolver = originalResolver })++ items := parseMarkdownToJSONItems(t.Context(), "ocr.md", "")+ if len(items) != 1 {+ t.Fatalf("items = %#v, want one Markdown image block", items)+ }+ select {+ case <-dnsLookup:+ t.Fatal("Markdown-to-JSON normalization performed a remote image DNS lookup")+ default:+ }+}++func TestLogParserOutputReportsNormalizationSource(t *testing.T) {+ core, logs := observer.New(zap.DebugLevel)+ originalLogger := common.Logger+ common.Logger = zap.New(core)+ t.Cleanup(func() { common.Logger = originalLogger })++ logParserOutput(parser.ParseResult{+ OutputFormat: "markdown",+ Markdown: "# Title\n\nBody",+ }, []map[string]any{{"text": "Title"}, {"text": "Body"}})++ entries := logs.FilterMessage("parser stage output").All()+ if len(entries) != 1 {+ t.Fatalf("parser output log count = %d, want 1", len(entries))+ }+ fields := entries[0].ContextMap()+ if _, ok := fields["output_format"]; ok {+ t.Errorf("output_format must be omitted because it is always json: %v", fields)+ }+ if got := fields["normalized_from"]; got != "markdown" {+ t.Errorf("normalized_from = %v, want markdown", got)+ }+ if got := fields["json_items"]; got != int64(2) {+ t.Errorf("json_items = %v, want 2", got)+ }+}++func TestWarnParserNormalizationFallbackReportsCause(t *testing.T) {+ core, logs := observer.New(zap.WarnLevel)+ originalLogger := common.Logger+ common.Logger = zap.New(core)+ t.Cleanup(func() { common.Logger = originalLogger })++ warnParserNormalizationFallback("broken.md", "markdown", fmt.Errorf("parse failed"))++ entries := logs.FilterMessage("parser normalization fell back to text").All()+ if len(entries) != 1 {+ t.Fatalf("normalization fallback log count = %d, want 1", len(entries))+ }+ fields := entries[0].ContextMap()+ if got := fields["filename"]; got != "broken.md" {+ t.Errorf("filename = %v, want broken.md", got)+ }+ if got := fields["normalized_from"]; got != "markdown" {+ t.Errorf("normalized_from = %v, want markdown", got)+ }+ if got := fields["error"]; got != "parse failed" {+ t.Errorf("error = %v, want parse failed", got)+ }+}++func TestDispatch_JSONOutput(t *testing.T) {setups := defaultSetups()- // Defaults already include Markdown → {text, json}.- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("# Title\n\nbody\n"),@@ -102,67 +268,42 @@ func TestDispatch_OutputFormatValidation_Allowed(t *testing.T) {if len(jsonItems) == 0 {t.Errorf("json payload empty; want at least 1 item")}- // Pages must still exist for chunker-side consumers.- pages, ok := out["pages"].([]schema.Page)- if !ok || len(pages) == 0 {- t.Errorf("pages slice missing or empty: %T", out["pages"])- }- if ok && len(pages) > 0 {- if got, _ := pages[0]["text"].(string); !strings.Contains(got, "Title") {- t.Errorf("pages[0].text = %q, want content containing Title", got)- }- }// File metadata is carried through dispatch.if fm, ok := out["file"].(map[string]any); !ok || fm["name"] != "doc.md" {t.Errorf("file metadata missing or wrong: %+v", out["file"])}}-// TestDispatch_OutputFormatValidation_Rejection pins the-// whitelist enforcement: a request for output_format=html on the-// Markdown family is rejected because Markdown's allowed list is-// {text, json}. The component must surface this as a hard error-// before any fallback so a misconfigured template cannot silently-// degrade.-func TestDispatch_OutputFormatValidation_Rejection(t *testing.T) {- param := schema.ParserParam{}.Defaults()+func TestDispatchNormalizesConfiguredOutputFormatToJSON(t *testing.T) {setups := defaultSetups()- // Override the Markdown setup to ask for an unsupported format.- // The key is "markdown" (the python-side family identifier),- // NOT "md" — utility.FileTypeMarkdown happens to be the string- // "md" but the setup key is the family name. resolveOutputFormat- // looks up setups[string(fileType)], so the fileType passed in- // here must match the setup key.- setups["markdown"] = schema.ParserSetup{"output_format": "html"}- // inputs["file_type"] must also be "markdown" so fileTypeFromInputs- // returns a FileType whose string form matches the setup key.- c := &ParserComponent{Param: param, Setups: setups}+ setups["email"]["output_format"] = "text"+ c := &ParserComponent{setups: setups}- _, err := c.Invoke(t.Context(), nil, map[string]any{- "binary": []byte("# Title\n"),- "file_type": "md",+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: Hello\r\n\r\nBody\r\n"),+ "name": "mail.eml",+ "file_type": "eml",})- if err == nil {- t.Fatal("Invoke: want error for unsupported output_format, got nil")+ if err != nil {+ t.Fatalf("Invoke: %v", err)}- if !strings.Contains(err.Error(), "output_format") {- t.Errorf("error %q must mention output_format", err.Error())+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)}- if !strings.Contains(err.Error(), "markdown") && !strings.Contains(err.Error(), "md") {- t.Errorf("error %q must mention the family", err.Error())+ if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {+ t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])}}// TestDispatch_TextPageMode_NoFileType pins the no-dispatch// path. When the upstream inputs supply neither file_type nor// file.name, the component degrades to text-page mode and-// emits output_format=text. This is the documented behavior for+// emits structured JSON items. This is the documented behavior for// canvas-bound invocations that wire the binary directly without// a family hint.func TestDispatch_TextPageMode_NoFileType(t *testing.T) {- param := schema.ParserParam{}.Defaults()setups := defaultSetups()- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("plain content\n"),@@ -171,12 +312,15 @@ func TestDispatch_TextPageMode_NoFileType(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if got, want := out["output_format"], "text"; got != want {- t.Errorf("output_format = %v, want %v (text-page mode)", got, want)+ if got, want := out["output_format"], "json"; got != want {+ t.Errorf("output_format = %v, want %v (raw-text mode)", got, want)}- pages, ok := out["pages"].([]schema.Page)- if !ok || len(pages) == 0 {- t.Fatalf("pages slice missing or empty: %T", out["pages"])+ jsonItems, ok := out["json"].([]map[string]any)+ if !ok || len(jsonItems) == 0 {+ t.Fatalf("json items missing or empty: %T", out["json"])+ }+ if _, ok := out["text"]; ok {+ t.Fatalf("output contains obsolete text payload: %#v", out["text"])}}@@ -185,9 +329,8 @@ func TestDispatch_TextPageMode_NoFileType(t *testing.T) {// resolution/execution failures must surface as errors instead of// silently degrading to text-page mode.func TestDispatch_SupportedFamilyFailure_HardErrors(t *testing.T) {- param := schema.ParserParam{}.Defaults()setups := defaultSetups()- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}_, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("PDF payload as bytes (not a real PDF — stub test)\n"),@@ -205,8 +348,8 @@ func TestDispatch_SupportedFamilyFailure_HardErrors(t *testing.T) {// rules documented on parser_dispatch.go:fileTypeFromInputs://// 1. inputs["file_type"] (explicit family hint)-// 2. inputs["file"].name (filename in the file descriptor)-// 3. inputs["name"] (last-resort filename)+// 2. inputs["name"] (the resolved source filename)+// 3. inputs["file"].name (filename in the file descriptor)// 4. FileTypeOTHER (text-page mode)func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {cases := []struct {@@ -222,6 +365,7 @@ func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {{"explicit slides (family name)", map[string]any{"file_type": "slides"}, "pptx"},{"explicit spreadsheet (family name)", map[string]any{"file_type": "spreadsheet"}, "xlsx"},{"explicit markdown (family form)", map[string]any{"file_type": "markdown"}, "md"},+ {"name wins over conflicting file.name", map[string]any{"name": "report.txt", "file": map[string]any{"name": "report.pdf"}}, "txt"},{"file.name docx", map[string]any{"file": map[string]any{"name": "report.docx"}}, "docx"},{"name fallback md", map[string]any{"name": "notes.md"}, "md"},{"unrelated inputs", map[string]any{"binary": []byte("x"), "doc_id": "abc"}, "other"},@@ -238,132 +382,7 @@ func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {}}-// TestResolveOutputFormat_DefaultsAndWhitelist pins the two-layer-// behavior of resolveOutputFormat: it returns the setup's-// output_format when present (or the per-family default when absent,-// e.g. markdown→json, spreadsheet→html), and rejects values not in-// the allowed_output_format list. Explicit image:text is rejected.-func TestResolveOutputFormat_DefaultsAndWhitelist(t *testing.T) {- allowed := map[string][]string{- "pdf": {"json", "markdown"},- "markdown": {"text", "json"},- "image": {"json"},- "spreadsheet": {"json", "markdown", "html"},- "email": {"text", "json"},- "audio": {"text", "json"},- }- cases := []struct {- name string- setups map[string]schema.ParserSetup- family string- want string- wantErr bool- }{- {- name: "no setup → empty (text-page mode)",- setups: nil,- family: "pdf",- want: "",- },- {- name: "setup with output_format=json → json",- setups: map[string]schema.ParserSetup{"pdf": {"output_format": "json"}},- family: "pdf",- want: "json",- },- {- name: "setup with output_format=markdown → markdown",- setups: map[string]schema.ParserSetup{"pdf": {"output_format": "markdown"}},- family: "pdf",- want: "markdown",- },- {- name: "setup without output_format → per-family default (markdown→json)",- setups: map[string]schema.ParserSetup{"markdown": {}},- family: "markdown",- want: "json",- },- {- name: "setup without output_format → per-family default (spreadsheet→html)",- setups: map[string]schema.ParserSetup{"spreadsheet": {}},- family: "spreadsheet",- want: "html",- },- {- name: "setup without output_format → per-family default (image→json)",- setups: map[string]schema.ParserSetup{"image": {}},- family: "image",- want: "json",- },- {- name: "setup without output_format → per-family default (email→text)",- setups: map[string]schema.ParserSetup{"email": {}},- family: "email",- want: "text",- },- {- name: "image explicit text (legacy) → strict reject",- setups: map[string]schema.ParserSetup{"image": {"output_format": "text"}},- family: "image",- wantErr: true,- },- {- name: "image explicit TEXT uppercase → strict reject",- setups: map[string]schema.ParserSetup{"image": {"output_format": "TEXT"}},- family: "image",- wantErr: true,- },- {- name: "pdf asking for html (not allowed) → reject",- setups: map[string]schema.ParserSetup{"pdf": {"output_format": "html"}},- family: "pdf",- wantErr: true,- },- {- name: "setup without output_format → per-family default (audio→json)",- setups: map[string]schema.ParserSetup{"audio": {}},- family: "audio",- want: "json",- },- {- name: "setup without output_format → per-family default (pdf→json)",- setups: map[string]schema.ParserSetup{"pdf": {}},- family: "pdf",- want: "json",- },- {- name: "family with no whitelist → accept setup value",- setups: map[string]schema.ParserSetup{"video": {"output_format": "json"}},- family: "video",- want: "json",- },- {- name: "family with no whitelist empty → default text",- setups: map[string]schema.ParserSetup{"video": {}},- family: "video",- want: "text",- },- }- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- got, err := resolveOutputFormat(tc.family, tc.setups, allowed)- if tc.wantErr {- if err == nil {- t.Fatalf("want error, got nil (value=%q)", got)- }- return- }- if err != nil {- t.Fatalf("unexpected error: %v", err)- }- if got != tc.want {- t.Errorf("got %q, want %q", got, tc.want)- }- })- }-}--func TestDefaultSetups_DOCX_OutputFormatMarkdown(t *testing.T) {+func TestDefaultSetups_DOCX_OutputFormatJSON(t *testing.T) {setups := defaultSetups()docx, ok := setups["docx"]if !ok {@@ -377,130 +396,7 @@ func TestDefaultSetups_DOCX_OutputFormatMarkdown(t *testing.T) {t.Errorf("docx.output_format = %q, want %q", got, "json")}}--// TestDefaultOutputFormatForFamily_Sync verifies the dispatch default-// stays in sync with the allowed whitelist and, except for the two-// intentional overrides (email:text, audio:json), with defaultSetups.-func TestDefaultOutputFormatForFamily_Sync(t *testing.T) {- allowed := schema.ParserParam{}.Defaults().AllowedOutputFormat- overrides := map[string]string{"email": "text", "audio": "json"}- for family, def := range map[string]string{- "pdf": "json",- "spreadsheet": "html",- "doc": "json",- "docx": "json",- "slides": "json",- "image": "json",- "markdown": "json",- "text&code": "json",- "html": "json",- "epub": "json",- "json": "json",- "email": "text",- "audio": "json",- "video": "text",- } {- got, ok := defaultOutputFormatForFamily(family)- if !ok {- t.Errorf("defaultOutputFormatForFamily(%q) missing", family)- continue- }- if got != def {- t.Errorf("defaultOutputFormatForFamily(%q)=%q, want %q", family, got, def)- }- if list, hasWL := allowed[family]; hasWL && len(list) > 0 {- found := false- for _, c := range list {- if strings.EqualFold(c, got) {- found = true- break- }- }- if !found {- t.Errorf("default %q for %q not in allowed %v", got, family, list)- }- }- if ov, isOverride := overrides[family]; isOverride {- if got != ov {- t.Errorf("override %q got %q want %q", family, got, ov)- }- continue- }- if ds, ok := defaultSetups()[family]; ok {- if want, ok := ds["output_format"].(string); ok && want != got {- t.Errorf("family %q dispatch default %q != defaultSetups %q (should be synced or listed as override)", family, got, want)- }- }- }- if _, ok := defaultOutputFormatForFamily("unknown"); ok {- t.Errorf("unknown family should return !ok")- }-}--// TestResolveOutputFormat_AudioOutputFormats pins the audio-family-// whitelist against the builtin audio template. The template-// (ingestion_pipeline_audio.json) and the Python default audio setup-// (rag/flow/parser/parser.py) both use output_format="text" — an audio-// transcription is inherently plain text. "text" must therefore pass-// the whitelist, "json" must stay accepted, and a format outside the-// whitelist must still be rejected.-func TestResolveOutputFormat_AudioOutputFormats(t *testing.T) {- allowed := schema.ParserParam{}.Defaults().AllowedOutputFormat- audioAllowed, ok := allowed["audio"]- if !ok {- t.Fatal("allowed_output_format: audio key missing")- }- has := func(want string) bool {- for _, v := range audioAllowed {- if strings.EqualFold(v, want) {- return true- }- }- return false- }- if !has("text") {- t.Errorf("allowed_output_format[audio] = %v, want it to include %q (builtin audio template and Python default use it)", audioAllowed, "text")- }- if !has("json") {- t.Errorf("allowed_output_format[audio] = %v, want it to include %q", audioAllowed, "json")- }-- cases := []struct {- name string- format string- wantErr bool- }{- {name: "text accepted", format: "text"},- {name: "json accepted", format: "json"},- {name: "html rejected", format: "html", wantErr: true},- }- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- setups := map[string]schema.ParserSetup{"audio": {"output_format": tc.format}}- got, err := resolveOutputFormat("audio", setups, allowed)- if tc.wantErr {- if err == nil {- t.Fatalf("want error for audio output_format=%q, got %q", tc.format, got)- }- return- }- if err != nil {- t.Fatalf("audio output_format=%q: unexpected error: %v", tc.format, err)- }- if got != tc.format {- t.Errorf("got %q, want %q", got, tc.format)- }- })- }-}--// TestDefaultSetups_Audio_OutputFormatText pins the audio default to-// "text", matching the Python default setup-// (rag/flow/parser/parser.py audio block) and the builtin audio-// template. The default feeds both the whitelist gate and the ASR-// dispatch, so it must not drift to a format the audio pipeline does-// not produce.-func TestDefaultSetups_Audio_OutputFormatText(t *testing.T) {+func TestDefaultSetups_Audio_OutputFormatJSON(t *testing.T) {setups := defaultSetups()audio, ok := setups["audio"]if !ok {@@ -510,8 +406,8 @@ func TestDefaultSetups_Audio_OutputFormatText(t *testing.T) {if !ok {t.Fatal("defaultSetups: audio.output_format missing or not a string")}- if got != "text" {- t.Errorf("audio.output_format = %q, want %q", got, "text")+ if got != "json" {+ t.Errorf("audio.output_format = %q, want %q", got, "json")}}@@ -527,7 +423,7 @@ func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) {}}-func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) {+func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {useMockDocAnalyzer(t)path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")@@ -536,10 +432,9 @@ func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) {t.Fatalf("ReadFile(%s): %v", path, err)}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["output_format"] = "markdown"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": data,@@ -549,15 +444,11 @@ func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if got, want := out["output_format"], "markdown"; got != want {- t.Fatalf("output_format = %v, want %v", got, want)- }- md, ok := out["markdown"].(string)- if !ok || md == "" {- t.Fatalf("markdown payload missing or empty: %T", out["markdown"])+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)}- if _, ok = out["json"]; ok {- t.Fatalf("json payload must be absent for markdown output: %+v", out["json"])+ if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {+ t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])}}@@ -568,11 +459,10 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {t.Fatalf("ReadFile(%s): %v", path, err)}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "plain_text"setups["pdf"]["output_format"] = "json"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": data,@@ -592,10 +482,9 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {}func TestDispatch_PDFUnsupportedParseMethod_HardErrors(t *testing.T) {- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}_, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -662,11 +551,10 @@ func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) {return &models.ChatResponse{Answer: &answer}, nil}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"setups["pdf"]["output_format"] = "json"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -728,11 +616,10 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) {return &models.ChatResponse{Answer: &answer}, nil}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "CustomVLM"setups["pdf"]["output_format"] = "json"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -778,11 +665,10 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "mineru"setups["pdf"]["output_format"] = "markdown"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -793,12 +679,57 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if got, want := out["output_format"], "markdown"; got != want {+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)+ }+ requireJSONText(t, out, "Title")+}++func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method == http.MethodPost && r.URL.Path == "/file_parse" {+ buf := new(bytes.Buffer)+ zw := zip.NewWriter(buf)+ f, _ := zw.Create("content_list.json")+ _, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody paragraph\n"}]`))+ _ = zw.Close()+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(buf.Bytes())+ return+ }+ http.NotFound(w, r)+ }))+ defer server.Close()++ origResolver := resolveTenantModelByType+ defer func() { resolveTenantModelByType = origResolver }()+ baseURL := server.URL+ apiKey := ""+ resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (models.ModelDriver, string, *models.APIConfig, int, error) {+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil+ }++ setups := defaultSetups()+ setups["pdf"]["parse_method"] = "mineru"+ setups["pdf"]["output_format"] = "json"+ c := &ParserComponent{setups: setups}++ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": []byte("%PDF-1.4"),+ "file_type": "pdf",+ "name": "sample.pdf",+ "tenant_id": "test-tenant",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if got, want := out["output_format"], "json"; got != want {t.Fatalf("output_format = %v, want %v", got, want)}- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "Title") {- t.Fatalf("markdown payload = %#v, want Title content", out["markdown"])+ items, ok := out["json"].([]map[string]any)+ if !ok || len(items) == 0 {+ t.Fatalf("json payload = %#v, want non-empty structured items", out["json"])}}@@ -854,9 +785,7 @@ func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if out["output_format"] != "markdown" || out["markdown"] != "MonkeyOCRv2 title" {- t.Fatalf("output=%#v", out)- }+ requireJSONText(t, out, "MonkeyOCRv2 title")}// mineruTestDriver is a minimal ModelDriver mock whose Name() returns "mineru".@@ -940,11 +869,10 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesTenantModel(t *testing.T) {return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "PaddleOCR"setups["pdf"]["output_format"] = "markdown"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -955,13 +883,10 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesTenantModel(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if got, want := out["output_format"], "markdown"; got != want {- t.Fatalf("output_format = %v, want %v", got, want)- }- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "Paddle Title") {- t.Fatalf("markdown payload = %#v, want Paddle Title content", out["markdown"])+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)}+ requireJSONText(t, out, "Paddle Title")}// TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload pins the cloud@@ -1008,11 +933,10 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload(t *testing.T) {return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &emptyBaseURL}, 0, nil}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "PaddleOCR"setups["pdf"]["output_format"] = "markdown"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1023,10 +947,7 @@ func TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "Unwrapped Title") {- t.Fatalf("markdown payload = %#v, want Unwrapped Title content", out["markdown"])- }+ requireJSONText(t, out, "Unwrapped Title")}func TestDispatch_PDFPaddleOCR_NoTenantModel_HardErrors(t *testing.T) {@@ -1037,11 +958,10 @@ func TestDispatch_PDFPaddleOCR_NoTenantModel_HardErrors(t *testing.T) {return nil, "", nil, 0, fmt.Errorf("no active PaddleOCR OCR model")}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "paddleocr"setups["pdf"]["output_format"] = "markdown"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}_, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1101,11 +1021,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel(t *testing.T) {return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["layout_recognizer"] = modelIDsetups["pdf"]["output_format"] = "markdown"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1116,13 +1035,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if got, want := out["output_format"], "markdown"; got != want {- t.Fatalf("output_format = %v, want %v", got, want)- }- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "Cloud Paddle Title") {- t.Fatalf("markdown payload = %#v, want Cloud Paddle Title content", out["markdown"])+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)}+ requireJSONText(t, out, "Cloud Paddle Title")}// TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod pins the routing of a@@ -1169,11 +1085,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {return &paddleocrTestDriver{}, "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil}- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = modelIDsetups["pdf"]["output_format"] = "markdown"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1184,13 +1099,10 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if got, want := out["output_format"], "markdown"; got != want {- t.Fatalf("output_format = %v, want %v", got, want)- }- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "Cloud Paddle Title") {- t.Fatalf("markdown payload = %#v, want Cloud Paddle Title content", out["markdown"])+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)}+ requireJSONText(t, out, "Cloud Paddle Title")}// paddleocrTestDriver is a minimal ModelDriver mock whose Name() returns "paddleocr".@@ -1345,13 +1257,12 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {}))defer server.Close()- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "Docling"setups["pdf"]["output_format"] = "markdown"setups["pdf"]["docling_server_url"] = server.URLsetups["pdf"]["docling_api_key"] = "doc-secret"- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1361,19 +1272,16 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if got, want := out["output_format"], "markdown"; got != want {- t.Fatalf("output_format = %v, want %v", got, want)- }- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "Docling Title") {- t.Fatalf("markdown payload = %#v, want Docling Title content", out["markdown"])+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)}+ requireJSONText(t, out, "Docling Title")if got, want := requestCount, 3; got != want {t.Fatalf("requestCount = %d, want %d", got, want)}}-func TestDispatch_PDFOpenDataLoaderMarkdown_UsesConfiguredBackend(t *testing.T) {+func TestDispatch_PDFOpenDataLoaderLegacyMarkdownEmitsJSON(t *testing.T) {withSSRFBypass(t)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {@@ -1385,12 +1293,11 @@ func TestDispatch_PDFOpenDataLoaderMarkdown_UsesConfiguredBackend(t *testing.T)}))defer server.Close()- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "OpenDataLoader"setups["pdf"]["output_format"] = "markdown"setups["pdf"]["opendataloader_apiserver"] = server.URL- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1400,13 +1307,10 @@ func TestDispatch_PDFOpenDataLoaderMarkdown_UsesConfiguredBackend(t *testing.T)if err != nil {t.Fatalf("Invoke: %v", err)}- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "ODL Title") {- t.Fatalf("markdown payload = %#v, want ODL Title", out["markdown"])- }+ requireJSONText(t, out, "ODL Title")}-func TestDispatch_PDFSoMarkMarkdown_UsesConfiguredBackend(t *testing.T) {+func TestDispatch_PDFSoMarkLegacyMarkdownEmitsJSON(t *testing.T) {withSSRFBypass(t)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {@@ -1420,12 +1324,11 @@ func TestDispatch_PDFSoMarkMarkdown_UsesConfiguredBackend(t *testing.T) {}))defer server.Close()- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "SoMark"setups["pdf"]["output_format"] = "markdown"setups["pdf"]["somark_base_url"] = server.URL- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1435,13 +1338,10 @@ func TestDispatch_PDFSoMarkMarkdown_UsesConfiguredBackend(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "SoMark Title") {- t.Fatalf("markdown payload = %#v, want SoMark Title", out["markdown"])- }+ requireJSONText(t, out, "SoMark Title")}-func TestDispatch_PDFTCADPMarkdown_UsesConfiguredBackend(t *testing.T) {+func TestDispatch_PDFTCADPLegacyMarkdownEmitsJSON(t *testing.T) {withSSRFBypass(t)zipPayload := tcadpZipFixtureForComponent(t)var server *httptest.Server@@ -1457,12 +1357,11 @@ func TestDispatch_PDFTCADPMarkdown_UsesConfiguredBackend(t *testing.T) {}))defer server.Close()- param := schema.ParserParam{}.Defaults()setups := defaultSetups()setups["pdf"]["parse_method"] = "TCADP parser"setups["pdf"]["output_format"] = "markdown"setups["pdf"]["tcadp_apiserver"] = server.URL- c := &ParserComponent{Param: param, Setups: setups}+ c := &ParserComponent{setups: setups}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("%PDF-1.4"),@@ -1472,10 +1371,7 @@ func TestDispatch_PDFTCADPMarkdown_UsesConfiguredBackend(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- md, ok := out["markdown"].(string)- if !ok || !strings.Contains(md, "Hello TCADP") {- t.Fatalf("markdown payload = %#v, want Hello TCADP", out["markdown"])- }+ requireJSONText(t, out, "Hello TCADP")}func tcadpZipFixtureForComponent(t *testing.T) []byte {@@ -1495,7 +1391,7 @@ func tcadpZipFixtureForComponent(t *testing.T) []byte {// TestPythonFamilyName_FileTypeConstants pins the contract that every// utility.FileType semantic constant resolves to the setups key used by-// defaultSetups / AllowedOutputFormat. Before the fix, FileTypeVISUAL mapped+// defaultSetups. Before the fix, FileTypeVISUAL mapped// to "picture" (mismatching the "image" setups key) and FileTypeAURAL matched// no case at all (returning ""), so output_format validation and// configureParserFromSetups were silently skipped for image/audio files.@@ -1527,16 +1423,11 @@ func TestPythonFamilyName_FileTypeConstants(t *testing.T) {if got != c.want {t.Errorf("pythonFamilyName(%q) = %q, want %q", c.ft, got, c.want)}- // Every mapped family must have a matching setups key and- // allowed_output_format entry, otherwise output_format validation- // is silently skipped.+ // Every mapped family must have a matching setup so its parser receives+ // the canonical output format and family-specific configuration.if _, ok := defaultSetups()[got]; !ok {t.Errorf("pythonFamilyName(%q) → %q has no defaultSetups entry", c.ft, got)}- allowed := schema.ParserParam{}.Defaults().AllowedOutputFormat- if _, ok := allowed[got]; !ok {- t.Errorf("pythonFamilyName(%q) → %q has no allowed_output_format entry", c.ft, got)- }}}diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.goindex 0d2900cc10..aecbd754e0 100644--- a/internal/ingestion/component/parser_test.go+++ b/internal/ingestion/component/parser_test.go@@ -18,14 +18,12 @@ package componentimport ("encoding/json"- "reflect""strings""testing""github.com/xuri/excelize/v2""ragflow/internal/agent/runtime""ragflow/internal/entity"- "ragflow/internal/ingestion/component/schema")func TestReportParserWarningsAsProgressMessages(t *testing.T) {@@ -79,13 +77,50 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {if len(out) == 0 {t.Errorf("Outputs() returned empty map")}- // The contract from the file header: at least "binary" in,- // "pages" out. Anything else is informational.- if _, ok := in["binary"]; !ok {- t.Errorf("Inputs() missing key %q", "binary")+ // The component catalog must expose every caller-provided input that+ // Invoke reads before dispatching.+ for _, key := range []string{"binary", "name", "file", "file_type", "lang", "doc_id", "bucket", "path"} {+ if _, ok := in[key]; !ok {+ t.Errorf("Inputs() missing runtime input key %q", key)+ }+ }+ if _, ok := out["output_format"]; !ok {+ t.Errorf("Outputs() missing key %q", "output_format")}- if _, ok := out["pages"]; !ok {- t.Errorf("Outputs() missing key %q", "pages")+ if _, ok := out["json"]; !ok {+ t.Errorf("Outputs() missing key %q", "json")+ }+ for _, key := range []string{"name", "lang", "file", "doc_id", "bucket", "path"} {+ if _, ok := out[key]; !ok {+ t.Errorf("Outputs() missing runtime output key %q", key)+ }+ }+ if _, ok := out["_ERROR"]; ok {+ t.Error("Outputs() must not advertise _ERROR; Parser failures return Go errors")+ }+}++func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {+ component, err := NewParserComponent(map[string]any{+ "pdf": map[string]any{"output_format": "markdown"},+ "spreadsheet": map[string]any{"output_format": "html"},+ "email": map[string]any{"output_format": "text"},+ "allowed_output_format": map[string]any{+ "pdf": []any{"json", "markdown"},+ },+ })+ if err != nil {+ t.Fatalf("NewParserComponent: %v", err)+ }++ parserComponent := component.(*ParserComponent)+ for _, family := range []string{"pdf", "spreadsheet", "email"} {+ if got := parserComponent.setups[family]["output_format"]; got != "json" {+ t.Errorf("%s output_format = %v, want json", family, got)+ }+ }+ if _, ok := parserComponent.setups["allowed_output_format"]; ok {+ t.Error("allowed_output_format must not be treated as a parser setup")}}@@ -96,22 +131,25 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {// owned by the canvas framework, not by this component, so they are// not asserted here.func TestParserComponent_Invoke_TextInput(t *testing.T) {- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": "hello world",})if err != nil {t.Fatalf("Invoke: %v", err)}- pages, ok := out["pages"].([]schema.Page)- if !ok {- t.Fatalf("pages: got %T, want []schema.Page", out["pages"])+ if got, want := out["output_format"], "json"; got != want {+ t.Errorf("output_format = %v, want %v", got, want)}- if len(pages) != 1 {- t.Fatalf("pages len = %d, want 1", len(pages))+ jsonItems, ok := out["json"].([]map[string]any)+ if !ok || len(jsonItems) != 1 {+ t.Fatalf("json: got %T (len %d), want 1 item", out["json"], len(jsonItems))}- if got := pages[0]["text"]; got != "hello world" {- t.Errorf("pages[0][text] = %q, want %q", got, "hello world")+ if got := jsonItems[0]["text"]; got != "hello world" {+ t.Errorf("json[0][text] = %q, want %q", got, "hello world")+ }+ if got := jsonItems[0]["doc_type_kwd"]; got != "text" {+ t.Errorf("json[0][doc_type_kwd] = %q, want %q", got, "text")}}@@ -125,7 +163,7 @@ func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {t.Fatalf("Close: %v", err)}- c := &ParserComponent{Param: schema.ParserParam{}.Defaults(), Setups: defaultSetups()}+ c := &ParserComponent{setups: defaultSetups()}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": data.Bytes(),"file_type": "xlsx",@@ -141,55 +179,45 @@ func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {if len(items) != 0 {t.Fatalf("empty XLSX JSON = %v, want no raw-binary items", items)}- pages, ok := out["pages"].([]schema.Page)- if !ok {- t.Fatalf("pages = %T, want []schema.Page", out["pages"])- }- if len(pages) != 0 {- t.Fatalf("empty XLSX pages = %v, want none", pages)- }}// TestParserComponent_Invoke_PageRangeFilter asserts that// form-feed boundaries are honored: "A\fB\fC" yields three-// pages, in input order, with text intact.+// items, in input order, with text intact.func TestParserComponent_Invoke_PageRangeFilter(t *testing.T) {- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": "pageA\fpageB\fpageC",})if err != nil {t.Fatalf("Invoke: %v", err)}- pages, ok := out["pages"].([]schema.Page)+ jsonItems, ok := out["json"].([]map[string]any)if !ok {- t.Fatalf("pages: got %T, want []schema.Page", out["pages"])+ t.Fatalf("json: got %T, want []map[string]any", out["json"])}- if len(pages) != 3 {- t.Fatalf("pages len = %d, want 3", len(pages))+ if len(jsonItems) != 3 {+ t.Fatalf("json len = %d, want 3", len(jsonItems))}want := []string{"pageA", "pageB", "pageC"}- for i, p := range pages {- if got := p["text"]; got != want[i] {- t.Errorf("pages[%d][text] = %q, want %q", i, got, want[i])+ for i, it := range jsonItems {+ if got := it["text"]; got != want[i] {+ t.Errorf("json[%d][text] = %q, want %q", i, got, want[i])}}}-// TestParserComponent_Invoke_DeterministicMerge is the-// golden-file test for plan §8 R8 (DETERMINISTIC MERGE).+// TestParserComponent_Invoke_DeterministicMerge verifies stable output order.//// We invoke the component 5 times with identical input and// assert byte-for-byte equality of the JSON-encoded output.// The test is expected to pass under `go test -count=10 -race`// — that flag is run separately in the verification block.//-// Text-page mode has no page_number key, so the input order is-// the output order; the deterministic sort keeps that order-// stable across runs, which is the contract the downstream-// chunker relies on for stable chunk IDs.+// Text-page mode preserves input order, which downstream chunkers rely on for+// stable chunk IDs.func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}// 8 form-feed-separated pages.input := "p1\fp2\fp3\fp4\fp5\fp6\fp7\fp8"@@ -200,7 +228,7 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {if err != nil {t.Fatalf("Invoke (first): %v", err)}- canonical, err := json.Marshal(first["pages"])+ canonical, err := json.Marshal(first["json"])if err != nil {t.Fatalf("Marshal canonical: %v", err)}@@ -213,7 +241,7 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {if err != nil {t.Fatalf("Invoke (run %d): %v", i, err)}- encoded, err := json.Marshal(got["pages"])+ encoded, err := json.Marshal(got["json"])if err != nil {t.Fatalf("Marshal run %d: %v", i, err)}@@ -224,10 +252,8 @@ func TestParserComponent_Invoke_DeterministicMerge(t *testing.T) {}}-// TestParserComponent_New_Defaults constructs a Parser from a-// nil param map and verifies the static Param is the-// Defaults() value (i.e., NewParserComponent does not mutate-// the schema default).+// TestParserComponent_New_Defaults constructs a Parser from a nil param map+// and verifies every parser family uses the canonical JSON output format.func TestParserComponent_New_Defaults(t *testing.T) {c, err := NewParserComponent(nil)if err != nil {@@ -237,9 +263,10 @@ func TestParserComponent_New_Defaults(t *testing.T) {if !ok {t.Fatalf("NewParserComponent returned %T, want *ParserComponent", c)}- defaults := schema.ParserParam{}.Defaults()- if !reflect.DeepEqual(pc.Param, defaults) {- t.Errorf("Param differs from Defaults:\n got=%+v\nwant=%+v", pc.Param, defaults)+ for family, setup := range pc.setups {+ if got := setup["output_format"]; got != "json" {+ t.Errorf("%s output_format = %v, want json", family, got)+ }}}@@ -259,7 +286,7 @@ func TestParserComponent_New_Overrides(t *testing.T) {if !ok {t.Fatalf("NewParserComponent returned %T", c)}- setup, ok := pc.Setups["text&code"]+ setup, ok := pc.setups["text&code"]if !ok {t.Fatalf("Setups[text&code] missing after override")}@@ -267,15 +294,38 @@ func TestParserComponent_New_Overrides(t *testing.T) {t.Errorf("Setups[text&code][chunk_token_size] = %v, want 256", setup["chunk_token_size"])}// Defaults must still be present for other file types.- if _, ok := pc.Setups["pdf"]; !ok {+ if _, ok := pc.setups["pdf"]; !ok {t.Errorf("Setups[pdf] missing; override should not erase defaults")}}+func TestParserComponent_NewOwnsNestedSetup(t *testing.T) {+ vlm := map[string]any{"llm_id": "original"}+ c, err := NewParserComponent(map[string]any{+ "pdf": map[string]any{"vlm": vlm},+ })+ if err != nil {+ t.Fatalf("NewParserComponent: %v", err)+ }+ pc, ok := c.(*ParserComponent)+ if !ok {+ t.Fatalf("NewParserComponent returned %T, want *ParserComponent", c)+ }++ vlm["llm_id"] = "mutated"+ gotVLM, ok := pc.setups["pdf"]["vlm"].(map[string]any)+ if !ok {+ t.Fatalf("pdf.vlm = %T, want map[string]any", pc.setups["pdf"]["vlm"])+ }+ if got := gotVLM["llm_id"]; got != "original" {+ t.Errorf("pdf.vlm.llm_id = %v after caller mutation, want original", got)+ }+}+// TestParserComponent_Invoke_DocIDCarried asserts the optional// doc_id input flows through to the "name" output.func TestParserComponent_Invoke_DocIDCarried(t *testing.T) {- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": "x","doc_id": "doc-123",@@ -311,17 +361,17 @@ func TestParserComponent_Invoke_ResolvesBinaryFromDocID(t *testing.T) {t.Fatalf("seed doc: %v", err)}- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}out, err := c.Invoke(ctx, db, map[string]any{"doc_id": "doc-parser"})if err != nil {t.Fatalf("Invoke: %v", err)}- pages, ok := out["pages"].([]schema.Page)- if !ok || len(pages) != 2 {- t.Fatalf("pages = %T/%v, want 2 schema.Page entries", out["pages"], out["pages"])+ jsonItems, ok := out["json"].([]map[string]any)+ if !ok || len(jsonItems) != 2 {+ t.Fatalf("json = %T/%v, want 2 items", out["json"], out["json"])}- if pages[0]["text"] != "alpha" || pages[1]["text"] != "beta" {- t.Fatalf("pages = %+v, want [alpha beta]", pages)+ if jsonItems[0]["text"] != "alpha" || jsonItems[1]["text"] != "beta" {+ t.Fatalf("json = %+v, want [alpha beta]", jsonItems)}if got, _ := out["name"].(string); got != "doc-parser" {t.Fatalf("name = %q, want %q", got, "doc-parser")@@ -335,7 +385,7 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {t.Fatalf("seed storage: %v", err)}- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}out, err := c.Invoke(ctx, nil, map[string]any{"bucket": "bucket-1","path": "docs/explicit.txt",@@ -343,12 +393,12 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- pages, ok := out["pages"].([]schema.Page)- if !ok || len(pages) != 1 {- t.Fatalf("pages = %T/%v, want 1 schema.Page entry", out["pages"], out["pages"])+ jsonItems, ok := out["json"].([]map[string]any)+ if !ok || len(jsonItems) != 1 {+ t.Fatalf("json = %T/%v, want 1 item", out["json"], out["json"])}- if got := pages[0]["text"]; got != "bucket content" {- t.Fatalf("pages[0][text] = %q, want %q", got, "bucket content")+ if got := jsonItems[0]["text"]; got != "bucket content" {+ t.Fatalf("json[0][text] = %q, want %q", got, "bucket content")}}@@ -358,7 +408,7 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {// look like this if a caller mistakenly handed a base64 string// without decoding it).func TestParserComponent_Invoke_RejectsInvalidUTF8(t *testing.T) {- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}_, err := c.Invoke(t.Context(), nil, map[string]any{// 0xFF alone is not valid UTF-8 start byte."binary": string([]byte{0xFF, 0xFE, 0xFD}),@@ -375,49 +425,21 @@ func TestParserComponent_Invoke_RejectsInvalidUTF8(t *testing.T) {// caller's normal form ([]byte) — the alternative to a UTF-8// string.func TestParserComponent_Invoke_AcceptsBytes(t *testing.T) {- c := &ParserComponent{Param: schema.ParserParam{}.Defaults()}+ c := &ParserComponent{}out, err := c.Invoke(t.Context(), nil, map[string]any{"binary": []byte("alpha\fbeta"),})if err != nil {t.Fatalf("Invoke: %v", err)}- pages, ok := out["pages"].([]schema.Page)+ jsonItems, ok := out["json"].([]map[string]any)if !ok {- t.Fatalf("pages: got %T", out["pages"])+ t.Fatalf("json: got %T", out["json"])}- if len(pages) != 2 {- t.Fatalf("pages len = %d, want 2", len(pages))+ if len(jsonItems) != 2 {+ t.Fatalf("json len = %d, want 2", len(jsonItems))}- if pages[0]["text"] != "alpha" || pages[1]["text"] != "beta" {- t.Errorf("pages = %+v, want [alpha beta]", pages)- }-}--// TestBuildPagesFromBytes_FormatAgnostic pins the page-builder-// contract: buildPagesFromBytes does not resolve parsers or inspect-// file families. It only wraps already-prepared page bytes into-// schema.Page items.-func TestBuildPagesFromBytes_FormatAgnostic(t *testing.T) {- got, err := buildPagesFromBytes(t.Context(), [][]byte{- []byte("first page from dispatch"),- []byte("<table>second page from html dispatch</table>"),- }, "")- if err != nil {- t.Fatalf("buildPagesFromBytes: %v", err)- }- if len(got) != 2 {- t.Fatalf("len(got) = %d, want 2", len(got))- }- if got[0]["text"] != "first page from dispatch" {- t.Fatalf("got[0][text] = %v, want first page from dispatch", got[0]["text"])- }- if got[1]["text"] != "<table>second page from html dispatch</table>" {- t.Fatalf("got[1][text] = %v, want HTML payload preserved verbatim", got[1]["text"])- }- for i := range got {- if got[i]["doc_type_kwd"] != "text" {- t.Fatalf("got[%d][doc_type_kwd] = %v, want text", i, got[i]["doc_type_kwd"])- }+ if jsonItems[0]["text"] != "alpha" || jsonItems[1]["text"] != "beta" {+ t.Errorf("json = %+v, want [alpha beta]", jsonItems)}}diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 40265d9451..b18a5e2d15 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -24,6 +24,7 @@ import ("ragflow/internal/entity"modelModule "ragflow/internal/entity/models""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser""ragflow/internal/utility""go.uber.org/zap"@@ -68,13 +69,13 @@ func maybeDispatchPDFVision(binary []byte,inputs map[string]any,setups map[string]schema.ParserSetup,-) (parserDispatchResult, bool, error) {+) (parser.ParseResult, bool, error) {if fileType != utility.FileTypePDF {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}setup, ok := setups["pdf"]if !ok {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}method := getStringOr(setup, "parse_method", "")@@ -113,12 +114,12 @@ func maybeDispatchPDFVision(zap.String("layout_recognizer", layout),zap.String("tenant_id", tenantID))if tenantID == "" {- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: MinerU requires tenant_id")}res, err := dispatchMinerUPDF(ctx, db, filename, binary, tenantID, setup)if err != nil {- return parserDispatchResult{}, true, err+ return parser.ParseResult{}, true, err}return res, true, nil}@@ -150,7 +151,7 @@ func maybeDispatchPDFVision(}if isPaddleOCRMatch || isPaddleOCRByUUID {if tenantID == "" {- return parserDispatchResult{}, true,+ return parser.ParseResult{}, true,fmt.Errorf("parser: PaddleOCR requires tenant_id")}// Only a resolved UUID may be passed to the model resolver; a string@@ -162,27 +163,27 @@ func maybeDispatchPDFVision(}res, err := dispatchPaddleOCRPdf(ctx, db, filename, binary, tenantID, setup, dispatchModelID)if err != nil {- return parserDispatchResult{}, true, err+ return parser.ParseResult{}, true, err}return res, true, nil}modelID, useVision := resolvePDFVisionModelID(setup)if !useVision {- return parserDispatchResult{}, false, nil+ return parser.ParseResult{}, false, nil}if tenantID == "" {- return parserDispatchResult{}, true, fmt.Errorf(+ return parser.ParseResult{}, true, fmt.Errorf(`parser: pdf parse_method %q requires tenant_id to resolve VLM model`, modelID)}- res, err := dispatchPDFVision(ctx, db, filename, binary, tenantID, modelID, setup)+ res, err := dispatchPDFVision(ctx, db, filename, binary, tenantID, modelID)if err != nil {- return parserDispatchResult{}, true, err+ return parser.ParseResult{}, true, err}return res, true, nil}-func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, binary []byte, tenantID string, setup schema.ParserSetup, modelRef string) (parserDispatchResult, error) {+func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, binary []byte, tenantID string, setup schema.ParserSetup, modelRef string) (parser.ParseResult, error) {baseURL := strings.TrimRight(getStringOr(setup, "monkeyocrv2_server_url", os.Getenv(common.EnvMonkeyOCRv2ServerURL)), "/")timeout := monkeyOCRv2RequestTimeout(setup, nil)if tenantID != "" {@@ -196,7 +197,7 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b}if err == nil {if !strings.EqualFold(driver.Name(), "monkeyocrv2") {- return parserDispatchResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires a MonkeyOCRv2 OCR model; found %q", driver.Name())+ return parser.ParseResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires a MonkeyOCRv2 OCR model; found %q", driver.Name())}if apiConfig != nil && apiConfig.BaseURL != nil && strings.TrimSpace(*apiConfig.BaseURL) != "" {baseURL = strings.TrimRight(*apiConfig.BaseURL, "/")@@ -205,21 +206,53 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b}timeout = monkeyOCRv2RequestTimeout(setup, apiConfig)} else if baseURL == "" {- return parserDispatchResult{}, fmt.Errorf("parser: MonkeyOCRv2 model: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: MonkeyOCRv2 model: %w", err)}}if baseURL == "" {- return parserDispatchResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires monkeyocrv2_server_url or MONKEYOCRV2_SERVER_URL")+ return parser.ParseResult{}, fmt.Errorf("parser: MonkeyOCRv2 requires monkeyocrv2_server_url or MONKEYOCRV2_SERVER_URL")}zipBytes, err := monkeyOCRv2ParseRequest(ctx, baseURL+"/parse", filename, binary, timeout)if err != nil {- return parserDispatchResult{}, err+ return parser.ParseResult{}, err}sections, err := monkeyOCRv2ExtractSections(zipBytes)if err != nil {- return parserDispatchResult{}, err+ return parser.ParseResult{}, err+ }+ md := strings.Join(sections, "\n\n")+ return buildMarkdownOCRDispatchResult(md), nil+}++func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []map[string]any {+ mp, err := parser.NewMarkdownParser(parser.GoMarkdown)+ if err != nil {+ warnParserNormalizationFallback(filename, "markdown", err)+ return []map[string]any{parser.NewTextJSONItem(mdText)}+ }+ // OCR Markdown is an untrusted backend response. Keep normalization+ // deterministic and local: embedded data URIs remain available, while+ // remote image URLs do not introduce network I/O at the Parser boundary.+ mp.FetchRemoteImages = false+ res := mp.ParseWithResult(ctx, filename, []byte(mdText))+ if res.Err != nil {+ warnParserNormalizationFallback(filename, "markdown", res.Err)+ return []map[string]any{parser.NewTextJSONItem(mdText)}+ }+ if len(res.JSON) == 0 {+ warnParserNormalizationFallback(filename, "markdown", fmt.Errorf("parser returned no JSON items"))+ return []map[string]any{parser.NewTextJSONItem(mdText)}+ }+ return res.JSON+}++func buildMarkdownOCRDispatchResult(md string) parser.ParseResult {+ // Keep the backend's Markdown intact here. buildParserOutputs is the+ // single normalization boundary for all non-JSON parser responses.+ return parser.ParseResult{+ OutputFormat: "markdown",+ Markdown: md,}- return parserDispatchResult{OutputFormat: "markdown", Markdown: strings.Join(sections, "\n\n")}, nil}var isMonkeyOCRv2LayoutModelID = defaultIsMonkeyOCRv2LayoutModelID@@ -447,17 +480,17 @@ func getAnyString(object map[string]any, keys ...string) string {func dispatchMinerUPDF(ctx context.Context,db *gorm.DB,- _ string,+ filename string,binary []byte,tenantID string,setup schema.ParserSetup,-) (parserDispatchResult, error) {+) (parser.ParseResult, error) {driver, _, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeOCR)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: MinerU model: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU model: %w", err)}if !isMinerUDriver(driver) {- return parserDispatchResult{}, fmt.Errorf(+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU requires a MinerU OCR model; found %q. Please add a MinerU OCR model to your tenant", driver.Name())}@@ -478,12 +511,12 @@ func dispatchMinerUPDF(zipBytes, err := mineruStreamParse(apiURL, apiConfig.ApiKey, binary, parseMethod, mineruLang, backend)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: MinerU stream: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)}sections, err := mineruExtractSections(zipBytes)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: MinerU extract: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", err)}var parts []string@@ -494,17 +527,7 @@ func dispatchMinerUPDF(}md := strings.Join(parts, "\n")- // MinerU always returns rendered markdown text (md), regardless of- // the requested output_format; label the payload as markdown so the- // downstream chunker consumes it instead of a nil JSONResult.- if format := strings.TrimSpace(getStringOr(setup, "output_format", "markdown")); !strings.EqualFold(format, "markdown") {- common.Warn("mineru parser: output_format %q requested but backend only returns markdown; treating result as markdown",- zap.String("output_format", format))- }- return parserDispatchResult{- OutputFormat: "markdown",- Markdown: md,- }, nil+ return buildMarkdownOCRDispatchResult(md), nil}// resolvePaddleOCRModelForDispatch resolves the OCR model used by the@@ -546,13 +569,13 @@ func dispatchPaddleOCRPdf(tenantID string,setup schema.ParserSetup,modelID string,-) (parserDispatchResult, error) {+) (parser.ParseResult, error) {driver, modelName, apiConfig, err := resolvePaddleOCRModelForDispatch(ctx, db, tenantID, modelID)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR model: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR model: %w", err)}if !isPaddleOCRDriver(driver) {- return parserDispatchResult{}, fmt.Errorf(+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR requires a PaddleOCR OCR model; found %q. Please add a PaddleOCR OCR model to your tenant", driver.Name())}@@ -581,7 +604,7 @@ func dispatchPaddleOCRPdf(baseURL = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRAPIURL))}if baseURL == "" {- return parserDispatchResult{}, fmt.Errorf(+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR requires a base url from the tenant PaddleOCR OCR model or PADDLEOCR_BASE_URL")}@@ -611,28 +634,16 @@ func dispatchPaddleOCRPdf(Algorithm: algorithm,}, nil)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR OCRFile: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR OCRFile: %w", err)}if resp == nil || resp.Text == nil {- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")}if strings.TrimSpace(*resp.Text) == "" {- return parserDispatchResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")+ return parser.ParseResult{}, fmt.Errorf("parser: PaddleOCR returned empty text")}- // PaddleOCR backends always return rendered markdown text- // (OCRFile.Text), regardless of the requested output_format. The- // payload MUST be labelled markdown so the downstream chunker- // consumes the text; a non-markdown setup value only means this- // backend cannot produce the requested layout format.- if format := strings.TrimSpace(getStringOr(setup, "output_format", "markdown")); !strings.EqualFold(format, "markdown") {- common.Warn("paddleocr parser: output_format %q requested but backend only returns markdown; treating result as markdown",- zap.String("output_format", format))- }- return parserDispatchResult{- OutputFormat: "markdown",- Markdown: *resp.Text,- }, nil+ return buildMarkdownOCRDispatchResult(*resp.Text), nil}// resolveMinerUBaseURL extracts the resolved base URL from a model driver.@@ -873,28 +884,26 @@ func dispatchPDFVision(binary []byte,tenantID string,modelID string,- setup schema.ParserSetup,-) (parserDispatchResult, error) {+) (parser.ParseResult, error) {renderedPages, err := pdfVisionPageRenderer(binary)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: pdf vision render: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: pdf vision render: %w", err)}driver, resolvedModelName, apiConfig, err := pdfVisionModelResolver(ctx, db, tenantID, modelID)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: pdf vision model %q: %w", modelID, err)+ return parser.ParseResult{}, fmt.Errorf("parser: pdf vision model %q: %w", modelID, err)}promptTemplate, err := pdfVisionPromptLoader("vision_llm_describe_prompt")if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: load vision prompt: %w", err)+ return parser.ParseResult{}, fmt.Errorf("parser: load vision prompt: %w", err)}items := make([]map[string]any, 0, len(renderedPages))- markdownParts := make([]string, 0, len(renderedPages))for _, page := range renderedPages {prompt := renderPDFVisionPrompt(promptTemplate, page.PageNumber)resp, err := pdfVisionChatInvoker(ctx, driver, resolvedModelName, buildPDFVisionMessages(prompt, page.ImageURL), apiConfig)if err != nil {- return parserDispatchResult{}, fmt.Errorf("parser: pdf vision page %d: %w", page.PageNumber, err)+ return parser.ParseResult{}, fmt.Errorf("parser: pdf vision page %d: %w", page.PageNumber, err)}text := extractVisionAnswer(resp)positions := [][]any{{page.PageNumber, 0.0, page.WidthPts, 0.0, page.HeightPts}}@@ -905,37 +914,19 @@ func dispatchPDFVision("_pdf_positions": positions,"positions": positions,})- if text != "" {- markdownParts = append(markdownParts, text)- }}- outputFormat := "json"- if v, ok := setup["output_format"].(string); ok && strings.TrimSpace(v) != "" {- outputFormat = strings.ToLower(strings.TrimSpace(v))- }fileMeta := map[string]any{"name": filename,"page_count": len(renderedPages),"outline": []map[string]any{},"parse_method": modelID,}- switch outputFormat {- case "json":- return parserDispatchResult{- OutputFormat: "json",- File: fileMeta,- JSON: items,- }, nil- case "markdown":- return parserDispatchResult{- OutputFormat: "markdown",- File: fileMeta,- Markdown: strings.TrimSpace(strings.Join(markdownParts, "\n\n")),- }, nil- default:- return parserDispatchResult{}, fmt.Errorf("parser: unsupported PDF output_format %q for vision parse_method %q", outputFormat, modelID)- }+ return parser.ParseResult{+ OutputFormat: "json",+ File: fileMeta,+ JSON: items,+ }, nil}func buildPDFVisionMessages(prompt string, imageURL string) []modelModule.Message {diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex 78d2cfacd8..899b3c166b 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -106,9 +106,30 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {if err != nil {t.Fatal(err)}- if result.OutputFormat != "markdown" || result.Markdown != "hello" {+ if result.OutputFormat != "markdown" || result.Markdown != "hello" || result.JSON != nil {t.Fatalf("result=%+v", result)}++ resultMD, err := dispatchMonkeyOCRv2PDF(context.Background(), nil, "sample.pdf", []byte("pdf"), "", schema.ParserSetup{"monkeyocrv2_server_url": server.URL, "output_format": "markdown"}, "")+ if err != nil {+ t.Fatal(err)+ }+ if resultMD.OutputFormat != "markdown" || resultMD.Markdown != "hello" {+ t.Fatalf("resultMD=%+v", resultMD)+ }+}++func TestBuildMarkdownOCRDispatchResultDefersJSONNormalization(t *testing.T) {+ result := buildMarkdownOCRDispatchResult("# Heading")+ if result.OutputFormat != "markdown" {+ t.Fatalf("OutputFormat = %q, want markdown", result.OutputFormat)+ }+ if result.Markdown != "# Heading" {+ t.Fatalf("Markdown = %q, want original OCR markdown", result.Markdown)+ }+ if result.JSON != nil {+ t.Fatalf("JSON = %#v, want nil until buildParserOutputs normalization", result.JSON)+ }}type monkeyOCRv2FakeDriver struct {@@ -160,14 +181,10 @@ func (d *paddleOCRFakeDriver) OCRFile(_ context.Context, _ *string, _ []byte, _return &modelModule.OCRFileResponse{Text: &d.text}, nil}-// TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown guards against the-// format-mismatch bug: PaddleOCR backends always return markdown text via-// OCRFile.Text, so the dispatch result MUST be labelled OutputFormat-// "markdown" regardless of what setup["output_format"] says (the pdf default-// is "json"). If the setup value leaked into OutputFormat, buildParserOutputs-// would emit a nil "json" payload and the downstream TokenChunker would-// consume an empty JSONResult -> "completed with 0 chunks".-func TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown(t *testing.T) {+// TestDispatchPaddleOCRPdfPreservesMarkdownForCentralNormalization verifies+// that PaddleOCR dispatch returns backend Markdown for the shared Parser+// output normalizer, regardless of the normalized component setup format.+func TestDispatchPaddleOCRPdfPreservesMarkdownForCentralNormalization(t *testing.T) {orig := resolvePaddleOCRModelForDispatcht.Cleanup(func() { resolvePaddleOCRModelForDispatch = orig })@@ -177,8 +194,7 @@ func TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown(t *testing.T) {return &paddleOCRFakeDriver{text: md}, "ocr-model", &modelModule.APIConfig{BaseURL: &baseURL}, nil}- // The real run had output_format=json in the pdf setup; the payload must- // still be labelled markdown because that is what the backend produced.+ // 1. output_format="json": preserve Markdown for central normalization.res, err := dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", schema.ParserSetup{"output_format": "json"}, "some-uuid")if err != nil {t.Fatalf("dispatchPaddleOCRPdf: %v", err)@@ -186,20 +202,41 @@ func TestDispatchPaddleOCRPdfLabelsPayloadAsMarkdown(t *testing.T) {if res.OutputFormat != "markdown" {t.Errorf("OutputFormat = %q, want markdown", res.OutputFormat)}+ if res.JSON != nil {+ t.Fatalf("res.JSON = %#v, want nil before central normalization", res.JSON)+ }if res.Markdown != md {t.Errorf("Markdown = %q, want %q", res.Markdown, md)}- // Default setup (no output_format key) must behave identically.- res, err = dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", nil, "some-uuid")+ // 2. output_format="markdown": the backend payload is unchanged.+ resMD, err := dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", schema.ParserSetup{"output_format": "markdown"}, "some-uuid")+ if err != nil {+ t.Fatalf("dispatchPaddleOCRPdf (markdown): %v", err)+ }+ if resMD.OutputFormat != "markdown" {+ t.Errorf("OutputFormat = %q, want markdown", resMD.OutputFormat)+ }+ if resMD.Markdown != md {+ t.Errorf("Markdown = %q, want %q", resMD.Markdown, md)+ }+ if resMD.JSON != nil {+ t.Errorf("resMD.JSON = %#v, want nil before central normalization", resMD.JSON)+ }++ // 3. Default setup (no output_format key) follows the same central path.+ resDef, err := dispatchPaddleOCRPdf(t.Context(), dao.DB, "test.pdf", []byte("%PDF-1.4"), "tenant", nil, "some-uuid")if err != nil {t.Fatalf("dispatchPaddleOCRPdf (default setup): %v", err)}- if res.OutputFormat != "markdown" {- t.Errorf("OutputFormat = %q, want markdown", res.OutputFormat)+ if resDef.OutputFormat != "markdown" {+ t.Errorf("OutputFormat = %q, want markdown", resDef.OutputFormat)}- if res.Markdown != md {- t.Errorf("Markdown = %q, want %q", res.Markdown, md)+ if resDef.JSON != nil {+ t.Fatalf("resDef.JSON = %#v, want nil before central normalization", resDef.JSON)+ }+ if resDef.Markdown != md {+ t.Errorf("Markdown = %q, want %q", resDef.Markdown, md)}}diff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.goindex 5f6e91d1fc..da429fa717 100644--- a/internal/ingestion/component/schema/parser.go+++ b/internal/ingestion/component/schema/parser.go@@ -16,121 +16,34 @@package schema-// ParserFromUpstream is the upstream payload consumed by the Parser-// component. It mirrors rag/flow/parser/schema.py:ParserFromUpstream-// (Pydantic BaseModel with populate_by_name, extra="forbid").-//-// created_time: float | None (alias _created_time)-// elapsed_time: float | None (alias _elapsed_time)-// name: str (required)-// file: dict | None-// abstract: bool = False-// author: bool = False-type ParserFromUpstream struct {- CreatedTime *float64 `json:"_created_time,omitempty"`- ElapsedTime *float64 `json:"_elapsed_time,omitempty"`-- Name string `json:"name"`-- // File is the optional upstream file descriptor. Python allows None- // when the parser is invoked via a canvas-bound doc_id path.- File map[string]any `json:"file,omitempty"`-- Abstract bool `json:"abstract,omitempty"`- Author bool `json:"author,omitempty"`-}--// Validate enforces the only required field in ParserFromUpstream: Name.-// Returns nil when Name is non-empty.-func (p *ParserFromUpstream) Validate() error {- if p.Name == "" {- return errRequiredField{Field: "name"}- }- return nil-}--// Page is one parsed document section. The Parser component does not emit-// a typed page model — Python code passes around `dict` literals with-// shape `{text, layout_type, doc_type_kwd, positions?, image?, ...}`. To-// keep the wire schema typed without overcommitting to a parser-specific-// shape, Page is left as a generic map and provided for forward-// documentation; downstream chunker code operates on the same dict shape.-type Page map[string]any--// ParserSetup is the per-filetype configuration block stored on-// ParserParam.setups[fileType]. The keys are heterogeneous (e.g.,+// ParserSetup is a per-filetype parser configuration block. The keys are heterogeneous (e.g.,// `parse_method`, `lang`, `output_format`, `suffix`, `fields`, `vlm`),// so a free-form map best mirrors the Python dict literal.type ParserSetup map[string]any-// ParserParam is the static configuration for the Parser component.-type ParserParam struct {- AllowedOutputFormat map[string][]string `json:"allowed_output_format"`-}--// Defaults returns a ParserParam populated with the Python defaults —-// the full setups table copied verbatim from-// rag/flow/parser/parser.py:ParserParam.__init__ and the corresponding-// allowed_output_format map.-func (ParserParam) Defaults() ParserParam {- return ParserParam{- AllowedOutputFormat: map[string][]string{- "pdf": {"json", "markdown"},- "spreadsheet": {"json", "markdown", "html"},- "doc": {"json", "markdown"},- "docx": {"json", "markdown"},- "slides": {"json"},- "image": {"json"},- "email": {"text", "json"},- "markdown": {"text", "json"},- "text&code": {"text", "json"},- "html": {"text", "json"},- "audio": {"text", "json"},- "video": {},- "epub": {"text", "json"},- "json": {"json"},- },- }-}--// Validate returns nil. ParserParam's field set is fully defaulted by-// Defaults(); the component's own `check()` method performs business-// validation (e.g., "parse_method" must be one of the allowed set), and-// that runs in the component implementation.-func (ParserParam) Validate() error { return nil }-// ParserOutputs is the result of invoking the Parser component. The-// Python component calls `self.set_output(...)` with a mix of-// string-typed, list-typed, and dict-typed values. The wire schema-// below is the typed surface consumed by downstream components.-//-// Mirrors what Parser sets at rag/flow/parser/parser.py:_invoke. The-// parser writes to EITHER ("json" | "markdown" | "text" | "html") and-// always sets "output_format" + "file" + "_ERROR".+// wire format is "json", with structured JSON items as the only payload.type ParserOutputs struct {- // OutputFormat is the active output format for this run- // (one of "json", "markdown", "text", "html"). The downstream- // Tokenizer branches on this field.- OutputFormat string `json:"output_format,omitempty"`-- // JSON holds the list of structured sections when output_format == "json".- JSON []map[string]any `json:"json,omitempty"`+ // Name is the resolved source filename.+ Name string `json:"name"`- // Markdown holds the rendered Markdown when output_format == "markdown".- Markdown string `json:"markdown,omitempty"`+ // OutputFormat is always "json". Downstream components consume structured JSON items.+ OutputFormat string `json:"output_format,omitempty"`- // Text holds the rendered plain text when output_format == "text".- Text string `json:"text,omitempty"`+ // JSON holds the list of structured sections (primary payload).+ JSON []map[string]any `json:"json"`- // HTML holds the rendered HTML when output_format == "html".- HTML string `json:"html,omitempty"`+ // Lang is the language used by downstream tokenization and model calls.+ Lang string `json:"lang,omitempty"`- // File is the upstream file descriptor with parser-derived metadata- // (e.g., outlines) merged in. Mirrors the Python `set_output("file", ...)`- // at parser.py:609, 791, 828.+ // File is backend-produced file metadata (for example page_count,+ // outline, format, or sheets). When present, it replaces rather than+ // merges with the upstream file descriptor.File map[string]any `json:"file,omitempty"`- // Error is set when the component short-circuits with an error- // message (Python: set_output("_ERROR", ...)).- Error string `json:"_ERROR,omitempty"`+ // DocID, Bucket, and Path let downstream chunkers reacquire the source PDF+ // for preview cropping without carrying its binary in the payload.+ DocID string `json:"doc_id,omitempty"`+ Bucket string `json:"bucket,omitempty"`+ Path string `json:"path,omitempty"`}diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex 609b496c1c..0943c23a73 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -120,98 +120,64 @@ func TestFileOutputsJSONRoundTrip(t *testing.T) {// Parser// ----------------------------------------------------------------------------func TestParserFromUpstreamValidate(t *testing.T) {- // Name is required.- if err := (&ParserFromUpstream{}).Validate(); err == nil {- t.Fatal("expected Validate to fail when Name is empty")- }- if err := (&ParserFromUpstream{Name: "doc.pdf"}).Validate(); err != nil {- t.Fatalf("Validate with Name unexpectedly failed: %v", err)- }-}--func TestParserParamDefaults(t *testing.T) {- p := ParserParam{}.Defaults()- if err := p.Validate(); err != nil {- t.Fatalf("default ParserParam failed Validate: %v", err)- }- if got := p.AllowedOutputFormat["pdf"]; len(got) != 2 || got[0] != "json" || got[1] != "markdown" {- t.Errorf("default pdf allowed_output_format = %v, want [json markdown]", got)+func TestParserOutputsJSONRoundTrip(t *testing.T) {+ original := ParserOutputs{+ Name: "input.pdf",+ OutputFormat: "json",+ JSON: []map[string]any{{"text": "hello", "doc_type_kwd": "text"}},+ Lang: "English",+ File: map[string]any{"name": "input.pdf", "page_count": float64(1)},+ DocID: "doc-1",+ Bucket: "bucket-1",+ Path: "tenant/doc-1.pdf",}-}--func TestParserParamJSONRoundTrip(t *testing.T) {- original := ParserParam{}.Defaults()data, err := json.Marshal(original)if err != nil {t.Fatalf("marshal: %v", err)}- if !strings.Contains(string(data), `"allowed_output_format"`) {- t.Errorf("expected allowed_output_format in JSON, got %s", data)+ if !strings.Contains(string(data), `"output_format":"json"`) {+ t.Errorf("expected output_format in JSON, got %s", data)}- var decoded ParserParam- if err = json.Unmarshal(data, &decoded); err != nil {+ var decoded ParserOutputs+ if err := json.Unmarshal(data, &decoded); err != nil {t.Fatalf("unmarshal: %v", err)}- if got := decoded.AllowedOutputFormat["pdf"]; len(got) != 2 || got[0] != "json" || got[1] != "markdown" {- t.Errorf("round-trip lost pdf allowed_output_format: got %v", got)- }-}--func TestParserFromUpstreamJSONRoundTrip(t *testing.T) {- original := ParserFromUpstream{- Name: "input.pdf",- Abstract: true,- Author: false,- }- data, err := json.Marshal(original)- if err != nil {- t.Fatalf("marshal: %v", err)+ if decoded.OutputFormat != "json" {+ t.Errorf("OutputFormat round-trip mismatch: got %q", decoded.OutputFormat)}- // abstract=true should be emitted; author=false has omitempty so it's- // dropped (zero-value bool with omitempty). We test the- // non-zero path.- if !strings.Contains(string(data), `"abstract":true`) {- t.Errorf("expected abstract=true in JSON, got %s", data)+ if len(decoded.JSON) != 1 {+ t.Errorf("JSON round-trip mismatch: got %d", len(decoded.JSON))}- var decoded ParserFromUpstream- if err := json.Unmarshal(data, &decoded); err != nil {- t.Fatalf("unmarshal: %v", err)+ if decoded.Name != original.Name || decoded.Lang != original.Lang {+ t.Errorf("parser identity round-trip mismatch: got name=%q lang=%q", decoded.Name, decoded.Lang)}- if decoded.Name != original.Name {- t.Errorf("Name round-trip mismatch: got %q", decoded.Name)+ if decoded.DocID != original.DocID || decoded.Bucket != original.Bucket || decoded.Path != original.Path {+ t.Errorf("parser storage round-trip mismatch: got doc_id=%q bucket=%q path=%q", decoded.DocID, decoded.Bucket, decoded.Path)}- if !decoded.Abstract {- t.Errorf("Abstract round-trip mismatch: got %v", decoded.Abstract)- }- // author field is omitempty, so JSON round-trip should leave it false- // (default value) on both sides.- if decoded.Author {- t.Errorf("Author should be false, got true")+ if decoded.File["name"] != "input.pdf" {+ t.Errorf("parser file metadata round-trip mismatch: got %#v", decoded.File)}}-func TestParserOutputsJSONRoundTrip(t *testing.T) {+func TestParserOutputsJSONRoundTripPreservesEmptyItems(t *testing.T) {original := ParserOutputs{+ Name: "empty.txt",OutputFormat: "json",- JSON: []map[string]any{{"text": "hello", "doc_type_kwd": "text"}},+ JSON: []map[string]any{},}data, err := json.Marshal(original)if err != nil {t.Fatalf("marshal: %v", err)}- if !strings.Contains(string(data), `"output_format":"json"`) {- t.Errorf("expected output_format in JSON, got %s", data)+ if !strings.Contains(string(data), `"json":[]`) {+ t.Fatalf("empty JSON payload omitted: %s", data)}var decoded ParserOutputsif err := json.Unmarshal(data, &decoded); err != nil {t.Fatalf("unmarshal: %v", err)}- if decoded.OutputFormat != "json" {- t.Errorf("OutputFormat round-trip mismatch: got %q", decoded.OutputFormat)- }- if len(decoded.JSON) != 1 {- t.Errorf("JSON round-trip mismatch: got %d", len(decoded.JSON))+ if decoded.JSON == nil || len(decoded.JSON) != 0 {+ t.Fatalf("empty JSON payload round-trip mismatch: %#v", decoded.JSON)}}diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex dc2f81308c..4823c97bb8 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -38,6 +38,7 @@ import ("ragflow/internal/entity"modelModule "ragflow/internal/entity/models""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser""ragflow/internal/utility""gorm.io/gorm"@@ -162,10 +163,10 @@ func maybeDispatchVisionEnhancement(ctx context.Context,db *gorm.DB,fileType utility.FileType,- dispatched parserDispatchResult,+ dispatched parser.ParseResult,inputs map[string]any,setups map[string]schema.ParserSetup,-) (parserDispatchResult, bool, error) {+) (parser.ParseResult, bool, error) {// 0. FileType allowlist guard.if !isVisionEnhancementAllowed(fileType) {return dispatched, false, nildiff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex 84b107f0cd..8d18b536ba 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -31,6 +31,7 @@ import ("ragflow/internal/entity"modelModule "ragflow/internal/entity/models""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser""ragflow/internal/utility""gorm.io/gorm"@@ -124,9 +125,8 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {return "describe the figure in " + language, nil})- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",- DocType: string(tc.fileType),JSON: []map[string]any{{"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},@@ -186,9 +186,8 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {nil,)- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "markdown",- DocType: "docx",Markdown: "",File: map[string]any{"figures": []map[string]any{{"image": "abc", "marker": "x"}}},}@@ -214,9 +213,8 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {}func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",- DocType: "other",JSON: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},},@@ -240,9 +238,8 @@ func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {}func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",- DocType: "docx",JSON: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},},@@ -267,7 +264,7 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {// returned unchanged — enhancement must not touch items when dispatched.Err != nil.func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {parseErr := errors.New("parse failed")- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{Err: parseErr,OutputFormat: "json",JSON: []map[string]any{@@ -305,7 +302,7 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())cancel() // pre-cancel- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},@@ -343,7 +340,7 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {fakePrompt,)- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{// non-string image field — filtered by target collector@@ -390,7 +387,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {"doc_type_kwd": "image",}}- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: items,}@@ -427,7 +424,7 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {fakePrompt,)- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},@@ -465,7 +462,7 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {},)- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},@@ -503,7 +500,7 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {fakePrompt,)- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},@@ -553,7 +550,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T"doc_type_kwd": "image",}}- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: items,}@@ -765,7 +762,7 @@ func TestVisionEnhancement_PerCallModelPreferred(t *testing.T) {setups := map[string]schema.ParserSetup{"pdf": {"vlm": map[string]any{"llm_id": "custom-vlm@provider"}},}- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},@@ -808,7 +805,7 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {fakePrompt,)- dispatched := parserDispatchResult{+ dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{{"text": "keep", "image": "!!!not-base64!!!", "doc_type_kwd": "image"},diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.goindex cacd43c796..7f50d792e4 100644--- a/internal/parser/parser/charset_test.go+++ b/internal/parser/parser/charset_test.go@@ -162,12 +162,16 @@ func TestCSVParser_GBK(t *testing.T) {if res.Err != nil {t.Fatalf("CSVParser.ParseWithResult failed: %v", res.Err)}- if strings.ContainsRune(res.HTML, '\ufffd') {- t.Errorf("CSV HTML output contains replacement runes: %s", res.HTML)+ if len(res.JSON) == 0 {+ t.Fatalf("CSVParser returned no JSON items")+ }+ tableHTML, _ := res.JSON[0]["text"].(string)+ if strings.ContainsRune(tableHTML, '\ufffd') {+ t.Errorf("CSV HTML output contains replacement runes: %s", tableHTML)}for _, want := range []string{"笔记本电脑", "电子", "特价销售,送鼠标", "无线耳机"} {- if !strings.Contains(res.HTML, want) {- t.Errorf("CSV HTML missing expected substring %q; got: %s", want, res.HTML)+ if !strings.Contains(tableHTML, want) {+ t.Errorf("CSV HTML missing expected substring %q; got: %s", want, tableHTML)}}if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" && enc != "gb2312" {diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 57d6208731..64a3d206d6 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -14,8 +14,7 @@// limitations under the License.//-// CSVParser renders CSV data as HTML tables, matching the spreadsheet-// family output_format == "html" convention from ParserParam.Defaults().+// CSVParser renders CSV data as HTML table text in structured JSON table items.//// Mirrors Python's deepdoc/parser/excel_parser.py:RAGFlowExcelParser.html():// - CSV data is rendered as an HTML <table> with <caption> "Data".@@ -40,7 +39,7 @@ import (const csvDefaultChunkRows = defaultTableChunkRowsconst csvSheetName = "Data"-// CSVParser reads RFC-4180 CSV data and emits HTML <table> payloads.+// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.type CSVParser struct {ParseMethod stringOutputFormat string@@ -128,14 +127,17 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [decoded, encName := DecodeToUTF8(data, "text/csv")text := string(decoded)if strings.TrimSpace(text) == "" {+ emptyHTML := "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>"return ParseResult{- OutputFormat: "html",+ OutputFormat: spreadsheetOutputFormat,File: map[string]any{"name": filename,"size": len(data),"encoding": encName,+ "format": "csv",+ "sheets": 1,},- HTML: "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>",+ JSON: []map[string]any{NewTableJSONItem(emptyHTML, csvSheetName, [][]float64{{1, 1, 1, 1, 1}})},}}@@ -157,13 +159,26 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [chunkRows = csvDefaultChunkRows}+ chunks := recordsToHTMLTableChunkList(records, chunkRows, csvSheetName, 1)+ items := make([]map[string]any, 0, len(chunks))+ for _, ch := range chunks {+ items = append(items, NewTableJSONItem(ch.HTML, csvSheetName, [][]float64{{+ 1,+ float64(ch.RowStart),+ float64(ch.RowEnd),+ float64(ch.ColStart),+ float64(ch.ColEnd),+ }}))+ }return ParseResult{- OutputFormat: "html",+ OutputFormat: spreadsheetOutputFormat,File: map[string]any{"name": filename,"size": len(data),"encoding": encName,+ "format": "csv",+ "sheets": 1,},- HTML: recordsToHTMLTableChunks(records, chunkRows, csvSheetName),+ JSON: items,}}diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.gonew file mode 100644index 0000000000..a8504ba47b--- /dev/null+++ b/internal/parser/parser/csv_parser_test.go@@ -0,0 +1,124 @@+package parser++import (+ "context"+ "strings"+ "testing"+)++func TestCSVParser_BasicHTMLAndJSON(t *testing.T) {+ csvData := []byte("Name,Age,City\nAlice,30,New York\nBob,25,San Francisco\n")+ p := NewCSVParser()++ res := p.ParseWithResult(context.Background(), "test.csv", csvData)+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }++ // Default output format is "json"+ if res.OutputFormat != "json" {+ t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, "json")+ }++ if len(res.JSON) != 1 {+ t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))+ }++ item := res.JSON[0]+ if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {+ t.Fatal("item text should contain rendered <table>")+ }+ if item["doc_type_kwd"] != "table" {+ t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])+ }+ if item["ck_type"] != "table" {+ t.Errorf("item ck_type = %v, want 'table'", item["ck_type"])+ }+ if item["sheet"] != "Data" {+ t.Errorf("item sheet = %v, want 'Data'", item["sheet"])+ }+ positions, ok := item["positions"].([][]float64)+ if !ok || len(positions) == 0 {+ t.Fatalf("item positions invalid: %v", item["positions"])+ }+ pos := positions[0]+ // [sheet, rowStart, rowEnd, colStart, colEnd]+ if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 3 || pos[3] != 1 || pos[4] != 3 {+ t.Errorf("unexpected positions: %v", pos)+ }+}++func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {+ csvData := []byte("col1,col2\nval1,val2\n")+ p := NewCSVParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ })++ res := p.ParseWithResult(context.Background(), "test.csv", csvData)+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }++ if res.OutputFormat != "json" {+ t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))+ }+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {+ t.Fatal("item text should contain rendered <table>")+ }+}++func TestCSVParser_EmptyContent(t *testing.T) {+ p := NewCSVParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ })++ res := p.ParseWithResult(context.Background(), "empty.csv", []byte(" \n\n "))+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }++ if res.OutputFormat != "json" {+ t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))+ }+ item := res.JSON[0]+ if item["doc_type_kwd"] != "table" {+ t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])+ }+}++func TestCSVParser_Chunking(t *testing.T) {+ // Header + 4 data rows, chunk size = 2+ csvData := []byte("Header\nr1\nr2\nr3\nr4\n")+ p := NewCSVParser()+ p.ChunkRows = 2++ res := p.ParseWithResult(context.Background(), "chunked.csv", csvData)+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }++ // 4 data rows with chunk_rows=2 => 2 chunks+ if len(res.JSON) != 2 {+ t.Fatalf("len(res.JSON) = %d, want 2", len(res.JSON))+ }++ pos1 := res.JSON[0]["positions"].([][]float64)[0]+ pos2 := res.JSON[1]["positions"].([][]float64)[0]++ // chunk 1: rows 2..3+ if pos1[1] != 2 || pos1[2] != 3 {+ t.Errorf("chunk 1 row range: [%v, %v], want [2, 3]", pos1[1], pos1[2])+ }+ // chunk 2: rows 4..5+ if pos2[1] != 4 || pos2[2] != 5 {+ t.Errorf("chunk 2 row range: [%v, %v], want [4, 5]", pos2[1], pos2[2])+ }+}diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex d4df6fa880..8bc8c4cdaa 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -26,27 +26,31 @@ import (officeOxide "github.com/yfedoseev/office_oxide/go")-type DOCParser struct{}+type DOCParser struct {+ OutputFormat string+}func NewDOCParser() *DOCParser {- return &DOCParser{}+ return &DOCParser{+ OutputFormat: "json",+ }}func (p *DOCParser) String() string {return "DOCParser"}-// ParseWithResult extracts text for the DOC family. The Go side uses-// office_oxide, which supports legacy .doc. We prefer the structured IR-// (flattened to plain text) and fall back to ToMarkdown, then to PlainText.-//-// office_oxide recovers table structure from legacy .doc through the IR-// (office_oxide/go v0.1.9, #116); list structure depends on whether the .doc-// reader surfaces list elements. Paragraph and heading (heuristically-// detected) lines are always present. When the IR carries table/list-// structure it is preferred so that content unique to the IR view is never-// shadowed by a longer but flatter view. OutputFormat stays "text" to keep-// the downstream contract unchanged.+func (p *DOCParser) ConfigureFromSetup(setup map[string]any) {+ if p == nil || setup == nil {+ return+ }+ if v, ok := setup["output_format"].(string); ok && v != "" {+ p.OutputFormat = v+ }+}++// ParseWithResult extracts text for the DOC family and converts it to the+// requested output format (defaulting to "json" to match Python's default).func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {doc, err := officeOxide.OpenFromBytes(data, "doc")if err != nil {@@ -54,16 +58,38 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [}defer doc.Close()- text, err := extractDocText(doc)+ text, mdText, err := extractDocText(doc)if err != nil {return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}}- return ParseResult{- OutputFormat: "text",+ outFmt := p.OutputFormat+ if outFmt == "" {+ outFmt = "json"+ }++ markdownPayload := text+ if strings.TrimSpace(mdText) != "" {+ markdownPayload = mdText+ }++ res := ParseResult{+ OutputFormat: outFmt,File: map[string]any{"name": filename, "format": "doc"},Text: text,+ Markdown: markdownPayload,}++ if strings.EqualFold(outFmt, "markdown") {+ return res+ }+ if strings.EqualFold(outFmt, "text") {+ return res+ }++ res.OutputFormat = "json"+ res.JSON = []map[string]any{NewTextJSONItem(text)}+ return res}// extractDocText returns the best-effort plain text for a legacy .doc@@ -81,7 +107,7 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [// is chosen so a sparser view never shadows a more complete one. A failure at// every stage degrades to the PlainText error, preserving the original "no// text at all" failure semantics.-func extractDocText(doc *officeOxide.Document) (string, error) {+func extractDocText(doc *officeOxide.Document) (string, string, error) {var irJSON, irText, mdText, plainText stringif j, err := doc.ToIRJSON(); err == nil {irJSON = j@@ -95,9 +121,9 @@ func extractDocText(doc *officeOxide.Document) (string, error) {} else if strings.TrimSpace(irText) == "" && strings.TrimSpace(mdText) == "" {// Every view failed (or produced nothing): keep the original// "no text at all" failure semantics.- return "", err+ return "", "", err}- return selectDocTextView(irJSON, irText, mdText, plainText), nil+ return selectDocTextView(irJSON, irText, mdText, plainText), mdText, nil}// selectDocTextView chooses the best plain-text rendering from office_oxide'sdiff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.goindex 65c6056c01..8c7d1bafed 100644--- a/internal/parser/parser/doc_parser_synth_test.go+++ b/internal/parser/parser/doc_parser_synth_test.go@@ -146,8 +146,11 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {if pr.Err != nil {t.Fatalf("ParseWithResult error: %v", pr.Err)}- if pr.OutputFormat != "text" {- t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "text")+ if pr.OutputFormat != "json" {+ t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "json")+ }+ if len(pr.JSON) != 1 {+ t.Fatalf("ParseWithResult returned %d JSON items, want 1", len(pr.JSON))}if pr.Text == "" {t.Fatal("ParseWithResult returned empty text — synthetic .doc was not parsed")@@ -157,4 +160,15 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)}}+ if got, _ := pr.JSON[0]["text"].(string); got != pr.Text {+ t.Errorf("JSON text = %q, want complete parser text %q", got, pr.Text)+ }++ // Also verify that explicit output_format="text" in setup is respected+ pText := NewDOCParser()+ pText.ConfigureFromSetup(map[string]any{"output_format": "text"})+ prText := pText.ParseWithResult(context.Background(), "synthetic.doc", data)+ if prText.OutputFormat != "text" {+ t.Errorf("configured text OutputFormat = %q, want text", prText.OutputFormat)+ }}diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex 99829273ef..825b6be234 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -47,7 +47,9 @@ type EmailParser struct {}func NewEmailParser() *EmailParser {- return &EmailParser{}+ return &EmailParser{+ outputFormat: "json",+ }}func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {@@ -110,12 +112,16 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by}content = msg} else {- content = parseEML(bytes.NewReader(data), p.fields)+ var err error+ content, err = parseEMLWithError(bytes.NewReader(data), p.fields)+ if err != nil {+ return ParseResult{Err: fmt.Errorf("email: .eml: %w", err)}+ }}outputFormat := p.outputFormatif outputFormat == "" {- outputFormat = "text"+ outputFormat = "json"}// Re-chunk attachments so their content becomes retrievable within the@@ -125,27 +131,13 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by// never breaks the whole email.extraItems, attachmentText := p.rechunkEmailAttachments(ctx, content, depth)- // attachments has been consumed by rechunkEmailAttachments (which- // re-parses each attachment by extension to make its content- // retrievable). It is otherwise dead weight: jsonItemsToPages copies- // every key into a schema.Page, but buildPagesFromBytes keeps only- // text+doc_type_kwd, so carrying the full attachment payloads through to- // the chunker would only bloat the intermediate pages before being- // discarded. Drop it from the result content.+ // Attachments have been consumed by rechunkEmailAttachments, which+ // re-parses each one by extension into a searchable item. Keeping their+ // raw payloads in the parent item would only bloat the parser result.delete(content, "attachments")- if outputFormat == "json" {- content["doc_type_kwd"] = "text"- items := []map[string]any{content}- items = append(items, extraItems...)- return ParseResult{- OutputFormat: "json",- File: map[string]any{"name": filename},- JSON: items,- }- }-- // Text output: flatten fields into a single string.+ // Text representation: flatten fields into a single string for explicit+ // text output and callers that need a display representation.var sb strings.Builderfor k, v := range content {// The metadata map (every non-basic header: Received chains,@@ -193,11 +185,46 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []bysb.WriteString(attachmentText)sb.WriteString("\n")}+ text := sb.String()++ if outputFormat == "text" {+ return ParseResult{+ OutputFormat: "text",+ File: map[string]any{"name": filename},+ Text: text,+ }+ }++ content["doc_type_kwd"] = "text"+ items := []map[string]any{content}+ if headerText := searchableEmailHeaders(content); headerText != "" {+ items = append(items, NewTextJSONItem(headerText))+ }+ items = append(items, extraItems...)return ParseResult{- OutputFormat: "text",+ OutputFormat: "json",File: map[string]any{"name": filename},- Text: sb.String(),+ JSON: items,+ Text: text,+ }+}++// searchableEmailHeaders returns the user-facing headers as a separate JSON+// text item. The main email item keeps its structured fields and body; the+// chunker indexes only item text, so headers need their own text item.+func searchableEmailHeaders(content map[string]any) string {+ var sb strings.Builder+ for _, key := range []string{"from", "to", "cc", "bcc", "date", "subject"} {+ value, ok := content[key].(string)+ if !ok || value == "" {+ continue+ }+ sb.WriteString(key)+ sb.WriteString(":")+ sb.WriteString(value)+ sb.WriteString("\n")}+ return sb.String()}// -- field set helpers --@@ -236,14 +263,13 @@ func decodeHeaderWord(val string) string {// -- .eml parsing (RFC 5322 with multipart support) ---func parseEML(r io.Reader, fields []string) map[string]any {+func parseEMLWithError(r io.Reader, fields []string) (map[string]any, error) {target := targetFieldsSet(fields)content := map[string]any{}msg, err := mail.ReadMessage(r)if err != nil {- content["error"] = fmt.Sprintf("email: parse error: %v", err)- return content+ return nil, err}// Headers. net/mail does not decode RFC 2047 encoded-words, so decode@@ -299,7 +325,7 @@ func parseEML(r io.Reader, fields []string) map[string]any {}}- return content+ return content, nil}// readMailBody reads the body of an email message, handlingdiff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex 3aa93711aa..acc1d07e4c 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -20,6 +20,7 @@ import ("bytes""context""encoding/base64"+ "io""mime/multipart""net/textproto""os"@@ -28,6 +29,15 @@ import ("time")+func mustParseEML(t *testing.T, reader io.Reader, fields []string) map[string]any {+ t.Helper()+ content, err := parseEMLWithError(reader, fields)+ if err != nil {+ t.Fatalf("parseEMLWithError: %v", err)+ }+ return content+}+func TestEmailParser_EmlJSON(t *testing.T) {ctx := t.Context()raw := strings.Join([]string{@@ -55,8 +65,8 @@ func TestEmailParser_EmlJSON(t *testing.T) {if result.OutputFormat != "json" {t.Fatalf("expected output_format json, got %q", result.OutputFormat)}- if len(result.JSON) != 1 {- t.Fatalf("expected 1 JSON item, got %d", len(result.JSON))+ if len(result.JSON) != 2 {+ t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))}item := result.JSON[0]@@ -82,6 +92,31 @@ func TestEmailParser_EmlJSON(t *testing.T) {if v, ok := item["doc_type_kwd"].(string); !ok || v != "text" {t.Errorf("doc_type_kwd: got %q", v)}++ header := result.JSON[1]+ headerText, ok := header["text"].(string)+ if !ok {+ t.Fatalf("header item text missing: %#v", header)+ }+ for _, want := range []string{"from:[email protected]", "to:[email protected]", "subject:Test Email"} {+ if !strings.Contains(headerText, want) {+ t.Errorf("header item missing %q: %q", want, headerText)+ }+ }+ if strings.Contains(headerText, "This is the body of the test email.") {+ t.Errorf("header item must not duplicate body text: %q", headerText)+ }+ if v, ok := header["doc_type_kwd"].(string); !ok || v != "text" {+ t.Errorf("header doc_type_kwd: got %q", v)+ }+}++func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {+ p := NewEmailParser()+ result := p.ParseWithResult(t.Context(), "broken.eml", []byte("not a valid RFC 5322 header"))+ if result.Err == nil {+ t.Fatalf("expected malformed EML error, got result %#v", result)+ }}func TestEmailParser_EmlText(t *testing.T) {@@ -233,8 +268,8 @@ func TestEmailParser_MsgSupported(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- if len(result.JSON) != 1 {- t.Fatalf("expected 1 JSON item, got %d", len(result.JSON))+ if len(result.JSON) != 2 {+ t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))}item := result.JSON[0]@@ -349,7 +384,7 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {t.Fatalf("read fixture: %v", err)}- p := NewEmailParser() // text output; default fields+ p := NewEmailParser() // json output; default fieldsp.ConfigureFromSetup(map[string]any{"fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},})@@ -357,8 +392,11 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- if result.OutputFormat != "text" {- t.Fatalf("output format = %q, want text", result.OutputFormat)+ if result.OutputFormat != "json" {+ t.Fatalf("output format = %q, want json", result.OutputFormat)+ }+ if len(result.JSON) == 0 {+ t.Fatalf("expected non-empty JSON items")}if !strings.Contains(result.Text, "subject:asdf") {@@ -421,7 +459,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {t.Error("attachments must be dropped from the final ParseResult")}// ...and verify the .eml branch still decodes the base64 attachment.- eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})+ eml := mustParseEML(t, bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})atts, ok := eml["attachments"].([]map[string]any)if !ok {t.Fatalf("attachments missing or wrong type: %T", eml["attachments"])@@ -507,7 +545,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {t.Error("attachments must be dropped from the final ParseResult")}// ...and verify the .eml branch still decodes the base64 attachment.- eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})+ eml := mustParseEML(t, bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})atts, ok := eml["attachments"].([]map[string]any)if !ok || len(atts) != 1 {t.Fatalf("expected 1 attachment, got %d", len(atts))@@ -933,7 +971,7 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {t.Error("attachments must be dropped from the final ParseResult")}// ...and verify the .eml branch still extracts them even without body.- eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "attachments"})+ eml := mustParseEML(t, bytes.NewReader([]byte(raw)), []string{"from", "attachments"})atts, ok := eml["attachments"].([]map[string]any)if !ok {t.Fatalf("attachments missing or wrong type: %T", eml["attachments"])@@ -1254,7 +1292,7 @@ func TestParseEML_GB2312EncodedSubject(t *testing.T) {"body",}, "\r\n")- content := parseEML(strings.NewReader(raw), []string{"subject"})+ content := mustParseEML(t, strings.NewReader(raw), []string{"subject"})if content["subject"] != "中文" {t.Errorf("subject = %q, want 中文", content["subject"])}diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 680db0a5c1..bb3dce85be 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -51,13 +51,18 @@ type MarkdownParser struct {OutputFormat stringVLM map[string]anyFlattenMediaToText bool+ // FetchRemoteImages controls whether HTTP(S) Markdown images are+ // downloaded. NewMarkdownParser enables it for direct Markdown parsing;+ // callers normalizing untrusted backend responses may disable it.+ FetchRemoteImages bool}func NewMarkdownParser(libType string) (*MarkdownParser, error) {switch libType {case GoMarkdown:return &MarkdownParser{- libType: GoMarkdown,+ libType: GoMarkdown,+ FetchRemoteImages: true,}, nildefault:return nil, fmt.Errorf("unsupported Markdown library type: %s", libType)@@ -103,7 +108,7 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, ddoc := markdownNew().Parse([]byte(rendered))var items []map[string]any- walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText)+ walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText, p.FetchRemoteImages)if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}@@ -275,7 +280,7 @@ func markdownTableCells(line string) []string {// doc_type_kwd:"table" to keep the table whole and attach table context// to neighbouring chunks (chunker/token.go). Non-table HTML blocks// (<div>, <style>, …) are emitted as ordinary text with no ck_type.-func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten bool) {+func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool) {for _, child := range doc.GetChildren() {var ckType stringvar docTypeKwd string@@ -342,10 +347,13 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten b// flatten is true, keep doc_type_kwd="text" (Python// parser.py:1034: flatten_media_to_text overrides image).if imgURL, ok := findBlockImage(child); ok {- if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {- item["image"] = imgData- if !flatten {- item["doc_type_kwd"] = "image"+ isRemote := strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")+ if fetchRemoteImages || !isRemote {+ if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {+ item["image"] = imgData+ if !flatten {+ item["doc_type_kwd"] = "image"+ }}}}diff --git a/internal/parser/parser/office_parsers_no_cgo.go b/internal/parser/parser/office_parsers_no_cgo.goindex 8f1443fde2..b00d0c6f60 100644--- a/internal/parser/parser/office_parsers_no_cgo.go+++ b/internal/parser/parser/office_parsers_no_cgo.go@@ -62,11 +62,6 @@ func NewDOCXParser() *DOCXParser {return &DOCXParser{}}-func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {- // No-op in the no-CGO stub: the real implementation in- // docx_parser.go reads output_format from setup.-}-func (p *DOCXParser) String() string {return "DOCXParser(no-cgo)"}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex dc50ba0c54..fd4b81f356 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -161,25 +161,7 @@ type htmlTableChunk struct {ColEnd int}-// recordsToHTMLTableChunks renders records as one or more self-contained HTML-// <table> chunks. The first row is always the header (<th>). Data rows are-// split into chunks of chunkRows, each chunk being a complete <table> with-// <caption> and a repeated header row. Chunks are joined with newlines.-//-// The tag schema is <table><caption>{caption}</caption><tr><th>…</th></tr>-// <tr><td>…</td></tr>…</table>. Rows are intentionally NOT wrapped in-// <thead>/<tbody>, so every <table> is one atomic chunk that downstream-// chunkers can consume independently.-func recordsToHTMLTableChunks(records [][]string, chunkRows int, caption string) string {- chunks := recordsToHTMLTableChunkList(records, chunkRows, caption, 1)- parts := make([]string, len(chunks))- for i, ch := range chunks {- parts[i] = ch.HTML- }- return strings.Join(parts, "")-}--// recordsToHTMLTableChunkList is the structured form of recordsToHTMLTableChunks.+// recordsToHTMLTableChunkList renders records as structured htmlTableChunk items.// headerRowAbs is the 1-based workbook row number of records[0] (normally 1).func recordsToHTMLTableChunkList(records [][]string, chunkRows int, caption string, headerRowAbs int) []htmlTableChunk {if headerRowAbs <= 0 {@@ -400,7 +382,7 @@ func mergeExtentCol(ranges []mergeRange) int {}// padRowToWidth grows a single row to at least maxCol, padding with empty-// strings. Only the header row is padded (see renderSheetTables): merged-master+// strings. Only the header row is padded: merged-master// text is inherited into the header alone, so data rows must not be widened —// widening them would emit a sea of empty <td> cells for every far merge in the// sheet and is the memory blow-up flagged in review.@@ -634,22 +616,6 @@ func decodeChunkRows(setup map[string]any) int {return defaultTableChunkRows}-// renderSheetTables renders a single workbook sheet into one or more-// self-contained <table> chunks using the shared spreadsheet-HTML contract:-// detect the header row, inherit merged-master text into the header, and split-// data into chunkRows-sized atomic tables each repeating the header.-func renderSheetTables(f *excelize.File, sheet string, chunkRows int) (string, []string, error) {- chunks, warnings, err := renderSheetTableChunks(f, sheet, chunkRows)- if err != nil {- return "", warnings, err- }- parts := make([]string, len(chunks))- for i, ch := range chunks {- parts[i] = ch.HTML- }- return strings.Join(parts, ""), warnings, nil-}-func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]htmlTableChunk, []string, error) {rows, err := f.GetRows(sheet)if err != nil {diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex 956afb5e62..1407301720 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -103,12 +103,16 @@ func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.}}-// TestRecordsToHTMLTableChunks_Alignment asserts the chunked output uses the+// TestRecordsToHTMLTableChunkList_Alignment asserts the chunked output uses the// shared schema: <caption>, first row as <th>, data as <td>, repeated header// per 256-row chunk, and NO <thead>/<tbody> wrapper.-func TestRecordsToHTMLTableChunks_Alignment(t *testing.T) {+func TestRecordsToHTMLTableChunkList_Alignment(t *testing.T) {records := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}- out := recordsToHTMLTableChunks(records, 256, "Sheet1")+ chunks := recordsToHTMLTableChunkList(records, 256, "Sheet1", 1)+ if len(chunks) != 1 {+ t.Fatalf("want 1 chunk, got %d", len(chunks))+ }+ out := chunks[0].HTMLif !strings.Contains(out, `<caption>Sheet1</caption>`) {t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", out)@@ -122,28 +126,26 @@ func TestRecordsToHTMLTableChunks_Alignment(t *testing.T) {if strings.Contains(out, "<thead>") || strings.Contains(out, "<tbody>") {t.Fatalf("must not emit <thead>/<tbody> to stay byte-compatible with Python/CSV, got:\n%s", out)}- // Exactly one <table> for <=256 data rows.- if n := strings.Count(out, "<table>"); n != 1 {- t.Fatalf("want 1 <table>, got %d:\n%s", n, out)- }}-// TestRecordsToHTMLTableChunks_Chunking asserts 256-row chunking with a repeated+// TestRecordsToHTMLTableChunkList_Chunking asserts 256-row chunking with a repeated// header (ceil(n_data / chunk_rows) chunks).-func TestRecordsToHTMLTableChunks_Chunking(t *testing.T) {+func TestRecordsToHTMLTableChunkList_Chunking(t *testing.T) {const dataRows = 300records := make([][]string, 0, dataRows+1)records = append(records, []string{"C1", "C2"})for i := 0; i < dataRows; i++ {records = append(records, []string{"x", "y"})}- out := recordsToHTMLTableChunks(records, 256, "S")+ chunks := recordsToHTMLTableChunkList(records, 256, "S", 1)// 300 data rows → ceil(300/256) = 2 chunks, each repeating the header.- if n := strings.Count(out, "<table>"); n != 2 {- t.Fatalf("want 2 <table> chunks, got %d", n)+ if len(chunks) != 2 {+ t.Fatalf("want 2 chunks, got %d", len(chunks))}- if n := strings.Count(out, "<tr><th>C1</th><th>C2</th></tr>"); n != 2 {- t.Fatalf("want header repeated in both chunks, got %d", n)+ for i, ch := range chunks {+ if !strings.Contains(ch.HTML, "<tr><th>C1</th><th>C2</th></tr>") {+ t.Fatalf("want header repeated in chunk %d, got %s", i, ch.HTML)+ }}}diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex 339e976d52..4904151a1c 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -19,9 +19,9 @@// parsers surface enough data to reconstruct a Python-compatible// stage-boundary payload://-// output_format ∈ {"json","markdown","text","html"}-// file (enriched metadata)-// exactly one payload family populated (matching output_format)+// output_format+// file (enriched metadata)+// backend payload to be normalized by the owning component// err//// Go parser callers now consume only the structured ParseResult@@ -33,28 +33,19 @@ package parserimport "context"-// ParseResult is the structured return value of a successful parse.-// Exactly one of the payload fields (JSON / Markdown / Text / HTML)-// is populated on success, matching the Python contract — see-// port-rag-flow-pipeline-to-go.md §4.2:-//-// - OutputFormat = "json" → JSON populated-// - OutputFormat = "markdown" → Markdown populated-// - OutputFormat = "text" → Text populated-// - OutputFormat = "html" → HTML populated-//-// On failure (Err != nil), all payload fields are zero values and-// OutputFormat is empty.+// ParseResult is the structured return value of a parse operation. JSON holds+// structured backend output when available; the rendered fields carry+// non-JSON backend responses to the Parser component's normalization boundary.+// On failure, Err is non-nil and OutputFormat is empty.type ParseResult struct {- // OutputFormat is the wire-compatible format the parser- // chose. Empty when Err is non-nil.+ // OutputFormat identifies the backend payload representation. The Parser+ // component normalizes successful results to JSON. Empty when Err is non-nil.OutputFormat string- // File is the enriched file metadata the parser emits. In- // Python this is the dict form of the original `file`- // descriptor, augmented with format-specific keys (e.g.- // `outline` on the PDF path, `page_count` for paginated- // formats). Nil when the parser did not enrich.+ // File is metadata produced by the parser backend (for example+ // `outline` on the PDF path or `page_count` for paginated formats).+ // Nil when the backend does not produce metadata. It is not an augmented+ // copy of the upstream file descriptor.//// For the office family (docx/doc, pptx/ppt), File["format"]// reflects the real container format detected via magic-byte@@ -66,25 +57,22 @@ type ParseResult struct {// echoed the requested extension.File map[string]any- // JSON is the structured payload when OutputFormat == "json".+ // JSON is the structured payload when available.// Shape depends on the parser family: PDF emits// `[]map[string]any` with `text` + `doc_type_kwd` keys (and// optional `image` / `layout` / `positions` fields);// Markdown / HTML / text emit normalized// `{text, doc_type_kwd}` items; image emits OCR/VLM result- // items. Exactly one payload family is populated on success.+ // items.JSON []map[string]any- // Markdown is the string payload when OutputFormat ==- // "markdown". Empty otherwise.+ // Markdown is a backend response awaiting normalization.Markdown string- // Text is the string payload when OutputFormat == "text".- // Empty otherwise.+ // Text is a backend response awaiting normalization.Text string- // HTML is the string payload when OutputFormat == "html".- // Empty otherwise.+ // HTML is a backend response awaiting normalization.HTML string// Err is the failure reason. On non-nil Err, all payload@@ -100,3 +88,38 @@ type ParseResult struct {type ParseResultProducer interface {ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult}++// Canonical document type identifiers used in structured JSON items.+// These align directly with Python's doc_type_kwd contract.+const (+ DocTypeKey = "doc_type_kwd"+ DocTypeText = "text"+ DocTypeTable = "table"+ DocTypeImage = "image"++ spreadsheetOutputFormat = "json"+)++// NewTextJSONItem constructs a canonical text JSON item for parser output.+func NewTextJSONItem(text string) map[string]any {+ return map[string]any{+ "text": text,+ DocTypeKey: DocTypeText,+ }+}++// NewTableJSONItem constructs a canonical table JSON item for parser output.+func NewTableJSONItem(html string, sheet string, positions [][]float64) map[string]any {+ item := map[string]any{+ "text": html,+ DocTypeKey: DocTypeTable,+ "ck_type": DocTypeTable,+ }+ if sheet != "" {+ item["sheet"] = sheet+ }+ if len(positions) > 0 {+ item["positions"] = positions+ }+ return item+}diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex ffb3707755..031e378dad 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -21,6 +21,17 @@ import ("testing")+func TestNewTableJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {+ item := NewTableJSONItem("<table><tr><td>value</td></tr></table>", "Sheet1", [][]float64{{1, 2, 2, 1, 1}})++ if got, want := item[DocTypeKey], DocTypeTable; got != want {+ t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)+ }+ if got, want := item["ck_type"], DocTypeTable; got != want {+ t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)+ }+}+// TestParseResult_Contract pins the wire-shape guarantees// port-rag-flow-pipeline-to-go.md §6.5 requires://diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.goindex 47406db6f6..558ea70e6c 100644--- a/internal/parser/parser/picture_parser.go+++ b/internal/parser/parser/picture_parser.go@@ -129,9 +129,8 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da// OutputFormat, VLMModelID, ImageContextSize, and LayoutRecognize// are consumed by maybeDispatchImage at the component layer.- // Image family only allows json (schema/parser.go, Python parser.py),- // so default empty to json. Strict: explicit text is kept as-is- // and will be rejected by the dispatch whitelist.+ // Parser output is normalized to JSON at the component boundary, so an+ // absent backend format defaults to JSON here as well.outFmt := p.OutputFormatif outFmt == "" {outFmt = "json"diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex e542d8c3ec..3dc510333b 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -17,12 +17,8 @@package parserimport (- "bytes""context""fmt"- "strings"-- "github.com/xuri/excelize/v2")type XLSParser struct {@@ -95,33 +91,20 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [}}- f, err := excelize.OpenReader(bytes.NewReader(data))- if err != nil {- return ParseResult{Err: fmt.Errorf("xls open: %w", err)}- }- defer f.Close()-- sheets := f.GetSheetList()chunkRows := p.ChunkRowsif chunkRows <= 0 {chunkRows = defaultTableChunkRows}- var html strings.Builder- var warnings []string- for _, sheet := range sheets {- rendered, sheetWarnings, err := renderSheetTables(f, sheet, chunkRows)- if err != nil {- return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}- }- html.WriteString(rendered)- warnings = append(warnings, sheetWarnings...)+ items, warnings, sheetsCount, err := parseXLSXBytes(data, chunkRows)+ if err != nil {+ return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}}return ParseResult{- OutputFormat: "html",- File: map[string]any{"name": filename, "format": "xls"},- HTML: html.String(),+ OutputFormat: spreadsheetOutputFormat,+ File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},+ JSON: items,Warnings: warnings,}}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.gonew file mode 100644index 0000000000..6aef5e68cd--- /dev/null+++ b/internal/parser/parser/xls_parser_test.go@@ -0,0 +1,112 @@+package parser++import (+ "bytes"+ "context"+ "strings"+ "testing"++ "github.com/xuri/excelize/v2"+)++func createTestExcelBytes(t *testing.T) []byte {+ t.Helper()+ f := excelize.NewFile()+ defer f.Close()++ sheet := "Sheet1"+ _ = f.SetCellValue(sheet, "A1", "Header1")+ _ = f.SetCellValue(sheet, "B1", "Header2")+ _ = f.SetCellValue(sheet, "A2", "Val1")+ _ = f.SetCellValue(sheet, "B2", "Val2")++ var buf bytes.Buffer+ if err := f.Write(&buf); err != nil {+ t.Fatalf("Write excel file failed: %v", err)+ }+ return buf.Bytes()+}++func TestXLSParser_HTMLAndJSONOutput(t *testing.T) {+ data := createTestExcelBytes(t)+ p, err := NewXLSParser("excelize")+ if err != nil {+ t.Fatalf("NewXLSParser failed: %v", err)+ }++ res := p.ParseWithResult(context.Background(), "test.xls", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }++ // Default output format for XLSParser is "json"+ if res.OutputFormat != "json" {+ t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)+ }+ if len(res.JSON) == 0 {+ t.Fatalf("res.JSON should have structured table items")+ }+ item := res.JSON[0]+ if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {+ t.Errorf("table item text should contain rendered <table>, got %v", item["text"])+ }+ if item["doc_type_kwd"] != "table" {+ t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])+ }+ if item["ck_type"] != "table" {+ t.Errorf("ck_type = %v, want 'table'", item["ck_type"])+ }++ // Configure output format to "json"+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ })+ resJSON := p.ParseWithResult(context.Background(), "test.xls", data)+ if resJSON.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", resJSON.Err)+ }+ if resJSON.OutputFormat != "json" {+ t.Errorf("resJSON.OutputFormat = %q, want 'json'", resJSON.OutputFormat)+ }+ if len(resJSON.JSON) == 0 {+ t.Errorf("resJSON.JSON should not be empty")+ }+}++func TestXLSXParser_JSONOutput(t *testing.T) {+ data := createTestExcelBytes(t)+ p, err := NewXLSXParser("excelize")+ if err != nil {+ t.Fatalf("NewXLSXParser failed: %v", err)+ }++ // Default output format is "json"+ res := p.ParseWithResult(context.Background(), "test.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }+ if res.OutputFormat != "json" {+ t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)+ }+ if len(res.JSON) == 0 {+ t.Fatalf("res.JSON should have items")+ }+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {+ t.Errorf("item text should contain HTML table, got %v", res.JSON[0]["text"])+ }++ // Even if configured to legacy "html", format is unified to "json"+ p.ConfigureFromSetup(map[string]any{+ "output_format": "html",+ })+ resHTML := p.ParseWithResult(context.Background(), "test.xlsx", data)+ if resHTML.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", resHTML.Err)+ }+ if resHTML.OutputFormat != "json" {+ t.Errorf("resHTML.OutputFormat = %q, want 'json'", resHTML.OutputFormat)+ }+ if len(resHTML.JSON) == 0 {+ t.Errorf("resHTML.JSON should have items")+ }+}diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.goindex 96b5a8bc73..bf3fe936d7 100644--- a/internal/parser/parser/xls_tcadp_test.go+++ b/internal/parser/parser/xls_tcadp_test.go@@ -3,6 +3,7 @@ package parserimport ("net/http""net/http/httptest"+ "strings""testing")@@ -216,11 +217,14 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if got, want := res.OutputFormat, "html"; got != want {+ if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if res.HTML == "" {- t.Fatal("HTML is empty; want rendered table")+ if len(res.JSON) == 0 {+ t.Fatal("JSON items is empty; want structured table items")+ }+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {+ t.Fatalf("JSON item text = %v; want rendered table", res.JSON[0]["text"])}}diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.goindex 2d69f8cf34..e91c5c97eb 100644--- a/internal/parser/parser/xlsx_parse_method_test.go+++ b/internal/parser/parser/xlsx_parse_method_test.go@@ -167,11 +167,14 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult(deepdoc): %v", res.Err)}- if got, want := res.OutputFormat, "html"; got != want {+ if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if !strings.Contains(res.HTML, "<table>") {- t.Fatalf("HTML = %q, want a rendered <table>", res.HTML)+ if len(res.JSON) == 0 {+ t.Fatal("JSON items is empty; want structured table items")+ }+ if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {+ t.Fatalf("JSON item text = %v, want a rendered <table>", res.JSON[0]["text"])}}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex e370623a91..eea92c925d 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -155,19 +155,13 @@ func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int}warnings = append(warnings, sheetWarnings...)for _, table := range tables {- items = append(items, map[string]any{- "text": table.HTML,- "doc_type_kwd": "table",- "ck_type": "table",- "sheet": sheet,- "positions": [][]float64{{- float64(sheetIdx + 1),- float64(table.RowStart),- float64(table.RowEnd),- float64(table.ColStart),- float64(table.ColEnd),- }},- })+ items = append(items, NewTableJSONItem(table.HTML, sheet, [][]float64{{+ float64(sheetIdx + 1),+ float64(table.RowStart),+ float64(table.RowEnd),+ float64(table.ColStart),+ float64(table.ColEnd),+ }}))}images, imageWarnings := extractXLSXImages(f, sheet)items = append(items, images...)@@ -178,7 +172,7 @@ func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, intfunc xlsxParseResult(filename string, items []map[string]any, warnings []string, sheets int) ParseResult {return ParseResult{- OutputFormat: "json",+ OutputFormat: spreadsheetOutputFormat,File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},JSON: items,Warnings: warnings,diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex e7cd72a19d..5e0d8b9c02 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -210,14 +210,28 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,if res.Err != nil {return "", res.Err}- switch res.OutputFormat {- case "text":- content = res.Text- case "markdown":- content = res.Markdown- case "html":- content = res.HTML- case "json":+ parsed, err := parseResultText(res)+ if err != nil {+ return "", err+ }+ content = parsed+ }+ return fmt.Sprintf("\n -----------------\nFile: %s\nContent as following: \n%s", filename, content), nil+}++// parseResultText converts a parser result into the readable text expected by+// sys.files. JSON results are flattened in item order, preferring each item's+// text field and serializing items without one as a final fallback.+func parseResultText(res parser.ParseResult) (string, error) {+ switch strings.ToLower(strings.TrimSpace(res.OutputFormat)) {+ case "text":+ return res.Text, nil+ case "markdown":+ return res.Markdown, nil+ case "html":+ return res.HTML, nil+ case "json":+ if len(res.JSON) > 0 {parts := make([]string, 0, len(res.JSON))for _, item := range res.JSON {if text, ok := item["text"].(string); ok {@@ -230,10 +244,20 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,}parts = append(parts, string(raw))}- content = strings.Join(parts, "\n")+ return strings.Join(parts, "\n"), nil}+ // Some legacy parsers mark the result as JSON while only populating a+ // rendered companion field. Preserve that content instead of returning+ // an empty sys.files value.+ for _, fallback := range []string{res.Markdown, res.HTML, res.Text} {+ if fallback != "" {+ return fallback, nil+ }+ }+ return "", nil+ default:+ return "", fmt.Errorf("unsupported parser output format %q", res.OutputFormat)}- return fmt.Sprintf("\n -----------------\nFile: %s\nContent as following: \n%s", filename, content), nil}// parseFileContent tries to parse a file's contents using the appropriate parser.@@ -251,16 +275,9 @@ func parseFileContent(ctx context.Context, filename string, data []byte) stringif res.Err != nil {return string(data)}- switch res.OutputFormat {- case "text":- return res.Text- case "markdown":- return res.Markdown- case "html":- return res.HTML- case "json":- return string(data)- default:+ content, err := parseResultText(res)+ if err != nil {return string(data)}+ return content}diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.goindex 9fc4ca7f40..795de53997 100644--- a/internal/service/file/file_content_test.go+++ b/internal/service/file/file_content_test.go@@ -1,6 +1,9 @@package fileimport (+ "strings"++ "ragflow/internal/parser/parser""ragflow/internal/utility""testing")@@ -38,16 +41,92 @@ func TestLooksLikeHTML(t *testing.T) {}}-func TestParseFileContent_HTMLOutputFormat(t *testing.T) {+func TestParseFileContent_JSONItems(t *testing.T) {ctx := t.Context()- // CSV parser produces OutputFormat "html".+ // CSV parser now produces JSON table items. sys.files still needs the+ // readable item text, not the original CSV bytes.result := parseFileContent(ctx, "data.csv", []byte("a,b,c\n1,2,3\n"))- if result == "" || result == string([]byte("a,b,c\n1,2,3\n")) {- t.Skip("CSV parser not available or returned raw text; integration-only test")+ if result == "" {+ t.Fatal("CSV parser returned empty content")+ }+ if strings.Contains(result, "a,b,c\n1,2,3\n") {+ t.Errorf("CSV content used original bytes instead of parsed item text: %q", result)+ }+ if !strings.Contains(result, "<table") || !strings.Contains(result, "1") {+ t.Errorf("CSV content lost readable table text: %q", result)+ }+}++func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T) {+ result, err := parseResultText(parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "first"},+ {"text": "second"},+ {"value": 3},+ },+ })+ if err != nil {+ t.Fatalf("parseResultText: %v", err)+ }+ want := "first\nsecond\n{\"value\":3}"+ if result != want {+ t.Fatalf("parseResultText = %q, want %q", result, want)+ }+}++func TestParseResultText_EmailJSONKeepsSearchableHeaders(t *testing.T) {+ raw := []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: Parser contract\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEmail body")+ p := parser.NewEmailParser()+ p.ConfigureFromSetup(map[string]any{"output_format": "json", "fields": []string{"from", "to", "subject", "body"}})+ parsed := p.ParseWithResult(t.Context(), "message.eml", raw)+ if parsed.Err != nil {+ t.Fatalf("ParseWithResult: %v", parsed.Err)+ }+ content, err := parseResultText(parsed)+ if err != nil {+ t.Fatalf("parseResultText: %v", err)+ }+ for _, want := range []string{"from:[email protected]", "to:[email protected]", "subject:Parser contract", "Email body"} {+ if !strings.Contains(content, want) {+ t.Errorf("sys.files text missing %q: %q", want, content)+ }+ }+}++func TestParseResultText_EmptyJSONUsesRenderedFallback(t *testing.T) {+ result, err := parseResultText(parser.ParseResult{+ OutputFormat: "json",+ Markdown: "legacy markdown",+ })+ if err != nil {+ t.Fatalf("parseResultText: %v", err)+ }+ if result != "legacy markdown" {+ t.Fatalf("parseResultText = %q, want rendered fallback", result)+ }+}++func TestParseResultText_EmptyJSONWithoutFallbackReturnsEmpty(t *testing.T) {+ result, err := parseResultText(parser.ParseResult{OutputFormat: "json"})+ if err != nil {+ t.Fatalf("parseResultText: %v", err)+ }+ if result != "" {+ t.Fatalf("parseResultText = %q, want empty text", result)+ }+}++func TestParseAgentUploadContent_JSONUsesParsedItemText(t *testing.T) {+ content, err := parseAgentUploadContent(t.Context(), "data.csv", []byte("a,b\n1,2\n"), "")+ if err != nil {+ t.Fatalf("parseAgentUploadContent: %v", err)+ }+ if strings.Contains(content, "a,b\n1,2\n") {+ t.Fatalf("upload content used original bytes instead of parsed item text: %q", content)}- // CSV parser emits an HTML table; must not contain raw CSV comma-separated rows.- if result == "a,b,c\n1,2,3\n" {- t.Errorf("CSV should produce HTML output, got raw CSV: %q", result)+ if !strings.Contains(content, "<table") || !strings.Contains(content, "1") {+ t.Fatalf("upload content lost readable table text: %q", content)}}