RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): unify PDF JSON post-processing (#21228)
jay7772132f0114c9598
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.goindex 912fccc09a..a88dba6da3 100644--- a/internal/ingestion/component/chunker/manual.go+++ b/internal/ingestion/component/chunker/manual.go@@ -26,6 +26,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/globals"+ "ragflow/internal/parser/parser""gorm.io/gorm")@@ -169,6 +170,10 @@ func sortRecordsByPosition(records []lineRecord) {// structured `_pdf_positions` key first and falls back to the legacy// `positions` key. Callers (pdfPosRowLess) index offsets 0/1/3, so we enforce// len(row) >= 5 here.+//+// The matrix is decoded through parser.PDFPositionRowsFromJSON, so the matrix+// form and the legacy flat 5-tuple are both understood and the chunker shares+// one position decoder with the parser boundary and the crop path.func firstPositionRow(r lineRecord) ([]float64, bool) {var raw json.RawMessageswitch {@@ -179,15 +184,11 @@ func firstPositionRow(r lineRecord) ([]float64, bool) {default:return nil, false}- var mat [][]float64- if err := json.Unmarshal(raw, &mat); err != nil || len(mat) == 0 {- return nil, false- }- row := mat[0]- if len(row) < 5 {+ rows := parser.PDFPositionRowsFromJSON(raw)+ if len(rows) == 0 {return nil, false}- return row, true+ return rows[0], true}// init registers ManualChunker under CategoryIngestion.diff --git a/internal/ingestion/component/chunker/pdf_postprocess_test.go b/internal/ingestion/component/chunker/pdf_postprocess_test.gonew file mode 100644index 0000000000..8b993c3116--- /dev/null+++ b/internal/ingestion/component/chunker/pdf_postprocess_test.go@@ -0,0 +1,36 @@+package chunker++import (+ "strings"+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/parser"+)++func TestPDFTableNotesDoNotHideCellsDuringChunking(t *testing.T) {+ result := parser.NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{+ parser.TableItem{Table: &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, Extra: map[string]any{"text": "Footnote"}},+ }})+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatal(err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{"name": "sample.pdf", "file_type": "pdf", "output_format": "json", "json": result.JSON})+ if err != nil {+ t.Fatal(err)+ }+ var text string+ for _, chunk := range out["chunks"].([]map[string]any) {+ value, _ := chunk["text"].(string)+ text += value+ }+ for _, want := range []string{"Cell value", "Footnote"} {+ if !strings.Contains(text, want) {+ t.Fatalf("missing %q in %q", want, text)+ }+ }+}diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.goindex 2e0b448177..1e4087d8b2 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go@@ -30,6 +30,7 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser""gorm.io/gorm")@@ -284,12 +285,33 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [func needsCrop(ck schema.ChunkDoc) bool {switch cropChunkType(ck) {case "image", "table", "text":- return len(ck.PDFPositions) > 0 || len(ck.Positions) > 0+ return hasCropRegion(ck)default:return false}}+// hasCropRegion reports whether the chunk carries a position with a+// positive-area box. A page-only position (page number, no geometry) carries+// page identity for the indexer but no region, so it must not trigger a crop.+//+// Every row of the matrix is inspected: mergePositionMatrix sorts page-only+// rows first, so a page-only first row must not hide a real region in a later+// row. The matrix is read exactly as the crop path reads it (the canonical+// _pdf_positions first, the legacy positions key as fallback).+func hasCropRegion(ck schema.ChunkDoc) bool {+ raw := ck.PDFPositions+ if len(raw) == 0 {+ raw = ck.Positions+ }+ for _, row := range parser.PDFPositionRowsFromJSON(raw) {+ if row[2] > row[1] && row[4] > row[3] {+ return true+ }+ }+ return false+}+// cropChunkType returns the content type that selects the crop. Callers must// pass normalized chunks: General/Token clear structural layout before// cropping, so passing raw parser items would lose their block-kinddiff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.goindex 897e8628ef..96af3114df 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go@@ -66,6 +66,16 @@ func jsonPositions(t *testing.T, rows ...[]float64) json.RawMessage {return b}+// jsonFlatPosition encodes one position row in the legacy flat wire form.+func jsonFlatPosition(t *testing.T, row []float64) json.RawMessage {+ t.Helper()+ b, err := json.Marshal(row)+ if err != nil {+ t.Fatalf("marshal flat position: %v", err)+ }+ return b+}+func TestNeedsCrop(t *testing.T) {cases := []struct {name string@@ -94,6 +104,19 @@ func TestNeedsCrop(t *testing.T) {// Unknown doc_type values normalize to text (itemDocType), matching// every other content-type dispatch.{"unknown doc type normalizes to text", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},++ // --- Page-only positions (TCADP wire form) carry page identity but+ // no crop region, so they must not trigger a crop.+ {"page-only image position", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{3, 0, 0, 0, 0})}, false},+ {"page-only text position", schema.ChunkDoc{DocType: "text", Positions: jsonPositions(t, []float64{3, 0, 0, 0, 0})}, false},+ {"zero-area box with coordinates", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{3, 10, 10, 5, 5})}, false},++ // --- A page-only row must not hide a real region in a later row:+ // mergePositionMatrix sorts page-only rows first.+ {"page-only row before a real region", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 0, 0, 0, 0}, []float64{2, 10, 100, 10, 100})}, true},+ {"real region before a page-only row", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100}, []float64{2, 0, 0, 0, 0})}, true},+ {"flat legacy position with a region", schema.ChunkDoc{DocType: "image", PDFPositions: jsonFlatPosition(t, []float64{1, 10, 100, 10, 100})}, true},+ {"flat legacy page-only position", schema.ChunkDoc{DocType: "image", PDFPositions: jsonFlatPosition(t, []float64{1, 0, 0, 0, 0})}, false},}for _, tc := range cases {if got := needsCrop(tc.ck); got != tc.want {diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex a66da12a33..fbabfd8449 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -513,6 +513,11 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// arms are keyed off the utility constants.//fileTypeExt := fileTypeFromInputs(inputs)+ parsePagesOn := pdfPageProgressSupported(fileTypeExt, setups)+ enhanceOn := c.enableVisionEnhancement+ split := progress.NewSplit(parsePagesOn, enhanceOn)+ pageWin := split.Take(parsePagesOn)+ visionWin := split.Take(enhanceOn)dispatched, handledVision, visionErr := maybeDispatchPDFVision(ctx, db, fileTypeExt, filename, binary, inputs, setups)if visionErr != nil {@@ -546,28 +551,25 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st}}if !handledVision && !handledMedia && !handledImage && !handledAudio {- parsePagesOn := pdfPageProgressSupported(fileTypeExt, setups)- enhanceOn := c.enableVisionEnhancement- split := progress.NewSplit(parsePagesOn, enhanceOn)- pageWin := split.Take(parsePagesOn)- visionWin := split.Take(enhanceOn)-dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups, pageWin)if dispatched.Err == nil && ctx.Err() == nil {pageWin.Report(ctx, 1.0)}-- if c.enableVisionEnhancement {- // Enhancement is optional; parser-provided text and image metadata- // remain available if a vision model cannot describe an image.- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.vision, visionWin)- }}// Known/supported families must fail loudly when dispatch or// parsing breaks. Only unknown families keep the raw-text fallback.if dispatched.Err != nil && fileTypeExt != utility.FileTypeOTHER {return nil, dispatched.Err}+ if enhanceOn && !handledMedia && !handledImage && !handledAudio && dispatched.Err == nil && ctx.Err() == nil {+ // PDF dispatch and ordinary parsers share enhancement after normalization.+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.vision, visionWin)+ } else {+ // Enhancement skipped entirely (disabled or non-PDF media handled):+ // still strip the transient backend-description marker so it never+ // reaches the chunker.+ dispatched = stripBackendDescribedResult(dispatched)+ }reportParserWarnings(ctx, dispatched.Warnings)if err := ctx.Err(); err != nil {diff --git a/internal/ingestion/component/pdf_postprocess_test.go b/internal/ingestion/component/pdf_postprocess_test.gonew file mode 100644index 0000000000..5416951833--- /dev/null+++ b/internal/ingestion/component/pdf_postprocess_test.go@@ -0,0 +1,190 @@+package component++import (+ "archive/zip"+ "bytes"+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "gorm.io/gorm"++ "ragflow/internal/entity"+ modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/progress"+ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser"+ "ragflow/internal/utility"+)++func TestPDFDispatchNormalizesBeforeOutputsAndEnhancement(t *testing.T) {+ image := visionTestPNGBase64(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var buf bytes.Buffer+ archive := zip.NewWriter(&buf)+ file, _ := archive.Create("sample/sample.json")+ _ = json.NewEncoder(file).Encode(map[string]any{"layouts": []any{map[string]any{"label": "Text", "content": ""}}})+ _ = archive.Close()+ _, _ = w.Write(buf.Bytes())+ }))+ defer server.Close()+ previousResolver := resolveTenantOCRModelByProvider+ t.Cleanup(func() { resolveTenantOCRModelByProvider = previousResolver })+ resolveTenantOCRModelByProvider = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return &monkeyOCRv2FakeDriver{}, "monkey", &modelModule.APIConfig{}, 0, nil+ }+ setup := schema.ParserSetup{"parse_method": "monkeyocrv2", "monkeyocrv2_server_url": server.URL}+ result, handled, err := maybeDispatchPDFVision(t.Context(), nil, utility.FileTypePDF, "sample.pdf", []byte("pdf"), nil, map[string]schema.ParserSetup{"pdf": setup})+ if err != nil || !handled || result.OutputFormat != "json" || len(result.JSON) == 0 {+ t.Fatalf("dispatch did not emit structured JSON: %+v, %v", result, err)+ }+ for _, enabled := range []bool{false, true} {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)+ component, err := NewParserComponent(map[string]any{"pdf": map[string]any(setup), "enable_vision_enhancement": enabled})+ if err != nil {+ t.Fatal(err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{"binary": []byte("pdf"), "file_type": "pdf", "name": "sample.pdf", "tenant_id": "tenant-1"})+ if err != nil {+ t.Fatal(err)+ }+ var text string+ for _, item := range out["json"].([]parser.DocItem) {+ value, _ := item.ToMap()["text"].(string)+ text += value+ }+ if strings.Contains(text, "a diagram of a pipeline") != enabled {+ t.Fatalf("enabled=%v output=%s", enabled, text)+ }+ want := 0+ if enabled {+ want = 1+ }+ if len(invoker.images) != want {+ t.Fatalf("enabled=%v model calls=%d, want %d", enabled, len(invoker.images), want)+ }+ }+}++func TestMinerUTableRetainsCaptionAndFootnote(t *testing.T) {+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><tr><td>Cell</td></tr></table>", "table_caption": []any{"Caption"}, "table_footnote": []any{"Footnote"}, "page_idx": 1, "bbox": []any{10.0, 20.0, 30.0, 40.0}}, nil)+ table, ok := item["table"].(*entity.TableData)+ if !ok || table == nil || table.Caption != "Caption" || len(table.CellTexts()) != 1 || len(table.CellTexts()[0]) != 1 || table.CellTexts()[0][0] != "Cell" || table.Page != 2 {+ t.Fatalf("structured table missing: %#v", item)+ }+ if item["text"] != "Footnote" {+ t.Fatalf("notes lost or table duplicated: %#v", item)+ }+}++func TestMinerUTablePreservesEmbeddedCaption(t *testing.T) {+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><caption>Embedded caption</caption><tr><td>Cell</td></tr></table>"}, nil)+ table, ok := item["table"].(*entity.TableData)+ if !ok || table == nil || table.Caption != "Embedded caption" {+ t.Fatalf("embedded caption lost: %#v", item)+ }+}++func TestPDFTableEnhancementKeepsDescriptionOutsideTableBody(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)+ result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{+ parser.NewTextJSONItem("Before"),+ parser.TableItem{Table: &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, Extra: map[string]any{"image": visionTestPNGBase64(t), "page_number": 3}},+ parser.NewTextJSONItem("After"),+ }}+ result, handled, err := maybeDispatchVisionEnhancement(t.Context(), nil, utility.FileTypePDF, result, map[string]any{"tenant_id": "tenant-1"}, visionSettings{}, progress.Window{})+ if err != nil || !handled {+ t.Fatalf("enhancement failed: %v", err)+ }+ if len(result.JSON) != 4 || result.JSON[1].ToMap()["text"] != nil || result.JSON[2].ToMap()["text"] != "a diagram of a pipeline" || result.JSON[2].ToMap()["page_number"] != 3 || result.JSON[3].ToMap()["text"] != "After" {+ t.Fatalf("table description broke table contract or order: %#v", result.JSON)+ }+}++func TestMinerUContentMarksBackendDescribedImages(t *testing.T) {+ withDesc := mineruContentItemToJSON(map[string]any{+ "type": "image", "image_caption": []any{"Caption"},+ "vlm_description": "backend desc",+ "page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},+ }, nil)+ if withDesc == nil {+ t.Fatal("image item dropped")+ }+ if withDesc[backendDescribedKey] != true {+ t.Fatalf("backend-described image not marked: %#v", withDesc)+ }+ if text, _ := withDesc["text"].(string); !strings.Contains(text, "backend desc") {+ t.Fatalf("backend description lost from text: %q", text)+ }+ withoutDesc := mineruContentItemToJSON(map[string]any{+ "type": "image", "image_caption": []any{"Caption"},+ "page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},+ }, nil)+ if withoutDesc == nil {+ t.Fatal("image item dropped")+ }+ if _, ok := withoutDesc[backendDescribedKey]; ok {+ t.Fatalf("undescribed image wrongly marked: %#v", withoutDesc)+ }+}++func TestVisionEnhancementSkipsBackendDescribedImages(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)+ png := "data:image/png;base64," + visionTestPNGBase64(t)+ marked := parser.DocItemFromMap(map[string]any{+ "doc_type_kwd": "image", "layout": "figure",+ "text": "Caption\nbackend desc", "image": png,+ backendDescribedKey: true,+ })+ plain := parser.DocItemFromMap(map[string]any{+ "doc_type_kwd": "image", "layout": "figure",+ "text": "Caption", "image": png,+ })+ result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{marked, plain}}+ enhanced, handled, err := maybeDispatchVisionEnhancement(t.Context(), nil, utility.FileTypePDF, result, map[string]any{"tenant_id": "tenant-1"}, visionSettings{}, progress.Window{})+ if err != nil || !handled {+ t.Fatalf("enhancement failed: %v", err)+ }+ if len(invoker.images) != 1 {+ t.Fatalf("backend-described image should be skipped: %d VLM calls", len(invoker.images))+ }+ if len(enhanced.JSON) != 2 {+ t.Fatalf("items lost: %#v", enhanced.JSON)+ }+ first := enhanced.JSON[0].ToMap()+ if text, _ := first["text"].(string); text != "Caption\nbackend desc" {+ t.Fatalf("backend-described text duplicated or changed: %q", text)+ }+ if _, ok := first[backendDescribedKey]; ok {+ t.Fatalf("marker leaked to output: %#v", first)+ }+ second := enhanced.JSON[1].ToMap()+ if text, _ := second["text"].(string); !strings.Contains(text, "a diagram of a pipeline") {+ t.Fatalf("undescribed image not enhanced: %q", text)+ }+ if _, ok := second[backendDescribedKey]; ok {+ t.Fatalf("marker leaked to output: %#v", second)+ }+}++func TestStripBackendDescribedResultClearsMarker(t *testing.T) {+ result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{+ parser.DocItemFromMap(map[string]any{"doc_type_kwd": "image", "text": "t", backendDescribedKey: true}),+ parser.NewTextJSONItem("plain"),+ }}+ stripped := stripBackendDescribedResult(result)+ for _, item := range stripped.JSON {+ if _, ok := item.ToMap()[backendDescribedKey]; ok {+ t.Fatalf("marker leaked: %#v", item.ToMap())+ }+ }+ if text, _ := stripped.JSON[0].ToMap()["text"].(string); text != "t" {+ t.Fatalf("text changed: %q", text)+ }+}diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 3ddb05df39..b65f90f476 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -24,6 +24,7 @@ import (modelModule "ragflow/internal/entity/models""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser"+ "ragflow/internal/parser/tableutil""ragflow/internal/rag/prompts""ragflow/internal/utility"@@ -79,16 +80,38 @@ func maybeDispatchPDFVision(if !ok {return parser.ParseResult{}, false, nil}+ tenantID := getStringOr(inputs, "tenant_id", "")+ // The tenant-selected parser and the built-in backends share one+ // normalization boundary so a future normalization fix cannot miss one+ // of the two paths. ParseWithResult already normalizes JSON internally,+ // but markdown stays unconverted there; running it through the same+ // NormalizeResult keeps markdown→JSON conversion in the parser package+ // and makes the call idempotent for already-normalized JSON.+ flatten, _ := setup["flatten_media_to_text"].(bool)+ normalize := func(result parser.ParseResult) parser.ParseResult {+ pdf := parser.PDFParser{FlattenMediaToText: flatten}+ return pdf.NormalizeResult(ctx, filename, result)+ }+ if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled {+ if err != nil {+ return result, true, err+ }+ result = normalize(result)+ return result, true, result.Err+ }+ result, handled, err := dispatchPDFBackend(ctx, db, filename, binary, tenantID, setup)+ if err != nil || !handled {+ return result, handled, err+ }+ result = normalize(result)+ return result, true, result.Err+}+func dispatchPDFBackend(ctx context.Context, db *gorm.DB, filename string, binary []byte, tenantID string, setup schema.ParserSetup) (parser.ParseResult, bool, error) {method := getStringOr(setup, "parse_method", "")layout := getStringOr(setup, "layout_recognizer", "")- tenantID := getStringOr(inputs, "tenant_id", "")layoutLower := strings.ToLower(strings.TrimSpace(layout))- if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled {- return result, true, err- }-monkeySelector := layoutif strings.TrimSpace(monkeySelector) == "" {monkeySelector = method@@ -254,6 +277,10 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b}func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []parser.DocItem {+ // Fallback markdown→JSON converter for non-PDF families via+ // buildParserOutputs/normalizeParserJSON. PDF dispatch results are+ // normalized earlier by PDFParser.NormalizeResult (parser.parsePDFMarkdown),+ // so this stays as the final safety net rather than a second PDF boundary.mp, err := parser.NewMarkdownParser(parser.GoMarkdown)if err != nil {warnParserNormalizationFallback(filename, "markdown", err)@@ -276,8 +303,7 @@ func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []pa}func buildMarkdownOCRDispatchResult(md string) parser.ParseResult {- // Keep the backend's Markdown intact here. buildParserOutputs is the- // single normalization boundary for all non-JSON parser responses.+ // The PDF dispatch boundary converts backend Markdown before enhancement.return parser.ParseResult{OutputFormat: "markdown",Markdown: md,@@ -922,6 +948,14 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) m"doc_type_kwd": docType,"layout": layout,}+ // MinerU folds its own VLM description into text for image/chart items.+ // Mark them so the shared vision enhancement skips them instead of+ // appending a second model description.+ if typ == "image" || typ == "chart" {+ if desc, _ := item["vlm_description"].(string); strings.TrimSpace(desc) != "" {+ jsonItem[backendDescribedKey] = true+ }+ }if imageURI != "" {jsonItem["image"] = imageURI}@@ -932,6 +966,20 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) mjsonItem["page_number"] = page}}+ if typ == "table" {+ body, _ := item["table_body"].(string)+ if table, err := tableutil.ParseTableHTML(body); err == nil && table != nil && len(table.Grid) > 0 {+ caption := strings.Join(stringSlice(item["table_caption"]), "\n")+ if caption != "" && caption != table.Caption {+ table.Caption = strings.TrimSpace(table.Caption + "\n" + caption)+ }+ if page, ok := jsonItem["page_number"].(int); ok {+ table.Page = page+ }+ jsonItem["table"] = table+ jsonItem["text"] = strings.Join(stringSlice(item["table_footnote"]), "\n")+ }+ }return jsonItem}diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex df652a8839..9cf69b4d3b 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -120,19 +120,6 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {}}-func TestBuildMarkdownOCRDispatchResultDefersJSONNormalization(t *testing.T) {- result := buildMarkdownOCRDispatchResult("# Heading")- if result.OutputFormat != "markdown" {- t.Fatalf("OutputFormat = %q, want markdown", result.OutputFormat)- }- if result.Markdown != "# Heading" {- t.Fatalf("Markdown = %q, want original OCR markdown", result.Markdown)- }- if result.JSON != nil {- t.Fatalf("JSON = %#v, want nil until buildParserOutputs normalization", result.JSON)- }-}-type monkeyOCRv2FakeDriver struct {modelModule.ModelDriver}diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex 3d37ce56c3..2856eceb7a 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -27,6 +27,7 @@ import ("fmt""image""io"+ "maps""strings""sync""sync/atomic"@@ -61,6 +62,56 @@ const (visionMediaItemBudget = 10 * time.Second)+// backendDescribedKey marks items whose text already carries a backend-provided+// vision description (MinerU folds vlm_description into text). The vision+// enhancement collector skips such items to avoid appending a second model+// description and spending an extra VLM call. The key is stripped before the+// result leaves the Parser component so it never reaches the chunker or index.+const backendDescribedKey = "_backend_described"++func isBackendDescribed(m map[string]any) bool {+ v, _ := m[backendDescribedKey].(bool)+ return v+}++// stripBackendDescribedResult removes the transient backend-description marker+// from every item. Enhancement skips marked items, but the marker must not leak+// to the chunker when enhancement is disabled or bails out early.+func stripBackendDescribedResult(result parser.ParseResult) parser.ParseResult {+ if len(result.JSON) == 0 {+ return result+ }+ needsStrip := false+ for _, item := range result.JSON {+ if item == nil {+ continue+ }+ if _, ok := item.ToMap()[backendDescribedKey]; ok {+ needsStrip = true+ break+ }+ }+ if !needsStrip {+ return result+ }+ stripped := make([]parser.DocItem, 0, len(result.JSON))+ for _, item := range result.JSON {+ if item == nil {+ continue+ }+ m := item.ToMap()+ if _, ok := m[backendDescribedKey]; !ok {+ stripped = append(stripped, item)+ continue+ }+ cloned := maps.Clone(m)+ delete(cloned, backendDescribedKey)+ stripped = append(stripped, parser.DocItemFromMap(cloned))+ }+ result.JSON = stripped+ return result+}+// isUsableVisionImage checks the encoded size and rejects raster dimensions// beyond the limits shared by Parser and VLM materialization.func isUsableVisionImage(raw string) bool {@@ -140,7 +191,7 @@ func maybeDispatchVisionEnhancement(// Only enhance successful JSON output format containing items.if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {- return dispatched, false, nil+ return stripBackendDescribedResult(dispatched), false, nil}tenantID := getStringOr(inputs, "tenant_id", "")@@ -151,23 +202,30 @@ func maybeDispatchVisionEnhancement(language := resolveVisionLanguage(inputs)// Collect visual resources, including Markdown images whose type is text- // because flatten_media_to_text is enabled.+ // because flatten_media_to_text is enabled. Items whose text already+ // carries a backend-provided description (MinerU vlm_description) are+ // skipped to avoid a duplicate second description and an extra VLM call.var items []intfor i, rawItem := range dispatched.JSON {m := rawItem.ToMap()+ if isBackendDescribed(m) {+ continue+ }if img, _ := m["image"].(string); img != "" {items = append(items, i)continue}kd, _ := m["doc_type_kwd"].(string)if kd == "image" || kd == "table" {- if _, ok := parser.ExtractPDFPositions(m); ok {+ // A page-only position carries no crop region, so it must not+ // reach the cropper.+ if parser.HasPDFCropRegion(m) {items = append(items, i)}}}if len(items) == 0 {- return dispatched, false, nil+ return stripBackendDescribedResult(dispatched), false, nil}// Resolve VLM before materializing image payloads, so disabled or// unavailable enhancement does not spend time loading media.@@ -204,7 +262,7 @@ func maybeDispatchVisionEnhancement(}}dispatched.Warnings = append(dispatched.Warnings, warning)- return dispatched, false, nil+ return stripBackendDescribedResult(dispatched), false, nil}// Prepare PDF sources outside image admission so slow storage reads do not@@ -213,7 +271,7 @@ func maybeDispatchVisionEnhancement(cropper, cerr := visionImageCropperFactory(ctx, db, inputs)if cerr != nil {dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement skipped: %v", cerr))- return dispatched, false, nil+ return stripBackendDescribedResult(dispatched), false, nil}// Close is invoked explicitly after materialization so the engine and the// source bytes are released while model requests are still in flight. Keep@@ -354,7 +412,7 @@ func maybeDispatchVisionEnhancement(}wg.Wait()if err := ctx.Err(); err != nil {- return dispatched, modified, err+ return stripBackendDescribedResult(dispatched), modified, err}w.Report(ctx, 1.0)failedItems := 0@@ -375,17 +433,42 @@ func maybeDispatchVisionEnhancement(zap.Int("failed_items", failedItems), zap.String("first_failure", firstFailure))}- // Append descriptions after parser-provided text and captions.- for slot, itemIdx := range items {+ // Candidate indices are in document order. Build output once so a table's+ // description can follow it without changing later candidates' indices.+ // The transient backend-description marker is stripped from every emitted+ // item so it never reaches the chunker.+ enhanced := make([]parser.DocItem, 0, len(dispatched.JSON)+len(items))+ slot := 0+ for itemIdx, item := range dispatched.JSON {+ clean := item+ if m := item.ToMap(); isBackendDescribed(m) {+ cloned := maps.Clone(m)+ delete(cloned, backendDescribedKey)+ clean = parser.DocItemFromMap(cloned)+ }+ if slot >= len(items) || items[slot] != itemIdx {+ enhanced = append(enhanced, clean)+ continue+ }desc := strings.TrimSpace(descriptions[slot])+ slot++if desc == "" {+ enhanced = append(enhanced, clean)continue}- m := dispatched.JSON[itemIdx].ToMap()- appendItemText(m, desc)- dispatched.JSON[itemIdx] = parser.DocItemFromMap(m)+ m := clean.ToMap()+ // clean.ToMap may share the underlying map for RawItem; clone before+ // mutating so the source item is never modified in place.+ m = maps.Clone(m)+ if table, ok := m["table"].(*entity.TableData); ok && table != nil && m["doc_type_kwd"] == "table" {+ enhanced = append(enhanced, clean, parser.NewTableNoteItem(m, desc))+ } else {+ appendItemText(m, desc)+ enhanced = append(enhanced, parser.DocItemFromMap(m))+ }modified = true}+ dispatched.JSON = enhancedreturn dispatched, modified, nil}diff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.goindex 0f5dbe2a1e..e3d73b742b 100644--- a/internal/ingestion/component/vision_enhancement_cgo.go+++ b/internal/ingestion/component/vision_enhancement_cgo.go@@ -101,6 +101,9 @@ func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]a// share this predicate so a future condition change cannot make Prepare skip a// fetch that Crop then performs under image admission, silently reintroducing// the admission block.+//+// A page-only position (page number, no geometry) carries no crop region and+// is skipped, so it never triggers a page render.func pdfSourcePositions(item map[string]any) ([]deepdoctype.Position, bool) {if img, _ := item["image"].(string); img != "" {return nil, false@@ -110,10 +113,16 @@ func pdfSourcePositions(item map[string]any) ([]deepdoctype.Position, bool) {return nil, false}positions := util.PositionsFromMatrix(matrix)- if len(positions) == 0 {+ regions := positions[:0]+ for _, pos := range positions {+ if pos.Right > pos.Left && pos.Bottom > pos.Top {+ regions = append(regions, pos)+ }+ }+ if len(regions) == 0 {return nil, false}- return positions, true+ return regions, true}func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visionImage, error) {diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.goindex fa7adf08a3..14d246515e 100644--- a/internal/parser/parser/doc_item.go+++ b/internal/parser/parser/doc_item.go@@ -184,7 +184,7 @@ func imageItemFromMap(m map[string]any) DocItem {if pos != nil {// Parsed into the typed field: it owns "positions", so strip it from// Extra. Both [][]float64 (office/xlsx) and [][]any (PDF crop regions- // from normalizePDFPositions) parse here, so the old [][]any-only+ // from normalizeDeepDocPositions) parse here, so the old [][]any-only// Extra fallback is gone.skip = append(skip, "positions")}@@ -385,7 +385,7 @@ func wireToPositions(raw any, hasSheet bool) Positions {// into the historical wire form for ToMap. The wire container type is preserved// per vocabulary so the parser output is byte-for-byte unchanged:// - AbsoluteBox (PDF) -> [][]any (int page + float64 coords), matching the-// shape normalizePDFPositions already emitted and keeping the page a+// shape normalizeDeepDocPositions already emitted and keeping the page a// genuine int (a [][]float64 container would coerce it to float64).// - RowColumnBox (xlsx/csv) -> [][]float64, matching the historical// spreadsheet wire the ingestion indexer consumes.diff --git a/internal/parser/parser/doc_item_positions_test.go b/internal/parser/parser/doc_item_positions_test.goindex c78fb2a303..d6872104b8 100644--- a/internal/parser/parser/doc_item_positions_test.go+++ b/internal/parser/parser/doc_item_positions_test.go@@ -56,7 +56,7 @@ func TestPositionKindMethod(t *testing.T) {}// PDF path positions arrive as [][]any (int page + float64 coords) from-// normalizePDFPositions. wireToPositions must build AbsoluteBox and round-trip+// normalizeDeepDocPositions. wireToPositions must build AbsoluteBox and round-trip// byte-exact, with the page preserved as int.func TestWireToPositionsAbsoluteAny(t *testing.T) {raw := [][]any{{1, 2.0, 3.0, 4.0, 5.0}}diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.goindex 59689f994f..3ab4d180b2 100644--- a/internal/parser/parser/doc_item_test.go+++ b/internal/parser/parser/doc_item_test.go@@ -244,7 +244,7 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {}opaqueMap := map[string]any{"foo": "bar", "nested": map[string]any{"a": 1}}- // PDF crop positions arrive as [][]any (normalizePDFPositions), not+ // PDF crop positions arrive as [][]any (normalizeDeepDocPositions), not// [][]float64; the typed conversion must preserve them verbatim. Regression// guard for the positions-dropping bug caught during review.imageMapAny := map[string]any{diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 2d08a4713a..aa57203814 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -33,7 +33,6 @@ import (pdflayout "ragflow/internal/deepdoc/parser/pdf/layout""ragflow/internal/deepdoc/parser/pdf/util"deepdoctype "ragflow/internal/deepdoc/parser/type"- "ragflow/internal/parser/tableutil""ragflow/internal/utility")@@ -353,6 +352,14 @@ func (p *PDFParser) validateParseMethod() error {// build-tag agnostic: under !cgo the deepdoc path transparently reports// ErrPDFEngineUnavailable while plain_text falls back to the pure-Go engine.func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {+ result := p.parseWithResult(ctx, filename, data)+ if result.Err == nil && result.OutputFormat == "json" {+ return p.NormalizeResult(ctx, filename, result)+ }+ return result+}++func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data []byte) ParseResult {if err := p.validateParseMethod(); err != nil {return ParseResult{Err: err}}@@ -527,10 +534,6 @@ func GetDocAnalyzer() (deepdoctype.DocAnalyzer, error) {return deepDocAnalyzerFromEnv()}-func pdfParseResultToJSON(filename string, parsed *deepdoctype.ParseResult) ParseResult {- return pdfParseResultToJSONWithOptions(filename, parsed, pdfPostProcessOptions{})-}-func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseResult, opts pdfPostProcessOptions) ParseResult {if parsed == nil {return ParseResult{Err: fmt.Errorf("parser: nil DeepDOC PDF result for %s", filename)}@@ -541,6 +544,8 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseRif opts.enableMultiColumn && opts.pageWidth <= 0 {opts.pageWidth = firstPDFPageWidth(processed.PageWidth)}+ // JSON media policy is applied once at the PDFParser result boundary.+ opts.flattenMediaToText = falseapplyPDFPostProcess(&processed, opts)defer processed.Close()// NOTE: PDF media (figure/table) is intentionally NOT cropped here under@@ -559,15 +564,13 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseRif layoutType, _ := items[i]["layout_type"].(string); layoutType != "" {items[i]["layout"] = layoutType}- if normalized := normalizePDFPositions(items[i]["_pdf_positions"]); len(normalized) > 0 {+ if normalized := normalizeDeepDocPositions(items[i]["_pdf_positions"]); len(normalized) > 0 {items[i]["_pdf_positions"] = normalizeditems[i]["positions"] = normalizedif _, ok := items[i]["page_number"]; !ok {items[i]["page_number"] = firstPageNumber(normalized)}}- normalizePDFDocType(items[i])- projectPDFTable(items[i])if img, _ := items[i]["image"].(string); img != "" {items[i]["image"] = pdflayout.InlinePNGDataURL(img)}@@ -897,7 +900,9 @@ func firstPDFPageWidth(pageWidths map[int]float64) float64 {return pageWidths[pg]}-func normalizePDFPositions(raw any) [][]any {+// normalizeDeepDocPositions converts DeepDoc source positions, including its+// zero-based pages. Other engines must not pass canonical positions here.+func normalizeDeepDocPositions(raw any) [][]any {positions, ok := raw.([][]any)if !ok || len(positions) == 0 {return nil@@ -907,7 +912,7 @@ func normalizePDFPositions(raw any) [][]any {if len(pos) < 5 {continue}- pageNumber, ok := normalizePDFPageNumber(pos[0])+ pageNumber, ok := normalizeDeepDocPageNumber(pos[0])if !ok {continue}@@ -923,7 +928,7 @@ func normalizePDFPositions(raw any) [][]any {return normalized}-// normalizePDFPageNumber converts a DeepDoc 0-indexed page number to the+// normalizeDeepDocPageNumber converts a DeepDoc 0-indexed page number to the// 1-indexed form stored in _pdf_positions / positions. It is the SINGLE// 0→1 conversion point: DeepDoc (pdf_oxide/pdfium) emits 0-indexed pages,// and every downstream consumer (addPDFPositions for ES storage,@@ -931,24 +936,24 @@ func normalizePDFPositions(raw any) [][]any {// Adding +1 unconditionally — instead of only for v<=0 — keeps all pages// consistent; the old heuristic left page>=1 unconverted, which addPDFPositions// then double-incremented and PositionsFromMatrix mis-decremented.-func normalizePDFPageNumber(raw any) (int, bool) {+func normalizeDeepDocPageNumber(raw any) (int, bool) {switch v := raw.(type) {case int:return v + 1, truecase int64:- return normalizePDFPageNumber(int(v))+ return normalizeDeepDocPageNumber(int(v))case float64:- return normalizePDFPageNumber(int(v))+ return normalizeDeepDocPageNumber(int(v))case []any:if len(v) == 0 {return 0, false}- return normalizePDFPageNumber(v[len(v)-1])+ return normalizeDeepDocPageNumber(v[len(v)-1])case []int:if len(v) == 0 {return 0, false}- return normalizePDFPageNumber(v[len(v)-1])+ return normalizeDeepDocPageNumber(v[len(v)-1])default:return 0, false}@@ -967,86 +972,11 @@ func numericAny(raw any) (float64, bool) {}}-func normalizePDFDocType(item map[string]any) {- if item == nil {- return- }- layoutType, _ := item["layout_type"].(string)- // A figure caption is a media section even when the parser no longer- // inlines its cropped image (cgo): it still carries PDF positions, so the- // downstream VLM/chunker crop it on demand. Classify it as image whenever- // it has positions (the inlined image was only a side effect of cropping).- _, hasMedia := ExtractPDFPositions(item)- if docType, _ := item["doc_type_kwd"].(string); docType != "" {- // A figure caption keeps its media classification so the downstream- // VLM enhancement and on-demand chunker crop it.- if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption && hasMedia {- item["doc_type_kwd"] = "image"- }- return- }- switch layoutType {- case "table":- item["doc_type_kwd"] = "table"- case "figure", "image":- item["doc_type_kwd"] = "image"- case deepdoctype.DLALabelFigureCaption:- if hasMedia {- item["doc_type_kwd"] = "image"- } else {- item["doc_type_kwd"] = "text"- }- default:- if img, _ := item["image"].(string); img != "" {- item["doc_type_kwd"] = "image"- return- }- item["doc_type_kwd"] = "text"- }-}--// projectPDFTable upgrades a legacy pdf table item — one whose doc_type_kwd is-// "table" and whose text is <table> markup — to the structured TableData-// contract, so the downstream chunker reads Rows directly instead of-// re-parsing HTML. ParseTableHTML + RenderTableHTML round-trips the legacy-// htmlTableRE extraction exactly (entity/html unescaping is left to the-// consumer), so dropping the original HTML string loses no information: the-// markdown and search paths render it on demand from `table`. The pdf page-// number (when present) is projected onto TableData.Page.-//-// The redundant HTML `text` is deleted once `table` is projected: keeping both-// would duplicate every table's cell text in the parse result (the storage-// cost that motivated moving to the structured contract). Non-table items, and-// table items whose text is not <table> markup, are left untouched so their-// `text` is preserved as a last-resort carrier.-func projectPDFTable(item map[string]any) {- if item == nil {- return- }- if dt, _ := item["doc_type_kwd"].(string); dt != "table" {- return- }- text, _ := item["text"].(string)- if !isTableHTML(text) {- return- }- td, err := tableutil.ParseTableHTML(text)- if err != nil || td == nil {- return- }- if page := numberValue(item["page_number"]); page > 0 {- td.Page = int(page)- }- item["table"] = td- delete(item, "text")-}-// ExtractPDFPositions is the single source of truth for "does this parsed item-// carry a usable PDF crop region". It returns the non-empty positions matrix-// (under the canonical _pdf_positions key or the legacy positions key),-// accepting either the typed [][]any form produced in-process or the-// JSON-decoded []any form whose elements are themselves []any rows. The bool-// reports whether a usable matrix was present.+// carry PDF positions". It returns the position rows (under the canonical+// _pdf_positions key or the legacy positions key), decoded through+// PDFPositionRows so every wire shape is understood. The bool reports whether+// any row was present.//// The parser (doc-type classification in normalizePDFDocType) and the on-demand// croppers (VLM vision_enhancement and the chunker) MUST agree on this@@ -1056,34 +986,13 @@ func projectPDFTable(item map[string]any) {// which escaped the figure-caption → image normalization and the crop path.func ExtractPDFPositions(item map[string]any) ([][]any, bool) {for _, key := range []string{"_pdf_positions", "positions"} {- switch v := item[key].(type) {- case [][]any:- if len(v) > 0 {- return v, true- }- case []any:- out := make([][]any, 0, len(v))- ok := true- for _, row := range v {- r, rOK := row.([]any)- if !rOK {- ok = false- break- }- out = append(out, r)- }- if ok && len(out) > 0 {- return out, true- }+ if rows := PDFPositionRows(item[key]); len(rows) > 0 {+ return rows, true}}return nil, false}-func parsePDFWithDeepDoc(ctx context.Context, filename string, data []byte, parseFn func(context.Context, []byte, deepdoctype.DocAnalyzer) (*deepdoctype.ParseResult, error)) ParseResult {- return parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{}, parseFn)-}-func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []byte, opts pdfPostProcessOptions, parseFn func(context.Context, []byte, deepdoctype.DocAnalyzer) (*deepdoctype.ParseResult, error)) ParseResult {if len(data) == 0 {return emptyPDFResult(filename)@@ -1105,12 +1014,5 @@ func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []bytdefault:return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", opts.outputFormat)}}- for i := range res.JSON {- m := res.JSON[i].ToMap()- if img, _ := m["image"].(string); img != "" {- m["image"] = pdflayout.InlinePNGDataURL(img)- res.JSON[i] = DocItemFromMap(m)- }- }return res}diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.goindex 07058df973..703ebd8f4d 100644--- a/internal/parser/parser/pdf_parser_common_test.go+++ b/internal/parser/parser/pdf_parser_common_test.go@@ -1,6 +1,7 @@package parserimport (+ "context""image""strings""testing"@@ -8,6 +9,10 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type")+func pdfParseResultToJSON(filename string, parsed *deepdoctype.ParseResult) ParseResult {+ return NewPDFParser().NormalizeResult(context.Background(), filename, pdfParseResultToJSONWithOptions(filename, parsed, pdfPostProcessOptions{}))+}+func TestPDFParseResultToJSON_NormalizesCoreFields(t *testing.T) {parsed := &deepdoctype.ParseResult{Sections: []deepdoctype.Section{@@ -154,7 +159,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {}// TestNormalizePDFPageNumber_UnconditionalIncrement pins the contract that-// DeepDoc emits 0-indexed page numbers and normalizePDFPageNumber is the+// DeepDoc emits 0-indexed page numbers and normalizeDeepDocPageNumber is the// SINGLE conversion point to 1-indexed. It must add +1 unconditionally —// not just for v<=0 — so that downstream addPDFPositions (a passthrough) and// PositionsFromMatrix (which subtracts 1 for the 0-indexed PDFium engine)@@ -178,7 +183,7 @@ func TestNormalizePDFPageNumber_UnconditionalIncrement(t *testing.T) {}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- got, ok := normalizePDFPageNumber(tc.in)+ got, ok := normalizeDeepDocPageNumber(tc.in)if ok != tc.ok {t.Fatalf("ok = %v, want %v", ok, tc.ok)}@@ -189,11 +194,11 @@ func TestNormalizePDFPageNumber_UnconditionalIncrement(t *testing.T) {}}-func TestPDFParseResultToJSON_PreservesPositivePageNumbers(t *testing.T) {+func TestPDFParseResultToJSONConvertsPositiveSourcePageNumbers(t *testing.T) {parsed := &deepdoctype.ParseResult{Sections: []deepdoctype.Section{{- Text: "Already one-based",+ Text: "<table><tr><td>Page four</td></tr></table>",LayoutType: deepdoctype.LayoutTypeTable,Positions: []deepdoctype.Position{{@@ -209,7 +214,7 @@ func TestPDFParseResultToJSON_PreservesPositivePageNumbers(t *testing.T) {}res := pdfParseResultToJSON("one-based.pdf", parsed)- // DeepDoc page 3 is 0-indexed (the 4th page); normalizePDFPageNumber+ // DeepDoc page 3 is 0-indexed (the 4th page); normalizeDeepDocPageNumber// converts it to 1-indexed page 4.if got, want := res.JSON[0].ToMap()["page_number"], 4; got != want {t.Fatalf("JSON[0].page_number = %v, want %v", got, want)diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.goindex b08022df2f..b99a84e107 100644--- a/internal/parser/parser/pdf_parser_docling.go+++ b/internal/parser/parser/pdf_parser_docling.go@@ -235,13 +235,13 @@ func parseDoclingStandardResult(ctx context.Context, filename string, body []byt}for _, doc := range docs {if md := strings.TrimSpace(doc.MDContent); md != "" {- return parseMinerUMarkdownResult(ctx, filename, md, outputFormat, len(docs)), true+ return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true}if txt := strings.TrimSpace(doc.TextContent); txt != "" {return doclingTextsToResult(filename, []string{txt}, outputFormat, len(docs)), true}if md, _ := doc.JSONContent["md_content"].(string); strings.TrimSpace(md) != "" {- return parseMinerUMarkdownResult(ctx, filename, md, outputFormat, len(docs)), true+ return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true}}return ParseResult{}, falsediff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.goindex 0b66f11d95..333ded8f3d 100644--- a/internal/parser/parser/pdf_parser_mineru.go+++ b/internal/parser/parser/pdf_parser_mineru.go@@ -72,7 +72,7 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parseif strings.TrimSpace(content) != "" {pageCount = 1}- return parseMinerUMarkdownResult(ctx, filename, content, parser.OutputFormat, pageCount)+ return pdfMarkdownToResult(ctx, filename, content, parser.OutputFormat, pageCount)}func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskID string, apiConfig *models.APIConfig, timeout time.Duration) (string, error) {@@ -102,25 +102,3 @@ func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskIDtime.Sleep(minerUPollInterval)}}--func parseMinerUMarkdownResult(ctx context.Context, filename, markdown, outputFormat string, pageCount int) ParseResult {- fileMeta := pdfFileMeta(filename, pageCount)- switch strings.ToLower(strings.TrimSpace(outputFormat)) {- case "", "json":- mp, _ := NewMarkdownParser(GoMarkdown)- res := mp.ParseWithResult(ctx, filename, []byte(markdown))- if res.Err != nil {- return res- }- res.File = fileMeta- return res- case "markdown":- return ParseResult{- OutputFormat: "markdown",- File: fileMeta,- Markdown: markdown,- }- default:- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}- }-}diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.goindex 7227752873..46e0b2b495 100644--- a/internal/parser/parser/pdf_parser_opendataloader.go+++ b/internal/parser/parser/pdf_parser_opendataloader.go@@ -68,7 +68,7 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byt}}if strings.TrimSpace(payload.MDText) != "" {- return parseMinerUMarkdownResult(ctx, filename, payload.MDText, parser.OutputFormat, 1)+ return pdfMarkdownToResult(ctx, filename, payload.MDText, parser.OutputFormat, 1)}return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader returned no parsed content")}}diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.goindex 57d6df306f..1073df5ab9 100644--- a/internal/parser/parser/pdf_parser_remote_common.go+++ b/internal/parser/parser/pdf_parser_remote_common.go@@ -1,6 +1,7 @@package parserimport (+ "context""fmt""strings"@@ -28,7 +29,11 @@ func pdfFileMeta(filename string, pageCount int) map[string]any {func markdownItemText(item map[string]any) string {if dt, _ := item["doc_type_kwd"].(string); dt == "table" {if td, ok := item["table"].(*entity.TableData); ok && td != nil {- return tableutil.RenderTableHTML(td)+ markup := tableutil.RenderTableHTML(td)+ if notes, _ := item["text"].(string); strings.TrimSpace(notes) != "" {+ return strings.TrimRight(markup, "\n") + "\n\n" + notes+ }+ return markup}return ""}@@ -150,3 +155,35 @@ func rowsHaveText(rows [][]string) bool {}return false}++func pdfMarkdownToResult(ctx context.Context, filename, markdown, outputFormat string, pageCount int) ParseResult {+ fileMeta := pdfFileMeta(filename, pageCount)+ switch strings.ToLower(strings.TrimSpace(outputFormat)) {+ case "", "json":+ res := parsePDFMarkdown(ctx, filename, markdown)+ if res.Err != nil {+ if ctx.Err() != nil {+ res.File = fileMeta+ return res+ }+ // Same text fallback as NormalizeResult: keep the backend+ // markdown as raw text instead of failing the task.+ return ParseResult{+ OutputFormat: "json",+ File: fileMeta,+ Warnings: append(res.Warnings, "markdown normalization fell back to text: "+res.Err.Error()),+ JSON: []DocItem{NewTextJSONItem(markdown)},+ }+ }+ res.File = fileMeta+ return res+ case "markdown":+ return ParseResult{+ OutputFormat: "markdown",+ File: fileMeta,+ Markdown: markdown,+ }+ default:+ return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}+ }+}diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.goindex b10b13334c..e52f173647 100644--- a/internal/parser/parser/pdf_parser_tcadp.go+++ b/internal/parser/parser/pdf_parser_tcadp.go@@ -84,6 +84,7 @@ func tcadpAnyToItems(raw any) []map[string]any {emit := func(text, docType, layout string) []map[string]any {m := map[string]any{"text": text, "doc_type_kwd": docType, "layout": layout}if page > 0 {+ m["page_number"] = page// 1-indexed 5-tuple. addPDFPositions is a passthrough so// the final position_int / page_num_int carry the same// 1-indexed page number the caller passes. Mirrorsdiff --git a/internal/parser/parser/pdf_parser_tcadp_test.go b/internal/parser/parser/pdf_parser_tcadp_test.goindex 37dd8148aa..dbaff9b772 100644--- a/internal/parser/parser/pdf_parser_tcadp_test.go+++ b/internal/parser/parser/pdf_parser_tcadp_test.go@@ -204,6 +204,41 @@ func TestTCADPAnyToItems_PropagatesPageNumber(t *testing.T) {}}+// TestTCADPItemsKeepPageIdentityThroughNormalization runs the TCADP adapter+// output through the shared PDF normalization boundary. The flat page-only+// tuple must survive as page identity (the indexer derives page_num_int from+// positions) while staying out of the crop path.+func TestTCADPItemsKeepPageIdentityThroughNormalization(t *testing.T) {+ for _, raw := range []map[string]any{+ {"content": "slide text", "type": "text", "page_number": 3},+ {"caption": "fig", "type": "image", "page_number": 5},+ {"type": "table", "table_data": map[string]any{"rows": []any{[]any{"a", "b"}}}, "page_number": 2},+ } {+ items := tcadpAnyToItems(raw)+ if len(items) == 0 {+ t.Fatalf("tcadpAnyToItems(%v) returned no items", raw)+ }+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ item := result.JSON[0].ToMap()+ if item["page_number"] == nil {+ t.Fatalf("page identity lost: %#v", item)+ }+ positions, ok := ExtractPDFPositions(item)+ if !ok || len(positions) != 1 {+ t.Fatalf("page-only position lost: %#v", item)+ }+ if positions[0][0] != item["page_number"] {+ t.Fatalf("position page %v != page_number %v", positions[0][0], item["page_number"])+ }+ if HasPDFCropRegion(item) {+ t.Fatalf("page-only position exposed as a crop region: %#v", item)+ }+ }+}+// TestTCADPAnyToItems_NestedArrayKeepsEachPage verifies that a nested// array of TCADP elements produces one item per element, each carrying// its own page-derived positions — not just the first element.diff --git a/internal/parser/parser/pdf_positions.go b/internal/parser/parser/pdf_positions.gonew file mode 100644index 0000000000..6968684ee2--- /dev/null+++ b/internal/parser/parser/pdf_positions.go@@ -0,0 +1,94 @@+package parser++import "encoding/json"++// This file owns PDF position decoding. The parser boundary, the vision+// cropper and the chunker all read PDF positions through these helpers, so+// they agree on what a position row is and on which wire shapes are+// supported.+//+// Supported shapes:+// - a matrix of 5-tuples: [][]any, []any of rows, [][]float64;+// - one flat 5-tuple: []float64, the legacy wire form some engines (TCADP)+// still emit.+//+// A row is [page, left, right, top, bottom]. The page is 1-based.++// PDFPositionRows decodes the position rows carried by raw. Rows that are not+// a 5-tuple are skipped; a flat tuple counts as one row.+func PDFPositionRows(raw any) [][]any {+ switch v := raw.(type) {+ case [][]any:+ return filterPositionRows(v, func(row []any) []any { return row })+ case []any:+ out := make([][]any, 0, len(v))+ for _, rawRow := range v {+ switch row := rawRow.(type) {+ case []any:+ if len(row) == 5 {+ out = append(out, row)+ }+ case []float64:+ if len(row) == 5 {+ out = append(out, floatRowToAny(row))+ }+ }+ }+ return out+ case [][]float64:+ out := make([][]any, 0, len(v))+ for _, row := range v {+ if len(row) == 5 {+ out = append(out, floatRowToAny(row))+ }+ }+ return out+ case []float64:+ if len(v) == 5 {+ return [][]any{floatRowToAny(v)}+ }+ }+ return nil+}++// PDFPositionRowsFromJSON decodes a PDF position matrix carried as raw JSON.+// It accepts the matrix form and the legacy flat 5-tuple and returns rows of+// five floats. Rows shorter than five values are skipped.+func PDFPositionRowsFromJSON(raw json.RawMessage) [][]float64 {+ if len(raw) == 0 {+ return nil+ }+ var matrix [][]float64+ if err := json.Unmarshal(raw, &matrix); err == nil {+ out := make([][]float64, 0, len(matrix))+ for _, row := range matrix {+ if len(row) >= 5 {+ out = append(out, row)+ }+ }+ return out+ }+ var flat []float64+ if err := json.Unmarshal(raw, &flat); err == nil && len(flat) >= 5 {+ return [][]float64{flat}+ }+ return nil+}++func filterPositionRows(rows [][]any, keep func([]any) []any) [][]any {+ out := make([][]any, 0, len(rows))+ for _, row := range rows {+ if len(row) == 5 {+ out = append(out, keep(row))+ }+ }+ return out+}++func floatRowToAny(row []float64) []any {+ out := make([]any, len(row))+ for i, v := range row {+ out[i] = v+ }+ return out+}diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.gonew file mode 100644index 0000000000..4cc77653cf--- /dev/null+++ b/internal/parser/parser/pdf_result.go@@ -0,0 +1,424 @@+package parser++import (+ "context"+ "maps"+ "math"+ "strings"++ "go.uber.org/zap"+ "golang.org/x/net/html"++ "ragflow/internal/common"+ "ragflow/internal/deepdoc/parser/pdf/util"+ deepdoctype "ragflow/internal/deepdoc/parser/type"+ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil"+)++// NormalizeResult converts a PDF backend response to JSON and applies common+// item policies. Producers own page-base and coordinate-unit conversion.+func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result ParseResult) ParseResult {+ if result.Err != nil {+ return result+ }+ if err := ctx.Err(); err != nil {+ result.Err = err+ result.File = ensurePDFResultFile(result.File, filename)+ return result+ }+ if len(result.JSON) == 0 {+ switch {+ case result.Markdown != "":+ parsed := parsePDFMarkdown(ctx, filename, result.Markdown)+ if parsed.Err != nil {+ if ctx.Err() != nil {+ // Cancellation is fatal; preserve file metadata for callers.+ parsed.File = ensurePDFResultFile(result.File, filename)+ return parsed+ }+ // Mirror the historical markdown→JSON fallback: a backend+ // markdown payload that cannot be structured is still indexed+ // as raw text instead of failing the task.+ common.Warn("parser normalization fell back to text",+ zap.String("filename", filename),+ zap.String("normalized_from", "markdown"),+ zap.Error(parsed.Err))+ result.Warnings = append(result.Warnings, parsed.Warnings...)+ result.Warnings = append(result.Warnings, "markdown normalization fell back to text: "+parsed.Err.Error())+ result.JSON = []DocItem{NewTextJSONItem(result.Markdown)}+ } else {+ result.Warnings = append(result.Warnings, parsed.Warnings...)+ if len(parsed.JSON) == 0 {+ result.JSON = []DocItem{NewTextJSONItem(result.Markdown)}+ } else {+ result.JSON = parsed.JSON+ }+ }+ case result.Text != "":+ result.JSON = []DocItem{NewTextJSONItem(result.Text)}+ }+ }+ items := make([]DocItem, 0, len(result.JSON))+ droppedPositions := 0+ for _, item := range result.JSON {+ if err := ctx.Err(); err != nil {+ result.Err = err+ result.File = ensurePDFResultFile(result.File, filename)+ return result+ }+ if item == nil {+ continue+ }+ fields := maps.Clone(item.ToMap())+ if fields == nil {+ continue+ }+ layout, _ := fields["layout"].(string)+ if strings.TrimSpace(layout) == "" {+ layout, _ = fields["layout_type"].(string)+ }+ if layout = strings.ToLower(strings.TrimSpace(layout)); layout != "" {+ fields["layout"] = layout+ if _, ok := fields["layout_type"]; ok {+ fields["layout_type"] = layout+ }+ }+ rawPrimary := fields["_pdf_positions"]+ positions := validPDFPositions(rawPrimary)+ droppedPositions += countPDFPositionRows(rawPrimary) - len(positions)+ if len(positions) == 0 {+ rawFallback := fields["positions"]+ positions = validPDFPositions(rawFallback)+ droppedPositions += countPDFPositionRows(rawFallback) - len(positions)+ }+ delete(fields, "_pdf_positions")+ delete(fields, "positions")+ if len(positions) > 0 {+ fields["_pdf_positions"], fields["positions"] = positions, positions+ if _, ok := fields["page_number"]; !ok {+ fields["page_number"] = positions[0][0]+ }+ }+ normalizePDFDocType(fields)+ projectPDFTable(fields)+ if table, ok := fields["table"].(*entity.TableData); ok && table != nil {+ if page := int(numberValue(fields["page_number"])); page > 0 && page != table.Page {+ normalizedTable := *table+ normalizedTable.Page = page+ fields["table"] = &normalizedTable+ }+ } else if fields["doc_type_kwd"] == "table" && len(positions) == 0 {+ if text, _ := fields["text"].(string); text != "" {+ fields["doc_type_kwd"] = "text"+ }+ }+ if p.FlattenMediaToText {+ // Flattened output keeps one text carrier per item. Structured+ // tables (MinerU content_list, projected HTML) carry cells in+ // `table` and footnotes in `text`; rendering the table preserves+ // cells, caption, and footnote in that order. The render+ // normalizes markup (header inference, escaping, whitespace), so+ // it is not byte-identical to the source HTML, but it is+ // content-preserving. Items without a table keep their text verbatim.+ if table, ok := fields["table"].(*entity.TableData); ok && table != nil {+ text, _ := fields["text"].(string)+ fields["text"] = strings.TrimSpace(tableutil.RenderTableHTML(table) + "\n" + text)+ delete(fields, "table")+ }+ fields["doc_type_kwd"] = "text"+ }+ // ck_type is retired: the content type lives on doc_type_kwd and the+ // block kind on layout. A legacy raw item can still carry the key, so+ // drop it here as well — the typed conversion already does.+ delete(fields, "ck_type")+ items = appendPDFItem(items, fields)+ }+ if droppedPositions > 0 {+ // Zero-area or otherwise invalid boxes are dropped so they never reach+ // the on-demand cropper. Log the count so a parser that stops emitting+ // regions is distinguishable from one that never emitted any.+ common.Warn("PDF normalization dropped invalid positions",+ zap.String("filename", filename),+ zap.Int("dropped_rows", droppedPositions))+ }+ result.JSON = items+ result.OutputFormat, result.Markdown, result.Text = "json", "", ""+ result.File = ensurePDFResultFile(result.File, filename)+ return result+}++// ensurePDFResultFile preserves backend file metadata while guaranteeing the+// filename is present, including on error paths where callers still need to+// report which document failed.+func ensurePDFResultFile(file map[string]any, filename string) map[string]any {+ file = maps.Clone(file)+ if file == nil {+ file = map[string]any{}+ }+ if _, ok := file["name"]; !ok {+ file["name"] = filename+ }+ return file+}++// NewTableNoteItem builds the text item that carries a table's description.+// The note keeps the table's page identity and ordering hint so downstream+// chunking preserves document order. Shared by NormalizeResult's table+// splitting and the vision enhancement path.+func NewTableNoteItem(fields map[string]any, text string) TextItem {+ note := TextItem{Text: text, Extra: map[string]any{"layout": "text"}}+ for _, key := range []string{"page_number", "_preserve_order"} {+ if value, ok := fields[key]; ok {+ note.Extra[key] = value+ }+ }+ return note+}++// appendPDFItem emits adjacent tables separately and keeps their descriptions+// in text items. Chunkers materialize a table only when its text field is empty.+// The isTableHTML branch handles one text carrier holding two adjacent tables+// plus a trailing note (projectPDFTable only projects the leading table, so the+// remainder can itself start with a table). Covered by+// TestPDFNormalizeResultPreservesAdjacentTables.+func appendPDFItem(items []DocItem, fields map[string]any) []DocItem {+ for {+ table, _ := fields["table"].(*entity.TableData)+ text, _ := fields["text"].(string)+ if fields["doc_type_kwd"] != "table" || table == nil || text == "" {+ return append(items, DocItemFromMap(fields))+ }+ fields = maps.Clone(fields)+ delete(fields, "text")+ items = append(items, DocItemFromMap(fields))+ if strings.TrimSpace(text) == "" {+ return items+ }+ if isTableHTML(text) {+ next := maps.Clone(fields)+ delete(next, "table")+ next["text"] = text+ projectPDFTable(next)+ if table, _ := next["table"].(*entity.TableData); table != nil {+ fields = next+ continue+ }+ }+ note := NewTableNoteItem(fields, text)+ return append(items, note)+ }+}++func parsePDFMarkdown(ctx context.Context, filename, markdown string) ParseResult {+ p, err := NewMarkdownParser(GoMarkdown)+ if err != nil {+ return ParseResult{Err: err}+ }+ p.FetchRemoteImages = false+ return p.ParseWithResult(ctx, filename, []byte(markdown))+}++// validPDFPositions validates canonical 1-based PDF positions without changing+// page identity or units. Invalid regions must never reach the cropper.+//+// Every wire shape is decoded through PDFPositionRows: a matrix of 5-tuples+// ([][]any, []any of rows, [][]float64) and the legacy flat 5-tuple+// ([]float64) some engines still emit.+//+// A page-only tuple — a page number with all-zero geometry — is valid: it+// carries page identity for the indexer but no crop region, so the crop+// consumers skip it (see HasPDFCropRegion). A box with real coordinates that is+// zero-area or inverted is invalid and is dropped; the caller logs the dropped+// count so the loss is observable instead of silent.+func validPDFPositions(raw any) [][]any {+ var result [][]any+ for _, row := range PDFPositionRows(raw) {+ var values [5]float64+ valid := true+ for i, value := range row {+ v, ok := numericAny(value)+ if !ok || math.IsNaN(v) || math.IsInf(v, 0) {+ valid = false+ break+ }+ values[i] = v+ }+ page := values[0]+ if !valid || page < 1 || page >= float64(^uint(0)>>1) || math.Trunc(page) != page {+ continue+ }+ if !pdfPositionIsUsable(values) {+ continue+ }+ result = append(result, []any{int(page), values[1], values[2], values[3], values[4]})+ }+ return result+}++// pdfPositionIsPageOnly reports whether a position carries page identity only:+// a page number with all-zero geometry and therefore no crop region.+func pdfPositionIsPageOnly(values [5]float64) bool {+ return values[1] == 0 && values[2] == 0 && values[3] == 0 && values[4] == 0+}++// pdfPositionIsUsable reports whether a position may be kept. A page-only+// tuple carries page identity and is kept; every other box must have a+// positive width and height, so a zero-area or inverted region is dropped.+func pdfPositionIsUsable(values [5]float64) bool {+ if pdfPositionIsPageOnly(values) {+ return true+ }+ return values[2] > values[1] && values[4] > values[3]+}++// pdfPositionHasRegion reports whether a position carries a usable crop region.+// A page-only tuple carries page identity only, so it is not a region.+func pdfPositionHasRegion(values [5]float64) bool {+ if pdfPositionIsPageOnly(values) {+ return false+ }+ return values[2] > values[1] && values[4] > values[3]+}++// HasPDFCropRegion reports whether the item carries at least one PDF crop+// region, i.e. a position with a positive-area box. A page-only position+// carries page identity but no region, so it must not trigger a crop.+//+// The crop consumers (the VLM vision path and the chunker) consult this+// predicate so a page-only position never reaches the cropper, while the+// indexer still receives the page identity through the positions matrix. The+// geometry is decoded exactly as the cropper decodes it, so the two agree on+// what a usable region is.+func HasPDFCropRegion(item map[string]any) bool {+ matrix, ok := ExtractPDFPositions(item)+ if !ok {+ return false+ }+ for _, pos := range util.PositionsFromMatrix(matrix) {+ if pos.Right > pos.Left && pos.Bottom > pos.Top {+ return true+ }+ }+ return false+}++// countPDFPositionRows counts candidate position rows before validation so the+// caller can report how many were dropped. It uses the same decoder as+// validPDFPositions, so the count matches the rows validation sees.+func countPDFPositionRows(raw any) int {+ return len(PDFPositionRows(raw))+}++// splitPDFTable separates the leading table from any following description.+// Tokenization handles nested tables without discarding the trailing text.+func splitPDFTable(text string) (string, string) {+ text = strings.TrimSpace(text)+ if !isTableHTML(text) {+ return "", text+ }+ z := html.NewTokenizer(strings.NewReader(text))+ depth, offset := 0, 0+ for {+ kind := z.Next()+ if kind == html.ErrorToken {+ if depth > 0 {+ return text, ""+ }+ return "", text+ }+ offset += len(z.Raw())+ if kind != html.StartTagToken && kind != html.EndTagToken {+ continue+ }+ name, _ := z.TagName()+ if string(name) != "table" {+ continue+ }+ if kind == html.StartTagToken {+ depth+++ } else {+ depth--+ }+ if depth == 0 {+ return text[:offset], strings.TrimSpace(text[offset:])+ }+ }+}++func normalizePDFDocType(item map[string]any) {+ if item == nil {+ return+ }+ layoutType, _ := item["layout_type"].(string)+ if layoutType == "" {+ layoutType, _ = item["layout"].(string)+ }+ // A figure caption is a media section even when the parser no longer+ // inlines its cropped image (cgo): it still carries PDF positions, so the+ // downstream VLM/chunker crop it on demand. Classify it as image whenever+ // it has positions (the inlined image was only a side effect of cropping).+ _, hasMedia := ExtractPDFPositions(item)+ if docType, _ := item["doc_type_kwd"].(string); docType == "text" || docType == "image" || docType == "table" {+ // A figure caption keeps its media classification so the downstream+ // VLM enhancement and on-demand chunker crop it.+ if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption && hasMedia {+ item["doc_type_kwd"] = "image"+ }+ return+ }+ if table, ok := item["table"].(*entity.TableData); ok && table != nil {+ item["doc_type_kwd"] = "table"+ return+ }+ switch layoutType {+ case "table":+ item["doc_type_kwd"] = "table"+ case "figure", "image":+ item["doc_type_kwd"] = "image"+ case deepdoctype.DLALabelFigureCaption:+ if hasMedia {+ item["doc_type_kwd"] = "image"+ } else {+ item["doc_type_kwd"] = "text"+ }+ default:+ if img, _ := item["image"].(string); img != "" {+ item["doc_type_kwd"] = "image"+ return+ }+ item["doc_type_kwd"] = "text"+ }+}++// projectPDFTable projects a leading HTML table into TableData, retaining any+// following description and leaving existing structured tables untouched.+func projectPDFTable(item map[string]any) {+ if item == nil {+ return+ }+ if dt, _ := item["doc_type_kwd"].(string); dt != "table" {+ return+ }+ if item["table"] != nil {+ return+ }+ text, _ := item["text"].(string)+ markup, notes := splitPDFTable(text)+ if markup == "" {+ return+ }+ td, err := tableutil.ParseTableHTML(markup)+ if err != nil || td == nil {+ return+ }+ if page := numberValue(item["page_number"]); page > 0 {+ td.Page = int(page)+ }+ item["table"] = td+ if notes == "" {+ delete(item, "text")+ } else {+ item["text"] = notes+ }+}diff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.gonew file mode 100644index 0000000000..d2b00d4812--- /dev/null+++ b/internal/parser/parser/pdf_result_test.go@@ -0,0 +1,520 @@+package parser++import (+ "context"+ "errors"+ "fmt"+ "math"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"++ "ragflow/internal/entity"+)++func TestPDFNormalizeResultPositionsAndMetadata(t *testing.T) {+ file := map[string]any{"outline": []string{"Chapter"}, "page_count": 20, "parse_method": "selected-model"}+ input := map[string]any{"text": "caption", "layout": "caption", "doc_type_kwd": "text", "page_number": 2,+ "positions": []any{[]any{2.0, 10.0, 30.0, 20.0, 40.0}, []any{3.0, math.NaN(), 30.0, 20.0, 40.0}, []any{4, 0, 0, 0, 0}}}+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", File: file, Warnings: []string{"source warning"}, JSON: MapsToItems([]map[string]any{input, {"text": "", "page_number": 4, "positions": []float64{4, 0, 0, 0, 0}}})})+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ item := result.JSON[0].ToMap()+ positions, ok := ExtractPDFPositions(item)+ if !ok || len(positions) != 2 || positions[0][0] != 2 || item["doc_type_kwd"] != "text" {+ t.Fatalf("positions/type = %#v", item)+ }+ // The NaN row is dropped; the page-only row survives as page identity.+ if positions[1][0] != 4 {+ t.Fatalf("page-only position lost: %#v", positions)+ }+ if len(result.JSON) != 2 || result.JSON[1].ToMap()["page_number"] != 4 {+ t.Fatalf("blank page lost: %#v", result.JSON)+ }+ if !HasPDFCropRegion(item) {+ t.Fatal("real crop region dropped")+ }+ if result.File["parse_method"] != "selected-model" || result.File["page_count"] != 20 || len(result.Warnings) != 1 {+ t.Fatalf("metadata lost: %+v", result)+ }+ if _, ok := input["_pdf_positions"]; ok {+ t.Fatal("input mutated")+ }+}++func TestPDFNormalizeResultMarkdownAndCancellation(t *testing.T) {+ previous := ssrfAllowLoopback+ ssrfAllowLoopback = true+ t.Cleanup(func() { ssrfAllowLoopback = previous })+ requests := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests++ }))+ defer server.Close()+ result := NewPDFParser().NormalizeResult(t.Context(), "ocr.pdf", ParseResult{OutputFormat: "markdown", Markdown: "| A | B |\n| --- | --- |\n| one | two |\n\n"})+ if result.Err != nil || result.OutputFormat != "json" || result.Markdown != "" || requests != 0 {+ t.Fatalf("result=%+v requests=%d", result, requests)+ }+ var table *entity.TableData+ for _, item := range result.JSON {+ if td, ok := item.ToMap()["table"].(*entity.TableData); ok {+ table = td+ }+ }+ if table == nil || len(table.CellTexts()) < 2 || len(table.CellTexts()[1]) == 0 || table.CellTexts()[1][0] != "one" {+ t.Fatalf("markdown table lost: %#v", result.JSON)+ }+ ctx, cancel := context.WithCancel(t.Context())+ cancel()+ result = NewPDFParser().NormalizeResult(ctx, "ocr.pdf", ParseResult{OutputFormat: "markdown", Markdown: "body"})+ if !errors.Is(result.Err, context.Canceled) {+ t.Fatalf("cancellation = %+v", result)+ }+ if result.File["name"] != "ocr.pdf" {+ t.Fatalf("cancellation lost file metadata: %+v", result)+ }+}++func TestPDFMarkdownBackendDoesNotFetchRemoteImages(t *testing.T) {+ withSSRFBypass(t)+ previous := ssrfAllowLoopback+ ssrfAllowLoopback = true+ t.Cleanup(func() { ssrfAllowLoopback = previous })+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(http.StatusNotFound) }))+ defer server.Close()+ result := pdfMarkdownToResult(t.Context(), "remote.pdf", "", "json", 1)+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ if requests.Load() != 0 {+ t.Fatal("PDF Markdown backend normalization fetched a remote image")+ }+}++func TestPDFNormalizeResultKeepsTCADPPageWithoutCrop(t *testing.T) {+ items := tcadpAnyToItems(map[string]any{"type": "text", "content": "page text", "page_num": 7})+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ item := result.JSON[0].ToMap()+ if item["page_number"] != 7 {+ t.Fatalf("page identity lost: %#v", item)+ }+ // The page-only tuple survives as page identity but is not a crop region.+ positions, ok := ExtractPDFPositions(item)+ if !ok || len(positions) != 1 || positions[0][0] != 7 {+ t.Fatalf("page-only position lost: %#v", item)+ }+ if HasPDFCropRegion(item) {+ t.Fatalf("unknown region exposed for cropping: %#v", item)+ }+}++func TestPDFNormalizeResultInfersStructuredTable(t *testing.T) {+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}}+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{"table": table, "page_number": 3}})})+ item := result.JSON[0].ToMap()+ if item["doc_type_kwd"] != "table" {+ t.Fatalf("table classified incorrectly: %#v", item)+ }+ if table.Page != 0 {+ t.Fatal("input table mutated")+ }+}++func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testing.T) {+ p := NewPDFParser()+ p.FlattenMediaToText = true+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}, Caption: "Caption"}+ result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: []DocItem{+ TableItem{Table: table, Extra: map[string]any{"text": "Footnote"}},+ ImageItem{Text: "Figure", Image: "image-data", Positions: Positions{AbsoluteBox{PageNumber: 2, Left: 1, Right: 5, Top: 2, Bottom: 6}}},+ TextItem{Text: "Title", Extra: map[string]any{"layout": "heading"}},+ }})+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ for i, item := range result.JSON[:2] {+ fields := item.ToMap()+ if fields["doc_type_kwd"] != "text" {+ t.Fatalf("flattened item %d still has media type: %#v", i, fields)+ }+ if _, ok := fields["ck_type"]; ok {+ t.Fatalf("flattened item %d still carries the retired ck_type: %#v", i, fields)+ }+ }+ text, _ := result.JSON[0].ToMap()["text"].(string)+ for _, want := range []string{"Cell", "Caption", "Footnote"} {+ if !strings.Contains(text, want) {+ t.Fatalf("missing %q in %q", want, text)+ }+ }+ if result.JSON[1].ToMap()["image"] != "image-data" {+ t.Fatal("flatten lost image")+ }+ if _, ok := ExtractPDFPositions(result.JSON[1].ToMap()); !ok {+ t.Fatal("flatten lost positions")+ }+ if result.JSON[2].ToMap()["layout"] != "heading" {+ t.Fatal("heading layout changed")+ }+}++func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{+ "doc_type_kwd": "table", "ck_type": "table", "layout": "table", "text": "Recognized text without cells",+ }})})+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ item := result.JSON[0].ToMap()+ if item["doc_type_kwd"] != "text" || item["text"] != "Recognized text without cells" {+ t.Fatalf("unstructured table kept an inconsistent subtype: %#v", item)+ }+ if _, ok := item["ck_type"]; ok {+ t.Fatalf("retired ck_type survived normalization: %#v", item)+ }+}++func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {+ for _, flatten := range []bool{false, true} {+ p := NewPDFParser()+ p.FlattenMediaToText = flatten+ result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{+ "doc_type_kwd": "table", "ck_type": "table", "text": "Recognized table without cells",+ "positions": [][]any{{2, 10, 30, 20, 40}},+ }})})+ if result.Err != nil || len(result.JSON) != 1 {+ t.Fatalf("unexpected result: %+v", result)+ }+ item := result.JSON[0].ToMap()+ want := "table"+ if flatten {+ want = "text"+ }+ if item["doc_type_kwd"] != want || item["text"] != "Recognized table without cells" {+ t.Fatalf("flatten=%v: table media identity or text lost: %#v", flatten, item)+ }+ if _, ok := ExtractPDFPositions(item); !ok {+ t.Fatalf("flatten=%v: table crop region lost: %#v", flatten, item)+ }+ }+}++func TestPDFNormalizeResultRecoversUnclosedTableCells(t *testing.T) {+ for _, markup := range []string{+ "<table><tr><td>Cell value</td></tr>",+ "<table><tr><td>Cell value",+ } {+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{+ "doc_type_kwd": "table", "text": markup,+ }})})+ if result.Err != nil || len(result.JSON) != 1 {+ t.Fatalf("unexpected result: %+v", result)+ }+ item := result.JSON[0].ToMap()+ table, _ := item["table"].(*entity.TableData)+ if table == nil || len(table.CellTexts()) != 1 || len(table.CellTexts()[0]) != 1 || table.CellTexts()[0][0] != "Cell value" {+ t.Fatalf("unclosed markup lost recoverable cells: %#v", item)+ }+ if text, _ := item["text"].(string); text != "" {+ t.Fatalf("recovered table still carries raw HTML: %q", text)+ }+ }+}++func TestPDFParserSoMarkFlattensStructuredMedia(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/parse/async" {+ _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task"}}`))+ return+ }+ _, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"figure","content":"Figure caption"},{"type":"table","content":"<table><tr><td>Cell value</td></tr></table>"}]}]}}}}}`))+ }))+ defer server.Close()+ p := NewPDFParser()+ p.ConfigureFromSetup(map[string]any{"parse_method": "somark", "somark_base_url": server.URL, "flatten_media_to_text": true})+ result := p.ParseWithResult(t.Context(), "sample.pdf", []byte("pdf"))+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ if len(result.JSON) != 2 {+ t.Fatalf("items = %#v", result.JSON)+ }+ for i, want := range []string{"Figure caption", "Cell value"} {+ item := result.JSON[i].ToMap()+ text, _ := item["text"].(string)+ if item["doc_type_kwd"] != "text" || !strings.Contains(text, want) {+ t.Fatalf("item %d = %#v; want text containing %q", i, item, want)+ }+ }+}++func TestProjectPDFTablePreservesExistingStructureAndNotes(t *testing.T) {+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Original cell"}}}}+ item := map[string]any{"doc_type_kwd": "table", "table": table, "text": "<table><tr><td>Other</td></tr></table>\nFootnote"}+ projectPDFTable(item)+ text, _ := item["text"].(string)+ if item["table"] != table || !strings.Contains(text, "Footnote") {+ t.Fatalf("existing table or notes overwritten: %#v", item)+ }+ mixed := map[string]any{"doc_type_kwd": "table", "text": "<table><tr><td>Cell</td></tr></table>\nFootnote"}+ projectPDFTable(mixed)+ text, _ = mixed["text"].(string)+ if !strings.Contains(text, "Footnote") {+ t.Fatalf("notes lost: %#v", mixed)+ }+}++func TestPDFTableDescriptionUsesTextItemForBothOutputs(t *testing.T) {+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}+ item := map[string]any{"table": table, "doc_type_kwd": "table", "text": "Footnote", "page_number": 3}+ for _, format := range []string{"json", "markdown"} {+ result := pdfItemsToResult("sample.pdf", []map[string]any{item}, format, 3)+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ if format == "json" {+ result = NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", result)+ if len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != nil || result.JSON[1].ToMap()["text"] != "Footnote" || result.JSON[1].ToMap()["page_number"] != 3 {+ t.Fatalf("table description not separated: %#v", result.JSON)+ }+ } else if !strings.Contains(result.Markdown, "Cell value") || !strings.Contains(result.Markdown, "Footnote") {+ t.Fatalf("table description lost: %s", result.Markdown)+ }+ }+ if item["text"] != "Footnote" {+ t.Fatal("source item mutated")+ }+}++func TestPDFWhitespaceTableDescriptionCannotHideCells(t *testing.T) {+ item := map[string]any{"table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, "doc_type_kwd": "table", "text": " \n "}+ results := []ParseResult{+ NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 1)),+ NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{item})}),+ }+ for _, result := range results {+ if result.Err != nil || len(result.JSON) != 1 {+ t.Fatalf("unexpected result: %+v", result)+ }+ if text, _ := result.JSON[0].ToMap()["text"].(string); text != "" {+ t.Fatalf("whitespace still suppresses table materialization: %q", text)+ }+ }+}++func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) {+ const markup = "<table><tr><td>First cell</td></tr></table>\n<table><tr><td>Second cell</td></tr></table>\nFootnote"+ for _, projected := range []bool{false, true} {+ for _, flatten := range []bool{false, true} {+ name := fmt.Sprintf("projected=%v/flatten=%v", projected, flatten)+ t.Run(name, func(t *testing.T) {+ item := map[string]any{+ "doc_type_kwd": "table", "text": markup, "page_number": 3,+ "positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,+ }+ if projected {+ projectPDFTable(item)+ }+ originalText := item["text"]+ p := NewPDFParser()+ p.FlattenMediaToText = flatten+ result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ if flatten {+ var text string+ for _, doc := range result.JSON {+ fields := doc.ToMap()+ if fields["doc_type_kwd"] != "text" || fields["table"] != nil {+ t.Fatalf("media not flattened: %#v", fields)+ }+ value, _ := fields["text"].(string)+ text += value+ }+ for _, want := range []string{"First cell", "Second cell", "Footnote"} {+ if strings.Count(text, want) != 1 {+ t.Fatalf("want %q exactly once in %q", want, text)+ }+ }+ } else {+ if len(result.JSON) != 3 {+ t.Fatalf("want two tables and a footnote, got %#v", result.JSON)+ }+ for i, want := range []string{"First cell", "Second cell"} {+ fields := result.JSON[i].ToMap()+ table, _ := fields["table"].(*entity.TableData)+ if table == nil || table.Page != 3 || len(table.Grid) != 1 || len(table.Grid[0]) != 1 || table.Grid[0][0].Text != want || fields["text"] != nil {+ t.Fatalf("table %d lost cells or metadata: %#v", i, fields)+ }+ if _, ok := ExtractPDFPositions(fields); !ok {+ t.Fatalf("table %d lost crop positions", i)+ }+ }+ note := result.JSON[2].ToMap()+ if note["text"] != "Footnote" || note["doc_type_kwd"] != "text" || note["page_number"] != 3 || note["_preserve_order"] != true {+ t.Fatalf("footnote lost text or metadata: %#v", note)+ }+ }+ if item["text"] != originalText {+ t.Fatal("source item mutated")+ }+ })+ }+ }+}++func TestPDFNormalizeResultIsIdempotent(t *testing.T) {+ const markup = "<table><tr><td>First cell</td></tr></table>\n<table><tr><td>Second cell</td></tr></table>\nFootnote"+ for _, flatten := range []bool{false, true} {+ p := NewPDFParser()+ p.FlattenMediaToText = flatten+ item := map[string]any{+ "doc_type_kwd": "table", "text": markup, "page_number": 3,+ "positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,+ }+ once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))+ if once.Err != nil {+ t.Fatal(once.Err)+ }+ twice := p.NormalizeResult(t.Context(), "sample.pdf", once)+ if twice.Err != nil {+ t.Fatal(twice.Err)+ }+ if len(once.JSON) != len(twice.JSON) {+ t.Fatalf("flatten=%v: second normalize changed item count %d -> %d", flatten, len(once.JSON), len(twice.JSON))+ }+ for i := range once.JSON {+ a, b := once.JSON[i].ToMap(), twice.JSON[i].ToMap()+ if a["doc_type_kwd"] != b["doc_type_kwd"] {+ t.Fatalf("flatten=%v: item %d type changed %v -> %v", flatten, i, a["doc_type_kwd"], b["doc_type_kwd"])+ }+ ta, _ := a["text"].(string)+ tb, _ := b["text"].(string)+ if ta != tb {+ t.Fatalf("flatten=%v: item %d text changed %q -> %q", flatten, i, ta, tb)+ }+ }+ }+}++// TestPDFNormalizeResultKeepsTCADPPageOnlyPosition covers the TCADP wire+// form: a flat []float64 page-only tuple. Normalization must keep the page+// identity (the indexer derives page_num_int from positions) while the tuple+// stays out of the crop path.+func TestPDFNormalizeResultKeepsTCADPPageOnlyPosition(t *testing.T) {+ items := tcadpAnyToItems(map[string]any{"type": "text", "content": "page text", "page_num": 7})+ if len(items) != 1 {+ t.Fatalf("items = %#v", items)+ }+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ item := result.JSON[0].ToMap()+ if item["page_number"] != 7 {+ t.Fatalf("page identity lost: %#v", item)+ }+ positions, ok := ExtractPDFPositions(item)+ if !ok || len(positions) != 1 {+ t.Fatalf("page-only position dropped: %#v", item)+ }+ if positions[0][0] != 7 {+ t.Fatalf("page-only position lost its page: %#v", positions)+ }+ if HasPDFCropRegion(item) {+ t.Fatalf("page-only position exposed as a crop region: %#v", item)+ }+}++func TestValidPDFPositionsAcceptsFlatTupleAndDropsBadBoxes(t *testing.T) {+ flat := validPDFPositions([]float64{7, 0, 0, 0, 0})+ if len(flat) != 1 || flat[0][0] != 7 {+ t.Fatalf("flat tuple dropped: %#v", flat)+ }+ if HasPDFCropRegion(map[string]any{"positions": flat}) {+ t.Fatal("page-only tuple exposed as a crop region")+ }+ degenerate := validPDFPositions([][]any{{1, 10, 10, 5, 5}})+ if len(degenerate) != 0 {+ t.Fatalf("zero-area box kept: %#v", degenerate)+ }+ region := validPDFPositions([][]any{{1, 10, 30, 5, 25}})+ if len(region) != 1 {+ t.Fatalf("valid region dropped: %#v", region)+ }+ if !HasPDFCropRegion(map[string]any{"positions": region}) {+ t.Fatal("valid region not exposed as a crop region")+ }+}++// TestHasPDFCropRegionReadsEveryRowAndShape covers the crop-region contract:+// every row of the matrix counts, and the legacy flat tuple is understood.+func TestHasPDFCropRegionReadsEveryRowAndShape(t *testing.T) {+ cases := []struct {+ name string+ item map[string]any+ want bool+ }{+ {"page-only row before a real region", map[string]any{"positions": [][]any{{1, 0, 0, 0, 0}, {2, 10, 100, 10, 100}}}, true},+ {"real region before a page-only row", map[string]any{"positions": [][]any{{1, 10, 100, 10, 100}, {2, 0, 0, 0, 0}}}, true},+ {"only page-only rows", map[string]any{"positions": [][]any{{1, 0, 0, 0, 0}}}, false},+ {"flat tuple with a region", map[string]any{"positions": []float64{1, 10, 100, 10, 100}}, true},+ {"flat page-only tuple", map[string]any{"positions": []float64{1, 0, 0, 0, 0}}, false},+ {"no positions", map[string]any{"positions": []any{}}, false},+ }+ for _, tc := range cases {+ if got := HasPDFCropRegion(tc.item); got != tc.want {+ t.Errorf("%s: HasPDFCropRegion = %v, want %v", tc.name, got, tc.want)+ }+ }+}++func TestCountPDFPositionRows(t *testing.T) {+ if got := countPDFPositionRows(nil); got != 0 {+ t.Fatalf("nil = %d, want 0", got)+ }+ if got := countPDFPositionRows([][]any{{1, 0, 1, 0, 1}, {2, 0, 1, 0, 1}}); got != 2 {+ t.Fatalf("[][]any = %d, want 2", got)+ }+ if got := countPDFPositionRows([]any{[]any{1, 0, 1, 0, 1}}); got != 1 {+ t.Fatalf("[]any = %d, want 1", got)+ }+ if got := countPDFPositionRows("bad"); got != 0 {+ t.Fatalf("bad type = %d, want 0", got)+ }+}++func TestPDFNormalizeResultCancellationPreservesFile(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ cancel()+ result := NewPDFParser().NormalizeResult(ctx, "cancel.pdf", ParseResult{+ OutputFormat: "json",+ File: map[string]any{"parse_method": "mineru"},+ JSON: MapsToItems([]map[string]any{{"text": "hi", "doc_type_kwd": "text"}}),+ })+ if !errors.Is(result.Err, context.Canceled) {+ t.Fatalf("err = %v", result.Err)+ }+ if result.File["name"] != "cancel.pdf" || result.File["parse_method"] != "mineru" {+ t.Fatalf("file lost: %+v", result.File)+ }+}++func TestPDFMarkdownToResultPreservesFileOnCancel(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ cancel()+ result := pdfMarkdownToResult(ctx, "cancel.pdf", "body", "json", 3)+ if !errors.Is(result.Err, context.Canceled) {+ t.Fatalf("err = %v", result.Err)+ }+ if result.File["name"] != "cancel.pdf" || result.File["page_count"] != 3 {+ t.Fatalf("file lost: %+v", result.File)+ }+}