Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): unify PDF JSON post-processing (#21228)

jay7772132f0114c9598
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go
index 912fccc09a..a88dba6da3 100644
--- a/internal/ingestion/component/chunker/manual.go
+++ b/internal/ingestion/component/chunker/manual.go
@@ -26,6 +26,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/globals"
+ "ragflow/internal/parser/parser"
"gorm.io/gorm"
)
@@ -169,6 +170,10 @@ func sortRecordsByPosition(records []lineRecord) {
// structured `_pdf_positions` key first and falls back to the legacy
// `positions` key. Callers (pdfPosRowLess) index offsets 0/1/3, so we enforce
// len(row) >= 5 here.
+//
+// The matrix is decoded through parser.PDFPositionRowsFromJSON, so the matrix
+// form and the legacy flat 5-tuple are both understood and the chunker shares
+// one position decoder with the parser boundary and the crop path.
func firstPositionRow(r lineRecord) ([]float64, bool) {
var raw json.RawMessage
switch {
@@ -179,15 +184,11 @@ func firstPositionRow(r lineRecord) ([]float64, bool) {
default:
return nil, false
}
- var mat [][]float64
- if err := json.Unmarshal(raw, &mat); err != nil || len(mat) == 0 {
- return nil, false
- }
- row := mat[0]
- if len(row) < 5 {
+ rows := parser.PDFPositionRowsFromJSON(raw)
+ if len(rows) == 0 {
return nil, false
}
- return row, true
+ return rows[0], true
}
// init registers ManualChunker under CategoryIngestion.
diff --git a/internal/ingestion/component/chunker/pdf_postprocess_test.go b/internal/ingestion/component/chunker/pdf_postprocess_test.go
new file mode 100644
index 0000000000..8b993c3116
--- /dev/null
+++ b/internal/ingestion/component/chunker/pdf_postprocess_test.go
@@ -0,0 +1,36 @@
+package chunker
+
+import (
+ "strings"
+ "testing"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/parser"
+)
+
+func TestPDFTableNotesDoNotHideCellsDuringChunking(t *testing.T) {
+ result := parser.NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{
+ parser.TableItem{Table: &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, Extra: map[string]any{"text": "Footnote"}},
+ }})
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{"name": "sample.pdf", "file_type": "pdf", "output_format": "json", "json": result.JSON})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var text string
+ for _, chunk := range out["chunks"].([]map[string]any) {
+ value, _ := chunk["text"].(string)
+ text += value
+ }
+ for _, want := range []string{"Cell value", "Footnote"} {
+ if !strings.Contains(text, want) {
+ t.Fatalf("missing %q in %q", want, text)
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.go
index 2e0b448177..1e4087d8b2 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go
@@ -30,6 +30,7 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"gorm.io/gorm"
)
@@ -284,12 +285,33 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
func needsCrop(ck schema.ChunkDoc) bool {
switch cropChunkType(ck) {
case "image", "table", "text":
- return len(ck.PDFPositions) > 0 || len(ck.Positions) > 0
+ return hasCropRegion(ck)
default:
return false
}
}
+// hasCropRegion reports whether the chunk carries a position with a
+// positive-area box. A page-only position (page number, no geometry) carries
+// page identity for the indexer but no region, so it must not trigger a crop.
+//
+// Every row of the matrix is inspected: mergePositionMatrix sorts page-only
+// rows first, so a page-only first row must not hide a real region in a later
+// row. The matrix is read exactly as the crop path reads it (the canonical
+// _pdf_positions first, the legacy positions key as fallback).
+func hasCropRegion(ck schema.ChunkDoc) bool {
+ raw := ck.PDFPositions
+ if len(raw) == 0 {
+ raw = ck.Positions
+ }
+ for _, row := range parser.PDFPositionRowsFromJSON(raw) {
+ if row[2] > row[1] && row[4] > row[3] {
+ return true
+ }
+ }
+ return false
+}
+
// cropChunkType returns the content type that selects the crop. Callers must
// pass normalized chunks: General/Token clear structural layout before
// cropping, so passing raw parser items would lose their block-kind
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
index 897e8628ef..96af3114df 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
@@ -66,6 +66,16 @@ func jsonPositions(t *testing.T, rows ...[]float64) json.RawMessage {
return b
}
+// jsonFlatPosition encodes one position row in the legacy flat wire form.
+func jsonFlatPosition(t *testing.T, row []float64) json.RawMessage {
+ t.Helper()
+ b, err := json.Marshal(row)
+ if err != nil {
+ t.Fatalf("marshal flat position: %v", err)
+ }
+ return b
+}
+
func TestNeedsCrop(t *testing.T) {
cases := []struct {
name string
@@ -94,6 +104,19 @@ func TestNeedsCrop(t *testing.T) {
// Unknown doc_type values normalize to text (itemDocType), matching
// every other content-type dispatch.
{"unknown doc type normalizes to text", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+
+ // --- Page-only positions (TCADP wire form) carry page identity but
+ // no crop region, so they must not trigger a crop.
+ {"page-only image position", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{3, 0, 0, 0, 0})}, false},
+ {"page-only text position", schema.ChunkDoc{DocType: "text", Positions: jsonPositions(t, []float64{3, 0, 0, 0, 0})}, false},
+ {"zero-area box with coordinates", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{3, 10, 10, 5, 5})}, false},
+
+ // --- A page-only row must not hide a real region in a later row:
+ // mergePositionMatrix sorts page-only rows first.
+ {"page-only row before a real region", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 0, 0, 0, 0}, []float64{2, 10, 100, 10, 100})}, true},
+ {"real region before a page-only row", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100}, []float64{2, 0, 0, 0, 0})}, true},
+ {"flat legacy position with a region", schema.ChunkDoc{DocType: "image", PDFPositions: jsonFlatPosition(t, []float64{1, 10, 100, 10, 100})}, true},
+ {"flat legacy page-only position", schema.ChunkDoc{DocType: "image", PDFPositions: jsonFlatPosition(t, []float64{1, 0, 0, 0, 0})}, false},
}
for _, tc := range cases {
if got := needsCrop(tc.ck); got != tc.want {
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index a66da12a33..fbabfd8449 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -513,6 +513,11 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
// arms are keyed off the utility constants.
//
fileTypeExt := fileTypeFromInputs(inputs)
+ parsePagesOn := pdfPageProgressSupported(fileTypeExt, setups)
+ enhanceOn := c.enableVisionEnhancement
+ split := progress.NewSplit(parsePagesOn, enhanceOn)
+ pageWin := split.Take(parsePagesOn)
+ visionWin := split.Take(enhanceOn)
dispatched, handledVision, visionErr := maybeDispatchPDFVision(ctx, db, fileTypeExt, filename, binary, inputs, setups)
if visionErr != nil {
@@ -546,28 +551,25 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
}
if !handledVision && !handledMedia && !handledImage && !handledAudio {
- parsePagesOn := pdfPageProgressSupported(fileTypeExt, setups)
- enhanceOn := c.enableVisionEnhancement
- split := progress.NewSplit(parsePagesOn, enhanceOn)
- pageWin := split.Take(parsePagesOn)
- visionWin := split.Take(enhanceOn)
-
dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups, pageWin)
if dispatched.Err == nil && ctx.Err() == nil {
pageWin.Report(ctx, 1.0)
}
-
- if c.enableVisionEnhancement {
- // Enhancement is optional; parser-provided text and image metadata
- // remain available if a vision model cannot describe an image.
- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.vision, visionWin)
- }
}
// Known/supported families must fail loudly when dispatch or
// parsing breaks. Only unknown families keep the raw-text fallback.
if dispatched.Err != nil && fileTypeExt != utility.FileTypeOTHER {
return nil, dispatched.Err
}
+ if enhanceOn && !handledMedia && !handledImage && !handledAudio && dispatched.Err == nil && ctx.Err() == nil {
+ // PDF dispatch and ordinary parsers share enhancement after normalization.
+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.vision, visionWin)
+ } else {
+ // Enhancement skipped entirely (disabled or non-PDF media handled):
+ // still strip the transient backend-description marker so it never
+ // reaches the chunker.
+ dispatched = stripBackendDescribedResult(dispatched)
+ }
reportParserWarnings(ctx, dispatched.Warnings)
if err := ctx.Err(); err != nil {
diff --git a/internal/ingestion/component/pdf_postprocess_test.go b/internal/ingestion/component/pdf_postprocess_test.go
new file mode 100644
index 0000000000..5416951833
--- /dev/null
+++ b/internal/ingestion/component/pdf_postprocess_test.go
@@ -0,0 +1,190 @@
+package component
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+ modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/ingestion/component/progress"
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
+ "ragflow/internal/utility"
+)
+
+func TestPDFDispatchNormalizesBeforeOutputsAndEnhancement(t *testing.T) {
+ image := visionTestPNGBase64(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var buf bytes.Buffer
+ archive := zip.NewWriter(&buf)
+ file, _ := archive.Create("sample/sample.json")
+ _ = json.NewEncoder(file).Encode(map[string]any{"layouts": []any{map[string]any{"label": "Text", "content": "![figure](data:image/png;base64," + image + ")"}}})
+ _ = archive.Close()
+ _, _ = w.Write(buf.Bytes())
+ }))
+ defer server.Close()
+ previousResolver := resolveTenantOCRModelByProvider
+ t.Cleanup(func() { resolveTenantOCRModelByProvider = previousResolver })
+ resolveTenantOCRModelByProvider = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return &monkeyOCRv2FakeDriver{}, "monkey", &modelModule.APIConfig{}, 0, nil
+ }
+ setup := schema.ParserSetup{"parse_method": "monkeyocrv2", "monkeyocrv2_server_url": server.URL}
+ result, handled, err := maybeDispatchPDFVision(t.Context(), nil, utility.FileTypePDF, "sample.pdf", []byte("pdf"), nil, map[string]schema.ParserSetup{"pdf": setup})
+ if err != nil || !handled || result.OutputFormat != "json" || len(result.JSON) == 0 {
+ t.Fatalf("dispatch did not emit structured JSON: %+v, %v", result, err)
+ }
+ for _, enabled := range []bool{false, true} {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+ component, err := NewParserComponent(map[string]any{"pdf": map[string]any(setup), "enable_vision_enhancement": enabled})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{"binary": []byte("pdf"), "file_type": "pdf", "name": "sample.pdf", "tenant_id": "tenant-1"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var text string
+ for _, item := range out["json"].([]parser.DocItem) {
+ value, _ := item.ToMap()["text"].(string)
+ text += value
+ }
+ if strings.Contains(text, "a diagram of a pipeline") != enabled {
+ t.Fatalf("enabled=%v output=%s", enabled, text)
+ }
+ want := 0
+ if enabled {
+ want = 1
+ }
+ if len(invoker.images) != want {
+ t.Fatalf("enabled=%v model calls=%d, want %d", enabled, len(invoker.images), want)
+ }
+ }
+}
+
+func TestMinerUTableRetainsCaptionAndFootnote(t *testing.T) {
+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><tr><td>Cell</td></tr></table>", "table_caption": []any{"Caption"}, "table_footnote": []any{"Footnote"}, "page_idx": 1, "bbox": []any{10.0, 20.0, 30.0, 40.0}}, nil)
+ table, ok := item["table"].(*entity.TableData)
+ if !ok || table == nil || table.Caption != "Caption" || len(table.CellTexts()) != 1 || len(table.CellTexts()[0]) != 1 || table.CellTexts()[0][0] != "Cell" || table.Page != 2 {
+ t.Fatalf("structured table missing: %#v", item)
+ }
+ if item["text"] != "Footnote" {
+ t.Fatalf("notes lost or table duplicated: %#v", item)
+ }
+}
+
+func TestMinerUTablePreservesEmbeddedCaption(t *testing.T) {
+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><caption>Embedded caption</caption><tr><td>Cell</td></tr></table>"}, nil)
+ table, ok := item["table"].(*entity.TableData)
+ if !ok || table == nil || table.Caption != "Embedded caption" {
+ t.Fatalf("embedded caption lost: %#v", item)
+ }
+}
+
+func TestPDFTableEnhancementKeepsDescriptionOutsideTableBody(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+ result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{
+ parser.NewTextJSONItem("Before"),
+ parser.TableItem{Table: &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, Extra: map[string]any{"image": visionTestPNGBase64(t), "page_number": 3}},
+ parser.NewTextJSONItem("After"),
+ }}
+ result, handled, err := maybeDispatchVisionEnhancement(t.Context(), nil, utility.FileTypePDF, result, map[string]any{"tenant_id": "tenant-1"}, visionSettings{}, progress.Window{})
+ if err != nil || !handled {
+ t.Fatalf("enhancement failed: %v", err)
+ }
+ if len(result.JSON) != 4 || result.JSON[1].ToMap()["text"] != nil || result.JSON[2].ToMap()["text"] != "a diagram of a pipeline" || result.JSON[2].ToMap()["page_number"] != 3 || result.JSON[3].ToMap()["text"] != "After" {
+ t.Fatalf("table description broke table contract or order: %#v", result.JSON)
+ }
+}
+
+func TestMinerUContentMarksBackendDescribedImages(t *testing.T) {
+ withDesc := mineruContentItemToJSON(map[string]any{
+ "type": "image", "image_caption": []any{"Caption"},
+ "vlm_description": "backend desc",
+ "page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},
+ }, nil)
+ if withDesc == nil {
+ t.Fatal("image item dropped")
+ }
+ if withDesc[backendDescribedKey] != true {
+ t.Fatalf("backend-described image not marked: %#v", withDesc)
+ }
+ if text, _ := withDesc["text"].(string); !strings.Contains(text, "backend desc") {
+ t.Fatalf("backend description lost from text: %q", text)
+ }
+ withoutDesc := mineruContentItemToJSON(map[string]any{
+ "type": "image", "image_caption": []any{"Caption"},
+ "page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},
+ }, nil)
+ if withoutDesc == nil {
+ t.Fatal("image item dropped")
+ }
+ if _, ok := withoutDesc[backendDescribedKey]; ok {
+ t.Fatalf("undescribed image wrongly marked: %#v", withoutDesc)
+ }
+}
+
+func TestVisionEnhancementSkipsBackendDescribedImages(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+ png := "data:image/png;base64," + visionTestPNGBase64(t)
+ marked := parser.DocItemFromMap(map[string]any{
+ "doc_type_kwd": "image", "layout": "figure",
+ "text": "Caption\nbackend desc", "image": png,
+ backendDescribedKey: true,
+ })
+ plain := parser.DocItemFromMap(map[string]any{
+ "doc_type_kwd": "image", "layout": "figure",
+ "text": "Caption", "image": png,
+ })
+ result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{marked, plain}}
+ enhanced, handled, err := maybeDispatchVisionEnhancement(t.Context(), nil, utility.FileTypePDF, result, map[string]any{"tenant_id": "tenant-1"}, visionSettings{}, progress.Window{})
+ if err != nil || !handled {
+ t.Fatalf("enhancement failed: %v", err)
+ }
+ if len(invoker.images) != 1 {
+ t.Fatalf("backend-described image should be skipped: %d VLM calls", len(invoker.images))
+ }
+ if len(enhanced.JSON) != 2 {
+ t.Fatalf("items lost: %#v", enhanced.JSON)
+ }
+ first := enhanced.JSON[0].ToMap()
+ if text, _ := first["text"].(string); text != "Caption\nbackend desc" {
+ t.Fatalf("backend-described text duplicated or changed: %q", text)
+ }
+ if _, ok := first[backendDescribedKey]; ok {
+ t.Fatalf("marker leaked to output: %#v", first)
+ }
+ second := enhanced.JSON[1].ToMap()
+ if text, _ := second["text"].(string); !strings.Contains(text, "a diagram of a pipeline") {
+ t.Fatalf("undescribed image not enhanced: %q", text)
+ }
+ if _, ok := second[backendDescribedKey]; ok {
+ t.Fatalf("marker leaked to output: %#v", second)
+ }
+}
+
+func TestStripBackendDescribedResultClearsMarker(t *testing.T) {
+ result := parser.ParseResult{OutputFormat: "json", JSON: []parser.DocItem{
+ parser.DocItemFromMap(map[string]any{"doc_type_kwd": "image", "text": "t", backendDescribedKey: true}),
+ parser.NewTextJSONItem("plain"),
+ }}
+ stripped := stripBackendDescribedResult(result)
+ for _, item := range stripped.JSON {
+ if _, ok := item.ToMap()[backendDescribedKey]; ok {
+ t.Fatalf("marker leaked: %#v", item.ToMap())
+ }
+ }
+ if text, _ := stripped.JSON[0].ToMap()["text"].(string); text != "t" {
+ t.Fatalf("text changed: %q", text)
+ }
+}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 3ddb05df39..b65f90f476 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -24,6 +24,7 @@ import (
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
+ "ragflow/internal/parser/tableutil"
"ragflow/internal/rag/prompts"
"ragflow/internal/utility"
@@ -79,16 +80,38 @@ func maybeDispatchPDFVision(
if !ok {
return parser.ParseResult{}, false, nil
}
+ tenantID := getStringOr(inputs, "tenant_id", "")
+ // The tenant-selected parser and the built-in backends share one
+ // normalization boundary so a future normalization fix cannot miss one
+ // of the two paths. ParseWithResult already normalizes JSON internally,
+ // but markdown stays unconverted there; running it through the same
+ // NormalizeResult keeps markdown→JSON conversion in the parser package
+ // and makes the call idempotent for already-normalized JSON.
+ flatten, _ := setup["flatten_media_to_text"].(bool)
+ normalize := func(result parser.ParseResult) parser.ParseResult {
+ pdf := parser.PDFParser{FlattenMediaToText: flatten}
+ return pdf.NormalizeResult(ctx, filename, result)
+ }
+ if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled {
+ if err != nil {
+ return result, true, err
+ }
+ result = normalize(result)
+ return result, true, result.Err
+ }
+ result, handled, err := dispatchPDFBackend(ctx, db, filename, binary, tenantID, setup)
+ if err != nil || !handled {
+ return result, handled, err
+ }
+ result = normalize(result)
+ return result, true, result.Err
+}
+func dispatchPDFBackend(ctx context.Context, db *gorm.DB, filename string, binary []byte, tenantID string, setup schema.ParserSetup) (parser.ParseResult, bool, error) {
method := getStringOr(setup, "parse_method", "")
layout := getStringOr(setup, "layout_recognizer", "")
- tenantID := getStringOr(inputs, "tenant_id", "")
layoutLower := strings.ToLower(strings.TrimSpace(layout))
- if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled {
- return result, true, err
- }
-
monkeySelector := layout
if strings.TrimSpace(monkeySelector) == "" {
monkeySelector = method
@@ -254,6 +277,10 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
}
func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []parser.DocItem {
+ // Fallback markdown→JSON converter for non-PDF families via
+ // buildParserOutputs/normalizeParserJSON. PDF dispatch results are
+ // normalized earlier by PDFParser.NormalizeResult (parser.parsePDFMarkdown),
+ // so this stays as the final safety net rather than a second PDF boundary.
mp, err := parser.NewMarkdownParser(parser.GoMarkdown)
if err != nil {
warnParserNormalizationFallback(filename, "markdown", err)
@@ -276,8 +303,7 @@ func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []pa
}
func buildMarkdownOCRDispatchResult(md string) parser.ParseResult {
- // Keep the backend's Markdown intact here. buildParserOutputs is the
- // single normalization boundary for all non-JSON parser responses.
+ // The PDF dispatch boundary converts backend Markdown before enhancement.
return parser.ParseResult{
OutputFormat: "markdown",
Markdown: md,
@@ -922,6 +948,14 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) m
"doc_type_kwd": docType,
"layout": layout,
}
+ // MinerU folds its own VLM description into text for image/chart items.
+ // Mark them so the shared vision enhancement skips them instead of
+ // appending a second model description.
+ if typ == "image" || typ == "chart" {
+ if desc, _ := item["vlm_description"].(string); strings.TrimSpace(desc) != "" {
+ jsonItem[backendDescribedKey] = true
+ }
+ }
if imageURI != "" {
jsonItem["image"] = imageURI
}
@@ -932,6 +966,20 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) m
jsonItem["page_number"] = page
}
}
+ if typ == "table" {
+ body, _ := item["table_body"].(string)
+ if table, err := tableutil.ParseTableHTML(body); err == nil && table != nil && len(table.Grid) > 0 {
+ caption := strings.Join(stringSlice(item["table_caption"]), "\n")
+ if caption != "" && caption != table.Caption {
+ table.Caption = strings.TrimSpace(table.Caption + "\n" + caption)
+ }
+ if page, ok := jsonItem["page_number"].(int); ok {
+ table.Page = page
+ }
+ jsonItem["table"] = table
+ jsonItem["text"] = strings.Join(stringSlice(item["table_footnote"]), "\n")
+ }
+ }
return jsonItem
}
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index df652a8839..9cf69b4d3b 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -120,19 +120,6 @@ func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {
}
}
-func TestBuildMarkdownOCRDispatchResultDefersJSONNormalization(t *testing.T) {
- result := buildMarkdownOCRDispatchResult("# Heading")
- if result.OutputFormat != "markdown" {
- t.Fatalf("OutputFormat = %q, want markdown", result.OutputFormat)
- }
- if result.Markdown != "# Heading" {
- t.Fatalf("Markdown = %q, want original OCR markdown", result.Markdown)
- }
- if result.JSON != nil {
- t.Fatalf("JSON = %#v, want nil until buildParserOutputs normalization", result.JSON)
- }
-}
-
type monkeyOCRv2FakeDriver struct {
modelModule.ModelDriver
}
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index 3d37ce56c3..2856eceb7a 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -27,6 +27,7 @@ import (
"fmt"
"image"
"io"
+ "maps"
"strings"
"sync"
"sync/atomic"
@@ -61,6 +62,56 @@ const (
visionMediaItemBudget = 10 * time.Second
)
+// backendDescribedKey marks items whose text already carries a backend-provided
+// vision description (MinerU folds vlm_description into text). The vision
+// enhancement collector skips such items to avoid appending a second model
+// description and spending an extra VLM call. The key is stripped before the
+// result leaves the Parser component so it never reaches the chunker or index.
+const backendDescribedKey = "_backend_described"
+
+func isBackendDescribed(m map[string]any) bool {
+ v, _ := m[backendDescribedKey].(bool)
+ return v
+}
+
+// stripBackendDescribedResult removes the transient backend-description marker
+// from every item. Enhancement skips marked items, but the marker must not leak
+// to the chunker when enhancement is disabled or bails out early.
+func stripBackendDescribedResult(result parser.ParseResult) parser.ParseResult {
+ if len(result.JSON) == 0 {
+ return result
+ }
+ needsStrip := false
+ for _, item := range result.JSON {
+ if item == nil {
+ continue
+ }
+ if _, ok := item.ToMap()[backendDescribedKey]; ok {
+ needsStrip = true
+ break
+ }
+ }
+ if !needsStrip {
+ return result
+ }
+ stripped := make([]parser.DocItem, 0, len(result.JSON))
+ for _, item := range result.JSON {
+ if item == nil {
+ continue
+ }
+ m := item.ToMap()
+ if _, ok := m[backendDescribedKey]; !ok {
+ stripped = append(stripped, item)
+ continue
+ }
+ cloned := maps.Clone(m)
+ delete(cloned, backendDescribedKey)
+ stripped = append(stripped, parser.DocItemFromMap(cloned))
+ }
+ result.JSON = stripped
+ return result
+}
+
// isUsableVisionImage checks the encoded size and rejects raster dimensions
// beyond the limits shared by Parser and VLM materialization.
func isUsableVisionImage(raw string) bool {
@@ -140,7 +191,7 @@ func maybeDispatchVisionEnhancement(
// Only enhance successful JSON output format containing items.
if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
- return dispatched, false, nil
+ return stripBackendDescribedResult(dispatched), false, nil
}
tenantID := getStringOr(inputs, "tenant_id", "")
@@ -151,23 +202,30 @@ func maybeDispatchVisionEnhancement(
language := resolveVisionLanguage(inputs)
// Collect visual resources, including Markdown images whose type is text
- // because flatten_media_to_text is enabled.
+ // because flatten_media_to_text is enabled. Items whose text already
+ // carries a backend-provided description (MinerU vlm_description) are
+ // skipped to avoid a duplicate second description and an extra VLM call.
var items []int
for i, rawItem := range dispatched.JSON {
m := rawItem.ToMap()
+ if isBackendDescribed(m) {
+ continue
+ }
if img, _ := m["image"].(string); img != "" {
items = append(items, i)
continue
}
kd, _ := m["doc_type_kwd"].(string)
if kd == "image" || kd == "table" {
- if _, ok := parser.ExtractPDFPositions(m); ok {
+ // A page-only position carries no crop region, so it must not
+ // reach the cropper.
+ if parser.HasPDFCropRegion(m) {
items = append(items, i)
}
}
}
if len(items) == 0 {
- return dispatched, false, nil
+ return stripBackendDescribedResult(dispatched), false, nil
}
// Resolve VLM before materializing image payloads, so disabled or
// unavailable enhancement does not spend time loading media.
@@ -204,7 +262,7 @@ func maybeDispatchVisionEnhancement(
}
}
dispatched.Warnings = append(dispatched.Warnings, warning)
- return dispatched, false, nil
+ return stripBackendDescribedResult(dispatched), false, nil
}
// Prepare PDF sources outside image admission so slow storage reads do not
@@ -213,7 +271,7 @@ func maybeDispatchVisionEnhancement(
cropper, cerr := visionImageCropperFactory(ctx, db, inputs)
if cerr != nil {
dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement skipped: %v", cerr))
- return dispatched, false, nil
+ return stripBackendDescribedResult(dispatched), false, nil
}
// Close is invoked explicitly after materialization so the engine and the
// source bytes are released while model requests are still in flight. Keep
@@ -354,7 +412,7 @@ func maybeDispatchVisionEnhancement(
}
wg.Wait()
if err := ctx.Err(); err != nil {
- return dispatched, modified, err
+ return stripBackendDescribedResult(dispatched), modified, err
}
w.Report(ctx, 1.0)
failedItems := 0
@@ -375,17 +433,42 @@ func maybeDispatchVisionEnhancement(
zap.Int("failed_items", failedItems), zap.String("first_failure", firstFailure))
}
- // Append descriptions after parser-provided text and captions.
- for slot, itemIdx := range items {
+ // Candidate indices are in document order. Build output once so a table's
+ // description can follow it without changing later candidates' indices.
+ // The transient backend-description marker is stripped from every emitted
+ // item so it never reaches the chunker.
+ enhanced := make([]parser.DocItem, 0, len(dispatched.JSON)+len(items))
+ slot := 0
+ for itemIdx, item := range dispatched.JSON {
+ clean := item
+ if m := item.ToMap(); isBackendDescribed(m) {
+ cloned := maps.Clone(m)
+ delete(cloned, backendDescribedKey)
+ clean = parser.DocItemFromMap(cloned)
+ }
+ if slot >= len(items) || items[slot] != itemIdx {
+ enhanced = append(enhanced, clean)
+ continue
+ }
desc := strings.TrimSpace(descriptions[slot])
+ slot++
if desc == "" {
+ enhanced = append(enhanced, clean)
continue
}
- m := dispatched.JSON[itemIdx].ToMap()
- appendItemText(m, desc)
- dispatched.JSON[itemIdx] = parser.DocItemFromMap(m)
+ m := clean.ToMap()
+ // clean.ToMap may share the underlying map for RawItem; clone before
+ // mutating so the source item is never modified in place.
+ m = maps.Clone(m)
+ if table, ok := m["table"].(*entity.TableData); ok && table != nil && m["doc_type_kwd"] == "table" {
+ enhanced = append(enhanced, clean, parser.NewTableNoteItem(m, desc))
+ } else {
+ appendItemText(m, desc)
+ enhanced = append(enhanced, parser.DocItemFromMap(m))
+ }
modified = true
}
+ dispatched.JSON = enhanced
return dispatched, modified, nil
}
diff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.go
index 0f5dbe2a1e..e3d73b742b 100644
--- a/internal/ingestion/component/vision_enhancement_cgo.go
+++ b/internal/ingestion/component/vision_enhancement_cgo.go
@@ -101,6 +101,9 @@ func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]a
// share this predicate so a future condition change cannot make Prepare skip a
// fetch that Crop then performs under image admission, silently reintroducing
// the admission block.
+//
+// A page-only position (page number, no geometry) carries no crop region and
+// is skipped, so it never triggers a page render.
func pdfSourcePositions(item map[string]any) ([]deepdoctype.Position, bool) {
if img, _ := item["image"].(string); img != "" {
return nil, false
@@ -110,10 +113,16 @@ func pdfSourcePositions(item map[string]any) ([]deepdoctype.Position, bool) {
return nil, false
}
positions := util.PositionsFromMatrix(matrix)
- if len(positions) == 0 {
+ regions := positions[:0]
+ for _, pos := range positions {
+ if pos.Right > pos.Left && pos.Bottom > pos.Top {
+ regions = append(regions, pos)
+ }
+ }
+ if len(regions) == 0 {
return nil, false
}
- return positions, true
+ return regions, true
}
func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visionImage, error) {
diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.go
index fa7adf08a3..14d246515e 100644
--- a/internal/parser/parser/doc_item.go
+++ b/internal/parser/parser/doc_item.go
@@ -184,7 +184,7 @@ func imageItemFromMap(m map[string]any) DocItem {
if pos != nil {
// Parsed into the typed field: it owns "positions", so strip it from
// Extra. Both [][]float64 (office/xlsx) and [][]any (PDF crop regions
- // from normalizePDFPositions) parse here, so the old [][]any-only
+ // from normalizeDeepDocPositions) parse here, so the old [][]any-only
// Extra fallback is gone.
skip = append(skip, "positions")
}
@@ -385,7 +385,7 @@ func wireToPositions(raw any, hasSheet bool) Positions {
// into the historical wire form for ToMap. The wire container type is preserved
// per vocabulary so the parser output is byte-for-byte unchanged:
// - AbsoluteBox (PDF) -> [][]any (int page + float64 coords), matching the
-// shape normalizePDFPositions already emitted and keeping the page a
+// shape normalizeDeepDocPositions already emitted and keeping the page a
// genuine int (a [][]float64 container would coerce it to float64).
// - RowColumnBox (xlsx/csv) -> [][]float64, matching the historical
// spreadsheet wire the ingestion indexer consumes.
diff --git a/internal/parser/parser/doc_item_positions_test.go b/internal/parser/parser/doc_item_positions_test.go
index c78fb2a303..d6872104b8 100644
--- a/internal/parser/parser/doc_item_positions_test.go
+++ b/internal/parser/parser/doc_item_positions_test.go
@@ -56,7 +56,7 @@ func TestPositionKindMethod(t *testing.T) {
}
// PDF path positions arrive as [][]any (int page + float64 coords) from
-// normalizePDFPositions. wireToPositions must build AbsoluteBox and round-trip
+// normalizeDeepDocPositions. wireToPositions must build AbsoluteBox and round-trip
// byte-exact, with the page preserved as int.
func TestWireToPositionsAbsoluteAny(t *testing.T) {
raw := [][]any{{1, 2.0, 3.0, 4.0, 5.0}}
diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.go
index 59689f994f..3ab4d180b2 100644
--- a/internal/parser/parser/doc_item_test.go
+++ b/internal/parser/parser/doc_item_test.go
@@ -244,7 +244,7 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
}
opaqueMap := map[string]any{"foo": "bar", "nested": map[string]any{"a": 1}}
- // PDF crop positions arrive as [][]any (normalizePDFPositions), not
+ // PDF crop positions arrive as [][]any (normalizeDeepDocPositions), not
// [][]float64; the typed conversion must preserve them verbatim. Regression
// guard for the positions-dropping bug caught during review.
imageMapAny := map[string]any{
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index 2d08a4713a..aa57203814 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -33,7 +33,6 @@ import (
pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
"ragflow/internal/deepdoc/parser/pdf/util"
deepdoctype "ragflow/internal/deepdoc/parser/type"
- "ragflow/internal/parser/tableutil"
"ragflow/internal/utility"
)
@@ -353,6 +352,14 @@ func (p *PDFParser) validateParseMethod() error {
// build-tag agnostic: under !cgo the deepdoc path transparently reports
// ErrPDFEngineUnavailable while plain_text falls back to the pure-Go engine.
func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
+ result := p.parseWithResult(ctx, filename, data)
+ if result.Err == nil && result.OutputFormat == "json" {
+ return p.NormalizeResult(ctx, filename, result)
+ }
+ return result
+}
+
+func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
if err := p.validateParseMethod(); err != nil {
return ParseResult{Err: err}
}
@@ -527,10 +534,6 @@ func GetDocAnalyzer() (deepdoctype.DocAnalyzer, error) {
return deepDocAnalyzerFromEnv()
}
-func pdfParseResultToJSON(filename string, parsed *deepdoctype.ParseResult) ParseResult {
- return pdfParseResultToJSONWithOptions(filename, parsed, pdfPostProcessOptions{})
-}
-
func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseResult, opts pdfPostProcessOptions) ParseResult {
if parsed == nil {
return ParseResult{Err: fmt.Errorf("parser: nil DeepDOC PDF result for %s", filename)}
@@ -541,6 +544,8 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
if opts.enableMultiColumn && opts.pageWidth <= 0 {
opts.pageWidth = firstPDFPageWidth(processed.PageWidth)
}
+ // JSON media policy is applied once at the PDFParser result boundary.
+ opts.flattenMediaToText = false
applyPDFPostProcess(&processed, opts)
defer processed.Close()
// NOTE: PDF media (figure/table) is intentionally NOT cropped here under
@@ -559,15 +564,13 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
if layoutType, _ := items[i]["layout_type"].(string); layoutType != "" {
items[i]["layout"] = layoutType
}
- if normalized := normalizePDFPositions(items[i]["_pdf_positions"]); len(normalized) > 0 {
+ if normalized := normalizeDeepDocPositions(items[i]["_pdf_positions"]); len(normalized) > 0 {
items[i]["_pdf_positions"] = normalized
items[i]["positions"] = normalized
if _, ok := items[i]["page_number"]; !ok {
items[i]["page_number"] = firstPageNumber(normalized)
}
}
- normalizePDFDocType(items[i])
- projectPDFTable(items[i])
if img, _ := items[i]["image"].(string); img != "" {
items[i]["image"] = pdflayout.InlinePNGDataURL(img)
}
@@ -897,7 +900,9 @@ func firstPDFPageWidth(pageWidths map[int]float64) float64 {
return pageWidths[pg]
}
-func normalizePDFPositions(raw any) [][]any {
+// normalizeDeepDocPositions converts DeepDoc source positions, including its
+// zero-based pages. Other engines must not pass canonical positions here.
+func normalizeDeepDocPositions(raw any) [][]any {
positions, ok := raw.([][]any)
if !ok || len(positions) == 0 {
return nil
@@ -907,7 +912,7 @@ func normalizePDFPositions(raw any) [][]any {
if len(pos) < 5 {
continue
}
- pageNumber, ok := normalizePDFPageNumber(pos[0])
+ pageNumber, ok := normalizeDeepDocPageNumber(pos[0])
if !ok {
continue
}
@@ -923,7 +928,7 @@ func normalizePDFPositions(raw any) [][]any {
return normalized
}
-// normalizePDFPageNumber converts a DeepDoc 0-indexed page number to the
+// normalizeDeepDocPageNumber converts a DeepDoc 0-indexed page number to the
// 1-indexed form stored in _pdf_positions / positions. It is the SINGLE
// 0→1 conversion point: DeepDoc (pdf_oxide/pdfium) emits 0-indexed pages,
// and every downstream consumer (addPDFPositions for ES storage,
@@ -931,24 +936,24 @@ func normalizePDFPositions(raw any) [][]any {
// Adding +1 unconditionally — instead of only for v<=0 — keeps all pages
// consistent; the old heuristic left page>=1 unconverted, which addPDFPositions
// then double-incremented and PositionsFromMatrix mis-decremented.
-func normalizePDFPageNumber(raw any) (int, bool) {
+func normalizeDeepDocPageNumber(raw any) (int, bool) {
switch v := raw.(type) {
case int:
return v + 1, true
case int64:
- return normalizePDFPageNumber(int(v))
+ return normalizeDeepDocPageNumber(int(v))
case float64:
- return normalizePDFPageNumber(int(v))
+ return normalizeDeepDocPageNumber(int(v))
case []any:
if len(v) == 0 {
return 0, false
}
- return normalizePDFPageNumber(v[len(v)-1])
+ return normalizeDeepDocPageNumber(v[len(v)-1])
case []int:
if len(v) == 0 {
return 0, false
}
- return normalizePDFPageNumber(v[len(v)-1])
+ return normalizeDeepDocPageNumber(v[len(v)-1])
default:
return 0, false
}
@@ -967,86 +972,11 @@ func numericAny(raw any) (float64, bool) {
}
}
-func normalizePDFDocType(item map[string]any) {
- if item == nil {
- return
- }
- layoutType, _ := item["layout_type"].(string)
- // A figure caption is a media section even when the parser no longer
- // inlines its cropped image (cgo): it still carries PDF positions, so the
- // downstream VLM/chunker crop it on demand. Classify it as image whenever
- // it has positions (the inlined image was only a side effect of cropping).
- _, hasMedia := ExtractPDFPositions(item)
- if docType, _ := item["doc_type_kwd"].(string); docType != "" {
- // A figure caption keeps its media classification so the downstream
- // VLM enhancement and on-demand chunker crop it.
- if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption && hasMedia {
- item["doc_type_kwd"] = "image"
- }
- return
- }
- switch layoutType {
- case "table":
- item["doc_type_kwd"] = "table"
- case "figure", "image":
- item["doc_type_kwd"] = "image"
- case deepdoctype.DLALabelFigureCaption:
- if hasMedia {
- item["doc_type_kwd"] = "image"
- } else {
- item["doc_type_kwd"] = "text"
- }
- default:
- if img, _ := item["image"].(string); img != "" {
- item["doc_type_kwd"] = "image"
- return
- }
- item["doc_type_kwd"] = "text"
- }
-}
-
-// projectPDFTable upgrades a legacy pdf table item — one whose doc_type_kwd is
-// "table" and whose text is <table> markup — to the structured TableData
-// contract, so the downstream chunker reads Rows directly instead of
-// re-parsing HTML. ParseTableHTML + RenderTableHTML round-trips the legacy
-// htmlTableRE extraction exactly (entity/html unescaping is left to the
-// consumer), so dropping the original HTML string loses no information: the
-// markdown and search paths render it on demand from `table`. The pdf page
-// number (when present) is projected onto TableData.Page.
-//
-// The redundant HTML `text` is deleted once `table` is projected: keeping both
-// would duplicate every table's cell text in the parse result (the storage
-// cost that motivated moving to the structured contract). Non-table items, and
-// table items whose text is not <table> markup, are left untouched so their
-// `text` is preserved as a last-resort carrier.
-func projectPDFTable(item map[string]any) {
- if item == nil {
- return
- }
- if dt, _ := item["doc_type_kwd"].(string); dt != "table" {
- return
- }
- text, _ := item["text"].(string)
- if !isTableHTML(text) {
- return
- }
- td, err := tableutil.ParseTableHTML(text)
- if err != nil || td == nil {
- return
- }
- if page := numberValue(item["page_number"]); page > 0 {
- td.Page = int(page)
- }
- item["table"] = td
- delete(item, "text")
-}
-
// ExtractPDFPositions is the single source of truth for "does this parsed item
-// carry a usable PDF crop region". It returns the non-empty positions matrix
-// (under the canonical _pdf_positions key or the legacy positions key),
-// accepting either the typed [][]any form produced in-process or the
-// JSON-decoded []any form whose elements are themselves []any rows. The bool
-// reports whether a usable matrix was present.
+// carry PDF positions". It returns the position rows (under the canonical
+// _pdf_positions key or the legacy positions key), decoded through
+// PDFPositionRows so every wire shape is understood. The bool reports whether
+// any row was present.
//
// The parser (doc-type classification in normalizePDFDocType) and the on-demand
// croppers (VLM vision_enhancement and the chunker) MUST agree on this
@@ -1056,34 +986,13 @@ func projectPDFTable(item map[string]any) {
// which escaped the figure-caption → image normalization and the crop path.
func ExtractPDFPositions(item map[string]any) ([][]any, bool) {
for _, key := range []string{"_pdf_positions", "positions"} {
- switch v := item[key].(type) {
- case [][]any:
- if len(v) > 0 {
- return v, true
- }
- case []any:
- out := make([][]any, 0, len(v))
- ok := true
- for _, row := range v {
- r, rOK := row.([]any)
- if !rOK {
- ok = false
- break
- }
- out = append(out, r)
- }
- if ok && len(out) > 0 {
- return out, true
- }
+ if rows := PDFPositionRows(item[key]); len(rows) > 0 {
+ return rows, true
}
}
return nil, false
}
-func parsePDFWithDeepDoc(ctx context.Context, filename string, data []byte, parseFn func(context.Context, []byte, deepdoctype.DocAnalyzer) (*deepdoctype.ParseResult, error)) ParseResult {
- return parsePDFWithDeepDocOptions(ctx, filename, data, pdfPostProcessOptions{}, parseFn)
-}
-
func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []byte, opts pdfPostProcessOptions, parseFn func(context.Context, []byte, deepdoctype.DocAnalyzer) (*deepdoctype.ParseResult, error)) ParseResult {
if len(data) == 0 {
return emptyPDFResult(filename)
@@ -1105,12 +1014,5 @@ func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []byt
default:
return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", opts.outputFormat)}
}
- for i := range res.JSON {
- m := res.JSON[i].ToMap()
- if img, _ := m["image"].(string); img != "" {
- m["image"] = pdflayout.InlinePNGDataURL(img)
- res.JSON[i] = DocItemFromMap(m)
- }
- }
return res
}
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index 07058df973..703ebd8f4d 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -1,6 +1,7 @@
package parser
import (
+ "context"
"image"
"strings"
"testing"
@@ -8,6 +9,10 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
)
+func pdfParseResultToJSON(filename string, parsed *deepdoctype.ParseResult) ParseResult {
+ return NewPDFParser().NormalizeResult(context.Background(), filename, pdfParseResultToJSONWithOptions(filename, parsed, pdfPostProcessOptions{}))
+}
+
func TestPDFParseResultToJSON_NormalizesCoreFields(t *testing.T) {
parsed := &deepdoctype.ParseResult{
Sections: []deepdoctype.Section{
@@ -154,7 +159,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {
}
// TestNormalizePDFPageNumber_UnconditionalIncrement pins the contract that
-// DeepDoc emits 0-indexed page numbers and normalizePDFPageNumber is the
+// DeepDoc emits 0-indexed page numbers and normalizeDeepDocPageNumber is the
// SINGLE conversion point to 1-indexed. It must add +1 unconditionally —
// not just for v<=0 — so that downstream addPDFPositions (a passthrough) and
// PositionsFromMatrix (which subtracts 1 for the 0-indexed PDFium engine)
@@ -178,7 +183,7 @@ func TestNormalizePDFPageNumber_UnconditionalIncrement(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- got, ok := normalizePDFPageNumber(tc.in)
+ got, ok := normalizeDeepDocPageNumber(tc.in)
if ok != tc.ok {
t.Fatalf("ok = %v, want %v", ok, tc.ok)
}
@@ -189,11 +194,11 @@ func TestNormalizePDFPageNumber_UnconditionalIncrement(t *testing.T) {
}
}
-func TestPDFParseResultToJSON_PreservesPositivePageNumbers(t *testing.T) {
+func TestPDFParseResultToJSONConvertsPositiveSourcePageNumbers(t *testing.T) {
parsed := &deepdoctype.ParseResult{
Sections: []deepdoctype.Section{
{
- Text: "Already one-based",
+ Text: "<table><tr><td>Page four</td></tr></table>",
LayoutType: deepdoctype.LayoutTypeTable,
Positions: []deepdoctype.Position{
{
@@ -209,7 +214,7 @@ func TestPDFParseResultToJSON_PreservesPositivePageNumbers(t *testing.T) {
}
res := pdfParseResultToJSON("one-based.pdf", parsed)
- // DeepDoc page 3 is 0-indexed (the 4th page); normalizePDFPageNumber
+ // DeepDoc page 3 is 0-indexed (the 4th page); normalizeDeepDocPageNumber
// converts it to 1-indexed page 4.
if got, want := res.JSON[0].ToMap()["page_number"], 4; got != want {
t.Fatalf("JSON[0].page_number = %v, want %v", got, want)
diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.go
index b08022df2f..b99a84e107 100644
--- a/internal/parser/parser/pdf_parser_docling.go
+++ b/internal/parser/parser/pdf_parser_docling.go
@@ -235,13 +235,13 @@ func parseDoclingStandardResult(ctx context.Context, filename string, body []byt
}
for _, doc := range docs {
if md := strings.TrimSpace(doc.MDContent); md != "" {
- return parseMinerUMarkdownResult(ctx, filename, md, outputFormat, len(docs)), true
+ return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true
}
if txt := strings.TrimSpace(doc.TextContent); txt != "" {
return doclingTextsToResult(filename, []string{txt}, outputFormat, len(docs)), true
}
if md, _ := doc.JSONContent["md_content"].(string); strings.TrimSpace(md) != "" {
- return parseMinerUMarkdownResult(ctx, filename, md, outputFormat, len(docs)), true
+ return pdfMarkdownToResult(ctx, filename, md, outputFormat, len(docs)), true
}
}
return ParseResult{}, false
diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.go
index 0b66f11d95..333ded8f3d 100644
--- a/internal/parser/parser/pdf_parser_mineru.go
+++ b/internal/parser/parser/pdf_parser_mineru.go
@@ -72,7 +72,7 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parse
if strings.TrimSpace(content) != "" {
pageCount = 1
}
- return parseMinerUMarkdownResult(ctx, filename, content, parser.OutputFormat, pageCount)
+ return pdfMarkdownToResult(ctx, filename, content, parser.OutputFormat, pageCount)
}
func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskID string, apiConfig *models.APIConfig, timeout time.Duration) (string, error) {
@@ -102,25 +102,3 @@ func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskID
time.Sleep(minerUPollInterval)
}
}
-
-func parseMinerUMarkdownResult(ctx context.Context, filename, markdown, outputFormat string, pageCount int) ParseResult {
- fileMeta := pdfFileMeta(filename, pageCount)
- switch strings.ToLower(strings.TrimSpace(outputFormat)) {
- case "", "json":
- mp, _ := NewMarkdownParser(GoMarkdown)
- res := mp.ParseWithResult(ctx, filename, []byte(markdown))
- if res.Err != nil {
- return res
- }
- res.File = fileMeta
- return res
- case "markdown":
- return ParseResult{
- OutputFormat: "markdown",
- File: fileMeta,
- Markdown: markdown,
- }
- default:
- return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}
- }
-}
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.go
index 7227752873..46e0b2b495 100644
--- a/internal/parser/parser/pdf_parser_opendataloader.go
+++ b/internal/parser/parser/pdf_parser_opendataloader.go
@@ -68,7 +68,7 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byt
}
}
if strings.TrimSpace(payload.MDText) != "" {
- return parseMinerUMarkdownResult(ctx, filename, payload.MDText, parser.OutputFormat, 1)
+ return pdfMarkdownToResult(ctx, filename, payload.MDText, parser.OutputFormat, 1)
}
return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader returned no parsed content")}
}
diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.go
index 57d6df306f..1073df5ab9 100644
--- a/internal/parser/parser/pdf_parser_remote_common.go
+++ b/internal/parser/parser/pdf_parser_remote_common.go
@@ -1,6 +1,7 @@
package parser
import (
+ "context"
"fmt"
"strings"
@@ -28,7 +29,11 @@ func pdfFileMeta(filename string, pageCount int) map[string]any {
func markdownItemText(item map[string]any) string {
if dt, _ := item["doc_type_kwd"].(string); dt == "table" {
if td, ok := item["table"].(*entity.TableData); ok && td != nil {
- return tableutil.RenderTableHTML(td)
+ markup := tableutil.RenderTableHTML(td)
+ if notes, _ := item["text"].(string); strings.TrimSpace(notes) != "" {
+ return strings.TrimRight(markup, "\n") + "\n\n" + notes
+ }
+ return markup
}
return ""
}
@@ -150,3 +155,35 @@ func rowsHaveText(rows [][]string) bool {
}
return false
}
+
+func pdfMarkdownToResult(ctx context.Context, filename, markdown, outputFormat string, pageCount int) ParseResult {
+ fileMeta := pdfFileMeta(filename, pageCount)
+ switch strings.ToLower(strings.TrimSpace(outputFormat)) {
+ case "", "json":
+ res := parsePDFMarkdown(ctx, filename, markdown)
+ if res.Err != nil {
+ if ctx.Err() != nil {
+ res.File = fileMeta
+ return res
+ }
+ // Same text fallback as NormalizeResult: keep the backend
+ // markdown as raw text instead of failing the task.
+ return ParseResult{
+ OutputFormat: "json",
+ File: fileMeta,
+ Warnings: append(res.Warnings, "markdown normalization fell back to text: "+res.Err.Error()),
+ JSON: []DocItem{NewTextJSONItem(markdown)},
+ }
+ }
+ res.File = fileMeta
+ return res
+ case "markdown":
+ return ParseResult{
+ OutputFormat: "markdown",
+ File: fileMeta,
+ Markdown: markdown,
+ }
+ default:
+ return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", outputFormat)}
+ }
+}
diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.go
index b10b13334c..e52f173647 100644
--- a/internal/parser/parser/pdf_parser_tcadp.go
+++ b/internal/parser/parser/pdf_parser_tcadp.go
@@ -84,6 +84,7 @@ func tcadpAnyToItems(raw any) []map[string]any {
emit := func(text, docType, layout string) []map[string]any {
m := map[string]any{"text": text, "doc_type_kwd": docType, "layout": layout}
if page > 0 {
+ m["page_number"] = page
// 1-indexed 5-tuple. addPDFPositions is a passthrough so
// the final position_int / page_num_int carry the same
// 1-indexed page number the caller passes. Mirrors
diff --git a/internal/parser/parser/pdf_parser_tcadp_test.go b/internal/parser/parser/pdf_parser_tcadp_test.go
index 37dd8148aa..dbaff9b772 100644
--- a/internal/parser/parser/pdf_parser_tcadp_test.go
+++ b/internal/parser/parser/pdf_parser_tcadp_test.go
@@ -204,6 +204,41 @@ func TestTCADPAnyToItems_PropagatesPageNumber(t *testing.T) {
}
}
+// TestTCADPItemsKeepPageIdentityThroughNormalization runs the TCADP adapter
+// output through the shared PDF normalization boundary. The flat page-only
+// tuple must survive as page identity (the indexer derives page_num_int from
+// positions) while staying out of the crop path.
+func TestTCADPItemsKeepPageIdentityThroughNormalization(t *testing.T) {
+ for _, raw := range []map[string]any{
+ {"content": "slide text", "type": "text", "page_number": 3},
+ {"caption": "fig", "type": "image", "page_number": 5},
+ {"type": "table", "table_data": map[string]any{"rows": []any{[]any{"a", "b"}}}, "page_number": 2},
+ } {
+ items := tcadpAnyToItems(raw)
+ if len(items) == 0 {
+ t.Fatalf("tcadpAnyToItems(%v) returned no items", raw)
+ }
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ item := result.JSON[0].ToMap()
+ if item["page_number"] == nil {
+ t.Fatalf("page identity lost: %#v", item)
+ }
+ positions, ok := ExtractPDFPositions(item)
+ if !ok || len(positions) != 1 {
+ t.Fatalf("page-only position lost: %#v", item)
+ }
+ if positions[0][0] != item["page_number"] {
+ t.Fatalf("position page %v != page_number %v", positions[0][0], item["page_number"])
+ }
+ if HasPDFCropRegion(item) {
+ t.Fatalf("page-only position exposed as a crop region: %#v", item)
+ }
+ }
+}
+
// TestTCADPAnyToItems_NestedArrayKeepsEachPage verifies that a nested
// array of TCADP elements produces one item per element, each carrying
// its own page-derived positions — not just the first element.
diff --git a/internal/parser/parser/pdf_positions.go b/internal/parser/parser/pdf_positions.go
new file mode 100644
index 0000000000..6968684ee2
--- /dev/null
+++ b/internal/parser/parser/pdf_positions.go
@@ -0,0 +1,94 @@
+package parser
+
+import "encoding/json"
+
+// This file owns PDF position decoding. The parser boundary, the vision
+// cropper and the chunker all read PDF positions through these helpers, so
+// they agree on what a position row is and on which wire shapes are
+// supported.
+//
+// Supported shapes:
+// - a matrix of 5-tuples: [][]any, []any of rows, [][]float64;
+// - one flat 5-tuple: []float64, the legacy wire form some engines (TCADP)
+// still emit.
+//
+// A row is [page, left, right, top, bottom]. The page is 1-based.
+
+// PDFPositionRows decodes the position rows carried by raw. Rows that are not
+// a 5-tuple are skipped; a flat tuple counts as one row.
+func PDFPositionRows(raw any) [][]any {
+ switch v := raw.(type) {
+ case [][]any:
+ return filterPositionRows(v, func(row []any) []any { return row })
+ case []any:
+ out := make([][]any, 0, len(v))
+ for _, rawRow := range v {
+ switch row := rawRow.(type) {
+ case []any:
+ if len(row) == 5 {
+ out = append(out, row)
+ }
+ case []float64:
+ if len(row) == 5 {
+ out = append(out, floatRowToAny(row))
+ }
+ }
+ }
+ return out
+ case [][]float64:
+ out := make([][]any, 0, len(v))
+ for _, row := range v {
+ if len(row) == 5 {
+ out = append(out, floatRowToAny(row))
+ }
+ }
+ return out
+ case []float64:
+ if len(v) == 5 {
+ return [][]any{floatRowToAny(v)}
+ }
+ }
+ return nil
+}
+
+// PDFPositionRowsFromJSON decodes a PDF position matrix carried as raw JSON.
+// It accepts the matrix form and the legacy flat 5-tuple and returns rows of
+// five floats. Rows shorter than five values are skipped.
+func PDFPositionRowsFromJSON(raw json.RawMessage) [][]float64 {
+ if len(raw) == 0 {
+ return nil
+ }
+ var matrix [][]float64
+ if err := json.Unmarshal(raw, &matrix); err == nil {
+ out := make([][]float64, 0, len(matrix))
+ for _, row := range matrix {
+ if len(row) >= 5 {
+ out = append(out, row)
+ }
+ }
+ return out
+ }
+ var flat []float64
+ if err := json.Unmarshal(raw, &flat); err == nil && len(flat) >= 5 {
+ return [][]float64{flat}
+ }
+ return nil
+}
+
+func filterPositionRows(rows [][]any, keep func([]any) []any) [][]any {
+ out := make([][]any, 0, len(rows))
+ for _, row := range rows {
+ if len(row) == 5 {
+ out = append(out, keep(row))
+ }
+ }
+ return out
+}
+
+func floatRowToAny(row []float64) []any {
+ out := make([]any, len(row))
+ for i, v := range row {
+ out[i] = v
+ }
+ return out
+}
diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.go
new file mode 100644
index 0000000000..4cc77653cf
--- /dev/null
+++ b/internal/parser/parser/pdf_result.go
@@ -0,0 +1,424 @@
+package parser
+
+import (
+ "context"
+ "maps"
+ "math"
+ "strings"
+
+ "go.uber.org/zap"
+ "golang.org/x/net/html"
+
+ "ragflow/internal/common"
+ "ragflow/internal/deepdoc/parser/pdf/util"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
+)
+
+// NormalizeResult converts a PDF backend response to JSON and applies common
+// item policies. Producers own page-base and coordinate-unit conversion.
+func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result ParseResult) ParseResult {
+ if result.Err != nil {
+ return result
+ }
+ if err := ctx.Err(); err != nil {
+ result.Err = err
+ result.File = ensurePDFResultFile(result.File, filename)
+ return result
+ }
+ if len(result.JSON) == 0 {
+ switch {
+ case result.Markdown != "":
+ parsed := parsePDFMarkdown(ctx, filename, result.Markdown)
+ if parsed.Err != nil {
+ if ctx.Err() != nil {
+ // Cancellation is fatal; preserve file metadata for callers.
+ parsed.File = ensurePDFResultFile(result.File, filename)
+ return parsed
+ }
+ // Mirror the historical markdown→JSON fallback: a backend
+ // markdown payload that cannot be structured is still indexed
+ // as raw text instead of failing the task.
+ common.Warn("parser normalization fell back to text",
+ zap.String("filename", filename),
+ zap.String("normalized_from", "markdown"),
+ zap.Error(parsed.Err))
+ result.Warnings = append(result.Warnings, parsed.Warnings...)
+ result.Warnings = append(result.Warnings, "markdown normalization fell back to text: "+parsed.Err.Error())
+ result.JSON = []DocItem{NewTextJSONItem(result.Markdown)}
+ } else {
+ result.Warnings = append(result.Warnings, parsed.Warnings...)
+ if len(parsed.JSON) == 0 {
+ result.JSON = []DocItem{NewTextJSONItem(result.Markdown)}
+ } else {
+ result.JSON = parsed.JSON
+ }
+ }
+ case result.Text != "":
+ result.JSON = []DocItem{NewTextJSONItem(result.Text)}
+ }
+ }
+ items := make([]DocItem, 0, len(result.JSON))
+ droppedPositions := 0
+ for _, item := range result.JSON {
+ if err := ctx.Err(); err != nil {
+ result.Err = err
+ result.File = ensurePDFResultFile(result.File, filename)
+ return result
+ }
+ if item == nil {
+ continue
+ }
+ fields := maps.Clone(item.ToMap())
+ if fields == nil {
+ continue
+ }
+ layout, _ := fields["layout"].(string)
+ if strings.TrimSpace(layout) == "" {
+ layout, _ = fields["layout_type"].(string)
+ }
+ if layout = strings.ToLower(strings.TrimSpace(layout)); layout != "" {
+ fields["layout"] = layout
+ if _, ok := fields["layout_type"]; ok {
+ fields["layout_type"] = layout
+ }
+ }
+ rawPrimary := fields["_pdf_positions"]
+ positions := validPDFPositions(rawPrimary)
+ droppedPositions += countPDFPositionRows(rawPrimary) - len(positions)
+ if len(positions) == 0 {
+ rawFallback := fields["positions"]
+ positions = validPDFPositions(rawFallback)
+ droppedPositions += countPDFPositionRows(rawFallback) - len(positions)
+ }
+ delete(fields, "_pdf_positions")
+ delete(fields, "positions")
+ if len(positions) > 0 {
+ fields["_pdf_positions"], fields["positions"] = positions, positions
+ if _, ok := fields["page_number"]; !ok {
+ fields["page_number"] = positions[0][0]
+ }
+ }
+ normalizePDFDocType(fields)
+ projectPDFTable(fields)
+ if table, ok := fields["table"].(*entity.TableData); ok && table != nil {
+ if page := int(numberValue(fields["page_number"])); page > 0 && page != table.Page {
+ normalizedTable := *table
+ normalizedTable.Page = page
+ fields["table"] = &normalizedTable
+ }
+ } else if fields["doc_type_kwd"] == "table" && len(positions) == 0 {
+ if text, _ := fields["text"].(string); text != "" {
+ fields["doc_type_kwd"] = "text"
+ }
+ }
+ if p.FlattenMediaToText {
+ // Flattened output keeps one text carrier per item. Structured
+ // tables (MinerU content_list, projected HTML) carry cells in
+ // `table` and footnotes in `text`; rendering the table preserves
+ // cells, caption, and footnote in that order. The render
+ // normalizes markup (header inference, escaping, whitespace), so
+ // it is not byte-identical to the source HTML, but it is
+ // content-preserving. Items without a table keep their text verbatim.
+ if table, ok := fields["table"].(*entity.TableData); ok && table != nil {
+ text, _ := fields["text"].(string)
+ fields["text"] = strings.TrimSpace(tableutil.RenderTableHTML(table) + "\n" + text)
+ delete(fields, "table")
+ }
+ fields["doc_type_kwd"] = "text"
+ }
+ // ck_type is retired: the content type lives on doc_type_kwd and the
+ // block kind on layout. A legacy raw item can still carry the key, so
+ // drop it here as well — the typed conversion already does.
+ delete(fields, "ck_type")
+ items = appendPDFItem(items, fields)
+ }
+ if droppedPositions > 0 {
+ // Zero-area or otherwise invalid boxes are dropped so they never reach
+ // the on-demand cropper. Log the count so a parser that stops emitting
+ // regions is distinguishable from one that never emitted any.
+ common.Warn("PDF normalization dropped invalid positions",
+ zap.String("filename", filename),
+ zap.Int("dropped_rows", droppedPositions))
+ }
+ result.JSON = items
+ result.OutputFormat, result.Markdown, result.Text = "json", "", ""
+ result.File = ensurePDFResultFile(result.File, filename)
+ return result
+}
+
+// ensurePDFResultFile preserves backend file metadata while guaranteeing the
+// filename is present, including on error paths where callers still need to
+// report which document failed.
+func ensurePDFResultFile(file map[string]any, filename string) map[string]any {
+ file = maps.Clone(file)
+ if file == nil {
+ file = map[string]any{}
+ }
+ if _, ok := file["name"]; !ok {
+ file["name"] = filename
+ }
+ return file
+}
+
+// NewTableNoteItem builds the text item that carries a table's description.
+// The note keeps the table's page identity and ordering hint so downstream
+// chunking preserves document order. Shared by NormalizeResult's table
+// splitting and the vision enhancement path.
+func NewTableNoteItem(fields map[string]any, text string) TextItem {
+ note := TextItem{Text: text, Extra: map[string]any{"layout": "text"}}
+ for _, key := range []string{"page_number", "_preserve_order"} {
+ if value, ok := fields[key]; ok {
+ note.Extra[key] = value
+ }
+ }
+ return note
+}
+
+// appendPDFItem emits adjacent tables separately and keeps their descriptions
+// in text items. Chunkers materialize a table only when its text field is empty.
+// The isTableHTML branch handles one text carrier holding two adjacent tables
+// plus a trailing note (projectPDFTable only projects the leading table, so the
+// remainder can itself start with a table). Covered by
+// TestPDFNormalizeResultPreservesAdjacentTables.
+func appendPDFItem(items []DocItem, fields map[string]any) []DocItem {
+ for {
+ table, _ := fields["table"].(*entity.TableData)
+ text, _ := fields["text"].(string)
+ if fields["doc_type_kwd"] != "table" || table == nil || text == "" {
+ return append(items, DocItemFromMap(fields))
+ }
+ fields = maps.Clone(fields)
+ delete(fields, "text")
+ items = append(items, DocItemFromMap(fields))
+ if strings.TrimSpace(text) == "" {
+ return items
+ }
+ if isTableHTML(text) {
+ next := maps.Clone(fields)
+ delete(next, "table")
+ next["text"] = text
+ projectPDFTable(next)
+ if table, _ := next["table"].(*entity.TableData); table != nil {
+ fields = next
+ continue
+ }
+ }
+ note := NewTableNoteItem(fields, text)
+ return append(items, note)
+ }
+}
+
+func parsePDFMarkdown(ctx context.Context, filename, markdown string) ParseResult {
+ p, err := NewMarkdownParser(GoMarkdown)
+ if err != nil {
+ return ParseResult{Err: err}
+ }
+ p.FetchRemoteImages = false
+ return p.ParseWithResult(ctx, filename, []byte(markdown))
+}
+
+// validPDFPositions validates canonical 1-based PDF positions without changing
+// page identity or units. Invalid regions must never reach the cropper.
+//
+// Every wire shape is decoded through PDFPositionRows: a matrix of 5-tuples
+// ([][]any, []any of rows, [][]float64) and the legacy flat 5-tuple
+// ([]float64) some engines still emit.
+//
+// A page-only tuple — a page number with all-zero geometry — is valid: it
+// carries page identity for the indexer but no crop region, so the crop
+// consumers skip it (see HasPDFCropRegion). A box with real coordinates that is
+// zero-area or inverted is invalid and is dropped; the caller logs the dropped
+// count so the loss is observable instead of silent.
+func validPDFPositions(raw any) [][]any {
+ var result [][]any
+ for _, row := range PDFPositionRows(raw) {
+ var values [5]float64
+ valid := true
+ for i, value := range row {
+ v, ok := numericAny(value)
+ if !ok || math.IsNaN(v) || math.IsInf(v, 0) {
+ valid = false
+ break
+ }
+ values[i] = v
+ }
+ page := values[0]
+ if !valid || page < 1 || page >= float64(^uint(0)>>1) || math.Trunc(page) != page {
+ continue
+ }
+ if !pdfPositionIsUsable(values) {
+ continue
+ }
+ result = append(result, []any{int(page), values[1], values[2], values[3], values[4]})
+ }
+ return result
+}
+
+// pdfPositionIsPageOnly reports whether a position carries page identity only:
+// a page number with all-zero geometry and therefore no crop region.
+func pdfPositionIsPageOnly(values [5]float64) bool {
+ return values[1] == 0 && values[2] == 0 && values[3] == 0 && values[4] == 0
+}
+
+// pdfPositionIsUsable reports whether a position may be kept. A page-only
+// tuple carries page identity and is kept; every other box must have a
+// positive width and height, so a zero-area or inverted region is dropped.
+func pdfPositionIsUsable(values [5]float64) bool {
+ if pdfPositionIsPageOnly(values) {
+ return true
+ }
+ return values[2] > values[1] && values[4] > values[3]
+}
+
+// pdfPositionHasRegion reports whether a position carries a usable crop region.
+// A page-only tuple carries page identity only, so it is not a region.
+func pdfPositionHasRegion(values [5]float64) bool {
+ if pdfPositionIsPageOnly(values) {
+ return false
+ }
+ return values[2] > values[1] && values[4] > values[3]
+}
+
+// HasPDFCropRegion reports whether the item carries at least one PDF crop
+// region, i.e. a position with a positive-area box. A page-only position
+// carries page identity but no region, so it must not trigger a crop.
+//
+// The crop consumers (the VLM vision path and the chunker) consult this
+// predicate so a page-only position never reaches the cropper, while the
+// indexer still receives the page identity through the positions matrix. The
+// geometry is decoded exactly as the cropper decodes it, so the two agree on
+// what a usable region is.
+func HasPDFCropRegion(item map[string]any) bool {
+ matrix, ok := ExtractPDFPositions(item)
+ if !ok {
+ return false
+ }
+ for _, pos := range util.PositionsFromMatrix(matrix) {
+ if pos.Right > pos.Left && pos.Bottom > pos.Top {
+ return true
+ }
+ }
+ return false
+}
+
+// countPDFPositionRows counts candidate position rows before validation so the
+// caller can report how many were dropped. It uses the same decoder as
+// validPDFPositions, so the count matches the rows validation sees.
+func countPDFPositionRows(raw any) int {
+ return len(PDFPositionRows(raw))
+}
+
+// splitPDFTable separates the leading table from any following description.
+// Tokenization handles nested tables without discarding the trailing text.
+func splitPDFTable(text string) (string, string) {
+ text = strings.TrimSpace(text)
+ if !isTableHTML(text) {
+ return "", text
+ }
+ z := html.NewTokenizer(strings.NewReader(text))
+ depth, offset := 0, 0
+ for {
+ kind := z.Next()
+ if kind == html.ErrorToken {
+ if depth > 0 {
+ return text, ""
+ }
+ return "", text
+ }
+ offset += len(z.Raw())
+ if kind != html.StartTagToken && kind != html.EndTagToken {
+ continue
+ }
+ name, _ := z.TagName()
+ if string(name) != "table" {
+ continue
+ }
+ if kind == html.StartTagToken {
+ depth++
+ } else {
+ depth--
+ }
+ if depth == 0 {
+ return text[:offset], strings.TrimSpace(text[offset:])
+ }
+ }
+}
+
+func normalizePDFDocType(item map[string]any) {
+ if item == nil {
+ return
+ }
+ layoutType, _ := item["layout_type"].(string)
+ if layoutType == "" {
+ layoutType, _ = item["layout"].(string)
+ }
+ // A figure caption is a media section even when the parser no longer
+ // inlines its cropped image (cgo): it still carries PDF positions, so the
+ // downstream VLM/chunker crop it on demand. Classify it as image whenever
+ // it has positions (the inlined image was only a side effect of cropping).
+ _, hasMedia := ExtractPDFPositions(item)
+ if docType, _ := item["doc_type_kwd"].(string); docType == "text" || docType == "image" || docType == "table" {
+ // A figure caption keeps its media classification so the downstream
+ // VLM enhancement and on-demand chunker crop it.
+ if docType == "text" && layoutType == deepdoctype.DLALabelFigureCaption && hasMedia {
+ item["doc_type_kwd"] = "image"
+ }
+ return
+ }
+ if table, ok := item["table"].(*entity.TableData); ok && table != nil {
+ item["doc_type_kwd"] = "table"
+ return
+ }
+ switch layoutType {
+ case "table":
+ item["doc_type_kwd"] = "table"
+ case "figure", "image":
+ item["doc_type_kwd"] = "image"
+ case deepdoctype.DLALabelFigureCaption:
+ if hasMedia {
+ item["doc_type_kwd"] = "image"
+ } else {
+ item["doc_type_kwd"] = "text"
+ }
+ default:
+ if img, _ := item["image"].(string); img != "" {
+ item["doc_type_kwd"] = "image"
+ return
+ }
+ item["doc_type_kwd"] = "text"
+ }
+}
+
+// projectPDFTable projects a leading HTML table into TableData, retaining any
+// following description and leaving existing structured tables untouched.
+func projectPDFTable(item map[string]any) {
+ if item == nil {
+ return
+ }
+ if dt, _ := item["doc_type_kwd"].(string); dt != "table" {
+ return
+ }
+ if item["table"] != nil {
+ return
+ }
+ text, _ := item["text"].(string)
+ markup, notes := splitPDFTable(text)
+ if markup == "" {
+ return
+ }
+ td, err := tableutil.ParseTableHTML(markup)
+ if err != nil || td == nil {
+ return
+ }
+ if page := numberValue(item["page_number"]); page > 0 {
+ td.Page = int(page)
+ }
+ item["table"] = td
+ if notes == "" {
+ delete(item, "text")
+ } else {
+ item["text"] = notes
+ }
+}
diff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.go
new file mode 100644
index 0000000000..d2b00d4812
--- /dev/null
+++ b/internal/parser/parser/pdf_result_test.go
@@ -0,0 +1,520 @@
+package parser
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+func TestPDFNormalizeResultPositionsAndMetadata(t *testing.T) {
+ file := map[string]any{"outline": []string{"Chapter"}, "page_count": 20, "parse_method": "selected-model"}
+ input := map[string]any{"text": "caption", "layout": "caption", "doc_type_kwd": "text", "page_number": 2,
+ "positions": []any{[]any{2.0, 10.0, 30.0, 20.0, 40.0}, []any{3.0, math.NaN(), 30.0, 20.0, 40.0}, []any{4, 0, 0, 0, 0}}}
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", File: file, Warnings: []string{"source warning"}, JSON: MapsToItems([]map[string]any{input, {"text": "", "page_number": 4, "positions": []float64{4, 0, 0, 0, 0}}})})
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ item := result.JSON[0].ToMap()
+ positions, ok := ExtractPDFPositions(item)
+ if !ok || len(positions) != 2 || positions[0][0] != 2 || item["doc_type_kwd"] != "text" {
+ t.Fatalf("positions/type = %#v", item)
+ }
+ // The NaN row is dropped; the page-only row survives as page identity.
+ if positions[1][0] != 4 {
+ t.Fatalf("page-only position lost: %#v", positions)
+ }
+ if len(result.JSON) != 2 || result.JSON[1].ToMap()["page_number"] != 4 {
+ t.Fatalf("blank page lost: %#v", result.JSON)
+ }
+ if !HasPDFCropRegion(item) {
+ t.Fatal("real crop region dropped")
+ }
+ if result.File["parse_method"] != "selected-model" || result.File["page_count"] != 20 || len(result.Warnings) != 1 {
+ t.Fatalf("metadata lost: %+v", result)
+ }
+ if _, ok := input["_pdf_positions"]; ok {
+ t.Fatal("input mutated")
+ }
+}
+
+func TestPDFNormalizeResultMarkdownAndCancellation(t *testing.T) {
+ previous := ssrfAllowLoopback
+ ssrfAllowLoopback = true
+ t.Cleanup(func() { ssrfAllowLoopback = previous })
+ requests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests++ }))
+ defer server.Close()
+ result := NewPDFParser().NormalizeResult(t.Context(), "ocr.pdf", ParseResult{OutputFormat: "markdown", Markdown: "| A | B |\n| --- | --- |\n| one | two |\n\n![remote](" + server.URL + "/image.png)"})
+ if result.Err != nil || result.OutputFormat != "json" || result.Markdown != "" || requests != 0 {
+ t.Fatalf("result=%+v requests=%d", result, requests)
+ }
+ var table *entity.TableData
+ for _, item := range result.JSON {
+ if td, ok := item.ToMap()["table"].(*entity.TableData); ok {
+ table = td
+ }
+ }
+ if table == nil || len(table.CellTexts()) < 2 || len(table.CellTexts()[1]) == 0 || table.CellTexts()[1][0] != "one" {
+ t.Fatalf("markdown table lost: %#v", result.JSON)
+ }
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ result = NewPDFParser().NormalizeResult(ctx, "ocr.pdf", ParseResult{OutputFormat: "markdown", Markdown: "body"})
+ if !errors.Is(result.Err, context.Canceled) {
+ t.Fatalf("cancellation = %+v", result)
+ }
+ if result.File["name"] != "ocr.pdf" {
+ t.Fatalf("cancellation lost file metadata: %+v", result)
+ }
+}
+
+func TestPDFMarkdownBackendDoesNotFetchRemoteImages(t *testing.T) {
+ withSSRFBypass(t)
+ previous := ssrfAllowLoopback
+ ssrfAllowLoopback = true
+ t.Cleanup(func() { ssrfAllowLoopback = previous })
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(http.StatusNotFound) }))
+ defer server.Close()
+ result := pdfMarkdownToResult(t.Context(), "remote.pdf", "![remote]("+server.URL+"/image.png)", "json", 1)
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ if requests.Load() != 0 {
+ t.Fatal("PDF Markdown backend normalization fetched a remote image")
+ }
+}
+
+func TestPDFNormalizeResultKeepsTCADPPageWithoutCrop(t *testing.T) {
+ items := tcadpAnyToItems(map[string]any{"type": "text", "content": "page text", "page_num": 7})
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ item := result.JSON[0].ToMap()
+ if item["page_number"] != 7 {
+ t.Fatalf("page identity lost: %#v", item)
+ }
+ // The page-only tuple survives as page identity but is not a crop region.
+ positions, ok := ExtractPDFPositions(item)
+ if !ok || len(positions) != 1 || positions[0][0] != 7 {
+ t.Fatalf("page-only position lost: %#v", item)
+ }
+ if HasPDFCropRegion(item) {
+ t.Fatalf("unknown region exposed for cropping: %#v", item)
+ }
+}
+
+func TestPDFNormalizeResultInfersStructuredTable(t *testing.T) {
+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}}
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{"table": table, "page_number": 3}})})
+ item := result.JSON[0].ToMap()
+ if item["doc_type_kwd"] != "table" {
+ t.Fatalf("table classified incorrectly: %#v", item)
+ }
+ if table.Page != 0 {
+ t.Fatal("input table mutated")
+ }
+}
+
+func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testing.T) {
+ p := NewPDFParser()
+ p.FlattenMediaToText = true
+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell"}}}, Caption: "Caption"}
+ result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: []DocItem{
+ TableItem{Table: table, Extra: map[string]any{"text": "Footnote"}},
+ ImageItem{Text: "Figure", Image: "image-data", Positions: Positions{AbsoluteBox{PageNumber: 2, Left: 1, Right: 5, Top: 2, Bottom: 6}}},
+ TextItem{Text: "Title", Extra: map[string]any{"layout": "heading"}},
+ }})
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ for i, item := range result.JSON[:2] {
+ fields := item.ToMap()
+ if fields["doc_type_kwd"] != "text" {
+ t.Fatalf("flattened item %d still has media type: %#v", i, fields)
+ }
+ if _, ok := fields["ck_type"]; ok {
+ t.Fatalf("flattened item %d still carries the retired ck_type: %#v", i, fields)
+ }
+ }
+ text, _ := result.JSON[0].ToMap()["text"].(string)
+ for _, want := range []string{"Cell", "Caption", "Footnote"} {
+ if !strings.Contains(text, want) {
+ t.Fatalf("missing %q in %q", want, text)
+ }
+ }
+ if result.JSON[1].ToMap()["image"] != "image-data" {
+ t.Fatal("flatten lost image")
+ }
+ if _, ok := ExtractPDFPositions(result.JSON[1].ToMap()); !ok {
+ t.Fatal("flatten lost positions")
+ }
+ if result.JSON[2].ToMap()["layout"] != "heading" {
+ t.Fatal("heading layout changed")
+ }
+}
+
+func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
+ "doc_type_kwd": "table", "ck_type": "table", "layout": "table", "text": "Recognized text without cells",
+ }})})
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ item := result.JSON[0].ToMap()
+ if item["doc_type_kwd"] != "text" || item["text"] != "Recognized text without cells" {
+ t.Fatalf("unstructured table kept an inconsistent subtype: %#v", item)
+ }
+ if _, ok := item["ck_type"]; ok {
+ t.Fatalf("retired ck_type survived normalization: %#v", item)
+ }
+}
+
+func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {
+ for _, flatten := range []bool{false, true} {
+ p := NewPDFParser()
+ p.FlattenMediaToText = flatten
+ result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
+ "doc_type_kwd": "table", "ck_type": "table", "text": "Recognized table without cells",
+ "positions": [][]any{{2, 10, 30, 20, 40}},
+ }})})
+ if result.Err != nil || len(result.JSON) != 1 {
+ t.Fatalf("unexpected result: %+v", result)
+ }
+ item := result.JSON[0].ToMap()
+ want := "table"
+ if flatten {
+ want = "text"
+ }
+ if item["doc_type_kwd"] != want || item["text"] != "Recognized table without cells" {
+ t.Fatalf("flatten=%v: table media identity or text lost: %#v", flatten, item)
+ }
+ if _, ok := ExtractPDFPositions(item); !ok {
+ t.Fatalf("flatten=%v: table crop region lost: %#v", flatten, item)
+ }
+ }
+}
+
+func TestPDFNormalizeResultRecoversUnclosedTableCells(t *testing.T) {
+ for _, markup := range []string{
+ "<table><tr><td>Cell value</td></tr>",
+ "<table><tr><td>Cell value",
+ } {
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
+ "doc_type_kwd": "table", "text": markup,
+ }})})
+ if result.Err != nil || len(result.JSON) != 1 {
+ t.Fatalf("unexpected result: %+v", result)
+ }
+ item := result.JSON[0].ToMap()
+ table, _ := item["table"].(*entity.TableData)
+ if table == nil || len(table.CellTexts()) != 1 || len(table.CellTexts()[0]) != 1 || table.CellTexts()[0][0] != "Cell value" {
+ t.Fatalf("unclosed markup lost recoverable cells: %#v", item)
+ }
+ if text, _ := item["text"].(string); text != "" {
+ t.Fatalf("recovered table still carries raw HTML: %q", text)
+ }
+ }
+}
+
+func TestPDFParserSoMarkFlattensStructuredMedia(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/parse/async" {
+ _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task"}}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"figure","content":"Figure caption"},{"type":"table","content":"<table><tr><td>Cell value</td></tr></table>"}]}]}}}}}`))
+ }))
+ defer server.Close()
+ p := NewPDFParser()
+ p.ConfigureFromSetup(map[string]any{"parse_method": "somark", "somark_base_url": server.URL, "flatten_media_to_text": true})
+ result := p.ParseWithResult(t.Context(), "sample.pdf", []byte("pdf"))
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ if len(result.JSON) != 2 {
+ t.Fatalf("items = %#v", result.JSON)
+ }
+ for i, want := range []string{"Figure caption", "Cell value"} {
+ item := result.JSON[i].ToMap()
+ text, _ := item["text"].(string)
+ if item["doc_type_kwd"] != "text" || !strings.Contains(text, want) {
+ t.Fatalf("item %d = %#v; want text containing %q", i, item, want)
+ }
+ }
+}
+
+func TestProjectPDFTablePreservesExistingStructureAndNotes(t *testing.T) {
+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Original cell"}}}}
+ item := map[string]any{"doc_type_kwd": "table", "table": table, "text": "<table><tr><td>Other</td></tr></table>\nFootnote"}
+ projectPDFTable(item)
+ text, _ := item["text"].(string)
+ if item["table"] != table || !strings.Contains(text, "Footnote") {
+ t.Fatalf("existing table or notes overwritten: %#v", item)
+ }
+ mixed := map[string]any{"doc_type_kwd": "table", "text": "<table><tr><td>Cell</td></tr></table>\nFootnote"}
+ projectPDFTable(mixed)
+ text, _ = mixed["text"].(string)
+ if !strings.Contains(text, "Footnote") {
+ t.Fatalf("notes lost: %#v", mixed)
+ }
+}
+
+func TestPDFTableDescriptionUsesTextItemForBothOutputs(t *testing.T) {
+ table := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}
+ item := map[string]any{"table": table, "doc_type_kwd": "table", "text": "Footnote", "page_number": 3}
+ for _, format := range []string{"json", "markdown"} {
+ result := pdfItemsToResult("sample.pdf", []map[string]any{item}, format, 3)
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ if format == "json" {
+ result = NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", result)
+ if len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != nil || result.JSON[1].ToMap()["text"] != "Footnote" || result.JSON[1].ToMap()["page_number"] != 3 {
+ t.Fatalf("table description not separated: %#v", result.JSON)
+ }
+ } else if !strings.Contains(result.Markdown, "Cell value") || !strings.Contains(result.Markdown, "Footnote") {
+ t.Fatalf("table description lost: %s", result.Markdown)
+ }
+ }
+ if item["text"] != "Footnote" {
+ t.Fatal("source item mutated")
+ }
+}
+
+func TestPDFWhitespaceTableDescriptionCannotHideCells(t *testing.T) {
+ item := map[string]any{"table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Cell value"}}}}, "doc_type_kwd": "table", "text": " \n "}
+ results := []ParseResult{
+ NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 1)),
+ NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{item})}),
+ }
+ for _, result := range results {
+ if result.Err != nil || len(result.JSON) != 1 {
+ t.Fatalf("unexpected result: %+v", result)
+ }
+ if text, _ := result.JSON[0].ToMap()["text"].(string); text != "" {
+ t.Fatalf("whitespace still suppresses table materialization: %q", text)
+ }
+ }
+}
+
+func TestPDFNormalizeResultPreservesAdjacentTables(t *testing.T) {
+ const markup = "<table><tr><td>First cell</td></tr></table>\n<table><tr><td>Second cell</td></tr></table>\nFootnote"
+ for _, projected := range []bool{false, true} {
+ for _, flatten := range []bool{false, true} {
+ name := fmt.Sprintf("projected=%v/flatten=%v", projected, flatten)
+ t.Run(name, func(t *testing.T) {
+ item := map[string]any{
+ "doc_type_kwd": "table", "text": markup, "page_number": 3,
+ "positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,
+ }
+ if projected {
+ projectPDFTable(item)
+ }
+ originalText := item["text"]
+ p := NewPDFParser()
+ p.FlattenMediaToText = flatten
+ result := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ if flatten {
+ var text string
+ for _, doc := range result.JSON {
+ fields := doc.ToMap()
+ if fields["doc_type_kwd"] != "text" || fields["table"] != nil {
+ t.Fatalf("media not flattened: %#v", fields)
+ }
+ value, _ := fields["text"].(string)
+ text += value
+ }
+ for _, want := range []string{"First cell", "Second cell", "Footnote"} {
+ if strings.Count(text, want) != 1 {
+ t.Fatalf("want %q exactly once in %q", want, text)
+ }
+ }
+ } else {
+ if len(result.JSON) != 3 {
+ t.Fatalf("want two tables and a footnote, got %#v", result.JSON)
+ }
+ for i, want := range []string{"First cell", "Second cell"} {
+ fields := result.JSON[i].ToMap()
+ table, _ := fields["table"].(*entity.TableData)
+ if table == nil || table.Page != 3 || len(table.Grid) != 1 || len(table.Grid[0]) != 1 || table.Grid[0][0].Text != want || fields["text"] != nil {
+ t.Fatalf("table %d lost cells or metadata: %#v", i, fields)
+ }
+ if _, ok := ExtractPDFPositions(fields); !ok {
+ t.Fatalf("table %d lost crop positions", i)
+ }
+ }
+ note := result.JSON[2].ToMap()
+ if note["text"] != "Footnote" || note["doc_type_kwd"] != "text" || note["page_number"] != 3 || note["_preserve_order"] != true {
+ t.Fatalf("footnote lost text or metadata: %#v", note)
+ }
+ }
+ if item["text"] != originalText {
+ t.Fatal("source item mutated")
+ }
+ })
+ }
+ }
+}
+
+func TestPDFNormalizeResultIsIdempotent(t *testing.T) {
+ const markup = "<table><tr><td>First cell</td></tr></table>\n<table><tr><td>Second cell</td></tr></table>\nFootnote"
+ for _, flatten := range []bool{false, true} {
+ p := NewPDFParser()
+ p.FlattenMediaToText = flatten
+ item := map[string]any{
+ "doc_type_kwd": "table", "text": markup, "page_number": 3,
+ "positions": [][]any{{3, 10, 30, 20, 40}}, "_preserve_order": true,
+ }
+ once := p.NormalizeResult(t.Context(), "sample.pdf", pdfItemsToResult("sample.pdf", []map[string]any{item}, "json", 3))
+ if once.Err != nil {
+ t.Fatal(once.Err)
+ }
+ twice := p.NormalizeResult(t.Context(), "sample.pdf", once)
+ if twice.Err != nil {
+ t.Fatal(twice.Err)
+ }
+ if len(once.JSON) != len(twice.JSON) {
+ t.Fatalf("flatten=%v: second normalize changed item count %d -> %d", flatten, len(once.JSON), len(twice.JSON))
+ }
+ for i := range once.JSON {
+ a, b := once.JSON[i].ToMap(), twice.JSON[i].ToMap()
+ if a["doc_type_kwd"] != b["doc_type_kwd"] {
+ t.Fatalf("flatten=%v: item %d type changed %v -> %v", flatten, i, a["doc_type_kwd"], b["doc_type_kwd"])
+ }
+ ta, _ := a["text"].(string)
+ tb, _ := b["text"].(string)
+ if ta != tb {
+ t.Fatalf("flatten=%v: item %d text changed %q -> %q", flatten, i, ta, tb)
+ }
+ }
+ }
+}
+
+// TestPDFNormalizeResultKeepsTCADPPageOnlyPosition covers the TCADP wire
+// form: a flat []float64 page-only tuple. Normalization must keep the page
+// identity (the indexer derives page_num_int from positions) while the tuple
+// stays out of the crop path.
+func TestPDFNormalizeResultKeepsTCADPPageOnlyPosition(t *testing.T) {
+ items := tcadpAnyToItems(map[string]any{"type": "text", "content": "page text", "page_num": 7})
+ if len(items) != 1 {
+ t.Fatalf("items = %#v", items)
+ }
+ result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems(items)})
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ item := result.JSON[0].ToMap()
+ if item["page_number"] != 7 {
+ t.Fatalf("page identity lost: %#v", item)
+ }
+ positions, ok := ExtractPDFPositions(item)
+ if !ok || len(positions) != 1 {
+ t.Fatalf("page-only position dropped: %#v", item)
+ }
+ if positions[0][0] != 7 {
+ t.Fatalf("page-only position lost its page: %#v", positions)
+ }
+ if HasPDFCropRegion(item) {
+ t.Fatalf("page-only position exposed as a crop region: %#v", item)
+ }
+}
+
+func TestValidPDFPositionsAcceptsFlatTupleAndDropsBadBoxes(t *testing.T) {
+ flat := validPDFPositions([]float64{7, 0, 0, 0, 0})
+ if len(flat) != 1 || flat[0][0] != 7 {
+ t.Fatalf("flat tuple dropped: %#v", flat)
+ }
+ if HasPDFCropRegion(map[string]any{"positions": flat}) {
+ t.Fatal("page-only tuple exposed as a crop region")
+ }
+ degenerate := validPDFPositions([][]any{{1, 10, 10, 5, 5}})
+ if len(degenerate) != 0 {
+ t.Fatalf("zero-area box kept: %#v", degenerate)
+ }
+ region := validPDFPositions([][]any{{1, 10, 30, 5, 25}})
+ if len(region) != 1 {
+ t.Fatalf("valid region dropped: %#v", region)
+ }
+ if !HasPDFCropRegion(map[string]any{"positions": region}) {
+ t.Fatal("valid region not exposed as a crop region")
+ }
+}
+
+// TestHasPDFCropRegionReadsEveryRowAndShape covers the crop-region contract:
+// every row of the matrix counts, and the legacy flat tuple is understood.
+func TestHasPDFCropRegionReadsEveryRowAndShape(t *testing.T) {
+ cases := []struct {
+ name string
+ item map[string]any
+ want bool
+ }{
+ {"page-only row before a real region", map[string]any{"positions": [][]any{{1, 0, 0, 0, 0}, {2, 10, 100, 10, 100}}}, true},
+ {"real region before a page-only row", map[string]any{"positions": [][]any{{1, 10, 100, 10, 100}, {2, 0, 0, 0, 0}}}, true},
+ {"only page-only rows", map[string]any{"positions": [][]any{{1, 0, 0, 0, 0}}}, false},
+ {"flat tuple with a region", map[string]any{"positions": []float64{1, 10, 100, 10, 100}}, true},
+ {"flat page-only tuple", map[string]any{"positions": []float64{1, 0, 0, 0, 0}}, false},
+ {"no positions", map[string]any{"positions": []any{}}, false},
+ }
+ for _, tc := range cases {
+ if got := HasPDFCropRegion(tc.item); got != tc.want {
+ t.Errorf("%s: HasPDFCropRegion = %v, want %v", tc.name, got, tc.want)
+ }
+ }
+}
+
+func TestCountPDFPositionRows(t *testing.T) {
+ if got := countPDFPositionRows(nil); got != 0 {
+ t.Fatalf("nil = %d, want 0", got)
+ }
+ if got := countPDFPositionRows([][]any{{1, 0, 1, 0, 1}, {2, 0, 1, 0, 1}}); got != 2 {
+ t.Fatalf("[][]any = %d, want 2", got)
+ }
+ if got := countPDFPositionRows([]any{[]any{1, 0, 1, 0, 1}}); got != 1 {
+ t.Fatalf("[]any = %d, want 1", got)
+ }
+ if got := countPDFPositionRows("bad"); got != 0 {
+ t.Fatalf("bad type = %d, want 0", got)
+ }
+}
+
+func TestPDFNormalizeResultCancellationPreservesFile(t *testing.T) {
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ result := NewPDFParser().NormalizeResult(ctx, "cancel.pdf", ParseResult{
+ OutputFormat: "json",
+ File: map[string]any{"parse_method": "mineru"},
+ JSON: MapsToItems([]map[string]any{{"text": "hi", "doc_type_kwd": "text"}}),
+ })
+ if !errors.Is(result.Err, context.Canceled) {
+ t.Fatalf("err = %v", result.Err)
+ }
+ if result.File["name"] != "cancel.pdf" || result.File["parse_method"] != "mineru" {
+ t.Fatalf("file lost: %+v", result.File)
+ }
+}
+
+func TestPDFMarkdownToResultPreservesFileOnCancel(t *testing.T) {
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ result := pdfMarkdownToResult(ctx, "cancel.pdf", "body", "json", 3)
+ if !errors.Is(result.Err, context.Canceled) {
+ t.Fatalf("err = %v", result.Err)
+ }
+ if result.File["name"] != "cancel.pdf" || result.File["page_count"] != 3 {
+ t.Fatalf("file lost: %+v", result.File)
+ }
+}