Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): scale MinerU content_list bboxes for PDF chunk highlights (#21498)

Sourav Rajvib22b250290a1
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 227d9d1d5b..a9664970cf 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "math"
"mime"
"mime/multipart"
"net/http"
@@ -588,9 +589,16 @@ func dispatchMinerUPDF(
v1Markdown = strings.TrimSpace(result.Markdown)
zipBytes = result.Zip
}
- items, extractErr := mineruExtractJSONItems(zipBytes)
+ var sizes map[int][2]float64
+ var geometryErr error
+ if len(zipBytes) > 0 {
+ sizes, geometryErr = mineruPageSizes(ctx, binary)
+ }
+ items, extractErr := mineruExtractJSONItems(zipBytes, sizes)
if extractErr == nil && len(items) > 0 {
- return mineruJSONParseResult(filename, items), nil
+ res := mineruJSONParseResult(filename, items)
+ appendMineruGeometryWarning(&res, geometryErr)
+ return res, nil
}
if v1Markdown != "" {
return normalizePDFMarkdown(ctx, filename, result.Markdown, setup), nil
@@ -607,12 +615,24 @@ func dispatchMinerUPDF(
return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)
}
- items, err := mineruExtractJSONItems(zipBytes)
+ sizes, geometryErr := mineruPageSizes(ctx, binary)
+ items, err := mineruExtractJSONItems(zipBytes, sizes)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", err)
}
- return mineruJSONParseResult(filename, items), nil
+ res := mineruJSONParseResult(filename, items)
+ appendMineruGeometryWarning(&res, geometryErr)
+ return res, nil
+}
+
+// appendMineruGeometryWarning records a failed page-size lookup. Without the
+// lookup MinerU items keep their text and page identity but carry no
+// positions, because content_list boxes are page-normalized, not PDF points.
+func appendMineruGeometryWarning(res *parser.ParseResult, err error) {
+ if err != nil {
+ res.Warnings = append(res.Warnings, "MinerU PDF geometry unavailable: "+err.Error())
+ }
}
// resolveMinerUModelForDispatch resolves the OCR model used by the MinerU PDF
@@ -825,12 +845,17 @@ const (
mineruMaxMiddleBytes = 64 << 20
)
+// mineruContentBBoxScale is the divisor MinerU applies when writing
+// content_list bboxes: each edge is normalized to 0-1000 over the page box.
+const mineruContentBBoxScale = 1000.0
+
// mineruExtractJSONItems reads MinerU content_list.json from a zip and
// emits JSON parser items with captions, embedded zip images, and
// 1-indexed _pdf_positions so jump-to-source works like DeepDoc.
-// Bounding boxes are scaled from MinerU's 0..1000 space to PDF points
-// using the per-page sizes in middle.json when that member is present.
-func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
+// pdfPageSizes comes from the source PDF (pdfium); middle.json in the zip
+// fills any missing pages. content_list bboxes are normalized to 0-1000, so
+// positions require a known page box in PDF points.
+func mineruExtractJSONItems(zipBytes []byte, pdfPageSizes map[int][2]float64) ([]map[string]any, error) {
zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
if err != nil {
return nil, fmt.Errorf("open zip: %w", err)
@@ -878,7 +903,7 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
return nil, fmt.Errorf("parse content_list.json: %w", err)
}
- pageSizes := mineruPageSizes(middleJSON)
+ pageSizes := mergeMineruPageSizes(pdfPageSizes, mineruPageSizesFromMiddleJSON(middleJSON))
out := make([]map[string]any, 0, len(rawItems))
for _, item := range rawItems {
converted := mineruContentItemToJSON(item, images, pageSizes)
@@ -892,14 +917,14 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
return out, nil
}
-// mineruPageSizes parses the per-page page_size ([width, height] in PDF
-// points, written by MinerU's pdfium page.get_size()) out of middle.json,
-// keyed by 0-based page_idx. A map rather than a slice because partial
-// jobs are legal and preserve the source page indices, so pdf_info can be
-// sparse (entries for pages 5 and 6 only). Entries without a usable
-// page_idx fall back to their position in pdf_info. Returns nil when
-// middle.json is absent or unparseable.
-func mineruPageSizes(middleJSON []byte) map[int][2]float64 {
+// mineruPageSizesFromMiddleJSON parses the per-page page_size ([width, height]
+// in PDF points, written by MinerU's pdfium page.get_size()) out of
+// middle.json, keyed by 0-based page_idx. A map rather than a slice because
+// partial jobs are legal and preserve the source page indices, so pdf_info can
+// be sparse (entries for pages 5 and 6 only). Entries without a usable page_idx
+// fall back to their position in pdf_info. Returns nil when middle.json is
+// absent or unparseable.
+func mineruPageSizesFromMiddleJSON(middleJSON []byte) map[int][2]float64 {
if len(middleJSON) == 0 {
return nil
}
@@ -929,6 +954,23 @@ func mineruPageSizes(middleJSON []byte) map[int][2]float64 {
return sizes
}
+func mergeMineruPageSizes(pdfSizes, middleSizes map[int][2]float64) map[int][2]float64 {
+ if len(pdfSizes) == 0 {
+ return middleSizes
+ }
+ if len(middleSizes) == 0 {
+ return pdfSizes
+ }
+ out := make(map[int][2]float64, len(pdfSizes)+len(middleSizes))
+ for idx, size := range middleSizes {
+ out[idx] = size
+ }
+ for idx, size := range pdfSizes {
+ out[idx] = size
+ }
+ return out
+}
+
func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, pageSizes map[int][2]float64) map[string]any {
typ, _ := item["type"].(string)
typ = strings.ToLower(strings.TrimSpace(typ))
@@ -990,12 +1032,14 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, p
if imageURI != "" {
jsonItem["image"] = imageURI
}
+ // Page identity does not depend on geometry: it drives table placement and
+ // page_count even when positions cannot be produced.
+ if pageIdx, ok := mineruJSONInt(item["page_idx"]); ok && pageIdx >= 0 {
+ jsonItem["page_number"] = pageIdx + 1
+ }
if positions := mineruItemPositions(item, pageSizes); len(positions) > 0 {
jsonItem["_pdf_positions"] = positions
jsonItem["positions"] = positions
- if page, ok := positions[0][0].(int); ok {
- jsonItem["page_number"] = page
- }
}
if typ == "table" {
body, _ := item["table_body"].(string)
@@ -1074,32 +1118,40 @@ func mineruItemPositions(item map[string]any, pageSizes map[int][2]float64) [][]
return nil
}
+// mineruBBoxPosition converts a content_list bbox ([x0, top, x1, bottom],
+// 0-1000 normalized) into the 1-based [page, left, right, top, bottom] row in
+// PDF points. The page size is required: emitting the normalized values as-is
+// misaligns jump-to-source highlights (issues #21220, #21488).
func mineruBBoxPosition(pageRaw, bboxRaw any, pageSizes map[int][2]float64) []any {
pageIdx, ok := mineruJSONInt(pageRaw)
- if !ok {
+ if !ok || pageIdx < 0 {
+ return nil
+ }
+ size, ok := pageSizes[pageIdx]
+ if !ok || size[0] <= 0 || size[1] <= 0 {
return nil
}
bbox := mineruJSONFloatSlice(bboxRaw)
if len(bbox) < 4 {
return nil
}
- // MinerU page_idx is 0-based; bbox is [x0, top, x1, bottom] in MinerU's
- // content_list coordinate space. Both the pipeline and vlm backends
- // normalize that space to 0..1000 against the page's point size
- // (pipeline _build_bbox / vlm mkcontent: int(x * 1000 / page_width)), so
- // a box is not comparable to the PDF-point space the rest of the
- // pipeline expects (position_int, the web highlighter, CropSection*)
- // until it is rescaled. When middle.json carried the page size, undo the
- // normalization exactly; without it, keep the pre-existing passthrough.
- left, right, top, bottom := bbox[0], bbox[2], bbox[1], bbox[3]
- if size, ok := pageSizes[pageIdx]; ok && size[0] > 0 && size[1] > 0 {
- left *= size[0] / 1000.0
- right *= size[0] / 1000.0
- top *= size[1] / 1000.0
- bottom *= size[1] / 1000.0
- }
- // Downstream position_int is 1-based [page, left, right, top, bottom].
- return []any{pageIdx + 1, left, right, top, bottom}
+ x0, top, x1, bott := bbox[0], bbox[1], bbox[2], bbox[3]
+ if x0 > x1 {
+ x0, x1 = x1, x0
+ }
+ if top > bott {
+ top, bott = bott, top
+ }
+ scaleX := size[0] / mineruContentBBoxScale
+ scaleY := size[1] / mineruContentBBoxScale
+ left := math.Max(0, math.Min(size[0], x0*scaleX))
+ right := math.Max(0, math.Min(size[0], x1*scaleX))
+ topPt := math.Max(0, math.Min(size[1], top*scaleY))
+ bottomPt := math.Max(0, math.Min(size[1], bott*scaleY))
+ if right <= left || bottomPt <= topPt {
+ return nil
+ }
+ return []any{pageIdx + 1, left, right, topPt, bottomPt}
}
func mineruJSONInt(raw any) (int, bool) {
diff --git a/internal/ingestion/component/pdf_vision_dispatch_cgo.go b/internal/ingestion/component/pdf_vision_dispatch_cgo.go
index 89158ef381..d7be6a97ba 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_cgo.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_cgo.go
@@ -4,16 +4,43 @@ package component
import (
"bytes"
+ "context"
"encoding/base64"
"fmt"
"image/png"
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ "ragflow/internal/deepdoc/parser/pdf/pdfium"
"ragflow/internal/utility"
)
const pdfVisionZoom = 3.0
+// mineruPageSizes maps each zero-based page index to its effective size in PDF
+// points. MinerU content_list boxes are normalized to 0-1000, so both axes
+// need the real page box before they can become positions.
+func mineruPageSizes(ctx context.Context, binary []byte) (map[int][2]float64, error) {
+ _, count, err := pdfium.PageSizes(ctx, binary, nil)
+ if err != nil {
+ return nil, err
+ }
+ if count <= 0 {
+ return nil, fmt.Errorf("PDF has no pages")
+ }
+ indexes := make([]int, count)
+ for i := range indexes {
+ indexes[i] = i
+ }
+ sizes, _, err := pdfium.PageSizes(ctx, binary, indexes)
+ if err != nil {
+ return sizes, err
+ }
+ if len(sizes) == 0 {
+ return nil, fmt.Errorf("no readable page sizes")
+ }
+ return sizes, nil
+}
+
func defaultRenderPDFVisionPages(binary []byte, ranges [][]int) ([]pdfVisionPage, error) {
engine, err := deepdocpdf.NewEngine(binary)
if err != nil {
diff --git a/internal/ingestion/component/pdf_vision_dispatch_mineru_cgo_test.go b/internal/ingestion/component/pdf_vision_dispatch_mineru_cgo_test.go
new file mode 100644
index 0000000000..e58cc995a9
--- /dev/null
+++ b/internal/ingestion/component/pdf_vision_dispatch_mineru_cgo_test.go
@@ -0,0 +1,40 @@
+//go:build cgo
+
+package component
+
+import (
+ "context"
+ "errors"
+ "math"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestMineruPageSizesReadsEveryPage(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join("..", "..", "deepdoc", "parser", "pdf", "pdfium", "testdata", "page_sizes.pdf"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ sizes, err := mineruPageSizes(t.Context(), data)
+ if err != nil {
+ t.Fatalf("mineruPageSizes: %v", err)
+ }
+ if len(sizes) != 3 {
+ t.Fatalf("sizes = %v, want 3 pages", sizes)
+ }
+ for idx, want := range map[int][2]float64{0: {595.5, 842}, 2: {300, 400}} {
+ got, ok := sizes[idx]
+ if !ok || math.Abs(got[0]-want[0]) > 1e-6 || math.Abs(got[1]-want[1]) > 1e-6 {
+ t.Errorf("page %d size = %v, want %v", idx, got, want)
+ }
+ }
+}
+
+func TestMineruPageSizesHonorsCancellation(t *testing.T) {
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ if _, err := mineruPageSizes(ctx, []byte("%PDF-1.4")); !errors.Is(err, context.Canceled) {
+ t.Fatalf("error = %v, want context.Canceled", err)
+ }
+}
diff --git a/internal/ingestion/component/pdf_vision_dispatch_nocgo.go b/internal/ingestion/component/pdf_vision_dispatch_nocgo.go
index 3881e527a0..586c98c167 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_nocgo.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_nocgo.go
@@ -2,8 +2,18 @@
package component
-import "fmt"
+import (
+ "context"
+ "fmt"
+)
func defaultRenderPDFVisionPages(_ []byte, _ [][]int) ([]pdfVisionPage, error) {
return nil, fmt.Errorf("tenant-aware PDF IMAGE2TEXT backend requires cgo rendering support")
}
+
+func mineruPageSizes(ctx context.Context, _ []byte) (map[int][2]float64, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ return nil, fmt.Errorf("PDF page geometry requires cgo")
+}
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 009d6ff561..9876b69cb1 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -20,9 +20,12 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
+ "math"
"net/http"
"net/http/httptest"
"os"
+ "path/filepath"
"strings"
"testing"
"time"
@@ -32,6 +35,7 @@ import (
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"gorm.io/gorm"
)
@@ -253,7 +257,8 @@ func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {
t.Fatal(err)
}
- items, err := mineruExtractJSONItems(buf.Bytes())
+ sizes := map[int][2]float64{0: {612, 792}, 1: {612, 792}}
+ items, err := mineruExtractJSONItems(buf.Bytes(), sizes)
if err != nil {
t.Fatalf("mineruExtractJSONItems: %v", err)
}
@@ -270,11 +275,11 @@ func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {
if page, _ := pos[0][0].(int); page != 1 {
t.Fatalf("text page = %#v, want 1-based page 1", pos[0])
}
- if left, _ := pos[0][1].(float64); left != 10 {
- t.Fatalf("bbox left = %#v, want 10 (x0)", pos[0])
- }
- if right, _ := pos[0][2].(float64); right != 110 {
- t.Fatalf("bbox right = %#v, want 110 (x1)", pos[0])
+ want := []float64{6.12, 67.32, 15.84, 31.68}
+ for i, w := range want {
+ if got, _ := pos[0][i+1].(float64); math.Abs(got-w) > 1e-6 {
+ t.Fatalf("bbox positions = %#v, want %v", pos[0], want)
+ }
}
imageItem := items[1]
@@ -297,10 +302,64 @@ func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {
}
}
+func TestMinerUBBoxPositionScalesAndClamps(t *testing.T) {
+ sizes := map[int][2]float64{0: {600, 800}}
+ for _, tc := range []struct {
+ name string
+ page any
+ bbox any
+ want []float64
+ wantNil bool
+ }{
+ {
+ name: "normalized to points",
+ page: 0,
+ bbox: []any{100.0, 200.0, 300.0, 400.0},
+ want: []float64{1, 60, 180, 160, 320},
+ },
+ {
+ name: "inverted edges are ordered",
+ page: 0,
+ bbox: []any{300.0, 400.0, 100.0, 200.0},
+ want: []float64{1, 60, 180, 160, 320},
+ },
+ {
+ name: "edges clamped to the page box",
+ page: 0,
+ bbox: []any{-50.0, -20.0, 1100.0, 1200.0},
+ want: []float64{1, 0, 600, 0, 800},
+ },
+ {name: "unknown page size", page: 3, bbox: []any{1.0, 2.0, 3.0, 4.0}, wantNil: true},
+ {name: "negative page index", page: -1, bbox: []any{1.0, 2.0, 3.0, 4.0}, wantNil: true},
+ {name: "short bbox", page: 0, bbox: []any{1.0, 2.0}, wantNil: true},
+ {name: "zero width", page: 0, bbox: []any{10.0, 20.0, 10.0, 40.0}, wantNil: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got := mineruBBoxPosition(tc.page, tc.bbox, sizes)
+ if tc.wantNil {
+ if got != nil {
+ t.Fatalf("position = %#v, want nil", got)
+ }
+ return
+ }
+ if len(got) != 5 {
+ t.Fatalf("position = %#v, want 5 values", got)
+ }
+ if page, ok := got[0].(int); !ok || page != int(tc.want[0]) {
+ t.Fatalf("page = %#v, want %v", got[0], tc.want[0])
+ }
+ for i, w := range tc.want[1:] {
+ if v, _ := got[i+1].(float64); math.Abs(v-w) > 1e-6 {
+ t.Fatalf("position = %#v, want %v", got, tc.want)
+ }
+ }
+ })
+ }
+}
+
// MinerU normalizes content_list bbox to 0..1000 against the page's point
-// size (pipeline _build_bbox / vlm mkcontent), so raw pass-through makes
-// highlights drift ~1.7x on A4. With middle.json in the zip the boxes must
-// come back as PDF points: bbox/1000 * page_size.
+// size (pipeline _build_bbox / vlm mkcontent). With middle.json in the zip the
+// boxes must come back as PDF points: bbox/1000 * page_size.
func TestMinerUExtractJSONItemsScalesBBoxToPoints(t *testing.T) {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
@@ -323,7 +382,7 @@ func TestMinerUExtractJSONItemsScalesBBoxToPoints(t *testing.T) {
t.Fatal(err)
}
- items, err := mineruExtractJSONItems(buf.Bytes())
+ items, err := mineruExtractJSONItems(buf.Bytes(), nil)
if err != nil {
t.Fatalf("mineruExtractJSONItems: %v", err)
}
@@ -338,8 +397,7 @@ func TestMinerUExtractJSONItemsScalesBBoxToPoints(t *testing.T) {
if page, _ := row[0].(int); page != 1 {
t.Fatalf("page = %#v, want 1", row[0])
}
- // page 0 is 500x1000pt: x*0.5, y*1.0 — exact in float64.
- want := []float64{50, 300, 200, 400} // left, right, top, bottom
+ want := []float64{50, 300, 200, 400}
for i, key := range []int{1, 2, 3, 4} {
v, _ := row[key].(float64)
if v != want[i] {
@@ -348,13 +406,41 @@ func TestMinerUExtractJSONItemsScalesBBoxToPoints(t *testing.T) {
}
}
+func TestMinerUExtractJSONItemsWithoutGeometryKeepsText(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ list, err := zw.Create("content_list.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, _ = list.Write([]byte(`[{"type":"text","text":"Hello","page_idx":1,"bbox":[10,20,110,40]}]`))
+ if err = zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ items, err := mineruExtractJSONItems(buf.Bytes(), nil)
+ if err != nil {
+ t.Fatalf("mineruExtractJSONItems: %v", err)
+ }
+ if len(items) != 1 {
+ t.Fatalf("len(items)=%d, want 1", len(items))
+ }
+ if items[0]["text"] != "Hello" {
+ t.Errorf("text = %#v", items[0]["text"])
+ }
+ if items[0]["page_number"] != 2 {
+ t.Errorf("page_number = %#v, want 2", items[0]["page_number"])
+ }
+ if _, ok := items[0]["_pdf_positions"]; ok {
+ t.Errorf("_pdf_positions = %#v, want none", items[0]["_pdf_positions"])
+ }
+}
+
// MinerU V1 accepts partial jobs and preserves the source page indices in
// its artifacts, so middle.json pdf_info can be sparse (page_idx 5 and 6
-// only). Sizes must be keyed by page_idx, not entry order: with two entries
-// the old slice lookup rejected index 5 and either passed the box through
-// unscaled or applied the wrong page's dimensions.
+// only). Sizes must be keyed by page_idx, not entry order.
func TestMinerUPageSizesKeysSparsePageIdx(t *testing.T) {
- sizes := mineruPageSizes([]byte(`{"pdf_info":[
+ sizes := mineruPageSizesFromMiddleJSON([]byte(`{"pdf_info":[
{"page_idx":5,"page_size":[400,800]},
{"page_idx":6,"page_size":[600,900]}
]}`))
@@ -365,28 +451,21 @@ func TestMinerUPageSizesKeysSparsePageIdx(t *testing.T) {
if pos == nil {
t.Fatal("expected a position, got nil")
}
- // page 6 is 600x900pt: x*0.6, y*0.9.
- want := []float64{300, 600, 450, 900} // left, right, top, bottom
+ want := []float64{300, 600, 450, 900}
for i, key := range []int{1, 2, 3, 4} {
v, _ := pos[key].(float64)
if v != want[i] {
t.Fatalf("position[%d] = %v, want %v (row=%v)", key, pos[key], want[i], pos)
}
}
- // A page missing from a partial job stays passthrough, not another
- // page's scale.
- missing := mineruBBoxPosition(4, []any{100, 100, 200, 200}, sizes)
- if missing == nil {
- t.Fatal("expected a position for missing page, got nil")
- }
- if left, _ := missing[1].(float64); left != 100 {
- t.Fatalf("left = %#v, want 100 (passthrough)", missing[1])
+ if mineruBBoxPosition(4, []any{100, 100, 200, 200}, sizes) != nil {
+ t.Fatal("expected nil position for page missing from sparse sizes")
}
}
// Entries with no page_idx at all fall back to their position in pdf_info.
func TestMinerUPageSizesFallsBackToEntryOrder(t *testing.T) {
- sizes := mineruPageSizes([]byte(`{"pdf_info":[
+ sizes := mineruPageSizesFromMiddleJSON([]byte(`{"pdf_info":[
{"page_size":[500,1000]},
{"page_size":[400,800]}
]}`))
@@ -395,18 +474,73 @@ func TestMinerUPageSizesFallsBackToEntryOrder(t *testing.T) {
}
}
-// Without middle.json there is no trustworthy page size, so boxes keep the
-// pre-existing passthrough behavior (a known degraded mode, not a failure).
-func TestMinerUBBoxPositionWithoutPageSizesIsPassthrough(t *testing.T) {
- pos := mineruBBoxPosition(2, []any{188, 142, 875, 393}, nil)
- if pos == nil {
- t.Fatal("expected a position, got nil")
+func TestMinerUBBoxPositionWithoutPageSizesReturnsNil(t *testing.T) {
+ if mineruBBoxPosition(2, []any{188, 142, 875, 393}, nil) != nil {
+ t.Fatal("expected nil position without page sizes")
+ }
+}
+
+func TestMinerURealContentListPositionsStayInPage(t *testing.T) {
+ list, err := os.ReadFile(filepath.Join("..", "..", "..", "test", "fixtures", "mineru", "bmw_page_chrome_content_list.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ f, err := zw.Create("content_list.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = f.Write(list); err != nil {
+ t.Fatal(err)
+ }
+ if err = zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ const (
+ pageWidth = 595.0
+ pageHeight = 842.0
+ )
+ items, err := mineruExtractJSONItems(buf.Bytes(), map[int][2]float64{0: {pageWidth, pageHeight}})
+ if err != nil {
+ t.Fatalf("mineruExtractJSONItems: %v", err)
}
- if page, _ := pos[0].(int); page != 3 {
- t.Fatalf("page = %#v, want 3", pos[0])
+ if len(items) == 0 {
+ t.Fatal("no items converted")
+ }
+ positioned := 0
+ for _, item := range items {
+ raw, ok := item["_pdf_positions"]
+ if !ok {
+ continue
+ }
+ positioned++
+ rows, ok := raw.([][]any)
+ if !ok || len(rows) != 1 {
+ t.Fatalf("_pdf_positions = %#v, want one row", raw)
+ }
+ row := rows[0]
+ left, right := row[1].(float64), row[2].(float64)
+ top, bottom := row[3].(float64), row[4].(float64)
+ if left < 0 || right > pageWidth || top < 0 || bottom > pageHeight || right <= left || bottom <= top {
+ t.Fatalf("position %v outside the %.0fx%.0f page", row, pageWidth, pageHeight)
+ }
+ }
+ if positioned == 0 {
+ t.Fatal("fixture produced no positions")
+ }
+}
+
+func TestAppendMineruGeometryWarningOnlyOnError(t *testing.T) {
+ res := parser.ParseResult{}
+ appendMineruGeometryWarning(&res, nil)
+ if len(res.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none on success", res.Warnings)
}
- if left, _ := pos[1].(float64); left != 188 {
- t.Fatalf("left = %#v, want 188 (unchanged)", pos[1])
+ appendMineruGeometryWarning(&res, errors.New("no pdfium"))
+ if len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], "MinerU PDF geometry unavailable: no pdfium") {
+ t.Fatalf("warnings = %v", res.Warnings)
}
}