Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser/mineru): rescale content_list bbox from 0..1000 to PDF points (#21267)

Riven6d553f3875b7
diff --git a/internal/ingestion/component/pdf_postprocess_test.go b/internal/ingestion/component/pdf_postprocess_test.go
index 6a1f705bb3..40caee46df 100644
--- a/internal/ingestion/component/pdf_postprocess_test.go
+++ b/internal/ingestion/component/pdf_postprocess_test.go
@@ -71,7 +71,7 @@ func TestPDFDispatchNormalizesBeforeOutputsAndEnhancement(t *testing.T) {
}
func TestMinerUTableRetainsCaptionAndFootnote(t *testing.T) {
- item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><tr><td>Cell</td></tr></table>", "table_caption": []any{"Caption"}, "table_footnote": []any{"Footnote"}, "page_idx": 1, "bbox": []any{10.0, 20.0, 30.0, 40.0}}, nil)
+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><tr><td>Cell</td></tr></table>", "table_caption": []any{"Caption"}, "table_footnote": []any{"Footnote"}, "page_idx": 1, "bbox": []any{10.0, 20.0, 30.0, 40.0}}, nil, nil)
table, ok := item["table"].(*entity.TableData)
if !ok || table == nil || table.Caption != "Caption" || len(table.CellTexts()) != 1 || len(table.CellTexts()[0]) != 1 || table.CellTexts()[0][0] != "Cell" || table.Page != 2 {
t.Fatalf("structured table missing: %#v", item)
@@ -82,7 +82,7 @@ func TestMinerUTableRetainsCaptionAndFootnote(t *testing.T) {
}
func TestMinerUTablePreservesEmbeddedCaption(t *testing.T) {
- item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><caption>Embedded caption</caption><tr><td>Cell</td></tr></table>"}, nil)
+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><caption>Embedded caption</caption><tr><td>Cell</td></tr></table>"}, nil, nil)
table, ok := item["table"].(*entity.TableData)
if !ok || table == nil || table.Caption != "Embedded caption" {
t.Fatalf("embedded caption lost: %#v", item)
@@ -111,7 +111,7 @@ func TestMinerUContentMarksBackendDescribedImages(t *testing.T) {
"type": "image", "image_caption": []any{"Caption"},
"vlm_description": "backend desc",
"page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},
- }, nil)
+ }, nil, nil)
if withDesc == nil {
t.Fatal("image item dropped")
}
@@ -124,7 +124,7 @@ func TestMinerUContentMarksBackendDescribedImages(t *testing.T) {
withoutDesc := mineruContentItemToJSON(map[string]any{
"type": "image", "image_caption": []any{"Caption"},
"page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},
- }, nil)
+ }, nil, nil)
if withoutDesc == nil {
t.Fatal("image item dropped")
}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index d9e7610021..227d9d1d5b 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -817,11 +817,19 @@ func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod,
return zipBytes, nil
}
-const mineruMaxImageBytes = 20 << 20
+const (
+ mineruMaxImageBytes = 20 << 20
+ // mineruMaxMiddleBytes bounds middle.json, which carries the full
+ // layout/span payload and can legitimately reach tens of megabytes on
+ // long documents, while still refusing a zip-bomb-sized member.
+ mineruMaxMiddleBytes = 64 << 20
+)
// mineruExtractJSONItems reads MinerU content_list.json from a zip and
// emits JSON parser items with captions, embedded zip images, and
// 1-indexed _pdf_positions so jump-to-source works like DeepDoc.
+// Bounding boxes are scaled from MinerU's 0..1000 space to PDF points
+// using the per-page sizes in middle.json when that member is present.
func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
if err != nil {
@@ -829,7 +837,7 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
}
images := make(map[string]*zip.File)
- var contentList []byte
+ var contentList, middleJSON []byte
for _, f := range zipReader.File {
if strings.HasSuffix(f.Name, "content_list.json") && len(contentList) == 0 {
rc, openErr := f.Open()
@@ -845,6 +853,15 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
return nil, fmt.Errorf("read %s: %w", f.Name, readErr)
}
}
+ // middle.json is optional: it only carries page_size, which lets
+ // positions be converted to points. A zip without it keeps the
+ // pre-existing passthrough behavior instead of failing the parse,
+ // and so does an oversized (or lying-header) member.
+ if strings.HasSuffix(f.Name, "middle.json") && len(middleJSON) == 0 {
+ if data, ok := mineruReadBounded(f, mineruMaxMiddleBytes); ok {
+ middleJSON = data
+ }
+ }
if strings.HasPrefix(strings.ToLower(mime.TypeByExtension(path.Ext(f.Name))), "image/") {
clean := strings.TrimPrefix(path.Clean(f.Name), "./")
images[path.Base(f.Name)] = f
@@ -861,9 +878,10 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
return nil, fmt.Errorf("parse content_list.json: %w", err)
}
+ pageSizes := mineruPageSizes(middleJSON)
out := make([]map[string]any, 0, len(rawItems))
for _, item := range rawItems {
- converted := mineruContentItemToJSON(item, images)
+ converted := mineruContentItemToJSON(item, images, pageSizes)
if converted != nil {
out = append(out, converted)
}
@@ -874,7 +892,44 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {
return out, nil
}
-func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) map[string]any {
+// mineruPageSizes parses the per-page page_size ([width, height] in PDF
+// points, written by MinerU's pdfium page.get_size()) out of middle.json,
+// keyed by 0-based page_idx. A map rather than a slice because partial
+// jobs are legal and preserve the source page indices, so pdf_info can be
+// sparse (entries for pages 5 and 6 only). Entries without a usable
+// page_idx fall back to their position in pdf_info. Returns nil when
+// middle.json is absent or unparseable.
+func mineruPageSizes(middleJSON []byte) map[int][2]float64 {
+ if len(middleJSON) == 0 {
+ return nil
+ }
+ var parsed struct {
+ PdfInfo []struct {
+ PageIdx *int `json:"page_idx"`
+ PageSize []float64 `json:"page_size"`
+ } `json:"pdf_info"`
+ }
+ if err := json.Unmarshal(middleJSON, &parsed); err != nil || len(parsed.PdfInfo) == 0 {
+ return nil
+ }
+ sizes := make(map[int][2]float64, len(parsed.PdfInfo))
+ for i, page := range parsed.PdfInfo {
+ if len(page.PageSize) < 2 {
+ continue
+ }
+ idx := i
+ if page.PageIdx != nil && *page.PageIdx >= 0 {
+ idx = *page.PageIdx
+ }
+ sizes[idx] = [2]float64{page.PageSize[0], page.PageSize[1]}
+ }
+ if len(sizes) == 0 {
+ return nil
+ }
+ return sizes
+}
+
+func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, pageSizes map[int][2]float64) map[string]any {
typ, _ := item["type"].(string)
typ = strings.ToLower(strings.TrimSpace(typ))
switch typ {
@@ -935,7 +990,7 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) m
if imageURI != "" {
jsonItem["image"] = imageURI
}
- if positions := mineruItemPositions(item); len(positions) > 0 {
+ if positions := mineruItemPositions(item, pageSizes); len(positions) > 0 {
jsonItem["_pdf_positions"] = positions
jsonItem["positions"] = positions
if page, ok := positions[0][0].(int); ok {
@@ -997,7 +1052,7 @@ func mineruItemText(item map[string]any, typ string) string {
return strings.TrimSpace(strings.Join(parts, "\n"))
}
-func mineruItemPositions(item map[string]any) [][]any {
+func mineruItemPositions(item map[string]any, pageSizes map[int][2]float64) [][]any {
if raw, ok := item["_mineru_positions"].([]any); ok {
var out [][]any
for _, entry := range raw {
@@ -1005,7 +1060,7 @@ func mineruItemPositions(item map[string]any) [][]any {
if !ok {
continue
}
- if pos := mineruBBoxPosition(posMap["page_idx"], posMap["bbox"]); pos != nil {
+ if pos := mineruBBoxPosition(posMap["page_idx"], posMap["bbox"], pageSizes); pos != nil {
out = append(out, pos)
}
}
@@ -1013,13 +1068,13 @@ func mineruItemPositions(item map[string]any) [][]any {
return out
}
}
- if pos := mineruBBoxPosition(item["page_idx"], item["bbox"]); pos != nil {
+ if pos := mineruBBoxPosition(item["page_idx"], item["bbox"], pageSizes); pos != nil {
return [][]any{pos}
}
return nil
}
-func mineruBBoxPosition(pageRaw, bboxRaw any) []any {
+func mineruBBoxPosition(pageRaw, bboxRaw any, pageSizes map[int][2]float64) []any {
pageIdx, ok := mineruJSONInt(pageRaw)
if !ok {
return nil
@@ -1028,9 +1083,23 @@ func mineruBBoxPosition(pageRaw, bboxRaw any) []any {
if len(bbox) < 4 {
return nil
}
- // MinerU page_idx is 0-based; bbox is [x0, top, x1, bottom].
+ // MinerU page_idx is 0-based; bbox is [x0, top, x1, bottom] in MinerU's
+ // content_list coordinate space. Both the pipeline and vlm backends
+ // normalize that space to 0..1000 against the page's point size
+ // (pipeline _build_bbox / vlm mkcontent: int(x * 1000 / page_width)), so
+ // a box is not comparable to the PDF-point space the rest of the
+ // pipeline expects (position_int, the web highlighter, CropSection*)
+ // until it is rescaled. When middle.json carried the page size, undo the
+ // normalization exactly; without it, keep the pre-existing passthrough.
+ left, right, top, bottom := bbox[0], bbox[2], bbox[1], bbox[3]
+ if size, ok := pageSizes[pageIdx]; ok && size[0] > 0 && size[1] > 0 {
+ left *= size[0] / 1000.0
+ right *= size[0] / 1000.0
+ top *= size[1] / 1000.0
+ bottom *= size[1] / 1000.0
+ }
// Downstream position_int is 1-based [page, left, right, top, bottom].
- return []any{pageIdx + 1, bbox[0], bbox[2], bbox[1], bbox[3]}
+ return []any{pageIdx + 1, left, right, top, bottom}
}
func mineruJSONInt(raw any) (int, bool) {
@@ -1085,6 +1154,26 @@ func mineruJSONFloatSlice(raw any) []float64 {
}
}
+// mineruReadBounded reads a zip member, refusing anything that is empty or
+// larger than limit bytes. The size is checked both against the declared
+// uncompressed size and against the bytes actually produced, since a
+// hostile member can understate the former.
+func mineruReadBounded(f *zip.File, limit int64) ([]byte, bool) {
+ if f == nil || f.UncompressedSize64 > uint64(limit) {
+ return nil, false
+ }
+ rc, err := f.Open()
+ if err != nil {
+ return nil, false
+ }
+ data, err := io.ReadAll(io.LimitReader(rc, limit+1))
+ rc.Close()
+ if err != nil || len(data) == 0 || int64(len(data)) > limit {
+ return nil, false
+ }
+ return data, true
+}
+
func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {
imgPath = strings.TrimSpace(strings.ReplaceAll(imgPath, "\\", "/"))
if imgPath == "" {
@@ -1098,16 +1187,8 @@ func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {
break
}
}
- if file == nil || file.UncompressedSize64 > mineruMaxImageBytes {
- return ""
- }
- rc, err := file.Open()
- if err != nil {
- return ""
- }
- data, err := io.ReadAll(io.LimitReader(rc, mineruMaxImageBytes+1))
- rc.Close()
- if err != nil || len(data) == 0 || len(data) > mineruMaxImageBytes {
+ data, ok := mineruReadBounded(file, mineruMaxImageBytes)
+ if !ok {
return ""
}
mediaType := mime.TypeByExtension(path.Ext(file.Name))
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index e119dec540..009d6ff561 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -218,7 +218,7 @@ func TestMinerUContentItemToJSONExplicitLayoutMapping(t *testing.T) {
{"table", map[string]any{"type": "table", "table_body": "<table></table>"}, "table", "table"},
} {
t.Run(tc.name, func(t *testing.T) {
- got := mineruContentItemToJSON(tc.item, nil)
+ got := mineruContentItemToJSON(tc.item, nil, nil)
if got == nil {
t.Fatal("item was dropped")
}
@@ -297,6 +297,119 @@ func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {
}
}
+// MinerU normalizes content_list bbox to 0..1000 against the page's point
+// size (pipeline _build_bbox / vlm mkcontent), so raw pass-through makes
+// highlights drift ~1.7x on A4. With middle.json in the zip the boxes must
+// come back as PDF points: bbox/1000 * page_size.
+func TestMinerUExtractJSONItemsScalesBBoxToPoints(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ list, err := zw.Create("output/doc/content_list.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, _ = list.Write([]byte(`[
+ {"type":"text","text":"Scaled","page_idx":0,"bbox":[100,200,600,400]}
+ ]`))
+ mid, err := zw.Create("output/doc/middle.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, _ = mid.Write([]byte(`{"pdf_info":[
+ {"page_idx":0,"page_size":[500,1000],"preproc_blocks":[]},
+ {"page_idx":1,"page_size":[400,800],"preproc_blocks":[]}
+ ]}`))
+ if err = zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ items, err := mineruExtractJSONItems(buf.Bytes())
+ if err != nil {
+ t.Fatalf("mineruExtractJSONItems: %v", err)
+ }
+ if len(items) != 1 {
+ t.Fatalf("len(items)=%d, want 1", len(items))
+ }
+ pos, ok := items[0]["_pdf_positions"].([][]any)
+ if !ok || len(pos) != 1 {
+ t.Fatalf("positions = %#v", items[0]["_pdf_positions"])
+ }
+ row := pos[0]
+ if page, _ := row[0].(int); page != 1 {
+ t.Fatalf("page = %#v, want 1", row[0])
+ }
+ // page 0 is 500x1000pt: x*0.5, y*1.0 — exact in float64.
+ want := []float64{50, 300, 200, 400} // left, right, top, bottom
+ for i, key := range []int{1, 2, 3, 4} {
+ v, _ := row[key].(float64)
+ if v != want[i] {
+ t.Fatalf("position[%d] = %v, want %v (row=%v)", key, row[key], want[i], row)
+ }
+ }
+}
+
+// MinerU V1 accepts partial jobs and preserves the source page indices in
+// its artifacts, so middle.json pdf_info can be sparse (page_idx 5 and 6
+// only). Sizes must be keyed by page_idx, not entry order: with two entries
+// the old slice lookup rejected index 5 and either passed the box through
+// unscaled or applied the wrong page's dimensions.
+func TestMinerUPageSizesKeysSparsePageIdx(t *testing.T) {
+ sizes := mineruPageSizes([]byte(`{"pdf_info":[
+ {"page_idx":5,"page_size":[400,800]},
+ {"page_idx":6,"page_size":[600,900]}
+ ]}`))
+ if len(sizes) != 2 {
+ t.Fatalf("len(sizes)=%d, want 2", len(sizes))
+ }
+ pos := mineruBBoxPosition(6, []any{500, 500, 1000, 1000}, sizes)
+ if pos == nil {
+ t.Fatal("expected a position, got nil")
+ }
+ // page 6 is 600x900pt: x*0.6, y*0.9.
+ want := []float64{300, 600, 450, 900} // left, right, top, bottom
+ for i, key := range []int{1, 2, 3, 4} {
+ v, _ := pos[key].(float64)
+ if v != want[i] {
+ t.Fatalf("position[%d] = %v, want %v (row=%v)", key, pos[key], want[i], pos)
+ }
+ }
+ // A page missing from a partial job stays passthrough, not another
+ // page's scale.
+ missing := mineruBBoxPosition(4, []any{100, 100, 200, 200}, sizes)
+ if missing == nil {
+ t.Fatal("expected a position for missing page, got nil")
+ }
+ if left, _ := missing[1].(float64); left != 100 {
+ t.Fatalf("left = %#v, want 100 (passthrough)", missing[1])
+ }
+}
+
+// Entries with no page_idx at all fall back to their position in pdf_info.
+func TestMinerUPageSizesFallsBackToEntryOrder(t *testing.T) {
+ sizes := mineruPageSizes([]byte(`{"pdf_info":[
+ {"page_size":[500,1000]},
+ {"page_size":[400,800]}
+ ]}`))
+ if got, ok := sizes[1]; !ok || got != [2]float64{400, 800} {
+ t.Fatalf("sizes[1] = %#v, want [400 800]", sizes)
+ }
+}
+
+// Without middle.json there is no trustworthy page size, so boxes keep the
+// pre-existing passthrough behavior (a known degraded mode, not a failure).
+func TestMinerUBBoxPositionWithoutPageSizesIsPassthrough(t *testing.T) {
+ pos := mineruBBoxPosition(2, []any{188, 142, 875, 393}, nil)
+ if pos == nil {
+ t.Fatal("expected a position, got nil")
+ }
+ if page, _ := pos[0].(int); page != 3 {
+ t.Fatalf("page = %#v, want 3", pos[0])
+ }
+ if left, _ := pos[1].(float64); left != 188 {
+ t.Fatalf("left = %#v, want 188 (unchanged)", pos[1])
+ }
+}
+
func TestDispatchMinerUPDFRequiresBaseURL(t *testing.T) {
t.Setenv("MINERU_APISERVER", "")
orig := resolveMinerUModelForDispatch