RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser/mineru): rescale content_list bbox from 0..1000 to PDF points (#21267)
Riven6d553f3875b7
diff --git a/internal/ingestion/component/pdf_postprocess_test.go b/internal/ingestion/component/pdf_postprocess_test.goindex 6a1f705bb3..40caee46df 100644--- a/internal/ingestion/component/pdf_postprocess_test.go+++ b/internal/ingestion/component/pdf_postprocess_test.go@@ -71,7 +71,7 @@ func TestPDFDispatchNormalizesBeforeOutputsAndEnhancement(t *testing.T) {}func TestMinerUTableRetainsCaptionAndFootnote(t *testing.T) {- item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><tr><td>Cell</td></tr></table>", "table_caption": []any{"Caption"}, "table_footnote": []any{"Footnote"}, "page_idx": 1, "bbox": []any{10.0, 20.0, 30.0, 40.0}}, nil)+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><tr><td>Cell</td></tr></table>", "table_caption": []any{"Caption"}, "table_footnote": []any{"Footnote"}, "page_idx": 1, "bbox": []any{10.0, 20.0, 30.0, 40.0}}, nil, nil)table, ok := item["table"].(*entity.TableData)if !ok || table == nil || table.Caption != "Caption" || len(table.CellTexts()) != 1 || len(table.CellTexts()[0]) != 1 || table.CellTexts()[0][0] != "Cell" || table.Page != 2 {t.Fatalf("structured table missing: %#v", item)@@ -82,7 +82,7 @@ func TestMinerUTableRetainsCaptionAndFootnote(t *testing.T) {}func TestMinerUTablePreservesEmbeddedCaption(t *testing.T) {- item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><caption>Embedded caption</caption><tr><td>Cell</td></tr></table>"}, nil)+ item := mineruContentItemToJSON(map[string]any{"type": "table", "table_body": "<table><caption>Embedded caption</caption><tr><td>Cell</td></tr></table>"}, nil, nil)table, ok := item["table"].(*entity.TableData)if !ok || table == nil || table.Caption != "Embedded caption" {t.Fatalf("embedded caption lost: %#v", item)@@ -111,7 +111,7 @@ func TestMinerUContentMarksBackendDescribedImages(t *testing.T) {"type": "image", "image_caption": []any{"Caption"},"vlm_description": "backend desc","page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},- }, nil)+ }, nil, nil)if withDesc == nil {t.Fatal("image item dropped")}@@ -124,7 +124,7 @@ func TestMinerUContentMarksBackendDescribedImages(t *testing.T) {withoutDesc := mineruContentItemToJSON(map[string]any{"type": "image", "image_caption": []any{"Caption"},"page_idx": 0, "bbox": []any{0.0, 0.0, 10.0, 10.0},- }, nil)+ }, nil, nil)if withoutDesc == nil {t.Fatal("image item dropped")}diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex d9e7610021..227d9d1d5b 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -817,11 +817,19 @@ func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod,return zipBytes, nil}-const mineruMaxImageBytes = 20 << 20+const (+ mineruMaxImageBytes = 20 << 20+ // mineruMaxMiddleBytes bounds middle.json, which carries the full+ // layout/span payload and can legitimately reach tens of megabytes on+ // long documents, while still refusing a zip-bomb-sized member.+ mineruMaxMiddleBytes = 64 << 20+)// mineruExtractJSONItems reads MinerU content_list.json from a zip and// emits JSON parser items with captions, embedded zip images, and// 1-indexed _pdf_positions so jump-to-source works like DeepDoc.+// Bounding boxes are scaled from MinerU's 0..1000 space to PDF points+// using the per-page sizes in middle.json when that member is present.func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))if err != nil {@@ -829,7 +837,7 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {}images := make(map[string]*zip.File)- var contentList []byte+ var contentList, middleJSON []bytefor _, f := range zipReader.File {if strings.HasSuffix(f.Name, "content_list.json") && len(contentList) == 0 {rc, openErr := f.Open()@@ -845,6 +853,15 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {return nil, fmt.Errorf("read %s: %w", f.Name, readErr)}}+ // middle.json is optional: it only carries page_size, which lets+ // positions be converted to points. A zip without it keeps the+ // pre-existing passthrough behavior instead of failing the parse,+ // and so does an oversized (or lying-header) member.+ if strings.HasSuffix(f.Name, "middle.json") && len(middleJSON) == 0 {+ if data, ok := mineruReadBounded(f, mineruMaxMiddleBytes); ok {+ middleJSON = data+ }+ }if strings.HasPrefix(strings.ToLower(mime.TypeByExtension(path.Ext(f.Name))), "image/") {clean := strings.TrimPrefix(path.Clean(f.Name), "./")images[path.Base(f.Name)] = f@@ -861,9 +878,10 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {return nil, fmt.Errorf("parse content_list.json: %w", err)}+ pageSizes := mineruPageSizes(middleJSON)out := make([]map[string]any, 0, len(rawItems))for _, item := range rawItems {- converted := mineruContentItemToJSON(item, images)+ converted := mineruContentItemToJSON(item, images, pageSizes)if converted != nil {out = append(out, converted)}@@ -874,7 +892,44 @@ func mineruExtractJSONItems(zipBytes []byte) ([]map[string]any, error) {return out, nil}-func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) map[string]any {+// mineruPageSizes parses the per-page page_size ([width, height] in PDF+// points, written by MinerU's pdfium page.get_size()) out of middle.json,+// keyed by 0-based page_idx. A map rather than a slice because partial+// jobs are legal and preserve the source page indices, so pdf_info can be+// sparse (entries for pages 5 and 6 only). Entries without a usable+// page_idx fall back to their position in pdf_info. Returns nil when+// middle.json is absent or unparseable.+func mineruPageSizes(middleJSON []byte) map[int][2]float64 {+ if len(middleJSON) == 0 {+ return nil+ }+ var parsed struct {+ PdfInfo []struct {+ PageIdx *int `json:"page_idx"`+ PageSize []float64 `json:"page_size"`+ } `json:"pdf_info"`+ }+ if err := json.Unmarshal(middleJSON, &parsed); err != nil || len(parsed.PdfInfo) == 0 {+ return nil+ }+ sizes := make(map[int][2]float64, len(parsed.PdfInfo))+ for i, page := range parsed.PdfInfo {+ if len(page.PageSize) < 2 {+ continue+ }+ idx := i+ if page.PageIdx != nil && *page.PageIdx >= 0 {+ idx = *page.PageIdx+ }+ sizes[idx] = [2]float64{page.PageSize[0], page.PageSize[1]}+ }+ if len(sizes) == 0 {+ return nil+ }+ return sizes+}++func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, pageSizes map[int][2]float64) map[string]any {typ, _ := item["type"].(string)typ = strings.ToLower(strings.TrimSpace(typ))switch typ {@@ -935,7 +990,7 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) mif imageURI != "" {jsonItem["image"] = imageURI}- if positions := mineruItemPositions(item); len(positions) > 0 {+ if positions := mineruItemPositions(item, pageSizes); len(positions) > 0 {jsonItem["_pdf_positions"] = positionsjsonItem["positions"] = positionsif page, ok := positions[0][0].(int); ok {@@ -997,7 +1052,7 @@ func mineruItemText(item map[string]any, typ string) string {return strings.TrimSpace(strings.Join(parts, "\n"))}-func mineruItemPositions(item map[string]any) [][]any {+func mineruItemPositions(item map[string]any, pageSizes map[int][2]float64) [][]any {if raw, ok := item["_mineru_positions"].([]any); ok {var out [][]anyfor _, entry := range raw {@@ -1005,7 +1060,7 @@ func mineruItemPositions(item map[string]any) [][]any {if !ok {continue}- if pos := mineruBBoxPosition(posMap["page_idx"], posMap["bbox"]); pos != nil {+ if pos := mineruBBoxPosition(posMap["page_idx"], posMap["bbox"], pageSizes); pos != nil {out = append(out, pos)}}@@ -1013,13 +1068,13 @@ func mineruItemPositions(item map[string]any) [][]any {return out}}- if pos := mineruBBoxPosition(item["page_idx"], item["bbox"]); pos != nil {+ if pos := mineruBBoxPosition(item["page_idx"], item["bbox"], pageSizes); pos != nil {return [][]any{pos}}return nil}-func mineruBBoxPosition(pageRaw, bboxRaw any) []any {+func mineruBBoxPosition(pageRaw, bboxRaw any, pageSizes map[int][2]float64) []any {pageIdx, ok := mineruJSONInt(pageRaw)if !ok {return nil@@ -1028,9 +1083,23 @@ func mineruBBoxPosition(pageRaw, bboxRaw any) []any {if len(bbox) < 4 {return nil}- // MinerU page_idx is 0-based; bbox is [x0, top, x1, bottom].+ // MinerU page_idx is 0-based; bbox is [x0, top, x1, bottom] in MinerU's+ // content_list coordinate space. Both the pipeline and vlm backends+ // normalize that space to 0..1000 against the page's point size+ // (pipeline _build_bbox / vlm mkcontent: int(x * 1000 / page_width)), so+ // a box is not comparable to the PDF-point space the rest of the+ // pipeline expects (position_int, the web highlighter, CropSection*)+ // until it is rescaled. When middle.json carried the page size, undo the+ // normalization exactly; without it, keep the pre-existing passthrough.+ left, right, top, bottom := bbox[0], bbox[2], bbox[1], bbox[3]+ if size, ok := pageSizes[pageIdx]; ok && size[0] > 0 && size[1] > 0 {+ left *= size[0] / 1000.0+ right *= size[0] / 1000.0+ top *= size[1] / 1000.0+ bottom *= size[1] / 1000.0+ }// Downstream position_int is 1-based [page, left, right, top, bottom].- return []any{pageIdx + 1, bbox[0], bbox[2], bbox[1], bbox[3]}+ return []any{pageIdx + 1, left, right, top, bottom}}func mineruJSONInt(raw any) (int, bool) {@@ -1085,6 +1154,26 @@ func mineruJSONFloatSlice(raw any) []float64 {}}+// mineruReadBounded reads a zip member, refusing anything that is empty or+// larger than limit bytes. The size is checked both against the declared+// uncompressed size and against the bytes actually produced, since a+// hostile member can understate the former.+func mineruReadBounded(f *zip.File, limit int64) ([]byte, bool) {+ if f == nil || f.UncompressedSize64 > uint64(limit) {+ return nil, false+ }+ rc, err := f.Open()+ if err != nil {+ return nil, false+ }+ data, err := io.ReadAll(io.LimitReader(rc, limit+1))+ rc.Close()+ if err != nil || len(data) == 0 || int64(len(data)) > limit {+ return nil, false+ }+ return data, true+}+func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {imgPath = strings.TrimSpace(strings.ReplaceAll(imgPath, "\\", "/"))if imgPath == "" {@@ -1098,16 +1187,8 @@ func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {break}}- if file == nil || file.UncompressedSize64 > mineruMaxImageBytes {- return ""- }- rc, err := file.Open()- if err != nil {- return ""- }- data, err := io.ReadAll(io.LimitReader(rc, mineruMaxImageBytes+1))- rc.Close()- if err != nil || len(data) == 0 || len(data) > mineruMaxImageBytes {+ data, ok := mineruReadBounded(file, mineruMaxImageBytes)+ if !ok {return ""}mediaType := mime.TypeByExtension(path.Ext(file.Name))diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex e119dec540..009d6ff561 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -218,7 +218,7 @@ func TestMinerUContentItemToJSONExplicitLayoutMapping(t *testing.T) {{"table", map[string]any{"type": "table", "table_body": "<table></table>"}, "table", "table"},} {t.Run(tc.name, func(t *testing.T) {- got := mineruContentItemToJSON(tc.item, nil)+ got := mineruContentItemToJSON(tc.item, nil, nil)if got == nil {t.Fatal("item was dropped")}@@ -297,6 +297,119 @@ func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {}}+// MinerU normalizes content_list bbox to 0..1000 against the page's point+// size (pipeline _build_bbox / vlm mkcontent), so raw pass-through makes+// highlights drift ~1.7x on A4. With middle.json in the zip the boxes must+// come back as PDF points: bbox/1000 * page_size.+func TestMinerUExtractJSONItemsScalesBBoxToPoints(t *testing.T) {+ var buf bytes.Buffer+ zw := zip.NewWriter(&buf)+ list, err := zw.Create("output/doc/content_list.json")+ if err != nil {+ t.Fatal(err)+ }+ _, _ = list.Write([]byte(`[+ {"type":"text","text":"Scaled","page_idx":0,"bbox":[100,200,600,400]}+ ]`))+ mid, err := zw.Create("output/doc/middle.json")+ if err != nil {+ t.Fatal(err)+ }+ _, _ = mid.Write([]byte(`{"pdf_info":[+ {"page_idx":0,"page_size":[500,1000],"preproc_blocks":[]},+ {"page_idx":1,"page_size":[400,800],"preproc_blocks":[]}+ ]}`))+ if err = zw.Close(); err != nil {+ t.Fatal(err)+ }++ items, err := mineruExtractJSONItems(buf.Bytes())+ if err != nil {+ t.Fatalf("mineruExtractJSONItems: %v", err)+ }+ if len(items) != 1 {+ t.Fatalf("len(items)=%d, want 1", len(items))+ }+ pos, ok := items[0]["_pdf_positions"].([][]any)+ if !ok || len(pos) != 1 {+ t.Fatalf("positions = %#v", items[0]["_pdf_positions"])+ }+ row := pos[0]+ if page, _ := row[0].(int); page != 1 {+ t.Fatalf("page = %#v, want 1", row[0])+ }+ // page 0 is 500x1000pt: x*0.5, y*1.0 — exact in float64.+ want := []float64{50, 300, 200, 400} // left, right, top, bottom+ for i, key := range []int{1, 2, 3, 4} {+ v, _ := row[key].(float64)+ if v != want[i] {+ t.Fatalf("position[%d] = %v, want %v (row=%v)", key, row[key], want[i], row)+ }+ }+}++// MinerU V1 accepts partial jobs and preserves the source page indices in+// its artifacts, so middle.json pdf_info can be sparse (page_idx 5 and 6+// only). Sizes must be keyed by page_idx, not entry order: with two entries+// the old slice lookup rejected index 5 and either passed the box through+// unscaled or applied the wrong page's dimensions.+func TestMinerUPageSizesKeysSparsePageIdx(t *testing.T) {+ sizes := mineruPageSizes([]byte(`{"pdf_info":[+ {"page_idx":5,"page_size":[400,800]},+ {"page_idx":6,"page_size":[600,900]}+ ]}`))+ if len(sizes) != 2 {+ t.Fatalf("len(sizes)=%d, want 2", len(sizes))+ }+ pos := mineruBBoxPosition(6, []any{500, 500, 1000, 1000}, sizes)+ if pos == nil {+ t.Fatal("expected a position, got nil")+ }+ // page 6 is 600x900pt: x*0.6, y*0.9.+ want := []float64{300, 600, 450, 900} // left, right, top, bottom+ for i, key := range []int{1, 2, 3, 4} {+ v, _ := pos[key].(float64)+ if v != want[i] {+ t.Fatalf("position[%d] = %v, want %v (row=%v)", key, pos[key], want[i], pos)+ }+ }+ // A page missing from a partial job stays passthrough, not another+ // page's scale.+ missing := mineruBBoxPosition(4, []any{100, 100, 200, 200}, sizes)+ if missing == nil {+ t.Fatal("expected a position for missing page, got nil")+ }+ if left, _ := missing[1].(float64); left != 100 {+ t.Fatalf("left = %#v, want 100 (passthrough)", missing[1])+ }+}++// Entries with no page_idx at all fall back to their position in pdf_info.+func TestMinerUPageSizesFallsBackToEntryOrder(t *testing.T) {+ sizes := mineruPageSizes([]byte(`{"pdf_info":[+ {"page_size":[500,1000]},+ {"page_size":[400,800]}+ ]}`))+ if got, ok := sizes[1]; !ok || got != [2]float64{400, 800} {+ t.Fatalf("sizes[1] = %#v, want [400 800]", sizes)+ }+}++// Without middle.json there is no trustworthy page size, so boxes keep the+// pre-existing passthrough behavior (a known degraded mode, not a failure).+func TestMinerUBBoxPositionWithoutPageSizesIsPassthrough(t *testing.T) {+ pos := mineruBBoxPosition(2, []any{188, 142, 875, 393}, nil)+ if pos == nil {+ t.Fatal("expected a position, got nil")+ }+ if page, _ := pos[0].(int); page != 3 {+ t.Fatalf("page = %#v, want 3", pos[0])+ }+ if left, _ := pos[1].(float64); left != 188 {+ t.Fatalf("left = %#v, want 188 (unchanged)", pos[1])+ }+}+func TestDispatchMinerUPDFRequiresBaseURL(t *testing.T) {t.Setenv("MINERU_APISERVER", "")orig := resolveMinerUModelForDispatch