Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): stop MinerU JSON path from inlining zip figure bytes (#21803)

Sourav Rajvi62b94c03a20c
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 5c934d3037..a9276e3cb0 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -374,19 +374,24 @@ func canonicalChunkID(docID string, ck schema.ChunkDoc) string {
}
// chunkIDSource returns the identity string hashed into a chunk id. For a
-// media chunk whose canonical text is empty — a captionless image — hashing
+// media chunk whose canonical text is empty - a captionless image - hashing
// the text alone would give every such chunk in a document the same id
// (ChunkID(docID, "")), silently overwriting images in MinIO and merging
-// index entries, so a stable per-image identity is folded in instead.
-// Non-empty media text also folds its region in: two figures can share a
-// caption ("Figure", "图"), and their previews must not collapse onto one id.
-// Text chunks hash their text unchanged.
+// index entries. Position-only figures use the crop region; otherwise a
+// stable per-image payload identity is folded in. Non-empty media text also
+// folds its region in: two figures can share a caption (for example "Figure"),
+// and their previews must not collapse onto one id. Text chunks hash their text
+// unchanged.
func chunkIDSource(ck schema.ChunkDoc) string {
text := canonicalChunkText(ck)
+ matrix := mergePositionMatrix(ck.PDFPositions, ck.Positions)
if text == "" {
+ if isMediaChunk(ck) && len(matrix) > 0 {
+ return mediaTextIdentity(text, true, matrix)
+ }
return mediaChunkIdentity(ck.Image, ck.ImgID, ck.MediaRef)
}
- return mediaTextIdentity(text, isMediaChunk(ck), mergePositionMatrix(ck.PDFPositions, ck.Positions))
+ return mediaTextIdentity(text, isMediaChunk(ck), matrix)
}
// mediaTextIdentity appends a region digest to a media chunk's text identity.
@@ -430,7 +435,8 @@ func chunkPositionMatrix(ck map[string]any) [][]any {
// a digest of the image payload when present (identical images collapse onto
// one id, which is harmless — they carry identical bytes), else the
// content-addressed media reference, else the uploaded image id. Empty when
-// the chunk has none of them, leaving the id the plain empty-text hash.
+// the chunk has none of them; callers with PDF crop regions use
+// mediaTextIdentity instead.
func mediaChunkIdentity(image, imgID string, ref *entity.MediaRef) string {
if image != "" {
sum := sha256.Sum256([]byte(image))
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index da99a68ae7..ab6ff4d21e 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -1461,10 +1461,19 @@ func sliceGeneralSplitPieces(unit schema.ChunkDoc, parts []string) []schema.Chun
return out
}
+// finalizeGeneralChunks drops empty chunks after media context and on-demand
+// PDF cropping have run, then applies the shared child-delimiter split.
func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Regexp) []schema.ChunkDoc {
visible := make([]schema.ChunkDoc, 0, len(chunks))
for _, chunk := range chunks {
chunk.Text = removeTag(chunk.Text)
+ // Keep a chunk whenever it still carries retrievable content. A
+ // streamed crop upload sets ImgID and clears Image, so an uploaded
+ // media chunk (text may be only a position tag, or empty) must be
+ // retained via its ImgID - otherwise the already-uploaded image is
+ // dropped and its MinIO object orphaned. Position-only media without
+ // a crop result is dropped here; cropImageChunks runs immediately
+ // before this pass.
if strings.TrimSpace(chunk.Text) == "" && !hasMediaPayload(chunk) {
continue
}
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.go
index 7f4a6de87e..db091477d5 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go
@@ -127,8 +127,8 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
if !needsCrop(ck) || ck.Image != "" {
continue
}
- // A media chunk whose finalized text is empty — no caption, no vision
- // description, no media context — is not retrievable: its embedding
+ // A media chunk whose finalized text is empty - no caption, no vision
+ // description, no media context - is not retrievable: its embedding
// would be the empty-string vector, so retrieval can never surface it,
// and every captionless image/table in a document would otherwise
// collapse onto the same canonical id (ChunkID(docID, "")), each
@@ -138,7 +138,8 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
// mirrors Python's token_chunker drop of empty-text media chunks.
// Text chunks are excluded: a text body always carries content, and the
// Chunker-1.3 preview restore must still run for positioned text
- // regions.
+ // regions. Region-based chunk ids still apply when context or captions
+ // make the crop path run.
if isMediaChunk(ck) && canonicalChunkText(ck) == "" {
continue
}
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
index ce3e519aa0..a876edad1d 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
@@ -572,8 +572,8 @@ func TestCropImageChunks_DistinctIdsForSameTextMedia(t *testing.T) {
}
// TestCropImageChunks_SkipsEmptyTextMedia verifies that a media chunk whose
-// finalized text is empty — no caption, no vision description, no media
-// context — is NOT cropped or uploaded. Such a chunk is not retrievable
+// finalized text is empty - no caption, no vision description, no media
+// context - is NOT cropped or uploaded. Such a chunk is not retrievable
// (its embedding would be the empty-string vector) and, under the old
// behavior, collapsed every captionless image/table in a document onto the
// same canonical id ChunkID(docID, ""), overwriting each other's MinIO
@@ -592,13 +592,13 @@ func TestCropImageChunks_SkipsEmptyTextMedia(t *testing.T) {
t.Cleanup(func() { ChunkImageUploader = orig })
eng := mockCropEngine{}
- pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ posA := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ posB := jsonPositions(t, []float64{1, 200, 300, 400, 500})
+ posC := jsonPositions(t, []float64{2, 10, 80, 20, 90})
emptyMedia := []schema.ChunkDoc{
- {DocType: "image", PDFPositions: pos}, // no text
- {DocType: "table", PDFPositions: pos}, // no text
- {DocType: "image", Text: "@@1\t2##", PDFPositions: pos}, // position tag only, no body
- {DocType: "table", ContextAbove: "", ContextBelow: "", PDFPositions: pos}, // explicit empty
- {DocType: "image", PDFPositions: pos}, // doc-type fallback, no text
+ {DocType: "image", PDFPositions: posA},
+ {DocType: "table", PDFPositions: posB},
+ {DocType: "image", Text: "@@1\t2##", PDFPositions: posC},
}
out := cropImageChunks(ctx, eng, emptyMedia)
if len(out) != len(emptyMedia) {
@@ -618,7 +618,7 @@ func TestCropImageChunks_SkipsEmptyTextMedia(t *testing.T) {
// Sanity: a captioned media chunk is still cropped and uploaded.
captioned := []schema.ChunkDoc{
- {DocType: "image", Text: "a real figure caption", PDFPositions: pos},
+ {DocType: "image", Text: "a real figure caption", PDFPositions: posA},
}
out2 := cropImageChunks(ctx, eng, captioned)
if len(out2) != 1 {
@@ -638,6 +638,28 @@ func TestCropImageChunks_SkipsEmptyTextMedia(t *testing.T) {
}
}
+func TestFinalizeGeneralChunks_DropsUncroppedPositionOnlyMedia(t *testing.T) {
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ uncropped := []schema.ChunkDoc{{DocType: "image", PDFPositions: pos}}
+ if kept := finalizeGeneralChunks(uncropped, nil); len(kept) != 0 {
+ t.Fatalf("finalize kept %d, want uncropped position-only media dropped", len(kept))
+ }
+}
+
+func TestCanonicalChunkIDUsesRegionForCaptionlessPositionedMedia(t *testing.T) {
+ pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
+ ck := schema.ChunkDoc{
+ DocType: "image",
+ PDFPositions: pos,
+ Image: "data:image/png;base64,QUFBQQ==",
+ }
+ regionID := canonicalChunkID("doc1", ck)
+ payloadID := common.ChunkID("doc1", mediaChunkIdentity(ck.Image, "", nil))
+ if regionID == payloadID {
+ t.Fatalf("region id %q must differ from payload hash id %q", regionID, payloadID)
+ }
+}
+
// TestCropImageChunks_StreamingUploadUsesCanonicalID is the regression test
// for the most common chunk shape: an image/table chunk whose Text still
// carries parser position tags (@@…##) but has NO media context
diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go
index d575614aab..845aae7d95 100644
--- a/internal/ingestion/component/chunker/register.go
+++ b/internal/ingestion/component/chunker/register.go
@@ -67,6 +67,8 @@ type imageUploadDecorator struct {
inner runtime.Component
}
+// Invoke runs the wrapped chunker, stamps deterministic chunk ids, and uploads
+// any remaining inline or deferred media before the chunks leave the component.
func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
out, err := d.inner.Invoke(ctx, db, inputs)
if err != nil {
@@ -140,9 +142,15 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
return nil, err
}
if text == "" {
- image, _ := ck["image"].(string)
- imgID, _ := ck["img_id"].(string)
- text = mediaChunkIdentity(image, imgID, entity.MediaRefFromAny(ck["media_ref"]))
+ docType, _ := ck["doc_type_kwd"].(string)
+ matrix := chunkPositionMatrix(ck)
+ if isMediaDocType(docType) && len(matrix) > 0 {
+ text = mediaTextIdentity("", true, matrix)
+ } else {
+ image, _ := ck["image"].(string)
+ imgID, _ := ck["img_id"].(string)
+ text = mediaChunkIdentity(image, imgID, entity.MediaRefFromAny(ck["media_ref"]))
+ }
} else {
docType, _ := ck["doc_type_kwd"].(string)
text = mediaTextIdentity(text, isMediaDocType(docType), chunkPositionMatrix(ck))
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index b2b6113f9d..700f798819 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -686,6 +686,8 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
// without duplicating the fold. Inline bytes, an uploaded ID or a
// deferred reference also make the chunk retrievable; dropping it here
// would lose the image before the materializer can resolve or reject it.
+ // Position-only media without a crop result is dropped here; crop above
+ // already attempted the on-demand PDF handoff.
if strings.TrimSpace(removeTag(schema.ContextualText(m))) == "" && !hasMediaPayload(m) {
continue
}
diff --git a/internal/ingestion/component/mineru_image_budget_test.go b/internal/ingestion/component/mineru_image_budget_test.go
new file mode 100644
index 0000000000..c204866e9c
--- /dev/null
+++ b/internal/ingestion/component/mineru_image_budget_test.go
@@ -0,0 +1,279 @@
+package component
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/base64"
+ "encoding/json"
+ "strconv"
+ "strings"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/parser/media"
+)
+
+func mineruTestZipImages(t *testing.T, files map[string][]byte) map[string]*zip.File {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ for name, data := range files {
+ w, err := zw.Create(name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = w.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ images := map[string]*zip.File{}
+ for _, f := range zr.File {
+ images[f.Name] = f
+ }
+ return images
+}
+
+func TestMinerUZipImageDataURIHonorsMediaBudget(t *testing.T) {
+ images := mineruTestZipImages(t, map[string][]byte{
+ "images/a.png": bytes.Repeat([]byte{1}, 100),
+ "images/b.png": bytes.Repeat([]byte{2}, 100),
+ })
+ budget := media.NewBudgetWithLimits(media.Limits{
+ ImageMaxBytes: 200,
+ MaxImagesPerRun: 10,
+ MaxBytesPerRun: 150,
+ })
+ first := mineruZipImageDataURI(images, "images/a.png", budget)
+ if !strings.HasPrefix(first, "data:image/png;base64,") {
+ t.Fatalf("first image not inlined: %q", first)
+ }
+ second := mineruZipImageDataURI(images, "images/b.png", budget)
+ if second != "" {
+ t.Fatalf("second image should exceed run byte budget, got %q", second)
+ }
+ warnings := budget.Warnings()
+ if len(warnings) == 0 {
+ t.Fatal("expected byte-budget warning")
+ }
+ if got := mineruZipImageDataURI(images, "images/b.png", nil); got == "" {
+ t.Fatal("nil budget must not apply run limits")
+ }
+}
+
+func TestMinerUImageItemKeptWhenZipMemberExists(t *testing.T) {
+ images := mineruTestZipImages(t, map[string][]byte{"images/a.png": bytes.Repeat([]byte{1}, 8)})
+ item := map[string]any{
+ "type": "image", "img_path": "images/a.png",
+ "page_idx": 0, "bbox": []any{1.0, 2.0, 3.0, 4.0},
+ }
+ got := mineruContentItemToJSONBudget(item, images, map[int][2]float64{0: {612, 792}}, media.NewBudget())
+ if got == nil {
+ t.Fatal("image item with a zip member must be kept without inlining")
+ }
+ if _, ok := got["image"]; ok {
+ t.Fatalf("image must not be inlined on JSON path: %#v", got)
+ }
+ if got["doc_type_kwd"] != "image" {
+ t.Fatalf("doc_type_kwd = %v, want image", got["doc_type_kwd"])
+ }
+ if _, ok := got["_pdf_positions"]; !ok {
+ t.Fatalf("positions must be kept: %#v", got)
+ }
+ missing := map[string]any{"type": "image", "img_path": "images/none.png"}
+ if mineruContentItemToJSONBudget(missing, images, nil, nil) != nil {
+ t.Fatal("image with no text and no zip member must still be dropped")
+ }
+ bboxOnly := map[string]any{
+ "type": "image", "page_idx": 0, "bbox": []any{1.0, 2.0, 3.0, 4.0},
+ }
+ gotBBox := mineruContentItemToJSONBudget(bboxOnly, nil, map[int][2]float64{0: {612, 792}}, nil)
+ if gotBBox == nil {
+ t.Fatal("captionless figure with a crop box must be kept for PDF cropping")
+ }
+ if _, ok := gotBBox["_pdf_positions"]; !ok {
+ t.Fatalf("positions must be kept: %#v", gotBBox)
+ }
+ zipOnly := map[string]any{"type": "image", "img_path": "images/a.png"}
+ if mineruContentItemToJSONBudget(zipOnly, images, nil, nil) != nil {
+ t.Fatal("zip-only figure without a crop region must be dropped")
+ }
+}
+
+func TestMinerUStructuredFigureKeptWithoutCaptionOrZip(t *testing.T) {
+ payload := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{7}, 16))
+ dataURI := "data:image/png;base64," + payload
+ item := map[string]any{
+ "type": "image", "image_data_uri": dataURI,
+ "page_idx": 0, "bbox": []any{1.0, 2.0, 3.0, 4.0},
+ }
+ got := mineruContentItemToJSONBudget(item, nil, map[int][2]float64{0: {612, 792}}, media.NewBudget())
+ if got == nil {
+ t.Fatal("structured figure with positions must be kept without caption or zip member")
+ }
+ if _, ok := got["image"]; ok {
+ t.Fatalf("structured data URI must not be copied: %#v", got)
+ }
+ if _, ok := got["_pdf_positions"]; !ok {
+ t.Fatalf("positions must be kept: %#v", got)
+ }
+}
+
+func TestMinerUStructuredDataURIOverImageMaxBytesWarns(t *testing.T) {
+ t.Setenv(common.EnvMediaImageMaxBytes, "100")
+ payload := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{9}, 200))
+ dataURI := "data:image/png;base64," + payload
+ budget := media.NewBudget()
+ mineruAccountStructuredDataURI(budget, dataURI)
+ warnings := budget.Warnings()
+ if len(warnings) == 0 {
+ t.Fatal("expected per-image limit warning")
+ }
+ if !strings.Contains(warnings[0], "100-byte per-image limit") {
+ t.Fatalf("warnings = %v", warnings)
+ }
+ mineruAccountStructuredDataURI(budget, "data:image/png;base64,????")
+ if got := budget.Warnings(); len(got) != len(warnings) {
+ t.Fatalf("invalid payload must not add a warning: %v", got)
+ }
+}
+
+func TestMinerUStructuredDataURINotCopiedAndWarnsOnBudget(t *testing.T) {
+ payload := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{9}, 200))
+ dataURI := "data:image/png;base64," + payload
+ budget := media.NewBudgetWithLimits(media.Limits{
+ ImageMaxBytes: 512,
+ MaxImagesPerRun: 1,
+ MaxBytesPerRun: 100,
+ })
+ item := map[string]any{
+ "type": "image", "image_data_uri": dataURI,
+ "image_caption": []any{"cap"},
+ "page_idx": 0, "bbox": []any{1.0, 2.0, 3.0, 4.0},
+ }
+ got := mineruContentItemToJSONBudget(item, nil, map[int][2]float64{0: {612, 792}}, budget)
+ if got == nil {
+ t.Fatal("expected image item")
+ }
+ if _, ok := got["image"]; ok {
+ t.Fatalf("structured data URI must not be copied: %#v", got)
+ }
+ if len(budget.Warnings()) == 0 {
+ t.Fatalf("expected budget warnings, got none")
+ }
+}
+
+func TestMinerUExtractJSONItemsWarnsOnZipOnlyFigure(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ list := []map[string]any{
+ {"type": "image", "img_path": "images/a.png"},
+ {"type": "image", "img_path": "images/a.png", "page_idx": 0, "bbox": []any{1.0, 2.0, 3.0, 4.0}},
+ }
+ raw, err := json.Marshal(list)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cl, err := zw.Create("content_list.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = cl.Write(raw); err != nil {
+ t.Fatal(err)
+ }
+ img, err := zw.Create("images/a.png")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = img.Write(bytes.Repeat([]byte{1}, 8)); err != nil {
+ t.Fatal(err)
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ items, warnings, err := mineruExtractJSONItems(buf.Bytes(), map[int][2]float64{0: {612, 792}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(items) != 1 {
+ t.Fatalf("len(items)=%d, want 1 positioned figure kept", len(items))
+ }
+ found := false
+ for _, w := range warnings {
+ if strings.Contains(w, "zip bytes but no PDF crop region") {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Fatalf("warnings = %v, want zip-only omission warning", warnings)
+ }
+}
+
+func TestMinerUExtractJSONItemsManyImagesStaySmall(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ list := make([]map[string]any, 0, 50)
+ for i := 0; i < 50; i++ {
+ name := "images/fig" + strconv.Itoa(i) + ".png"
+ list = append(list, map[string]any{
+ "type": "image", "img_path": name,
+ "image_caption": []any{"caption"},
+ "page_idx": 0, "bbox": []any{10.0, 10.0, 20.0, 20.0},
+ })
+ }
+ raw, err := json.Marshal(list)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cl, err := zw.Create("content_list.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = cl.Write(raw); err != nil {
+ t.Fatal(err)
+ }
+ for i := 0; i < 50; i++ {
+ name := "images/fig" + strconv.Itoa(i) + ".png"
+ img, err := zw.Create(name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = img.Write(bytes.Repeat([]byte{1, 2, 3}, 4000)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ items, warnings, err := mineruExtractJSONItems(buf.Bytes(), map[int][2]float64{0: {612, 792}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(items) != 50 {
+ t.Fatalf("len(items)=%d, want 50", len(items))
+ }
+ encoded, err := json.Marshal(items)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(encoded) > 512*1024 {
+ t.Fatalf("serialized items too large (%d bytes); zip images were likely inlined", len(encoded))
+ }
+ for _, item := range items {
+ if _, ok := item["image"]; ok {
+ t.Fatalf("unexpected inlined image on item %#v", item)
+ }
+ }
+ if len(warnings) != 0 {
+ t.Fatalf("unexpected warnings: %v", warnings)
+ }
+}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index b9e48f17b8..4da7e32312 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -24,6 +24,7 @@ import (
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/media"
"ragflow/internal/parser/parser"
"ragflow/internal/parser/tableutil"
"ragflow/internal/rag/prompts"
@@ -641,13 +642,14 @@ func dispatchMinerUPDF(
}
sizes, geometryErr := mineruPageSizes(ctx, binary)
- items, err := mineruExtractJSONItems(zipBytes, sizes)
+ items, mediaWarnings, err := mineruExtractJSONItems(zipBytes, sizes)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", err)
}
res := mineruJSONParseResult(filename, items)
appendMineruGeometryWarning(&res, geometryErr)
+ res.Warnings = append(res.Warnings, mediaWarnings...)
return res, nil
}
@@ -663,10 +665,11 @@ func finishMinerURemoteResult(ctx context.Context, filename string, binary []byt
if len(zipBytes) > 0 {
sizes, geometryErr = mineruPageSizes(ctx, binary)
}
- items, extractErr := mineruExtractJSONItems(zipBytes, sizes)
+ items, mediaWarnings, extractErr := mineruExtractJSONItems(zipBytes, sizes)
if extractErr == nil && len(items) > 0 {
res := mineruJSONParseResult(filename, items)
appendMineruGeometryWarning(&res, geometryErr)
+ res.Warnings = append(res.Warnings, mediaWarnings...)
return res, nil
}
if markdown != "" {
@@ -906,7 +909,6 @@ func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary
}
const (
- mineruMaxImageBytes = 20 << 20
// mineruMaxMiddleBytes bounds middle.json, which carries the full
// layout/span payload and can legitimately reach tens of megabytes on
// long documents, while still refusing a zip-bomb-sized member.
@@ -919,17 +921,16 @@ const mineruContentBBoxScale = 1000.0
// mineruExtractJSONItems reads MinerU content_list.json, or MinerU 4
// structured_content.json when the zip has no content list, and emits JSON
-// parser items with captions, embedded zip images, and 1-indexed
-// _pdf_positions so jump-to-source works like DeepDoc. pdfPageSizes comes
-// from the source PDF (pdfium); middle.json in the zip fills any missing
-// pages. content_list bboxes are normalized to 0-1000, so positions require
-// a known page box in PDF points. Structured content boxes are 0-1 and are
-// scaled onto that same 0-1000 space before conversion.
-func mineruExtractJSONItems(zipBytes []byte, pdfPageSizes map[int][2]float64) ([]map[string]any, error) {
+// parser items with captions and 1-indexed _pdf_positions so jump-to-source
+// works like DeepDoc. Zip figure bytes are not inlined here; the chunker crops
+// from the source PDF using positions. pdfPageSizes comes from the source PDF
+// (pdfium); middle.json in the zip fills any missing pages.
+func mineruExtractJSONItems(zipBytes []byte, pdfPageSizes map[int][2]float64) ([]map[string]any, []string, error) {
zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
if err != nil {
- return nil, fmt.Errorf("open zip: %w", err)
+ return nil, nil, fmt.Errorf("open zip: %w", err)
}
+ budget := media.NewBudget()
images := make(map[string]*zip.File)
var contentList, middleJSON, structured []byte
@@ -942,7 +943,7 @@ func mineruExtractJSONItems(zipBytes []byte, pdfPageSizes map[int][2]float64) ([
structured, err = mineruReadZipFile(f)
}
if err != nil {
- return nil, err
+ return nil, nil, err
}
// middle.json is optional: it only carries page_size, which lets
// positions be converted to points. A zip without it keeps the
@@ -955,9 +956,12 @@ func mineruExtractJSONItems(zipBytes []byte, pdfPageSizes map[int][2]float64) ([
}
if strings.HasPrefix(strings.ToLower(mime.TypeByExtension(path.Ext(name))), "image/") {
clean := strings.TrimPrefix(path.Clean(name), "./")
- images[path.Base(name)] = f
images[clean] = f
images[name] = f
+ base := path.Base(name)
+ if existing, ok := images[base]; !ok || existing.Name == f.Name {
+ images[base] = f
+ }
}
}
@@ -965,31 +969,44 @@ func mineruExtractJSONItems(zipBytes []byte, pdfPageSizes map[int][2]float64) ([
switch {
case len(contentList) > 0:
if err = json.Unmarshal(contentList, &rawItems); err != nil {
- return nil, fmt.Errorf("parse content_list.json: %w", err)
+ return nil, nil, fmt.Errorf("parse content_list.json: %w", err)
}
case len(structured) > 0:
rawItems, err = mineruItemsFromStructuredContent(structured)
if err != nil {
- return nil, err
+ return nil, nil, err
}
default:
- return nil, fmt.Errorf("content_list.json not found in MinerU zip")
+ return nil, nil, fmt.Errorf("content_list.json not found in MinerU zip")
}
pageSizes := mergeMineruPageSizes(pdfPageSizes, mineruPageSizesFromMiddleJSON(middleJSON))
out := make([]map[string]any, 0, len(rawItems))
+ var zipOnlyOmitted int
for _, item := range rawItems {
- converted := mineruContentItemToJSON(item, images, pageSizes)
+ converted := mineruContentItemToJSONBudget(item, images, pageSizes, budget)
if converted != nil {
out = append(out, converted)
+ continue
+ }
+ if mineruZipOnlyFigureSkipped(item, images, pageSizes) {
+ zipOnlyOmitted++
}
}
if len(out) == 0 {
- return nil, fmt.Errorf("MinerU zip contained no usable content_list items")
+ return nil, nil, fmt.Errorf("MinerU zip contained no usable content_list items")
}
- return out, nil
+ warnings := budget.Warnings()
+ if zipOnlyOmitted > 0 {
+ warnings = append(warnings, fmt.Sprintf(
+ "omitted %d MinerU figure(s) with zip bytes but no PDF crop region",
+ zipOnlyOmitted,
+ ))
+ }
+ return out, warnings, nil
}
+// mineruReadZipFile reads one MinerU zip member in full for JSON extraction.
func mineruReadZipFile(f *zip.File) ([]byte, error) {
rc, err := f.Open()
if err != nil {
@@ -1194,7 +1211,18 @@ func mergeMineruPageSizes(pdfSizes, middleSizes map[int][2]float64) map[int][2]f
return out
}
+// mineruContentItemToJSON converts one MinerU content_list item without a
+// media budget. Tests and helpers that do not collect run warnings use this
+// wrapper; production extraction goes through mineruContentItemToJSONBudget.
func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, pageSizes map[int][2]float64) map[string]any {
+ return mineruContentItemToJSONBudget(item, images, pageSizes, nil)
+}
+
+// mineruContentItemToJSONBudget converts one content_list item. Figure bytes
+// from the MinerU zip are not inlined on this JSON path; when budget is set
+// (extract path) structured data-URI payloads are checked against the shared
+// media limits and omitted with warnings instead of being copied into items.
+func mineruContentItemToJSONBudget(item map[string]any, images map[string]*zip.File, pageSizes map[int][2]float64, budget *media.Budget) map[string]any {
typ, _ := item["type"].(string)
typ = strings.ToLower(strings.TrimSpace(typ))
switch typ {
@@ -1203,7 +1231,6 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, p
}
text := mineruItemText(item, typ)
- imageURI := ""
docType := "text"
// External MinerU labels are mapped explicitly. Unknown labels must not
// acquire parser block-kind semantics implicitly; list/code are real
@@ -1221,15 +1248,15 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, p
case "image", "chart":
docType = "image"
layout = "figure"
- if imgPath, _ := item["img_path"].(string); imgPath != "" {
- imageURI = mineruZipImageDataURI(images, imgPath)
- }
- if imageURI == "" {
- if dataURI, _ := item["image_data_uri"].(string); strings.HasPrefix(dataURI, "data:") {
- imageURI = dataURI
+ if dataURI, _ := item["image_data_uri"].(string); strings.HasPrefix(dataURI, "data:") {
+ if budget != nil {
+ mineruAccountStructuredDataURI(budget, dataURI)
}
}
- if text == "" && imageURI == "" {
+ hasCropRegion := len(mineruItemPositions(item, pageSizes)) > 0
+ // Zip figure bytes are not inlined on this path; keep captionless items
+ // only when they still have a PDF crop box (or non-empty text above).
+ if text == "" && !hasCropRegion {
return nil
}
case "table":
@@ -1257,9 +1284,6 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, p
jsonItem[backendDescribedKey] = true
}
}
- if imageURI != "" {
- jsonItem["image"] = imageURI
- }
// Page identity does not depend on geometry: it drives table placement and
// page_count even when positions cannot be produced.
if pageIdx, ok := mineruJSONInt(item["page_idx"]); ok && pageIdx >= 0 {
@@ -1286,6 +1310,27 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File, p
return jsonItem
}
+// mineruZipOnlyFigureSkipped reports a captionless MinerU figure that had a zip
+// member but no PDF crop region. Those bytes are intentionally not inlined, so
+// the item is omitted with a parse warning instead of producing an empty chunk.
+func mineruZipOnlyFigureSkipped(item map[string]any, images map[string]*zip.File, pageSizes map[int][2]float64) bool {
+ typ, _ := item["type"].(string)
+ typ = strings.ToLower(strings.TrimSpace(typ))
+ if typ != "image" && typ != "chart" {
+ return false
+ }
+ if mineruItemText(item, typ) != "" {
+ return false
+ }
+ if len(mineruItemPositions(item, pageSizes)) > 0 {
+ return false
+ }
+ imgPath, _ := item["img_path"].(string)
+ return imgPath != "" && mineruZipImageFile(images, imgPath) != nil
+}
+
+// mineruItemText assembles the searchable text MinerU attaches to one
+// content_list item before it is converted into a parser JSON item.
func mineruItemText(item map[string]any, typ string) string {
var parts []string
switch typ {
@@ -1324,6 +1369,8 @@ func mineruItemText(item map[string]any, typ string) string {
return strings.TrimSpace(strings.Join(parts, "\n"))
}
+// mineruItemPositions converts MinerU page/bbox fields into the shared
+// _pdf_positions matrix used by jump-to-source and on-demand PDF cropping.
func mineruItemPositions(item map[string]any, pageSizes map[int][2]float64) [][]any {
if raw, ok := item["_mineru_positions"].([]any); ok {
var out [][]any
@@ -1454,23 +1501,68 @@ func mineruReadBounded(f *zip.File, limit int64) ([]byte, bool) {
return data, true
}
-func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {
+// mineruZipImageFile resolves a MinerU img_path against the zip image index.
+func mineruZipImageFile(images map[string]*zip.File, imgPath string) *zip.File {
imgPath = strings.TrimSpace(strings.ReplaceAll(imgPath, "\\", "/"))
if imgPath == "" {
- return ""
+ return nil
}
candidates := []string{imgPath, path.Base(imgPath), strings.TrimPrefix(path.Clean(imgPath), "./")}
- var file *zip.File
for _, key := range candidates {
if f, ok := images[key]; ok {
- file = f
- break
+ return f
}
}
- data, ok := mineruReadBounded(file, mineruMaxImageBytes)
+ return nil
+}
+
+// mineruAccountStructuredDataURI applies the shared media budget to a
+// structured image_data_uri without copying it into parser items. Valid payloads
+// above the per-image limit are recorded as omissions; malformed payloads are
+// ignored.
+func mineruAccountStructuredDataURI(budget *media.Budget, dataURI string) {
+ if budget == nil {
+ return
+ }
+ decodedSize, ok := media.DataImageDecodedSize(dataURI)
+ if !ok {
+ return
+ }
+ if decodedSize > media.ImageMaxBytes() {
+ _, _ = budget.ReserveOccurrence(media.Size(decodedSize))
+ return
+ }
+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)
+ if admission != media.Admitted {
+ return
+ }
+ _ = reservation.Commit(media.Size(decodedSize))
+}
+
+// mineruZipImageDataURI inlines a zip image as a data URI. Production MinerU
+// JSON extraction does not call this; it is used by unit tests and keeps the
+// shared media.Budget semantics when budget is non-nil.
+func mineruZipImageDataURI(images map[string]*zip.File, imgPath string, budget *media.Budget) string {
+ file := mineruZipImageFile(images, imgPath)
+ limit := int64(media.ImageMaxBytes())
+ if file == nil || file.UncompressedSize64 > uint64(limit) {
+ return ""
+ }
+ var reservation *media.Reservation
+ if budget != nil {
+ var admission media.Admission
+ reservation, admission = budget.ReserveOccurrence(media.UnknownSize)
+ if admission != media.Admitted {
+ return ""
+ }
+ }
+ data, ok := mineruReadBounded(file, limit)
if !ok {
return ""
}
+ if budget != nil && !reservation.Commit(media.Size(len(data))) {
+ return ""
+ }
mediaType := mime.TypeByExtension(path.Ext(file.Name))
if mediaType == "" {
mediaType = http.DetectContentType(data)
@@ -1478,6 +1570,8 @@ func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {
return fmt.Sprintf("data:%s;base64,%s", mediaType, base64.StdEncoding.EncodeToString(data))
}
+// mineruJSONParseResult wraps extracted MinerU JSON items in the shared parser
+// ParseResult envelope, including page_count derived from item positions.
func mineruJSONParseResult(filename string, items []map[string]any) parser.ParseResult {
return parser.ParseResult{
File: map[string]any{
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 90184227c0..53be7146c2 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -260,7 +260,7 @@ func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {
}
sizes := map[int][2]float64{0: {612, 792}, 1: {612, 792}}
- items, err := mineruExtractJSONItems(buf.Bytes(), sizes)
+ items, _, err := mineruExtractJSONItems(buf.Bytes(), sizes)
if err != nil {
t.Fatalf("mineruExtractJSONItems: %v", err)
}
@@ -291,9 +291,8 @@ func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {
if imageItem["text"] != "A figure" {
t.Fatalf("image text = %#v", imageItem["text"])
}
- uri, _ := imageItem["image"].(string)
- if !strings.HasPrefix(uri, "data:image/png;base64,") {
- t.Fatalf("image data URI = %q", uri)
+ if _, ok := imageItem["image"]; ok {
+ t.Fatalf("zip figures must not be inlined on the MinerU JSON path, got image=%q", imageItem["image"])
}
imgPos, _ := imageItem["_pdf_positions"].([][]any)
if len(imgPos) != 1 {
@@ -384,7 +383,7 @@ func TestMinerUExtractJSONItemsScalesBBoxToPoints(t *testing.T) {
t.Fatal(err)
}
- items, err := mineruExtractJSONItems(buf.Bytes(), nil)
+ items, _, err := mineruExtractJSONItems(buf.Bytes(), nil)
if err != nil {
t.Fatalf("mineruExtractJSONItems: %v", err)
}
@@ -420,7 +419,7 @@ func TestMinerUExtractJSONItemsWithoutGeometryKeepsText(t *testing.T) {
t.Fatal(err)
}
- items, err := mineruExtractJSONItems(buf.Bytes(), nil)
+ items, _, err := mineruExtractJSONItems(buf.Bytes(), nil)
if err != nil {
t.Fatalf("mineruExtractJSONItems: %v", err)
}
@@ -504,7 +503,7 @@ func TestMinerURealContentListPositionsStayInPage(t *testing.T) {
pageWidth = 595.0
pageHeight = 842.0
)
- items, err := mineruExtractJSONItems(buf.Bytes(), map[int][2]float64{0: {pageWidth, pageHeight}})
+ items, _, err := mineruExtractJSONItems(buf.Bytes(), map[int][2]float64{0: {pageWidth, pageHeight}})
if err != nil {
t.Fatalf("mineruExtractJSONItems: %v", err)
}
@@ -632,7 +631,7 @@ func TestMineruExtractJSONItemsStructuredContent(t *testing.T) {
t.Fatal(err)
}
- items, err := mineruExtractJSONItems(buf.Bytes(), map[int][2]float64{0: {1000, 1000}})
+ items, _, err := mineruExtractJSONItems(buf.Bytes(), map[int][2]float64{0: {1000, 1000}})
if err != nil {
t.Fatal(err)
}
@@ -649,10 +648,12 @@ func TestMineruExtractJSONItemsStructuredContent(t *testing.T) {
if items[2]["doc_type_kwd"] != "table" || !strings.Contains(items[2]["text"].(string), "表1") {
t.Fatalf("table item = %#v", items[2])
}
- image, _ := items[3]["image"].(string)
- if items[3]["doc_type_kwd"] != "image" || !strings.Contains(items[3]["text"].(string), "图1") || !strings.HasPrefix(image, "data:image/png;base64,") {
+ if items[3]["doc_type_kwd"] != "image" || !strings.Contains(items[3]["text"].(string), "图1") {
t.Fatalf("image item = %#v", items[3])
}
+ if _, ok := items[3]["image"]; ok {
+ t.Fatalf("structured zip image must not be inlined: %#v", items[3])
+ }
}
func TestFinishMinerURemoteResultPrefersStructuredZip(t *testing.T) {