Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): bound MinerU result packages and their expansion (#4396)

ordinary being1fe804bd5cab
diff --git a/internal/infrastructure/docparser/mineru_cloud_converter.go b/internal/infrastructure/docparser/mineru_cloud_converter.go
index 046b63874..64136319f 100644
--- a/internal/infrastructure/docparser/mineru_cloud_converter.go
+++ b/internal/infrastructure/docparser/mineru_cloud_converter.go
@@ -376,7 +376,7 @@ func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, []byte, err
return "", nil, nil, fmt.Errorf("download zip status %d", resp.StatusCode)
}
- zipData, err := io.ReadAll(resp.Body)
+ zipData, err := readResultPackage(resp.Body, maxResultPackageBytes)
if err != nil {
return "", nil, nil, fmt.Errorf("read zip body: %w", err)
}
@@ -390,6 +390,13 @@ func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, []byte, err
// extractMarkdownZip reads the shallowest .md file of a MinerU result package
// and the images it references, via markdown or inline HTML <img> syntax.
func extractMarkdownZip(zipData []byte, logLabel string) (string, []types.ImageRef, error) {
+ return extractMarkdownZipBounded(zipData, logLabel, maxZipImagesTotalBytes)
+}
+
+// extractMarkdownZipBounded is extractMarkdownZip with the accumulated-image
+// budget as a parameter, so a test can exercise the limit without materialising
+// the production figure.
+func extractMarkdownZipBounded(zipData []byte, logLabel string, maxImages int64) (string, []types.ImageRef, error) {
zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
if err != nil {
return "", nil, fmt.Errorf("open zip: %w", err)
@@ -424,6 +431,10 @@ func extractMarkdownZip(zipData []byte, logLabel string) (string, []types.ImageR
// Extract referenced images
var imageRefs []types.ImageRef
+ // Every image below is kept in memory until the caller hands the slice on,
+ // so the package needs a total budget as well as a per-entry cap: a package
+ // of thousands of small entries expands without ever tripping the latter.
+ var imageBytes int64
seen := map[string]bool{}
for _, imgPath := range extractImageRefsFromContent(mdText) {
if strings.HasPrefix(imgPath, "http://") || strings.HasPrefix(imgPath, "https://") || strings.HasPrefix(imgPath, "data:") {
@@ -445,6 +456,11 @@ func extractMarkdownZip(zipData []byte, logLabel string) (string, []types.ImageR
logger.Errorf(context.Background(), "[%s] failed to read zip image %s: %v", logLabel, imgPath, err)
continue
}
+ if imageBytes+int64(len(imgData)) > maxImages {
+ return "", nil, fmt.Errorf("%w: images in %q exceed %d bytes",
+ errResultPackageTooLarge, logLabel, maxImages)
+ }
+ imageBytes += int64(len(imgData))
ext := strings.ToLower(filepath.Ext(resolved.Name))
if ext == "" {
@@ -485,26 +501,19 @@ func resolveInZip(imgPath, mdDir string, entries map[string]*zip.File) *zip.File
return nil
}
+// readZipEntry reads a text entry (the package markdown) under the per-entry cap.
func readZipEntry(f *zip.File) (string, error) {
- rc, err := f.Open()
- if err != nil {
- return "", err
- }
- defer rc.Close()
- data, err := io.ReadAll(rc)
+ data, err := readZipEntryCapped(f, maxZipEntryBytes)
if err != nil {
return "", err
}
return string(data), nil
}
+// readZipEntryBytes reads a binary entry (an extracted image) under the
+// per-entry cap.
func readZipEntryBytes(f *zip.File) ([]byte, error) {
- rc, err := f.Open()
- if err != nil {
- return nil, err
- }
- defer rc.Close()
- return io.ReadAll(rc)
+ return readZipEntryCapped(f, maxZipEntryBytes)
}
// PingMinerUCloud checks if the MinerU Cloud API is reachable with the given API key.
diff --git a/internal/infrastructure/docparser/mineru_layout.go b/internal/infrastructure/docparser/mineru_layout.go
index 169f21ebf..aa8190241 100644
--- a/internal/infrastructure/docparser/mineru_layout.go
+++ b/internal/infrastructure/docparser/mineru_layout.go
@@ -118,7 +118,7 @@ func minerUContentListFromZip(zipData []byte) []byte {
return candidates[i].Name < candidates[j].Name
})
for _, candidate := range candidates {
- data, err := readZipEntryBytes(candidate)
+ data, err := readZipEntryCapped(candidate, maxZipEntryBytes)
if err != nil {
continue
}
diff --git a/internal/infrastructure/docparser/zip_limits.go b/internal/infrastructure/docparser/zip_limits.go
new file mode 100644
index 000000000..f861d7608
--- /dev/null
+++ b/internal/infrastructure/docparser/zip_limits.go
@@ -0,0 +1,81 @@
+package docparser
+
+import (
+ "archive/zip"
+ "errors"
+ "fmt"
+ "io"
+)
+
+// A MinerU result package arrives over the network from a third-party parse API
+// and is buffered before anything about it is validated, so both the download
+// and what the archive expands to need a ceiling. The V1 client already caps the
+// download (mineruV1MaxZipBytes); the Cloud converter did not, and neither path
+// capped decompression.
+const (
+ // maxResultPackageBytes caps a downloaded result package. Same ceiling the
+ // V1 client applies to the same artifact.
+ maxResultPackageBytes = mineruV1MaxZipBytes
+
+ // maxZipEntryBytes caps one decompressed entry. A markdown page or a single
+ // extracted image is far below this; an entry that reaches it is a
+ // compression bomb rather than document content.
+ maxZipEntryBytes = 64 << 20
+
+ // maxZipImagesTotalBytes caps the images extracted from one package. Every
+ // image is held in memory at once, so the total is what actually decides the
+ // footprint - bounding each entry alone does not bound a package holding
+ // thousands of small ones.
+ maxZipImagesTotalBytes = 512 << 20
+)
+
+// errResultPackageTooLarge marks a result package over one of the caps above.
+var errResultPackageTooLarge = errors.New("result package exceeds maximum size")
+
+// readResultPackage reads a downloaded result package, refusing anything larger
+// than limit instead of buffering it. A non-positive limit falls back to
+// maxResultPackageBytes.
+//
+// Mirrors readCapped in the datasource connectors and internal/infrastructure
+// /web_search: read one byte past the cap so an oversized body is distinguishable
+// from one that merely failed to read.
+func readResultPackage(body io.Reader, limit int64) ([]byte, error) {
+ if limit <= 0 {
+ limit = maxResultPackageBytes
+ }
+ data, err := io.ReadAll(io.LimitReader(body, limit+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > limit {
+ return nil, fmt.Errorf("%w (%d bytes)", errResultPackageTooLarge, limit)
+ }
+ return data, nil
+}
+
+// readZipEntryCapped reads one archive entry under limit.
+//
+// It checks UncompressedSize64 first so an obviously oversized entry is never
+// decompressed at all, then caps the read anyway: the declared size is attacker
+// or vendor controlled and is allowed to understate what the entry expands to.
+func readZipEntryCapped(f *zip.File, limit int64) ([]byte, error) {
+ if limit <= 0 {
+ limit = maxZipEntryBytes
+ }
+ if f.UncompressedSize64 > uint64(limit) {
+ return nil, fmt.Errorf("%w: entry %q declares %d bytes", errResultPackageTooLarge, f.Name, f.UncompressedSize64)
+ }
+ rc, err := f.Open()
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = rc.Close() }()
+ data, err := io.ReadAll(io.LimitReader(rc, limit+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > limit {
+ return nil, fmt.Errorf("%w: entry %q expands past %d bytes", errResultPackageTooLarge, f.Name, limit)
+ }
+ return data, nil
+}
diff --git a/internal/infrastructure/docparser/zip_limits_test.go b/internal/infrastructure/docparser/zip_limits_test.go
new file mode 100644
index 000000000..8167f30cc
--- /dev/null
+++ b/internal/infrastructure/docparser/zip_limits_test.go
@@ -0,0 +1,158 @@
+package docparser
+
+import (
+ "archive/zip"
+ "bytes"
+ "errors"
+ "strings"
+ "testing"
+)
+
+// buildZip returns an in-memory archive holding the given name -> content pairs.
+func buildZip(t *testing.T, files map[string]string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ for name, content := range files {
+ w, err := zw.Create(name)
+ if err != nil {
+ t.Fatalf("create %s: %v", name, err)
+ }
+ if _, err := w.Write([]byte(content)); err != nil {
+ t.Fatalf("write %s: %v", name, err)
+ }
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatalf("close zip: %v", err)
+ }
+ return buf.Bytes()
+}
+
+// openEntry returns the first archive entry matching name.
+func openEntry(t *testing.T, zipData []byte, name string) *zip.File {
+ t.Helper()
+ zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
+ if err != nil {
+ t.Fatalf("open zip: %v", err)
+ }
+ for _, f := range zr.File {
+ if f.Name == name {
+ return f
+ }
+ }
+ t.Fatalf("entry %q not found", name)
+ return nil
+}
+
+func TestReadResultPackage(t *testing.T) {
+ t.Run("rejects over the cap", func(t *testing.T) {
+ const limit = 1 << 10
+ _, err := readResultPackage(strings.NewReader(strings.Repeat("a", limit+1)), limit)
+ if !errors.Is(err, errResultPackageTooLarge) {
+ t.Fatalf("err = %v, want errResultPackageTooLarge", err)
+ }
+ })
+
+ t.Run("accepts exactly the cap", func(t *testing.T) {
+ const limit = 1 << 10
+ got, err := readResultPackage(strings.NewReader(strings.Repeat("a", limit)), limit)
+ if err != nil {
+ t.Fatalf("readResultPackage() = %v, want nil at exactly the cap", err)
+ }
+ if len(got) != limit {
+ t.Errorf("len = %d, want %d", len(got), limit)
+ }
+ })
+
+ t.Run("non-positive limit falls back to the production cap", func(t *testing.T) {
+ if _, err := readResultPackage(strings.NewReader("zip"), 0); err != nil {
+ t.Fatalf("readResultPackage(_, 0) = %v, want nil", err)
+ }
+ })
+}
+
+// The declared UncompressedSize64 comes from the archive and is allowed to
+// understate the expansion, so the read has to be capped as well as the header
+// checked. A header that understates it is the actual bomb.
+func TestReadZipEntryCappedRejectsBombs(t *testing.T) {
+ t.Run("declares over the cap", func(t *testing.T) {
+ data := buildZip(t, map[string]string{"big.txt": strings.Repeat("a", 4096)})
+ entry := openEntry(t, data, "big.txt")
+
+ _, err := readZipEntryCapped(entry, 1024)
+ if !errors.Is(err, errResultPackageTooLarge) {
+ t.Fatalf("err = %v, want errResultPackageTooLarge", err)
+ }
+ if !strings.Contains(err.Error(), "declares") {
+ t.Errorf("error = %q, want it to name the declared size", err)
+ }
+ })
+
+ t.Run("expands past the cap even with an honest header", func(t *testing.T) {
+ // 64 KiB of zeroes deflates to well under 1 KiB, so the declared size is
+ // truthful and passes the header check; only the capped read catches it.
+ data := buildZip(t, map[string]string{"bomb.txt": strings.Repeat("\x00", 64<<10)})
+ entry := openEntry(t, data, "bomb.txt")
+ if entry.UncompressedSize64 > 1024 {
+ t.Skipf("fixture is not compressed enough to exercise the read cap")
+ }
+
+ got, err := readZipEntryCapped(entry, 1024)
+ if !errors.Is(err, errResultPackageTooLarge) {
+ t.Fatalf("err = %v, want errResultPackageTooLarge (read %d bytes)", err, len(got))
+ }
+ if !strings.Contains(err.Error(), "expands past") {
+ t.Errorf("error = %q, want it to name the expansion", err)
+ }
+ })
+
+ t.Run("accepts an entry at the cap", func(t *testing.T) {
+ data := buildZip(t, map[string]string{"ok.txt": strings.Repeat("a", 2048)})
+ entry := openEntry(t, data, "ok.txt")
+
+ got, err := readZipEntryCapped(entry, 2048)
+ if err != nil {
+ t.Fatalf("readZipEntryCapped() = %v, want nil at exactly the cap", err)
+ }
+ if len(got) != 2048 {
+ t.Errorf("len = %d, want 2048", len(got))
+ }
+ })
+}
+
+// Every extracted image is held in memory at once, so a package of many small
+// entries expands without ever tripping a per-entry cap.
+func TestExtractMarkdownZipEnforcesTotalImageBudget(t *testing.T) {
+ files := map[string]string{
+ "doc.md": "![a](img/a.png)\n![b](img/b.png)\n![c](img/c.png)\n![d](img/d.png)",
+ }
+ for _, name := range []string{"img/a.png", "img/b.png", "img/c.png", "img/d.png"} {
+ files[name] = strings.Repeat("i", 1024)
+ }
+ data := buildZip(t, files)
+
+ t.Run("refuses a package over the budget", func(t *testing.T) {
+ _, _, err := extractMarkdownZipBounded(data, "Test", 3*1024)
+ if !errors.Is(err, errResultPackageTooLarge) {
+ t.Fatalf("err = %v, want errResultPackageTooLarge", err)
+ }
+ })
+
+ t.Run("still extracts a package within budget", func(t *testing.T) {
+ md, refs, err := extractMarkdownZipBounded(data, "Test", 8*1024)
+ if err != nil {
+ t.Fatalf("extractMarkdownZipBounded() = %v, want nil", err)
+ }
+ if len(refs) != 4 {
+ t.Errorf("extracted %d images, want 4", len(refs))
+ }
+ for _, r := range refs {
+ if len(r.ImageData) != 1024 {
+ t.Errorf("%s: %d bytes, want 1024", r.Filename, len(r.ImageData))
+ }
+ }
+ if !strings.Contains(md, "img/a.png") {
+ t.Errorf("markdown = %q, want the image references preserved", md)
+ }
+ })
+}