Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docparser): bound MinerU result packages and their expansion (#4396)
ordinary being1fe804bd5cab
diff --git a/internal/infrastructure/docparser/mineru_cloud_converter.go b/internal/infrastructure/docparser/mineru_cloud_converter.goindex 046b63874..64136319f 100644--- a/internal/infrastructure/docparser/mineru_cloud_converter.go+++ b/internal/infrastructure/docparser/mineru_cloud_converter.go@@ -376,7 +376,7 @@ func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, []byte, errreturn "", nil, nil, fmt.Errorf("download zip status %d", resp.StatusCode)}- zipData, err := io.ReadAll(resp.Body)+ zipData, err := readResultPackage(resp.Body, maxResultPackageBytes)if err != nil {return "", nil, nil, fmt.Errorf("read zip body: %w", err)}@@ -390,6 +390,13 @@ func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, []byte, err// extractMarkdownZip reads the shallowest .md file of a MinerU result package// and the images it references, via markdown or inline HTML <img> syntax.func extractMarkdownZip(zipData []byte, logLabel string) (string, []types.ImageRef, error) {+ return extractMarkdownZipBounded(zipData, logLabel, maxZipImagesTotalBytes)+}++// extractMarkdownZipBounded is extractMarkdownZip with the accumulated-image+// budget as a parameter, so a test can exercise the limit without materialising+// the production figure.+func extractMarkdownZipBounded(zipData []byte, logLabel string, maxImages int64) (string, []types.ImageRef, error) {zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))if err != nil {return "", nil, fmt.Errorf("open zip: %w", err)@@ -424,6 +431,10 @@ func extractMarkdownZip(zipData []byte, logLabel string) (string, []types.ImageR// Extract referenced imagesvar imageRefs []types.ImageRef+ // Every image below is kept in memory until the caller hands the slice on,+ // so the package needs a total budget as well as a per-entry cap: a package+ // of thousands of small entries expands without ever tripping the latter.+ var imageBytes int64seen := map[string]bool{}for _, imgPath := range extractImageRefsFromContent(mdText) {if strings.HasPrefix(imgPath, "http://") || strings.HasPrefix(imgPath, "https://") || strings.HasPrefix(imgPath, "data:") {@@ -445,6 +456,11 @@ func extractMarkdownZip(zipData []byte, logLabel string) (string, []types.ImageRlogger.Errorf(context.Background(), "[%s] failed to read zip image %s: %v", logLabel, imgPath, err)continue}+ if imageBytes+int64(len(imgData)) > maxImages {+ return "", nil, fmt.Errorf("%w: images in %q exceed %d bytes",+ errResultPackageTooLarge, logLabel, maxImages)+ }+ imageBytes += int64(len(imgData))ext := strings.ToLower(filepath.Ext(resolved.Name))if ext == "" {@@ -485,26 +501,19 @@ func resolveInZip(imgPath, mdDir string, entries map[string]*zip.File) *zip.Filereturn nil}+// readZipEntry reads a text entry (the package markdown) under the per-entry cap.func readZipEntry(f *zip.File) (string, error) {- rc, err := f.Open()- if err != nil {- return "", err- }- defer rc.Close()- data, err := io.ReadAll(rc)+ data, err := readZipEntryCapped(f, maxZipEntryBytes)if err != nil {return "", err}return string(data), nil}+// readZipEntryBytes reads a binary entry (an extracted image) under the+// per-entry cap.func readZipEntryBytes(f *zip.File) ([]byte, error) {- rc, err := f.Open()- if err != nil {- return nil, err- }- defer rc.Close()- return io.ReadAll(rc)+ return readZipEntryCapped(f, maxZipEntryBytes)}// PingMinerUCloud checks if the MinerU Cloud API is reachable with the given API key.diff --git a/internal/infrastructure/docparser/mineru_layout.go b/internal/infrastructure/docparser/mineru_layout.goindex 169f21ebf..aa8190241 100644--- a/internal/infrastructure/docparser/mineru_layout.go+++ b/internal/infrastructure/docparser/mineru_layout.go@@ -118,7 +118,7 @@ func minerUContentListFromZip(zipData []byte) []byte {return candidates[i].Name < candidates[j].Name})for _, candidate := range candidates {- data, err := readZipEntryBytes(candidate)+ data, err := readZipEntryCapped(candidate, maxZipEntryBytes)if err != nil {continue}diff --git a/internal/infrastructure/docparser/zip_limits.go b/internal/infrastructure/docparser/zip_limits.gonew file mode 100644index 000000000..f861d7608--- /dev/null+++ b/internal/infrastructure/docparser/zip_limits.go@@ -0,0 +1,81 @@+package docparser++import (+ "archive/zip"+ "errors"+ "fmt"+ "io"+)++// A MinerU result package arrives over the network from a third-party parse API+// and is buffered before anything about it is validated, so both the download+// and what the archive expands to need a ceiling. The V1 client already caps the+// download (mineruV1MaxZipBytes); the Cloud converter did not, and neither path+// capped decompression.+const (+ // maxResultPackageBytes caps a downloaded result package. Same ceiling the+ // V1 client applies to the same artifact.+ maxResultPackageBytes = mineruV1MaxZipBytes++ // maxZipEntryBytes caps one decompressed entry. A markdown page or a single+ // extracted image is far below this; an entry that reaches it is a+ // compression bomb rather than document content.+ maxZipEntryBytes = 64 << 20++ // maxZipImagesTotalBytes caps the images extracted from one package. Every+ // image is held in memory at once, so the total is what actually decides the+ // footprint - bounding each entry alone does not bound a package holding+ // thousands of small ones.+ maxZipImagesTotalBytes = 512 << 20+)++// errResultPackageTooLarge marks a result package over one of the caps above.+var errResultPackageTooLarge = errors.New("result package exceeds maximum size")++// readResultPackage reads a downloaded result package, refusing anything larger+// than limit instead of buffering it. A non-positive limit falls back to+// maxResultPackageBytes.+//+// Mirrors readCapped in the datasource connectors and internal/infrastructure+// /web_search: read one byte past the cap so an oversized body is distinguishable+// from one that merely failed to read.+func readResultPackage(body io.Reader, limit int64) ([]byte, error) {+ if limit <= 0 {+ limit = maxResultPackageBytes+ }+ data, err := io.ReadAll(io.LimitReader(body, limit+1))+ if err != nil {+ return nil, err+ }+ if int64(len(data)) > limit {+ return nil, fmt.Errorf("%w (%d bytes)", errResultPackageTooLarge, limit)+ }+ return data, nil+}++// readZipEntryCapped reads one archive entry under limit.+//+// It checks UncompressedSize64 first so an obviously oversized entry is never+// decompressed at all, then caps the read anyway: the declared size is attacker+// or vendor controlled and is allowed to understate what the entry expands to.+func readZipEntryCapped(f *zip.File, limit int64) ([]byte, error) {+ if limit <= 0 {+ limit = maxZipEntryBytes+ }+ if f.UncompressedSize64 > uint64(limit) {+ return nil, fmt.Errorf("%w: entry %q declares %d bytes", errResultPackageTooLarge, f.Name, f.UncompressedSize64)+ }+ rc, err := f.Open()+ if err != nil {+ return nil, err+ }+ defer func() { _ = rc.Close() }()+ data, err := io.ReadAll(io.LimitReader(rc, limit+1))+ if err != nil {+ return nil, err+ }+ if int64(len(data)) > limit {+ return nil, fmt.Errorf("%w: entry %q expands past %d bytes", errResultPackageTooLarge, f.Name, limit)+ }+ return data, nil+}diff --git a/internal/infrastructure/docparser/zip_limits_test.go b/internal/infrastructure/docparser/zip_limits_test.gonew file mode 100644index 000000000..8167f30cc--- /dev/null+++ b/internal/infrastructure/docparser/zip_limits_test.go@@ -0,0 +1,158 @@+package docparser++import (+ "archive/zip"+ "bytes"+ "errors"+ "strings"+ "testing"+)++// buildZip returns an in-memory archive holding the given name -> content pairs.+func buildZip(t *testing.T, files map[string]string) []byte {+ t.Helper()+ var buf bytes.Buffer+ zw := zip.NewWriter(&buf)+ for name, content := range files {+ w, err := zw.Create(name)+ if err != nil {+ t.Fatalf("create %s: %v", name, err)+ }+ if _, err := w.Write([]byte(content)); err != nil {+ t.Fatalf("write %s: %v", name, err)+ }+ }+ if err := zw.Close(); err != nil {+ t.Fatalf("close zip: %v", err)+ }+ return buf.Bytes()+}++// openEntry returns the first archive entry matching name.+func openEntry(t *testing.T, zipData []byte, name string) *zip.File {+ t.Helper()+ zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))+ if err != nil {+ t.Fatalf("open zip: %v", err)+ }+ for _, f := range zr.File {+ if f.Name == name {+ return f+ }+ }+ t.Fatalf("entry %q not found", name)+ return nil+}++func TestReadResultPackage(t *testing.T) {+ t.Run("rejects over the cap", func(t *testing.T) {+ const limit = 1 << 10+ _, err := readResultPackage(strings.NewReader(strings.Repeat("a", limit+1)), limit)+ if !errors.Is(err, errResultPackageTooLarge) {+ t.Fatalf("err = %v, want errResultPackageTooLarge", err)+ }+ })++ t.Run("accepts exactly the cap", func(t *testing.T) {+ const limit = 1 << 10+ got, err := readResultPackage(strings.NewReader(strings.Repeat("a", limit)), limit)+ if err != nil {+ t.Fatalf("readResultPackage() = %v, want nil at exactly the cap", err)+ }+ if len(got) != limit {+ t.Errorf("len = %d, want %d", len(got), limit)+ }+ })++ t.Run("non-positive limit falls back to the production cap", func(t *testing.T) {+ if _, err := readResultPackage(strings.NewReader("zip"), 0); err != nil {+ t.Fatalf("readResultPackage(_, 0) = %v, want nil", err)+ }+ })+}++// The declared UncompressedSize64 comes from the archive and is allowed to+// understate the expansion, so the read has to be capped as well as the header+// checked. A header that understates it is the actual bomb.+func TestReadZipEntryCappedRejectsBombs(t *testing.T) {+ t.Run("declares over the cap", func(t *testing.T) {+ data := buildZip(t, map[string]string{"big.txt": strings.Repeat("a", 4096)})+ entry := openEntry(t, data, "big.txt")++ _, err := readZipEntryCapped(entry, 1024)+ if !errors.Is(err, errResultPackageTooLarge) {+ t.Fatalf("err = %v, want errResultPackageTooLarge", err)+ }+ if !strings.Contains(err.Error(), "declares") {+ t.Errorf("error = %q, want it to name the declared size", err)+ }+ })++ t.Run("expands past the cap even with an honest header", func(t *testing.T) {+ // 64 KiB of zeroes deflates to well under 1 KiB, so the declared size is+ // truthful and passes the header check; only the capped read catches it.+ data := buildZip(t, map[string]string{"bomb.txt": strings.Repeat("\x00", 64<<10)})+ entry := openEntry(t, data, "bomb.txt")+ if entry.UncompressedSize64 > 1024 {+ t.Skipf("fixture is not compressed enough to exercise the read cap")+ }++ got, err := readZipEntryCapped(entry, 1024)+ if !errors.Is(err, errResultPackageTooLarge) {+ t.Fatalf("err = %v, want errResultPackageTooLarge (read %d bytes)", err, len(got))+ }+ if !strings.Contains(err.Error(), "expands past") {+ t.Errorf("error = %q, want it to name the expansion", err)+ }+ })++ t.Run("accepts an entry at the cap", func(t *testing.T) {+ data := buildZip(t, map[string]string{"ok.txt": strings.Repeat("a", 2048)})+ entry := openEntry(t, data, "ok.txt")++ got, err := readZipEntryCapped(entry, 2048)+ if err != nil {+ t.Fatalf("readZipEntryCapped() = %v, want nil at exactly the cap", err)+ }+ if len(got) != 2048 {+ t.Errorf("len = %d, want 2048", len(got))+ }+ })+}++// Every extracted image is held in memory at once, so a package of many small+// entries expands without ever tripping a per-entry cap.+func TestExtractMarkdownZipEnforcesTotalImageBudget(t *testing.T) {+ files := map[string]string{+ "doc.md": "\n\n\n",+ }+ for _, name := range []string{"img/a.png", "img/b.png", "img/c.png", "img/d.png"} {+ files[name] = strings.Repeat("i", 1024)+ }+ data := buildZip(t, files)++ t.Run("refuses a package over the budget", func(t *testing.T) {+ _, _, err := extractMarkdownZipBounded(data, "Test", 3*1024)+ if !errors.Is(err, errResultPackageTooLarge) {+ t.Fatalf("err = %v, want errResultPackageTooLarge", err)+ }+ })++ t.Run("still extracts a package within budget", func(t *testing.T) {+ md, refs, err := extractMarkdownZipBounded(data, "Test", 8*1024)+ if err != nil {+ t.Fatalf("extractMarkdownZipBounded() = %v, want nil", err)+ }+ if len(refs) != 4 {+ t.Errorf("extracted %d images, want 4", len(refs))+ }+ for _, r := range refs {+ if len(r.ImageData) != 1024 {+ t.Errorf("%s: %d bytes, want 1024", r.Filename, len(r.ImageData))+ }+ }+ if !strings.Contains(md, "img/a.png") {+ t.Errorf("markdown = %q, want the image references preserved", md)+ }+ })+}