Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

feat(parsing): auto-slice oversized PDFs for MinerU cloud parsing

polaris-smart4fe036231b42
diff --git a/deeptutor/services/parsing/engines/mineru/cloud.py b/deeptutor/services/parsing/engines/mineru/cloud.py
index e9cc9cde7..213057736 100644
--- a/deeptutor/services/parsing/engines/mineru/cloud.py
+++ b/deeptutor/services/parsing/engines/mineru/cloud.py
@@ -20,18 +20,23 @@ simplest correct choice (no nested event loop).
from __future__ import annotations
from collections.abc import Callable
+from dataclasses import dataclass
import io
+import json
import logging
from pathlib import Path
+import shutil
+import tempfile
import time
import zipfile
import httpx
+from pypdf import PdfReader, PdfWriter
from deeptutor.services.keypool import KeyPool
from .config import MinerUConfig, MinerUError
-from .formats import MINERU_SUPPORTED_FORMATS
+from .formats import MINERU_PDF_FORMATS, MINERU_SUPPORTED_FORMATS
logger = logging.getLogger(__name__)
@@ -91,22 +96,31 @@ def parse_cloud(
logger.debug("on_progress callback failed", exc_info=True)
with httpx.Client(base_url=base_url, headers={"Accept": "application/json"}) as client:
+ if source_path.suffix.lower() in MINERU_PDF_FORMATS:
+ page_count = _pdf_page_count(source_path)
+ if page_count is not None and page_count > config.max_pages_per_part:
+ return _parse_pdf_in_parts(
+ client,
+ source_path,
+ output_base,
+ config,
+ key_pool,
+ page_count=page_count,
+ report=report,
+ poll_interval=poll_interval,
+ timeout=timeout,
+ )
+
report(f"MinerU cloud: requesting upload slot for {source_path.name}")
- batch_id, upload_url = _request_upload(client, source_path, config, key_pool)
- size_mb = source_path.stat().st_size / (1024 * 1024)
- report(f"MinerU cloud: uploading {source_path.name} ({size_mb:.1f} MB)")
- _upload_file(source_path, upload_url)
- zip_url = _poll_for_zip(
+ archive_bytes = _upload_and_fetch_archive(
client,
- batch_id,
- source_path.name,
+ source_path,
+ config,
+ key_pool,
+ report=report,
poll_interval=poll_interval,
timeout=timeout,
- on_progress=on_progress,
- key_pool=key_pool,
)
- report("MinerU cloud: downloading parsed result archive")
- archive_bytes = _download(zip_url)
report("MinerU cloud: extracting archive")
working_dir = output_base / source_path.stem
@@ -121,6 +135,229 @@ def parse_cloud(
# ---------------------------------------------------------------------------
+def _upload_and_fetch_archive(
+ client: httpx.Client,
+ part_path: Path,
+ config: MinerUConfig,
+ key_pool: KeyPool,
+ *,
+ report: Callable[[str], None],
+ poll_interval: float,
+ timeout: float,
+) -> bytes:
+ """Run the upload → poll → download trio for one input file and return the
+ result archive bytes. Shared by the single-file path and each slice of the
+ auto-split path."""
+ report(f"MinerU cloud: requesting upload slot for {part_path.name}")
+ batch_id, upload_url = _request_upload(client, part_path, config, key_pool)
+ size_mb = part_path.stat().st_size / (1024 * 1024)
+ report(f"MinerU cloud: uploading {part_path.name} ({size_mb:.1f} MB)")
+ _upload_file(part_path, upload_url)
+ zip_url = _poll_for_zip(
+ client,
+ batch_id,
+ part_path.name,
+ poll_interval=poll_interval,
+ timeout=timeout,
+ on_progress=report,
+ key_pool=key_pool,
+ )
+ report("MinerU cloud: downloading parsed result archive")
+ return _download(zip_url)
+
+
+def _parse_pdf_in_parts(
+ client: httpx.Client,
+ source_path: Path,
+ output_base: Path,
+ config: MinerUConfig,
+ key_pool: KeyPool,
+ *,
+ page_count: int,
+ report: Callable[[str], None],
+ poll_interval: float,
+ timeout: float,
+) -> Path:
+ """Slice an oversized PDF, parse each part through the normal cloud flow,
+ and merge the per-part artifacts into one working dir that is isomorphic to
+ the single-file output (one ``*.md`` + ``*_content_list.json`` + ``images/``)."""
+ working_dir = output_base / source_path.stem
+ _reset_dir(working_dir)
+ markdown_chunks: list[str] = []
+ content_items: list = []
+ has_content_list = False
+
+ with tempfile.TemporaryDirectory(prefix="mineru-slice-") as tmp:
+ parts = _split_pdf(source_path, config.max_pages_per_part, Path(tmp))
+ total = len(parts)
+ report(
+ f"MinerU cloud: {source_path.name} has {page_count} pages, exceeding the "
+ f"{config.max_pages_per_part}-page per-file limit; auto-splitting into "
+ f"{total} part(s)"
+ )
+ for index, part in enumerate(parts, start=1):
+ report(
+ f"MinerU cloud: parsing part {index}/{total} (pages {part.start + 1}-{part.end})"
+ )
+ archive_bytes = _upload_and_fetch_archive(
+ client,
+ part.path,
+ config,
+ key_pool,
+ report=report,
+ poll_interval=poll_interval,
+ timeout=timeout,
+ )
+ report(f"MinerU cloud: merging part {index}/{total} artifacts")
+ with tempfile.TemporaryDirectory(prefix="mineru-part-") as part_tmp:
+ part_dir = Path(part_tmp) / "out"
+ part_dir.mkdir()
+ _extract_archive(archive_bytes, part_dir)
+ _merge_part_artifacts(
+ part_dir,
+ working_dir,
+ markdown_chunks=markdown_chunks,
+ content_items=content_items,
+ part_index=index,
+ )
+ has_content_list = True
+
+ merged_md = "".join(
+ chunk if chunk.endswith("\n") else chunk + "\n" for chunk in markdown_chunks
+ )
+ (working_dir / f"{source_path.stem}.md").write_text(merged_md, encoding="utf-8")
+ if has_content_list:
+ content_list_path = working_dir / f"{source_path.stem}_content_list.json"
+ content_list_path.write_text(
+ json.dumps(content_items, ensure_ascii=False), encoding="utf-8"
+ )
+ logger.info(
+ "MinerU cloud parse complete (%d parts): %s → %s", total, source_path.name, working_dir
+ )
+ return working_dir
+
+
+def _pdf_page_count(source_path: Path) -> int | None:
+ """Number of pages in a PDF, or ``None`` when pypdf cannot read it (e.g. a
+ truncated/encrypted file). ``None`` keeps the legacy single-file behaviour —
+ an unreadable PDF goes to MinerU as-is and fails there if it is truly broken."""
+ try:
+ with PdfReader(str(source_path)) as reader:
+ return len(reader.pages)
+ except Exception:
+ logger.warning("Could not count pages of %s; skipping auto-slicing", source_path.name)
+ return None
+
+
+def _split_pdf(source_path: Path, max_pages_per_part: int, work_dir: Path) -> list[_PdfPart]:
+ """Evenly cut ``source_path`` into parts of at most ``max_pages_per_part``
+ pages (the last part may be shorter) and write each part PDF into
+ ``work_dir``. 250 pages @180 → [180, 70]."""
+ try:
+ with PdfReader(str(source_path)) as reader:
+ total = len(reader.pages)
+ parts: list[_PdfPart] = []
+ for index, start in enumerate(range(0, total, max_pages_per_part), start=1):
+ end = min(start + max_pages_per_part, total)
+ part_path = work_dir / f"{source_path.stem}_part{index:02d}.pdf"
+ writer = PdfWriter()
+ for page_number in range(start, end):
+ writer.add_page(reader.pages[page_number])
+ with open(part_path, "wb") as out:
+ writer.write(out)
+ parts.append(_PdfPart(path=part_path, start=start, end=end))
+ return parts
+ except Exception as exc:
+ raise MinerUError(f"Failed to split PDF {source_path.name} into parts: {exc}") from exc
+
+
+def _merge_part_artifacts(
+ part_dir: Path,
+ working_dir: Path,
+ *,
+ markdown_chunks: list[str],
+ content_items: list,
+ part_index: int,
+) -> None:
+ """Accumulate one part's artifacts: markdown text, ``images/`` (renamed with
+ a part prefix so names stay globally unique) and ``content_list`` entries
+ (with ``img_path`` rewritten to match)."""
+ md_files = sorted(path for path in part_dir.rglob("*.md") if path.is_file())
+ if not md_files:
+ raise MinerUError(f"MinerU part {part_index} archive contains no markdown output.")
+ content_root = md_files[0].parent
+
+ rename_map = _copy_part_images(content_root / "images", working_dir / "images", part_index)
+
+ markdown = md_files[0].read_text(encoding="utf-8")
+ for original, unique in rename_map.items():
+ markdown = markdown.replace(f"images/{original}", f"images/{unique}")
+ markdown_chunks.append(markdown)
+
+ for content_list_path in sorted(content_root.glob("*_content_list.json")):
+ try:
+ payload = json.loads(content_list_path.read_text(encoding="utf-8"))
+ except (OSError, ValueError) as exc:
+ raise MinerUError(
+ f"MinerU part {part_index} content list is unreadable: {exc}"
+ ) from exc
+ if isinstance(payload, list):
+ _rewrite_image_paths(payload, rename_map)
+ content_items.extend(payload)
+
+ # Keep the remaining artifacts (middle/model/layout files, origin PDF, …)
+ # on disk with a part prefix, mirroring the single-file layout as closely
+ # as their per-part nature allows.
+ for extra in sorted(content_root.iterdir()):
+ if not extra.is_file() or extra.suffix.lower() == ".md":
+ continue
+ if extra.name.endswith("_content_list.json"):
+ continue
+ shutil.copyfile(extra, working_dir / f"part{part_index:02d}_{extra.name}")
+
+
+def _copy_part_images(images_dir: Path, target_dir: Path, part_index: int) -> dict[str, str]:
+ """Copy one part's ``images/`` into the merged ``images/`` dir, prefixing
+ each name with the part index so names stay globally unique across parts.
+ Returns ``{original_name: unique_name}``."""
+ if not images_dir.is_dir():
+ return {}
+ target_dir.mkdir(parents=True, exist_ok=True)
+ rename_map: dict[str, str] = {}
+ for image in sorted(images_dir.iterdir()):
+ if not image.is_file():
+ continue
+ unique_name = f"part{part_index:02d}_{image.name}"
+ shutil.copyfile(image, target_dir / unique_name)
+ rename_map[image.name] = unique_name
+ return rename_map
+
+
+def _rewrite_image_paths(node: object, rename_map: dict[str, str]) -> None:
+ """Rewrite ``img_path``/``image_path`` values in a content-list tree to the
+ renamed (part-prefixed) image files."""
+ if isinstance(node, dict):
+ for key, value in node.items():
+ if key in ("img_path", "image_path") and isinstance(value, str):
+ name = value.rsplit("/", 1)[-1]
+ if name in rename_map:
+ node[key] = value.replace(name, rename_map[name])
+ else:
+ _rewrite_image_paths(value, rename_map)
+ elif isinstance(node, list):
+ for item in node:
+ _rewrite_image_paths(item, rename_map)
+
+
+@dataclass(frozen=True)
+class _PdfPart:
+ """One slice of an oversized PDF: 0-based ``start``, exclusive ``end``."""
+
+ path: Path
+ start: int
+ end: int
+
+
def _request_upload(
client: httpx.Client, source_path: Path, config: MinerUConfig, key_pool: KeyPool
) -> tuple[str, str]:
diff --git a/deeptutor/services/parsing/engines/mineru/config.py b/deeptutor/services/parsing/engines/mineru/config.py
index ac69a2ba3..389adc967 100644
--- a/deeptutor/services/parsing/engines/mineru/config.py
+++ b/deeptutor/services/parsing/engines/mineru/config.py
@@ -19,6 +19,12 @@ from deeptutor.services.config.runtime_settings import (
load_mineru_settings,
)
+# The MinerU cloud API rejects any single file above 200 pages, so the PDF
+# auto-slicer must never emit a part larger than that even if a setting asks
+# for it.
+MAX_PAGES_PER_PART_CEILING = 200
+DEFAULT_MAX_PAGES_PER_PART = 180
+
class MinerUError(RuntimeError):
"""Raised when a MinerU parse fails (local CLI missing, cloud API error,
@@ -54,6 +60,17 @@ class MinerUConfig:
# opts in explicitly (Settings → Document Parsing) or via the one-click
# download button. Cloud mode ignores this (no local models).
allow_local_model_download: bool = False
+ # Cloud-mode PDF auto-slicing. PDFs longer than this many pages are split
+ # into per-part PDFs, parsed one by one, and their artifacts merged back
+ # into a single working dir (see ``cloud.parse_cloud``). Values above the
+ # MinerU cloud ceiling are clamped in ``__post_init__``.
+ max_pages_per_part: int = DEFAULT_MAX_PAGES_PER_PART
+
+ def __post_init__(self) -> None:
+ if self.max_pages_per_part > MAX_PAGES_PER_PART_CEILING:
+ object.__setattr__(self, "max_pages_per_part", MAX_PAGES_PER_PART_CEILING)
+ elif self.max_pages_per_part < 1:
+ object.__setattr__(self, "max_pages_per_part", 1)
@property
def is_cloud(self) -> bool:
@@ -95,7 +112,18 @@ def resolve_mineru_config() -> MinerUConfig:
enable_table=bool(settings.get("enable_table", True)),
is_ocr=bool(settings.get("is_ocr", False)),
allow_local_model_download=bool(settings.get("allow_local_model_download", False)),
+ max_pages_per_part=(
+ DEFAULT_MAX_PAGES_PER_PART
+ if settings.get("max_pages_per_part") is None
+ else int(settings.get("max_pages_per_part")) # type: ignore[arg-type]
+ ),
)
-__all__ = ["MinerUConfig", "MinerUError", "resolve_mineru_config"]
+__all__ = [
+ "DEFAULT_MAX_PAGES_PER_PART",
+ "MAX_PAGES_PER_PART_CEILING",
+ "MinerUConfig",
+ "MinerUError",
+ "resolve_mineru_config",
+]
diff --git a/tests/services/parsing/test_mineru_slicer.py b/tests/services/parsing/test_mineru_slicer.py
new file mode 100644
index 000000000..277dfc767
--- /dev/null
+++ b/tests/services/parsing/test_mineru_slicer.py
@@ -0,0 +1,252 @@
+"""Tests for the MinerU cloud auto-slicer (oversized PDF → parts → merged artifacts)."""
+
+from __future__ import annotations
+
+import io
+import json
+from pathlib import Path
+from typing import Callable
+import zipfile
+
+from pypdf import PdfReader, PdfWriter
+import pytest
+
+from deeptutor.services.parsing.engines.mineru import cloud as mineru_cloud
+from deeptutor.services.parsing.engines.mineru import config as mineru_config
+from deeptutor.services.parsing.engines.mineru.config import (
+ MAX_PAGES_PER_PART_CEILING,
+ MinerUConfig,
+)
+
+CLOUD_CFG = MinerUConfig(mode="cloud", api_token="tok")
+
+# Progress substrings that only the auto-slicing path emits.
+_SLICING_MARKERS = ("auto-splitting", "parsing part", "merging part")
+
+
+def _make_pdf(path: Path, pages: int) -> Path:
+ writer = PdfWriter()
+ for _ in range(pages):
+ writer.add_blank_page(width=612, height=792)
+ with open(path, "wb") as out:
+ writer.write(out)
+ return path
+
+
+def _fake_archive_bytes(
+ name: str,
+ md_text: str,
+ images: dict[str, bytes] | None = None,
+ content_items: list | None = None,
+) -> bytes:
+ """Build a zip shaped like a MinerU cloud result for one input file."""
+ buffer = io.BytesIO()
+ with zipfile.ZipFile(buffer, "w") as archive:
+ archive.writestr(f"{name}.md", md_text)
+ archive.writestr(f"{name}_content_list.json", json.dumps(content_items or []))
+ for image_name, data in (images or {}).items():
+ archive.writestr(f"images/{image_name}", data)
+ return buffer.getvalue()
+
+
+def _install_fake_parse(
+ monkeypatch: pytest.MonkeyPatch,
+ md_for: Callable[[Path], str] | None = None,
+ extras: Callable[[Path], tuple[dict[str, bytes], list]] | None = None,
+ record: list | None = None,
+) -> list[Path]:
+ """Replace the cloud upload/poll/download trio with a fake that records the
+ file it was handed and returns a synthetic MinerU archive for it. When
+ ``record`` is given, each call appends ``(part_path, page_count)`` — page
+ counts must be read at call time because part PDFs live in a temp dir that
+ is gone by the time the test asserts."""
+ calls: list[Path] = []
+
+ def fake_upload_and_fetch(
+ client, part_path: Path, config, key_pool, *, report, poll_interval, timeout
+ ) -> bytes:
+ calls.append(part_path)
+ if record is not None:
+ record.append((part_path, len(PdfReader(str(part_path)).pages)))
+ images, content_items = extras(part_path) if extras else ({}, [])
+ return _fake_archive_bytes(
+ part_path.stem,
+ md_for(part_path) if md_for else f"# {part_path.stem}",
+ images,
+ content_items,
+ )
+
+ monkeypatch.setattr(mineru_cloud, "_upload_and_fetch_archive", fake_upload_and_fetch)
+ return calls
+
+
+# ---------------------------------------------------------------------------
+# Config: max_pages_per_part
+# ---------------------------------------------------------------------------
+
+
+def test_max_pages_per_part_defaults_and_clamps() -> None:
+ assert MinerUConfig().max_pages_per_part == 180
+ assert MinerUConfig(max_pages_per_part=500).max_pages_per_part == MAX_PAGES_PER_PART_CEILING
+ assert MinerUConfig(max_pages_per_part=0).max_pages_per_part == 1
+
+
+def test_resolve_mineru_config_reads_max_pages_per_part(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ mineru_config,
+ "load_mineru_settings",
+ lambda: {"mode": "cloud", "api_token": "tok", "max_pages_per_part": 120},
+ )
+ assert mineru_config.resolve_mineru_config().max_pages_per_part == 120
+
+ monkeypatch.setattr(
+ mineru_config, "load_mineru_settings", lambda: {"mode": "cloud", "api_token": "tok"}
+ )
+ assert mineru_config.resolve_mineru_config().max_pages_per_part == 180
+
+
+# ---------------------------------------------------------------------------
+# Slicing decision
+# ---------------------------------------------------------------------------
+
+
+def test_oversized_pdf_is_sliced_into_180_plus_70(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ pdf = _make_pdf(tmp_path / "big.pdf", 250)
+ record: list[tuple[Path, int]] = []
+ calls = _install_fake_parse(monkeypatch, record=record)
+ progress: list[str] = []
+
+ working_dir = mineru_cloud.parse_cloud(
+ pdf, tmp_path / "out", CLOUD_CFG, on_progress=progress.append
+ )
+
+ assert len(record) == 2
+ assert [count for _, count in record] == [180, 70]
+ assert calls[0].name == "big_part01.pdf"
+ assert calls[1].name == "big_part02.pdf"
+ assert working_dir == tmp_path / "out" / "big"
+ assert (working_dir / "big.md").is_file()
+ assert any("part 1/2" in message for message in progress)
+ assert any("part 2/2" in message for message in progress)
+
+
+def test_small_pdf_uses_original_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ pdf = _make_pdf(tmp_path / "small.pdf", 40)
+ calls = _install_fake_parse(monkeypatch)
+ progress: list[str] = []
+
+ working_dir = mineru_cloud.parse_cloud(
+ pdf, tmp_path / "out", CLOUD_CFG, on_progress=progress.append
+ )
+
+ # Exactly one parse round-trip, on the original file — no slicing.
+ assert len(calls) == 1
+ assert calls[0] == pdf
+ assert working_dir == tmp_path / "out" / "small"
+ assert (working_dir / "small.md").is_file()
+ assert not any(marker in message for message in progress for marker in _SLICING_MARKERS)
+
+
+def test_non_pdf_is_never_sliced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ docx = tmp_path / "notes.docx"
+ docx.write_bytes(b"fake docx bytes")
+ calls = _install_fake_parse(monkeypatch)
+ page_counts = []
+
+ real_page_count = mineru_cloud._pdf_page_count
+ monkeypatch.setattr(
+ mineru_cloud, "_pdf_page_count", lambda path: page_counts.append(real_page_count(path)) or 0
+ )
+
+ working_dir = mineru_cloud.parse_cloud(docx, tmp_path / "out", CLOUD_CFG)
+
+ assert len(calls) == 1
+ assert calls[0] == docx
+ assert page_counts == [] # the slicer never even counted pages
+ assert working_dir == tmp_path / "out" / "notes"
+
+
+def test_unreadable_pdf_falls_back_to_original_path(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A PDF pypdf cannot read keeps the legacy single-file behaviour instead of
+ failing at the page-count step."""
+ broken = tmp_path / "broken.pdf"
+ broken.write_bytes(b"%PDF-1.4") # no EOF marker / no page tree
+ calls = _install_fake_parse(monkeypatch)
+ progress: list[str] = []
+
+ working_dir = mineru_cloud.parse_cloud(
+ broken, tmp_path / "out", CLOUD_CFG, on_progress=progress.append
+ )
+
+ assert len(calls) == 1
+ assert calls[0] == broken
+ assert not any(marker in message for message in progress for marker in _SLICING_MARKERS)
+ assert working_dir == tmp_path / "out" / "broken"
+
+
+# ---------------------------------------------------------------------------
+# Merged artifacts
+# ---------------------------------------------------------------------------
+
+
+def test_merged_artifacts_md_order_and_unique_images(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ pdf = _make_pdf(tmp_path / "book.pdf", 250)
+
+ part_bodies = {
+ "book_part01.pdf": (
+ "Alpha intro\n![p1](images/a.jpg)\n![p3](images/b.jpg)\n",
+ {"a.jpg": b"one", "b.jpg": b"two"},
+ ),
+ "book_part02.pdf": ("Beta outro\n![p2](images/a.jpg)\n", {"a.jpg": b"three"}),
+ }
+
+ def md_for(part_path: Path) -> str:
+ return part_bodies[part_path.name][0]
+
+ def extras(part_path: Path) -> tuple[dict[str, bytes], list]:
+ _, images = part_bodies[part_path.name]
+ items = [{"type": "image", "img_path": f"images/{name}"} for name in images]
+ return images, items
+
+ _install_fake_parse(monkeypatch, md_for=md_for, extras=extras)
+
+ working_dir = mineru_cloud.parse_cloud(pdf, tmp_path / "out", CLOUD_CFG)
+
+ # Markdown is concatenated in part order, no separators.
+ merged_md = (working_dir / "book.md").read_text(encoding="utf-8")
+ assert merged_md.index("Alpha intro") < merged_md.index("Beta outro")
+ assert "Alpha intro" in merged_md and "Beta outro" in merged_md
+ assert "book_part01" not in merged_md
+
+ # Every image lands in the merged images/ dir with a globally unique name.
+ images_dir = working_dir / "images"
+ assert sorted(path.name for path in images_dir.iterdir()) == [
+ "part01_a.jpg",
+ "part01_b.jpg",
+ "part02_a.jpg",
+ ]
+ assert (images_dir / "part01_a.jpg").read_bytes() == b"one"
+ assert (images_dir / "part02_a.jpg").read_bytes() == b"three"
+
+ # Markdown references follow the renamed files (no dangling originals).
+ for original in ("images/a.jpg", "images/b.jpg"):
+ assert original not in merged_md
+ assert "images/part01_a.jpg" in merged_md
+ assert "images/part02_a.jpg" in merged_md
+ assert "images/part01_b.jpg" in merged_md
+
+ # content_list is merged in part order with rewritten img_path values.
+ content_list = json.loads((working_dir / "book_content_list.json").read_text(encoding="utf-8"))
+ assert [item["img_path"] for item in content_list] == [
+ "images/part01_a.jpg",
+ "images/part01_b.jpg",
+ "images/part02_a.jpg",
+ ]