DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
feat(parsing): auto-slice oversized PDFs for MinerU cloud parsing
polaris-smart4fe036231b42
diff --git a/deeptutor/services/parsing/engines/mineru/cloud.py b/deeptutor/services/parsing/engines/mineru/cloud.pyindex e9cc9cde7..213057736 100644--- a/deeptutor/services/parsing/engines/mineru/cloud.py+++ b/deeptutor/services/parsing/engines/mineru/cloud.py@@ -20,18 +20,23 @@ simplest correct choice (no nested event loop).from __future__ import annotationsfrom collections.abc import Callable+from dataclasses import dataclassimport io+import jsonimport loggingfrom pathlib import Path+import shutil+import tempfileimport timeimport zipfileimport httpx+from pypdf import PdfReader, PdfWriterfrom deeptutor.services.keypool import KeyPoolfrom .config import MinerUConfig, MinerUError-from .formats import MINERU_SUPPORTED_FORMATS+from .formats import MINERU_PDF_FORMATS, MINERU_SUPPORTED_FORMATSlogger = logging.getLogger(__name__)@@ -91,22 +96,31 @@ def parse_cloud(logger.debug("on_progress callback failed", exc_info=True)with httpx.Client(base_url=base_url, headers={"Accept": "application/json"}) as client:+ if source_path.suffix.lower() in MINERU_PDF_FORMATS:+ page_count = _pdf_page_count(source_path)+ if page_count is not None and page_count > config.max_pages_per_part:+ return _parse_pdf_in_parts(+ client,+ source_path,+ output_base,+ config,+ key_pool,+ page_count=page_count,+ report=report,+ poll_interval=poll_interval,+ timeout=timeout,+ )+report(f"MinerU cloud: requesting upload slot for {source_path.name}")- batch_id, upload_url = _request_upload(client, source_path, config, key_pool)- size_mb = source_path.stat().st_size / (1024 * 1024)- report(f"MinerU cloud: uploading {source_path.name} ({size_mb:.1f} MB)")- _upload_file(source_path, upload_url)- zip_url = _poll_for_zip(+ archive_bytes = _upload_and_fetch_archive(client,- batch_id,- source_path.name,+ source_path,+ config,+ key_pool,+ report=report,poll_interval=poll_interval,timeout=timeout,- on_progress=on_progress,- key_pool=key_pool,)- report("MinerU cloud: downloading parsed result archive")- archive_bytes = _download(zip_url)report("MinerU cloud: extracting archive")working_dir = output_base / source_path.stem@@ -121,6 +135,229 @@ def parse_cloud(# ---------------------------------------------------------------------------+def _upload_and_fetch_archive(+ client: httpx.Client,+ part_path: Path,+ config: MinerUConfig,+ key_pool: KeyPool,+ *,+ report: Callable[[str], None],+ poll_interval: float,+ timeout: float,+) -> bytes:+ """Run the upload → poll → download trio for one input file and return the+ result archive bytes. Shared by the single-file path and each slice of the+ auto-split path."""+ report(f"MinerU cloud: requesting upload slot for {part_path.name}")+ batch_id, upload_url = _request_upload(client, part_path, config, key_pool)+ size_mb = part_path.stat().st_size / (1024 * 1024)+ report(f"MinerU cloud: uploading {part_path.name} ({size_mb:.1f} MB)")+ _upload_file(part_path, upload_url)+ zip_url = _poll_for_zip(+ client,+ batch_id,+ part_path.name,+ poll_interval=poll_interval,+ timeout=timeout,+ on_progress=report,+ key_pool=key_pool,+ )+ report("MinerU cloud: downloading parsed result archive")+ return _download(zip_url)+++def _parse_pdf_in_parts(+ client: httpx.Client,+ source_path: Path,+ output_base: Path,+ config: MinerUConfig,+ key_pool: KeyPool,+ *,+ page_count: int,+ report: Callable[[str], None],+ poll_interval: float,+ timeout: float,+) -> Path:+ """Slice an oversized PDF, parse each part through the normal cloud flow,+ and merge the per-part artifacts into one working dir that is isomorphic to+ the single-file output (one ``*.md`` + ``*_content_list.json`` + ``images/``)."""+ working_dir = output_base / source_path.stem+ _reset_dir(working_dir)+ markdown_chunks: list[str] = []+ content_items: list = []+ has_content_list = False++ with tempfile.TemporaryDirectory(prefix="mineru-slice-") as tmp:+ parts = _split_pdf(source_path, config.max_pages_per_part, Path(tmp))+ total = len(parts)+ report(+ f"MinerU cloud: {source_path.name} has {page_count} pages, exceeding the "+ f"{config.max_pages_per_part}-page per-file limit; auto-splitting into "+ f"{total} part(s)"+ )+ for index, part in enumerate(parts, start=1):+ report(+ f"MinerU cloud: parsing part {index}/{total} (pages {part.start + 1}-{part.end})"+ )+ archive_bytes = _upload_and_fetch_archive(+ client,+ part.path,+ config,+ key_pool,+ report=report,+ poll_interval=poll_interval,+ timeout=timeout,+ )+ report(f"MinerU cloud: merging part {index}/{total} artifacts")+ with tempfile.TemporaryDirectory(prefix="mineru-part-") as part_tmp:+ part_dir = Path(part_tmp) / "out"+ part_dir.mkdir()+ _extract_archive(archive_bytes, part_dir)+ _merge_part_artifacts(+ part_dir,+ working_dir,+ markdown_chunks=markdown_chunks,+ content_items=content_items,+ part_index=index,+ )+ has_content_list = True++ merged_md = "".join(+ chunk if chunk.endswith("\n") else chunk + "\n" for chunk in markdown_chunks+ )+ (working_dir / f"{source_path.stem}.md").write_text(merged_md, encoding="utf-8")+ if has_content_list:+ content_list_path = working_dir / f"{source_path.stem}_content_list.json"+ content_list_path.write_text(+ json.dumps(content_items, ensure_ascii=False), encoding="utf-8"+ )+ logger.info(+ "MinerU cloud parse complete (%d parts): %s → %s", total, source_path.name, working_dir+ )+ return working_dir+++def _pdf_page_count(source_path: Path) -> int | None:+ """Number of pages in a PDF, or ``None`` when pypdf cannot read it (e.g. a+ truncated/encrypted file). ``None`` keeps the legacy single-file behaviour —+ an unreadable PDF goes to MinerU as-is and fails there if it is truly broken."""+ try:+ with PdfReader(str(source_path)) as reader:+ return len(reader.pages)+ except Exception:+ logger.warning("Could not count pages of %s; skipping auto-slicing", source_path.name)+ return None+++def _split_pdf(source_path: Path, max_pages_per_part: int, work_dir: Path) -> list[_PdfPart]:+ """Evenly cut ``source_path`` into parts of at most ``max_pages_per_part``+ pages (the last part may be shorter) and write each part PDF into+ ``work_dir``. 250 pages @180 → [180, 70]."""+ try:+ with PdfReader(str(source_path)) as reader:+ total = len(reader.pages)+ parts: list[_PdfPart] = []+ for index, start in enumerate(range(0, total, max_pages_per_part), start=1):+ end = min(start + max_pages_per_part, total)+ part_path = work_dir / f"{source_path.stem}_part{index:02d}.pdf"+ writer = PdfWriter()+ for page_number in range(start, end):+ writer.add_page(reader.pages[page_number])+ with open(part_path, "wb") as out:+ writer.write(out)+ parts.append(_PdfPart(path=part_path, start=start, end=end))+ return parts+ except Exception as exc:+ raise MinerUError(f"Failed to split PDF {source_path.name} into parts: {exc}") from exc+++def _merge_part_artifacts(+ part_dir: Path,+ working_dir: Path,+ *,+ markdown_chunks: list[str],+ content_items: list,+ part_index: int,+) -> None:+ """Accumulate one part's artifacts: markdown text, ``images/`` (renamed with+ a part prefix so names stay globally unique) and ``content_list`` entries+ (with ``img_path`` rewritten to match)."""+ md_files = sorted(path for path in part_dir.rglob("*.md") if path.is_file())+ if not md_files:+ raise MinerUError(f"MinerU part {part_index} archive contains no markdown output.")+ content_root = md_files[0].parent++ rename_map = _copy_part_images(content_root / "images", working_dir / "images", part_index)++ markdown = md_files[0].read_text(encoding="utf-8")+ for original, unique in rename_map.items():+ markdown = markdown.replace(f"images/{original}", f"images/{unique}")+ markdown_chunks.append(markdown)++ for content_list_path in sorted(content_root.glob("*_content_list.json")):+ try:+ payload = json.loads(content_list_path.read_text(encoding="utf-8"))+ except (OSError, ValueError) as exc:+ raise MinerUError(+ f"MinerU part {part_index} content list is unreadable: {exc}"+ ) from exc+ if isinstance(payload, list):+ _rewrite_image_paths(payload, rename_map)+ content_items.extend(payload)++ # Keep the remaining artifacts (middle/model/layout files, origin PDF, …)+ # on disk with a part prefix, mirroring the single-file layout as closely+ # as their per-part nature allows.+ for extra in sorted(content_root.iterdir()):+ if not extra.is_file() or extra.suffix.lower() == ".md":+ continue+ if extra.name.endswith("_content_list.json"):+ continue+ shutil.copyfile(extra, working_dir / f"part{part_index:02d}_{extra.name}")+++def _copy_part_images(images_dir: Path, target_dir: Path, part_index: int) -> dict[str, str]:+ """Copy one part's ``images/`` into the merged ``images/`` dir, prefixing+ each name with the part index so names stay globally unique across parts.+ Returns ``{original_name: unique_name}``."""+ if not images_dir.is_dir():+ return {}+ target_dir.mkdir(parents=True, exist_ok=True)+ rename_map: dict[str, str] = {}+ for image in sorted(images_dir.iterdir()):+ if not image.is_file():+ continue+ unique_name = f"part{part_index:02d}_{image.name}"+ shutil.copyfile(image, target_dir / unique_name)+ rename_map[image.name] = unique_name+ return rename_map+++def _rewrite_image_paths(node: object, rename_map: dict[str, str]) -> None:+ """Rewrite ``img_path``/``image_path`` values in a content-list tree to the+ renamed (part-prefixed) image files."""+ if isinstance(node, dict):+ for key, value in node.items():+ if key in ("img_path", "image_path") and isinstance(value, str):+ name = value.rsplit("/", 1)[-1]+ if name in rename_map:+ node[key] = value.replace(name, rename_map[name])+ else:+ _rewrite_image_paths(value, rename_map)+ elif isinstance(node, list):+ for item in node:+ _rewrite_image_paths(item, rename_map)+++@dataclass(frozen=True)+class _PdfPart:+ """One slice of an oversized PDF: 0-based ``start``, exclusive ``end``."""++ path: Path+ start: int+ end: int++def _request_upload(client: httpx.Client, source_path: Path, config: MinerUConfig, key_pool: KeyPool) -> tuple[str, str]:diff --git a/deeptutor/services/parsing/engines/mineru/config.py b/deeptutor/services/parsing/engines/mineru/config.pyindex ac69a2ba3..389adc967 100644--- a/deeptutor/services/parsing/engines/mineru/config.py+++ b/deeptutor/services/parsing/engines/mineru/config.py@@ -19,6 +19,12 @@ from deeptutor.services.config.runtime_settings import (load_mineru_settings,)+# The MinerU cloud API rejects any single file above 200 pages, so the PDF+# auto-slicer must never emit a part larger than that even if a setting asks+# for it.+MAX_PAGES_PER_PART_CEILING = 200+DEFAULT_MAX_PAGES_PER_PART = 180+class MinerUError(RuntimeError):"""Raised when a MinerU parse fails (local CLI missing, cloud API error,@@ -54,6 +60,17 @@ class MinerUConfig:# opts in explicitly (Settings → Document Parsing) or via the one-click# download button. Cloud mode ignores this (no local models).allow_local_model_download: bool = False+ # Cloud-mode PDF auto-slicing. PDFs longer than this many pages are split+ # into per-part PDFs, parsed one by one, and their artifacts merged back+ # into a single working dir (see ``cloud.parse_cloud``). Values above the+ # MinerU cloud ceiling are clamped in ``__post_init__``.+ max_pages_per_part: int = DEFAULT_MAX_PAGES_PER_PART++ def __post_init__(self) -> None:+ if self.max_pages_per_part > MAX_PAGES_PER_PART_CEILING:+ object.__setattr__(self, "max_pages_per_part", MAX_PAGES_PER_PART_CEILING)+ elif self.max_pages_per_part < 1:+ object.__setattr__(self, "max_pages_per_part", 1)@propertydef is_cloud(self) -> bool:@@ -95,7 +112,18 @@ def resolve_mineru_config() -> MinerUConfig:enable_table=bool(settings.get("enable_table", True)),is_ocr=bool(settings.get("is_ocr", False)),allow_local_model_download=bool(settings.get("allow_local_model_download", False)),+ max_pages_per_part=(+ DEFAULT_MAX_PAGES_PER_PART+ if settings.get("max_pages_per_part") is None+ else int(settings.get("max_pages_per_part")) # type: ignore[arg-type]+ ),)-__all__ = ["MinerUConfig", "MinerUError", "resolve_mineru_config"]+__all__ = [+ "DEFAULT_MAX_PAGES_PER_PART",+ "MAX_PAGES_PER_PART_CEILING",+ "MinerUConfig",+ "MinerUError",+ "resolve_mineru_config",+]diff --git a/tests/services/parsing/test_mineru_slicer.py b/tests/services/parsing/test_mineru_slicer.pynew file mode 100644index 000000000..277dfc767--- /dev/null+++ b/tests/services/parsing/test_mineru_slicer.py@@ -0,0 +1,252 @@+"""Tests for the MinerU cloud auto-slicer (oversized PDF → parts → merged artifacts)."""++from __future__ import annotations++import io+import json+from pathlib import Path+from typing import Callable+import zipfile++from pypdf import PdfReader, PdfWriter+import pytest++from deeptutor.services.parsing.engines.mineru import cloud as mineru_cloud+from deeptutor.services.parsing.engines.mineru import config as mineru_config+from deeptutor.services.parsing.engines.mineru.config import (+ MAX_PAGES_PER_PART_CEILING,+ MinerUConfig,+)++CLOUD_CFG = MinerUConfig(mode="cloud", api_token="tok")++# Progress substrings that only the auto-slicing path emits.+_SLICING_MARKERS = ("auto-splitting", "parsing part", "merging part")+++def _make_pdf(path: Path, pages: int) -> Path:+ writer = PdfWriter()+ for _ in range(pages):+ writer.add_blank_page(width=612, height=792)+ with open(path, "wb") as out:+ writer.write(out)+ return path+++def _fake_archive_bytes(+ name: str,+ md_text: str,+ images: dict[str, bytes] | None = None,+ content_items: list | None = None,+) -> bytes:+ """Build a zip shaped like a MinerU cloud result for one input file."""+ buffer = io.BytesIO()+ with zipfile.ZipFile(buffer, "w") as archive:+ archive.writestr(f"{name}.md", md_text)+ archive.writestr(f"{name}_content_list.json", json.dumps(content_items or []))+ for image_name, data in (images or {}).items():+ archive.writestr(f"images/{image_name}", data)+ return buffer.getvalue()+++def _install_fake_parse(+ monkeypatch: pytest.MonkeyPatch,+ md_for: Callable[[Path], str] | None = None,+ extras: Callable[[Path], tuple[dict[str, bytes], list]] | None = None,+ record: list | None = None,+) -> list[Path]:+ """Replace the cloud upload/poll/download trio with a fake that records the+ file it was handed and returns a synthetic MinerU archive for it. When+ ``record`` is given, each call appends ``(part_path, page_count)`` — page+ counts must be read at call time because part PDFs live in a temp dir that+ is gone by the time the test asserts."""+ calls: list[Path] = []++ def fake_upload_and_fetch(+ client, part_path: Path, config, key_pool, *, report, poll_interval, timeout+ ) -> bytes:+ calls.append(part_path)+ if record is not None:+ record.append((part_path, len(PdfReader(str(part_path)).pages)))+ images, content_items = extras(part_path) if extras else ({}, [])+ return _fake_archive_bytes(+ part_path.stem,+ md_for(part_path) if md_for else f"# {part_path.stem}",+ images,+ content_items,+ )++ monkeypatch.setattr(mineru_cloud, "_upload_and_fetch_archive", fake_upload_and_fetch)+ return calls+++# ---------------------------------------------------------------------------+# Config: max_pages_per_part+# ---------------------------------------------------------------------------+++def test_max_pages_per_part_defaults_and_clamps() -> None:+ assert MinerUConfig().max_pages_per_part == 180+ assert MinerUConfig(max_pages_per_part=500).max_pages_per_part == MAX_PAGES_PER_PART_CEILING+ assert MinerUConfig(max_pages_per_part=0).max_pages_per_part == 1+++def test_resolve_mineru_config_reads_max_pages_per_part(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ mineru_config,+ "load_mineru_settings",+ lambda: {"mode": "cloud", "api_token": "tok", "max_pages_per_part": 120},+ )+ assert mineru_config.resolve_mineru_config().max_pages_per_part == 120++ monkeypatch.setattr(+ mineru_config, "load_mineru_settings", lambda: {"mode": "cloud", "api_token": "tok"}+ )+ assert mineru_config.resolve_mineru_config().max_pages_per_part == 180+++# ---------------------------------------------------------------------------+# Slicing decision+# ---------------------------------------------------------------------------+++def test_oversized_pdf_is_sliced_into_180_plus_70(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ pdf = _make_pdf(tmp_path / "big.pdf", 250)+ record: list[tuple[Path, int]] = []+ calls = _install_fake_parse(monkeypatch, record=record)+ progress: list[str] = []++ working_dir = mineru_cloud.parse_cloud(+ pdf, tmp_path / "out", CLOUD_CFG, on_progress=progress.append+ )++ assert len(record) == 2+ assert [count for _, count in record] == [180, 70]+ assert calls[0].name == "big_part01.pdf"+ assert calls[1].name == "big_part02.pdf"+ assert working_dir == tmp_path / "out" / "big"+ assert (working_dir / "big.md").is_file()+ assert any("part 1/2" in message for message in progress)+ assert any("part 2/2" in message for message in progress)+++def test_small_pdf_uses_original_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ pdf = _make_pdf(tmp_path / "small.pdf", 40)+ calls = _install_fake_parse(monkeypatch)+ progress: list[str] = []++ working_dir = mineru_cloud.parse_cloud(+ pdf, tmp_path / "out", CLOUD_CFG, on_progress=progress.append+ )++ # Exactly one parse round-trip, on the original file — no slicing.+ assert len(calls) == 1+ assert calls[0] == pdf+ assert working_dir == tmp_path / "out" / "small"+ assert (working_dir / "small.md").is_file()+ assert not any(marker in message for message in progress for marker in _SLICING_MARKERS)+++def test_non_pdf_is_never_sliced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ docx = tmp_path / "notes.docx"+ docx.write_bytes(b"fake docx bytes")+ calls = _install_fake_parse(monkeypatch)+ page_counts = []++ real_page_count = mineru_cloud._pdf_page_count+ monkeypatch.setattr(+ mineru_cloud, "_pdf_page_count", lambda path: page_counts.append(real_page_count(path)) or 0+ )++ working_dir = mineru_cloud.parse_cloud(docx, tmp_path / "out", CLOUD_CFG)++ assert len(calls) == 1+ assert calls[0] == docx+ assert page_counts == [] # the slicer never even counted pages+ assert working_dir == tmp_path / "out" / "notes"+++def test_unreadable_pdf_falls_back_to_original_path(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A PDF pypdf cannot read keeps the legacy single-file behaviour instead of+ failing at the page-count step."""+ broken = tmp_path / "broken.pdf"+ broken.write_bytes(b"%PDF-1.4") # no EOF marker / no page tree+ calls = _install_fake_parse(monkeypatch)+ progress: list[str] = []++ working_dir = mineru_cloud.parse_cloud(+ broken, tmp_path / "out", CLOUD_CFG, on_progress=progress.append+ )++ assert len(calls) == 1+ assert calls[0] == broken+ assert not any(marker in message for message in progress for marker in _SLICING_MARKERS)+ assert working_dir == tmp_path / "out" / "broken"+++# ---------------------------------------------------------------------------+# Merged artifacts+# ---------------------------------------------------------------------------+++def test_merged_artifacts_md_order_and_unique_images(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ pdf = _make_pdf(tmp_path / "book.pdf", 250)++ part_bodies = {+ "book_part01.pdf": (+ "Alpha intro\n\n\n",+ {"a.jpg": b"one", "b.jpg": b"two"},+ ),+ "book_part02.pdf": ("Beta outro\n\n", {"a.jpg": b"three"}),+ }++ def md_for(part_path: Path) -> str:+ return part_bodies[part_path.name][0]++ def extras(part_path: Path) -> tuple[dict[str, bytes], list]:+ _, images = part_bodies[part_path.name]+ items = [{"type": "image", "img_path": f"images/{name}"} for name in images]+ return images, items++ _install_fake_parse(monkeypatch, md_for=md_for, extras=extras)++ working_dir = mineru_cloud.parse_cloud(pdf, tmp_path / "out", CLOUD_CFG)++ # Markdown is concatenated in part order, no separators.+ merged_md = (working_dir / "book.md").read_text(encoding="utf-8")+ assert merged_md.index("Alpha intro") < merged_md.index("Beta outro")+ assert "Alpha intro" in merged_md and "Beta outro" in merged_md+ assert "book_part01" not in merged_md++ # Every image lands in the merged images/ dir with a globally unique name.+ images_dir = working_dir / "images"+ assert sorted(path.name for path in images_dir.iterdir()) == [+ "part01_a.jpg",+ "part01_b.jpg",+ "part02_a.jpg",+ ]+ assert (images_dir / "part01_a.jpg").read_bytes() == b"one"+ assert (images_dir / "part02_a.jpg").read_bytes() == b"three"++ # Markdown references follow the renamed files (no dangling originals).+ for original in ("images/a.jpg", "images/b.jpg"):+ assert original not in merged_md+ assert "images/part01_a.jpg" in merged_md+ assert "images/part02_a.jpg" in merged_md+ assert "images/part01_b.jpg" in merged_md++ # content_list is merged in part order with rewritten img_path values.+ content_list = json.loads((working_dir / "book_content_list.json").read_text(encoding="utf-8"))+ assert [item["img_path"] for item in content_list] == [+ "images/part01_a.jpg",+ "images/part01_b.jpg",+ "images/part02_a.jpg",+ ]