DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""MinerU cloud (mineru.net) v4 API backend.
Implements the token-required *Precision API* flow for one local input:
1. ``POST /api/v4/file-urls/batch`` → ``{batch_id, file_urls: [signed_url]}``
2. ``PUT`` the raw file bytes to ``signed_url`` (no auth, no Content-Type)
3. Poll ``GET /api/v4/extract-results/batch/{batch_id}`` until the file's
``state`` reaches ``done`` / ``failed``
4. Download the ``full_zip_url`` archive and extract it into a working dir
whose layout matches the local CLI output (``*.md`` +
``*_content_list.json`` + ``images/``), so the downstream question
extractor is backend-agnostic.
The module is synchronous on purpose: it runs inside the worker thread that
:func:`deeptutor.agents.question.mimic_source.parse_exam_paper_to_templates`
spawns via ``asyncio.to_thread``, so a blocking ``httpx.Client`` is the
simplest correct choice (no nested event loop).
"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
import io
import json
import logging
from pathlib import Path
import shutil
import tempfile
import time
import zipfile
import httpx
from pypdf import PdfReader, PdfWriter
from deeptutor.services.keypool import KeyPool
from .checkpoints import SliceCheckpoint, job_directory
from .config import MinerUConfig, MinerUError
from .formats import MINERU_PDF_FORMATS, MINERU_SUPPORTED_FORMATS
logger = logging.getLogger(__name__)
# Async polling defaults. MinerU recommends a 3–5s interval; parsing a typical
# exam paper completes well under a few minutes.
DEFAULT_POLL_INTERVAL_SECONDS = 4.0
DEFAULT_TIMEOUT_SECONDS = 300.0
_SUBMIT_TIMEOUT_SECONDS = 60.0
_UPLOAD_TIMEOUT_SECONDS = 300.0
_DOWNLOAD_TIMEOUT_SECONDS = 300.0
_TERMINAL_OK = "done"
_TERMINAL_FAIL = "failed"
# Bounds for the extracted archive (defends a hostile/buggy CDN response).
_MAX_TOTAL_BYTES = 500 * 1024 * 1024
_MAX_ENTRIES = 5000
def parse_cloud(
source_path: Path,
output_base: Path,
config: MinerUConfig,
*,
poll_interval: float = DEFAULT_POLL_INTERVAL_SECONDS,
timeout: float = DEFAULT_TIMEOUT_SECONDS,
on_progress: Callable[[str], None] | None = None,
) -> Path:
"""Parse ``source_path`` via the MinerU cloud API; return the working dir.
The working dir sits under ``output_base`` (named after the input stem) and
holds the unzipped MinerU artifacts. ``on_progress`` (if given) receives a
short status line whenever the polled task state / page count changes.
Raises :class:`MinerUError` on any misconfiguration, API error, timeout,
or extraction failure.
"""
if not config.api_keys:
raise MinerUError(
"MinerU cloud mode is selected but no API token is configured. "
"Add a token in Settings → MinerU, or switch to local mode."
)
source_path = Path(source_path)
if not source_path.is_file():
raise MinerUError(f"Input file not found: {source_path}")
if source_path.suffix.lower() not in MINERU_SUPPORTED_FORMATS:
raise MinerUError(f"Unsupported MinerU cloud input format: {source_path.suffix or 'none'}")
base_url = config.api_base_url.rstrip("/")
key_pool = KeyPool(config.api_keys)
def report(message: str) -> None:
if on_progress is None:
return
try:
on_progress(message)
except Exception:
logger.debug("on_progress callback failed", exc_info=True)
with httpx.Client(base_url=base_url, headers={"Accept": "application/json"}) as client:
if source_path.suffix.lower() in MINERU_PDF_FORMATS:
page_count = _pdf_page_count(source_path)
if page_count is not None and page_count > config.max_pages_per_part:
return _parse_pdf_in_parts(
client,
source_path,
output_base,
config,
key_pool,
page_count=page_count,
report=report,
poll_interval=poll_interval,
timeout=timeout,
)
report(f"MinerU cloud: requesting upload slot for {source_path.name}")
archive_bytes = _upload_and_fetch_archive(
client,
source_path,
config,
key_pool,
report=report,
poll_interval=poll_interval,
timeout=timeout,
)
report("MinerU cloud: extracting archive")
working_dir = output_base / source_path.stem
_reset_dir(working_dir)
_extract_archive(archive_bytes, working_dir)
logger.info("MinerU cloud parse complete: %s → %s", source_path.name, working_dir)
return working_dir
# ---------------------------------------------------------------------------
# Steps
# ---------------------------------------------------------------------------
def _upload_and_fetch_archive(
client: httpx.Client,
part_path: Path,
config: MinerUConfig,
key_pool: KeyPool,
*,
report: Callable[[str], None],
poll_interval: float,
timeout: float,
) -> bytes:
"""Run the upload → poll → download trio for one input file and return the
result archive bytes. Shared by the single-file path and each slice of the
auto-split path."""
report(f"MinerU cloud: requesting upload slot for {part_path.name}")
batch_id, upload_url = _request_upload(client, part_path, config, key_pool)
size_mb = part_path.stat().st_size / (1024 * 1024)
report(f"MinerU cloud: uploading {part_path.name} ({size_mb:.1f} MB)")
_upload_file(part_path, upload_url)
zip_url = _poll_for_zip(
client,
batch_id,
part_path.name,
poll_interval=poll_interval,
timeout=timeout,
on_progress=report,
key_pool=key_pool,
)
report("MinerU cloud: downloading parsed result archive")
return _download(zip_url)
def _parse_pdf_in_parts(
client: httpx.Client,
source_path: Path,
output_base: Path,
config: MinerUConfig,
key_pool: KeyPool,
*,
page_count: int,
report: Callable[[str], None],
poll_interval: float,
timeout: float,
) -> Path:
"""Slice an oversized PDF, parse each part through the normal cloud flow,
and merge the per-part artifacts into one working dir that is isomorphic to
the single-file output (one ``*.md`` + ``*_content_list.json`` + ``images/``)."""
working_dir = output_base / source_path.stem
_reset_dir(working_dir)
markdown_chunks: list[str] = []
content_items: list = []
has_content_list = False
checkpoint_job = job_directory(source_path, config)
with tempfile.TemporaryDirectory(prefix="mineru-slice-") as tmp:
parts = _split_pdf(source_path, config.max_pages_per_part, Path(tmp))
total = len(parts)
report(
f"MinerU cloud: {source_path.name} has {page_count} pages, exceeding the "
f"{config.max_pages_per_part}-page per-file limit; auto-splitting into "
f"{total} part(s)"
)
for index, part in enumerate(parts, start=1):
report(
f"MinerU cloud: parsing part {index}/{total} (pages {part.start + 1}-{part.end})"
)
checkpoint = SliceCheckpoint(checkpoint_job, part.start, part.end)
archive_bytes = checkpoint.load()
if archive_bytes is None:
archive_bytes = _upload_and_fetch_archive(
client,
part.path,
config,
key_pool,
report=report,
poll_interval=poll_interval,
timeout=timeout,
)
else:
report(f"MinerU cloud: reusing completed part {index}/{total}")
report(f"MinerU cloud: merging part {index}/{total} artifacts")
with tempfile.TemporaryDirectory(prefix="mineru-part-") as part_tmp:
part_dir = Path(part_tmp) / "out"
part_dir.mkdir()
_extract_archive(archive_bytes, part_dir)
_merge_part_artifacts(
part_dir,
working_dir,
markdown_chunks=markdown_chunks,
content_items=content_items,
part_index=index,
page_offset=part.start,
page_count=part.end - part.start,
)
has_content_list = True
try:
checkpoint.save(archive_bytes)
except OSError:
logger.warning("Could not save MinerU slice checkpoint; parsing continues")
merged_md = "".join(
chunk if chunk.endswith("\n") else chunk + "\n" for chunk in markdown_chunks
)
(working_dir / f"{source_path.stem}.md").write_text(merged_md, encoding="utf-8")
if has_content_list:
content_list_path = working_dir / f"{source_path.stem}_content_list.json"
content_list_path.write_text(
json.dumps(content_items, ensure_ascii=False), encoding="utf-8"
)
logger.info(
"MinerU cloud parse complete (%d parts): %s → %s", total, source_path.name, working_dir
)
return working_dir
def _pdf_page_count(source_path: Path) -> int | None:
"""Number of pages in a PDF, or ``None`` when pypdf cannot read it (e.g. a
truncated/encrypted file). ``None`` keeps the legacy single-file behaviour —
an unreadable PDF goes to MinerU as-is and fails there if it is truly broken."""
try:
with PdfReader(str(source_path)) as reader:
return len(reader.pages)
except Exception:
logger.warning("Could not count pages of %s; skipping auto-slicing", source_path.name)
return None
def _split_pdf(source_path: Path, max_pages_per_part: int, work_dir: Path) -> list[_PdfPart]:
"""Evenly cut ``source_path`` into parts of at most ``max_pages_per_part``
pages (the last part may be shorter) and write each part PDF into
``work_dir``. 250 pages @180 → [180, 70]."""
try:
with PdfReader(str(source_path)) as reader:
total = len(reader.pages)
parts: list[_PdfPart] = []
for index, start in enumerate(range(0, total, max_pages_per_part), start=1):
end = min(start + max_pages_per_part, total)
part_path = work_dir / f"{source_path.stem}_part{index:02d}.pdf"
writer = PdfWriter()
for page_number in range(start, end):
writer.add_page(reader.pages[page_number])
with open(part_path, "wb") as out:
writer.write(out)
parts.append(_PdfPart(path=part_path, start=start, end=end))
return parts
except Exception as exc:
raise MinerUError(f"Failed to split PDF {source_path.name} into parts: {exc}") from exc
def _merge_part_artifacts(
part_dir: Path,
working_dir: Path,
*,
markdown_chunks: list[str],
content_items: list,
part_index: int,
page_offset: int = 0,
page_count: int | None = None,
) -> None:
"""Accumulate one part's artifacts: markdown text, ``images/`` (renamed with
a part prefix so names stay globally unique) and ``content_list`` entries
(with ``img_path`` rewritten to match)."""
md_files = sorted(path for path in part_dir.rglob("*.md") if path.is_file())
if not md_files:
raise MinerUError(f"MinerU part {part_index} archive contains no markdown output.")
content_root = md_files[0].parent
rename_map = _copy_part_images(content_root / "images", working_dir / "images", part_index)
markdown = md_files[0].read_text(encoding="utf-8")
for original, unique in rename_map.items():
markdown = markdown.replace(f"images/{original}", f"images/{unique}")
markdown_chunks.append(markdown)
for content_list_path in sorted(content_root.glob("*_content_list.json")):
try:
payload = json.loads(content_list_path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
raise MinerUError(
f"MinerU part {part_index} content list is unreadable: {exc}"
) from exc
if isinstance(payload, list):
_offset_page_indices(payload, page_offset, page_count)
_rewrite_image_paths(payload, rename_map)
content_items.extend(payload)
# Keep the remaining artifacts (middle/model/layout files, origin PDF, …)
# on disk with a part prefix, mirroring the single-file layout as closely
# as their per-part nature allows.
for extra in sorted(content_root.iterdir()):
if not extra.is_file() or extra.suffix.lower() == ".md":
continue
if extra.name.endswith("_content_list.json"):
continue
shutil.copyfile(extra, working_dir / f"part{part_index:02d}_{extra.name}")
def _offset_page_indices(node: object, offset: int, page_count: int | None) -> None:
"""Restore zero-based source page indices, including nested content blocks."""
if isinstance(node, dict):
if "page_idx" in node:
index = node["page_idx"]
if (
type(index) is not int
or index < 0
or (page_count is not None and index >= page_count)
):
raise MinerUError("MinerU slice returned an invalid page_idx")
node["page_idx"] = index + offset
for value in node.values():
_offset_page_indices(value, offset, page_count)
elif isinstance(node, list):
for value in node:
_offset_page_indices(value, offset, page_count)
def _copy_part_images(images_dir: Path, target_dir: Path, part_index: int) -> dict[str, str]:
"""Copy one part's ``images/`` into the merged ``images/`` dir, prefixing
each name with the part index so names stay globally unique across parts.
Returns ``{original_name: unique_name}``."""
if not images_dir.is_dir():
return {}
target_dir.mkdir(parents=True, exist_ok=True)
rename_map: dict[str, str] = {}
for image in sorted(images_dir.iterdir()):
if not image.is_file():
continue
unique_name = f"part{part_index:02d}_{image.name}"
shutil.copyfile(image, target_dir / unique_name)
rename_map[image.name] = unique_name
return rename_map
def _rewrite_image_paths(node: object, rename_map: dict[str, str]) -> None:
"""Rewrite ``img_path``/``image_path`` values in a content-list tree to the
renamed (part-prefixed) image files."""
if isinstance(node, dict):
for key, value in node.items():
if key in ("img_path", "image_path") and isinstance(value, str):
name = value.rsplit("/", 1)[-1]
if name in rename_map:
node[key] = value.replace(name, rename_map[name])
else:
_rewrite_image_paths(value, rename_map)
elif isinstance(node, list):
for item in node:
_rewrite_image_paths(item, rename_map)
@dataclass(frozen=True)
class _PdfPart:
"""One slice of an oversized PDF: 0-based ``start``, exclusive ``end``."""
path: Path
start: int
end: int
def _request_upload(
client: httpx.Client, source_path: Path, config: MinerUConfig, key_pool: KeyPool
) -> tuple[str, str]:
"""POST file-urls/batch → ``(batch_id, signed_upload_url)``."""
file_entry: dict[str, object] = {"name": source_path.name, "is_ocr": config.is_ocr}
body: dict[str, object] = {
"files": [file_entry],
"model_version": config.model_version,
"enable_formula": config.enable_formula,
"enable_table": config.enable_table,
}
if config.api_language:
body["language"] = config.api_language
payload = _post_json(client, "/api/v4/file-urls/batch", body, key_pool)
data = payload.get("data") or {}
batch_id = str(data.get("batch_id") or "").strip()
file_urls = data.get("file_urls") or []
if not batch_id or not isinstance(file_urls, list) or not file_urls:
raise MinerUError("MinerU API did not return an upload URL (missing batch_id/file_urls).")
return batch_id, str(file_urls[0])
def _upload_file(source_path: Path, upload_url: str) -> None:
"""PUT the input bytes to the signed URL.
The signed URL carries its own auth; per MinerU's docs we must NOT send an
``Authorization`` or ``Content-Type`` header (a stray Content-Type breaks
the OSS signature).
"""
data = source_path.read_bytes()
try:
response = httpx.put(upload_url, content=data, timeout=_UPLOAD_TIMEOUT_SECONDS)
response.raise_for_status()
except httpx.HTTPError as exc:
raise MinerUError(f"Failed to upload file to MinerU: {exc}") from exc
def _poll_for_zip(
client: httpx.Client,
batch_id: str,
file_name: str,
*,
key_pool: KeyPool,
poll_interval: float,
timeout: float,
on_progress: Callable[[str], None] | None = None,
) -> str:
"""Poll the batch results until our file is ``done``; return full_zip_url."""
deadline = time.monotonic() + timeout
last_state = ""
last_report = ""
while True:
payload = _get_json(client, f"/api/v4/extract-results/batch/{batch_id}", key_pool)
results = (payload.get("data") or {}).get("extract_result") or []
entry = _match_entry(results, file_name)
if entry is not None:
state = str(entry.get("state") or "").strip().lower()
last_state = state or last_state
if on_progress is not None:
progress = entry.get("extract_progress") or {}
total_pages = progress.get("total_pages")
report = f"MinerU cloud: {state or 'queued'}"
if total_pages:
report += f" ({progress.get('extracted_pages') or 0}/{total_pages} pages)"
if report != last_report:
last_report = report
try:
on_progress(report)
except Exception:
on_progress = None
if state == _TERMINAL_OK:
zip_url = str(entry.get("full_zip_url") or "").strip()
if not zip_url:
raise MinerUError("MinerU reported done but returned no full_zip_url.")
return zip_url
if state == _TERMINAL_FAIL:
err = str(entry.get("err_msg") or "unknown error")
raise MinerUError(f"MinerU failed to parse the document: {err}")
if time.monotonic() >= deadline:
raise MinerUError(
f"MinerU parsing timed out after {int(timeout)}s "
f"(last state: {last_state or 'unknown'})."
)
time.sleep(poll_interval)
def verify_credentials(config: MinerUConfig) -> None:
"""Best-effort connectivity / token check for the Settings → MinerU "Test"
button. Requests an upload slot (which does not consume parsing quota and
is never followed by an upload, so it simply expires) and validates the
business code. Raises :class:`MinerUError` with a user-facing message on
any failure."""
if not config.api_keys:
raise MinerUError("No API token configured.")
base_url = config.api_base_url.rstrip("/")
key_pool = KeyPool(config.api_keys)
body: dict[str, object] = {
"files": [{"name": "connectivity-check.pdf", "is_ocr": False}],
"model_version": config.model_version,
"enable_formula": config.enable_formula,
"enable_table": config.enable_table,
}
if config.api_language:
body["language"] = config.api_language
with httpx.Client(base_url=base_url, headers={"Accept": "application/json"}) as client:
_post_json(client, "/api/v4/file-urls/batch", body, key_pool)
def _download(zip_url: str) -> bytes:
try:
response = httpx.get(zip_url, timeout=_DOWNLOAD_TIMEOUT_SECONDS, follow_redirects=True)
response.raise_for_status()
return response.content
except httpx.HTTPError as exc:
raise MinerUError(f"Failed to download MinerU result archive: {exc}") from exc
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _match_entry(results: list, file_name: str) -> dict | None:
"""Pick our file's result row. Single-file batch → first row is ours, but
match on ``file_name`` when present to be safe."""
rows = [r for r in results if isinstance(r, dict)]
if not rows:
return None
for row in rows:
if str(row.get("file_name") or "") == file_name:
return row
return rows[0]
def _request_json(
request: Callable[..., httpx.Response],
path: str,
key_pool: KeyPool,
**kwargs,
) -> dict:
max_attempts = max(2, len(key_pool))
for attempt in range(max_attempts):
api_key = key_pool.next()
try:
response = request(
path,
timeout=_SUBMIT_TIMEOUT_SECONDS,
headers={"Authorization": f"Bearer {api_key}"},
**kwargs,
)
response.raise_for_status()
payload = response.json()
except httpx.HTTPStatusError as exc:
if exc.response.status_code == 429:
key_pool.mark_429(api_key)
if attempt < max_attempts - 1:
continue
raise MinerUError(_http_error_message(exc)) from exc
except httpx.HTTPError as exc:
raise MinerUError(f"MinerU API request failed: {exc}") from exc
_check_code(payload)
return payload
raise MinerUError("MinerU API key rotation exhausted.")
def _post_json(client: httpx.Client, path: str, body: dict, key_pool: KeyPool) -> dict:
return _request_json(client.post, path, key_pool, json=body)
def _get_json(client: httpx.Client, path: str, key_pool: KeyPool) -> dict:
return _request_json(client.get, path, key_pool)
def _check_code(payload: dict) -> None:
"""MinerU wraps errors in ``{"code": <non-zero>, "msg": ...}`` even on
HTTP 200, so the business code must be inspected explicitly."""
if not isinstance(payload, dict):
raise MinerUError("MinerU API returned an unexpected (non-JSON) response.")
code = payload.get("code")
if code not in (0, None):
msg = str(payload.get("msg") or "unknown error")
raise MinerUError(f"MinerU API error (code {code}): {msg}")
def _http_error_message(exc: httpx.HTTPStatusError) -> str:
status = exc.response.status_code
if status in (401, 403):
return "MinerU API rejected the token (401/403). Check the API token in Settings → MinerU."
if status == 429:
return "MinerU API rate limit hit (429). Try again later or reduce request volume."
return f"MinerU API returned HTTP {status}."
def _reset_dir(path: Path) -> None:
if path.exists():
import shutil
shutil.rmtree(path)
path.mkdir(parents=True, exist_ok=True)
def _extract_archive(archive_bytes: bytes, target_dir: Path) -> None:
"""Extract the MinerU zip into ``target_dir``, preserving its directory
tree (the ``images/`` subdir matters) while defending against Zip Slip and
zip bombs. Unlike :func:`safe_extract_zip`, this keeps subdirectories and
does not apply a document-extension whitelist — the archive is a trusted
MinerU artifact, not a user upload."""
target_root = target_dir.resolve()
total = 0
try:
with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
members = [m for m in archive.infolist() if not m.is_dir()]
if len(members) > _MAX_ENTRIES:
raise MinerUError(f"MinerU archive has too many entries ({len(members)}).")
for member in members:
# Collapse to a POSIX-relative path and reject traversal.
rel = Path(member.filename.replace("\\", "/"))
if rel.is_absolute() or ".." in rel.parts:
logger.warning("Skipping unsafe zip member: %s", member.filename)
continue
dest = (target_root / rel).resolve()
if target_root not in dest.parents and dest != target_root:
logger.warning("Skipping zip member escaping root: %s", member.filename)
continue
total += member.file_size
if total > _MAX_TOTAL_BYTES:
raise MinerUError("MinerU archive exceeds the size limit.")
dest.parent.mkdir(parents=True, exist_ok=True)
with archive.open(member) as src, open(dest, "wb") as out:
out.write(src.read())
except zipfile.BadZipFile as exc:
raise MinerUError(f"MinerU returned an invalid archive: {exc}") from exc
__all__ = ["parse_cloud", "verify_credentials"]