Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(parsing): restore source pages and resume completed MinerU slices

ZyPulse056e6a81728a
diff --git a/deeptutor/api/routers/settings.py b/deeptutor/api/routers/settings.py
index 41fd591c5..93f192389 100644
--- a/deeptutor/api/routers/settings.py
+++ b/deeptutor/api/routers/settings.py
@@ -1242,6 +1242,7 @@ async def update_mineru_settings(payload: MinerUSettingsUpdate):
"enable_formula": payload.enable_formula,
"enable_table": payload.enable_table,
"is_ocr": payload.is_ocr,
+ "max_pages_per_part": current.get("max_pages_per_part", 180),
"allow_local_model_download": payload.allow_local_model_download,
}
)
diff --git a/deeptutor/services/config/runtime_settings.py b/deeptutor/services/config/runtime_settings.py
index 03313f63e..9cb46224c 100644
--- a/deeptutor/services/config/runtime_settings.py
+++ b/deeptutor/services/config/runtime_settings.py
@@ -185,6 +185,7 @@ _DEFAULT_MINERU_ENGINE: dict[str, Any] = {
"enable_formula": True,
"enable_table": True,
"is_ocr": False,
+ "max_pages_per_part": 180,
"allow_local_model_download": False,
}
@@ -1154,6 +1155,9 @@ class RuntimeSettingsService:
"enable_formula": _coerce_bool(settings.get("enable_formula"), True),
"enable_table": _coerce_bool(settings.get("enable_table"), True),
"is_ocr": _coerce_bool(settings.get("is_ocr"), False),
+ "max_pages_per_part": _coerce_clamped_int(
+ settings.get("max_pages_per_part"), 180, 1, 200
+ ),
"allow_local_model_download": _coerce_bool(
settings.get("allow_local_model_download"), False
),
diff --git a/deeptutor/services/parsing/engines/mineru/checkpoints.py b/deeptutor/services/parsing/engines/mineru/checkpoints.py
new file mode 100644
index 000000000..75acf6fb8
--- /dev/null
+++ b/deeptutor/services/parsing/engines/mineru/checkpoints.py
@@ -0,0 +1,98 @@
+"""Workspace-scoped, integrity-checked archives for completed PDF slices.
+
+Each archive is immutable and addressed by its full SHA-256. A tiny atomic
+pointer is written only after extraction and merge validation. Concurrent jobs
+may duplicate an upload, but cannot expose a partially written checkpoint.
+No API tokens or signed URLs are added to checkpoint metadata. Archives may
+include source copies supplied by the parser.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import tempfile
+
+from .config import MinerUConfig
+
+_VERSION = "slices-v1"
+_MAX_ARCHIVE_BYTES = 500 * 1024 * 1024
+
+
+def checkpoint_root() -> Path:
+ from deeptutor.services.path_service import get_path_service
+
+ return get_path_service().get_parse_cache_root() / ".mineru-segments"
+
+
+def job_directory(source: Path, config: MinerUConfig) -> Path:
+ digest = hashlib.sha256()
+ with source.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ identity = {
+ "version": _VERSION,
+ "source_sha256": digest.hexdigest(),
+ "name": source.name,
+ "endpoint": config.api_base_url.rstrip("/"),
+ "model": config.model_version,
+ "language": config.language,
+ "formula": config.enable_formula,
+ "table": config.enable_table,
+ "ocr": config.is_ocr,
+ "pages_per_part": config.max_pages_per_part,
+ }
+ key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
+ return checkpoint_root() / key
+
+
+class SliceCheckpoint:
+ def __init__(self, job: Path, start: int, end: int):
+ self.directory = job / f"{start}-{end}"
+ self.pointer = self.directory / "ready.json"
+
+ def load(self) -> bytes | None:
+ try:
+ if self.pointer.is_symlink() or self.directory.is_symlink():
+ return None
+ info = json.loads(self.pointer.read_text(encoding="utf-8"))
+ digest = info["sha256"]
+ if (
+ not isinstance(digest, str)
+ or len(digest) != 64
+ or any(c not in "0123456789abcdef" for c in digest)
+ ):
+ return None
+ archive = self.directory / f"{digest}.zip"
+ if archive.is_symlink() or archive.stat().st_size > _MAX_ARCHIVE_BYTES:
+ return None
+ data = archive.read_bytes()
+ if hashlib.sha256(data).hexdigest() == digest:
+ return data
+ except (OSError, ValueError, KeyError, TypeError):
+ pass
+ return None
+
+ def save(self, archive: bytes) -> None:
+ if len(archive) > _MAX_ARCHIVE_BYTES:
+ return
+ self.directory.mkdir(parents=True, exist_ok=True)
+ digest = hashlib.sha256(archive).hexdigest()
+ self._write(self.directory / f"{digest}.zip", archive)
+ self._write(self.pointer, json.dumps({"sha256": digest}).encode())
+
+ @staticmethod
+ def _write(target: Path, data: bytes) -> None:
+ temporary = None
+ try:
+ with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as handle:
+ temporary = Path(handle.name)
+ handle.write(data)
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temporary, target)
+ finally:
+ if temporary is not None:
+ temporary.unlink(missing_ok=True)
diff --git a/deeptutor/services/parsing/engines/mineru/cloud.py b/deeptutor/services/parsing/engines/mineru/cloud.py
index 213057736..a1711504e 100644
--- a/deeptutor/services/parsing/engines/mineru/cloud.py
+++ b/deeptutor/services/parsing/engines/mineru/cloud.py
@@ -35,6 +35,7 @@ from pypdf import PdfReader, PdfWriter
from deeptutor.services.keypool import KeyPool
+from .checkpoints import SliceCheckpoint, job_directory
from .config import MinerUConfig, MinerUError
from .formats import MINERU_PDF_FORMATS, MINERU_SUPPORTED_FORMATS
@@ -186,6 +187,7 @@ def _parse_pdf_in_parts(
markdown_chunks: list[str] = []
content_items: list = []
has_content_list = False
+ checkpoint_job = job_directory(source_path, config)
with tempfile.TemporaryDirectory(prefix="mineru-slice-") as tmp:
parts = _split_pdf(source_path, config.max_pages_per_part, Path(tmp))
@@ -199,15 +201,20 @@ def _parse_pdf_in_parts(
report(
f"MinerU cloud: parsing part {index}/{total} (pages {part.start + 1}-{part.end})"
)
- archive_bytes = _upload_and_fetch_archive(
- client,
- part.path,
- config,
- key_pool,
- report=report,
- poll_interval=poll_interval,
- timeout=timeout,
- )
+ checkpoint = SliceCheckpoint(checkpoint_job, part.start, part.end)
+ archive_bytes = checkpoint.load()
+ if archive_bytes is None:
+ archive_bytes = _upload_and_fetch_archive(
+ client,
+ part.path,
+ config,
+ key_pool,
+ report=report,
+ poll_interval=poll_interval,
+ timeout=timeout,
+ )
+ else:
+ report(f"MinerU cloud: reusing completed part {index}/{total}")
report(f"MinerU cloud: merging part {index}/{total} artifacts")
with tempfile.TemporaryDirectory(prefix="mineru-part-") as part_tmp:
part_dir = Path(part_tmp) / "out"
@@ -219,8 +226,14 @@ def _parse_pdf_in_parts(
markdown_chunks=markdown_chunks,
content_items=content_items,
part_index=index,
+ page_offset=part.start,
+ page_count=part.end - part.start,
)
has_content_list = True
+ try:
+ checkpoint.save(archive_bytes)
+ except OSError:
+ logger.warning("Could not save MinerU slice checkpoint; parsing continues")
merged_md = "".join(
chunk if chunk.endswith("\n") else chunk + "\n" for chunk in markdown_chunks
@@ -278,6 +291,8 @@ def _merge_part_artifacts(
markdown_chunks: list[str],
content_items: list,
part_index: int,
+ page_offset: int = 0,
+ page_count: int | None = None,
) -> None:
"""Accumulate one part's artifacts: markdown text, ``images/`` (renamed with
a part prefix so names stay globally unique) and ``content_list`` entries
@@ -302,6 +317,7 @@ def _merge_part_artifacts(
f"MinerU part {part_index} content list is unreadable: {exc}"
) from exc
if isinstance(payload, list):
+ _offset_page_indices(payload, page_offset, page_count)
_rewrite_image_paths(payload, rename_map)
content_items.extend(payload)
@@ -316,6 +332,25 @@ def _merge_part_artifacts(
shutil.copyfile(extra, working_dir / f"part{part_index:02d}_{extra.name}")
+def _offset_page_indices(node: object, offset: int, page_count: int | None) -> None:
+ """Restore zero-based source page indices, including nested content blocks."""
+ if isinstance(node, dict):
+ if "page_idx" in node:
+ index = node["page_idx"]
+ if (
+ type(index) is not int
+ or index < 0
+ or (page_count is not None and index >= page_count)
+ ):
+ raise MinerUError("MinerU slice returned an invalid page_idx")
+ node["page_idx"] = index + offset
+ for value in node.values():
+ _offset_page_indices(value, offset, page_count)
+ elif isinstance(node, list):
+ for value in node:
+ _offset_page_indices(value, offset, page_count)
+
+
def _copy_part_images(images_dir: Path, target_dir: Path, part_index: int) -> dict[str, str]:
"""Copy one part's ``images/`` into the merged ``images/`` dir, prefixing
each name with the part index so names stay globally unique across parts.
diff --git a/deeptutor/services/parsing/engines/mineru/engine.py b/deeptutor/services/parsing/engines/mineru/engine.py
index 02cc579a3..d6cd6f040 100644
--- a/deeptutor/services/parsing/engines/mineru/engine.py
+++ b/deeptutor/services/parsing/engines/mineru/engine.py
@@ -48,6 +48,14 @@ class MinerUParser:
"enable_formula": config.enable_formula,
"enable_table": config.enable_table,
"is_ocr": config.is_ocr,
+ **(
+ {
+ "segmentation": "source-pages-v1",
+ "max_pages_per_part": config.max_pages_per_part,
+ }
+ if config.is_cloud
+ else {}
+ ),
},
)
diff --git a/docs-for-user/SOURCE_VISUALS.md b/docs-for-user/SOURCE_VISUALS.md
index 674b8d1fc..17482ed4d 100644
--- a/docs-for-user/SOURCE_VISUALS.md
+++ b/docs-for-user/SOURCE_VISUALS.md
@@ -26,3 +26,32 @@ source image assets. Images that are only vector drawing commands, or pages
that require OCR when no usable OCR engine is configured, may be absent.
Other RAG providers do not yet index this visual manifest. Interactive
exercises, visual annotations, and mastery updates are separate future work.
+
+## Resuming MinerU cloud PDF slices
+
+Large cloud PDFs are sliced using `engines.mineru.max_pages_per_part` in
+Document Parsing settings (default 180; clamped to 1–200). This advanced setting
+is preserved when saving the legacy MinerU settings form. Local parsing and
+small/non-PDF inputs keep their existing behavior.
+
+Merged content-list `page_idx` values refer to the original PDF, including
+nested blocks. For example, page index 0 of the second 180-page slice becomes
+180. Invalid local indices fail the parse instead of silently mislabeling
+source pages. Other per-part diagnostic artifacts retain their original local
+numbering and `partNN_` filename prefix.
+
+Completed slice archives are saved under the active workspace's parse cache in
+`.mineru-segments/`, outside disposable failed-parse directories. Retrying the
+same source and parser settings reuses completed slices; a changed document,
+endpoint, model, language, OCR/formula/table setting, or slice size starts a new
+checkpoint set. API tokens and signed URLs are never written to checkpoints.
+Each archive is SHA-256 checked before reuse; an incomplete or corrupt checkpoint
+is downloaded again. Concurrent jobs may still duplicate a cloud request, but
+checkpoint writes are atomic. Cache write failures do not fail a successful parse.
+
+These archives contain parsed document content, images, and any source copies
+returned by MinerU, just like the normal parse cache. They are retained until the workspace cache is cleared; deleting
+`.mineru-segments/` while no parse is running only discards resumable progress.
+This change versions the cloud parser signature so older merged page indices
+are not reused from the normal parse cache. Existing knowledge-base indexes
+need an explicit rebuild to consume corrected pages.
diff --git a/tests/services/parsing/test_mineru_slicer.py b/tests/services/parsing/test_mineru_slicer.py
index 277dfc767..96b314a55 100644
--- a/tests/services/parsing/test_mineru_slicer.py
+++ b/tests/services/parsing/test_mineru_slicer.py
@@ -24,6 +24,13 @@ CLOUD_CFG = MinerUConfig(mode="cloud", api_token="tok")
_SLICING_MARKERS = ("auto-splitting", "parsing part", "merging part")
[email protected](autouse=True)
+def isolated_checkpoints(tmp_path, monkeypatch):
+ from deeptutor.services.parsing.engines.mineru import checkpoints
+
+ monkeypatch.setattr(checkpoints, "checkpoint_root", lambda: tmp_path / "checkpoints")
+
+
def _make_pdf(path: Path, pages: int) -> Path:
writer = PdfWriter()
for _ in range(pages):
@@ -250,3 +257,152 @@ def test_merged_artifacts_md_order_and_unique_images(
"images/part01_b.jpg",
"images/part02_a.jpg",
]
+
+
+def test_original_page_indices_include_nested_blocks(tmp_path, monkeypatch):
+ pdf = _make_pdf(tmp_path / "book.pdf", 5)
+ config = MinerUConfig(mode="cloud", api_token="tok", max_pages_per_part=2)
+ _install_fake_parse(
+ monkeypatch,
+ extras=lambda part: (
+ {"a.jpg": b"image"},
+ [
+ {
+ "type": "image",
+ "img_path": "images/a.jpg",
+ "page_idx": 0,
+ "children": [{"page_idx": 0, "text": "caption"}],
+ }
+ ],
+ ),
+ )
+ out = mineru_cloud.parse_cloud(pdf, tmp_path / "out", config)
+ rows = json.loads((out / "book_content_list.json").read_text())
+ assert [row["page_idx"] for row in rows] == [0, 2, 4]
+ assert [row["children"][0]["page_idx"] for row in rows] == [0, 2, 4]
+ assert all((out / row["img_path"]).is_file() for row in rows)
+
+
[email protected]("bad_index", [-1, 2, True, "0", None, 1.5])
+def test_invalid_page_index_does_not_create_ready_checkpoint(tmp_path, monkeypatch, bad_index):
+ from deeptutor.services.parsing.engines.mineru.config import MinerUError
+
+ pdf = _make_pdf(tmp_path / "bad.pdf", 3)
+ config = MinerUConfig(mode="cloud", api_token="tok", max_pages_per_part=2)
+ _install_fake_parse(monkeypatch, extras=lambda part: ({}, [{"page_idx": bad_index}]))
+ with pytest.raises(MinerUError, match="page_idx"):
+ mineru_cloud.parse_cloud(pdf, tmp_path / "out", config)
+ assert not list((tmp_path / "checkpoints").rglob("ready.json"))
+
+
+def test_retry_reuses_completed_slice_after_output_cleanup(tmp_path, monkeypatch):
+ import shutil
+
+ from deeptutor.services.parsing.engines.mineru.config import MinerUError
+
+ pdf = _make_pdf(tmp_path / "retry.pdf", 5)
+ config = MinerUConfig(mode="cloud", api_token="tok", max_pages_per_part=2)
+ calls = []
+ fail = True
+
+ def upload(client, part, config, key_pool, **kwargs):
+ calls.append(part.name)
+ if fail and part.name.endswith("02.pdf"):
+ raise MinerUError("synthetic transport interruption")
+ return _fake_archive_bytes(part.stem, "text", content_items=[{"page_idx": 0}])
+
+ monkeypatch.setattr(mineru_cloud, "_upload_and_fetch_archive", upload)
+ with pytest.raises(MinerUError, match="interruption"):
+ mineru_cloud.parse_cloud(pdf, tmp_path / "out", config)
+ shutil.rmtree(tmp_path / "out") # ParseService removes failed work directories
+ fail = False
+ out = mineru_cloud.parse_cloud(pdf, tmp_path / "out", config)
+ assert calls == ["retry_part01.pdf", "retry_part02.pdf", "retry_part02.pdf", "retry_part03.pdf"]
+ assert [
+ row["page_idx"] for row in json.loads((out / "retry_content_list.json").read_text())
+ ] == [0, 2, 4]
+
+
+def test_corrupt_checkpoint_reparses_only_affected_slice(tmp_path, monkeypatch):
+ pdf = _make_pdf(tmp_path / "book.pdf", 3)
+ config = MinerUConfig(mode="cloud", api_token="tok", max_pages_per_part=2)
+ calls = _install_fake_parse(monkeypatch)
+ mineru_cloud.parse_cloud(pdf, tmp_path / "out", config)
+ archive = next((tmp_path / "checkpoints").glob("*/0-2/*.zip"))
+ archive.write_bytes(b"truncated")
+ calls.clear()
+ mineru_cloud.parse_cloud(pdf, tmp_path / "out", config)
+ assert [p.name for p in calls] == ["book_part01.pdf"]
+
+
+def test_checkpoint_identity_tracks_source_and_parser_but_not_credentials(tmp_path):
+ from dataclasses import replace
+
+ from deeptutor.services.parsing.engines.mineru.checkpoints import job_directory
+
+ pdf = _make_pdf(tmp_path / "book.pdf", 3)
+ config = MinerUConfig(mode="cloud", api_token="secret-one", max_pages_per_part=2)
+ key = job_directory(pdf, config)
+ assert job_directory(pdf, replace(config, api_token="secret-two")) == key
+ for change in [
+ dict(language="ch"),
+ dict(model_version="vlm"),
+ dict(is_ocr=True),
+ dict(enable_formula=False),
+ dict(enable_table=False),
+ dict(max_pages_per_part=1),
+ dict(api_base_url="https://example.test"),
+ ]:
+ assert job_directory(pdf, replace(config, **change)) != key
+ _make_pdf(pdf, 4)
+ assert job_directory(pdf, config) != key
+
+
+def test_slice_setting_persists_and_cloud_signature_changes(tmp_path):
+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService
+ from deeptutor.services.parsing.engines.mineru.engine import MinerUParser
+
+ service = RuntimeSettingsService(tmp_path / "settings", process_env={})
+ service.save_mineru({"max_pages_per_part": 120})
+ assert service.load_mineru()["max_pages_per_part"] == 120
+ service.save_mineru({"max_pages_per_part": 500})
+ assert service.load_mineru()["max_pages_per_part"] == 200
+ parser = MinerUParser()
+ assert parser.signature(MinerUConfig(mode="cloud", max_pages_per_part=2)) != parser.signature(
+ MinerUConfig(mode="cloud", max_pages_per_part=3)
+ )
+ assert parser.signature(MinerUConfig(mode="local", max_pages_per_part=2)) == parser.signature(
+ MinerUConfig(mode="local", max_pages_per_part=3)
+ )
+
+
+def test_default_180_page_boundary_is_rebased(tmp_path, monkeypatch):
+ pdf = _make_pdf(tmp_path / "large.pdf", 181)
+ _install_fake_parse(monkeypatch, extras=lambda part: ({}, [{"page_idx": 0}]))
+ out = mineru_cloud.parse_cloud(pdf, tmp_path / "out", CLOUD_CFG)
+ rows = json.loads((out / "large_content_list.json").read_text())
+ assert [row["page_idx"] for row in rows] == [0, 180]
+
+
+def test_checkpoint_write_failure_does_not_fail_parse(tmp_path, monkeypatch):
+ from deeptutor.services.parsing.engines.mineru.checkpoints import SliceCheckpoint
+
+ def fail_save(self, archive):
+ raise OSError("synthetic read-only cache")
+
+ monkeypatch.setattr(SliceCheckpoint, "save", fail_save)
+ pdf = _make_pdf(tmp_path / "book.pdf", 3)
+ config = MinerUConfig(mode="cloud", api_token="tok", max_pages_per_part=2)
+ _install_fake_parse(monkeypatch)
+ out = mineru_cloud.parse_cloud(pdf, tmp_path / "out", config)
+ assert (out / "book.md").is_file()
+
+
[email protected]("pointer", ["{", "[]", '{"sha256":"../../escape"}'])
+def test_invalid_checkpoint_pointer_is_a_miss(tmp_path, pointer):
+ from deeptutor.services.parsing.engines.mineru.checkpoints import SliceCheckpoint
+
+ checkpoint = SliceCheckpoint(tmp_path, 0, 2)
+ checkpoint.directory.mkdir()
+ checkpoint.pointer.write_text(pointer)
+ assert checkpoint.load() is None