Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(parsing): preserve failed MinerU attempts and usable outputs (#1612)

Bingxi Zhao (Frank)428287fb16a8
diff --git a/deeptutor/services/parsing/cache.py b/deeptutor/services/parsing/cache.py
index 94bea9574..10876b9c0 100644
--- a/deeptutor/services/parsing/cache.py
+++ b/deeptutor/services/parsing/cache.py
@@ -6,7 +6,7 @@ the same engine config is a directory lookup; a different engine/version/knob
lands in a different signature dir and re-parses. Layout::
parse_cache/<hash[:2]>/<source_hash>/<signature>/
- manifest.json # written last → presence == "ready"
+ manifest.json # atomically written last → completed
<stem>.md
<stem>_content_list.json # optional (engines that emit structure)
images/ # optional
@@ -23,8 +23,10 @@ import hashlib
import json
import logging
from pathlib import Path
-import shutil
from typing import Any, Optional
+from uuid import uuid4
+
+from deeptutor.services.file_io import atomic_write_json
logger = logging.getLogger(__name__)
@@ -48,7 +50,13 @@ def signature_dir(cache_root: Path, source_hash: str, sig_hash: str) -> Path:
def is_ready(workdir: Optional[Path]) -> bool:
- return bool(workdir) and (workdir / MANIFEST_FILENAME).is_file()
+ if workdir is None:
+ return False
+ try:
+ manifest = json.loads((workdir / MANIFEST_FILENAME).read_text(encoding="utf-8"))
+ return isinstance(manifest, dict) and bool(manifest.get("created_at"))
+ except (OSError, ValueError):
+ return False
def lookup(cache_root: Path, source_hash: str, sig_hash: str) -> Optional[Path]:
@@ -60,12 +68,14 @@ def lookup(cache_root: Path, source_hash: str, sig_hash: str) -> Optional[Path]:
def reserve(cache_root: Path, source_hash: str, sig_hash: str) -> Path:
"""Create (or reuse) the signature dir the engine writes its artifacts into.
- Stale incomplete dirs (no manifest, e.g. a previous crash) are cleared so a
- retry starts clean.
+ Incomplete attempts are retained beside the cache entry for diagnosis, but
+ never reused as completed parser output (#1612).
"""
target = signature_dir(cache_root, source_hash, sig_hash)
if target.exists() and not is_ready(target):
- shutil.rmtree(target, ignore_errors=True)
+ cleanup_failed(target)
+ if target.exists():
+ raise OSError(f"Could not retain incomplete parse output: {target}")
target.mkdir(parents=True, exist_ok=True)
return target
@@ -77,15 +87,16 @@ def write_manifest(workdir: Path, meta: dict[str, Any]) -> None:
**meta,
"created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
}
- with open(workdir / MANIFEST_FILENAME, "w", encoding="utf-8") as handle:
- json.dump(payload, handle, indent=2, ensure_ascii=False)
+ atomic_write_json(workdir / MANIFEST_FILENAME, payload)
def cleanup_failed(workdir: Path) -> None:
- """Best-effort removal of an unfinished (manifest-less) cache dir."""
+ """Retain unfinished work outside the active signature directory."""
try:
if workdir.is_dir() and not is_ready(workdir):
- shutil.rmtree(workdir, ignore_errors=True)
+ retained = workdir.with_name(f".{workdir.name}.failed-{uuid4().hex}")
+ workdir.rename(retained)
+ logger.warning("Retained incomplete parse output at %s", retained)
except Exception as exc: # pragma: no cover - best-effort
logger.warning("Could not clean up failed parse dir %s: %s", workdir, exc)
diff --git a/deeptutor/services/parsing/engines/mineru/backend.py b/deeptutor/services/parsing/engines/mineru/backend.py
index d19a794c8..b0a0035c4 100644
--- a/deeptutor/services/parsing/engines/mineru/backend.py
+++ b/deeptutor/services/parsing/engines/mineru/backend.py
@@ -238,8 +238,18 @@ def _parse_local(
)
if not result.ok:
reason = result.reason or LocalParseReason.EXCEPTION
+ message = _LOCAL_FAILURE_MESSAGES[reason].format(detail=result.detail or "no output")
+ if reason in {
+ LocalParseReason.NONZERO_EXIT,
+ LocalParseReason.NO_ARTIFACTS,
+ LocalParseReason.EXCEPTION,
+ }:
+ message += (
+ " Retry restarts this document from the beginning; completed compatible "
+ "document parses will be reused. Incomplete output is retained for diagnosis."
+ )
raise MinerUError(
- _LOCAL_FAILURE_MESSAGES[reason].format(detail=result.detail or "no output"),
+ message,
code=str(reason),
detail=result.detail,
)
diff --git a/deeptutor/services/parsing/engines/mineru/local.py b/deeptutor/services/parsing/engines/mineru/local.py
index f7518a992..236d3fa7f 100644
--- a/deeptutor/services/parsing/engines/mineru/local.py
+++ b/deeptutor/services/parsing/engines/mineru/local.py
@@ -11,7 +11,12 @@ from pathlib import Path
import shutil
import subprocess
import sys
+import tempfile
import time
+from uuid import uuid4
+
+from deeptutor.services.file_io import atomic_write_json
+from deeptutor.services.parsing.cache import load_ir
from .formats import MINERU_SUPPORTED_FORMATS
@@ -186,20 +191,21 @@ def parse_document_with_mineru_result(
source_name = source_file.stem
output_dir = base_dir / source_name
- if output_dir.exists():
- print(f"⚠️ Directory already exists, replacing: {output_dir.name}")
- shutil.rmtree(output_dir)
-
print(f"📄 Input file: {source_file}")
print(f"📁 Output directory: {output_dir}")
print("→ Starting parsing...")
+ # Each CLI owns its attempt. Failed/interrupted attempts stay available,
+ # while a prior usable output survives until a validated replacement (#1612).
+ attempt = Path(tempfile.mkdtemp(prefix=".mineru-attempt-", dir=base_dir))
+ temp_output = attempt / "output"
+ temp_output.mkdir()
+ state_path = attempt / "state.json"
+ atomic_write_json(state_path, {"source": source_file.name, "state": "running"})
+ process = None
+ process_finished = False
+ result = LocalParseResult.failure(LocalParseReason.EXCEPTION, "parse interrupted")
try:
- temp_output = base_dir / "temp_mineru_output"
- if temp_output.exists():
- shutil.rmtree(temp_output)
- temp_output.mkdir(parents=True, exist_ok=True)
-
cmd = [mineru_cmd, "-p", str(source_file), "-o", str(temp_output)]
print(f"🔧 Executing command: {' '.join(cmd)}")
@@ -236,55 +242,52 @@ def parse_document_with_mineru_result(
# reporting and keep going.
on_output = None
returncode = process.wait()
+ process_finished = True
if returncode != 0:
print("✗ MinerU parsing failed:")
print("\n".join(tail))
- if temp_output.exists():
- shutil.rmtree(temp_output)
- return LocalParseResult.failure(
+ result = LocalParseResult.failure(
LocalParseReason.NONZERO_EXIT,
f"exit code {returncode}\n" + "\n".join(tail),
)
+ return result
print("✓ MinerU parsing completed!")
- generated_folders = list(temp_output.iterdir())
+ generated_folders = sorted(temp_output.iterdir())
if not generated_folders:
print("⚠️ Warning: No generated files found in temp directory")
- if temp_output.exists():
- shutil.rmtree(temp_output)
- return LocalParseResult.failure(
+ result = LocalParseResult.failure(
LocalParseReason.NO_ARTIFACTS,
f"no files were produced in {temp_output}",
)
+ return result
- source_folder = generated_folders[0] if generated_folders[0].is_dir() else temp_output
-
- # Create target directory and move content
- output_dir.mkdir(parents=True, exist_ok=True)
-
- # Move MinerU-generated content to target directory
- if source_folder.exists() and source_folder.is_dir():
- # If source_folder is the source-named directory, move its contents
- for item in source_folder.iterdir():
- dest_item = output_dir / item.name
- if dest_item.exists():
- if dest_item.is_dir():
- shutil.rmtree(dest_item)
- else:
- dest_item.unlink()
- shutil.move(str(item), str(dest_item))
- print(f"📦 Files saved to: {output_dir}")
- else:
- if output_dir.exists():
- shutil.rmtree(output_dir)
- shutil.move(str(source_folder), str(output_dir))
- print(f"📦 Files saved to: {output_dir}")
-
- if temp_output.exists():
- shutil.rmtree(temp_output)
+ named_folder = temp_output / source_name
+ source_folder = named_folder if named_folder.is_dir() else temp_output
+ markdown, blocks, _assets = load_ir(source_folder)
+ if not markdown.strip() and not blocks:
+ result = LocalParseResult.failure(
+ LocalParseReason.NO_ARTIFACTS,
+ "MinerU produced no usable markdown or content blocks",
+ )
+ return result
+
+ backup = base_dir / f".{source_name}.previous-{uuid4().hex}"
+ had_previous = output_dir.exists()
+ if had_previous:
+ output_dir.rename(backup)
+ try:
+ source_folder.rename(output_dir)
+ except BaseException:
+ if had_previous:
+ backup.rename(output_dir)
+ raise
+ if had_previous:
+ shutil.rmtree(backup)
+ print(f"📦 Files saved to: {output_dir}")
print("\n📋 Generated files:")
for item in output_dir.rglob("*"):
@@ -292,17 +295,43 @@ def parse_document_with_mineru_result(
rel_path = item.relative_to(output_dir)
print(f" - {rel_path}")
- return LocalParseResult.success()
+ result = LocalParseResult.success()
+ return result
except Exception as e:
print(f"✗ Error occurred during parsing: {e!s}")
import traceback
traceback.print_exc()
- return LocalParseResult.failure(
+ result = LocalParseResult.failure(
LocalParseReason.EXCEPTION,
f"{type(e).__name__}: {e}",
)
+ return result
+ finally:
+ # Stop an interrupted child before another attempt can publish output.
+ if process is not None and not process_finished:
+ try:
+ process.terminate()
+ process.wait(timeout=3)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ process.wait()
+ if result.ok:
+ shutil.rmtree(attempt, ignore_errors=True)
+ else:
+ try:
+ atomic_write_json(
+ state_path,
+ {
+ "source": source_file.name,
+ "state": "incomplete",
+ "reason": str(result.reason),
+ },
+ )
+ except OSError:
+ # A diagnostic write must not hide the original failure.
+ pass
def parse_document_with_mineru(
diff --git a/deeptutor/services/parsing/service.py b/deeptutor/services/parsing/service.py
index 0daa05811..2db211b24 100644
--- a/deeptutor/services/parsing/service.py
+++ b/deeptutor/services/parsing/service.py
@@ -155,17 +155,24 @@ class ParseService:
hit = cache.lookup(cache_root, source_hash, sig)
if hit is not None:
- logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)
- markdown, blocks, asset_dir = cache.load_ir(hit)
- return ParsedDocument(
- markdown=markdown,
- blocks=blocks,
- asset_dir=asset_dir,
- source_hash=source_hash,
- parser_signature=sig,
- engine=engine_name,
- workdir=hit,
- )
+ try:
+ markdown, blocks, asset_dir = cache.load_ir(hit)
+ except (OSError, UnicodeError, ValueError):
+ markdown, blocks, asset_dir = "", None, None
+ if markdown.strip() or blocks:
+ logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)
+ return ParsedDocument(
+ markdown=markdown,
+ blocks=blocks,
+ asset_dir=asset_dir,
+ source_hash=source_hash,
+ parser_signature=sig,
+ engine=engine_name,
+ workdir=hit,
+ )
+ # A ready stamp alone cannot make missing/empty output usable (#1612).
+ logger.warning("Invalid parse cache for %s; restarting this document", source_path.name)
+ (hit / cache.MANIFEST_FILENAME).unlink(missing_ok=True)
report = parser.is_ready(config)
if not report.ready:
@@ -176,7 +183,7 @@ class ParseService:
try:
parser.parse(source_path, workdir, config=config, on_output=on_output)
markdown, blocks, asset_dir = cache.load_ir(workdir)
- if not markdown and not blocks:
+ if not markdown.strip() and not blocks:
raise ParserError(
f"The '{engine_name}' engine produced no content for {source_path.name}."
)
diff --git a/docs-for-user/SOURCE_VISUALS.md b/docs-for-user/SOURCE_VISUALS.md
index 7f5ecc06f..68a7c28d6 100644
--- a/docs-for-user/SOURCE_VISUALS.md
+++ b/docs-for-user/SOURCE_VISUALS.md
@@ -71,6 +71,22 @@ This change versions the cloud parser signature so older merged page indices
are not reused from the normal parse cache. Existing knowledge-base indexes
need an explicit rebuild to consume corrected pages.
+## Retrying local MinerU documents
+
+Local MinerU retries restart the interrupted document from its beginning;
+the CLI does not expose a reliable checkpoint inside an inference call.
+Compatible completed document parses remain reusable after retry or restart,
+including when only the embedding configuration changes. Empty or unreadable
+cached output is reparsed rather than accepted as complete.
+
+New local output is checked for usable markdown or content blocks before it
+replaces an existing parse. Failed attempts retain their artifacts for
+diagnosis. Within the workspace parse cache these are moved to hidden
+`.failed-` directories beside the affected signature, outside the next retry's
+working directory. They are never treated as completed cache entries. Clearing
+the workspace parse cache also removes these diagnostic artifacts. An interrupted
+local child process is stopped before the caller starts another attempt.
+
## Tiny scanned PDF pages with MinerU
Some scanned PDFs encode a full-resolution page in an unusually small physical
diff --git a/tests/services/parsing/test_cache.py b/tests/services/parsing/test_cache.py
index 00650c521..ed97f98e5 100644
--- a/tests/services/parsing/test_cache.py
+++ b/tests/services/parsing/test_cache.py
@@ -38,6 +38,13 @@ def test_reserve_clears_stale_incomplete_dir(tmp_path: Path) -> None:
second = cache.reserve(root, "h", "s")
assert second == first
assert not (second / "junk.txt").exists()
+ assert len(list(second.parent.glob(".s.failed-*/junk.txt"))) == 1
+
+
+def test_truncated_manifest_never_marks_output_ready(tmp_path: Path) -> None:
+ target = cache.reserve(tmp_path, "h", "s")
+ (target / "manifest.json").write_text('{"created_at":', encoding="utf-8")
+ assert cache.lookup(tmp_path, "h", "s") is None
def test_load_ir_reads_markdown_blocks_images(tmp_path: Path) -> None:
diff --git a/tests/services/parsing/test_mineru_local_failures.py b/tests/services/parsing/test_mineru_local_failures.py
index 682bf6847..b98d59180 100644
--- a/tests/services/parsing/test_mineru_local_failures.py
+++ b/tests/services/parsing/test_mineru_local_failures.py
@@ -164,6 +164,97 @@ def test_failure_detail_is_bounded() -> None:
assert len(result.detail) <= mineru_local._FAILURE_DETAIL_MAX_CHARS + 1
[email protected]("returncode,artifacts", [(7, ("exam.md",)), (0, ("junk.txt",))])
+def test_failed_replacement_retains_previous_output_and_attempt(
+ pdf: Path,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ returncode: int,
+ artifacts: tuple[str, ...],
+) -> None:
+ output = tmp_path / "out"
+ previous = output / "exam" / "exam.md"
+ previous.parent.mkdir(parents=True)
+ previous.write_text("previous usable parse", encoding="utf-8")
+ _install_fake_popen(monkeypatch, returncode=returncode, artifacts=artifacts)
+
+ result = parse_document_with_mineru_result(pdf, output, cli_command="mineru")
+
+ assert not result.ok
+ assert previous.read_text(encoding="utf-8") == "previous usable parse"
+ attempts = list(output.glob(".mineru-attempt-*"))
+ assert len(attempts) == 1
+ assert (attempts[0] / "output" / artifacts[0]).is_file()
+ assert '"incomplete"' in (attempts[0] / "state.json").read_text(encoding="utf-8")
+
+ _install_fake_popen(monkeypatch, artifacts=("exam.md",))
+ assert parse_document_with_mineru_result(pdf, output, cli_command="mineru").ok
+ assert previous.read_text(encoding="utf-8") == "# parsed"
+ # The successful retry clears its own attempt, not the failed diagnostic output.
+ assert list(output.glob(".mineru-attempt-*")) == attempts
+
+
+def test_failed_publication_restores_previous_output(
+ pdf: Path,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ output = tmp_path / "out"
+ previous = output / "exam" / "exam.md"
+ previous.parent.mkdir(parents=True)
+ previous.write_text("previous", encoding="utf-8")
+ _install_fake_popen(monkeypatch, artifacts=("exam.md",))
+ rename = Path.rename
+
+ def fail_publication(path, target):
+ if path.name == "output":
+ raise OSError("controlled publication failure")
+ return rename(path, target)
+
+ monkeypatch.setattr(Path, "rename", fail_publication)
+ result = parse_document_with_mineru_result(pdf, output, cli_command="mineru")
+ assert result.reason is LocalParseReason.EXCEPTION
+ assert previous.read_text(encoding="utf-8") == "previous"
+
+
+def test_interrupted_attempt_stops_child_and_retains_partial_output(
+ pdf: Path,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class InterruptedProcess:
+ stopped = False
+
+ def __init__(self, cmd, **kwargs):
+ target = Path(cmd[cmd.index("-o") + 1])
+ (target / "partial.md").write_text("partial", encoding="utf-8")
+ self.stdout = self.lines()
+
+ def lines(self):
+ yield "parsing page 2"
+ raise KeyboardInterrupt
+
+ def terminate(self):
+ self.stopped = True
+
+ def wait(self, timeout=None):
+ assert self.stopped
+ return -15
+
+ child = None
+
+ def start(*args, **kwargs):
+ nonlocal child
+ child = InterruptedProcess(*args, **kwargs)
+ return child
+
+ monkeypatch.setattr(mineru_local.subprocess, "Popen", start)
+ with pytest.raises(KeyboardInterrupt):
+ parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
+ assert child.stopped
+ assert len(list((tmp_path / "out").glob(".mineru-attempt-*/output/partial.md"))) == 1
+
+
# ---------------------------------------------------------------------------
# backend mapping and legacy contract
# ---------------------------------------------------------------------------
diff --git a/tests/services/parsing/test_parse_service.py b/tests/services/parsing/test_parse_service.py
index 91c9a603a..2e9638dc4 100644
--- a/tests/services/parsing/test_parse_service.py
+++ b/tests/services/parsing/test_parse_service.py
@@ -80,6 +80,56 @@ def test_signature_change_busts_cache(tmp_path: Path, monkeypatch: pytest.Monkey
assert len(p1.calls) == 1 and len(p2.calls) == 1 # different signature → re-parse
+def test_restart_reuses_completed_documents_and_restarts_only_failed_document(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class FailingParser(_FakeParser):
+ fail = True
+
+ def parse(self, source_path, workdir, **kwargs):
+ super().parse(source_path, workdir, **kwargs)
+ if source_path.name == "second.pdf" and self.fail:
+ raise ParserError("controlled interruption")
+
+ parser = FailingParser()
+ _use(monkeypatch, parser)
+ root = tmp_path / "cache"
+ first = _pdf(tmp_path, b"first", "first.pdf")
+ second = _pdf(tmp_path, b"second", "second.pdf")
+ service = ParseService(cache_root=root)
+ completed = service.parse(first, engine="fake")
+ with pytest.raises(ParserError, match="controlled interruption"):
+ service.parse(second, engine="fake")
+ assert len(list(root.glob("*/*/.*.failed-*/second.md"))) == 1
+
+ parser.fail = False
+ restarted = ParseService(cache_root=root)
+ assert restarted.parse(first, engine="fake").workdir == completed.workdir
+ assert restarted.parse(second, engine="fake").markdown == "# md"
+ assert parser.calls == [first, second, second]
+
+
[email protected]("corruption", [b"", b"\xff"])
+def test_unusable_cache_reparses_only_affected_document(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ corruption: bytes,
+) -> None:
+ parser = _FakeParser()
+ _use(monkeypatch, parser)
+ service = ParseService(cache_root=tmp_path / "cache")
+ first = _pdf(tmp_path, b"first", "first.pdf")
+ second = _pdf(tmp_path, b"second", "second.pdf")
+ first_result = service.parse(first, engine="fake")
+ second_result = service.parse(second, engine="fake")
+ (second_result.workdir / "second.md").write_bytes(corruption)
+
+ assert service.parse(second, engine="fake").markdown == "# md"
+ assert service.parse(first, engine="fake").workdir == first_result.workdir
+ assert parser.calls == [first, second, second]
+
+
def test_same_bytes_different_name_share_cache(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None: