DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(parsing): preserve failed MinerU attempts and usable outputs (#1612)
Bingxi Zhao (Frank)428287fb16a8
diff --git a/deeptutor/services/parsing/cache.py b/deeptutor/services/parsing/cache.pyindex 94bea9574..10876b9c0 100644--- a/deeptutor/services/parsing/cache.py+++ b/deeptutor/services/parsing/cache.py@@ -6,7 +6,7 @@ the same engine config is a directory lookup; a different engine/version/knoblands in a different signature dir and re-parses. Layout::parse_cache/<hash[:2]>/<source_hash>/<signature>/- manifest.json # written last → presence == "ready"+ manifest.json # atomically written last → completed<stem>.md<stem>_content_list.json # optional (engines that emit structure)images/ # optional@@ -23,8 +23,10 @@ import hashlibimport jsonimport loggingfrom pathlib import Path-import shutilfrom typing import Any, Optional+from uuid import uuid4++from deeptutor.services.file_io import atomic_write_jsonlogger = logging.getLogger(__name__)@@ -48,7 +50,13 @@ def signature_dir(cache_root: Path, source_hash: str, sig_hash: str) -> Path:def is_ready(workdir: Optional[Path]) -> bool:- return bool(workdir) and (workdir / MANIFEST_FILENAME).is_file()+ if workdir is None:+ return False+ try:+ manifest = json.loads((workdir / MANIFEST_FILENAME).read_text(encoding="utf-8"))+ return isinstance(manifest, dict) and bool(manifest.get("created_at"))+ except (OSError, ValueError):+ return Falsedef lookup(cache_root: Path, source_hash: str, sig_hash: str) -> Optional[Path]:@@ -60,12 +68,14 @@ def lookup(cache_root: Path, source_hash: str, sig_hash: str) -> Optional[Path]:def reserve(cache_root: Path, source_hash: str, sig_hash: str) -> Path:"""Create (or reuse) the signature dir the engine writes its artifacts into.- Stale incomplete dirs (no manifest, e.g. a previous crash) are cleared so a- retry starts clean.+ Incomplete attempts are retained beside the cache entry for diagnosis, but+ never reused as completed parser output (#1612)."""target = signature_dir(cache_root, source_hash, sig_hash)if target.exists() and not is_ready(target):- shutil.rmtree(target, ignore_errors=True)+ cleanup_failed(target)+ if target.exists():+ raise OSError(f"Could not retain incomplete parse output: {target}")target.mkdir(parents=True, exist_ok=True)return target@@ -77,15 +87,16 @@ def write_manifest(workdir: Path, meta: dict[str, Any]) -> None:**meta,"created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",}- with open(workdir / MANIFEST_FILENAME, "w", encoding="utf-8") as handle:- json.dump(payload, handle, indent=2, ensure_ascii=False)+ atomic_write_json(workdir / MANIFEST_FILENAME, payload)def cleanup_failed(workdir: Path) -> None:- """Best-effort removal of an unfinished (manifest-less) cache dir."""+ """Retain unfinished work outside the active signature directory."""try:if workdir.is_dir() and not is_ready(workdir):- shutil.rmtree(workdir, ignore_errors=True)+ retained = workdir.with_name(f".{workdir.name}.failed-{uuid4().hex}")+ workdir.rename(retained)+ logger.warning("Retained incomplete parse output at %s", retained)except Exception as exc: # pragma: no cover - best-effortlogger.warning("Could not clean up failed parse dir %s: %s", workdir, exc)diff --git a/deeptutor/services/parsing/engines/mineru/backend.py b/deeptutor/services/parsing/engines/mineru/backend.pyindex d19a794c8..b0a0035c4 100644--- a/deeptutor/services/parsing/engines/mineru/backend.py+++ b/deeptutor/services/parsing/engines/mineru/backend.py@@ -238,8 +238,18 @@ def _parse_local()if not result.ok:reason = result.reason or LocalParseReason.EXCEPTION+ message = _LOCAL_FAILURE_MESSAGES[reason].format(detail=result.detail or "no output")+ if reason in {+ LocalParseReason.NONZERO_EXIT,+ LocalParseReason.NO_ARTIFACTS,+ LocalParseReason.EXCEPTION,+ }:+ message += (+ " Retry restarts this document from the beginning; completed compatible "+ "document parses will be reused. Incomplete output is retained for diagnosis."+ )raise MinerUError(- _LOCAL_FAILURE_MESSAGES[reason].format(detail=result.detail or "no output"),+ message,code=str(reason),detail=result.detail,)diff --git a/deeptutor/services/parsing/engines/mineru/local.py b/deeptutor/services/parsing/engines/mineru/local.pyindex f7518a992..236d3fa7f 100644--- a/deeptutor/services/parsing/engines/mineru/local.py+++ b/deeptutor/services/parsing/engines/mineru/local.py@@ -11,7 +11,12 @@ from pathlib import Pathimport shutilimport subprocessimport sys+import tempfileimport time+from uuid import uuid4++from deeptutor.services.file_io import atomic_write_json+from deeptutor.services.parsing.cache import load_irfrom .formats import MINERU_SUPPORTED_FORMATS@@ -186,20 +191,21 @@ def parse_document_with_mineru_result(source_name = source_file.stemoutput_dir = base_dir / source_name- if output_dir.exists():- print(f"⚠️ Directory already exists, replacing: {output_dir.name}")- shutil.rmtree(output_dir)-print(f"📄 Input file: {source_file}")print(f"📁 Output directory: {output_dir}")print("→ Starting parsing...")+ # Each CLI owns its attempt. Failed/interrupted attempts stay available,+ # while a prior usable output survives until a validated replacement (#1612).+ attempt = Path(tempfile.mkdtemp(prefix=".mineru-attempt-", dir=base_dir))+ temp_output = attempt / "output"+ temp_output.mkdir()+ state_path = attempt / "state.json"+ atomic_write_json(state_path, {"source": source_file.name, "state": "running"})+ process = None+ process_finished = False+ result = LocalParseResult.failure(LocalParseReason.EXCEPTION, "parse interrupted")try:- temp_output = base_dir / "temp_mineru_output"- if temp_output.exists():- shutil.rmtree(temp_output)- temp_output.mkdir(parents=True, exist_ok=True)-cmd = [mineru_cmd, "-p", str(source_file), "-o", str(temp_output)]print(f"🔧 Executing command: {' '.join(cmd)}")@@ -236,55 +242,52 @@ def parse_document_with_mineru_result(# reporting and keep going.on_output = Nonereturncode = process.wait()+ process_finished = Trueif returncode != 0:print("✗ MinerU parsing failed:")print("\n".join(tail))- if temp_output.exists():- shutil.rmtree(temp_output)- return LocalParseResult.failure(+ result = LocalParseResult.failure(LocalParseReason.NONZERO_EXIT,f"exit code {returncode}\n" + "\n".join(tail),)+ return resultprint("✓ MinerU parsing completed!")- generated_folders = list(temp_output.iterdir())+ generated_folders = sorted(temp_output.iterdir())if not generated_folders:print("⚠️ Warning: No generated files found in temp directory")- if temp_output.exists():- shutil.rmtree(temp_output)- return LocalParseResult.failure(+ result = LocalParseResult.failure(LocalParseReason.NO_ARTIFACTS,f"no files were produced in {temp_output}",)+ return result- source_folder = generated_folders[0] if generated_folders[0].is_dir() else temp_output-- # Create target directory and move content- output_dir.mkdir(parents=True, exist_ok=True)-- # Move MinerU-generated content to target directory- if source_folder.exists() and source_folder.is_dir():- # If source_folder is the source-named directory, move its contents- for item in source_folder.iterdir():- dest_item = output_dir / item.name- if dest_item.exists():- if dest_item.is_dir():- shutil.rmtree(dest_item)- else:- dest_item.unlink()- shutil.move(str(item), str(dest_item))- print(f"📦 Files saved to: {output_dir}")- else:- if output_dir.exists():- shutil.rmtree(output_dir)- shutil.move(str(source_folder), str(output_dir))- print(f"📦 Files saved to: {output_dir}")-- if temp_output.exists():- shutil.rmtree(temp_output)+ named_folder = temp_output / source_name+ source_folder = named_folder if named_folder.is_dir() else temp_output+ markdown, blocks, _assets = load_ir(source_folder)+ if not markdown.strip() and not blocks:+ result = LocalParseResult.failure(+ LocalParseReason.NO_ARTIFACTS,+ "MinerU produced no usable markdown or content blocks",+ )+ return result++ backup = base_dir / f".{source_name}.previous-{uuid4().hex}"+ had_previous = output_dir.exists()+ if had_previous:+ output_dir.rename(backup)+ try:+ source_folder.rename(output_dir)+ except BaseException:+ if had_previous:+ backup.rename(output_dir)+ raise+ if had_previous:+ shutil.rmtree(backup)+ print(f"📦 Files saved to: {output_dir}")print("\n📋 Generated files:")for item in output_dir.rglob("*"):@@ -292,17 +295,43 @@ def parse_document_with_mineru_result(rel_path = item.relative_to(output_dir)print(f" - {rel_path}")- return LocalParseResult.success()+ result = LocalParseResult.success()+ return resultexcept Exception as e:print(f"✗ Error occurred during parsing: {e!s}")import tracebacktraceback.print_exc()- return LocalParseResult.failure(+ result = LocalParseResult.failure(LocalParseReason.EXCEPTION,f"{type(e).__name__}: {e}",)+ return result+ finally:+ # Stop an interrupted child before another attempt can publish output.+ if process is not None and not process_finished:+ try:+ process.terminate()+ process.wait(timeout=3)+ except subprocess.TimeoutExpired:+ process.kill()+ process.wait()+ if result.ok:+ shutil.rmtree(attempt, ignore_errors=True)+ else:+ try:+ atomic_write_json(+ state_path,+ {+ "source": source_file.name,+ "state": "incomplete",+ "reason": str(result.reason),+ },+ )+ except OSError:+ # A diagnostic write must not hide the original failure.+ passdef parse_document_with_mineru(diff --git a/deeptutor/services/parsing/service.py b/deeptutor/services/parsing/service.pyindex 0daa05811..2db211b24 100644--- a/deeptutor/services/parsing/service.py+++ b/deeptutor/services/parsing/service.py@@ -155,17 +155,24 @@ class ParseService:hit = cache.lookup(cache_root, source_hash, sig)if hit is not None:- logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)- markdown, blocks, asset_dir = cache.load_ir(hit)- return ParsedDocument(- markdown=markdown,- blocks=blocks,- asset_dir=asset_dir,- source_hash=source_hash,- parser_signature=sig,- engine=engine_name,- workdir=hit,- )+ try:+ markdown, blocks, asset_dir = cache.load_ir(hit)+ except (OSError, UnicodeError, ValueError):+ markdown, blocks, asset_dir = "", None, None+ if markdown.strip() or blocks:+ logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)+ return ParsedDocument(+ markdown=markdown,+ blocks=blocks,+ asset_dir=asset_dir,+ source_hash=source_hash,+ parser_signature=sig,+ engine=engine_name,+ workdir=hit,+ )+ # A ready stamp alone cannot make missing/empty output usable (#1612).+ logger.warning("Invalid parse cache for %s; restarting this document", source_path.name)+ (hit / cache.MANIFEST_FILENAME).unlink(missing_ok=True)report = parser.is_ready(config)if not report.ready:@@ -176,7 +183,7 @@ class ParseService:try:parser.parse(source_path, workdir, config=config, on_output=on_output)markdown, blocks, asset_dir = cache.load_ir(workdir)- if not markdown and not blocks:+ if not markdown.strip() and not blocks:raise ParserError(f"The '{engine_name}' engine produced no content for {source_path.name}.")diff --git a/docs-for-user/SOURCE_VISUALS.md b/docs-for-user/SOURCE_VISUALS.mdindex 7f5ecc06f..68a7c28d6 100644--- a/docs-for-user/SOURCE_VISUALS.md+++ b/docs-for-user/SOURCE_VISUALS.md@@ -71,6 +71,22 @@ This change versions the cloud parser signature so older merged page indicesare not reused from the normal parse cache. Existing knowledge-base indexesneed an explicit rebuild to consume corrected pages.+## Retrying local MinerU documents++Local MinerU retries restart the interrupted document from its beginning;+the CLI does not expose a reliable checkpoint inside an inference call.+Compatible completed document parses remain reusable after retry or restart,+including when only the embedding configuration changes. Empty or unreadable+cached output is reparsed rather than accepted as complete.++New local output is checked for usable markdown or content blocks before it+replaces an existing parse. Failed attempts retain their artifacts for+diagnosis. Within the workspace parse cache these are moved to hidden+`.failed-` directories beside the affected signature, outside the next retry's+working directory. They are never treated as completed cache entries. Clearing+the workspace parse cache also removes these diagnostic artifacts. An interrupted+local child process is stopped before the caller starts another attempt.+## Tiny scanned PDF pages with MinerUSome scanned PDFs encode a full-resolution page in an unusually small physicaldiff --git a/tests/services/parsing/test_cache.py b/tests/services/parsing/test_cache.pyindex 00650c521..ed97f98e5 100644--- a/tests/services/parsing/test_cache.py+++ b/tests/services/parsing/test_cache.py@@ -38,6 +38,13 @@ def test_reserve_clears_stale_incomplete_dir(tmp_path: Path) -> None:second = cache.reserve(root, "h", "s")assert second == firstassert not (second / "junk.txt").exists()+ assert len(list(second.parent.glob(".s.failed-*/junk.txt"))) == 1+++def test_truncated_manifest_never_marks_output_ready(tmp_path: Path) -> None:+ target = cache.reserve(tmp_path, "h", "s")+ (target / "manifest.json").write_text('{"created_at":', encoding="utf-8")+ assert cache.lookup(tmp_path, "h", "s") is Nonedef test_load_ir_reads_markdown_blocks_images(tmp_path: Path) -> None:diff --git a/tests/services/parsing/test_mineru_local_failures.py b/tests/services/parsing/test_mineru_local_failures.pyindex 682bf6847..b98d59180 100644--- a/tests/services/parsing/test_mineru_local_failures.py+++ b/tests/services/parsing/test_mineru_local_failures.py@@ -164,6 +164,97 @@ def test_failure_detail_is_bounded() -> None:assert len(result.detail) <= mineru_local._FAILURE_DETAIL_MAX_CHARS + 1[email protected]("returncode,artifacts", [(7, ("exam.md",)), (0, ("junk.txt",))])+def test_failed_replacement_retains_previous_output_and_attempt(+ pdf: Path,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ returncode: int,+ artifacts: tuple[str, ...],+) -> None:+ output = tmp_path / "out"+ previous = output / "exam" / "exam.md"+ previous.parent.mkdir(parents=True)+ previous.write_text("previous usable parse", encoding="utf-8")+ _install_fake_popen(monkeypatch, returncode=returncode, artifacts=artifacts)++ result = parse_document_with_mineru_result(pdf, output, cli_command="mineru")++ assert not result.ok+ assert previous.read_text(encoding="utf-8") == "previous usable parse"+ attempts = list(output.glob(".mineru-attempt-*"))+ assert len(attempts) == 1+ assert (attempts[0] / "output" / artifacts[0]).is_file()+ assert '"incomplete"' in (attempts[0] / "state.json").read_text(encoding="utf-8")++ _install_fake_popen(monkeypatch, artifacts=("exam.md",))+ assert parse_document_with_mineru_result(pdf, output, cli_command="mineru").ok+ assert previous.read_text(encoding="utf-8") == "# parsed"+ # The successful retry clears its own attempt, not the failed diagnostic output.+ assert list(output.glob(".mineru-attempt-*")) == attempts+++def test_failed_publication_restores_previous_output(+ pdf: Path,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ output = tmp_path / "out"+ previous = output / "exam" / "exam.md"+ previous.parent.mkdir(parents=True)+ previous.write_text("previous", encoding="utf-8")+ _install_fake_popen(monkeypatch, artifacts=("exam.md",))+ rename = Path.rename++ def fail_publication(path, target):+ if path.name == "output":+ raise OSError("controlled publication failure")+ return rename(path, target)++ monkeypatch.setattr(Path, "rename", fail_publication)+ result = parse_document_with_mineru_result(pdf, output, cli_command="mineru")+ assert result.reason is LocalParseReason.EXCEPTION+ assert previous.read_text(encoding="utf-8") == "previous"+++def test_interrupted_attempt_stops_child_and_retains_partial_output(+ pdf: Path,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ class InterruptedProcess:+ stopped = False++ def __init__(self, cmd, **kwargs):+ target = Path(cmd[cmd.index("-o") + 1])+ (target / "partial.md").write_text("partial", encoding="utf-8")+ self.stdout = self.lines()++ def lines(self):+ yield "parsing page 2"+ raise KeyboardInterrupt++ def terminate(self):+ self.stopped = True++ def wait(self, timeout=None):+ assert self.stopped+ return -15++ child = None++ def start(*args, **kwargs):+ nonlocal child+ child = InterruptedProcess(*args, **kwargs)+ return child++ monkeypatch.setattr(mineru_local.subprocess, "Popen", start)+ with pytest.raises(KeyboardInterrupt):+ parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")+ assert child.stopped+ assert len(list((tmp_path / "out").glob(".mineru-attempt-*/output/partial.md"))) == 1++# ---------------------------------------------------------------------------# backend mapping and legacy contract# ---------------------------------------------------------------------------diff --git a/tests/services/parsing/test_parse_service.py b/tests/services/parsing/test_parse_service.pyindex 91c9a603a..2e9638dc4 100644--- a/tests/services/parsing/test_parse_service.py+++ b/tests/services/parsing/test_parse_service.py@@ -80,6 +80,56 @@ def test_signature_change_busts_cache(tmp_path: Path, monkeypatch: pytest.Monkeyassert len(p1.calls) == 1 and len(p2.calls) == 1 # different signature → re-parse+def test_restart_reuses_completed_documents_and_restarts_only_failed_document(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ class FailingParser(_FakeParser):+ fail = True++ def parse(self, source_path, workdir, **kwargs):+ super().parse(source_path, workdir, **kwargs)+ if source_path.name == "second.pdf" and self.fail:+ raise ParserError("controlled interruption")++ parser = FailingParser()+ _use(monkeypatch, parser)+ root = tmp_path / "cache"+ first = _pdf(tmp_path, b"first", "first.pdf")+ second = _pdf(tmp_path, b"second", "second.pdf")+ service = ParseService(cache_root=root)+ completed = service.parse(first, engine="fake")+ with pytest.raises(ParserError, match="controlled interruption"):+ service.parse(second, engine="fake")+ assert len(list(root.glob("*/*/.*.failed-*/second.md"))) == 1++ parser.fail = False+ restarted = ParseService(cache_root=root)+ assert restarted.parse(first, engine="fake").workdir == completed.workdir+ assert restarted.parse(second, engine="fake").markdown == "# md"+ assert parser.calls == [first, second, second]++[email protected]("corruption", [b"", b"\xff"])+def test_unusable_cache_reparses_only_affected_document(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ corruption: bytes,+) -> None:+ parser = _FakeParser()+ _use(monkeypatch, parser)+ service = ParseService(cache_root=tmp_path / "cache")+ first = _pdf(tmp_path, b"first", "first.pdf")+ second = _pdf(tmp_path, b"second", "second.pdf")+ first_result = service.parse(first, engine="fake")+ second_result = service.parse(second, engine="fake")+ (second_result.workdir / "second.md").write_bytes(corruption)++ assert service.parse(second, engine="fake").markdown == "# md"+ assert service.parse(first, engine="fake").workdir == first_result.workdir+ assert parser.calls == [first, second, second]++def test_same_bytes_different_name_share_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: