DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(parsing): align MinerU deadlines and retain terminal errors (#1982)
Bingxi Zhao (Frank)c3b98baf346a
diff --git a/deeptutor/knowledge/indexing_run.py b/deeptutor/knowledge/indexing_run.pyindex d1f5d7d7e..8a3a73774 100644--- a/deeptutor/knowledge/indexing_run.py+++ b/deeptutor/knowledge/indexing_run.py@@ -29,6 +29,10 @@ from deeptutor.services.file_io import atomic_write_json, read_text_CURRENT: ContextVar[IndexingRun | None] = ContextVar("indexing_run", default=None)_TERMINAL = {"completed", "partial", "failed", "cancelled", "interrupted"}+_RECOVERY_DESCRIPTION = (+ "Retry reuses validated parser outputs and completed cloud-parsing slices. "+ "Uncheckpointed chunking and embedding stages restart."+)class IndexingBusyError(RuntimeError):@@ -158,7 +162,7 @@ def visible_run(kb_dir: Path) -> dict[str, Any] | None:**value,"state": "interrupted","documents": documents,- "recovery": "Retry reuses validated parser outputs and completed slices. The interrupted uncheckpointed stage restarts.",+ "recovery": _RECOVERY_DESCRIPTION,}@@ -247,7 +251,7 @@ class IndexingRun:"last_progress_at": now,"documents": {},"previous_task_id": previous.get("task_id") if previous else None,- "recovery": "Validated parser caches and completed slices are reused. Uncheckpointed stages restart.",+ "recovery": _RECOVERY_DESCRIPTION,"cancel_requested": False,"usable_version": None,}diff --git a/deeptutor/services/parsing/engines/mineru/local.py b/deeptutor/services/parsing/engines/mineru/local.pyindex d502260d8..7ef98d402 100644--- a/deeptutor/services/parsing/engines/mineru/local.py+++ b/deeptutor/services/parsing/engines/mineru/local.py@@ -7,6 +7,7 @@ from collections.abc import Callablefrom dataclasses import dataclassfrom enum import StrEnumimport logging+import mathimport osfrom pathlib import Pathimport shutil@@ -39,7 +40,11 @@ def _bounded_detail(text: str) -> str:clean = str(text or "").strip()if len(clean) <= _FAILURE_DETAIL_MAX_CHARS:return clean- return clean[:_FAILURE_DETAIL_MAX_CHARS].rstrip() + "…"+ # Keep the exception/exit-code prefix and the terminal cause. Progress+ # bars can otherwise consume the whole excerpt before the real error (#1982).+ head = clean[:80].rstrip()+ tail_size = _FAILURE_DETAIL_MAX_CHARS - len(head) - 3+ return head + "\n…\n" + clean[-tail_size:].lstrip()class LocalParseReason(StrEnum):@@ -225,17 +230,24 @@ def parse_document_with_mineru_result(quiet_reported = Falsetotal_timeout = _LOCAL_PARSE_TIMEOUT_SECONDStry:- configured_timeout = int(+ configured_timeout = float((extra_env or {}).get("MINERU_TASK_RESULT_TIMEOUT_SECONDS",os.environ.get("MINERU_TASK_RESULT_TIMEOUT_SECONDS", str(total_timeout)),))- if configured_timeout > 0:+ if math.isfinite(configured_timeout) and configured_timeout >= 1:total_timeout = configured_timeoutexcept (TypeError, ValueError):pass- env = {**os.environ, **(extra_env or {}), "PYTHONUTF8": "1"}+ env = {+ **os.environ,+ **(extra_env or {}),+ "PYTHONUTF8": "1",+ # MinerU's nested API client otherwise times out at 3600 seconds,+ # before our default deadline. Both owners use the resolved value.+ "MINERU_TASK_RESULT_TIMEOUT_SECONDS": f"{total_timeout:g}",+ }log_path = attempt / "cli.log"# File reads do not block on a pipe inherited by a surviving worker.# Settle the owned process tree before validating/publishing artifacts.diff --git a/tests/services/parsing/test_mineru_local_failures.py b/tests/services/parsing/test_mineru_local_failures.pyindex df745f3f2..33d8e77c7 100644--- a/tests/services/parsing/test_mineru_local_failures.py+++ b/tests/services/parsing/test_mineru_local_failures.py@@ -11,6 +11,7 @@ from __future__ import annotationsfrom contextlib import contextmanagerfrom pathlib import Pathimport subprocess+from types import SimpleNamespaceimport pytest@@ -131,6 +132,58 @@ def test_nonzero_exit_keeps_bounded_stderr_tail(assert "second problem" in result.detail+def test_long_progress_log_preserves_terminal_timeout_and_full_log(+ pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """#1982: tqdm output must not push the actionable final error out of the summary."""+ terminal = "Timed out waiting for result of task textbook after 3600 seconds"+ lines = tuple(f"Predict {index}/89 " + "." * 80 for index in range(40)) + (terminal,)+ _install_fake_popen(monkeypatch, returncode=1, lines=lines)++ result = parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")++ assert result.reason is LocalParseReason.NONZERO_EXIT+ assert "exit code 1" in result.detail+ assert terminal in result.detail+ assert len(result.detail) <= mineru_local._FAILURE_DETAIL_MAX_CHARS+ (attempt,) = (tmp_path / "out").glob(".mineru-attempt-*")+ assert (attempt / "cli.log").read_text() == "\n".join(lines)+++ "inherited,extra,expected",+ [+ (None, {}, "7200"),+ ("10800", {}, "10800"),+ ("10800", {"MINERU_TASK_RESULT_TIMEOUT_SECONDS": "14400"}, "14400"),+ (None, {"MINERU_TASK_RESULT_TIMEOUT_SECONDS": "1.5"}, "1.5"),+ (None, {"MINERU_TASK_RESULT_TIMEOUT_SECONDS": "invalid"}, "7200"),+ ],+)+def test_cli_receives_the_effective_parent_deadline(+ pdf, tmp_path, monkeypatch, inherited, extra, expected+) -> None:+ """The nested API client must not silently retain its shorter one-hour default."""+ if inherited is None:+ monkeypatch.delenv("MINERU_TASK_RESULT_TIMEOUT_SECONDS", raising=False)+ else:+ monkeypatch.setenv("MINERU_TASK_RESULT_TIMEOUT_SECONDS", inherited)+ seen = []++ @contextmanager+ def cli(cmd, log_path, env):+ seen.append(env["MINERU_TASK_RESULT_TIMEOUT_SECONDS"])+ log_path.write_text("controlled failure")+ yield SimpleNamespace(poll=lambda: 1)++ monkeypatch.setattr(mineru_local, "cli_process", cli)+ result = parse_document_with_mineru_result(+ pdf, tmp_path / "out", cli_command="mineru", extra_env=extra+ )+ assert result.reason is LocalParseReason.NONZERO_EXIT+ assert seen == [expected]++def test_no_artifacts_reports_reason(pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/tests/services/parsing/test_mineru_process_lifecycle.py b/tests/services/parsing/test_mineru_process_lifecycle.pyindex f8777f686..a53ce9e2f 100644--- a/tests/services/parsing/test_mineru_process_lifecycle.py+++ b/tests/services/parsing/test_mineru_process_lifecycle.py@@ -13,7 +13,7 @@ from deeptutor.knowledge import indexing_runfrom deeptutor.services.parsing.engines.mineru import local-def run_fixture(tmp_path, monkeypatch, code, run=None, idle=0.05, timeout=30):+def run_fixture(tmp_path, monkeypatch, code, run=None, idle=0.05, timeout=30, extra_env=None):source = tmp_path / "lesson.pdf"source.write_bytes(b"controlled fixture")popen = subprocess.Popen@@ -34,6 +34,7 @@ def run_fixture(tmp_path, monkeypatch, code, run=None, idle=0.05, timeout=30):source,tmp_path / "out",cli_command=sys.executable,+ extra_env=extra_env,)return result, time.monotonic() - started@@ -48,6 +49,25 @@ def test_quiet_inference_is_not_killed_for_log_silence(tmp_path, monkeypatch):assert elapsed < 2+def test_explicit_client_timeout_also_bounds_parent_runtime(tmp_path, monkeypatch):+ code = (+ "import os,sys,time,pathlib; p=pathlib.Path(sys.argv[1]); "+ "(p/'deadline.txt').write_text(os.environ['MINERU_TASK_RESULT_TIMEOUT_SECONDS']); "+ "time.sleep(10)"+ )+ result, elapsed = run_fixture(+ tmp_path,+ monkeypatch,+ code,+ extra_env={"MINERU_TASK_RESULT_TIMEOUT_SECONDS": "1.2"},+ )+ assert result.reason is local.LocalParseReason.TIMEOUT+ assert "1.2 seconds" in result.detail+ assert elapsed < 2.5+ (deadline,) = (tmp_path / "out").glob(".mineru-attempt-*/output/deadline.txt")+ assert deadline.read_text() == "1.2"++def test_parent_exit_does_not_wait_for_descendant_stdout(tmp_path, monkeypatch):# A finite child keeps the red test bounded even with the old broken reader.code = (diff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.pyindex 63ab686c4..dd19ebf4d 100644--- a/tests/services/rag/test_llamaindex_document_loader.py+++ b/tests/services/rag/test_llamaindex_document_loader.py@@ -200,6 +200,33 @@ def test_loader_skips_document_when_active_engine_cannot_parse(assert "Settings" in caplog.text+def test_loader_continues_after_mineru_task_failure(tmp_path, monkeypatch):+ """#1982: a failed CLI document cannot prevent later documents from loading."""+ pytest.importorskip("llama_index.core")+ from deeptutor.knowledge.indexing_run import IndexingRun+ from deeptutor.services.parsing.types import ParsedDocument, ParserError+ from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader++ failed = tmp_path / "failed.pdf"+ good = tmp_path / "next.docx"+ failed.write_bytes(b"fixture")+ good.write_bytes(b"fixture")+ _install_stub_parse_service(+ monkeypatch,+ {+ failed.name: ParserError(+ "1 task(s) failed while processing documents: Timed out waiting for result"+ ),+ good.name: ParsedDocument(markdown="The next document is still usable."),+ },+ )+ with IndexingRun(tmp_path, [str(failed), str(good)]) as run:+ documents = asyncio.run(LlamaIndexDocumentLoader().load([str(failed), str(good)]))+ assert [doc.metadata["file_name"] for doc in documents] == [good.name]+ assert run.data["documents"][run.source_key(failed)]["status"] == "failed"+ assert run.data["documents"][run.source_key(good)]["status"] == "embedding"++def test_loader_explains_scanned_pdf_when_parser_yields_images_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None: