DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""Local MinerU failure classification (issue #1612, outcome 4).
Covers the reason codes returned by
:func:`deeptutor.services.parsing.engines.mineru.local.parse_document_with_mineru_result`,
how ``backend`` maps them to distinct :class:`MinerUError` messages, and the
unchanged bool contract of the legacy helpers.
"""
from __future__ import annotations
from contextlib import contextmanager
from pathlib import Path
import subprocess
from types import SimpleNamespace
import pytest
from deeptutor.services.parsing.engines.mineru import backend as mineru_backend
from deeptutor.services.parsing.engines.mineru import local as mineru_local
from deeptutor.services.parsing.engines.mineru.config import MinerUConfig, MinerUError
from deeptutor.services.parsing.engines.mineru.local import (
LocalParseReason,
LocalParseResult,
parse_document_with_mineru,
parse_document_with_mineru_result,
)
REASONS = list(LocalParseReason)
@pytest.fixture()
def pdf(tmp_path: Path) -> Path:
source = tmp_path / "exam.pdf"
source.write_bytes(b"%PDF-1.4")
return source
def _install_fake_popen(
monkeypatch: pytest.MonkeyPatch,
*,
returncode: int = 0,
lines: tuple[str, ...] = (),
artifacts: tuple[str, ...] = (),
) -> None:
"""Patch the subprocess factory so the local CLI is simulated in-process."""
class FakeProcess:
def __init__(self, cmd, **_kwargs) -> None: # noqa: ANN001
target = Path(cmd[cmd.index("-o") + 1])
for name in artifacts:
path = target / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("# parsed", encoding="utf-8")
self.stdout = iter(lines)
def poll(self) -> int:
return returncode
@contextmanager
def fake_cli_process(cmd, log_path, env):
process = FakeProcess(cmd)
log_path.write_text("\n".join(lines), encoding="utf-8")
yield process
monkeypatch.setattr(mineru_local, "cli_process", fake_cli_process)
# ---------------------------------------------------------------------------
# parse_document_with_mineru_result — reason codes
# ---------------------------------------------------------------------------
def test_cli_missing_reports_reason(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(mineru_local, "check_mineru_installed", lambda: None)
result = parse_document_with_mineru_result(pdf, tmp_path / "out")
assert result.ok is False
assert result.reason is LocalParseReason.CLI_MISSING
assert result.detail
def test_input_missing_reports_reason(pdf: Path, tmp_path: Path) -> None:
missing = tmp_path / "gone.pdf"
result = parse_document_with_mineru_result(missing, tmp_path / "out", cli_command="mineru")
assert result.reason is LocalParseReason.INPUT_MISSING
assert result.detail == str(missing.resolve())
def test_unsupported_input_reports_reason(tmp_path: Path) -> None:
source = tmp_path / "notes.xyz"
source.write_text("x", encoding="utf-8")
result = parse_document_with_mineru_result(source, tmp_path / "out", cli_command="mineru")
assert result.reason is LocalParseReason.UNSUPPORTED_INPUT
assert result.detail == ".xyz"
def test_legacy_cli_input_reports_reason(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "lesson.docx"
source.write_bytes(b"office")
monkeypatch.setattr(
mineru_local.subprocess,
"Popen",
lambda *_args, **_kwargs: pytest.fail("legacy CLI must not be invoked"),
)
result = parse_document_with_mineru_result(source, tmp_path / "out", cli_command="magic-pdf")
assert result.reason is LocalParseReason.LEGACY_CLI_INPUT
def test_nonzero_exit_keeps_bounded_stderr_tail(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_install_fake_popen(
monkeypatch,
returncode=7,
lines=("first problem", "second problem"),
)
result = parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
assert result.reason is LocalParseReason.NONZERO_EXIT
assert "exit code 7" in result.detail
assert "first problem" in result.detail
assert "second problem" in result.detail
def test_long_progress_log_preserves_terminal_timeout_and_full_log(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""#1982: tqdm output must not push the actionable final error out of the summary."""
terminal = "Timed out waiting for result of task textbook after 3600 seconds"
lines = tuple(f"Predict {index}/89 " + "." * 80 for index in range(40)) + (terminal,)
_install_fake_popen(monkeypatch, returncode=1, lines=lines)
result = parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
assert result.reason is LocalParseReason.NONZERO_EXIT
assert "exit code 1" in result.detail
assert terminal in result.detail
assert len(result.detail) <= mineru_local._FAILURE_DETAIL_MAX_CHARS
(attempt,) = (tmp_path / "out").glob(".mineru-attempt-*")
assert (attempt / "cli.log").read_text() == "\n".join(lines)
@pytest.mark.parametrize(
"inherited,extra,expected",
[
(None, {}, "7200"),
("10800", {}, "10800"),
("10800", {"MINERU_TASK_RESULT_TIMEOUT_SECONDS": "14400"}, "14400"),
(None, {"MINERU_TASK_RESULT_TIMEOUT_SECONDS": "1.5"}, "1.5"),
(None, {"MINERU_TASK_RESULT_TIMEOUT_SECONDS": "invalid"}, "7200"),
],
)
def test_cli_receives_the_effective_parent_deadline(
pdf, tmp_path, monkeypatch, inherited, extra, expected
) -> None:
"""The nested API client must not silently retain its shorter one-hour default."""
if inherited is None:
monkeypatch.delenv("MINERU_TASK_RESULT_TIMEOUT_SECONDS", raising=False)
else:
monkeypatch.setenv("MINERU_TASK_RESULT_TIMEOUT_SECONDS", inherited)
seen = []
@contextmanager
def cli(cmd, log_path, env):
seen.append(env["MINERU_TASK_RESULT_TIMEOUT_SECONDS"])
log_path.write_text("controlled failure")
yield SimpleNamespace(poll=lambda: 1)
monkeypatch.setattr(mineru_local, "cli_process", cli)
result = parse_document_with_mineru_result(
pdf, tmp_path / "out", cli_command="mineru", extra_env=extra
)
assert result.reason is LocalParseReason.NONZERO_EXIT
assert seen == [expected]
def test_no_artifacts_reports_reason(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_install_fake_popen(monkeypatch, returncode=0, artifacts=())
result = parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
assert result.reason is LocalParseReason.NO_ARTIFACTS
def test_internal_exception_reports_reason(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
def boom(*_args, **_kwargs): # noqa: ANN002, ANN003
raise subprocess.TimeoutExpired(cmd="mineru", timeout=1)
monkeypatch.setattr(mineru_local.subprocess, "Popen", boom)
result = parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
assert result.reason is LocalParseReason.EXCEPTION
assert "TimeoutExpired" in result.detail
def test_success_reports_ok(pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
_install_fake_popen(monkeypatch, artifacts=("exam.md",))
result = parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
assert result.ok is True
assert result.reason is None
assert (tmp_path / "out" / "exam" / "exam.md").exists()
def test_failure_detail_is_bounded() -> None:
result = LocalParseResult.failure(LocalParseReason.EXCEPTION, "x" * 1000)
assert len(result.detail) <= mineru_local._FAILURE_DETAIL_MAX_CHARS + 1
@pytest.mark.parametrize("returncode,artifacts", [(7, ("exam.md",)), (0, ("junk.txt",))])
def test_failed_replacement_retains_previous_output_and_attempt(
pdf: Path,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
returncode: int,
artifacts: tuple[str, ...],
) -> None:
output = tmp_path / "out"
previous = output / "exam" / "exam.md"
previous.parent.mkdir(parents=True)
previous.write_text("previous usable parse", encoding="utf-8")
_install_fake_popen(monkeypatch, returncode=returncode, artifacts=artifacts)
result = parse_document_with_mineru_result(pdf, output, cli_command="mineru")
assert not result.ok
assert previous.read_text(encoding="utf-8") == "previous usable parse"
attempts = list(output.glob(".mineru-attempt-*"))
assert len(attempts) == 1
assert (attempts[0] / "output" / artifacts[0]).is_file()
assert '"incomplete"' in (attempts[0] / "state.json").read_text(encoding="utf-8")
_install_fake_popen(monkeypatch, artifacts=("exam.md",))
assert parse_document_with_mineru_result(pdf, output, cli_command="mineru").ok
assert previous.read_text(encoding="utf-8") == "# parsed"
# The successful retry clears its own attempt, not the failed diagnostic output.
assert list(output.glob(".mineru-attempt-*")) == attempts
def test_failed_publication_restores_previous_output(
pdf: Path,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
output = tmp_path / "out"
previous = output / "exam" / "exam.md"
previous.parent.mkdir(parents=True)
previous.write_text("previous", encoding="utf-8")
_install_fake_popen(monkeypatch, artifacts=("exam.md",))
rename = Path.rename
def fail_publication(path, target):
if path.name == "output":
raise OSError("controlled publication failure")
return rename(path, target)
monkeypatch.setattr(Path, "rename", fail_publication)
result = parse_document_with_mineru_result(pdf, output, cli_command="mineru")
assert result.reason is LocalParseReason.EXCEPTION
assert previous.read_text(encoding="utf-8") == "previous"
def test_interrupted_attempt_stops_child_and_retains_partial_output(
pdf: Path,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
class InterruptedProcess:
stopped = False
def __init__(self, cmd, **kwargs):
target = Path(cmd[cmd.index("-o") + 1])
(target / "partial.md").write_text("partial", encoding="utf-8")
def poll(self):
raise KeyboardInterrupt
def terminate(self):
self.stopped = True
def wait(self, timeout=None):
assert self.stopped
return -15
child = None
@contextmanager
def start(cmd, log_path, env):
nonlocal child
child = InterruptedProcess(cmd)
log_path.write_text("parsing page 2\n", encoding="utf-8")
try:
yield child
finally:
child.terminate()
child.wait()
monkeypatch.setattr(mineru_local, "cli_process", start)
with pytest.raises(KeyboardInterrupt):
parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
assert child.stopped
assert len(list((tmp_path / "out").glob(".mineru-attempt-*/output/partial.md"))) == 1
# ---------------------------------------------------------------------------
# backend mapping and legacy contract
# ---------------------------------------------------------------------------
def test_backend_maps_every_reason_to_a_distinct_error(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
exe = tmp_path / "mineru"
exe.write_text("#!/bin/sh\n", encoding="utf-8")
exe.chmod(0o755)
config = MinerUConfig(mode="local", local_cli_path=str(exe))
messages: dict[str, str] = {}
for reason in REASONS:
monkeypatch.setattr(
mineru_backend,
"parse_document_with_mineru_result",
lambda *_args, _reason=reason, **_kwargs: LocalParseResult(
ok=False, reason=_reason, detail="excerpt-here"
),
)
with pytest.raises(MinerUError) as exc:
mineru_backend.parse_document_to_workdir(pdf, tmp_path / f"out-{reason}", config=config)
assert exc.value.code == str(reason)
assert exc.value.detail == "excerpt-here"
messages[str(reason)] = str(exc.value)
assert len(set(messages.values())) == len(messages)
install_hits = [
key for key, message in messages.items() if "Ensure MinerU is installed" in message
]
assert install_hits == ["cli_missing"]
def test_backend_runtime_failure_does_not_claim_missing_install(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
mineru_backend,
"parse_document_with_mineru_result",
lambda *_args, **_kwargs: LocalParseResult(
ok=False, reason=LocalParseReason.NONZERO_EXIT, detail="exit code 9: broken"
),
)
with pytest.raises(MinerUError) as exc:
mineru_backend.parse_document_to_workdir(
pdf, tmp_path / "out", config=MinerUConfig(mode="local")
)
assert exc.value.code == "nonzero_exit"
assert "Ensure MinerU is installed" not in str(exc.value)
assert "exit code 9" in str(exc.value)
def test_legacy_bool_helpers_keep_their_contract(
pdf: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(mineru_local, "check_mineru_installed", lambda: None)
assert parse_document_with_mineru(pdf, tmp_path / "out") is False
source = tmp_path / "lesson.docx"
source.write_bytes(b"office")
assert mineru_local.parse_pdf_with_mineru(str(source)) is False
def test_error_without_classification_keeps_old_shape() -> None:
error = MinerUError("boom")
assert error.code is None
assert error.detail == ""
assert str(error) == "boom"