DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(parsing): isolate native MinerU failures during KB indexing (#1982)
Bingxi Zhao (Frank)35cea5618731
diff --git a/deeptutor/services/parsing/engines/mineru/config.py b/deeptutor/services/parsing/engines/mineru/config.pyindex 779e56272..771151d4c 100644--- a/deeptutor/services/parsing/engines/mineru/config.py+++ b/deeptutor/services/parsing/engines/mineru/config.py@@ -18,6 +18,7 @@ from deeptutor.services.config.runtime_settings import (MINERU_MODE_LOCAL,load_mineru_settings,)+from deeptutor.services.parsing.types import ParserError# The MinerU cloud API rejects any single file above 200 pages, so the PDF# auto-slicer must never emit a part larger than that even if a setting asks@@ -26,10 +27,11 @@ MAX_PAGES_PER_PART_CEILING = 200DEFAULT_MAX_PAGES_PER_PART = 180-class MinerUError(RuntimeError):+class MinerUError(ParserError):"""Raised when a MinerU parse fails (local CLI missing, cloud API error,misconfiguration). Carries a user-facing message; the capability layer- surfaces it as a stream error."""+ surfaces it as a stream error. The shared ParserError boundary lets batch+ consumers isolate this document instead of aborting the whole KB (#1982)."""def __init__(self, message: str, *, code: str | None = None, detail: str = "") -> None:super().__init__(message)diff --git a/tests/knowledge/test_indexing_run.py b/tests/knowledge/test_indexing_run.pyindex 6b1b9874a..2f60b1add 100644--- a/tests/knowledge/test_indexing_run.py+++ b/tests/knowledge/test_indexing_run.py@@ -142,6 +142,87 @@ def test_real_persisted_index_reloads_and_missing_vectors_fail(tmp_path, monkeypstorage.verify_persisted_index(tmp_path)[email protected]("mode", ["local", "cloud"])[email protected]("cancelled", [False, True])+async def test_mineru_failure_publishes_remaining_documents_as_partial(+ tmp_path, monkeypatch, mode, cancelled+):+ """#1982: the real parser error crosses loading, persistence and the run journal."""+ from llama_index.core import Settings+ from llama_index.core.embeddings import MockEmbedding++ from deeptutor.services import parsing+ from deeptutor.services.parsing import service as parse_module+ from deeptutor.services.parsing.base import ReadinessReport+ from deeptutor.services.parsing.engines.mineru import backend+ from deeptutor.services.parsing.engines.mineru.config import MinerUConfig, MinerUError+ from deeptutor.services.parsing.engines.mineru.engine import MinerUParser+ from deeptutor.services.rag.pipelines.llamaindex import storage, vector_store+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline+ from deeptutor.services.rag.service import RAGService++ kb_dir = tmp_path / "kb"+ raw = kb_dir / "raw"+ raw.mkdir(parents=True)+ bad, good = raw / "failed.docx", raw / "next.docx"+ bad.write_bytes(b"failed source")+ good.write_bytes(b"good source")+ parser = MinerUParser()+ monkeypatch.setattr(parser, "resolve_config", lambda: MinerUConfig(mode=mode))+ monkeypatch.setattr(parser, "is_ready", lambda config: ReadinessReport(True))+ monkeypatch.setattr(parse_module, "get_parser", lambda name: parser)+ service = parse_module.ParseService(cache_root=tmp_path / "parse_cache")+ monkeypatch.setattr(service, "active_engine", lambda: "mineru")+ monkeypatch.setattr(parsing, "get_parse_service", lambda: service)+ attempts = []++ def parse(source, workdir, **kwargs):+ attempts.append(source.name)+ if source == bad:+ if cancelled:+ assert request_cancel(kb_dir, current_run().task_id)+ raise MinerUError("Local MinerU parsing was cancelled", code="cancelled")+ raise MinerUError("1 task(s) failed: Timed out waiting for result", code="timeout")+ (workdir / "full.md").write_text("The next document explains Fourier transforms.")++ monkeypatch.setattr(backend, "parse_document_to_workdir", parse)+ monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=4))+ pipeline = LlamaIndexPipeline(str(tmp_path), signature_provider=lambda: None)+ monkeypatch.setattr(pipeline, "_configure_settings", lambda: None)++ async def offline_connectivity():+ return None++ monkeypatch.setattr(pipeline, "_verify_embedding_connectivity", offline_connectivity)+ rag = RAGService(str(tmp_path), provider="llamaindex")+ rag._pipelines["llamaindex"] = pipeline+ if cancelled:+ with pytest.raises(IndexingCancelled):+ await rag._index("kb", [str(bad), str(good)], "initialize", {})+ assert attempts == [bad.name]+ assert load_run(kb_dir)["state"] == "cancelled"+ assert not load_run(kb_dir).get("usable_version")+ return+ assert await rag._index("kb", [str(bad), str(good)], "initialize", {})++ journal = load_run(kb_dir)+ assert attempts == [bad.name, good.name]+ assert journal["state"] == "partial"+ assert journal["documents"][bad.name]["status"] == "failed"+ assert "Timed out waiting" in journal["documents"][bad.name]["detail"]+ assert journal["documents"][good.name]["status"] == "completed"+ published = kb_dir / journal["usable_version"]+ storage.verify_persisted_index(published)+ reloaded = vector_store.load_index(published)+ assert any("Fourier" in node.text for node in reloaded.docstore.docs.values())++ # Retry must reuse the good parse even though its previous run was partial.+ assert await rag._index("kb", [str(bad), str(good)], "initialize", {})+ assert attempts == [bad.name, good.name, bad.name]+ assert load_run(kb_dir)["documents"][good.name]["parse_reused"] is True++@pytest.mark.asyncioasync def test_readiness_is_local_and_cancel_requires_writable_owned_task(tmp_path, monkeypatch):from fastapi import HTTPExceptiondiff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.pyindex dd19ebf4d..f0abdd19b 100644--- a/tests/services/rag/test_llamaindex_document_loader.py+++ b/tests/services/rag/test_llamaindex_document_loader.py@@ -204,7 +204,8 @@ def test_loader_continues_after_mineru_task_failure(tmp_path, monkeypatch):"""#1982: a failed CLI document cannot prevent later documents from loading."""pytest.importorskip("llama_index.core")from deeptutor.knowledge.indexing_run import IndexingRun- from deeptutor.services.parsing.types import ParsedDocument, ParserError+ from deeptutor.services.parsing.engines.mineru.config import MinerUError+ from deeptutor.services.parsing.types import ParsedDocumentfrom deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoaderfailed = tmp_path / "failed.pdf"@@ -214,7 +215,7 @@ def test_loader_continues_after_mineru_task_failure(tmp_path, monkeypatch):_install_stub_parse_service(monkeypatch,{- failed.name: ParserError(+ failed.name: MinerUError("1 task(s) failed while processing documents: Timed out waiting for result"),good.name: ParsedDocument(markdown="The next document is still usable."),