Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(parsing): isolate native MinerU failures during KB indexing (#1982)

Bingxi Zhao (Frank)35cea5618731
diff --git a/deeptutor/services/parsing/engines/mineru/config.py b/deeptutor/services/parsing/engines/mineru/config.py
index 779e56272..771151d4c 100644
--- a/deeptutor/services/parsing/engines/mineru/config.py
+++ b/deeptutor/services/parsing/engines/mineru/config.py
@@ -18,6 +18,7 @@ from deeptutor.services.config.runtime_settings import (
MINERU_MODE_LOCAL,
load_mineru_settings,
)
+from deeptutor.services.parsing.types import ParserError
# The MinerU cloud API rejects any single file above 200 pages, so the PDF
# auto-slicer must never emit a part larger than that even if a setting asks
@@ -26,10 +27,11 @@ MAX_PAGES_PER_PART_CEILING = 200
DEFAULT_MAX_PAGES_PER_PART = 180
-class MinerUError(RuntimeError):
+class MinerUError(ParserError):
"""Raised when a MinerU parse fails (local CLI missing, cloud API error,
misconfiguration). Carries a user-facing message; the capability layer
- surfaces it as a stream error."""
+ surfaces it as a stream error. The shared ParserError boundary lets batch
+ consumers isolate this document instead of aborting the whole KB (#1982)."""
def __init__(self, message: str, *, code: str | None = None, detail: str = "") -> None:
super().__init__(message)
diff --git a/tests/knowledge/test_indexing_run.py b/tests/knowledge/test_indexing_run.py
index 6b1b9874a..2f60b1add 100644
--- a/tests/knowledge/test_indexing_run.py
+++ b/tests/knowledge/test_indexing_run.py
@@ -142,6 +142,87 @@ def test_real_persisted_index_reloads_and_missing_vectors_fail(tmp_path, monkeyp
storage.verify_persisted_index(tmp_path)
[email protected]("mode", ["local", "cloud"])
[email protected]("cancelled", [False, True])
+async def test_mineru_failure_publishes_remaining_documents_as_partial(
+ tmp_path, monkeypatch, mode, cancelled
+):
+ """#1982: the real parser error crosses loading, persistence and the run journal."""
+ from llama_index.core import Settings
+ from llama_index.core.embeddings import MockEmbedding
+
+ from deeptutor.services import parsing
+ from deeptutor.services.parsing import service as parse_module
+ from deeptutor.services.parsing.base import ReadinessReport
+ from deeptutor.services.parsing.engines.mineru import backend
+ from deeptutor.services.parsing.engines.mineru.config import MinerUConfig, MinerUError
+ from deeptutor.services.parsing.engines.mineru.engine import MinerUParser
+ from deeptutor.services.rag.pipelines.llamaindex import storage, vector_store
+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline
+ from deeptutor.services.rag.service import RAGService
+
+ kb_dir = tmp_path / "kb"
+ raw = kb_dir / "raw"
+ raw.mkdir(parents=True)
+ bad, good = raw / "failed.docx", raw / "next.docx"
+ bad.write_bytes(b"failed source")
+ good.write_bytes(b"good source")
+ parser = MinerUParser()
+ monkeypatch.setattr(parser, "resolve_config", lambda: MinerUConfig(mode=mode))
+ monkeypatch.setattr(parser, "is_ready", lambda config: ReadinessReport(True))
+ monkeypatch.setattr(parse_module, "get_parser", lambda name: parser)
+ service = parse_module.ParseService(cache_root=tmp_path / "parse_cache")
+ monkeypatch.setattr(service, "active_engine", lambda: "mineru")
+ monkeypatch.setattr(parsing, "get_parse_service", lambda: service)
+ attempts = []
+
+ def parse(source, workdir, **kwargs):
+ attempts.append(source.name)
+ if source == bad:
+ if cancelled:
+ assert request_cancel(kb_dir, current_run().task_id)
+ raise MinerUError("Local MinerU parsing was cancelled", code="cancelled")
+ raise MinerUError("1 task(s) failed: Timed out waiting for result", code="timeout")
+ (workdir / "full.md").write_text("The next document explains Fourier transforms.")
+
+ monkeypatch.setattr(backend, "parse_document_to_workdir", parse)
+ monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=4))
+ pipeline = LlamaIndexPipeline(str(tmp_path), signature_provider=lambda: None)
+ monkeypatch.setattr(pipeline, "_configure_settings", lambda: None)
+
+ async def offline_connectivity():
+ return None
+
+ monkeypatch.setattr(pipeline, "_verify_embedding_connectivity", offline_connectivity)
+ rag = RAGService(str(tmp_path), provider="llamaindex")
+ rag._pipelines["llamaindex"] = pipeline
+ if cancelled:
+ with pytest.raises(IndexingCancelled):
+ await rag._index("kb", [str(bad), str(good)], "initialize", {})
+ assert attempts == [bad.name]
+ assert load_run(kb_dir)["state"] == "cancelled"
+ assert not load_run(kb_dir).get("usable_version")
+ return
+ assert await rag._index("kb", [str(bad), str(good)], "initialize", {})
+
+ journal = load_run(kb_dir)
+ assert attempts == [bad.name, good.name]
+ assert journal["state"] == "partial"
+ assert journal["documents"][bad.name]["status"] == "failed"
+ assert "Timed out waiting" in journal["documents"][bad.name]["detail"]
+ assert journal["documents"][good.name]["status"] == "completed"
+ published = kb_dir / journal["usable_version"]
+ storage.verify_persisted_index(published)
+ reloaded = vector_store.load_index(published)
+ assert any("Fourier" in node.text for node in reloaded.docstore.docs.values())
+
+ # Retry must reuse the good parse even though its previous run was partial.
+ assert await rag._index("kb", [str(bad), str(good)], "initialize", {})
+ assert attempts == [bad.name, good.name, bad.name]
+ assert load_run(kb_dir)["documents"][good.name]["parse_reused"] is True
+
+
@pytest.mark.asyncio
async def test_readiness_is_local_and_cancel_requires_writable_owned_task(tmp_path, monkeypatch):
from fastapi import HTTPException
diff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.py
index dd19ebf4d..f0abdd19b 100644
--- a/tests/services/rag/test_llamaindex_document_loader.py
+++ b/tests/services/rag/test_llamaindex_document_loader.py
@@ -204,7 +204,8 @@ def test_loader_continues_after_mineru_task_failure(tmp_path, monkeypatch):
"""#1982: a failed CLI document cannot prevent later documents from loading."""
pytest.importorskip("llama_index.core")
from deeptutor.knowledge.indexing_run import IndexingRun
- from deeptutor.services.parsing.types import ParsedDocument, ParserError
+ from deeptutor.services.parsing.engines.mineru.config import MinerUError
+ from deeptutor.services.parsing.types import ParsedDocument
from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader
failed = tmp_path / "failed.pdf"
@@ -214,7 +215,7 @@ def test_loader_continues_after_mineru_task_failure(tmp_path, monkeypatch):
_install_stub_parse_service(
monkeypatch,
{
- failed.name: ParserError(
+ failed.name: MinerUError(
"1 task(s) failed while processing documents: Timed out waiting for result"
),
good.name: ParsedDocument(markdown="The next document is still usable."),