Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): isolate unreadable PDF evidence during KB indexing (#1982)

Bingxi Zhao (Frank)e98e3d75587b
diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
index 6cbd96049..78608014a 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
@@ -28,6 +28,7 @@ from deeptutor.services.llm.image_caption_batch import ImageCaptionBatcher
from deeptutor.services.llm.image_caption_cache import complete_image_caption
from deeptutor.services.llm.image_description import get_image_description_client
from deeptutor.services.rag.file_routing import FileTypeRouter
+from deeptutor.services.rag.source_pages import SourcePageError
from deeptutor.services.rag.visual_assets import VisualAssetCandidate, collect_visual_assets
from deeptutor.utils.document_validator import DocumentValidator
@@ -92,31 +93,42 @@ class LlamaIndexDocumentLoader:
# archive download) on a synchronous httpx.Client — running it on
# the event loop stalls every other request for the whole PDF
# (same class of bug as upstream #761/#777). Hand it to a thread.
- text, extracted_images, parse_engine = await asyncio.to_thread(
- self._parse_document, file_path, kb_dir=kb_dir
- )
- scanned_pdf_needs_ocr = (
- file_path.suffix.lower() == ".pdf"
- and not text.strip()
- and parse_engine != _PDF_OCR_FALLBACK_ENGINE
- )
- if scanned_pdf_needs_ocr:
- fallback = await asyncio.to_thread(
- self._parse_document,
- file_path,
- None,
- _PDF_OCR_FALLBACK_ENGINE,
- kb_dir=kb_dir,
+ try:
+ text, extracted_images, parse_engine = await asyncio.to_thread(
+ self._parse_document, file_path, kb_dir=kb_dir
)
- if fallback[0].strip() or fallback[1]:
- text, extracted_images, parse_engine = fallback
- scanned_pdf_needs_ocr = False
- else:
- self._log_scanned_pdf_without_ocr(
+ scanned_pdf_needs_ocr = (
+ file_path.suffix.lower() == ".pdf"
+ and not text.strip()
+ and parse_engine != _PDF_OCR_FALLBACK_ENGINE
+ )
+ if scanned_pdf_needs_ocr:
+ fallback = await asyncio.to_thread(
+ self._parse_document,
file_path,
- parse_engine,
- len(extracted_images),
+ None,
+ _PDF_OCR_FALLBACK_ENGINE,
+ kb_dir=kb_dir,
)
+ if fallback[0].strip() or fallback[1]:
+ text, extracted_images, parse_engine = fallback
+ scanned_pdf_needs_ocr = False
+ else:
+ self._log_scanned_pdf_without_ocr(
+ file_path,
+ parse_engine,
+ len(extracted_images),
+ )
+ except SourcePageError as exc:
+ # Parsing can succeed while the original PDF is unreadable.
+ # Keep that document's failure isolated and visible (#1982).
+ from deeptutor.knowledge.indexing_run import current_run
+
+ self._pdf_documents.pop(str(file_path), None)
+ self.logger.warning("Skipped %s: %s", file_path.name, exc)
+ if run := current_run():
+ run.document(file_path, "failed", reason="invalid_source", detail=str(exc))
+ continue
if not scanned_pdf_needs_ocr:
page_documents = self._pdf_documents.pop(str(file_path), [])
if page_documents:
@@ -239,9 +251,14 @@ class LlamaIndexDocumentLoader:
text = parsed.markdown.strip() or self._text_from_blocks(parsed.blocks)
if kb_dir is not None and file_path.suffix.lower() == ".pdf":
+ import pymupdf
+
from deeptutor.services.rag.source_pages import build_pdf_documents
- self._pdf_documents[str(file_path)] = build_pdf_documents(file_path, parsed, kb_dir)
+ try:
+ self._pdf_documents[str(file_path)] = build_pdf_documents(file_path, parsed, kb_dir)
+ except (pymupdf.FileDataError, OSError) as exc:
+ raise SourcePageError(f"Cannot read PDF page evidence: {exc}") from exc
images = self._collect_asset_images(parsed.asset_dir, origin=file_path)
if kb_dir is not None:
from deeptutor.services.rag.visual_coverage import record_coverage
diff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.py
index f0abdd19b..07f566d8c 100644
--- a/tests/services/rag/test_llamaindex_document_loader.py
+++ b/tests/services/rag/test_llamaindex_document_loader.py
@@ -228,6 +228,47 @@ def test_loader_continues_after_mineru_task_failure(tmp_path, monkeypatch):
assert run.data["documents"][run.source_key(good)]["status"] == "embedding"
[email protected]("encrypted", [False, True])
+def test_unusable_pdf_evidence_does_not_abort_other_documents(tmp_path, monkeypatch, encrypted):
+ """A parser success is not a guarantee that the PDF can supply page evidence."""
+ import pymupdf
+
+ from deeptutor.knowledge.indexing_run import IndexingRun
+ from deeptutor.services.parsing.types import ParsedDocument
+ from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader
+
+ raw = tmp_path / "raw"
+ raw.mkdir()
+ bad = raw / "bad.pdf"
+ if encrypted:
+ with pymupdf.open() as pdf:
+ pdf.new_page().insert_text((72, 72), "Private source")
+ pdf.save(
+ bad, encryption=pymupdf.PDF_ENCRYPT_AES_256, owner_pw="owner", user_pw="reader"
+ )
+ else:
+ bad.write_bytes(b"%PDF-1.7\ntruncated")
+ good = raw / "good.docx"
+ good.write_bytes(b"parser fixture")
+ _install_stub_parse_service(
+ monkeypatch,
+ {
+ bad.name: ParsedDocument(markdown="Parser supplied text", engine="mineru"),
+ good.name: ParsedDocument(markdown="Usable next document"),
+ },
+ )
+ with IndexingRun(tmp_path, [str(bad), str(good)]) as run:
+ documents = asyncio.run(
+ LlamaIndexDocumentLoader().load([str(bad), str(good)], kb_dir=tmp_path)
+ )
+ assert [doc.metadata["file_name"] for doc in documents] == [good.name]
+ failure = run.data["documents"][run.source_key(bad)]
+ assert failure["status"] == "failed"
+ assert failure["reason"] == "invalid_source"
+ assert failure["detail"]
+ assert run.data["documents"][run.source_key(good)]["status"] == "embedding"
+
+
def test_loader_explains_scanned_pdf_when_parser_yields_images_only(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None: