DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(rag): isolate unreadable PDF evidence during KB indexing (#1982)
Bingxi Zhao (Frank)e98e3d75587b
diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.pyindex 6cbd96049..78608014a 100644--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py@@ -28,6 +28,7 @@ from deeptutor.services.llm.image_caption_batch import ImageCaptionBatcherfrom deeptutor.services.llm.image_caption_cache import complete_image_captionfrom deeptutor.services.llm.image_description import get_image_description_clientfrom deeptutor.services.rag.file_routing import FileTypeRouter+from deeptutor.services.rag.source_pages import SourcePageErrorfrom deeptutor.services.rag.visual_assets import VisualAssetCandidate, collect_visual_assetsfrom deeptutor.utils.document_validator import DocumentValidator@@ -92,31 +93,42 @@ class LlamaIndexDocumentLoader:# archive download) on a synchronous httpx.Client — running it on# the event loop stalls every other request for the whole PDF# (same class of bug as upstream #761/#777). Hand it to a thread.- text, extracted_images, parse_engine = await asyncio.to_thread(- self._parse_document, file_path, kb_dir=kb_dir- )- scanned_pdf_needs_ocr = (- file_path.suffix.lower() == ".pdf"- and not text.strip()- and parse_engine != _PDF_OCR_FALLBACK_ENGINE- )- if scanned_pdf_needs_ocr:- fallback = await asyncio.to_thread(- self._parse_document,- file_path,- None,- _PDF_OCR_FALLBACK_ENGINE,- kb_dir=kb_dir,+ try:+ text, extracted_images, parse_engine = await asyncio.to_thread(+ self._parse_document, file_path, kb_dir=kb_dir)- if fallback[0].strip() or fallback[1]:- text, extracted_images, parse_engine = fallback- scanned_pdf_needs_ocr = False- else:- self._log_scanned_pdf_without_ocr(+ scanned_pdf_needs_ocr = (+ file_path.suffix.lower() == ".pdf"+ and not text.strip()+ and parse_engine != _PDF_OCR_FALLBACK_ENGINE+ )+ if scanned_pdf_needs_ocr:+ fallback = await asyncio.to_thread(+ self._parse_document,file_path,- parse_engine,- len(extracted_images),+ None,+ _PDF_OCR_FALLBACK_ENGINE,+ kb_dir=kb_dir,)+ if fallback[0].strip() or fallback[1]:+ text, extracted_images, parse_engine = fallback+ scanned_pdf_needs_ocr = False+ else:+ self._log_scanned_pdf_without_ocr(+ file_path,+ parse_engine,+ len(extracted_images),+ )+ except SourcePageError as exc:+ # Parsing can succeed while the original PDF is unreadable.+ # Keep that document's failure isolated and visible (#1982).+ from deeptutor.knowledge.indexing_run import current_run++ self._pdf_documents.pop(str(file_path), None)+ self.logger.warning("Skipped %s: %s", file_path.name, exc)+ if run := current_run():+ run.document(file_path, "failed", reason="invalid_source", detail=str(exc))+ continueif not scanned_pdf_needs_ocr:page_documents = self._pdf_documents.pop(str(file_path), [])if page_documents:@@ -239,9 +251,14 @@ class LlamaIndexDocumentLoader:text = parsed.markdown.strip() or self._text_from_blocks(parsed.blocks)if kb_dir is not None and file_path.suffix.lower() == ".pdf":+ import pymupdf+from deeptutor.services.rag.source_pages import build_pdf_documents- self._pdf_documents[str(file_path)] = build_pdf_documents(file_path, parsed, kb_dir)+ try:+ self._pdf_documents[str(file_path)] = build_pdf_documents(file_path, parsed, kb_dir)+ except (pymupdf.FileDataError, OSError) as exc:+ raise SourcePageError(f"Cannot read PDF page evidence: {exc}") from excimages = self._collect_asset_images(parsed.asset_dir, origin=file_path)if kb_dir is not None:from deeptutor.services.rag.visual_coverage import record_coveragediff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.pyindex f0abdd19b..07f566d8c 100644--- a/tests/services/rag/test_llamaindex_document_loader.py+++ b/tests/services/rag/test_llamaindex_document_loader.py@@ -228,6 +228,47 @@ def test_loader_continues_after_mineru_task_failure(tmp_path, monkeypatch):assert run.data["documents"][run.source_key(good)]["status"] == "embedding"[email protected]("encrypted", [False, True])+def test_unusable_pdf_evidence_does_not_abort_other_documents(tmp_path, monkeypatch, encrypted):+ """A parser success is not a guarantee that the PDF can supply page evidence."""+ import pymupdf++ from deeptutor.knowledge.indexing_run import IndexingRun+ from deeptutor.services.parsing.types import ParsedDocument+ from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader++ raw = tmp_path / "raw"+ raw.mkdir()+ bad = raw / "bad.pdf"+ if encrypted:+ with pymupdf.open() as pdf:+ pdf.new_page().insert_text((72, 72), "Private source")+ pdf.save(+ bad, encryption=pymupdf.PDF_ENCRYPT_AES_256, owner_pw="owner", user_pw="reader"+ )+ else:+ bad.write_bytes(b"%PDF-1.7\ntruncated")+ good = raw / "good.docx"+ good.write_bytes(b"parser fixture")+ _install_stub_parse_service(+ monkeypatch,+ {+ bad.name: ParsedDocument(markdown="Parser supplied text", engine="mineru"),+ good.name: ParsedDocument(markdown="Usable next document"),+ },+ )+ with IndexingRun(tmp_path, [str(bad), str(good)]) as run:+ documents = asyncio.run(+ LlamaIndexDocumentLoader().load([str(bad), str(good)], kb_dir=tmp_path)+ )+ assert [doc.metadata["file_name"] for doc in documents] == [good.name]+ failure = run.data["documents"][run.source_key(bad)]+ assert failure["status"] == "failed"+ assert failure["reason"] == "invalid_source"+ assert failure["detail"]+ assert run.data["documents"][run.source_key(good)]["status"] == "embedding"++def test_loader_explains_scanned_pdf_when_parser_yields_images_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None: