OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: detect password-protected PDFs when docling omits the reason (#2540)
Mike Pawlowski1da5a09e9567
diff --git a/src/services/docling_service.py b/src/services/docling_service.pyindex a96fca248..0d3b3022d 100644--- a/src/services/docling_service.py+++ b/src/services/docling_service.py@@ -1,6 +1,7 @@import asyncioimport jsonimport platform+import refrom dataclasses import dataclassfrom enum import StrEnumfrom pathlib import Path@@ -232,6 +233,96 @@ def _format_docling_error(payload: dict[str, Any]) -> str:return result+# Keeps the "incorrect password" substring that task_service._infer_failure_metadata+# maps to the user-facing password-protected message.+PASSWORD_PROTECTED_PDF_DETAIL = "PDF is password-protected (Incorrect password)"++# The trailer / xref-stream dictionary holding /Encrypt is never compressed, so a+# byte scan finds it without a PDF library. Applied only to the active trailer+# dictionary (_active_trailer_dict): /Encrypt elsewhere in the file, such as page+# text or an attached PDF, does not make the document encrypted. Matches an+# indirect reference (/Encrypt 5 0 R) or a direct dictionary (/Encrypt <<).+_PDF_ENCRYPT_RE = re.compile(rb"/Encrypt\s*(?:\d+\s+\d+\s+R|<<)")+_STARTXREF_RE = re.compile(rb"startxref\s+(\d+)")+# What a startxref offset may point at: a classic `xref` table (group 1) or the+# header of an indirect object that should be the xref stream.+_XREF_SECTION_START_RE = re.compile(rb"\s*(?:(xref)|\d+\s+\d+\s+obj\b)")+# The object's dictionary ends where its stream data or the object itself ends.+_OBJ_DICT_END_RE = re.compile(rb"stream|endobj")+_XREF_TYPE_RE = re.compile(rb"/Type\s*/XRef\b")+++def is_docling_load_failure(message: str) -> bool:+ """True when docling reports that its backend could not open the document."""+ return "could not load document" in message.lower()+++def _active_trailer_dict(content: bytes) -> bytes | None:+ """Return the trailer or xref-stream dictionary that the final ``startxref`` names.++ PDF encryption is declared only there. Incremental updates repeat /Encrypt in+ the newest trailer, and a linearized file's final ``startxref`` points at the+ first-page section, which carries the full trailer. Offsets are tried both from+ the start of the file and from the ``%PDF-`` header, because some producers+ write junk before the header. An object at the offset counts only if it is an+ xref stream (``/Type /XRef``), so a stale offset in a damaged file cannot pick+ up an unrelated dictionary. Returns ``None`` when the content is not a PDF or+ the trailer cannot be located.++ Searches ``content`` in place and slices out only the dictionary, because a+ linearized file's offset is near the start and a tail copy would duplicate+ almost the whole upload.+ """+ header = content.find(b"%PDF-", 0, 1024)+ if header < 0:+ return None+ matches = list(_STARTXREF_RE.finditer(content, max(0, len(content) - 1024)))+ if not matches:+ logger.debug("No startxref found; not treating the PDF as encrypted")+ return None+ offset = int(matches[-1].group(1))+ for start in dict.fromkeys((offset, offset + header)):+ section = _XREF_SECTION_START_RE.match(content, start)+ if section is None:+ continue+ if section.group(1):+ # Classic table: the dictionary follows the `trailer` keyword.+ trailer = content.find(b"trailer", section.end())+ if trailer < 0:+ logger.debug("Cross-reference table has no trailer", offset=start)+ return None+ end = content.find(b"startxref", trailer)+ return content[trailer : end if end >= 0 else None]+ # Xref stream: the dictionary precedes `stream`; the data is excluded.+ dict_end = _OBJ_DICT_END_RE.search(content, section.end())+ xref_dict = content[section.end() : dict_end.start() if dict_end else None]+ if _XREF_TYPE_RE.search(xref_dict):+ return xref_dict+ logger.debug("startxref offset does not point at a cross-reference section", offset=offset)+ return None+++def is_encrypted_pdf(content: bytes) -> bool:+ """True when ``content`` is a PDF whose active trailer declares an encryption dictionary."""+ trailer = _active_trailer_dict(content)+ return trailer is not None and _PDF_ENCRYPT_RE.search(trailer) is not None+++def password_protected_pdf_error(message: str, content: bytes) -> str | None:+ """Return the password-protected detail for a docling load failure on an encrypted PDF.++ docling 2.108 pre-opened PDFs with pypdfium2, whose error read "Incorrect+ password". Later versions load through docling-parse first, which reports only+ "Failed to load document with key ...", so the reason has to be recovered from+ the bytes. This is checked only after docling failed to load the file: a PDF+ with just an owner password also declares /Encrypt but converts fine. A file+ whose trailer cannot be located keeps docling's own message.+ """+ if is_docling_load_failure(message) and is_encrypted_pdf(content):+ return PASSWORD_PROTECTED_PDF_DETAIL+ return None++def _extract_document_json_content(payload: dict[str, Any], *, task_id: str) -> dict[str, Any]:"""Return ``document.json_content`` from a ``/v1/result`` payload.@@ -851,8 +942,8 @@ class DoclingService:ocr=ocr,picture_descriptions=picture_descriptions,)- return await self.get_docling_result_async(- task_id, user_id=user_id, auth_header=auth_header+ return await self._get_result_for_content(+ task_id, file_bytes, user_id=user_id, auth_header=auth_header)async def convert_bytes(@@ -868,6 +959,30 @@ class DoclingService:task_id = await self.upload_to_docling_direct_async(filename, content, user_id=user_id, auth_header=auth_header)- return await self.get_docling_result_async(- task_id, user_id=user_id, auth_header=auth_header+ return await self._get_result_for_content(+ task_id, content, user_id=user_id, auth_header=auth_header)++ async def _get_result_for_content(+ self,+ task_id: str,+ content: bytes,+ *,+ user_id: str | None,+ auth_header: str | None,+ ) -> dict[str, Any]:+ """Fetch the conversion result, naming a password-protected PDF on load failure."""+ try:+ return await self.get_docling_result_async(+ task_id, user_id=user_id, auth_header=auth_header+ )+ except DoclingServeError as exc:+ detail = password_protected_pdf_error(str(exc), content)+ if detail is None:+ raise+ logger.warning(+ "Docling could not load an encrypted PDF",+ task_id=task_id,+ docling_error=str(exc),+ )+ raise DoclingServeError(f"Docling processing failed: {detail}") from excdiff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex c2119e835..d980a0879 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -1178,9 +1178,18 @@ class LangflowFileService:"elapsed_seconds": round(poll_result.elapsed_seconds, 2),},)+ from services.docling_service import password_protected_pdf_error++ detail = poll_result.detail+ if (+ poll_result.outcome == PollOutcome.FAILED+ and detail+ and isinstance(content, bytes)+ ):+ detail = password_protected_pdf_error(detail, content) or detailraise Exception(f"Docling conversion did not complete ({poll_result.outcome.value}): "- f"{poll_result.detail or 'no detail provided'}"+ f"{detail or 'no detail provided'}")if file_task is not None:diff --git a/src/services/task_service.py b/src/services/task_service.pyindex 2741bbe8c..79c7e9fef 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -1365,7 +1365,9 @@ class TaskService:if opensearch_meta:return opensearch_meta- if "incorrect password" in error.lower(): # for password protected pdf cases+ # Password-protected PDFs. Newer docling no longer names the reason, so the+ # backend recovers it from the file bytes (docling_service.password_protected_pdf_error).+ if "incorrect password" in error.lower():return {"component": "docling","failure_phase": "parsing",diff --git a/tests/unit/test_docling_service.py b/tests/unit/test_docling_service.pyindex 945954572..85ba5cab0 100644--- a/tests/unit/test_docling_service.py+++ b/tests/unit/test_docling_service.py@@ -8,7 +8,14 @@ from unittest.mock import AsyncMock, MagicMock, patchimport httpximport pytest-from services.docling_service import DoclingPayloadTooLarge, DoclingServeError, DoclingService+from services.docling_service import (+ PASSWORD_PROTECTED_PDF_DETAIL,+ DoclingPayloadTooLarge,+ DoclingServeError,+ DoclingService,+ is_encrypted_pdf,+ password_protected_pdf_error,+)from utils.upload_limits import UploadTooLarge# Shape of a /v1/result body for a password-protected PDF: the task ran to@@ -166,6 +173,196 @@ async def test_poll_result_surfaces_conversion_failure_from_result_payload(assert "missing document.json_content" not in message+# ── Password-protected PDFs on newer docling ───────────────────────++# docling >= 2.141 loads through docling-parse first, whose load failure no longer+# names the reason (seen on CPD, tracker #99817).+_DOCLING_PARSE_LOAD_FAILURE = (+ "Docling processing failed: docling-parse could not load document 8fc525: "+ "Failed to load document with key key=8fc525"+)+_XREF_TABLE = b"xref\n0 1\n0000000000 65535 f \n"+++def _pdf(body: bytes, trailer: bytes, *, junk: bytes = b"") -> bytes:+ """Minimal classic-xref PDF; with ``junk``, offsets are relative to the header."""+ head = b"%PDF-1.7\n" + body+ return junk + head + _XREF_TABLE + trailer + b"\nstartxref\n%d\n%%%%EOF\n" % len(head)+++def _append_revision(pdf: bytes, body: bytes, trailer: bytes) -> bytes:+ """Append an incremental update whose trailer becomes the active one."""+ head = pdf + body+ return head + _XREF_TABLE + trailer + b"\nstartxref\n%d\n%%%%EOF\n" % len(head)+++def _xref_stream_pdf(entries: bytes, data: bytes) -> bytes:+ """Minimal PDF whose active cross-reference section is an xref stream."""+ head = b"%PDF-1.5\n"+ xref = b"9 0 obj\n<< /Type /XRef %s /Length %d >>\nstream\n%s\nendstream\nendobj\n" % (+ entries,+ len(data),+ data,+ )+ return head + xref + b"startxref\n%d\n%%%%EOF\n" % len(head)+++def _stale_startxref_pdf(objects: bytes) -> bytes:+ """PDF whose ``startxref`` names the first of ``objects`` rather than an xref section."""+ head = b"%PDF-1.5\n"+ return head + objects + b"startxref\n%d\n%%%%EOF\n" % len(head)+++_CATALOG = b"1 0 obj\n<< /Type /Catalog >>\nendobj\n"+_ENCRYPTED_PDF = _pdf(_CATALOG, b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R /Size 6 >>")+_PLAIN_PDF = _pdf(_CATALOG, b"trailer\n<< /Root 1 0 R >>")+# Unencrypted, but a content stream shows the literal text "/Encrypt 1 0 R".+_PLAIN_PDF_WITH_ENCRYPT_TEXT = _pdf(+ _CATALOG + b"4 0 obj\n<< /Length 25 >>\nstream\nBT (/Encrypt 1 0 R) Tj ET\nendstream\nendobj\n",+ b"trailer\n<< /Root 1 0 R >>",+)+++ ("content", "expected"),+ [+ pytest.param(_ENCRYPTED_PDF, True, id="classic-trailer-encrypted"),+ pytest.param(+ _xref_stream_pdf(b"/Encrypt << /Filter /Standard >>", b"abc"),+ True,+ id="xref-stream-encrypted",+ ),+ pytest.param(+ _append_revision(+ _ENCRYPTED_PDF,+ b"2 0 obj\n<< /Type /Page >>\nendobj\n",+ b"trailer\n<< /Root 1 0 R /Prev %d /Encrypt 5 0 R /Size 6 >>"+ % _ENCRYPTED_PDF.index(b"xref"),+ ),+ True,+ id="incremental-update-encrypted",+ ),+ pytest.param(+ _pdf(_CATALOG, b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R >>", junk=b"JUNK\n"),+ True,+ id="junk-before-header",+ ),+ pytest.param(_PLAIN_PDF, False, id="classic-trailer-plain"),+ pytest.param(_PLAIN_PDF_WITH_ENCRYPT_TEXT, False, id="encrypt-in-content-stream"),+ pytest.param(+ _xref_stream_pdf(b"", b"/Encrypt 1 0 R"), False, id="encrypt-in-xref-stream-data"+ ),+ pytest.param(+ b"%PDF-1.7\n" + _CATALOG + b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R >>\n",+ False,+ id="truncated-no-startxref",+ ),+ pytest.param(+ b"%PDF-1.7\ntrailer\n<< /Encrypt 5 0 R >>\nstartxref\n3\n%%EOF\n",+ False,+ id="startxref-points-at-garbage",+ ),+ pytest.param(+ _stale_startxref_pdf(b"3 0 obj\n<< /Type /Filespec /Encrypt 5 0 R >>\nendobj\n"),+ False,+ id="startxref-points-at-non-xref-object",+ ),+ pytest.param(+ # The non-xref dictionary ends at `endobj`; the later xref stream's+ # /Type /XRef and /Encrypt must not be read as belonging to it.+ _stale_startxref_pdf(+ b"3 0 obj\n<< /Type /Filespec >>\nendobj\n"+ b"9 0 obj\n<< /Type /XRef /Encrypt 5 0 R /Length 3 >>\nstream\nabc\n"+ b"endstream\nendobj\n"+ ),+ False,+ id="startxref-points-at-object-before-xref-stream",+ ),+ pytest.param(+ # Offsets are relative to the header, as in the classic junk case.+ b"JUNK\n" + _xref_stream_pdf(b"/Encrypt 5 0 R", b"abc"),+ True,+ id="xref-stream-junk-before-header",+ ),+ pytest.param(b"not a pdf /Encrypt 5 0 R startxref 0", False, id="not-a-pdf"),+ ],+)+def test_is_encrypted_pdf(content, expected):+ assert is_encrypted_pdf(content) is expected+++def test_password_protected_pdf_error_needs_load_failure_and_encryption():+ assert (+ password_protected_pdf_error(_DOCLING_PARSE_LOAD_FAILURE, _ENCRYPTED_PDF)+ == PASSWORD_PROTECTED_PDF_DETAIL+ )+ # A corrupt but unencrypted PDF keeps docling's own message.+ assert password_protected_pdf_error(_DOCLING_PARSE_LOAD_FAILURE, _PLAIN_PDF) is None+ # An owner-password-only PDF that failed for another reason is not relabelled.+ assert (+ password_protected_pdf_error("Docling processing failed: OCR crashed", _ENCRYPTED_PDF)+ is None+ )+++async def test_convert_bytes_names_password_protected_pdf(docling_service):+ with (+ patch.object(+ docling_service, "upload_to_docling_direct_async", AsyncMock(return_value="t1")+ ),+ patch.object(+ docling_service,+ "get_docling_result_async",+ AsyncMock(side_effect=DoclingServeError(_DOCLING_PARSE_LOAD_FAILURE)),+ ),+ pytest.raises(DoclingServeError) as exc_info,+ ):+ await docling_service.convert_bytes(_ENCRYPTED_PDF, "secret.pdf")++ assert str(exc_info.value) == f"Docling processing failed: {PASSWORD_PROTECTED_PDF_DETAIL}"+ assert str(exc_info.value.__cause__) == _DOCLING_PARSE_LOAD_FAILURE+++async def test_convert_bytes_keeps_docling_error_when_encrypt_appears_outside_trailer(+ docling_service,+):+ """Literal /Encrypt text in page content must not relabel an unrelated load failure."""+ with (+ patch.object(+ docling_service, "upload_to_docling_direct_async", AsyncMock(return_value="t1")+ ),+ patch.object(+ docling_service,+ "get_docling_result_async",+ AsyncMock(side_effect=DoclingServeError(_DOCLING_PARSE_LOAD_FAILURE)),+ ),+ pytest.raises(DoclingServeError) as exc_info,+ ):+ await docling_service.convert_bytes(_PLAIN_PDF_WITH_ENCRYPT_TEXT, "about-pdf.pdf")++ assert str(exc_info.value) == _DOCLING_PARSE_LOAD_FAILURE+++async def test_convert_file_keeps_docling_error_for_unencrypted_pdf(docling_service, tmp_path):+ pdf = tmp_path / "broken.pdf"+ pdf.write_bytes(_PLAIN_PDF)+ with (+ patch.object(+ docling_service, "upload_to_docling_direct_async", AsyncMock(return_value="t1")+ ),+ patch.object(+ docling_service,+ "get_docling_result_async",+ AsyncMock(side_effect=DoclingServeError(_DOCLING_PARSE_LOAD_FAILURE)),+ ),+ pytest.raises(DoclingServeError, match="Failed to load document with key"),+ ):+ await docling_service.convert_file(str(pdf))++@pytest.mark.asyncioasync def test_poll_result_http_error(docling_service, mock_httpx_client):"""Propagates HTTP errors during polling as DoclingServeError."""diff --git a/tests/unit/test_langflow_file_service_two_phase.py b/tests/unit/test_langflow_file_service_two_phase.pyindex ac1f543ae..4105e53e8 100644--- a/tests/unit/test_langflow_file_service_two_phase.py+++ b/tests/unit/test_langflow_file_service_two_phase.py@@ -113,6 +113,58 @@ async def test_langflow_not_invoked_on_docling_failure(assert file_task.phase == IngestionPhase.DOCLING+async def test_docling_load_failure_on_encrypted_pdf_names_password_protection(+ langflow_service, mock_polling_service, file_task+):+ """Newer docling drops the PDFium reason; the bytes still show the PDF is encrypted."""+ head = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n"+ pdf = (+ head+ + b"xref\n0 1\n0000000000 65535 f \ntrailer\n<< /Root 1 0 R /Encrypt 5 0 R >>\n"+ + b"startxref\n%d\n%%%%EOF\n" % len(head)+ )+ encrypted = ("secret.pdf", pdf, "application/pdf")+ mock_polling_service.poll_until_ready.return_value = DoclingPollResult(+ outcome=PollOutcome.FAILED,+ detail=(+ "Docling result unavailable after SUCCESS status: Docling processing failed: "+ "docling-parse could not load document 8fc525: "+ "Failed to load document with key key=8fc525"+ ),+ )++ with pytest.raises(Exception) as exc_info:+ await langflow_service.upload_and_ingest_file(+ file_tuple=encrypted,+ docling_polling_service=mock_polling_service,+ file_task=file_task,+ )++ message = str(exc_info.value)+ assert message.startswith("Docling conversion did not complete (failed): ")+ assert "Incorrect password" in message+ assert "with key" not in message+ assert langflow_service.run_ingestion_flow.call_count == 0+++async def test_docling_load_failure_on_unencrypted_pdf_keeps_docling_detail(+ langflow_service, mock_polling_service, file_tuple, file_task+):+ detail = "docling-parse could not load document 8fc525: Failed to load document with key k"+ mock_polling_service.poll_until_ready.return_value = DoclingPollResult(+ outcome=PollOutcome.FAILED, detail=detail+ )++ with pytest.raises(Exception, match="Failed to load document with key k"):+ await langflow_service.upload_and_ingest_file(+ file_tuple=file_tuple,+ docling_polling_service=mock_polling_service,+ file_task=file_task,+ )++@pytest.mark.asyncioasync def test_langflow_not_invoked_on_docling_expired(langflow_service, mock_polling_service, file_tuple, file_taskdiff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.pyindex 32c5a5444..e21ee6742 100644--- a/tests/unit/test_task_service_get_task_status2.py+++ b/tests/unit/test_task_service_get_task_status2.py@@ -14,6 +14,7 @@ from unittest.mock import Mockimport pytestfrom models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTask+from services.docling_service import PASSWORD_PROTECTED_PDF_DETAILfrom services.task_service import TaskService, _opensearch_failure_metadata# ---------------------------------------------------------------------------@@ -203,6 +204,31 @@ class TestInferFailureMetadata:assert "docling-parse" not in msg.lower()assert "cbfc231c" not in msg+ @pytest.mark.parametrize(+ "error",+ [+ # Langflow path (upload_and_ingest_file)+ f"Docling conversion did not complete (failed): {PASSWORD_PROTECTED_PDF_DETAIL}",+ # Non-Langflow path (DoclingService.convert_file / convert_bytes)+ f"Docling processing failed: {PASSWORD_PROTECTED_PDF_DETAIL}",+ ],+ )+ def test_backend_detected_password_protected_pdf_maps_to_clean_message(+ self, task_service, error+ ):+ """docling >= 2.141 no longer names the reason; the backend's remap must still match."""+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.FAILED,+ error=error,+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["actionable_by"] == "USER_ACTIONABLE"+ assert meta["component"] == "docling"+ assert meta["failure_phase"] == "parsing"+ assert meta["user_facing_message"].startswith("This PDF is password-protected.")+def test_langflow_empty_content_not_retryable(self, task_service):ft = _make_file_task(phase=IngestionPhase.LANGFLOW,