A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Fix: don't skip OCR/archive for tagged PDFs with no actual text (#13351)
Trenton Hdcff067dc126
diff --git a/docs/configuration.md b/docs/configuration.mdindex 83bd25138..6529791f0 100644--- a/docs/configuration.md+++ b/docs/configuration.md@@ -932,8 +932,8 @@ for display in the web interface.| Document type | `never` | `auto` (default) | `always` || -------------------------- | ------- | -------------------------- | -------- || Scanned image (TIFF, JPEG) | No | **Yes** | Yes |- | Image-based PDF | No | **Yes** (short/no text, untagged) | Yes |- | Born-digital PDF | No | No (tagged or has embedded text) | Yes |+ | Image-based PDF | No | **Yes** (no embedded text) | Yes |+ | Born-digital PDF | No | No (has embedded text, optionally confirmed by tag) | Yes || Plain text, email, HTML | No | No | No || DOCX / ODT (via Tika) | Yes\* | Yes\* | Yes\* |diff --git a/src/documents/consumer.py b/src/documents/consumer.pyindex 7c01853c5..e7f026e71 100644--- a/src/documents/consumer.py+++ b/src/documents/consumer.py@@ -160,13 +160,14 @@ def should_produce_archive(_log.debug("Archive: yes — image document, ARCHIVE_FILE_GENERATION=auto")return Trueif mime_type == "application/pdf":- if is_tagged_pdf(document_path):+ text = extract_pdf_text(document_path)+ has_text = text is not None and len(text) > 0+ if has_text and is_tagged_pdf(document_path):_log.debug("Archive: no — born-digital PDF (structure tags detected),"" ARCHIVE_FILE_GENERATION=auto",)return False- text = extract_pdf_text(document_path)if text is None or len(text) <= PDF_TEXT_MIN_LENGTH:_log.debug("Archive: yes — scanned PDF (text_length=%d ≤ %d),"diff --git a/src/documents/tests/test_consumer_archive.py b/src/documents/tests/test_consumer_archive.pyindex 265bd7bc6..c179162db 100644--- a/src/documents/tests/test_consumer_archive.py+++ b/src/documents/tests/test_consumer_archive.py@@ -166,24 +166,28 @@ class TestShouldProduceArchive:mocker: MockerFixture,settings,) -> None:- """Tagged PDFs (e.g. Word exports) are treated as born-digital regardless of text length."""+ """Tagged PDFs (e.g. Word exports) with real text are treated as born-digital, even below PDF_TEXT_MIN_LENGTH."""settings.ARCHIVE_FILE_GENERATION = "auto"mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)+ mocker.patch("documents.consumer.extract_pdf_text", return_value="tiny")parser = _parser_instance(can_produce=True, requires_rendition=False)assert (should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))is False)- def test_tagged_pdf_does_not_call_pdftotext(+ def test_tagged_pdf_without_text_produces_archive(self,mocker: MockerFixture,settings,) -> None:- """When a PDF is tagged, pdftotext is not invoked (fast path)."""+ """A tagged PDF with no actual extractable text (e.g. some scanner firmware) is not+ trusted as born-digital — the tag alone must not bypass OCR."""settings.ARCHIVE_FILE_GENERATION = "auto"mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)- mock_extract = mocker.patch("documents.consumer.extract_pdf_text")+ mocker.patch("documents.consumer.extract_pdf_text", return_value=None)parser = _parser_instance(can_produce=True, requires_rendition=False)- should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))- mock_extract.assert_not_called()+ assert (+ should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))+ is True+ )diff --git a/src/paperless/parsers/tesseract.py b/src/paperless/parsers/tesseract.pyindex 2e0d791ea..78ce4c5c4 100644--- a/src/paperless/parsers/tesseract.py+++ b/src/paperless/parsers/tesseract.py@@ -510,8 +510,10 @@ class RasterisedDocumentParser:if mime_type == "application/pdf":text_original = self.extract_text(None, document_path)- original_has_text = is_tagged_pdf(document_path, log=self.log) or (- text_original is not None and len(text_original) > PDF_TEXT_MIN_LENGTH+ has_text = text_original is not None and len(text_original) > 0+ original_has_text = has_text and (+ is_tagged_pdf(document_path, log=self.log)+ or len(text_original) > PDF_TEXT_MIN_LENGTH)else:text_original = Nonediff --git a/src/paperless/tests/parsers/test_tesseract_parser.py b/src/paperless/tests/parsers/test_tesseract_parser.pyindex 0c086745c..e56992d0b 100644--- a/src/paperless/tests/parsers/test_tesseract_parser.py+++ b/src/paperless/tests/parsers/test_tesseract_parser.py@@ -906,6 +906,34 @@ class TestSkipArchive:assert tesseract_parser.archive_path is Noneassert tesseract_parser.get_text()+ def test_tagged_pdf_without_text_does_not_skip_ocr(+ self,+ mocker: MockerFixture,+ tesseract_parser: RasterisedDocumentParser,+ tesseract_samples_dir: Path,+ ) -> None:+ """+ GIVEN:+ - A PDF that reports itself as tagged (/MarkInfo /Marked true) but+ has no actual extractable text (some scanner firmware produces+ this — see GitHub issue #13349)+ - Mode: auto, produce_archive=False+ WHEN:+ - Document is parsed+ THEN:+ - The tag alone is not trusted as "has text"; OCRmyPDF still runs+ """+ tesseract_parser.settings.mode = ModeChoices.AUTO+ mocker.patch("paperless.parsers.tesseract.is_tagged_pdf", return_value=True)+ mocker.patch.object(tesseract_parser, "extract_text", return_value=None)+ mock_ocr = mocker.patch("ocrmypdf.ocr")+ tesseract_parser.parse(+ tesseract_samples_dir / "multi-page-images.pdf",+ "application/pdf",+ produce_archive=False,+ )+ mock_ocr.assert_called()+def test_tagged_pdf_produces_pdfa_archive_without_ocr(self,tesseract_parser: RasterisedDocumentParser,