Skip to content

A community-supported supercharged document management system: scan, index and archive all your documents

Feature: Generate PDF thumbnails via poppler utils (#14420)

Trenton Ha05afd654ca7
diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
index 1e0ba23b1..331168ba2 100644
--- a/.github/workflows/ci-backend.yml
+++ b/.github/workflows/ci-backend.yml
@@ -111,7 +111,7 @@ jobs:
timeout-minutes: 12
uses: $/.github/actions/apt-install
with:
- packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils
+ packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils qpdf
- name: Configure ImageMagick
run: |
sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml
diff --git a/docs/configuration.md b/docs/configuration.md
index 3257a815c..a6f8afa89 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1315,29 +1315,17 @@ valid crontab(5) expression describing when to run.
#### [`PAPERLESS_CONVERT_MEMORY_LIMIT=<num>`](#PAPERLESS_CONVERT_MEMORY_LIMIT) {#PAPERLESS_CONVERT_MEMORY_LIMIT}
-: On smaller systems, or even in the case of Very Large Documents, the
-consumer may explode, complaining about how it's "unable to extend
-pixel cache". In such cases, try setting this to a reasonably low
-value, like 32. The default is to use whatever is necessary to do
-everything without writing to disk, and units are in megabytes.
-
- For more information on how to use this value, you should search the
- web for "MAGICK_MEMORY_LIMIT".
+!!! warning
- Defaults to 0, which disables the limit.
+ Deprecated and has no effect, since PDF thumbnails no longer use
+ ImageMagick. It will be removed in a future release.
#### [`PAPERLESS_CONVERT_TMPDIR=<path>`](#PAPERLESS_CONVERT_TMPDIR) {#PAPERLESS_CONVERT_TMPDIR}
-: Similar to the memory limit, if you've got a small system and your
-OS mounts /tmp as tmpfs, you should set this to a path that's on a
-physical disk, like /home/your_user/tmp or something. ImageMagick
-will use this as scratch space when crunching through very large
-documents.
-
- For more information on how to use this value, you should search the
- web for "MAGICK_TMPDIR".
+!!! warning
- Default is none, which disables the temporary directory.
+ Deprecated and has no effect, since PDF thumbnails no longer use
+ ImageMagick. It will be removed in a future release.
#### [`PAPERLESS_APPS=<string>`](#PAPERLESS_APPS) {#PAPERLESS_APPS}
@@ -1889,7 +1877,10 @@ the literal path for that program.
#### [`PAPERLESS_GS_BINARY=<path>`](#PAPERLESS_GS_BINARY) {#PAPERLESS_GS_BINARY}
-: Defaults to "gs".
+!!! warning
+
+ Deprecated and has no effect, since nothing in Paperless-ngx calls
+ Ghostscript by this path. It will be removed in a future release.
## Docker-specific options {#docker}
diff --git a/docs/setup.md b/docs/setup.md
index e666cb58d..514cf424c 100644
--- a/docs/setup.md
+++ b/docs/setup.md
@@ -177,12 +177,12 @@ to a positive number to enable polling and disable native filesystem notificatio
- `pkg-config` for mysqlclient (python dependency)
- `fonts-liberation` for generating thumbnails for plain text
files
- - `imagemagick` >= 6 for PDF conversion
+ - `imagemagick` >= 6 for image alpha handling
- `gnupg` for decrypting GPG-encrypted email
- `libpq-dev` for PostgreSQL
- `libmagic-dev` for mime type detection
- `mariadb-client` for MariaDB compile time
- - `poppler-utils` for barcode detection
+ - `poppler-utils` for thumbnail generation and barcode detection
Use this list for your preferred package management:
@@ -416,11 +416,8 @@ to a positive number to enable polling and disable native filesystem notificatio
You may need to change the path in the files. Example:
`ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
-12. Configure ImageMagick to allow processing of PDF documents and disable
- formats that Paperless-ngx does not use. Most distributions disable PDF
- processing by default, since PDF documents can contain malware. If you
- don't enable it, Paperless-ngx will fall back to Ghostscript for certain
- steps such as thumbnail generation.
+12. Harden ImageMagick by disabling formats that Paperless-ngx does not use.
+ PDF processing is not needed and should stay disabled.
Configure the active ImageMagick policy file (commonly
`/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and
diff --git a/paperless.conf.example b/paperless.conf.example
index e2cedf63c..7e6960c82 100644
--- a/paperless.conf.example
+++ b/paperless.conf.example
@@ -50,8 +50,6 @@ PAPERLESS_SECRET_KEY=change-me
#PAPERLESS_OCR_ROTATE_PAGES=true
#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
#PAPERLESS_OCR_USER_ARGS={}
-#PAPERLESS_CONVERT_MEMORY_LIMIT=0
-#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
# Software tweaks
@@ -91,4 +89,3 @@ PAPERLESS_SECRET_KEY=change-me
# Binaries
#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
-#PAPERLESS_GS_BINARY=/usr/bin/gs
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index 69ee4e285..48983bb23 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -2,12 +2,12 @@ from __future__ import annotations
import logging
import mimetypes
-import os
import shutil
import subprocess
import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
+from typing import Final
from django.conf import settings
@@ -68,105 +68,188 @@ def get_supported_file_extensions() -> set[str]:
return extensions
-def run_convert(
- input_file,
- output_file,
+def get_default_thumbnail() -> Path:
+ """
+ Returns the path to a generic thumbnail
+ """
+ return (Path(__file__).parent / "resources" / "document.webp").resolve()
+
+
+_THUMBNAIL_MAX_WIDTH: Final[int] = 500
+_THUMBNAIL_MAX_HEIGHT: Final[int] = 5000
+# Used only when the page geometry cannot be read
+_THUMBNAIL_FALLBACK_DPI: Final[int] = 150
+# Applied before supersampling, so tiny pages are not enlarged
+_THUMBNAIL_MAX_DPI: Final[int] = 300
+# Rendering at a multiple and downsampling keeps text crisper
+_THUMBNAIL_SUPERSAMPLE: Final[int] = 2
+
+
+def rasterize_pdf_page_to_png(
+ in_path: Path,
+ out_path: Path,
*,
- density=None,
- scale=None,
- alpha=None,
- strip=False,
- trim=False,
- type=None,
- depth=None,
- auto_orient=False,
- use_cropbox=False,
- extra=None,
+ dpi: float,
logging_group=None,
) -> None:
- environment = os.environ.copy()
- if settings.CONVERT_MEMORY_LIMIT:
- # MAGICK_MEMORY_LIMIT sets the maximum amount of RAM the pixel cache can use.
- # MAGICK_MAP_LIMIT sets the maximum amount of memory-mapped I/O allowed.
- #
- # For large-format documents ImageMagick will hit the RAM limit and
- # immediately try to "map" the remaining data. If MAGICK_MAP_LIMIT isn't
- # also set, the process may trigger an OOM kill because the default
- # system/policy map limit is often too restrictive for these massive bitmaps.
- environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
- environment["MAGICK_MAP_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
- if settings.CONVERT_TMPDIR:
- environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR
-
- args = [settings.CONVERT_BINARY]
- args += ["-density", str(density)] if density else []
- args += ["-scale", str(scale)] if scale else []
- args += ["-alpha", str(alpha)] if alpha else []
- args += ["-strip"] if strip else []
- args += ["-trim"] if trim else []
- args += ["-type", str(type)] if type else []
- args += ["-depth", str(depth)] if depth else []
- args += ["-auto-orient"] if auto_orient else []
- args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
- args += [str(input_file), str(output_file)]
+ """
+ Rasterizes the first page of a PDF to a PNG with pdftoppm.
+ """
+ args = [
+ "pdftoppm",
+ # Only the first page is needed, first and last page are both 1
+ "-f",
+ "1",
+ "-l",
+ "1",
+ # Resolution in DPI, which may be fractional
+ "-r",
+ str(dpi),
+ # Write a PNG
+ "-png",
+ # Write exactly one file with no page number in its name. -png appends
+ # ".png" to the output root, so the path is passed without its suffix
+ "-singlefile",
+ # Render the CropBox instead of the MediaBox, as a viewer would. This
+ # must be the same box get_pdf_first_page_size_points measures, or the
+ # DPI is computed for the wrong size
+ "-cropbox",
+ str(in_path),
+ str(out_path.with_suffix("")),
+ ]
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
try:
- run_subprocess(args, environment, logger)
+ run_subprocess(args, logger=logger)
except subprocess.CalledProcessError as e:
- raise ParseError(f"Convert failed at {args}") from e
+ raise ParseError(f"pdftoppm failed at {args}") from e
except Exception as e: # pragma: no cover
- raise ParseError("Unknown error running convert") from e
+ raise ParseError("Unknown error running pdftoppm") from e
-def get_default_thumbnail() -> Path:
+def encode_thumbnail_webp(
+ png_path: Path,
+ out_path: Path,
+ *,
+ supersample: int = 1,
+) -> None:
"""
- Returns the path to a generic thumbnail
+ Flattens alpha onto white, undoes supersampling, shrinks to fit and saves as WebP.
"""
- return (Path(__file__).parent / "resources" / "document.webp").resolve()
+ from PIL import Image
+ try:
+ with Image.open(png_path) as im:
+ if im.mode in ("RGBA", "LA"):
+ flattened = Image.new("RGB", im.size, (255, 255, 255))
+ flattened.paste(im, mask=im.split()[-1])
+ else:
+ flattened = im.convert("RGB")
+
+ # One resize avoids a second, softer pass: undo the supersample and
+ # clamp to the bounds together, never enlarging
+ scale = min(
+ 1 / supersample,
+ _THUMBNAIL_MAX_WIDTH / flattened.width,
+ _THUMBNAIL_MAX_HEIGHT / flattened.height,
+ )
+ if scale < 1:
+ flattened = flattened.resize(
+ (
+ max(1, round(flattened.width * scale)),
+ max(1, round(flattened.height * scale)),
+ ),
+ Image.Resampling.LANCZOS,
+ )
-def make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group=None) -> Path:
- out_path: Path = Path(temp_dir) / "convert_gs.webp"
+ flattened.save(out_path, format="WEBP")
+ except (OSError, Image.DecompressionBombError) as e:
+ raise ParseError(f"Unable to encode thumbnail from {png_path}") from e
- # if convert fails, fall back to extracting
- # the first PDF page as a PNG using Ghostscript
- logger.warning(
- "Thumbnail generation with ImageMagick failed, falling back "
- "to ghostscript. Check your /etc/ImageMagick-x/policy.xml!",
- extra={"group": logging_group},
+
+def _compute_thumbnail_dpi(in_path: Path, logging_group=None) -> tuple[float, int]:
+ """
+ Returns (dpi, supersample). Unknown geometry is not supersampled, since
+ the render size cannot be bounded.
+
+ The DPI is left fractional rather than floored at 1, so an oversized page
+ still renders at about the thumbnail size instead of an unbounded raster.
+ pdftoppm rounds pixel dimensions up, so the render is never short of the
+ target; the shrink-only clamp in encode_thumbnail_webp trims any excess.
+ """
+ from paperless.parsers.utils import get_pdf_first_page_size_points
+
+ size = get_pdf_first_page_size_points(in_path)
+ if size is None:
+ logger.debug(
+ "Could not read PDF page size, using fallback DPI",
+ extra={"group": logging_group},
+ )
+ return _THUMBNAIL_FALLBACK_DPI, 1
+
+ width_pts, height_pts = size
+ dpi_for_width = _THUMBNAIL_MAX_WIDTH * 72 / width_pts
+ dpi_for_height = _THUMBNAIL_MAX_HEIGHT * 72 / height_pts
+ return (
+ min(_THUMBNAIL_MAX_DPI, dpi_for_width, dpi_for_height),
+ _THUMBNAIL_SUPERSAMPLE,
)
- # Ghostscript doesn't handle WebP outputs
- gs_out_path: Path = Path(temp_dir) / "gs_out.png"
- cmd = [settings.GS_BINARY, "-q", "-sDEVICE=pngalpha", "-o", gs_out_path, in_path]
+
+def _render_pdf_thumbnail(
+ in_path: Path,
+ png_path: Path,
+ out_path: Path,
+ logging_group=None,
+) -> None:
+ dpi, supersample = _compute_thumbnail_dpi(in_path, logging_group=logging_group)
+ rasterize_pdf_page_to_png(
+ in_path,
+ png_path,
+ dpi=dpi * supersample,
+ logging_group=logging_group,
+ )
+ encode_thumbnail_webp(png_path, out_path, supersample=supersample)
+
+
+def _repair_pdf_with_qpdf(in_path: Path, out_path: Path) -> None:
+ # qpdf exits 3 after a repair; --warning-exit-0 keeps that from failing
try:
- try:
- run_subprocess(cmd, logger=logger)
- except subprocess.CalledProcessError as e:
- raise ParseError(f"Thumbnail (gs) failed at {cmd}") from e
- # then run convert on the output from gs to make WebP
- run_convert(
- density=300,
- scale="500x5000>",
- alpha="remove",
- strip=True,
- trim=False,
- auto_orient=True,
- input_file=gs_out_path,
- output_file=out_path,
- logging_group=logging_group,
+ shutil.copy(in_path, out_path)
+ run_subprocess(
+ ["qpdf", "--warning-exit-0", "--replace-input", str(out_path)],
+ logger=logger,
)
+ except (subprocess.CalledProcessError, OSError) as e:
+ raise ParseError(f"qpdf repair failed for {in_path}") from e
+
+def make_thumbnail_from_pdf_qpdf_fallback(
+ in_path: Path,
+ temp_dir: Path,
+ logging_group=None,
+) -> Path:
+ png_path = temp_dir / "page1_repaired.png"
+ out_path = temp_dir / "convert_qpdf.webp"
+ repaired_path = temp_dir / "repaired.pdf"
+
+ logger.warning(
+ "Thumbnail generation with pdftoppm failed, attempting qpdf repair and retry.",
+ extra={"group": logging_group},
+ )
+
+ try:
+ _repair_pdf_with_qpdf(in_path, repaired_path)
+ _render_pdf_thumbnail(repaired_path, png_path, out_path, logging_group)
return out_path
except ParseError as e:
- logger.error(f"Unable to make thumbnail with Ghostscript: {e}")
+ logger.error(f"Unable to make thumbnail after qpdf repair: {e}")
# The caller might expect a generated thumbnail that can be moved,
# so we need to copy it before it gets moved.
# https://github.com/paperless-ngx/paperless-ngx/issues/3631
- default_thumbnail_path: Path = Path(temp_dir) / "document.webp"
+ default_thumbnail_path = temp_dir / "document.webp"
copy_file_with_basic_stats(get_default_thumbnail(), default_thumbnail_path)
return default_thumbnail_path
@@ -175,25 +258,18 @@ def make_thumbnail_from_pdf(in_path: Path, temp_dir: Path, logging_group=None) -
"""
The thumbnail of a PDF is just a 500px wide image of the first page.
"""
+ png_path: Path = temp_dir / "page1.png"
out_path: Path = temp_dir / "convert.webp"
- # Run convert to get a decent thumbnail
try:
- run_convert(
- density=300,
- scale="500x5000>",
- alpha="remove",
- strip=True,
- trim=False,
- auto_orient=True,
- use_cropbox=True,
- input_file=f"{in_path}[0]",
- output_file=str(out_path),
- logging_group=logging_group,
- )
+ _render_pdf_thumbnail(in_path, png_path, out_path, logging_group)
except ParseError as e:
- logger.error(f"Unable to make thumbnail with convert: {e}")
- out_path = make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group)
+ logger.error(f"Unable to make thumbnail with pdftoppm: {e}")
+ out_path = make_thumbnail_from_pdf_qpdf_fallback(
+ in_path,
+ temp_dir,
+ logging_group,
+ )
return out_path
diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py
index 5f3893112..cf27994c1 100644
--- a/src/documents/tests/test_parsers.py
+++ b/src/documents/tests/test_parsers.py
@@ -1,11 +1,22 @@
+import subprocess
from collections.abc import Generator
+from pathlib import Path
+import pikepdf
import pytest
+from PIL import Image
from pytest_django.fixtures import Settings
+from pytest_mock import MockerFixture
+from documents.parsers import ParseError
+from documents.parsers import _compute_thumbnail_dpi
+from documents.parsers import encode_thumbnail_webp
from documents.parsers import get_default_file_extension
+from documents.parsers import get_default_thumbnail
from documents.parsers import get_supported_file_extensions
from documents.parsers import is_file_ext_supported
+from documents.parsers import make_thumbnail_from_pdf
+from documents.parsers import rasterize_pdf_page_to_png
from paperless.parsers.registry import get_parser_registry
from paperless.parsers.registry import reset_parser_registry
from paperless.parsers.tesseract import RasterisedDocumentParser
@@ -125,3 +136,387 @@ class TestParserAvailability:
assert is_file_ext_supported(".pdf")
assert not is_file_ext_supported(".hsdfh")
assert not is_file_ext_supported("")
+
+
+class TestComputeThumbnailDpi:
+ @pytest.mark.parametrize(
+ ("size", "expected"),
+ [
+ pytest.param((612.0, 792.0), (500 * 72 / 612, 2), id="letter-width-bound"),
+ pytest.param(
+ (792.0, 612.0),
+ (500 * 72 / 792, 2),
+ id="landscape-width-bound",
+ ),
+ pytest.param(
+ (612.0, 100000.0),
+ (5000 * 72 / 100000, 2),
+ id="tall-strip-height-bound",
+ ),
+ pytest.param((200.0, 300.0), (180, 2), id="small-page-width-bound"),
+ pytest.param((72.0, 72.0), (300, 2), id="tiny-page-capped-at-300"),
+ pytest.param(
+ (1000000.0, 1000000.0),
+ (500 * 72 / 1000000, 2),
+ id="huge-page-fractional",
+ ),
+ pytest.param(None, (150, 1), id="unreadable-geometry-fallback"),
+ ],
+ )
+ def test_dpi_from_page_size(
+ self,
+ mocker: MockerFixture,
+ tmp_path: Path,
+ size: tuple[float, float] | None,
+ expected: tuple[float, int],
+ ) -> None:
+ """
+ GIVEN:
+ - A first page of the given size, or unreadable geometry
+ WHEN:
+ - The thumbnail DPI is computed
+ THEN:
+ - The expected DPI and supersample factor are returned
+ """
+ mocker.patch(
+ "paperless.parsers.utils.get_pdf_first_page_size_points",
+ return_value=size,
+ )
+ assert _compute_thumbnail_dpi(tmp_path / "doc.pdf") == pytest.approx(expected)
+
+
+class TestMakeThumbnailFromPdf:
+ @pytest.fixture
+ def work_dir(self, tmp_path: Path) -> Path:
+ path = tmp_path / "work"
+ path.mkdir()
+ return path
+
+ @staticmethod
+ def _write_blank_pdf(path: Path, page_size: tuple[int, int] = (612, 792)) -> Path:
+ pdf = pikepdf.new()
+ pdf.add_blank_page(page_size=page_size)
+ pdf.save(path, object_stream_mode=pikepdf.ObjectStreamMode.disable)
+ return path
+
+ @pytest.mark.parametrize(
+ ("size", "expected_dpi", "expected_supersample"),
+ [
+ pytest.param(
+ (612.0, 792.0),
+ 2 * 500 * 72 / 612,
+ 2,
+ id="known-geometry-2x",
+ ),
+ pytest.param(None, 150, 1, id="unreadable-geometry-plain-fallback"),
+ ],
+ )
+ def test_render_dpi_requested(
+ self,
+ mocker: MockerFixture,
+ tmp_path: Path,
+ work_dir: Path,
+ size: tuple[float, float] | None,
+ expected_dpi: float,
+ expected_supersample: int,
+ ) -> None:
+ """
+ GIVEN:
+ - Readable or unreadable page geometry
+ WHEN:
+ - A thumbnail is made
+ THEN:
+ - Rasterize and encode get the matching DPI and supersample factor
+ """
+ mocker.patch(
+ "paperless.parsers.utils.get_pdf_first_page_size_points",
+ return_value=size,
+ )
+ rasterize = mocker.patch("documents.parsers.rasterize_pdf_page_to_png")
+ encode = mocker.patch("documents.parsers.encode_thumbnail_webp")
+
+ make_thumbnail_from_pdf(tmp_path / "in.pdf", work_dir)
+
+ assert rasterize.call_args.kwargs["dpi"] == pytest.approx(expected_dpi)
+ assert encode.call_args.kwargs["supersample"] == expected_supersample
+
+ @pytest.mark.parametrize(
+ ("page_size", "expected_width"),
+ [
+ pytest.param((612, 792), 500, id="letter"),
+ pytest.param((792, 612), 500, id="landscape-letter"),
+ pytest.param((595, 842), 500, id="a4"),
+ pytest.param((200, 300), 500, id="small-page"),
+ pytest.param((72, 72), 300, id="tiny-page-capped"),
+ ],
+ )
+ def test_thumbnail_width(
+ self,
+ tmp_path: Path,
+ work_dir: Path,
+ page_size: tuple[int, int],
+ expected_width: int,
+ ) -> None:
+ """
+ GIVEN:
+ - A PDF whose first page has the given size in points
+ WHEN:
+ - A thumbnail is made from it
+ THEN:
+ - The thumbnail has the expected width
+ """
+ pdf_path = self._write_blank_pdf(tmp_path / "in.pdf", page_size)
+
+ thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
+
+ assert thumb == work_dir / "convert.webp"
+ with Image.open(thumb) as im:
+ assert im.format == "WEBP"
+ assert im.width == expected_width
+
+ @classmethod
+ def _write_pdf_without_xref(cls, path: Path) -> Path:
+ """
+ Cuts off the xref and trailer, which pdftoppm cannot recover from but qpdf can.
+ """
+ cls._write_blank_pdf(path)
+ data = path.read_bytes()
+ path.write_bytes(data[: data.rindex(b"\nxref")])
+ return path
+
+ def test_qpdf_repair_produces_real_thumbnail(
+ self,
+ tmp_path: Path,
+ work_dir: Path,
+ ) -> None:
+ """
+ GIVEN:
+ - A PDF with its xref table and trailer cut off
+ WHEN:
+ - A thumbnail is made from it
+ THEN:
+ - The thumbnail is rendered from a qpdf repaired copy
+ - The original file is unchanged
+ """
+ pdf_path = self._write_pdf_without_xref(tmp_path / "broken.pdf")
+ original_bytes = pdf_path.read_bytes()
+
+ with pytest.raises(ParseError):
+ rasterize_pdf_page_to_png(pdf_path, work_dir / "probe.png", dpi=50)
+
+ thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
+
+ assert thumb == work_dir / "convert_qpdf.webp"
+ with Image.open(thumb) as im:
+ assert im.format == "WEBP"
+ assert im.width == 500
+ assert pdf_path.read_bytes() == original_bytes
+
+ @pytest.mark.parametrize(
+ "qpdf_error",
+ [
+ pytest.param(subprocess.CalledProcessError(2, "qpdf"), id="qpdf-fails"),
+ pytest.param(None, id="repaired-still-unrenderable"),
+ ],
+ )
+ def test_double_failure_uses_default_thumbnail(
+ self,
+ mocker: MockerFixture,
+ tmp_path: Path,
+ work_dir: Path,
+ qpdf_error: subprocess.CalledProcessError | None,
+ ) -> None:
+ """
+ GIVEN:
+ - A PDF that cannot be rendered, even after qpdf repair
+ WHEN:
+ - A thumbnail is made from it
+ THEN:
+ - A copy of the default thumbnail is returned
+ """
+ mocker.patch(
+ "documents.parsers.rasterize_pdf_page_to_png",
+ side_effect=ParseError("Does not compute."),
+ )
+ if qpdf_error is not None:
+ mocker.patch("documents.parsers.run_subprocess", side_effect=qpdf_error)
+ pdf_path = self._write_blank_pdf(tmp_path / "in.pdf")
+
+ thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
+
+ assert thumb == work_dir / "document.webp"
+ assert thumb.read_bytes() == get_default_thumbnail().read_bytes()
+
+
+class TestRasterizePdfPageToPng:
+ @staticmethod
+ def _write_pdf(
+ path: Path,
+ *,
+ crop_box: tuple[float, float, float, float] | None = None,
+ rotate: int | None = None,
+ ) -> Path:
+ pdf = pikepdf.new()
+ pdf.add_blank_page(page_size=(144, 72))
+ pdf.add_blank_page(page_size=(300, 300))
+ page = pdf.pages[0]
+ if crop_box is not None:
+ page.obj.CropBox = pikepdf.Array(crop_box)
+ if rotate is not None:
+ page.obj.Rotate = rotate
+ pdf.save(path)
+ return path
+
+ @pytest.mark.parametrize(
+ ("crop_box", "rotate", "expected_size"),
+ [
+ pytest.param(None, None, (144, 72), id="plain"),
+ pytest.param(None, 90, (72, 144), id="rotated-90"),
+ pytest.param((0, 0, 72, 36), None, (72, 36), id="crop-box"),
+ ],
+ )
+ def test_renders_first_page_to_exact_path(
+ self,
+ tmp_path: Path,
+ crop_box: tuple[float, float, float, float] | None,
+ rotate: int | None,
+ expected_size: tuple[int, int],
+ ) -> None:
+ """
+ GIVEN:
+ - A two page PDF, first page optionally cropped or rotated
+ WHEN:
+ - The first page is rasterized at 72 DPI
+ THEN:
+ - Only out_path is written, sized to the first page's crop and rotation
+ """
+ pdf_path = self._write_pdf(
+ tmp_path / "in.pdf",
+ crop_box=crop_box,
+ rotate=rotate,
+ )
+ out_dir = tmp_path / "out"
+ out_dir.mkdir()
+ out_path = out_dir / "page1.png"
+
+ rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72)
+
+ assert list(out_dir.iterdir()) == [out_path]
+ with Image.open(out_path) as im:
+ assert im.format == "PNG"
+ assert im.size == expected_size
+
+ def test_failure_raises_parse_error(self, tmp_path: Path) -> None:
+ """
+ GIVEN:
+ - A file that is not a PDF
+ WHEN:
+ - Rasterization is attempted
+ THEN:
+ - A ParseError is raised
+ """
+ bad = tmp_path / "bad.pdf"
+ bad.write_bytes(b"not a pdf")
+
+ with pytest.raises(ParseError):
+ rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72)
+
+
+class TestEncodeThumbnailWebp:
+ @pytest.mark.parametrize(
+ ("mode", "color"),
+ [
+ pytest.param("RGBA", (0, 0, 0, 0), id="rgba"),
+ pytest.param("LA", (0, 0), id="la"),
+ ],
+ )
+ def test_alpha_flattened_onto_white(
+ self,
+ tmp_path: Path,
+ mode: str,
+ color: tuple[int, ...],
+ ) -> None:
+ """
+ GIVEN:
+ - A fully transparent PNG with an alpha channel
+ WHEN:
+ - It is encoded as a thumbnail
+ THEN:
+ - The WebP output is RGB with the transparency flattened to white
+ """
+ png_path = tmp_path / "in.png"
+ Image.new(mode, (20, 10), color).save(png_path)
+ out_path = tmp_path / "out.webp"
+
+ encode_thumbnail_webp(png_path, out_path)
+
+ with Image.open(out_path) as im:
+ assert im.format == "WEBP"
+ assert im.mode == "RGB"
+ assert im.size == (20, 10)
+ red, green, blue = im.getpixel((10, 5))
+ assert min(red, green, blue) >= 250
+
+ @pytest.mark.parametrize(
+ ("in_size", "supersample", "expected_size"),
+ [
+ pytest.param((1000, 2000), 1, (500, 1000), id="too-wide-shrunk"),
+ pytest.param((100, 10000), 1, (50, 5000), id="too-tall-shrunk"),
+ pytest.param((100, 200), 1, (100, 200), id="small-not-enlarged"),
+ pytest.param((1000, 1400), 2, (500, 700), id="2x-halved"),
+ pytest.param((1001, 1401), 2, (500, 700), id="2x-odd-rounded"),
+ pytest.param((600, 800), 2, (300, 400), id="2x-small-not-enlarged"),
+ pytest.param((900, 1200), 2, (450, 600), id="2x-below-clamp"),
+ ],
+ )
+ def test_size_clamped_and_downsampled(
+ self,
+ tmp_path: Path,
+ in_size: tuple[int, int],
+ supersample: int,
+ expected_size: tuple[int, int],
+ ) -> None:
+ """
+ GIVEN:
+ - A rendered image and its supersample factor
+ WHEN:
+ - It is encoded as a thumbnail with that factor
+ THEN:
+ - It is downsampled, fit within 500x5000 and never enlarged
+ """
+ png_path = tmp_path / "in.png"
+ Image.new("RGB", in_size, (255, 255, 255)).save(png_path)
+ out_path = tmp_path / "out.webp"
+
+ encode_thumbnail_webp(png_path, out_path, supersample=supersample)
+
+ with Image.open(out_path) as im:
+ assert im.size == expected_size
+
+ @pytest.mark.parametrize(
+ "error",
+ [
+ pytest.param(OSError("broken image"), id="os-error"),
+ pytest.param(Image.DecompressionBombError("too large"), id="bomb"),
+ ],
+ )
+ def test_decode_failure_raises_parse_error(
+ self,
+ tmp_path: Path,
+ mocker: MockerFixture,
+ error: Exception,
+ ) -> None:
+ """
+ GIVEN:
+ - Opening the rendered image fails
+ WHEN:
+ - It is encoded as a thumbnail
+ THEN:
+ - A ParseError is raised
+ """
+ png_path = tmp_path / "in.png"
+ Image.new("RGB", (10, 10)).save(png_path)
+ mocker.patch("PIL.Image.open", side_effect=error)
+
+ with pytest.raises(ParseError):
+ encode_thumbnail_webp(png_path, tmp_path / "out.webp")
diff --git a/src/paperless/checks.py b/src/paperless/checks.py
index 16bbff55e..8c333eefc 100644
--- a/src/paperless/checks.py
+++ b/src/paperless/checks.py
@@ -82,7 +82,7 @@ def binaries_check(app_configs: Any, **kwargs: Any) -> list[Error]:
error = "Paperless can't find {}. Without it, consumption is impossible."
hint = "Either it's not in your ${PATH} or it's not installed."
- binaries = (settings.CONVERT_BINARY, "tesseract", "gs")
+ binaries = (settings.CONVERT_BINARY, "tesseract", "gs", "pdftoppm", "pdfinfo")
check_messages = []
for binary in binaries:
diff --git a/src/paperless/parsers/utils.py b/src/paperless/parsers/utils.py
index 9fe1d4908..b01e1678e 100644
--- a/src/paperless/parsers/utils.py
+++ b/src/paperless/parsers/utils.py
@@ -10,7 +10,9 @@ from __future__ import annotations
import codecs
import logging
+import os
import re
+import subprocess
import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
@@ -25,6 +27,15 @@ logger = logging.getLogger("paperless.parsers.utils")
# Used by both the consumer (archive decision) and the tesseract parser (skip-OCR decision).
PDF_TEXT_MIN_LENGTH: Final[int] = 50
+_PDFINFO_PAGE_SIZE: Final[re.Pattern[str]] = re.compile(
+ r"^Page\s+1 size:\s+([\d.eE+-]+) x ([\d.eE+-]+) pts",
+ re.MULTILINE,
+)
+_PDFINFO_PAGE_ROTATION: Final[re.Pattern[str]] = re.compile(
+ r"^Page\s+1 rot:\s+(\d+)",
+ re.MULTILINE,
+)
+
def is_tagged_pdf(
path: Path,
@@ -265,6 +276,61 @@ def get_page_count_for_pdf(
return None
+def get_pdf_first_page_size_points(
+ path: Path,
+ log: logging.Logger | None = None,
+) -> tuple[float, float] | None:
+ """Return the first page's (width, height) in PDF points, post-rotation.
+
+ Asks ``pdfinfo`` rather than reading the page boxes directly, so the size
+ matches what ``pdftoppm -cropbox`` renders, including poppler's clipping of
+ the CropBox to the MediaBox. pdfinfo reports the box size before rotation,
+ so width and height are swapped for 90/270 rotation.
+
+ Parameters
+ ----------
+ path:
+ Absolute path to the PDF file.
+ log:
+ Logger for warnings. Falls back to the module-level logger when omitted.
+
+ Returns
+ -------
+ tuple[float, float] | None
+ ``(width_points, height_points)``, or ``None`` if pdfinfo fails, the
+ file has no pages, or the page size is degenerate.
+ """
+ _log = log or logger
+ try:
+ proc = subprocess.run(
+ ["pdfinfo", "-f", "1", "-l", "1", str(path)],
+ capture_output=True,
+ check=True,
+ text=True,
+ errors="ignore",
+ # The labels parsed below are localized
+ env={**os.environ, "LC_ALL": "C"},
+ )
+ except (OSError, subprocess.CalledProcessError) as e:
+ _log.warning("Could not determine PDF page size for %s: %s", path, e)
+ return None
+
+ size_match = _PDFINFO_PAGE_SIZE.search(proc.stdout)
+ if size_match is None:
+ _log.warning("Could not determine PDF page size for %s", path)
+ return None
+
+ width = float(size_match.group(1))
+ height = float(size_match.group(2))
+ if width <= 0 or height <= 0:
+ return None
+
+ rotation_match = _PDFINFO_PAGE_ROTATION.search(proc.stdout)
+ if rotation_match is not None and int(rotation_match.group(1)) in (90, 270):
+ width, height = height, width
+ return width, height
+
+
def extract_pdf_metadata(
document_path: Path,
log: logging.Logger | None = None,
diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py
index 004b23feb..23e700461 100644
--- a/src/paperless/settings/__init__.py
+++ b/src/paperless/settings/__init__.py
@@ -978,10 +978,6 @@ GNUPG_HOME = os.getenv("HOME", "/tmp")
# Convert is part of the ImageMagick package
CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert")
-CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
-CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
-
-GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
# Fallback layout for .eml consumption
EMAIL_PARSE_DEFAULT_LAYOUT = get_int_from_env(
diff --git a/src/paperless/tests/parsers/test_tesseract_parser.py b/src/paperless/tests/parsers/test_tesseract_parser.py
index bb6a85031..f057ad086 100644
--- a/src/paperless/tests/parsers/test_tesseract_parser.py
+++ b/src/paperless/tests/parsers/test_tesseract_parser.py
@@ -15,9 +15,10 @@ from typing import TYPE_CHECKING
import pytest
from ocrmypdf import SubprocessOutputError
+from PIL import Image
from documents.parsers import ParseError
-from documents.parsers import run_convert
+from documents.parsers import rasterize_pdf_page_to_png
from paperless.models import ModeChoices
from paperless.parsers import ParserProtocol
from paperless.parsers.tesseract import RasterisedDocumentParser
@@ -280,24 +281,71 @@ class TestGetThumbnail:
)
assert thumb.is_file()
- def test_thumbnail_fallback_on_convert_error(
+ @pytest.mark.parametrize(
+ ("filename", "expected_height"),
+ [
+ pytest.param("simple-digital.pdf", 647, id="portrait-letter"),
+ pytest.param("rotated.pdf", 386, id="landscape"),
+ ],
+ )
+ def test_thumbnail_is_correct_format_and_size(
+ self,
+ tesseract_parser: RasterisedDocumentParser,
+ tesseract_samples_dir: Path,
+ filename: str,
+ expected_height: int,
+ ) -> None:
+ """
+ GIVEN:
+ - A portrait or landscape PDF
+ WHEN:
+ - A thumbnail is generated
+ THEN:
+ - A 500px wide WebP keeping the page's aspect ratio
+ """
+ thumb = tesseract_parser.get_thumbnail(
+ tesseract_samples_dir / filename,
+ "application/pdf",
+ )
+ with Image.open(thumb) as im:
+ assert im.format == "WEBP"
+ assert im.width == 500
+ assert im.height == pytest.approx(expected_height, abs=2)
+
+ def test_thumbnail_fallback_on_pdftoppm_error(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
- def _raise_on_pdf(input_file, output_file, **kwargs) -> None:
- if ".pdf" in str(input_file):
- raise ParseError("Does not compute.")
- run_convert(input_file=input_file, output_file=output_file, **kwargs)
+ """
+ GIVEN:
+ - Rasterizing the original PDF fails
+ WHEN:
+ - A thumbnail is generated
+ THEN:
+ - The PDF is repaired with qpdf and a real thumbnail is rendered
+ """
+ original = tesseract_samples_dir / "simple-digital.pdf"
- mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf)
+ def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None:
+ if in_path == original:
+ raise ParseError("Does not compute.")
+ rasterize_pdf_page_to_png(in_path, out_path, **kwargs)
- thumb = tesseract_parser.get_thumbnail(
- tesseract_samples_dir / "simple-digital.pdf",
- "application/pdf",
+ rasterize = mocker.patch(
+ "documents.parsers.rasterize_pdf_page_to_png",
+ side_effect=_fail_on_original,
)
+
+ thumb = tesseract_parser.get_thumbnail(original, "application/pdf")
+
+ assert rasterize.call_count == 2
assert thumb.is_file()
+ assert thumb.name == "convert_qpdf.webp"
+ with Image.open(thumb) as im:
+ assert im.format == "WEBP"
+ assert im.width == 500
def test_thumbnail_encrypted_pdf(
self,
diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py
index 4d8a5bfa6..0ea129adf 100644
--- a/src/paperless/tests/test_checks.py
+++ b/src/paperless/tests/test_checks.py
@@ -1,4 +1,5 @@
import os
+import shutil
from collections.abc import Callable
from pathlib import Path
from unittest import mock
@@ -29,6 +30,19 @@ class TestChecks:
settings.CONVERT_BINARY = "uuuhh"
assert len(binaries_check(None)) == 1
+ @pytest.mark.parametrize("binary", ["pdftoppm", "pdfinfo"])
+ def test_binaries_fail_poppler(self, mocker: MockerFixture, binary: str) -> None:
+ real_which = shutil.which
+ mocker.patch(
+ "paperless.checks.shutil.which",
+ side_effect=lambda name: None if name == binary else real_which(name),
+ )
+
+ msgs = binaries_check(None)
+
+ assert len(msgs) == 1
+ assert binary in msgs[0].msg
+
@pytest.mark.usefixtures("paperless_dirs")
def test_paths_check(self) -> None:
assert paths_check(None) == []
diff --git a/src/paperless/tests/test_parser_utils.py b/src/paperless/tests/test_parser_utils.py
index f0a63ea44..7bae1f78d 100644
--- a/src/paperless/tests/test_parser_utils.py
+++ b/src/paperless/tests/test_parser_utils.py
@@ -6,8 +6,10 @@ import codecs
from pathlib import Path
from typing import TYPE_CHECKING
+import pikepdf
import pytest
+from paperless.parsers.utils import get_pdf_first_page_size_points
from paperless.parsers.utils import is_tagged_pdf
from paperless.parsers.utils import pdf_born_digital_text
from paperless.parsers.utils import post_process_text
@@ -70,6 +72,197 @@ class TestIsTaggedPdf:
assert is_tagged_pdf(bad) is False
+class TestGetPdfFirstPageSizePoints:
+ @staticmethod
+ def _write_pdf(
+ path: Path,
+ *,
+ media_box: tuple[float, float, float, float] = (0, 0, 600, 800),
+ crop_box: tuple[float, float, float, float] | None = None,
+ page_rotate: int | None = None,
+ inherited_rotate: int | None = None,
+ ) -> Path:
+ pdf = pikepdf.new()
+ pdf.add_blank_page(page_size=(media_box[2], media_box[3]))
+ page = pdf.pages[0]
+ page.obj.MediaBox = pikepdf.Array(media_box)
+ if crop_box is not None:
+ page.obj.CropBox = pikepdf.Array(crop_box)
+ if page_rotate is not None:
+ page.obj.Rotate = page_rotate
+ if inherited_rotate is not None:
+ pdf.Root.Pages.Rotate = inherited_rotate
+ pdf.save(path)
+ return path
+
+ def test_letter_sample(self) -> None:
+ """
+ GIVEN:
+ - A US Letter sample PDF with no CropBox and no rotation
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - The MediaBox size in points is returned
+ """
+ assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == (
+ 612.0,
+ 792.0,
+ )
+
+ @pytest.mark.parametrize(
+ ("rotate", "expected"),
+ [
+ pytest.param(0, (600.0, 800.0), id="rotate-0"),
+ pytest.param(90, (800.0, 600.0), id="rotate-90"),
+ pytest.param(180, (600.0, 800.0), id="rotate-180"),
+ pytest.param(270, (800.0, 600.0), id="rotate-270"),
+ pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"),
+ ],
+ )
+ def test_page_rotation_swaps_dimensions(
+ self,
+ tmp_path: Path,
+ rotate: int,
+ expected: tuple[float, float],
+ ) -> None:
+ """
+ GIVEN:
+ - A portrait PDF page with /Rotate set directly on the page
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - Width and height are swapped for quarter-turn rotations only
+ """
+ pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate)
+ assert get_pdf_first_page_size_points(pdf_path) == expected
+
+ def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None:
+ """
+ GIVEN:
+ - A page inheriting /Rotate 90 from the /Pages node
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - Width and height are swapped
+ """
+ pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90)
+ assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0)
+
+ def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None:
+ """
+ GIVEN:
+ - A PDF page with a CropBox smaller than its MediaBox
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - The CropBox size is returned
+ """
+ pdf_path = self._write_pdf(
+ tmp_path / "cropped.pdf",
+ crop_box=(50, 100, 350, 500),
+ )
+ assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0)
+
+ def test_crop_box_clipped_to_media_box(self, tmp_path: Path) -> None:
+ """
+ GIVEN:
+ - A PDF page whose CropBox extends well past its MediaBox
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - The size is the CropBox clipped to the MediaBox, as pdftoppm renders it
+ """
+ pdf_path = self._write_pdf(
+ tmp_path / "oversized-crop.pdf",
+ media_box=(0, 0, 200, 200),
+ crop_box=(-200, -200, 600, 600),
+ )
+ assert get_pdf_first_page_size_points(pdf_path) == (200.0, 200.0)
+
+ def test_crop_box_with_rotation(self, tmp_path: Path) -> None:
+ """
+ GIVEN:
+ - A rotated PDF page with a CropBox smaller than its MediaBox
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - The CropBox size is returned with width and height swapped
+ """
+ pdf_path = self._write_pdf(
+ tmp_path / "cropped-rotated.pdf",
+ media_box=(0, 0, 600, 300),
+ crop_box=(100, 50, 400, 250),
+ page_rotate=270,
+ )
+ assert get_pdf_first_page_size_points(pdf_path) == (200.0, 300.0)
+
+ def test_pdfinfo_not_installed_returns_none(
+ self,
+ mocker: MockerFixture,
+ ) -> None:
+ """
+ GIVEN:
+ - The pdfinfo binary cannot be executed
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - None is returned and nothing is raised
+ """
+ mocker.patch("subprocess.run", side_effect=FileNotFoundError("pdfinfo"))
+ assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") is None
+
+ def test_degenerate_box_returns_none(self, tmp_path: Path) -> None:
+ """
+ GIVEN:
+ - A PDF page whose box has zero width
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - None is returned
+ """
+ pdf_path = self._write_pdf(
+ tmp_path / "degenerate.pdf",
+ media_box=(0, 0, 600, 800),
+ crop_box=(100, 0, 100, 800),
+ )
+ assert get_pdf_first_page_size_points(pdf_path) is None
+
+ def test_nonexistent_path_returns_none(self) -> None:
+ """
+ GIVEN:
+ - A path that does not exist
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - None is returned and nothing is raised
+ """
+ assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None
+
+ def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None:
+ """
+ GIVEN:
+ - A file that is not a PDF
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - None is returned and nothing is raised
+ """
+ bad = tmp_path / "bad.pdf"
+ bad.write_bytes(b"not a pdf")
+ assert get_pdf_first_page_size_points(bad) is None
+
+ def test_encrypted_pdf_returns_none(self) -> None:
+ """
+ GIVEN:
+ - A password protected PDF
+ WHEN:
+ - The first page size is requested
+ THEN:
+ - None is returned and nothing is raised
+ """
+ assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None
+
+
class TestPostProcessText:
@pytest.mark.parametrize(
("source", "expected"),