Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix(render): the PDF manifest's `complete` flag must mean something too (#152)

Aakash Suresha9b9bc39b218
diff --git a/render/src/pixelrag_render/backends/pdf.py b/render/src/pixelrag_render/backends/pdf.py
index 1c3d04e..7241105 100644
--- a/render/src/pixelrag_render/backends/pdf.py
+++ b/render/src/pixelrag_render/backends/pdf.py
@@ -113,7 +113,15 @@ def render_pdf(
"dpi": dpi,
"total_pages": len(saved_tiles),
"tiles": saved_tiles,
- "complete": True,
+ # The selection the render ran with, recorded for the same reason the
+ # URL manifests record their tile height: so a consumer reading
+ # tiles.json can see what was asked for rather than be told out of band.
+ "requested_pages": sorted(pages) if pages is not None else None,
+ # A page selection means these tiles are some of the document rather
+ # than all of it, and nothing here has checked the selection against
+ # the document's length. Only a whole-document render may claim to be
+ # complete — see #139 on what an unconditional flag costs downstream.
+ "complete": pages is None,
}
with open(tile_dir / "tiles.json", "w") as f:
json.dump(manifest, f)
diff --git a/tests/test_pdf_manifest.py b/tests/test_pdf_manifest.py
new file mode 100644
index 0000000..d998853
--- /dev/null
+++ b/tests/test_pdf_manifest.py
@@ -0,0 +1,73 @@
+"""The PDF manifest's `complete` flag must mean what the URL manifests' does.
+
+#139 made the case that a manifest which always claims completeness is worse
+than no flag at all: a consumer cannot tell a whole capture from a fragment,
+so it reads the fragment as the whole. #141 fixed that for the two URL
+backends. The PDF backend writes the same key and still hardcodes it, so a
+caller that asked for a subset of pages gets a directory that claims to be the
+entire document.
+"""
+
+import json
+import shutil
+from pathlib import Path
+
+import pytest
+from PIL import Image
+from pixelrag_render import render_pdf
+
+# pdf2image ships in the optional `pdf` extra and needs poppler's rasteriser on
+# PATH; CI syncs only `--extra dev`. Skip rather than fail, matching how the
+# suite treats the other extras (see test_serve_backends.py).
+pytest.importorskip("pdf2image", reason="pdf extra not installed")
+
+pytestmark = pytest.mark.skipif(
+ shutil.which("pdftoppm") is None,
+ reason="poppler (pdftoppm) not on PATH; pdf2image cannot rasterise",
+)
+
+
+def three_page_pdf(tmp_path):
+ pdf = tmp_path / "doc.pdf"
+ pages = [Image.new("RGB", (612, 792), "white") for _ in range(3)]
+ pages[0].save(pdf, save_all=True, append_images=pages[1:])
+ return pdf
+
+
+def _manifest(tile_dirs):
+ return json.loads((Path(tile_dirs[0]) / "tiles.json").read_text())
+
+
+def test_pdf_manifest_reports_a_full_render_as_complete(three_page_pdf, tmp_path):
+ dirs = render_pdf(three_page_pdf, tmp_path / "out", dpi=50)
+ manifest = _manifest(dirs)
+
+ assert len(manifest["tiles"]) == 3
+ assert manifest["complete"] is True
+ assert manifest["requested_pages"] is None
+
+
+def test_pdf_manifest_reports_a_page_subset_as_incomplete(three_page_pdf, tmp_path):
+ """The whole point of the flag: two of three pages is not the document."""
+ dirs = render_pdf(three_page_pdf, tmp_path / "out", dpi=50, pages=[1, 3])
+ manifest = _manifest(dirs)
+
+ assert len(manifest["tiles"]) == 2, f"test setup rendered wrong: {manifest}"
+ assert manifest["complete"] is False
+ assert manifest["requested_pages"] == [1, 3]
+
+
+def test_pdf_manifest_records_a_full_page_range_as_incomplete(three_page_pdf, tmp_path):
+ """An explicit range that happens to cover everything is still a request.
+
+ The caller asked for specific pages; the backend has not checked them
+ against the document's length, so it must not claim to have captured the
+ document.
+ """
+ dirs = render_pdf(three_page_pdf, tmp_path / "out", dpi=50, pages=[1, 2, 3])
+ manifest = _manifest(dirs)
+
+ assert len(manifest["tiles"]) == 3
+ assert manifest["complete"] is False
+ assert manifest["requested_pages"] == [1, 2, 3]