https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
fix(render): measure page height from content, not the body box (#131)
Zhifei Liae809153b58c
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.pyindex f727048..22924c1 100644--- a/render/src/pixelrag_render/backends/cdp.py+++ b/render/src/pixelrag_render/backends/cdp.py@@ -35,6 +35,8 @@ from pathlib import Pathfrom PIL import Image+from .page_metrics import CONTENT_BOTTOM_JS+logger = logging.getLogger("pixelrag_render.backends.cdp")VIEWPORT_W = 875@@ -344,6 +346,7 @@ def _readiness_expr() -> str:Returns an async-IIFE expression resolving to the page height to tile."""return f"""(async () => {{+ {CONTENT_BOTTOM_JS}await new Promise(res => {{if (document.readyState === 'complete') return res();const t = setTimeout(res, {LOAD_TIMEOUT_MS});@@ -361,8 +364,7 @@ def _readiness_expr() -> str:const sh = document.documentElement.scrollHeight;const body = document.body;if (body) {{- const bottom = Math.ceil(body.getBoundingClientRect().bottom);- return Math.min(sh, Math.max(bottom, 1));+ return Math.min(sh, Math.max(contentBottom(body), 1));}}return sh;}})()"""diff --git a/render/src/pixelrag_render/backends/fast_cdp.py b/render/src/pixelrag_render/backends/fast_cdp.pyindex 251195a..844b2b6 100644--- a/render/src/pixelrag_render/backends/fast_cdp.py+++ b/render/src/pixelrag_render/backends/fast_cdp.py@@ -34,6 +34,8 @@ import timeimport urllib.requestfrom pathlib import Path+from .page_metrics import CONTENT_BOTTOM_JS+logger = logging.getLogger("pixelrag_render.backends.fast_cdp")VIEWPORT_WIDTH = 875@@ -58,8 +60,12 @@ CHROME_ARGS = ["--disable-features=Translate,MediaRouter,OptimizationHints",]-# JS: wait for fonts + eager images, then return scrollHeight-_WAIT_FONTS_IMGS = """new Promise(resolve => {+# JS: wait for fonts + eager images, then return the page height to tile+_WAIT_FONTS_IMGS = (+ """new Promise(resolve => {+ """+ + CONTENT_BOTTOM_JS+ + """const waitEagerImgs = Promise.all(Array.from(document.images).filter(i => !i.complete && i.loading !== 'lazy')@@ -79,12 +85,13 @@ _WAIT_FONTS_IMGS = """new Promise(resolve => {const sh = document.documentElement.scrollHeight;const body = document.body;resolve(body- ? Math.min(sh, Math.max(Math.ceil(body.getBoundingClientRect().bottom), 1))+ ? Math.min(sh, Math.max(contentBottom(body), 1)): sh);});});});})"""+)# ---------------------------------------------------------------------------diff --git a/render/src/pixelrag_render/backends/page_metrics.py b/render/src/pixelrag_render/backends/page_metrics.pynew file mode 100644index 0000000..1800e95--- /dev/null+++ b/render/src/pixelrag_render/backends/page_metrics.py@@ -0,0 +1,44 @@+"""Shared in-page measurement JS for the capture backends.++Kept in one place because both backends measure page height the same way and+must agree: the standard (``cdp``) and turbo (``fast_cdp``) paths each embed+this snippet, and a divergence between them silently changes how much of a page+gets captured depending on which Chrome binary is installed.++No imports — a plain string constant, so either backend can use it without a+dependency edge between them.+"""++# How tall is the document's *content*?+#+# `documentElement.scrollHeight` alone over-reports: padding on the root element+# or a trailing margin inflates it, buying a run of blank tiles at the bottom of+# every such page. So it is clamped to where the content actually ends.+#+# That bound has to be measured from the content, not from the body box. A body+# is only as tall as the document when the page lets it size to its content;+# sites that pin it to the viewport (`html, body { height: 100% }` — Wikipedia's+# Vector 2022 skin among them) leave the content overflowing a one-viewport box,+# and clamping to that box truncates a 20,000px article to a single tile+# (issue #124). Taking the lowest edge among the body and its element children+# reads the same on a self-sizing body and survives a pinned one.+#+# Coordinates are viewport-relative, so scroll offset is added back: a page+# navigated to a `#fragment` lands scrolled down, where a raw rect bottom would+# under-report by exactly the scrolled distance.+CONTENT_BOTTOM_JS = """+ function contentBottom(body) {+ const offset = window.scrollY || window.pageYOffset || 0;+ let bottom = body.getBoundingClientRect().bottom;+ for (let el = body.firstElementChild; el; el = el.nextElementSibling) {+ const r = el.getBoundingClientRect();+ // Skip elements with no box at all (display:none, empty <script>);+ // theirs is a zero rect at the origin and would not move `bottom`,+ // but skipping keeps the intent explicit.+ if (r.width > 0 || r.height > 0) {+ bottom = Math.max(bottom, r.bottom);+ }+ }+ return Math.ceil(bottom + offset);+ }+"""diff --git a/tests/test_render.py b/tests/test_render.pyindex f37a379..05b91b3 100644--- a/tests/test_render.py+++ b/tests/test_render.py@@ -6,6 +6,7 @@ This is the one user-facing capability the README promises, so it is tested endto end rather than mocked."""+import jsonfrom pathlib import Pathfrom pixelrag_render import render_file@@ -29,3 +30,36 @@ def test_render_local_html_to_tiles(tmp_path):f"no tile images produced in {tile_dir} "f"(contents: {[p.name for p in tile_dir.iterdir()]})")+++def test_page_taller_than_a_viewport_bounded_body_is_fully_tiled(tmp_path):+ """A page whose body is pinned to the viewport must still tile in full.++ Regression for issue #124. Sites that set ``html, body { height: 100% }``+ (Wikipedia's Vector 2022 skin among them) leave the article content+ overflowing the body box visibly, so ``body.getBoundingClientRect()`` is one+ viewport tall on a 20,000px page. The readiness probe used to clamp the page+ height to that rect, capturing a single tile and reporting the viewport as+ the page height.+ """+ body = "".join(f"<p>line {i:03d}</p>" for i in range(400))+ html = tmp_path / "viewport_bounded_body.html"+ html.write_text(+ '<!DOCTYPE html><html style="height:100%"><body style="height:100%">'+ f"{body}</body></html>"+ )+ out = tmp_path / "tiles"++ dirs = render_file(html, out, tile_height=1000, viewport_width=1280)++ tile_dir = Path(dirs[0])+ manifest = json.loads((tile_dir / "tiles.json").read_text())+ tiles = sorted(tile_dir.glob("tile_*.jpg"))++ # 400 paragraphs are several viewports tall whatever the default font is;+ # assert against the viewport rather than a brittle exact pixel count.+ assert manifest["page_height"] > 3000, (+ f"page_height {manifest['page_height']} is about one viewport — the "+ "content below the fold was never measured"+ )+ assert len(tiles) > 1, f"expected multiple tiles, got {[t.name for t in tiles]}"