Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix(render): measure page height from content, not the body box (#131)

Zhifei Liae809153b58c
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.py
index f727048..22924c1 100644
--- a/render/src/pixelrag_render/backends/cdp.py
+++ b/render/src/pixelrag_render/backends/cdp.py
@@ -35,6 +35,8 @@ from pathlib import Path
from PIL import Image
+from .page_metrics import CONTENT_BOTTOM_JS
+
logger = logging.getLogger("pixelrag_render.backends.cdp")
VIEWPORT_W = 875
@@ -344,6 +346,7 @@ def _readiness_expr() -> str:
Returns an async-IIFE expression resolving to the page height to tile.
"""
return f"""(async () => {{
+ {CONTENT_BOTTOM_JS}
await new Promise(res => {{
if (document.readyState === 'complete') return res();
const t = setTimeout(res, {LOAD_TIMEOUT_MS});
@@ -361,8 +364,7 @@ def _readiness_expr() -> str:
const sh = document.documentElement.scrollHeight;
const body = document.body;
if (body) {{
- const bottom = Math.ceil(body.getBoundingClientRect().bottom);
- return Math.min(sh, Math.max(bottom, 1));
+ return Math.min(sh, Math.max(contentBottom(body), 1));
}}
return sh;
}})()"""
diff --git a/render/src/pixelrag_render/backends/fast_cdp.py b/render/src/pixelrag_render/backends/fast_cdp.py
index 251195a..844b2b6 100644
--- a/render/src/pixelrag_render/backends/fast_cdp.py
+++ b/render/src/pixelrag_render/backends/fast_cdp.py
@@ -34,6 +34,8 @@ import time
import urllib.request
from pathlib import Path
+from .page_metrics import CONTENT_BOTTOM_JS
+
logger = logging.getLogger("pixelrag_render.backends.fast_cdp")
VIEWPORT_WIDTH = 875
@@ -58,8 +60,12 @@ CHROME_ARGS = [
"--disable-features=Translate,MediaRouter,OptimizationHints",
]
-# JS: wait for fonts + eager images, then return scrollHeight
-_WAIT_FONTS_IMGS = """new Promise(resolve => {
+# JS: wait for fonts + eager images, then return the page height to tile
+_WAIT_FONTS_IMGS = (
+ """new Promise(resolve => {
+ """
+ + CONTENT_BOTTOM_JS
+ + """
const waitEagerImgs = Promise.all(
Array.from(document.images)
.filter(i => !i.complete && i.loading !== 'lazy')
@@ -79,12 +85,13 @@ _WAIT_FONTS_IMGS = """new Promise(resolve => {
const sh = document.documentElement.scrollHeight;
const body = document.body;
resolve(body
- ? Math.min(sh, Math.max(Math.ceil(body.getBoundingClientRect().bottom), 1))
+ ? Math.min(sh, Math.max(contentBottom(body), 1))
: sh);
});
});
});
})"""
+)
# ---------------------------------------------------------------------------
diff --git a/render/src/pixelrag_render/backends/page_metrics.py b/render/src/pixelrag_render/backends/page_metrics.py
new file mode 100644
index 0000000..1800e95
--- /dev/null
+++ b/render/src/pixelrag_render/backends/page_metrics.py
@@ -0,0 +1,44 @@
+"""Shared in-page measurement JS for the capture backends.
+
+Kept in one place because both backends measure page height the same way and
+must agree: the standard (``cdp``) and turbo (``fast_cdp``) paths each embed
+this snippet, and a divergence between them silently changes how much of a page
+gets captured depending on which Chrome binary is installed.
+
+No imports — a plain string constant, so either backend can use it without a
+dependency edge between them.
+"""
+
+# How tall is the document's *content*?
+#
+# `documentElement.scrollHeight` alone over-reports: padding on the root element
+# or a trailing margin inflates it, buying a run of blank tiles at the bottom of
+# every such page. So it is clamped to where the content actually ends.
+#
+# That bound has to be measured from the content, not from the body box. A body
+# is only as tall as the document when the page lets it size to its content;
+# sites that pin it to the viewport (`html, body { height: 100% }` — Wikipedia's
+# Vector 2022 skin among them) leave the content overflowing a one-viewport box,
+# and clamping to that box truncates a 20,000px article to a single tile
+# (issue #124). Taking the lowest edge among the body and its element children
+# reads the same on a self-sizing body and survives a pinned one.
+#
+# Coordinates are viewport-relative, so scroll offset is added back: a page
+# navigated to a `#fragment` lands scrolled down, where a raw rect bottom would
+# under-report by exactly the scrolled distance.
+CONTENT_BOTTOM_JS = """
+ function contentBottom(body) {
+ const offset = window.scrollY || window.pageYOffset || 0;
+ let bottom = body.getBoundingClientRect().bottom;
+ for (let el = body.firstElementChild; el; el = el.nextElementSibling) {
+ const r = el.getBoundingClientRect();
+ // Skip elements with no box at all (display:none, empty <script>);
+ // theirs is a zero rect at the origin and would not move `bottom`,
+ // but skipping keeps the intent explicit.
+ if (r.width > 0 || r.height > 0) {
+ bottom = Math.max(bottom, r.bottom);
+ }
+ }
+ return Math.ceil(bottom + offset);
+ }
+"""
diff --git a/tests/test_render.py b/tests/test_render.py
index f37a379..05b91b3 100644
--- a/tests/test_render.py
+++ b/tests/test_render.py
@@ -6,6 +6,7 @@ This is the one user-facing capability the README promises, so it is tested end
to end rather than mocked.
"""
+import json
from pathlib import Path
from pixelrag_render import render_file
@@ -29,3 +30,36 @@ def test_render_local_html_to_tiles(tmp_path):
f"no tile images produced in {tile_dir} "
f"(contents: {[p.name for p in tile_dir.iterdir()]})"
)
+
+
+def test_page_taller_than_a_viewport_bounded_body_is_fully_tiled(tmp_path):
+ """A page whose body is pinned to the viewport must still tile in full.
+
+ Regression for issue #124. Sites that set ``html, body { height: 100% }``
+ (Wikipedia's Vector 2022 skin among them) leave the article content
+ overflowing the body box visibly, so ``body.getBoundingClientRect()`` is one
+ viewport tall on a 20,000px page. The readiness probe used to clamp the page
+ height to that rect, capturing a single tile and reporting the viewport as
+ the page height.
+ """
+ body = "".join(f"<p>line {i:03d}</p>" for i in range(400))
+ html = tmp_path / "viewport_bounded_body.html"
+ html.write_text(
+ '<!DOCTYPE html><html style="height:100%"><body style="height:100%">'
+ f"{body}</body></html>"
+ )
+ out = tmp_path / "tiles"
+
+ dirs = render_file(html, out, tile_height=1000, viewport_width=1280)
+
+ tile_dir = Path(dirs[0])
+ manifest = json.loads((tile_dir / "tiles.json").read_text())
+ tiles = sorted(tile_dir.glob("tile_*.jpg"))
+
+ # 400 paragraphs are several viewports tall whatever the default font is;
+ # assert against the viewport rather than a brittle exact pixel count.
+ assert manifest["page_height"] > 3000, (
+ f"page_height {manifest['page_height']} is about one viewport — the "
+ "content below the fold was never measured"
+ )
+ assert len(tiles) > 1, f"expected multiple tiles, got {[t.name for t in tiles]}"