Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat(render): networkidle2 in-flight wait, replacing the completion-reset probe (#120)

Zhifei Li1ee3d8e18de2
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.py
index 2e8b9c0..584f4fd 100644
--- a/render/src/pixelrag_render/backends/cdp.py
+++ b/render/src/pixelrag_render/backends/cdp.py
@@ -183,11 +183,149 @@ async def _cdp_send(ws, msg_id_ref: list, method: str, params: dict | None = Non
# before giving up and capturing whatever is there. Keeps a hanging page from
# stalling a worker.
LOAD_TIMEOUT_MS = 12_000
-# Network is considered idle once no new resource has been fetched for this long.
+# The network is considered idle once at most NET_IDLE_MAX_INFLIGHT requests
+# have been in flight for NET_QUIET_MS (Puppeteer's "networkidle2" semantics).
+# Tolerating 2 in-flight requests is what makes this usable on arbitrary web
+# pages: analytics beacons / long-polling keep 1-2 connections busy forever,
+# and a strict zero-in-flight wait would sit at the hard cap on every such
+# page. Content loads (SPA hydration, image sets) burst well past 2.
NET_QUIET_MS = 500
+NET_IDLE_MAX_INFLIGHT = 2
-def _readiness_expr(wait_network_idle: bool) -> str:
+class _NetIdleState:
+ """networkidle2 bookkeeping over raw CDP frames.
+
+ Feed every incoming CDP frame to :meth:`on_frame`; the state tracks the
+ set of in-flight HTTP requests (``Network.requestWillBeSent`` opens one,
+ ``Network.loadingFinished`` / ``Network.loadingFailed`` closes it) and
+ whether the page has loaded (``Page.loadEventFired``). WebSocket traffic
+ uses different CDP methods and is deliberately not counted, so persistent
+ sockets never block readiness.
+
+ The quiet clock runs whenever <= ``max_inflight`` requests are pending —
+ including before load — so a page that is already quiet when `load` fires
+ becomes ready immediately, with no mandatory post-load wait. Pure and
+ clock-injected for testability; the async I/O lives in
+ :func:`_wait_load_and_network_idle`.
+ """
+
+ _OPEN = "Network.requestWillBeSent"
+ _CLOSE = ("Network.loadingFinished", "Network.loadingFailed")
+
+ def __init__(
+ self,
+ now: float,
+ *,
+ max_inflight: int = NET_IDLE_MAX_INFLIGHT,
+ quiet_ms: int = NET_QUIET_MS,
+ ):
+ self.max_inflight = max_inflight
+ self.quiet_s = quiet_ms / 1000
+ self.loaded = False
+ self.inflight: set[str] = set()
+ self.quiet_since: float | None = now # nothing in flight at start
+
+ def on_frame(self, msg: dict, now: float) -> None:
+ method = msg.get("method")
+ if method == "Page.loadEventFired":
+ self.loaded = True
+ elif method == self._OPEN:
+ self.inflight.add(msg["params"]["requestId"])
+ elif method in self._CLOSE:
+ self.inflight.discard(msg["params"]["requestId"])
+ else:
+ return
+ if len(self.inflight) <= self.max_inflight:
+ if self.quiet_since is None:
+ self.quiet_since = now
+ else:
+ self.quiet_since = None
+
+ def ready(self, now: float) -> bool:
+ return (
+ self.loaded
+ and self.quiet_since is not None
+ and now - self.quiet_since >= self.quiet_s
+ )
+
+ def next_deadline(self, now: float) -> float | None:
+ """Earliest future moment ready() could flip true, or None (event-bound)."""
+ if self.loaded and self.quiet_since is not None:
+ return self.quiet_since + self.quiet_s
+ return None
+
+
+async def _wait_load_and_network_idle(
+ ws,
+ msg_id_ref: list,
+ *,
+ max_inflight: int = NET_IDLE_MAX_INFLIGHT,
+ quiet_ms: int = NET_QUIET_MS,
+ cap_ms: int = LOAD_TIMEOUT_MS,
+) -> None:
+ """Wait until the page has loaded AND the network is networkidle2-quiet.
+
+ Owns the websocket while it runs (the per-page CDP flow is sequential, so
+ there is no competing reader) and feeds every frame to a
+ :class:`_NetIdleState`. Returns when the state is ready or after
+ ``cap_ms`` regardless, so a page that never settles (>2 permanently busy
+ connections) can stall a worker for at most the cap — capture then
+ proceeds with whatever is rendered, exactly like the load timeout.
+
+ One frame race is tolerated by design: ``requestWillBeSent`` events emitted
+ before this coroutine starts reading (during the ``Page.navigate``
+ round-trip) are lost, which can only *under*-count in-flight requests. The
+ document request itself is covered by the load gate — an initial
+ ``document.readyState`` probe catches pages whose `load` fired before we
+ started listening (e.g. instant file:// renders).
+ """
+ state = _NetIdleState(
+ time.monotonic(), max_inflight=max_inflight, quiet_ms=quiet_ms
+ )
+ cap_deadline = time.monotonic() + cap_ms / 1000
+
+ # readyState probe, sent raw so the frames that arrive while we wait for
+ # its response are counted rather than discarded by _cdp_send's loop.
+ msg_id_ref[0] += 1
+ probe_id = msg_id_ref[0]
+ await ws.send(
+ json.dumps(
+ {
+ "id": probe_id,
+ "method": "Runtime.evaluate",
+ "params": {
+ "expression": "document.readyState",
+ "returnByValue": True,
+ },
+ }
+ )
+ )
+
+ while True:
+ now = time.monotonic()
+ if state.ready(now) or now >= cap_deadline:
+ return
+ deadline = state.next_deadline(now)
+ timeout = (
+ cap_deadline if deadline is None else min(deadline, cap_deadline)
+ ) - now
+ try:
+ frame = await asyncio.wait_for(ws.recv(), timeout=max(timeout, 0.001))
+ except asyncio.TimeoutError:
+ continue # re-evaluate ready()/cap
+ msg = json.loads(frame)
+ if msg.get("id") == probe_id:
+ try:
+ if msg["result"]["result"]["value"] == "complete":
+ state.loaded = True
+ except (KeyError, TypeError):
+ pass
+ continue
+ state.on_frame(msg, time.monotonic())
+
+
+def _readiness_expr() -> str:
"""Build the in-page readiness probe.
Always waits for the `load` event before measuring (with a
@@ -197,38 +335,20 @@ def _readiness_expr(wait_network_idle: bool) -> str:
layout — often much taller than the settled page — producing blank tiles. SSR
pages (e.g. Wikipedia) fire `load` almost immediately, so this adds ~no cost.
- When ``wait_network_idle`` is set, also waits (after load) until no new
- resource has been fetched for ``NET_QUIET_MS`` — for SPAs that fetch their
- content *after* load. This costs a quiet window (>= NET_QUIET_MS, up to
- LOAD_TIMEOUT_MS) per page and disqualifies the turbo capture path, so it
- is off by default here; the pixelbrowse skill and the index pipeline's
- `web` source (arbitrary external URLs) enable it, while kiwix/localhost
- and file:// batch renders stay on the fast path.
+ Network-idle waiting is NOT done in-page: it runs Python-side over CDP
+ Network events (see :func:`_wait_load_and_network_idle`), which can see
+ in-flight requests — a PerformanceObserver only sees completions, so the
+ old in-page variant reset its quiet timer on every analytics beacon and
+ rode the hard cap on any page with sub-500ms background traffic.
Returns an async-IIFE expression resolving to the page height to tile.
"""
- idle_step = ""
- if wait_network_idle:
- idle_step = f"""
- await new Promise(res => {{
- let timer;
- let obs;
- const finish = () => {{ try {{ obs && obs.disconnect(); }} catch (e) {{}}
- clearTimeout(timer); clearTimeout(hard); res(); }};
- const bump = () => {{ clearTimeout(timer); timer = setTimeout(finish, {NET_QUIET_MS}); }};
- try {{
- obs = new PerformanceObserver(bump);
- obs.observe({{ type: 'resource', buffered: true }});
- }} catch (e) {{}}
- const hard = setTimeout(finish, {LOAD_TIMEOUT_MS});
- bump();
- }});"""
return f"""(async () => {{
await new Promise(res => {{
if (document.readyState === 'complete') return res();
const t = setTimeout(res, {LOAD_TIMEOUT_MS});
window.addEventListener('load', () => {{ clearTimeout(t); res(); }}, {{ once: true }});
- }});{idle_step}
+ }});
await document.fonts.ready;
// Let layout settle over two frames — but cap it: requestAnimationFrame
// never ticks in some headless modes (e.g. google-chrome --headless=new
@@ -295,14 +415,22 @@ async def capture_url(
await _cdp_send(ws, msg_id_ref, "Page.navigate", {"url": url})
- # Wait for load (+ optional network-idle) + fonts + layout to stabilize,
- # return the page height to tile in one call. See _readiness_expr.
+ # Optional networkidle2 wait, done Python-side over CDP Network events so
+ # it can see in-flight requests (and therefore tolerate persistent
+ # analytics/long-poll connections). Must run before the in-page probe:
+ # frames arriving during a _cdp_send round-trip are discarded, so the
+ # watcher has to be the socket's reader while the page settles.
+ if wait_network_idle:
+ await _wait_load_and_network_idle(ws, msg_id_ref)
+
+ # Wait for load + fonts + layout to stabilize, return the page height to
+ # tile in one call. See _readiness_expr.
result = await _cdp_send(
ws,
msg_id_ref,
"Runtime.evaluate",
{
- "expression": _readiness_expr(wait_network_idle),
+ "expression": _readiness_expr(),
"awaitPromise": True,
"returnByValue": True,
},
@@ -389,8 +517,9 @@ async def _setup_page(
"""Enable the CDP domains and fix the viewport for a page ws before capture."""
await _cdp_send(ws, msg_id_ref, "Page.enable")
if wait_network_idle:
- # PerformanceObserver (used by the idle wait) needs no CDP domain, but
- # enabling Network keeps resource timing reliable across navigations.
+ # The networkidle2 watcher (_wait_load_and_network_idle) counts
+ # Network.requestWillBeSent / loadingFinished / loadingFailed events;
+ # without this domain enabled Chrome emits none of them.
await _cdp_send(ws, msg_id_ref, "Network.enable")
await _cdp_send(
ws,
@@ -811,6 +940,13 @@ def render_urls(
or wait_network_idle
or not from_surface
):
+ if wait_network_idle:
+ # Be loud about the trade: users with a turbo-capable Chrome would
+ # otherwise silently lose ~half their capture throughput.
+ logger.info(
+ "wait_network_idle is set: using the standard capture path "
+ "(the turbo path does not implement the idle wait yet)"
+ )
use_turbo = False
if use_turbo:
diff --git a/render/src/pixelrag_render/render.py b/render/src/pixelrag_render/render.py
index 722f587..ee76afb 100644
--- a/render/src/pixelrag_render/render.py
+++ b/render/src/pixelrag_render/render.py
@@ -284,10 +284,12 @@ def main() -> None:
parser.add_argument(
"--wait-network-idle",
action="store_true",
- help="After the page's load event, also wait until the network is quiet "
- "(~500ms) before capturing. Helps JS/SPA pages that fetch content after "
- "load; adds a quiet window per page, so off by default. Recommended for "
- "single-page renders (e.g. the pixelbrowse skill).",
+ help="After the page's load event, also wait until at most 2 network "
+ "requests have been in flight for ~500ms (networkidle2) before "
+ "capturing, capped at 12s. Helps JS/SPA pages that fetch content "
+ "after load; tolerates persistent analytics/long-poll connections. "
+ "Off by default; the index pipeline's `web` source and the "
+ "pixelbrowse skill enable it.",
)
parser.add_argument(
"--dpi",
diff --git a/tests/test_network_idle.py b/tests/test_network_idle.py
new file mode 100644
index 0000000..e7b9e27
--- /dev/null
+++ b/tests/test_network_idle.py
@@ -0,0 +1,192 @@
+"""networkidle2 readiness: in-flight tolerance, load gating, and the hard cap.
+
+The old in-page probe reset its quiet timer on every resource *completion*,
+so any page with sub-500ms analytics beacons rode the 12s hard cap. The
+Python-side watcher counts *in-flight* requests instead (Puppeteer's
+networkidle2 semantics): <=2 pending requests count as quiet, persistent
+connections never block, and >2 permanently-busy connections still exit at
+the cap rather than hanging.
+"""
+
+import asyncio
+import json
+import time
+
+from pixelrag_render.backends.cdp import (
+ _NetIdleState,
+ _wait_load_and_network_idle,
+)
+
+
+def _open(state, rid, now):
+ state.on_frame(
+ {"method": "Network.requestWillBeSent", "params": {"requestId": rid}}, now
+ )
+
+
+def _close(state, rid, now):
+ state.on_frame(
+ {"method": "Network.loadingFinished", "params": {"requestId": rid}}, now
+ )
+
+
+def _loaded(state, now):
+ state.on_frame({"method": "Page.loadEventFired", "params": {}}, now)
+
+
+def test_quiet_page_ready_immediately_after_load():
+ # Subresources finished well before load: no mandatory post-load wait.
+ s = _NetIdleState(now=0.0)
+ _open(s, "doc", 0.0)
+ _close(s, "doc", 0.1)
+ _loaded(s, 0.8)
+ assert s.ready(0.8), "already-quiet page must be ready the moment load fires"
+
+
+def test_not_ready_before_load_even_when_quiet():
+ s = _NetIdleState(now=0.0)
+ assert not s.ready(5.0), "quiet alone is not readiness — load must have fired"
+
+
+def test_spa_hydration_burst_blocks_then_clears():
+ s = _NetIdleState(now=0.0)
+ _loaded(s, 0.5)
+ for i in range(5): # hydration burst: 5 concurrent fetches
+ _open(s, f"xhr{i}", 0.6)
+ assert not s.ready(2.0), ">2 in-flight must hold readiness open"
+ for i in range(3): # down to 2 pending -> quiet clock restarts
+ _close(s, f"xhr{i}", 2.0)
+ assert not s.ready(2.4), "quiet window must elapse after the burst clears"
+ assert s.ready(2.51), "<=2 in-flight for quiet_ms is ready (networkidle2)"
+
+
+def test_persistent_connections_are_tolerated():
+ # The defining networkidle2 case: 2 long-poll/analytics connections that
+ # never finish. The old completion-based probe never went quiet here.
+ s = _NetIdleState(now=0.0)
+ _open(s, "longpoll", 0.0)
+ _open(s, "analytics", 0.0)
+ _loaded(s, 0.3)
+ assert s.ready(0.9), "2 permanently-open connections must not block"
+
+
+def test_frequent_beacon_completions_do_not_reset_quiet():
+ # Beacons completing every 200ms: completions used to reset the timer;
+ # in-flight counting stays <=2 throughout, so the quiet clock never stops.
+ s = _NetIdleState(now=0.0)
+ _loaded(s, 0.0)
+ t = 0.0
+ for i in range(10):
+ _open(s, f"beacon{i}", t)
+ _close(s, f"beacon{i}", t + 0.05)
+ t += 0.2
+ assert s.ready(0.51), "sub-500ms beacon traffic must not defeat the quiet window"
+
+
+class _FakeWs:
+ """Minimal CDP websocket: scripted frames with delays, echoes probe replies."""
+
+ def __init__(self, script):
+ # script: list of (delay_s, frame_dict) relative to previous frame
+ self._queue = asyncio.Queue()
+ self._script = script
+ self._feeder = None
+
+ async def send(self, raw):
+ msg = json.loads(raw)
+ if msg.get("id") is not None: # readyState probe -> not complete yet
+ await self._queue.put(
+ {"id": msg["id"], "result": {"result": {"value": "loading"}}}
+ )
+ if self._feeder is None:
+ self._feeder = asyncio.ensure_future(self._feed())
+
+ async def _feed(self):
+ for delay, frame in self._script:
+ await asyncio.sleep(delay)
+ await self._queue.put(frame)
+
+ async def recv(self):
+ return json.dumps(await self._queue.get())
+
+
+def _run(coro):
+ return asyncio.run(coro)
+
+
+def test_watcher_exits_promptly_despite_beacons():
+ # Load fires at 100ms, then beacons complete every 100ms forever (well,
+ # 30 of them). Old behavior: wait the full cap. New: ready ~600ms.
+ script = [(0.1, {"method": "Page.loadEventFired", "params": {}})]
+ for i in range(30):
+ script.append(
+ (
+ 0.05,
+ {
+ "method": "Network.requestWillBeSent",
+ "params": {"requestId": f"b{i}"},
+ },
+ )
+ )
+ script.append(
+ (
+ 0.05,
+ {
+ "method": "Network.loadingFinished",
+ "params": {"requestId": f"b{i}"},
+ },
+ )
+ )
+
+ async def scenario():
+ ws = _FakeWs(script)
+ t0 = time.monotonic()
+ await _wait_load_and_network_idle(ws, [0], cap_ms=12_000)
+ return time.monotonic() - t0
+
+ elapsed = _run(scenario())
+ assert elapsed < 3.0, f"beacon page must not ride the 12s cap (took {elapsed:.2f}s)"
+
+
+def test_watcher_respects_hard_cap_when_never_quiet():
+ # 5 requests open immediately and never finish: must exit at the cap,
+ # not hang.
+ script = [(0.0, {"method": "Page.loadEventFired", "params": {}})] + [
+ (
+ 0.0,
+ {"method": "Network.requestWillBeSent", "params": {"requestId": f"r{i}"}},
+ )
+ for i in range(5)
+ ]
+
+ async def scenario():
+ ws = _FakeWs(script)
+ t0 = time.monotonic()
+ await _wait_load_and_network_idle(ws, [0], cap_ms=1_000)
+ return time.monotonic() - t0
+
+ elapsed = _run(scenario())
+ assert 0.9 <= elapsed < 2.0, f"expected ~1s cap exit, took {elapsed:.2f}s"
+
+
+def test_watcher_handles_preloaded_page_via_readystate_probe():
+ # load fired before the watcher attached (instant file:// render): the
+ # readyState probe must catch it; no Page.loadEventFired ever arrives.
+ class _CompleteWs(_FakeWs):
+ async def send(self, raw):
+ msg = json.loads(raw)
+ if msg.get("id") is not None:
+ await self._queue.put(
+ {"id": msg["id"], "result": {"result": {"value": "complete"}}}
+ )
+
+ async def scenario():
+ ws = _CompleteWs([])
+ t0 = time.monotonic()
+ await _wait_load_and_network_idle(ws, [0], cap_ms=5_000)
+ return time.monotonic() - t0
+
+ elapsed = _run(scenario())
+ assert elapsed < 1.5, (
+ f"preloaded page must exit after one quiet window ({elapsed:.2f}s)"
+ )