https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(render): networkidle2 in-flight wait, replacing the completion-reset probe (#120)
Zhifei Li1ee3d8e18de2
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.pyindex 2e8b9c0..584f4fd 100644--- a/render/src/pixelrag_render/backends/cdp.py+++ b/render/src/pixelrag_render/backends/cdp.py@@ -183,11 +183,149 @@ async def _cdp_send(ws, msg_id_ref: list, method: str, params: dict | None = Non# before giving up and capturing whatever is there. Keeps a hanging page from# stalling a worker.LOAD_TIMEOUT_MS = 12_000-# Network is considered idle once no new resource has been fetched for this long.+# The network is considered idle once at most NET_IDLE_MAX_INFLIGHT requests+# have been in flight for NET_QUIET_MS (Puppeteer's "networkidle2" semantics).+# Tolerating 2 in-flight requests is what makes this usable on arbitrary web+# pages: analytics beacons / long-polling keep 1-2 connections busy forever,+# and a strict zero-in-flight wait would sit at the hard cap on every such+# page. Content loads (SPA hydration, image sets) burst well past 2.NET_QUIET_MS = 500+NET_IDLE_MAX_INFLIGHT = 2-def _readiness_expr(wait_network_idle: bool) -> str:+class _NetIdleState:+ """networkidle2 bookkeeping over raw CDP frames.++ Feed every incoming CDP frame to :meth:`on_frame`; the state tracks the+ set of in-flight HTTP requests (``Network.requestWillBeSent`` opens one,+ ``Network.loadingFinished`` / ``Network.loadingFailed`` closes it) and+ whether the page has loaded (``Page.loadEventFired``). WebSocket traffic+ uses different CDP methods and is deliberately not counted, so persistent+ sockets never block readiness.++ The quiet clock runs whenever <= ``max_inflight`` requests are pending —+ including before load — so a page that is already quiet when `load` fires+ becomes ready immediately, with no mandatory post-load wait. Pure and+ clock-injected for testability; the async I/O lives in+ :func:`_wait_load_and_network_idle`.+ """++ _OPEN = "Network.requestWillBeSent"+ _CLOSE = ("Network.loadingFinished", "Network.loadingFailed")++ def __init__(+ self,+ now: float,+ *,+ max_inflight: int = NET_IDLE_MAX_INFLIGHT,+ quiet_ms: int = NET_QUIET_MS,+ ):+ self.max_inflight = max_inflight+ self.quiet_s = quiet_ms / 1000+ self.loaded = False+ self.inflight: set[str] = set()+ self.quiet_since: float | None = now # nothing in flight at start++ def on_frame(self, msg: dict, now: float) -> None:+ method = msg.get("method")+ if method == "Page.loadEventFired":+ self.loaded = True+ elif method == self._OPEN:+ self.inflight.add(msg["params"]["requestId"])+ elif method in self._CLOSE:+ self.inflight.discard(msg["params"]["requestId"])+ else:+ return+ if len(self.inflight) <= self.max_inflight:+ if self.quiet_since is None:+ self.quiet_since = now+ else:+ self.quiet_since = None++ def ready(self, now: float) -> bool:+ return (+ self.loaded+ and self.quiet_since is not None+ and now - self.quiet_since >= self.quiet_s+ )++ def next_deadline(self, now: float) -> float | None:+ """Earliest future moment ready() could flip true, or None (event-bound)."""+ if self.loaded and self.quiet_since is not None:+ return self.quiet_since + self.quiet_s+ return None+++async def _wait_load_and_network_idle(+ ws,+ msg_id_ref: list,+ *,+ max_inflight: int = NET_IDLE_MAX_INFLIGHT,+ quiet_ms: int = NET_QUIET_MS,+ cap_ms: int = LOAD_TIMEOUT_MS,+) -> None:+ """Wait until the page has loaded AND the network is networkidle2-quiet.++ Owns the websocket while it runs (the per-page CDP flow is sequential, so+ there is no competing reader) and feeds every frame to a+ :class:`_NetIdleState`. Returns when the state is ready or after+ ``cap_ms`` regardless, so a page that never settles (>2 permanently busy+ connections) can stall a worker for at most the cap — capture then+ proceeds with whatever is rendered, exactly like the load timeout.++ One frame race is tolerated by design: ``requestWillBeSent`` events emitted+ before this coroutine starts reading (during the ``Page.navigate``+ round-trip) are lost, which can only *under*-count in-flight requests. The+ document request itself is covered by the load gate — an initial+ ``document.readyState`` probe catches pages whose `load` fired before we+ started listening (e.g. instant file:// renders).+ """+ state = _NetIdleState(+ time.monotonic(), max_inflight=max_inflight, quiet_ms=quiet_ms+ )+ cap_deadline = time.monotonic() + cap_ms / 1000++ # readyState probe, sent raw so the frames that arrive while we wait for+ # its response are counted rather than discarded by _cdp_send's loop.+ msg_id_ref[0] += 1+ probe_id = msg_id_ref[0]+ await ws.send(+ json.dumps(+ {+ "id": probe_id,+ "method": "Runtime.evaluate",+ "params": {+ "expression": "document.readyState",+ "returnByValue": True,+ },+ }+ )+ )++ while True:+ now = time.monotonic()+ if state.ready(now) or now >= cap_deadline:+ return+ deadline = state.next_deadline(now)+ timeout = (+ cap_deadline if deadline is None else min(deadline, cap_deadline)+ ) - now+ try:+ frame = await asyncio.wait_for(ws.recv(), timeout=max(timeout, 0.001))+ except asyncio.TimeoutError:+ continue # re-evaluate ready()/cap+ msg = json.loads(frame)+ if msg.get("id") == probe_id:+ try:+ if msg["result"]["result"]["value"] == "complete":+ state.loaded = True+ except (KeyError, TypeError):+ pass+ continue+ state.on_frame(msg, time.monotonic())+++def _readiness_expr() -> str:"""Build the in-page readiness probe.Always waits for the `load` event before measuring (with a@@ -197,38 +335,20 @@ def _readiness_expr(wait_network_idle: bool) -> str:layout — often much taller than the settled page — producing blank tiles. SSRpages (e.g. Wikipedia) fire `load` almost immediately, so this adds ~no cost.- When ``wait_network_idle`` is set, also waits (after load) until no new- resource has been fetched for ``NET_QUIET_MS`` — for SPAs that fetch their- content *after* load. This costs a quiet window (>= NET_QUIET_MS, up to- LOAD_TIMEOUT_MS) per page and disqualifies the turbo capture path, so it- is off by default here; the pixelbrowse skill and the index pipeline's- `web` source (arbitrary external URLs) enable it, while kiwix/localhost- and file:// batch renders stay on the fast path.+ Network-idle waiting is NOT done in-page: it runs Python-side over CDP+ Network events (see :func:`_wait_load_and_network_idle`), which can see+ in-flight requests — a PerformanceObserver only sees completions, so the+ old in-page variant reset its quiet timer on every analytics beacon and+ rode the hard cap on any page with sub-500ms background traffic.Returns an async-IIFE expression resolving to the page height to tile."""- idle_step = ""- if wait_network_idle:- idle_step = f"""- await new Promise(res => {{- let timer;- let obs;- const finish = () => {{ try {{ obs && obs.disconnect(); }} catch (e) {{}}- clearTimeout(timer); clearTimeout(hard); res(); }};- const bump = () => {{ clearTimeout(timer); timer = setTimeout(finish, {NET_QUIET_MS}); }};- try {{- obs = new PerformanceObserver(bump);- obs.observe({{ type: 'resource', buffered: true }});- }} catch (e) {{}}- const hard = setTimeout(finish, {LOAD_TIMEOUT_MS});- bump();- }});"""return f"""(async () => {{await new Promise(res => {{if (document.readyState === 'complete') return res();const t = setTimeout(res, {LOAD_TIMEOUT_MS});window.addEventListener('load', () => {{ clearTimeout(t); res(); }}, {{ once: true }});- }});{idle_step}+ }});await document.fonts.ready;// Let layout settle over two frames — but cap it: requestAnimationFrame// never ticks in some headless modes (e.g. google-chrome --headless=new@@ -295,14 +415,22 @@ async def capture_url(await _cdp_send(ws, msg_id_ref, "Page.navigate", {"url": url})- # Wait for load (+ optional network-idle) + fonts + layout to stabilize,- # return the page height to tile in one call. See _readiness_expr.+ # Optional networkidle2 wait, done Python-side over CDP Network events so+ # it can see in-flight requests (and therefore tolerate persistent+ # analytics/long-poll connections). Must run before the in-page probe:+ # frames arriving during a _cdp_send round-trip are discarded, so the+ # watcher has to be the socket's reader while the page settles.+ if wait_network_idle:+ await _wait_load_and_network_idle(ws, msg_id_ref)++ # Wait for load + fonts + layout to stabilize, return the page height to+ # tile in one call. See _readiness_expr.result = await _cdp_send(ws,msg_id_ref,"Runtime.evaluate",{- "expression": _readiness_expr(wait_network_idle),+ "expression": _readiness_expr(),"awaitPromise": True,"returnByValue": True,},@@ -389,8 +517,9 @@ async def _setup_page("""Enable the CDP domains and fix the viewport for a page ws before capture."""await _cdp_send(ws, msg_id_ref, "Page.enable")if wait_network_idle:- # PerformanceObserver (used by the idle wait) needs no CDP domain, but- # enabling Network keeps resource timing reliable across navigations.+ # The networkidle2 watcher (_wait_load_and_network_idle) counts+ # Network.requestWillBeSent / loadingFinished / loadingFailed events;+ # without this domain enabled Chrome emits none of them.await _cdp_send(ws, msg_id_ref, "Network.enable")await _cdp_send(ws,@@ -811,6 +940,13 @@ def render_urls(or wait_network_idleor not from_surface):+ if wait_network_idle:+ # Be loud about the trade: users with a turbo-capable Chrome would+ # otherwise silently lose ~half their capture throughput.+ logger.info(+ "wait_network_idle is set: using the standard capture path "+ "(the turbo path does not implement the idle wait yet)"+ )use_turbo = Falseif use_turbo:diff --git a/render/src/pixelrag_render/render.py b/render/src/pixelrag_render/render.pyindex 722f587..ee76afb 100644--- a/render/src/pixelrag_render/render.py+++ b/render/src/pixelrag_render/render.py@@ -284,10 +284,12 @@ def main() -> None:parser.add_argument("--wait-network-idle",action="store_true",- help="After the page's load event, also wait until the network is quiet "- "(~500ms) before capturing. Helps JS/SPA pages that fetch content after "- "load; adds a quiet window per page, so off by default. Recommended for "- "single-page renders (e.g. the pixelbrowse skill).",+ help="After the page's load event, also wait until at most 2 network "+ "requests have been in flight for ~500ms (networkidle2) before "+ "capturing, capped at 12s. Helps JS/SPA pages that fetch content "+ "after load; tolerates persistent analytics/long-poll connections. "+ "Off by default; the index pipeline's `web` source and the "+ "pixelbrowse skill enable it.",)parser.add_argument("--dpi",diff --git a/tests/test_network_idle.py b/tests/test_network_idle.pynew file mode 100644index 0000000..e7b9e27--- /dev/null+++ b/tests/test_network_idle.py@@ -0,0 +1,192 @@+"""networkidle2 readiness: in-flight tolerance, load gating, and the hard cap.++The old in-page probe reset its quiet timer on every resource *completion*,+so any page with sub-500ms analytics beacons rode the 12s hard cap. The+Python-side watcher counts *in-flight* requests instead (Puppeteer's+networkidle2 semantics): <=2 pending requests count as quiet, persistent+connections never block, and >2 permanently-busy connections still exit at+the cap rather than hanging.+"""++import asyncio+import json+import time++from pixelrag_render.backends.cdp import (+ _NetIdleState,+ _wait_load_and_network_idle,+)+++def _open(state, rid, now):+ state.on_frame(+ {"method": "Network.requestWillBeSent", "params": {"requestId": rid}}, now+ )+++def _close(state, rid, now):+ state.on_frame(+ {"method": "Network.loadingFinished", "params": {"requestId": rid}}, now+ )+++def _loaded(state, now):+ state.on_frame({"method": "Page.loadEventFired", "params": {}}, now)+++def test_quiet_page_ready_immediately_after_load():+ # Subresources finished well before load: no mandatory post-load wait.+ s = _NetIdleState(now=0.0)+ _open(s, "doc", 0.0)+ _close(s, "doc", 0.1)+ _loaded(s, 0.8)+ assert s.ready(0.8), "already-quiet page must be ready the moment load fires"+++def test_not_ready_before_load_even_when_quiet():+ s = _NetIdleState(now=0.0)+ assert not s.ready(5.0), "quiet alone is not readiness — load must have fired"+++def test_spa_hydration_burst_blocks_then_clears():+ s = _NetIdleState(now=0.0)+ _loaded(s, 0.5)+ for i in range(5): # hydration burst: 5 concurrent fetches+ _open(s, f"xhr{i}", 0.6)+ assert not s.ready(2.0), ">2 in-flight must hold readiness open"+ for i in range(3): # down to 2 pending -> quiet clock restarts+ _close(s, f"xhr{i}", 2.0)+ assert not s.ready(2.4), "quiet window must elapse after the burst clears"+ assert s.ready(2.51), "<=2 in-flight for quiet_ms is ready (networkidle2)"+++def test_persistent_connections_are_tolerated():+ # The defining networkidle2 case: 2 long-poll/analytics connections that+ # never finish. The old completion-based probe never went quiet here.+ s = _NetIdleState(now=0.0)+ _open(s, "longpoll", 0.0)+ _open(s, "analytics", 0.0)+ _loaded(s, 0.3)+ assert s.ready(0.9), "2 permanently-open connections must not block"+++def test_frequent_beacon_completions_do_not_reset_quiet():+ # Beacons completing every 200ms: completions used to reset the timer;+ # in-flight counting stays <=2 throughout, so the quiet clock never stops.+ s = _NetIdleState(now=0.0)+ _loaded(s, 0.0)+ t = 0.0+ for i in range(10):+ _open(s, f"beacon{i}", t)+ _close(s, f"beacon{i}", t + 0.05)+ t += 0.2+ assert s.ready(0.51), "sub-500ms beacon traffic must not defeat the quiet window"+++class _FakeWs:+ """Minimal CDP websocket: scripted frames with delays, echoes probe replies."""++ def __init__(self, script):+ # script: list of (delay_s, frame_dict) relative to previous frame+ self._queue = asyncio.Queue()+ self._script = script+ self._feeder = None++ async def send(self, raw):+ msg = json.loads(raw)+ if msg.get("id") is not None: # readyState probe -> not complete yet+ await self._queue.put(+ {"id": msg["id"], "result": {"result": {"value": "loading"}}}+ )+ if self._feeder is None:+ self._feeder = asyncio.ensure_future(self._feed())++ async def _feed(self):+ for delay, frame in self._script:+ await asyncio.sleep(delay)+ await self._queue.put(frame)++ async def recv(self):+ return json.dumps(await self._queue.get())+++def _run(coro):+ return asyncio.run(coro)+++def test_watcher_exits_promptly_despite_beacons():+ # Load fires at 100ms, then beacons complete every 100ms forever (well,+ # 30 of them). Old behavior: wait the full cap. New: ready ~600ms.+ script = [(0.1, {"method": "Page.loadEventFired", "params": {}})]+ for i in range(30):+ script.append(+ (+ 0.05,+ {+ "method": "Network.requestWillBeSent",+ "params": {"requestId": f"b{i}"},+ },+ )+ )+ script.append(+ (+ 0.05,+ {+ "method": "Network.loadingFinished",+ "params": {"requestId": f"b{i}"},+ },+ )+ )++ async def scenario():+ ws = _FakeWs(script)+ t0 = time.monotonic()+ await _wait_load_and_network_idle(ws, [0], cap_ms=12_000)+ return time.monotonic() - t0++ elapsed = _run(scenario())+ assert elapsed < 3.0, f"beacon page must not ride the 12s cap (took {elapsed:.2f}s)"+++def test_watcher_respects_hard_cap_when_never_quiet():+ # 5 requests open immediately and never finish: must exit at the cap,+ # not hang.+ script = [(0.0, {"method": "Page.loadEventFired", "params": {}})] + [+ (+ 0.0,+ {"method": "Network.requestWillBeSent", "params": {"requestId": f"r{i}"}},+ )+ for i in range(5)+ ]++ async def scenario():+ ws = _FakeWs(script)+ t0 = time.monotonic()+ await _wait_load_and_network_idle(ws, [0], cap_ms=1_000)+ return time.monotonic() - t0++ elapsed = _run(scenario())+ assert 0.9 <= elapsed < 2.0, f"expected ~1s cap exit, took {elapsed:.2f}s"+++def test_watcher_handles_preloaded_page_via_readystate_probe():+ # load fired before the watcher attached (instant file:// render): the+ # readyState probe must catch it; no Page.loadEventFired ever arrives.+ class _CompleteWs(_FakeWs):+ async def send(self, raw):+ msg = json.loads(raw)+ if msg.get("id") is not None:+ await self._queue.put(+ {"id": msg["id"], "result": {"result": {"value": "complete"}}}+ )++ async def scenario():+ ws = _CompleteWs([])+ t0 = time.monotonic()+ await _wait_load_and_network_idle(ws, [0], cap_ms=5_000)+ return time.monotonic() - t0++ elapsed = _run(scenario())+ assert elapsed < 1.5, (+ f"preloaded page must exit after one quiet window ({elapsed:.2f}s)"+ )