Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(smoke): wait for navigation loader readiness before evaluation
ldm0b05dbacc3554
diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.py b/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.pyindex 4b4e9136a..66d6a8d9c 100644--- a/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.py+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.py@@ -1,8 +1,12 @@from __future__ import annotations+import asynciofrom typing import Anyfrom urllib.parse import urlsplitfrom . import SmokeState-from ..assertions import SmokeError, wait_until+from ..assertions import SmokeError+++_NAVIGATION_TIMEOUT_SECONDS = 10.0def _alternate_loopback_origin(url: str) -> str:@@ -18,31 +22,53 @@ async def _performance_metrics(cdp: Any) -> dict[str, float]:return {metric.get("name"): metric.get("value") for metric in metrics if metric.get("name")}async def _navigate_with_cdp_until_dom_ready(state: SmokeState, url: str) -> None:- result = await state.cdp.send("Page.navigate", {"url": url})- if result.get("errorText"):- raise SmokeError(f"Page.navigate failed for {url}: {result}")-- async def is_ready() -> bool:- ready_state = await state.cdp.send(- "Runtime.evaluate",- {- "expression": "document.readyState",- "returnByValue": True,- },- )- location = await state.cdp.send(- "Runtime.evaluate",- {- "expression": "location.href",+ ready: set[tuple[str, Any, Any]] = set()+ changed = asyncio.Event()++ def lifecycle(params: dict[str, Any]) -> None:+ if params.get("name") == "DOMContentLoaded":+ ready.add(("lifecycle", params.get("frameId"), params.get("loaderId")))+ changed.set()++ def within_document(params: dict[str, Any]) -> None:+ ready.add(("within-document", params.get("frameId"), params.get("url")))+ changed.set()++ state.cdp.on("Page.lifecycleEvent", lifecycle)+ state.cdp.on("Page.navigatedWithinDocument", within_document)+ phase = "enabling Page lifecycle events"+ try:+ async with asyncio.timeout(_NAVIGATION_TIMEOUT_SECONDS):+ await state.cdp.send("Page.enable")+ await state.cdp.send("Page.setLifecycleEventsEnabled", {"enabled": True})+ phase = "Page.navigate"+ result = await state.cdp.send("Page.navigate", {"url": url})+ if result.get("errorText"):+ raise SmokeError(f"Page.navigate failed for {url}: {result}")+ frame, loader = result["frameId"], result.get("loaderId")+ phase = f"DOM ready for frame={frame}, loader={loader}"++ # A lifecycle event can precede the navigate reply. Keep those+ # events, but reject old loaders and child-frame notifications.+ expected = (("lifecycle", frame, loader) if loader is not None+ else ("within-document", frame, url))+ while expected not in ready:+ changed.clear()+ await changed.wait()+ phase = "Runtime.evaluate after DOM ready"+ snapshot = await state.cdp.send("Runtime.evaluate", {+ "expression": "({url: location.href, readyState: document.readyState})","returnByValue": True,- },- )- return (- location.get("result", {}).get("value") == url- and ready_state.get("result", {}).get("value") in ["interactive", "complete"]- )-- await wait_until(is_ready, f"CDP navigation DOM ready for {url}")+ })+ value = snapshot.get("result", {}).get("value")+ if (not isinstance(value, dict) or value.get("url") != url+ or value.get("readyState") not in ("interactive", "complete")):+ raise SmokeError(f"Unexpected document after CDP navigation to {url}: {snapshot}")+ except TimeoutError as error:+ raise SmokeError(f"CDP navigation timed out during {phase}: {url}") from error+ finally:+ state.cdp.remove_listener("Page.lifecycleEvent", lifecycle)+ state.cdp.remove_listener("Page.navigatedWithinDocument", within_document)async def _send_cdp_expect_optional_error(cdp: Any, method: str, params: dict[str, Any]) -> dict[str, Any] | None:try:diff --git a/moli-cdp-smoke/tests/test_chromium_navigation.py b/moli-cdp-smoke/tests/test_chromium_navigation.pynew file mode 100644index 000000000..b4fc374ad--- /dev/null+++ b/moli-cdp-smoke/tests/test_chromium_navigation.py@@ -0,0 +1,119 @@+from __future__ import annotations++import asyncio+import unittest+from types import SimpleNamespace+from typing import Any+from unittest.mock import patch++from moli_cdp_smoke.assertions import SmokeError+from moli_cdp_smoke.groups import chromium_cdp_support as navigation+++URL = "http://fixture.test/plain?requested"+FRAME = "main-frame"+LOADER = "requested-loader"+++class _NavigationSession:+ def __init__(self, *, early: bool = False, complete: bool = True,+ same_document: bool = False, error: str | None = None,+ block_method: str | None = None) -> None:+ self.early = early+ self.complete = complete+ self.same_document = same_document+ self.error = error+ self.block_method = block_method+ self.listeners: dict[str, list[Any]] = {}+ self.commands: list[str] = []+ self.ready = False++ def on(self, method: str, listener: Any) -> None:+ self.listeners.setdefault(method, []).append(listener)++ def remove_listener(self, method: str, listener: Any) -> None:+ self.listeners[method].remove(listener)++ def emit(self, method: str, params: dict[str, Any]) -> None:+ for listener in self.listeners.get(method, []):+ listener(params)++ def document_ready(self) -> None:+ self.ready = True+ if self.same_document:+ self.emit("Page.navigatedWithinDocument", {"frameId": FRAME, "url": URL})+ else:+ self.emit("Page.lifecycleEvent", {+ "frameId": FRAME, "loaderId": LOADER, "name": "DOMContentLoaded",+ })++ async def send(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]:+ self.commands.append(method)+ if method == self.block_method:+ await asyncio.Event().wait()+ if method == "Page.setLifecycleEventsEnabled":+ self.emit("Page.lifecycleEvent", {+ "frameId": FRAME, "loaderId": "old-loader", "name": "DOMContentLoaded",+ })+ if method == "Page.navigate":+ if self.error:+ return {"frameId": FRAME, "errorText": self.error}+ self.emit("Page.lifecycleEvent", {+ "frameId": "child-frame", "loaderId": LOADER, "name": "DOMContentLoaded",+ })+ self.emit("Page.lifecycleEvent", {+ "frameId": FRAME, "loaderId": "old-loader", "name": "DOMContentLoaded",+ })+ if self.complete:+ if self.early:+ self.document_ready()+ else:+ asyncio.get_running_loop().call_soon(self.document_ready)+ return {"frameId": FRAME, **({} if self.same_document else {"loaderId": LOADER})}+ if method == "Runtime.evaluate":+ if not self.ready:+ raise AssertionError("evaluated while the requested loader was still loading")+ return {"result": {"value": {"url": URL, "readyState": "interactive"}}}+ return {}+++class ChromiumNavigationTests(unittest.IsolatedAsyncioTestCase):+ async def test_does_not_evaluate_until_requested_loader_dom_ready(self) -> None:+ session = _NavigationSession()+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)+ self.assertEqual(session.commands.count("Runtime.evaluate"), 1)+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))++ async def test_dom_ready_before_navigate_reply_is_retained(self) -> None:+ session = _NavigationSession(early=True)+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)+ self.assertEqual(session.commands.count("Runtime.evaluate"), 1)++ async def test_same_document_navigation_uses_its_frame_and_url(self) -> None:+ session = _NavigationSession(same_document=True)+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)+ self.assertEqual(session.commands.count("Runtime.evaluate"), 1)++ async def test_stale_and_child_events_cannot_satisfy_the_navigation(self) -> None:+ session = _NavigationSession(complete=False)+ with patch.object(navigation, "_NAVIGATION_TIMEOUT_SECONDS", 0.01):+ with self.assertRaisesRegex(SmokeError, "requested-loader"):+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)+ self.assertNotIn("Runtime.evaluate", session.commands)+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))++ async def test_navigation_error_is_not_replaced_by_a_readiness_timeout(self) -> None:+ session = _NavigationSession(error="net::ERR_ABORTED")+ with self.assertRaisesRegex(SmokeError, "net::ERR_ABORTED"):+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)+ self.assertNotIn("Runtime.evaluate", session.commands)+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))++ async def test_pending_cdp_commands_respect_the_navigation_deadline(self) -> None:+ for method in ("Page.navigate", "Runtime.evaluate"):+ with self.subTest(method=method):+ session = _NavigationSession(block_method=method)+ with patch.object(navigation, "_NAVIGATION_TIMEOUT_SECONDS", 0.01):+ with self.assertRaisesRegex(SmokeError, method):+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))