Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(smoke): wait for navigation loader readiness before evaluation

ldm0b05dbacc3554
diff --git a/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.py b/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.py
index 4b4e9136a..66d6a8d9c 100644
--- a/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.py
+++ b/moli-cdp-smoke/moli_cdp_smoke/groups/chromium_cdp_support.py
@@ -1,8 +1,12 @@
from __future__ import annotations
+import asyncio
from typing import Any
from urllib.parse import urlsplit
from . import SmokeState
-from ..assertions import SmokeError, wait_until
+from ..assertions import SmokeError
+
+
+_NAVIGATION_TIMEOUT_SECONDS = 10.0
def _alternate_loopback_origin(url: str) -> str:
@@ -18,31 +22,53 @@ async def _performance_metrics(cdp: Any) -> dict[str, float]:
return {metric.get("name"): metric.get("value") for metric in metrics if metric.get("name")}
async def _navigate_with_cdp_until_dom_ready(state: SmokeState, url: str) -> None:
- result = await state.cdp.send("Page.navigate", {"url": url})
- if result.get("errorText"):
- raise SmokeError(f"Page.navigate failed for {url}: {result}")
-
- async def is_ready() -> bool:
- ready_state = await state.cdp.send(
- "Runtime.evaluate",
- {
- "expression": "document.readyState",
- "returnByValue": True,
- },
- )
- location = await state.cdp.send(
- "Runtime.evaluate",
- {
- "expression": "location.href",
+ ready: set[tuple[str, Any, Any]] = set()
+ changed = asyncio.Event()
+
+ def lifecycle(params: dict[str, Any]) -> None:
+ if params.get("name") == "DOMContentLoaded":
+ ready.add(("lifecycle", params.get("frameId"), params.get("loaderId")))
+ changed.set()
+
+ def within_document(params: dict[str, Any]) -> None:
+ ready.add(("within-document", params.get("frameId"), params.get("url")))
+ changed.set()
+
+ state.cdp.on("Page.lifecycleEvent", lifecycle)
+ state.cdp.on("Page.navigatedWithinDocument", within_document)
+ phase = "enabling Page lifecycle events"
+ try:
+ async with asyncio.timeout(_NAVIGATION_TIMEOUT_SECONDS):
+ await state.cdp.send("Page.enable")
+ await state.cdp.send("Page.setLifecycleEventsEnabled", {"enabled": True})
+ phase = "Page.navigate"
+ result = await state.cdp.send("Page.navigate", {"url": url})
+ if result.get("errorText"):
+ raise SmokeError(f"Page.navigate failed for {url}: {result}")
+ frame, loader = result["frameId"], result.get("loaderId")
+ phase = f"DOM ready for frame={frame}, loader={loader}"
+
+ # A lifecycle event can precede the navigate reply. Keep those
+ # events, but reject old loaders and child-frame notifications.
+ expected = (("lifecycle", frame, loader) if loader is not None
+ else ("within-document", frame, url))
+ while expected not in ready:
+ changed.clear()
+ await changed.wait()
+ phase = "Runtime.evaluate after DOM ready"
+ snapshot = await state.cdp.send("Runtime.evaluate", {
+ "expression": "({url: location.href, readyState: document.readyState})",
"returnByValue": True,
- },
- )
- return (
- location.get("result", {}).get("value") == url
- and ready_state.get("result", {}).get("value") in ["interactive", "complete"]
- )
-
- await wait_until(is_ready, f"CDP navigation DOM ready for {url}")
+ })
+ value = snapshot.get("result", {}).get("value")
+ if (not isinstance(value, dict) or value.get("url") != url
+ or value.get("readyState") not in ("interactive", "complete")):
+ raise SmokeError(f"Unexpected document after CDP navigation to {url}: {snapshot}")
+ except TimeoutError as error:
+ raise SmokeError(f"CDP navigation timed out during {phase}: {url}") from error
+ finally:
+ state.cdp.remove_listener("Page.lifecycleEvent", lifecycle)
+ state.cdp.remove_listener("Page.navigatedWithinDocument", within_document)
async def _send_cdp_expect_optional_error(cdp: Any, method: str, params: dict[str, Any]) -> dict[str, Any] | None:
try:
diff --git a/moli-cdp-smoke/tests/test_chromium_navigation.py b/moli-cdp-smoke/tests/test_chromium_navigation.py
new file mode 100644
index 000000000..b4fc374ad
--- /dev/null
+++ b/moli-cdp-smoke/tests/test_chromium_navigation.py
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import asyncio
+import unittest
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import patch
+
+from moli_cdp_smoke.assertions import SmokeError
+from moli_cdp_smoke.groups import chromium_cdp_support as navigation
+
+
+URL = "http://fixture.test/plain?requested"
+FRAME = "main-frame"
+LOADER = "requested-loader"
+
+
+class _NavigationSession:
+ def __init__(self, *, early: bool = False, complete: bool = True,
+ same_document: bool = False, error: str | None = None,
+ block_method: str | None = None) -> None:
+ self.early = early
+ self.complete = complete
+ self.same_document = same_document
+ self.error = error
+ self.block_method = block_method
+ self.listeners: dict[str, list[Any]] = {}
+ self.commands: list[str] = []
+ self.ready = False
+
+ def on(self, method: str, listener: Any) -> None:
+ self.listeners.setdefault(method, []).append(listener)
+
+ def remove_listener(self, method: str, listener: Any) -> None:
+ self.listeners[method].remove(listener)
+
+ def emit(self, method: str, params: dict[str, Any]) -> None:
+ for listener in self.listeners.get(method, []):
+ listener(params)
+
+ def document_ready(self) -> None:
+ self.ready = True
+ if self.same_document:
+ self.emit("Page.navigatedWithinDocument", {"frameId": FRAME, "url": URL})
+ else:
+ self.emit("Page.lifecycleEvent", {
+ "frameId": FRAME, "loaderId": LOADER, "name": "DOMContentLoaded",
+ })
+
+ async def send(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
+ self.commands.append(method)
+ if method == self.block_method:
+ await asyncio.Event().wait()
+ if method == "Page.setLifecycleEventsEnabled":
+ self.emit("Page.lifecycleEvent", {
+ "frameId": FRAME, "loaderId": "old-loader", "name": "DOMContentLoaded",
+ })
+ if method == "Page.navigate":
+ if self.error:
+ return {"frameId": FRAME, "errorText": self.error}
+ self.emit("Page.lifecycleEvent", {
+ "frameId": "child-frame", "loaderId": LOADER, "name": "DOMContentLoaded",
+ })
+ self.emit("Page.lifecycleEvent", {
+ "frameId": FRAME, "loaderId": "old-loader", "name": "DOMContentLoaded",
+ })
+ if self.complete:
+ if self.early:
+ self.document_ready()
+ else:
+ asyncio.get_running_loop().call_soon(self.document_ready)
+ return {"frameId": FRAME, **({} if self.same_document else {"loaderId": LOADER})}
+ if method == "Runtime.evaluate":
+ if not self.ready:
+ raise AssertionError("evaluated while the requested loader was still loading")
+ return {"result": {"value": {"url": URL, "readyState": "interactive"}}}
+ return {}
+
+
+class ChromiumNavigationTests(unittest.IsolatedAsyncioTestCase):
+ async def test_does_not_evaluate_until_requested_loader_dom_ready(self) -> None:
+ session = _NavigationSession()
+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)
+ self.assertEqual(session.commands.count("Runtime.evaluate"), 1)
+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))
+
+ async def test_dom_ready_before_navigate_reply_is_retained(self) -> None:
+ session = _NavigationSession(early=True)
+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)
+ self.assertEqual(session.commands.count("Runtime.evaluate"), 1)
+
+ async def test_same_document_navigation_uses_its_frame_and_url(self) -> None:
+ session = _NavigationSession(same_document=True)
+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)
+ self.assertEqual(session.commands.count("Runtime.evaluate"), 1)
+
+ async def test_stale_and_child_events_cannot_satisfy_the_navigation(self) -> None:
+ session = _NavigationSession(complete=False)
+ with patch.object(navigation, "_NAVIGATION_TIMEOUT_SECONDS", 0.01):
+ with self.assertRaisesRegex(SmokeError, "requested-loader"):
+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)
+ self.assertNotIn("Runtime.evaluate", session.commands)
+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))
+
+ async def test_navigation_error_is_not_replaced_by_a_readiness_timeout(self) -> None:
+ session = _NavigationSession(error="net::ERR_ABORTED")
+ with self.assertRaisesRegex(SmokeError, "net::ERR_ABORTED"):
+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)
+ self.assertNotIn("Runtime.evaluate", session.commands)
+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))
+
+ async def test_pending_cdp_commands_respect_the_navigation_deadline(self) -> None:
+ for method in ("Page.navigate", "Runtime.evaluate"):
+ with self.subTest(method=method):
+ session = _NavigationSession(block_method=method)
+ with patch.object(navigation, "_NAVIGATION_TIMEOUT_SECONDS", 0.01):
+ with self.assertRaisesRegex(SmokeError, method):
+ await navigation._navigate_with_cdp_until_dom_ready(SimpleNamespace(cdp=session), URL)
+ self.assertTrue(all(not listeners for listeners in session.listeners.values()))