Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Fix navigation readiness detection: per-target lifecycle event storage
Saurav Pandac5f0fa767c3b
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex 72d5a2045..e4933a7ed 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -98,9 +98,9 @@ class CDPSession(BaseModel):target_id: TargetIDsession_id: SessionID- # Lifecycle monitoring (populated by SessionManager)+ # Lifecycle monitoring: reference to SessionManager's per-target event buffer+ # (assigned in _enable_page_monitoring; used by readiness checks)_lifecycle_events: Any = PrivateAttr(default=None)- _lifecycle_lock: Any = PrivateAttr(default=None)class ResilientEventBus(EventBus):@@ -959,7 +959,7 @@ class BrowserSession(BaseModel):await self.event_bus.dispatch(NavigationStartedEvent(target_id=target_id, url=event.url))# Navigate to URL with proper lifecycle waiting- await self._navigate_and_wait(+ loading_status = await self._navigate_and_wait(event.url,target_id,timeout=event.timeout_ms / 1000 if event.timeout_ms is not None else None,@@ -977,6 +977,7 @@ class BrowserSession(BaseModel):target_id=target_id,url=event.url,status=None, # CDP doesn't provide status directly+ loading_status=loading_status, # non-None when readiness timed out))await self.event_bus.dispatch(AgentFocusChangedEvent(target_id=target_id, url=event.url))@@ -1009,12 +1010,18 @@ class BrowserSession(BaseModel):timeout: float | None = None,wait_until: str = 'load',nav_timeout: float | None = None,- ) -> None:+ ) -> str | None:"""Navigate to URL and wait for page readiness using CDP lifecycle events.- Polls stored lifecycle events (registered once per session in SessionManager).+ Polls the per-target lifecycle event buffer (fed by SessionManager's single+ global Page.lifecycleEvent handler).wait_until controls the minimum acceptable signal: 'commit', 'domcontentloaded', 'load', 'networkidle'.nav_timeout controls the timeout for the CDP Page.navigate() call itself (defaults to 20.0s).++ Returns None when the requested readiness signal was observed, or a+ 'timeout...' status string when the wait timed out — callers surface it via+ NavigationCompleteEvent.loading_status so downstream consumers know the page+ may not be fully loaded."""cdp_session = await self.get_or_create_cdp_session(target_id, focus=False)@@ -1052,18 +1059,15 @@ class BrowserSession(BaseModel):if wait_until == 'commit':duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000self.logger.debug(f'✅ Page ready for {url} (commit, {duration_ms:.0f}ms)')- return+ return Nonenavigation_id = nav_result.get('loaderId')start_time = asyncio.get_event_loop().time()seen_events = []- if not hasattr(cdp_session, '_lifecycle_events'):- raise RuntimeError(- f'❌ Lifecycle monitoring not enabled for {cdp_session.target_id[:8]}! '- f'This is a bug - SessionManager should have initialized it. '- f'Session: {cdp_session}'- )+ # Per-target buffer owned by SessionManager — NOT a per-session attribute, whose+ # feeding handler used to get replaced whenever another target attached.+ lifecycle_events = self.session_manager.get_lifecycle_events(target_id)# Acceptable events by readiness level (higher is always acceptable)acceptable_events: set[str] = {'networkIdle'}@@ -1075,7 +1079,7 @@ class BrowserSession(BaseModel):poll_interval = 0.05while (asyncio.get_event_loop().time() - start_time) < timeout:try:- for event_data in list(cdp_session._lifecycle_events):+ for event_data in list(lifecycle_events):event_name = event_data.get('name')event_loader_id = event_data.get('loaderId')@@ -1083,13 +1087,20 @@ class BrowserSession(BaseModel):if event_str not in seen_events:seen_events.append(event_str)+ # Skip events from a previous document in this frame (stale entries+ # carry the old loaderId; the buffer may hold pre-navigation events).if event_loader_id and navigation_id and event_loader_id != navigation_id:continue+ # Defense for events without a usable loaderId: only trust them if+ # they arrived after this navigation started.+ if event_data.get('timestamp', 0) < nav_start_time and (not event_loader_id or not navigation_id):+ continue+if event_name in acceptable_events:duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000self.logger.debug(f'✅ Page ready for {url} ({event_name}, {duration_ms:.0f}ms)')- return+ return Noneexcept Exception as e:self.logger.debug(f'Error polling lifecycle events: {e}')@@ -1102,8 +1113,9 @@ class BrowserSession(BaseModel):f'❌ No lifecycle events received for {url} after {duration_ms:.0f}ms! 'f'Monitoring may have failed. Target: {cdp_session.target_id[:8]}')- else:- self.logger.warning(f'⚠️ Page readiness timeout ({timeout}s, {duration_ms:.0f}ms) for {url}')+ return f'timeout after {timeout}s: no lifecycle events received (monitoring may have failed)'+ self.logger.warning(f'⚠️ Page readiness timeout ({timeout}s, {duration_ms:.0f}ms) for {url}')+ return f'timeout after {timeout}s waiting for {wait_until!r} (saw: {", ".join(seen_events[-5:])})'async def on_SwitchTabEvent(self, event: SwitchTabEvent) -> TargetID:"""Handle tab switching - core browser functionality."""diff --git a/browser_use/browser/session_manager.py b/browser_use/browser/session_manager.pyindex b380dabed..110de272b 100644--- a/browser_use/browser/session_manager.py+++ b/browser_use/browser/session_manager.py@@ -5,7 +5,8 @@ events, ensuring the session pool always reflects the current browser state."""import asyncio-from typing import TYPE_CHECKING+from collections import deque+from typing import TYPE_CHECKING, Anyfrom cdp_use.cdp.target import AttachedToTargetEvent, DetachedFromTargetEvent, SessionID, TargetID@@ -45,6 +46,12 @@ class SessionManager:# Reverse mapping: session -> target it belongs toself._session_to_target: dict[SessionID, TargetID] = {}+ # Page lifecycle events per target, fed by ONE global Page.lifecycleEvent handler+ # registered in start_monitoring(). cdp-use's event registry is single-slot per+ # CDP method, so per-session handler registrations would replace each other and+ # leave every tab but the most recently attached one without lifecycle events.+ self._lifecycle_events: dict[TargetID, deque[dict[str, Any]]] = {}+self._lock = asyncio.Lock()self._recovery_lock = asyncio.Lock()@@ -102,15 +109,41 @@ class SessionManager:suppress_exceptions=True,)+ def on_lifecycle_event(event, session_id: SessionID | None = None):+ # ONE global handler for all targets: route by session_id -> target_id.+ # Registering per-session closures instead would clobber each other in+ # cdp-use's single-slot registry (one handler per CDP method).+ if not session_id:+ return+ target_id = self.get_target_id_from_session_id(session_id)+ if not target_id:+ return+ self.get_lifecycle_events(target_id).append(+ {+ 'name': event.get('name', 'unknown'),+ 'loaderId': event.get('loaderId'),+ 'timestamp': asyncio.get_event_loop().time(),+ }+ )+cdp_client.register.Target.attachedToTarget(on_attached)cdp_client.register.Target.detachedFromTarget(on_detached)cdp_client.register.Target.targetInfoChanged(on_target_info_changed)+ cdp_client.register.Page.lifecycleEvent(on_lifecycle_event)self.logger.debug('[SessionManager] Event monitoring started')# Discover and initialize ALL existing targetsawait self._initialize_existing_targets()+ def get_lifecycle_events(self, target_id: TargetID) -> 'deque[dict[str, Any]]':+ """Get (creating if needed) the lifecycle event buffer for a target."""+ events = self._lifecycle_events.get(target_id)+ if events is None:+ events = deque(maxlen=50)+ self._lifecycle_events[target_id] = events+ return events+def _get_session_for_target(self, target_id: TargetID) -> 'CDPSession | None':"""Internal: Get ANY valid session for a target (picks first available).@@ -558,6 +591,7 @@ class SessionManager:# Clean up trackingdel self._target_sessions[target_id]+ self._lifecycle_events.pop(target_id, None)else:# Target not tracked - already removed or never attachedself.logger.debug(@@ -864,39 +898,12 @@ class SessionManager:# Enable network monitoring for networkIdle detectionawait cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)- # Initialize lifecycle event storage for this session (thread-safe)- from collections import deque-- cdp_session._lifecycle_events = deque(maxlen=50) # Keep last 50 events- cdp_session._lifecycle_lock = asyncio.Lock()-- # Register ONE handler per session that stores events- def on_lifecycle_event(event, session_id=None):- event_name = event.get('name', 'unknown')- event_loader_id = event.get('loaderId', 'none')-- # Find which target this session belongs to- target_id_from_event = None- if session_id:- target_id_from_event = self.get_target_id_from_session_id(session_id)-- # Check if this event is for our target- if target_id_from_event == cdp_session.target_id:- # Store event for navigations to consume- event_data = {- 'name': event_name,- 'loaderId': event_loader_id,- 'timestamp': asyncio.get_event_loop().time(),- }- # Append is atomic in CPython- try:- cdp_session._lifecycle_events.append(event_data)- except Exception as e:- # Only log errors, not every event- self.logger.error(f'[SessionManager] Failed to store lifecycle event: {e}')-- # Register the handler ONCE (this is the only place we register)- cdp_session.cdp_client.register.Page.lifecycleEvent(on_lifecycle_event)+ # Event storage and the Page.lifecycleEvent handler live in SessionManager+ # (one global handler registered in start_monitoring, routed by session_id):+ # cdp-use's registry is single-slot per method, so a per-session registration+ # here would replace the previous tab's handler and freeze its event buffer.+ # Expose the shared per-target buffer on the session for readiness checks.+ cdp_session._lifecycle_events = self.get_lifecycle_events(cdp_session.target_id)except Exception as e:# Don't fail - target might be short-lived or already detacheddiff --git a/tests/ci/browser/test_navigation_readiness.py b/tests/ci/browser/test_navigation_readiness.pynew file mode 100644index 000000000..7fb6ed0a6--- /dev/null+++ b/tests/ci/browser/test_navigation_readiness.py@@ -0,0 +1,112 @@+"""Regression tests: navigation readiness detection must actually detect page load.++cdp-use's event registry is single-slot per CDP method: registering a per-session+Page.lifecycleEvent closure replaces the previous one, so only the most recently+attached target recorded lifecycle events. Any earlier tab's event deque went silent,+and every navigation on it burned the full readiness timeout (3s same-domain /+8s cross-domain) before proceeding on a page in unknown load state.++Lifecycle events are now stored per-target in SessionManager, fed by one global+handler registered once on the root CDP client.+"""++import asyncio+import time++import pytest++from browser_use.browser.events import NavigateToUrlEvent+from browser_use.browser.profile import BrowserProfile+from browser_use.browser.session import BrowserSession++SIMPLE_HTML = '<html><head><title>fast page</title></head><body>hello</body></html>'++# Generous bound for a local page load: well above real load time (~100ms),+# well below the 3s that a readiness-timeout fallback burns.+FAST_NAVIGATION_BOUND_S = 2.5+++async def browser_session():+ session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=True))+ await session.start()+ yield session+ await session.kill()+++async def test_navigation_detects_readiness_without_burning_timeout(httpserver, browser_session: BrowserSession):+ """A local page load must complete as soon as the load event fires, not after the fallback timeout."""+ httpserver.expect_request('/fast').respond_with_data(SIMPLE_HTML, content_type='text/html')++ start = time.monotonic()+ await browser_session.navigate_to(httpserver.url_for('/fast'))+ elapsed = time.monotonic() - start++ assert elapsed < FAST_NAVIGATION_BOUND_S, (+ f'navigation took {elapsed:.2f}s — readiness detection failed and the fallback timeout was burned'+ )+++async def test_first_tab_navigation_still_works_after_second_tab_opens(httpserver, browser_session: BrowserSession):+ """Opening a new tab must not disable lifecycle monitoring on existing tabs.++ Pre-fix: the second tab's per-session handler registration replaced the first+ tab's on the shared root client, freezing the first tab's event deque — every+ subsequent navigation on tab A hit the full readiness timeout.+ """+ httpserver.expect_request('/a').respond_with_data(SIMPLE_HTML, content_type='text/html')+ httpserver.expect_request('/b').respond_with_data(SIMPLE_HTML, content_type='text/html')+ httpserver.expect_request('/a2').respond_with_data(SIMPLE_HTML, content_type='text/html')++ await browser_session.navigate_to(httpserver.url_for('/a'))+ tab_a = browser_session.agent_focus_target_id+ assert tab_a is not None++ # Open a second tab — its target attach re-runs page monitoring setup+ event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/b'), new_tab=True))+ await event+ await asyncio.sleep(0.5)++ # Navigate the FIRST tab again (same domain -> 3s fallback timeout pre-fix)+ start = time.monotonic()+ await browser_session._navigate_and_wait(httpserver.url_for('/a2'), tab_a, wait_until='load')+ elapsed = time.monotonic() - start++ assert elapsed < FAST_NAVIGATION_BOUND_S, (+ f'tab A navigation took {elapsed:.2f}s after tab B opened — its lifecycle monitoring was clobbered'+ )+++async def test_readiness_timeout_is_reported_not_swallowed(httpserver, browser_session: BrowserSession):+ """When readiness genuinely times out, _navigate_and_wait must say so instead of+ returning indistinguishably from success (NavigationCompleteEvent.loading_status+ exists for exactly this)."""++ def slow_image(request):+ time.sleep(5)+ from werkzeug.wrappers import Response++ return Response(b'', content_type='image/png')++ # DOMContentLoaded fires immediately; 'load' is held hostage by the stalled image.+ httpserver.expect_request('/hanging').respond_with_data(+ '<html><body><img src="/slow-img">never finishes loading</body></html>', content_type='text/html'+ )+ httpserver.expect_request('/slow-img').respond_with_handler(slow_image)++ target_id = browser_session.agent_focus_target_id+ assert target_id is not None++ status = await browser_session._navigate_and_wait(httpserver.url_for('/hanging'), target_id, timeout=1.0, wait_until='load')++ assert status is not None and 'timeout' in status, f'readiness timeout was swallowed, got status={status!r}'+++async def test_successful_navigation_returns_no_timeout_status(httpserver, browser_session: BrowserSession):+ httpserver.expect_request('/ok').respond_with_data(SIMPLE_HTML, content_type='text/html')++ target_id = browser_session.agent_focus_target_id+ assert target_id is not None++ status = await browser_session._navigate_and_wait(httpserver.url_for('/ok'), target_id, wait_until='load')+ assert status is None