Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Fix navigation readiness detection: per-target lifecycle event storage

Saurav Pandac5f0fa767c3b
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 72d5a2045..e4933a7ed 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -98,9 +98,9 @@ class CDPSession(BaseModel):
target_id: TargetID
session_id: SessionID
- # Lifecycle monitoring (populated by SessionManager)
+ # Lifecycle monitoring: reference to SessionManager's per-target event buffer
+ # (assigned in _enable_page_monitoring; used by readiness checks)
_lifecycle_events: Any = PrivateAttr(default=None)
- _lifecycle_lock: Any = PrivateAttr(default=None)
class ResilientEventBus(EventBus):
@@ -959,7 +959,7 @@ class BrowserSession(BaseModel):
await self.event_bus.dispatch(NavigationStartedEvent(target_id=target_id, url=event.url))
# Navigate to URL with proper lifecycle waiting
- await self._navigate_and_wait(
+ loading_status = await self._navigate_and_wait(
event.url,
target_id,
timeout=event.timeout_ms / 1000 if event.timeout_ms is not None else None,
@@ -977,6 +977,7 @@ class BrowserSession(BaseModel):
target_id=target_id,
url=event.url,
status=None, # CDP doesn't provide status directly
+ loading_status=loading_status, # non-None when readiness timed out
)
)
await self.event_bus.dispatch(AgentFocusChangedEvent(target_id=target_id, url=event.url))
@@ -1009,12 +1010,18 @@ class BrowserSession(BaseModel):
timeout: float | None = None,
wait_until: str = 'load',
nav_timeout: float | None = None,
- ) -> None:
+ ) -> str | None:
"""Navigate to URL and wait for page readiness using CDP lifecycle events.
- Polls stored lifecycle events (registered once per session in SessionManager).
+ Polls the per-target lifecycle event buffer (fed by SessionManager's single
+ global Page.lifecycleEvent handler).
wait_until controls the minimum acceptable signal: 'commit', 'domcontentloaded', 'load', 'networkidle'.
nav_timeout controls the timeout for the CDP Page.navigate() call itself (defaults to 20.0s).
+
+ Returns None when the requested readiness signal was observed, or a
+ 'timeout...' status string when the wait timed out — callers surface it via
+ NavigationCompleteEvent.loading_status so downstream consumers know the page
+ may not be fully loaded.
"""
cdp_session = await self.get_or_create_cdp_session(target_id, focus=False)
@@ -1052,18 +1059,15 @@ class BrowserSession(BaseModel):
if wait_until == 'commit':
duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000
self.logger.debug(f'✅ Page ready for {url} (commit, {duration_ms:.0f}ms)')
- return
+ return None
navigation_id = nav_result.get('loaderId')
start_time = asyncio.get_event_loop().time()
seen_events = []
- if not hasattr(cdp_session, '_lifecycle_events'):
- raise RuntimeError(
- f'❌ Lifecycle monitoring not enabled for {cdp_session.target_id[:8]}! '
- f'This is a bug - SessionManager should have initialized it. '
- f'Session: {cdp_session}'
- )
+ # Per-target buffer owned by SessionManager — NOT a per-session attribute, whose
+ # feeding handler used to get replaced whenever another target attached.
+ lifecycle_events = self.session_manager.get_lifecycle_events(target_id)
# Acceptable events by readiness level (higher is always acceptable)
acceptable_events: set[str] = {'networkIdle'}
@@ -1075,7 +1079,7 @@ class BrowserSession(BaseModel):
poll_interval = 0.05
while (asyncio.get_event_loop().time() - start_time) < timeout:
try:
- for event_data in list(cdp_session._lifecycle_events):
+ for event_data in list(lifecycle_events):
event_name = event_data.get('name')
event_loader_id = event_data.get('loaderId')
@@ -1083,13 +1087,20 @@ class BrowserSession(BaseModel):
if event_str not in seen_events:
seen_events.append(event_str)
+ # Skip events from a previous document in this frame (stale entries
+ # carry the old loaderId; the buffer may hold pre-navigation events).
if event_loader_id and navigation_id and event_loader_id != navigation_id:
continue
+ # Defense for events without a usable loaderId: only trust them if
+ # they arrived after this navigation started.
+ if event_data.get('timestamp', 0) < nav_start_time and (not event_loader_id or not navigation_id):
+ continue
+
if event_name in acceptable_events:
duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000
self.logger.debug(f'✅ Page ready for {url} ({event_name}, {duration_ms:.0f}ms)')
- return
+ return None
except Exception as e:
self.logger.debug(f'Error polling lifecycle events: {e}')
@@ -1102,8 +1113,9 @@ class BrowserSession(BaseModel):
f'❌ No lifecycle events received for {url} after {duration_ms:.0f}ms! '
f'Monitoring may have failed. Target: {cdp_session.target_id[:8]}'
)
- else:
- self.logger.warning(f'⚠️ Page readiness timeout ({timeout}s, {duration_ms:.0f}ms) for {url}')
+ return f'timeout after {timeout}s: no lifecycle events received (monitoring may have failed)'
+ self.logger.warning(f'⚠️ Page readiness timeout ({timeout}s, {duration_ms:.0f}ms) for {url}')
+ return f'timeout after {timeout}s waiting for {wait_until!r} (saw: {", ".join(seen_events[-5:])})'
async def on_SwitchTabEvent(self, event: SwitchTabEvent) -> TargetID:
"""Handle tab switching - core browser functionality."""
diff --git a/browser_use/browser/session_manager.py b/browser_use/browser/session_manager.py
index b380dabed..110de272b 100644
--- a/browser_use/browser/session_manager.py
+++ b/browser_use/browser/session_manager.py
@@ -5,7 +5,8 @@ events, ensuring the session pool always reflects the current browser state.
"""
import asyncio
-from typing import TYPE_CHECKING
+from collections import deque
+from typing import TYPE_CHECKING, Any
from cdp_use.cdp.target import AttachedToTargetEvent, DetachedFromTargetEvent, SessionID, TargetID
@@ -45,6 +46,12 @@ class SessionManager:
# Reverse mapping: session -> target it belongs to
self._session_to_target: dict[SessionID, TargetID] = {}
+ # Page lifecycle events per target, fed by ONE global Page.lifecycleEvent handler
+ # registered in start_monitoring(). cdp-use's event registry is single-slot per
+ # CDP method, so per-session handler registrations would replace each other and
+ # leave every tab but the most recently attached one without lifecycle events.
+ self._lifecycle_events: dict[TargetID, deque[dict[str, Any]]] = {}
+
self._lock = asyncio.Lock()
self._recovery_lock = asyncio.Lock()
@@ -102,15 +109,41 @@ class SessionManager:
suppress_exceptions=True,
)
+ def on_lifecycle_event(event, session_id: SessionID | None = None):
+ # ONE global handler for all targets: route by session_id -> target_id.
+ # Registering per-session closures instead would clobber each other in
+ # cdp-use's single-slot registry (one handler per CDP method).
+ if not session_id:
+ return
+ target_id = self.get_target_id_from_session_id(session_id)
+ if not target_id:
+ return
+ self.get_lifecycle_events(target_id).append(
+ {
+ 'name': event.get('name', 'unknown'),
+ 'loaderId': event.get('loaderId'),
+ 'timestamp': asyncio.get_event_loop().time(),
+ }
+ )
+
cdp_client.register.Target.attachedToTarget(on_attached)
cdp_client.register.Target.detachedFromTarget(on_detached)
cdp_client.register.Target.targetInfoChanged(on_target_info_changed)
+ cdp_client.register.Page.lifecycleEvent(on_lifecycle_event)
self.logger.debug('[SessionManager] Event monitoring started')
# Discover and initialize ALL existing targets
await self._initialize_existing_targets()
+ def get_lifecycle_events(self, target_id: TargetID) -> 'deque[dict[str, Any]]':
+ """Get (creating if needed) the lifecycle event buffer for a target."""
+ events = self._lifecycle_events.get(target_id)
+ if events is None:
+ events = deque(maxlen=50)
+ self._lifecycle_events[target_id] = events
+ return events
+
def _get_session_for_target(self, target_id: TargetID) -> 'CDPSession | None':
"""Internal: Get ANY valid session for a target (picks first available).
@@ -558,6 +591,7 @@ class SessionManager:
# Clean up tracking
del self._target_sessions[target_id]
+ self._lifecycle_events.pop(target_id, None)
else:
# Target not tracked - already removed or never attached
self.logger.debug(
@@ -864,39 +898,12 @@ class SessionManager:
# Enable network monitoring for networkIdle detection
await cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)
- # Initialize lifecycle event storage for this session (thread-safe)
- from collections import deque
-
- cdp_session._lifecycle_events = deque(maxlen=50) # Keep last 50 events
- cdp_session._lifecycle_lock = asyncio.Lock()
-
- # Register ONE handler per session that stores events
- def on_lifecycle_event(event, session_id=None):
- event_name = event.get('name', 'unknown')
- event_loader_id = event.get('loaderId', 'none')
-
- # Find which target this session belongs to
- target_id_from_event = None
- if session_id:
- target_id_from_event = self.get_target_id_from_session_id(session_id)
-
- # Check if this event is for our target
- if target_id_from_event == cdp_session.target_id:
- # Store event for navigations to consume
- event_data = {
- 'name': event_name,
- 'loaderId': event_loader_id,
- 'timestamp': asyncio.get_event_loop().time(),
- }
- # Append is atomic in CPython
- try:
- cdp_session._lifecycle_events.append(event_data)
- except Exception as e:
- # Only log errors, not every event
- self.logger.error(f'[SessionManager] Failed to store lifecycle event: {e}')
-
- # Register the handler ONCE (this is the only place we register)
- cdp_session.cdp_client.register.Page.lifecycleEvent(on_lifecycle_event)
+ # Event storage and the Page.lifecycleEvent handler live in SessionManager
+ # (one global handler registered in start_monitoring, routed by session_id):
+ # cdp-use's registry is single-slot per method, so a per-session registration
+ # here would replace the previous tab's handler and freeze its event buffer.
+ # Expose the shared per-target buffer on the session for readiness checks.
+ cdp_session._lifecycle_events = self.get_lifecycle_events(cdp_session.target_id)
except Exception as e:
# Don't fail - target might be short-lived or already detached
diff --git a/tests/ci/browser/test_navigation_readiness.py b/tests/ci/browser/test_navigation_readiness.py
new file mode 100644
index 000000000..7fb6ed0a6
--- /dev/null
+++ b/tests/ci/browser/test_navigation_readiness.py
@@ -0,0 +1,112 @@
+"""Regression tests: navigation readiness detection must actually detect page load.
+
+cdp-use's event registry is single-slot per CDP method: registering a per-session
+Page.lifecycleEvent closure replaces the previous one, so only the most recently
+attached target recorded lifecycle events. Any earlier tab's event deque went silent,
+and every navigation on it burned the full readiness timeout (3s same-domain /
+8s cross-domain) before proceeding on a page in unknown load state.
+
+Lifecycle events are now stored per-target in SessionManager, fed by one global
+handler registered once on the root CDP client.
+"""
+
+import asyncio
+import time
+
+import pytest
+
+from browser_use.browser.events import NavigateToUrlEvent
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+
+SIMPLE_HTML = '<html><head><title>fast page</title></head><body>hello</body></html>'
+
+# Generous bound for a local page load: well above real load time (~100ms),
+# well below the 3s that a readiness-timeout fallback burns.
+FAST_NAVIGATION_BOUND_S = 2.5
+
+
+async def browser_session():
+ session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=True))
+ await session.start()
+ yield session
+ await session.kill()
+
+
+async def test_navigation_detects_readiness_without_burning_timeout(httpserver, browser_session: BrowserSession):
+ """A local page load must complete as soon as the load event fires, not after the fallback timeout."""
+ httpserver.expect_request('/fast').respond_with_data(SIMPLE_HTML, content_type='text/html')
+
+ start = time.monotonic()
+ await browser_session.navigate_to(httpserver.url_for('/fast'))
+ elapsed = time.monotonic() - start
+
+ assert elapsed < FAST_NAVIGATION_BOUND_S, (
+ f'navigation took {elapsed:.2f}s — readiness detection failed and the fallback timeout was burned'
+ )
+
+
+async def test_first_tab_navigation_still_works_after_second_tab_opens(httpserver, browser_session: BrowserSession):
+ """Opening a new tab must not disable lifecycle monitoring on existing tabs.
+
+ Pre-fix: the second tab's per-session handler registration replaced the first
+ tab's on the shared root client, freezing the first tab's event deque — every
+ subsequent navigation on tab A hit the full readiness timeout.
+ """
+ httpserver.expect_request('/a').respond_with_data(SIMPLE_HTML, content_type='text/html')
+ httpserver.expect_request('/b').respond_with_data(SIMPLE_HTML, content_type='text/html')
+ httpserver.expect_request('/a2').respond_with_data(SIMPLE_HTML, content_type='text/html')
+
+ await browser_session.navigate_to(httpserver.url_for('/a'))
+ tab_a = browser_session.agent_focus_target_id
+ assert tab_a is not None
+
+ # Open a second tab — its target attach re-runs page monitoring setup
+ event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/b'), new_tab=True))
+ await event
+ await asyncio.sleep(0.5)
+
+ # Navigate the FIRST tab again (same domain -> 3s fallback timeout pre-fix)
+ start = time.monotonic()
+ await browser_session._navigate_and_wait(httpserver.url_for('/a2'), tab_a, wait_until='load')
+ elapsed = time.monotonic() - start
+
+ assert elapsed < FAST_NAVIGATION_BOUND_S, (
+ f'tab A navigation took {elapsed:.2f}s after tab B opened — its lifecycle monitoring was clobbered'
+ )
+
+
+async def test_readiness_timeout_is_reported_not_swallowed(httpserver, browser_session: BrowserSession):
+ """When readiness genuinely times out, _navigate_and_wait must say so instead of
+ returning indistinguishably from success (NavigationCompleteEvent.loading_status
+ exists for exactly this)."""
+
+ def slow_image(request):
+ time.sleep(5)
+ from werkzeug.wrappers import Response
+
+ return Response(b'', content_type='image/png')
+
+ # DOMContentLoaded fires immediately; 'load' is held hostage by the stalled image.
+ httpserver.expect_request('/hanging').respond_with_data(
+ '<html><body><img src="/slow-img">never finishes loading</body></html>', content_type='text/html'
+ )
+ httpserver.expect_request('/slow-img').respond_with_handler(slow_image)
+
+ target_id = browser_session.agent_focus_target_id
+ assert target_id is not None
+
+ status = await browser_session._navigate_and_wait(httpserver.url_for('/hanging'), target_id, timeout=1.0, wait_until='load')
+
+ assert status is not None and 'timeout' in status, f'readiness timeout was swallowed, got status={status!r}'
+
+
+async def test_successful_navigation_returns_no_timeout_status(httpserver, browser_session: BrowserSession):
+ httpserver.expect_request('/ok').respond_with_data(SIMPLE_HTML, content_type='text/html')
+
+ target_id = browser_session.agent_focus_target_id
+ assert target_id is not None
+
+ status = await browser_session._navigate_and_wait(httpserver.url_for('/ok'), target_id, wait_until='load')
+ assert status is None