Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

updated _navigate_and_wait

Saurav Panda6713afc63667
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.py
index afc492de3..ec045bbfa 100644
--- a/browser_use/browser/events.py
+++ b/browser_use/browser/events.py
@@ -119,7 +119,7 @@ class NavigateToUrlEvent(BaseEvent[None]):
# existing_tab: PageHandle | None = None # TODO
# time limits enforced by bubus, not exposed to LLM:
- event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigateToUrlEvent', 15.0)) # seconds
+ event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigateToUrlEvent', 30.0)) # seconds
class ClickElementEvent(ElementSelectedEvent[dict[str, Any] | None]):
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 5fd6f596e..ce8999f44 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -850,7 +850,7 @@ class BrowserSession(BaseModel):
await self.event_bus.dispatch(NavigationStartedEvent(target_id=target_id, url=event.url))
# Navigate to URL with proper lifecycle waiting
- await self._navigate_and_wait(event.url, target_id)
+ await self._navigate_and_wait(event.url, target_id, wait_until=event.wait_until)
# Close any extension options pages that might have opened
await self._close_extension_options_pages()
@@ -887,17 +887,17 @@ class BrowserSession(BaseModel):
await self.event_bus.dispatch(AgentFocusChangedEvent(target_id=target_id, url=event.url))
raise
- async def _navigate_and_wait(self, url: str, target_id: str, timeout: float | None = None) -> None:
+ async def _navigate_and_wait(
+ self,
+ url: str,
+ target_id: str,
+ timeout: float | None = None,
+ wait_until: str = 'load',
+ ) -> None:
"""Navigate to URL and wait for page readiness using CDP lifecycle events.
- Two-strategy approach optimized for speed with robust fallback:
- 1. networkIdle - Returns ASAP when no network activity (~50-200ms for cached pages)
- 2. load - Fallback when page has ongoing network activity (all resources loaded)
-
- This gives us instant returns for cached content while being robust for dynamic pages.
-
- NO handler registration here - handlers are registered ONCE per session in SessionManager.
- We poll stored events instead to avoid handler accumulation.
+ Polls stored lifecycle events (registered once per session in SessionManager).
+ wait_until controls the minimum acceptable signal: 'commit', 'domcontentloaded', 'load', 'networkidle'.
"""
cdp_session = await self.get_or_create_cdp_session(target_id, focus=False)
@@ -909,28 +909,37 @@ class BrowserSession(BaseModel):
if url.startswith('http') and current_url.startswith('http')
else False
)
- timeout = 2.0 if same_domain else 4.0
+ timeout = 3.0 if same_domain else 8.0
- # Start performance tracking
nav_start_time = asyncio.get_event_loop().time()
- nav_result = await cdp_session.cdp_client.send.Page.navigate(
- params={'url': url, 'transitionType': 'address_bar'},
- session_id=cdp_session.session_id,
- )
+ # Wrap Page.navigate() with timeout — heavy sites can block here for 10s+
+ nav_timeout = 20.0
+ try:
+ nav_result = await asyncio.wait_for(
+ cdp_session.cdp_client.send.Page.navigate(
+ params={'url': url, 'transitionType': 'address_bar'},
+ session_id=cdp_session.session_id,
+ ),
+ timeout=nav_timeout,
+ )
+ except TimeoutError:
+ duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000
+ self.logger.warning(f'⚠️ Page.navigate() timed out after {nav_timeout}s ({duration_ms:.0f}ms) for {url}')
+ return
- # Check for immediate navigation errors
if nav_result.get('errorText'):
raise RuntimeError(f'Navigation failed: {nav_result["errorText"]}')
- # Track this specific navigation
+ if wait_until == 'commit':
+ duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000
+ self.logger.debug(f'✅ Page ready for {url} (commit, {duration_ms:.0f}ms)')
+ return
+
navigation_id = nav_result.get('loaderId')
start_time = asyncio.get_event_loop().time()
+ seen_events = []
- # Poll stored lifecycle events
- seen_events = [] # Track events for timeout diagnostics
-
- # Check if session has lifecycle monitoring enabled
if not hasattr(cdp_session, '_lifecycle_events'):
raise RuntimeError(
f'❌ Lifecycle monitoring not enabled for {cdp_session.target_id[:8]}! '
@@ -938,42 +947,37 @@ class BrowserSession(BaseModel):
f'Session: {cdp_session}'
)
- # Poll for lifecycle events until timeout
- poll_interval = 0.05 # Poll every 50ms
+ # Acceptable events by readiness level (higher is always acceptable)
+ acceptable_events: set[str] = {'networkIdle'}
+ if wait_until in ('load', 'domcontentloaded'):
+ acceptable_events.add('load')
+ if wait_until == 'domcontentloaded':
+ acceptable_events.add('DOMContentLoaded')
+
+ poll_interval = 0.05
while (asyncio.get_event_loop().time() - start_time) < timeout:
- # Check stored events
try:
- # Get recent events matching our navigation
for event_data in list(cdp_session._lifecycle_events):
event_name = event_data.get('name')
event_loader_id = event_data.get('loaderId')
- # Track events
event_str = f'{event_name}(loader={event_loader_id[:8] if event_loader_id else "none"})'
if event_str not in seen_events:
seen_events.append(event_str)
- # Only respond to events from our navigation (or accept all if no loaderId)
if event_loader_id and navigation_id and event_loader_id != navigation_id:
continue
- if event_name == 'networkIdle':
- duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000
- self.logger.debug(f'✅ Page ready for {url} (networkIdle, {duration_ms:.0f}ms)')
- return
-
- elif event_name == 'load':
+ if event_name in acceptable_events:
duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000
- self.logger.debug(f'✅ Page ready for {url} (load, {duration_ms:.0f}ms)')
+ self.logger.debug(f'✅ Page ready for {url} ({event_name}, {duration_ms:.0f}ms)')
return
except Exception as e:
self.logger.debug(f'Error polling lifecycle events: {e}')
- # Wait before next poll
await asyncio.sleep(poll_interval)
- # Timeout - continue anyway with detailed diagnostics
duration_ms = (asyncio.get_event_loop().time() - nav_start_time) * 1000
if not seen_events:
self.logger.error(
diff --git a/tests/ci/browser/test_navigation_slow_pages.py b/tests/ci/browser/test_navigation_slow_pages.py
new file mode 100644
index 000000000..ed5fadb0c
--- /dev/null
+++ b/tests/ci/browser/test_navigation_slow_pages.py
@@ -0,0 +1,194 @@
+"""
+Test navigation on heavy/slow-loading pages (e.g. e-commerce PDPs).
+
+Reproduces the issue where navigating to heavy pages like stevemadden.com PDPs
+fails due to NavigateToUrlEvent timing out.
+
+Usage:
+ uv run pytest tests/ci/browser/test_navigation_slow_pages.py -v -s
+"""
+
+import asyncio
+import time
+
+import pytest
+from pytest_httpserver import HTTPServer
+from werkzeug import Response
+
+from browser_use.agent.service import Agent
+from browser_use.browser import BrowserSession
+from browser_use.browser.events import NavigateToUrlEvent
+from browser_use.browser.profile import BrowserProfile
+from tests.ci.conftest import create_mock_llm
+
+
+HEAVY_PDP_HTML = """
+<!DOCTYPE html>
+<html>
+<head><title>Frosting Black Velvet - Steve Madden</title></head>
+<body>
+ <h1>FROSTING</h1>
+ <p class="price">$129.95</p>
+ <button id="add-to-cart">ADD TO BAG</button>
+</body>
+</html>
+"""
+
+
[email protected](scope='session')
+def heavy_page_server():
+ server = HTTPServer()
+ server.start()
+
+ def slow_initial_response(request):
+ time.sleep(6)
+ return Response(HEAVY_PDP_HTML, content_type='text/html')
+
+ server.expect_request('/slow-server-pdp').respond_with_handler(slow_initial_response)
+
+ def redirect_step1(request):
+ return Response('', status=302, headers={'Location': f'http://{server.host}:{server.port}/redirect-step2'})
+
+ def redirect_step2(request):
+ return Response('', status=302, headers={'Location': f'http://{server.host}:{server.port}/redirect-final'})
+
+ def redirect_final(request):
+ time.sleep(3)
+ return Response(HEAVY_PDP_HTML, content_type='text/html')
+
+ server.expect_request('/redirect-step1').respond_with_handler(redirect_step1)
+ server.expect_request('/redirect-step2').respond_with_handler(redirect_step2)
+ server.expect_request('/redirect-final').respond_with_handler(redirect_final)
+
+ server.expect_request('/fast-dom-slow-load').respond_with_data(HEAVY_PDP_HTML, content_type='text/html')
+ server.expect_request('/quick-page').respond_with_data(
+ '<html><body><h1>Quick Page</h1></body></html>', content_type='text/html'
+ )
+
+ yield server
+ server.stop()
+
+
[email protected](scope='session')
+def heavy_base_url(heavy_page_server):
+ return f'http://{heavy_page_server.host}:{heavy_page_server.port}'
+
+
[email protected](scope='function')
+async def browser_session():
+ session = BrowserSession(
+ browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=True)
+ )
+ await session.start()
+ yield session
+ await session.kill()
+
+
+def _nav_actions(url: str, msg: str = 'Done') -> list[str]:
+ """Helper to build a navigate-then-done action sequence."""
+ return [
+ f"""
+ {{
+ "thinking": "Navigate to the page",
+ "evaluation_previous_goal": "Starting task",
+ "memory": "Navigating",
+ "next_goal": "Navigate",
+ "action": [{{"navigate": {{"url": "{url}"}}}}]
+ }}
+ """,
+ f"""
+ {{
+ "thinking": "Page loaded",
+ "evaluation_previous_goal": "Navigation completed",
+ "memory": "Page loaded",
+ "next_goal": "Done",
+ "action": [{{"done": {{"text": "{msg}", "success": true}}}}]
+ }}
+ """,
+ ]
+
+
+class TestHeavyPageNavigation:
+
+ async def test_slow_server_response_completes(self, browser_session, heavy_base_url):
+ """Navigation succeeds even when server takes 6s to respond."""
+ url = f'{heavy_base_url}/slow-server-pdp'
+ agent = Agent(
+ task=f'Navigate to {url}',
+ llm=create_mock_llm(actions=_nav_actions(url)),
+ browser_session=browser_session,
+ )
+ start = time.time()
+ history = await asyncio.wait_for(agent.run(max_steps=3), timeout=60)
+ assert len(history) > 0
+ assert history.final_result() is not None
+ assert time.time() - start >= 5, 'Should have waited for slow server'
+
+ async def test_redirect_chain_completes(self, browser_session, heavy_base_url):
+ """Navigation handles multi-step redirects + slow final response."""
+ url = f'{heavy_base_url}/redirect-step1'
+ agent = Agent(
+ task=f'Navigate to {url}',
+ llm=create_mock_llm(actions=_nav_actions(url)),
+ browser_session=browser_session,
+ )
+ history = await asyncio.wait_for(agent.run(max_steps=3), timeout=60)
+ assert len(history) > 0
+ assert history.final_result() is not None
+
+ async def test_navigate_event_accepts_domcontentloaded(self, browser_session, heavy_base_url):
+ """NavigateToUrlEvent with fast page should complete quickly via DOMContentLoaded/load."""
+ url = f'{heavy_base_url}/fast-dom-slow-load'
+ event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=url))
+ await asyncio.wait_for(event, timeout=15)
+ await event.event_result(raise_if_any=True, raise_if_none=False)
+
+ async def test_recovery_after_slow_navigation(self, browser_session, heavy_base_url):
+ """Agent recovers and navigates to a fast page after a slow one."""
+ slow_url = f'{heavy_base_url}/slow-server-pdp'
+ quick_url = f'{heavy_base_url}/quick-page'
+ actions = [
+ f"""
+ {{
+ "thinking": "Navigate to slow page",
+ "evaluation_previous_goal": "Starting",
+ "memory": "Going to slow page",
+ "next_goal": "Navigate",
+ "action": [{{"navigate": {{"url": "{slow_url}"}}}}]
+ }}
+ """,
+ f"""
+ {{
+ "thinking": "Now navigate to quick page",
+ "evaluation_previous_goal": "Slow page loaded",
+ "memory": "Trying quick page",
+ "next_goal": "Navigate",
+ "action": [{{"navigate": {{"url": "{quick_url}"}}}}]
+ }}
+ """,
+ """
+ {
+ "thinking": "Both done",
+ "evaluation_previous_goal": "Quick page loaded",
+ "memory": "Recovery successful",
+ "next_goal": "Done",
+ "action": [{"done": {"text": "Recovery succeeded", "success": true}}]
+ }
+ """,
+ ]
+ agent = Agent(
+ task='Navigate to slow then quick page',
+ llm=create_mock_llm(actions=actions),
+ browser_session=browser_session,
+ )
+ history = await asyncio.wait_for(agent.run(max_steps=4), timeout=90)
+ assert len(history) >= 2
+ assert history.final_result() is not None
+
+ async def test_event_timeout_sufficient_for_heavy_pages(self, browser_session):
+ """event_timeout should be >= 30s to handle slow servers + redirect chains."""
+ event = NavigateToUrlEvent(url='http://example.com')
+ assert event.event_timeout is not None
+ assert event.event_timeout >= 30.0, (
+ f'event_timeout={event.event_timeout}s is too low for heavy pages (need >= 30s)'
+ )