Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix(browser): expose minimal state after capture timeout

MagMuellere7cc40f5adee
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index 00c8c946d..4d8bcd958 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -298,6 +298,10 @@ class AgentMessagePrompt:
current_tab_text = f'Current tab: {current_target_id[-4:]}' if current_target_id is not None else ''
+ state_error_text = ''
+ if state_error := getattr(self.browser_state, 'state_error', None):
+ state_error_text = f'<browser_state_error>{state_error}</browser_state_error>\n'
+
# Check if current page is a PDF viewer and add appropriate message
pdf_message = ''
if self.browser_state.is_pdf_viewer:
@@ -325,7 +329,7 @@ class AgentMessagePrompt:
Available tabs:
{tabs_text}
{page_info_text}
-{recent_events_text}{closed_popups_text}{pdf_message}Interactive elements{truncated_text}:
+{state_error_text}{recent_events_text}{closed_popups_text}{pdf_message}Interactive elements{truncated_text}:
{elements_text}
"""
return browser_state
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 55424f82f..466b4a928 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -54,7 +54,7 @@ from browser_use.browser.events import (
)
from browser_use.browser.profile import BrowserProfile, ProxySettings
from browser_use.browser.views import BrowserStateSummary, TabInfo
-from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, TargetInfo
+from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, SerializedDOMState, TargetInfo
from browser_use.observability import observe_debug
from browser_use.utils import _log_pretty_url, create_task_with_error_handling, is_new_tab_page
@@ -1617,8 +1617,58 @@ class BrowserSession(BaseModel):
),
)
- # The handler returns the BrowserStateSummary directly
- result = await event.event_result(raise_if_none=True, raise_if_any=True)
+ # The handler returns the BrowserStateSummary directly. If the complete state
+ # request times out, return a non-actionable state so the model can recover
+ # without exposing selectors from an earlier page.
+ try:
+ result = await event.event_result(raise_if_none=True, raise_if_any=True)
+ except TimeoutError:
+ state_error = (
+ 'Browser state capture timed out. The current DOM and screenshot are unavailable, '
+ 'so no element indices are safe to use. Recover with navigation, waiting, or another non-indexed action.'
+ )
+ empty_dom_state = SerializedDOMState(_root=None, selector_map={})
+
+ # Clear every action lookup path before calling the model.
+ self.update_cached_selector_map({})
+ if self._dom_watchdog is not None:
+ self._dom_watchdog.selector_map = {}
+ self._dom_watchdog.current_dom_state = empty_dom_state
+ self._dom_watchdog.enhanced_dom_tree = None
+
+ cached_state = self._cached_browser_state_summary
+ current_target = (
+ self.session_manager.get_target(self.agent_focus_target_id)
+ if self.session_manager is not None and self.agent_focus_target_id is not None
+ else None
+ )
+ url = (
+ current_target.url
+ if current_target and current_target.url
+ else cached_state.url
+ if cached_state
+ else 'about:blank'
+ )
+ title = (
+ current_target.title
+ if current_target and current_target.title
+ else cached_state.title
+ if cached_state
+ else 'Browser state unavailable'
+ )
+ tabs = [TabInfo(url=url, title=title, target_id=current_target.target_id)] if current_target else []
+
+ result = BrowserStateSummary(
+ dom_state=empty_dom_state,
+ url=url,
+ title=title,
+ tabs=tabs,
+ screenshot=None,
+ browser_errors=[state_error],
+ state_error=state_error,
+ )
+ self._cached_browser_state_summary = result
+
assert result is not None and result.dom_state is not None
return result
diff --git a/browser_use/browser/views.py b/browser_use/browser/views.py
index 962a3c84c..98cb26239 100644
--- a/browser_use/browser/views.py
+++ b/browser_use/browser/views.py
@@ -109,6 +109,7 @@ class BrowserStateSummary:
pending_network_requests: list[NetworkRequest] = field(default_factory=list) # Currently loading network requests
pagination_buttons: list[PaginationButton] = field(default_factory=list) # Detected pagination buttons
closed_popup_messages: list[str] = field(default_factory=list) # Messages from auto-closed JavaScript dialogs
+ state_error: str | None = None # Safe, model-visible explanation when the current state could not be captured
@dataclass
diff --git a/tests/ci/browser/test_dom_listener_detection.py b/tests/ci/browser/test_dom_listener_detection.py
index 37eed4b3d..3a85977ca 100644
--- a/tests/ci/browser/test_dom_listener_detection.py
+++ b/tests/ci/browser/test_dom_listener_detection.py
@@ -5,6 +5,8 @@ from collections import Counter
import pytest
+from browser_use.agent.service import Agent
+from browser_use.browser.events import BrowserStateRequestEvent
from browser_use.browser.profile import BrowserProfile
from browser_use.browser.session import BrowserSession
from browser_use.browser.watchdogs import dom_watchdog
@@ -113,8 +115,10 @@ async def test_screenshot_timeout_preserves_structural_dom(httpserver, browser_s
assert any(node.attributes.get('id') == 'continue' for node in state.dom_state.selector_map.values())
-async def test_whole_state_timeout_does_not_reuse_cached_actionable_dom(httpserver, browser_session: BrowserSession, monkeypatch):
- """A whole-state timeout must not expose selectors from an earlier page state."""
+async def test_whole_state_timeout_returns_model_visible_non_actionable_state(
+ httpserver, browser_session: BrowserSession, monkeypatch
+):
+ """A whole-state timeout must call the model without exposing earlier selectors."""
httpserver.expect_request('/cached-state').respond_with_data(
'<html><body><button id="continue">Continue</button></body></html>',
content_type='text/html',
@@ -127,7 +131,66 @@ async def test_whole_state_timeout_does_not_reuse_cached_actionable_dom(httpserv
async def event_result(self, **_kwargs):
raise TimeoutError
+ original_dispatch = browser_session.event_bus.dispatch
+ monkeypatch.setattr(
+ browser_session.event_bus,
+ 'dispatch',
+ lambda event: TimedOutStateEvent() if isinstance(event, BrowserStateRequestEvent) else original_dispatch(event),
+ )
+
+ state = await browser_session.get_browser_state_summary(include_screenshot=True)
+
+ assert state.dom_state.selector_map == {}
+ assert await browser_session.get_selector_map() == {}
+ assert state.screenshot is None
+ assert state.state_error is not None
+ assert 'no element indices are safe to use' in state.state_error
+ assert browser_session._cached_browser_state_summary is state
+
+
+async def test_initial_state_timeout_still_returns_model_visible_state(browser_session: BrowserSession, monkeypatch):
+ """The first state timeout must not require an earlier cached state."""
+
+ class TimedOutStateEvent:
+ async def event_result(self, **_kwargs):
+ raise TimeoutError
+
+ assert browser_session._cached_browser_state_summary is None
monkeypatch.setattr(browser_session.event_bus, 'dispatch', lambda _event: TimedOutStateEvent())
- with pytest.raises(TimeoutError):
- await browser_session.get_browser_state_summary(include_screenshot=True)
+ state = await browser_session.get_browser_state_summary(include_screenshot=True)
+
+ assert state.dom_state.selector_map == {}
+ assert state.screenshot is None
+ assert state.state_error is not None
+ assert state.url
+
+
+async def test_agent_still_calls_model_after_state_timeout(browser_session: BrowserSession, mock_llm, monkeypatch):
+ """Returning the minimal state must let the normal model/action loop continue."""
+
+ class TimedOutStateEvent:
+ async def event_result(self, **_kwargs):
+ raise TimeoutError
+
+ original_dispatch = browser_session.event_bus.dispatch
+ monkeypatch.setattr(
+ browser_session.event_bus,
+ 'dispatch',
+ lambda event: TimedOutStateEvent() if isinstance(event, BrowserStateRequestEvent) else original_dispatch(event),
+ )
+ model_call_count = 0
+ original_ainvoke = mock_llm.ainvoke
+
+ async def counted_ainvoke(*args, **kwargs):
+ nonlocal model_call_count
+ model_call_count += 1
+ return await original_ainvoke(*args, **kwargs)
+
+ monkeypatch.setattr(mock_llm, 'ainvoke', counted_ainvoke)
+ agent = Agent(task='Finish after inspecting the available state.', llm=mock_llm, browser_session=browser_session)
+
+ history = await agent.run(max_steps=1)
+
+ assert model_call_count >= 1
+ assert history.is_done() is True
diff --git a/tests/ci/test_prompt_step_meta_suffix.py b/tests/ci/test_prompt_step_meta_suffix.py
index 91e415b03..80116a0e3 100644
--- a/tests/ci/test_prompt_step_meta_suffix.py
+++ b/tests/ci/test_prompt_step_meta_suffix.py
@@ -104,3 +104,16 @@ def test_agent_state_block_unaffected_by_step_change(tmp_path):
return s[s.index('<agent_state>') : s.index('</agent_state>')]
assert agent_state_block(a) == agent_state_block(b)
+
+
+def test_browser_state_error_is_visible_to_model(tmp_path):
+ prompt = _make_prompt(tmp_path)
+ prompt.browser_state.state_error = (
+ 'Browser state capture timed out. The current DOM and screenshot are unavailable, so no element indices are safe to use.'
+ )
+
+ content = prompt.get_user_message(use_vision=False).content
+
+ assert isinstance(content, str)
+ assert '<browser_state_error>' in content
+ assert 'no element indices are safe to use' in content