Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fixed backend_node_id based element counting.
Saurav Panda2aa23f01dcb0
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 9b1eec41d..820e8cb7d 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -2709,6 +2709,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):max_step_interval: float = 45.0,summary_llm: BaseChatModel | None = None,ai_step_llm: BaseChatModel | None = None,+ wait_for_elements: bool = False,) -> list[ActionResult]:"""Rerun a saved history of actions with error handling and retry logic.@@ -2723,6 +2724,9 @@ class Agent(Generic[Context, AgentStructuredOutput]):max_step_interval: Maximum delay from saved step_interval (caps LLM time from original run)summary_llm: Optional LLM to use for generating the final summary. If not provided, uses the agent's LLMai_step_llm: Optional LLM to use for AI steps (extract actions). If not provided, uses the agent's LLM+ wait_for_elements: If True, wait for minimum number of elements before attempting element+ matching. Useful for SPA pages where shadow DOM content loads dynamically.+ Default is False.Returns:List of action results (including AI summary as the final result)@@ -2812,7 +2816,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):max_retry_delay = 30.0while retry_count < max_retries:try:- result = await self._execute_history_step(history_item, step_delay, ai_step_llm)+ result = await self._execute_history_step(history_item, step_delay, ai_step_llm, wait_for_elements)results.extend(result)step_succeeded = Truebreak@@ -2944,54 +2948,69 @@ class Agent(Generic[Context, AgentStructuredOutput]):def _count_expected_elements_from_history(self, history_item: AgentHistory) -> int:"""Estimate the minimum number of elements expected based on history.- Looks at the interacted elements and their indices to determine- a reasonable minimum element count for the page.+ Uses the action indices from the history to determine the minimum+ number of elements the page should have. If an action targets index N,+ the page needs at least N+1 elements in the selector_map."""- if not history_item.state.interacted_element:+ if not history_item.model_output or not history_item.model_output.action:return 0max_index = 0- for elem in history_item.state.interacted_element:- if elem and elem.backend_node_id:- # Use backend_node_id as a rough indicator of element count- # Elements with higher indices suggest more elements on the page- max_index = max(max_index, elem.backend_node_id)+ for action in history_item.model_output.action:+ # Get the element index this action targets+ index = action.get_index()+ if index is not None:+ max_index = max(max_index, index)- # Return a reasonable minimum - at least the max index we're trying to interact with- # but cap it to avoid waiting forever for very high indices- return min(max_index, 50) if max_index > 0 else 0+ # Need at least max_index + 1 elements (indices are 0-based)+ # Cap at 50 to avoid waiting forever for very high indices+ return min(max_index + 1, 50) if max_index > 0 else 0async def _execute_history_step(- self, history_item: AgentHistory, delay: float, ai_step_llm: BaseChatModel | None = None+ self,+ history_item: AgentHistory,+ delay: float,+ ai_step_llm: BaseChatModel | None = None,+ wait_for_elements: bool = False,) -> list[ActionResult]:"""Execute a single step from history with element validation.For extract actions, uses AI to re-evaluate the content since page content may have changed.++ Args:+ history_item: The history step to execute+ delay: Delay before executing the step+ ai_step_llm: Optional LLM to use for AI steps+ wait_for_elements: If True, wait for minimum elements before element matching"""assert self.browser_session is not None, 'BrowserSession is not set up'await asyncio.sleep(delay)- # Determine if we need to wait for elements (actions that interact with DOM elements)- needs_element_matching = False- if history_item.model_output:- for i, action in enumerate(history_item.model_output.action):- action_data = action.model_dump(exclude_unset=True)- action_name = next(iter(action_data.keys()), None)- # Actions that need element matching- if action_name in ('click', 'input', 'hover', 'select_option', 'drag_and_drop'):- historical_elem = (- history_item.state.interacted_element[i] if i < len(history_item.state.interacted_element) else None- )- if historical_elem is not None:- needs_element_matching = True- break+ # Optionally wait for minimum elements before element matching (useful for SPAs)+ if wait_for_elements:+ # Determine if we need to wait for elements (actions that interact with DOM elements)+ needs_element_matching = False+ if history_item.model_output:+ for i, action in enumerate(history_item.model_output.action):+ action_data = action.model_dump(exclude_unset=True)+ action_name = next(iter(action_data.keys()), None)+ # Actions that need element matching+ if action_name in ('click', 'input', 'hover', 'select_option', 'drag_and_drop'):+ historical_elem = (+ history_item.state.interacted_element[i] if i < len(history_item.state.interacted_element) else None+ )+ if historical_elem is not None:+ needs_element_matching = True+ break- # If we need element matching, wait for minimum elements before proceeding- if needs_element_matching:- min_elements = self._count_expected_elements_from_history(history_item)- if min_elements > 0:- state = await self._wait_for_minimum_elements(min_elements, timeout=15.0, poll_interval=1.0)+ # If we need element matching, wait for minimum elements before proceeding+ if needs_element_matching:+ min_elements = self._count_expected_elements_from_history(history_item)+ if min_elements > 0:+ state = await self._wait_for_minimum_elements(min_elements, timeout=15.0, poll_interval=1.0)+ else:+ state = await self.browser_session.get_browser_state_summary(include_screenshot=False)else:state = await self.browser_session.get_browser_state_summary(include_screenshot=False)else:diff --git a/tests/ci/test_rerun_ai_summary.py b/tests/ci/test_rerun_ai_summary.pyindex b10ffb0ab..253395416 100644--- a/tests/ci/test_rerun_ai_summary.py+++ b/tests/ci/test_rerun_ai_summary.py@@ -865,55 +865,13 @@ async def test_is_redundant_retry_step_detection():async def test_count_expected_elements_from_history():- """Test that _count_expected_elements_from_history correctly estimates element count."""- from browser_use.dom.views import DOMInteractedElement-+ """Test that _count_expected_elements_from_history correctly estimates element count based on action indices."""llm = create_mock_llm(actions=None)agent = Agent(task='Test task', llm=llm)AgentOutput = agent.AgentOutput- # Create elements with different backend_node_ids- element_low_id = DOMInteractedElement(- node_id=1,- backend_node_id=5,- frame_id=None,- node_type=NodeType.ELEMENT_NODE,- node_value='',- node_name='INPUT',- attributes={'name': 'username'},- x_path='html/body/input[1]',- element_hash=12345,- bounds=DOMRect(x=0, y=0, width=100, height=30),- )-- element_high_id = DOMInteractedElement(- node_id=2,- backend_node_id=25,- frame_id=None,- node_type=NodeType.ELEMENT_NODE,- node_value='',- node_name='BUTTON',- attributes={'id': 'submit'},- x_path='html/body/button',- element_hash=67890,- bounds=DOMRect(x=0, y=0, width=100, height=30),- )-- element_very_high_id = DOMInteractedElement(- node_id=3,- backend_node_id=100, # Very high - should be capped at 50- frame_id=None,- node_type=NodeType.ELEMENT_NODE,- node_value='',- node_name='DIV',- attributes={},- x_path='html/body/div',- element_hash=11111,- bounds=DOMRect(x=0, y=0, width=100, height=30),- )-- # Test 1: History with low backend_node_id- step_low = AgentHistory(+ # Test 1: Action with low index (5) -> needs at least 6 elements (index + 1)+ step_low_index = AgentHistory(model_output=AgentOutput(evaluation_previous_goal=None,memory='Test',@@ -925,13 +883,13 @@ async def test_count_expected_elements_from_history():url='http://test.com',title='Test',tabs=[],- interacted_element=[element_low_id],+ interacted_element=[None],),metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),)- # Test 2: History with higher backend_node_id- step_high = AgentHistory(+ # Test 2: Action with higher index (25) -> needs at least 26 elements+ step_high_index = AgentHistory(model_output=AgentOutput(evaluation_previous_goal=None,memory='Test',@@ -943,13 +901,13 @@ async def test_count_expected_elements_from_history():url='http://test.com',title='Test',tabs=[],- interacted_element=[element_high_id],+ interacted_element=[None],),metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=2, step_interval=0.1),)- # Test 3: History with very high backend_node_id (should be capped)- step_very_high = AgentHistory(+ # Test 3: Action with very high index (100) -> capped at 50+ step_very_high_index = AgentHistory(model_output=AgentOutput(evaluation_previous_goal=None,memory='Test',@@ -961,13 +919,13 @@ async def test_count_expected_elements_from_history():url='http://test.com',title='Test',tabs=[],- interacted_element=[element_very_high_id],+ interacted_element=[None],),metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=3, step_interval=0.1),)- # Test 4: History with no interacted elements- step_no_elements = AgentHistory(+ # Test 4: Navigate action (no index) -> returns 0+ step_no_index = AgentHistory(model_output=AgentOutput(evaluation_previous_goal=None,memory='Test',@@ -984,18 +942,42 @@ async def test_count_expected_elements_from_history():metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=4, step_interval=0.1),)+ # Test 5: Multiple actions - uses max index+ step_multiple_actions = AgentHistory(+ model_output=AgentOutput(+ evaluation_previous_goal=None,+ memory='Test',+ next_goal=None,+ action=[+ {'click': {'index': 3}}, # type: ignore[arg-type]+ {'input': {'index': 10, 'text': 'test'}}, # type: ignore[arg-type]+ ],+ ),+ result=[ActionResult(long_term_memory='Done'), ActionResult(long_term_memory='Done')],+ state=BrowserStateHistory(+ url='http://test.com',+ title='Test',+ tabs=[],+ interacted_element=[None, None],+ ),+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=5, step_interval=0.1),+ )+try:- # Test 1: Low backend_node_id returns that id- assert agent._count_expected_elements_from_history(step_low) == 5+ # Test 1: Action index 5 -> needs 6 elements (index + 1)+ assert agent._count_expected_elements_from_history(step_low_index) == 6++ # Test 2: Action index 25 -> needs 26 elements+ assert agent._count_expected_elements_from_history(step_high_index) == 26- # Test 2: Higher backend_node_id returns that id- assert agent._count_expected_elements_from_history(step_high) == 25+ # Test 3: Action index 100 -> capped at 50+ assert agent._count_expected_elements_from_history(step_very_high_index) == 50- # Test 3: Very high backend_node_id is capped at 50- assert agent._count_expected_elements_from_history(step_very_high) == 50+ # Test 4: Navigate has no index -> returns 0+ assert agent._count_expected_elements_from_history(step_no_index) == 0- # Test 4: No interacted elements returns 0- assert agent._count_expected_elements_from_history(step_no_elements) == 0+ # Test 5: Multiple actions -> uses max index (10) + 1 = 11+ assert agent._count_expected_elements_from_history(step_multiple_actions) == 11finally:await agent.close()@@ -1154,12 +1136,13 @@ async def test_rerun_waits_for_elements_before_matching(httpserver):history = AgentHistoryList(history=[navigate_step, click_step])try:- # Run rerun - it should wait for elements before trying to match+ # Run rerun with wait_for_elements=True - should wait for elements before trying to matchresults = await agent.rerun_history(history,skip_failures=True,max_retries=1,summary_llm=mock_summary_llm,+ wait_for_elements=True, # Enable element waiting)# Should have results: navigate + click (or error if element not found) + summary