Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fixed backend_node_id based element counting.

Saurav Panda2aa23f01dcb0
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 9b1eec41d..820e8cb7d 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -2709,6 +2709,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
max_step_interval: float = 45.0,
summary_llm: BaseChatModel | None = None,
ai_step_llm: BaseChatModel | None = None,
+ wait_for_elements: bool = False,
) -> list[ActionResult]:
"""
Rerun a saved history of actions with error handling and retry logic.
@@ -2723,6 +2724,9 @@ class Agent(Generic[Context, AgentStructuredOutput]):
max_step_interval: Maximum delay from saved step_interval (caps LLM time from original run)
summary_llm: Optional LLM to use for generating the final summary. If not provided, uses the agent's LLM
ai_step_llm: Optional LLM to use for AI steps (extract actions). If not provided, uses the agent's LLM
+ wait_for_elements: If True, wait for minimum number of elements before attempting element
+ matching. Useful for SPA pages where shadow DOM content loads dynamically.
+ Default is False.
Returns:
List of action results (including AI summary as the final result)
@@ -2812,7 +2816,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
max_retry_delay = 30.0
while retry_count < max_retries:
try:
- result = await self._execute_history_step(history_item, step_delay, ai_step_llm)
+ result = await self._execute_history_step(history_item, step_delay, ai_step_llm, wait_for_elements)
results.extend(result)
step_succeeded = True
break
@@ -2944,54 +2948,69 @@ class Agent(Generic[Context, AgentStructuredOutput]):
def _count_expected_elements_from_history(self, history_item: AgentHistory) -> int:
"""Estimate the minimum number of elements expected based on history.
- Looks at the interacted elements and their indices to determine
- a reasonable minimum element count for the page.
+ Uses the action indices from the history to determine the minimum
+ number of elements the page should have. If an action targets index N,
+ the page needs at least N+1 elements in the selector_map.
"""
- if not history_item.state.interacted_element:
+ if not history_item.model_output or not history_item.model_output.action:
return 0
max_index = 0
- for elem in history_item.state.interacted_element:
- if elem and elem.backend_node_id:
- # Use backend_node_id as a rough indicator of element count
- # Elements with higher indices suggest more elements on the page
- max_index = max(max_index, elem.backend_node_id)
+ for action in history_item.model_output.action:
+ # Get the element index this action targets
+ index = action.get_index()
+ if index is not None:
+ max_index = max(max_index, index)
- # Return a reasonable minimum - at least the max index we're trying to interact with
- # but cap it to avoid waiting forever for very high indices
- return min(max_index, 50) if max_index > 0 else 0
+ # Need at least max_index + 1 elements (indices are 0-based)
+ # Cap at 50 to avoid waiting forever for very high indices
+ return min(max_index + 1, 50) if max_index > 0 else 0
async def _execute_history_step(
- self, history_item: AgentHistory, delay: float, ai_step_llm: BaseChatModel | None = None
+ self,
+ history_item: AgentHistory,
+ delay: float,
+ ai_step_llm: BaseChatModel | None = None,
+ wait_for_elements: bool = False,
) -> list[ActionResult]:
"""Execute a single step from history with element validation.
For extract actions, uses AI to re-evaluate the content since page content may have changed.
+
+ Args:
+ history_item: The history step to execute
+ delay: Delay before executing the step
+ ai_step_llm: Optional LLM to use for AI steps
+ wait_for_elements: If True, wait for minimum elements before element matching
"""
assert self.browser_session is not None, 'BrowserSession is not set up'
await asyncio.sleep(delay)
- # Determine if we need to wait for elements (actions that interact with DOM elements)
- needs_element_matching = False
- if history_item.model_output:
- for i, action in enumerate(history_item.model_output.action):
- action_data = action.model_dump(exclude_unset=True)
- action_name = next(iter(action_data.keys()), None)
- # Actions that need element matching
- if action_name in ('click', 'input', 'hover', 'select_option', 'drag_and_drop'):
- historical_elem = (
- history_item.state.interacted_element[i] if i < len(history_item.state.interacted_element) else None
- )
- if historical_elem is not None:
- needs_element_matching = True
- break
+ # Optionally wait for minimum elements before element matching (useful for SPAs)
+ if wait_for_elements:
+ # Determine if we need to wait for elements (actions that interact with DOM elements)
+ needs_element_matching = False
+ if history_item.model_output:
+ for i, action in enumerate(history_item.model_output.action):
+ action_data = action.model_dump(exclude_unset=True)
+ action_name = next(iter(action_data.keys()), None)
+ # Actions that need element matching
+ if action_name in ('click', 'input', 'hover', 'select_option', 'drag_and_drop'):
+ historical_elem = (
+ history_item.state.interacted_element[i] if i < len(history_item.state.interacted_element) else None
+ )
+ if historical_elem is not None:
+ needs_element_matching = True
+ break
- # If we need element matching, wait for minimum elements before proceeding
- if needs_element_matching:
- min_elements = self._count_expected_elements_from_history(history_item)
- if min_elements > 0:
- state = await self._wait_for_minimum_elements(min_elements, timeout=15.0, poll_interval=1.0)
+ # If we need element matching, wait for minimum elements before proceeding
+ if needs_element_matching:
+ min_elements = self._count_expected_elements_from_history(history_item)
+ if min_elements > 0:
+ state = await self._wait_for_minimum_elements(min_elements, timeout=15.0, poll_interval=1.0)
+ else:
+ state = await self.browser_session.get_browser_state_summary(include_screenshot=False)
else:
state = await self.browser_session.get_browser_state_summary(include_screenshot=False)
else:
diff --git a/tests/ci/test_rerun_ai_summary.py b/tests/ci/test_rerun_ai_summary.py
index b10ffb0ab..253395416 100644
--- a/tests/ci/test_rerun_ai_summary.py
+++ b/tests/ci/test_rerun_ai_summary.py
@@ -865,55 +865,13 @@ async def test_is_redundant_retry_step_detection():
async def test_count_expected_elements_from_history():
- """Test that _count_expected_elements_from_history correctly estimates element count."""
- from browser_use.dom.views import DOMInteractedElement
-
+ """Test that _count_expected_elements_from_history correctly estimates element count based on action indices."""
llm = create_mock_llm(actions=None)
agent = Agent(task='Test task', llm=llm)
AgentOutput = agent.AgentOutput
- # Create elements with different backend_node_ids
- element_low_id = DOMInteractedElement(
- node_id=1,
- backend_node_id=5,
- frame_id=None,
- node_type=NodeType.ELEMENT_NODE,
- node_value='',
- node_name='INPUT',
- attributes={'name': 'username'},
- x_path='html/body/input[1]',
- element_hash=12345,
- bounds=DOMRect(x=0, y=0, width=100, height=30),
- )
-
- element_high_id = DOMInteractedElement(
- node_id=2,
- backend_node_id=25,
- frame_id=None,
- node_type=NodeType.ELEMENT_NODE,
- node_value='',
- node_name='BUTTON',
- attributes={'id': 'submit'},
- x_path='html/body/button',
- element_hash=67890,
- bounds=DOMRect(x=0, y=0, width=100, height=30),
- )
-
- element_very_high_id = DOMInteractedElement(
- node_id=3,
- backend_node_id=100, # Very high - should be capped at 50
- frame_id=None,
- node_type=NodeType.ELEMENT_NODE,
- node_value='',
- node_name='DIV',
- attributes={},
- x_path='html/body/div',
- element_hash=11111,
- bounds=DOMRect(x=0, y=0, width=100, height=30),
- )
-
- # Test 1: History with low backend_node_id
- step_low = AgentHistory(
+ # Test 1: Action with low index (5) -> needs at least 6 elements (index + 1)
+ step_low_index = AgentHistory(
model_output=AgentOutput(
evaluation_previous_goal=None,
memory='Test',
@@ -925,13 +883,13 @@ async def test_count_expected_elements_from_history():
url='http://test.com',
title='Test',
tabs=[],
- interacted_element=[element_low_id],
+ interacted_element=[None],
),
metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),
)
- # Test 2: History with higher backend_node_id
- step_high = AgentHistory(
+ # Test 2: Action with higher index (25) -> needs at least 26 elements
+ step_high_index = AgentHistory(
model_output=AgentOutput(
evaluation_previous_goal=None,
memory='Test',
@@ -943,13 +901,13 @@ async def test_count_expected_elements_from_history():
url='http://test.com',
title='Test',
tabs=[],
- interacted_element=[element_high_id],
+ interacted_element=[None],
),
metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=2, step_interval=0.1),
)
- # Test 3: History with very high backend_node_id (should be capped)
- step_very_high = AgentHistory(
+ # Test 3: Action with very high index (100) -> capped at 50
+ step_very_high_index = AgentHistory(
model_output=AgentOutput(
evaluation_previous_goal=None,
memory='Test',
@@ -961,13 +919,13 @@ async def test_count_expected_elements_from_history():
url='http://test.com',
title='Test',
tabs=[],
- interacted_element=[element_very_high_id],
+ interacted_element=[None],
),
metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=3, step_interval=0.1),
)
- # Test 4: History with no interacted elements
- step_no_elements = AgentHistory(
+ # Test 4: Navigate action (no index) -> returns 0
+ step_no_index = AgentHistory(
model_output=AgentOutput(
evaluation_previous_goal=None,
memory='Test',
@@ -984,18 +942,42 @@ async def test_count_expected_elements_from_history():
metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=4, step_interval=0.1),
)
+ # Test 5: Multiple actions - uses max index
+ step_multiple_actions = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Test',
+ next_goal=None,
+ action=[
+ {'click': {'index': 3}}, # type: ignore[arg-type]
+ {'input': {'index': 10, 'text': 'test'}}, # type: ignore[arg-type]
+ ],
+ ),
+ result=[ActionResult(long_term_memory='Done'), ActionResult(long_term_memory='Done')],
+ state=BrowserStateHistory(
+ url='http://test.com',
+ title='Test',
+ tabs=[],
+ interacted_element=[None, None],
+ ),
+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=5, step_interval=0.1),
+ )
+
try:
- # Test 1: Low backend_node_id returns that id
- assert agent._count_expected_elements_from_history(step_low) == 5
+ # Test 1: Action index 5 -> needs 6 elements (index + 1)
+ assert agent._count_expected_elements_from_history(step_low_index) == 6
+
+ # Test 2: Action index 25 -> needs 26 elements
+ assert agent._count_expected_elements_from_history(step_high_index) == 26
- # Test 2: Higher backend_node_id returns that id
- assert agent._count_expected_elements_from_history(step_high) == 25
+ # Test 3: Action index 100 -> capped at 50
+ assert agent._count_expected_elements_from_history(step_very_high_index) == 50
- # Test 3: Very high backend_node_id is capped at 50
- assert agent._count_expected_elements_from_history(step_very_high) == 50
+ # Test 4: Navigate has no index -> returns 0
+ assert agent._count_expected_elements_from_history(step_no_index) == 0
- # Test 4: No interacted elements returns 0
- assert agent._count_expected_elements_from_history(step_no_elements) == 0
+ # Test 5: Multiple actions -> uses max index (10) + 1 = 11
+ assert agent._count_expected_elements_from_history(step_multiple_actions) == 11
finally:
await agent.close()
@@ -1154,12 +1136,13 @@ async def test_rerun_waits_for_elements_before_matching(httpserver):
history = AgentHistoryList(history=[navigate_step, click_step])
try:
- # Run rerun - it should wait for elements before trying to match
+ # Run rerun with wait_for_elements=True - should wait for elements before trying to match
results = await agent.rerun_history(
history,
skip_failures=True,
max_retries=1,
summary_llm=mock_summary_llm,
+ wait_for_elements=True, # Enable element waiting
)
# Should have results: navigate + click (or error if element not found) + summary