Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

added a fix for multi menu dropdown

Saurav Panda97d71b84fa69
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 448c410bc..82b8e5db4 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -2811,6 +2811,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
retry_count = 0
step_succeeded = False
+ menu_reopened = False # Track if we've already tried reopening the menu
# Exponential backoff: 5s base, doubling each retry, capped at 30s
base_retry_delay = 5.0
max_retry_delay = 30.0
@@ -2822,9 +2823,36 @@ class Agent(Generic[Context, AgentStructuredOutput]):
break
except Exception as e:
+ error_str = str(e)
retry_count += 1
+
+ # Check if this is a "Could not find matching element" error for a menu item
+ # If so, try to re-open the dropdown from the previous step before retrying
+ if (
+ not menu_reopened
+ and 'Could not find matching element' in error_str
+ and previous_item is not None
+ and self._is_menu_opener_step(previous_item)
+ ):
+ # Check if current step targets a menu item element
+ curr_elements = history_item.state.interacted_element if history_item.state else []
+ curr_elem = curr_elements[0] if curr_elements else None
+ if self._is_menu_item_element(curr_elem):
+ self.logger.info(
+ '🔄 Dropdown may have closed. Attempting to re-open by re-executing previous step...'
+ )
+ reopened = await self._reexecute_menu_opener(previous_item, ai_step_llm)
+ if reopened:
+ menu_reopened = True
+ # Don't increment retry_count for the menu reopen attempt
+ # Retry immediately with minimal delay
+ retry_count -= 1
+ step_delay = 0.5 # Use short delay after reopening
+ self.logger.info('🔄 Dropdown re-opened, retrying element match...')
+ continue
+
if retry_count == max_retries:
- error_msg = f'{step_name} failed after {max_retries} attempts: {str(e)}'
+ error_msg = f'{step_name} failed after {max_retries} attempts: {error_str}'
self.logger.error(error_msg)
# Always record the error in results so AI summary counts it correctly
results.append(ActionResult(error=error_msg))
@@ -3346,6 +3374,102 @@ class Agent(Generic[Context, AgentStructuredOutput]):
return True
+ def _is_menu_opener_step(self, history_item: AgentHistory | None) -> bool:
+ """
+ Detect if a step opens a dropdown/menu.
+
+ Checks for common patterns indicating a menu opener:
+ - Element has aria-haspopup attribute
+ - Element has data-gw-click="toggleSubMenu" (Guidewire pattern)
+ - Element has expand-button in class name
+ - Element role is "menuitem" with aria-expanded
+
+ Returns True if the step appears to open a dropdown/submenu.
+ """
+ if not history_item or not history_item.state.interacted_element:
+ return False
+
+ elem = history_item.state.interacted_element[0] if history_item.state.interacted_element else None
+ if not elem:
+ return False
+
+ attrs = elem.attributes or {}
+
+ # Check for common menu opener indicators
+ if attrs.get('aria-haspopup') in ('true', 'menu', 'listbox'):
+ return True
+ if attrs.get('data-gw-click') == 'toggleSubMenu':
+ return True
+ if 'expand-button' in attrs.get('class', ''):
+ return True
+ if attrs.get('role') == 'menuitem' and attrs.get('aria-expanded') in ('false', 'true'):
+ return True
+ if attrs.get('role') == 'button' and attrs.get('aria-expanded') in ('false', 'true'):
+ return True
+
+ return False
+
+ def _is_menu_item_element(self, elem: 'DOMInteractedElement | None') -> bool:
+ """
+ Detect if an element is a menu item that appears inside a dropdown/menu.
+
+ Checks for:
+ - role="menuitem", "option", "menuitemcheckbox", "menuitemradio"
+ - Element is inside a menu structure (has menu-related parent indicators)
+ - ax_name is set (menu items typically have accessible names)
+
+ Returns True if the element appears to be a menu item.
+ """
+ if not elem:
+ return False
+
+ attrs = elem.attributes or {}
+
+ # Check for menu item roles
+ role = attrs.get('role', '')
+ if role in ('menuitem', 'option', 'menuitemcheckbox', 'menuitemradio', 'treeitem'):
+ return True
+
+ # Elements in Guidewire menus have these patterns
+ if 'gw-action--inner' in attrs.get('class', ''):
+ return True
+ if 'menuitem' in attrs.get('class', '').lower():
+ return True
+
+ # If element has an ax_name and looks like it could be in a menu
+ # This is a softer check - only used if the previous step was a menu opener
+ if elem.ax_name and elem.ax_name not in ('', None):
+ # Common menu container classes
+ elem_class = attrs.get('class', '').lower()
+ if any(x in elem_class for x in ['dropdown', 'popup', 'menu', 'submenu', 'action']):
+ return True
+
+ return False
+
+ async def _reexecute_menu_opener(
+ self,
+ opener_item: AgentHistory,
+ ai_step_llm: 'BaseChatModel | None' = None,
+ ) -> bool:
+ """
+ Re-execute a menu opener step to re-open a closed dropdown.
+
+ This is used when a menu item can't be found because the dropdown
+ closed during the wait between steps.
+
+ Returns True if re-execution succeeded, False otherwise.
+ """
+ try:
+ self.logger.info('🔄 Re-opening dropdown/menu by re-executing previous step...')
+ # Use a minimal delay - we want to quickly re-open the menu
+ await self._execute_history_step(opener_item, delay=0.5, ai_step_llm=ai_step_llm, wait_for_elements=False)
+ # Small delay to let the menu render
+ await asyncio.sleep(0.3)
+ return True
+ except Exception as e:
+ self.logger.warning(f'Failed to re-open dropdown: {e}')
+ return False
+
async def load_and_rerun(
self,
history_file: str | Path | None = None,
diff --git a/tests/ci/test_ax_name_matching.py b/tests/ci/test_ax_name_matching.py
index 7e00538ca..9734f6c47 100644
--- a/tests/ci/test_ax_name_matching.py
+++ b/tests/ci/test_ax_name_matching.py
@@ -4,6 +4,9 @@ This tests Level 4 matching which uses the accessibility tree's name property
to match elements when hash, stable_hash, and xpath all fail.
This is particularly useful for dynamic SPAs where DOM structure changes
but accessible names remain stable.
+
+Also tests dropdown/menu re-opening behavior when menu items can't be found
+because the dropdown closed during the wait between steps.
"""
from unittest.mock import AsyncMock
@@ -355,3 +358,201 @@ async def test_ax_name_matching_before_attribute_matching(httpserver):
finally:
await agent.close()
+
+
+# Tests for dropdown/menu re-opening behavior
+
+
+def test_is_menu_opener_step_detects_aria_haspopup():
+ """Test that _is_menu_opener_step detects aria-haspopup elements."""
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ AgentOutput = agent.AgentOutput
+
+ # Element with aria-haspopup="true" should be detected as menu opener
+ opener_element = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='DIV',
+ attributes={'aria-haspopup': 'true', 'class': 'dropdown-trigger'},
+ x_path='html/body/div',
+ element_hash=12345,
+ stable_hash=12345,
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='Contact',
+ )
+
+ history_item = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Click dropdown',
+ next_goal=None,
+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Clicked')],
+ state=BrowserStateHistory(
+ url='http://test.com',
+ title='Test',
+ tabs=[],
+ interacted_element=[opener_element],
+ ),
+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),
+ )
+
+ assert agent._is_menu_opener_step(history_item) is True
+
+
+def test_is_menu_opener_step_detects_guidewire_toggle():
+ """Test that _is_menu_opener_step detects Guidewire toggleSubMenu pattern."""
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ AgentOutput = agent.AgentOutput
+
+ # Element with data-gw-click="toggleSubMenu" should be detected
+ opener_element = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='DIV',
+ attributes={'data-gw-click': 'toggleSubMenu', 'class': 'gw-action--expand-button'},
+ x_path='html/body/div',
+ element_hash=12345,
+ stable_hash=12345,
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name=None,
+ )
+
+ history_item = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Toggle menu',
+ next_goal=None,
+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Toggled')],
+ state=BrowserStateHistory(
+ url='http://test.com',
+ title='Test',
+ tabs=[],
+ interacted_element=[opener_element],
+ ),
+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),
+ )
+
+ assert agent._is_menu_opener_step(history_item) is True
+
+
+def test_is_menu_opener_step_returns_false_for_regular_element():
+ """Test that _is_menu_opener_step returns False for non-menu elements."""
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ AgentOutput = agent.AgentOutput
+
+ # Regular button without menu attributes
+ regular_element = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='BUTTON',
+ attributes={'class': 'submit-btn', 'type': 'submit'},
+ x_path='html/body/button',
+ element_hash=12345,
+ stable_hash=12345,
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='Submit',
+ )
+
+ history_item = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Click submit',
+ next_goal=None,
+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Clicked')],
+ state=BrowserStateHistory(
+ url='http://test.com',
+ title='Test',
+ tabs=[],
+ interacted_element=[regular_element],
+ ),
+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),
+ )
+
+ assert agent._is_menu_opener_step(history_item) is False
+
+
+def test_is_menu_item_element_detects_role_menuitem():
+ """Test that _is_menu_item_element detects role=menuitem."""
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+
+ menu_item = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='DIV',
+ attributes={'role': 'menuitem', 'class': 'menu-option'},
+ x_path='html/body/div/div',
+ element_hash=12345,
+ stable_hash=12345,
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='New Contact',
+ )
+
+ assert agent._is_menu_item_element(menu_item) is True
+
+
+def test_is_menu_item_element_detects_guidewire_class():
+ """Test that _is_menu_item_element detects Guidewire gw-action--inner class."""
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+
+ menu_item = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='DIV',
+ attributes={'class': 'gw-action--inner gw-hasDivider', 'aria-haspopup': 'true'},
+ x_path='html/body/div/div',
+ element_hash=12345,
+ stable_hash=12345,
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='New Contact',
+ )
+
+ assert agent._is_menu_item_element(menu_item) is True
+
+
+def test_is_menu_item_element_returns_false_for_regular_element():
+ """Test that _is_menu_item_element returns False for non-menu elements."""
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+
+ regular_element = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='BUTTON',
+ attributes={'class': 'submit-btn', 'type': 'submit'},
+ x_path='html/body/button',
+ element_hash=12345,
+ stable_hash=12345,
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='Submit',
+ )
+
+ assert agent._is_menu_item_element(regular_element) is False