Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
added a fix for multi menu dropdown
Saurav Panda97d71b84fa69
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 448c410bc..82b8e5db4 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -2811,6 +2811,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):retry_count = 0step_succeeded = False+ menu_reopened = False # Track if we've already tried reopening the menu# Exponential backoff: 5s base, doubling each retry, capped at 30sbase_retry_delay = 5.0max_retry_delay = 30.0@@ -2822,9 +2823,36 @@ class Agent(Generic[Context, AgentStructuredOutput]):breakexcept Exception as e:+ error_str = str(e)retry_count += 1++ # Check if this is a "Could not find matching element" error for a menu item+ # If so, try to re-open the dropdown from the previous step before retrying+ if (+ not menu_reopened+ and 'Could not find matching element' in error_str+ and previous_item is not None+ and self._is_menu_opener_step(previous_item)+ ):+ # Check if current step targets a menu item element+ curr_elements = history_item.state.interacted_element if history_item.state else []+ curr_elem = curr_elements[0] if curr_elements else None+ if self._is_menu_item_element(curr_elem):+ self.logger.info(+ '🔄 Dropdown may have closed. Attempting to re-open by re-executing previous step...'+ )+ reopened = await self._reexecute_menu_opener(previous_item, ai_step_llm)+ if reopened:+ menu_reopened = True+ # Don't increment retry_count for the menu reopen attempt+ # Retry immediately with minimal delay+ retry_count -= 1+ step_delay = 0.5 # Use short delay after reopening+ self.logger.info('🔄 Dropdown re-opened, retrying element match...')+ continue+if retry_count == max_retries:- error_msg = f'{step_name} failed after {max_retries} attempts: {str(e)}'+ error_msg = f'{step_name} failed after {max_retries} attempts: {error_str}'self.logger.error(error_msg)# Always record the error in results so AI summary counts it correctlyresults.append(ActionResult(error=error_msg))@@ -3346,6 +3374,102 @@ class Agent(Generic[Context, AgentStructuredOutput]):return True+ def _is_menu_opener_step(self, history_item: AgentHistory | None) -> bool:+ """+ Detect if a step opens a dropdown/menu.++ Checks for common patterns indicating a menu opener:+ - Element has aria-haspopup attribute+ - Element has data-gw-click="toggleSubMenu" (Guidewire pattern)+ - Element has expand-button in class name+ - Element role is "menuitem" with aria-expanded++ Returns True if the step appears to open a dropdown/submenu.+ """+ if not history_item or not history_item.state.interacted_element:+ return False++ elem = history_item.state.interacted_element[0] if history_item.state.interacted_element else None+ if not elem:+ return False++ attrs = elem.attributes or {}++ # Check for common menu opener indicators+ if attrs.get('aria-haspopup') in ('true', 'menu', 'listbox'):+ return True+ if attrs.get('data-gw-click') == 'toggleSubMenu':+ return True+ if 'expand-button' in attrs.get('class', ''):+ return True+ if attrs.get('role') == 'menuitem' and attrs.get('aria-expanded') in ('false', 'true'):+ return True+ if attrs.get('role') == 'button' and attrs.get('aria-expanded') in ('false', 'true'):+ return True++ return False++ def _is_menu_item_element(self, elem: 'DOMInteractedElement | None') -> bool:+ """+ Detect if an element is a menu item that appears inside a dropdown/menu.++ Checks for:+ - role="menuitem", "option", "menuitemcheckbox", "menuitemradio"+ - Element is inside a menu structure (has menu-related parent indicators)+ - ax_name is set (menu items typically have accessible names)++ Returns True if the element appears to be a menu item.+ """+ if not elem:+ return False++ attrs = elem.attributes or {}++ # Check for menu item roles+ role = attrs.get('role', '')+ if role in ('menuitem', 'option', 'menuitemcheckbox', 'menuitemradio', 'treeitem'):+ return True++ # Elements in Guidewire menus have these patterns+ if 'gw-action--inner' in attrs.get('class', ''):+ return True+ if 'menuitem' in attrs.get('class', '').lower():+ return True++ # If element has an ax_name and looks like it could be in a menu+ # This is a softer check - only used if the previous step was a menu opener+ if elem.ax_name and elem.ax_name not in ('', None):+ # Common menu container classes+ elem_class = attrs.get('class', '').lower()+ if any(x in elem_class for x in ['dropdown', 'popup', 'menu', 'submenu', 'action']):+ return True++ return False++ async def _reexecute_menu_opener(+ self,+ opener_item: AgentHistory,+ ai_step_llm: 'BaseChatModel | None' = None,+ ) -> bool:+ """+ Re-execute a menu opener step to re-open a closed dropdown.++ This is used when a menu item can't be found because the dropdown+ closed during the wait between steps.++ Returns True if re-execution succeeded, False otherwise.+ """+ try:+ self.logger.info('🔄 Re-opening dropdown/menu by re-executing previous step...')+ # Use a minimal delay - we want to quickly re-open the menu+ await self._execute_history_step(opener_item, delay=0.5, ai_step_llm=ai_step_llm, wait_for_elements=False)+ # Small delay to let the menu render+ await asyncio.sleep(0.3)+ return True+ except Exception as e:+ self.logger.warning(f'Failed to re-open dropdown: {e}')+ return False+async def load_and_rerun(self,history_file: str | Path | None = None,diff --git a/tests/ci/test_ax_name_matching.py b/tests/ci/test_ax_name_matching.pyindex 7e00538ca..9734f6c47 100644--- a/tests/ci/test_ax_name_matching.py+++ b/tests/ci/test_ax_name_matching.py@@ -4,6 +4,9 @@ This tests Level 4 matching which uses the accessibility tree's name propertyto match elements when hash, stable_hash, and xpath all fail.This is particularly useful for dynamic SPAs where DOM structure changesbut accessible names remain stable.++Also tests dropdown/menu re-opening behavior when menu items can't be found+because the dropdown closed during the wait between steps."""from unittest.mock import AsyncMock@@ -355,3 +358,201 @@ async def test_ax_name_matching_before_attribute_matching(httpserver):finally:await agent.close()+++# Tests for dropdown/menu re-opening behavior+++def test_is_menu_opener_step_detects_aria_haspopup():+ """Test that _is_menu_opener_step detects aria-haspopup elements."""+ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)+ AgentOutput = agent.AgentOutput++ # Element with aria-haspopup="true" should be detected as menu opener+ opener_element = DOMInteractedElement(+ node_id=1,+ backend_node_id=1,+ frame_id=None,+ node_type=NodeType.ELEMENT_NODE,+ node_value='',+ node_name='DIV',+ attributes={'aria-haspopup': 'true', 'class': 'dropdown-trigger'},+ x_path='html/body/div',+ element_hash=12345,+ stable_hash=12345,+ bounds=DOMRect(x=0, y=0, width=100, height=50),+ ax_name='Contact',+ )++ history_item = AgentHistory(+ model_output=AgentOutput(+ evaluation_previous_goal=None,+ memory='Click dropdown',+ next_goal=None,+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]+ ),+ result=[ActionResult(long_term_memory='Clicked')],+ state=BrowserStateHistory(+ url='http://test.com',+ title='Test',+ tabs=[],+ interacted_element=[opener_element],+ ),+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),+ )++ assert agent._is_menu_opener_step(history_item) is True+++def test_is_menu_opener_step_detects_guidewire_toggle():+ """Test that _is_menu_opener_step detects Guidewire toggleSubMenu pattern."""+ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)+ AgentOutput = agent.AgentOutput++ # Element with data-gw-click="toggleSubMenu" should be detected+ opener_element = DOMInteractedElement(+ node_id=1,+ backend_node_id=1,+ frame_id=None,+ node_type=NodeType.ELEMENT_NODE,+ node_value='',+ node_name='DIV',+ attributes={'data-gw-click': 'toggleSubMenu', 'class': 'gw-action--expand-button'},+ x_path='html/body/div',+ element_hash=12345,+ stable_hash=12345,+ bounds=DOMRect(x=0, y=0, width=100, height=50),+ ax_name=None,+ )++ history_item = AgentHistory(+ model_output=AgentOutput(+ evaluation_previous_goal=None,+ memory='Toggle menu',+ next_goal=None,+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]+ ),+ result=[ActionResult(long_term_memory='Toggled')],+ state=BrowserStateHistory(+ url='http://test.com',+ title='Test',+ tabs=[],+ interacted_element=[opener_element],+ ),+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),+ )++ assert agent._is_menu_opener_step(history_item) is True+++def test_is_menu_opener_step_returns_false_for_regular_element():+ """Test that _is_menu_opener_step returns False for non-menu elements."""+ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)+ AgentOutput = agent.AgentOutput++ # Regular button without menu attributes+ regular_element = DOMInteractedElement(+ node_id=1,+ backend_node_id=1,+ frame_id=None,+ node_type=NodeType.ELEMENT_NODE,+ node_value='',+ node_name='BUTTON',+ attributes={'class': 'submit-btn', 'type': 'submit'},+ x_path='html/body/button',+ element_hash=12345,+ stable_hash=12345,+ bounds=DOMRect(x=0, y=0, width=100, height=50),+ ax_name='Submit',+ )++ history_item = AgentHistory(+ model_output=AgentOutput(+ evaluation_previous_goal=None,+ memory='Click submit',+ next_goal=None,+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]+ ),+ result=[ActionResult(long_term_memory='Clicked')],+ state=BrowserStateHistory(+ url='http://test.com',+ title='Test',+ tabs=[],+ interacted_element=[regular_element],+ ),+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),+ )++ assert agent._is_menu_opener_step(history_item) is False+++def test_is_menu_item_element_detects_role_menuitem():+ """Test that _is_menu_item_element detects role=menuitem."""+ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)++ menu_item = DOMInteractedElement(+ node_id=1,+ backend_node_id=1,+ frame_id=None,+ node_type=NodeType.ELEMENT_NODE,+ node_value='',+ node_name='DIV',+ attributes={'role': 'menuitem', 'class': 'menu-option'},+ x_path='html/body/div/div',+ element_hash=12345,+ stable_hash=12345,+ bounds=DOMRect(x=0, y=0, width=100, height=50),+ ax_name='New Contact',+ )++ assert agent._is_menu_item_element(menu_item) is True+++def test_is_menu_item_element_detects_guidewire_class():+ """Test that _is_menu_item_element detects Guidewire gw-action--inner class."""+ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)++ menu_item = DOMInteractedElement(+ node_id=1,+ backend_node_id=1,+ frame_id=None,+ node_type=NodeType.ELEMENT_NODE,+ node_value='',+ node_name='DIV',+ attributes={'class': 'gw-action--inner gw-hasDivider', 'aria-haspopup': 'true'},+ x_path='html/body/div/div',+ element_hash=12345,+ stable_hash=12345,+ bounds=DOMRect(x=0, y=0, width=100, height=50),+ ax_name='New Contact',+ )++ assert agent._is_menu_item_element(menu_item) is True+++def test_is_menu_item_element_returns_false_for_regular_element():+ """Test that _is_menu_item_element returns False for non-menu elements."""+ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)++ regular_element = DOMInteractedElement(+ node_id=1,+ backend_node_id=1,+ frame_id=None,+ node_type=NodeType.ELEMENT_NODE,+ node_value='',+ node_name='BUTTON',+ attributes={'class': 'submit-btn', 'type': 'submit'},+ x_path='html/body/button',+ element_hash=12345,+ stable_hash=12345,+ bounds=DOMRect(x=0, y=0, width=100, height=50),+ ax_name='Submit',+ )++ assert agent._is_menu_item_element(regular_element) is False