Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Added ax-name fallback for history rerun.

Saurav Panda2c06eabf5ffa
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index c1a58ed9a..448c410bc 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -3088,7 +3088,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
f'Could not find matching element for action {i} in current page.\n'
f' Looking for: {elem_info}\n'
f' Page has {selector_count} interactive elements.{diagnostic}\n'
- f' Tried: EXACT hash → STABLE hash → XPATH → ATTRIBUTE matching'
+ f' Tried: EXACT hash → STABLE hash → XPATH → AX_NAME → ATTRIBUTE matching'
)
pending_actions.append(updated_action)
@@ -3113,7 +3113,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):
1. EXACT: Full element_hash match (includes all attributes + ax_name)
2. STABLE: Hash with dynamic CSS classes filtered out (focus, hover, animation, etc.)
3. XPATH: XPath string match (structural position in DOM)
- 4. ATTRIBUTE: Unique attribute match (name, id, aria-label) for old history files
+ 4. AX_NAME: Accessible name match from accessibility tree (robust for dynamic menus)
+ 5. ATTRIBUTE: Unique attribute match (name, id, aria-label) for old history files
"""
if not historical_element or not browser_state_summary.dom_state.selector_map:
return action
@@ -3175,7 +3176,34 @@ class Agent(Generic[Context, AgentStructuredOutput]):
if highlight_index is None:
self.logger.debug(f'XPATH match failed for: {historical_element.x_path[-60:]}')
- # Level 4: Unique attribute fallback (for old history files without stable_hash)
+ # Level 4: ax_name (accessible name) match - robust for dynamic SPAs with menus
+ # This uses the accessible name from the accessibility tree which is stable
+ # even when DOM structure changes (e.g., dynamically generated menu items)
+ if highlight_index is None and historical_element.ax_name:
+ hist_name = historical_element.node_name.lower()
+ hist_ax_name = historical_element.ax_name
+ for idx, elem in selector_map.items():
+ # Match by node type and accessible name
+ elem_ax_name = elem.ax_node.name if elem.ax_node else None
+ if elem.node_name.lower() == hist_name and elem_ax_name == hist_ax_name:
+ highlight_index = idx
+ match_level = MatchLevel.AX_NAME
+ self.logger.info(f'Element matched at AX_NAME level: "{hist_ax_name}"')
+ break
+ if highlight_index is None:
+ # Log available ax_names for debugging
+ same_type_ax_names = [
+ (idx, elem.ax_node.name if elem.ax_node else None)
+ for idx, elem in selector_map.items()
+ if elem.node_name.lower() == hist_name and elem.ax_node and elem.ax_node.name
+ ]
+ self.logger.debug(
+ f'AX_NAME match failed for <{hist_name.upper()}> ax_name="{hist_ax_name}". '
+ f'Page has {len(same_type_ax_names)} <{hist_name.upper()}> with ax_names: '
+ f'{same_type_ax_names[:5]}{"..." if len(same_type_ax_names) > 5 else ""}'
+ )
+
+ # Level 5: Unique attribute fallback (for old history files without stable_hash)
if highlight_index is None and historical_element.attributes:
hist_attrs = historical_element.attributes
hist_name = historical_element.node_name.lower()
@@ -3190,7 +3218,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
and elem.attributes.get(attr_key) == hist_attrs[attr_key]
):
highlight_index = idx
- match_level = MatchLevel.XPATH # Reuse XPATH level for logging
+ match_level = MatchLevel.ATTRIBUTE
self.logger.info(f'Element matched via {attr_key} attribute: {hist_attrs[attr_key]}')
break
if highlight_index is not None:
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 911cdd0e4..2d1f0f0f7 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -164,6 +164,8 @@ class MatchLevel(Enum):
EXACT = 1 # Full hash with all attributes (current behavior)
STABLE = 2 # Hash with dynamic classes filtered out
XPATH = 3 # XPath string comparison
+ AX_NAME = 4 # Accessible name (ax_name) from accessibility tree
+ ATTRIBUTE = 5 # Unique attribute match (name, id, aria-label)
def filter_dynamic_classes(class_str: str | None) -> str:
diff --git a/examples/features/rerun_history.py b/examples/features/rerun_history.py
index 3b38eb069..a3db07f08 100644
--- a/examples/features/rerun_history.py
+++ b/examples/features/rerun_history.py
@@ -122,6 +122,8 @@ async def main():
variables=new_values,
ai_step_llm=ai_step_llm, # For extract action re-evaluation
summary_llm=summary_llm, # For final summary
+ max_step_interval=20,
+ delay_between_actions=1,
)
# Display AI-generated summary (last result)
diff --git a/tests/ci/test_ax_name_matching.py b/tests/ci/test_ax_name_matching.py
new file mode 100644
index 000000000..7e00538ca
--- /dev/null
+++ b/tests/ci/test_ax_name_matching.py
@@ -0,0 +1,357 @@
+"""Tests for ax_name (accessible name) element matching in history rerun.
+
+This tests Level 4 matching which uses the accessibility tree's name property
+to match elements when hash, stable_hash, and xpath all fail.
+This is particularly useful for dynamic SPAs where DOM structure changes
+but accessible names remain stable.
+"""
+
+from unittest.mock import AsyncMock
+
+from browser_use.agent.service import Agent
+from browser_use.agent.views import ActionResult, AgentHistory, AgentHistoryList, RerunSummaryAction, StepMetadata
+from browser_use.browser.views import BrowserStateHistory
+from browser_use.dom.views import DOMInteractedElement, DOMRect, MatchLevel, NodeType
+from tests.ci.conftest import create_mock_llm
+
+
+async def test_ax_name_matching_succeeds_when_hash_fails(httpserver):
+ """Test that ax_name matching finds elements when hash/xpath matching fails.
+
+ This simulates a dynamic SPA where the element hash and xpath change between
+ sessions, but the accessible name (ax_name) remains stable.
+ """
+ # Set up a test page with a menu item that has an aria-label
+ # The aria-label becomes the accessible name (ax_name)
+ test_html = """<!DOCTYPE html>
+ <html>
+ <body>
+ <div role="menuitem" aria-label="New Contact" id="menu-1">New Contact</div>
+ <div role="menuitem" aria-label="Search" id="menu-2">Search</div>
+ </body>
+ </html>"""
+ httpserver.expect_request('/test').respond_with_data(test_html, content_type='text/html')
+ test_url = httpserver.url_for('/test')
+
+ # Create a mock LLM for summary
+ summary_action = RerunSummaryAction(
+ summary='Rerun completed',
+ success=True,
+ completion_status='complete',
+ )
+
+ async def custom_ainvoke(*args, **kwargs):
+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')
+ if output_format is RerunSummaryAction:
+ from browser_use.llm.views import ChatInvokeCompletion
+
+ return ChatInvokeCompletion(completion=summary_action, usage=None)
+ raise ValueError('Unexpected output_format')
+
+ mock_summary_llm = AsyncMock()
+ mock_summary_llm.ainvoke.side_effect = custom_ainvoke
+
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ AgentOutput = agent.AgentOutput
+
+ # Create an element with DIFFERENT hash/xpath but SAME ax_name as the real element
+ # This simulates what happens in dynamic SPAs where the DOM changes but
+ # accessible names remain stable
+ historical_element = DOMInteractedElement(
+ node_id=9999, # Different node_id
+ backend_node_id=9999, # Different backend_node_id
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='DIV', # Same node type
+ # Note: aria-label is NOT in attributes - this tests that ax_name matching
+ # is used as a fallback when attribute matching fails
+ attributes={'role': 'menuitem', 'class': 'dynamic-class-12345'},
+ x_path='html/body/div[1]/div[4]/div[4]/div[1]', # Different xpath
+ element_hash=123456789, # Different hash (won't match)
+ stable_hash=987654321, # Different stable_hash (won't match)
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='New Contact', # SAME ax_name - this should match!
+ )
+
+ # Step 1: Navigate to test page
+ navigate_step = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Navigate to test page',
+ next_goal=None,
+ action=[{'navigate': {'url': test_url}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Navigated')],
+ state=BrowserStateHistory(
+ url=test_url,
+ title='Test Page',
+ tabs=[],
+ interacted_element=[None],
+ ),
+ metadata=StepMetadata(
+ step_start_time=0,
+ step_end_time=1,
+ step_number=1,
+ step_interval=0.1,
+ ),
+ )
+
+ # Step 2: Click on element that has different hash/xpath but same ax_name
+ click_step = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Click New Contact menu',
+ next_goal=None,
+ action=[{'click': {'index': 100}}], # type: ignore[arg-type] # Original index doesn't matter
+ ),
+ result=[ActionResult(long_term_memory='Clicked New Contact')],
+ state=BrowserStateHistory(
+ url=test_url,
+ title='Test Page',
+ tabs=[],
+ interacted_element=[historical_element],
+ ),
+ metadata=StepMetadata(
+ step_start_time=1,
+ step_end_time=2,
+ step_number=2,
+ step_interval=0.1,
+ ),
+ )
+
+ history = AgentHistoryList(history=[navigate_step, click_step])
+
+ try:
+ # Run rerun - should succeed because ax_name matching finds the element
+ results = await agent.rerun_history(
+ history,
+ skip_failures=False,
+ max_retries=1,
+ summary_llm=mock_summary_llm,
+ )
+
+ # Should have 3 results: navigate + click + AI summary
+ assert len(results) == 3
+
+ # First result should be navigation success
+ nav_result = results[0]
+ assert nav_result.error is None
+
+ # Second result should be click success (matched via ax_name)
+ click_result = results[1]
+ assert click_result.error is None, f'Click should succeed via ax_name matching, got error: {click_result.error}'
+
+ # Third result should be AI summary
+ summary_result = results[2]
+ assert summary_result.is_done is True
+
+ finally:
+ await agent.close()
+
+
+async def test_ax_name_matching_requires_same_node_type(httpserver):
+ """Test that ax_name matching also requires matching node type.
+
+ Even if ax_name matches, the node type (DIV, BUTTON, etc.) must also match.
+ """
+ test_html = """<!DOCTYPE html>
+ <html>
+ <body>
+ <button aria-label="Submit">Submit</button>
+ <div aria-label="Submit">Submit Label</div>
+ </body>
+ </html>"""
+ httpserver.expect_request('/test').respond_with_data(test_html, content_type='text/html')
+ test_url = httpserver.url_for('/test')
+
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ AgentOutput = agent.AgentOutput
+
+ # Historical element is a SPAN with ax_name "Submit"
+ # Page has BUTTON and DIV with same ax_name, but no SPAN
+ historical_element = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='SPAN', # SPAN - won't match BUTTON or DIV
+ attributes={},
+ x_path='html/body/span',
+ element_hash=111,
+ stable_hash=111,
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='Submit', # Same ax_name, but wrong node type
+ )
+
+ navigate_step = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Navigate',
+ next_goal=None,
+ action=[{'navigate': {'url': test_url}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Navigated')],
+ state=BrowserStateHistory(
+ url=test_url,
+ title='Test',
+ tabs=[],
+ interacted_element=[None],
+ ),
+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),
+ )
+
+ click_step = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Click Submit',
+ next_goal=None,
+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Clicked')],
+ state=BrowserStateHistory(
+ url=test_url,
+ title='Test',
+ tabs=[],
+ interacted_element=[historical_element],
+ ),
+ metadata=StepMetadata(step_start_time=1, step_end_time=2, step_number=2, step_interval=0.1),
+ )
+
+ history = AgentHistoryList(history=[navigate_step, click_step])
+
+ try:
+ # Should fail because no SPAN with ax_name "Submit" exists
+ await agent.rerun_history(
+ history,
+ skip_failures=False,
+ max_retries=1,
+ )
+ assert False, 'Expected RuntimeError - no matching SPAN element'
+ except RuntimeError as e:
+ # Expected - no SPAN element with ax_name "Submit"
+ assert 'failed after 1 attempts' in str(e)
+ finally:
+ await agent.close()
+
+
+def test_match_level_enum_includes_ax_name():
+ """Test that MatchLevel enum includes AX_NAME level."""
+ assert hasattr(MatchLevel, 'AX_NAME')
+ assert MatchLevel.AX_NAME.value == 4
+ assert MatchLevel.ATTRIBUTE.value == 5 # AX_NAME comes before ATTRIBUTE
+
+
+async def test_ax_name_matching_before_attribute_matching(httpserver):
+ """Test that ax_name matching (Level 4) is tried before attribute matching (Level 5).
+
+ This ensures the correct matching order: EXACT -> STABLE -> XPATH -> AX_NAME -> ATTRIBUTE
+ """
+ # Page has element with text content that becomes its ax_name
+ # The DIV has role="menuitem" and text "Contact" which becomes its accessible name
+ # but NO aria-label/id/name attributes - so attribute matching will fail but ax_name should work
+ test_html = """<!DOCTYPE html>
+ <html>
+ <body>
+ <div role="menuitem">Contact</div>
+ </body>
+ </html>"""
+ httpserver.expect_request('/test').respond_with_data(test_html, content_type='text/html')
+ test_url = httpserver.url_for('/test')
+
+ summary_action = RerunSummaryAction(
+ summary='Rerun completed',
+ success=True,
+ completion_status='complete',
+ )
+
+ async def custom_ainvoke(*args, **kwargs):
+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')
+ if output_format is RerunSummaryAction:
+ from browser_use.llm.views import ChatInvokeCompletion
+
+ return ChatInvokeCompletion(completion=summary_action, usage=None)
+ raise ValueError('Unexpected output_format')
+
+ mock_summary_llm = AsyncMock()
+ mock_summary_llm.ainvoke.side_effect = custom_ainvoke
+
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ AgentOutput = agent.AgentOutput
+
+ # Historical element has NO aria-label attribute (attribute matching will fail)
+ # but HAS ax_name (ax_name matching should work)
+ historical_element = DOMInteractedElement(
+ node_id=1,
+ backend_node_id=1,
+ frame_id=None,
+ node_type=NodeType.ELEMENT_NODE,
+ node_value='',
+ node_name='DIV',
+ # No aria-label, id, or name - attribute matching will fail
+ attributes={'role': 'menuitem'},
+ x_path='html/body/div[99]', # Wrong xpath
+ element_hash=12345, # Wrong hash
+ stable_hash=12345, # Wrong stable hash
+ bounds=DOMRect(x=0, y=0, width=100, height=50),
+ ax_name='Contact', # ax_name from accessibility tree
+ )
+
+ navigate_step = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Navigate',
+ next_goal=None,
+ action=[{'navigate': {'url': test_url}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Navigated')],
+ state=BrowserStateHistory(
+ url=test_url,
+ title='Test',
+ tabs=[],
+ interacted_element=[None],
+ ),
+ metadata=StepMetadata(step_start_time=0, step_end_time=1, step_number=1, step_interval=0.1),
+ )
+
+ click_step = AgentHistory(
+ model_output=AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Click Contact',
+ next_goal=None,
+ action=[{'click': {'index': 1}}], # type: ignore[arg-type]
+ ),
+ result=[ActionResult(long_term_memory='Clicked')],
+ state=BrowserStateHistory(
+ url=test_url,
+ title='Test',
+ tabs=[],
+ interacted_element=[historical_element],
+ ),
+ metadata=StepMetadata(step_start_time=1, step_end_time=2, step_number=2, step_interval=0.1),
+ )
+
+ history = AgentHistoryList(history=[navigate_step, click_step])
+
+ try:
+ # Should succeed via ax_name matching (Level 4)
+ # since hash, stable_hash, xpath all fail but ax_name matches
+ results = await agent.rerun_history(
+ history,
+ skip_failures=False,
+ max_retries=1,
+ summary_llm=mock_summary_llm,
+ )
+
+ # Navigation + click + summary
+ assert len(results) == 3
+ # Click should succeed (matched via ax_name)
+ click_result = results[1]
+ assert click_result.error is None, f'Expected ax_name match to succeed, got: {click_result.error}'
+
+ finally:
+ await agent.close()