Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Migration from highlight index to backendnode id
Magnus Müllerb678f27a3e79
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.pyindex 5fe5c1b98..8980aed83 100644--- a/browser_use/agent/prompts.py+++ b/browser_use/agent/prompts.py@@ -142,7 +142,7 @@ class AgentMessagePrompt:stats['scroll_containers'] += 1# Check if interactive- if node.interactive_index is not None:+ if node.is_interactive:stats['interactive_elements'] += 1# Check if this element hosts shadow DOMdiff --git a/browser_use/browser/events.py b/browser_use/browser/events.pyindex c0cccdd62..ffb712f74 100644--- a/browser_use/browser/events.py+++ b/browser_use/browser/events.py@@ -59,7 +59,6 @@ class ElementSelectedEvent(BaseEvent[T_EventResultType]):if data is None:return Nonereturn EnhancedDOMTreeNode(- element_index=data.element_index,node_id=data.node_id,backend_node_id=data.backend_node_id,session_id=data.session_id,diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.pyindex a8b741afa..2d41b7e27 100644--- a/browser_use/browser/python_highlights.py+++ b/browser_use/browser/python_highlights.py@@ -100,9 +100,9 @@ def get_element_color(tag_name: str, element_type: str | None = None) -> str:return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])-def should_show_index_overlay(element_index: int | None) -> bool:+def should_show_index_overlay(backend_node_id: int | None) -> bool:"""Determine if index overlay should be shown."""- return element_index is not None+ return backend_node_id is not Nonedef draw_enhanced_bounding_box_with_text(@@ -381,19 +381,19 @@ def process_element_highlight(color = get_element_color(tag_name, element_type)# Get element index for overlay and apply filtering- element_index = getattr(element, 'element_index', None)+ backend_node_id = getattr(element, 'backend_node_id', None)index_text = None- if element_index is not None:+ if backend_node_id is not None:if filter_highlight_ids:# Use the meaningful text that matches what the LLM seesmeaningful_text = element.get_meaningful_text_for_llm()# Show ID only if meaningful text is less than 5 charactersif len(meaningful_text) < 3:- index_text = str(element_index)+ index_text = str(backend_node_id)else:# Always show ID when filter is disabled- index_text = str(element_index)+ index_text = str(backend_node_id)# Draw enhanced bounding box with bigger indexdraw_enhanced_bounding_box_with_text(diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex 4571e3605..6d327cbbe 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -2131,7 +2131,7 @@ class BrowserSession(BaseModel):# Convert selector_map to the format expected by the highlighting scriptelements_data = []- for interactive_index, node in selector_map.items():+ for _, node in selector_map.items():# Get bounding box using absolute position (includes iframe translations) if availableif node.absolute_position:# Use absolute position which includes iframe coordinate translations@@ -2145,7 +2145,6 @@ class BrowserSession(BaseModel):'y': bbox['y'],'width': bbox['width'],'height': bbox['height'],- 'interactive_index': interactive_index,'element_name': node.node_name,'is_clickable': node.snapshot_node.is_clickable if node.snapshot_node else True,'is_scrollable': getattr(node, 'is_scrollable', False),@@ -2238,7 +2237,7 @@ class BrowserSession(BaseModel):interactiveElements.forEach((element, index) => {{const highlight = document.createElement('div');highlight.setAttribute('data-browser-use-highlight', 'element');- highlight.setAttribute('data-element-id', element.interactive_index);+ highlight.setAttribute('data-element-id', element.backend_node_id);highlight.style.cssText = `position: absolute;left: ${{element.x}}px;@@ -2256,8 +2255,8 @@ class BrowserSession(BaseModel):border: none;`;- // Enhanced label with interactive index- const label = createTextElement('div', element.interactive_index, `+ // Enhanced label with backend node ID+ const label = createTextElement('div', element.backend_node_id, `position: absolute;top: -20px;left: 0;@@ -2896,7 +2895,7 @@ class BrowserSession(BaseModel):object_id = result.get('object', {}).get('objectId')if not object_id:raise ValueError(- f'Could not find #{node.element_index} backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}'+ f'Could not find backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}')return cdp_sessionexcept (ValueError, Exception) as e:@@ -2913,7 +2912,7 @@ class BrowserSession(BaseModel):object_id = result.get('object', {}).get('objectId')if not object_id:raise ValueError(- f'Could not find #{node.element_index} backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}'+ f'Could not find backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}')except Exception as e:self.logger.debug(f'Failed to get CDP client for target {node.target_id}: {e}, using main session')diff --git a/browser_use/browser/views.py b/browser_use/browser/views.pyindex bf4fe7fb4..da222b61d 100644--- a/browser_use/browser/views.py+++ b/browser_use/browser/views.py@@ -80,7 +80,7 @@ class PaginationButton:"""Information about a pagination button detected on the page"""button_type: str # 'next', 'prev', 'first', 'last', 'page_number'- element_index: int # Interactive element index for clicking+ backend_node_id: int # Backend node ID for clickingtext: str # Button text/labelselector: str # XPath or other selector to locate the elementis_disabled: bool = False # Whether the button appears disableddiff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.pyindex f2e7b3b7d..0c281d2e1 100644--- a/browser_use/browser/watchdogs/default_action_watchdog.py+++ b/browser_use/browser/watchdogs/default_action_watchdog.py@@ -50,7 +50,7 @@ class DefaultActionWatchdog(BaseWatchdog):# Use the provided nodeelement_node = event.node- index_for_logging = element_node.element_index or 'unknown'+ index_for_logging = element_node.backend_node_id or 'unknown'starting_target_id = self.browser_session.agent_focus.target_id# Check if element is a file input (should not be clicked)@@ -85,10 +85,10 @@ class DefaultActionWatchdog(BaseWatchdog):try:# Use the provided nodeelement_node = event.node- index_for_logging = element_node.element_index or 'unknown'+ index_for_logging = element_node.backend_node_id or 'unknown'# Check if this is index 0 or a falsy index - type to the page (whatever has focus)- if not element_node.element_index or element_node.element_index == 0:+ if not element_node.backend_node_id or element_node.backend_node_id == 0:# Type to the page without focusing any specific elementawait self._type_to_page(event.text)# Log with sensitive data protection@@ -301,7 +301,7 @@ class DefaultActionWatchdog(BaseWatchdog):if tag_name == 'select':msg = (- f'Cannot click on <select> elements. Use dropdown_options(index={element_node.element_index}) action instead.'+ f'Cannot click on <select> elements. Use dropdown_options(index={element_node.backend_node_id}) action instead.')self.logger.warning(msg)raise BrowserError(@@ -310,7 +310,7 @@ class DefaultActionWatchdog(BaseWatchdog):)if tag_name == 'input' and element_type == 'file':- msg = f'Cannot click on file input element (index={element_node.element_index}). File uploads must be handled using upload_file_to_element action.'+ msg = f'Cannot click on file input element (index={element_node.backend_node_id}). File uploads must be handled using upload_file_to_element action.'raise BrowserError(message=msg,long_term_memory=msg,@@ -560,16 +560,16 @@ class DefaultActionWatchdog(BaseWatchdog):except Exception as e:# Extract key element info for error messageelement_info = f'<{element_node.tag_name or "unknown"}'- if element_node.element_index:- element_info += f' index={element_node.element_index}'+ if element_node.backend_node_id:+ element_info += f' index={element_node.backend_node_id}'element_info += '>'# Create helpful error message based on contexterror_detail = f'Failed to click element {element_info}. The element may not be interactable or visible.'# Add hint if element has index (common in code-use mode)- if element_node.element_index:- error_detail += f' If the page changed after navigation/interaction, the index [{element_node.element_index}] may be stale. Get fresh browser state before retrying.'+ if element_node.backend_node_id:+ error_detail += f' If the page changed after navigation/interaction, the index [{element_node.backend_node_id}] may be stale. Get fresh browser state before retrying.'raise BrowserError(message=f'Failed to click element: {e}',@@ -1651,7 +1651,7 @@ class DefaultActionWatchdog(BaseWatchdog):try:# Use the provided nodeelement_node = event.node- index_for_logging = element_node.element_index or 'unknown'+ index_for_logging = element_node.backend_node_id or 'unknown'# Check if it's a file inputif not self.browser_session.is_file_input(element_node):@@ -1776,7 +1776,7 @@ class DefaultActionWatchdog(BaseWatchdog):try:# Use the provided nodeelement_node = event.node- index_for_logging = element_node.element_index or 'unknown'+ index_for_logging = element_node.backend_node_id or 'unknown'# Get CDP session for this nodecdp_session = await self.browser_session.cdp_client_for_node(element_node)@@ -1936,7 +1936,7 @@ class DefaultActionWatchdog(BaseWatchdog):'error': msg,'short_term_memory': msg,'long_term_memory': msg,- 'element_index': str(index_for_logging),+ 'backend_node_id': str(index_for_logging),}# Format options for display@@ -1980,7 +1980,7 @@ class DefaultActionWatchdog(BaseWatchdog):'message': msg,'short_term_memory': short_term_memory,'long_term_memory': long_term_memory,- 'element_index': str(index_for_logging),+ 'backend_node_id': str(index_for_logging),}except BrowserError:@@ -2003,7 +2003,7 @@ class DefaultActionWatchdog(BaseWatchdog):try:# Use the provided nodeelement_node = event.node- index_for_logging = element_node.element_index or 'unknown'+ index_for_logging = element_node.backend_node_id or 'unknown'target_text = event.text# Get CDP session for this node@@ -2240,7 +2240,7 @@ class DefaultActionWatchdog(BaseWatchdog):'success': 'true','message': msg,'value': selection_result.get('value', target_text),- 'element_index': str(index_for_logging),+ 'backend_node_id': str(index_for_logging),}else:error_msg = selection_result.get('error', f'Failed to select option: {target_text}')@@ -2275,14 +2275,14 @@ class DefaultActionWatchdog(BaseWatchdog):'error': error_msg,'short_term_memory': short_term_memory,'long_term_memory': long_term_memory,- 'element_index': str(index_for_logging),+ 'backend_node_id': str(index_for_logging),}# Fallback to regular error result if no available optionsreturn {'success': 'false','error': error_msg,- 'element_index': str(index_for_logging),+ 'backend_node_id': str(index_for_logging),}except Exception as e:diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.pyindex 370e184ea..51ad3d035 100644--- a/browser_use/browser/watchdogs/dom_watchdog.py+++ b/browser_use/browser/watchdogs/dom_watchdog.py@@ -673,7 +673,7 @@ class DOMWatchdog(BaseWatchdog):pagination_buttons_data = [PaginationButton(button_type=btn['button_type'], # type: ignore- element_index=btn['element_index'], # type: ignore+ backend_node_id=btn['backend_node_id'], # type: ignoretext=btn['text'], # type: ignoreselector=btn['selector'], # type: ignoreis_disabled=btn['is_disabled'], # type: ignorediff --git a/browser_use/dom/serializer/eval_serializer.py b/browser_use/dom/serializer/eval_serializer.pyindex 4f349699f..64116d255 100644--- a/browser_use/dom/serializer/eval_serializer.py+++ b/browser_use/dom/serializer/eval_serializer.py@@ -157,7 +157,7 @@ class DOMEvalSerializer:if tag == 'svg':line = f'{depth_str}'# Add [i_X] for interactive SVG elements only- if node.interactive_index is not None:+ if node.is_interactive:line += f'[i_{node.original_node.backend_node_id}] 'line += '<svg'attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)@@ -182,7 +182,7 @@ class DOMEvalSerializer:# Build compact element representationline = f'{depth_str}'# Add backend node ID notation - [i_X] for interactive elements only- if node.interactive_index is not None:+ if node.is_interactive:line += f'[i_{node.original_node.backend_node_id}] '# Non-interactive elements don't get an index notationline += f'<{tag}'diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.pyindex 9dd6d3784..ae5bf4ef3 100644--- a/browser_use/dom/serializer/serializer.py+++ b/browser_use/dom/serializer/serializer.py@@ -553,10 +553,10 @@ class DOMTreeSerializer:is_visible = node.original_node.snapshot_node and node.original_node.is_visible# Only add to selector map if element is both interactive AND visible- # KEY CHANGE: Use backend_node_id instead of interactive_counter as the keyif is_interactive_assign and is_visible:- node.interactive_index = self._interactive_counter- node.original_node.element_index = self._interactive_counter+ # Mark node as interactive+ node.is_interactive = True+ # Store backend_node_id in selector map (model outputs backend_node_id)self._selector_map[node.original_node.backend_node_id] = node.original_nodeself._interactive_counter += 1@@ -754,11 +754,11 @@ class DOMTreeSerializer:formatted_text.append(child_text)return '\n'.join(formatted_text)- # Add element with interactive_index if clickable, scrollable, or iframe+ # Add element if clickable, scrollable, or iframeis_any_scrollable = node.original_node.is_actually_scrollable or node.original_node.is_scrollableshould_show_scroll = node.original_node.should_show_scroll_infoif (- node.interactive_index is not None+ node.is_interactiveor is_any_scrollableor node.original_node.tag_name.upper() == 'IFRAME'or node.original_node.tag_name.upper() == 'FRAME'@@ -818,14 +818,14 @@ class DOMTreeSerializer:)shadow_prefix = '|SHADOW(closed)|' if has_closed_shadow else '|SHADOW(open)|'- if should_show_scroll and node.interactive_index is None:+ if should_show_scroll and not node.is_interactive:# Scrollable container but not clickableline = f'{depth_str}{shadow_prefix}|SCROLL|<{node.original_node.tag_name}'- elif node.interactive_index is not None:- # Clickable (and possibly scrollable)+ elif node.is_interactive:+ # Clickable (and possibly scrollable) - show backend_node_idnew_prefix = '*' if node.is_new else ''scroll_prefix = '|SCROLL[' if should_show_scroll else '['- line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}{node.interactive_index}]<{node.original_node.tag_name}'+ line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}{node.original_node.backend_node_id}]<{node.original_node.tag_name}'elif node.original_node.tag_name.upper() == 'IFRAME':# Iframe element (not interactive)line = f'{depth_str}{shadow_prefix}|IFRAME|<{node.original_node.tag_name}'diff --git a/browser_use/dom/service.py b/browser_use/dom/service.pyindex 836b1054d..bdd0327a4 100644--- a/browser_use/dom/service.py+++ b/browser_use/dom/service.py@@ -560,7 +560,6 @@ class DomService:snapshot_node=snapshot_data,is_visible=None,absolute_position=absolute_position,- element_index=None,)enhanced_dom_tree_node_lookup[node['nodeId']] = dom_tree_node@@ -759,7 +758,7 @@ class DomService:Returns:List of pagination button information dicts with:- button_type: 'next', 'prev', 'first', 'last', 'page_number'- - element_index: Interactive element index for clicking+ - backend_node_id: Backend node ID for clicking- text: Button text/label- selector: XPath selector- is_disabled: Whether the button appears disabled@@ -816,7 +815,7 @@ class DomService:pagination_buttons.append({'button_type': button_type,- 'element_index': index,+ 'backend_node_id': index,'text': node.get_all_children_text().strip() or aria_label or title,'selector': node.xpath,'is_disabled': is_disabled,diff --git a/browser_use/dom/views.py b/browser_use/dom/views.pyindex 22475376a..55b579aeb 100644--- a/browser_use/dom/views.py+++ b/browser_use/dom/views.py@@ -156,7 +156,7 @@ class SimplifiedNode:original_node: 'EnhancedDOMTreeNode'children: list['SimplifiedNode']should_display: bool = True- interactive_index: int | None = None+ is_interactive: bool = False # True if element is in selector_mapis_new: bool = False@@ -185,7 +185,7 @@ class SimplifiedNode:cleaned_original_node_json = self._clean_original_node_json(original_node_json)return {'should_display': self.should_display,- 'interactive_index': self.interactive_index,+ 'is_interactive': self.is_interactive,'ignored_by_paint_order': self.ignored_by_paint_order,'excluded_by_parent': self.excluded_by_parent,'original_node': cleaned_original_node_json,@@ -375,9 +375,6 @@ class EnhancedDOMTreeNode:# endregion - Snapshot Node data- # Interactive element index- element_index: int | None = None-# Compound control child components information_compound_children: list[dict[str, Any]] = field(default_factory=list)@@ -746,7 +743,7 @@ class EnhancedDOMTreeNode:return hash(self)def __str__(self) -> str:- return f'[<{self.tag_name}>#{self.frame_id[-4:] if self.frame_id else "?"}:{self.element_index}]'+ return f'[<{self.tag_name}>#{self.frame_id[-4:] if self.frame_id else "?"}:{self.backend_node_id}]'def __hash__(self) -> int:"""diff --git a/browser_use/llm/tests/test_single_step.py b/browser_use/llm/tests/test_single_step.pyindex 0286a0a64..99cb76ddc 100644--- a/browser_use/llm/tests/test_single_step.py+++ b/browser_use/llm/tests/test_single_step.py@@ -69,8 +69,8 @@ def create_mock_state_message(temp_dir: str):snapshot_node=None,)- # Create selector map- selector_map: DOMSelectorMap = {1: mock_button}+ # Create selector map (keyed by backend_node_id)+ selector_map: DOMSelectorMap = {mock_button.backend_node_id: mock_button}# Create mock tab info with proper target_idmock_tab = TabInfo(@@ -84,7 +84,7 @@ def create_mock_state_message(temp_dir: str):original_node=mock_button,children=[],should_display=True,- interactive_index=1,+ is_interactive=True,),selector_map=selector_map,)