Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Migration from highlight index to backendnode id

Magnus Müllerb678f27a3e79
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index 5fe5c1b98..8980aed83 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -142,7 +142,7 @@ class AgentMessagePrompt:
stats['scroll_containers'] += 1
# Check if interactive
- if node.interactive_index is not None:
+ if node.is_interactive:
stats['interactive_elements'] += 1
# Check if this element hosts shadow DOM
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.py
index c0cccdd62..ffb712f74 100644
--- a/browser_use/browser/events.py
+++ b/browser_use/browser/events.py
@@ -59,7 +59,6 @@ class ElementSelectedEvent(BaseEvent[T_EventResultType]):
if data is None:
return None
return EnhancedDOMTreeNode(
- element_index=data.element_index,
node_id=data.node_id,
backend_node_id=data.backend_node_id,
session_id=data.session_id,
diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.py
index a8b741afa..2d41b7e27 100644
--- a/browser_use/browser/python_highlights.py
+++ b/browser_use/browser/python_highlights.py
@@ -100,9 +100,9 @@ def get_element_color(tag_name: str, element_type: str | None = None) -> str:
return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])
-def should_show_index_overlay(element_index: int | None) -> bool:
+def should_show_index_overlay(backend_node_id: int | None) -> bool:
"""Determine if index overlay should be shown."""
- return element_index is not None
+ return backend_node_id is not None
def draw_enhanced_bounding_box_with_text(
@@ -381,19 +381,19 @@ def process_element_highlight(
color = get_element_color(tag_name, element_type)
# Get element index for overlay and apply filtering
- element_index = getattr(element, 'element_index', None)
+ backend_node_id = getattr(element, 'backend_node_id', None)
index_text = None
- if element_index is not None:
+ if backend_node_id is not None:
if filter_highlight_ids:
# Use the meaningful text that matches what the LLM sees
meaningful_text = element.get_meaningful_text_for_llm()
# Show ID only if meaningful text is less than 5 characters
if len(meaningful_text) < 3:
- index_text = str(element_index)
+ index_text = str(backend_node_id)
else:
# Always show ID when filter is disabled
- index_text = str(element_index)
+ index_text = str(backend_node_id)
# Draw enhanced bounding box with bigger index
draw_enhanced_bounding_box_with_text(
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 4571e3605..6d327cbbe 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -2131,7 +2131,7 @@ class BrowserSession(BaseModel):
# Convert selector_map to the format expected by the highlighting script
elements_data = []
- for interactive_index, node in selector_map.items():
+ for _, node in selector_map.items():
# Get bounding box using absolute position (includes iframe translations) if available
if node.absolute_position:
# Use absolute position which includes iframe coordinate translations
@@ -2145,7 +2145,6 @@ class BrowserSession(BaseModel):
'y': bbox['y'],
'width': bbox['width'],
'height': bbox['height'],
- 'interactive_index': interactive_index,
'element_name': node.node_name,
'is_clickable': node.snapshot_node.is_clickable if node.snapshot_node else True,
'is_scrollable': getattr(node, 'is_scrollable', False),
@@ -2238,7 +2237,7 @@ class BrowserSession(BaseModel):
interactiveElements.forEach((element, index) => {{
const highlight = document.createElement('div');
highlight.setAttribute('data-browser-use-highlight', 'element');
- highlight.setAttribute('data-element-id', element.interactive_index);
+ highlight.setAttribute('data-element-id', element.backend_node_id);
highlight.style.cssText = `
position: absolute;
left: ${{element.x}}px;
@@ -2256,8 +2255,8 @@ class BrowserSession(BaseModel):
border: none;
`;
- // Enhanced label with interactive index
- const label = createTextElement('div', element.interactive_index, `
+ // Enhanced label with backend node ID
+ const label = createTextElement('div', element.backend_node_id, `
position: absolute;
top: -20px;
left: 0;
@@ -2896,7 +2895,7 @@ class BrowserSession(BaseModel):
object_id = result.get('object', {}).get('objectId')
if not object_id:
raise ValueError(
- f'Could not find #{node.element_index} backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}'
+ f'Could not find backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}'
)
return cdp_session
except (ValueError, Exception) as e:
@@ -2913,7 +2912,7 @@ class BrowserSession(BaseModel):
object_id = result.get('object', {}).get('objectId')
if not object_id:
raise ValueError(
- f'Could not find #{node.element_index} backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}'
+ f'Could not find backendNodeId={node.backend_node_id} in target_id={cdp_session.target_id}'
)
except Exception as e:
self.logger.debug(f'Failed to get CDP client for target {node.target_id}: {e}, using main session')
diff --git a/browser_use/browser/views.py b/browser_use/browser/views.py
index bf4fe7fb4..da222b61d 100644
--- a/browser_use/browser/views.py
+++ b/browser_use/browser/views.py
@@ -80,7 +80,7 @@ class PaginationButton:
"""Information about a pagination button detected on the page"""
button_type: str # 'next', 'prev', 'first', 'last', 'page_number'
- element_index: int # Interactive element index for clicking
+ backend_node_id: int # Backend node ID for clicking
text: str # Button text/label
selector: str # XPath or other selector to locate the element
is_disabled: bool = False # Whether the button appears disabled
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.py
index f2e7b3b7d..0c281d2e1 100644
--- a/browser_use/browser/watchdogs/default_action_watchdog.py
+++ b/browser_use/browser/watchdogs/default_action_watchdog.py
@@ -50,7 +50,7 @@ class DefaultActionWatchdog(BaseWatchdog):
# Use the provided node
element_node = event.node
- index_for_logging = element_node.element_index or 'unknown'
+ index_for_logging = element_node.backend_node_id or 'unknown'
starting_target_id = self.browser_session.agent_focus.target_id
# Check if element is a file input (should not be clicked)
@@ -85,10 +85,10 @@ class DefaultActionWatchdog(BaseWatchdog):
try:
# Use the provided node
element_node = event.node
- index_for_logging = element_node.element_index or 'unknown'
+ index_for_logging = element_node.backend_node_id or 'unknown'
# Check if this is index 0 or a falsy index - type to the page (whatever has focus)
- if not element_node.element_index or element_node.element_index == 0:
+ if not element_node.backend_node_id or element_node.backend_node_id == 0:
# Type to the page without focusing any specific element
await self._type_to_page(event.text)
# Log with sensitive data protection
@@ -301,7 +301,7 @@ class DefaultActionWatchdog(BaseWatchdog):
if tag_name == 'select':
msg = (
- f'Cannot click on <select> elements. Use dropdown_options(index={element_node.element_index}) action instead.'
+ f'Cannot click on <select> elements. Use dropdown_options(index={element_node.backend_node_id}) action instead.'
)
self.logger.warning(msg)
raise BrowserError(
@@ -310,7 +310,7 @@ class DefaultActionWatchdog(BaseWatchdog):
)
if tag_name == 'input' and element_type == 'file':
- msg = f'Cannot click on file input element (index={element_node.element_index}). File uploads must be handled using upload_file_to_element action.'
+ msg = f'Cannot click on file input element (index={element_node.backend_node_id}). File uploads must be handled using upload_file_to_element action.'
raise BrowserError(
message=msg,
long_term_memory=msg,
@@ -560,16 +560,16 @@ class DefaultActionWatchdog(BaseWatchdog):
except Exception as e:
# Extract key element info for error message
element_info = f'<{element_node.tag_name or "unknown"}'
- if element_node.element_index:
- element_info += f' index={element_node.element_index}'
+ if element_node.backend_node_id:
+ element_info += f' index={element_node.backend_node_id}'
element_info += '>'
# Create helpful error message based on context
error_detail = f'Failed to click element {element_info}. The element may not be interactable or visible.'
# Add hint if element has index (common in code-use mode)
- if element_node.element_index:
- error_detail += f' If the page changed after navigation/interaction, the index [{element_node.element_index}] may be stale. Get fresh browser state before retrying.'
+ if element_node.backend_node_id:
+ error_detail += f' If the page changed after navigation/interaction, the index [{element_node.backend_node_id}] may be stale. Get fresh browser state before retrying.'
raise BrowserError(
message=f'Failed to click element: {e}',
@@ -1651,7 +1651,7 @@ class DefaultActionWatchdog(BaseWatchdog):
try:
# Use the provided node
element_node = event.node
- index_for_logging = element_node.element_index or 'unknown'
+ index_for_logging = element_node.backend_node_id or 'unknown'
# Check if it's a file input
if not self.browser_session.is_file_input(element_node):
@@ -1776,7 +1776,7 @@ class DefaultActionWatchdog(BaseWatchdog):
try:
# Use the provided node
element_node = event.node
- index_for_logging = element_node.element_index or 'unknown'
+ index_for_logging = element_node.backend_node_id or 'unknown'
# Get CDP session for this node
cdp_session = await self.browser_session.cdp_client_for_node(element_node)
@@ -1936,7 +1936,7 @@ class DefaultActionWatchdog(BaseWatchdog):
'error': msg,
'short_term_memory': msg,
'long_term_memory': msg,
- 'element_index': str(index_for_logging),
+ 'backend_node_id': str(index_for_logging),
}
# Format options for display
@@ -1980,7 +1980,7 @@ class DefaultActionWatchdog(BaseWatchdog):
'message': msg,
'short_term_memory': short_term_memory,
'long_term_memory': long_term_memory,
- 'element_index': str(index_for_logging),
+ 'backend_node_id': str(index_for_logging),
}
except BrowserError:
@@ -2003,7 +2003,7 @@ class DefaultActionWatchdog(BaseWatchdog):
try:
# Use the provided node
element_node = event.node
- index_for_logging = element_node.element_index or 'unknown'
+ index_for_logging = element_node.backend_node_id or 'unknown'
target_text = event.text
# Get CDP session for this node
@@ -2240,7 +2240,7 @@ class DefaultActionWatchdog(BaseWatchdog):
'success': 'true',
'message': msg,
'value': selection_result.get('value', target_text),
- 'element_index': str(index_for_logging),
+ 'backend_node_id': str(index_for_logging),
}
else:
error_msg = selection_result.get('error', f'Failed to select option: {target_text}')
@@ -2275,14 +2275,14 @@ class DefaultActionWatchdog(BaseWatchdog):
'error': error_msg,
'short_term_memory': short_term_memory,
'long_term_memory': long_term_memory,
- 'element_index': str(index_for_logging),
+ 'backend_node_id': str(index_for_logging),
}
# Fallback to regular error result if no available options
return {
'success': 'false',
'error': error_msg,
- 'element_index': str(index_for_logging),
+ 'backend_node_id': str(index_for_logging),
}
except Exception as e:
diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.py
index 370e184ea..51ad3d035 100644
--- a/browser_use/browser/watchdogs/dom_watchdog.py
+++ b/browser_use/browser/watchdogs/dom_watchdog.py
@@ -673,7 +673,7 @@ class DOMWatchdog(BaseWatchdog):
pagination_buttons_data = [
PaginationButton(
button_type=btn['button_type'], # type: ignore
- element_index=btn['element_index'], # type: ignore
+ backend_node_id=btn['backend_node_id'], # type: ignore
text=btn['text'], # type: ignore
selector=btn['selector'], # type: ignore
is_disabled=btn['is_disabled'], # type: ignore
diff --git a/browser_use/dom/serializer/eval_serializer.py b/browser_use/dom/serializer/eval_serializer.py
index 4f349699f..64116d255 100644
--- a/browser_use/dom/serializer/eval_serializer.py
+++ b/browser_use/dom/serializer/eval_serializer.py
@@ -157,7 +157,7 @@ class DOMEvalSerializer:
if tag == 'svg':
line = f'{depth_str}'
# Add [i_X] for interactive SVG elements only
- if node.interactive_index is not None:
+ if node.is_interactive:
line += f'[i_{node.original_node.backend_node_id}] '
line += '<svg'
attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
@@ -182,7 +182,7 @@ class DOMEvalSerializer:
# Build compact element representation
line = f'{depth_str}'
# Add backend node ID notation - [i_X] for interactive elements only
- if node.interactive_index is not None:
+ if node.is_interactive:
line += f'[i_{node.original_node.backend_node_id}] '
# Non-interactive elements don't get an index notation
line += f'<{tag}'
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index 9dd6d3784..ae5bf4ef3 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -553,10 +553,10 @@ class DOMTreeSerializer:
is_visible = node.original_node.snapshot_node and node.original_node.is_visible
# Only add to selector map if element is both interactive AND visible
- # KEY CHANGE: Use backend_node_id instead of interactive_counter as the key
if is_interactive_assign and is_visible:
- node.interactive_index = self._interactive_counter
- node.original_node.element_index = self._interactive_counter
+ # Mark node as interactive
+ node.is_interactive = True
+ # Store backend_node_id in selector map (model outputs backend_node_id)
self._selector_map[node.original_node.backend_node_id] = node.original_node
self._interactive_counter += 1
@@ -754,11 +754,11 @@ class DOMTreeSerializer:
formatted_text.append(child_text)
return '\n'.join(formatted_text)
- # Add element with interactive_index if clickable, scrollable, or iframe
+ # Add element if clickable, scrollable, or iframe
is_any_scrollable = node.original_node.is_actually_scrollable or node.original_node.is_scrollable
should_show_scroll = node.original_node.should_show_scroll_info
if (
- node.interactive_index is not None
+ node.is_interactive
or is_any_scrollable
or node.original_node.tag_name.upper() == 'IFRAME'
or node.original_node.tag_name.upper() == 'FRAME'
@@ -818,14 +818,14 @@ class DOMTreeSerializer:
)
shadow_prefix = '|SHADOW(closed)|' if has_closed_shadow else '|SHADOW(open)|'
- if should_show_scroll and node.interactive_index is None:
+ if should_show_scroll and not node.is_interactive:
# Scrollable container but not clickable
line = f'{depth_str}{shadow_prefix}|SCROLL|<{node.original_node.tag_name}'
- elif node.interactive_index is not None:
- # Clickable (and possibly scrollable)
+ elif node.is_interactive:
+ # Clickable (and possibly scrollable) - show backend_node_id
new_prefix = '*' if node.is_new else ''
scroll_prefix = '|SCROLL[' if should_show_scroll else '['
- line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}{node.interactive_index}]<{node.original_node.tag_name}'
+ line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}{node.original_node.backend_node_id}]<{node.original_node.tag_name}'
elif node.original_node.tag_name.upper() == 'IFRAME':
# Iframe element (not interactive)
line = f'{depth_str}{shadow_prefix}|IFRAME|<{node.original_node.tag_name}'
diff --git a/browser_use/dom/service.py b/browser_use/dom/service.py
index 836b1054d..bdd0327a4 100644
--- a/browser_use/dom/service.py
+++ b/browser_use/dom/service.py
@@ -560,7 +560,6 @@ class DomService:
snapshot_node=snapshot_data,
is_visible=None,
absolute_position=absolute_position,
- element_index=None,
)
enhanced_dom_tree_node_lookup[node['nodeId']] = dom_tree_node
@@ -759,7 +758,7 @@ class DomService:
Returns:
List of pagination button information dicts with:
- button_type: 'next', 'prev', 'first', 'last', 'page_number'
- - element_index: Interactive element index for clicking
+ - backend_node_id: Backend node ID for clicking
- text: Button text/label
- selector: XPath selector
- is_disabled: Whether the button appears disabled
@@ -816,7 +815,7 @@ class DomService:
pagination_buttons.append(
{
'button_type': button_type,
- 'element_index': index,
+ 'backend_node_id': index,
'text': node.get_all_children_text().strip() or aria_label or title,
'selector': node.xpath,
'is_disabled': is_disabled,
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 22475376a..55b579aeb 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -156,7 +156,7 @@ class SimplifiedNode:
original_node: 'EnhancedDOMTreeNode'
children: list['SimplifiedNode']
should_display: bool = True
- interactive_index: int | None = None
+ is_interactive: bool = False # True if element is in selector_map
is_new: bool = False
@@ -185,7 +185,7 @@ class SimplifiedNode:
cleaned_original_node_json = self._clean_original_node_json(original_node_json)
return {
'should_display': self.should_display,
- 'interactive_index': self.interactive_index,
+ 'is_interactive': self.is_interactive,
'ignored_by_paint_order': self.ignored_by_paint_order,
'excluded_by_parent': self.excluded_by_parent,
'original_node': cleaned_original_node_json,
@@ -375,9 +375,6 @@ class EnhancedDOMTreeNode:
# endregion - Snapshot Node data
- # Interactive element index
- element_index: int | None = None
-
# Compound control child components information
_compound_children: list[dict[str, Any]] = field(default_factory=list)
@@ -746,7 +743,7 @@ class EnhancedDOMTreeNode:
return hash(self)
def __str__(self) -> str:
- return f'[<{self.tag_name}>#{self.frame_id[-4:] if self.frame_id else "?"}:{self.element_index}]'
+ return f'[<{self.tag_name}>#{self.frame_id[-4:] if self.frame_id else "?"}:{self.backend_node_id}]'
def __hash__(self) -> int:
"""
diff --git a/browser_use/llm/tests/test_single_step.py b/browser_use/llm/tests/test_single_step.py
index 0286a0a64..99cb76ddc 100644
--- a/browser_use/llm/tests/test_single_step.py
+++ b/browser_use/llm/tests/test_single_step.py
@@ -69,8 +69,8 @@ def create_mock_state_message(temp_dir: str):
snapshot_node=None,
)
- # Create selector map
- selector_map: DOMSelectorMap = {1: mock_button}
+ # Create selector map (keyed by backend_node_id)
+ selector_map: DOMSelectorMap = {mock_button.backend_node_id: mock_button}
# Create mock tab info with proper target_id
mock_tab = TabInfo(
@@ -84,7 +84,7 @@ def create_mock_state_message(temp_dir: str):
original_node=mock_button,
children=[],
should_display=True,
- interactive_index=1,
+ is_interactive=True,
),
selector_map=selector_map,
)