Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Enhance element interaction handling

Magnus Müllerd80549038b25
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index b2007c552..c75663c58 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -1494,6 +1494,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):
new_element_hashes = {e.parent_branch_hash() for e in new_selector_map.values()}
if check_for_new_elements and not new_element_hashes.issubset(cached_element_hashes):
# next action requires index but there are new elements on the page
+ # log difference in len debug
+ self.logger.debug(f'New elements: {abs(len(new_element_hashes) - len(cached_element_hashes))}')
remaining_actions_str = get_remaining_actions_str(actions, i)
msg = f'Something new appeared after action {i} / {total_actions}: actions {remaining_actions_str} were not executed'
logger.info(msg)
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.py
index 836ef2456..132ecb3ea 100644
--- a/browser_use/browser/watchdogs/default_action_watchdog.py
+++ b/browser_use/browser/watchdogs/default_action_watchdog.py
@@ -3,7 +3,6 @@
import asyncio
import json
import platform
-from typing import Any
from browser_use.browser.events import (
ClickElementEvent,
@@ -634,78 +633,139 @@ class DefaultActionWatchdog(BaseWatchdog):
# Fallback for unknown characters
return f'Key{char.upper()}'
- async def _check_element_focusability(self, element_node, object_id: str, session_id: str) -> dict[str, Any]:
- """
- Check if an element is likely to be focusable and visible.
-
- Returns:
- Dict with keys: 'visible', 'focusable', 'interactive', 'disabled'
- """
+ async def _clear_text_field(self, object_id: str, cdp_session) -> None:
+ """Clear text field using human-like Ctrl+A + Backspace approach."""
try:
- cdp_client = self.browser_session.cdp_client
+ self.logger.debug('🧹 Clearing text field using Ctrl+A + Backspace')
- # Run comprehensive element checks via JavaScript
- check_result = await cdp_client.send.Runtime.callFunctionOn(
+ # Select all text (Ctrl+A)
+ await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
params={
- 'functionDeclaration': """
- function() {
- const element = this;
- const computedStyle = window.getComputedStyle(element);
- const rect = element.getBoundingClientRect();
-
- // Check basic visibility
- const isVisible = rect.width > 0 && rect.height > 0 &&
- computedStyle.visibility !== 'hidden' &&
- computedStyle.display !== 'none' &&
- computedStyle.opacity !== '0';
-
- // Check if element is disabled
- const isDisabled = element.disabled || element.hasAttribute('disabled') ||
- element.getAttribute('aria-disabled') === 'true';
-
- // Check if element is focusable by tag and attributes
- const focusableTags = ['input', 'textarea', 'select', 'button', 'a'];
- const hasFocusableTag = focusableTags.includes(element.tagName.toLowerCase());
- const hasTabIndex = element.hasAttribute('tabindex') && element.tabIndex >= 0;
- const isContentEditable = element.contentEditable === 'true';
-
- const isFocusable = !isDisabled && (hasFocusableTag || hasTabIndex || isContentEditable);
-
- // Check if element is interactive (clickable/editable)
- const isInteractive = isFocusable || element.onclick !== null ||
- element.getAttribute('role') === 'button' ||
- element.classList.contains('clickable');
-
- return {
- visible: isVisible,
- focusable: isFocusable,
- interactive: isInteractive,
- disabled: isDisabled,
- bounds: {
- x: rect.left,
- y: rect.top,
- width: rect.width,
- height: rect.height
- },
- tagName: element.tagName.toLowerCase(),
- type: element.type || null
- };
- }
- """,
- 'objectId': object_id,
- 'returnByValue': True,
+ 'type': 'keyDown',
+ 'key': 'a',
+ 'code': 'KeyA',
+ 'modifiers': 2, # Ctrl modifier
+ 'windowsVirtualKeyCode': 65,
},
- session_id=session_id,
+ session_id=cdp_session.session_id,
+ )
+ await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
+ params={
+ 'type': 'keyUp',
+ 'key': 'a',
+ 'code': 'KeyA',
+ 'modifiers': 2, # Ctrl modifier
+ 'windowsVirtualKeyCode': 65,
+ },
+ session_id=cdp_session.session_id,
)
- if 'result' in check_result and 'value' in check_result['result']:
- return check_result['result']['value']
- else:
- self.logger.debug('Element focusability check returned no results')
- return {'visible': False, 'focusable': False, 'interactive': False, 'disabled': True}
+ # Small delay
+ await asyncio.sleep(0.01)
+
+ # Delete selected text (Backspace)
+ await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
+ params={
+ 'type': 'keyDown',
+ 'key': 'Backspace',
+ 'code': 'Backspace',
+ 'windowsVirtualKeyCode': 8,
+ },
+ session_id=cdp_session.session_id,
+ )
+ await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
+ params={
+ 'type': 'keyUp',
+ 'key': 'Backspace',
+ 'code': 'Backspace',
+ 'windowsVirtualKeyCode': 8,
+ },
+ session_id=cdp_session.session_id,
+ )
+
+ self.logger.debug('✅ Text field cleared successfully')
+
except Exception as e:
- self.logger.debug(f'Element focusability check failed: {e}')
- return {'visible': False, 'focusable': False, 'interactive': False, 'disabled': True}
+ self.logger.debug(f'Failed to clear text field: {e}')
+ # Try JavaScript fallback
+ try:
+ await cdp_session.cdp_client.send.Runtime.callFunctionOn(
+ params={
+ 'functionDeclaration': 'function() { if (this.value !== undefined) this.value = ""; }',
+ 'objectId': object_id,
+ },
+ session_id=cdp_session.session_id,
+ )
+ self.logger.debug('✅ Text field cleared using JavaScript fallback')
+ except Exception as js_e:
+ self.logger.debug(f'JavaScript clear also failed: {js_e}')
+
+ async def _focus_element_simple(
+ self, backend_node_id: int, object_id: str, cdp_session, input_coordinates: dict | None = None
+ ) -> bool:
+ """Simple focus strategy: CDP first, then click if failed."""
+
+ # Strategy 1: Try CDP DOM.focus first
+ try:
+ result = await cdp_session.cdp_client.send.DOM.focus(
+ params={'backendNodeId': backend_node_id},
+ session_id=cdp_session.session_id,
+ )
+ self.logger.debug(f'CDP DOM.focus result: {result}')
+ self.logger.debug('✅ Element focused using CDP DOM.focus')
+ return True
+ except Exception as e:
+ # Check for specific CDP "Element is not focusable" error
+ error_str = str(e).lower()
+ if (
+ 'element is not focusable' in error_str
+ or "code': -32000" in error_str
+ or '-32000' in error_str
+ or 'not focusable' in error_str
+ ):
+ self.logger.debug('❌ CDP DOM.focus failed - element not focusable')
+ else:
+ self.logger.debug(f'CDP DOM.focus failed: {e}')
+
+ # Strategy 2: Try click to focus if CDP failed
+ if input_coordinates and 'input_x' in input_coordinates and 'input_y' in input_coordinates:
+ try:
+ click_x = input_coordinates['input_x']
+ click_y = input_coordinates['input_y']
+
+ self.logger.debug(f'🎯 Attempting click-to-focus at ({click_x:.1f}, {click_y:.1f})')
+
+ # Click to focus
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mousePressed',
+ 'x': click_x,
+ 'y': click_y,
+ 'button': 'left',
+ 'clickCount': 1,
+ },
+ session_id=cdp_session.session_id,
+ )
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mouseReleased',
+ 'x': click_x,
+ 'y': click_y,
+ 'button': 'left',
+ 'clickCount': 1,
+ },
+ session_id=cdp_session.session_id,
+ )
+
+ self.logger.debug('✅ Element focused using click method')
+ return True
+
+ except Exception as e:
+ self.logger.debug(f'Click focus failed: {e}')
+
+ # Both strategies failed
+ self.logger.warning('⚠️ All focus strategies failed')
+ return False
async def _input_text_element_node_impl(self, element_node, text: str, clear_existing: bool = True) -> dict | None:
"""
@@ -749,117 +809,82 @@ class DefaultActionWatchdog(BaseWatchdog):
)
object_id = result['object']['objectId']
- # Check element focusability before attempting focus
- element_info = await self._check_element_focusability(element_node, object_id, cdp_session.session_id)
- self.logger.debug(f'Element focusability check: {element_info}')
-
- # Extract coordinates from element bounds for metadata
- bounds = element_info.get('bounds', {})
- if bounds.get('width', 0) > 0 and bounds.get('height', 0) > 0:
- center_x = bounds['x'] + bounds['width'] / 2
- center_y = bounds['y'] + bounds['height'] / 2
- input_coordinates = {'input_x': center_x, 'input_y': center_y}
- self.logger.debug(f'Input coordinates: x={center_x:.1f}, y={center_y:.1f}')
-
- # Provide helpful warnings for common issues
- if not element_info.get('visible', False):
- self.logger.warning('⚠️ Target element appears to be invisible or has zero dimensions')
- if element_info.get('disabled', False):
- self.logger.warning('⚠️ Target element appears to be disabled')
- if not element_info.get('focusable', False):
- self.logger.warning('⚠️ Target element may not be focusable by standard criteria')
-
- # Clear existing text if requested
- if clear_existing:
- await cdp_session.cdp_client.send.Runtime.callFunctionOn(
- params={
- 'functionDeclaration': 'function() { if (this.value !== undefined) this.value = ""; if (this.textContent !== undefined) this.textContent = ""; }',
- 'objectId': object_id,
- },
- session_id=cdp_session.session_id,
- )
-
- # Try multiple focus strategies
- focused_successfully = False
+ # Check if this is a label with 'for' attribute - if so, find the actual input
+ actual_element = element_node
+ if element_node.tag_name.lower() == 'label' and element_node.attributes and element_node.attributes.get('for'):
+ input_id = element_node.attributes['for']
+ self.logger.debug(f'🏷️ Label detected with for="{input_id}", searching for associated input element')
- # Strategy 1: Try CDP DOM.focus (original method)
- try:
- result = await cdp_session.cdp_client.send.DOM.focus(
- params={'backendNodeId': backend_node_id},
- session_id=cdp_session.session_id,
- )
- focused_successfully = True
- self.logger.debug('✅ Element focused using CDP DOM.focus')
- except Exception as e:
- self.logger.debug(f'CDP DOM.focus failed: {e}')
-
- # Strategy 2: Try JavaScript focus as fallback
+ # Find the input element by ID using JavaScript
try:
- await cdp_session.cdp_client.send.Runtime.callFunctionOn(
+ input_result = await cdp_session.cdp_client.send.Runtime.evaluate(
params={
- 'functionDeclaration': 'function() { this.focus(); }',
- 'objectId': object_id,
+ 'expression': f'document.getElementById({json.dumps(input_id)})',
+ 'returnByValue': False, # Return object reference
},
session_id=cdp_session.session_id,
)
- focused_successfully = True
- self.logger.debug('✅ Element focused using JavaScript focus()')
- except Exception as js_e:
- self.logger.debug(f'JavaScript focus failed: {js_e}')
- # Strategy 3: Try click-to-focus for stubborn elements
- try:
- await cdp_session.cdp_client.send.Runtime.callFunctionOn(
+ input_object_id = input_result.get('result', {}).get('objectId')
+ if input_object_id:
+ self.logger.debug(f'✅ Found associated input element with id="{input_id}"')
+ # Use the input element instead of the label
+ object_id = input_object_id
+
+ # Get the input element's bounding box for coordinates
+ bbox_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
params={
- 'functionDeclaration': 'function() { this.click(); this.focus(); }',
- 'objectId': object_id,
+ 'functionDeclaration': 'function() { return this.getBoundingClientRect(); }',
+ 'objectId': input_object_id,
+ 'returnByValue': True,
},
session_id=cdp_session.session_id,
)
- focused_successfully = True
- self.logger.debug('✅ Element focused using click + focus combination')
- except Exception as click_e:
- self.logger.debug(f'Click + focus failed: {click_e}')
-
- # Strategy 4: Try simulated mouse click for maximum compatibility
- try:
- # Use coordinates already calculated from element bounds
- if input_coordinates and 'input_x' in input_coordinates and 'input_y' in input_coordinates:
- click_x = input_coordinates['input_x']
- click_y = input_coordinates['input_y']
-
- await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
- params={
- 'type': 'mousePressed',
- 'x': click_x,
- 'y': click_y,
- 'button': 'left',
- 'clickCount': 1,
- },
- session_id=cdp_session.session_id,
- )
- await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
- params={
- 'type': 'mouseReleased',
- 'x': click_x,
- 'y': click_y,
- 'button': 'left',
- 'clickCount': 1,
- },
- session_id=cdp_session.session_id,
- )
- focused_successfully = True
- self.logger.debug('✅ Element focused using simulated mouse click')
- else:
- self.logger.debug('Element bounds not available for mouse click')
- except Exception as mouse_e:
- self.logger.debug(f'Simulated mouse click failed: {mouse_e}')
-
- # Log focus result
- if not focused_successfully:
- self.logger.warning('⚠️ All focus strategies failed, typing without explicit focus')
-
- # Type the text character by character using proper human-like key events
+
+ bbox_value = bbox_result.get('result', {}).get('value')
+ if bbox_value:
+ center_x = bbox_value['x'] + bbox_value['width'] / 2
+ center_y = bbox_value['y'] + bbox_value['height'] / 2
+ input_coordinates = {'input_x': center_x, 'input_y': center_y}
+ self.logger.debug(f'✅ Using input coordinates: x={center_x:.1f}, y={center_y:.1f}')
+ else:
+ self.logger.warning(f'❌ Could not find input element with id="{input_id}", using label coordinates')
+ except Exception as e:
+ self.logger.warning(f'❌ Error finding input element: {e}, using label coordinates')
+
+ # Use actual_element coordinates directly (it already has them) if not already set by label-to-input mapping
+ if input_coordinates is None:
+ if hasattr(actual_element, 'center_x') and hasattr(actual_element, 'center_y'):
+ center_x = actual_element.center_x
+ center_y = actual_element.center_y
+ input_coordinates = {'input_x': center_x, 'input_y': center_y}
+ self.logger.debug(f'Using element coordinates: x={center_x:.1f}, y={center_y:.1f}')
+ else:
+ # Fallback: calculate from bounding box if available
+ if hasattr(actual_element, 'bounding_box') and actual_element.bounding_box:
+ bbox = actual_element.bounding_box
+ center_x = bbox.x + bbox.width / 2
+ center_y = bbox.y + bbox.height / 2
+ input_coordinates = {'input_x': center_x, 'input_y': center_y}
+ self.logger.debug(f'Calculated coordinates from bbox: x={center_x:.1f}, y={center_y:.1f}')
+ else:
+ input_coordinates = None
+ self.logger.warning('⚠️ No coordinates available for element')
+
+ # Ensure we have a valid object_id before proceeding
+ if not object_id:
+ raise ValueError('Could not get object_id for element')
+
+ # Step 1: Focus the element using simple strategy
+ focused_successfully = await self._focus_element_simple(
+ backend_node_id=backend_node_id, object_id=object_id, cdp_session=cdp_session, input_coordinates=input_coordinates
+ )
+
+ # Step 2: Clear existing text if requested
+ if clear_existing:
+ await self._clear_text_field(object_id=object_id, cdp_session=cdp_session)
+
+ # Step 3: Type the text character by character using proper human-like key events
# This emulates exactly how a human would type, which modern websites expect
self.logger.debug(f'🎯 Typing text character by character: "{text}"')
diff --git a/browser_use/dom/serializer/clickable_elements.py b/browser_use/dom/serializer/clickable_elements.py
index 836dba8ca..4e33dafbb 100644
--- a/browser_use/dom/serializer/clickable_elements.py
+++ b/browser_use/dom/serializer/clickable_elements.py
@@ -101,7 +101,6 @@ class ClickableElementDetector:
'select',
'textarea',
'a',
- 'label',
'details',
'summary',
'option',
@@ -110,6 +109,16 @@ class ClickableElementDetector:
if node.tag_name in interactive_tags:
return True
+ # Special handling for labels: only interactive if they DON'T have a 'for' attribute
+ # Labels with 'for' attribute should not be interactive - their associated input should be
+ if node.tag_name == 'label':
+ if node.attributes and node.attributes.get('for'):
+ # This label points to an input - it should NOT be interactive
+ return False
+ else:
+ # This label doesn't point to an input - it might be clickable itself
+ return True
+
# SVG elements need special handling - only interactive if they have explicit handlers
# svg_tags = {'svg', 'path', 'circle', 'rect', 'polygon', 'ellipse', 'line', 'polyline', 'g'}
# if node.tag_name in svg_tags: