Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

record coordinates for input_text

mertunsallcc1244201452
diff --git a/browser_use/browser/default_action_watchdog.py b/browser_use/browser/default_action_watchdog.py
index 267bad52d..f58131516 100644
--- a/browser_use/browser/default_action_watchdog.py
+++ b/browser_use/browser/default_action_watchdog.py
@@ -111,7 +111,7 @@ class DefaultActionWatchdog(BaseWatchdog):
except Exception as e:
raise
- async def on_TypeTextEvent(self, event: TypeTextEvent) -> None:
+ async def on_TypeTextEvent(self, event: TypeTextEvent) -> dict | None:
"""Handle text input request with CDP."""
try:
# Use the provided node
@@ -123,14 +123,16 @@ class DefaultActionWatchdog(BaseWatchdog):
# Type to the page without focusing any specific element
await self._type_to_page(event.text)
self.logger.info(f'⌨️ Typed "{event.text}" to the page (current focus)')
+ return None # No coordinates available for page typing
else:
try:
# Try to type to the specific element
- await self._input_text_element_node_impl(
+ input_metadata = await self._input_text_element_node_impl(
element_node, event.text, clear_existing=event.clear_existing or (not event.text)
)
self.logger.info(f'⌨️ Typed "{event.text}" into element with index {index_for_logging}')
self.logger.debug(f'Element xpath: {element_node.xpath}')
+ return input_metadata # Return coordinates if available
except Exception as e:
# Element not found or error - fall back to typing to the page
self.logger.warning(f'Failed to type to element {index_for_logging}: {e}. Falling back to page typing.')
@@ -140,11 +142,10 @@ class DefaultActionWatchdog(BaseWatchdog):
pass
await self._type_to_page(event.text)
self.logger.info(f'⌨️ Typed "{event.text}" to the page as fallback')
+ return None # No coordinates available for fallback typing
# Note: We don't clear cached state here - let multi_act handle DOM change detection
# by explicitly rebuilding and comparing when needed
-
- return None
except Exception as e:
raise
@@ -655,7 +656,7 @@ class DefaultActionWatchdog(BaseWatchdog):
self.logger.debug(f'Element focusability check failed: {e}')
return {'visible': False, 'focusable': False, 'interactive': False, 'disabled': True}
- async def _input_text_element_node_impl(self, element_node, text: str, clear_existing: bool = True):
+ async def _input_text_element_node_impl(self, element_node, text: str, clear_existing: bool = True) -> dict | None:
"""
Input text into an element using pure CDP with improved focus fallbacks.
"""
@@ -672,6 +673,9 @@ class DefaultActionWatchdog(BaseWatchdog):
# Get element info
backend_node_id = element_node.backend_node_id
+
+ # Track coordinates for metadata
+ input_coordinates = None
# Scroll element into view
try:
@@ -698,6 +702,14 @@ class DefaultActionWatchdog(BaseWatchdog):
element_info = await self._check_element_focusability(element_node, object_id, cdp_session.session_id)
self.logger.debug(f'Element focusability check: {element_info}')
+ # Extract coordinates from element bounds for metadata
+ bounds = element_info.get('bounds', {})
+ if bounds.get('width', 0) > 0 and bounds.get('height', 0) > 0:
+ center_x = bounds['x'] + bounds['width'] / 2
+ center_y = bounds['y'] + bounds['height'] / 2
+ input_coordinates = {"input_x": center_x, "input_y": center_y}
+ self.logger.debug(f'📍 Input coordinates: x={center_x:.1f}, y={center_y:.1f}')
+
# Provide helpful warnings for common issues
if not element_info.get('visible', False):
self.logger.warning('⚠️ Target element appears to be invisible or has zero dimensions')
@@ -760,11 +772,10 @@ class DefaultActionWatchdog(BaseWatchdog):
# Strategy 4: Try simulated mouse click for maximum compatibility
try:
- # Use bounds from focusability check if available
- bounds = element_info.get('bounds', {})
- if bounds.get('width', 0) > 0 and bounds.get('height', 0) > 0:
- click_x = bounds['x'] + bounds['width'] / 2
- click_y = bounds['y'] + bounds['height'] / 2
+ # Use coordinates already calculated from element bounds
+ if input_coordinates and 'input_x' in input_coordinates and 'input_y' in input_coordinates:
+ click_x = input_coordinates['input_x']
+ click_y = input_coordinates['input_y']
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
params={
@@ -826,6 +837,9 @@ class DefaultActionWatchdog(BaseWatchdog):
)
# Small delay between characters
await asyncio.sleep(0.01)
+
+ # Return coordinates metadata if available
+ return input_coordinates
except Exception as e:
self.logger.error(f'Failed to input text via CDP: {type(e).__name__}: {e}')
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.py
index b28695b2f..1f89d4f14 100644
--- a/browser_use/browser/events.py
+++ b/browser_use/browser/events.py
@@ -105,7 +105,7 @@ class ClickElementEvent(ElementSelectedEvent[dict | None]):
event_timeout: float | None = 15.0 # seconds
-class TypeTextEvent(ElementSelectedEvent[None]):
+class TypeTextEvent(ElementSelectedEvent[dict | None]):
"""Type text into an element."""
node: 'EnhancedDOMTreeNode'
diff --git a/browser_use/controller/service.py b/browser_use/controller/service.py
index 1fc590ff6..4104882e5 100644
--- a/browser_use/controller/service.py
+++ b/browser_use/controller/service.py
@@ -329,13 +329,16 @@ class Controller(Generic[Context]):
TypeTextEvent(node=node, text=params.text, clear_existing=params.clear_existing)
)
await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
+ input_metadata = await event.event_result(raise_if_any=True, raise_if_none=False)
msg = f"Input '{params.text}' into element {params.index}."
logger.info(msg)
+
+ # Include input coordinates in metadata if available
return ActionResult(
extracted_content=msg,
include_in_memory=True,
long_term_memory=f"Input '{params.text}' into element {params.index}.",
+ metadata=input_metadata if isinstance(input_metadata, dict) else None
)
except Exception as e:
# Log the full error for debugging