Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

add clicking by coordinate

mertunsall4dca53282c71
diff --git a/browser_use/agent/system_prompt_flash.md b/browser_use/agent/system_prompt_flash.md
index e5254fddc..34113a869 100644
--- a/browser_use/agent/system_prompt_flash.md
+++ b/browser_use/agent/system_prompt_flash.md
@@ -2,7 +2,7 @@ You are an AI agent designed to operate in an iterative loop to automate browser
<language_settings>Default: English. Match user's language.</language_settings>
<user_request>Ultimate objective. Specific tasks: follow each step. Open-ended: plan approach.</user_request>
<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.</browser_state>
-<file_system>- PDFs auto-download to available_file_paths. Read file or scroll viewer. Persistent file system for progress tracking. Long tasks <10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. CSV: use double quotes for commas. available_file_paths: downloaded/user files (read/upload only).</file_system>
+<file_system>- PDFs auto-download to available_file_paths. Read file or scroll viewer. Persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. CSV: use double quotes for commas. available_file_paths: downloaded/user files (read/upload only).</file_system>
<output>You must respond with a valid JSON in this exact format:
{{
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 60c975147..91df40d2c 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -741,6 +741,9 @@ class BrowserSession(BaseModel):
viewport_height = self.browser_profile.viewport.height
device_scale_factor = self.browser_profile.device_scale_factor or 1.0
+ self.logger.info(
+ f'Setting viewport to {viewport_width}x{viewport_height} with device scale factor {device_scale_factor} whereas original device scale factor was {self.browser_profile.device_scale_factor}'
+ )
# Use the helper method with the new tab's target_id
await self._cdp_set_viewport(viewport_width, viewport_height, device_scale_factor, target_id=event.target_id)
@@ -2129,6 +2132,115 @@ class BrowserSession(BaseModel):
# Don't fail the action if highlighting fails
self.logger.debug(f'Failed to highlight interaction element: {e}')
+ async def highlight_coordinate_click(self, x: int, y: int) -> None:
+ """Temporarily highlight a coordinate click position for user visibility.
+
+ This creates a visual highlight at the specified coordinates showing where
+ the click action occurred. The highlight automatically fades after the configured duration.
+
+ Args:
+ x: Horizontal coordinate relative to viewport left edge
+ y: Vertical coordinate relative to viewport top edge
+ """
+ if not self.browser_profile.highlight_elements:
+ return
+
+ try:
+ import json
+
+ cdp_session = await self.get_or_create_cdp_session()
+
+ color = self.browser_profile.interaction_highlight_color
+ duration_ms = int(self.browser_profile.interaction_highlight_duration * 1000)
+
+ # Create animated crosshair and circle at the click coordinates
+ script = f"""
+ (function() {{
+ const x = {x};
+ const y = {y};
+ const color = {json.dumps(color)};
+ const duration = {duration_ms};
+
+ // Get current scroll position
+ const scrollX = window.pageXOffset || document.documentElement.scrollLeft || 0;
+ const scrollY = window.pageYOffset || document.documentElement.scrollTop || 0;
+
+ // Create container
+ const container = document.createElement('div');
+ container.setAttribute('data-browser-use-coordinate-highlight', 'true');
+ container.style.cssText = `
+ position: absolute;
+ left: ${{x + scrollX}}px;
+ top: ${{y + scrollY}}px;
+ width: 0;
+ height: 0;
+ pointer-events: none;
+ z-index: 2147483647;
+ `;
+
+ // Create outer circle
+ const outerCircle = document.createElement('div');
+ outerCircle.style.cssText = `
+ position: absolute;
+ left: -15px;
+ top: -15px;
+ width: 30px;
+ height: 30px;
+ border: 3px solid ${{color}};
+ border-radius: 50%;
+ opacity: 0;
+ transform: scale(0.3);
+ transition: all 0.2s ease-out;
+ `;
+ container.appendChild(outerCircle);
+
+ // Create center dot
+ const centerDot = document.createElement('div');
+ centerDot.style.cssText = `
+ position: absolute;
+ left: -4px;
+ top: -4px;
+ width: 8px;
+ height: 8px;
+ background: ${{color}};
+ border-radius: 50%;
+ opacity: 0;
+ transform: scale(0);
+ transition: all 0.15s ease-out;
+ `;
+ container.appendChild(centerDot);
+
+ document.body.appendChild(container);
+
+ // Animate in
+ setTimeout(() => {{
+ outerCircle.style.opacity = '0.8';
+ outerCircle.style.transform = 'scale(1)';
+ centerDot.style.opacity = '1';
+ centerDot.style.transform = 'scale(1)';
+ }}, 10);
+
+ // Animate out and remove
+ setTimeout(() => {{
+ outerCircle.style.opacity = '0';
+ outerCircle.style.transform = 'scale(1.5)';
+ centerDot.style.opacity = '0';
+ setTimeout(() => container.remove(), 300);
+ }}, duration);
+
+ return {{ created: true }};
+ }})();
+ """
+
+ # Fire and forget - don't wait for completion
+ await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={'expression': script, 'returnByValue': True}, session_id=cdp_session.session_id
+ )
+
+ except Exception as e:
+ # Don't fail the action if highlighting fails
+ self.logger.debug(f'Failed to highlight coordinate click: {e}')
+
async def add_highlights(self, selector_map: dict[int, 'EnhancedDOMTreeNode']) -> None:
"""Add visual highlights to the browser DOM for user visibility."""
if not self.browser_profile.dom_highlight_elements or not selector_map:
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 9f84d07ee..4380b6667 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -232,13 +232,37 @@ class Tools(Generic[Context]):
return ActionResult(extracted_content=memory, long_term_memory=memory)
# Element Interaction Actions
+ async def _click_by_coordinate(params: ClickElementAction, browser_session: BrowserSession) -> ActionResult:
+ # Ensure coordinates are provided (type safety)
+ if params.coordinate_x is None or params.coordinate_y is None:
+ return ActionResult(error='Both coordinate_x and coordinate_y must be provided')
- @self.registry.action(
- '',
- param_model=ClickElementAction,
- )
- async def click(params: ClickElementAction, browser_session: BrowserSession):
- # Dispatch click event with node
+ try:
+ # Highlight the coordinate being clicked (truly non-blocking)
+ asyncio.create_task(browser_session.highlight_coordinate_click(params.coordinate_x, params.coordinate_y))
+
+ # Use Actor (page.mouse.click) for coordinate-based clicking
+ page = await browser_session.get_current_page()
+ if page is None:
+ return ActionResult(error='No active page found')
+
+ mouse = await page.mouse
+ await mouse.click(params.coordinate_x, params.coordinate_y)
+
+ memory = f'Clicked on coordinate {params.coordinate_x}, {params.coordinate_y}'
+ msg = f'🖱️ {memory}'
+ logger.info(msg)
+
+ return ActionResult(
+ extracted_content=memory,
+ metadata={'click_x': params.coordinate_x, 'click_y': params.coordinate_y},
+ )
+ except Exception as e:
+ error_msg = f'Failed to click at coordinates ({params.coordinate_x}, {params.coordinate_y}).'
+ return ActionResult(error=error_msg)
+
+ async def _click_by_index(params: ClickElementAction, browser_session: BrowserSession) -> ActionResult:
+ assert params.index is not None
try:
assert params.index != 0, (
'Cannot click on element with index 0. If there are no interactive elements use scroll(), wait(), refresh(), etc. to troubleshoot'
@@ -285,6 +309,22 @@ class Tools(Generic[Context]):
error_msg = f'Failed to click element {params.index}: {str(e)}'
return ActionResult(error=error_msg)
+ @self.registry.action(
+ 'Click element by index or coordinates. Prefer index over coordinates when possible.',
+ param_model=ClickElementAction,
+ )
+ async def click(params: ClickElementAction, browser_session: BrowserSession):
+ # Validate that either index or coordinates are provided
+ if params.index is None and (params.coordinate_x is None or params.coordinate_y is None):
+ return ActionResult(error='Must provide either index or both coordinate_x and coordinate_y')
+
+ # Try index-based clicking first if index is provided
+ if params.index is not None:
+ return await _click_by_index(params, browser_session)
+ # Coordinate-based clicking when index is not provided
+ else:
+ return await _click_by_coordinate(params, browser_session)
+
@self.registry.action(
'',
param_model=InputTextAction,
@@ -564,11 +604,6 @@ class Tools(Generic[Context]):
long_term_memory=memory,
)
- # Content Actions
-
- # TODO: Refactor to use events instead of direct page access
- # This action is temporarily disabled as it needs refactoring to use events
-
@self.registry.action(
"""LLM extracts structured data from page markdown. Use when: on right page, know what to extract, haven't called before on same page+query. Can't get interactive elements. Set extract_links=True for URLs. Use start_from_char if truncated. If fails, use find_text/scroll instead.""",
)
diff --git a/browser_use/tools/views.py b/browser_use/tools/views.py
index 5775175a0..d90f6cc33 100644
--- a/browser_use/tools/views.py
+++ b/browser_use/tools/views.py
@@ -25,7 +25,9 @@ GoToUrlAction = NavigateAction
class ClickElementAction(BaseModel):
- index: int = Field(ge=1, description='from browser_state')
+ index: int | None = Field(default=None, ge=1, description='Element index from browser_state')
+ coordinate_x: int | None = Field(default=None, description='Horizontal coordinate relative to viewport left edge')
+ coordinate_y: int | None = Field(default=None, description='Vertical coordinate relative to viewport top edge')
# expect_download: bool = Field(default=False, description='set True if expecting a download, False otherwise') # moved to downloads_watchdog.py
# click_count: int = 1 # TODO