Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
allowing MUCH MORE with coordinates
mertunsall46cb1a0ae19a
diff --git a/browser_use/agent/message_manager/views.py b/browser_use/agent/message_manager/views.pyindex 88076ba52..3f8fe7bc0 100644--- a/browser_use/agent/message_manager/views.py+++ b/browser_use/agent/message_manager/views.py@@ -32,7 +32,7 @@ class HistoryItem(BaseModel):def to_string(self) -> str:"""Get string representation of the history item"""- step_str = 'step' if self.step_number is not None else 'step_unknown'+ step_str = f'step_{self.step_number}' if self.step_number is not None else 'step_unknown'if self.error:return f"""<{step_str}>diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.pyindex b2729b248..3a002f08b 100644--- a/browser_use/agent/prompts.py+++ b/browser_use/agent/prompts.py@@ -43,8 +43,13 @@ class SystemPrompt:"""Load the prompt template from the markdown file."""try:# Choose the appropriate template based on flash_mode, use_thinking, and is_anthropic- if self.flash_mode and self.is_anthropic:- template_filename = 'system_prompt_flash_anthropic.md'+ if self.is_anthropic:+ if self.flash_mode:+ template_filename = 'system_prompt_flash_anthropic.md'+ elif self.use_thinking:+ template_filename = 'system_prompt_anthropic.md'+ else:+ template_filename = 'system_prompt_no_thinking.md'elif self.flash_mode:template_filename = 'system_prompt_flash.md'elif self.use_thinking:diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 8b36d6a64..df3dc6c63 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -213,12 +213,16 @@ class Agent(Generic[Context, AgentStructuredOutput]):if llm.provider == 'browser-use':flash_mode = True- # Auto-configure llm_screenshot_size for Claude Sonnet models+ # Auto-configure llm_screenshot_size for specific modelsif llm_screenshot_size is None:model_name = getattr(llm, 'model', '')- if isinstance(model_name, str) and model_name.startswith('claude-sonnet'):- llm_screenshot_size = (1400, 850)- logger.info('🖼️ Auto-configured LLM screenshot size for Claude Sonnet: 1400x850')+ if isinstance(model_name, str):+ if model_name.startswith('claude-sonnet'):+ llm_screenshot_size = (1400, 850)+ logger.info('🖼️ Auto-configured LLM screenshot size for Claude Sonnet: 1400x850')+ elif 'gemini' in model_name.lower():+ llm_screenshot_size = (1024, 720)+ logger.info('🖼️ Auto-configured LLM screenshot size for Gemini: 1024x720')if page_extraction_llm is None:page_extraction_llm = llmdiff --git a/browser_use/agent/system_prompt_anthropic.md b/browser_use/agent/system_prompt_anthropic.mdnew file mode 100644index 000000000..e8dff7267--- /dev/null+++ b/browser_use/agent/system_prompt_anthropic.md@@ -0,0 +1,34 @@+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.+<user_request>+User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.+</user_request>+<browser_state>+Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.+</browser_state>+<file_system>+PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or scroll and look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.+</file_system>+<action_rules>+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.++Default to element indices for browser interaction. If the target index is missing or a prior index-based action failed, use screenshot coordinates instead—DOM extraction doesn't capture everything. Coordinate interaction is useful when DOM extraction fails such as interacting with Canvas, scrolling on sidebars, etc.+</action_rules>+<output>You must call the AgentOutput tool with the following schema for the arguments:++{{+ "thinking": "A structured <think>-style reasoning block to analyze the current state, agent history, and plan the next goals. Analyze what happened in the last few steps.",+ "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",+ "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",+ "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence."+ "action": [+ {{+ "action_name": {{+ "parameter1": "value1",+ "parameter2": "value2"+ }}+ }}+ ]+}}++Action list should NEVER be empty.+</output>diff --git a/browser_use/agent/system_prompt_anthropic_no_thinking.md b/browser_use/agent/system_prompt_anthropic_no_thinking.mdnew file mode 100644index 000000000..204b80d92--- /dev/null+++ b/browser_use/agent/system_prompt_anthropic_no_thinking.md@@ -0,0 +1,33 @@+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.+<user_request>+User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.+</user_request>+<browser_state>+Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.+</browser_state>+<file_system>+PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or scroll and look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.+</file_system>+<action_rules>+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.++Default to element indices for browser interaction. If the target index is missing or a prior index-based action failed, use screenshot coordinates instead—DOM extraction doesn't capture everything. Coordinate interaction is useful when DOM extraction fails such as interacting with Canvas, scrolling on sidebars, etc.+</action_rules>+<output>You must call the AgentOutput tool with the following schema for the arguments:++{{+ "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",+ "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",+ "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence."+ "action": [+ {{+ "action_name": {{+ "parameter1": "value1",+ "parameter2": "value2"+ }}+ }}+ ]+}}++Action list should NEVER be empty.+</output>diff --git a/browser_use/agent/system_prompt_flash_anthropic.md b/browser_use/agent/system_prompt_flash_anthropic.mdindex 01182c55b..82b3a65ec 100644--- a/browser_use/agent/system_prompt_flash_anthropic.md+++ b/browser_use/agent/system_prompt_flash_anthropic.md@@ -10,6 +10,8 @@ PDFs are auto-downloaded to available_file_paths - use read_file to read the doc</file_system><action_rules>You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.++Default to element indices for browser interaction. If the target index is missing or a prior index-based action failed, use screenshot coordinates instead—DOM extraction doesn't capture everything. Coordinate interaction is useful when DOM extraction fails such as interacting with Canvas, scrolling on sidebars, etc.</action_rules><output>You must call the AgentOutput tool with the following schema for the arguments:diff --git a/browser_use/browser/events.py b/browser_use/browser/events.pyindex 4c40cb428..af99cf31a 100644--- a/browser_use/browser/events.py+++ b/browser_use/browser/events.py@@ -166,6 +166,17 @@ class ScrollEvent(ElementSelectedEvent[None]):event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollEvent', 8.0)) # seconds+class ScrollAtCoordinateEvent(BaseEvent[None]):+ """Scroll at specific coordinates using mouse wheel."""++ coordinate_x: int+ coordinate_y: int+ scroll_x: int = 0 # deltaX (positive=right, negative=left)+ scroll_y: int = 0 # deltaY (positive=down, negative=up)++ event_timeout: float | None = _get_timeout('TIMEOUT_ScrollAtCoordinateEvent', 8.0) # seconds++class SwitchTabEvent(BaseEvent[TargetID]):"""Switch to a different tab."""diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.pyindex a87664fe8..90745b968 100644--- a/browser_use/browser/watchdogs/default_action_watchdog.py+++ b/browser_use/browser/watchdogs/default_action_watchdog.py@@ -13,6 +13,7 @@ from browser_use.browser.events import (GoBackEvent,GoForwardEvent,RefreshEvent,+ ScrollAtCoordinateEvent,ScrollEvent,ScrollToTextEvent,SelectDropdownOptionEvent,@@ -383,6 +384,46 @@ class DefaultActionWatchdog(BaseWatchdog):except Exception as e:raise+ async def on_ScrollAtCoordinateEvent(self, event: ScrollAtCoordinateEvent) -> None:+ """Handle scroll at specific coordinates using CDP synthesizeScrollGesture."""+ # Check if we have a current target for scrolling+ if not self.browser_session.agent_focus_target_id:+ error_msg = 'No active target for scrolling'+ raise BrowserError(error_msg)++ try:+ # Get focused CDP session+ cdp_session = await self.browser_session.get_or_create_cdp_session()+ cdp_client = cdp_session.cdp_client+ session_id = cdp_session.session_id++ # Convert scroll deltas to gesture distances+ # Note: synthesizeScrollGesture uses opposite directions:+ # - positive yDistance = scroll UP (opposite of mouseWheel deltaY)+ # - positive xDistance = scroll LEFT (opposite of mouseWheel deltaX)+ # So we negate the values to maintain the same behavior as before+ params: dict[str, float] = {+ 'x': float(event.coordinate_x),+ 'y': float(event.coordinate_y),+ }+ if event.scroll_x != 0:+ params['xDistance'] = float(-event.scroll_x)+ if event.scroll_y != 0:+ params['yDistance'] = float(-event.scroll_y)++ # Synthesize scroll gesture at the specified coordinates+ await cdp_client.send.Input.synthesizeScrollGesture(+ params=params, # type: ignore[arg-type]+ session_id=session_id,+ )++ self.logger.debug(+ f'📄 Scrolled at ({event.coordinate_x}, {event.coordinate_y}) by deltaX={event.scroll_x}, deltaY={event.scroll_y}'+ )+ return None+ except Exception as e:+ raise+# ========== Implementation Methods ==========async def _check_element_occlusion(self, backend_node_id: int, x: float, y: float, cdp_session) -> bool:diff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex 95739f49b..74590b0f9 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -19,6 +19,7 @@ from browser_use.browser.events import (GetDropdownOptionsEvent,GoBackEvent,NavigateToUrlEvent,+ ScrollAtCoordinateEvent,ScrollEvent,ScrollToTextEvent,SendKeysEvent,@@ -44,6 +45,7 @@ from browser_use.tools.views import (NavigateAction,NoParamsAction,ScrollAction,+ ScrollAtCoordinateAction,SearchAction,SelectDropdownOptionAction,SendKeysAction,@@ -60,6 +62,7 @@ logger = logging.getLogger(__name__)ClickElementEvent.model_rebuild()TypeTextEvent.model_rebuild()ScrollEvent.model_rebuild()+ScrollAtCoordinateEvent.model_rebuild()UploadFileEvent.model_rebuild()Context = TypeVar('Context')@@ -251,6 +254,25 @@ class Tools(Generic[Context]):return actual_x, actual_yreturn llm_x, llm_y+ def _convert_llm_scroll_deltas_to_viewport(+ llm_scroll_x: int, llm_scroll_y: int, browser_session: BrowserSession+ ) -> tuple[int, int]:+ """Convert scroll deltas from LLM screenshot size to original viewport size."""+ if browser_session.llm_screenshot_size and browser_session._original_viewport_size:+ original_width, original_height = browser_session._original_viewport_size+ llm_width, llm_height = browser_session.llm_screenshot_size++ # Scale scroll deltas using the same ratio as coordinates+ actual_scroll_x = int((llm_scroll_x / llm_width) * original_width)+ actual_scroll_y = int((llm_scroll_y / llm_height) * original_height)++ logger.info(+ f'🔄 Scaling scroll deltas: LLM ({llm_scroll_x}, {llm_scroll_y}) @ {llm_width}x{llm_height} '+ f'→ Viewport ({actual_scroll_x}, {actual_scroll_y}) @ {original_width}x{original_height}'+ )+ return actual_scroll_x, actual_scroll_y+ return llm_scroll_x, llm_scroll_y+# Element Interaction Actionsasync def _click_by_coordinate(params: ClickElementAction, browser_session: BrowserSession) -> ActionResult:# Ensure coordinates are provided (type safety)@@ -352,7 +374,7 @@ class Tools(Generic[Context]):return ActionResult(error=error_msg)@self.registry.action(- 'Click element by index or coordinates. Prefer index over coordinates when possible. Either provide coordinates or index.',+ 'Click element by index or coordinates',param_model=ClickElementAction,)async def click(params: ClickElementAction, browser_session: BrowserSession):@@ -788,25 +810,14 @@ You will be given a query and the markdown of a webpage that has been filtered traise RuntimeError(str(e))@self.registry.action(- """Scroll by pages. REQUIRED: down=True/False (True=scroll down, False=scroll up, default=True). Optional: pages=0.5-10.0 (default 1.0). Use index for scroll containers (dropdowns/custom UI). High pages (10) reaches bottom. Multi-page scrolls sequentially. Viewport-based height, fallback 1000px/page.""",+ """Scroll by pages where one page = viewport height. Set down=True to scroll down, down=False to scroll up. Defaults to scrolling down one page.""",param_model=ScrollAction,)async def scroll(params: ScrollAction, browser_session: BrowserSession):try:- # Look up the node from the selector map if index is provided- # Special case: index 0 means scroll the whole page (root/body element)- node = None- if params.index is not None and params.index != 0:- node = await browser_session.get_element_by_index(params.index)- if node is None:- # Element does not exist- msg = f'Element index {params.index} not found in browser state'- return ActionResult(error=msg)-direction = 'down' if params.down else 'up'- target = f'element {params.index}' if params.index is not None and params.index != 0 else ''- # Get actual viewport height for more accurate scrolling+ # Get actual viewport height for scrollingtry:cdp_session = await browser_session.get_or_create_cdp_session()metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)@@ -823,6 +834,12 @@ You will be given a query and the markdown of a webpage that has been filtered tviewport_height = 1000 # Fallback to 1000pxlogger.debug(f'Failed to get viewport height, using fallback 1000px: {e}')+ # For reporting to LLM, use LLM screenshot height if available (so LLM's mental model matches)+ if browser_session.llm_screenshot_size:+ _, llm_viewport_height = browser_session.llm_screenshot_size+ else:+ llm_viewport_height = viewport_height+# For multiple pages (>=1.0), scroll one page at a time to ensure each scroll completesif params.pages >= 1.0:import asyncio@@ -840,7 +857,7 @@ You will be given a query and the markdown of a webpage that has been filtered tpixels = -pixelsevent = browser_session.event_bus.dispatch(- ScrollEvent(direction=direction, amount=abs(pixels), node=node)+ ScrollEvent(direction=direction, amount=abs(pixels), node=None))await eventawait event.event_result(raise_if_any=True, raise_if_none=False)@@ -861,7 +878,7 @@ You will be given a query and the markdown of a webpage that has been filtered tpixels = -pixelsevent = browser_session.event_bus.dispatch(- ScrollEvent(direction=direction, amount=abs(pixels), node=node)+ ScrollEvent(direction=direction, amount=abs(pixels), node=None))await eventawait event.event_result(raise_if_any=True, raise_if_none=False)@@ -871,18 +888,19 @@ You will be given a query and the markdown of a webpage that has been filtered tlogger.warning(f'Fractional scroll failed: {e}')if params.pages == 1.0:- long_term_memory = f'Scrolled {direction} {target} {viewport_height}px'.replace(' ', ' ')+ long_term_memory = f'Scrolled {direction} {llm_viewport_height}px'else:- long_term_memory = f'Scrolled {direction} {target} {completed_scrolls:.1f} pages'.replace(' ', ' ')+ long_term_memory = f'Scrolled {direction} {completed_scrolls:.1f} pages'else:# For fractional pages <1.0, do single scrollpixels = int(params.pages * viewport_height)event = browser_session.event_bus.dispatch(- ScrollEvent(direction='down' if params.down else 'up', amount=pixels, node=node)+ ScrollEvent(direction='down' if params.down else 'up', amount=pixels, node=None))await eventawait event.event_result(raise_if_any=True, raise_if_none=False)- long_term_memory = f'Scrolled {direction} {target} {params.pages} pages'.replace(' ', ' ')+ llm_pixels = int(params.pages * llm_viewport_height)+ long_term_memory = f'Scrolled {direction} {llm_pixels}px'msg = f'🔍 {long_term_memory}'logger.info(msg)@@ -892,6 +910,54 @@ You will be given a query and the markdown of a webpage that has been filtered terror_msg = 'Failed to execute scroll action.'return ActionResult(error=error_msg)+ @self.registry.action(+ 'Scroll at specific coordinates. Use when you need to scroll within a specific area (e.g., scrollable containers, maps, modals) not reachable via element index.',+ param_model=ScrollAtCoordinateAction,+ )+ async def scroll_at_coordinates(params: ScrollAtCoordinateAction, browser_session: BrowserSession):+ try:+ # Convert coordinates from LLM screenshot size to viewport size+ actual_x, actual_y = _convert_llm_coordinates_to_viewport(+ params.coordinate_x, params.coordinate_y, browser_session+ )++ # Convert scroll deltas from LLM screenshot size to viewport size+ actual_scroll_x, actual_scroll_y = _convert_llm_scroll_deltas_to_viewport(+ params.scroll_x, params.scroll_y, browser_session+ )++ # Dispatch scroll at coordinate event+ event = browser_session.event_bus.dispatch(+ ScrollAtCoordinateEvent(+ coordinate_x=actual_x,+ coordinate_y=actual_y,+ scroll_x=actual_scroll_x,+ scroll_y=actual_scroll_y,+ )+ )+ await event+ await event.event_result(raise_if_any=True, raise_if_none=False)++ # Build memory with scroll amounts+ direction_parts = []+ if params.scroll_y > 0:+ direction_parts.append(f'down {params.scroll_y}px')+ elif params.scroll_y < 0:+ direction_parts.append(f'up {abs(params.scroll_y)}px')+ if params.scroll_x > 0:+ direction_parts.append(f'right {params.scroll_x}px')+ elif params.scroll_x < 0:+ direction_parts.append(f'left {abs(params.scroll_x)}px')+ scroll_desc = ' and '.join(direction_parts) if direction_parts else 'zero'++ memory = f'Scrolled {scroll_desc} at ({params.coordinate_x}, {params.coordinate_y})'+ msg = f'🔍 {memory}'+ logger.info(msg)+ return ActionResult(extracted_content=msg, long_term_memory=memory)+ except Exception as e:+ logger.error(f'Failed to scroll at coordinates: {type(e).__name__}: {e}')+ return ActionResult(error=f'Failed to scroll at ({params.coordinate_x}, {params.coordinate_y})')+@self.registry.action('',param_model=SendKeysAction,diff --git a/browser_use/tools/views.py b/browser_use/tools/views.pyindex fc3eaea93..ac657388e 100644--- a/browser_use/tools/views.py+++ b/browser_use/tools/views.py@@ -73,8 +73,7 @@ class CloseTabAction(BaseModel):class ScrollAction(BaseModel):down: bool = Field(default=True, description='down=True=scroll down, down=False scroll up')- pages: float = Field(default=1.0, description='0.5=half page, 1=full page, 10=to bottom/top')- index: int | None = Field(default=None, description='Optional element index to scroll within specific container')+ pages: float = Field(default=1.0, description='0.5=half page, 1=full page, 2=two pages, etc.')class SendKeysAction(BaseModel):@@ -97,3 +96,10 @@ class GetDropdownOptionsAction(BaseModel):class SelectDropdownOptionAction(BaseModel):index: inttext: str = Field(description='exact text/value')+++class ScrollAtCoordinateAction(BaseModel):+ coordinate_x: int = Field(description='Horizontal coordinate relative to viewport left edge')+ coordinate_y: int = Field(description='Vertical coordinate relative to viewport top edge')+ scroll_x: int = Field(default=0, description='Horizontal scroll delta (positive=right, negative=left)')+ scroll_y: int = Field(default=0, description='Vertical scroll delta (positive=down, negative=up)')diff --git a/pyproject.toml b/pyproject.tomlindex 34500544c..d3c6bfe8a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -166,6 +166,9 @@ include = ["browser_use/agent/system_prompt_no_thinking.md","browser_use/agent/system_prompt_flash.md","browser_use/agent/system_prompt_flash_anthropic.md",+ "browser-use/agent/system_prompt_anthropic_no_thinking.md",+ "browser_use/agent/system_prompt_flash_anthropic.md",+ "browser-use/agent/system_prompt_anthropic.md","browser_use/code_use/system_prompt.md","browser_use/cli_templates/*.py","browser_use/py.typed",