Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Revert "allowing MUCH MORE with coordinates"

Mert Unsal02cf38e864e1
diff --git a/browser_use/agent/message_manager/views.py b/browser_use/agent/message_manager/views.py
index 3f8fe7bc0..88076ba52 100644
--- a/browser_use/agent/message_manager/views.py
+++ b/browser_use/agent/message_manager/views.py
@@ -32,7 +32,7 @@ class HistoryItem(BaseModel):
def to_string(self) -> str:
"""Get string representation of the history item"""
- step_str = f'step_{self.step_number}' if self.step_number is not None else 'step_unknown'
+ step_str = 'step' if self.step_number is not None else 'step_unknown'
if self.error:
return f"""<{step_str}>
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index 3a002f08b..b2729b248 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -43,13 +43,8 @@ class SystemPrompt:
"""Load the prompt template from the markdown file."""
try:
# Choose the appropriate template based on flash_mode, use_thinking, and is_anthropic
- if self.is_anthropic:
- if self.flash_mode:
- template_filename = 'system_prompt_flash_anthropic.md'
- elif self.use_thinking:
- template_filename = 'system_prompt_anthropic.md'
- else:
- template_filename = 'system_prompt_no_thinking.md'
+ if self.flash_mode and self.is_anthropic:
+ template_filename = 'system_prompt_flash_anthropic.md'
elif self.flash_mode:
template_filename = 'system_prompt_flash.md'
elif self.use_thinking:
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index df3dc6c63..8b36d6a64 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -213,16 +213,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):
if llm.provider == 'browser-use':
flash_mode = True
- # Auto-configure llm_screenshot_size for specific models
+ # Auto-configure llm_screenshot_size for Claude Sonnet models
if llm_screenshot_size is None:
model_name = getattr(llm, 'model', '')
- if isinstance(model_name, str):
- if model_name.startswith('claude-sonnet'):
- llm_screenshot_size = (1400, 850)
- logger.info('🖼️ Auto-configured LLM screenshot size for Claude Sonnet: 1400x850')
- elif 'gemini' in model_name.lower():
- llm_screenshot_size = (1024, 720)
- logger.info('🖼️ Auto-configured LLM screenshot size for Gemini: 1024x720')
+ if isinstance(model_name, str) and model_name.startswith('claude-sonnet'):
+ llm_screenshot_size = (1400, 850)
+ logger.info('🖼️ Auto-configured LLM screenshot size for Claude Sonnet: 1400x850')
if page_extraction_llm is None:
page_extraction_llm = llm
diff --git a/browser_use/agent/system_prompt.md b/browser_use/agent/system_prompt.md
index d8f2c7c59..058849cf2 100644
--- a/browser_use/agent/system_prompt.md
+++ b/browser_use/agent/system_prompt.md
@@ -178,7 +178,7 @@ You must ALWAYS respond with a valid JSON in this exact format:
"thinking": "A structured <think>-style reasoning block that applies the <reasoning_rules> provided above.",
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
"memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
- "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+ "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence."
"action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
}}
Action list should NEVER be empty.
diff --git a/browser_use/agent/system_prompt_anthropic.md b/browser_use/agent/system_prompt_anthropic.md
deleted file mode 100644
index fdd20df93..000000000
--- a/browser_use/agent/system_prompt_anthropic.md
+++ /dev/null
@@ -1,34 +0,0 @@
-You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
-<user_request>
-User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.
-</user_request>
-<browser_state>
-Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.
-</browser_state>
-<file_system>
-PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or scroll and look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
-</file_system>
-<action_rules>
-You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
-
-Default to element indices for browser interaction. If the target index is missing or a prior index-based action failed, use screenshot coordinates instead—DOM extraction doesn't capture everything. Coordinate interaction is useful when DOM extraction fails such as interacting with Canvas, scrolling on sidebars, etc.
-</action_rules>
-<output>You must call the AgentOutput tool with the following schema for the arguments:
-
-{{
- "thinking": "A structured <think>-style reasoning block to analyze the current state, agent history, and plan the next goals. Analyze what happened in the last few steps.",
- "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
- "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
- "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
- "action": [
- {{
- "action_name": {{
- "parameter1": "value1",
- "parameter2": "value2"
- }}
- }}
- ]
-}}
-
-Action list should NEVER be empty.
-</output>
diff --git a/browser_use/agent/system_prompt_anthropic_no_thinking.md b/browser_use/agent/system_prompt_anthropic_no_thinking.md
deleted file mode 100644
index 09136a212..000000000
--- a/browser_use/agent/system_prompt_anthropic_no_thinking.md
+++ /dev/null
@@ -1,33 +0,0 @@
-You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
-<user_request>
-User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.
-</user_request>
-<browser_state>
-Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.
-</browser_state>
-<file_system>
-PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or scroll and look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
-</file_system>
-<action_rules>
-You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
-
-Default to element indices for browser interaction. If the target index is missing or a prior index-based action failed, use screenshot coordinates instead—DOM extraction doesn't capture everything. Coordinate interaction is useful when DOM extraction fails such as interacting with Canvas, scrolling on sidebars, etc.
-</action_rules>
-<output>You must call the AgentOutput tool with the following schema for the arguments:
-
-{{
- "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
- "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
- "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
- "action": [
- {{
- "action_name": {{
- "parameter1": "value1",
- "parameter2": "value2"
- }}
- }}
- ]
-}}
-
-Action list should NEVER be empty.
-</output>
diff --git a/browser_use/agent/system_prompt_flash_anthropic.md b/browser_use/agent/system_prompt_flash_anthropic.md
index 82b3a65ec..01182c55b 100644
--- a/browser_use/agent/system_prompt_flash_anthropic.md
+++ b/browser_use/agent/system_prompt_flash_anthropic.md
@@ -10,8 +10,6 @@ PDFs are auto-downloaded to available_file_paths - use read_file to read the doc
</file_system>
<action_rules>
You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
-
-Default to element indices for browser interaction. If the target index is missing or a prior index-based action failed, use screenshot coordinates instead—DOM extraction doesn't capture everything. Coordinate interaction is useful when DOM extraction fails such as interacting with Canvas, scrolling on sidebars, etc.
</action_rules>
<output>You must call the AgentOutput tool with the following schema for the arguments:
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.py
index af99cf31a..4c40cb428 100644
--- a/browser_use/browser/events.py
+++ b/browser_use/browser/events.py
@@ -166,17 +166,6 @@ class ScrollEvent(ElementSelectedEvent[None]):
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollEvent', 8.0)) # seconds
-class ScrollAtCoordinateEvent(BaseEvent[None]):
- """Scroll at specific coordinates using mouse wheel."""
-
- coordinate_x: int
- coordinate_y: int
- scroll_x: int = 0 # deltaX (positive=right, negative=left)
- scroll_y: int = 0 # deltaY (positive=down, negative=up)
-
- event_timeout: float | None = _get_timeout('TIMEOUT_ScrollAtCoordinateEvent', 8.0) # seconds
-
-
class SwitchTabEvent(BaseEvent[TargetID]):
"""Switch to a different tab."""
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.py
index 90745b968..a87664fe8 100644
--- a/browser_use/browser/watchdogs/default_action_watchdog.py
+++ b/browser_use/browser/watchdogs/default_action_watchdog.py
@@ -13,7 +13,6 @@ from browser_use.browser.events import (
GoBackEvent,
GoForwardEvent,
RefreshEvent,
- ScrollAtCoordinateEvent,
ScrollEvent,
ScrollToTextEvent,
SelectDropdownOptionEvent,
@@ -384,46 +383,6 @@ class DefaultActionWatchdog(BaseWatchdog):
except Exception as e:
raise
- async def on_ScrollAtCoordinateEvent(self, event: ScrollAtCoordinateEvent) -> None:
- """Handle scroll at specific coordinates using CDP synthesizeScrollGesture."""
- # Check if we have a current target for scrolling
- if not self.browser_session.agent_focus_target_id:
- error_msg = 'No active target for scrolling'
- raise BrowserError(error_msg)
-
- try:
- # Get focused CDP session
- cdp_session = await self.browser_session.get_or_create_cdp_session()
- cdp_client = cdp_session.cdp_client
- session_id = cdp_session.session_id
-
- # Convert scroll deltas to gesture distances
- # Note: synthesizeScrollGesture uses opposite directions:
- # - positive yDistance = scroll UP (opposite of mouseWheel deltaY)
- # - positive xDistance = scroll LEFT (opposite of mouseWheel deltaX)
- # So we negate the values to maintain the same behavior as before
- params: dict[str, float] = {
- 'x': float(event.coordinate_x),
- 'y': float(event.coordinate_y),
- }
- if event.scroll_x != 0:
- params['xDistance'] = float(-event.scroll_x)
- if event.scroll_y != 0:
- params['yDistance'] = float(-event.scroll_y)
-
- # Synthesize scroll gesture at the specified coordinates
- await cdp_client.send.Input.synthesizeScrollGesture(
- params=params, # type: ignore[arg-type]
- session_id=session_id,
- )
-
- self.logger.debug(
- f'📄 Scrolled at ({event.coordinate_x}, {event.coordinate_y}) by deltaX={event.scroll_x}, deltaY={event.scroll_y}'
- )
- return None
- except Exception as e:
- raise
-
# ========== Implementation Methods ==========
async def _check_element_occlusion(self, backend_node_id: int, x: float, y: float, cdp_session) -> bool:
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 74590b0f9..95739f49b 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -19,7 +19,6 @@ from browser_use.browser.events import (
GetDropdownOptionsEvent,
GoBackEvent,
NavigateToUrlEvent,
- ScrollAtCoordinateEvent,
ScrollEvent,
ScrollToTextEvent,
SendKeysEvent,
@@ -45,7 +44,6 @@ from browser_use.tools.views import (
NavigateAction,
NoParamsAction,
ScrollAction,
- ScrollAtCoordinateAction,
SearchAction,
SelectDropdownOptionAction,
SendKeysAction,
@@ -62,7 +60,6 @@ logger = logging.getLogger(__name__)
ClickElementEvent.model_rebuild()
TypeTextEvent.model_rebuild()
ScrollEvent.model_rebuild()
-ScrollAtCoordinateEvent.model_rebuild()
UploadFileEvent.model_rebuild()
Context = TypeVar('Context')
@@ -254,25 +251,6 @@ class Tools(Generic[Context]):
return actual_x, actual_y
return llm_x, llm_y
- def _convert_llm_scroll_deltas_to_viewport(
- llm_scroll_x: int, llm_scroll_y: int, browser_session: BrowserSession
- ) -> tuple[int, int]:
- """Convert scroll deltas from LLM screenshot size to original viewport size."""
- if browser_session.llm_screenshot_size and browser_session._original_viewport_size:
- original_width, original_height = browser_session._original_viewport_size
- llm_width, llm_height = browser_session.llm_screenshot_size
-
- # Scale scroll deltas using the same ratio as coordinates
- actual_scroll_x = int((llm_scroll_x / llm_width) * original_width)
- actual_scroll_y = int((llm_scroll_y / llm_height) * original_height)
-
- logger.info(
- f'🔄 Scaling scroll deltas: LLM ({llm_scroll_x}, {llm_scroll_y}) @ {llm_width}x{llm_height} '
- f'→ Viewport ({actual_scroll_x}, {actual_scroll_y}) @ {original_width}x{original_height}'
- )
- return actual_scroll_x, actual_scroll_y
- return llm_scroll_x, llm_scroll_y
-
# Element Interaction Actions
async def _click_by_coordinate(params: ClickElementAction, browser_session: BrowserSession) -> ActionResult:
# Ensure coordinates are provided (type safety)
@@ -374,7 +352,7 @@ class Tools(Generic[Context]):
return ActionResult(error=error_msg)
@self.registry.action(
- 'Click element by index or coordinates',
+ 'Click element by index or coordinates. Prefer index over coordinates when possible. Either provide coordinates or index.',
param_model=ClickElementAction,
)
async def click(params: ClickElementAction, browser_session: BrowserSession):
@@ -810,14 +788,25 @@ You will be given a query and the markdown of a webpage that has been filtered t
raise RuntimeError(str(e))
@self.registry.action(
- """Scroll by pages where one page = viewport height. Set down=True to scroll down, down=False to scroll up. Defaults to scrolling down one page.""",
+ """Scroll by pages. REQUIRED: down=True/False (True=scroll down, False=scroll up, default=True). Optional: pages=0.5-10.0 (default 1.0). Use index for scroll containers (dropdowns/custom UI). High pages (10) reaches bottom. Multi-page scrolls sequentially. Viewport-based height, fallback 1000px/page.""",
param_model=ScrollAction,
)
async def scroll(params: ScrollAction, browser_session: BrowserSession):
try:
+ # Look up the node from the selector map if index is provided
+ # Special case: index 0 means scroll the whole page (root/body element)
+ node = None
+ if params.index is not None and params.index != 0:
+ node = await browser_session.get_element_by_index(params.index)
+ if node is None:
+ # Element does not exist
+ msg = f'Element index {params.index} not found in browser state'
+ return ActionResult(error=msg)
+
direction = 'down' if params.down else 'up'
+ target = f'element {params.index}' if params.index is not None and params.index != 0 else ''
- # Get actual viewport height for scrolling
+ # Get actual viewport height for more accurate scrolling
try:
cdp_session = await browser_session.get_or_create_cdp_session()
metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
@@ -834,12 +823,6 @@ You will be given a query and the markdown of a webpage that has been filtered t
viewport_height = 1000 # Fallback to 1000px
logger.debug(f'Failed to get viewport height, using fallback 1000px: {e}')
- # For reporting to LLM, use LLM screenshot height if available (so LLM's mental model matches)
- if browser_session.llm_screenshot_size:
- _, llm_viewport_height = browser_session.llm_screenshot_size
- else:
- llm_viewport_height = viewport_height
-
# For multiple pages (>=1.0), scroll one page at a time to ensure each scroll completes
if params.pages >= 1.0:
import asyncio
@@ -857,7 +840,7 @@ You will be given a query and the markdown of a webpage that has been filtered t
pixels = -pixels
event = browser_session.event_bus.dispatch(
- ScrollEvent(direction=direction, amount=abs(pixels), node=None)
+ ScrollEvent(direction=direction, amount=abs(pixels), node=node)
)
await event
await event.event_result(raise_if_any=True, raise_if_none=False)
@@ -878,7 +861,7 @@ You will be given a query and the markdown of a webpage that has been filtered t
pixels = -pixels
event = browser_session.event_bus.dispatch(
- ScrollEvent(direction=direction, amount=abs(pixels), node=None)
+ ScrollEvent(direction=direction, amount=abs(pixels), node=node)
)
await event
await event.event_result(raise_if_any=True, raise_if_none=False)
@@ -888,19 +871,18 @@ You will be given a query and the markdown of a webpage that has been filtered t
logger.warning(f'Fractional scroll failed: {e}')
if params.pages == 1.0:
- long_term_memory = f'Scrolled {direction} {llm_viewport_height}px'
+ long_term_memory = f'Scrolled {direction} {target} {viewport_height}px'.replace(' ', ' ')
else:
- long_term_memory = f'Scrolled {direction} {completed_scrolls:.1f} pages'
+ long_term_memory = f'Scrolled {direction} {target} {completed_scrolls:.1f} pages'.replace(' ', ' ')
else:
# For fractional pages <1.0, do single scroll
pixels = int(params.pages * viewport_height)
event = browser_session.event_bus.dispatch(
- ScrollEvent(direction='down' if params.down else 'up', amount=pixels, node=None)
+ ScrollEvent(direction='down' if params.down else 'up', amount=pixels, node=node)
)
await event
await event.event_result(raise_if_any=True, raise_if_none=False)
- llm_pixels = int(params.pages * llm_viewport_height)
- long_term_memory = f'Scrolled {direction} {llm_pixels}px'
+ long_term_memory = f'Scrolled {direction} {target} {params.pages} pages'.replace(' ', ' ')
msg = f'🔍 {long_term_memory}'
logger.info(msg)
@@ -910,54 +892,6 @@ You will be given a query and the markdown of a webpage that has been filtered t
error_msg = 'Failed to execute scroll action.'
return ActionResult(error=error_msg)
- @self.registry.action(
- 'Scroll at specific coordinates. Use when you need to scroll within a specific area (e.g., scrollable containers, maps, modals) not reachable via element index.',
- param_model=ScrollAtCoordinateAction,
- )
- async def scroll_at_coordinates(params: ScrollAtCoordinateAction, browser_session: BrowserSession):
- try:
- # Convert coordinates from LLM screenshot size to viewport size
- actual_x, actual_y = _convert_llm_coordinates_to_viewport(
- params.coordinate_x, params.coordinate_y, browser_session
- )
-
- # Convert scroll deltas from LLM screenshot size to viewport size
- actual_scroll_x, actual_scroll_y = _convert_llm_scroll_deltas_to_viewport(
- params.scroll_x, params.scroll_y, browser_session
- )
-
- # Dispatch scroll at coordinate event
- event = browser_session.event_bus.dispatch(
- ScrollAtCoordinateEvent(
- coordinate_x=actual_x,
- coordinate_y=actual_y,
- scroll_x=actual_scroll_x,
- scroll_y=actual_scroll_y,
- )
- )
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Build memory with scroll amounts
- direction_parts = []
- if params.scroll_y > 0:
- direction_parts.append(f'down {params.scroll_y}px')
- elif params.scroll_y < 0:
- direction_parts.append(f'up {abs(params.scroll_y)}px')
- if params.scroll_x > 0:
- direction_parts.append(f'right {params.scroll_x}px')
- elif params.scroll_x < 0:
- direction_parts.append(f'left {abs(params.scroll_x)}px')
- scroll_desc = ' and '.join(direction_parts) if direction_parts else 'zero'
-
- memory = f'Scrolled {scroll_desc} at ({params.coordinate_x}, {params.coordinate_y})'
- msg = f'🔍 {memory}'
- logger.info(msg)
- return ActionResult(extracted_content=msg, long_term_memory=memory)
- except Exception as e:
- logger.error(f'Failed to scroll at coordinates: {type(e).__name__}: {e}')
- return ActionResult(error=f'Failed to scroll at ({params.coordinate_x}, {params.coordinate_y})')
-
@self.registry.action(
'',
param_model=SendKeysAction,
diff --git a/browser_use/tools/views.py b/browser_use/tools/views.py
index ac657388e..fc3eaea93 100644
--- a/browser_use/tools/views.py
+++ b/browser_use/tools/views.py
@@ -73,7 +73,8 @@ class CloseTabAction(BaseModel):
class ScrollAction(BaseModel):
down: bool = Field(default=True, description='down=True=scroll down, down=False scroll up')
- pages: float = Field(default=1.0, description='0.5=half page, 1=full page, 2=two pages, etc.')
+ pages: float = Field(default=1.0, description='0.5=half page, 1=full page, 10=to bottom/top')
+ index: int | None = Field(default=None, description='Optional element index to scroll within specific container')
class SendKeysAction(BaseModel):
@@ -96,10 +97,3 @@ class GetDropdownOptionsAction(BaseModel):
class SelectDropdownOptionAction(BaseModel):
index: int
text: str = Field(description='exact text/value')
-
-
-class ScrollAtCoordinateAction(BaseModel):
- coordinate_x: int = Field(description='Horizontal coordinate relative to viewport left edge')
- coordinate_y: int = Field(description='Vertical coordinate relative to viewport top edge')
- scroll_x: int = Field(default=0, description='Horizontal scroll delta (positive=right, negative=left)')
- scroll_y: int = Field(default=0, description='Vertical scroll delta (positive=down, negative=up)')
diff --git a/pyproject.toml b/pyproject.toml
index d3c6bfe8a..34500544c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -166,9 +166,6 @@ include = [
"browser_use/agent/system_prompt_no_thinking.md",
"browser_use/agent/system_prompt_flash.md",
"browser_use/agent/system_prompt_flash_anthropic.md",
- "browser-use/agent/system_prompt_anthropic_no_thinking.md",
- "browser_use/agent/system_prompt_flash_anthropic.md",
- "browser-use/agent/system_prompt_anthropic.md",
"browser_use/code_use/system_prompt.md",
"browser_use/cli_templates/*.py",
"browser_use/py.typed",