Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

image resizing the make sonnet 4.5 work

mertunsalld06d8fa8a2b1
diff --git a/browser_use/agent/message_manager/service.py b/browser_use/agent/message_manager/service.py
index a8b71253e..99034c2a3 100644
--- a/browser_use/agent/message_manager/service.py
+++ b/browser_use/agent/message_manager/service.py
@@ -110,6 +110,7 @@ class MessageManager:
include_tool_call_examples: bool = False,
include_recent_events: bool = False,
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
+ llm_screenshot_size: tuple[int, int] | None = None,
):
self.task = task
self.state = state
@@ -122,6 +123,7 @@ class MessageManager:
self.include_tool_call_examples = include_tool_call_examples
self.include_recent_events = include_recent_events
self.sample_images = sample_images
+ self.llm_screenshot_size = llm_screenshot_size
assert max_history_items is None or max_history_items > 5, 'max_history_items must be None or greater than 5'
@@ -361,6 +363,7 @@ class MessageManager:
include_recent_events=self.include_recent_events,
sample_images=self.sample_images,
read_state_images=self.state.read_state_images,
+ llm_screenshot_size=self.llm_screenshot_size,
).get_user_message(effective_use_vision)
# Store state message text for history
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index 86fc0c1ac..d0367295d 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -85,6 +85,7 @@ class AgentMessagePrompt:
include_recent_events: bool = False,
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
read_state_images: list[dict] | None = None,
+ llm_screenshot_size: tuple[int, int] | None = None,
):
self.browser_state: 'BrowserStateSummary' = browser_state_summary
self.file_system: 'FileSystem | None' = file_system
@@ -102,6 +103,7 @@ class AgentMessagePrompt:
self.include_recent_events = include_recent_events
self.sample_images = sample_images or []
self.read_state_images = read_state_images or []
+ self.llm_screenshot_size = llm_screenshot_size
assert self.browser_state
def _extract_page_statistics(self) -> dict[str, int]:
@@ -318,6 +320,34 @@ Available tabs:
agent_state += f'<available_file_paths>{available_file_paths_text}\nUse with absolute paths</available_file_paths>\n'
return agent_state
+ def _resize_screenshot(self, screenshot_b64: str) -> str:
+ """Resize screenshot to llm_screenshot_size if configured."""
+ if not self.llm_screenshot_size:
+ return screenshot_b64
+
+ try:
+ import base64
+ import logging
+ from io import BytesIO
+
+ from PIL import Image
+
+ img = Image.open(BytesIO(base64.b64decode(screenshot_b64)))
+ if img.size == self.llm_screenshot_size:
+ return screenshot_b64
+
+ logging.getLogger(__name__).info(
+ f'πŸ”„ Resizing screenshot from {img.size[0]}x{img.size[1]} to {self.llm_screenshot_size[0]}x{self.llm_screenshot_size[1]} for LLM'
+ )
+
+ img_resized = img.resize(self.llm_screenshot_size, Image.Resampling.LANCZOS)
+ buffer = BytesIO()
+ img_resized.save(buffer, format='PNG')
+ return base64.b64encode(buffer.getvalue()).decode('utf-8')
+ except Exception as e:
+ logging.getLogger(__name__).warning(f'Failed to resize screenshot: {e}, using original')
+ return screenshot_b64
+
@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')
def get_user_message(self, use_vision: bool = True) -> UserMessage:
"""Get complete state as a single cached message"""
@@ -369,11 +399,14 @@ Available tabs:
# Add label as text content
content_parts.append(ContentPartTextParam(text=label))
+ # Resize screenshot if llm_screenshot_size is configured
+ processed_screenshot = self._resize_screenshot(screenshot)
+
# Add the screenshot
content_parts.append(
ContentPartImageParam(
image_url=ImageURL(
- url=f'data:image/png;base64,{screenshot}',
+ url=f'data:image/png;base64,{processed_screenshot}',
media_type='image/png',
detail=self.vision_detail_level,
),
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index b6db1a5f2..eebf9648a 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -180,9 +180,20 @@ class Agent(Generic[Context, AgentStructuredOutput]):
include_recent_events: bool = False,
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
final_response_after_failure: bool = True,
+ llm_screenshot_size: tuple[int, int] | None = None,
_url_shortening_limit: int = 25,
**kwargs,
):
+ # Validate llm_screenshot_size
+ if llm_screenshot_size is not None:
+ if not isinstance(llm_screenshot_size, tuple) or len(llm_screenshot_size) != 2:
+ raise ValueError('llm_screenshot_size must be a tuple of (width, height)')
+ width, height = llm_screenshot_size
+ if not isinstance(width, int) or not isinstance(height, int):
+ raise ValueError('llm_screenshot_size dimensions must be integers')
+ if width < 100 or height < 100:
+ raise ValueError('llm_screenshot_size dimensions must be at least 100 pixels')
+ logger.info(f'πŸ–ΌοΈ LLM screenshot resizing enabled: {width}x{height}')
if llm is None:
default_llm_name = CONFIG.DEFAULT_LLM
if default_llm_name:
@@ -349,6 +360,9 @@ class Agent(Generic[Context, AgentStructuredOutput]):
f'{" +file_system" if self.file_system else ""}'
)
+ # Store llm_screenshot_size in browser_session so tools can access it
+ self.browser_session.llm_screenshot_size = llm_screenshot_size
+
# Initialize message manager with state
# Initial system prompt with all actions - will be updated during each step
self._message_manager = MessageManager(
@@ -371,6 +385,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
include_tool_call_examples=self.settings.include_tool_call_examples,
include_recent_events=self.include_recent_events,
sample_images=self.sample_images,
+ llm_screenshot_size=llm_screenshot_size,
)
if self.sensitive_data:
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 047443bd8..2ce00d81d 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -432,6 +432,15 @@ class BrowserSession(BaseModel):
description='BrowserProfile() options to use for the session, otherwise a default profile will be used',
)
+ # LLM screenshot resizing configuration
+ llm_screenshot_size: tuple[int, int] | None = Field(
+ default=None,
+ description='Target size (width, height) to resize screenshots before sending to LLM. Coordinates from LLM will be scaled back to original viewport size.',
+ )
+
+ # Cache of original viewport size for coordinate conversion (set when browser state is captured)
+ _original_viewport_size: tuple[int, int] | None = PrivateAttr(default=None)
+
# Convenience properties for common browser settings
@property
def cdp_url(self) -> str | None:
diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.py
index a82bd9266..bfce6bd19 100644
--- a/browser_use/browser/watchdogs/dom_watchdog.py
+++ b/browser_use/browser/watchdogs/dom_watchdog.py
@@ -511,6 +511,10 @@ class DOMWatchdog(BaseWatchdog):
# Cache the state
self.browser_session._cached_browser_state_summary = browser_state
+ # Cache viewport size for coordinate conversion (if llm_screenshot_size is enabled)
+ if self.browser_session.llm_screenshot_size and page_info:
+ self.browser_session._original_viewport_size = (page_info.viewport_width, page_info.viewport_height)
+
self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: βœ… COMPLETED - Returning browser state')
return browser_state
@@ -736,6 +740,10 @@ class DOMWatchdog(BaseWatchdog):
viewport_width = int(css_layout_viewport.get('clientWidth') or layout_viewport.get('clientWidth', 1280))
viewport_height = int(css_layout_viewport.get('clientHeight') or layout_viewport.get('clientHeight', 720))
+ self.logger.info(
+ f'πŸ“ Viewport dimensions from CDP: {viewport_width}x{viewport_height} CSS pixels (device_pixel_ratio: {device_pixel_ratio:.2f})'
+ )
+
# For total page dimensions, content size is typically in device pixels, so convert to CSS pixels
# by dividing by device pixel ratio
raw_page_width = content_size.get('width', viewport_width * device_pixel_ratio)
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 009910277..adfad5e2c 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -233,6 +233,24 @@ class Tools(Generic[Context]):
await asyncio.sleep(actual_seconds)
return ActionResult(extracted_content=memory, long_term_memory=memory)
+ # Helper function for coordinate conversion
+ def _convert_llm_coordinates_to_viewport(llm_x: int, llm_y: int, browser_session: BrowserSession) -> tuple[int, int]:
+ """Convert coordinates from LLM screenshot size to original viewport size."""
+ if browser_session.llm_screenshot_size and browser_session._original_viewport_size:
+ original_width, original_height = browser_session._original_viewport_size
+ llm_width, llm_height = browser_session.llm_screenshot_size
+
+ # Convert coordinates using fractions
+ actual_x = int((llm_x / llm_width) * original_width)
+ actual_y = int((llm_y / llm_height) * original_height)
+
+ logger.info(
+ f'πŸ”„ Converting coordinates: LLM ({llm_x}, {llm_y}) @ {llm_width}x{llm_height} '
+ f'β†’ Viewport ({actual_x}, {actual_y}) @ {original_width}x{original_height}'
+ )
+ return actual_x, actual_y
+ return llm_x, llm_y
+
# Element Interaction Actions
async def _click_by_coordinate(params: ClickElementAction, browser_session: BrowserSession) -> ActionResult:
# Ensure coordinates are provided (type safety)
@@ -240,8 +258,13 @@ class Tools(Generic[Context]):
return ActionResult(error='Both coordinate_x and coordinate_y must be provided')
try:
+ # Convert coordinates from LLM size to original viewport size if resizing was used
+ actual_x, actual_y = _convert_llm_coordinates_to_viewport(
+ params.coordinate_x, params.coordinate_y, browser_session
+ )
+
# Highlight the coordinate being clicked (truly non-blocking)
- asyncio.create_task(browser_session.highlight_coordinate_click(params.coordinate_x, params.coordinate_y))
+ asyncio.create_task(browser_session.highlight_coordinate_click(actual_x, actual_y))
# Use Actor (page.mouse.click) for coordinate-based clicking
page = await browser_session.get_current_page()
@@ -249,15 +272,15 @@ class Tools(Generic[Context]):
return ActionResult(error='No active page found')
mouse = await page.mouse
- await mouse.click(params.coordinate_x, params.coordinate_y)
+ await mouse.click(actual_x, actual_y)
- memory = f'Clicked on coordinate {params.coordinate_x}, {params.coordinate_y}'
+ memory = f'Clicked on coordinate {actual_x}, {actual_y}'
msg = f'πŸ–±οΈ {memory}'
logger.info(msg)
return ActionResult(
extracted_content=memory,
- metadata={'click_x': params.coordinate_x, 'click_y': params.coordinate_y},
+ metadata={'click_x': actual_x, 'click_y': actual_y},
)
except Exception as e:
error_msg = f'Failed to click at coordinates ({params.coordinate_x}, {params.coordinate_y}).'