Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Add sample_images to enable browseruse to learn how to operate a platform.

zhcnf68c0865f709
diff --git a/browser_use/agent/message_manager/service.py b/browser_use/agent/message_manager/service.py
index e4455186a..db4693bd2 100644
--- a/browser_use/agent/message_manager/service.py
+++ b/browser_use/agent/message_manager/service.py
@@ -19,6 +19,7 @@ from browser_use.llm.messages import (
BaseMessage,
ContentPartTextParam,
SystemMessage,
+ UserMessage,
)
from browser_use.observability import observe_debug
from browser_use.utils import match_url_with_domain_pattern, time_execution_sync
@@ -108,6 +109,7 @@ class MessageManager:
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
include_tool_call_examples: bool = False,
include_recent_events: bool = False,
+ sample_images: list[UserMessage] = None,
):
self.task = task
self.state = state
@@ -119,6 +121,7 @@ class MessageManager:
self.vision_detail_level = vision_detail_level
self.include_tool_call_examples = include_tool_call_examples
self.include_recent_events = include_recent_events
+ self.sample_images = sample_images
assert max_history_items is None or max_history_items > 5, 'max_history_items must be None or greater than 5'
@@ -306,6 +309,7 @@ class MessageManager:
screenshots=screenshots,
vision_detail_level=self.vision_detail_level,
include_recent_events=self.include_recent_events,
+ sample_images=self.sample_images,
).get_user_message(use_vision)
# Set the state message with caching enabled
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index d1e859206..14b8c7a0f 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -93,6 +93,7 @@ class AgentMessagePrompt:
screenshots: list[str] | None = None,
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
include_recent_events: bool = False,
+ sample_images: list[dict] | None = None,
):
self.browser_state: 'BrowserStateSummary' = browser_state_summary
self.file_system: 'FileSystem | None' = file_system
@@ -108,6 +109,7 @@ class AgentMessagePrompt:
self.screenshots = screenshots or []
self.vision_detail_level = vision_detail_level
self.include_recent_events = include_recent_events
+ self.sample_images = sample_images or []
assert self.browser_state
@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')
@@ -258,6 +260,9 @@ Available tabs:
# Start with text description
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=state_description)]
+ # Add sample images
+ content_parts.extend(self.sample_images)
+
# Add screenshots with labels
for i, screenshot in enumerate(self.screenshots):
if i == len(self.screenshots) - 1:
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index a6fb4eadf..2f274ce40 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -179,6 +179,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
step_timeout: int = 120,
directly_open_url: bool = True,
include_recent_events: bool = False,
+ sample_images: list[UserMessage] = None,
**kwargs,
):
if page_extraction_llm is None:
@@ -224,6 +225,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self.sensitive_data = sensitive_data
+ self.sample_images = sample_images
+
self.settings = AgentSettings(
use_vision=use_vision,
vision_detail_level=vision_detail_level,
@@ -330,6 +333,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
vision_detail_level=self.settings.vision_detail_level,
include_tool_call_examples=self.settings.include_tool_call_examples,
include_recent_events=self.include_recent_events,
+ sample_images=self.sample_images,
)
if self.sensitive_data: