Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Include use_vision auto

Magnus Müllerd3abbcb2c741
diff --git a/browser_use/agent/message_manager/service.py b/browser_use/agent/message_manager/service.py
index 2d922a125..521e10c64 100644
--- a/browser_use/agent/message_manager/service.py
+++ b/browser_use/agent/message_manager/service.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import logging
-from typing import Literal
+from typing import Literal, Union
from browser_use.agent.message_manager.views import (
HistoryItem,
@@ -285,7 +285,7 @@ class MessageManager:
model_output: AgentOutput | None = None,
result: list[ActionResult] | None = None,
step_info: AgentStepInfo | None = None,
- use_vision=True,
+ use_vision: Union[bool, Literal['auto']] = 'auto',
page_filtered_actions: str | None = None,
sensitive_data=None,
available_file_paths: list[str] | None = None, # Always pass current available_file_paths
@@ -317,12 +317,24 @@ class MessageManager:
logger.debug('Screenshot inclusion requested by action result')
break
- # Include screenshot if either use_vision is True, or if explicitly requested by an action
- if (use_vision or include_screenshot_requested) and browser_state_summary.screenshot:
+ # Handle different use_vision modes:
+ # - "auto": Only include screenshot if explicitly requested by action (e.g., take_screenshot)
+ # - True: Always include screenshot
+ # - False: Never include screenshot
+ include_screenshot = False
+ if use_vision is True:
+ # Always include screenshot when use_vision=True
+ include_screenshot = True
+ elif use_vision == 'auto':
+ # Only include screenshot if explicitly requested by action when use_vision="auto"
+ include_screenshot = include_screenshot_requested
+ # else: use_vision is False, never include screenshot (include_screenshot stays False)
+
+ if include_screenshot and browser_state_summary.screenshot:
screenshots.append(browser_state_summary.screenshot)
- # Override use_vision if screenshot was explicitly requested
- effective_use_vision = use_vision or include_screenshot_requested
+ # Use vision in the user message if screenshots are included
+ effective_use_vision = len(screenshots) > 0
# Create single state message with all content
assert browser_state_summary
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index c9d314a4f..21c65edbc 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -9,7 +9,7 @@ import time
from collections.abc import Awaitable, Callable
from datetime import datetime
from pathlib import Path
-from typing import Any, Generic, Literal, TypeVar
+from typing import Any, Generic, Literal, TypeVar, Union
from urllib.parse import urlparse
from dotenv import load_dotenv
@@ -154,7 +154,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
register_should_stop_callback: Callable[[], Awaitable[bool]] | None = None,
# Agent settings
output_model_schema: type[AgentStructuredOutput] | None = None,
- use_vision: bool = True,
+ use_vision: Union[bool, Literal['auto']] = 'auto',
save_conversation_path: str | Path | None = None,
save_conversation_path_encoding: str | None = 'utf-8',
max_failures: int = 3,
@@ -255,7 +255,9 @@ class Agent(Generic[Context, AgentStructuredOutput]):
elif controller is not None:
self.tools = controller
else:
- self.tools = Tools(display_files_in_done_text=display_files_in_done_text)
+ # Exclude take_screenshot tool when use_vision=False
+ exclude_actions = ['take_screenshot'] if use_vision is False else []
+ self.tools = Tools(exclude_actions=exclude_actions, display_files_in_done_text=display_files_in_done_text)
# Structured output
self.output_model_schema = output_model_schema
diff --git a/browser_use/agent/system_prompt.md b/browser_use/agent/system_prompt.md
index 3944bcf8d..b48a68cb6 100644
--- a/browser_use/agent/system_prompt.md
+++ b/browser_use/agent/system_prompt.md
@@ -20,7 +20,7 @@ At every step, your input will consist of:
1. <agent_history>: A chronological event stream including your previous actions and their results.
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
-4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before
+4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before, this will contain a screenshot.
5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.
</input>
diff --git a/browser_use/agent/system_prompt_flash.md b/browser_use/agent/system_prompt_flash.md
index 32795764f..9f14aad29 100644
--- a/browser_use/agent/system_prompt_flash.md
+++ b/browser_use/agent/system_prompt_flash.md
@@ -20,7 +20,7 @@ At every step, your input will consist of:
1. <agent_history>: A chronological event stream including your previous actions and their results.
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
-4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before
+4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before, this will contain a screenshot.
5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.
</input>
diff --git a/browser_use/agent/system_prompt_no_thinking.md b/browser_use/agent/system_prompt_no_thinking.md
index fdfaf57c3..c905e5bb6 100644
--- a/browser_use/agent/system_prompt_no_thinking.md
+++ b/browser_use/agent/system_prompt_no_thinking.md
@@ -20,7 +20,7 @@ At every step, your input will consist of:
1. <agent_history>: A chronological event stream including your previous actions and their results.
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
-4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before
+4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before, this will contain a screenshot.
5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.
</input>
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.py
index daa6ee78d..c433935b3 100644
--- a/browser_use/agent/views.py
+++ b/browser_use/agent/views.py
@@ -5,7 +5,7 @@ import logging
import traceback
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Generic, Literal
+from typing import Any, Generic, Literal, Union
from openai import RateLimitError
from pydantic import BaseModel, ConfigDict, Field, ValidationError, create_model, model_validator
@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)
class AgentSettings(BaseModel):
"""Configuration options for the Agent"""
- use_vision: bool = True
+ use_vision: Union[bool, Literal['auto']] = 'auto'
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto'
save_conversation_path: str | Path | None = None
save_conversation_path_encoding: str | None = 'utf-8'
diff --git a/browser_use/telemetry/views.py b/browser_use/telemetry/views.py
index a5fd8ee83..48b3a8e6f 100644
--- a/browser_use/telemetry/views.py
+++ b/browser_use/telemetry/views.py
@@ -1,7 +1,7 @@
from abc import ABC, abstractmethod
from collections.abc import Sequence
from dataclasses import asdict, dataclass
-from typing import Any
+from typing import Any, Literal, Union
from browser_use.config import is_running_in_docker
@@ -29,7 +29,7 @@ class AgentTelemetryEvent(BaseTelemetryEvent):
model_provider: str
max_steps: int
max_actions_per_step: int
- use_vision: bool
+ use_vision: Union[bool, Literal['auto']]
version: str
source: str
cdp_url: str | None
diff --git a/docs/customize/agent/all-parameters.mdx b/docs/customize/agent/all-parameters.mdx
index 05f6e217e..0f31dbb6e 100644
--- a/docs/customize/agent/all-parameters.mdx
+++ b/docs/customize/agent/all-parameters.mdx
@@ -13,7 +13,7 @@ mode: "wide"
- `output_model_schema`: Pydantic model class for structured output validation. [Example](https://github.com/browser-use/browser-use/blob/main/examples/features/custom_output.py)
### Vision & Processing
-- `use_vision` (default: `True`): Enable/disable vision capabilities for processing screenshots
+- `use_vision` (default: `"auto"`): Vision mode - `"auto"` includes take_screenshot tool but only uses vision when requested, `True` always includes screenshots, `False` never includes screenshots and excludes take_screenshot tool
- `vision_detail_level` (default: `'auto'`): Screenshot detail level - `'low'`, `'high'`, or `'auto'`
- `page_extraction_llm`: Separate LLM model for page content extraction. You can choose a small & fast model because it only needs to extract text from the page (default: same as `llm`)
diff --git a/docs/customize/tools/available.mdx b/docs/customize/tools/available.mdx
index 1bd6f4dfe..c045f3dca 100644
--- a/docs/customize/tools/available.mdx
+++ b/docs/customize/tools/available.mdx
@@ -32,6 +32,9 @@ mode: "wide"
### Content Extraction
- **`extract_structured_data`** - Extract data from webpages using LLM
+### Visual Analysis
+- **`take_screenshot`** - Request a screenshot in your next browser state for visual confirmation
+
### Form Controls
- **`get_dropdown_options`** - Get dropdown option values
- **`select_dropdown_option`** - Select dropdown options