Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Include use_vision auto
Magnus Müllerd3abbcb2c741
diff --git a/browser_use/agent/message_manager/service.py b/browser_use/agent/message_manager/service.pyindex 2d922a125..521e10c64 100644--- a/browser_use/agent/message_manager/service.py+++ b/browser_use/agent/message_manager/service.py@@ -1,7 +1,7 @@from __future__ import annotationsimport logging-from typing import Literal+from typing import Literal, Unionfrom browser_use.agent.message_manager.views import (HistoryItem,@@ -285,7 +285,7 @@ class MessageManager:model_output: AgentOutput | None = None,result: list[ActionResult] | None = None,step_info: AgentStepInfo | None = None,- use_vision=True,+ use_vision: Union[bool, Literal['auto']] = 'auto',page_filtered_actions: str | None = None,sensitive_data=None,available_file_paths: list[str] | None = None, # Always pass current available_file_paths@@ -317,12 +317,24 @@ class MessageManager:logger.debug('Screenshot inclusion requested by action result')break- # Include screenshot if either use_vision is True, or if explicitly requested by an action- if (use_vision or include_screenshot_requested) and browser_state_summary.screenshot:+ # Handle different use_vision modes:+ # - "auto": Only include screenshot if explicitly requested by action (e.g., take_screenshot)+ # - True: Always include screenshot+ # - False: Never include screenshot+ include_screenshot = False+ if use_vision is True:+ # Always include screenshot when use_vision=True+ include_screenshot = True+ elif use_vision == 'auto':+ # Only include screenshot if explicitly requested by action when use_vision="auto"+ include_screenshot = include_screenshot_requested+ # else: use_vision is False, never include screenshot (include_screenshot stays False)++ if include_screenshot and browser_state_summary.screenshot:screenshots.append(browser_state_summary.screenshot)- # Override use_vision if screenshot was explicitly requested- effective_use_vision = use_vision or include_screenshot_requested+ # Use vision in the user message if screenshots are included+ effective_use_vision = len(screenshots) > 0# Create single state message with all contentassert browser_state_summarydiff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex c9d314a4f..21c65edbc 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -9,7 +9,7 @@ import timefrom collections.abc import Awaitable, Callablefrom datetime import datetimefrom pathlib import Path-from typing import Any, Generic, Literal, TypeVar+from typing import Any, Generic, Literal, TypeVar, Unionfrom urllib.parse import urlparsefrom dotenv import load_dotenv@@ -154,7 +154,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):register_should_stop_callback: Callable[[], Awaitable[bool]] | None = None,# Agent settingsoutput_model_schema: type[AgentStructuredOutput] | None = None,- use_vision: bool = True,+ use_vision: Union[bool, Literal['auto']] = 'auto',save_conversation_path: str | Path | None = None,save_conversation_path_encoding: str | None = 'utf-8',max_failures: int = 3,@@ -255,7 +255,9 @@ class Agent(Generic[Context, AgentStructuredOutput]):elif controller is not None:self.tools = controllerelse:- self.tools = Tools(display_files_in_done_text=display_files_in_done_text)+ # Exclude take_screenshot tool when use_vision=False+ exclude_actions = ['take_screenshot'] if use_vision is False else []+ self.tools = Tools(exclude_actions=exclude_actions, display_files_in_done_text=display_files_in_done_text)# Structured outputself.output_model_schema = output_model_schemadiff --git a/browser_use/agent/system_prompt.md b/browser_use/agent/system_prompt.mdindex 3944bcf8d..b48a68cb6 100644--- a/browser_use/agent/system_prompt.md+++ b/browser_use/agent/system_prompt.md@@ -20,7 +20,7 @@ At every step, your input will consist of:1. <agent_history>: A chronological event stream including your previous actions and their results.2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.-4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before+4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before, this will contain a screenshot.5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.</input>diff --git a/browser_use/agent/system_prompt_flash.md b/browser_use/agent/system_prompt_flash.mdindex 32795764f..9f14aad29 100644--- a/browser_use/agent/system_prompt_flash.md+++ b/browser_use/agent/system_prompt_flash.md@@ -20,7 +20,7 @@ At every step, your input will consist of:1. <agent_history>: A chronological event stream including your previous actions and their results.2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.-4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before+4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before, this will contain a screenshot.5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.</input>diff --git a/browser_use/agent/system_prompt_no_thinking.md b/browser_use/agent/system_prompt_no_thinking.mdindex fdfaf57c3..c905e5bb6 100644--- a/browser_use/agent/system_prompt_no_thinking.md+++ b/browser_use/agent/system_prompt_no_thinking.md@@ -20,7 +20,7 @@ At every step, your input will consist of:1. <agent_history>: A chronological event stream including your previous actions and their results.2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.-4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before+4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used take_screenshot before, this will contain a screenshot.5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.</input>diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex daa6ee78d..c433935b3 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -5,7 +5,7 @@ import loggingimport tracebackfrom dataclasses import dataclassfrom pathlib import Path-from typing import Any, Generic, Literal+from typing import Any, Generic, Literal, Unionfrom openai import RateLimitErrorfrom pydantic import BaseModel, ConfigDict, Field, ValidationError, create_model, model_validator@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)class AgentSettings(BaseModel):"""Configuration options for the Agent"""- use_vision: bool = True+ use_vision: Union[bool, Literal['auto']] = 'auto'vision_detail_level: Literal['auto', 'low', 'high'] = 'auto'save_conversation_path: str | Path | None = Nonesave_conversation_path_encoding: str | None = 'utf-8'diff --git a/browser_use/telemetry/views.py b/browser_use/telemetry/views.pyindex a5fd8ee83..48b3a8e6f 100644--- a/browser_use/telemetry/views.py+++ b/browser_use/telemetry/views.py@@ -1,7 +1,7 @@from abc import ABC, abstractmethodfrom collections.abc import Sequencefrom dataclasses import asdict, dataclass-from typing import Any+from typing import Any, Literal, Unionfrom browser_use.config import is_running_in_docker@@ -29,7 +29,7 @@ class AgentTelemetryEvent(BaseTelemetryEvent):model_provider: strmax_steps: intmax_actions_per_step: int- use_vision: bool+ use_vision: Union[bool, Literal['auto']]version: strsource: strcdp_url: str | Nonediff --git a/docs/customize/agent/all-parameters.mdx b/docs/customize/agent/all-parameters.mdxindex 05f6e217e..0f31dbb6e 100644--- a/docs/customize/agent/all-parameters.mdx+++ b/docs/customize/agent/all-parameters.mdx@@ -13,7 +13,7 @@ mode: "wide"- `output_model_schema`: Pydantic model class for structured output validation. [Example](https://github.com/browser-use/browser-use/blob/main/examples/features/custom_output.py)### Vision & Processing-- `use_vision` (default: `True`): Enable/disable vision capabilities for processing screenshots+- `use_vision` (default: `"auto"`): Vision mode - `"auto"` includes take_screenshot tool but only uses vision when requested, `True` always includes screenshots, `False` never includes screenshots and excludes take_screenshot tool- `vision_detail_level` (default: `'auto'`): Screenshot detail level - `'low'`, `'high'`, or `'auto'`- `page_extraction_llm`: Separate LLM model for page content extraction. You can choose a small & fast model because it only needs to extract text from the page (default: same as `llm`)diff --git a/docs/customize/tools/available.mdx b/docs/customize/tools/available.mdxindex 1bd6f4dfe..c045f3dca 100644--- a/docs/customize/tools/available.mdx+++ b/docs/customize/tools/available.mdx@@ -32,6 +32,9 @@ mode: "wide"### Content Extraction- **`extract_structured_data`** - Extract data from webpages using LLM+### Visual Analysis+- **`take_screenshot`** - Request a screenshot in your next browser state for visual confirmation+### Form Controls- **`get_dropdown_options`** - Get dropdown option values- **`select_dropdown_option`** - Select dropdown options