Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix: respect use_vision=False in judge to prevent screenshot leakage (#3726)

Mr-Neutr0nd73bca47e8ec
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.py
index 3f5ddb069..db2416cf8 100644
--- a/browser_use/agent/judge.py
+++ b/browser_use/agent/judge.py
@@ -4,6 +4,8 @@ import base64
import logging
from pathlib import Path
+from typing import Literal
+
from browser_use.llm.messages import (
BaseMessage,
ContentPartImageParam,
@@ -46,6 +48,7 @@ def construct_judge_messages(
screenshot_paths: list[str],
max_images: int = 10,
ground_truth: str | None = None,
+ use_vision: bool | Literal['auto'] = True,
) -> list[BaseMessage]:
"""
Construct messages for judge evaluation of agent trace.
@@ -66,22 +69,24 @@ def construct_judge_messages(
steps_text = '\n'.join(agent_steps)
steps_text_truncated = _truncate_text(steps_text, 40000)
- # Select last N screenshots
- selected_screenshots = screenshot_paths[-max_images:] if len(screenshot_paths) > max_images else screenshot_paths
-
- # Encode screenshots
+ # Only include screenshots if use_vision is not False
encoded_images: list[ContentPartImageParam] = []
- for img_path in selected_screenshots:
- encoded = _encode_image(img_path)
- if encoded:
- encoded_images.append(
- ContentPartImageParam(
- image_url=ImageURL(
- url=f'data:image/png;base64,{encoded}',
- media_type='image/png',
+ if use_vision is not False:
+ # Select last N screenshots
+ selected_screenshots = screenshot_paths[-max_images:] if len(screenshot_paths) > max_images else screenshot_paths
+
+ # Encode screenshots
+ for img_path in selected_screenshots:
+ encoded = _encode_image(img_path)
+ if encoded:
+ encoded_images.append(
+ ContentPartImageParam(
+ image_url=ImageURL(
+ url=f'data:image/png;base64,{encoded}',
+ media_type='image/png',
+ )
)
)
- )
# System prompt for judge - conditionally add ground truth section
ground_truth_section = ''
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 448c410bc..586196b6f 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -1267,6 +1267,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
screenshot_paths=screenshot_paths,
max_images=10,
ground_truth=self.settings.ground_truth,
+ use_vision=self.settings.use_vision,
)
# Call LLM with JudgementResult as output format
diff --git a/tests/ci/test_judge_use_vision.py b/tests/ci/test_judge_use_vision.py
new file mode 100644
index 000000000..83f19d4c7
--- /dev/null
+++ b/tests/ci/test_judge_use_vision.py
@@ -0,0 +1,142 @@
+"""Tests for judge respecting use_vision setting."""
+
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from browser_use.agent.judge import construct_judge_messages
+from browser_use.llm.messages import ContentPartImageParam
+
+
+class TestJudgeUseVision:
+ """Test that construct_judge_messages respects use_vision parameter."""
+
+ @pytest.fixture
+ def sample_screenshot_paths(self):
+ """Create temporary screenshot files for testing."""
+ temp_files = []
+ for i in range(3):
+ with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as f:
+ # Write a minimal valid PNG header
+ f.write(b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR')
+ f.write(b'\x00' * 20) # Minimal image data
+ temp_files.append(f.name)
+ yield temp_files
+ # Cleanup
+ for f in temp_files:
+ Path(f).unlink(missing_ok=True)
+
+ def test_use_vision_true_includes_screenshots(self, sample_screenshot_paths):
+ """When use_vision=True, screenshots should be included."""
+ messages = construct_judge_messages(
+ task="Test task",
+ final_result="Test result",
+ agent_steps=["Step 1", "Step 2"],
+ screenshot_paths=sample_screenshot_paths,
+ use_vision=True,
+ )
+
+ # Check that the user message contains image parts
+ user_message = messages[1]
+ image_parts = [
+ part for part in user_message.content
+ if isinstance(part, ContentPartImageParam)
+ ]
+ assert len(image_parts) > 0, "Screenshots should be included when use_vision=True"
+
+ def test_use_vision_false_excludes_screenshots(self, sample_screenshot_paths):
+ """When use_vision=False, screenshots should NOT be included."""
+ messages = construct_judge_messages(
+ task="Test task",
+ final_result="Test result",
+ agent_steps=["Step 1", "Step 2"],
+ screenshot_paths=sample_screenshot_paths,
+ use_vision=False,
+ )
+
+ # Check that the user message contains NO image parts
+ user_message = messages[1]
+ image_parts = [
+ part for part in user_message.content
+ if isinstance(part, ContentPartImageParam)
+ ]
+ assert len(image_parts) == 0, "Screenshots should NOT be included when use_vision=False"
+
+ def test_use_vision_auto_includes_screenshots(self, sample_screenshot_paths):
+ """When use_vision='auto', screenshots should be included (same as True for judge)."""
+ messages = construct_judge_messages(
+ task="Test task",
+ final_result="Test result",
+ agent_steps=["Step 1", "Step 2"],
+ screenshot_paths=sample_screenshot_paths,
+ use_vision='auto',
+ )
+
+ # Check that the user message contains image parts
+ user_message = messages[1]
+ image_parts = [
+ part for part in user_message.content
+ if isinstance(part, ContentPartImageParam)
+ ]
+ assert len(image_parts) > 0, "Screenshots should be included when use_vision='auto'"
+
+ def test_use_vision_default_includes_screenshots(self, sample_screenshot_paths):
+ """Default behavior (use_vision not specified) should include screenshots."""
+ messages = construct_judge_messages(
+ task="Test task",
+ final_result="Test result",
+ agent_steps=["Step 1", "Step 2"],
+ screenshot_paths=sample_screenshot_paths,
+ # use_vision not specified, defaults to True
+ )
+
+ # Check that the user message contains image parts
+ user_message = messages[1]
+ image_parts = [
+ part for part in user_message.content
+ if isinstance(part, ContentPartImageParam)
+ ]
+ assert len(image_parts) > 0, "Screenshots should be included by default"
+
+ def test_use_vision_false_message_shows_zero_screenshots(self, sample_screenshot_paths):
+ """When use_vision=False, message should indicate 0 screenshots attached."""
+ messages = construct_judge_messages(
+ task="Test task",
+ final_result="Test result",
+ agent_steps=["Step 1", "Step 2"],
+ screenshot_paths=sample_screenshot_paths,
+ use_vision=False,
+ )
+
+ # Check that the user message text indicates 0 screenshots
+ user_message = messages[1]
+ text_parts = [
+ part for part in user_message.content
+ if hasattr(part, 'text')
+ ]
+ assert len(text_parts) > 0
+ text_content = text_parts[0].text
+ assert "0 screenshots from execution are attached" in text_content
+
+ def test_empty_screenshot_paths_no_error(self):
+ """Empty screenshot_paths should not cause errors regardless of use_vision."""
+ # Should not raise any errors
+ messages_true = construct_judge_messages(
+ task="Test task",
+ final_result="Test result",
+ agent_steps=["Step 1"],
+ screenshot_paths=[],
+ use_vision=True,
+ )
+ messages_false = construct_judge_messages(
+ task="Test task",
+ final_result="Test result",
+ agent_steps=["Step 1"],
+ screenshot_paths=[],
+ use_vision=False,
+ )
+
+ assert len(messages_true) == 2
+ assert len(messages_false) == 2
+