Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

implement simple judge directly into the library on done action

Alezander90f1946cb5473
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.py
new file mode 100644
index 000000000..1a6554e75
--- /dev/null
+++ b/browser_use/agent/judge.py
@@ -0,0 +1,170 @@
+"""Judge system for evaluating browser-use agent execution traces."""
+
+import base64
+import logging
+from pathlib import Path
+
+from browser_use.llm.messages import (
+ BaseMessage,
+ ContentPartImageParam,
+ ContentPartTextParam,
+ ImageURL,
+ SystemMessage,
+ UserMessage,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _encode_image(image_path: str) -> str | None:
+ """Encode image to base64 string."""
+ try:
+ path = Path(image_path)
+ if not path.exists():
+ return None
+ with open(path, 'rb') as f:
+ return base64.b64encode(f.read()).decode('utf-8')
+ except Exception as e:
+ logger.warning(f'Failed to encode image {image_path}: {e}')
+ return None
+
+
+def _truncate_text(text: str, max_length: int, from_beginning: bool = False) -> str:
+ """Truncate text to maximum length with eval system indicator."""
+ if len(text) <= max_length:
+ return text
+ if from_beginning:
+ return '...[text truncated]' + text[-max_length + 23 :]
+ else:
+ return text[: max_length - 23] + '...[text truncated]...'
+
+
+def construct_judge_messages(
+ task: str,
+ final_result: str,
+ agent_steps: list[str],
+ screenshot_paths: list[str],
+ max_images: int = 50,
+) -> list[BaseMessage]:
+ """
+ Construct messages for judge evaluation of agent trace.
+
+ Args:
+ task: The original task description
+ final_result: The final result returned to the user
+ agent_steps: List of formatted agent step descriptions
+ screenshot_paths: List of screenshot file paths
+ max_images: Maximum number of screenshots to include
+
+ Returns:
+ List of messages for LLM judge evaluation
+ """
+ task_truncated = _truncate_text(task, 40000)
+ final_result_truncated = _truncate_text(final_result, 40000)
+ steps_text = '\n'.join(agent_steps)
+ steps_text_truncated = _truncate_text(steps_text, 40000)
+
+ # Select last N screenshots
+ selected_screenshots = screenshot_paths[-max_images:] if len(screenshot_paths) > max_images else screenshot_paths
+
+ # Encode screenshots
+ encoded_images: list[ContentPartImageParam] = []
+ for img_path in selected_screenshots:
+ encoded = _encode_image(img_path)
+ if encoded:
+ encoded_images.append(
+ ContentPartImageParam(
+ image_url=ImageURL(
+ url=f'data:image/jpeg;base64,{encoded}',
+ media_type='image/jpeg',
+ )
+ )
+ )
+
+ # System prompt for judge
+ system_prompt = """You are an expert judge evaluating browser automation agent performance.
+
+<evaluation_framework>
+**PRIMARY EVALUATION CRITERIA (in order of importance):**
+1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome.
+2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested?
+3. **Tool Effectiveness**: Did the browser interactions work as expected? Were tools used appropriately? How many % of the tools failed?
+4. **Agent Reasoning**: Quality of decision-making, planning, and problem-solving throughout the trajectory.
+5. **Browser Handling**: Navigation stability, error recovery, and technical execution. If the browser crashes, does not load or a captcha blocks the task, the score must be very low.
+
+**VERDICT GUIDELINES:**
+- true: Task completed as requested, human-like execution, all of the users criteria were met and the agent did not make up any information.
+- false: Task not completed, or only partially completed.
+
+**Examples of task completion verdict:**
+- If task asks for 10 items and agent finds 4 items correctly: false
+- If task completed to full user requirements but with some errors to improve in the trajectory: true
+- If task impossible due to captcha/login requirements: false
+- If the trajectory is ideal and the output is perfect: true
+- If the task asks to search all headphones in amazon under $100 but the agent searches all headphones and the lowest price is $150: false
+- If the task asks to research a property and create a google doc with the result but the agents only returns the results in text: false
+- If the task asks to complete an action on the page, and the agent reports that the action is completed but the screenshot or page shows the action is not actually complete: false
+- If the task asks to use a certain tool or site to complete the task but the agent completes the task without using it: false
+- If the task asks to look for a section of a page that does not exist: false
+- If the agent concludes the task is impossible but it is not: false
+- If the agent concludes the task is impossible and it truly is impossible: false
+- If the agent is unable to complete the task because no login information was provided and it is truly needed to complete the task: false
+
+**FAILURE CONDITIONS (automatically set verdict to false):**
+- Blocked by captcha or missing authentication
+- Output format completely wrong or missing
+- Infinite loops or severe technical failures
+- Critical user requirements ignored
+- Page not loaded
+- Browser crashed
+- Agent could not interact with required UI elements
+- The agent moved on from a important step in the task without completing it
+- The agent made up content that is not in the screenshot or the page state
+- The agent calls done action before completing all key points of the task
+
+**IMPORTANT EVALUATION NOTES:**
+- **evaluate for action** - For each key step of the trace, double check whether the action that the agent tried to performed actually happened. If the required action did not actually occur, the verdict should be false.
+- **screenshot is not entire content** - The agent has the entire DOM content, but the screenshot is only part of the content. If the agent extracts information from the page, but you do not see it in the screenshot, you can assume this information is there.
+- **Penalize poor tool usage** - Wrong tools, inefficient approaches, ignoring available information.
+- **ignore unexpected dates and times** - These agent traces are from varying dates, you can assume the dates the agent uses for search or filtering are correct.
+- **IMPORTANT**: be very picky about the user's request - Have very high standard for the agent completing the task exactly to the user's request.
+- **IMPORTANT**: be initially doubtful of the agent's self reported success, be sure to verify that its methods are valid and fulfill the user's desires to a tee.
+
+</evaluation_framework>
+
+<response_format>
+Respond with EXACTLY this JSON structure (no additional text before or after):
+
+{{
+ "reasoning": "Breakdown of user task into key points. Detailed analysis covering: what went well, what didn't work, trajectory quality assessment, tool usage evaluation, output quality review, and overall user satisfaction prediction",
+ "verdict": true or false,
+ "failure_reason": "If verdict is false, provide the key reason why the task was not completed successfully. If verdict is true, use an empty string."
+}}
+</response_format>
+"""
+
+ user_prompt = f"""
+<task>
+{task_truncated or 'No task provided'}
+</task>
+
+<agent_trajectory>
+{steps_text_truncated or 'No agent trajectory provided'}
+</agent_trajectory>
+
+<final_result>
+{final_result_truncated or 'No final result provided'}
+</final_result>
+
+{len(encoded_images)} screenshots from execution are attached.
+
+Evaluate this agent execution given the criteria and respond with the exact JSON structure requested."""
+
+ # Build messages with screenshots
+ content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=user_prompt)]
+ content_parts.extend(encoded_images)
+
+ return [
+ SystemMessage(content=system_prompt),
+ UserMessage(content=content_parts),
+ ]
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 389238691..af16d164f 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -7,7 +7,6 @@ import re
import tempfile
import time
from collections.abc import Awaitable, Callable
-from datetime import datetime
from pathlib import Path
from typing import Any, Generic, Literal, TypeVar
from urllib.parse import urlparse
@@ -33,6 +32,7 @@ from pydantic import BaseModel, ValidationError
from uuid_extensions import uuid7str
from browser_use import Browser, BrowserProfile, BrowserSession
+from browser_use.agent.judge import construct_judge_messages
# Lazy import for gif to avoid heavy agent.views import at startup
# from browser_use.agent.gif import create_history_gif
@@ -51,6 +51,7 @@ from browser_use.agent.views import (
AgentStepInfo,
AgentStructuredOutput,
BrowserStateHistory,
+ JudgementResult,
StepMetadata,
)
from browser_use.browser.session import DEFAULT_BROWSER_PROFILE
@@ -68,7 +69,6 @@ from browser_use.utils import (
_log_pretty_path,
check_latest_browser_use_version,
get_browser_use_version,
- get_git_info,
time_execution_async,
time_execution_sync,
)
@@ -178,6 +178,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
include_recent_events: bool = False,
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
final_response_after_failure: bool = True,
+ use_judge: bool = True,
_url_shortening_limit: int = 25,
**kwargs,
):
@@ -282,6 +283,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
llm_timeout=llm_timeout,
step_timeout=step_timeout,
final_response_after_failure=final_response_after_failure,
+ use_judge=use_judge,
)
# Token cost service
@@ -905,6 +907,30 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self._message_manager._add_context_message(UserMessage(content=msg))
self.AgentOutput = self.DoneAgentOutput
+ async def _judge_trace(self) -> JudgementResult:
+ """Judge the trace of the agent"""
+ task = self.task
+ final_result = self.history.final_result() or ''
+ agent_steps = self.history.agent_steps()
+ screenshot_paths = [p for p in self.history.screenshot_paths() if p is not None]
+
+ # Construct input messages for judge evaluation
+ input_messages = construct_judge_messages(
+ task=task,
+ final_result=final_result,
+ agent_steps=agent_steps,
+ screenshot_paths=screenshot_paths,
+ max_images=50,
+ )
+
+ # Call LLM with JudgementResult as output format
+ kwargs: dict = {'output_format': JudgementResult}
+
+ response = await self.llm.ainvoke(input_messages, **kwargs)
+ judgement: JudgementResult = response.completion # type: ignore[assignment]
+
+ return judgement
+
async def _get_model_output_with_retry(self, input_messages: list[BaseMessage]) -> AgentOutput:
"""Get model output with retry logic for empty actions"""
model_output = await self.get_model_output(input_messages)
@@ -1689,7 +1715,13 @@ class Agent(Generic[Context, AgentStructuredOutput]):
is_done = await self._execute_step(step, max_steps, step_info, on_step_start, on_step_end)
if is_done:
- # Task completed
+ # Agent has marked the task as done
+ if self.settings.use_judge:
+ judgement = await self._judge_trace()
+ # Modify the last action result (that should have is_done=True) to include the judgement
+ if self.history.history[-1].result[-1].is_done:
+ self.history.history[-1].result[-1].judgement = judgement
+
break
else:
agent_run_error = 'Failed to complete task in maximum steps'
@@ -2205,82 +2237,6 @@ class Agent(Generic[Context, AgentStructuredOutput]):
else:
self.DoneAgentOutput = AgentOutput.type_with_custom_actions_no_thinking(self.DoneActionModel)
- def get_trace_object(self) -> dict[str, Any]:
- """Get the trace and trace_details objects for the agent"""
-
- def extract_task_website(task_text: str) -> str | None:
- url_pattern = r'https?://[^\s<>"\']+|www\.[^\s<>"\']+|[^\s<>"\']+\.[a-z]{2,}(?:/[^\s<>"\']*)?'
- match = re.search(url_pattern, task_text, re.IGNORECASE)
- return match.group(0) if match else None
-
- def _get_complete_history_without_screenshots(history_data: dict[str, Any]) -> str:
- if 'history' in history_data:
- for item in history_data['history']:
- if 'state' in item and 'screenshot' in item['state']:
- item['state']['screenshot'] = None
-
- return json.dumps(history_data)
-
- # Generate autogenerated fields
- trace_id = uuid7str()
- timestamp = datetime.now().isoformat()
-
- # Only declare variables that are used multiple times
- structured_output = self.history.structured_output
- structured_output_json = json.dumps(structured_output.model_dump()) if structured_output else None
- final_result = self.history.final_result()
- git_info = get_git_info()
- action_history = self.history.action_history()
- action_errors = self.history.errors()
- urls = self.history.urls()
- usage = self.history.usage
-
- return {
- 'trace': {
- # Autogenerated fields
- 'trace_id': trace_id,
- 'timestamp': timestamp,
- 'browser_use_version': get_browser_use_version(),
- 'git_info': json.dumps(git_info) if git_info else None,
- # Direct agent properties
- 'model': self.llm.model,
- 'settings': json.dumps(self.settings.model_dump()) if self.settings else None,
- 'task_id': self.task_id,
- 'task_truncated': self.task[:20000] if len(self.task) > 20000 else self.task,
- 'task_website': extract_task_website(self.task),
- # AgentHistoryList methods
- 'structured_output_truncated': (
- structured_output_json[:20000]
- if structured_output_json and len(structured_output_json) > 20000
- else structured_output_json
- ),
- 'action_history_truncated': json.dumps(action_history) if action_history else None,
- 'action_errors': json.dumps(action_errors) if action_errors else None,
- 'urls': json.dumps(urls) if urls else None,
- 'final_result_response_truncated': (
- final_result[:20000] if final_result and len(final_result) > 20000 else final_result
- ),
- 'self_report_completed': 1 if self.history.is_done() else 0,
- 'self_report_success': 1 if self.history.is_successful() else 0,
- 'duration': self.history.total_duration_seconds(),
- 'steps_taken': self.history.number_of_steps(),
- 'usage': json.dumps(usage.model_dump()) if usage else None,
- },
- 'trace_details': {
- # Autogenerated fields (ensure same as trace)
- 'trace_id': trace_id,
- 'timestamp': timestamp,
- # Direct agent properties
- 'task': self.task,
- # AgentHistoryList methods
- 'structured_output': structured_output_json,
- 'final_result_response': final_result,
- 'complete_history': _get_complete_history_without_screenshots(
- self.history.model_dump(sensitive_data=self.sensitive_data)
- ),
- },
- }
-
async def authenticate_cloud_sync(self, show_instructions: bool = True) -> bool:
"""
Authenticate with cloud service for future runs.
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.py
index 662a4d000..376c1a0c5 100644
--- a/browser_use/agent/views.py
+++ b/browser_use/agent/views.py
@@ -45,6 +45,7 @@ class AgentSettings(BaseModel):
max_actions_per_step: int = 4
use_thinking: bool = True
flash_mode: bool = False # If enabled, disables evaluation_previous_goal and next_goal, and sets use_thinking = False
+ use_judge: bool = True
max_history_items: int | None = None
page_extraction_llm: BaseChatModel | None = None
@@ -87,6 +88,14 @@ class AgentStepInfo:
return self.step_number >= self.max_steps - 1
+class JudgementResult(BaseModel):
+ """LLM judgement of agent trace"""
+
+ reasoning: str | None = Field(default=None, description='Explanation of the judgement')
+ verdict: bool = Field(description='Whether the trace was successful or not')
+ failure_reason: str | None = Field(default=None, description='If the trace was not successful, the reason why')
+
+
class ActionResult(BaseModel):
"""Result of executing an action"""
@@ -94,6 +103,9 @@ class ActionResult(BaseModel):
is_done: bool | None = False
success: bool | None = None
+ # For trace judgement
+ judgement: JudgementResult | None = None
+
# Error handling - always include in long term memory
error: str | None = None
@@ -619,6 +631,36 @@ class AgentHistoryList(BaseModel, Generic[AgentStructuredOutput]):
"""Get the number of steps in the history"""
return len(self.history)
+ def agent_steps(self) -> list[str]:
+ """Format agent history as readable step descriptions for judge evaluation."""
+ steps = []
+
+ # Iterate through history items (each is an AgentHistory)
+ for i, h in enumerate(self.history):
+ step_text = f'Step {i + 1}:\n'
+
+ # Get actions from model_output
+ if h.model_output and h.model_output.action:
+ # Use existing model_dump to get action dicts
+ actions_list = [action.model_dump(exclude_none=True) for action in h.model_output.action]
+ action_json = json.dumps(actions_list, indent=1)
+ step_text += f'Actions: {action_json}\n'
+
+ # Get results (already a list[ActionResult] in h.result)
+ if h.result:
+ for j, result in enumerate(h.result):
+ if result.extracted_content:
+ content = str(result.extracted_content)
+ step_text += f'Result {j + 1}: {content}\n'
+
+ if result.error:
+ error = str(result.error)
+ step_text += f'Error {j + 1}: {error}\n'
+
+ steps.append(step_text)
+
+ return steps
+
@property
def structured_output(self) -> AgentStructuredOutput | None:
"""Get the structured output from the history
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 78004db1a..261b08d48 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -1249,7 +1249,6 @@ Validated Code (after quote fixing):
self,
action: ActionModel,
browser_session: BrowserSession,
- #
page_extraction_llm: BaseChatModel | None = None,
sensitive_data: dict[str, str | dict[str, str]] | None = None,
available_file_paths: list[str] | None = None,