Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

updated agent success with simple judge

Saurav Panda8565c7d26731
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.py
index 1f91bcd7f..8b0cb7a3b 100644
--- a/browser_use/agent/judge.py
+++ b/browser_use/agent/judge.py
@@ -220,3 +220,50 @@ Evaluate this agent execution given the criteria and respond with the exact JSON
SystemMessage(content=system_prompt),
UserMessage(content=content_parts),
]
+
+
+def construct_simple_judge_messages(
+ task: str,
+ final_result: str,
+) -> list[BaseMessage]:
+ """Construct lightweight judge messages to validate agent success claims.
+
+ Always runs regardless of use_judge setting. Text-only — no screenshots,
+ no trajectory. Just task + final result.
+ """
+ task_truncated = _truncate_text(task, 20000)
+ final_result_truncated = _truncate_text(final_result, 20000)
+
+ system_prompt = """You are a strict verifier checking whether a browser automation agent actually completed its task.
+
+Given the task and the agent's final response, determine if the response genuinely satisfies ALL requirements.
+
+Check for these common failure patterns:
+1. **Incorrect data**: Wrong number of items, missing filters/criteria, wrong format
+2. **Unverified actions**: Agent claims to have submitted a form, posted a comment, or saved a file but there's no evidence
+3. **Incomplete results**: Some requirements from the task are not addressed in the response
+4. **Fabricated content**: Data that looks plausible but wasn't actually extracted from any page
+5. **Partial completion reported as success**: Response acknowledges failure or blockers (captcha, access denied, etc.) but still claims success
+
+Respond with EXACTLY this JSON structure:
+{
+ "is_correct": true or false,
+ "reason": "Brief explanation if not correct, empty string if correct"
+}
+
+Be strict: if the response doesn't clearly satisfy every requirement, set is_correct to false."""
+
+ user_prompt = f"""<task>
+{task_truncated or 'No task provided'}
+</task>
+
+<agent_final_response>
+{final_result_truncated or 'No response provided'}
+</agent_final_response>
+
+Does the agent's response fully satisfy all requirements of the task? Respond with the JSON structure."""
+
+ return [
+ SystemMessage(content=system_prompt),
+ UserMessage(content=user_prompt),
+ ]
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 1da5e252f..957acd883 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -36,7 +36,7 @@ from pydantic import BaseModel, ValidationError
from uuid_extensions import uuid7str
from browser_use import Browser, BrowserProfile, BrowserSession
-from browser_use.agent.judge import construct_judge_messages
+from browser_use.agent.judge import construct_judge_messages, construct_simple_judge_messages
# Lazy import for gif to avoid heavy agent.views import at startup
# from browser_use.agent.gif import create_history_gif
@@ -57,6 +57,7 @@ from browser_use.agent.views import (
BrowserStateHistory,
DetectedVariable,
JudgementResult,
+ SimpleJudgeResult,
StepMetadata,
)
from browser_use.browser.events import _get_timeout
@@ -1286,6 +1287,40 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self._message_manager._add_context_message(UserMessage(content=msg))
self.AgentOutput = self.DoneAgentOutput
+ async def _run_simple_judge(self) -> None:
+ """Lightweight always-on judge that overrides agent success when it overclaims.
+
+ Runs regardless of use_judge setting. Only checks tasks where the agent
+ claimed success — if the agent already reports failure, there's nothing to correct.
+ """
+ last_result = self.history.history[-1].result[-1]
+ if not last_result.is_done or not last_result.success:
+ return
+
+ task = self.task
+ final_result = self.history.final_result() or ''
+
+ messages = construct_simple_judge_messages(
+ task=task,
+ final_result=final_result,
+ )
+
+ try:
+ response = await self.llm.ainvoke(messages, output_format=SimpleJudgeResult)
+ result: SimpleJudgeResult = response.completion # type: ignore[assignment]
+ if not result.is_correct:
+ reason = result.reason or 'Task requirements not fully met'
+ self.logger.info(f'⚠️ Simple judge overriding success to failure: {reason}')
+ last_result.success = False
+ note = f'[Simple judge: {reason}]'
+ if last_result.extracted_content:
+ last_result.extracted_content += f'\n\n{note}'
+ else:
+ last_result.extracted_content = note
+ except Exception as e:
+ self.logger.warning(f'Simple judge failed with error: {e}')
+ # Don't override on error — keep the agent's self-report
+
@observe(ignore_input=True, ignore_output=False)
async def _judge_trace(self) -> JudgementResult | None:
"""Judge the trace of the agent"""
@@ -1322,7 +1357,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):
return None
async def _judge_and_log(self) -> None:
- """Run judge evaluation, override agent success with judge verdict, and log."""
+ """Run judge evaluation and log the verdict.
+
+ The judge verdict is attached to the action result but does NOT override
+ last_result.success — that stays as the agent's self-report. Telemetry
+ sends both values so the eval platform can compare agent vs judge.
+ """
judgement = await self._judge_trace()
# Attach judgement to last action result
@@ -1333,11 +1373,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):
# Get self-reported success
self_reported_success = last_result.success
+ # Log the verdict based on self-reported success and judge verdict
if judgement:
- # Override agent's success with judge verdict
- if judgement.verdict is not None:
- last_result.success = judgement.verdict
-
# If both self-reported and judge agree on success, don't log
if self_reported_success is True and judgement.verdict is True:
return
@@ -1345,7 +1382,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
judge_log = '\n'
# If agent reported success but judge thinks it failed, show warning
if self_reported_success is True and judgement.verdict is False:
- judge_log += '⚠️ \033[33mAgent reported success but judge thinks task failed — overriding to failure\033[0m\n'
+ judge_log += '⚠️ \033[33mAgent reported success but judge thinks task failed\033[0m\n'
# Otherwise, show full judge result
verdict_color = '\033[32m' if judgement.verdict else '\033[31m'
@@ -1965,9 +2002,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):
await self.step(step_info)
if self.history.is_done():
+ # Always run simple judge to align agent success with reality
+ await self._run_simple_judge()
+
await self.log_completion()
- # Run judge before done callback if enabled
+ # Run full judge before done callback if enabled
if self.settings.use_judge:
await self._judge_and_log()
@@ -2166,9 +2206,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):
await on_step_end(self)
if self.history.is_done():
+ # Always run simple judge to align agent success with reality
+ await self._run_simple_judge()
+
await self.log_completion()
- # Run judge before done callback if enabled
+ # Run full judge before done callback if enabled
if self.settings.use_judge:
await self._judge_and_log()
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.py
index dcc6ab8e1..3b41fd0dd 100644
--- a/browser_use/agent/views.py
+++ b/browser_use/agent/views.py
@@ -107,6 +107,13 @@ class JudgementResult(BaseModel):
)
+class SimpleJudgeResult(BaseModel):
+ """Result of lightweight always-on judge that validates agent success claims."""
+
+ is_correct: bool = Field(description='True if the agent response genuinely satisfies the task requirements')
+ reason: str = Field(default='', description='Brief explanation if not correct')
+
+
class ActionResult(BaseModel):
"""Result of executing an action"""