Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
updated agent success with simple judge
Saurav Panda8565c7d26731
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.pyindex 1f91bcd7f..8b0cb7a3b 100644--- a/browser_use/agent/judge.py+++ b/browser_use/agent/judge.py@@ -220,3 +220,50 @@ Evaluate this agent execution given the criteria and respond with the exact JSONSystemMessage(content=system_prompt),UserMessage(content=content_parts),]+++def construct_simple_judge_messages(+ task: str,+ final_result: str,+) -> list[BaseMessage]:+ """Construct lightweight judge messages to validate agent success claims.++ Always runs regardless of use_judge setting. Text-only — no screenshots,+ no trajectory. Just task + final result.+ """+ task_truncated = _truncate_text(task, 20000)+ final_result_truncated = _truncate_text(final_result, 20000)++ system_prompt = """You are a strict verifier checking whether a browser automation agent actually completed its task.++Given the task and the agent's final response, determine if the response genuinely satisfies ALL requirements.++Check for these common failure patterns:+1. **Incorrect data**: Wrong number of items, missing filters/criteria, wrong format+2. **Unverified actions**: Agent claims to have submitted a form, posted a comment, or saved a file but there's no evidence+3. **Incomplete results**: Some requirements from the task are not addressed in the response+4. **Fabricated content**: Data that looks plausible but wasn't actually extracted from any page+5. **Partial completion reported as success**: Response acknowledges failure or blockers (captcha, access denied, etc.) but still claims success++Respond with EXACTLY this JSON structure:+{+ "is_correct": true or false,+ "reason": "Brief explanation if not correct, empty string if correct"+}++Be strict: if the response doesn't clearly satisfy every requirement, set is_correct to false."""++ user_prompt = f"""<task>+{task_truncated or 'No task provided'}+</task>++<agent_final_response>+{final_result_truncated or 'No response provided'}+</agent_final_response>++Does the agent's response fully satisfy all requirements of the task? Respond with the JSON structure."""++ return [+ SystemMessage(content=system_prompt),+ UserMessage(content=user_prompt),+ ]diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 1da5e252f..957acd883 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -36,7 +36,7 @@ from pydantic import BaseModel, ValidationErrorfrom uuid_extensions import uuid7strfrom browser_use import Browser, BrowserProfile, BrowserSession-from browser_use.agent.judge import construct_judge_messages+from browser_use.agent.judge import construct_judge_messages, construct_simple_judge_messages# Lazy import for gif to avoid heavy agent.views import at startup# from browser_use.agent.gif import create_history_gif@@ -57,6 +57,7 @@ from browser_use.agent.views import (BrowserStateHistory,DetectedVariable,JudgementResult,+ SimpleJudgeResult,StepMetadata,)from browser_use.browser.events import _get_timeout@@ -1286,6 +1287,40 @@ class Agent(Generic[Context, AgentStructuredOutput]):self._message_manager._add_context_message(UserMessage(content=msg))self.AgentOutput = self.DoneAgentOutput+ async def _run_simple_judge(self) -> None:+ """Lightweight always-on judge that overrides agent success when it overclaims.++ Runs regardless of use_judge setting. Only checks tasks where the agent+ claimed success — if the agent already reports failure, there's nothing to correct.+ """+ last_result = self.history.history[-1].result[-1]+ if not last_result.is_done or not last_result.success:+ return++ task = self.task+ final_result = self.history.final_result() or ''++ messages = construct_simple_judge_messages(+ task=task,+ final_result=final_result,+ )++ try:+ response = await self.llm.ainvoke(messages, output_format=SimpleJudgeResult)+ result: SimpleJudgeResult = response.completion # type: ignore[assignment]+ if not result.is_correct:+ reason = result.reason or 'Task requirements not fully met'+ self.logger.info(f'⚠️ Simple judge overriding success to failure: {reason}')+ last_result.success = False+ note = f'[Simple judge: {reason}]'+ if last_result.extracted_content:+ last_result.extracted_content += f'\n\n{note}'+ else:+ last_result.extracted_content = note+ except Exception as e:+ self.logger.warning(f'Simple judge failed with error: {e}')+ # Don't override on error — keep the agent's self-report+@observe(ignore_input=True, ignore_output=False)async def _judge_trace(self) -> JudgementResult | None:"""Judge the trace of the agent"""@@ -1322,7 +1357,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):return Noneasync def _judge_and_log(self) -> None:- """Run judge evaluation, override agent success with judge verdict, and log."""+ """Run judge evaluation and log the verdict.++ The judge verdict is attached to the action result but does NOT override+ last_result.success — that stays as the agent's self-report. Telemetry+ sends both values so the eval platform can compare agent vs judge.+ """judgement = await self._judge_trace()# Attach judgement to last action result@@ -1333,11 +1373,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):# Get self-reported successself_reported_success = last_result.success+ # Log the verdict based on self-reported success and judge verdictif judgement:- # Override agent's success with judge verdict- if judgement.verdict is not None:- last_result.success = judgement.verdict-# If both self-reported and judge agree on success, don't logif self_reported_success is True and judgement.verdict is True:return@@ -1345,7 +1382,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):judge_log = '\n'# If agent reported success but judge thinks it failed, show warningif self_reported_success is True and judgement.verdict is False:- judge_log += '⚠️ \033[33mAgent reported success but judge thinks task failed — overriding to failure\033[0m\n'+ judge_log += '⚠️ \033[33mAgent reported success but judge thinks task failed\033[0m\n'# Otherwise, show full judge resultverdict_color = '\033[32m' if judgement.verdict else '\033[31m'@@ -1965,9 +2002,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):await self.step(step_info)if self.history.is_done():+ # Always run simple judge to align agent success with reality+ await self._run_simple_judge()+await self.log_completion()- # Run judge before done callback if enabled+ # Run full judge before done callback if enabledif self.settings.use_judge:await self._judge_and_log()@@ -2166,9 +2206,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):await on_step_end(self)if self.history.is_done():+ # Always run simple judge to align agent success with reality+ await self._run_simple_judge()+await self.log_completion()- # Run judge before done callback if enabled+ # Run full judge before done callback if enabledif self.settings.use_judge:await self._judge_and_log()diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex dcc6ab8e1..3b41fd0dd 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -107,6 +107,13 @@ class JudgementResult(BaseModel):)+class SimpleJudgeResult(BaseModel):+ """Result of lightweight always-on judge that validates agent success claims."""++ is_correct: bool = Field(description='True if the agent response genuinely satisfies the task requirements')+ reason: str = Field(default='', description='Brief explanation if not correct')++class ActionResult(BaseModel):"""Result of executing an action"""