Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
align agent success with simple judge
Saurav Pandac33e9494c5f7
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.pyindex 1f91bcd7f..2b7721717 100644--- a/browser_use/agent/judge.py+++ b/browser_use/agent/judge.py@@ -220,3 +220,67 @@ Evaluate this agent execution given the criteria and respond with the exact JSONSystemMessage(content=system_prompt),UserMessage(content=content_parts),]+++def construct_self_verify_messages(+ task: str,+ final_result: str,+ last_screenshot_path: str | None = None,+ use_vision: bool | Literal['auto'] = True,+) -> list[BaseMessage]:+ """+ Construct lightweight verification messages to check if the agent's claimed success is justified.++ Much lighter than the full judge — just task + final result + optional last screenshot.+ """+ task_truncated = _truncate_text(task, 20000)+ final_result_truncated = _truncate_text(final_result, 20000)++ system_prompt = """You are a strict verifier checking whether a browser automation agent actually completed its task.++Your job is simple: given the task and the agent's final response, determine if the response genuinely satisfies ALL requirements of the task.++Check for these common failure patterns:+1. **Incorrect data**: Wrong number of items, missing filters/criteria, wrong format+2. **Unverified actions**: Agent claims to have submitted a form, posted a comment, or saved a file but there's no evidence it happened+3. **Incomplete results**: Some requirements from the task are not addressed in the response+4. **Fabricated content**: Data that looks plausible but wasn't actually extracted from any page++Respond with EXACTLY this JSON structure:+{+ "verified": true or false,+ "reason": "Brief explanation if not verified, empty string if verified"+}++Be strict: if the response doesn't clearly satisfy every requirement, set verified to false."""++ user_prompt = f"""<task>+{task_truncated}+</task>++<agent_final_response>+{final_result_truncated or 'No response provided'}+</agent_final_response>++Does the agent's response fully satisfy all requirements of the task? Respond with the JSON structure."""++ # Build content parts+ content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=user_prompt)]++ # Add last screenshot if available and vision is enabled+ if use_vision is not False and last_screenshot_path:+ encoded = _encode_image(last_screenshot_path)+ if encoded:+ content_parts.append(+ ContentPartImageParam(+ image_url=ImageURL(+ url=f'data:image/png;base64,{encoded}',+ media_type='image/png',+ )+ )+ )++ return [+ SystemMessage(content=system_prompt),+ UserMessage(content=content_parts),+ ]diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex f9b8c9c22..1c16d34b2 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -36,7 +36,7 @@ from pydantic import BaseModel, ValidationErrorfrom uuid_extensions import uuid7strfrom browser_use import Browser, BrowserProfile, BrowserSession-from browser_use.agent.judge import construct_judge_messages+from browser_use.agent.judge import construct_judge_messages, construct_self_verify_messages# Lazy import for gif to avoid heavy agent.views import at startup# from browser_use.agent.gif import create_history_gif@@ -57,6 +57,7 @@ from browser_use.agent.views import (BrowserStateHistory,DetectedVariable,JudgementResult,+ SelfVerifyResult,StepMetadata,)from browser_use.browser.events import _get_timeout@@ -178,6 +179,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):page_extraction_llm: BaseChatModel | None = None,fallback_llm: BaseChatModel | None = None,use_judge: bool = True,+ use_self_verification: bool = True,ground_truth: str | None = None,judge_llm: BaseChatModel | None = None,injected_agent_state: AgentState | None = None,@@ -386,6 +388,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):step_timeout=step_timeout,final_response_after_failure=final_response_after_failure,use_judge=use_judge,+ use_self_verification=use_self_verification,ground_truth=ground_truth,)@@ -1286,6 +1289,40 @@ class Agent(Generic[Context, AgentStructuredOutput]):self._message_manager._add_context_message(UserMessage(content=msg))self.AgentOutput = self.DoneAgentOutput+ async def _self_verify_success(self) -> bool:+ """Lightweight self-verification when agent claims success.++ Uses the agent's own LLM to verify that the final response actually satisfies the task.+ Returns True if verification passes, False if success should be overridden to False.+ """+ last_result = self.history.history[-1].result[-1]+ if not last_result.success:+ return True # Agent already says failure, no need to verify++ task = self.task+ final_result = self.history.final_result() or ''+ screenshot_paths = [p for p in self.history.screenshot_paths() if p is not None]++ verify_messages = construct_self_verify_messages(+ task=task,+ final_result=final_result,+ last_screenshot_path=screenshot_paths[-1] if screenshot_paths else None,+ use_vision=self.settings.use_vision,+ )++ try:+ response = await self.llm.ainvoke(verify_messages, output_format=SelfVerifyResult)+ result: SelfVerifyResult = response.completion # type: ignore[assignment]+ if not result.verified:+ self.logger.info(f'⚠️ Self-verification failed: {result.reason}')+ last_result.success = False+ if last_result.extracted_content:+ last_result.extracted_content += f'\n\n[Self-verification: {result.reason}]'+ return result.verified+ except Exception as e:+ self.logger.warning(f'Self-verification failed with error: {e}')+ return True # Don't override on error+@observe(ignore_input=True, ignore_output=False)async def _judge_trace(self) -> JudgementResult | None:"""Judge the trace of the agent"""@@ -1962,6 +1999,10 @@ class Agent(Generic[Context, AgentStructuredOutput]):await self.step(step_info)if self.history.is_done():+ # Self-verify before logging completion+ if self.settings.use_self_verification:+ await self._self_verify_success()+await self.log_completion()# Run judge before done callback if enabled@@ -2163,6 +2204,10 @@ class Agent(Generic[Context, AgentStructuredOutput]):await on_step_end(self)if self.history.is_done():+ # Self-verify before logging completion+ if self.settings.use_self_verification:+ await self._self_verify_success()+await self.log_completion()# Run judge before done callback if enableddiff --git a/browser_use/agent/system_prompts/system_prompt.md b/browser_use/agent/system_prompts/system_prompt.mdindex f19b6ed32..98ef349ce 100644--- a/browser_use/agent/system_prompts/system_prompt.md+++ b/browser_use/agent/system_prompts/system_prompt.md@@ -118,6 +118,21 @@ The `done` action is your opportunity to terminate and share your findings withThis ensures that when you do call `done` (at max_steps or earlier), you have meaningful partial results to deliver.- For large multi-item tasks (e.g. "search 50 items"), estimate the per-item cost from the first few items.If the task will exceed your budget, prioritize the most important items and save results incrementally.+<pre_done_verification>+BEFORE calling `done` with `success=true`, you MUST perform this verification:+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).+2. **Check each requirement against your results:**+ - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)+ - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)+ - Does your output match the requested format exactly?+3. **Verify actions actually completed:**+ - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.+ - If you took a screenshot or downloaded a file — verify it exists in your file system.+4. **Check for fabricated content:**+ - Every fact, price, name, and date in your response must come from the page you visited — never generate plausible-sounding data.+5. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**+ Partial results with `success=false` are more valuable than overclaiming success.+</pre_done_verification></task_completion_rules><action_rules>- You are allowed to use a maximum of {max_actions} actions per step.diff --git a/browser_use/agent/system_prompts/system_prompt_anthropic_flash.md b/browser_use/agent/system_prompts/system_prompt_anthropic_flash.mdindex f4241715c..524b006be 100644--- a/browser_use/agent/system_prompts/system_prompt_anthropic_flash.md+++ b/browser_use/agent/system_prompts/system_prompt_anthropic_flash.md@@ -83,6 +83,21 @@ The `done` action is your opportunity to terminate and share your findings with- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!+<pre_done_verification>+BEFORE calling `done` with `success=true`, you MUST perform this verification:+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).+2. **Check each requirement against your results:**+ - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)+ - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)+ - Does your output match the requested format exactly?+3. **Verify actions actually completed:**+ - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.+ - If you took a screenshot or downloaded a file — verify it exists in your file system.+4. **Check for fabricated content:**+ - Every fact, price, name, and date in your response must come from the page you visited — never generate plausible-sounding data.+5. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**+ Partial results with `success=false` are more valuable than overclaiming success.+</pre_done_verification></task_completion_rules><input>At every step, your input will consist of:diff --git a/browser_use/agent/system_prompts/system_prompt_flash.md b/browser_use/agent/system_prompts/system_prompt_flash.mdindex 7cefa3003..4ef82501c 100644--- a/browser_use/agent/system_prompts/system_prompt_flash.md+++ b/browser_use/agent/system_prompts/system_prompt_flash.md@@ -10,4 +10,6 @@ You are allowed to use a maximum of {max_actions} actions per step. Check the br{{"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.","action":[{{"navigate": {{ "url": "url_value"}}}}]-}}</output>+}}+Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.+</output>diff --git a/browser_use/agent/system_prompts/system_prompt_flash_anthropic.md b/browser_use/agent/system_prompts/system_prompt_flash_anthropic.mdindex 158a32e89..fa0291b12 100644--- a/browser_use/agent/system_prompts/system_prompt_flash_anthropic.md+++ b/browser_use/agent/system_prompts/system_prompt_flash_anthropic.md@@ -26,4 +26,5 @@ You are allowed to use a maximum of {max_actions} actions per step. Check the br}}Always put `memory` field before the `action` field.+Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.</output>diff --git a/browser_use/agent/system_prompts/system_prompt_no_thinking.md b/browser_use/agent/system_prompts/system_prompt_no_thinking.mdindex 07391acb1..3c9b1dc7b 100644--- a/browser_use/agent/system_prompts/system_prompt_no_thinking.md+++ b/browser_use/agent/system_prompts/system_prompt_no_thinking.md@@ -110,6 +110,21 @@ The `done` action is your opportunity to terminate and share your findings with- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!+<pre_done_verification>+BEFORE calling `done` with `success=true`, you MUST perform this verification:+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).+2. **Check each requirement against your results:**+ - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)+ - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)+ - Does your output match the requested format exactly?+3. **Verify actions actually completed:**+ - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.+ - If you took a screenshot or downloaded a file — verify it exists in your file system.+4. **Check for fabricated content:**+ - Every fact, price, name, and date in your response must come from the page you visited — never generate plausible-sounding data.+5. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**+ Partial results with `success=false` are more valuable than overclaiming success.+</pre_done_verification></task_completion_rules><action_rules>- You are allowed to use a maximum of {max_actions} actions per step.diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex 0131a0f7e..4237bc6e3 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -46,6 +46,7 @@ class AgentSettings(BaseModel):flash_mode: bool = False # If enabled, disables evaluation_previous_goal and next_goal, and sets use_thinking = Falseuse_judge: bool = Trueground_truth: str | None = None # Ground truth answer or criteria for judge validation+ use_self_verification: bool = True # Run lightweight self-check before accepting done(success=True)max_history_items: int | None = Nonepage_extraction_llm: BaseChatModel | None = None@@ -107,6 +108,13 @@ class JudgementResult(BaseModel):)+class SelfVerifyResult(BaseModel):+ """Result of lightweight self-verification before done."""++ verified: bool = Field(description='True if the task appears completed correctly')+ reason: str = Field(default='', description='Brief explanation if not verified')++class ActionResult(BaseModel):"""Result of executing an action"""