Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
add ground truth to judge
mertunsalldfbba0e062fc
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.pyindex 2f102241b..3f5ddb069 100644--- a/browser_use/agent/judge.py+++ b/browser_use/agent/judge.py@@ -45,6 +45,7 @@ def construct_judge_messages(agent_steps: list[str],screenshot_paths: list[str],max_images: int = 10,+ ground_truth: str | None = None,) -> list[BaseMessage]:"""Construct messages for judge evaluation of agent trace.@@ -55,6 +56,7 @@ def construct_judge_messages(agent_steps: List of formatted agent step descriptionsscreenshot_paths: List of screenshot file pathsmax_images: Maximum number of screenshots to include+ ground_truth: Optional ground truth answer or criteria that must be satisfied for successReturns:List of messages for LLM judge evaluation@@ -81,10 +83,23 @@ def construct_judge_messages())- # System prompt for judge- system_prompt = """You are an expert judge evaluating browser automation agent performance.+ # System prompt for judge - conditionally add ground truth section+ ground_truth_section = ''+ if ground_truth:+ ground_truth_section = """+**GROUND TRUTH VALIDATION (HIGHEST PRIORITY):**+The <ground_truth> section contains verified correct information for this task. This can be:+- **Evaluation criteria**: Specific conditions that must be met (e.g., "The success popup should show up", "Must extract exactly 5 items")+- **Factual answers**: The correct answer to a question or information retrieval task (e.g. "10/11/24", "Paris")+- **Expected outcomes**: What should happen after task completion (e.g., "Google Doc must be created", "File should be downloaded")++The ground truth takes ABSOLUTE precedence over all other evaluation criteria. If the ground truth is not satisfied by the agent's execution and final response, the verdict MUST be false.+"""++ system_prompt = f"""You are an expert judge evaluating browser automation agent performance.<evaluation_framework>+{ground_truth_section}**PRIMARY EVALUATION CRITERIA (in order of importance):**1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome.2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested?@@ -165,13 +180,22 @@ Respond with EXACTLY this JSON structure (no additional text before or after):"reached_captcha": true or false}}</response_format>+"""++ # Build user prompt with conditional ground truth section+ ground_truth_prompt = ''+ if ground_truth:+ ground_truth_prompt = f"""+<ground_truth>+{ground_truth}+</ground_truth>"""user_prompt = f"""<task>{task_truncated or 'No task provided'}</task>-+{ground_truth_prompt}<agent_trajectory>{steps_text_truncated or 'No agent trajectory provided'}</agent_trajectory>diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 15c9b8efa..a14fa4e48 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -165,6 +165,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):max_history_items: int | None = None,page_extraction_llm: BaseChatModel | None = None,use_judge: bool = True,+ ground_truth: str | None = None,judge_llm: BaseChatModel | None = None,injected_agent_state: AgentState | None = None,source: str | None = None,@@ -306,6 +307,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):step_timeout=step_timeout,final_response_after_failure=final_response_after_failure,use_judge=use_judge,+ ground_truth=ground_truth,)# Token cost service@@ -960,6 +962,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):agent_steps=agent_steps,screenshot_paths=screenshot_paths,max_images=10,+ ground_truth=self.settings.ground_truth,)# Call LLM with JudgementResult as output formatdiff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex b29be6361..303fcd4f1 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -46,6 +46,7 @@ class AgentSettings(BaseModel):use_thinking: bool = Trueflash_mode: bool = False # If enabled, disables evaluation_previous_goal and next_goal, and sets use_thinking = Falseuse_judge: bool = True+ ground_truth: str | None = None # Ground truth answer or criteria for judge validationmax_history_items: int | None = Nonepage_extraction_llm: BaseChatModel | None = Nonediff --git a/examples/features/judge_trace.py b/examples/features/judge_trace.pyindex d89673668..48a5d9d2e 100644--- a/examples/features/judge_trace.py+++ b/examples/features/judge_trace.py@@ -21,7 +21,13 @@ from browser_use import Agent, ChatBrowserUseasync def main():llm = ChatBrowserUse()task = "Search Google for 'what is browser automation' and tell me the top 3 results"- agent = Agent(task=task, llm=llm, use_judge=True, judge_llm=llm)+ agent = Agent(+ task=task,+ llm=llm,+ use_judge=True,+ judge_llm=llm,+ # Optional: Add ground_truth="specific answer or criteria" to validate against ground truth+ )history = await agent.run()# Get the judgement result