Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fix bugs in library judge and make it off by default
Alezander915911c8c72bf
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.pyindex 1a6554e75..c5c7631e0 100644--- a/browser_use/agent/judge.py+++ b/browser_use/agent/judge.py@@ -44,7 +44,7 @@ def construct_judge_messages(final_result: str,agent_steps: list[str],screenshot_paths: list[str],- max_images: int = 50,+ max_images: int = 10,) -> list[BaseMessage]:"""Construct messages for judge evaluation of agent trace.diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex af16d164f..e30fd06e8 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -164,6 +164,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):flash_mode: bool = False,max_history_items: int | None = None,page_extraction_llm: BaseChatModel | None = None,+ use_judge: bool = False,+ judge_llm: BaseChatModel | None = None,injected_agent_state: AgentState | None = None,source: str | None = None,file_system_path: str | None = None,@@ -178,7 +180,6 @@ class Agent(Generic[Context, AgentStructuredOutput]):include_recent_events: bool = False,sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,final_response_after_failure: bool = True,- use_judge: bool = True,_url_shortening_limit: int = 25,**kwargs,):@@ -200,6 +201,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):if page_extraction_llm is None:page_extraction_llm = llm+ if judge_llm is None:+ judge_llm = llmif available_file_paths is None:available_file_paths = []@@ -242,6 +245,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):# Core componentsself.task = self._enhance_task_with_schema(task, output_model_schema)self.llm = llm+ self.judge_llm = judge_llmself.directly_open_url = directly_open_urlself.include_recent_events = include_recent_eventsself._url_shortening_limit = _url_shortening_limit@@ -290,6 +294,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):self.token_cost_service = TokenCost(include_cost=calculate_cost)self.token_cost_service.register_llm(llm)self.token_cost_service.register_llm(page_extraction_llm)+ self.token_cost_service.register_llm(judge_llm)# Initialize stateself.state = injected_agent_state or AgentState()@@ -907,7 +912,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):self._message_manager._add_context_message(UserMessage(content=msg))self.AgentOutput = self.DoneAgentOutput- async def _judge_trace(self) -> JudgementResult:+ async def _judge_trace(self) -> JudgementResult | None:"""Judge the trace of the agent"""task = self.taskfinal_result = self.history.final_result() or ''@@ -920,16 +925,20 @@ class Agent(Generic[Context, AgentStructuredOutput]):final_result=final_result,agent_steps=agent_steps,screenshot_paths=screenshot_paths,- max_images=50,+ max_images=10,)# Call LLM with JudgementResult as output formatkwargs: dict = {'output_format': JudgementResult}- response = await self.llm.ainvoke(input_messages, **kwargs)- judgement: JudgementResult = response.completion # type: ignore[assignment]-- return judgement+ try:+ response = await self.judge_llm.ainvoke(input_messages, **kwargs)+ judgement: JudgementResult = response.completion # type: ignore[assignment]+ return judgement+ except Exception as e:+ self.logger.error(f'Judge trace failed: {e}')+ # Return a default judgement on failure+ return Noneasync def _get_model_output_with_retry(self, input_messages: list[BaseMessage]) -> AgentOutput:"""Get model output with retry logic for empty actions"""diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex 376c1a0c5..bc3ca59fe 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -511,6 +511,29 @@ class AgentHistoryList(BaseModel, Generic[AgentStructuredOutput]):"""Check if the agent has any non-None errors"""return any(error is not None for error in self.errors())+ def judgement(self) -> dict | None:+ """Get the judgement result as a dictionary if it exists"""+ if self.history and len(self.history[-1].result) > 0:+ last_result = self.history[-1].result[-1]+ if last_result.judgement:+ return last_result.judgement.model_dump()+ return None++ def is_judged(self) -> bool:+ """Check if the agent trace has been judged"""+ if self.history and len(self.history[-1].result) > 0:+ last_result = self.history[-1].result[-1]+ return last_result.judgement is not None+ return False++ def is_validated(self) -> bool | None:+ """Check if the judge validated the agent execution (verdict is True). Returns None if not judged yet."""+ if self.history and len(self.history[-1].result) > 0:+ last_result = self.history[-1].result[-1]+ if last_result.judgement:+ return last_result.judgement.verdict+ return None+def urls(self) -> list[str | None]:"""Get all unique URLs from history"""return [h.state.url if h.state.url is not None else None for h in self.history]diff --git a/examples/features/judge_trace.py b/examples/features/judge_trace.pynew file mode 100644index 000000000..b17489063--- /dev/null+++ b/examples/features/judge_trace.py@@ -0,0 +1,41 @@+"""+Setup:+1. Get your API key from https://cloud.browser-use.com/new-api-key+2. Set environment variable: export BROWSER_USE_API_KEY="your-key"+"""++import asyncio+import os+import sys++# Add the parent directory to the path so we can import browser_use+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))++from dotenv import load_dotenv++load_dotenv()++from browser_use import Agent, ChatBrowserUse+from browser_use.llm import ChatGoogle+++async def main():+ llm = ChatBrowserUse()+ # Use Claude for judging since it supports vision + structured output+ judge_llm = ChatGoogle(model='gemini-flash-latest')+ task = "Search Google for 'what is browser automation' and tell me the top 3 results"+ agent = Agent(task=task, llm=llm, judge_llm=judge_llm)+ history = await agent.run()++ # Print the judgement result+ if history.is_judged():+ judgement = history.judgement()+ print('\n' + '=' * 80)+ print('JUDGE EVALUATION')+ print(judgement)+ else:+ print('\nNo judgement available (task may not have completed or use_judge=False)')+++if __name__ == '__main__':+ asyncio.run(main())