Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix bugs in library judge and make it off by default

Alezander915911c8c72bf
diff --git a/browser_use/agent/judge.py b/browser_use/agent/judge.py
index 1a6554e75..c5c7631e0 100644
--- a/browser_use/agent/judge.py
+++ b/browser_use/agent/judge.py
@@ -44,7 +44,7 @@ def construct_judge_messages(
final_result: str,
agent_steps: list[str],
screenshot_paths: list[str],
- max_images: int = 50,
+ max_images: int = 10,
) -> list[BaseMessage]:
"""
Construct messages for judge evaluation of agent trace.
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index af16d164f..e30fd06e8 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -164,6 +164,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):
flash_mode: bool = False,
max_history_items: int | None = None,
page_extraction_llm: BaseChatModel | None = None,
+ use_judge: bool = False,
+ judge_llm: BaseChatModel | None = None,
injected_agent_state: AgentState | None = None,
source: str | None = None,
file_system_path: str | None = None,
@@ -178,7 +180,6 @@ class Agent(Generic[Context, AgentStructuredOutput]):
include_recent_events: bool = False,
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
final_response_after_failure: bool = True,
- use_judge: bool = True,
_url_shortening_limit: int = 25,
**kwargs,
):
@@ -200,6 +201,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):
if page_extraction_llm is None:
page_extraction_llm = llm
+ if judge_llm is None:
+ judge_llm = llm
if available_file_paths is None:
available_file_paths = []
@@ -242,6 +245,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
# Core components
self.task = self._enhance_task_with_schema(task, output_model_schema)
self.llm = llm
+ self.judge_llm = judge_llm
self.directly_open_url = directly_open_url
self.include_recent_events = include_recent_events
self._url_shortening_limit = _url_shortening_limit
@@ -290,6 +294,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self.token_cost_service = TokenCost(include_cost=calculate_cost)
self.token_cost_service.register_llm(llm)
self.token_cost_service.register_llm(page_extraction_llm)
+ self.token_cost_service.register_llm(judge_llm)
# Initialize state
self.state = injected_agent_state or AgentState()
@@ -907,7 +912,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self._message_manager._add_context_message(UserMessage(content=msg))
self.AgentOutput = self.DoneAgentOutput
- async def _judge_trace(self) -> JudgementResult:
+ async def _judge_trace(self) -> JudgementResult | None:
"""Judge the trace of the agent"""
task = self.task
final_result = self.history.final_result() or ''
@@ -920,16 +925,20 @@ class Agent(Generic[Context, AgentStructuredOutput]):
final_result=final_result,
agent_steps=agent_steps,
screenshot_paths=screenshot_paths,
- max_images=50,
+ max_images=10,
)
# Call LLM with JudgementResult as output format
kwargs: dict = {'output_format': JudgementResult}
- response = await self.llm.ainvoke(input_messages, **kwargs)
- judgement: JudgementResult = response.completion # type: ignore[assignment]
-
- return judgement
+ try:
+ response = await self.judge_llm.ainvoke(input_messages, **kwargs)
+ judgement: JudgementResult = response.completion # type: ignore[assignment]
+ return judgement
+ except Exception as e:
+ self.logger.error(f'Judge trace failed: {e}')
+ # Return a default judgement on failure
+ return None
async def _get_model_output_with_retry(self, input_messages: list[BaseMessage]) -> AgentOutput:
"""Get model output with retry logic for empty actions"""
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.py
index 376c1a0c5..bc3ca59fe 100644
--- a/browser_use/agent/views.py
+++ b/browser_use/agent/views.py
@@ -511,6 +511,29 @@ class AgentHistoryList(BaseModel, Generic[AgentStructuredOutput]):
"""Check if the agent has any non-None errors"""
return any(error is not None for error in self.errors())
+ def judgement(self) -> dict | None:
+ """Get the judgement result as a dictionary if it exists"""
+ if self.history and len(self.history[-1].result) > 0:
+ last_result = self.history[-1].result[-1]
+ if last_result.judgement:
+ return last_result.judgement.model_dump()
+ return None
+
+ def is_judged(self) -> bool:
+ """Check if the agent trace has been judged"""
+ if self.history and len(self.history[-1].result) > 0:
+ last_result = self.history[-1].result[-1]
+ return last_result.judgement is not None
+ return False
+
+ def is_validated(self) -> bool | None:
+ """Check if the judge validated the agent execution (verdict is True). Returns None if not judged yet."""
+ if self.history and len(self.history[-1].result) > 0:
+ last_result = self.history[-1].result[-1]
+ if last_result.judgement:
+ return last_result.judgement.verdict
+ return None
+
def urls(self) -> list[str | None]:
"""Get all unique URLs from history"""
return [h.state.url if h.state.url is not None else None for h in self.history]
diff --git a/examples/features/judge_trace.py b/examples/features/judge_trace.py
new file mode 100644
index 000000000..b17489063
--- /dev/null
+++ b/examples/features/judge_trace.py
@@ -0,0 +1,41 @@
+"""
+Setup:
+1. Get your API key from https://cloud.browser-use.com/new-api-key
+2. Set environment variable: export BROWSER_USE_API_KEY="your-key"
+"""
+
+import asyncio
+import os
+import sys
+
+# Add the parent directory to the path so we can import browser_use
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from browser_use import Agent, ChatBrowserUse
+from browser_use.llm import ChatGoogle
+
+
+async def main():
+ llm = ChatBrowserUse()
+ # Use Claude for judging since it supports vision + structured output
+ judge_llm = ChatGoogle(model='gemini-flash-latest')
+ task = "Search Google for 'what is browser automation' and tell me the top 3 results"
+ agent = Agent(task=task, llm=llm, judge_llm=judge_llm)
+ history = await agent.run()
+
+ # Print the judgement result
+ if history.is_judged():
+ judgement = history.judgement()
+ print('\n' + '=' * 80)
+ print('JUDGE EVALUATION')
+ print(judgement)
+ else:
+ print('\nNo judgement available (task may not have completed or use_judge=False)')
+
+
+if __name__ == '__main__':
+ asyncio.run(main())