Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
feat: added AI summary for rerun task
Saurav Panda5e356108f9c1
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.pyindex fd283714e..3e7830e2e 100644--- a/browser_use/agent/prompts.py+++ b/browser_use/agent/prompts.py@@ -451,3 +451,50 @@ Available tabs:return UserMessage(content=content_parts, cache=True)return UserMessage(content=state_description, cache=True)+++def get_rerun_summary_prompt(original_task: str, total_steps: int, success_count: int, error_count: int) -> str:+ return f'''You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary.++Original task: {original_task}++Execution statistics:+- Total steps: {total_steps}+- Successful steps: {success_count}+- Failed steps: {error_count}++Analyze the screenshot to determine:+1. Whether the task completed successfully+2. What the final state shows+3. Overall completion status (complete/partial/failed)++Respond with:+- summary: A clear, concise summary of what happened during the rerun+- success: Whether the task completed successfully (true/false)+- completion_status: One of "complete", "partial", or "failed"'''+++def get_rerun_summary_message(prompt: str, screenshot_b64: str | None = None) -> UserMessage:+ """+ Build a UserMessage for rerun summary generation.++ Args:+ prompt: The prompt text+ screenshot_b64: Optional base64-encoded screenshot++ Returns:+ UserMessage with prompt and optional screenshot+ """+ if screenshot_b64:+ # With screenshot: use multi-part content+ content_parts: list[ContentPartTextParam | ContentPartImageParam] = [+ ContentPartTextParam(type='text', text=prompt),+ ContentPartImageParam(+ type='image_url',+ image_url=ImageURL(url=f'data:image/png;base64,{screenshot_b64}'),+ ),+ ]+ return UserMessage(content=content_parts)+ else:+ # Without screenshot: use simple string content+ return UserMessage(content=prompt)diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex cf0fdad3d..5d164b726 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -2165,12 +2165,104 @@ class Agent(Generic[Context, AgentStructuredOutput]):self.logger.info('✅ Task completed successfully')await self._demo_mode_log('Task completed successfully', 'success', {'tag': 'task'})+ async def _generate_rerun_summary(+ self, original_task: str, results: list[ActionResult], summary_llm: BaseChatModel | None = None+ ) -> ActionResult:+ """Generate AI summary of rerun completion using screenshot and last step info"""+ from browser_use.agent.views import RerunSummaryAction++ # Get current screenshot+ screenshot_b64 = None+ try:+ screenshot = await self.browser_session.take_screenshot(full_page=False)+ if screenshot:+ import base64++ screenshot_b64 = base64.b64encode(screenshot).decode('utf-8')+ except Exception as e:+ self.logger.warning(f'Failed to capture screenshot for rerun summary: {e}')++ # Build summary prompt and message+ error_count = sum(1 for r in results if r.error)+ success_count = len(results) - error_count++ from browser_use.agent.prompts import get_rerun_summary_message, get_rerun_summary_prompt++ prompt = get_rerun_summary_prompt(+ original_task=original_task,+ total_steps=len(results),+ success_count=success_count,+ error_count=error_count,+ )++ # Use provided LLM, agent's LLM, or fall back to OpenAI with structured output+ try:+ # Determine which LLM to use+ if summary_llm is None:+ # Try to use the agent's LLM first+ summary_llm = self.llm+ self.logger.debug('Using agent LLM for rerun summary')+ else:+ self.logger.debug(f'Using provided LLM for rerun summary: {summary_llm.model}')++ # Build message with prompt and optional screenshot+ from browser_use.llm.messages import BaseMessage++ message = get_rerun_summary_message(prompt, screenshot_b64)+ messages: list[BaseMessage] = [message] # type: ignore[list-item]++ # Try calling with structured output first+ self.logger.debug(f'Calling LLM for rerun summary with {len(messages)} message(s)')+ try:+ kwargs: dict = {'output_format': RerunSummaryAction}+ response = await summary_llm.ainvoke(messages, **kwargs)+ summary: RerunSummaryAction = response.completion # type: ignore[assignment]+ self.logger.debug(f'LLM response type: {type(summary)}')+ self.logger.debug(f'LLM response: {summary}')+ except Exception as structured_error:+ # If structured output fails (e.g., Browser-Use LLM doesn't support it for this type),+ # fall back to text response without parsing+ self.logger.debug(f'Structured output failed: {structured_error}, falling back to text response')++ response = await summary_llm.ainvoke(messages, None)+ response_text = response.completion+ self.logger.debug(f'LLM text response: {response_text}')++ # Use the text response directly as the summary+ summary = RerunSummaryAction(+ summary=response_text if isinstance(response_text, str) else str(response_text),+ success=error_count == 0,+ completion_status='complete' if error_count == 0 else ('partial' if success_count > 0 else 'failed'),+ )++ self.logger.info(f'📊 Rerun Summary: {summary.summary}')+ self.logger.info(f'📊 Status: {summary.completion_status} (success={summary.success})')++ return ActionResult(+ is_done=True,+ success=summary.success,+ extracted_content=summary.summary,+ long_term_memory=f'Rerun completed with status: {summary.completion_status}. {summary.summary[:100]}',+ )++ except Exception as e:+ self.logger.warning(f'Failed to generate AI summary: {e.__class__.__name__}: {e}')+ self.logger.debug('Full error traceback:', exc_info=True)+ # Fallback to simple summary+ return ActionResult(+ is_done=True,+ success=error_count == 0,+ extracted_content=f'Rerun completed: {success_count}/{len(results)} steps succeeded',+ long_term_memory=f'Rerun completed: {success_count} steps succeeded, {error_count} errors',+ )+async def rerun_history(self,history: AgentHistoryList,max_retries: int = 3,skip_failures: bool = True,delay_between_actions: float = 2.0,+ summary_llm: BaseChatModel | None = None,) -> list[ActionResult]:"""Rerun a saved history of actions with error handling and retry logic.@@ -2180,9 +2272,10 @@ class Agent(Generic[Context, AgentStructuredOutput]):max_retries: Maximum number of retries per actionskip_failures: Whether to skip failed actions or stop executiondelay_between_actions: Delay between actions in seconds+ summary_llm: Optional LLM to use for generating the final summary. If not provided, uses the agent's LLMReturns:- List of action results+ List of action results (including AI summary as the final result)"""# Skip cloud sync session events for rerunning (we're replaying, not starting new)self.state.session_initialized = True@@ -2244,6 +2337,11 @@ class Agent(Generic[Context, AgentStructuredOutput]):self.logger.warning(f'{step_name} failed (attempt {retry_count}/{max_retries}), retrying...')await asyncio.sleep(delay_between_actions)+ # Generate AI summary of rerun completion+ self.logger.info('🤖 Generating AI summary of rerun completion...')+ summary_result = await self._generate_rerun_summary(self.task, results, summary_llm)+ results.append(summary_result)+await self.close()return resultsdiff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex 85add952b..537f3b184 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -153,6 +153,16 @@ class ActionResult(BaseModel):return self+class RerunSummaryAction(BaseModel):+ """AI-generated summary for rerun completion"""++ summary: str = Field(description='Summary of what happened during the rerun')+ success: bool = Field(description='Whether the rerun completed successfully based on visual inspection')+ completion_status: Literal['complete', 'partial', 'failed'] = Field(+ description='Status of rerun completion: complete (all steps succeeded), partial (some steps succeeded), failed (task did not complete)'+ )++class StepMetadata(BaseModel):"""Metadata for a single step including timing and token information"""diff --git a/examples/features/rerun_history.py b/examples/features/rerun_history.pyindex 92d9ae0c0..a3eebbdb3 100644--- a/examples/features/rerun_history.py+++ b/examples/features/rerun_history.py@@ -4,18 +4,42 @@ Example: Rerunning saved agent history with variable detection and substitutionThis example shows how to:1. Run an agent and save its history (including initial URL navigation)2. Detect variables in the saved history (emails, names, dates, etc.)-3. Rerun the history with original values-4. Rerun the history with substituted values (different data)+3. Rerun the history with substituted values (different data)+4. Get AI-generated summary of rerun completion (with screenshot analysis)Useful for:- Debugging agent behavior- Testing changes with consistent scenarios- Replaying successful workflows with different data- Understanding what values can be substituted in reruns+- Getting automated verification of rerun successNote: Initial actions (like opening URLs from tasks) are now automaticallysaved to history and will be replayed during rerun, so you don't need toworry about manually specifying URLs when rerunning.++AI Summary:+The rerun will automatically generate an AI summary at the end that analyzes+the final screenshot and execution statistics. By default it uses the agent's LLM,+but you can pass a custom LLM via the summary_llm parameter:++ # Option 1: Use agent's LLM (default)+ results = await agent.load_and_rerun(history_file)++ # Option 2: Use a specific LLM for summary generation+ from browser_use.llm.openai import ChatOpenAI+ summary_llm = ChatOpenAI(model='gpt-4o')+ results = await agent.load_and_rerun(history_file, summary_llm=summary_llm)++ # Option 3: Use Anthropic, Google, or any other supported LLM+ from browser_use.llm.anthropic import ChatAnthropic+ summary_llm = ChatAnthropic(model='claude-3-5-sonnet-20241022')+ results = await agent.load_and_rerun(history_file, summary_llm=summary_llm)++The AI summary will be the last item in results and will have:+ - extracted_content: The summary text+ - success: Whether rerun was successful+ - is_done: Always True for summary"""import asyncio@@ -31,6 +55,13 @@ async def main():task = 'Go to https://browser-use.github.io/stress-tests/challenges/reference-number-form.html and fill the form with example data and submit.'llm = ChatBrowserUse()+ # Optional: Use a custom LLM for AI summary generation+ # By default, the agent's LLM is used, but you can specify a different one+ # Uncomment to use a custom LLM for summaries:+ # from browser_use.llm.openai import ChatOpenAI+ # summary_llm = ChatOpenAI(model='gpt-4o-mini')+ summary_llm = None # Set to None to use agent's LLM (default)+# Step 1: Run the agent and save historyprint('=== Running Agent ===')agent = Agent(task=task, llm=llm, max_actions_per_step=1)@@ -49,13 +80,7 @@ async def main():else:print('No variables detected in history')- # Step 3: Rerun the history with original values- print('\n=== Rerunning History (Original Values) ===')- rerun_agent = Agent(task='', llm=llm)- await rerun_agent.load_and_rerun(history_file)- print('✓ History rerun complete')-- # Step 4: Rerun the history with substituted values+ # Step 3: Rerun the history with substituted valuesif variables:print('\n=== Rerunning History (Substituted Values) ===')# Create new values for the detected variables@@ -67,7 +92,7 @@ async def main():elif var_name == 'full_name':new_values[var_name] = 'Jane Smith'elif var_name.startswith('full_name_'):- new_values[var_name] = 'Sales Department'+ new_values[var_name] = 'General Information'elif var_name == 'first_name':new_values[var_name] = 'Jane'elif var_name == 'date':@@ -82,10 +107,19 @@ async def main():old_value = variables[var_name].original_valueprint(f' • {var_name}: "{old_value}" → "{new_value}"')- # Rerun with substituted values+ # Rerun with substituted values and optional custom summary LLMsubstitute_agent = Agent(task='', llm=llm)- await substitute_agent.load_and_rerun(history_file, variables=new_values)+ results = await substitute_agent.load_and_rerun(history_file, variables=new_values, summary_llm=summary_llm)++ # Display AI-generated summary (last result)+ if results and results[-1].is_done:+ summary = results[-1]+ print('\n📊 AI Summary:')+ print(f' Summary: {summary.extracted_content}')+ print(f' Success: {summary.success}')print('✓ History rerun with substituted values complete')+ else:+ print('\n⚠️ No variables detected, skipping substitution rerun')if __name__ == '__main__':diff --git a/tests/ci/test_rerun_ai_summary.py b/tests/ci/test_rerun_ai_summary.pynew file mode 100644index 000000000..1fbd187a7--- /dev/null+++ b/tests/ci/test_rerun_ai_summary.py@@ -0,0 +1,169 @@+"""Tests for AI summary generation during rerun"""++from unittest.mock import AsyncMock++from browser_use.agent.service import Agent+from browser_use.agent.views import ActionResult, RerunSummaryAction+from tests.ci.conftest import create_mock_llm+++async def test_generate_rerun_summary_success():+ """Test that _generate_rerun_summary generates an AI summary for successful rerun"""+ # Create mock LLM that returns RerunSummaryAction+ summary_action = RerunSummaryAction(+ summary='Form filled successfully',+ success=True,+ completion_status='complete',+ )++ async def custom_ainvoke(*args, **kwargs):+ # Get output_format from second positional arg or kwargs+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')+ assert output_format is RerunSummaryAction+ from browser_use.llm.views import ChatInvokeCompletion++ return ChatInvokeCompletion(completion=summary_action, usage=None)++ # Mock ChatOpenAI class+ mock_openai = AsyncMock()+ mock_openai.ainvoke.side_effect = custom_ainvoke++ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)+ await agent.browser_session.start()++ try:+ # Create some successful results+ results = [+ ActionResult(long_term_memory='Step 1 completed'),+ ActionResult(long_term_memory='Step 2 completed'),+ ]++ # Pass the mock LLM directly as summary_llm+ summary = await agent._generate_rerun_summary('Test task', results, summary_llm=mock_openai)++ # Check that result is the AI summary+ assert summary.is_done is True+ assert summary.success is True+ assert summary.extracted_content == 'Form filled successfully'+ assert 'Rerun completed' in (summary.long_term_memory or '')++ finally:+ await agent.close()+++async def test_generate_rerun_summary_with_errors():+ """Test that AI summary correctly reflects errors in execution"""+ # Create mock LLM for summary+ summary_action = RerunSummaryAction(+ summary='Rerun had errors',+ success=False,+ completion_status='failed',+ )++ async def custom_ainvoke(*args, **kwargs):+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')+ assert output_format is RerunSummaryAction+ from browser_use.llm.views import ChatInvokeCompletion++ return ChatInvokeCompletion(completion=summary_action, usage=None)++ mock_openai = AsyncMock()+ mock_openai.ainvoke.side_effect = custom_ainvoke++ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)+ await agent.browser_session.start()++ try:+ # Create results with errors+ results_with_errors = [+ ActionResult(error='Failed to find element'),+ ActionResult(error='Timeout'),+ ]++ # Pass the mock LLM directly as summary_llm+ summary = await agent._generate_rerun_summary('Test task', results_with_errors, summary_llm=mock_openai)++ # Verify summary reflects errors+ assert summary.is_done is True+ assert summary.success is False+ assert summary.extracted_content == 'Rerun had errors'++ finally:+ await agent.close()+++async def test_generate_rerun_summary_fallback_on_error():+ """Test that a fallback summary is generated if LLM fails"""+ # Mock ChatOpenAI to throw an error+ mock_openai = AsyncMock()+ mock_openai.ainvoke.side_effect = Exception('LLM service unavailable')++ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)+ await agent.browser_session.start()++ try:+ # Create some results+ results = [+ ActionResult(long_term_memory='Step 1 completed'),+ ActionResult(long_term_memory='Step 2 completed'),+ ]++ # Pass the mock LLM directly as summary_llm+ summary = await agent._generate_rerun_summary('Test task', results, summary_llm=mock_openai)++ # Verify fallback summary+ assert summary.is_done is True+ assert summary.success is True # No errors, so success=True+ assert 'Rerun completed' in (summary.extracted_content or '')+ assert '2/2' in (summary.extracted_content or '') # Should show stats++ finally:+ await agent.close()+++async def test_generate_rerun_summary_statistics():+ """Test that summary includes execution statistics in the prompt"""+ # Create mock LLM+ summary_action = RerunSummaryAction(+ summary='3 of 5 steps succeeded',+ success=False,+ completion_status='partial',+ )++ async def custom_ainvoke(*args, **kwargs):+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')+ assert output_format is RerunSummaryAction+ from browser_use.llm.views import ChatInvokeCompletion++ return ChatInvokeCompletion(completion=summary_action, usage=None)++ mock_openai = AsyncMock()+ mock_openai.ainvoke.side_effect = custom_ainvoke++ llm = create_mock_llm(actions=None)+ agent = Agent(task='Test task', llm=llm)+ await agent.browser_session.start()++ try:+ # Create results with mix of success and errors+ results = [+ ActionResult(long_term_memory='Step 1 completed'),+ ActionResult(error='Step 2 failed'),+ ActionResult(long_term_memory='Step 3 completed'),+ ActionResult(error='Step 4 failed'),+ ActionResult(long_term_memory='Step 5 completed'),+ ]++ # Pass the mock LLM directly as summary_llm+ summary = await agent._generate_rerun_summary('Test task', results, summary_llm=mock_openai)++ # Verify summary+ assert summary.is_done is True+ assert summary.success is False # partial completion+ assert '3 of 5' in (summary.extracted_content or '')++ finally:+ await agent.close()