Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

feat: added AI summary for rerun task

Saurav Panda5e356108f9c1
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index fd283714e..3e7830e2e 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -451,3 +451,50 @@ Available tabs:
return UserMessage(content=content_parts, cache=True)
return UserMessage(content=state_description, cache=True)
+
+
+def get_rerun_summary_prompt(original_task: str, total_steps: int, success_count: int, error_count: int) -> str:
+ return f'''You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary.
+
+Original task: {original_task}
+
+Execution statistics:
+- Total steps: {total_steps}
+- Successful steps: {success_count}
+- Failed steps: {error_count}
+
+Analyze the screenshot to determine:
+1. Whether the task completed successfully
+2. What the final state shows
+3. Overall completion status (complete/partial/failed)
+
+Respond with:
+- summary: A clear, concise summary of what happened during the rerun
+- success: Whether the task completed successfully (true/false)
+- completion_status: One of "complete", "partial", or "failed"'''
+
+
+def get_rerun_summary_message(prompt: str, screenshot_b64: str | None = None) -> UserMessage:
+ """
+ Build a UserMessage for rerun summary generation.
+
+ Args:
+ prompt: The prompt text
+ screenshot_b64: Optional base64-encoded screenshot
+
+ Returns:
+ UserMessage with prompt and optional screenshot
+ """
+ if screenshot_b64:
+ # With screenshot: use multi-part content
+ content_parts: list[ContentPartTextParam | ContentPartImageParam] = [
+ ContentPartTextParam(type='text', text=prompt),
+ ContentPartImageParam(
+ type='image_url',
+ image_url=ImageURL(url=f'data:image/png;base64,{screenshot_b64}'),
+ ),
+ ]
+ return UserMessage(content=content_parts)
+ else:
+ # Without screenshot: use simple string content
+ return UserMessage(content=prompt)
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index cf0fdad3d..5d164b726 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -2165,12 +2165,104 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self.logger.info('✅ Task completed successfully')
await self._demo_mode_log('Task completed successfully', 'success', {'tag': 'task'})
+ async def _generate_rerun_summary(
+ self, original_task: str, results: list[ActionResult], summary_llm: BaseChatModel | None = None
+ ) -> ActionResult:
+ """Generate AI summary of rerun completion using screenshot and last step info"""
+ from browser_use.agent.views import RerunSummaryAction
+
+ # Get current screenshot
+ screenshot_b64 = None
+ try:
+ screenshot = await self.browser_session.take_screenshot(full_page=False)
+ if screenshot:
+ import base64
+
+ screenshot_b64 = base64.b64encode(screenshot).decode('utf-8')
+ except Exception as e:
+ self.logger.warning(f'Failed to capture screenshot for rerun summary: {e}')
+
+ # Build summary prompt and message
+ error_count = sum(1 for r in results if r.error)
+ success_count = len(results) - error_count
+
+ from browser_use.agent.prompts import get_rerun_summary_message, get_rerun_summary_prompt
+
+ prompt = get_rerun_summary_prompt(
+ original_task=original_task,
+ total_steps=len(results),
+ success_count=success_count,
+ error_count=error_count,
+ )
+
+ # Use provided LLM, agent's LLM, or fall back to OpenAI with structured output
+ try:
+ # Determine which LLM to use
+ if summary_llm is None:
+ # Try to use the agent's LLM first
+ summary_llm = self.llm
+ self.logger.debug('Using agent LLM for rerun summary')
+ else:
+ self.logger.debug(f'Using provided LLM for rerun summary: {summary_llm.model}')
+
+ # Build message with prompt and optional screenshot
+ from browser_use.llm.messages import BaseMessage
+
+ message = get_rerun_summary_message(prompt, screenshot_b64)
+ messages: list[BaseMessage] = [message] # type: ignore[list-item]
+
+ # Try calling with structured output first
+ self.logger.debug(f'Calling LLM for rerun summary with {len(messages)} message(s)')
+ try:
+ kwargs: dict = {'output_format': RerunSummaryAction}
+ response = await summary_llm.ainvoke(messages, **kwargs)
+ summary: RerunSummaryAction = response.completion # type: ignore[assignment]
+ self.logger.debug(f'LLM response type: {type(summary)}')
+ self.logger.debug(f'LLM response: {summary}')
+ except Exception as structured_error:
+ # If structured output fails (e.g., Browser-Use LLM doesn't support it for this type),
+ # fall back to text response without parsing
+ self.logger.debug(f'Structured output failed: {structured_error}, falling back to text response')
+
+ response = await summary_llm.ainvoke(messages, None)
+ response_text = response.completion
+ self.logger.debug(f'LLM text response: {response_text}')
+
+ # Use the text response directly as the summary
+ summary = RerunSummaryAction(
+ summary=response_text if isinstance(response_text, str) else str(response_text),
+ success=error_count == 0,
+ completion_status='complete' if error_count == 0 else ('partial' if success_count > 0 else 'failed'),
+ )
+
+ self.logger.info(f'📊 Rerun Summary: {summary.summary}')
+ self.logger.info(f'📊 Status: {summary.completion_status} (success={summary.success})')
+
+ return ActionResult(
+ is_done=True,
+ success=summary.success,
+ extracted_content=summary.summary,
+ long_term_memory=f'Rerun completed with status: {summary.completion_status}. {summary.summary[:100]}',
+ )
+
+ except Exception as e:
+ self.logger.warning(f'Failed to generate AI summary: {e.__class__.__name__}: {e}')
+ self.logger.debug('Full error traceback:', exc_info=True)
+ # Fallback to simple summary
+ return ActionResult(
+ is_done=True,
+ success=error_count == 0,
+ extracted_content=f'Rerun completed: {success_count}/{len(results)} steps succeeded',
+ long_term_memory=f'Rerun completed: {success_count} steps succeeded, {error_count} errors',
+ )
+
async def rerun_history(
self,
history: AgentHistoryList,
max_retries: int = 3,
skip_failures: bool = True,
delay_between_actions: float = 2.0,
+ summary_llm: BaseChatModel | None = None,
) -> list[ActionResult]:
"""
Rerun a saved history of actions with error handling and retry logic.
@@ -2180,9 +2272,10 @@ class Agent(Generic[Context, AgentStructuredOutput]):
max_retries: Maximum number of retries per action
skip_failures: Whether to skip failed actions or stop execution
delay_between_actions: Delay between actions in seconds
+ summary_llm: Optional LLM to use for generating the final summary. If not provided, uses the agent's LLM
Returns:
- List of action results
+ List of action results (including AI summary as the final result)
"""
# Skip cloud sync session events for rerunning (we're replaying, not starting new)
self.state.session_initialized = True
@@ -2244,6 +2337,11 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self.logger.warning(f'{step_name} failed (attempt {retry_count}/{max_retries}), retrying...')
await asyncio.sleep(delay_between_actions)
+ # Generate AI summary of rerun completion
+ self.logger.info('🤖 Generating AI summary of rerun completion...')
+ summary_result = await self._generate_rerun_summary(self.task, results, summary_llm)
+ results.append(summary_result)
+
await self.close()
return results
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.py
index 85add952b..537f3b184 100644
--- a/browser_use/agent/views.py
+++ b/browser_use/agent/views.py
@@ -153,6 +153,16 @@ class ActionResult(BaseModel):
return self
+class RerunSummaryAction(BaseModel):
+ """AI-generated summary for rerun completion"""
+
+ summary: str = Field(description='Summary of what happened during the rerun')
+ success: bool = Field(description='Whether the rerun completed successfully based on visual inspection')
+ completion_status: Literal['complete', 'partial', 'failed'] = Field(
+ description='Status of rerun completion: complete (all steps succeeded), partial (some steps succeeded), failed (task did not complete)'
+ )
+
+
class StepMetadata(BaseModel):
"""Metadata for a single step including timing and token information"""
diff --git a/examples/features/rerun_history.py b/examples/features/rerun_history.py
index 92d9ae0c0..a3eebbdb3 100644
--- a/examples/features/rerun_history.py
+++ b/examples/features/rerun_history.py
@@ -4,18 +4,42 @@ Example: Rerunning saved agent history with variable detection and substitution
This example shows how to:
1. Run an agent and save its history (including initial URL navigation)
2. Detect variables in the saved history (emails, names, dates, etc.)
-3. Rerun the history with original values
-4. Rerun the history with substituted values (different data)
+3. Rerun the history with substituted values (different data)
+4. Get AI-generated summary of rerun completion (with screenshot analysis)
Useful for:
- Debugging agent behavior
- Testing changes with consistent scenarios
- Replaying successful workflows with different data
- Understanding what values can be substituted in reruns
+- Getting automated verification of rerun success
Note: Initial actions (like opening URLs from tasks) are now automatically
saved to history and will be replayed during rerun, so you don't need to
worry about manually specifying URLs when rerunning.
+
+AI Summary:
+The rerun will automatically generate an AI summary at the end that analyzes
+the final screenshot and execution statistics. By default it uses the agent's LLM,
+but you can pass a custom LLM via the summary_llm parameter:
+
+ # Option 1: Use agent's LLM (default)
+ results = await agent.load_and_rerun(history_file)
+
+ # Option 2: Use a specific LLM for summary generation
+ from browser_use.llm.openai import ChatOpenAI
+ summary_llm = ChatOpenAI(model='gpt-4o')
+ results = await agent.load_and_rerun(history_file, summary_llm=summary_llm)
+
+ # Option 3: Use Anthropic, Google, or any other supported LLM
+ from browser_use.llm.anthropic import ChatAnthropic
+ summary_llm = ChatAnthropic(model='claude-3-5-sonnet-20241022')
+ results = await agent.load_and_rerun(history_file, summary_llm=summary_llm)
+
+The AI summary will be the last item in results and will have:
+ - extracted_content: The summary text
+ - success: Whether rerun was successful
+ - is_done: Always True for summary
"""
import asyncio
@@ -31,6 +55,13 @@ async def main():
task = 'Go to https://browser-use.github.io/stress-tests/challenges/reference-number-form.html and fill the form with example data and submit.'
llm = ChatBrowserUse()
+ # Optional: Use a custom LLM for AI summary generation
+ # By default, the agent's LLM is used, but you can specify a different one
+ # Uncomment to use a custom LLM for summaries:
+ # from browser_use.llm.openai import ChatOpenAI
+ # summary_llm = ChatOpenAI(model='gpt-4o-mini')
+ summary_llm = None # Set to None to use agent's LLM (default)
+
# Step 1: Run the agent and save history
print('=== Running Agent ===')
agent = Agent(task=task, llm=llm, max_actions_per_step=1)
@@ -49,13 +80,7 @@ async def main():
else:
print('No variables detected in history')
- # Step 3: Rerun the history with original values
- print('\n=== Rerunning History (Original Values) ===')
- rerun_agent = Agent(task='', llm=llm)
- await rerun_agent.load_and_rerun(history_file)
- print('✓ History rerun complete')
-
- # Step 4: Rerun the history with substituted values
+ # Step 3: Rerun the history with substituted values
if variables:
print('\n=== Rerunning History (Substituted Values) ===')
# Create new values for the detected variables
@@ -67,7 +92,7 @@ async def main():
elif var_name == 'full_name':
new_values[var_name] = 'Jane Smith'
elif var_name.startswith('full_name_'):
- new_values[var_name] = 'Sales Department'
+ new_values[var_name] = 'General Information'
elif var_name == 'first_name':
new_values[var_name] = 'Jane'
elif var_name == 'date':
@@ -82,10 +107,19 @@ async def main():
old_value = variables[var_name].original_value
print(f' • {var_name}: "{old_value}" → "{new_value}"')
- # Rerun with substituted values
+ # Rerun with substituted values and optional custom summary LLM
substitute_agent = Agent(task='', llm=llm)
- await substitute_agent.load_and_rerun(history_file, variables=new_values)
+ results = await substitute_agent.load_and_rerun(history_file, variables=new_values, summary_llm=summary_llm)
+
+ # Display AI-generated summary (last result)
+ if results and results[-1].is_done:
+ summary = results[-1]
+ print('\n📊 AI Summary:')
+ print(f' Summary: {summary.extracted_content}')
+ print(f' Success: {summary.success}')
print('✓ History rerun with substituted values complete')
+ else:
+ print('\n⚠️ No variables detected, skipping substitution rerun')
if __name__ == '__main__':
diff --git a/tests/ci/test_rerun_ai_summary.py b/tests/ci/test_rerun_ai_summary.py
new file mode 100644
index 000000000..1fbd187a7
--- /dev/null
+++ b/tests/ci/test_rerun_ai_summary.py
@@ -0,0 +1,169 @@
+"""Tests for AI summary generation during rerun"""
+
+from unittest.mock import AsyncMock
+
+from browser_use.agent.service import Agent
+from browser_use.agent.views import ActionResult, RerunSummaryAction
+from tests.ci.conftest import create_mock_llm
+
+
+async def test_generate_rerun_summary_success():
+ """Test that _generate_rerun_summary generates an AI summary for successful rerun"""
+ # Create mock LLM that returns RerunSummaryAction
+ summary_action = RerunSummaryAction(
+ summary='Form filled successfully',
+ success=True,
+ completion_status='complete',
+ )
+
+ async def custom_ainvoke(*args, **kwargs):
+ # Get output_format from second positional arg or kwargs
+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')
+ assert output_format is RerunSummaryAction
+ from browser_use.llm.views import ChatInvokeCompletion
+
+ return ChatInvokeCompletion(completion=summary_action, usage=None)
+
+ # Mock ChatOpenAI class
+ mock_openai = AsyncMock()
+ mock_openai.ainvoke.side_effect = custom_ainvoke
+
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ await agent.browser_session.start()
+
+ try:
+ # Create some successful results
+ results = [
+ ActionResult(long_term_memory='Step 1 completed'),
+ ActionResult(long_term_memory='Step 2 completed'),
+ ]
+
+ # Pass the mock LLM directly as summary_llm
+ summary = await agent._generate_rerun_summary('Test task', results, summary_llm=mock_openai)
+
+ # Check that result is the AI summary
+ assert summary.is_done is True
+ assert summary.success is True
+ assert summary.extracted_content == 'Form filled successfully'
+ assert 'Rerun completed' in (summary.long_term_memory or '')
+
+ finally:
+ await agent.close()
+
+
+async def test_generate_rerun_summary_with_errors():
+ """Test that AI summary correctly reflects errors in execution"""
+ # Create mock LLM for summary
+ summary_action = RerunSummaryAction(
+ summary='Rerun had errors',
+ success=False,
+ completion_status='failed',
+ )
+
+ async def custom_ainvoke(*args, **kwargs):
+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')
+ assert output_format is RerunSummaryAction
+ from browser_use.llm.views import ChatInvokeCompletion
+
+ return ChatInvokeCompletion(completion=summary_action, usage=None)
+
+ mock_openai = AsyncMock()
+ mock_openai.ainvoke.side_effect = custom_ainvoke
+
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ await agent.browser_session.start()
+
+ try:
+ # Create results with errors
+ results_with_errors = [
+ ActionResult(error='Failed to find element'),
+ ActionResult(error='Timeout'),
+ ]
+
+ # Pass the mock LLM directly as summary_llm
+ summary = await agent._generate_rerun_summary('Test task', results_with_errors, summary_llm=mock_openai)
+
+ # Verify summary reflects errors
+ assert summary.is_done is True
+ assert summary.success is False
+ assert summary.extracted_content == 'Rerun had errors'
+
+ finally:
+ await agent.close()
+
+
+async def test_generate_rerun_summary_fallback_on_error():
+ """Test that a fallback summary is generated if LLM fails"""
+ # Mock ChatOpenAI to throw an error
+ mock_openai = AsyncMock()
+ mock_openai.ainvoke.side_effect = Exception('LLM service unavailable')
+
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ await agent.browser_session.start()
+
+ try:
+ # Create some results
+ results = [
+ ActionResult(long_term_memory='Step 1 completed'),
+ ActionResult(long_term_memory='Step 2 completed'),
+ ]
+
+ # Pass the mock LLM directly as summary_llm
+ summary = await agent._generate_rerun_summary('Test task', results, summary_llm=mock_openai)
+
+ # Verify fallback summary
+ assert summary.is_done is True
+ assert summary.success is True # No errors, so success=True
+ assert 'Rerun completed' in (summary.extracted_content or '')
+ assert '2/2' in (summary.extracted_content or '') # Should show stats
+
+ finally:
+ await agent.close()
+
+
+async def test_generate_rerun_summary_statistics():
+ """Test that summary includes execution statistics in the prompt"""
+ # Create mock LLM
+ summary_action = RerunSummaryAction(
+ summary='3 of 5 steps succeeded',
+ success=False,
+ completion_status='partial',
+ )
+
+ async def custom_ainvoke(*args, **kwargs):
+ output_format = args[1] if len(args) > 1 else kwargs.get('output_format')
+ assert output_format is RerunSummaryAction
+ from browser_use.llm.views import ChatInvokeCompletion
+
+ return ChatInvokeCompletion(completion=summary_action, usage=None)
+
+ mock_openai = AsyncMock()
+ mock_openai.ainvoke.side_effect = custom_ainvoke
+
+ llm = create_mock_llm(actions=None)
+ agent = Agent(task='Test task', llm=llm)
+ await agent.browser_session.start()
+
+ try:
+ # Create results with mix of success and errors
+ results = [
+ ActionResult(long_term_memory='Step 1 completed'),
+ ActionResult(error='Step 2 failed'),
+ ActionResult(long_term_memory='Step 3 completed'),
+ ActionResult(error='Step 4 failed'),
+ ActionResult(long_term_memory='Step 5 completed'),
+ ]
+
+ # Pass the mock LLM directly as summary_llm
+ summary = await agent._generate_rerun_summary('Test task', results, summary_llm=mock_openai)
+
+ # Verify summary
+ assert summary.is_done is True
+ assert summary.success is False # partial completion
+ assert '3 of 5' in (summary.extracted_content or '')
+
+ finally:
+ await agent.close()