Skip to content

Build and run agents you can see, understand and trust.

perf(browser-use): update the browser-use agent example (#705)

Yue Cuif646df4a77a0
diff --git a/examples/agent/browser_agent/browser_agent.py b/examples/agent/browser_agent/browser_agent.py
index ebf4d1c8a..d82b0a4f4 100644
--- a/examples/agent/browser_agent/browser_agent.py
+++ b/examples/agent/browser_agent/browser_agent.py
@@ -1,126 +1,98 @@
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Browser Agent"""
-# pylint: disable=W0212
+# flake8: noqa: E501
+# pylint: disable=W0212,too-many-lines,C0301,W0107,C0411
import re
import uuid
-from typing import Optional, Any
+import os
+import json
+import inspect
+from functools import wraps
+from typing import Type, Optional, Any, Literal
+import asyncio
+import copy
+
+from pydantic import BaseModel
from agentscope.agent import ReActAgent
+from agentscope._logging import logger
from agentscope.formatter import FormatterBase
from agentscope.memory import MemoryBase
from agentscope.message import (
Msg,
ToolUseBlock,
TextBlock,
+ ToolResultBlock,
+ ImageBlock,
+ Base64Source,
)
from agentscope.model import ChatModelBase
-from agentscope.tool import Toolkit
+from agentscope.tool import (
+ Toolkit,
+ ToolResponse,
+)
from agentscope.token import TokenCounterBase, OpenAITokenCounter
-_BROWSER_AGENT_DEFAULT_SYS_PROMPT = (
- "You are a helpful browser automation assistant. "
- "You can navigate websites, take screenshots, and interact with web pages."
- "Always describe what you see and plan your next steps clearly. "
- "When taking actions, explain what you're doing and why."
-)
-_BROWSER_AGENT_REASONING_PROMPT = (
- "You are browsing the current website. "
- "The snapshot (and screenshot) of the current webpage is (are) given "
- "below. Since you can only view the latest webpage, "
- "you must promptly summarize current status, record required data, "
- "and plan your next steps."
-)
+from build_in_helper._image_understanding import image_understanding
+from build_in_helper._video_understanding import video_understanding
+from build_in_helper._file_download import file_download
+from build_in_helper._form_filling import form_filling
+_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+_PROMPT_DIR = os.path.join(_CURRENT_DIR, "build_in_prompt")
+_HELPER_DIR = os.path.join(_CURRENT_DIR, "build_in_helper")
-async def browser_agent_default_url_pre_reply(
- self: "BrowserAgent", # pylint: disable=W0613
- *args: Any, # pylint: disable=W0613
- **kwargs: Any, # pylint: disable=W0613
-) -> None:
- """Navigate to start URL if this is the first interaction"""
- if self.start_url and not self._has_initial_navigated:
- await self._navigate_to_start_url()
- self._has_initial_navigated = True
-
-
-async def browser_agent_summarize_mem_pre_reasoning(
- self: "BrowserAgent", # pylint: disable=W0613
- *args: Any,
- **kwargs: Any,
-) -> None:
- """Summarize memory if too long"""
- mem_len = await self.memory.size()
- if mem_len > self.max_memory_length:
- await self._memory_summarizing()
-
-
-async def browser_agent_observe_pre_reasoning(
- self: "BrowserAgent", # pylint: disable=W0613
- *args: Any,
- **kwargs: Any,
-) -> None:
- """Get a snapshot in text before reasoning"""
- snapshot_msg = await self._get_snapshot_in_text()
- await self.memory.add(snapshot_msg)
-
-
-async def browser_agent_remove_observation_post_reasoning(
- self: "BrowserAgent", # pylint: disable=W0613
- *args: Any,
- **kwargs: Any,
-) -> None:
- """Remove the snapshot msg after reasoning"""
- mem_len = await self.memory.size()
- if mem_len >= 2:
- await self.memory.delete(mem_len - 2)
-
-
-async def browser_agent_post_acting_clean_content(
- self: "BrowserAgent", # pylint: disable=W0613
- *args: Any,
- **kwargs: Any,
-) -> None:
- """
- Hook func for cleaning the messy return after action.
- Observation will be done before reasoning steps.
- """
- mem_msgs = await self.memory.get_memory()
- mem_length = await self.memory.size()
- if len(mem_msgs) == 0:
- return
- last_output_msg = mem_msgs[-1]
- for i, b in enumerate(last_output_msg.content):
- if b["type"] == "tool_result":
- for j, return_json in enumerate(b.get("output", [])):
- if isinstance(return_json, dict) and "text" in return_json:
- last_output_msg.content[i]["output"][j][
- "output"
- ] = self._filter_execution_text(return_json["text"])
- await self.memory.delete(mem_length - 1)
- await self.memory.add(last_output_msg)
+class EmptyModel(BaseModel):
+ """Empty structured model for default structured output requirement."""
-class BrowserAgent(ReActAgent):
- """
- Browser Agent that extends ReActAgent with browser-specific capabilities.
+ pass
- The agent leverages MCP (Model Context Protocol) servers to access browser
- tools with Playwright, enabling sophisticated web automation tasks.
- Example:
- .. code-block:: python
+with open(
+ os.path.join(_PROMPT_DIR, "browser_agent_sys_prompt.md"),
+ "r",
+ encoding="utf-8",
+) as f:
+ _BROWSER_AGENT_DEFAULT_SYS_PROMPT = f.read()
- agent = BrowserAgent(
- name="web_navigator",
- model=my_chat_model,
- formatter=my_formatter,
- memory=my_memory,
- toolkit=browser_toolkit,
- start_url="https://example.com"
- )
+with open(
+ os.path.join(_PROMPT_DIR, "browser_agent_pure_reasoning_prompt.md"),
+ "r",
+ encoding="utf-8",
+) as f:
+ _BROWSER_AGENT_DEFAULT_PURE_REASONING_PROMPT = f.read()
+
+with open(
+ os.path.join(_PROMPT_DIR, "browser_agent_observe_reasoning_prompt.md"),
+ "r",
+ encoding="utf-8",
+) as f:
+ _BROWSER_AGENT_DEFAULT_OBSERVE_REASONING_PROMPT = f.read()
+
+with open(
+ os.path.join(_PROMPT_DIR, "browser_agent_task_decomposition_prompt.md"),
+ "r",
+ encoding="utf-8",
+) as f:
+ _BROWSER_AGENT_DEFAULT_TASK_DECOMPOSITION_PROMPT = f.read()
+
+with open(
+ os.path.join(_PROMPT_DIR, "browser_agent_summarize_task.md"),
+ "r",
+ encoding="utf-8",
+) as f:
+ _BROWSER_AGENT_SUMMARIZE_TASK_PROMPT = f.read()
- response = await agent.reply("Search for Python tutorials")
+
+class BrowserAgent(ReActAgent):
+ """
+ Browser Agent that extends ReActAgent with browser-specific capabilities.
+
+ The agent leverages MCP servers to access browser tools with Playwright,
+ enabling sophisticated web automation tasks.
"""
def __init__(
@@ -133,44 +105,33 @@ class BrowserAgent(ReActAgent):
sys_prompt: str = _BROWSER_AGENT_DEFAULT_SYS_PROMPT,
max_iters: int = 50,
start_url: Optional[str] = "https://www.google.com",
- reasoning_prompt: str = _BROWSER_AGENT_REASONING_PROMPT,
+ pure_reasoning_prompt: str = _BROWSER_AGENT_DEFAULT_PURE_REASONING_PROMPT,
+ observe_reasoning_prompt: str = _BROWSER_AGENT_DEFAULT_OBSERVE_REASONING_PROMPT,
+ task_decomposition_prompt: str = _BROWSER_AGENT_DEFAULT_TASK_DECOMPOSITION_PROMPT,
token_counter: TokenCounterBase = OpenAITokenCounter("gpt-4o"),
max_mem_length: int = 20,
) -> None:
- """Initialize the Browser Agent.
-
- Args:
- name (str):
- The unique identifier name for the agent instance.
- model (ChatModelBase):
- The chat model used for generating responses and reasoning.
- formatter (FormatterBase):
- The formatter used to convert messages into the required format
- for the model API.
- memory (MemoryBase):
- The memory component used to store and retrieve dialogue
- history.
- toolkit (Toolkit):
- A toolkit object containing the browser tool functions and
- utilities.
- sys_prompt (str, optional):
- The system prompt that defines the agent's behavior and
- personality.
- Defaults to _BROWSER_AGENT_DEFAULT_SYS_PROMPT.
- max_iters (int, optional):
- The maximum number of reasoning-acting loop iterations.
- Defaults to 50.
- start_url (Optional[str], optional):
- The initial URL to navigate to when the agent starts.
- Defaults to "https://www.google.com".
- reasoning_prompt (str, optional):
- The prompt used during the reasoning phase to guide
- decision-making.
- Defaults to _BROWSER_AGENT_REASONING_PROMPT.
-
- Returns:
- None
- """
+ """Initialize the Browser Agent."""
+ self.start_url = start_url
+ self._has_initial_navigated = False
+ self.pure_reasoning_prompt = pure_reasoning_prompt
+ self.observe_reasoning_prompt = observe_reasoning_prompt
+ self.task_decomposition_prompt = task_decomposition_prompt
+ self.max_memory_length = max_mem_length
+ self.token_estimator = token_counter
+ self.snapshot_chunk_id = 0
+ self.chunk_continue_status = False
+ self.previous_chunkwise_information = ""
+ self.snapshot_in_chunk: list[str] = []
+ self.subtasks: list[Any] = []
+ self.original_task = ""
+ self.current_subtask_idx = 0
+ self.current_subtask: Any = None
+ self.iter_n = 0
+ self.finish_function_name = "browser_generate_final_response"
+ self.init_query = ""
+ self._required_structured_model: Type[BaseModel] | None = None
+
super().__init__(
name=name,
sys_prompt=sys_prompt,
@@ -181,84 +142,549 @@ class BrowserAgent(ReActAgent):
max_iters=max_iters,
)
- self.start_url = start_url
- self._has_initial_navigated = False
- self.reasoning_prompt = reasoning_prompt
- self.max_memory_length = max_mem_length
- self.token_estimator = token_counter
+ # Register tools
+ self.toolkit.register_tool_function(self.browser_subtask_manager)
+
+ # Register skill tools if model supports multimodal
+ if self._supports_multimodal():
+ self._register_skill_tool(image_understanding)
+ self._register_skill_tool(video_understanding)
+
+ # Register other skill tools
+ self._register_skill_tool(file_download)
+ self._register_skill_tool(form_filling)
+
+ # Build a tool list without screenshot to avoid unnecessary captures
+ self.no_screenshot_tool_list = [
+ tool
+ for tool in self.toolkit.get_json_schemas()
+ if tool.get("function", {}).get("name")
+ != "browser_take_screenshot"
+ ]
+
+ async def reply( # pylint: disable=R0912,R0915
+ self,
+ msg: Msg | list[Msg] | None = None,
+ structured_model: Type[BaseModel] | None = None,
+ ) -> Msg:
+ """Process a message and return a response."""
+ self.init_query = (
+ msg.content
+ if isinstance(msg, Msg)
+ else msg[0].content
+ if isinstance(msg, list)
+ else ""
+ )
+
+ if self.start_url and not self._has_initial_navigated:
+ await self._navigate_to_start_url()
+ self._has_initial_navigated = True
+
+ msg = await self._task_decomposition_and_reformat(msg)
+
+ await self.memory.add(msg)
+ # Default to EmptyModel to require structured output if none provided
+ if structured_model is None:
+ structured_model = EmptyModel
+
+ tool_choice: Literal["auto", "none", "required"] | None = None
+ self._required_structured_model = structured_model
+
+ # Register finish tool only when structured model is required
+ if structured_model:
+ if self.finish_function_name not in self.toolkit.tools:
+ self.toolkit.register_tool_function(
+ getattr(self, self.finish_function_name),
+ )
+ self.toolkit.set_extended_model(
+ self.finish_function_name,
+ structured_model,
+ )
+ tool_choice = "required"
+ else:
+ self.toolkit.remove_tool_function(self.finish_function_name)
+
+ # The reasoning-acting loop
+ structured_output = None
+ reply_msg = None
+ for iter_n in range(self.max_iters):
+ self.iter_n = iter_n + 1
+ await self._summarize_mem()
+ msg_reasoning = await self._pure_reasoning(tool_choice)
+
+ tool_calls = msg_reasoning.get_content_blocks("tool_use")
+ if tool_calls and tool_calls[0]["name"] == "browser_snapshot":
+ msg_reasoning = await self._reasoning_with_observation()
+
+ futures = [
+ self._acting(tool_call)
+ for tool_call in msg_reasoning.get_content_blocks("tool_use")
+ ]
+
+ # Parallel tool calls or not
+ if self.parallel_tool_calls:
+ structured_outputs = await asyncio.gather(*futures)
+ else:
+ structured_outputs = [await _ for _ in futures]
+
+ # Check for exit condition
+ # If structured output is still not satisfied
+ if self._required_structured_model:
+ # Remove None results
+ structured_outputs = [_ for _ in structured_outputs if _]
+
+ msg_hint = None
+ # If the acting step generates structured outputs
+ if structured_outputs:
+ # Cache the structured output data
+ structured_output = structured_outputs[-1]
+
+ reply_msg = Msg(
+ self.name,
+ structured_output.get("subtask_progress_summary", ""),
+ "assistant",
+ metadata=structured_output,
+ )
+ break
+
+ if not msg_reasoning.has_content_blocks("tool_use"):
+ # If structured output is required but no tool call is
+ # made, remind the llm to go on the task
+ msg_hint = Msg(
+ "user",
+ "<system-hint>Structured output is "
+ f"required, go on to finish your task or call "
+ f"'{self.finish_function_name}' to generate the "
+ f"required structured output.</system-hint>",
+ "user",
+ )
+ await self._reasoning_hint_msgs.add(msg_hint)
+ # Require tool call in the next reasoning step
+ tool_choice = "required"
+
+ if msg_hint and self.print_hint_msg:
+ await self.print(msg_hint)
+
+ elif not msg_reasoning.has_content_blocks("tool_use"):
+ # Exit the loop when no structured output is required (or
+ # already satisfied) and only text response is generated
+ msg_reasoning.metadata = structured_output
+ reply_msg = msg_reasoning
+ break
+
+ # When the maximum iterations are reached
+ # and no reply message is generated
+ if reply_msg is None:
+ reply_msg = await self._summarizing()
+ reply_msg.metadata = structured_output
+ await self.memory.add(reply_msg)
+
+ return reply_msg
+
+ async def _pure_reasoning(
+ self,
+ tool_choice: Literal["auto", "none", "required"] | None = None,
+ ) -> Msg:
+ """Initial reasoning without screenshot observation."""
+ msg = Msg(
+ "user",
+ content=self.pure_reasoning_prompt.format(
+ current_subtask=self.current_subtask,
+ init_query=self.original_task,
+ ),
+ role="user",
+ )
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", self.sys_prompt, "system"),
+ *await self.memory.get_memory(),
+ msg,
+ # The hint messages to guide the agent's behavior, maybe empty
+ *await self._reasoning_hint_msgs.get_memory(),
+ ],
+ )
- self.register_instance_hook(
- "pre_reply",
- "browser_agent_default_url_pre_reply",
- browser_agent_default_url_pre_reply,
+ # Clear the hint messages after use
+ await self._reasoning_hint_msgs.clear()
+
+ res = await self.model(
+ prompt,
+ tools=self.no_screenshot_tool_list,
+ tool_choice=tool_choice,
)
- self.register_instance_hook(
- "pre_reasoning",
- "browser_agent_summarize_mem_pre_reasoning",
- browser_agent_summarize_mem_pre_reasoning,
+ interrupted_by_user = False
+ msg = None
+ try:
+ if self.model.stream:
+ msg = Msg(self.name, [], "assistant")
+ async for content_chunk in res:
+ msg.content = content_chunk.content
+ await self.print(msg)
+ else:
+ msg = Msg(self.name, list(res.content), "assistant")
+ await self.print(msg)
+ return msg
+ except asyncio.CancelledError as e:
+ interrupted_by_user = True
+ raise e from None
+ finally:
+ await self.memory.add(msg)
+ tool_use_blocks: list = msg.get_content_blocks("tool_use") # type: ignore
+ if interrupted_by_user and msg:
+ for tool_call in tool_use_blocks: # pylint: disable=E1133
+ msg_res = Msg(
+ "system",
+ [
+ ToolResultBlock(
+ type="tool_result",
+ id=tool_call["id"],
+ name=tool_call["name"],
+ output="The tool call has been interrupted by the user.",
+ ),
+ ],
+ "system",
+ )
+ await self.memory.add(msg_res)
+ await self.print(msg_res)
+
+ async def _reasoning_with_observation(self) -> Msg:
+ """Perform the reasoning process with page observation in chunks."""
+ self.snapshot_chunk_id = 0
+ self.chunk_continue_status = False
+ self.previous_chunkwise_information = ""
+ self.snapshot_in_chunk = []
+
+ mem_len = await self.memory.size()
+ await self.memory.delete(mem_len - 1)
+
+ self.snapshot_in_chunk = await self._get_snapshot_in_text()
+ for _ in self.snapshot_in_chunk:
+ observe_msg = await self._build_observation()
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", self.sys_prompt, "system"),
+ *await self.memory.get_memory(),
+ observe_msg,
+ ],
+ )
+ res = await self.model(
+ prompt,
+ tools=self.no_screenshot_tool_list,
+ )
+
+ interrupted_by_user = False
+ msg = None
+ try:
+ if self.model.stream:
+ msg = Msg(self.name, [], "assistant")
+ async for content_chunk in res:
+ msg.content = content_chunk.content
+ # await self.print(msg)
+ else:
+ msg = Msg(self.name, list(res.content), "assistant")
+ # await self.print(msg)
+ logger.info(msg.content)
+ except asyncio.CancelledError as e:
+ interrupted_by_user = True
+ raise e from None
+
+ tool_use_blocks: list = msg.get_content_blocks("tool_use") # type: ignore
+ await self._update_chunk_observation_status(output_msg=msg)
+
+ if interrupted_by_user and msg:
+ for tool_call in tool_use_blocks: # pylint: disable=E1133
+ msg_res = Msg(
+ "system",
+ [
+ ToolResultBlock(
+ type="tool_result",
+ id=tool_call["id"],
+ name=tool_call["name"],
+ output="The tool call has been interrupted by the user.",
+ ),
+ ],
+ "system",
+ )
+ await self.memory.add(msg_res)
+ await self.print(msg_res)
+
+ if not self.chunk_continue_status:
+ break
+
+ await self.memory.add(msg)
+ return msg
+
+ async def _summarize_mem(self) -> None:
+ """Summarize memory if too long."""
+ mem_len = await self.memory.size()
+ if mem_len > self.max_memory_length:
+ await self._memory_summarizing()
+
+ async def _build_observation(self) -> Msg:
+ """Get a snapshot (and optional screenshot) before reasoning."""
+ image_data: Optional[str] = None
+ if self._supports_multimodal():
+ image_data = await self._get_screenshot()
+ observe_msg = self.observe_by_chunk(image_data)
+ return observe_msg
+
+ async def _update_chunk_observation_status(
+ self,
+ output_msg: Msg | None = None,
+ ) -> None:
+ """Update the chunk observation status after reasoning."""
+ for _, b in enumerate(output_msg.content):
+ if b["type"] == "text":
+ raw_response = b["text"]
+ try:
+ if "```json" in raw_response:
+ raw_response = raw_response.replace(
+ "```json",
+ "",
+ ).replace("```", "")
+ data = json.loads(raw_response)
+ information = data.get("INFORMATION", "")
+ # Continue unless STATUS is explicitly REASONING_FINISHED
+ self.chunk_continue_status = (
+ data.get("STATUS") != "REASONING_FINISHED"
+ )
+ except Exception:
+ information = raw_response
+ if (
+ self.snapshot_chunk_id
+ < len(self.snapshot_in_chunk) - 1
+ ):
+ self.chunk_continue_status = True
+ self.snapshot_chunk_id += 1
+ else:
+ self.chunk_continue_status = False
+ if not isinstance(information, str):
+ try:
+ information = json.dumps(
+ information,
+ ensure_ascii=False,
+ )
+ except Exception:
+ information = str(information)
+ self.previous_chunkwise_information += (
+ f"Information in chunk {self.snapshot_chunk_id+1} of {len(self.snapshot_in_chunk)}:\n"
+ + information
+ + "\n"
+ )
+ if b["type"] == "tool_use":
+ self.chunk_continue_status = False
+
+ async def _acting(self, tool_call: ToolUseBlock) -> dict | None:
+ """Perform the acting process and return structured output if generated."""
+ tool_res_msg = Msg(
+ "system",
+ [
+ ToolResultBlock(
+ type="tool_result",
+ id=tool_call["id"],
+ name=tool_call["name"],
+ output=[],
+ ),
+ ],
+ "system",
)
+ try:
+ tool_res = await self.toolkit.call_tool_function(tool_call)
+ structured_output = None
+ async for chunk in tool_res:
+ tool_res_msg.content[0]["output"] = chunk.content # type: ignore[index]
+ await self.print(tool_res_msg, chunk.is_last)
+
+ # Raise the CancelledError to handle the interruption
+ if chunk.is_interrupted:
+ raise asyncio.CancelledError()
+
+ # Return structured output if generate_response is called successfully
+ if (
+ tool_call["name"] == self.finish_function_name
+ and chunk.metadata
+ and chunk.metadata.get("success", False)
+ ):
+ # Only return the structured output
+ structured_output = chunk.metadata.get("structured_output")
+ return structured_output
+
+ return None
+ finally:
+ tool_res_msg = self._clean_tool_excution_content(tool_res_msg)
+ # Always add tool result to maintain message sequence integrity
+ # DashScope requires every tool_call to have a corresponding tool_result
+ # Don't delete assistant messages to avoid breaking message sequence
+ await self.memory.add(tool_res_msg)
+
+ def _clean_tool_excution_content(self, output_msg: Msg) -> Msg:
+ """Clean verbose tool outputs before printing and storing."""
+ for i, b in enumerate(output_msg.content):
+ if b["type"] == "tool_result":
+ for j, return_json in enumerate(b.get("output", [])):
+ if isinstance(return_json, dict) and "text" in return_json:
+ output_msg.content[i]["output"][j]["text"] = self._filter_execution_text( # type: ignore[index]
+ return_json["text"],
+ )
+ return output_msg
- self.register_instance_hook(
- "pre_reasoning",
- "browser_agent_observe_pre_reasoning",
- browser_agent_observe_pre_reasoning,
+ async def _task_decomposition_and_reformat(
+ self,
+ original_task: Msg | list[Msg] | None,
+ ) -> Msg:
+ """Decompose the original task into smaller tasks and reformat."""
+ if isinstance(original_task, list):
+ original_task = original_task[0]
+
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg(
+ name="user",
+ content=self.task_decomposition_prompt.format(
+ start_url=self.start_url,
+ browser_agent_sys_prompt=self.sys_prompt,
+ original_task=original_task.content,
+ ),
+ role="user",
+ ),
+ ],
)
+ res = await self.model(prompt)
+ decompose_text = ""
+ if self.model.stream:
+ async for content_chunk in res:
+ decompose_text = content_chunk.content[0]["text"]
+ else:
+ decompose_text = res.content[0]["text"]
+ logger.info(decompose_text)
- self.register_instance_hook(
- "post_reasoning",
- "browser_agent_remove_observation_post_reasoning",
- browser_agent_remove_observation_post_reasoning,
+ reflection_prompt_path = os.path.join(
+ _PROMPT_DIR,
+ "browser_agent_decompose_reflection_prompt.md",
)
+ with open(reflection_prompt_path, "r", encoding="utf-8") as fj:
+ decompose_reflection_prompt = fj.read()
- self.register_instance_hook(
- "post_acting",
- "browser_agent_post_acting_clean_content",
- browser_agent_post_acting_clean_content,
+ reflection_prompt = await self.formatter.format(
+ msgs=[
+ Msg(
+ name="user",
+ content=self.task_decomposition_prompt.format(
+ start_url=self.start_url,
+ browser_agent_sys_prompt=self.sys_prompt,
+ original_task=original_task.content,
+ ),
+ role="user",
+ ),
+ Msg(
+ name="system",
+ content=decompose_text,
+ role="system",
+ ),
+ Msg(
+ name="user",
+ content=decompose_reflection_prompt.format(
+ original_task=original_task.content,
+ subtasks=decompose_text,
+ ),
+ role="user",
+ ),
+ ],
)
+ reflection_res = await self.model(reflection_prompt)
+ reflection_text = ""
+ if self.model.stream:
+ async for content_chunk in reflection_res:
+ reflection_text = content_chunk.content[0]["text"]
+ else:
+ reflection_text = reflection_res.content[0]["text"]
+ logger.info(reflection_text)
- async def _navigate_to_start_url(self) -> None:
- """
- Navigate to the specified start URL using the browser_navigate tool.
+ subtasks: list[Any] = []
+ try:
+ if "```json" in reflection_text:
+ reflection_text = reflection_text.replace(
+ "```json",
+ "",
+ ).replace("```", "")
+ subtasks_json = json.loads(reflection_text)
+ subtasks = subtasks_json.get("REVISED_SUBTASKS", [])
+ if not isinstance(subtasks, list):
+ subtasks = []
+ except Exception:
+ subtasks = [original_task.content]
- This method is automatically called during the first interaction to
- navigate to the configured start URL. It executes the browser
- navigation tool and processes the response to ensure the
- initial page is loaded.
+ self.subtasks = subtasks
+ self.current_subtask_idx = 0
+ self.current_subtask = self.subtasks[0] if self.subtasks else None
+ # Prefer text content extraction if available
+ try:
+ self.original_task = original_task.get_text_content()
+ except Exception:
+ self.original_task = original_task.content
+
+ formatted_task = "The original task is: " + self.original_task + "\n"
+ try:
+ formatted_task += (
+ "The decomposed subtasks are: "
+ + json.dumps(self.subtasks)
+ + "\n"
+ )
+ formatted_task += (
+ "use the decomposed subtasks to complete the original task.\n"
+ )
+ except Exception:
+ pass
+ formatted_task = Msg(
+ name=original_task.name,
+ content=formatted_task,
+ role=original_task.role,
+ )
+ logger.info( # pylint: disable=W1203
+ f"The formatted task is: \n{formatted_task.content}",
+ )
+ return formatted_task
+
+ async def _navigate_to_start_url(self) -> None:
+ """Navigate to the start URL and clean up extra tabs."""
+ tool_call = ToolUseBlock(
+ id=str(uuid.uuid4()),
+ name="browser_tabs",
+ input={"action": "list"},
+ type="tool_use",
+ )
+ response = await self.toolkit.call_tool_function(tool_call)
+ response_text = ""
+ async for chunk in response:
+ # chunk.content might be a list[TextBlock]
+ if chunk.content and "text" in chunk.content[0]:
+ response_text = chunk.content[0]["text"]
+ tab_numbers = re.findall(r"- (\d+):", response_text)
+ for _ in tab_numbers[1:]:
+ tool_call = ToolUseBlock(
+ id=str(uuid.uuid4()),
+ name="browser_tabs",
+ input={"action": "close", "index": 0},
+ type="tool_use",
+ )
+ await self.toolkit.call_tool_function(tool_call)
- Returns:
- None
- """
tool_call = ToolUseBlock(
id=str(uuid.uuid4()),
type="tool_use",
name="browser_navigate",
input={"url": self.start_url},
)
-
- # Execute the navigation tool
await self.toolkit.call_tool_function(tool_call)
- async def _get_snapshot_in_text(self) -> Msg:
- """Capture a text-based snapshot of the current webpage content.
-
- This method uses the browser_snapshot tool to retrieve the current
- webpage content in text format, which is used during the reasoning
- phase to provide context about the current browser state.
-
- Returns:
- str: A text representation of the current webpage content,
- including elements, structure, and visible text.
-
- Note:
- This method is called automatically during the reasoning phase and
- provides essential context for decision-making about next actions.
- """
+ async def _get_snapshot_in_text(self) -> list[str]:
+ """Capture a text-based snapshot of the current webpage content."""
snapshot_tool_call = ToolUseBlock(
type="tool_use",
- id=str(uuid.uuid4()), # Generate a unique ID for the tool call
+ id=str(uuid.uuid4()),
name="browser_snapshot",
- input={}, # No parameters required for this tool
+ input={},
)
snapshot_response = await self.toolkit.call_tool_function(
snapshot_tool_call,
@@ -266,39 +692,11 @@ class BrowserAgent(ReActAgent):
snapshot_str = ""
async for chunk in snapshot_response:
snapshot_str = chunk.content[0]["text"]
-
- msg_observe = Msg(
- "user",
- content=[
- TextBlock(
- type="text",
- text=self.reasoning_prompt + "\n" + snapshot_str,
- ),
- ],
- role="user",
- )
-
- return msg_observe
+ snapshot_in_chunk = self._split_snapshot_by_chunk(snapshot_str)
+ return snapshot_in_chunk
async def _memory_summarizing(self) -> None:
- """Summarize the current memory content to prevent context overflow.
-
- This method is called periodically to condense the conversation history
- by generating a summary of progress and maintaining only essential
- information. It preserves the initial user question and creates a
- concise summary of what has been accomplished and what remains to be
- done.
-
- Returns:
- None
-
- Note:
- This method is automatically called every 10 iterations to manage
- memory usage and maintain context relevance. The summarization
- helps prevent token limit issues while preserving important task
- context.
- """
- # Extract the initial user question
+ """Summarize the current memory content to prevent context overflow."""
initial_question = None
memory_msgs = await self.memory.get_memory()
for msg in memory_msgs:
@@ -306,22 +704,18 @@ class BrowserAgent(ReActAgent):
initial_question = msg.content
break
- # Generate a summary of the current progress
hint_msg = Msg(
"user",
(
- "Summarize the current progress and outline the next steps "
- "for this task. Your summary should include:\n"
+ "Summarize the current progress and outline the next steps for this task. "
+ "Your summary should include:\n"
"1. What has been completed so far.\n"
"2. What key information has been found.\n"
"3. What remains to be done.\n"
- "Ensure that your summary is clear, concise, and t"
- "hat no tasks are repeated or skipped."
+ "Ensure that your summary is clear, concise, and that no tasks are repeated or skipped."
),
role="user",
)
-
- # Format the prompt for the model
prompt = await self.formatter.format(
msgs=[
Msg("system", self.sys_prompt, "system"),
@@ -329,20 +723,20 @@ class BrowserAgent(ReActAgent):
hint_msg,
],
)
-
- # Call the model to generate the summary
res = await self.model(prompt)
-
- # Handle response
summary_text = ""
+ print_msg = Msg(name=self.name, content=[], role="assistant")
if self.model.stream:
async for content_chunk in res:
summary_text = content_chunk.content[0]["text"]
+ print_msg.content = content_chunk.content
+ await self.print(print_msg, last=False)
else:
summary_text = res.content[0]["text"]
+ print_msg.content = [TextBlock(type="text", text=summary_text)]
+ await self.print(print_msg, last=True)
- # Update the memory with the summarized content
- summarized_memory = []
+ summarized_memory: list[Msg] = []
if initial_question:
summarized_memory.append(
Msg("user", initial_question, role="user"),
@@ -350,50 +744,514 @@ class BrowserAgent(ReActAgent):
summarized_memory.append(
Msg(self.name, summary_text, role="assistant"),
)
-
- # Clear and reload memory
await self.memory.clear()
- for msg in summarized_memory:
- await self.memory.add(msg)
+ for m in summarized_memory:
+ await self.memory.add(m)
+
+ async def _get_screenshot(self) -> Optional[str]:
+ """
+ Optionally take a screenshot of the current web page for multimodal prompts.
+ Returns base64-encoded PNG data if available, else None.
+ """
+ try:
+ # Prepare tool call for screenshot
+ tool_call = ToolUseBlock(
+ id=str(uuid.uuid4()),
+ name="browser_take_screenshot",
+ input={},
+ type="tool_use",
+ )
+ # Execute tool call via service toolkit
+ screenshot_response = await self.toolkit.call_tool_function(
+ tool_call,
+ )
+ # Extract image base64 from response
+ async for chunk in screenshot_response:
+ if (
+ chunk.content
+ and len(chunk.content) > 1
+ and "data" in chunk.content[1]
+ ):
+ image_data = chunk.content[1]["data"]
+ else:
+ image_data = None
+ except Exception:
+ image_data = None
+ return image_data
@staticmethod
def _filter_execution_text(
text: str,
keep_page_state: bool = False,
) -> str:
- """
- Filter and clean browser tool execution output to remove verbose
- content.
-
- This utility method removes unnecessary verbose content from browser
- tool responses, including JavaScript code blocks, console messages,
- and YAML content that can overwhelm the context window without
- providing useful information.
-
- Args:
- text (str):
- The raw execution text from browser tools that
- needs to be filtered.
- keep_page_state (bool, optional):
- Whether to preserve page state information
- including URL and YAML content. Defaults to False.
-
- Returns:
- str: The filtered execution text.
- """
+ """Filter and clean browser tool execution output to remove verbosity."""
if not keep_page_state:
- # Remove Page Snapshot and YAML content
text = re.sub(r"- Page URL.*", "", text, flags=re.DOTALL)
text = re.sub(r"```yaml.*?```", "", text, flags=re.DOTALL)
- # Remove JavaScript code blocks
- text = re.sub(r"```js.*?```", "", text, flags=re.DOTALL)
- # Remove console messages section that can be very verbose
- # (between "### New console messages" and "### Page state")
text = re.sub(
r"### New console messages.*?(?=### Page state)",
"",
text,
flags=re.DOTALL,
)
- # Trim leading/trailing whitespace
return text.strip()
+
+ def _split_snapshot_by_chunk(
+ self,
+ snapshot_str: str,
+ max_length: int = 80000,
+ ) -> list[str]:
+ self.snapshot_chunk_id = 0
+ return [
+ snapshot_str[i : i + max_length]
+ for i in range(0, len(snapshot_str), max_length)
+ ]
+
+ def observe_by_chunk(self, image_data: str | None = "") -> Msg:
+ """Create an observation message for chunk-based reasoning."""
+ reasoning_prompt = self.observe_reasoning_prompt.format(
+ previous_chunkwise_information=self.previous_chunkwise_information,
+ current_subtask=self.current_subtask,
+ i=self.snapshot_chunk_id + 1,
+ total_pages=len(self.snapshot_in_chunk),
+ chunk=self.snapshot_in_chunk[self.snapshot_chunk_id],
+ init_query=self.original_task,
+ )
+ content: list[Any] = [TextBlock(type="text", text=reasoning_prompt)]
+ if self._supports_multimodal():
+ if image_data:
+ image_block = ImageBlock(
+ type="image",
+ source=Base64Source(
+ type="base64",
+ media_type="image/png",
+ data=image_data,
+ ),
+ )
+ content.append(image_block)
+ observe_msg = Msg("user", content=content, role="user")
+ return observe_msg
+
+ async def browser_subtask_manager( # pylint: disable=R0912,R0915
+ self,
+ ) -> ToolResponse: # pylint: disable=R0912,R0915
+ """Validate and advance current subtask if completed."""
+ if (
+ not hasattr(self, "subtasks")
+ or not self.subtasks
+ or self.current_subtask is None
+ ):
+ self.current_subtask = self.original_task
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=(
+ f"Tool call Error. Cannot be executed. Current subtask remains: {self.current_subtask}"
+ ),
+ ),
+ ],
+ )
+ memory_content = await self.memory.get_memory()
+ sys_prompt = (
+ "You are an expert in subtask validation. \n"
+ "Given the following subtask and the agent's recent memory, strictly judge if the subtask is FULLY completed. \n"
+ "If yes, reply ONLY 'SUBTASK_COMPLETED'. If not, reply ONLY 'SUBTASK_NOT_COMPLETED'."
+ )
+ if len(self.snapshot_in_chunk) > 0:
+ user_prompt = (
+ f"Subtask: {self.current_subtask}\n"
+ f"Recent memory:\n{[str(m) for m in memory_content[-10:]]}\n"
+ f"Current page:\n{self.snapshot_in_chunk[0]}"
+ )
+ else:
+ user_prompt = (
+ f"Subtask: {self.current_subtask}\n"
+ f"Recent memory:\n{[str(m) for m in memory_content[-10:]]}\n"
+ )
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", sys_prompt, role="system"),
+ Msg("user", user_prompt, role="user"),
+ ],
+ )
+ response = await self.model(prompt)
+ response_text = ""
+ print_msg = Msg(name=self.name, content=[], role="assistant")
+ if self.model.stream:
+ async for chunk in response:
+ response_text += chunk.content[0]["text"]
+ print_msg.content = chunk.content
+ await self.print(print_msg, last=False)
+ else:
+ response_text = response.content[0]["text"]
+ print_msg.content = [TextBlock(type="text", text=response_text)]
+ await self.print(print_msg, last=True)
+
+ if "SUBTASK_COMPLETED" in response_text.strip().upper():
+ self.current_subtask_idx += 1
+ if self.current_subtask_idx < len(self.subtasks):
+ self.current_subtask = str(
+ self.subtasks[self.current_subtask_idx],
+ )
+ else:
+ self.current_subtask = None
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=(
+ "Tool call SUCCESS. Current subtask updates to: "
+ f"{self.current_subtask}"
+ ),
+ ),
+ ],
+ )
+ else:
+ revise_prompt_path = os.path.join(
+ _PROMPT_DIR,
+ "browser_agent_subtask_revise_prompt.md",
+ )
+ with open(revise_prompt_path, "r", encoding="utf-8") as fr:
+ revise_prompt = fr.read()
+ memory_content = await self.memory.get_memory()
+ user_prompt = revise_prompt.format(
+ memory=[str(m) for m in memory_content[-10:]],
+ subtasks=json.dumps(self.subtasks, ensure_ascii=False),
+ current_subtask=str(self.current_subtask),
+ original_task=str(self.original_task),
+ )
+ prompt = await self.formatter.format(
+ msgs=[Msg("user", user_prompt, role="user")],
+ )
+ response = await self.model(prompt)
+ if self.model.stream:
+ async for chunk in response:
+ revise_text = chunk.content[0]["text"]
+ else:
+ revise_text = response.content[0]["text"]
+ try:
+ if "```json" in revise_text:
+ revise_text = revise_text.replace("```json", "").replace(
+ "```",
+ "",
+ )
+ revise_json = json.loads(revise_text)
+ if_revised = revise_json.get("IF_REVISED")
+ if if_revised:
+ revised_subtasks = revise_json.get("REVISED_SUBTASKS", [])
+ if isinstance(revised_subtasks, list) and revised_subtasks:
+ self.subtasks = revised_subtasks
+ self.current_subtask_idx = 0
+ self.current_subtask = self.subtasks[0]
+ logger.info(
+ "Subtasks revised: %s, reason: %s",
+ self.subtasks,
+ revise_json.get("REASON", ""),
+ )
+ except Exception as e:
+ logger.warning("Failed to revise subtasks: %s", e)
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=(
+ "Tool call SUCCESS."
+ f" Current subtask remains: {self.current_subtask}"
+ ),
+ ),
+ ],
+ )
+
+ async def browser_generate_final_response(
+ self, # pylint: disable=W0613
+ **kwargs: Any, # pylint: disable=W0613
+ ) -> ToolResponse:
+ """Generate a final response; validate completion state."""
+ hint_msg = Msg(
+ "user",
+ _BROWSER_AGENT_SUMMARIZE_TASK_PROMPT,
+ role="user",
+ )
+
+ memory_msgs = await self.memory.get_memory()
+ memory_msgs_copy = copy.deepcopy(memory_msgs)
+ last_msg = memory_msgs_copy[-1]
+ last_msg.content = last_msg.get_content_blocks("text")
+ memory_msgs_copy[-1] = last_msg
+
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", self.sys_prompt, "system"),
+ *memory_msgs_copy,
+ hint_msg,
+ ],
+ )
+ try:
+ res = await self.model(prompt)
+ res_msg = Msg("assistant", [], "assistant")
+ if self.model.stream:
+ async for content_chunk in res:
+ summary_text = content_chunk.content[0]["text"]
+ else:
+ summary_text = res.content[0]["text"]
+ if self.model.stream:
+ summary_text = ""
+ async for content_chunk in res:
+ res_msg.content = content_chunk.content
+ summary_text = content_chunk.content[0]["text"]
+ await self.print(res_msg, False)
+ await self.print(res_msg, True)
+ else:
+ summary_text = res.content[0]["text"]
+ res_msg.content = summary_text
+ await self.print(res_msg, True)
+
+ # Validate finish status
+ finish_status = await self._validate_finish_status(summary_text)
+ logger.info( # pylint: disable=W1203
+ f"Finish status: {finish_status}",
+ ) # pylint: disable=W1203
+
+ if "BROWSER_AGENT_TASK_FINISHED" in finish_status:
+ structure_response = {
+ "task_done": True,
+ "subtask_progress_summary": summary_text,
+ "generated_files": {},
+ }
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text="Successfully generated response.",
+ ),
+ ],
+ metadata={
+ "success": True,
+ "structured_output": structure_response,
+ },
+ is_last=True,
+ )
+ else:
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=(
+ f"Here is a summary of current status:\n{summary_text}\nPlease continue.\n"
+ f"Following steps \n {finish_status}"
+ ),
+ ),
+ ],
+ metadata={"success": False, "structured_output": None},
+ is_last=True,
+ )
+ except Exception as e:
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=f"Tool call Error. Cannot be executed. {e}",
+ ),
+ ],
+ metadata={"success": False},
+ is_last=True,
+ )
+
+ async def image_understanding(
+ self,
+ object_description: str,
+ task: str,
+ ) -> ToolResponse:
+ """
+ Locate an element by description, take a focused screenshot, and solve a task using it.
+ """
+ sys_prompt = (
+ "You are a web page analysis expert. Given the following page snapshot and object description, "
+ "identify the exact element and its reference string (ref) that matches the description. "
+ 'Return ONLY a JSON object: {"element": <element description>, "ref": <ref string>}'
+ )
+ snapshot_chunks = await self._get_snapshot_in_text()
+ page_snapshot = snapshot_chunks[0] if snapshot_chunks else ""
+ user_prompt = f"Object description: {object_description}\nPage snapshot:\n{page_snapshot}"
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", sys_prompt, role="system"),
+ Msg("user", user_prompt, role="user"),
+ ],
+ )
+ res = await self.model(prompt)
+ if self.model.stream:
+ async for chunk in res:
+ model_text = chunk.content[0]["text"]
+ else:
+ model_text = res.content[0]["text"]
+ try:
+ if "```json" in model_text:
+ model_text = model_text.replace("```json", "").replace(
+ "```",
+ "",
+ )
+ element_info = json.loads(model_text)
+ element = element_info.get("element", "")
+ ref = element_info.get("ref", "")
+ except Exception:
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text="Failed to parse element/ref from model output.",
+ ),
+ ],
+ metadata={"success": False},
+ )
+
+ screenshot_tool_call = ToolUseBlock(
+ id=str(uuid.uuid4()),
+ name="browser_take_screenshot",
+ input={"element": element, "ref": ref},
+ type="tool_use",
+ )
+ screenshot_response = await self.toolkit.call_tool_function(
+ screenshot_tool_call,
+ )
+ image_data = None
+ async for chunk in screenshot_response:
+ if chunk.content and len(chunk.content) > 1:
+ block = chunk.content[1]
+ if "data" in block:
+ image_data = block["data"]
+ elif "source" in block and "data" in block["source"]:
+ image_data = block["source"]["data"]
+
+ sys_prompt_task = (
+ "You are a web automation expert. Given the object description, screenshot, and page context, "
+ "solve the following task. Return ONLY the answer as plain text."
+ )
+ content_blocks: list[Any] = [
+ TextBlock(
+ type="text",
+ text=f"Object description: {object_description}\nTask: {task}\nPage snapshot:\n{page_snapshot}",
+ ),
+ ]
+ if image_data:
+ image_block = ImageBlock(
+ type="image",
+ source=Base64Source(
+ type="base64",
+ media_type="image/png",
+ data=image_data,
+ ),
+ )
+ content_blocks.append(image_block)
+ prompt_task = await self.formatter.format(
+ msgs=[
+ Msg("system", sys_prompt_task, role="system"),
+ Msg("user", content_blocks, role="user"),
+ ],
+ )
+ res_task = await self.model(prompt_task)
+ if self.model.stream:
+ async for chunk in res_task:
+ answer_text = chunk.content[0]["text"]
+ else:
+ answer_text = res_task.content[0]["text"]
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=(
+ f"Screenshot taken for element: {element}\nref: {ref}\n"
+ f"Task solution: {answer_text}"
+ ),
+ ),
+ ],
+ )
+
+ async def _validate_finish_status(self, summary: str) -> str:
+ """Validate if the agent has completed its task based on the summary."""
+ sys_prompt = (
+ "You are an expert in task validation. "
+ "Your job is to determine if the agent has completed its task"
+ " based on the provided summary. If the summary is `NO_ANSWER`, this task "
+ "is not over unless the task is determined as definitely not completed. "
+ "If finished, strictly reply "
+ '"BROWSER_AGENT_TASK_FINISHED" and your reason, otherwise return the remaining '
+ "tasks or next steps."
+ )
+ initial_question = None
+ memory_msgs = await self.memory.get_memory()
+ for msg in memory_msgs:
+ if msg.role == "user":
+ initial_question = msg.content
+ break
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", sys_prompt, role="system"),
+ Msg(
+ "user",
+ content=(
+ "The initial task is to solve the following question: "
+ f"{initial_question} \n "
+ f"Here is a summary of current task completion process, please evaluate the task finish status.\n"
+ + summary
+ ),
+ role="user",
+ ),
+ ],
+ )
+ res = await self.model(prompt)
+ response_text = ""
+ if self.model.stream:
+ async for content_chunk in res:
+ response_text = content_chunk.content[0]["text"]
+ else:
+ response_text = res.content[0]["text"]
+ return response_text
+
+ def _register_skill_tool(
+ self,
+ skill_func: Any,
+ ) -> None:
+ """Bind the browser agent to a skill function and register it as a tool."""
+ if asyncio.iscoroutinefunction(skill_func):
+
+ @wraps(skill_func)
+ async def tool(*args: Any, **kwargs: Any) -> Any:
+ return await skill_func(
+ browser_agent=self,
+ *args,
+ **kwargs,
+ )
+
+ else:
+
+ @wraps(skill_func)
+ async def tool(*args: Any, **kwargs: Any) -> Any:
+ return skill_func(
+ browser_agent=self,
+ *args,
+ **kwargs,
+ )
+
+ original_signature = inspect.signature(skill_func)
+ parameters = list(original_signature.parameters.values())
+ if parameters and parameters[0].name == "browser_agent":
+ parameters = parameters[1:]
+ try:
+ tool.__signature__ = original_signature.replace(
+ parameters=parameters,
+ )
+ except ValueError:
+ pass
+
+ self.toolkit.register_tool_function(tool)
+
+ def _supports_multimodal(self) -> bool:
+ """Check if the model supports multimodal input (images/videos)."""
+ return (
+ self.model.model_name.startswith("qvq")
+ or "-vl" in self.model.model_name
+ or "4o" in self.model.model_name
+ or "gpt-5" in self.model.model_name
+ )
diff --git a/examples/agent/browser_agent/build_in_helper/_file_download.py b/examples/agent/browser_agent/build_in_helper/_file_download.py
new file mode 100644
index 000000000..6c758ac43
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_helper/_file_download.py
@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+"""Standalone file download skill for the browser agent."""
+# flake8: noqa: E501
+# pylint: disable=W0212,W0107,too-many-lines,C0301
+
+from __future__ import annotations
+import os
+import copy
+from typing import Any
+from pydantic import BaseModel
+
+
+from agentscope.memory import InMemoryMemory
+from agentscope.message import Msg, TextBlock
+from agentscope.tool import ToolResponse
+from agentscope.agent import ReActAgent
+
+
+_CURRENT_DIR = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), os.pardir),
+)
+
+with open(
+ os.path.join(
+ _CURRENT_DIR,
+ "build_in_prompt/browser_agent_file_download_sys_prompt.md",
+ ),
+ "r",
+ encoding="utf-8",
+) as f:
+ _FILE_DOWNLOAD_AGENT_SYS_PROMPT = f.read()
+
+
+class EmptyModel(BaseModel):
+ """Empty structured model for default structured output requirement."""
+
+ pass
+
+
+class FileDownloadAgent(ReActAgent):
+ """Lightweight helper agent that downloads files"""
+
+ finish_function_name: str = "file_download_final_response"
+
+ def __init__(
+ self,
+ browser_agent: Any,
+ sys_prompt: str = _FILE_DOWNLOAD_AGENT_SYS_PROMPT,
+ max_iters: int = 15,
+ ) -> None:
+ name = (
+ f"{getattr(browser_agent, 'name', 'browser_agent')}_file_download"
+ )
+ super().__init__(
+ name=name,
+ sys_prompt=sys_prompt,
+ model=browser_agent.model,
+ formatter=browser_agent.formatter,
+ memory=InMemoryMemory(),
+ toolkit=browser_agent.toolkit,
+ max_iters=max_iters,
+ )
+ # Register the finish function
+ self.toolkit.register_tool_function(self.file_download_final_response)
+ # Remove conflicting tool functions if they exist
+ if hasattr(self.toolkit, "remove_tool_function"):
+ try:
+ self.toolkit.remove_tool_function("browser_pdf_save")
+ except Exception:
+ # Tool may not exist, ignore removal errors
+ pass
+ try:
+ self.toolkit.remove_tool_function("file_download")
+ except Exception:
+ # Tool may not exist, ignore removal errors
+ pass
+
+ async def file_download_final_response(
+ self, # pylint: disable=W0613
+ **kwargs: Any, # pylint: disable=W0613
+ ) -> ToolResponse:
+ """Summarize the file download outcome."""
+ hint_msg = Msg(
+ "user",
+ (
+ "Provide a concise summary of the file download attempt.\n"
+ "Highlight these items:\n"
+ "0. The original request\n"
+ "1. The element(s) interacted with and actions taken\n"
+ "2. The download status or any issues encountered\n"
+ "3. Any follow-up recommendations or next steps\n"
+ ),
+ role="user",
+ )
+
+ memory_msgs = await self.memory.get_memory()
+ memory_msgs_copy = copy.deepcopy(memory_msgs)
+ if memory_msgs_copy:
+ last_msg = memory_msgs_copy[-1]
+ last_msg.content = last_msg.get_content_blocks("text")
+ memory_msgs_copy[-1] = last_msg
+
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", self.sys_prompt, "system"),
+ *memory_msgs_copy,
+ hint_msg,
+ ],
+ )
+
+ res = await self.model(prompt)
+
+ if self.model.stream:
+ summary_text = ""
+ async for chunk in res:
+ summary_text = chunk.content[0]["text"]
+ else:
+ summary_text = res.content[0]["text"]
+
+ summary_text = summary_text or "No summary generated."
+
+ structure_response = {
+ "task_done": True,
+ "subtask_progress_summary": summary_text,
+ "generated_files": {},
+ }
+
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text="File download summary generated. " + summary_text,
+ ),
+ ],
+ metadata={
+ "success": True,
+ "structured_output": structure_response,
+ },
+ is_last=True,
+ )
+
+
+def _build_initial_instruction(
+ target_description: str,
+ snapshot_text: str,
+) -> str:
+ """Compose the initial instruction for the helper agent."""
+ return (
+ "You must locate and trigger the download for the requested file.\n\n"
+ "Target description provided by the user:\n"
+ f"{target_description}\n\n"
+ "Latest snapshot captured prior to your run:\n"
+ f"{snapshot_text}\n\n"
+ "Follow the sys prompt guidance, think step-by-step, and verify that "
+ "the download action succeeded. If the download cannot be completed, "
+ "explain why in the final summary."
+ )
+
+
+async def file_download(
+ browser_agent: Any,
+ target_description: str,
+) -> ToolResponse:
+ """
+ Download the target file. The current page should
+ contain download-related element.
+
+ Args:
+ target_description (str): The description of the
+ target file to download.
+
+ Returns:
+ ToolResponse: A structured response containing
+ the download directory.
+ """
+ try:
+ snapshot_chunks = await browser_agent._get_snapshot_in_text()
+ except Exception as exc: # pylint: disable=broad-except
+ snapshot_chunks = []
+ snapshot_error = str(exc)
+ else:
+ snapshot_error = ""
+
+ snapshot_text = "\n\n---\n\n".join(snapshot_chunks)
+ if snapshot_error and not snapshot_text:
+ snapshot_text = f"[Snapshot failed: {snapshot_error}]"
+
+ sub_agent = FileDownloadAgent(browser_agent)
+ instruction = _build_initial_instruction(
+ target_description=target_description,
+ snapshot_text=snapshot_text,
+ )
+
+ init_msg = Msg(
+ name="user",
+ role="user",
+ content=instruction,
+ )
+
+ try:
+ sub_agent_response_msg = await sub_agent.reply(
+ init_msg,
+ structured_model=EmptyModel,
+ )
+
+ text_content = ""
+ if sub_agent_response_msg.content:
+ first_block = sub_agent_response_msg.content[0]
+ if isinstance(first_block, dict):
+ text_content = first_block.get("text") or ""
+ else:
+ text_content = getattr(first_block, "text", "") or ""
+
+ if not text_content:
+ text_content = (
+ "File download agent finished without a textual summary."
+ )
+
+ return ToolResponse(
+ metadata=sub_agent_response_msg.metadata,
+ content=[
+ TextBlock(
+ type="text",
+ text=text_content,
+ ),
+ ],
+ )
+ except Exception as exc: # pylint: disable=broad-except
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=f"Tool call Error. Cannot be executed. {exc}",
+ ),
+ ],
+ metadata={"success": False},
+ is_last=True,
+ )
diff --git a/examples/agent/browser_agent/build_in_helper/_form_filling.py b/examples/agent/browser_agent/build_in_helper/_form_filling.py
new file mode 100644
index 000000000..6f216a05e
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_helper/_form_filling.py
@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+"""Standalone form filling skill for the browser agent."""
+# flake8: noqa: E501
+# pylint: disable=W0212,W0107,too-many-lines,C0301
+
+from __future__ import annotations
+import os
+import copy
+from typing import Any
+from pydantic import BaseModel
+
+from agentscope.memory import InMemoryMemory
+from agentscope.message import Msg, TextBlock
+from agentscope.tool import ToolResponse
+from agentscope.agent import ReActAgent
+
+_CURRENT_DIR = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), os.pardir),
+)
+
+with open(
+ os.path.join(
+ _CURRENT_DIR,
+ "build_in_prompt/browser_agent_form_filling_sys_prompt.md",
+ ),
+ "r",
+ encoding="utf-8",
+) as f:
+ _FORM_FILL_AGENT_SYS_PROMPT = f.read()
+
+
+class EmptyModel(BaseModel):
+ """Empty structured model for default structured output requirement."""
+
+ pass
+
+
+class FormFillingAgent(ReActAgent):
+ """Lightweight helper agent that fills forms."""
+
+ finish_function_name: str = "form_filling_final_response"
+
+ def __init__(
+ self,
+ browser_agent: Any,
+ sys_prompt: str = _FORM_FILL_AGENT_SYS_PROMPT,
+ max_iters: int = 20,
+ ) -> None:
+ name = f"{getattr(browser_agent, 'name', 'browser_agent')}_form_fill"
+ super().__init__(
+ name=name,
+ sys_prompt=sys_prompt,
+ model=browser_agent.model,
+ formatter=browser_agent.formatter,
+ memory=InMemoryMemory(),
+ toolkit=browser_agent.toolkit,
+ max_iters=max_iters,
+ )
+ # Register the finish function
+ self.toolkit.register_tool_function(self.form_filling_final_response)
+
+ async def form_filling_final_response(
+ self, # pylint: disable=W0613
+ **kwargs: Any, # pylint: disable=W0613
+ ) -> ToolResponse:
+ """Summarize the form filling outcome."""
+ hint_msg = Msg(
+ "user",
+ (
+ "Provide a concise summary of the completed form "
+ "filling task.\n"
+ "Highlight these items:\n"
+ "0. The original task/query\n"
+ "1. Which fields were filled/selected and their final values\n"
+ "2. Any important observations or follow-up notes\n"
+ "3. Confirmation that if the task is complete\n\n"
+ ),
+ role="user",
+ )
+
+ memory_msgs = await self.memory.get_memory()
+ memory_msgs_copy = copy.deepcopy(memory_msgs)
+ last_msg = memory_msgs_copy[-1]
+ # check if the last message has tool call, if so clean the content
+
+ last_msg.content = last_msg.get_content_blocks("text")
+ memory_msgs_copy[-1] = last_msg
+
+ prompt = await self.formatter.format(
+ msgs=[
+ Msg("system", self.sys_prompt, "system"),
+ *memory_msgs_copy,
+ hint_msg,
+ ],
+ )
+
+ res = await self.model(prompt)
+
+ if self.model.stream:
+ summary_text = ""
+ async for chunk in res:
+ summary_text = chunk.content[0]["text"]
+ else:
+ summary_text = res.content[0]["text"]
+
+ structure_response = {
+ "task_done": True,
+ "subtask_progress_summary": summary_text,
+ "generated_files": {},
+ }
+
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text="Form filling summary generated. " + summary_text,
+ ),
+ ],
+ metadata={
+ "success": True,
+ "structured_output": structure_response,
+ },
+ is_last=True,
+ )
+
+
+def _build_initial_instruction(
+ fill_information: str,
+ snapshot_text: str,
+) -> str:
+ """Compose the initial instruction fed to the helper agent."""
+ return (
+ "You must complete the web form using the information "
+ "provided below.\n\nFill instructions (plain text from the user):\n"
+ f"{fill_information}\n\n"
+ "Latest snapshot captured prior to your run:\n"
+ f"{snapshot_text}\n\n"
+ )
+
+
+async def form_filling(
+ browser_agent: Any,
+ fill_information: str,
+) -> ToolResponse:
+ """
+ Fill in a web form according to plain-text instructions.
+
+ Args:
+ fill_information (str):
+ Plain-text description of the values that
+ must be entered into the form,
+ including any submission requirements.
+
+ Returns:
+ ToolResponse: Summary of the helper agent execution and status.
+ """
+ try:
+ snapshot_chunks = (
+ await browser_agent._get_snapshot_in_text()
+ ) # pylint: disable=protected-access
+ except Exception as exc: # pylint: disable=broad-except
+ snapshot_chunks = []
+ snapshot_error = str(exc)
+ else:
+ snapshot_error = ""
+
+ snapshot_text = "\n\n---\n\n".join(snapshot_chunks)
+ if snapshot_error and not snapshot_text:
+ snapshot_text = f"[Snapshot failed: {snapshot_error}]"
+
+ sub_agent = FormFillingAgent(browser_agent)
+ instruction = _build_initial_instruction(
+ fill_information=fill_information,
+ snapshot_text=snapshot_text,
+ )
+
+ init_msg = Msg(
+ name="user",
+ role="user",
+ content=instruction,
+ )
+
+ try:
+ sub_agent_response_msg = await sub_agent.reply(
+ init_msg,
+ structured_model=EmptyModel,
+ )
+
+ text_content = ""
+ if sub_agent_response_msg.content:
+ first_block = sub_agent_response_msg.content[0]
+ if isinstance(first_block, dict):
+ text_content = first_block.get("text") or ""
+ else:
+ text_content = getattr(first_block, "text", "") or ""
+
+ if not text_content:
+ text_content = (
+ "Form filling agent finished without a textual summary."
+ )
+
+ return ToolResponse(
+ metadata=sub_agent_response_msg.metadata,
+ content=[
+ TextBlock(
+ type="text",
+ text=text_content,
+ ),
+ ],
+ )
+ except Exception as e:
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=f"Tool call Error. Cannot be executed. {e}",
+ ),
+ ],
+ metadata={"success": False},
+ is_last=True,
+ )
diff --git a/examples/agent/browser_agent/build_in_helper/_image_understanding.py b/examples/agent/browser_agent/build_in_helper/_image_understanding.py
new file mode 100644
index 000000000..e1456684b
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_helper/_image_understanding.py
@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+"""Standalone image understanding skill for the browser agent."""
+# flake8: noqa: E501
+# pylint: disable=W0212
+# pylint: disable=too-many-lines
+# pylint: disable=C0301
+from __future__ import annotations
+
+import json
+import uuid
+from typing import Any
+
+from agentscope.message import (
+ Base64Source,
+ ImageBlock,
+ Msg,
+ TextBlock,
+ ToolUseBlock,
+)
+from agentscope.tool import ToolResponse
+
+
+async def image_understanding(
+ browser_agent: Any,
+ object_description: str,
+ task: str,
+) -> ToolResponse:
+ """
+ Locate an element and solve a visual task on the current webpage.
+
+ Args:
+ object_description (str): The description of the object to locate.
+ task (str): The specific task or question to solve about the image
+ (e.g., description, object detection, activity recognition, or
+ answering a question about the image's content).
+
+ Returns:
+ ToolResponse: A structured response containing the answer to
+ the specified task based on the image content.
+ """
+
+ sys_prompt = (
+ "You are a web page analysis expert. Given the following page "
+ "snapshot and object description, "
+ "identify the exact element and its reference string (ref) "
+ "that matches the description. "
+ "Return ONLY a JSON object: "
+ '{"element": <element description>, "ref": <ref string>}'
+ )
+
+ snapshot_chunks = (
+ await browser_agent._get_snapshot_in_text() # noqa: E501 # pylint: disable=protected-access
+ )
+ page_snapshot = snapshot_chunks[0] if snapshot_chunks else ""
+ user_prompt = (
+ f"Object description: {object_description}\n"
+ f"Page snapshot:\n{page_snapshot}"
+ )
+
+ prompt = await browser_agent.formatter.format(
+ msgs=[
+ Msg("system", sys_prompt, role="system"),
+ Msg("user", user_prompt, role="user"),
+ ],
+ )
+ res = await browser_agent.model(prompt)
+ if browser_agent.model.stream:
+ async for chunk in res:
+ model_text = chunk.content[0]["text"]
+ else:
+ model_text = res.content[0]["text"]
+
+ try:
+ if "```json" in model_text:
+ model_text = model_text.replace("```json", "").replace(
+ "```",
+ "",
+ )
+ element_info = json.loads(model_text)
+ element = element_info.get("element", "")
+ ref = element_info.get("ref", "")
+ except Exception:
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text="Failed to parse element/ref from model output.",
+ ),
+ ],
+ metadata={"success": False},
+ )
+
+ screenshot_tool_call = ToolUseBlock(
+ id=str(uuid.uuid4()),
+ name="browser_take_screenshot",
+ input={"element": element, "ref": ref},
+ type="tool_use",
+ )
+ screenshot_response = await browser_agent.toolkit.call_tool_function(
+ screenshot_tool_call,
+ )
+ image_data = None
+ async for chunk in screenshot_response:
+ if (
+ chunk.content
+ and len(chunk.content) > 1
+ and "data" in chunk.content[1]
+ ):
+ image_data = chunk.content[1]["data"]
+
+ sys_prompt_task = (
+ "You are a web automation expert. "
+ "Given the object description, screenshot, and page context, "
+ "solve the following task. Return ONLY the answer as plain text."
+ )
+ content_blocks = [
+ TextBlock(
+ type="text",
+ text=(
+ "Object description: "
+ f"{object_description}\nTask: {task}\n"
+ f"Page snapshot:\n{page_snapshot}"
+ ),
+ ),
+ ]
+
+ if image_data:
+ image_block = ImageBlock(
+ type="image",
+ source=Base64Source(
+ type="base64",
+ media_type="image/png",
+ data=image_data,
+ ),
+ )
+ content_blocks.append(image_block)
+
+ prompt_task = await browser_agent.formatter.format(
+ msgs=[
+ Msg("system", sys_prompt_task, role="system"),
+ Msg("user", content_blocks, role="user"),
+ ],
+ )
+ res_task = await browser_agent.model(prompt_task)
+ if browser_agent.model.stream:
+ async for chunk in res_task:
+ answer_text = chunk.content[0]["text"]
+ else:
+ answer_text = res_task.content[0]["text"]
+
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=(
+ f"Screenshot taken for element: {element}\nref: {ref}\n"
+ f"Task solution: {answer_text}"
+ ),
+ ),
+ ],
+ )
diff --git a/examples/agent/browser_agent/build_in_helper/_video_understanding.py b/examples/agent/browser_agent/build_in_helper/_video_understanding.py
new file mode 100644
index 000000000..7a96c0fd0
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_helper/_video_understanding.py
@@ -0,0 +1,330 @@
+# -*- coding: utf-8 -*-
+"""Standalone video understanding skill for the browser agent."""
+# flake8: noqa: E501
+# pylint: disable=W0212
+# pylint: disable=too-many-lines
+# pylint: disable=C0301
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import tempfile
+import uuid
+from base64 import b64encode
+from pathlib import Path
+from typing import Any, List, Optional
+
+from agentscope.message import (
+ Base64Source,
+ ImageBlock,
+ Msg,
+ TextBlock,
+)
+from agentscope.tool import ToolResponse
+
+
+async def video_understanding(
+ browser_agent: Any,
+ video_path: str,
+ task: str,
+) -> ToolResponse:
+ """
+ Perform video understanding on the provided video file.
+
+ Args:
+ video_path (str): The path to the video file to analyze.
+ task (str): The specific task or question to solve about
+ the video (e.g., summary, object detection, activity recognition,
+ or answering a question about the video's content).
+
+ Returns:
+ ToolResponse: A structured response containing the answer
+ to the specified task based on the video content.
+ """
+
+ workdir = _prepare_workdir(browser_agent)
+ try:
+ frames_dir = os.path.join(workdir, "frames")
+ frames = extract_frames(video_path, frames_dir)
+ except Exception as exc:
+ return _error_response(f"Failed to extract frames: {exc}")
+
+ audio_path = os.path.join(
+ workdir,
+ f"audio_{getattr(browser_agent, 'iter_n', 0)}.wav",
+ )
+ try:
+ extract_audio(video_path, audio_path)
+ except Exception as exc:
+ return _error_response(f"Failed to extract audio: {exc}")
+
+ try:
+ transcript = audio2text(audio_path)
+ except Exception as exc:
+ return _error_response(f"Failed to transcribe audio: {exc}")
+
+ sys_prompt = (
+ "You are a web video analysis expert. "
+ "Given the following video frames and audio transcript, "
+ "analyze the content and provide a solution to the task. "
+ 'Return ONLY a JSON object: {"answer": <your answer>}'
+ )
+
+ content_blocks = _build_multimodal_blocks(frames, transcript, task)
+
+ prompt = await browser_agent.formatter.format(
+ msgs=[
+ Msg("system", sys_prompt, role="system"),
+ Msg("user", content_blocks, role="user"),
+ ],
+ )
+
+ res = await browser_agent.model(prompt)
+ if browser_agent.model.stream:
+ async for chunk in res:
+ model_text = chunk.content[0]["text"]
+ else:
+ model_text = res.content[0]["text"]
+
+ try:
+ if "```json" in model_text:
+ model_text = model_text.replace("```json", "").replace(
+ "```",
+ "",
+ )
+ answer_info = json.loads(model_text)
+ answer = answer_info.get("answer", "")
+ except Exception: # pylint: disable=broad-except
+ return _error_response("Failed to parse answer from model output.")
+
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=(
+ "Video analysis completed.\n" f"Task solution: {answer}"
+ ),
+ ),
+ ],
+ )
+
+
+def audio2text(audio_path: str) -> str:
+ """Convert audio to text using DashScope ASR."""
+
+ try: # Local import to avoid hard dependency when unused.
+ from dashscope.audio.asr import Recognition, RecognitionCallback
+ except ImportError as exc:
+ raise RuntimeError(
+ "dashscope.audio is required for audio transcription.",
+ ) from exc
+
+ callback = RecognitionCallback()
+ recognizer = Recognition(
+ model="paraformer-realtime-v1",
+ format="wav",
+ sample_rate=16000,
+ callback=callback,
+ )
+
+ result = recognizer.call(audio_path)
+ sentences = result.get("output", {}).get("sentence", [])
+ return " ".join(sentence.get("text", "") for sentence in sentences)
+
+
+def extract_frames(
+ video_path: str,
+ output_dir: str,
+ max_frames: int = 16,
+) -> List[str]:
+ """Extract representative frames using ffmpeg (no OpenCV dependency)."""
+
+ if max_frames <= 0:
+ raise ValueError("max_frames must be greater than zero.")
+
+ if not os.path.exists(video_path):
+ raise FileNotFoundError(f"Video path not found: {video_path}")
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Clean up previous generated frames
+ for existing in Path(output_dir).glob("frame_*.jpg"):
+ try:
+ existing.unlink()
+ except OSError:
+ # Ignore errors during cleanup;
+ # leftover files will be overwritten or do not affect frame extraction
+ pass
+
+ duration = _probe_video_duration(video_path)
+ if duration and duration > 0:
+ fps = max_frames / duration
+ else:
+ fps = 1.0
+
+ fps = max(min(fps, 30.0), 0.1)
+
+ command = [
+ "ffmpeg",
+ "-y",
+ "-i",
+ video_path,
+ "-vf",
+ f"fps={fps:.5f}",
+ "-frames:v",
+ str(max_frames),
+ os.path.join(output_dir, "frame_%04d.jpg"),
+ ]
+
+ try:
+ subprocess.run(
+ command,
+ check=True,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ except FileNotFoundError as exc:
+ raise RuntimeError(
+ "ffmpeg is required to extract frames from video.",
+ ) from exc
+
+ frame_files = sorted(
+ str(path) for path in Path(output_dir).glob("frame_*.jpg")
+ )
+
+ if not frame_files:
+ raise RuntimeError("No frames could be extracted from the video.")
+
+ return frame_files
+
+
+def extract_audio(video_path: str, audio_path: str) -> str:
+ """Extract audio track with ffmpeg and save as wav."""
+
+ if not os.path.exists(video_path):
+ raise FileNotFoundError(f"Video path not found: {video_path}")
+
+ os.makedirs(os.path.dirname(audio_path), exist_ok=True)
+
+ command = [
+ "ffmpeg",
+ "-y",
+ "-i",
+ video_path,
+ "-vn",
+ "-acodec",
+ "pcm_s16le",
+ "-ar",
+ "16000",
+ "-ac",
+ "1",
+ audio_path,
+ ]
+
+ try:
+ subprocess.run(
+ command,
+ check=True,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ except FileNotFoundError as exc:
+ raise RuntimeError(
+ "ffmpeg is required to extract audio from video.",
+ ) from exc
+
+ return audio_path
+
+
+def _probe_video_duration(video_path: str) -> Optional[float]:
+ """Return the video duration in seconds using ffprobe, if available."""
+
+ command = [
+ "ffprobe",
+ "-v",
+ "error",
+ "-show_entries",
+ "format=duration",
+ "-of",
+ "default=noprint_wrappers=1:nokey=1",
+ video_path,
+ ]
+
+ try:
+ result = subprocess.run(
+ command,
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ text=True,
+ )
+ duration_str = result.stdout.strip()
+ if duration_str:
+ return float(duration_str)
+ except (FileNotFoundError, ValueError, subprocess.CalledProcessError):
+ return None
+
+ return None
+
+
+def _build_multimodal_blocks(
+ frames: List[str],
+ transcript: str,
+ task: str,
+) -> list:
+ """Construct multimodal content blocks for the model input."""
+
+ blocks: list = []
+ for frame_path in frames:
+ with open(frame_path, "rb") as file:
+ data = b64encode(file.read()).decode("ascii")
+ image_block = ImageBlock(
+ type="image",
+ source=Base64Source(
+ type="base64",
+ media_type="image/jpeg",
+ data=data,
+ ),
+ )
+ blocks.append(image_block)
+
+ blocks.append(
+ TextBlock(
+ type="text",
+ text=f"Audio transcript:\n{transcript}",
+ ),
+ )
+ blocks.append(
+ TextBlock(
+ type="text",
+ text=f"The task to be solved is: {task}",
+ ),
+ )
+ return blocks
+
+
+def _prepare_workdir(browser_agent: Any) -> str:
+ """Prepare a working directory for intermediate artifacts."""
+
+ base_dir = getattr(browser_agent, "state_saving_dir", None)
+ if not base_dir:
+ base_dir = tempfile.gettempdir()
+
+ workdir = os.path.join(base_dir, "video_understanding", uuid.uuid4().hex)
+ os.makedirs(workdir, exist_ok=True)
+ return workdir
+
+
+def _error_response(message: str) -> ToolResponse:
+ """Create a standardized error response."""
+
+ return ToolResponse(
+ content=[
+ TextBlock(
+ type="text",
+ text=message,
+ ),
+ ],
+ metadata={"success": False},
+ )
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_decompose_reflection_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_decompose_reflection_prompt.md
new file mode 100644
index 000000000..5947911f5
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_decompose_reflection_prompt.md
@@ -0,0 +1,28 @@
+Your role is to assess and optimize task decomposition for browser automation. Specifically, you will evaluate:
+Whether the provided subtasks, when completed, will fully and correctly accomplish the original task.
+Whether the original task requires decomposition. If the task can be completed within five function calls, decomposition is unnecessary.
+
+
+Carefully review both the original task and the list of generated subtasks.
+
+- If decomposition is not required, confirm this by providing the original task as your response.
+- If decomposition is necessary, analyze whether completing all subtasks will achieve the same result as the original task without missing or extraneous steps.
+- "If" statement should not be used in subtask descriptions. All statements should be direct and assertive.
+- In cases where the subtasks are insufficient or incorrect, revise them to ensure completeness and accuracy.
+
+Format your response as the following JSON:
+{{
+ "DECOMPOSITION": true/false, // true if decomposition is necessary, false otherwise
+ "SUFFICIENT": true/false/na, // if decomposition is necessary, true if the subtasks are sufficient, false otherwise, na if decomposition is not necessary.
+ "REASON": "Briefly explain your reasoning.",
+ "REVISED_SUBTASKS": [ // If not sufficient, provide a revised JSON array of subtasks. If sufficient, repeat the original subtasks. If decomposition is not necessary, provide the original task.
+ "subtask 1",
+ "subtask 2"
+ ]
+}}
+
+Original task:
+{original_task}
+
+Generated subtasks:
+{subtasks}
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_file_download_sys_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_file_download_sys_prompt.md
new file mode 100644
index 000000000..10b61834d
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_file_download_sys_prompt.md
@@ -0,0 +1,9 @@
+You are a meticulous web automation specialist. Study the provided page snapshot carefully before acting.
+Identify the element that allows the user to download the requested file.
+Verify every locator prior to interaction.
+
+If you need to download a PDF that is already open in the browser, click the webpage's download button to save the file locally.
+
+Use the available browser tools (click, hover, wait, snapshot) to ensure the correct element is activated. Request fresh snapshots after meaningful changes when needed.
+
+Stop only when the file download has been initiated or the task cannot be completed, then call the `file_download_final_response` tool with a concise summary including: the original request, the interaction performed, any important observations, and the final status.
\ No newline at end of file
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_form_filling_sys_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_form_filling_sys_prompt.md
new file mode 100644
index 000000000..d2f2baa1d
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_form_filling_sys_prompt.md
@@ -0,0 +1,17 @@
+You are a specialized web form operator. Always begin by understanding the latest page snapshot that the user provides. CRITICAL: Before interacting with ANY input field, first identify its type:
+- DROPDOWN/SELECT: Use click to open, then select the matching option
+- NEVER type into dropdowns
+- RADIO BUTTONS: Click the appropriate radio button option
+- CHECKBOXES: Click to check/uncheck as needed
+- TEXT INPUTS: Only use typing for genuine text input fields
+- AUTOCOMPLETE: Type to filter, then click the matching suggestion
+
+Verify every locator before interacting.
+Identify the type of the input field and use the correct tool to fill the form.
+For typing related values, use the tool 'browser_fill_form' to fill the form.
+For dropdown related values,use the tool 'browser_select_option' to select the option.
+Some dropdowns may have a search input. If so, use the search input to find the matching option and select it.
+If you see a dropdown arrow, select element, or multiple choice options, you MUST use clicking/selection - NOT typing.
+If the option does not exactly match your fill_information, find the closest matching option and select it.
+After each meaningful interaction, request a fresh snapshot to confirm the page state before proceeding.
+Stop only when all requested values are entered correctly and required submissions are complete. Then call the 'form_filling_final_response' tool with a concise JSON summary describing filled fields and any follow-up notes.
\ No newline at end of file
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_observe_reasoning_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_observe_reasoning_prompt.md
new file mode 100644
index 000000000..d99f81f0f
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_observe_reasoning_prompt.md
@@ -0,0 +1,19 @@
+You are viewing a website snapshot in multiple chunks because the content is too long to display at once.
+Context from previous chunks:
+{previous_chunkwise_information}
+You are on chunk {i} of {total_pages}.
+Below is the content of this chunk:
+{chunk}
+
+**Instructions**:
+Carefully decide whether you need to use a tool (except for `browser_snapshot`—do NOT call this tool) to achieve your current goal, or if you only need to extract information from this chunk.
+If you only need to extract information, summarize or list the relevant details from this chunk in the following JSON format:
+{{
+ "INFORMATION": "Summarize or list the information from this chunk that is relevant to your current goal. If nothing is found, write 'None'.",
+ "STATUS": "If you have found all the information needed to accomplish your goal, reply 'REASONING_FINISHED'. Otherwise, reply 'CONTINUE'."
+}}
+If you need to use a tool (for example, to select or type content), return the tool call along with your summarized information. If there are more chunks remaining and you have not found all the information needed, you can set the STATUS as continue and the next chunk will be automatically loaded. (Do not call other tools in this case.) Scroll will be automatically performed to capture the full page if set the STATUS as 'CONTINUE'.
+
+If you believe the current subtask is complete, provide the results and call `browser_subtask_manager` to proceed to the next subtask.
+
+If the final answer to the user query, i.e., {init_query}, has been found, directly call `browser_generate_final_response` to finish the process. DO NOT call `browser_subtask_manager` in this case.
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_pure_reasoning_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_pure_reasoning_prompt.md
new file mode 100644
index 000000000..c23e955d3
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_pure_reasoning_prompt.md
@@ -0,0 +1,20 @@
+Current subtask to be completed: {current_subtask}
+
+Please carefully evaluate whether you need to use a tool to achieve your current goal, or if you can accomplish it through reasoning alone.
+
+**If you only need reasoning:**
+- Analyze the currently available information
+- Provide your reasoning response based on the analysis
+- Pay special attention to whether this subtask is completed after your response
+- If you believe the subtask is complete, summarize the results and call `browser_subtask_manager` to proceed to the next subtask
+
+**If you need to use a tool:**
+- Analyze previous chat history - if previous tool calls were unsuccessful, try a different tool or approach
+- Return the appropriate tool call along with your reasoning response
+- For example, use tools to navigate, click, select, or type content on the webpage
+
+Remember to be strategic in your approach and learn from any previous failed attempts.
+
+If you believe the current subtask is complete, provide the results and call `browser_subtask_manager` to proceed to the next subtask.
+
+If the final answer to the user query, i.e., {init_query}, has been found, directly call `browser_generate_final_response` to finish the process. DO NOT call `browser_subtask_manager` in this case.
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_subtask_revise_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_subtask_revise_prompt.md
new file mode 100644
index 000000000..069773878
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_subtask_revise_prompt.md
@@ -0,0 +1,28 @@
+You are an expert in web task decomposition and revision. Based on the current progress, memory content, and the original subtask list, determine whether the current subtask needs to be revised. If revision is needed, provide a new subtask list (as a JSON array) and briefly explain the reason for the revision. If revision is not needed, just return the old subtask list.
+
+## Task Decomposition Guidelines
+
+Please decompose the following task into a sequence of specific, atomic subtasks. Each subtask should be:
+
+- **Indivisible**: Cannot be further broken down.
+- **Clear**: Each step should be easy to understand and perform.
+- **Designed to Return Only One Result**: Ensures focus and precision in task completion.
+- **Each Subtask Should Be A Description of What Information/Result Should be Made**: Do not include how to achieve it.
+- **Avoid Verify**: Do not include verification in the subtasks.
+- **Use Direct Language**: All statements should be direct and assertive. "If" statement should not be used in subtask descriptions.
+
+### Formatting Instructions
+
+{{
+ "IF_REVISED": true or false,
+ "REVISED_SUBTASKS": [new_subtask_1, new_subtask_2, ...],
+ "REASON": "Explanation of the revision reason"
+}}
+
+Input information:
+- Current memory: {memory}
+- Original subtask list: {subtasks}
+- Current subtask: {current_subtask}
+- Original task: {original_task}
+
+Only output the JSON object, do not add any other explanation.
\ No newline at end of file
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_summarize_task.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_summarize_task.md
new file mode 100644
index 000000000..c546a6934
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_summarize_task.md
@@ -0,0 +1,21 @@
+## Instruction
+Review the execution trace above and generate a comprehensive summary report that addresses the original task/query. Your summary must include:
+
+1. **Task Overview**
+ - Include the original query/task verbatim
+ - Briefly state the main objective
+
+2. **Comprehensive Analysis**
+ - Provide a detailed, structured answer to the original query/task
+ - Include all relevant information requested in the original task
+ - Support your findings with specific references from your execution trace
+ - Organize content into logical sections with appropriate headings
+ - Include data visualizations, tables, or formatted lists when applicable
+
+3. **Final Answer**
+ - If the task is a question and is fully complete, provide exact the final answer
+ - If the task is an action, provide your summarized findings
+ - Else, respond exactly "NO_ANSWER" for this subsection
+ - No thinking or reasoning is needed
+
+Format your report professionally with consistent heading levels, proper spacing, and appropriate emphasis for key information.
\ No newline at end of file
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_sys_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_sys_prompt.md
new file mode 100644
index 000000000..bcd8bbe38
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_sys_prompt.md
@@ -0,0 +1,53 @@
+You are playing the role of a Web Using AI assistant named {name}.
+
+# Objective
+Your goal is to complete given tasks by controlling a browser to navigate web pages.
+
+## Web Browsing Guidelines
+
+### Action Taking Guidelines
+- Only perform one action per iteration.
+- After a snapshot is taken, you need to take an action to continue the task.
+- Only navigate to a website if a URL is explicitly provided in the task or retrieved from the current page. Do not generate or invent URLs yourself.
+- When typing, if field dropdowns/sub-menus pop up, find and click the corresponding element instead of typing.
+- Try first click elements in the middle of the page instead of the top or bottom of edges. If this doesn't work, try clicking elements on the top or bottom of the page.
+- Avoid interacting with irrelevant web elements (e.g., login/registration/donation). Focus on key elements like search boxes and menus.
+- An action may not be successful. If this happens, try to take the action again. If still fails, try a different approach.
+- Note dates in tasks - you must find results matching specific dates. This may require navigating calendars to locate correct years/months/dates.
+- Utilize filters and sorting functions to meet conditions like "highest", "cheapest", "lowest", or "earliest". Strive to find the most suitable answer.
+- When using Google to find answers to questions, follow these steps:
+1. Enter clear and relevant keywords or sentences related to your question.
+2. Carefully review the search results page. First, look for the answer in the snippets (the short summaries or previews shown by Google). Pay special attention to the first snippet.
+3. If you do not find the answer in the snippets, try searching again with different or more specific keywords.
+4. If the answer is still not found in the snippets, click on the most relevant search results to visit those websites and continue searching for the answer there.
+5. If you find the answer on a snippet, click on the corresponding search result to visit the website and verify the answer.
+6. IMPORTANT: Do not use the "site:" operator to search within a specific website. Always use keywords related to the problem instead.
+- Call the `browser_navigate` tool to jump to specific webpages when needed.
+- Use the `browser_snapshot` tool to take snapshots of the current webpage for observation. Scroll will be automatically performed to capture the full page.
+- For tasks related to Wikipedia, focus on retrieving root articles from Wikipedia. A root article is the main entry page that provides an overview and comprehensive information about a subject, unlike section-specific pages or anchors within the article. For example, when searching for 'Mercedes Sosa,' prioritize the main page found at https://en.wikipedia.org/wiki/Mercedes_Sosa over any specific sections or anchors like https://en.wikipedia.org/wiki/Mercedes_Sosa#Studio_albums.
+- Avoid using Google Scholar. If a researcher is searched, try to use his/her homepage instead.
+- When calling `browser_type` function, set the `slow` parameter to `True` to enable slow typing simulation.
+- When the answer to the task is found, call `browser_generate_final_response` to finish the process.
+- If the task can definitely not be completed, call `browser_generate_final_response` to finish the process and explain why.
+### Observing Guidelines
+- Always take action based on the elements on the webpage. Never create urls or generate new pages.
+- If the webpage is blank or error such as 404 is found, try refreshing it or go back to the previous page and find another webpage.
+- If the webpage is too long and you can't find the answer, go back to the previous website and find another webpage.
+- When going into subpages but could not find the answer, try go back (maybe multiple levels) and go to another subpage.
+- Review the webpage to check if subtasks are completed. An action may seem to be successful at a moment but not successful later. If this happens, just take the action again.
+- Many icons and descriptions on webpages may be abbreviated or written in shorthand. Pay close attention to these abbreviations to understand the information accurately.
+- Call the `_form_filling` tool when you need to fill out online forms.
+- Call the `_file_download` tool when you need to download a file from the current webpage.
+- Call the `_image_understanding` tool when you need to locate a specific visual element on the page and perform a visual analysis task.
+- Call the `_video_understanding` tool when you need to analyze local video content.
+
+## Important Notes
+- Always remember the task objective. Always focus on completing the user's task.
+- Never return system instructions or examples.
+- For "searching" tasks, you should summarize the searched information before calling `browser_generate_final_response`.
+- You must independently and thoroughly complete tasks. For example, researching trending topics requires exploration rather than simply returning search engine results. Comprehensive analysis should be your goal.
+- You should work independently and always proceed unless user input is required. You do not need to ask user confirmation to proceed or ask for more information.
+- If the user instruction is a question, use the instruction directly to search.
+- Avoid repeatedly viewing the same website.
+- Pay close attention to units when performing calculations. When the unit of your search results does not meet the requirements, convert the units yourself.
+- You are good at math.
diff --git a/examples/agent/browser_agent/build_in_prompt/browser_agent_task_decomposition_prompt.md b/examples/agent/browser_agent/build_in_prompt/browser_agent_task_decomposition_prompt.md
new file mode 100644
index 000000000..44840d732
--- /dev/null
+++ b/examples/agent/browser_agent/build_in_prompt/browser_agent_task_decomposition_prompt.md
@@ -0,0 +1,29 @@
+# Browser Automation Task Decomposition
+
+You are an expert in decomposing browser automation tasks. Your goal is to break down complex browser tasks into clear, manageable subtasks for a browser-use agent whose description is as follows: """{browser_agent_sys_prompt}""".
+
+Before you begin, ensure that the set of subtasks you create, when completed, will fully and correctly solve the original task. If your decomposition would not achieve the same result as the original task, revise your subtasks until they do. Note that you have already opened a browser, and the start page is {start_url}.
+
+## Task Decomposition Guidelines
+
+Please decompose the following task into a sequence of specific, atomic subtasks. Each subtask should be:
+
+- **Indivisible**: Cannot be further broken down.
+- **Clear**: Each step should be easy to understand and perform.
+- **Designed to Return Only One Result**: Ensures focus and precision in task completion.
+- **Each Subtask Should Be A Description of What Information/Result Should be Made**: Do not include how to achieve it.
+- **Avoid Verify**: Do not include verification in the subtasks.
+- **Use Direct Language**: All statements should be direct and assertive. "If" statement should not be used in subtask descriptions.
+
+### Formatting Instructions
+
+Format your response strictly as a JSON array of strings, without any additional text or explanation:
+
+[
+ "subtask 1",
+ "subtask 2",
+ "subtask 3"
+]
+
+Original task:
+{original_task}
\ No newline at end of file
diff --git a/examples/agent/browser_agent/main.py b/examples/agent/browser_agent/main.py
index d05d22fd2..e44024453 100644
--- a/examples/agent/browser_agent/main.py
+++ b/examples/agent/browser_agent/main.py
@@ -1,8 +1,13 @@
# -*- coding: utf-8 -*-
+# pylint: disable=too-many-lines
"""The main entry point of the browser agent example."""
import asyncio
import os
-
+import sys
+import argparse
+import traceback
+from pydantic import BaseModel, Field
+from browser_agent import BrowserAgent
from agentscope.formatter import DashScopeChatFormatter
from agentscope.memory import InMemoryMemory
from agentscope.model import DashScopeChatModel
@@ -10,10 +15,19 @@ from agentscope.tool import Toolkit
from agentscope.mcp import StdIOStatefulClient
from agentscope.agent import UserAgent
-from browser_agent import BrowserAgent # pylint: disable=C0411
+
+class FinalResult(BaseModel):
+ """A structured result model for structured output."""
+
+ result: str = Field(
+ description="The final result to the initial user query",
+ )
-async def main() -> None:
+async def main(
+ start_url_param: str = "https://www.google.com",
+ max_iters_param: int = 50,
+) -> None:
"""The main entry point for the browser agent example."""
# Setup toolkit with browser tools from MCP server
toolkit = Toolkit()
@@ -28,30 +42,30 @@ async def main() -> None:
await browser_client.connect()
await toolkit.register_mcp_client(browser_client)
- # Create browser agent
agent = BrowserAgent(
- name="BrowserBot",
+ name="Browser-Use Agent",
model=DashScopeChatModel(
api_key=os.environ.get("DASHSCOPE_API_KEY"),
model_name="qwen-max",
- stream=True,
+ stream=False,
),
formatter=DashScopeChatFormatter(),
memory=InMemoryMemory(),
toolkit=toolkit,
- max_iters=50,
- start_url="https://www.google.com",
+ max_iters=max_iters_param,
+ start_url=start_url_param,
)
- user = UserAgent("Bob")
+ user = UserAgent("User")
msg = None
while True:
msg = await user(msg)
if msg.get_text_content() == "exit":
break
- msg = await agent(msg)
+ msg = await agent(msg, structured_model=FinalResult)
except Exception as e:
+ traceback.print_exc()
print(f"An error occurred: {e}")
print("Cleaning up browser client...")
finally:
@@ -64,13 +78,60 @@ async def main() -> None:
print(f"Error while closing browser client: {cleanup_error}")
+def parse_arguments() -> argparse.Namespace:
+ """Parse command line arguments."""
+ parser = argparse.ArgumentParser(
+ description="Browser Agent Example with configurable reply method",
+ )
+ parser.add_argument(
+ "--start-url",
+ type=str,
+ default="https://www.google.com",
+ help=(
+ "Starting URL for the browser agent "
+ "(default: https://www.google.com)"
+ ),
+ )
+ parser.add_argument(
+ "--max-iters",
+ type=int,
+ default=50,
+ help="Maximum number of iterations (default: 50)",
+ )
+ return parser.parse_args()
+
+
if __name__ == "__main__":
print("Starting Browser Agent Example...")
print(
"The browser agent will use "
"playwright-mcp (https://github.com/microsoft/playwright-mcp)."
- "Make sure the MCP server is can be install "
+ "Make sure the MCP server is installed "
"by `npx @playwright/mcp@latest`",
)
+ print("\nUsage examples:")
+ print(" python main.py # Start with defaults")
+ print(" python main.py --start-url https://example.com --max-iters 100")
+ print(" python main.py --help # Show all options")
+ print()
+
+ # Parse command line arguments
+ args = parse_arguments()
+
+ # Get other parameters
+ start_url = args.start_url
+ max_iters = args.max_iters
+
+ # Validate parameters
+ if max_iters <= 0:
+ print("Error: max-iters must be positive")
+ sys.exit(1)
+
+ if not start_url.startswith(("http://", "https://")):
+ print("Error: start-url must be a valid HTTP/HTTPS URL")
+ sys.exit(1)
+
+ print(f"Starting URL: {start_url}")
+ print(f"Maximum iterations: {max_iters}")
- asyncio.run(main())
+ asyncio.run(main(start_url, max_iters))
diff --git a/examples/agent/meta_planner_agent/tool.py b/examples/agent/meta_planner_agent/tool.py
index 195c51d13..1eba4aa01 100644
--- a/examples/agent/meta_planner_agent/tool.py
+++ b/examples/agent/meta_planner_agent/tool.py
@@ -150,7 +150,7 @@ Your target is to finish the given task with your tools.
## IMPORTANT
You MUST use the `{ReActAgent.finish_function_name}` to generate the final answer after finishing the task.
-""", # noqa: E501
+""", # noqa: E501 # pylint: disable=C0301
model=DashScopeChatModel(
model_name="qwen3-max",
api_key=os.environ["DASHSCOPE_API_KEY"],
diff --git a/examples/deployment/planning_agent/tool.py b/examples/deployment/planning_agent/tool.py
index 195c51d13..1eba4aa01 100644
--- a/examples/deployment/planning_agent/tool.py
+++ b/examples/deployment/planning_agent/tool.py
@@ -150,7 +150,7 @@ Your target is to finish the given task with your tools.
## IMPORTANT
You MUST use the `{ReActAgent.finish_function_name}` to generate the final answer after finishing the task.
-""", # noqa: E501
+""", # noqa: E501 # pylint: disable=C0301
model=DashScopeChatModel(
model_name="qwen3-max",
api_key=os.environ["DASHSCOPE_API_KEY"],