Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
refactor(msg): simplify core building blocks and Msg class structure (#1440)
DavdGaob158fc2e0ccd
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex b26df494c..5f34d4102 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -99,6 +99,9 @@ repos:--disable=W0632,--disable=W0123,--disable=C3001,+ --max-branches=30,+ --max-nested-blocks=7,+ --max-statements=100,]- repo: https://github.com/regebro/pyromarev: "5.0"diff --git a/src/agentscope/__init__.py b/src/agentscope/__init__.pyindex d52e1385b..31cda97ce 100644--- a/src/agentscope/__init__.py+++ b/src/agentscope/__init__.py@@ -1,6 +1,5 @@# -*- coding: utf-8 -*-"""The agentscope serialization module"""-import warningsfrom . import exception@@ -11,7 +10,6 @@ from . import formatterfrom . import agentfrom . import embeddingfrom . import tracing-from ._logging import (logger,setup_logger,@@ -33,7 +31,6 @@ __all__ = ["logger","embedding","tracing",- "a2a",# functions"setup_logger","__version__",diff --git a/src/agentscope/a2a/__init__.py b/src/agentscope/a2a/__init__.pydeleted file mode 100644index 828f2621d..000000000--- a/src/agentscope/a2a/__init__.py+++ /dev/null@@ -1,14 +0,0 @@-# -*- coding: utf-8 -*--"""The A2A related modules."""-from ._base import AgentCardResolverBase-from ._file_resolver import FileAgentCardResolver-from ._well_known_resolver import WellKnownAgentCardResolver-from ._nacos_resolver import NacosAgentCardResolver---__all__ = [- "AgentCardResolverBase",- "FileAgentCardResolver",- "WellKnownAgentCardResolver",- "NacosAgentCardResolver",-]diff --git a/src/agentscope/a2a/_base.py b/src/agentscope/a2a/_base.pydeleted file mode 100644index 23ff1ea7e..000000000--- a/src/agentscope/a2a/_base.py+++ /dev/null@@ -1,25 +0,0 @@-# -*- coding: utf-8 -*--"""The A2A agent card resolver base class."""-from abc import abstractmethod-from typing import Any, TYPE_CHECKING--if TYPE_CHECKING:- from a2a.types import AgentCard-else:- AgentCard = "a2a.types.AgentCard"---class AgentCardResolverBase:- """Base class for A2A agent card resolvers, responsible for fetching- agent cards from various sources. Implementations must provide the- `get_agent_card` method to retrieve the agent card.- """-- @abstractmethod- async def get_agent_card(self, *args: Any, **kwargs: Any) -> AgentCard:- """Get Agent Card from the configured source.-- Returns:- `AgentCard`:- The resolved agent card object.- """diff --git a/src/agentscope/a2a/_file_resolver.py b/src/agentscope/a2a/_file_resolver.pydeleted file mode 100644index e47114096..000000000--- a/src/agentscope/a2a/_file_resolver.py+++ /dev/null@@ -1,78 +0,0 @@-# -*- coding: utf-8 -*--"""The JSON file based A2A agent card resolver."""-import json-from pathlib import Path-from typing import TYPE_CHECKING--from ._base import AgentCardResolverBase--if TYPE_CHECKING:- from a2a.types import AgentCard-else:- AgentCard = "a2a.types.AgentCard"---class FileAgentCardResolver(AgentCardResolverBase):- """Agent card resolver that loads AgentCard from a JSON file.-- The JSON file should contain an AgentCard object with the following- required fields:-- - name (str): The name of the agent- - url (str): The URL of the agent- - version (str): The version of the agent- - capabilities (dict): The capabilities of the agent- - default_input_modes (list[str]): Default input modes- - default_output_modes (list[str]): Default output modes- - skills (list): List of agent skills-- Example JSON file content:-- .. code-block:: json-- {- "name": "RemoteAgent",- "url": "http://localhost:8000",- "description": "A remote A2A agent",- "version": "1.0.0",- "capabilities": {},- "default_input_modes": ["text/plain"],- "default_output_modes": ["text/plain"],- "skills": []- }-- """-- def __init__(- self,- file_path: str,- ) -> None:- """Initialize the FileAgentCardResolver with the path to the JSON file.-- Args:- file_path (`str`):- The path to the JSON file containing the agent card.- """- self._file_path = file_path-- async def get_agent_card(self) -> AgentCard:- """Get the agent card from the JSON file.-- Returns:- `AgentCard`:- The agent card loaded from the file.- """- from a2a.types import AgentCard-- path = Path(self._file_path)- if not path.exists():- raise FileNotFoundError(- f"Agent card file not found: {self._file_path}",- )-- if not path.is_file():- raise ValueError(f"Path is not a file: {self._file_path}")-- with path.open("r", encoding="utf-8") as f:- agent_json_data = json.load(f)- return AgentCard.model_validate(agent_json_data)diff --git a/src/agentscope/a2a/_nacos_resolver.py b/src/agentscope/a2a/_nacos_resolver.pydeleted file mode 100644index 5a5690234..000000000--- a/src/agentscope/a2a/_nacos_resolver.py+++ /dev/null@@ -1,98 +0,0 @@-# -*- coding: utf-8 -*--"""The Nacos-based A2A Agent Card resolver."""--from typing import TYPE_CHECKING--from ._base import AgentCardResolverBase-from .._logging import logger--if TYPE_CHECKING:- from a2a.types import AgentCard- from v2.nacos.common.client_config import ClientConfig-else:- AgentCard = "a2a.types.AgentCard"- ClientConfig = "v2.nacos.common.client_config.ClientConfig"---class NacosAgentCardResolver(AgentCardResolverBase):- """Nacos-based A2A Agent Card resolver.-- Nacos is a dynamic service discovery, configuration and service- management platform for building cloud native applications. This resolver- fetches the agent card from a Nacos server and subscribes to updates.- """-- def __init__(- self,- remote_agent_name: str,- nacos_client_config: ClientConfig,- version: str | None = None,- ) -> None:- """Initialize the nacos agent card resolver.-- Args:- remote_agent_name (`str`):- Name of the remote agent in Nacos.- nacos_client_config (`ClientConfig | None`, optional):- Nacos client configuration, where a `server_addresses`- parameter is required.- version (`str | None`, optional):- Version of the agent card to fetch. If None, fetches the- latest version. This version is also used when subscribing- to agent card updates.- Defaults to None (latest version).- """- if not remote_agent_name:- raise ValueError(- "The remote_agent_name cannot be empty.",- )-- if not nacos_client_config:- raise ValueError(- "The nacos_client_config cannot be None.",- )-- self._nacos_client_config = nacos_client_config- self._remote_agent_name = remote_agent_name- self._version = version-- async def get_agent_card(self) -> AgentCard:- """Get agent card from Nacos with lazy initialization.-- Returns:- `AgentCard`:- The resolved agent card from Nacos.- """- try:- from v2.nacos.ai.model.ai_param import GetAgentCardParam- from v2.nacos.ai.nacos_ai_service import NacosAIService- except ImportError as e:- raise ImportError(- "Please install the nacos sdk by running `pip install "- "nacos-sdk-python>=3.0.0` first.",- ) from e-- client = None- try:- client = await NacosAIService.create_ai_service(- self._nacos_client_config,- )-- await client.start()- return await client.get_agent_card(- GetAgentCardParam(- agent_name=self._remote_agent_name,- version=self._version,- ),- )-- finally:- if client:- # Close the Nacos client to free resources- try:- await client.shutdown()- except Exception as e:- logger.warning(- "Failed to shutdown Nacos client: %s",- str(e),- )diff --git a/src/agentscope/a2a/_well_known_resolver.py b/src/agentscope/a2a/_well_known_resolver.pydeleted file mode 100644index 2f9445884..000000000--- a/src/agentscope/a2a/_well_known_resolver.py+++ /dev/null@@ -1,90 +0,0 @@-# -*- coding: utf-8 -*--"""The A2A well-known agent card resolver."""-from typing import TYPE_CHECKING-from urllib.parse import urlparse--from ._base import AgentCardResolverBase-from .._logging import logger--if TYPE_CHECKING:- from a2a.types import AgentCard-else:- AgentCard = "a2a.types.AgentCard"---class WellKnownAgentCardResolver(AgentCardResolverBase):- """Agent card resolver that loads AgentCard from a well-known URL."""-- def __init__(- self,- base_url: str,- agent_card_path: str | None = None,- ) -> None:- """Initialize the WellKnownAgentCardResolver.-- Args:- base_url (`str`):- The base URL to resolve the agent card from.- agent_card_path (`str | None`, optional):- The path to the agent card relative to the base URL.- Defaults to AGENT_CARD_WELL_KNOWN_PATH from a2a.utils.- """- self._base_url = base_url- self._agent_card_path = agent_card_path-- async def get_agent_card(self) -> AgentCard:- """Get the agent card from the well-known URL.-- Returns:- `AgentCard`:- The agent card loaded from the URL.- """- import httpx- from a2a.client import A2ACardResolver- from a2a.utils import AGENT_CARD_WELL_KNOWN_PATH-- try:- parsed_url = urlparse(self._base_url)- if not parsed_url.scheme or not parsed_url.netloc:- logger.error(- "[%s] Invalid URL format: %s",- self.__class__.__name__,- self._base_url,- )- raise ValueError(- f"Invalid URL format: {self._base_url}",- )-- base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"- relative_card_path = parsed_url.path-- # Use default path if not specified- agent_card_path = (- self._agent_card_path- if self._agent_card_path is not None- else AGENT_CARD_WELL_KNOWN_PATH- )-- # Use async context manager to ensure proper cleanup- async with httpx.AsyncClient(- timeout=httpx.Timeout(timeout=600),- ) as _http_client:- resolver = A2ACardResolver(- httpx_client=_http_client,- base_url=base_url,- agent_card_path=agent_card_path,- )- return await resolver.get_agent_card(- relative_card_path=relative_card_path,- )- except Exception as e:- logger.error(- "[%s] Failed to resolve agent card from URL %s: %s",- self.__class__.__name__,- self._base_url,- e,- )- raise RuntimeError(- f"Failed to resolve AgentCard from URL "- f"{self._base_url}: {e}",- ) from ediff --git a/src/agentscope/agent/__init__.py b/src/agentscope/agent/__init__.pyindex e63c4c420..d65420dde 100644--- a/src/agentscope/agent/__init__.py+++ b/src/agentscope/agent/__init__.py@@ -1,28 +1,5 @@# -*- coding: utf-8 -*--"""The agent base class."""-from ._agent_base import AgentBase-from ._react_agent_base import ReActAgentBase-from ._react_agent import ReActAgent-from ._user_input import (- UserInputBase,- UserInputData,- TerminalUserInput,- StudioUserInput,-)-from ._user_agent import UserAgent-from ._a2a_agent import A2AAgent-from ._realtime_agent import RealtimeAgent+"""Initialize the agent module."""+from ._agent import Agent--__all__ = [- "AgentBase",- "ReActAgentBase",- "ReActAgent",- "UserInputData",- "UserInputBase",- "TerminalUserInput",- "StudioUserInput",- "UserAgent",- "A2AAgent",- "RealtimeAgent",-]+__all__ = ["Agent"]diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.pynew file mode 100644index 000000000..dae40adda--- /dev/null+++ b/src/agentscope/agent/_agent.py@@ -0,0 +1,594 @@+# -*- coding: utf-8 -*-+"""The unified agent class in AgentScope library."""+import uuid+from dataclasses import dataclass+from typing import AsyncGenerator, Literal++from ..event import (+ AgentEvent,+ EventType,+ ModelCallEndedEvent,+ ModelCallStartedEvent,+ RunFinishedEvent,+ RunStartedEvent,+ TextBlockDeltaEvent,+ TextBlockEndEvent,+ TextBlockStartEvent,+ ThinkingBlockDeltaEvent,+ ThinkingBlockEndEvent,+ ThinkingBlockStartEvent,+ ToolCallDeltaEvent,+ ToolCallEndEvent,+ ToolCallStartEvent,+ ToolResultBinaryDeltaEvent,+ ToolResultEndEvent,+ ToolResultStartEvent,+ ToolResultTextDeltaEvent,+ RequireUserConfirmEvent,+ RequireExternalExecutionEvent,+ ExternalExecutionResultEvent,+ UserConfirmResultEvent,+)+from ..message import Msg+from ..model import ChatResponse+from ..model import ChatUsage++from ..message import (+ TextBlock,+ ThinkingBlock,+ ToolCallBlock,+ ToolResultBlock,+ DataBlock,+ Base64Source,+ URLSource,+)++from ..model import ChatModelBase+++DEFAULT_COMPRESSION_PROMPT = (+ "<system-hint>You have been working on the task described above but have "+ "not yet completed it. "+ "Now write a continuation summary that will allow you to resume work "+ "efficiently in a future context window "+ "where the conversation history will be replaced with this summary. "+ "Your summary should be structured, concise, and actionable.</system-hint>"+)+++@dataclass+class AgentState:+ """The agent state that should be saved and loaded from storage."""++ context: list[Msg]+ """The uncompressed conversation context, that will be feed into the LLM"""++ reply_id: str+ """The id of the current reply, which is also used as the id of the+ final message of the reply."""++ cur_iter: int = 0+ """The current iteration of the agent's reasoning-acting loop."""++ cur_summary: str = ""+ """The current compressed summary of the context, which will be prepended+ to the context when feed into the LLM."""+++class Agent:+ """The unified agent class in AgentScope library."""++ def __init__(+ self,+ name: str,+ sys_prompt: str,+ model: "ChatModelBase",+ max_iters: int = 20,+ ) -> None:+ """Initialize an agent instance."""+ if max_iters <= 0:+ raise ValueError("max_iters must be greater than 0")++ self.name = name+ self._sys_prompt = sys_prompt+ self.model = model+ self.max_iters = max_iters++ # TODO: storage - these should be loaded from storage+ self._loaded = False+ self.context: list[Msg] = []+ self.reply_id: str = ""+ self.cur_iter: int = 0+ self.confirmed_tool_call_ids: list[str] = []+ self.cur_summary: str = ""++ @property+ def sys_prompt(self) -> str:+ """Get the system prompt of the agent."""+ # TODO: Add toolkit skills prompt when toolkit is implemented+ return self._sys_prompt++ async def load_state(self) -> None:+ """Load the state from storage if available."""+ # TODO: Implement storage loading++ async def save_state(self) -> None:+ """Save the state to storage if available."""+ # TODO: Implement storage saving++ async def reply_stream(+ self,+ msgs: Msg | list[Msg] | None = None,+ event: AgentEvent | None = None,+ ) -> AsyncGenerator[AgentEvent, None]:+ """Reply to the given message and stream agent events."""+ await self.load_state()+ try:+ async for chunk in self._reply(msgs=msgs, event=event):+ if not isinstance(chunk, Msg):+ yield chunk+ finally:+ await self.save_state()++ async def reply(+ self,+ msgs: Msg | list[Msg] | None = None,+ event: AgentEvent | None = None,+ ) -> Msg:+ """Reply to the given message, consuming all streamed events."""+ await self.load_state()+ try:+ final_msg: Msg | None = None+ async for chunk in self._reply(msgs=msgs, event=event):+ if chunk.is_last:+ final_msg = chunk.msg+ if final_msg is None:+ raise RuntimeError("Agent did not produce a final message.")+ return final_msg+ finally:+ await self.save_state()++ def _save_to_context(+ self,+ blocks: list,+ _usage: ChatUsage | None = None,+ ) -> None:+ """Save content blocks into the context."""+ if len(self.context) == 0:+ self.context.append(+ Msg(name=self.name, content=list(blocks), role="assistant"),+ )+ else:+ last_msg = self.context[-1]+ if last_msg.role == "assistant" and last_msg.name == self.name:+ if isinstance(last_msg.content, str):+ last_msg.content = [TextBlock(text=last_msg.content)]+ last_msg.content.extend(blocks)+ # TODO: Merge usage if needed+ else:+ self.context.append(+ Msg(+ name=self.name,+ content=list(blocks),+ role="assistant",+ ),+ )++ def _get_pending_tool_calls(self) -> list[ToolCallBlock]:+ """Get tool calls from the last assistant message that have no result+ yet."""+ if len(self.context) == 0:+ return []+ last_msg = self.context[-1]+ if last_msg.role != "assistant":+ return []+ content = (+ last_msg.content if isinstance(last_msg.content, list) else []+ )+ tool_calls = [b for b in content if isinstance(b, ToolCallBlock)]+ result_ids = {b.id for b in content if isinstance(b, ToolResultBlock)}+ return [tc for tc in tool_calls if tc.id not in result_ids]++ def _get_awaiting_tool_calls(self) -> dict:+ """Get the awaiting tool calls that need user confirmation or external+ execution."""+ pending_tool_calls = self._get_pending_tool_calls()+ pre_tool_calls: list = []++ for tool_call in enumerate(pending_tool_calls):+ # TODO: toolkit checks for user confirmation and external execution+ pre_tool_calls.append(tool_call)++ return {+ "awaiting_type": None,+ "expected_event_type": None,+ "awaiting_tool_calls": [],+ "pre_tool_calls": pre_tool_calls,+ }++ async def _reply(+ self,+ msgs: Msg | list[Msg] | None = None,+ event: AgentEvent | None = None,+ ) -> AsyncGenerator[AgentEvent | Msg, None]:+ """Core reply logic. Yields chunks with is_last flag."""+ awaiting_info = self._get_awaiting_tool_calls()+ expected_event_type = awaiting_info["expected_event_type"]++ if expected_event_type is not None:+ if event is None or event.type != expected_event_type:+ raise ValueError(+ f"Agent is awaiting '{expected_event_type}' but received "+ f"event of type '{event.type if event else 'none'}'.",+ )++ if isinstance(event, ExternalExecutionResultEvent):+ self._save_to_context(list(event.execution_results))++ elif isinstance(event, UserConfirmResultEvent):+ for result in event.confirm_results:+ confirmed = result.confirmed+ tool_call = result.tool_call+ if confirmed:+ self.confirmed_tool_call_ids.append(tool_call["id"])+ else:+ rejection_text = (+ f"<system-info>**Note** the user rejected the "+ f"execution of tool "+ f'"{tool_call["name"]}"!</system-info>'+ )+ yield ToolResultStartEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call["id"],+ tool_call_name=tool_call["name"],+ )+ yield ToolResultTextDeltaEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call["id"],+ delta=rejection_text,+ )+ yield ToolResultEndEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call["id"],+ state="interrupted",+ )+ self._save_to_context(+ [+ ToolResultBlock(+ id=tool_call["id"],+ name=tool_call["name"],+ output=[TextBlock(text=rejection_text)],+ state="interrupted",+ ),+ ],+ )++ processed_ids = {+ r.get("tool_call", {}).get("id")+ for r in event.confirm_results+ }+ if self.context:+ last_content = self.context[-1].content+ if isinstance(last_content, list):+ for block in last_content:+ if (+ isinstance(block, ToolCallBlock)+ and block.id in processed_ids+ ):+ block.await_user_confirmation = False++ else:+ self.cur_iter = 0+ self.reply_id = str(uuid.uuid4())+ self.confirmed_tool_call_ids = []++ yield RunStartedEvent(+ session_id="",+ reply_id=self.reply_id,+ name=self.name,+ role="assistant",+ )++ if isinstance(msgs, list):+ self.context.extend(msgs)+ elif msgs is not None:+ self.context.append(msgs)++ while self.cur_iter < self.max_iters:+ pending_tool_calls = self._get_pending_tool_calls()+ if len(pending_tool_calls) == 0:+ await self._compress_memory_if_needed()+ async for evt in self._reasoning(tool_choice="auto"):+ yield evt++ awaiting_info = self._get_awaiting_tool_calls()+ awaiting_type = awaiting_info["awaiting_type"]+ awaiting_tool_calls = awaiting_info["awaiting_tool_calls"]+ pre_tool_calls = awaiting_info["pre_tool_calls"]++ for tool_call in pre_tool_calls:+ async for evt in self._acting(tool_call=tool_call):+ yield evt+ self.confirmed_tool_call_ids = [+ cid+ for cid in self.confirmed_tool_call_ids+ if cid != tool_call.id+ ]++ if awaiting_type is not None:+ if awaiting_type == EventType.REQUIRE_USER_CONFIRM:+ yield RequireUserConfirmEvent(+ reply_id=self.reply_id,+ tool_calls=awaiting_tool_calls,+ )+ else:+ yield RequireExternalExecutionEvent(+ reply_id=self.reply_id,+ tool_calls=awaiting_tool_calls,+ )+ waiting_text = (+ "Waiting for user confirmation ..."+ if awaiting_type == EventType.REQUIRE_USER_CONFIRM+ else "Waiting for external execution ..."+ )+ yield Msg(+ name=self.name,+ content=[TextBlock(text=waiting_text)],+ role="assistant",+ )+ return++ if len(pre_tool_calls) == 0:+ break++ self.cur_iter += 1++ last_block = None+ if self.context:+ last_content = self.context[-1].content+ if isinstance(last_content, list) and last_content:+ last_block = last_content[-1]+ if last_block is None or not isinstance(last_block, TextBlock):+ async for evt in self._reasoning(tool_choice="none"):+ yield evt++ yield RunFinishedEvent(+ session_id="",+ reply_id=self.reply_id,+ )++ final_block = None+ if self.context:+ last_content = self.context[-1].content+ if isinstance(last_content, list) and last_content:+ final_block = last_content[-1]+ yield Msg(+ id=self.reply_id,+ name=self.name,+ content=[final_block] if final_block else [],+ role="assistant",+ )++ async def _reasoning(+ self,+ tool_choice: Literal["auto", "none", "required"] = "auto",+ ) -> AsyncGenerator:+ """Core reasoning logic. Yields chunks with is_last flag."""+ # TODO: Pass tool schemas from toolkit when toolkit is implemented++ yield ModelCallStartedEvent(+ reply_id=self.reply_id,+ model_name=self.model.model_name,+ )++ system_msg = Msg(+ name="system",+ content=[TextBlock(text=self.sys_prompt)],+ role="system",+ )+ messages = [system_msg]+ if self.cur_summary:+ messages.append(+ Msg(+ name="user",+ content=[TextBlock(text=self.cur_summary)],+ role="user",+ ),+ )+ messages.extend(self.context)++ # TODO: Pass tools and tool_choice when toolkit is implemented+ res = await self.model(+ messages=messages,+ tool_choice=tool_choice,+ )++ block_ids: dict = {+ "text_block_id": None,+ "thinking_block_id": None,+ "tool_call_ids": [],+ }+ completed_response: ChatResponse | None = None++ if self.model.stream:+ async for chunk in res:+ if chunk.is_last:+ completed_response = chunk+ break+ async for evt in self._convert_chat_response_to_event(+ block_ids,+ chunk,+ ):+ yield evt+ elif isinstance(res, ChatResponse):+ completed_response = res+ async for evt in self._convert_chat_response_to_event(+ block_ids,+ res,+ ):+ yield evt++ if block_ids["text_block_id"] is not None:+ yield TextBlockEndEvent(+ reply_id=self.reply_id,+ block_id=block_ids["text_block_id"],+ )+ if block_ids["thinking_block_id"] is not None:+ yield ThinkingBlockEndEvent(+ reply_id=self.reply_id,+ block_id=block_ids["thinking_block_id"],+ )+ for tool_call_id in block_ids["tool_call_ids"]:+ yield ToolCallEndEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call_id,+ )++ yield ModelCallEndedEvent(+ reply_id=self.reply_id,+ input_tokens=completed_response.usage.input_tokens+ if completed_response.usage+ else 0,+ output_tokens=completed_response.usage.output_tokens+ if completed_response.usage+ else 0,+ )++ self._save_to_context(+ list(completed_response.content),+ completed_response.usage,+ )++ async def _acting(+ self,+ tool_call: ToolCallBlock,+ ) -> AsyncGenerator:+ """Core acting logic. Yields chunks with is_last flag."""+ yield ToolResultStartEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call.id,+ tool_call_name=tool_call.name,+ )++ # TODO: Implement toolkit execution when toolkit is available+ # res = self.toolkit.call_tool_function(tool_call)+ # async for tool_res in res:+ # if tool_res.is_last:+ # result_block = {+ # "type": "tool_result",+ # "id": tool_call["id"],+ # "name": tool_call["name"],+ # "output": tool_res.content,+ # "state": tool_res.state,+ # }+ # self._save_to_context([result_block])+ # return+ # async for evt in self._convert_tool_response_to_event(+ # tool_call, tool_res+ # ):+ # yield evt+ raise NotImplementedError("Toolkit not yet implemented.")++ async def _observe(self, msgs: Msg | list[Msg] | None = None) -> None:+ """Receive external observation message(s) and save them into+ context."""+ await self.load_state()+ if isinstance(msgs, list):+ self.context.extend(msgs)+ elif msgs is not None:+ self.context.append(msgs)++ async def _convert_chat_response_to_event(+ self,+ block_ids: dict,+ chunk: ChatResponse,+ ) -> AsyncGenerator:+ """Convert a ChatResponse chunk into a sequence of agent events."""+ for block in chunk.content:+ if isinstance(block, TextBlock):+ if block_ids["text_block_id"] is None:+ block_ids["text_block_id"] = str(uuid.uuid4())+ yield TextBlockStartEvent(+ reply_id=self.reply_id,+ block_id=block_ids["text_block_id"],+ )+ yield TextBlockDeltaEvent(+ reply_id=self.reply_id,+ block_id=block_ids["text_block_id"],+ delta=block.text,+ )++ elif isinstance(block, ThinkingBlock):+ if block_ids["thinking_block_id"] is None:+ block_ids["thinking_block_id"] = str(uuid.uuid4())+ yield ThinkingBlockStartEvent(+ reply_id=self.reply_id,+ block_id=block_ids["thinking_block_id"],+ )+ yield ThinkingBlockDeltaEvent(+ reply_id=self.reply_id,+ block_id=block_ids["thinking_block_id"],+ delta=block.thinking,+ )++ elif isinstance(block, ToolCallBlock):+ if block.id not in block_ids["tool_call_ids"]:+ block_ids["tool_call_ids"].append(block.id)+ yield ToolCallStartEvent(+ reply_id=self.reply_id,+ tool_call_id=block.id,+ tool_call_name=block.name,+ )+ yield ToolCallDeltaEvent(+ reply_id=self.reply_id,+ tool_call_id=block.id,+ delta=block.input,+ )++ async def _convert_tool_response_to_event(+ self,+ tool_call: ToolCallBlock,+ tool_res: object,+ ) -> AsyncGenerator:+ """Convert a ToolResponse into a sequence of agent events."""+ # TODO: Properly typed once toolkit is implemented+ for block in tool_res.content: # type: ignore[attr-defined]+ if isinstance(block, TextBlock):+ yield ToolResultTextDeltaEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call.id,+ delta=block.text,+ )++ elif isinstance(block, DataBlock):+ if isinstance(block.source, Base64Source):+ yield ToolResultBinaryDeltaEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call.id,+ media_type=block.source.media_type,+ data=block.source.data,+ )+ elif isinstance(block.source, URLSource):+ yield ToolResultBinaryDeltaEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call.id,+ media_type=block.source.media_type,+ url=block.source.url,+ )++ yield ToolResultEndEvent(+ reply_id=self.reply_id,+ tool_call_id=tool_call.id,+ state=tool_res.state, # type: ignore[attr-defined]+ )++ def _split_context_for_compression(self) -> tuple[list[Msg], list[Msg]]:+ """Split context into parts to compress and parts to keep recent."""+ # TODO: Implement full splitting logic when compression_config is+ # available+ return [], list(self.context)++ async def _compress_memory_if_needed(self) -> None:+ """Compress the agent's memory if the token count exceeds the+ threshold."""+ # TODO: Implement when compression_config is availablediff --git a/src/agentscope/embedding/_dashscope_multimodal_embedding.py b/src/agentscope/embedding/_dashscope_multimodal_embedding.pyindex f8a50a156..e63bfc612 100644--- a/src/agentscope/embedding/_dashscope_multimodal_embedding.py+++ b/src/agentscope/embedding/_dashscope_multimodal_embedding.py@@ -8,8 +8,7 @@ from ._embedding_response import EmbeddingResponsefrom ._embedding_usage import EmbeddingUsagefrom ._embedding_base import EmbeddingModelBasefrom ..message import (- VideoBlock,- ImageBlock,+ DataBlock,TextBlock,)@@ -90,14 +89,14 @@ class DashScopeMultiModalEmbedding(EmbeddingModelBase):async def __call__(self,- inputs: list[TextBlock | ImageBlock | VideoBlock],+ inputs: list[TextBlock | DataBlock],**kwargs: Any,) -> EmbeddingResponse:"""Call the DashScope multimodal embedding API, which accepts text,image, and video data.Args:- inputs (`list[TextBlock | ImageBlock | VideoBlock]`):+ inputs (`list[TextBlock | DataBlock]`):The input data to be embedded. It can be a list of text,image, and video blocks.diff --git a/src/agentscope/event/__init__.py b/src/agentscope/event/__init__.pynew file mode 100644index 000000000..4d045dc8f--- /dev/null+++ b/src/agentscope/event/__init__.py@@ -0,0 +1,65 @@+# -*- coding: utf-8 -*-+"""The event module of agentscope."""++from ._event import (+ EventType,+ EventBase,+ RunStartedEvent,+ RunFinishedEvent,+ ModelCallStartedEvent,+ ModelCallEndedEvent,+ TextBlockStartEvent,+ TextBlockDeltaEvent,+ TextBlockEndEvent,+ BinaryBlockStartEvent,+ BinaryBlockDeltaEvent,+ BinaryBlockEndEvent,+ ThinkingBlockStartEvent,+ ThinkingBlockDeltaEvent,+ ThinkingBlockEndEvent,+ ToolCallStartEvent,+ ToolCallDeltaEvent,+ ToolCallEndEvent,+ ToolResultStartEvent,+ ToolResultTextDeltaEvent,+ ToolResultBinaryDeltaEvent,+ ToolResultEndEvent,+ ExceedMaxItersEvent,+ RequireUserConfirmEvent,+ RequireExternalExecutionEvent,+ UserConfirmResultEvent,+ ExternalExecutionResultEvent,+ AgentEvent,+)+++__all__ = [+ "EventType",+ "EventBase",+ "RunStartedEvent",+ "RunFinishedEvent",+ "ModelCallStartedEvent",+ "ModelCallEndedEvent",+ "TextBlockStartEvent",+ "TextBlockDeltaEvent",+ "TextBlockEndEvent",+ "BinaryBlockStartEvent",+ "BinaryBlockDeltaEvent",+ "BinaryBlockEndEvent",+ "ThinkingBlockStartEvent",+ "ThinkingBlockDeltaEvent",+ "ThinkingBlockEndEvent",+ "ToolCallStartEvent",+ "ToolCallDeltaEvent",+ "ToolCallEndEvent",+ "ToolResultStartEvent",+ "ToolResultTextDeltaEvent",+ "ToolResultBinaryDeltaEvent",+ "ToolResultEndEvent",+ "ExceedMaxItersEvent",+ "RequireUserConfirmEvent",+ "RequireExternalExecutionEvent",+ "UserConfirmResultEvent",+ "ExternalExecutionResultEvent",+ "AgentEvent",+]diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.pynew file mode 100644index 000000000..fcb8460f2--- /dev/null+++ b/src/agentscope/event/_event.py@@ -0,0 +1,421 @@+# -*- coding: utf-8 -*-+"""Event types for agent execution."""+import uuid+from datetime import datetime+from enum import Enum+from typing import Literal, List, TypeAlias++from pydantic import BaseModel, Field++from ..message import ToolCallBlock, ToolResultBlock+++class EventType(str, Enum):+ """Event type enumeration."""++ RUN_STARTED = "RUN_STARTED"+ RUN_FINISHED = "RUN_FINISHED"++ MODEL_CALL_STARTED = "MODEL_CALL_STARTED"+ MODEL_CALL_ENDED = "MODEL_CALL_ENDED"++ TEXT_BLOCK_START = "TEXT_BLOCK_START"+ TEXT_BLOCK_DELTA = "TEXT_BLOCK_DELTA"+ TEXT_BLOCK_END = "TEXT_BLOCK_END"++ BINARY_BLOCK_START = "BINARY_BLOCK_START"+ BINARY_BLOCK_DELTA = "BINARY_BLOCK_DELTA"+ BINARY_BLOCK_END = "BINARY_BLOCK_END"++ THINKING_BLOCK_START = "THINKING_BLOCK_START"+ THINKING_BLOCK_DELTA = "THINKING_BLOCK_DELTA"+ THINKING_BLOCK_END = "THINKING_BLOCK_END"++ TOOL_CALL_START = "TOOL_CALL_START"+ TOOL_CALL_DELTA = "TOOL_CALL_DELTA"+ TOOL_CALL_END = "TOOL_CALL_END"++ TOOL_RESULT_START = "TOOL_RESULT_START"+ TOOL_RESULT_TEXT_DELTA = "TOOL_RESULT_TEXT_DELTA"+ TOOL_RESULT_BINARY_DELTA = "TOOL_RESULT_BINARY_DELTA"+ TOOL_RESULT_END = "TOOL_RESULT_END"++ EXCEED_MAX_ITERS = "EXCEED_MAX_ITERS"++ REQUIRE_USER_CONFIRM = "REQUIRE_USER_CONFIRM"+ REQUIRE_EXTERNAL_EXECUTION = "REQUIRE_EXTERNAL_EXECUTION"++ USER_CONFIRM_RESULT = "USER_CONFIRM_RESULT"+ EXTERNAL_EXECUTION_RESULT = "EXTERNAL_EXECUTION_RESULT"+++class EventBase(BaseModel):+ """Base event class."""++ id: str = Field(default_factory=lambda: uuid.uuid4().hex)+ """Unique event identifier."""+ created_at: str = Field(default_factory=lambda: datetime.now().isoformat())+ """ISO 8601 timestamp of when the event was created."""+++class RunStartedEvent(EventBase):+ """Run started event."""++ type: Literal[EventType.RUN_STARTED] = EventType.RUN_STARTED+ """Event type."""+ session_id: str+ """ID of the session this run belongs to."""+ reply_id: str+ """ID of the reply message produced by this run."""+ name: str+ """Name of the agent."""+ role: Literal["user", "assistant", "system"] = "assistant"+ """Role of the agent."""+++class RunFinishedEvent(EventBase):+ """Run finished event."""++ type: Literal[EventType.RUN_FINISHED] = EventType.RUN_FINISHED+ """Event type."""+ session_id: str+ """ID of the session this run belongs to."""+ reply_id: str+ """ID of the reply message produced by this run."""+++class ModelCallStartedEvent(EventBase):+ """Model call started event."""++ type: Literal[EventType.MODEL_CALL_STARTED] = EventType.MODEL_CALL_STARTED+ """Event type."""+ reply_id: str+ """ID of the reply message this model call belongs to."""+ model_name: str+ """Name of the model being called."""+++class ModelCallEndedEvent(EventBase):+ """Model call ended event."""++ type: Literal[EventType.MODEL_CALL_ENDED] = EventType.MODEL_CALL_ENDED+ """Event type."""+ reply_id: str+ """ID of the reply message this model call belongs to."""+ input_tokens: int+ """Number of input tokens consumed."""+ output_tokens: int+ """Number of output tokens generated."""+++class TextBlockStartEvent(EventBase):+ """Text block start event."""++ type: Literal[EventType.TEXT_BLOCK_START] = EventType.TEXT_BLOCK_START+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the text block."""+++class TextBlockDeltaEvent(EventBase):+ """Text block delta event."""++ type: Literal[EventType.TEXT_BLOCK_DELTA] = EventType.TEXT_BLOCK_DELTA+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the text block."""+ delta: str+ """Incremental text content."""+++class TextBlockEndEvent(EventBase):+ """Text block end event."""++ type: Literal[EventType.TEXT_BLOCK_END] = EventType.TEXT_BLOCK_END+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the text block."""+++class BinaryBlockStartEvent(EventBase):+ """Binary block start event."""++ type: Literal[EventType.BINARY_BLOCK_START] = EventType.BINARY_BLOCK_START+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the binary block."""+ media_type: str+ """MIME type of the binary content (e.g. "image/png")."""+++class BinaryBlockDeltaEvent(EventBase):+ """Binary block delta event."""++ type: Literal[EventType.BINARY_BLOCK_DELTA] = EventType.BINARY_BLOCK_DELTA+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the binary block."""+ data: str+ """Incremental base64-encoded binary data."""+ media_type: str+ """MIME type of the binary content."""+++class BinaryBlockEndEvent(EventBase):+ """Binary block end event."""++ type: Literal[EventType.BINARY_BLOCK_END] = EventType.BINARY_BLOCK_END+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the binary block."""+++class ThinkingBlockStartEvent(EventBase):+ """Thinking block start event."""++ type: Literal[+ EventType.THINKING_BLOCK_START+ ] = EventType.THINKING_BLOCK_START+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the thinking block."""+++class ThinkingBlockDeltaEvent(EventBase):+ """Thinking block delta event."""++ type: Literal[+ EventType.THINKING_BLOCK_DELTA+ ] = EventType.THINKING_BLOCK_DELTA+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the thinking block."""+ delta: str+ """Incremental thinking text content."""+++class ThinkingBlockEndEvent(EventBase):+ """Thinking block end event."""++ type: Literal[EventType.THINKING_BLOCK_END] = EventType.THINKING_BLOCK_END+ """Event type."""+ reply_id: str+ """ID of the reply message this block belongs to."""+ block_id: str+ """Unique identifier of the thinking block."""+++class ToolCallStartEvent(EventBase):+ """Tool call start event."""++ type: Literal[EventType.TOOL_CALL_START] = EventType.TOOL_CALL_START+ """Event type."""+ reply_id: str+ """ID of the reply message this tool call belongs to."""+ tool_call_id: str+ """Unique identifier of the tool call."""+ tool_call_name: str+ """Name of the tool being called."""+++class ToolCallDeltaEvent(EventBase):+ """Tool call delta event."""++ type: Literal[EventType.TOOL_CALL_DELTA] = EventType.TOOL_CALL_DELTA+ """Event type."""+ reply_id: str+ """ID of the reply message this tool call belongs to."""+ tool_call_id: str+ """Unique identifier of the tool call."""+ delta: str+ """Incremental tool call arguments (JSON fragment)."""+++class ToolCallEndEvent(EventBase):+ """Tool call end event."""++ type: Literal[EventType.TOOL_CALL_END] = EventType.TOOL_CALL_END+ """Event type."""+ reply_id: str+ """ID of the reply message this tool call belongs to."""+ tool_call_id: str+ """Unique identifier of the tool call."""+++class ToolResultStartEvent(EventBase):+ """Tool result start event."""++ type: Literal[EventType.TOOL_RESULT_START] = EventType.TOOL_RESULT_START+ """Event type."""+ reply_id: str+ """ID of the reply message this tool result belongs to."""+ tool_call_id: str+ """ID of the corresponding tool call."""+ tool_call_name: str+ """Name of the tool that was called."""+++class ToolResultTextDeltaEvent(EventBase):+ """Tool result text delta event."""++ type: Literal[+ EventType.TOOL_RESULT_TEXT_DELTA+ ] = EventType.TOOL_RESULT_TEXT_DELTA+ """Event type."""+ reply_id: str+ """ID of the reply message this tool result belongs to."""+ tool_call_id: str+ """ID of the corresponding tool call."""+ delta: str+ """Incremental text content of the tool result."""+++class ToolResultBinaryDeltaEvent(EventBase):+ """Tool result binary delta event."""++ type: Literal[+ EventType.TOOL_RESULT_BINARY_DELTA+ ] = EventType.TOOL_RESULT_BINARY_DELTA+ """Event type."""+ reply_id: str+ """ID of the reply message this tool result belongs to."""+ tool_call_id: str+ """ID of the corresponding tool call."""+ media_type: str+ """MIME type of the binary content."""+ data: str | None = None+ """Base64-encoded binary data, mutually exclusive with `url`."""+ url: str | None = None+ """URL pointing to the binary content, mutually exclusive with `data`."""+++class ToolResultEndEvent(EventBase):+ """Tool result end event."""++ type: Literal[EventType.TOOL_RESULT_END] = EventType.TOOL_RESULT_END+ """Event type."""+ reply_id: str+ """ID of the reply message this tool result belongs to."""+ tool_call_id: str+ """ID of the corresponding tool call."""+ state: Literal["success", "error", "interrupted", "running"]+ """Final execution state of the tool call."""+++class ExceedMaxItersEvent(EventBase):+ """Exceed max iterations event."""++ type: Literal[EventType.EXCEED_MAX_ITERS] = EventType.EXCEED_MAX_ITERS+ """Event type."""+ reply_id: str+ """ID of the reply message associated with this run."""+ agent_id: str+ """ID of the agent that exceeded the iteration limit."""+ name: str+ """Name of the agent."""+++class RequireUserConfirmEvent(EventBase):+ """Require user confirm event."""++ type: Literal[+ EventType.REQUIRE_USER_CONFIRM+ ] = EventType.REQUIRE_USER_CONFIRM+ """Event type."""+ reply_id: str+ """ID of the reply message associated with this run."""+ tool_calls: List[ToolCallBlock]+ """Tool calls pending user confirmation."""+++class RequireExternalExecutionEvent(EventBase):+ """Require external execution event."""++ type: Literal[+ EventType.REQUIRE_EXTERNAL_EXECUTION+ ] = EventType.REQUIRE_EXTERNAL_EXECUTION+ """Event type."""+ reply_id: str+ """ID of the reply message associated with this run."""+ tool_calls: List[ToolCallBlock]+ """Tool calls to be executed externally."""+++class ConfirmResult(BaseModel):+ """Confirm result for a tool call."""++ confirmed: bool+ """Whether the user confirmed the tool call."""+ tool_call: ToolCallBlock+ """The tool call that was confirmed or rejected."""+++class UserConfirmResultEvent(EventBase):+ """User confirm result event."""++ type: Literal[+ EventType.USER_CONFIRM_RESULT+ ] = EventType.USER_CONFIRM_RESULT+ """Event type."""+ reply_id: str+ """ID of the reply message associated with this run."""+ confirm_results: list[ConfirmResult]+ """Confirmation results for each pending tool call."""+++class ExternalExecutionResultEvent(EventBase):+ """External execution result event."""++ type: Literal[+ EventType.EXTERNAL_EXECUTION_RESULT+ ] = EventType.EXTERNAL_EXECUTION_RESULT+ """Event type."""+ reply_id: str+ """ID of the reply message associated with this run."""+ execution_results: List[ToolResultBlock]+ """Results returned by the external executor."""+++AgentEvent: TypeAlias = (+ RunStartedEvent+ | RunFinishedEvent+ | ExceedMaxItersEvent+ | RequireUserConfirmEvent+ | RequireExternalExecutionEvent+ | ModelCallStartedEvent+ | ModelCallEndedEvent+ | TextBlockStartEvent+ | TextBlockDeltaEvent+ | TextBlockEndEvent+ | BinaryBlockStartEvent+ | BinaryBlockDeltaEvent+ | BinaryBlockEndEvent+ | ThinkingBlockStartEvent+ | ThinkingBlockDeltaEvent+ | ThinkingBlockEndEvent+ | ToolCallStartEvent+ | ToolCallDeltaEvent+ | ToolCallEndEvent+ | ToolResultStartEvent+ | ToolResultTextDeltaEvent+ | ToolResultBinaryDeltaEvent+ | ToolResultEndEvent+ | UserConfirmResultEvent+ | ExternalExecutionResultEvent+)diff --git a/src/agentscope/formatter/__init__.py b/src/agentscope/formatter/__init__.pyindex 320229813..974880ce8 100644--- a/src/agentscope/formatter/__init__.py+++ b/src/agentscope/formatter/__init__.py@@ -2,7 +2,6 @@"""The formatter module in agentscope."""from ._formatter_base import FormatterBase-from ._truncated_formatter_base import TruncatedFormatterBasefrom ._dashscope_formatter import (DashScopeChatFormatter,DashScopeMultiAgentFormatter,@@ -27,11 +26,9 @@ from ._deepseek_formatter import (DeepSeekChatFormatter,DeepSeekMultiAgentFormatter,)-from ._a2a_formatter import A2AChatFormatter__all__ = ["FormatterBase",- "TruncatedFormatterBase","DashScopeChatFormatter","DashScopeMultiAgentFormatter","OpenAIChatFormatter",@@ -44,5 +41,4 @@ __all__ = ["OllamaMultiAgentFormatter","DeepSeekChatFormatter","DeepSeekMultiAgentFormatter",- "A2AChatFormatter",]diff --git a/src/agentscope/formatter/_a2a_formatter.py b/src/agentscope/formatter/_a2a_formatter.pydeleted file mode 100644index 4382dd89d..000000000--- a/src/agentscope/formatter/_a2a_formatter.py+++ /dev/null@@ -1,364 +0,0 @@-# -*- coding: utf-8 -*--"""The A2A message formatter class."""-import mimetypes-import uuid-from typing import Literal, TYPE_CHECKING---from .._logging import logger-from ._formatter_base import FormatterBase-from ..message import (- Msg,- TextBlock,- URLSource,- Base64Source,- ContentBlock,-)---if TYPE_CHECKING:- from a2a.types import (- Message,- Task,- Part,- )-else:- Message = "a2a.types.Message"- Task = "a2a.types.Task"- Part = "a2a.types.Part"---class A2AChatFormatter(FormatterBase):- """A2A message formatter class, which convert AgentScope messages into- A2A message format."""-- async def format(self, msgs: list[Msg]) -> Message:- """Convert AgentScope messages into a A2A message object. Note that- A2A server only supports single request message, so the input msgs- list will be merged into a single A2A Message.-- .. note:: Note the A2A protocol receives a single message per request,- so multi-message inputs will be merged into one A2A Message with role- 'user'.-- Args:- msgs (`list[Msg]`):- List of AgentScope Msg objects to be converted.-- Returns:- `Message`:- The converted A2A Message object.- """-- from a2a.types import (- Part,- TextPart,- FilePart,- FileWithUri,- FileWithBytes,- DataPart,- Role,- Message,- )-- self.assert_list_of_msgs(msgs)-- parts = []- for msg in msgs:- for block in msg.get_content_blocks():- block_type = block.get("type")- if block_type == "text" and block.get("text"):- parts.append(- Part(- root=TextPart(- text=block.get("text"),- ),- ),- )-- elif block_type == "thinking" and block.get("thinking"):- parts.append(- Part(- root=TextPart(- text=block.get("thinking"),- ),- ),- )-- elif block_type in [- "image",- "video",- "audio",- ] and block.get("source"):- source = block.get("source", {})- source_type = source.get("type")-- if source_type == "url":- parts.append(- Part(- root=FilePart(- file=FileWithUri(- uri=source.get("url"),- ),- ),- ),- )-- elif source_type == "base64":- parts.append(- Part(- root=FilePart(- file=FileWithBytes(- bytes=source.get("data"),- mime_type=source.get("media_type"),- ),- ),- ),- )-- else:- raise ValueError(- f"Unsupported source type: {source_type}",- )-- elif block_type in ["tool_use", "tool_result"]:- parts.append(- Part(- root=DataPart(- data=block,- ),- ),- )-- else:- logger.error(- "Unsupported block type %s in A2AFormatter.",- block_type,- )-- a2a_message = Message(- message_id=str(uuid.uuid4()),- role=Role.user,- parts=parts,- )-- return a2a_message-- async def format_a2a_message(self, name: str, message: Message) -> Msg:- """Convert A2A Message object back to AgentScope Msg format.-- Args:- name (`str`):- The name of the message sender.- message (`Message`):- The A2A Message object to be converted.-- Returns:- `list[Msg]`:- List of converted AgentScope Msg objects.- """-- from a2a.types import Role-- content = []- metadata = None- for part in message.parts:- content.append(- await self._format_a2a_part(part),- )-- if message.role == Role.user:- role: Literal["user", "assistant"] = "user"- elif message.role == Role.agent:- role = "assistant"- else:- raise ValueError(- f"Unsupported role: {message.role} in A2A message.",- )-- return Msg(- name=name,- role=role,- content=content,- metadata=metadata,- )-- @staticmethod- def _guess_type(- uri: str | None = None,- mime_type: str | None = None,- ) -> Literal["image", "video", "audio", "unknown"]:- """Guess the content type from the uri or mime type.-- Args:- uri (`str | None`, optional):- The uri of the content.- mime_type (`str | None`, optional):- The mime type of the content.-- Returns:- `Literal["image", "video", "audio", "unknown"]`:- The guessed content type.- """- if mime_type is None and uri is None:- raise ValueError(- "Either uri or mime_type must be provided to guess the"- " content type.",- )-- if mime_type is None:- mime_type, _encoding = mimetypes.guess_type(uri or "")-- if isinstance(mime_type, str):- if mime_type.startswith("image/"):- return "image"-- if mime_type.startswith("video/"):- return "video"-- if mime_type.startswith("audio/"):- return "audio"-- return "unknown"-- async def format_a2a_task(self, name: str, task: Task) -> list[Msg]:- """Convert A2A Task object back to AgentScope Msg format.-- Args:- name (`str`):- The name of the message sender.- task (`Task`):- The A2A Task object to be converted.-- Returns:- `list[Msg]`:- Converted AgentScope Msg objects.- """- msgs = []- if task.status and task.status.message:- msgs.append(- await self.format_a2a_message(name, task.status.message),- )-- merged_msgs = []- for msg in msgs:- if merged_msgs and merged_msgs[-1].role == msg.role:- merged_msgs[-1].content.extend(msg.content)-- else:- merged_msgs.append(msg)-- if task.artifacts:- for artifact in task.artifacts:- artifact_content = [- await self._format_a2a_part(_) for _ in artifact.parts- ]-- if merged_msgs and merged_msgs[-1].role == "assistant":- merged_msgs[-1].content.extend(artifact_content)- merged_msgs[-1].metadata = artifact.metadata-- else:- merged_msgs.append(- Msg(- name=name,- role="assistant",- content=artifact_content,- metadata=artifact.metadata,- ),- )-- return merged_msgs-- async def _format_a2a_part(self, part: Part) -> ContentBlock:- """Convert a single A2A Part object into AgentScope ContentBlock.-- .. note:: We will try to convert the `DataPart` into tool use and tool- result blocks if possible.-- Args:- part (`Part`):- The A2A Part object to be converted.-- Returns:- `ContentBlock`:- The converted AgentScope ContentBlock.- """-- from a2a.types import (- TextPart,- FilePart,- FileWithUri,- FileWithBytes,- DataPart,- )-- if isinstance(part.root, TextPart):- return TextBlock(- type="text",- text=part.root.text,- )-- if isinstance(part.root, FilePart):- if isinstance(part.root.file, FileWithUri):- return { # type: ignore[return-value, misc]- "type": self._guess_type(- part.root.file.uri,- part.root.file.mime_type,- ),- "source": URLSource(- type="url",- url=part.root.file.uri,- ),- }-- if isinstance(part.root.file, FileWithBytes):- return { # type: ignore[return-value, misc]- "type": self._guess_type(- mime_type=part.root.file.mime_type,- ),- "source": Base64Source(- type="base64",- media_type=part.root.file.mime_type- or "application/octet-stream",- data=part.root.file.bytes,- ),- }-- raise ValueError(- f"Unsupported File type: {type(part.root.file)} in A2A"- "message.",- )-- if isinstance(part.root, DataPart):- # Maybe the tool use and tool result blocks- if {- "type",- "name",- "input",- "id",- } <= part.root.data.keys() and part.root.data[- "type"- ] == "tool_use":- return part.root.data-- if {- "type",- "name",- "output",- "id",- } <= part.root.data.keys() and part.root.data[- "type"- ] == "tool_result":- return part.root.data-- # TODO: what about the other data parts?- return TextBlock(- type="text",- text=str(part.root.data),- )-- raise ValueError(- f"Unsupported Part type: {type(part.root)} in A2A message"- f": {part.root}",- )diff --git a/src/agentscope/formatter/_anthropic_formatter.py b/src/agentscope/formatter/_anthropic_formatter.pyindex 3a1818325..0d2061d29 100644--- a/src/agentscope/formatter/_anthropic_formatter.py+++ b/src/agentscope/formatter/_anthropic_formatter.py@@ -1,41 +1,34 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches"""The Anthropic formatter module."""-+import base64+import fnmatch+from abc import ABCfrom typing import Any-from ._truncated_formatter_base import TruncatedFormatterBase-from .._logging import logger-from ..message import Msg, TextBlock, ImageBlock, ToolUseBlock, ToolResultBlock-from ..token import TokenCounterBase---class AnthropicChatFormatter(TruncatedFormatterBase):- """The Anthropic formatter class for chatbot scenario, where only a user- and an agent are involved. We use the `role` field to identify different- entities in the conversation.- """-- support_tools_api: bool = True- """Whether support tools API"""+import requests- support_multiagent: bool = False- """Whether support multi-agent conversations"""-- support_vision: bool = True- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Multimodal- ImageBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""-- async def _format(+from ._formatter_base import FormatterBase+from .._logging import logger+from ..message import (+ Msg,+ TextBlock,+ ThinkingBlock,+ HintBlock,+ DataBlock,+ ToolCallBlock,+ ToolResultBlock,+ URLSource,+ Base64Source,+)+++class _AnthropicFormatterBase(FormatterBase, ABC):+ """Mixin for formatting Anthropic formatters to avoid duplication between+ AnthropicChatFormatter and AnthropicMultiAgentFormatter."""++ supported_input_media_types: list[str]++ async def _format_messages(self,msgs: list[Msg],) -> list[dict[str, Any]]:@@ -58,92 +51,263 @@ class AnthropicChatFormatter(TruncatedFormatterBase):self.assert_list_of_msgs(msgs)messages: list[dict] = []- for index, msg in enumerate(msgs):- content_blocks = []+ for msg in msgs:+ content_blocks: list = []for block in msg.get_content_blocks():- typ = block.get("type")- if typ in ["thinking", "text", "image"]:- content_blocks.append({**block})+ if isinstance(block, TextBlock):+ content_blocks.append(+ {"type": "text", "text": block.text},+ )- elif typ == "tool_use":+ elif isinstance(block, ThinkingBlock):+ content_blocks.append(+ {"type": "thinking", "thinking": block.thinking},+ )++ elif isinstance(block, HintBlock):+ pass # Anthropic does not support hint blocks++ elif isinstance(block, DataBlock):+ formatted_block = self._format_anthropic_data_block(block)+ if formatted_block:+ content_blocks.append(formatted_block)++ elif isinstance(block, ToolCallBlock):content_blocks.append({- "id": block.get("id"),"type": "tool_use",- "name": block.get("name"),- "input": block.get("input", {}),+ "id": block.id,+ "name": block.name,+ "input": block.input,},)- elif typ == "tool_result":- output = block.get("output")- if output is None:- content_value = [{"type": "text", "text": None}]- elif isinstance(output, list):- content_value = output- else:- content_value = [{"type": "text", "text": str(output)}]- messages.append(- {- "role": "user",- "content": [+ elif isinstance(block, ToolResultBlock):+ (+ textual_output,+ multimodal_data,+ ) = self.convert_tool_result_to_string(block.output)++ tool_result_content = []+ if textual_output:+ tool_result_content.append(+ {"type": "text", "text": textual_output},+ )++ for data_block in multimodal_data:+ formatted_block = self._format_anthropic_data_block(+ data_block,+ )+ if formatted_block:+ tool_result_content.append(formatted_block)++ # If there's multimodal data, promote to UserMsg+ if multimodal_data:+ # Add tool result first+ content_blocks.append(+ {+ "type": "tool_result",+ "tool_use_id": block.id,+ "content": tool_result_content,+ },+ )++ # Flush current message+ if content_blocks:+ messages.append({- "type": "tool_result",- "tool_use_id": block.get("id"),- "content": content_value,+ "role": msg.role,+ "content": content_blocks,},- ],- },- )+ )+ content_blocks = []++ # Insert UserMsg with multimodal data+ user_content = []+ for data_block in multimodal_data:+ formatted_block = (+ self._format_anthropic_data_block(data_block)+ )+ if formatted_block:+ user_content.append(formatted_block)++ if user_content:+ messages.append(+ {+ "role": "user",+ "content": user_content,+ },+ )+ else:+ # No multimodal data, just add tool result+ content_blocks.append(+ {+ "type": "tool_result",+ "tool_use_id": block.id,+ "content": tool_result_content,+ },+ )+else:logger.warning("Unsupported block type %s in the message, skipped.",- typ,+ type(block),)- # Claude only allow the first message to be system message- if msg.role == "system" and index != 0:- role = "user"- else:- role = msg.role+ if content_blocks:+ messages.append(+ {+ "role": msg.role,+ "content": content_blocks,+ },+ )- msg_anthropic = {- "role": role,- "content": content_blocks or None,- }+ return messages++ def _format_anthropic_data_block(+ self,+ block: DataBlock,+ ) -> dict[str, Any] | None:+ """Format a DataBlock into Anthropic API format.- # When both content and tool_calls are None, skipped- if msg_anthropic["content"] or msg_anthropic.get("tool_calls"):- messages.append(msg_anthropic)+ Args:+ block (`DataBlock`):+ The data block to format.- return messages+ Returns:+ `dict[str, Any] | None`:+ The formatted data block, or None if the media type is not+ supported.+ """+ source = block.source+ media_type = source.media_type++ # Check if media type is supported+ if not any(+ fnmatch.fnmatch(media_type, pattern)+ for pattern in self.supported_input_media_types+ ):+ logger.warning(+ "Media type %s is not supported, skipped.",+ media_type,+ )+ return None+ # Anthropic only supports images+ if not media_type.startswith("image/"):+ logger.warning(+ "Anthropic only supports image data, got %s, skipped.",+ media_type,+ )+ return None-class AnthropicMultiAgentFormatter(TruncatedFormatterBase):- """- Anthropic formatter for multi-agent conversations, where more than- a user and an agent are involved.+ return self._format_image_source(source)++ @staticmethod+ def _format_image_source(+ source: URLSource | Base64Source,+ ) -> dict[str, Any]:+ """Format an image source into Anthropic API format.++ Args:+ source (`URLSource | Base64Source`):+ The image source to format.++ Returns:+ `dict[str, Any]`:+ The formatted image source.+ """+ if isinstance(source, Base64Source):+ return {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": source.media_type,+ "data": source.data,+ },+ }+ elif isinstance(source, URLSource):+ url = source.url+ if url.startswith("file://"):+ # Local file - read and convert to base64+ file_path = url.removeprefix("file://")+ with open(file_path, "rb") as f:+ data = base64.b64encode(f.read()).decode("utf-8")+ return {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": source.media_type,+ "data": data,+ },+ }+ else:+ # Remote URL - download and convert to base64+ response = requests.get(url, timeout=30)+ response.raise_for_status()+ data = base64.b64encode(response.content).decode("utf-8")+ return {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": source.media_type,+ "data": data,+ },+ }+ else:+ raise ValueError(f"Unsupported source type: {type(source)}")+++class AnthropicChatFormatter(_AnthropicFormatterBase):+ """The Anthropic formatter class for chatbot scenario, where only a user+ and an agent are involved. We use the `role` field to identify different+ entities in the conversation."""- support_tools_api: bool = True- """Whether support tools API"""+ def __init__(+ self,+ supported_input_media_types: list[str] | None = None,+ ) -> None:+ """Initialize the Anthropic chat formatter.++ Args:+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types. Defaults to+ ``["image/*"]``.+ """+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*"],+ )++ async def format(+ self,+ msgs: list[Msg],+ ) -> list[dict[str, Any]]:+ """Format message objects into Anthropic API format.++ Args:+ msgs (`list[Msg]`):+ The list of message objects to format.- support_multiagent: bool = True- """Whether support multi-agent conversations"""+ Returns:+ `list[dict[str, Any]]`:+ The formatted messages as a list of dictionaries.++ .. note:: Anthropic suggests always passing all previous thinking+ blocks back to the API in subsequent calls to maintain reasoning+ continuity. For more details, please refer to+ `Anthropic's documentation+ <https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#preserving-thinking-blocks>`_.+ """+ return await self._format_messages(msgs)- support_vision: bool = True- """Whether support vision data"""- supported_blocks: list[type] = [- TextBlock,- # Multimodal- ImageBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""+class AnthropicMultiAgentFormatter(_AnthropicFormatterBase):+ """Anthropic formatter for multi-agent conversations, where more than+ a user and an agent are involved.+ """def __init__(self,@@ -152,102 +316,143 @@ class AnthropicMultiAgentFormatter(TruncatedFormatterBase):"The content between <history></history> tags contains ""your conversation history\n"),- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:- """Initialize the DashScope multi-agent formatter.+ """Initialize the Anthropic multi-agent formatter.Args:conversation_history_prompt (`str`):The prompt to use for the conversation history section.+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types. Defaults to+ ``["image/*"]``."""- super().__init__(token_counter=token_counter, max_tokens=max_tokens)+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*"],+ )self.conversation_history_prompt = conversation_history_prompt- async def _format_tool_sequence(- self,- msgs: list[Msg],- ) -> list[dict[str, Any]]:- """Given a sequence of tool call/result messages, format them into- the required format for the Anthropic API."""- return await AnthropicChatFormatter().format(msgs)+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:+ """Format input messages into the structure required by the Anthropic+ API for multi-agent conversations."""+ self.assert_list_of_msgs(msgs)++ formatted_msgs = []+ start_index = 0+ if len(msgs) > 0 and msgs[0].role == "system":+ formatted_msgs.append(+ await self._format_system_message(msgs[0]),+ )+ start_index = 1++ is_first_agent_message = True+ async for typ, group in self._group_messages(msgs[start_index:]):+ match typ:+ case "tool_sequence":+ formatted_msgs.extend(+ await self._format_messages(group),+ )+ case "agent_message":+ formatted_msgs.extend(+ await self._format_agent_message(+ group,+ is_first_agent_message,+ ),+ )+ is_first_agent_message = False++ return formatted_msgsasync def _format_agent_message(self,msgs: list[Msg],- is_first: bool = True,+ is_first: bool,) -> list[dict[str, Any]]:- """Given a sequence of messages without tool calls/results, format- them into the required format for the Anthropic API."""-- if is_first:- conversation_history_prompt = self.conversation_history_prompt- else:- conversation_history_prompt = ""-- # Format into required Anthropic format- formatted_msgs: list[dict] = []-- # Collect the multimodal files- conversation_blocks: list = []+ """Format agent messages into conversation history."""+ conversation_blocks = []accumulated_text = []+for msg in msgs:- for block in msg.get_content_blocks():- if block["type"] == "text":- accumulated_text.append(f"{msg.name}: {block['text']}")+ agent_name = msg.name or "Agent"+ agent_text_parts = []- elif block["type"] == "image":- # Handle the accumulated text as a single block- if accumulated_text:- conversation_blocks.append(- {- "text": "\n".join(accumulated_text),- "type": "text",- },- )- accumulated_text.clear()+ for block in msg.get_content_blocks():+ if isinstance(block, TextBlock):+ agent_text_parts.append(block.text)+ elif isinstance(block, DataBlock):+ formatted_block = self._format_anthropic_data_block(block)+ if formatted_block:+ if accumulated_text:+ conversation_blocks.append(+ {+ "type": "text",+ "text": "\n".join(accumulated_text),+ },+ )+ accumulated_text = []+ conversation_blocks.append(formatted_block)- conversation_blocks.append({**block})+ if agent_text_parts:+ agent_message = f"{agent_name}: {' '.join(agent_text_parts)}"+ accumulated_text.append(agent_message)if accumulated_text:conversation_blocks.append({- "text": "\n".join(accumulated_text),"type": "text",+ "text": "\n".join(accumulated_text),},)- if conversation_blocks:+ if conversation_blocks and is_first:if conversation_blocks[0].get("text"):conversation_blocks[0]["text"] = (- conversation_history_prompt+ self.conversation_history_prompt+ "<history>\n"+ conversation_blocks[0]["text"])-else:conversation_blocks.insert(0,{"type": "text",- "text": conversation_history_prompt + "<history>\n",+ "text": self.conversation_history_prompt+ + "<history>\n",},)if conversation_blocks[-1].get("text"):conversation_blocks[-1]["text"] += "\n</history>"-else:conversation_blocks.append({"type": "text", "text": "</history>"},)if conversation_blocks:- formatted_msgs.append(+ return [{"role": "user","content": conversation_blocks,},- )+ ]- return formatted_msgs+ return []++ @staticmethod+ async def _format_system_message(msg: Msg) -> dict[str, Any]:+ """Format a system message."""+ text_parts = []+ for block in msg.get_content_blocks():+ if isinstance(block, TextBlock):+ text_parts.append(block.text)++ return {+ "role": "system",+ "content": [+ {+ "type": "text",+ "text": "\n".join(text_parts),+ },+ ],+ }diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.pyindex 766876b99..56913d23e 100644--- a/src/agentscope/formatter/_dashscope_formatter.py+++ b/src/agentscope/formatter/_dashscope_formatter.py@@ -1,150 +1,73 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches"""The dashscope formatter module."""-import json-import os.pathfrom typing import Any+from fnmatch import fnmatch+from abc import ABC-from ._truncated_formatter_base import TruncatedFormatterBase+from . import FormatterBasefrom .._logging import logger-from .._utils._common import _is_accessible_local_filefrom ..message import (Msg,TextBlock,- ImageBlock,- AudioBlock,- VideoBlock,- ToolUseBlock,ToolResultBlock,URLSource,+ DataBlock,+ ToolCallBlock,+ Base64Source,+ UserMsg,+ HintBlock,)-from ..token import TokenCounterBase-def _format_dashscope_media_block(- block: ImageBlock | AudioBlock,-) -> dict[str, str]:- """Format an image or audio block for DashScope API.+class _DashScopeFormatterBase(FormatterBase, ABC):+ """Base class for DashScope formatters, providing shared data block+ formatting logic."""- Args:- block (`ImageBlock` | `AudioBlock`):- The image or audio block to format.-- Returns:- `dict[str, str]`:- A dictionary with "image" or "audio" key and the formatted URL or- data URI as value.-- Raises:- `NotImplementedError`:- If the source type is not supported.- """- typ = block["type"]- source = block["source"]- if source["type"] == "url":- url = source["url"]- if _is_accessible_local_file(url):- return {typ: "file://" + os.path.abspath(url)}- else:- # treat as web url- return {typ: url}-- elif source["type"] == "base64":- media_type = source["media_type"]- base64_data = source["data"]- return {- typ: f"data:{media_type};base64,{base64_data}",- }-- else:- raise NotImplementedError(- f"Unsupported source type '{source.get('type')}' "- f"for {typ} block.",- )+ supported_input_media_types: list[str]+ def _format_dashscope_data_block(+ self,+ block: DataBlock,+ ) -> dict[str, Any] | None:+ """Format a DataBlock into the required format for DashScope API.-def _reformat_messages(- messages: list[dict[str, Any]],-) -> list[dict[str, Any]]:- """Reformat the content to be compatible with HuggingFaceTokenCounter.-- This function processes a list of messages and converts multi-part- text content into single string content when all parts are plain text.- This is necessary for compatibility with HuggingFaceTokenCounter which- expects simple string content rather than structured content with- multiple parts.-- Args:- messages (list[dict[str, Any]]):- A list of message dictionaries where each message may contain a- "content" field. The content can be either:- - A string (unchanged)- - A list of content items, where each item is a dict that may- contain "text", "type", and other fields-- Returns:- list[dict[str, Any]]:- A list of reformatted messages. For messages where all content- items are plain text (have "text" field and either no "type"- field or "type" == "text"), the content list is converted to a- single newline-joined string. Other messages remain unchanged.-- Example:- .. code-block:: python-- # Case 1: All text content - will be converted- messages = [- {- "role": "user",- "content": [- {"text": "Hello", "type": "text"},- {"text": "World", "type": "text"}- ]- }- ]- result = _reformat_messages(messages)- print(result[0]["content"])- # Output: "Hello\nWorld"-- # Case 2: Mixed content - will remain unchanged- messages = [- {- "role": "user",- "content": [- {"text": "Hello", "type": "text"},- {"image_url": "...", "type": "image"}- ]- }- ]+ Args:+ block (`DataBlock`):+ The DataBlock to format.- result = _reformat_messages(messages) # remain unchanged- print(type(result[0]["content"]))- # Output: <class 'list'>+ Returns:+ `dict[str, Any] | None`:+ A dictionary representing the formatted DataBlock for+ DashScope API.+ """+ if not any(+ fnmatch(block.source.media_type, pattern)+ for pattern in self.supported_input_media_types+ ):+ logger.warning(+ "Unsupported media type %s for DashScope API. Supported "+ "types: %s. This block will be skipped.",+ block.source.media_type,+ ", ".join(self.supported_input_media_types),+ )+ return None- """- for message in messages:- content = message.get("content", [])+ main_type = block.source.media_type.split("/")[0]- is_all_text = True- texts = []- for item in content:- if not isinstance(item, dict) or "text" not in item:- is_all_text = False- break- if "type" in item and item["type"] != "text":- is_all_text = False- break- if item["text"]:- texts.append(item["text"])+ if isinstance(block.source, URLSource):+ return {main_type: block.source.url}- if is_all_text and texts:- message["content"] = "\n".join(texts)+ if isinstance(block.source, Base64Source):+ return {+ main_type: f"data:{block.source.media_type};"+ f"base64,{block.source.data}",+ }- return messages+ return None-class DashScopeChatFormatter(TruncatedFormatterBase):+class DashScopeChatFormatter(_DashScopeFormatterBase):"""The DashScope formatter class for chatbot scenario, where only a userand an agent are involved. We use the `role` field to identify differententities in the conversation.@@ -165,71 +88,25 @@ class DashScopeChatFormatter(TruncatedFormatterBase):To avoid these issues, this formatter assigns content as an emptylist ``[]`` for messages without valid content blocks.-"""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = False- """Whether support multi-agent conversations"""-- support_vision: bool = True- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- ImageBlock,- AudioBlock,- VideoBlock,- ToolUseBlock,- ToolResultBlock,- ]-def __init__(self,- promote_tool_result_images: bool = False,- promote_tool_result_audios: bool = False,- promote_tool_result_videos: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_type: list[str] | None = None,) -> None:"""Initialize the DashScope chat formatter.Args:- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- promote_tool_result_audios (`bool`, defaults to `False`):- Whether to promote audios from tool results to user messages.- Most LLM APIs don't support audios in tool result blocks, but- do support them in user message blocks. When `True`, audios are- extracted and appended as a separate user message with- explanatory text indicating their source.- promote_tool_result_videos (`bool`, defaults to `False`):- Whether to promote videos from tool results to user messages.- Most LLM APIs don't support videos in tool result blocks, but- do support them in user message blocks. When `True`, videos are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- A token counter instance used to count tokens in the messages.- If not provided, the formatter will format the messages- without considering token limits.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If not provided, the formatter will not truncate- the messages.+ supported_input_media_type (`list[str] | None`, optional):+ The list of supported input media types. Defaults to+ ["image/*", "audio/*", "video/*"]."""- super().__init__(token_counter, max_tokens)- self.promote_tool_result_images = promote_tool_result_images- self.promote_tool_result_audios = promote_tool_result_audios- self.promote_tool_result_videos = promote_tool_result_videos+ super().__init__(+ supported_input_media_types=supported_input_media_type+ or ["image/*", "audio/*", "video/*"],+ )- async def _format(+ async def format(self,msgs: list[Msg],) -> list[dict[str, Any]]:@@ -246,153 +123,79 @@ class DashScopeChatFormatter(TruncatedFormatterBase):self.assert_list_of_msgs(msgs)formatted_msgs: list[dict] = []-i = 0while i < len(msgs):msg = msgs[i]- content_blocks: list[dict[str, Any]] = []+ content_blocks: list[dict] = []tool_calls = []for block in msg.get_content_blocks():- typ = block.get("type")-- if typ == "text":- content_blocks.append(- {- "text": block.get("text"),- },- )-- elif typ in ["image", "audio", "video"]:- content_blocks.append(- _format_dashscope_media_block(- block, # type: ignore[arg-type]- ),- )+ if isinstance(block, TextBlock):+ content_blocks.append({"text": block.text})++ elif isinstance(block, DataBlock):+ formatted_block = self._format_dashscope_data_block(block)+ if formatted_block:+ content_blocks.append(formatted_block)++ elif isinstance(block, HintBlock):+ # Insert a new user message with the hint content right+ # after the current message, and go on processing the+ # rest of the blocks in the current message+ if content_blocks or tool_calls:+ formatted_msgs.append(+ {+ "role": "user",+ "content": content_blocks,+ "tool_calls": tool_calls+ if tool_calls+ else None,+ },+ )+ # Refresh content_blocks and tool_calls for the last+ # blocks+ content_blocks = []+ tool_calls = []- elif typ == "tool_use":+ elif isinstance(block, ToolCallBlock):tool_calls.append({- "id": block.get("id"),+ "id": block.id,"type": "function","function": {- "name": block.get("name"),- "arguments": json.dumps(- block.get("input", {}),- ensure_ascii=False,- ),+ "name": block.name,+ "arguments": block.input,},},)- elif typ == "tool_result":+ elif isinstance(block, ToolResultBlock):(textual_output,multimodal_data,- ) = self.convert_tool_result_to_string(block["output"])+ ) = self.convert_tool_result_to_string(block.output)# First add the tool result message in DashScope API formatformatted_msgs.append({"role": "tool",- "tool_call_id": block.get("id"),+ "tool_call_id": block.id,"content": textual_output,- "name": block.get("name"),+ "name": block.name,},)- # Then, handle the multimodal data if any- promoted_blocks: list = []- for url, multimodal_block in multimodal_data:- if (- multimodal_block["type"] == "image"- and self.promote_tool_result_images- ):- promoted_blocks.extend(- [- TextBlock(- type="text",- text=f"\n- The image from '{url}': ",- ),- ImageBlock(- type="image",- source=URLSource(- type="url",- url=url,- ),- ),- ],- )- elif (- multimodal_block["type"] == "audio"- and self.promote_tool_result_audios- ):- promoted_blocks.extend(- [- TextBlock(- type="text",- text=f"\n- The audio from '{url}': ",- ),- AudioBlock(- type="audio",- source=URLSource(- type="url",- url=url,- ),- ),- ],- )- elif (- multimodal_block["type"] == "video"- and self.promote_tool_result_videos- ):- promoted_blocks.extend(- [- TextBlock(- type="text",- text=f"\n- The video from '{url}': ",- ),- VideoBlock(- type="video",- source=URLSource(- type="url",- url=url,- ),- ),- ],- )-- if promoted_blocks:- # Insert promoted blocks as new user message(s)- promoted_blocks = [- TextBlock(- type="text",- text="<system-info>The following are "- f"the media contents from the tool "- f"result of '{block['name']}':",- ),- *promoted_blocks,- TextBlock(- type="text",- text="</system-info>",- ),- ]-+ # If we have multimodal data that needs to be prompted to+ # the out-tool-result message+ if multimodal_data:msgs.insert(i + 1,- Msg(- name="user",- content=promoted_blocks,- role="user",+ UserMsg(+ name="system-reminder",+ content=multimodal_data,),)- else:- logger.warning(- "Unsupported block type %s in the message, skipped.",- typ,- )-msg_dashscope = {"role": msg.role,"content": content_blocks,@@ -407,10 +210,26 @@ class DashScopeChatFormatter(TruncatedFormatterBase):# Move to next messagei += 1- return _reformat_messages(formatted_msgs)+ # Merge adjacent text block into one block to avoid API issues+ cleaned_msgs: list = []+ for msg in formatted_msgs:+ new_content = []+ for block in msg["content"]:+ if (+ block.get("text")+ and new_content+ and new_content[-1].get("text")+ ):+ new_content[-1]["text"] += "\n" + block["text"]+ else:+ new_content.append(block)+ msg["content"] = new_content+ cleaned_msgs.append(msg)++ return cleaned_msgs-class DashScopeMultiAgentFormatter(TruncatedFormatterBase):+class DashScopeMultiAgentFormatter(_DashScopeFormatterBase):"""DashScope formatter for multi-agent conversations, where more thana user and an agent are involved.@@ -429,27 +248,6 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):"""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = True- """Whether support multi-agent conversations"""-- support_vision: bool = True- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Multimodal- ImageBlock,- AudioBlock,- VideoBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""-def __init__(self,conversation_history_prompt: str = (@@ -457,46 +255,67 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):"The content between <history></history> tags contains ""your conversation history\n"),- promote_tool_result_images: bool = False,- promote_tool_result_audios: bool = False,- promote_tool_result_videos: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:"""Initialize the DashScope multi-agent formatter.Args:conversation_history_prompt (`str`):The prompt to use for the conversation history section.- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- promote_tool_result_audios (`bool`, defaults to `False`):- Whether to promote audios from tool results to user messages.- Most LLM APIs don't support audios in tool result blocks, but- do support them in user message blocks. When `True`, audios are- extracted and appended as a separate user message with- explanatory text indicating their source.- promote_tool_result_videos (`bool`, defaults to `False`):- Whether to promote videos from tool results to user messages.- Most LLM APIs don't support videos in tool result blocks, but- do support them in user message blocks. When `True`, videos are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- The token counter used for truncation.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If `None`, no truncation will be applied.+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types. Defaults to+ ["image/*", "audio/*", "video/*"]."""- super().__init__(token_counter=token_counter, max_tokens=max_tokens)+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*", "audio/*", "video/*"],+ )self.conversation_history_prompt = conversation_history_prompt- self.promote_tool_result_images = promote_tool_result_images- self.promote_tool_result_audios = promote_tool_result_audios- self.promote_tool_result_videos = promote_tool_result_videos++ async def format(self, msgs: list[Msg]) -> list[dict]:+ """Format input messages into the structure required by the DashScope+ API.++ To support multi-agent conversations, this formatter processes messages+ as follows:++ - Prepends an instruction before the first conversation history+ section.+ - Combines conversation turns into a history section, where each entry+ is formatted as `{name}: {content}`.+ - Wraps the conversation history with `<history>` and `</history>`+ tags.++ Returns:+ `list[dict[str, Any]]`:+ A list of dictionaries formatted for the DashScope API.+ """++ formatted_msgs = []+ start_index = 0+ if len(msgs) > 0 and msgs[0].role == "system":+ formatted_msgs.append(+ await self._format_system_message(msgs[0]),+ )+ start_index = 1++ is_first_agent_message = True+ async for typ, group in self._group_messages(msgs[start_index:]):+ match typ:+ case "tool_sequence":+ formatted_msgs.extend(+ await self._format_tool_sequence(group),+ )+ case "agent_message":+ formatted_msgs.extend(+ await self._format_agent_message(+ group,+ is_first_agent_message,+ ),+ )+ is_first_agent_message = False++ return formatted_msgsasync def _format_tool_sequence(self,@@ -514,9 +333,7 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):A list of dictionaries formatted for the DashScope API."""return await DashScopeChatFormatter(- promote_tool_result_images=self.promote_tool_result_images,- promote_tool_result_audios=self.promote_tool_result_audios,- promote_tool_result_videos=self.promote_tool_result_videos,+ supported_input_media_type=self.supported_input_media_types,).format(msgs)async def _format_agent_message(@@ -539,7 +356,6 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):`list[dict[str, Any]]`:A list of dictionaries formatted for the DashScope API."""-if is_first:conversation_history_prompt = self.conversation_history_promptelse:@@ -553,10 +369,10 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):accumulated_text = []for msg in msgs:for block in msg.get_content_blocks():- if block["type"] == "text":+ if isinstance(block, TextBlock):accumulated_text.append(f"{msg.name}: {block['text']}")- elif block["type"] in ["image", "audio", "video"]:+ elif isinstance(block, DataBlock):# Handle the accumulated text as a single blockif accumulated_text:conversation_blocks.append(@@ -564,35 +380,9 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):)accumulated_text.clear()- if block["source"]["type"] == "url":- url = block["source"]["url"]- if _is_accessible_local_file(url):- conversation_blocks.append(- {- block["type"]: "file://"- + os.path.abspath(url),- },- )- else:- conversation_blocks.append({block["type"]: url})-- elif block["source"]["type"] == "base64":- media_type = block["source"]["media_type"]- base64_data = block["source"]["data"]- conversation_blocks.append(- {- block[- "type"- ]: f"data:{media_type};base64,{base64_data}",- },- )-- else:- logger.warning(- "Unsupported block type %s in the message, "- "skipped.",- block["type"],- )+ formatted_block = self._format_dashscope_data_block(block)+ if formatted_block is not None:+ conversation_blocks.append(formatted_block)if accumulated_text:conversation_blocks.append({"text": "\n".join(accumulated_text)})@@ -613,23 +403,32 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):},)- if conversation_blocks[-1].get("text"):- conversation_blocks[-1]["text"] += "\n</history>"-- else:- conversation_blocks.append({"text": "</history>"})+ conversation_blocks.append({"text": "</history>"})++ # Merge the adjacent text blocks into one text blocks to avoid API+ # issues+ new_content = []+ for block in conversation_blocks:+ if (+ block.get("text")+ and new_content+ and new_content[-1].get("text")+ ):+ new_content[-1]["text"] += "\n" + block["text"]+ else:+ new_content.append(block)formatted_msgs.append({"role": "user",- "content": conversation_blocks,+ "content": new_content,},)- return _reformat_messages(formatted_msgs)+ return formatted_msgs+ @staticmethodasync def _format_system_message(- self,msg: Msg,) -> dict[str, Any]:"""Format system message for DashScope API."""diff --git a/src/agentscope/formatter/_deepseek_formatter.py b/src/agentscope/formatter/_deepseek_formatter.pyindex f9dce3953..9328961ee 100644--- a/src/agentscope/formatter/_deepseek_formatter.py+++ b/src/agentscope/formatter/_deepseek_formatter.py@@ -1,39 +1,34 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches"""The DeepSeek formatter module."""-import jsonfrom typing import Any-from ._truncated_formatter_base import TruncatedFormatterBase+from ._formatter_base import FormatterBasefrom .._logging import logger-from ..message import Msg, TextBlock, ToolUseBlock, ToolResultBlock-from ..token import TokenCounterBase+from ..message import (+ Msg,+ TextBlock,+ ThinkingBlock,+ HintBlock,+ ToolCallBlock,+ ToolResultBlock,+)-class DeepSeekChatFormatter(TruncatedFormatterBase):+class DeepSeekChatFormatter(FormatterBase):"""The DeepSeek formatter class for chatbot scenario, where only a userand an agent are involved. We use the `role` field to identify differententities in the conversation."""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = False- """Whether support multi-agent conversations"""-- support_vision: bool = False- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""+ def __init__(+ self,+ supported_input_media_types: list[str] | None = None,+ ) -> None:+ super().__init__(+ supported_input_media_types=supported_input_media_types or [],+ )- async def _format(+ async def format(self,msgs: list[Msg],) -> list[dict[str, Any]]:@@ -56,60 +51,57 @@ class DeepSeekChatFormatter(TruncatedFormatterBase):tool_calls = []for block in msg.get_content_blocks():- typ = block.get("type")- if typ == "text":- content_blocks.append({**block})- elif typ == "thinking":- reasoning_content_blocks.append({**block})+ if isinstance(block, TextBlock):+ content_blocks.append({"type": "text", "text": block.text})- elif typ == "tool_use":+ elif isinstance(block, ThinkingBlock):+ reasoning_content_blocks.append(block.thinking)++ elif isinstance(block, HintBlock):+ pass # DeepSeek does not support hint blocks++ elif isinstance(block, ToolCallBlock):tool_calls.append({- "id": block.get("id"),+ "id": block.id,"type": "function","function": {- "name": block.get("name"),- "arguments": json.dumps(- block.get("input", {}),- ensure_ascii=False,- ),+ "name": block.name,+ "arguments": block.input,},},)- elif typ == "tool_result":+ elif isinstance(block, ToolResultBlock):textual_output, _ = self.convert_tool_result_to_string(- block.get("output"), # type: ignore[arg-type]+ block.output,)messages.append({"role": "tool",- "tool_call_id": block.get("id"),+ "tool_call_id": block.id,"content": textual_output,- "name": block.get("name"),+ "name": block.name,},)else:logger.warning("Unsupported block type %s in the message, skipped.",- typ,+ type(block),)- content_msg = "\n".join(- content.get("text", "") for content in content_blocks- )- reasoning_msg = "\n".join(- reasoning.get("thinking", "")- for reasoning in reasoning_content_blocks- )- msg_deepseek = {+ content_msg = "\n".join(b.get("text", "") for b in content_blocks)++ msg_deepseek: dict[str, Any] = {"role": msg.role,"content": content_msg or None,}- if reasoning_msg:- msg_deepseek["reasoning_content"] = reasoning_msg+ if reasoning_content_blocks:+ msg_deepseek["reasoning_content"] = "\n".join(+ reasoning_content_blocks,+ )if tool_calls:msg_deepseek["tool_calls"] = tool_calls@@ -120,29 +112,12 @@ class DeepSeekChatFormatter(TruncatedFormatterBase):return messages-class DeepSeekMultiAgentFormatter(TruncatedFormatterBase):+class DeepSeekMultiAgentFormatter(FormatterBase):"""DeepSeek formatter for multi-agent conversations, where more thana user and an agent are involved."""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = True- """Whether support multi-agent conversations"""-- support_vision: bool = False- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""-def __init__(self,conversation_history_prompt: str = (@@ -150,42 +125,62 @@ class DeepSeekMultiAgentFormatter(TruncatedFormatterBase):"The content between <history></history> tags contains ""your conversation history\n"),- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:"""Initialize the DeepSeek multi-agent formatter.Args:conversation_history_prompt (`str`):The prompt to use for the conversation history section.- token_counter (`TokenCounterBase | None`, optional):- A token counter instance used to count tokens in the messages.- If not provided, the formatter will format the messages- without considering token limits.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If not provided, the formatter will not truncate- the messages.+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types. Defaults to ``[]``+ (DeepSeek does not support multimodal input)."""- super().__init__(token_counter=token_counter, max_tokens=max_tokens)+ super().__init__(+ supported_input_media_types=supported_input_media_types or [],+ )self.conversation_history_prompt = conversation_history_prompt+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:+ """Format input messages into the structure required by the DeepSeek+ API for multi-agent conversations."""+ self.assert_list_of_msgs(msgs)++ formatted_msgs = []+ start_index = 0+ if len(msgs) > 0 and msgs[0].role == "system":+ formatted_msgs.append(+ await self._format_system_message(msgs[0]),+ )+ start_index = 1++ is_first_agent_message = True+ async for typ, group in self._group_messages(msgs[start_index:]):+ match typ:+ case "tool_sequence":+ formatted_msgs.extend(+ await self._format_tool_sequence(group),+ )+ case "agent_message":+ formatted_msgs.extend(+ await self._format_agent_message(+ group,+ is_first_agent_message,+ ),+ )+ is_first_agent_message = False++ return formatted_msgs+async def _format_tool_sequence(self,msgs: list[Msg],) -> list[dict[str, Any]]:"""Given a sequence of tool call/result messages, format them into- the required format for the DeepSeek API.-- Args:- msgs (`list[Msg]`):- The list of messages containing tool calls/results to format.-- Returns:- `list[dict[str, Any]]`:- A list of dictionaries formatted for the DeepSeek API.- """- return await DeepSeekChatFormatter().format(msgs)+ the required format for the DeepSeek API."""+ return await DeepSeekChatFormatter(+ supported_input_media_types=self.supported_input_media_types,+ ).format(msgs)async def _format_agent_message(self,@@ -193,73 +188,46 @@ class DeepSeekMultiAgentFormatter(TruncatedFormatterBase):is_first: bool = True,) -> list[dict[str, Any]]:"""Given a sequence of messages without tool calls/results, format- them into the required format for the DeepSeek API.-- Args:- msgs (`list[Msg]`):- A list of Msg objects to be formatted.- is_first (`bool`, defaults to `True`):- Whether this is the first agent message in the conversation.- If `True`, the conversation history prompt will be included.-- Returns:- `list[dict[str, Any]]`:- A list of dictionaries formatted for the DeepSeek API.- """+ them into the required format for the DeepSeek API."""if is_first:conversation_history_prompt = self.conversation_history_promptelse:conversation_history_prompt = ""- # Format into required DeepSeek formatformatted_msgs: list[dict] = []-- conversation_blocks: list = []accumulated_text = []+for msg in msgs:for block in msg.get_content_blocks():- if block["type"] == "text":- accumulated_text.append(f"{msg.name}: {block['text']}")+ if isinstance(block, TextBlock):+ accumulated_text.append(f"{msg.name}: {block.text}")+ conversation_blocks_text = ""if accumulated_text:- conversation_blocks.append(- {"text": "\n".join(accumulated_text)},+ conversation_blocks_text = (+ conversation_history_prompt+ + "<history>\n"+ + "\n".join(accumulated_text)+ + "\n</history>")- if conversation_blocks:- if conversation_blocks[0].get("text"):- conversation_blocks[0]["text"] = (- conversation_history_prompt- + "<history>\n"- + conversation_blocks[0]["text"]- )-- else:- conversation_blocks.insert(- 0,- {- "text": conversation_history_prompt + "<history>\n",- },- )-- if conversation_blocks[-1].get("text"):- conversation_blocks[-1]["text"] += "\n</history>"-- else:- conversation_blocks.append({"text": "</history>"})+ if conversation_blocks_text:+ formatted_msgs.append(+ {+ "role": "user",+ "content": conversation_blocks_text,+ },+ )- conversation_blocks_text = "\n".join(- conversation_block.get("text", "")- for conversation_block in conversation_blocks- )+ return formatted_msgs- user_message = {- "role": "user",- "content": conversation_blocks_text,+ @staticmethod+ async def _format_system_message(+ msg: Msg,+ ) -> dict[str, Any]:+ """Format system message for DeepSeek API."""+ return {+ "role": "system",+ "content": msg.get_text_content(),}-- if conversation_blocks:- formatted_msgs.append(user_message)-- return formatted_msgsdiff --git a/src/agentscope/formatter/_formatter_base.py b/src/agentscope/formatter/_formatter_base.pyindex 6abb05a99..3ac02590e 100644--- a/src/agentscope/formatter/_formatter_base.py+++ b/src/agentscope/formatter/_formatter_base.py@@ -1,16 +1,42 @@# -*- coding: utf-8 -*-"""The formatter module."""-+import base64+import mimetypes+import tempfilefrom abc import abstractmethod-from typing import Any, List, Tuple, Sequence+from fnmatch import fnmatch+from typing import Any, List, AsyncGenerator++import shortuuid-from .._utils._common import _save_base64_data-from ..message import Msg, AudioBlock, ImageBlock, TextBlock, VideoBlock+from ..message import (+ Msg,+ DataBlock,+ TextBlock,+ URLSource,+ Base64Source,+)class FormatterBase:"""The base class for formatters."""+ supported_input_media_types: list[str]+ """The supported media types for multimodal data in the input messages,+ supporting mime types like "image/*", "image/png", "audio/*", "video/*",+ etc."""++ def __init__(self, supported_input_media_types: list[str]) -> None:+ """Initialize the formatter base class.++ Args:+ supported_input_media_types (`list[str]`):+ The supported media types for multimodal data in the input+ messages, supporting mime types like "image/*", "image/png",+ "audio/*", "video/*", etc.+ """+ self.supported_input_media_types = supported_input_media_types+@abstractmethodasync def format(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:"""Format the Msg objects to a list of dictionaries that satisfy the@@ -33,18 +59,10 @@ class FormatterBase:f"Expected Msg object, got {type(msg)} instead.",)- @staticmethoddef convert_tool_result_to_string(- output: str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock],- ) -> tuple[- str,- Sequence[- Tuple[- str,- ImageBlock | AudioBlock | TextBlock | VideoBlock,- ]- ],- ]:+ self,+ output: str | List[TextBlock | DataBlock],+ ) -> tuple[str, list[TextBlock | DataBlock]]:"""Turn the tool result list into a textual output to be compatiblewith the LLM API that doesn't support multimodal data in the toolresult.@@ -54,76 +72,139 @@ class FormatterBase:is included in the returned list.Args:- output (`str | List[TextBlock | ImageBlock | AudioBlock | \- VideoBlock]`):+ output (`str | List[TextBlock | DataBlock]`):The output of the tool response, including text and multimodaldata like images and audio.Returns:- `tuple[str, list[Tuple[str, ImageBlock | AudioBlock | VideoBlock \- TextBlock]]]`:+ `tuple[str, list[TextBlock | DataBlock]]`:A tuple containing the textual representation of the tool- result and a list of tuples. The first element of each tuple- is the local file path or URL of the multimodal data, and the- second element is the corresponding block.+ result and a list of blocks to be promoted as a user message."""if isinstance(output, str):return output, []textual_output = []- multimodal_data = []+ multimodal_data: list = []+for block in output:- assert isinstance(block, dict) and "type" in block, (- f"Invalid block: {block}, a TextBlock, ImageBlock, "- f"AudioBlock, or VideoBlock is expected."- )- if block["type"] == "text":- textual_output.append(block["text"])-- elif block["type"] in ["image", "audio", "video"]:- assert "source" in block, (- f"Invalid {block['type']} block: {block}, 'source' key "- "is required."- )- source = block["source"]- # Save the image locally and return the file path- if source["type"] == "url":- textual_output.append(- f"The returned {block['type']} can be found "- f"at: {source['url']}",- )+ if isinstance(block, TextBlock):+ textual_output.append(block.text)++ elif isinstance(block, DataBlock):+ main_type = block.source.media_type.split("/")[0]++ if any(+ fnmatch(block.source.media_type, _)+ for _ in self.supported_input_media_types+ ):+ # If supported, promote the block- path_multimodal_file = source["url"]+ # Create an identifier for such multimodal data for+ # accurate reference (in terms of order, position, etc.)+ identifier = shortuuid.uuid()- elif source["type"] == "base64":- path_multimodal_file = _save_base64_data(- source["media_type"],- source["data"],+ textual_output.append(+ f"<system-reminder>A(n) {main_type} file is returned "+ f"and will be presented to you with the identifier "+ f"[{identifier}].</system-reminder>",+ )+ multimodal_data.extend(+ [+ TextBlock(+ text=f"- {identifier} ({main_type} file): ",+ ),+ block,+ ],)+ multimodal_data.append(block)++ # For unsupported media types, if it's a URL, include it in+ # the textual output; if it's base64 data, save it locally+ # and include the file path in the textual output.+ # Note if you don't want to save the local file, you should+ # transform the base64 data in the tool execution hook+ # rather than changing the formatter.+ elif isinstance(block.source, URLSource):textual_output.append(- f"The returned {block['type']} can be found "- f"at: {path_multimodal_file}",+ f"<system-reminder>A(n) {main_type} file is returned "+ f"and can be accessed at the URL: {block.source.url}."+ f"</system-reminder>",)- else:- raise ValueError(- f"Invalid image source: {block['source']}, "- "expected 'url' or 'base64'.",+ elif isinstance(block.source, Base64Source):+ # Have to save the base64 data locally+ extension = mimetypes.guess_extension(+ block.source.media_type,)+ with tempfile.NamedTemporaryFile(+ suffix=extension,+ delete=False,+ ) as temp_file:+ decoded_data = base64.b64decode(block.source.data)+ temp_file.write(decoded_data)+ textual_output.append(+ f"<system-reminder>A(n) {main_type} file is "+ f"returned and saved locally at: {temp_file.name}."+ f"</system-reminder>",+ )++ # Add system reminder tags if there is multimodal data to be promoted+ if multimodal_data:+ multimodal_data = [+ TextBlock(+ text="<system-reminder>The multimodal data and their "+ "identifiers are listed as follows:",+ ),+ *multimodal_data,+ TextBlock(+ text="</system-reminder>",+ ),+ ]- multimodal_data.append(- (path_multimodal_file, block),- )+ return "\n".join(textual_output), multimodal_data- else:- raise ValueError(- f"Unsupported block type: {block['type']}, "- "expected 'text', 'image', 'audio', or 'video'.",- )+ @staticmethod+ async def _group_messages(msgs: list[Msg]) -> AsyncGenerator:+ """Group messages into tool sequences and agent messages.- if len(textual_output) == 1:- return textual_output[0], multimodal_data+ Args:+ msgs (`list[Msg]`):+ A list of Msg objects to be grouped.+ """+ group_type = None+ group = []+ for msg in msgs:+ if group_type is None:+ if msg.get_content_blocks(+ "tool_call",+ ) or msg.get_content_blocks("tool_result"):+ group_type = "tool_sequence"+ else:+ group_type = "agent_message"+ group.append(msg)+ continue++ if group_type == "tool_sequence":+ if msg.has_content_blocks(+ "tool_call",+ ) or msg.has_content_blocks("tool_result"):+ group.append(msg)+ else:+ yield group_type, group+ group = [msg]+ group_type = "agent_message"++ elif group_type == "agent_message":+ if msg.has_content_blocks(+ "tool_call",+ ) or msg.has_content_blocks("tool_result"):+ yield group_type, group+ group = [msg]+ group_type = "tool_sequence"+ else:+ group.append(msg)- else:- return "\n".join("- " + _ for _ in textual_output), multimodal_data+ if group_type:+ yield group_type, groupdiff --git a/src/agentscope/formatter/_gemini_formatter.py b/src/agentscope/formatter/_gemini_formatter.pyindex 81d2dfbd2..d31b2f17c 100644--- a/src/agentscope/formatter/_gemini_formatter.py+++ b/src/agentscope/formatter/_gemini_formatter.py@@ -1,229 +1,192 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches-"""Google gemini API formatter in agentscope."""+"""Google Gemini API formatter in agentscope."""import base64-import os+import fnmatch+from abc import ABCfrom typing import Any-from urllib.parse import urlparse-from ._truncated_formatter_base import TruncatedFormatterBase-from .._utils._common import _get_bytes_from_web_url+import requests++from ._formatter_base import FormatterBase+from .._logging import loggerfrom ..message import (Msg,TextBlock,- ImageBlock,- AudioBlock,- ToolUseBlock,+ ThinkingBlock,+ HintBlock,+ DataBlock,+ ToolCallBlock,ToolResultBlock,- VideoBlock,+ UserMsg,URLSource,+ Base64Source,)-from .._logging import logger-from ..token import TokenCounterBase--def _format_gemini_media_block(- media_block: ImageBlock | AudioBlock | VideoBlock,-) -> dict[str, Any]:- """Format an image/audio/video block for Gemini API.- Args:- media_block (`ImageBlock | AudioBlock | VideoBlock`):- The media block to format.+class _GeminiFormatterBase(FormatterBase, ABC):+ """Base class for Gemini formatters, providing shared data block+ formatting logic."""- Returns:- `dict[str, Any]`:- A dictionary with "inline_data" key in Gemini format.+ supported_input_media_types: list[str]- Raises:- `ValueError`:- If the source type is not supported.- """- source = media_block["source"]- if source["type"] == "base64":- return {- "inline_data": {- "data": source["data"],- "mime_type": source["media_type"],- },- }- elif source["type"] == "url":- return {- "inline_data": _to_gemini_inline_data(source["url"]),- }- else:- raise ValueError(- f"Unsupported source type: {source['type']}",- )+ def _format_gemini_data_block(+ self,+ block: DataBlock,+ ) -> dict[str, Any] | None:+ """Format a DataBlock into Gemini API format.+ Args:+ block (`DataBlock`):+ The data block to format.-def _to_gemini_inline_data(url: str) -> dict:- """Convert url into the Gemini API required format."""- raw_url = url.removeprefix("file://")- parsed_url = urlparse(raw_url)- extension = raw_url.split(".")[-1].lower()-- # Pre-calculate media type from extension (image/audio/video).- typ = None- for k, v in GeminiChatFormatter.supported_extensions.items():- if extension in v:- typ = k- break-- if os.path.exists(raw_url):- # Local file- if typ is None:- raise TypeError(- f"Unsupported file extension: {extension}, expected "- f"{GeminiChatFormatter.supported_extensions}",+ Returns:+ `dict[str, Any] | None`:+ The formatted data block in Gemini ``inline_data`` format,+ or None if the media type is not supported.+ """+ source = block.source+ media_type = source.media_type++ # Check if media type is supported+ if not any(+ fnmatch.fnmatch(media_type, pattern)+ for pattern in self.supported_input_media_types+ ):+ logger.warning(+ "Media type %s is not supported, skipped.",+ media_type,)+ return None- with open(raw_url, "rb") as f:- data = base64.b64encode(f.read()).decode("utf-8")+ return self._format_media_source(source)- return {- "data": data,- "mime_type": f"{typ}/{extension}",- }-- elif parsed_url.scheme not in ["", "file"]:- # Web url- if typ is None:- raise TypeError(- f"Unsupported file extension: {extension}, expected "- f"{GeminiChatFormatter.supported_extensions}",- )+ @staticmethod+ def _format_media_source(+ source: URLSource | Base64Source,+ ) -> dict[str, Any]:+ """Format a media source into Gemini API ``inline_data`` format.- data = _get_bytes_from_web_url(url)- return {- "data": data,- "mime_type": f"{typ}/{extension}",- }+ Args:+ source (`URLSource | Base64Source`):+ The media source to format.- raise ValueError(- f"The URL `{url}` is not a valid image URL or local file.",- )+ Returns:+ `dict[str, Any]`:+ The formatted media source.+ """+ if isinstance(source, Base64Source):+ return {+ "inline_data": {+ "data": source.data,+ "mime_type": source.media_type,+ },+ }+ elif isinstance(source, URLSource):+ url = source.url+ if url.startswith("file://"):+ # Local file - read and convert to base64+ file_path = url.removeprefix("file://")+ with open(file_path, "rb") as f:+ data = base64.b64encode(f.read()).decode("utf-8")+ return {+ "inline_data": {+ "data": data,+ "mime_type": source.media_type,+ },+ }+ else:+ # Remote URL - download and convert to base64+ response = requests.get(url, timeout=30)+ response.raise_for_status()+ data = base64.b64encode(response.content).decode("utf-8")+ return {+ "inline_data": {+ "data": data,+ "mime_type": source.media_type,+ },+ }+ else:+ raise ValueError(f"Unsupported source type: {type(source)}")-class GeminiChatFormatter(TruncatedFormatterBase):+class GeminiChatFormatter(_GeminiFormatterBase):"""The Gemini formatter class for chatbot scenario, where only a userand an agent are involved. We use the `role` field to identify differententities in the conversation."""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = False- """Whether support multi-agent conversations"""-- support_vision: bool = True- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Multimodal- ImageBlock,- VideoBlock,- AudioBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""-- supported_extensions: dict[str, list[str]] = {- "image": ["png", "jpeg", "webp", "heic", "heif"],- "video": [- "mp4",- "mpeg",- "mov",- "avi",- "x-flv",- "mpg",- "webm",- "wmv",- "3gpp",- ],- "audio": ["mp3", "wav", "aiff", "aac", "ogg", "flac"],- }-def __init__(self,- promote_tool_result_images: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:- """Initialize the Gemini chat formatter.-- Args:- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- A token counter instance used to count tokens in the messages.- If not provided, the formatter will format the messages- without considering token limits.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If not provided, the formatter will not truncate- the messages.- """- super().__init__(token_counter, max_tokens)- self.promote_tool_result_images = promote_tool_result_images+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*", "audio/*", "video/*"],+ )- async def _format(+ async def format(self,msgs: list[Msg],- ) -> list[dict]:- """Format message objects into Gemini API required format."""+ ) -> list[dict[str, Any]]:+ """Format message objects into Gemini API required format.++ Args:+ msgs (`list[Msg]`):+ The list of message objects to format.++ Returns:+ `list[dict[str, Any]]`:+ The formatted messages as a list of dictionaries.+ """self.assert_list_of_msgs(msgs)- messages: list = []+ messages: list[dict] = []i = 0while i < len(msgs):msg = msgs[i]- parts = []+ parts: list = []for block in msg.get_content_blocks():- typ = block.get("type")- if typ == "text":- parts.append(- {- "text": block.get("text"),- },- )+ if isinstance(block, TextBlock):+ parts.append({"text": block.text})++ elif isinstance(block, ThinkingBlock):+ parts.append({"text": block.thinking})- elif typ == "tool_use":+ elif isinstance(block, HintBlock):+ pass # Gemini does not support hint blocks++ elif isinstance(block, DataBlock):+ formatted = self._format_gemini_data_block(block)+ if formatted:+ parts.append(formatted)++ elif isinstance(block, ToolCallBlock):parts.append({"function_call": {"id": None,- "name": block["name"],- "args": block["input"],+ "name": block.name,+ "args": block.input,},- "thought_signature": block.get("id", None),+ "thought_signature": block.id,},)- elif typ == "tool_result":+ elif isinstance(block, ToolResultBlock):(textual_output,multimodal_data,- ) = self.convert_tool_result_to_string(block["output"])+ ) = self.convert_tool_result_to_string(block.output)- # First add the tool result message in DashScope API formatmessages.append({"role": "user","parts": [{"function_response": {- "id": block["id"],- "name": block["name"],+ "id": block.id,+ "name": block.name,"response": {"output": textual_output,},@@ -233,66 +196,22 @@ class GeminiChatFormatter(TruncatedFormatterBase):},)- promoted_blocks: list = []- for url, multimodal_block in multimodal_data:- if (- multimodal_block["type"] == "image"- and self.promote_tool_result_images- ):- promoted_blocks.extend(- [- TextBlock(- type="text",- text=f"\n- The image from '{url}': ",- ),- ImageBlock(- type="image",- source=URLSource(- type="url",- url=url,- ),- ),- ],- )-- if promoted_blocks:- # Insert promoted blocks as new user message(s)- promoted_blocks = [- TextBlock(- type="text",- text="<system-info>The following are "- "the image contents from the tool "- f"result of '{block['name']}':",- ),- *promoted_blocks,- TextBlock(- type="text",- text="</system-info>",- ),- ]-+ if multimodal_data:msgs.insert(i + 1,- Msg(- name="user",- content=promoted_blocks,- role="user",+ UserMsg(+ name="system-reminder",+ content=multimodal_data,),)- elif typ in ["image", "audio", "video"]:- parts.append(- _format_gemini_media_block(- block, # type: ignore[arg-type]- ),- )-else:logger.warning(- "Unsupported block type: %s in the message, skipped. ",- typ,+ "Unsupported block type %s in the message, skipped.",+ type(block),)+ # Gemini uses "model" instead of "assistant"role = "model" if msg.role == "assistant" else "user"if parts:@@ -303,14 +222,12 @@ class GeminiChatFormatter(TruncatedFormatterBase):},)- # Move to next message (including inserted messages, which will- # be processed in subsequent iterations)i += 1return messages-class GeminiMultiAgentFormatter(TruncatedFormatterBase):+class GeminiMultiAgentFormatter(_GeminiFormatterBase):"""The multi-agent formatter for Google Gemini API, where more than auser and an agent are involved.@@ -329,27 +246,6 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):"""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = True- """Whether support multi-agent conversations"""-- support_vision: bool = True- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Multimodal- ImageBlock,- VideoBlock,- AudioBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""-def __init__(self,conversation_history_prompt: str = (@@ -357,62 +253,62 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):"The content between <history></history> tags contains ""your conversation history\n"),- promote_tool_result_images: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:"""Initialize the Gemini multi-agent formatter.Args:conversation_history_prompt (`str`):The prompt to be used for the conversation history section.- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- The token counter used for truncation.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If `None`, no truncation will be applied.+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types. Defaults to+ ``["image/*", "audio/*", "video/*"]``."""- super().__init__(token_counter=token_counter, max_tokens=max_tokens)+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*", "audio/*", "video/*"],+ )self.conversation_history_prompt = conversation_history_prompt- self.promote_tool_result_images = promote_tool_result_images- async def _format_system_message(- self,- msg: Msg,- ) -> dict[str, Any]:- """Format system message for the Gemini API."""- return {- "role": "user",- "parts": [- {- "text": msg.get_text_content(),- },- ],- }+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:+ """Format input messages into the structure required by the Gemini+ API for multi-agent conversations."""+ self.assert_list_of_msgs(msgs)++ formatted_msgs = []+ start_index = 0+ if len(msgs) > 0 and msgs[0].role == "system":+ formatted_msgs.append(+ await self._format_system_message(msgs[0]),+ )+ start_index = 1++ is_first_agent_message = True+ async for typ, group in self._group_messages(msgs[start_index:]):+ match typ:+ case "tool_sequence":+ formatted_msgs.extend(+ await self._format_tool_sequence(group),+ )+ case "agent_message":+ formatted_msgs.extend(+ await self._format_agent_message(+ group,+ is_first_agent_message,+ ),+ )+ is_first_agent_message = False++ return formatted_msgsasync def _format_tool_sequence(self,msgs: list[Msg],) -> list[dict[str, Any]]:"""Given a sequence of tool call/result messages, format them into- the required format for the Gemini API.-- Args:- msgs (`list[Msg]`):- The list of messages containing tool calls/results to format.-- Returns:- `list[dict[str, Any]]`:- A list of dictionaries formatted for the Gemini API.- """+ the required format for the Gemini API."""return await GeminiChatFormatter(- promote_tool_result_images=self.promote_tool_result_images,+ supported_input_media_types=self.supported_input_media_types,).format(msgs)async def _format_agent_message(@@ -421,58 +317,37 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):is_first: bool = True,) -> list[dict[str, Any]]:"""Given a sequence of messages without tool calls/results, format- them into the required format for the Gemini API.-- Args:- msgs (`list[Msg]`):- A list of Msg objects to be formatted.- is_first (`bool`, defaults to `True`):- Whether this is the first agent message in the conversation.- If `True`, the conversation history prompt will be included.-- Returns:- `list[dict[str, Any]]`:- A list of dictionaries formatted for the Gemini API.- """+ them into the required format for the Gemini API."""if is_first:conversation_history_prompt = self.conversation_history_promptelse:conversation_history_prompt = ""- # Format into Gemini API required format- formatted_msgs: list = []+ formatted_msgs: list[dict] = []+ conversation_parts: list[dict] = []+ accumulated_text: list[str] = []- # Collect the multimodal files- conversation_parts: list = []- accumulated_text = []for msg in msgs:for block in msg.get_content_blocks():- if block["type"] == "text":- accumulated_text.append(f"{msg.name}: {block['text']}")+ if isinstance(block, TextBlock):+ accumulated_text.append(f"{msg.name}: {block.text}")- elif block["type"] in ["image", "video", "audio"]:- # handle the accumulated text as a single part if exists+ elif isinstance(block, DataBlock):+ # Flush accumulated text firstif accumulated_text:conversation_parts.append(- {- "text": "\n".join(accumulated_text),- },+ {"text": "\n".join(accumulated_text)},)- accumulated_text.clear()+ accumulated_text = []- # handle the multimodal data- conversation_parts.append(- _format_gemini_media_block(- block, # type: ignore[arg-type]- ),- )+ formatted = self._format_gemini_data_block(block)+ if formatted:+ conversation_parts.append(formatted)if accumulated_text:conversation_parts.append(- {- "text": "\n".join(accumulated_text),- },+ {"text": "\n".join(accumulated_text)},)# Add prompt and <history></history> tags around conversation history@@ -480,23 +355,19 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):if conversation_parts[0].get("text"):conversation_parts[0]["text"] = (conversation_history_prompt- + "<history>"+ + "<history>\n"+ conversation_parts[0]["text"])-else:conversation_parts.insert(0,- {"text": conversation_history_prompt + "<history>"},+ {"text": conversation_history_prompt + "<history>\n"},)if conversation_parts[-1].get("text"):conversation_parts[-1]["text"] += "\n</history>"-else:- conversation_parts.append(- {"text": "</history>"},- )+ conversation_parts.append({"text": "</history>"})formatted_msgs.append({@@ -506,3 +377,15 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):)return formatted_msgs++ @staticmethod+ async def _format_system_message(msg: Msg) -> dict[str, Any]:+ """Format system message for the Gemini API."""+ return {+ "role": "user",+ "parts": [+ {+ "text": msg.get_text_content(),+ },+ ],+ }diff --git a/src/agentscope/formatter/_ollama_formatter.py b/src/agentscope/formatter/_ollama_formatter.pyindex 1b0c31502..43f51196d 100644--- a/src/agentscope/formatter/_ollama_formatter.py+++ b/src/agentscope/formatter/_ollama_formatter.py@@ -1,128 +1,119 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches"""The Ollama formatter module."""import base64-import os+import fnmatch+from abc import ABCfrom typing import Any-from urllib.parse import urlparse-from ._truncated_formatter_base import TruncatedFormatterBase+import requests++from ._formatter_base import FormatterBasefrom .._logging import logger-from .._utils._common import _get_bytes_from_web_urlfrom ..message import (Msg,TextBlock,- ImageBlock,- ToolUseBlock,+ HintBlock,+ DataBlock,+ ToolCallBlock,ToolResultBlock,URLSource,+ Base64Source,)-from ..token import TokenCounterBase-def _format_ollama_image_block(- image_block: ImageBlock,-) -> str:- """Format an image block for Ollama API.+class _OllamaFormatterBase(FormatterBase, ABC):+ """Base class for Ollama formatters, providing shared data block+ formatting logic."""- Args:- image_block (`ImageBlock`):- The image block to format.+ supported_input_media_types: list[str]- Returns:- `str`:- Base64 encoded image data as a string.+ def _format_ollama_data_block(+ self,+ block: DataBlock,+ ) -> str | None:+ """Format a DataBlock into Ollama API format (base64 string).- Raises:- `ValueError`:- If the source type is not supported.- """- source = image_block["source"]- if source["type"] == "url":- return _convert_ollama_image_url_to_base64_data(source["url"])- elif source["type"] == "base64":- return source["data"]- else:- raise ValueError(- f"Unsupported image source type: {source['type']}",- )+ Args:+ block (`DataBlock`):+ The data block to format.+ Returns:+ `str | None`:+ Base64 encoded data as a string, or None if the media type+ is not supported.+ """+ source = block.source+ media_type = source.media_type++ # Check if media type is supported+ if not any(+ fnmatch.fnmatch(media_type, pattern)+ for pattern in self.supported_input_media_types+ ):+ logger.warning(+ "Media type %s is not supported, skipped.",+ media_type,+ )+ return None-def _convert_ollama_image_url_to_base64_data(url: str) -> str:- """Convert image url to base64."""- raw_url = url.removeprefix("file://")- parsed_url = urlparse(raw_url)+ # Ollama only supports images+ if not media_type.startswith("image/"):+ logger.warning(+ "Ollama only supports image data, got %s, skipped.",+ media_type,+ )+ return None- if os.path.exists(raw_url):- # Local file- with open(raw_url, "rb") as f:- data = base64.b64encode(f.read()).decode("utf-8")+ return self._format_image_source(source)- return data+ @staticmethod+ def _format_image_source(source: URLSource | Base64Source) -> str:+ """Format an image source into Ollama API format (base64 string).- if parsed_url.scheme not in ["", "file"]:- # Web url- data = _get_bytes_from_web_url(url)- return data+ Args:+ source (`URLSource | Base64Source`):+ The image source to format.- raise ValueError(- f"The URL `{url}` is not a valid image URL or local file.",- )+ Returns:+ `str`:+ Base64 encoded image data.+ """+ if isinstance(source, Base64Source):+ return source.data+ elif isinstance(source, URLSource):+ url = source.url+ if url.startswith("file://"):+ # Local file - read and convert to base64+ file_path = url.removeprefix("file://")+ with open(file_path, "rb") as f:+ data = base64.b64encode(f.read()).decode("utf-8")+ return data+ else:+ # Remote URL - download and convert to base64+ response = requests.get(url, timeout=30)+ response.raise_for_status()+ data = base64.b64encode(response.content).decode("utf-8")+ return data+ else:+ raise ValueError(f"Unsupported source type: {type(source)}")-class OllamaChatFormatter(TruncatedFormatterBase):+class OllamaChatFormatter(_OllamaFormatterBase):"""The Ollama formatter class for chatbot scenario, where only a userand an agent are involved. We use the `role` field to identify differentparticipants in the conversation."""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = False- """Whether support multi-agent conversations"""-- support_vision: bool = True- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Multimodal- ImageBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""-def __init__(self,- promote_tool_result_images: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:- """Initialize the Ollama chat formatter.-- Args:- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- A token counter instance used to count tokens in the messages.- If not provided, the formatter will format the messages- without considering token limits.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If not provided, the formatter will not truncate- the messages.- """- super().__init__(token_counter, max_tokens)- self.promote_tool_result_images = promote_tool_result_images+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*"],+ )- async def _format(+ async def format(self,msgs: list[Msg],) -> list[dict[str, Any]]:@@ -138,158 +129,105 @@ class OllamaChatFormatter(TruncatedFormatterBase):"""self.assert_list_of_msgs(msgs)- messages: list = []- i = 0- while i < len(msgs):- msg = msgs[i]- content_blocks: list = []- tool_calls = []+ messages: list[dict] = []+ for msg in msgs:+ content_parts = []images = []for block in msg.get_content_blocks():- typ = block.get("type")- if typ == "text":- content_blocks.append({**block})+ if isinstance(block, TextBlock):+ content_parts.append(block.text)- elif typ == "tool_use":- tool_calls.append(- {- "id": block.get("id"),- "type": "function",- "function": {- "name": block.get("name"),- "arguments": block.get("input", {}),- },- },- )+ elif isinstance(block, HintBlock):+ pass # Ollama does not support hint blocks- elif typ == "tool_result":- (- textual_output,- multimodal_data,- ) = self.convert_tool_result_to_string(block["output"])+ elif isinstance(block, DataBlock):+ formatted_image = self._format_ollama_data_block(block)+ if formatted_image:+ images.append(formatted_image)+ elif isinstance(block, ToolCallBlock):messages.append({- "role": "tool",- "tool_call_id": block.get("id"),- "content": textual_output,- "name": block.get("name"),+ "role": msg.role,+ "content": "\n".join(content_parts)+ if content_parts+ else "",+ "tool_calls": [+ {+ "function": {+ "name": block.name,+ "arguments": block.input,+ },+ },+ ],},)+ content_parts = []+ images = []- # Then, handle the multimodal data if any- promoted_blocks: list = []- for url, multimodal_block in multimodal_data:- if (- multimodal_block["type"] == "image"- and self.promote_tool_result_images- ):- promoted_blocks.extend(- [- TextBlock(- type="text",- text=f"\n- The image from '{url}': ",- ),- ImageBlock(- type="image",- source=URLSource(- type="url",- url=url,- ),- ),- ],- )-- if promoted_blocks:- # Insert promoted blocks as new user message(s)- promoted_blocks = [- TextBlock(- type="text",- text="<system-info>The following are "- "the image contents from the tool "- f"result of '{block['name']}':",- ),- *promoted_blocks,- TextBlock(- type="text",- text="</system-info>",- ),- ]-- msgs.insert(- i + 1,- Msg(- name="user",- content=promoted_blocks,- role="user",- ),- )-- elif typ == "image":- images.append(- _format_ollama_image_block(- block, # type: ignore[arg-type]- ),- )+ elif isinstance(block, ToolResultBlock):+ (+ textual_output,+ multimodal_data,+ ) = self.convert_tool_result_to_string(block.output)++ # If there's multimodal data, insert a user message+ if multimodal_data:+ user_images = []+ user_content_parts = []+ for data_block in multimodal_data:+ if isinstance(data_block, DataBlock):+ formatted_image = (+ self._format_ollama_data_block(+ data_block,+ )+ )+ if formatted_image:+ user_images.append(formatted_image)+ elif isinstance(data_block, TextBlock):+ user_content_parts.append(data_block.text)++ user_msg = {+ "role": "user",+ "content": "\n".join(user_content_parts)+ if user_content_parts+ else textual_output,+ }+ if user_images:+ user_msg["images"] = user_images+ messages.append(user_msg)+ else:+ # No multimodal data, just add text to content+ content_parts.append(textual_output)else:logger.warning("Unsupported block type %s in the message, skipped.",- typ,+ type(block),)- content_msg = "\n".join(- content.get("text", "") for content in content_blocks- )- msg_ollama = {- "role": msg.role,- "content": content_msg or None,- }-- if tool_calls:- msg_ollama["tool_calls"] = tool_calls-- if images:- msg_ollama["images"] = images-- if (- msg_ollama["content"]- or msg_ollama.get("images")- or msg_ollama.get("tool_calls")- ):- messages.append(msg_ollama)- # Move to next message- i += 1+ # Add the message if there's content or images+ if content_parts or images:+ msg_ollama: dict[str, Any] = {+ "role": msg.role,+ "content": "\n".join(content_parts)+ if content_parts+ else "",+ }+ if images:+ msg_ollama["images"] = images+ messages.append(msg_ollama)return messages-class OllamaMultiAgentFormatter(TruncatedFormatterBase):+class OllamaMultiAgentFormatter(_OllamaFormatterBase):"""Ollama formatter for multi-agent conversations, where more thana user and an agent are involved."""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = True- """Whether support multi-agent conversations"""-- support_vision: bool = True- """Whether support vision data"""-- supported_blocks: list[type] = [- TextBlock,- # Multimodal- ImageBlock,- # Tool use- ToolUseBlock,- ToolResultBlock,- ]- """The list of supported message blocks"""-def __init__(self,conversation_history_prompt: str = (@@ -297,117 +235,105 @@ class OllamaMultiAgentFormatter(TruncatedFormatterBase):"The content between <history></history> tags contains ""your conversation history\n"),- promote_tool_result_images: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:"""Initialize the Ollama multi-agent formatter.Args:conversation_history_prompt (`str`):The prompt to use for the conversation history section.- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- The token counter used for truncation.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If `None`, no truncation will be applied.+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types. Defaults to+ ``["image/*"]``."""- super().__init__(token_counter=token_counter, max_tokens=max_tokens)+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*"],+ )self.conversation_history_prompt = conversation_history_prompt- self.promote_tool_result_images = promote_tool_result_images- async def _format_system_message(- self,- msg: Msg,- ) -> dict[str, Any]:- """Format system message for the Ollama API."""- return {- "role": "system",- "content": msg.get_text_content(),- }+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:+ """Format input messages into the structure required by the Ollama+ API for multi-agent conversations."""+ self.assert_list_of_msgs(msgs)++ formatted_msgs = []+ start_index = 0+ if len(msgs) > 0 and msgs[0].role == "system":+ formatted_msgs.append(+ await self._format_system_message(msgs[0]),+ )+ start_index = 1++ is_first_agent_message = True+ async for typ, group in self._group_messages(msgs[start_index:]):+ match typ:+ case "tool_sequence":+ formatted_msgs.extend(+ await self._format_tool_sequence(group),+ )+ case "agent_message":+ formatted_msgs.extend(+ await self._format_agent_message(+ group,+ is_first_agent_message,+ ),+ )+ is_first_agent_message = False++ return formatted_msgsasync def _format_tool_sequence(self,msgs: list[Msg],) -> list[dict[str, Any]]:- """Given a sequence of tool call/result messages, format them into- the required format for the Ollama API.-- Args:- msgs (`list[Msg]`):- The list of messages containing tool calls/results to format.-- Returns:- `list[dict[str, Any]]`:- A list of dictionaries formatted for the Ollama API.- """+ """Format a sequence of tool-related messages."""return await OllamaChatFormatter(- promote_tool_result_images=self.promote_tool_result_images,+ supported_input_media_types=self.supported_input_media_types,).format(msgs)async def _format_agent_message(self,msgs: list[Msg],- is_first: bool = True,+ is_first: bool,) -> list[dict[str, Any]]:- """Given a sequence of messages without tool calls/results, format- them into the required format for the Ollama API.-- Args:- msgs (`list[Msg]`):- A list of Msg objects to be formatted.- is_first (`bool`, defaults to `True`):- Whether this is the first agent message in the conversation.- If `True`, the conversation history prompt will be included.-- Returns:- `list[dict[str, Any]]`:- A list of dictionaries formatted for the ollama API.- """+ """Format agent messages into conversation history format."""+ conversation_blocks: list[dict] = []+ accumulated_text: list[str] = []+ images: list[str] = []- if is_first:- conversation_history_prompt = self.conversation_history_prompt- else:- conversation_history_prompt = ""-- # Format into required Ollama format- formatted_msgs: list[dict] = []-- # Collect the multimodal files- conversation_blocks: list = []- accumulated_text = []- images = []for msg in msgs:- for block in msg.get_content_blocks():- if block["type"] == "text":- accumulated_text.append(f"{msg.name}: {block['text']}")+ msg_text_parts = []+ if msg.name:+ msg_text_parts.append(f"{msg.name}:")- elif block["type"] == "image":- # Handle the accumulated text as a single block- if accumulated_text:- conversation_blocks.append(- {"text": "\n".join(accumulated_text)},- )- accumulated_text.clear()+ for block in msg.get_content_blocks():+ if isinstance(block, TextBlock):+ msg_text_parts.append(block.text)+ elif isinstance(block, DataBlock):+ formatted_image = self._format_ollama_data_block(block)+ if formatted_image:+ images.append(formatted_image)+ elif isinstance(block, HintBlock):+ pass # Ollama does not support hint blocks+ else:+ logger.warning(+ "Unsupported block type %s in agent message, skipped.",+ type(block),+ )- images.append(_format_ollama_image_block(block))- conversation_blocks.append({**block})+ if msg_text_parts:+ accumulated_text.append("\n".join(msg_text_parts))if accumulated_text:conversation_blocks.append({"text": "\n".join(accumulated_text)},)- if conversation_blocks:+ if conversation_blocks and is_first:if conversation_blocks[0].get("text"):conversation_blocks[0]["text"] = (- conversation_history_prompt+ self.conversation_history_prompt+ "<history>\n"+ conversation_blocks[0]["text"])@@ -416,7 +342,8 @@ class OllamaMultiAgentFormatter(TruncatedFormatterBase):conversation_blocks.insert(0,{- "text": conversation_history_prompt + "<history>\n",+ "text": self.conversation_history_prompt+ + "<history>\n",},)@@ -431,13 +358,32 @@ class OllamaMultiAgentFormatter(TruncatedFormatterBase):for conversation_block in conversation_blocks)- user_message = {+ user_message: dict[str, Any] = {"role": "user","content": conversation_blocks_text,}if images:user_message["images"] = images++ formatted_msgs = []if conversation_blocks:formatted_msgs.append(user_message)return formatted_msgs++ @staticmethod+ async def _format_system_message(msg: Msg) -> dict[str, Any]:+ """Format a system message."""+ text_parts = []+ for block in msg.get_content_blocks():+ if isinstance(block, TextBlock):+ text_parts.append(block.text)+ else:+ logger.warning(+ "Unsupported block type %s in system message, skipped.",+ type(block),+ )+ return {+ "role": "system",+ "content": "\n".join(text_parts),+ }diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.pyindex fef61e60a..490023cab 100644--- a/src/agentscope/formatter/_openai_formatter.py+++ b/src/agentscope/formatter/_openai_formatter.py@@ -1,222 +1,220 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches, too-many-nested-blocks"""The OpenAI formatter for agentscope."""import base64-import json-import os+from abc import ABC+from fnmatch import fnmatchfrom typing import Anyfrom urllib.parse import urlparseimport requests-from ._truncated_formatter_base import TruncatedFormatterBase+from . import FormatterBasefrom .._logging import loggerfrom ..message import (Msg,URLSource,TextBlock,- ImageBlock,- AudioBlock,+ DataBlock,Base64Source,- ToolUseBlock,+ ToolCallBlock,ToolResultBlock,+ HintBlock,+ UserMsg,)-from ..token import TokenCounterBase-def _format_openai_image_block(- image_block: ImageBlock,-) -> dict[str, Any]:- """Format an image block for OpenAI API.+class _OpenAIFormatterBase(FormatterBase, ABC):+ """Base class for OpenAI formatters, providing shared data block+ formatting logic."""- Args:- image_block (`ImageBlock`):- The image block to format.+ supported_input_media_types: list[str]- Returns:- `dict[str, Any]`:- A dictionary with "type" and "image_url" keys in OpenAI format.+ def _format_openai_data_block(+ self,+ block: DataBlock,+ role: str = "user",+ ) -> dict[str, Any] | None:+ """Format a DataBlock into the required format for OpenAI API.- Raises:- `ValueError`:- If the source type is not supported.- """- source = image_block["source"]- if source["type"] == "url":- url = _to_openai_image_url(source["url"])- elif source["type"] == "base64":- data = source["data"]- media_type = source["media_type"]- url = f"data:{media_type};base64,{data}"- else:- raise ValueError(- f"Unsupported image source type: {source['type']}",- )+ For image blocks, URLs are returned as-is (or converted to base64 for+ local ``file://`` paths). For audio blocks, data is always converted+ to base64 as required by the OpenAI input_audio format.- return {- "type": "image_url",- "image_url": {- "url": url,- },- }+ Args:+ block (`DataBlock`):+ The DataBlock to format.+ role (`str`, defaults to ``"user"``):+ The role of the message that contains this block. Audio blocks+ in assistant messages are skipped to avoid errors in subsequent+ model calls.+ Returns:+ `dict[str, Any] | None`:+ A dictionary in OpenAI API format, or ``None`` if the block+ should be skipped.+ """+ if not any(+ fnmatch(block.source.media_type, pattern)+ for pattern in self.supported_input_media_types+ ):+ logger.warning(+ "Unsupported media type %s for OpenAI API. "+ "Supported types: %s. This block will be skipped.",+ block.source.media_type,+ ", ".join(self.supported_input_media_types),+ )+ return None-def _to_openai_image_url(url: str) -> str:- """Convert an image url to openai format. If the given url is a local- file, it will be converted to base64 format. Otherwise, it will be- returned directly.+ main_type = block.source.media_type.split("/")[0]- Args:- url (`str`):- The local or public url of the image.- """- import filetype-- # See https://platform.openai.com/docs/guides/vision for details of- # support image extensions.- support_image_extensions = (- ".png",- ".jpg",- ".jpeg",- ".gif",- ".webp",- )-- raw_url = url.removeprefix("file://")- # For local files- if os.path.exists(raw_url) and os.path.isfile(raw_url):- if any(raw_url.endswith(_) for _ in support_image_extensions):- with open(raw_url, "rb") as image_file:- base64_image = base64.b64encode(image_file.read()).decode(- "utf-8",- )- extension = raw_url.lower().split(".")[-1]- mime_type = f"image/{extension}"- return f"data:{mime_type};base64,{base64_image}"-- # No extension - detect file type using filetype- kind = filetype.guess(raw_url)- if kind is not None and kind.mime.startswith("image/"):- with open(raw_url, "rb") as image_file:- base64_image = base64.b64encode(image_file.read()).decode(- "utf-8",- )- return f"data:{kind.mime};base64,{base64_image}"-- # For web urls- parsed_url = urlparse(raw_url)- if parsed_url.scheme not in ["", "file"]:- return url-- raise ValueError(- f'Invalid image URL: "{url}". It should be a local file or a web URL.',- )---def _to_openai_audio_data(source: URLSource | Base64Source) -> dict:- """Covert an audio source to OpenAI format."""- if source["type"] == "url":- raw_url = source["url"].removeprefix("file://")- extension = raw_url.split(".")[-1].lower()- if extension not in ["wav", "mp3"]:- raise TypeError(- f"Unsupported audio file extension: {extension}, "- "wav and mp3 are supported.",- )+ if main_type == "image":+ return self._format_image_source(block.source)++ if main_type == "audio":+ # Filter out audio content when the multimodal model outputs both+ # text and audio, to prevent errors in subsequent model calls+ if role == "assistant":+ return None+ return self._format_audio_source(block.source)++ logger.warning(+ "Unsupported main media type %s for OpenAI API. "+ "This block will be skipped.",+ main_type,+ )+ return None++ @staticmethod+ def _format_image_source(+ source: URLSource | Base64Source,+ ) -> dict[str, Any]:+ """Convert an image source to OpenAI image_url format.- parsed_url = urlparse(raw_url)+ Local ``file://`` URLs are read from disk and converted to base64+ data URIs. Remote URLs are passed through unchanged.- if os.path.exists(raw_url):- with open(raw_url, "rb") as audio_file:- data = base64.b64encode(audio_file.read()).decode("utf-8")+ Args:+ source (`URLSource | Base64Source`):+ The image source to convert.- # web url- elif parsed_url.scheme not in ["", "file"]:- response = requests.get(source["url"])- response.raise_for_status()- data = base64.b64encode(response.content).decode("utf-8")+ Returns:+ `dict[str, Any]`:+ A dictionary with ``"type": "image_url"`` in OpenAI format.+ """+ if isinstance(source, Base64Source):+ url = f"data:{source.media_type};base64,{source.data}"++ elif isinstance(source, URLSource):+ if source.url.startswith("file://"):+ # Local file — read and encode as base64 data URI+ local_path = source.url.removeprefix("file://")+ with open(local_path, "rb") as f:+ encoded = base64.b64encode(f.read()).decode("utf-8")+ url = f"data:{source.media_type};base64,{encoded}"+ else:+ # Remote URL — pass through as-is+ url = source.urlelse:- raise ValueError(- f"Unsupported audio source: {source['url']}, "- "it should be a local file or a web URL.",- )+ raise ValueError(f"Unsupported image source type: {type(source)}")return {- "data": data,- "format": extension,+ "type": "image_url",+ "image_url": {"url": url},}- if source["type"] == "base64":- data = source["data"]- media_type = source["media_type"]+ @staticmethod+ def _format_audio_source(+ source: URLSource | Base64Source,+ ) -> dict[str, Any]:+ """Convert an audio source to OpenAI input_audio format.- if media_type not in ["audio/wav", "audio/mp3"]:- raise TypeError(- f"Unsupported audio media type: {media_type}, "- "only audio/wav and audio/mp3 are supported.",- )+ Local ``file://`` URLs are read from disk. Remote URLs are downloaded.+ Only ``wav`` and ``mp3`` formats are supported by the OpenAI API.- return {- "data": data,- "format": media_type.split("/")[-1],- }+ Args:+ source (`URLSource | Base64Source`):+ The audio source to convert.++ Returns:+ `dict[str, Any]`:+ A dictionary with ``"type": "input_audio"`` in OpenAI format.+ """+ if isinstance(source, Base64Source):+ media_type = source.media_type+ if media_type not in ["audio/wav", "audio/mp3"]:+ raise TypeError(+ f"Unsupported audio media type: {media_type}, "+ "only audio/wav and audio/mp3 are supported.",+ )+ return {+ "type": "input_audio",+ "input_audio": {+ "data": source.data,+ "format": media_type.split("/")[-1],+ },+ }- raise TypeError(f"Unsupported audio source: {source['type']}.")+ if isinstance(source, URLSource):+ if source.url.startswith("file://"):+ # Local file+ local_path = source.url.removeprefix("file://")+ extension = local_path.rsplit(".", 1)[-1].lower()+ if extension not in ["wav", "mp3"]:+ raise TypeError(+ f"Unsupported audio file extension: {extension}, "+ "wav and mp3 are supported.",+ )+ with open(local_path, "rb") as f:+ data = base64.b64encode(f.read()).decode("utf-8")+ else:+ # Remote URL — download and encode+ parsed = urlparse(source.url)+ extension = parsed.path.rsplit(".", 1)[-1].lower()+ if extension not in ["wav", "mp3"]:+ raise TypeError(+ f"Unsupported audio file extension: {extension}, "+ "wav and mp3 are supported.",+ )+ response = requests.get(source.url)+ response.raise_for_status()+ data = base64.b64encode(response.content).decode("utf-8")++ return {+ "type": "input_audio",+ "input_audio": {+ "data": data,+ "format": extension,+ },+ }+ raise TypeError(f"Unsupported audio source type: {type(source)}.")-class OpenAIChatFormatter(TruncatedFormatterBase):++class OpenAIChatFormatter(_OpenAIFormatterBase):"""The OpenAI formatter class for chatbot scenario, where only a userand an agent are involved. We use the `name` field in OpenAI API toidentify different entities in the conversation."""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = True- """Whether support multi-agent conversation"""-- support_vision: bool = True- """Whether support vision models"""-- supported_blocks: list[type] = [- TextBlock,- ImageBlock,- AudioBlock,- ToolUseBlock,- ToolResultBlock,- ]- """Supported message blocks for OpenAI API"""-def __init__(self,- promote_tool_result_images: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:"""Initialize the OpenAI chat formatter.Args:- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- A token counter instance used to count tokens in the messages.- If not provided, the formatter will format the messages- without considering token limits.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If not provided, the formatter will not truncate- the messages.+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types, using glob-style+ patterns (e.g. ``"image/*"``, ``"audio/mp3"``). Defaults to+ ``["image/*", "audio/*"]``."""- super().__init__(token_counter=token_counter, max_tokens=max_tokens)- self.promote_tool_result_images = promote_tool_result_images+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*", "audio/*"],+ )- async def _format(+ async def format(self,msgs: list[Msg],) -> list[dict[str, Any]]:@@ -241,115 +239,73 @@ class OpenAIChatFormatter(TruncatedFormatterBase):tool_calls = []for block in msg.get_content_blocks():- typ = block.get("type")- if typ == "text":- content_blocks.append({**block})+ if isinstance(block, TextBlock):+ content_blocks.append({"type": "text", "text": block.text})- elif typ == "tool_use":+ elif isinstance(block, DataBlock):+ formatted = self._format_openai_data_block(+ block,+ role=msg.role,+ )+ if formatted is not None:+ content_blocks.append(formatted)++ elif isinstance(block, HintBlock):+ # Insert a new user message with the hint content right+ # after the current message, and go on processing the+ # rest of the blocks in the current message+ if content_blocks or tool_calls:+ msg_openai = {+ "role": msg.role,+ "name": msg.name,+ "content": content_blocks or None,+ }+ if tool_calls:+ msg_openai["tool_calls"] = tool_calls+ messages.append(msg_openai)+ content_blocks = []+ tool_calls = []++ elif isinstance(block, ToolCallBlock):tool_calls.append({- "id": block.get("id"),+ "id": block.id,"type": "function","function": {- "name": block.get("name"),- "arguments": json.dumps(- block.get("input", {}),- ensure_ascii=False,- ),+ "name": block.name,+ "arguments": block.input,},},)- elif typ == "tool_result":+ elif isinstance(block, ToolResultBlock):(textual_output,multimodal_data,- ) = self.convert_tool_result_to_string(block["output"])+ ) = self.convert_tool_result_to_string(block.output)messages.append({"role": "tool",- "tool_call_id": block.get("id"),- "content": ( # type: ignore[arg-type]- textual_output- ),- "name": block.get("name"),+ "tool_call_id": block.id,+ "content": textual_output,+ "name": block.name,},)- # Then, handle the multimodal data if any- promoted_blocks: list = []- for url, multimodal_block in multimodal_data:- if (- multimodal_block["type"] == "image"- and self.promote_tool_result_images- ):- promoted_blocks.extend(- [- TextBlock(- type="text",- text=f"\n- The image from '{url}': ",- ),- ImageBlock(- type="image",- source=URLSource(- type="url",- url=url,- ),- ),- ],- )-- if promoted_blocks:- # Insert promoted blocks as new user message(s)- promoted_blocks = [- TextBlock(- type="text",- text="<system-info>The following are "- "the image contents from the tool "- f"result of '{block['name']}':",- ),- *promoted_blocks,- TextBlock(- type="text",- text="</system-info>",- ),- ]-+ if multimodal_data:msgs.insert(i + 1,- Msg(- name="user",- content=promoted_blocks,- role="user",+ UserMsg(+ name="system-reminder",+ content=multimodal_data,),)- elif typ == "image":- content_blocks.append(- _format_openai_image_block(- block, # type: ignore[arg-type]- ),- )-- elif typ == "audio":- # Filter out audio content when the multimodal model- # outputs both text and audio, to prevent errors in- # subsequent model calls- if msg.role == "assistant":- continue- input_audio = _to_openai_audio_data(block["source"])- content_blocks.append(- {- "type": "input_audio",- "input_audio": input_audio,- },- )-else:logger.warning("Unsupported block type %s in the message, skipped.",- typ,+ type(block),)msg_openai = {@@ -371,32 +327,15 @@ class OpenAIChatFormatter(TruncatedFormatterBase):return messages-class OpenAIMultiAgentFormatter(TruncatedFormatterBase):+class OpenAIMultiAgentFormatter(_OpenAIFormatterBase):"""OpenAI formatter for multi-agent conversations, where more thana user and an agent are involved.+.. tip:: This formatter is compatible with OpenAI API and- OpenAI-compatible services like vLLM, Azure OpenAI, and others.+ OpenAI-compatible services like vLLM, Azure OpenAI, and others."""- support_tools_api: bool = True- """Whether support tools API"""-- support_multiagent: bool = True- """Whether support multi-agent conversation"""-- support_vision: bool = True- """Whether support vision models"""-- supported_blocks: list[type] = [- TextBlock,- ImageBlock,- AudioBlock,- ToolUseBlock,- ToolResultBlock,- ]- """Supported message blocks for OpenAI API"""-def __init__(self,conversation_history_prompt: str = (@@ -404,33 +343,54 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):"The content between <history></history> tags contains ""your conversation history\n"),- promote_tool_result_images: bool = False,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,+ supported_input_media_types: list[str] | None = None,) -> None:"""Initialize the OpenAI multi-agent formatter.Args:conversation_history_prompt (`str`):The prompt to use for the conversation history section.- promote_tool_result_images (`bool`, defaults to `False`):- Whether to promote images from tool results to user messages.- Most LLM APIs don't support images in tool result blocks, but- do support them in user message blocks. When `True`, images are- extracted and appended as a separate user message with- explanatory text indicating their source.- token_counter (`TokenCounterBase | None`, optional):- A token counter instance used to count tokens in the messages.- If not provided, the formatter will format the messages- without considering token limits.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If not provided, the formatter will not truncate- the messages.+ supported_input_media_types (`list[str] | None`, optional):+ The list of supported input media types, using glob-style+ patterns (e.g. ``"image/*"``, ``"audio/mp3"``). Defaults to+ ``["image/*", "audio/*"]``."""- super().__init__(token_counter=token_counter, max_tokens=max_tokens)+ super().__init__(+ supported_input_media_types=supported_input_media_types+ or ["image/*", "audio/*"],+ )self.conversation_history_prompt = conversation_history_prompt- self.promote_tool_result_images = promote_tool_result_images++ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:+ """Format input messages into the structure required by the OpenAI API+ for multi-agent conversations."""+ self.assert_list_of_msgs(msgs)++ formatted_msgs = []+ start_index = 0+ if len(msgs) > 0 and msgs[0].role == "system":+ formatted_msgs.append(+ await self._format_system_message(msgs[0]),+ )+ start_index = 1++ is_first_agent_message = True+ async for typ, group in self._group_messages(msgs[start_index:]):+ match typ:+ case "tool_sequence":+ formatted_msgs.extend(+ await self._format_tool_sequence(group),+ )+ case "agent_message":+ formatted_msgs.extend(+ await self._format_agent_message(+ group,+ is_first_agent_message,+ ),+ )+ is_first_agent_message = False++ return formatted_msgsasync def _format_tool_sequence(self,@@ -439,7 +399,7 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):"""Given a sequence of tool call/result messages, format them intothe required format for the OpenAI API."""return await OpenAIChatFormatter(- promote_tool_result_images=self.promote_tool_result_images,+ supported_input_media_types=self.supported_input_media_types,).format(msgs)async def _format_agent_message(@@ -455,34 +415,23 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):else:conversation_history_prompt = ""- # Format into required OpenAI formatformatted_msgs: list[dict] = []-conversation_blocks: list = []accumulated_text = []- images = []- audios = []+ media_blocks: list[dict] = []for msg in msgs:for block in msg.get_content_blocks():- if block["type"] == "text":- accumulated_text.append(f"{msg.name}: {block['text']}")-- elif block["type"] == "image":- images.append(_format_openai_image_block(block))- elif block["type"] == "audio":- # Filter out audio content when the multimodal model- # outputs both text and audio, to prevent errors in- # subsequent model calls- if msg.role == "assistant":- continue- input_audio = _to_openai_audio_data(block["source"])- audios.append(- {- "type": "input_audio",- "input_audio": input_audio,- },+ if isinstance(block, TextBlock):+ accumulated_text.append(f"{msg.name}: {block.text}")++ elif isinstance(block, DataBlock):+ formatted = self._format_openai_data_block(+ block,+ role=msg.role,)+ if formatted is not None:+ media_blocks.append(formatted)if accumulated_text:conversation_blocks.append(@@ -496,45 +445,39 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):+ "<history>\n"+ conversation_blocks[0]["text"])-else:conversation_blocks.insert(0,- {- "text": conversation_history_prompt + "<history>\n",- },+ {"text": conversation_history_prompt + "<history>\n"},)if conversation_blocks[-1].get("text"):conversation_blocks[-1]["text"] += "\n</history>"-else:conversation_blocks.append({"text": "</history>"})conversation_blocks_text = "\n".join(- conversation_block.get("text", "")- for conversation_block in conversation_blocks+ b.get("text", "") for b in conversation_blocks)content_list: list[dict[str, Any]] = []if conversation_blocks_text:content_list.append(- {- "type": "text",- "text": conversation_blocks_text,- },+ {"type": "text", "text": conversation_blocks_text},)- if images:- content_list.extend(images)- if audios:- content_list.extend(audios)-- user_message = {- "role": "user",- "content": content_list,- }+ content_list.extend(media_blocks)if content_list:- formatted_msgs.append(user_message)+ formatted_msgs.append({"role": "user", "content": content_list})return formatted_msgs++ @staticmethod+ async def _format_system_message(+ msg: Msg,+ ) -> dict[str, Any]:+ """Format system message for OpenAI API."""+ return {+ "role": "system",+ "content": msg.get_text_content(),+ }diff --git a/src/agentscope/formatter/_truncated_formatter_base.py b/src/agentscope/formatter/_truncated_formatter_base.pydeleted file mode 100644index 86d42ad41..000000000--- a/src/agentscope/formatter/_truncated_formatter_base.py+++ /dev/null@@ -1,297 +0,0 @@-# -*- coding: utf-8 -*--"""The truncated formatter base class, which allows to truncate the input-messages."""-from abc import ABC-from copy import deepcopy-from typing import (- Any,- Tuple,- Literal,- AsyncGenerator,-)--from ._formatter_base import FormatterBase-from ..message import Msg-from ..token import TokenCounterBase-from ..tracing import trace_format---class TruncatedFormatterBase(FormatterBase, ABC):- """Base class for truncated formatters, which formats input messages into- required formats with tokens under a specified limit."""-- def __init__(- self,- token_counter: TokenCounterBase | None = None,- max_tokens: int | None = None,- ) -> None:- """Initialize the TruncatedFormatterBase.-- Args:- token_counter (`TokenCounterBase | None`, optional):- A token counter instance used to count tokens in the messages.- If not provided, the formatter will format the messages- without considering token limits.- max_tokens (`int | None`, optional):- The maximum number of tokens allowed in the formatted- messages. If not provided, the formatter will not truncate- the messages.- """- self.token_counter = token_counter-- assert (- max_tokens is None or 0 < max_tokens- ), "max_tokens must be greater than 0"- self.max_tokens = max_tokens-- @trace_format- async def format(- self,- msgs: list[Msg],- **kwargs: Any,- ) -> list[dict[str, Any]]:- """Format the input messages into the required format. If token- counter and max token limit are provided, the messages will be- truncated to fit the limit.-- Args:- msgs (`list[Msg]`):- The input messages to be formatted.-- Returns:- `list[dict[str, Any]]`:- The formatted messages in the required format.- """-- # Check if the input messages are valid- self.assert_list_of_msgs(msgs)-- msgs = deepcopy(msgs)-- while True:- formatted_msgs = await self._format(msgs)- n_tokens = await self._count(formatted_msgs)-- if (- n_tokens is None- or self.max_tokens is None- or n_tokens <= self.max_tokens- ):- return formatted_msgs-- # truncate the input messages- msgs = await self._truncate(msgs)-- async def _format(self, msgs: list[Msg]) -> list[dict[str, Any]]:- """Format the input messages into the required format. This method- should be implemented by the subclasses."""-- formatted_msgs = []- start_index = 0- if len(msgs) > 0 and msgs[0].role == "system":- formatted_msgs.append(- await self._format_system_message(msgs[0]),- )- start_index = 1-- is_first_agent_message = True- async for typ, group in self._group_messages(msgs[start_index:]):- match typ:- case "tool_sequence":- formatted_msgs.extend(- await self._format_tool_sequence(group),- )- case "agent_message":- formatted_msgs.extend(- await self._format_agent_message(- group,- is_first_agent_message,- ),- )- is_first_agent_message = False-- return formatted_msgs-- async def _format_system_message(- self,- msg: Msg,- ) -> dict[str, Any]:- """Format system message for the LLM API.-- .. note:: This is the default implementation. For certain LLM APIs- with specific requirements, you may need to implement a custom- formatting function to accommodate those particular needs.- """- return {- "role": "system",- "content": msg.get_content_blocks("text"),- }-- async def _format_tool_sequence(- self,- msgs: list[Msg],- ) -> list[dict[str, Any]]:- """Given a sequence of tool call/result messages, format them into- the required format for the LLM API."""- raise NotImplementedError(- "_format_tool_sequence is not implemented",- )-- async def _format_agent_message(- self,- msgs: list[Msg],- is_first: bool = True,- ) -> list[dict[str, Any]]:- """Given a sequence of messages without tool calls/results, format- them into the required format for the LLM API."""- raise NotImplementedError(- "_format_agent_message is not implemented",- )-- async def _truncate(self, msgs: list[Msg]) -> list[Msg]:- """Truncate the input messages, so that it can fit the token limit.- This function is called only when-- - both `token_counter` and `max_tokens` are provided,- - the formatted output of the input messages exceeds the token limit.-- .. tip:: This function only provides a simple strategy, and developers- can override this method to implement more sophisticated- truncation strategies.-- .. note:: The tool call message should be truncated together with- its corresponding tool result message to satisfy the LLM API- requirements.-- Args:- msgs (`list[Msg]`):- The input messages to be truncated.-- Raises:- `ValueError`:- If the system prompt message already exceeds the token limit,- or if there are tool calls without corresponding tool results.-- Returns:- `list[Msg]`:- The truncated messages.- """- start_index = 0- if len(msgs) > 0 and msgs[0].role == "system":- if len(msgs) == 1:- # If the system prompt already exceeds the token limit, we- # raise an error.- raise ValueError(- f"The system prompt message already exceeds the token "- f"limit ({self.max_tokens} tokens).",- )-- start_index = 1-- # Create a tool call IDs queues to delete the corresponding tool- # result message- tool_call_ids = set()- for i in range(start_index, len(msgs)):- msg = msgs[i]- for block in msg.get_content_blocks("tool_use"):- tool_call_ids.add(block["id"])-- for block in msg.get_content_blocks("tool_result"):- try:- tool_call_ids.remove(block["id"])- except KeyError:- pass-- # We can stop truncating if the queue is empty- if len(tool_call_ids) == 0:- return msgs[:start_index] + msgs[i + 1 :]-- if len(tool_call_ids) > 0:- raise ValueError(- "The input messages contains tool call(s) that do not have "- f"the corresponding tool result(s): {tool_call_ids}. ",- )-- return msgs[:start_index]-- async def _count(self, msgs: list[dict[str, Any]]) -> int | None:- """Count the number of tokens in the input messages. If token counter- is not provided, `None` will be returned.-- Args:- msgs (`list[Msg]`):- The input messages to count tokens for.- """- if self.token_counter is None:- return None-- return await self.token_counter.count(msgs)-- @staticmethod- async def _group_messages(- msgs: list[Msg],- ) -> AsyncGenerator[- Tuple[Literal["tool_sequence", "agent_message"], list[Msg]],- None,- ]:- """Group the input messages into two types and yield them as a- generator. The two types are:-- - agent message that doesn't contain tool calls/results, and- - tool sequence that consisted of a sequence of tool calls/results-- .. note:: The group operation is used in multi-agent scenario, where- multiple entities are involved in the input messages. So that to be- compatible with tools API, we have to group the messages and format- them with different strategies.-- Args:- msgs (`list[Msg]`):- The input messages to be grouped, where the system prompt- message shouldn't be included.-- Yields:- `AsyncGenerator[Tuple[str, list[Msg]], None]`:- A generator that yields tuples of group type and the list of- messages in that group. The group type can be either- "tool_sequence" or "agent_message".- """-- group_type: Literal["tool_sequence", "agent_message"] | None = None- group = []- for msg in msgs:- if group_type is None:- if msg.has_content_blocks(- "tool_use",- ) or msg.has_content_blocks("tool_result"):- group_type = "tool_sequence"- else:- group_type = "agent_message"-- group.append(msg)- continue-- # determine if this msg has the same type as the current group- if group_type == "tool_sequence":- if msg.has_content_blocks(- "tool_use",- ) or msg.has_content_blocks("tool_result"):- group.append(msg)-- else:- yield group_type, group- group = [msg]- group_type = "agent_message"-- elif group_type == "agent_message":- if msg.has_content_blocks(- "tool_use",- ) or msg.has_content_blocks("tool_result"):- yield group_type, group- group = [msg]- group_type = "tool_sequence"-- else:- group.append(msg)- if group_type:- yield group_type, groupdiff --git a/src/agentscope/mcp/_client_base.py b/src/agentscope/mcp/_client_base.pyindex 368887034..c9a70a627 100644--- a/src/agentscope/mcp/_client_base.py+++ b/src/agentscope/mcp/_client_base.py@@ -7,11 +7,9 @@ import mcp.typesfrom .._logging import loggerfrom ..message import (- ImageBlock,Base64Source,- AudioBlock,TextBlock,- VideoBlock,+ DataBlock,)@@ -39,33 +37,19 @@ class MCPClientBase:@staticmethoddef _convert_mcp_content_to_as_blocks(mcp_content_blocks: list,- ) -> List[TextBlock | ImageBlock | AudioBlock | VideoBlock]:+ ) -> List[TextBlock | DataBlock]:"""Convert MCP content to AgentScope blocks."""as_content: list = []for content in mcp_content_blocks:if isinstance(content, mcp.types.TextContent):+ as_content.append(TextBlock(text=content.text))+ elif isinstance(+ content,+ (mcp.types.ImageContent, mcp.types.AudioContent),+ ):as_content.append(- TextBlock(- type="text",- text=content.text,- ),- )- elif isinstance(content, mcp.types.ImageContent):- as_content.append(- ImageBlock(- type="image",- source=Base64Source(- type="base64",- media_type=content.mimeType,- data=content.data,- ),- ),- )- elif isinstance(content, mcp.types.AudioContent):- as_content.append(- AudioBlock(- type="audio",+ DataBlock(source=Base64Source(type="base64",media_type=content.mimeType,@@ -80,7 +64,6 @@ class MCPClientBase:):as_content.append(TextBlock(- type="text",text=content.resource.model_dump_json(indent=2),),)@@ -93,6 +76,7 @@ class MCPClientBase:"Skipping this content.",type(content.resource),)+else:logger.warning("Unsupported content type: %s. Skipping this content.",diff --git a/src/agentscope/mcp/_http_stateful_client.py b/src/agentscope/mcp/_http_stateful_client.pyindex 9c13d8c84..610471e53 100644--- a/src/agentscope/mcp/_http_stateful_client.py+++ b/src/agentscope/mcp/_http_stateful_client.py@@ -3,7 +3,7 @@from typing import Any, Literalfrom mcp.client.sse import sse_client-from mcp.client.streamable_http import streamablehttp_client+from mcp.client.streamable_http import streamable_http_clientfrom ._stateful_client_base import StatefulClientBase@@ -67,7 +67,7 @@ class HttpStatefulClient(StatefulClientBase):self.transport = transportif self.transport == "streamable_http":- self.client = streamablehttp_client(+ self.client = streamable_http_client(url=url,headers=headers,timeout=timeout,diff --git a/src/agentscope/mcp/_http_stateless_client.py b/src/agentscope/mcp/_http_stateless_client.pyindex c90f965df..dbeab5686 100644--- a/src/agentscope/mcp/_http_stateless_client.py+++ b/src/agentscope/mcp/_http_stateless_client.py@@ -6,7 +6,7 @@ from typing import Any, Callable, Awaitable, Literal, Listimport mcp.typesfrom mcp import ClientSessionfrom mcp.client.sse import sse_client-from mcp.client.streamable_http import streamablehttp_client+from mcp.client.streamable_http import streamable_http_clientfrom . import MCPToolFunctionfrom ._client_base import MCPClientBase@@ -82,7 +82,7 @@ class HttpStatelessClient(MCPClientBase):return sse_client(**self.client_config)if self.transport == "streamable_http":- return streamablehttp_client(**self.client_config)+ return streamable_http_client(**self.client_config)raise ValueError(f"Unsupported transport type: {self.transport}. "diff --git a/src/agentscope/mcp/_mcp_function.py b/src/agentscope/mcp/_mcp_function.pyindex 44bfe6d62..d205a1e5b 100644--- a/src/agentscope/mcp/_mcp_function.py+++ b/src/agentscope/mcp/_mcp_function.py@@ -5,13 +5,35 @@ from datetime import timedeltafrom typing import Any, Callableimport mcp-from mcp import ClientSession+from mcp import ClientSession, Toolfrom ._client_base import MCPClientBase-from .._utils._common import _extract_json_schema_from_mcp_toolfrom ..tool import ToolResponse+def _extract_json_schema_from_mcp_tool(tool: Tool) -> dict[str, Any]:+ """Extract JSON schema from MCP tool."""++ return {+ "type": "function",+ "function": {+ "name": tool.name,+ "description": tool.description,+ "parameters": {+ "type": "object",+ "properties": tool.inputSchema.get(+ "properties",+ {},+ ),+ "required": tool.inputSchema.get(+ "required",+ [],+ ),+ },+ },+ }++class MCPToolFunction:"""An MCP tool function class that can be called directly."""diff --git a/src/agentscope/message/__init__.py b/src/agentscope/message/__init__.pyindex 1dfe69f5b..178e08417 100644--- a/src/agentscope/message/__init__.py+++ b/src/agentscope/message/__init__.py@@ -1,31 +1,34 @@# -*- coding: utf-8 -*-"""The message module in agentscope."""-from ._message_block import (+from ._block import (ContentBlock,+ ContentBlockTypes,TextBlock,ThinkingBlock,- ToolUseBlock,+ HintBlock,+ ToolCallBlock,ToolResultBlock,- ImageBlock,- AudioBlock,- VideoBlock,+ DataBlock,Base64Source,URLSource,)-from ._message_base import Msg+from ._base import Msg, UserMsg, AssistantMsg, SystemMsg__all__ = ["TextBlock","ThinkingBlock",+ "HintBlock",+ "ToolCallBlock",+ "ToolResultBlock",+ "DataBlock","Base64Source","URLSource",- "ImageBlock",- "AudioBlock",- "VideoBlock",- "ToolUseBlock",- "ToolResultBlock","ContentBlock",+ "ContentBlockTypes","Msg",+ "UserMsg",+ "AssistantMsg",+ "SystemMsg",]diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.pynew file mode 100644index 000000000..26e859499--- /dev/null+++ b/src/agentscope/message/_base.py@@ -0,0 +1,275 @@+# -*- coding: utf-8 -*-+"""The message class in agentscope."""+from datetime import datetime+from typing import Literal, List, overload, Sequence++import shortuuid+from pydantic import BaseModel, Field++from ._block import (+ TextBlock,+ ToolCallBlock,+ DataBlock,+ ContentBlock,+ ToolResultBlock,+ ContentBlockTypes,+ HintBlock,+ ThinkingBlock,+)+++def _assert_user_content_blocks(content: str | Sequence[ContentBlock]) -> None:+ """Assert that the content blocks in user message are valid."""+ if isinstance(content, str):+ return++ for block in content:+ if block.type not in ["text", "data"]:+ raise ValueError(+ "User message can only contain text blocks or data blocks.",+ )+++def _assert_system_content_blocks(+ content: str | Sequence[ContentBlock],+) -> None:+ """Assert that the content blocks in system message are valid."""+ if isinstance(content, str):+ return++ for block in content:+ if block.type not in ["text"]:+ raise ValueError("System message can only contain text blocks.")+++class Msg(BaseModel):+ """The message class in agentscope."""++ name: str+ content: str | list[ContentBlock]+ role: Literal["user", "assistant", "system"]++ id: str = Field(default_factory=shortuuid.uuid)+ metadata: dict = Field(default_factory=dict)+ created_at: str = Field(default_factory=datetime.now().isoformat)++ def has_content_blocks(+ self,+ block_type: ContentBlockTypes | None = None,+ ) -> bool:+ """Check if the message has content blocks of the given type.++ Args:+ block_type (ContentBlockTypes | None, defaults to None):+ The type of the block to be checked. If `None`, it will+ check if there are any content blocks.+ """+ return len(self.get_content_blocks(block_type)) > 0++ def get_text_content(self, separator: str = "\n") -> str | None:+ """Get the pure text blocks from the message content.++ Args:+ separator (`str`, defaults to `\n`):+ The separator to use when concatenating multiple text blocks.+ Defaults to newline character.++ Returns:+ `str | None`:+ The concatenated text content, or `None` if there is no text+ content.+ """+ if isinstance(self.content, str):+ return self.content++ gathered_text = []+ for block in self.content:+ if block.type == "text":+ gathered_text.append(block.text)++ if gathered_text:+ return separator.join(gathered_text)++ return None++ @overload+ def get_content_blocks(+ self,+ block_type: Literal["text"],+ ) -> list[TextBlock]:+ ...++ @overload+ def get_content_blocks(+ self,+ block_type: Literal["thinking"],+ ) -> list[ThinkingBlock]:+ ...++ @overload+ def get_content_blocks(+ self,+ block_type: Literal["tool_call"],+ ) -> list[ToolCallBlock]:+ ...++ @overload+ def get_content_blocks(+ self,+ block_type: Literal["tool_result"],+ ) -> list[ToolResultBlock]:+ ...++ @overload+ def get_content_blocks(+ self,+ block_type: Literal["data"],+ ) -> list[DataBlock]:+ ...++ @overload+ def get_content_blocks(+ self,+ block_type: None = None,+ ) -> list[ContentBlock]:+ ...++ @overload+ def get_content_blocks(+ self,+ block_type: Literal["hint"],+ ) -> list[HintBlock]:+ ...++ def get_content_blocks(+ self,+ block_type: ContentBlockTypes | List[ContentBlockTypes] | None = None,+ ) -> Sequence[ContentBlock]:+ """Get the content in block format. If the content is a string,+ it will be converted to a text block.++ Args:+ block_type (`ContentBlockTypes | List[ContentBlockTypes] | None`, \+ optional):+ The type of the block to be extracted. If `None`, all blocks+ will be returned.++ Returns:+ `List[ContentBlock]`:+ The content blocks.+ """+ blocks = []+ if isinstance(self.content, str):+ blocks.append(+ TextBlock(text=self.content),+ )+ else:+ blocks = self.content or []++ if isinstance(block_type, str):+ blocks = [_ for _ in blocks if _.type == block_type]++ elif isinstance(block_type, list):+ blocks = [_ for _ in blocks if _.type in block_type]++ return blocks+++def UserMsg(+ name: str,+ content: str | list[TextBlock | DataBlock],+ metadata: dict | None = None,+ created_at: str | None = None,+) -> Msg:+ """Create a user message with role "user".++ Args:+ name (`str`):+ The name of the message sender.+ content (`str | list[TextBlock | DataBlock]`):+ The content of the message. It can be a string or a list of+ TextBlock or DataBlock.+ metadata (`dict | None`, optional):+ The metadata of the message. Defaults to `None`.+ created_at (`str | None`, optional):+ The creation time of the message in ISO format. Defaults to `None`.++ Returns:+ `Msg`:+ The created user message.+ """+ _assert_user_content_blocks(content)++ return Msg(+ name=name,+ content=content,+ role="user",+ metadata=metadata or {},+ created_at=created_at or datetime.now().isoformat(),+ )+++def AssistantMsg(+ name: str,+ content: str | list[ContentBlock],+ metadata: dict | None = None,+ created_at: str | None = None,+) -> Msg:+ """Create an assistant message with role "assistant".++ Args:+ name (`str`):+ The name of the message sender.+ content (`str | list[ContentBlock]`):+ The content of the message. It can be a string or a list of+ ContentBlock.+ metadata (`dict | None`, optional):+ The metadata of the message. Defaults to `None`.+ created_at (`str | None`, optional):+ The creation time of the message in ISO format. Defaults to `None`.++ Returns:+ `Msg`:+ The created assistant message.+ """+ return Msg(+ name=name,+ content=content,+ role="assistant",+ metadata=metadata or {},+ created_at=created_at or datetime.now().isoformat(),+ )+++def SystemMsg(+ name: str,+ content: str | list[TextBlock],+ metadata: dict | None = None,+ created_at: str | None = None,+) -> Msg:+ """Create a system message with role "system".++ Args:+ name (`str`):+ The name of the message sender.+ content (`str | list[TextBlock]`):+ The content of the message. It can be a string or a list of+ TextBlock.+ metadata (`dict | None`, optional):+ The metadata of the message. Defaults to `None`.+ created_at (`str | None`, optional):+ The creation time of the message in ISO format. Defaults to `None`.++ Returns:+ `Msg`:+ The created system message.+ """+ _assert_system_content_blocks(content)++ return Msg(+ name=name,+ content=content,+ role="system",+ metadata=metadata or {},+ created_at=created_at or datetime.now().isoformat(),+ )diff --git a/src/agentscope/message/_block.py b/src/agentscope/message/_block.pynew file mode 100644index 000000000..366091491--- /dev/null+++ b/src/agentscope/message/_block.py@@ -0,0 +1,94 @@+# -*- coding: utf-8 -*-+"""The content blocks of messages."""+import uuid+from typing import Literal, List, TypeAlias+from pydantic import BaseModel, Field+++class TextBlock(BaseModel):+ """The text block."""++ type: Literal["text"] = "text"+ text: str+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)+++class ThinkingBlock(BaseModel):+ """The thinking block."""++ type: Literal["thinking"] = "thinking"+ thinking: str+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)+++class HintBlock(BaseModel):+ """The hint block."""++ type: Literal["hint"] = "hint"+ hint: str+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)+++class Base64Source(BaseModel):+ """The base64 source."""++ type: Literal["base64"] = "base64"+ data: str+ media_type: str+++class URLSource(BaseModel):+ """The URL source."""++ type: Literal["url"] = "url"+ url: str+ media_type: str+++class DataBlock(BaseModel):+ """The data block for binary content (images, audio, video, etc.)."""++ type: Literal["data"] = "data"+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)+ source: Base64Source | URLSource+ name: str | None = None+++class ToolCallBlock(BaseModel):+ """The tool call block."""++ type: Literal["tool_call"] = "tool_call"+ id: str+ name: str+ input: str+ """The raw JSON string input of the tool, accumulated during streaming."""+ await_user_confirmation: bool = False+++class ToolResultBlock(BaseModel):+ """The tool result block."""++ type: Literal["tool_result"] = "tool_result"+ id: str+ name: str+ output: str | List[TextBlock | DataBlock]+ state: Literal["success", "error", "interrupted", "running"]+++ContentBlock: TypeAlias = (+ TextBlock+ | ThinkingBlock+ | HintBlock+ | ToolCallBlock+ | ToolResultBlock+ | DataBlock+)++ContentBlockTypes: TypeAlias = Literal[+ "text",+ "thinking",+ "hint",+ "tool_call",+ "tool_result",+ "data",+]diff --git a/src/agentscope/message/_message_base.py b/src/agentscope/message/_message_base.pydeleted file mode 100644index 6a69a9c58..000000000--- a/src/agentscope/message/_message_base.py+++ /dev/null@@ -1,241 +0,0 @@-# -*- coding: utf-8 -*--"""The message class in agentscope."""-from datetime import datetime-from typing import Literal, List, overload, Sequence--import shortuuid--from ._message_block import (- TextBlock,- ToolUseBlock,- ImageBlock,- AudioBlock,- ContentBlock,- VideoBlock,- ToolResultBlock,- ContentBlockTypes,-)-from ..types import JSONSerializableObject---class Msg:- """The message class in agentscope."""-- def __init__(- self,- name: str,- content: str | Sequence[ContentBlock],- role: Literal["user", "assistant", "system"],- metadata: dict[str, JSONSerializableObject] | None = None,- timestamp: str | None = None,- invocation_id: str | None = None,- ) -> None:- """Initialize the Msg object.-- Args:- name (`str`):- The name of the message sender.- content (`str | list[ContentBlock]`):- The content of the message.- role (`Literal["user", "assistant", "system"]`):- The role of the message sender.- metadata (`dict[str, JSONSerializableObject] | None`, optional):- The metadata of the message, e.g. structured output.- timestamp (`str | None`, optional):- The created timestamp of the message. If not given, the- timestamp will be set automatically.- invocation_id (`str | None`, optional):- The related API invocation id, if any. This is useful for- tracking the message in the context of an API call.- """-- self.name = name-- assert isinstance(- content,- (list, str),- ), "The content must be a string or a list of content blocks."-- self.content = content-- assert role in ["user", "assistant", "system"]- self.role = role-- self.metadata = metadata or {}-- self.id = shortuuid.uuid()- self.timestamp = (- timestamp- or datetime.now().strftime(- "%Y-%m-%d %H:%M:%S.%f",- )[:-3]- )- self.invocation_id = invocation_id-- def to_dict(self) -> dict:- """Convert the message into JSON dict data."""- return {- "id": self.id,- "name": self.name,- "role": self.role,- "content": self.content,- "metadata": self.metadata,- "timestamp": self.timestamp,- }-- @classmethod- def from_dict(cls, json_data: dict) -> "Msg":- """Load a message object from the given JSON data."""- new_obj = cls(- name=json_data["name"],- content=json_data["content"],- role=json_data["role"],- metadata=json_data.get("metadata", None),- timestamp=json_data.get("timestamp", None),- invocation_id=json_data.get("invocation_id", None),- )-- new_obj.id = json_data.get("id", new_obj.id)- return new_obj-- def has_content_blocks(- self,- block_type: Literal[- "text",- "tool_use",- "tool_result",- "image",- "audio",- "video",- ]- | None = None,- ) -> bool:- """Check if the message has content blocks of the given type.-- Args:- block_type (Literal["text", "tool_use", "tool_result", "image", \- "audio", "video"] | None, defaults to None):- The type of the block to be checked. If `None`, it will- check if there are any content blocks.- """- return len(self.get_content_blocks(block_type)) > 0-- def get_text_content(self, separator: str = "\n") -> str | None:- """Get the pure text blocks from the message content.-- Args:- separator (`str`, defaults to `\n`):- The separator to use when concatenating multiple text blocks.- Defaults to newline character.-- Returns:- `str | None`:- The concatenated text content, or `None` if there is no text- content.- """- if isinstance(self.content, str):- return self.content-- gathered_text = []- for block in self.content:- if block.get("type") == "text":- gathered_text.append(block["text"])-- if gathered_text:- return separator.join(gathered_text)-- return None-- @overload- def get_content_blocks(- self,- block_type: Literal["text"],- ) -> Sequence[TextBlock]:- ...-- @overload- def get_content_blocks(- self,- block_type: Literal["tool_use"],- ) -> Sequence[ToolUseBlock]:- ...-- @overload- def get_content_blocks(- self,- block_type: Literal["tool_result"],- ) -> Sequence[ToolResultBlock]:- ...-- @overload- def get_content_blocks(- self,- block_type: Literal["image"],- ) -> Sequence[ImageBlock]:- ...-- @overload- def get_content_blocks(- self,- block_type: Literal["audio"],- ) -> Sequence[AudioBlock]:- ...-- @overload- def get_content_blocks(- self,- block_type: Literal["video"],- ) -> Sequence[VideoBlock]:- ...-- @overload- def get_content_blocks(- self,- block_type: None = None,- ) -> Sequence[ContentBlock]:- ...-- def get_content_blocks(- self,- block_type: ContentBlockTypes | List[ContentBlockTypes] | None = None,- ) -> Sequence[ContentBlock]:- """Get the content in block format. If the content is a string,- it will be converted to a text block.-- Args:- block_type (`ContentBlockTypes | List[ContentBlockTypes] | None`, \- optional):- The type of the block to be extracted. If `None`, all blocks- will be returned.-- Returns:- `List[ContentBlock]`:- The content blocks.- """- blocks = []- if isinstance(self.content, str):- blocks.append(- TextBlock(type="text", text=self.content),- )- else:- blocks = self.content or []-- if isinstance(block_type, str):- blocks = [_ for _ in blocks if _["type"] == block_type]-- elif isinstance(block_type, list):- blocks = [_ for _ in blocks if _["type"] in block_type]-- return blocks-- def __repr__(self) -> str:- """Get the string representation of the message."""- return (- f"Msg(id='{self.id}', "- f"name='{self.name}', "- f"content={repr(self.content)}, "- f"role='{self.role}', "- f"metadata={repr(self.metadata)}, "- f"timestamp='{self.timestamp}', "- f"invocation_id='{self.invocation_id}')"- )diff --git a/src/agentscope/message/_message_block.py b/src/agentscope/message/_message_block.pydeleted file mode 100644index 2adb683ec..000000000--- a/src/agentscope/message/_message_block.py+++ /dev/null@@ -1,128 +0,0 @@-# -*- coding: utf-8 -*--# pylint: disable=R0901-"""The content blocks of messages"""--from typing import Literal, List-from typing_extensions import TypedDict, Required---class TextBlock(TypedDict, total=False):- """The text block."""-- type: Required[Literal["text"]]- """The type of the block"""- text: str- """The text content"""---class ThinkingBlock(TypedDict, total=False):- """The thinking block."""-- type: Required[Literal["thinking"]]- """The type of the block"""- thinking: str---class Base64Source(TypedDict, total=False):- """The base64 source"""-- type: Required[Literal["base64"]]- """The type of the src, must be `base64`"""-- media_type: Required[str]- """The media type of the data, e.g. `image/jpeg` or `audio/mpeg`"""-- data: Required[str]- """The base64 data, in format of RFC 2397"""---class URLSource(TypedDict, total=False):- """The URL source"""-- type: Required[Literal["url"]]- """The type of the src, must be `url`"""-- url: Required[str]- """The URL of the image or audio"""---class ImageBlock(TypedDict, total=False):- """The image block"""-- type: Required[Literal["image"]]- """The type of the block"""-- source: Required[Base64Source | URLSource]- """The src of the image"""---class AudioBlock(TypedDict, total=False):- """The audio block"""-- type: Required[Literal["audio"]]- """The type of the block"""-- source: Required[Base64Source | URLSource]- """The src of the audio"""---class VideoBlock(TypedDict, total=False):- """The video block"""-- type: Required[Literal["video"]]- """The type of the block"""-- source: Required[Base64Source | URLSource]- """The src of the audio"""---class ToolUseBlock(TypedDict, total=False):- """The tool use block."""-- type: Required[Literal["tool_use"]]- """The type of the block, must be `tool_use`"""- id: Required[str]- """The identity of the tool call"""- name: Required[str]- """The name of the tool"""- input: Required[dict[str, object]]- """The input of the tool"""- raw_input: str- """The raw string input of the tool from the model API"""---class ToolResultBlock(TypedDict, total=False):- """The tool result block."""-- type: Required[Literal["tool_result"]]- """The type of the block"""- id: Required[str]- """The identity of the tool call result"""- output: Required[- str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock]- ]- """The output of the tool function"""- name: Required[str]- """The name of the tool function"""---# The content block-ContentBlock = (- ToolUseBlock- | ToolResultBlock- | TextBlock- | ThinkingBlock- | ImageBlock- | AudioBlock- | VideoBlock-)--ContentBlockTypes = Literal[- "text",- "thinking",- "tool_use",- "tool_result",- "image",- "audio",- "video",-]diff --git a/src/agentscope/model/__init__.py b/src/agentscope/model/__init__.pyindex 7cd0b7a8f..221272767 100644--- a/src/agentscope/model/__init__.py+++ b/src/agentscope/model/__init__.py@@ -9,8 +9,10 @@ from ._anthropic_model import AnthropicChatModelfrom ._ollama_model import OllamaChatModelfrom ._gemini_model import GeminiChatModelfrom ._trinity_model import TrinityChatModel+from ._model_usage import ChatUsage__all__ = [+ "ChatUsage","ChatModelBase","ChatResponse","DashScopeChatModel",diff --git a/src/agentscope/model/_anthropic_model.py b/src/agentscope/model/_anthropic_model.pyindex 1c62fbe0c..744c7526e 100644--- a/src/agentscope/model/_anthropic_model.py+++ b/src/agentscope/model/_anthropic_model.py@@ -1,8 +1,5 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches, too-many-statements"""The Anthropic API model classes."""-import copy-import warningsfrom datetime import datetimefrom typing import (Any,@@ -10,7 +7,6 @@ from typing import (TYPE_CHECKING,List,Literal,- Type,)from collections import OrderedDict@@ -19,15 +15,10 @@ from pydantic import BaseModelfrom ._model_base import ChatModelBasefrom ._model_response import ChatResponsefrom ._model_usage import ChatUsage-from .._logging import logger-from .._utils._common import (- _json_loads_with_repair,- _parse_streaming_json_dict,- _create_tool_from_base_model,-)-from ..message import TextBlock, ToolUseBlock, ThinkingBlock+from ..formatter import FormatterBase, AnthropicChatFormatter+from ..message import TextBlock, ToolCallBlock, ThinkingBlockfrom ..tracing import trace_llm-from ..types._json import JSONSerializableObject+from ..types import JSONSerializableObject, ToolChoiceif TYPE_CHECKING:from anthropic.types.message import Message@@ -40,17 +31,25 @@ else:class AnthropicChatModel(ChatModelBase):"""The Anthropic model wrapper for AgentScope."""+ class ThinkingConfig(BaseModel):+ """Configuration for Claude's internal reasoning process."""++ enable_thinking: bool+ budget_tokens: int = 1024+def __init__(self,model_name: str,api_key: str | None = None,max_tokens: int = 2048,stream: bool = True,- thinking: dict | None = None,+ max_retries: int = 0,+ fallback_model_name: str | None = None,+ formatter: FormatterBase | None = None,+ thinking_config: ThinkingConfig | None = None,stream_tool_parsing: bool = True,client_kwargs: dict[str, JSONSerializableObject] | None = None,generate_kwargs: dict[str, JSONSerializableObject] | None = None,- **kwargs: Any,) -> None:"""Initialize the Anthropic chat model.@@ -59,21 +58,18 @@ class AnthropicChatModel(ChatModelBase):The model names.api_key (`str`):The anthropic API key.- stream (`bool`):- The streaming output or notmax_tokens (`int`):Limit the maximum token count the model can generate.- thinking (`dict | None`, default `None`):+ stream (`bool`):+ The streaming output or not+ max_retries (`int`, optional):+ Maximum number of retries on failure. Defaults to 0.+ fallback_model_name (`str | None`, optional):+ Fallback model name to use after all retries fail.+ formatter (`FormatterBase | None`, optional):+ Formatter for message preprocessing.+ thinking_config (`ThinkingConfig | None`, default `None`):Configuration for Claude's internal reasoning process.-- .. code-block:: python- :caption: Example of thinking-- {- "type": "enabled" | "disabled",- "budget_tokens": 1024- }-stream_tool_parsing (`bool`, default to `True`):Whether to parse incomplete tool use JSON during streamingwith auto-repair. If True, partial JSON (e.g., `'{"a": "x'`)@@ -87,33 +83,8 @@ class AnthropicChatModel(ChatModelBase):optional):The extra keyword arguments used in Anthropic API generation,e.g. `temperature`, `seed`.- **kwargs (`Any`):- Additional keyword arguments."""- # Handle deprecated client_args parameter from kwargs- client_args = kwargs.pop("client_args", None)- if client_args is not None and client_kwargs is not None:- raise ValueError(- "Cannot specify both 'client_args' and 'client_kwargs'. "- "Please use only 'client_kwargs' (client_args is deprecated).",- )-- if client_args is not None:- logger.warning(- "The parameter 'client_args' is deprecated and will be "- "removed in a future version. Please use 'client_kwargs' "- "instead. Automatically converting 'client_args' to "- "'client_kwargs'.",- )- client_kwargs = client_args-- if kwargs:- logger.warning(- "Unknown keyword arguments: %s. These will be ignored.",- list(kwargs.keys()),- )-try:import anthropicexcept ImportError as e:@@ -122,127 +93,75 @@ class AnthropicChatModel(ChatModelBase):"`pip install anthropic`.",) from e- super().__init__(model_name, stream)+ super().__init__(+ model_name=model_name,+ stream=stream,+ max_retries=max_retries,+ fallback_model_name=fallback_model_name,+ formatter=formatter or AnthropicChatFormatter(),+ )self.client = anthropic.AsyncAnthropic(api_key=api_key,**(client_kwargs or {}),)self.max_tokens = max_tokens- self.thinking = thinking+ self.thinking_config = thinking_configself.stream_tool_parsing = stream_tool_parsingself.generate_kwargs = generate_kwargs or {}@trace_llm- async def __call__(+ async def _call_api(self,+ model_name: str,messages: list[dict[str, Any]],tools: list[dict] | None = None,tool_choice: Literal["auto", "none", "required"] | str | None = None,- structured_model: Type[BaseModel] | None = None,**generate_kwargs: Any,) -> ChatResponse | AsyncGenerator[ChatResponse, None]:"""Get the response from Anthropic chat completions API by the givenarguments.Args:+ model_name (`str`):+ The model name to use for this call.messages (`list[dict]`):A list of dictionaries, where `role` and `content` fields arerequired, and `name` field is optional.tools (`list[dict]`, default `None`):- The tools JSON schemas that in format of:-- .. code-block:: python- :caption: Example of tools JSON schemas-- [- {- "type": "function",- "function": {- "name": "xxx",- "description": "xxx",- "parameters": {- "type": "object",- "properties": {- "param1": {- "type": "string",- "description": "..."- },- # Add more parameters as needed- },- "required": ["param1"]- }- },- # More schemas here- ]-+ The tools JSON schemas.tool_choice (`Literal["auto", "none", "required"] | str \| None`, default `None`):Controls which (if any) tool is called by the model.- Can be "auto", "none", "required", or specific tool- name. For more details, please refer to- https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/implement-tool-use- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output. When provided, the model will be forced- to return data that conforms to this schema by automatically- converting the BaseModel to a tool function and setting- `tool_choice` to enforce its usage. This enables structured- output generation.-- .. note:: When `structured_model` is specified,- both `tools` and `tool_choice` parameters are ignored,- and the model will only perform structured output- generation without calling any other tools.-**generate_kwargs (`Any`):- The keyword arguments for Anthropic chat completions API,- e.g. `temperature`, `top_p`, etc. Please- refer to the Anthropic API documentation for more details.+ The keyword arguments for Anthropic chat completions API.Returns:`ChatResponse | AsyncGenerator[ChatResponse, None]`:The response from the Anthropic chat completions API."""kwargs: dict[str, Any] = {- "model": self.model_name,+ "model": model_name,"max_tokens": self.max_tokens,"stream": self.stream,**self.generate_kwargs,**generate_kwargs,}- if self.thinking and "thinking" not in kwargs:- kwargs["thinking"] = self.thinking+ if self.thinking_config and "thinking" not in kwargs:+ kwargs["thinking"] = {+ "type": "enabled"+ if self.thinking_config.enable_thinking+ else "disabled",+ "budget_tokens": self.thinking_config.budget_tokens,+ }if tools:kwargs["tools"] = self._format_tools_json_schemas(tools)if tool_choice:- # Handle deprecated "any" option with warning- if tool_choice == "any":- warnings.warn(- '"any" is deprecated and will be removed in a future '- "version.",- DeprecationWarning,- )- tool_choice = "required"- self._validate_tool_choice(tool_choice, tools)- kwargs["tool_choice"] = self._format_tool_choice(tool_choice)-- if structured_model:- if tools or tool_choice:- logger.warning(- "structured_model is provided. Both 'tools' and "- "'tool_choice' parameters will be overridden and "- "ignored. The model will only perform structured output "- "generation without calling any other tools.",- )- format_tool = _create_tool_from_base_model(structured_model)- kwargs["tools"] = self._format_tools_json_schemas(- [format_tool],- )kwargs["tool_choice"] = self._format_tool_choice(- format_tool["function"]["name"],+ tool_choice,+ tools,)# Extract the system message@@ -260,14 +179,12 @@ class AnthropicChatModel(ChatModelBase):return self._parse_anthropic_stream_completion_response(start_datetime,response,- structured_model,)# Non-streaming responseparsed_response = await self._parse_anthropic_completion_response(start_datetime,response,- structured_model,)return parsed_response@@ -276,7 +193,6 @@ class AnthropicChatModel(ChatModelBase):self,start_datetime: datetime,response: Message,- structured_model: Type[BaseModel] | None = None,) -> ChatResponse:"""Given an Anthropic Message object, extract the content blocks andusages from it.@@ -286,20 +202,12 @@ class AnthropicChatModel(ChatModelBase):The start datetime of the response generation.response (`Message`):Anthropic Message object to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:ChatResponse (`ChatResponse`):A ChatResponse object containing the content blocks and usage.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""- content_blocks: List[ThinkingBlock | TextBlock | ToolUseBlock] = []- metadata = None+ content_blocks: List[ThinkingBlock | TextBlock | ToolCallBlock] = []if hasattr(response, "content") and response.content:for content_block in response.content:@@ -308,7 +216,6 @@ class AnthropicChatModel(ChatModelBase):and content_block.type == "thinking"):thinking_block = ThinkingBlock(- type="thinking",thinking=content_block.thinking,)thinking_block["signature"] = content_block.signature@@ -319,26 +226,22 @@ class AnthropicChatModel(ChatModelBase):and content_block.type == "text"):content_blocks.append(- TextBlock(- type="text",- text=content_block.text,- ),+ TextBlock(text=content_block.text),)elif (hasattr(content_block, "type")and content_block.type == "tool_use"):+ import json+content_blocks.append(- ToolUseBlock(- type="tool_use",+ ToolCallBlock(id=content_block.id,name=content_block.name,- input=content_block.input,+ input=json.dumps(content_block.input),),)- if structured_model:- metadata = content_block.inputusage = Noneif response.usage:@@ -350,8 +253,8 @@ class AnthropicChatModel(ChatModelBase):resp_kwargs: dict[str, Any] = {"content": content_blocks,+ "is_last": True,"usage": usage,- "metadata": metadata,}response_id = getattr(response, "id", None)if response_id:@@ -363,7 +266,6 @@ class AnthropicChatModel(ChatModelBase):self,start_datetime: datetime,response: AsyncStream,- structured_model: Type[BaseModel] | None = None,) -> AsyncGenerator[ChatResponse, None]:"""Given an Anthropic streaming response, extract the content blocksand usages from it and yield ChatResponse objects.@@ -373,38 +275,26 @@ class AnthropicChatModel(ChatModelBase):The start datetime of the response generation.response (`AsyncStream`):Anthropic AsyncStream object to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:`AsyncGenerator[ChatResponse, None]`:An async generator that yields ChatResponse objects containingthe content blocks and usage information for each chunk inthe streaming response.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""usage = Noneresponse_id: str | None = None- text_buffer = ""- thinking_buffer = ""+ # Accumulated state+ acc_text = ""+ acc_thinking = ""thinking_signature = ""- tool_calls = OrderedDict()- tool_call_buffers = {}- last_input_objs = {} # Store last input_obj for each tool_call- res = None- metadata = None-- # Record the last yielded content to parse the tools' input- last_content = None+ acc_tool_calls: OrderedDict = (+ OrderedDict()+ ) # index -> {id, name, input}async for event in response:- content_changed = False- thinking_changed = False+ delta_content: list = []if event.type == "message_start":message = event.message@@ -425,117 +315,80 @@ class AnthropicChatModel(ChatModelBase):if event.content_block.type == "tool_use":block_index = event.indextool_block = event.content_block- tool_calls[block_index] = {- "type": "tool_use",+ acc_tool_calls[block_index] = {"id": tool_block.id,"name": tool_block.name,"input": "",}- tool_call_buffers[block_index] = ""- content_changed = Trueelif event.type == "content_block_delta":block_index = event.indexdelta = event.deltaif delta.type == "text_delta":- text_buffer += delta.text- content_changed = True+ acc_text += delta.text+ delta_content.append(TextBlock(text=delta.text))elif delta.type == "thinking_delta":- thinking_buffer += delta.thinking- thinking_changed = True+ acc_thinking += delta.thinking+ delta_content.append(+ ThinkingBlock(thinking=delta.thinking),+ )elif delta.type == "signature_delta":thinking_signature = delta.signatureelif (delta.type == "input_json_delta"- and block_index in tool_calls+ and block_index in acc_tool_calls):- tool_call_buffers[block_index] += delta.partial_json or ""- tool_calls[block_index]["input"] = tool_call_buffers[- block_index- ]- content_changed = True+ fragment = delta.partial_json or ""+ acc_tool_calls[block_index]["input"] += fragment+ tc = acc_tool_calls[block_index]+ delta_content.append(+ ToolCallBlock(+ id=tc["id"],+ name=tc["name"],+ input=fragment,+ ),+ )elif event.type == "message_delta":if event.usage and usage:usage.output_tokens = event.usage.output_tokens- if (thinking_changed or content_changed) and usage:- contents: list = []- if thinking_buffer:- thinking_block = ThinkingBlock(- type="thinking",- thinking=thinking_buffer,- )- thinking_block["signature"] = thinking_signature- contents.append(thinking_block)- if text_buffer:- contents.append(- TextBlock(- type="text",- text=text_buffer,- ),- )- for block_index, tool_call in tool_calls.items():- input_str = tool_call["input"]- tool_id = tool_call["id"]-- # If parsing the tool input in streaming mode- if self.stream_tool_parsing:- repaired_input = _parse_streaming_json_dict(- input_str,- last_input_objs.get(tool_id),- )- last_input_objs[tool_id] = repaired_input-- else:- repaired_input = {}-- contents.append(- ToolUseBlock(- type=tool_call["type"],- id=tool_call["id"],- name=tool_call["name"],- input=repaired_input,- raw_input=input_str,- ),- )-- if structured_model:- metadata = repaired_input-- if contents:- _kwargs: dict[str, Any] = {- "content": contents,- "usage": usage,- "metadata": metadata,- }- if response_id:- _kwargs["id"] = response_id- res = ChatResponse(**_kwargs)- yield res- last_content = copy.deepcopy(contents)-- # If stream_tool_parsing is False, yield last contents- if not self.stream_tool_parsing and last_content and tool_calls:- metadata = None- # Update tool use blocks in last_contents inplace- for block in last_content:- if block.get("type") == "tool_use":- block["input"] = input_obj = _json_loads_with_repair(- block.get("raw_input") or "{}",- )-- if structured_model:- metadata = input_obj+ if delta_content:+ _kwargs: dict[str, Any] = {+ "content": delta_content,+ "is_last": False,+ "usage": usage,+ }+ if response_id:+ _kwargs["id"] = response_id+ yield ChatResponse(**_kwargs)++ # Build final accumulated content+ final_content: list = []+ if acc_thinking:+ thinking_block = ThinkingBlock(thinking=acc_thinking)+ thinking_block["signature"] = thinking_signature+ final_content.append(thinking_block)+ if acc_text:+ final_content.append(TextBlock(text=acc_text))+ for tc in acc_tool_calls.values():+ input_str = tc["input"]+ final_content.append(+ ToolCallBlock(+ id=tc["id"],+ name=tc["name"],+ input=input_str,+ ),+ )- _final_kwargs: dict[str, Any] = {- "content": last_content,- "usage": usage,- "metadata": metadata,- }- if response_id:- _final_kwargs["id"] = response_id- yield ChatResponse(**_final_kwargs)+ _final_kwargs: dict[str, Any] = {+ "content": final_content,+ "is_last": True,+ "usage": usage,+ }+ if response_id:+ _final_kwargs["id"] = response_id+ yield ChatResponse(**_final_kwargs)def _format_tools_json_schemas(self,@@ -566,22 +419,26 @@ class AnthropicChatModel(ChatModelBase):def _format_tool_choice(self,- tool_choice: Literal["auto", "none", "required"] | str | None,+ tool_choice: ToolChoice | None,+ tools: list[dict] | None,) -> dict | None:"""Format tool_choice parameter for API compatibility.Args:- tool_choice (`Literal["auto", "none", "required"] | str \- | None`, default `None`):- Controls which (if any) tool is called by the model.- Can be "auto", "none", "required", or specific tool- name. For more details, please refer to- https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/implement-tool-use+ tool_choice (`ToolChoice | None`):+ The unified tool choice parameter which can be a mode ("auto",+ "none", "required") or a specific function name.+ tools (`list[dict] | None`):+ The list of available tools, used for validation if+ tool_choice is a specific function name.+Returns:`dict | None`:The formatted tool choice configuration dict, or None iftool_choice is None."""+ self._validate_tool_choice(tool_choice, tools)+if tool_choice is None:return Nonediff --git a/src/agentscope/model/_dashscope_model.py b/src/agentscope/model/_dashscope_model.pyindex c37f1dc6f..255448e76 100644--- a/src/agentscope/model/_dashscope_model.py+++ b/src/agentscope/model/_dashscope_model.py@@ -1,9 +1,6 @@# -*- coding: utf-8 -*-"""The dashscope API model classes."""-import copy-import collections-import json-import os+import uuidimport warningsfrom datetime import datetimefrom http import HTTPStatus@@ -14,25 +11,21 @@ from typing import (Union,TYPE_CHECKING,List,- Literal,- Type,)+from pydantic import BaseModelfrom aioitertools import iter as giterfrom ._model_base import ChatModelBasefrom ._model_response import ChatResponsefrom ._model_usage import ChatUsage-from .._utils._common import (- _json_loads_with_repair,- _parse_streaming_json_dict,- _create_tool_from_base_model,-)-from ..message import TextBlock, ToolUseBlock, ThinkingBlock+from ..formatter import FormatterBase, DashScopeChatFormatter+from ..message import TextBlock, ToolCallBlock, ThinkingBlock, Msgfrom ..tracing import trace_llm-from ..types import JSONSerializableObject+from ..types import JSONSerializableObject, ToolChoicefrom .._logging import logger+if TYPE_CHECKING:from dashscope.api_entities.dashscope_response import GenerationResponsefrom dashscope.api_entities.dashscope_response import (@@ -70,16 +63,25 @@ class DashScopeChatModel(ChatModelBase):DashScope's diverse model offerings."""+ class ThinkingConfig(BaseModel):+ """The configuration for the thinking process in DashScope API."""++ enable_thinking: bool+ thinking_budget: int = 2000+ preserve_thinking: bool = False+def __init__(self,model_name: str,api_key: str,stream: bool = True,- enable_thinking: bool | None = None,- multimodality: bool | None = None,+ max_retries: int = 0,+ fallback_model_name: str | None = None,+ formatter: FormatterBase | None = None,+ thinking_config: ThinkingConfig | None = None,+ multimodality: bool = False,generate_kwargs: dict[str, JSONSerializableObject] | None = None,base_http_api_url: str | None = None,- stream_tool_parsing: bool = True,**_kwargs: Any,) -> None:"""Initialize the DashScope chat model.@@ -120,65 +122,58 @@ class DashScopeChatModel(ChatModelBase):**_kwargs (`Any`):Additional keyword arguments."""- if enable_thinking and not stream:++ self.thinking_config = (+ thinking_config+ or DashScopeChatModel.ThinkingConfig(+ enable_thinking=False,+ )+ )++ if self.thinking_config.enable_thinking and not stream:logger.info("In DashScope API, `stream` must be True when ""`enable_thinking` is True. ",)stream = True- super().__init__(model_name, stream)+ super().__init__(+ model_name,+ stream,+ max_retries,+ fallback_model_name,+ formatter or DashScopeChatFormatter(),+ )self.api_key = api_key- self.enable_thinking = enable_thinkingself.multimodality = multimodalityself.generate_kwargs = generate_kwargs or {}- self.stream_tool_parsing = stream_tool_parsingif base_http_api_url is not None:import dashscopedashscope.base_http_api_url = base_http_api_url- # Load headers from environment variable if exists- headers = os.getenv("DASHSCOPE_API_HEADERS")- if headers:- try:- headers = json.loads(str(headers))- if not isinstance(headers, dict):- raise json.JSONDecodeError("", "", 0)-- if self.generate_kwargs.get("headers"):- headers.update(self.generate_kwargs["headers"])-- self.generate_kwargs["headers"] = headers-- except json.JSONDecodeError:- logger.warning(- "Failed to parse DASHSCOPE_API_HEADERS environment "- "variable as JSON. It should be a JSON object.",- )-@trace_llm- async def __call__(+ async def _call_api(self,- messages: list[dict[str, Any]],+ model_name: str,+ messages: list[Msg],tools: list[dict] | None = None,- tool_choice: Literal["auto", "none", "required"] | str | None = None,- structured_model: Type[BaseModel] | None = None,+ tool_choice: ToolChoice | None = None,**kwargs: Any,) -> ChatResponse | AsyncGenerator[ChatResponse, None]:"""Get the response from the dashscopeGeneration/MultimodalConversation API by the given arguments.- .. note:: We unify the dashscope generation and multimodal conversation+ .. note:: We unify the DashScope generation and multimodal conversationAPIs into one method, since they support similar arguments and sharethe same functionality.Args:- messages (`list[dict[str, Any]]`):- A list of dictionaries, where `role` and `content` fields are- required.+ messages (`list[Msg]`):+ The Msg object that will be formatted and sent to the LLM API+ as the conversation messages.tools (`list[dict] | None`, default `None`):The tools JSON schemas that the model can use.tool_choice (`Literal["auto", "none", "required"] | str \@@ -189,19 +184,6 @@ class DashScopeChatModel(ChatModelBase):"required" will be converted to "auto".For more details, please refer tohttps://help.aliyun.com/zh/model-studio/qwen-function-calling- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output. When provided, the model will be forced- to return data that conforms to this schema by automatically- converting the BaseModel to a tool function and setting- `tool_choice` to enforce its usage. This enables structured- output generation.-- .. note:: When `structured_model` is specified,- both `tools` and `tool_choice` parameters are ignored,- and the model will only perform structured output- generation without calling any other tools.-**kwargs (`Any`):The keyword arguments for DashScope chat completions API,e.g. `temperature`, `max_tokens`, `top_p`, etc. Please@@ -211,9 +193,13 @@ class DashScopeChatModel(ChatModelBase):"""import dashscope+ # 1. Format the messages to the format required by DashScope API+ formatted_msg = await self.formatter.format(messages)++ # 2. Prepare the generation keyword argumentskwargs = {- "messages": messages,- "model": self.model_name,+ "messages": formatted_msg,+ "model": model_name,"stream": self.stream,"result_format": "message",# In agentscope, the `incremental_output` must be `True` when@@ -223,53 +209,27 @@ class DashScopeChatModel(ChatModelBase):**kwargs,}+ # toolsif tools:kwargs["tools"] = self._format_tools_json_schemas(tools)+ # tool choice optionsif tool_choice:- # Handle deprecated "any" option with warning- if tool_choice in ["any", "required"]:- warnings.warn(- f"'{tool_choice}' is not supported by DashScope API. "- "It will be converted to 'auto'.",- DeprecationWarning,- )- tool_choice = "auto"-- self._validate_tool_choice(tool_choice, tools)- kwargs["tool_choice"] = self._format_tool_choice(tool_choice)-- if (- self.enable_thinking is not None- and "enable_thinking" not in kwargs- ):- kwargs["enable_thinking"] = self.enable_thinking-- if structured_model:- if tools or tool_choice:- logger.warning(- "structured_model is provided. Both 'tools' and "- "'tool_choice' parameters will be overridden and "- "ignored. The model will only perform structured output "- "generation without calling any other tools.",- )- format_tool = _create_tool_from_base_model(structured_model)- kwargs["tools"] = self._format_tools_json_schemas(- [format_tool],- )kwargs["tool_choice"] = self._format_tool_choice(- format_tool["function"]["name"],+ tool_choice,+ tools,)+ # thinking related options+ kwargs = {**kwargs, **self.thinking_config.model_dump()}++ # 3. Call the API and parse the responsestart_datetime = datetime.now()+ # Use MultiModalConversation API if multimodality is True, or if the+ # model is multimodal based on the model name.if self.multimodality or (self.multimodality is None- and (- self.model_name.startswith(- "qvq",- )- or "-vl" in self.model_name- )+ and ("qvq" in self.model_name or "-vl" in self.model_name)):response = await dashscope.AioMultiModalConversation.call(api_key=self.api_key,@@ -286,27 +246,21 @@ class DashScopeChatModel(ChatModelBase):return self._parse_dashscope_stream_response(start_datetime,response,- structured_model,)parsed_response = await self._parse_dashscope_generation_response(start_datetime,response,- structured_model,)return parsed_response- # pylint: disable=too-many-branches, too-many-statementsasync def _parse_dashscope_stream_response(self,start_datetime: datetime,- response: Union[- AsyncGenerator[GenerationResponse, None],- AsyncGenerator[MultiModalConversationResponse, None],- Generator[MultiModalConversationResponse, None, None],- ],- structured_model: Type[BaseModel] | None = None,+ response: AsyncGenerator[GenerationResponse, None]+ | AsyncGenerator[MultiModalConversationResponse, None]+ | Generator[MultiModalConversationResponse, None, None],) -> AsyncGenerator[ChatResponse, Any]:"""Given a DashScope streaming response generator, extract the contentblocks and usages from it and yield ChatResponse objects.@@ -315,132 +269,117 @@ class DashScopeChatModel(ChatModelBase):start_datetime (`datetime`):The start datetime of the response generation.response (- `Union[AsyncGenerator[GenerationResponse, None], \- AsyncGenerator[MultiModalConversationResponse, None], \- Generator[MultiModalConversationResponse, None, None]]`+ `AsyncGenerator[GenerationResponse, None] | \+ AsyncGenerator[MultiModalConversationResponse, None] | \+ Generator[MultiModalConversationResponse, None, None]`):DashScope streaming response (async) generator(GenerationResponse or MultiModalConversationResponse).- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:- AsyncGenerator[ChatResponse, Any]:+ `AsyncGenerator[ChatResponse, Any]`:An async generator that yields ChatResponse objects containingthe content blocks and usage information for each chunk in thestreaming response.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""- acc_content, acc_thinking_content = "", ""- acc_tool_calls = collections.defaultdict(dict)- last_input_objs = {} # Store last input_obj for each tool_call- metadata = None- last_content = None+ # All delta should have the same block identifier+ acc_text = TextBlock(text="")+ acc_thinking = ThinkingBlock(thinking="")+ acc_tool_calls = {}usage = Noneresponse_id: str | None = Noneasync for chunk in giter(response):+ # The yield delta content blocks in the current chunk+ delta_content: list = []+if chunk.status_code != HTTPStatus.OK:raise RuntimeError(- f"Failed to get response from _ API: {chunk}",+ f"Failed to get response from DashScope API: {chunk}",)- if response_id is None:- response_id = getattr(chunk, "request_id", None)+ # Capture response_id from the first chunk if not already set+ response_id = response_id or chunk.request_id+ # The message field in the chunkmessage = chunk.output.choices[0].message# Update reasoning contentif isinstance(message.get("reasoning_content"), str):- acc_thinking_content += message["reasoning_content"]+ acc_thinking.thinking += message["reasoning_content"]+ delta_content.append(+ ThinkingBlock(+ id=acc_thinking.id,+ thinking=message["reasoning_content"],+ ),+ )# Update text contentif isinstance(message.content, str):- acc_content += message.content+ acc_text.text += message.content+ delta_content.append(+ TextBlock(+ id=acc_text.id,+ text=message.content,+ ),+ )elif isinstance(message.content, list):- for item in message.content:- if isinstance(item, dict) and "text" in item:- acc_content += item["text"]+ delta_text = "".join(+ [+ _["text"]+ for _ in message.content+ if isinstance(_, dict)+ and isinstance(_.get("text"), str)+ ],+ )+ acc_text.text += delta_text+ delta_content.append(+ ThinkingBlock(id=acc_text.id, thinking=delta_text),+ )# Update tool calls- for tool_call in message.get("tool_calls", []):+ for tool_call in message.get("tool_calls") or []:+ # Must be a dictionary+ if not isinstance(tool_call, dict):+ continue++ # Use the index to identify different tool callsindex = tool_call.get("index", 0)+ # Avoid appending duplicate idif "id" in tool_call and tool_call["id"] != acc_tool_calls[index].get("id"):- acc_tool_calls[index]["id"] = (- acc_tool_calls[index].get("id", "") + tool_call["id"]- )+ acc_tool_calls[index]["id"] = tool_call["id"]- if "function" in tool_call:+ # Arguments+ if isinstance(tool_call.get("function"), dict):func = tool_call["function"]++ # Function nameif "name" in func:- acc_tool_calls[index]["name"] = (- acc_tool_calls[index].get("name", "")- + func["name"]- )+ acc_tool_calls[index]["name"] = func["name"]- if "arguments" in func:+ # Function input arguments+ if isinstance(func.get("arguments"), str):acc_tool_calls[index]["arguments"] = (acc_tool_calls[index].get("arguments", "")+ func["arguments"])- # Build content blocks (always include thinking and text)- content_blocks: list[TextBlock | ToolUseBlock | ThinkingBlock] = []-- if acc_thinking_content:- content_blocks.append(- ThinkingBlock(- type="thinking",- thinking=acc_thinking_content,- ),- )-- if acc_content:- content_blocks.append(- TextBlock(- type="text",- text=acc_content,- ),- )-- for tool_call in acc_tool_calls.values():- # Only add intermediate tool use blocks if- # stream_tool_parsing is True- tool_id = tool_call.get("id", "")- input_str = tool_call.get("arguments")-- # If parsing the tool input in streaming mode- if self.stream_tool_parsing:- repaired_input = _parse_streaming_json_dict(- input_str,- last_input_objs.get(tool_id),+ # Append the tool call block for this chunk+ delta_content.append(+ ToolCallBlock(+ id=acc_tool_calls[index].get(+ "id",+ uuid.uuid4().hex,+ ),+ name=func.get("name", ""),+ input=func.get("arguments", ""),+ ),)- last_input_objs[tool_id] = repaired_input-- else:- # Otherwise, keep input as empty dict until the final chunk- repaired_input = {}-- content_blocks.append(- ToolUseBlock(- type="tool_use",- id=tool_id,- name=tool_call.get("name", ""),- input=repaired_input,- raw_input=input_str,- ),- )-- if structured_model:- metadata = repaired_input+ # The chunk usageif chunk.usage:usage = ChatUsage(input_tokens=chunk.usage.input_tokens,@@ -449,40 +388,32 @@ class DashScopeChatModel(ChatModelBase):metadata=chunk.usage,)- if content_blocks:- _kwargs: dict[str, Any] = {- "content": content_blocks,- "usage": usage,- "metadata": metadata,- }- if response_id:- _kwargs["id"] = response_id- parsed_chunk = ChatResponse(**_kwargs)- yield parsed_chunk- last_content = copy.deepcopy(content_blocks)-- # If stream_tool_parsing is False, we need to parse the final tool- # use inputs here- if not self.stream_tool_parsing and last_content and acc_tool_calls:- metadata = None- # Update tool use blocks in last_contents inplace- for block in last_content:- if block.get("type") == "tool_use":- block["input"] = input_obj = _json_loads_with_repair(- str(block.get("raw_input") or "{}"),- )+ # Yield the chat response+ yield ChatResponse(+ id=response_id,+ content=delta_content,+ is_last=False,+ usage=usage,+ )++ # Yield a final complete response with the accumulated content and+ # final usage+ content = []+ if acc_text.text:+ content.append(acc_text)- if structured_model:- metadata = input_obj+ if acc_thinking.thinking:+ content.append(acc_thinking)- _final_kwargs: dict[str, Any] = {- "content": last_content,- "usage": usage,- "metadata": metadata,- }- if response_id:- _final_kwargs["id"] = response_id- yield ChatResponse(**_final_kwargs)+ if acc_tool_calls:+ content.extend(acc_tool_calls.values())++ yield ChatResponse(+ id=response_id or uuid.uuid4().hex,+ content=content,+ is_last=True,+ usage=usage,+ )async def _parse_dashscope_generation_response(self,@@ -491,7 +422,6 @@ class DashScopeChatModel(ChatModelBase):GenerationResponse,MultiModalConversationResponse,],- structured_model: Type[BaseModel] | None = None,) -> ChatResponse:"""Given a DashScope GenerationResponse object, extract the contentblocks and usages from it.@@ -504,71 +434,55 @@ class DashScopeChatModel(ChatModelBase):):Dashscope GenerationResponse | MultiModalConversationResponseobject to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:- ChatResponse (`ChatResponse`):+ `ChatResponse`:A ChatResponse object containing the content blocks and usage.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""# Collect the content blocks from the response.- if response.status_code != 200:- raise RuntimeError(response)+ if response.status_code != HTTPStatus.OK:+ raise RuntimeError(+ f"Failed to get response from DashScope API: {response}",+ )- content_blocks: List[TextBlock | ToolUseBlock] = []- metadata: dict | None = None+ content_blocks: List[TextBlock | ToolCallBlock] = []message = response.output.choices[0].message- content = message.get("content")-- if response.output.choices[0].message.get("content") not in [- None,- "",- [],- ]:- if isinstance(content, list):- for item in content:- if isinstance(item, dict) and "text" in item:- content_blocks.append(- TextBlock(- type="text",- text=item["text"],- ),- )- else:- content_blocks.append(- TextBlock(- type="text",- text=content,++ # The content and tool calls in the message+ content, tool_calls = message.get("content"), message.get("tool_calls")++ if isinstance(content, str):+ content_blocks.append(TextBlock(text=content))++ elif isinstance(content, list):+ content_blocks.append(+ TextBlock(+ text="".join(+ [+ _["text"]+ for _ in content+ if isinstance(_, dict)+ and isinstance(_.get("text"), str)+ ],),- )+ ),+ )++ if isinstance(tool_calls, list):+ for tool_call in tool_calls:+ function = tool_call.get("function") or {}+ name = function.get("name") or ""+ input_ = function.get("arguments") or "{}"- if message.get("tool_calls"):- for tool_call in message["tool_calls"]:- input_ = _json_loads_with_repair(- tool_call["function"].get(- "arguments",- "{}",- )- or "{}",- )content_blocks.append(- ToolUseBlock(- type="tool_use",- name=tool_call["function"]["name"],+ ToolCallBlock(+ id=tool_call.get("id", uuid.uuid4().hex),+ name=name,input=input_,- id=tool_call["id"],),)- if structured_model:- metadata = input_-# Usage informationusage = Noneif response.usage:@@ -579,19 +493,15 @@ class DashScopeChatModel(ChatModelBase):metadata=response.usage,)- resp_kwargs: dict[str, Any] = {- "content": content_blocks,- "usage": usage,- "metadata": metadata,- }- response_id = getattr(response, "request_id", None)- if response_id:- resp_kwargs["id"] = response_id-- return ChatResponse(**resp_kwargs)+ return ChatResponse(+ id=getattr(response, "request_id", uuid.uuid4().hex),+ content=content_blocks,+ is_last=True,+ usage=usage,+ )+ @staticmethoddef _format_tools_json_schemas(- self,schemas: list[dict[str, Any]],) -> list[dict[str, Any]]:"""Format the tools JSON schema into required format for DashScope API.@@ -617,26 +527,37 @@ class DashScopeChatModel(ChatModelBase):def _format_tool_choice(self,- tool_choice: Literal["auto", "none", "required"] | str | None,+ tool_choice: ToolChoice | None,+ tools: list[dict] | None,) -> str | dict | None:"""Format tool_choice parameter for API compatibility.Args:- tool_choice (`Literal["auto", "none", "required"] | str \- | None`, default `None`):- Controls which (if any) tool is called by the model. For more- details, please refer to- https://help.aliyun.com/zh/model-studio/qwen-function-calling+ tool_choice (`ToolChoice | None`):+ The unified tool choice parameter which can be a mode ("auto",+ "none", "required") or a specific function name.+ tools (`list[dict] | None`):+ The list of available tools, used for validation if+ tool_choice is a specific function name.Returns:`dict | None`:The formatted tool choice configuration dict, or None iftool_choice is None."""- if tool_choice is None:- return None+ # Validate the tool_choice value+ self._validate_tool_choice(tool_choice, tools)++ # DashScope API specific validation and formattingif tool_choice in ["auto", "none"]:return tool_choice+if tool_choice == "required":+ warnings.warn(+ f"'{tool_choice}' is not supported by DashScope API. "+ "It will be converted to 'auto'.",+ DeprecationWarning,+ )return "auto"+return {"type": "function", "function": {"name": tool_choice}}diff --git a/src/agentscope/model/_gemini_model.py b/src/agentscope/model/_gemini_model.pyindex 0a393b69d..e5d2bd33e 100644--- a/src/agentscope/model/_gemini_model.py+++ b/src/agentscope/model/_gemini_model.py@@ -1,9 +1,7 @@# -*- coding: utf-8 -*--# mypy: disable-error-code="dict-item""""The Google Gemini model in agentscope."""import base64import copy-import warningsfrom datetime import datetimeimport jsonfrom typing import (@@ -12,20 +10,19 @@ from typing import (TYPE_CHECKING,AsyncIterator,Literal,- Type,List,)from pydantic import BaseModelfrom .._logging import logger-from .._utils._common import _json_loads_with_repair-from ..message import ToolUseBlock, TextBlock, ThinkingBlock+from ..formatter import FormatterBase+from ..message import ToolCallBlock, TextBlock, ThinkingBlockfrom ._model_usage import ChatUsagefrom ._model_base import ChatModelBasefrom ._model_response import ChatResponsefrom ..tracing import trace_llm-from ..types import JSONSerializableObject+from ..types import JSONSerializableObject, ToolChoiceif TYPE_CHECKING:from google.genai.types import GenerateContentResponse@@ -115,15 +112,23 @@ def _flatten_json_schema(schema: dict) -> dict:class GeminiChatModel(ChatModelBase):"""The Google Gemini chat model class in agentscope."""+ class ThinkingConfig(BaseModel):+ """Configuration for enabling thinking blocks in Gemini responses."""++ enable_thinking: bool+ thinking_budget: int = 1024+def __init__(self,model_name: str,api_key: str,stream: bool = True,- thinking_config: dict | None = None,+ max_retries: int = 0,+ fallback_model_name: str | None = None,+ formatter: FormatterBase | None = None,+ thinking_config: ThinkingConfig | None = None,client_kwargs: dict[str, JSONSerializableObject] | None = None,generate_kwargs: dict[str, JSONSerializableObject] | None = None,- **kwargs: Any,) -> None:"""Initialize the Gemini chat model.@@ -134,19 +139,14 @@ class GeminiChatModel(ChatModelBase):The API key for Google Gemini.stream (`bool`, default `True`):Whether to use streaming output or not.- thinking_config (`dict | None`, optional):+ max_retries (`int`, optional):+ Maximum number of retries on failure. Defaults to 0.+ fallback_model_name (`str | None`, optional):+ Fallback model name to use after all retries fail.+ formatter (`FormatterBase | None`, optional):+ Formatter for message preprocessing.+ thinking_config (`ThinkingConfig | None`, optional):Thinking config, supported models are 2.5 Pro, 2.5 Flash, etc.- Refer to https://ai.google.dev/gemini-api/docs/thinking for- more details.-- .. code-block:: python- :caption: Example of thinking_config-- {- "include_thoughts": True, # enable thoughts or not- "thinking_budget": 1024 # Max tokens for reasoning- }-client_kwargs (`dict[str, JSONSerializableObject] | None`, \optional):The extra keyword arguments to initialize the Gemini client.@@ -154,33 +154,8 @@ class GeminiChatModel(ChatModelBase):optional):The extra keyword arguments used in Gemini API generation,e.g. `temperature`, `seed`.- **kwargs (`Any`):- Additional keyword arguments."""- # Handle deprecated client_args parameter from kwargs- client_args = kwargs.pop("client_args", None)- if client_args is not None and client_kwargs is not None:- raise ValueError(- "Cannot specify both 'client_args' and 'client_kwargs'. "- "Please use only 'client_kwargs' (client_args is deprecated).",- )-- if client_args is not None:- logger.warning(- "The parameter 'client_args' is deprecated and will be "- "removed in a future version. Please use 'client_kwargs' "- "instead. Automatically converting 'client_args' to "- "'client_kwargs'.",- )- client_kwargs = client_args-- if kwargs:- logger.warning(- "Unknown keyword arguments: %s. These will be ignored.",- list(kwargs.keys()),- )-try:from google import genaiexcept ImportError as e:@@ -189,7 +164,13 @@ class GeminiChatModel(ChatModelBase):"`pip install -q -U google-genai`",) from e- super().__init__(model_name, stream)+ super().__init__(+ model_name=model_name,+ stream=stream,+ max_retries=max_retries,+ fallback_model_name=fallback_model_name,+ formatter=formatter,+ )self.client = genai.Client(api_key=api_key,@@ -199,12 +180,12 @@ class GeminiChatModel(ChatModelBase):self.generate_kwargs = generate_kwargs or {}@trace_llm- async def __call__(+ async def _call_api(self,+ model_name: str,messages: list[dict],tools: list[dict] | None = None,tool_choice: Literal["auto", "none", "required"] | str | None = None,- structured_model: Type[BaseModel] | None = None,**config_kwargs: Any,) -> ChatResponse | AsyncGenerator[ChatResponse, None]:"""Call the Gemini model with the provided arguments.@@ -221,59 +202,32 @@ class GeminiChatModel(ChatModelBase):Can be "auto", "none", "required", or specific tool name.For more details, please refer tohttps://ai.google.dev/gemini-api/docs/function-calling?hl=en&example=meeting#function_calling_modes- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.-- .. note:: When `structured_model` is specified,- both `tools` and `tool_choice` parameters are ignored,- and the model will only perform structured output- generation without calling any other tools.-- For more details, please refer to- https://ai.google.dev/gemini-api/docs/structured-output-**config_kwargs (`Any`):The keyword arguments for Gemini chat completions API."""config: dict = {- "thinking_config": self.thinking_config,**self.generate_kwargs,**config_kwargs,}+ if self.thinking_config:+ config["thinking_config"] = {+ "include_thoughts": self.thinking_config.enable_thinking,+ "thinking_budget": self.thinking_config.thinking_budget,+ }if tools:config["tools"] = self._format_tools_json_schemas(tools)if tool_choice:- # Handle deprecated "any" option with warning- if tool_choice == "any":- warnings.warn(- '"any" is deprecated and will be removed in a future '- "version.",- DeprecationWarning,- )- tool_choice = "required"- self._validate_tool_choice(tool_choice, tools)- config["tool_config"] = self._format_tool_choice(tool_choice)-- if structured_model:- if tools or tool_choice:- logger.warning(- "structured_model is provided. Both 'tools' and "- "'tool_choice' parameters will be overridden and "- "ignored. The model will only perform structured output "- "generation without calling any other tools.",- )- config.pop("tools", None)- config.pop("tool_config", None)- config["response_mime_type"] = "application/json"- config["response_schema"] = structured_model+ config["tool_config"] = self._format_tool_choice(+ tool_choice,+ tools,+ )# Prepare the arguments for the Gemini API call- kwargs: dict[str, JSONSerializableObject] = {- "model": self.model_name,+ kwargs: dict = {+ "model": model_name,"contents": messages,"config": config,}@@ -287,7 +241,6 @@ class GeminiChatModel(ChatModelBase):return self._parse_gemini_stream_generation_response(start_datetime,response,- structured_model,)# non-streaming@@ -298,7 +251,6 @@ class GeminiChatModel(ChatModelBase):parsed_response = self._parse_gemini_generation_response(start_datetime,response,- structured_model,)return parsed_response@@ -338,7 +290,6 @@ class GeminiChatModel(ChatModelBase):self,start_datetime: datetime,response: AsyncIterator[GenerateContentResponse],- structured_model: Type[BaseModel] | None = None,) -> AsyncGenerator[ChatResponse, None]:"""Given a Gemini streaming generation response, extract thecontent blocks and usages from it and yield ChatResponse objects.@@ -348,27 +299,23 @@ class GeminiChatModel(ChatModelBase):The start datetime of the response generation.response (`AsyncIterator[GenerateContentResponse]`):Gemini GenerateContentResponse async iterator to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:`AsyncGenerator[ChatResponse, None]`:An async generator that yields ChatResponse objects containingthe content blocks and usage information for each chunk in thestreaming response.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""- text = ""- thinking = ""- tool_calls: list[ToolUseBlock] = []- metadata: dict | None = None- response_id: str | None = None+ # Accumulated state+ acc_text = ""+ acc_thinking = ""+ acc_tool_calls: dict = {} # call_id -> {name, input}+ usage = None+async for chunk in response:+ delta_content: list = []+if (chunk.candidatesand chunk.candidates[0].content@@ -377,20 +324,16 @@ class GeminiChatModel(ChatModelBase):for part in chunk.candidates[0].content.parts:if part.text:if part.thought:- thinking += part.text+ acc_thinking += part.text+ delta_content.append(+ ThinkingBlock(thinking=part.text),+ )else:- text += part.text+ acc_text += part.text+ delta_content.append(TextBlock(text=part.text))if part.function_call:keyword_args = part.function_call.args or {}- # .. note:: Gemini API always returns None for- # function_call.id, so we use thought_signature- # as the unique identifier for tool- # calls when available. That maybe- # infeasible someday, but Gemini- # requires the thought_signature for some- # llms like gemini-3-pro-if part.thought_signature:call_id = base64.b64encode(part.thought_signature,@@ -398,61 +341,56 @@ class GeminiChatModel(ChatModelBase):else:call_id = part.function_call.id- tool_calls.append(- ToolUseBlock(- type="tool_use",+ input_str = json.dumps(+ keyword_args,+ ensure_ascii=False,+ )+ acc_tool_calls[call_id] = {+ "name": part.function_call.name,+ "input": input_str,+ }+ delta_content.append(+ ToolCallBlock(id=call_id,name=part.function_call.name,- input=keyword_args,- raw_input=json.dumps(- keyword_args,- ensure_ascii=False,- ),+ input=input_str,),)- # Text parts- if text and structured_model:- metadata = _json_loads_with_repair(text)-usage = self._extract_usage(chunk.usage_metadata, start_datetime)- # The content blocks for the current chunk- content_blocks: list = []-- if thinking:- content_blocks.append(- ThinkingBlock(- type="thinking",- thinking=thinking,- ),- )-- if text:- content_blocks.append(- TextBlock(- type="text",- text=text,- ),+ if delta_content:+ yield ChatResponse(+ content=delta_content,+ is_last=False,+ usage=usage,)- if response_id is None:- response_id = getattr(chunk, "response_id", None)+ # Build final accumulated content+ final_content: list = []+ if acc_thinking:+ final_content.append(ThinkingBlock(thinking=acc_thinking))+ if acc_text:+ final_content.append(TextBlock(text=acc_text))+ for call_id, tc in acc_tool_calls.items():+ final_content.append(+ ToolCallBlock(+ id=call_id,+ name=tc["name"],+ input=tc["input"],+ ),+ )- _kwargs: dict[str, Any] = {- "content": content_blocks + tool_calls,- "usage": usage,- "metadata": metadata,- }- if response_id:- _kwargs["id"] = response_id- yield ChatResponse(**_kwargs)+ yield ChatResponse(+ content=final_content,+ is_last=True,+ usage=usage,+ )def _parse_gemini_generation_response(self,start_datetime: datetime,response: GenerateContentResponse,- structured_model: Type[BaseModel] | None = None,) -> ChatResponse:"""Given a Gemini chat completion response object, extract the contentblocks and usages from it.@@ -462,21 +400,12 @@ class GeminiChatModel(ChatModelBase):The start datetime of the response generation.response (`GenerateContentResponse`):The Gemini generation response object to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:ChatResponse (`ChatResponse`):A ChatResponse object containing the content blocks and usage.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""- content_blocks: List[TextBlock | ToolUseBlock | ThinkingBlock] = []- metadata: dict | None = None- tool_calls: list = []+ content_blocks: List[TextBlock | ToolCallBlock | ThinkingBlock] = []if (response.candidates@@ -487,28 +416,15 @@ class GeminiChatModel(ChatModelBase):if part.text:if part.thought:content_blocks.append(- ThinkingBlock(- type="thinking",- thinking=part.text,- ),+ ThinkingBlock(thinking=part.text),)else:content_blocks.append(- TextBlock(- type="text",- text=part.text,- ),+ TextBlock(text=part.text),)if part.function_call:keyword_args = part.function_call.args or {}- # .. note:: Gemini API always returns None for- # function_call.id, so we use thought_signature- # as the unique identifier for tool- # calls when available. That maybe infeasible- # someday, but Gemini requires the thought_signature- # for some llms like gemini-3-pro-if part.thought_signature:call_id = base64.b64encode(part.thought_signature,@@ -516,35 +432,21 @@ class GeminiChatModel(ChatModelBase):else:call_id = part.function_call.id- tool_calls.append(- ToolUseBlock(- type="tool_use",+ content_blocks.append(+ ToolCallBlock(id=call_id,name=part.function_call.name,- input=keyword_args,- raw_input=json.dumps(- keyword_args,- ensure_ascii=False,- ),+ input=json.dumps(keyword_args, ensure_ascii=False),),)- # For the structured output case- if response.text and structured_model:- metadata = _json_loads_with_repair(response.text)-usage = self._extract_usage(response.usage_metadata, start_datetime)- resp_kwargs: dict[str, Any] = {- "content": content_blocks + tool_calls,- "usage": usage,- "metadata": metadata,- }- response_id = getattr(response, "response_id", None)- if response_id:- resp_kwargs["id"] = response_id-- return ChatResponse(**resp_kwargs)+ return ChatResponse(+ content=content_blocks,+ is_last=True,+ usage=usage,+ )def _format_tools_json_schemas(self,@@ -637,23 +539,26 @@ class GeminiChatModel(ChatModelBase):def _format_tool_choice(self,- tool_choice: Literal["auto", "none", "required"] | str | None,+ tool_choice: ToolChoice | None,+ tools: list[dict] | None,) -> dict | None:"""Format tool_choice parameter for API compatibility.Args:- tool_choice (`Literal["auto", "none", "required"] | str | None`, \- default `None`):- Controls which (if any) tool is called by the model.- Can be "auto", "none", "required", or specific tool name.- For more details, please refer to- https://ai.google.dev/gemini-api/docs/function-calling?hl=en&example=meeting#function_calling_modes+ tool_choice (`ToolChoice | None`):+ The unified tool choice parameter which can be a mode ("auto",+ "none", "required") or a specific function name.+ tools (`list[dict] | None`):+ The list of available tools, used for validation if+ tool_choice is a specific function name.Returns:`dict | None`:The formatted tool choice configuration dict, or None iftool_choice is None."""+ self._validate_tool_choice(tool_choice, tools)+if tool_choice is None:return Nonediff --git a/src/agentscope/model/_model_base.py b/src/agentscope/model/_model_base.pyindex ff2503cad..7553e2057 100644--- a/src/agentscope/model/_model_base.py+++ b/src/agentscope/model/_model_base.py@@ -5,7 +5,10 @@ from abc import abstractmethodfrom typing import AsyncGenerator, Anyfrom ._model_response import ChatResponse-+from .._logging import logger+from ..formatter import FormatterBase+from ..message import Msg+from ..types import ToolChoice_TOOL_CHOICE_MODES = ["auto", "none", "required"]@@ -19,10 +22,23 @@ class ChatModelBase:stream: bool"""Is the model output streaming or not"""+ max_retries: int+ """Maximum number of retries on failure"""++ fallback_model_name: str | None+ """Fallback model name to use after all retries fail"""++ formatter: Any | None+ """The API formatter that format the messages into the required format for+ the underlying API."""+def __init__(self,model_name: str,stream: bool,+ max_retries: int = 0,+ fallback_model_name: str | None = None,+ formatter: FormatterBase | None = None,) -> None:"""Initialize the chat model base class.@@ -31,37 +47,130 @@ class ChatModelBase:The name of the modelstream (`bool`):Whether the model output is streaming or not+ max_retries (`int`, optional):+ Maximum number of retries on failure. Defaults to 0.+ fallback_model_name (`str | None`, optional):+ Fallback model name to use after all retries fail.+ formatter (`FormatterBase | None`, optional):+ Formatter for message preprocessing."""self.model_name = model_nameself.stream = stream+ self.max_retries = max_retries+ self.fallback_model_name = fallback_model_name+ self.formatter = formatter- @abstractmethodasync def __call__(self,- *args: Any,+ messages: list[Msg],+ tools: list[dict] | None = None,+ tool_choice: ToolChoice | None = None,+ **kwargs: Any,+ ) -> ChatResponse | AsyncGenerator[ChatResponse, None]:+ """Call the model with retry and fallback logic.++ Formats messages using formatter if available, then attempts+ to call the model up to max_retries + 1 times. If all attempts+ fail and a fallback model is configured, retries with that model.++ Args:+ messages (`list[Msg]`):+ The messages to send to the model.+ tools (`list[dict] | None`, optional):+ The tools available to the model.+ tool_choice (`ToolChoice | None`, optional):+ The tool choice mode or function name.+ **kwargs:+ Additional keyword arguments passed to the underlying API.+ """+ if self.formatter is not None:+ messages = await self.formatter.format(messages)++ last_error: Exception | None = None++ for model_name in self._models_to_try():+ for attempt in range(self.max_retries + 1):+ try:+ return await self._call_api(+ model_name,+ messages=messages,+ tools=tools,+ tool_choice=tool_choice,+ **kwargs,+ )+ except Exception as e:+ last_error = e+ if attempt < self.max_retries:+ logger.warning(+ "Attempt %d failed for model %s: %s. Retrying...",+ attempt + 1,+ model_name,+ str(e),+ )+ else:+ logger.warning(+ "All %d attempt(s) failed for model %s.",+ self.max_retries + 1,+ model_name,+ )++ if last_error is not None:+ raise last_error+ raise RuntimeError("No models to try")++ def _models_to_try(self) -> list[str]:+ """Return the ordered list of model names to try."""+ models = [self.model_name]+ if self.fallback_model_name:+ models.append(self.fallback_model_name)+ return models++ @abstractmethod+ async def _call_api(+ self,+ model_name: str,+ messages: list[Msg],+ tools: list[dict] | None = None,+ tool_choice: ToolChoice | None = None,**kwargs: Any,) -> ChatResponse | AsyncGenerator[ChatResponse, None]:- pass+ """Call the underlying API. Subclasses must implement this method.++ Args:+ model_name (`str`):+ The model name to use for this call.+ messages (`list[Msg]`):+ The messages to send to the model.+ tools (`list[dict] | None`, optional):+ The tools available to the model.+ tool_choice (`ToolChoice | None`, optional):+ The tool choice mode or function name.+ **kwargs:+ Additional keyword arguments for the underlying API.+ """def _validate_tool_choice(self,- tool_choice: str,+ tool_choice: ToolChoice | None,tools: list[dict] | None,) -> None:- """- Validate tool_choice parameter.+ """Validate tool_choice parameter.Args:- tool_choice (`str`):+ tool_choice (`ToolChoice | None`):Tool choice mode or function nametools (`list[dict] | None`):Available tools list+Raises:- TypeError: If tool_choice is not string- ValueError: If tool_choice is invalid+ `ValueError`:+ If tool_choice is not a valid mode or function name."""+ if tool_choice is None:+ return+if not isinstance(tool_choice, str):- raise TypeError(+ raise ValueError(f"tool_choice must be str, got {type(tool_choice)}",)if tool_choice in _TOOL_CHOICE_MODES:diff --git a/src/agentscope/model/_model_response.py b/src/agentscope/model/_model_response.pyindex 6f8e52f61..985091b55 100644--- a/src/agentscope/model/_model_response.py+++ b/src/agentscope/model/_model_response.py@@ -1,17 +1,17 @@# -*- coding: utf-8 -*-"""The model response module."""-+import uuidfrom dataclasses import dataclass, field+from datetime import datetimefrom typing import Literal, Sequencefrom ._model_usage import ChatUsage-from .._utils._common import _get_timestampfrom .._utils._mixin import DictMixinfrom ..message import (TextBlock,- ToolUseBlock,+ ToolCallBlock,ThinkingBlock,- AudioBlock,+ DataBlock,)from ..types import JSONSerializableObject@@ -20,14 +20,18 @@ from ..types import JSONSerializableObjectclass ChatResponse(DictMixin):"""The response of chat models."""- content: Sequence[TextBlock | ToolUseBlock | ThinkingBlock | AudioBlock]+ content: Sequence[TextBlock | ToolCallBlock | ThinkingBlock | DataBlock]"""The content of the chat response, which can include text blocks,tool use blocks, or thinking blocks."""- id: str = field(default_factory=lambda: _get_timestamp(True))+ is_last: bool+ """Whether this response is the last response, if `Ture`, the content will+ be the complete response, otherwise the content is a partial response"""++ id: str = field(default_factory=lambda: uuid.uuid4().hex)"""The unique identifier formatter """- created_at: str = field(default_factory=_get_timestamp)+ created_at: str = field(default_factory=datetime.now().isoformat)"""When the response was created"""type: Literal["chat"] = field(default_factory=lambda: "chat")diff --git a/src/agentscope/model/_ollama_model.py b/src/agentscope/model/_ollama_model.pyindex 6a86fa1a1..09d3190bb 100644--- a/src/agentscope/model/_ollama_model.py+++ b/src/agentscope/model/_ollama_model.py@@ -9,9 +9,7 @@ from typing import (AsyncGenerator,AsyncIterator,Literal,- Type,)-from collections import OrderedDictfrom pydantic import BaseModel@@ -19,8 +17,8 @@ from . import ChatResponsefrom ._model_base import ChatModelBasefrom ._model_usage import ChatUsagefrom .._logging import logger-from .._utils._common import _json_loads_with_repair-from ..message import ToolUseBlock, TextBlock, ThinkingBlock+from ..formatter import FormatterBase, OllamaChatFormatter+from ..message import ToolCallBlock, TextBlock, ThinkingBlockfrom ..tracing import trace_llmfrom ..types import JSONSerializableObject@@ -33,17 +31,24 @@ else:class OllamaChatModel(ChatModelBase):"""The Ollama chat model class in agentscope."""+ class ThinkingConfig(BaseModel):+ """Configuration for thinking in Ollama chat models."""++ enable_thinking: bool+def __init__(self,model_name: str,stream: bool = False,+ max_retries: int = 0,+ fallback_model_name: str | None = None,+ formatter: FormatterBase | None = None,+ thinking_config: ThinkingConfig | None = None,options: dict = None,keep_alive: str = "5m",- enable_thinking: bool | None = None,host: str | None = None,client_kwargs: dict[str, JSONSerializableObject] | None = None,generate_kwargs: dict[str, JSONSerializableObject] | None = None,- **kwargs: Any,) -> None:"""Initialize the Ollama chat model.@@ -52,6 +57,15 @@ class OllamaChatModel(ChatModelBase):The name of the model.stream (`bool`, default `True`):Streaming mode or not.+ max_retries (`int`, optional):+ Maximum number of retries on failure. Defaults to 0.+ fallback_model_name (`str | None`, optional):+ Fallback model name to use after all retries fail.+ formatter (`FormatterBase | None`, optional):+ Formatter for message preprocessing.+ thinking_config (`ThinkingConfig | None`, default `None`)+ Configuration for thinking, only for models such as qwen3,+ deepseek-r1, etc.options (`dict`, default `None`):Additional parameters to pass to the Ollama API. These caninclude temperature etc.@@ -59,10 +73,6 @@ class OllamaChatModel(ChatModelBase):Duration to keep the model loaded in memory. The format is anumber followed by a unit suffix (s for seconds, m for minutes, h for hours).- enable_thinking (`bool | None`, default `None`)- Whether enable thinking or not, only for models such as qwen3,- deepseek-r1, etc. For more details, please refer to- https://ollama.com/search?c=thinkinghost (`str | None`, default `None`):The host address of the Ollama server. If None, uses thedefault address (typically http://localhost:11434).@@ -72,9 +82,6 @@ class OllamaChatModel(ChatModelBase):generate_kwargs (`dict[str, JSONSerializableObject] | None`, \optional):The extra keyword arguments used in Ollama API generation.- **kwargs (`Any`):- Additional keyword arguments to pass to the base chat model- class."""try:@@ -85,31 +92,38 @@ class OllamaChatModel(ChatModelBase):'running command `pip install "ollama>=0.1.7"`',) from e- super().__init__(model_name, stream)+ super().__init__(+ model_name=model_name,+ stream=stream,+ max_retries=max_retries,+ fallback_model_name=fallback_model_name,+ formatter=formatter or OllamaChatFormatter(),+ )self.client = ollama.AsyncClient(host=host,**(client_kwargs or {}),- **kwargs,)self.options = optionsself.keep_alive = keep_alive- self.think = enable_thinking+ self.thinking_config = thinking_configself.generate_kwargs = generate_kwargs or {}@trace_llm- async def __call__(+ async def _call_api(self,+ model_name: str,messages: list[dict[str, Any]],tools: list[dict] | None = None,tool_choice: Literal["auto", "none", "required"] | str | None = None,- structured_model: Type[BaseModel] | None = None,**kwargs: Any,) -> ChatResponse | AsyncGenerator[ChatResponse, None]:"""Get the response from Ollama chat completions API by the givenarguments.Args:+ model_name (`str`):+ The model name to use for this call.messages (`list[dict]`):A list of dictionaries, where `role` and `content` fields arerequired, and `name` field is optional.@@ -118,9 +132,6 @@ class OllamaChatModel(ChatModelBase):tool_choice (`Literal["auto", "none", "required"] | str \| None`, default `None`):Ollama doesn't support `tool_choice` argument yet.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.**kwargs (`Any`):The keyword arguments for Ollama chat completions API,e.g. `think`etc. Please refer to the Ollama API@@ -132,7 +143,7 @@ class OllamaChatModel(ChatModelBase):"""kwargs = {- "model": self.model_name,+ "model": model_name,"messages": messages,"stream": self.stream,"options": self.options,@@ -141,8 +152,8 @@ class OllamaChatModel(ChatModelBase):**kwargs,}- if self.think is not None and "think" not in kwargs:- kwargs["think"] = self.think+ if self.thinking_config and "think" not in kwargs:+ kwargs["think"] = self.thinking_config.enable_thinkingif tools:kwargs["tools"] = self._format_tools_json_schemas(tools)@@ -150,9 +161,6 @@ class OllamaChatModel(ChatModelBase):if tool_choice:logger.warning("Ollama does not support tool_choice yet, ignored.")- if structured_model:- kwargs["format"] = structured_model.model_json_schema()-start_datetime = datetime.now()response = await self.client.chat(**kwargs)@@ -160,13 +168,11 @@ class OllamaChatModel(ChatModelBase):return self._parse_ollama_stream_completion_response(start_datetime,response,- structured_model,)parsed_response = await self._parse_ollama_completion_response(start_datetime,response,- structured_model,)return parsed_response@@ -175,7 +181,6 @@ class OllamaChatModel(ChatModelBase):self,start_datetime: datetime,response: AsyncIterator[OllamaChatResponse],- structured_model: Type[BaseModel] | None = None,) -> AsyncGenerator[ChatResponse, None]:"""Given an Ollama streaming completion response, extract thecontent blocks and usages from it and yield ChatResponse objects.@@ -185,9 +190,6 @@ class OllamaChatModel(ChatModelBase):The start datetime of the response generation.response (`AsyncIterator[OllamaChatResponse]`):Ollama streaming response async iterator to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:AsyncGenerator[ChatResponse, None]:@@ -195,94 +197,94 @@ class OllamaChatModel(ChatModelBase):the content blocks and usage information for each chunk in thestreaming response.- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`.-"""- accumulated_text = ""- acc_thinking_content = ""- tool_calls = OrderedDict() # Store tool calls- metadata: dict | None = None+ # Accumulated state+ acc_text = ""+ acc_thinking = ""+ acc_tool_calls: dict = {} # tool_id -> {name, input}response_id: str | None = None+ usage = Noneasync for chunk in response:- # Handle text content+ delta_content: list = []msg = chunk.message- acc_thinking_content += msg.thinking or ""- accumulated_text += msg.content or ""++ # Handle thinking delta+ if msg.thinking:+ acc_thinking += msg.thinking+ delta_content.append(ThinkingBlock(thinking=msg.thinking))++ # Handle text delta+ if msg.content:+ acc_text += msg.content+ delta_content.append(TextBlock(text=msg.content))# Handle tool callsfor idx, tool_call in enumerate(msg.tool_calls or []):function = tool_call.functiontool_id = f"{idx}_{function.name}"- tool_calls[tool_id] = {- "type": "tool_use",- "id": tool_id,+ input_str = json.dumps(function.arguments)+ acc_tool_calls[tool_id] = {"name": function.name,- "input": function.arguments,- "raw_input": json.dumps(function.arguments),+ "input": input_str,}- # Calculate usage statistics+ delta_content.append(+ ToolCallBlock(+ id=tool_id,+ name=function.name,+ input=input_str,+ ),+ )++ if response_id is None:+ response_id = getattr(chunk, "id", None)++ # Calculate usagecurrent_time = (datetime.now() - start_datetime).total_seconds()usage = ChatUsage(input_tokens=getattr(chunk, "prompt_eval_count", 0) or 0,output_tokens=getattr(chunk, "eval_count", 0) or 0,time=current_time,)- # Create content blocks- contents: list = []-- if acc_thinking_content:- contents.append(- ThinkingBlock(- type="thinking",- thinking=acc_thinking_content,- ),- )- if accumulated_text:- contents.append(TextBlock(type="text", text=accumulated_text))- if structured_model:- metadata = _json_loads_with_repair(accumulated_text)-- # Add tool call blocks- for tool_call in tool_calls.values():- try:- input_data = tool_call["input"]- if isinstance(input_data, str):- input_data = _json_loads_with_repair(input_data)- contents.append(- ToolUseBlock(- type=tool_call["type"],- id=tool_call["id"],- name=tool_call["name"],- input=input_data,- ),- )- except Exception as e:- print(f"Error parsing tool call input: {e}")-- if response_id is None:- response_id = getattr(chunk, "id", None)-- # Generate response when there's new content or at final chunk- if chunk.done or contents:+ if delta_content:_kwargs: dict[str, Any] = {- "content": contents,+ "content": delta_content,+ "is_last": False,"usage": usage,- "metadata": metadata,}if response_id:_kwargs["id"] = response_id- res = ChatResponse(**_kwargs)- yield res+ yield ChatResponse(**_kwargs)++ # Build final accumulated content+ final_content: list = []+ if acc_thinking:+ final_content.append(ThinkingBlock(thinking=acc_thinking))+ if acc_text:+ final_content.append(TextBlock(text=acc_text))+ for tool_id, tc in acc_tool_calls.items():+ final_content.append(+ ToolCallBlock(+ id=tool_id,+ name=tc["name"],+ input=tc["input"],+ ),+ )++ _final_kwargs: dict[str, Any] = {+ "content": final_content,+ "is_last": True,+ "usage": usage,+ }+ if response_id:+ _final_kwargs["id"] = response_id+ yield ChatResponse(**_final_kwargs)async def _parse_ollama_completion_response(self,start_datetime: datetime,response: OllamaChatResponse,- structured_model: Type[BaseModel] | None = None,) -> ChatResponse:"""Given an Ollama chat completion response object, extract the contentblocks and usages from it.@@ -292,49 +294,29 @@ class OllamaChatModel(ChatModelBase):The start datetime of the response generation.response (`OllamaChatResponse`):Ollama OllamaChatResponse object to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:`ChatResponse`:A ChatResponse object containing the content blocks and usage.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""- content_blocks: List[TextBlock | ToolUseBlock | ThinkingBlock] = []- metadata: dict | None = None+ content_blocks: List[TextBlock | ToolCallBlock | ThinkingBlock] = []if response.message.thinking:content_blocks.append(- ThinkingBlock(- type="thinking",- thinking=response.message.thinking,- ),+ ThinkingBlock(thinking=response.message.thinking),)if response.message.content:content_blocks.append(- TextBlock(- type="text",- text=response.message.content,- ),+ TextBlock(text=response.message.content),)- if structured_model:- metadata = _json_loads_with_repair(- response.message.content,- )for idx, tool_call in enumerate(response.message.tool_calls or []):content_blocks.append(- ToolUseBlock(- type="tool_use",+ ToolCallBlock(id=f"{idx}_{tool_call.function.name}",name=tool_call.function.name,- input=tool_call.function.arguments,- raw_input=json.dumps(tool_call.function.arguments),+ input=json.dumps(tool_call.function.arguments),),)@@ -348,8 +330,8 @@ class OllamaChatModel(ChatModelBase):resp_kwargs: dict[str, Any] = {"content": content_blocks,+ "is_last": True,"usage": usage,- "metadata": metadata,}response_id = getattr(response, "id", None)if response_id:diff --git a/src/agentscope/model/_openai_model.py b/src/agentscope/model/_openai_model.pyindex 19752c1fa..1b6da8fb7 100644--- a/src/agentscope/model/_openai_model.py+++ b/src/agentscope/model/_openai_model.py@@ -1,7 +1,5 @@# -*- coding: utf-8 -*--# pylint: disable=too-many-branches"""OpenAI Chat model class."""-import copyimport warningsfrom datetime import datetimefrom typing import (@@ -10,29 +8,25 @@ from typing import (List,AsyncGenerator,Literal,- Type,)from collections import OrderedDict+from openai.types import ReasoningEffortfrom pydantic import BaseModelfrom . import ChatResponsefrom ._model_base import ChatModelBasefrom ._model_usage import ChatUsage-from .._logging import logger-from .._utils._common import (- _json_loads_with_repair,- _parse_streaming_json_dict,-)+from ..formatter import FormatterBasefrom ..message import (- ToolUseBlock,TextBlock,ThinkingBlock,- AudioBlock,Base64Source,+ DataBlock,+ ToolCallBlock,)from ..tracing import trace_llm-from ..types import JSONSerializableObject+from ..types import JSONSerializableObject, ToolChoiceif TYPE_CHECKING:from openai.types.chat import ChatCompletion@@ -70,18 +64,25 @@ def _format_audio_data_for_qwen_omni(messages: list[dict]) -> None:class OpenAIChatModel(ChatModelBase):"""The OpenAI chat model class."""+ class ThinkingConfig(BaseModel):+ """Configuration for reasoning effort levels."""++ enable_thinking: bool+ reasoning_effect: ReasoningEffort | None = None+def __init__(self,model_name: str,api_key: str | None = None,stream: bool = True,- reasoning_effort: Literal["low", "medium", "high"] | None = None,+ max_retries: int = 0,+ fallback_model_name: str | None = None,+ formatter: FormatterBase | None = None,+ thinking_config: ThinkingConfig | None = None,organization: str = None,- stream_tool_parsing: bool = True,client_type: Literal["openai", "azure"] = "openai",client_kwargs: dict[str, JSONSerializableObject] | None = None,generate_kwargs: dict[str, JSONSerializableObject] | None = None,- **kwargs: Any,) -> None:"""Initialize the openai client.@@ -93,21 +94,16 @@ class OpenAIChatModel(ChatModelBase):be read from the environment variable `OPENAI_API_KEY`.stream (`bool`, default `True`):Whether to use streaming output or not.- reasoning_effort (`Literal["low", "medium", "high"] | None`, \- optional):- Reasoning effort, supported for o3, o4, etc. Please refer to- `OpenAI documentation- <https://platform.openai.com/docs/guides/reasoning?api-mode=chat>`_- for more details.+ max_retries (`int`, optional):+ Maximum number of retries on failure. Defaults to 0.+ fallback_model_name (`str | None`, optional):+ Fallback model name to use after all retries fail.+ formatter (`FormatterBase | None`, optional):+ Formatter for message preprocessing.+ thinking_config (`ThinkingConfig | None`, optional):+ Configuration for reasoning effort levels.organization (`str`, default `None`):- The organization ID for OpenAI API. If not specified, it will- be read from the environment variable `OPENAI_ORGANIZATION`.- stream_tool_parsing (`bool`, default to `True`):- Whether to parse incomplete tool use JSON during streaming- with auto-repair. If True, partial JSON (e.g., `'{"a": "x'`)- is repaired to valid dicts ({"a": "x"}) in real-time for- immediate tool function input. Otherwise, the input field- remains {} until the final chunk arrives.+ The organization ID for OpenAI API.client_type (`Literal["openai", "azure"]`, default `openai`):Selects which OpenAI-compatible client to initialize.client_kwargs (`dict[str, JSONSerializableObject] | None`, \@@ -117,34 +113,19 @@ class OpenAIChatModel(ChatModelBase):optional):The extra keyword arguments used in OpenAI API generation,e.g. `temperature`, `seed`.- **kwargs (`Any`):- Additional keyword arguments."""+ self.thinking_config = (+ thinking_config+ or OpenAIChatModel.ThinkingConfig(enable_thinking=False)+ )- # Handle deprecated client_args parameter from kwargs- client_args = kwargs.pop("client_args", None)- if client_args is not None and client_kwargs is not None:- raise ValueError(- "Cannot specify both 'client_args' and 'client_kwargs'. "- "Please use only 'client_kwargs' (client_args is deprecated).",- )-- if client_args is not None:- logger.warning(- "The parameter 'client_args' is deprecated and will be "- "removed in a future version. Please use 'client_kwargs' "- "instead. Automatically converting 'client_args' to "- "'client_kwargs'.",- )- client_kwargs = client_args-- if kwargs:- logger.warning(- "Unknown keyword arguments: %s. These will be ignored.",- list(kwargs.keys()),- )-- super().__init__(model_name, stream)+ super().__init__(+ model_name=model_name,+ stream=stream,+ max_retries=max_retries,+ fallback_model_name=fallback_model_name,+ formatter=formatter,+ )import openai@@ -166,23 +147,23 @@ class OpenAIChatModel(ChatModelBase):**(client_kwargs or {}),)- self.reasoning_effort = reasoning_effort- self.stream_tool_parsing = stream_tool_parsingself.generate_kwargs = generate_kwargs or {}@trace_llm- async def __call__(+ async def _call_api(self,+ model_name: str,messages: list[dict],tools: list[dict] | None = None,tool_choice: Literal["auto", "none", "required"] | str | None = None,- structured_model: Type[BaseModel] | None = None,**kwargs: Any,) -> ChatResponse | AsyncGenerator[ChatResponse, None]:"""Get the response from OpenAI chat completions API by the givenarguments.Args:+ model_name (`str`):+ The model name to use for this call.messages (`list[dict]`):A list of dictionaries, where `role` and `content` fields arerequired, and `name` field is optional.@@ -191,29 +172,8 @@ class OpenAIChatModel(ChatModelBase):tool_choice (`Literal["auto", "none", "required"] | str \| None`, default `None`):Controls which (if any) tool is called by the model.- Can be "auto", "none", "required", or specific tool- name. For more details, please refer to- https://platform.openai.com/docs/api-reference/responses/create#responses_create-tool_choice- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output. When provided, the model will be forced- to return data that conforms to this schema by automatically- converting the BaseModel to a tool function and setting- `tool_choice` to enforce its usage. This enables structured- output generation.-- .. note:: When `structured_model` is specified,- both `tools` and `tool_choice` parameters are ignored,- and the model will only perform structured output- generation without calling any other tools.-- For more details, please refer to the `official document- <https://platform.openai.com/docs/guides/structured-outputs>`_-**kwargs (`Any`):- The keyword arguments for OpenAI chat completions API,- e.g. `temperature`, `max_tokens`, `top_p`, etc. Please- refer to the OpenAI API documentation for more details.+ The keyword arguments for OpenAI chat completions API.Returns:`ChatResponse | AsyncGenerator[ChatResponse, None]`:@@ -233,18 +193,21 @@ class OpenAIChatModel(ChatModelBase):)# Qwen-omni requires different base64 audio format from openai- if "omni" in self.model_name.lower():+ if "omni" in model_name.lower():_format_audio_data_for_qwen_omni(messages)kwargs = {- "model": self.model_name,+ "model": model_name,"messages": messages,"stream": self.stream,**self.generate_kwargs,**kwargs,}- if self.reasoning_effort and "reasoning_effort" not in kwargs:- kwargs["reasoning_effort"] = self.reasoning_effort+ if (+ self.thinking_config.enable_thinking+ and self.thinking_config.reasoning_effect+ ):+ kwargs["reasoning_effort"] = self.thinking_config.reasoning_effectif tools:kwargs["tools"] = self._format_tools_json_schemas(tools)@@ -258,60 +221,36 @@ class OpenAIChatModel(ChatModelBase):DeprecationWarning,)tool_choice = "required"- self._validate_tool_choice(tool_choice, tools)- kwargs["tool_choice"] = self._format_tool_choice(tool_choice)+ kwargs["tool_choice"] = self._format_tool_choice(+ tool_choice,+ tools,+ )if self.stream:kwargs["stream_options"] = {"include_usage": True}start_datetime = datetime.now()- if structured_model:- if tools or tool_choice:- logger.warning(- "structured_model is provided. Both 'tools' and "- "'tool_choice' parameters will be overridden and "- "ignored. The model will only perform structured output "- "generation without calling any other tools.",- )- kwargs.pop("stream", None)- kwargs.pop("tools", None)- kwargs.pop("tool_choice", None)- kwargs["response_format"] = structured_model- if not self.stream:- response = await self.client.chat.completions.parse(**kwargs)- else:- response = self.client.chat.completions.stream(**kwargs)- return self._parse_openai_stream_response(- start_datetime,- response,- structured_model,- )- else:- response = await self.client.chat.completions.create(**kwargs)+ response = await self.client.chat.completions.create(**kwargs)if self.stream:return self._parse_openai_stream_response(start_datetime,response,- structured_model,)# Non-streaming responseparsed_response = self._parse_openai_completion_response(start_datetime,response,- structured_model,)return parsed_response- # pylint: disable=too-many-statementsasync def _parse_openai_stream_response(self,start_datetime: datetime,response: AsyncStream,- structured_model: Type[BaseModel] | None = None,) -> AsyncGenerator[ChatResponse, None]:"""Given an OpenAI streaming completion response, extract the contentblocks and usages from it and yield ChatResponse objects.@@ -321,45 +260,23 @@ class OpenAIChatModel(ChatModelBase):The start datetime of the response generation.response (`AsyncStream`):OpenAI AsyncStream object to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:`AsyncGenerator[ChatResponse, None]`:An async generator that yields ChatResponse objects containingthe content blocks and usage information for each chunk inthe streaming response.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""- usage, res = None, None- response_id: str | None = None- text = ""- thinking = ""- audio = ""- tool_calls = OrderedDict()- last_input_objs = {} # Store last input_obj for each tool_call- metadata: dict | None = None- contents: List[- TextBlock | ToolUseBlock | ThinkingBlock | AudioBlock- ] = []- last_contents = None-- async with response as stream:- async for item in stream:- if structured_model:- if item.type != "chunk":- continue- chunk = item.chunk- else:- chunk = item+ usage = None- if response_id is None:- response_id = getattr(chunk, "id", None)+ # Accumulators+ acc_text = ""+ acc_thinking = ""+ acc_audio = ""+ acc_tool_calls = OrderedDict()+ async with response as stream:+ async for chunk in stream:if chunk.usage:usage = ChatUsage(input_tokens=chunk.usage.prompt_tokens,@@ -369,164 +286,133 @@ class OpenAIChatModel(ChatModelBase):)if not chunk.choices:- if usage and contents:- _kwargs: dict[str, Any] = {- "content": contents,- "usage": usage,- "metadata": metadata,- }- if response_id:- _kwargs["id"] = response_id- res = ChatResponse(**_kwargs)- yield rescontinuechoice = chunk.choices[0]- delta_reasoning = getattr(+ # Extract delta content+ delta_thinking = getattr(choice.delta,"reasoning_content",None,)- if not isinstance(delta_reasoning, str):- delta_reasoning = getattr(choice.delta, "reasoning", None)- if not isinstance(delta_reasoning, str):- delta_reasoning = ""-- thinking += delta_reasoning- text += getattr(choice.delta, "content", None) or ""-- if (- hasattr(choice.delta, "audio")- and "data" in choice.delta.audio- ):- audio += choice.delta.audio["data"]- if (- hasattr(choice.delta, "audio")- and "transcript" in choice.delta.audio- ):- text += choice.delta.audio["transcript"]-+ if not isinstance(delta_thinking, str):+ delta_thinking = getattr(choice.delta, "reasoning", None)+ if not isinstance(delta_thinking, str):+ delta_thinking = ""++ delta_text = getattr(choice.delta, "content", None) or ""+ delta_audio = ""++ if isinstance(getattr(choice.delta, "audio", None), dict):+ if "data" in choice.delta.audio:+ delta_audio = choice.delta.audio["data"]+ if "transcript" in choice.delta.audio:+ delta_text += choice.delta.audio["transcript"]++ # Accumulate+ acc_thinking += delta_thinking+ acc_text += delta_text+ acc_audio += delta_audio++ # Handle tool callsfor tool_call in (getattr(choice.delta, "tool_calls", None) or []):- if tool_call.index in tool_calls:+ if tool_call.index in acc_tool_calls:if tool_call.function.arguments is not None:- tool_calls[tool_call.index][+ acc_tool_calls[tool_call.index]["input"] += tool_call.function.arguments-else:- tool_calls[tool_call.index] = {- "type": "tool_use",+ acc_tool_calls[tool_call.index] = {"id": tool_call.id,"name": tool_call.function.name,"input": tool_call.function.arguments or "",}- contents = []+ # Build delta content blocks+ delta_contents: List[+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock+ ] = []- if thinking:- contents.append(- ThinkingBlock(- type="thinking",- thinking=thinking,- ),+ if delta_thinking:+ delta_contents.append(+ ThinkingBlock(thinking=delta_thinking),)- if audio:+ if delta_audio:media_type = self.generate_kwargs.get("audio", {}).get("format","wav",)- contents.append(- AudioBlock(- type="audio",+ delta_contents.append(+ DataBlock(source=Base64Source(- data=audio,+ data=delta_audio,media_type=f"audio/{media_type}",- type="base64",),),)- if text:- contents.append(- TextBlock(- type="text",- text=text,- ),+ if delta_text:+ delta_contents.append(+ TextBlock(text=delta_text),)- if structured_model:- metadata = _json_loads_with_repair(text)+ # Yield delta response+ if delta_contents:+ yield ChatResponse(+ content=delta_contents,+ usage=usage,+ is_last=False,+ )- for tool_call in tool_calls.values():- input_str = tool_call["input"]- tool_id = tool_call["id"]+ # Build final accumulated content blocks+ final_contents: List[+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock+ ] = []- # If parsing the tool input in streaming mode- if self.stream_tool_parsing:- repaired_input = _parse_streaming_json_dict(- input_str,- last_input_objs.get(tool_id),- )- last_input_objs[tool_id] = repaired_input+ if acc_thinking:+ final_contents.append(ThinkingBlock(thinking=acc_thinking))- else:- # Otherwise, keep input as empty dict until the final- # chunk- repaired_input = {}-- contents.append(- ToolUseBlock(- type=tool_call["type"],- id=tool_id,- name=tool_call["name"],- input=repaired_input,- raw_input=input_str,- ),- )+ if acc_audio:+ media_type = self.generate_kwargs.get("audio", {}).get(+ "format",+ "wav",+ )+ final_contents.append(+ DataBlock(+ source=Base64Source(+ data=acc_audio,+ media_type=f"audio/{media_type}",+ ),+ ),+ )- if contents:- _kwargs = {- "content": contents,- "usage": usage,- "metadata": metadata,- }- if response_id:- _kwargs["id"] = response_id- res = ChatResponse(**_kwargs)- yield res- last_contents = copy.deepcopy(contents)-- # If stream_tool_parsing is False, yield last contents- if not self.stream_tool_parsing and tool_calls and last_contents:- metadata = None- # Update tool use blocks in last_contents inplace- for block in last_contents:- if block.get("type") == "tool_use":- block["input"] = input_obj = _json_loads_with_repair(- str(block.get("raw_input") or "{}"),- )+ if acc_text:+ final_contents.append(TextBlock(text=acc_text))- if structured_model:- metadata = input_obj+ for tool_call in acc_tool_calls.values():+ final_contents.append(+ ToolCallBlock(+ id=tool_call["id"],+ name=tool_call["name"],+ input=tool_call["input"],+ ),+ )- _kwargs = {- "content": last_contents,- "usage": usage,- "metadata": metadata,- }- if response_id:- _kwargs["id"] = response_id- yield ChatResponse(**_kwargs)+ # Yield final accumulated response+ yield ChatResponse(+ content=final_contents,+ usage=usage,+ is_last=True,+ )def _parse_openai_completion_response(self,start_datetime: datetime,response: ChatCompletion,- structured_model: Type[BaseModel] | None = None,) -> ChatResponse:"""Given an OpenAI chat completion response object, extract the contentblocks and usages from it.@@ -536,22 +422,14 @@ class OpenAIChatModel(ChatModelBase):The start datetime of the response generation.response (`ChatCompletion`):OpenAI ChatCompletion object to parse.- structured_model (`Type[BaseModel] | None`, default `None`):- A Pydantic BaseModel class that defines the expected structure- for the model's output.Returns:ChatResponse (`ChatResponse`):A ChatResponse object containing the content blocks and usage.-- .. note::- If `structured_model` is not `None`, the expected structured output- will be stored in the metadata of the `ChatResponse`."""content_blocks: List[- TextBlock | ToolUseBlock | ThinkingBlock | AudioBlock+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock] = []- metadata: dict | None = Noneif response.choices:choice = response.choices[0]@@ -563,18 +441,12 @@ class OpenAIChatModel(ChatModelBase):if reasoning is not None:content_blocks.append(- ThinkingBlock(- type="thinking",- thinking=reasoning,- ),+ ThinkingBlock(thinking=reasoning),)if choice.message.content:content_blocks.append(- TextBlock(- type="text",- text=response.choices[0].message.content,- ),+ TextBlock(text=response.choices[0].message.content),)if choice.message.audio:media_type = self.generate_kwargs.get("audio", {}).get(@@ -582,39 +454,28 @@ class OpenAIChatModel(ChatModelBase):"mp3",)content_blocks.append(- AudioBlock(- type="audio",+ DataBlock(source=Base64Source(data=choice.message.audio.data,media_type=f"audio/{media_type}",- type="base64",),),)if choice.message.audio.transcript:content_blocks.append(- TextBlock(- type="text",- text=choice.message.audio.transcript,- ),+ TextBlock(text=choice.message.audio.transcript),)for tool_call in choice.message.tool_calls or []:content_blocks.append(- ToolUseBlock(- type="tool_use",+ ToolCallBlock(id=tool_call.id,name=tool_call.function.name,- input=_json_loads_with_repair(- tool_call.function.arguments,- ),+ input=tool_call.function.arguments,),)- if structured_model:- metadata = choice.message.parsed.model_dump()-usage = Noneif response.usage:usage = ChatUsage(@@ -626,8 +487,8 @@ class OpenAIChatModel(ChatModelBase):resp_kwargs: dict[str, Any] = {"content": content_blocks,+ "is_last": True,"usage": usage,- "metadata": metadata,}response_id = getattr(response, "id", None)if response_id:@@ -644,30 +505,30 @@ class OpenAIChatModel(ChatModelBase):def _format_tool_choice(self,- tool_choice: Literal["auto", "none", "required"] | str | None,+ tool_choice: ToolChoice | None,+ tools: list[dict] | None,) -> str | dict | None:"""Format tool_choice parameter for API compatibility.Args:- tool_choice (`Literal["auto", "none", "required"] | str \- | None`, default `None`):- Controls which (if any) tool is called by the model.- Can be "auto", "none", "required", or specific tool name.- For more details, please refer to- https://platform.openai.com/docs/api-reference/responses/create#responses_create-tool_choice+ tool_choice (`ToolChoice | None`):+ The unified tool choice parameter which can be a mode ("auto",+ "none", "required") or a specific function name.+ tools (`list[dict] | None`):+ The list of available tools, used for validation if+ tool_choice is a specific function name.+Returns:`dict | None`:The formatted tool choice configuration dict, or None iftool_choice is None."""+ super()._validate_tool_choice(tool_choice, tools)+if tool_choice is None:return None- mode_mapping = {- "auto": "auto",- "none": "none",- "required": "required",- }- if tool_choice in mode_mapping:- return mode_mapping[tool_choice]+ if tool_choice in ["auto", "none", "required"]:+ return tool_choice+return {"type": "function", "function": {"name": tool_choice}}diff --git a/src/agentscope/tool/_coding/__init__.py b/src/agentscope/tool/_coding/__init__.pydeleted file mode 100644index acad79d70..000000000--- a/src/agentscope/tool/_coding/__init__.py+++ /dev/null@@ -1,10 +0,0 @@-# -*- coding: utf-8 -*--"""The coding-related tools module in agentscope."""--from ._python import execute_python_code-from ._shell import execute_shell_command--__all__ = [- "execute_python_code",- "execute_shell_command",-]diff --git a/src/agentscope/tool/_coding/_python.py b/src/agentscope/tool/_coding/_python.pydeleted file mode 100644index d9f58976c..000000000--- a/src/agentscope/tool/_coding/_python.py+++ /dev/null@@ -1,90 +0,0 @@-# -*- coding: utf-8 -*--# pylint: disable=unused-argument-"""The Python code execution tool in agentscope."""--import asyncio-import os-import sys-import tempfile-from typing import Any--import shortuuid--from ...message import TextBlock-from .._response import ToolResponse---async def execute_python_code(- code: str,- timeout: float = 300,- **kwargs: Any,-) -> ToolResponse:- """Execute the given python code in a temp file and capture the return- code, standard output and error. Note you must `print` the output to get- the result, and the tmp file will be removed right after the execution.-- Args:- code (`str`):- The Python code to be executed.- timeout (`float`, defaults to `300`):- The maximum time (in seconds) allowed for the code to run.-- Returns:- `ToolResponse`:- The response containing the return code, standard output, and- standard error of the executed code.- """-- with tempfile.TemporaryDirectory() as temp_dir:- temp_file = os.path.join(temp_dir, f"tmp_{shortuuid.uuid()}.py")- with open(temp_file, "w", encoding="utf-8") as f:- f.write(code)-- env = os.environ.copy()- env["PYTHONUTF8"] = "1"- env["PYTHONIOENCODING"] = "utf-8"- proc = await asyncio.create_subprocess_exec(- sys.executable,- "-u",- temp_file,- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- env=env,- )-- try:- await asyncio.wait_for(proc.wait(), timeout=timeout)- stdout, stderr = await proc.communicate()- stdout_str = stdout.decode("utf-8")- stderr_str = stderr.decode("utf-8")- returncode = proc.returncode-- except asyncio.TimeoutError:- stderr_suffix = (- f"TimeoutError: The code execution exceeded "- f"the timeout of {timeout} seconds."- )- returncode = -1- try:- proc.terminate()- stdout, stderr = await proc.communicate()- stdout_str = stdout.decode("utf-8")- stderr_str = stderr.decode("utf-8")- if stderr_str:- stderr_str += f"\n{stderr_suffix}"- else:- stderr_str = stderr_suffix- except ProcessLookupError:- stdout_str = ""- stderr_str = stderr_suffix-- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"<returncode>{returncode}</returncode>"- f"<stdout>{stdout_str}</stdout>"- f"<stderr>{stderr_str}</stderr>",- ),- ],- )diff --git a/src/agentscope/tool/_coding/_shell.py b/src/agentscope/tool/_coding/_shell.pydeleted file mode 100644index 653eeca14..000000000--- a/src/agentscope/tool/_coding/_shell.py+++ /dev/null@@ -1,77 +0,0 @@-# -*- coding: utf-8 -*--# pylint: disable=unused-argument-"""The shell command tool in agentscope."""--import asyncio-from typing import Any--from .._response import ToolResponse-from ...message import TextBlock---async def execute_shell_command(- command: str,- timeout: int = 300,- **kwargs: Any,-) -> ToolResponse:- """Execute given command and return the return code, standard output and- error within <returncode></returncode>, <stdout></stdout> and- <stderr></stderr> tags.-- Args:- command (`str`):- The shell command to execute.- timeout (`float`, defaults to `300`):- The maximum time (in seconds) allowed for the command to run.-- Returns:- `ToolResponse`:- The tool response containing the return code, standard output, and- standard error of the executed command.- """-- proc = await asyncio.create_subprocess_shell(- command,- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- bufsize=0,- )-- try:- await asyncio.wait_for(proc.wait(), timeout=timeout)- stdout, stderr = await proc.communicate()- stdout_str = stdout.decode("utf-8")- stderr_str = stderr.decode("utf-8")- returncode = proc.returncode-- except asyncio.TimeoutError:- stderr_suffix = (- f"TimeoutError: The command execution exceeded "- f"the timeout of {timeout} seconds."- )- returncode = -1- try:- proc.terminate()- stdout, stderr = await proc.communicate()- stdout_str = stdout.decode("utf-8")- stderr_str = stderr.decode("utf-8")- if stderr_str:- stderr_str += f"\n{stderr_suffix}"- else:- stderr_str = stderr_suffix- except ProcessLookupError:- stdout_str = ""- stderr_str = stderr_suffix-- return ToolResponse(- content=[- TextBlock(- type="text",- text=(- f"<returncode>{returncode}</returncode>"- f"<stdout>{stdout_str}</stdout>"- f"<stderr>{stderr_str}</stderr>"- ),- ),- ],- )diff --git a/src/agentscope/tool/_multi_modality/__init__.py b/src/agentscope/tool/_multi_modality/__init__.pydeleted file mode 100644index 766c169b9..000000000--- a/src/agentscope/tool/_multi_modality/__init__.py+++ /dev/null@@ -1,27 +0,0 @@-# -*- coding: utf-8 -*--"""The multi-modal-related tools module in agentscope."""-from ._dashscope_tools import (- dashscope_image_to_text,- dashscope_text_to_audio,- dashscope_text_to_image,-)-from ._openai_tools import (- openai_text_to_image,- openai_edit_image,- openai_text_to_audio,- openai_create_image_variation,- openai_image_to_text,- openai_audio_to_text,-)--__all__ = [- "dashscope_image_to_text",- "dashscope_text_to_audio",- "dashscope_text_to_image",- "openai_text_to_image",- "openai_text_to_audio",- "openai_edit_image",- "openai_create_image_variation",- "openai_image_to_text",- "openai_audio_to_text",-]diff --git a/src/agentscope/tool/_multi_modality/_dashscope_tools.py b/src/agentscope/tool/_multi_modality/_dashscope_tools.pydeleted file mode 100644index cc8861599..000000000--- a/src/agentscope/tool/_multi_modality/_dashscope_tools.py+++ /dev/null@@ -1,302 +0,0 @@-# -*- coding: utf-8 -*--"""Use DashScope API to generate images,-convert text to audio, and convert images to text.-Please refer to the `official documentation <https://dashscope.aliyun.com/>`_- for more details.-"""-import base64-from typing import Literal, Sequence--import os---from ..._utils._common import _get_bytes_from_web_url-from ...message import ImageBlock, TextBlock, AudioBlock-from ...tool import ToolResponse---def dashscope_text_to_image(- prompt: str,- api_key: str,- n: int = 1,- size: Literal["1024*1024", "720*1280", "1280*720"] = "1024*1024",- model: str = "wanx-v1",- use_base64: bool = False,-) -> ToolResponse:- """Generate image(s) based on the given prompt, and return image url(s)- or base64 data.-- Args:- prompt (`str`):- The text prompt to generate image.- api_key (`str`):- The api key for the dashscope api.- n (`int`, defaults to `1`):- The number of images to generate.- size (`Literal["1024*1024", "720*1280", "1280*720"]`, defaults to \- `"1024*1024"`):- Size of the image.- model (`str`, defaults to '"wanx-v1"'):- The model to use, such as "wanx-v1", "qwen-image",- "wan2.2-t2i-flash", etc.- use_base64 (`bool`, defaults to 'False'):- Whether to use base64 data for images.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.- """- try:- import dashscope-- response = dashscope.ImageSynthesis.call(- model=model,- prompt=prompt,- api_key=api_key,- n=n,- size=size,- )- images = response.output["results"]- urls = [_["url"] for _ in images]-- image_blocks: list = []-- if urls is not None:- for url in urls:- if use_base64:- extension = url.split(".")[-1].lower()-- image_data = _get_bytes_from_web_url(url)- image_blocks.append(- ImageBlock(- type="image",- source={- "type": "base64",- "media_type": f"image/{extension}",- "data": image_data,- },- ),- )- else:- image_blocks.append(- ImageBlock(- type="image",- source={- "type": "url",- "url": url,- },- ),- )-- return ToolResponse(- content=image_blocks,- )-- else:- return ToolResponse(- [- TextBlock(- type="text",- text="Error: Failed to generate images",- ),- ],- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Failed to generate images: {str(e)}",- ),- ],- )---def dashscope_image_to_text(- image_urls: str | Sequence[str],- api_key: str,- prompt: str = "Describe the image",- model: str = "qwen-vl-plus",-) -> ToolResponse:- """Generate text based on the given images.-- Args:- image_urls (`str | Sequence[str]`):- The url of single or multiple images.- api_key (`str`):- The api key for the dashscope api.- prompt (`str`, defaults to 'Describe the image' ):- The text prompt.- model (`str`, defaults to 'qwen-vl-plus'):- The model to use in DashScope MultiModal API.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.- """-- if isinstance(image_urls, str):- image_urls = [image_urls]-- # Check if the local url is valid- img_abs_urls = []- for url in image_urls:- if os.path.exists(url):- if os.path.isfile(url):- img_abs_urls.append(os.path.abspath(url))- else:- return ToolResponse(- [- TextBlock(- type="text",- text=f'Error: The input image url "{url}" is '- f"not a file.",- ),- ],- )- else:- # Maybe a web url or an invalid url, we leave it to the API- # to handle- img_abs_urls.append(url)-- # Convert image paths according to the model requirements- contents = []- for url in img_abs_urls:- contents.append(- {- "image": url,- },- )-- contents.append({"text": prompt})-- # currently only support one round of conversation- # if multiple rounds of conversation are needed,- # it would be better to implement an Agent class- sys_message = {- "role": "system",- "content": [{"text": "You are a helpful assistant."}],- }- user_message = {- "role": "user",- "content": contents,- }- messages = [sys_message, user_message]- try:- import dashscope-- response = dashscope.MultiModalConversation.call(- model=model,- messages=messages,- api_key=api_key,- )- content = response.output["choices"][0]["message"]["content"]- if isinstance(content, list):- content = content[0]["text"]- if content is not None:- return ToolResponse(- [- TextBlock(- type="text",- text=content,- ),- ],- )- else:- return ToolResponse(- [- TextBlock(- type="text",- text="Error: Failed to generate text",- ),- ],- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Failed to generate text: {str(e)}",- ),- ],- )---def dashscope_text_to_audio(- text: str,- api_key: str,- model: str = "sambert-zhichu-v1",- sample_rate: int = 48000,-) -> ToolResponse:- """Convert the given text to audio.-- Args:- text (`str`):- The text to be converted into audio.- api_key (`str`):- The api key for the dashscope API.- model (`str`, defaults to 'sambert-zhichu-v1'):- The model to use. Full model list can be found in the- `official document- <https://help.aliyun.com/zh/model-studio/sambert-python-sdk>`_.- sample_rate (`int`, defaults to 48000):- Sample rate of the audio.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.-- """- try:- import dashscope-- dashscope.api_key = api_key-- res = dashscope.audio.tts.SpeechSynthesizer.call(- model=model,- text=text,- sample_rate=sample_rate,- format="wav",- )-- audio_data = res.get_audio_data()-- if audio_data is not None:- audio_base64 = base64.b64encode(audio_data).decode("utf-8")-- return ToolResponse(- [- AudioBlock(- type="audio",- source={- "type": "base64",- "media_type": "audio/wav",- "data": audio_base64,- },- ),- ],- )- else:- return ToolResponse(- [- TextBlock(- type="text",- text="Error: Failed to generate audio",- ),- ],- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Failed to generate audio: {str(e)}",- ),- ],- )diff --git a/src/agentscope/tool/_multi_modality/_openai_tools.py b/src/agentscope/tool/_multi_modality/_openai_tools.pydeleted file mode 100644index 2c77c4238..000000000--- a/src/agentscope/tool/_multi_modality/_openai_tools.py+++ /dev/null@@ -1,672 +0,0 @@-# -*- coding: utf-8 -*--"""-Wrap OpenAI API calls as tools. Refer the official-`OpenAI API documentation <https://platform.openai.com/docs/overview>`_ for-more details.-"""-import base64-from io import BytesIO-import os-from typing import Literal, IO-import requests--from .. import ToolResponse-from ...formatter._openai_formatter import _to_openai_image_url-from ...message import (- ImageBlock,- TextBlock,- Base64Source,- URLSource,- AudioBlock,-)---def _parse_url(url: str) -> BytesIO | IO[bytes]:- """- If url is a local file path, return a BytesIO of the file content.- If url is a web URL, fetch the content and return as BytesIO.- """- if url.startswith(("http://", "https://")):- response = requests.get(url)- response.raise_for_status() # Raise an exception for HTTP errors- return BytesIO(response.content)- else:- if not os.path.exists(url):- raise FileNotFoundError(f"File not found: {url}")- return open(os.path.abspath(url), "rb")---def openai_text_to_image(- prompt: str,- api_key: str,- n: int = 1,- model: Literal["dall-e-2", "dall-e-3", "gpt-image-1"] = "dall-e-2",- size: Literal[- "256x256",- "512x512",- "1024x1024",- "1792x1024",- "1024x1792",- ] = "256x256",- quality: Literal[- "auto",- "standard",- "hd",- "high",- "medium",- "low",- ] = "auto",- style: Literal["vivid", "natural"] = "vivid",- response_format: Literal["url", "b64_json"] = "url",-) -> ToolResponse:- """- Generate image(s) based on the given prompt, and return image URL(s) or- base64 data.-- Args:- prompt (`str`):- The text prompt to generate images.- api_key (`str`):- The API key for the OpenAI API.- n (`int`, defaults to `1`):- The number of images to generate.- model (`Literal["dall-e-2", "dall-e-3"]`, defaults to `"dall-e-2"`):- The model to use for image generation.- size (`Literal["256x256", "512x512", "1024x1024", "1792x1024", \- "1024x1792"]`, defaults to `"256x256"`):- The size of the generated images.- Must be one of 1024x1024, 1536x1024 (landscape), 1024x1536 (- portrait), or auto (default value) for gpt-image-1,- one of 256x256, 512x512, or 1024x1024 for dall-e-2,- and one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3.- quality (`Literal["auto", "standard", "hd", "high", "medium", \- "low"]`, defaults to `"auto"`):- The quality of the image that will be generated.-- - `auto` (default value) will automatically select the best- quality for the given model.- - `high`, `medium` and `low` are supported for gpt-image-1.- - `hd` and `standard` are supported for dall-e-3.- - `standard` is the only option for dall-e-2.- style (`Literal["vivid", "natural"]`, defaults to `"vivid"`):- The style of the generated images.- This parameter is only supported for dall-e-3.- Must be one of `vivid` or `natural`.-- - `Vivid` causes the model to lean towards generating hyper-real- and dramatic images.- - `Natural` causes the model to produce more natural,- less hyper-real looking images.- response_format (`Literal["url", "b64_json"]`, defaults to `"url"`):- The format in which generated images with dall-e-2 and dall-e-3- are returned.-- - Must be one of "url" or "b64_json".- - URLs are only valid for 60 minutes after the image has been- generated.- - This parameter isn't supported for gpt-image-1 which will always- return base64-encoded images.- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.- """-- kwargs = {- "model": model,- "prompt": prompt,- "n": n,- "size": size,- }- if model == "dall-e-3":- kwargs["style"] = style- if model != "dall-e-2":- kwargs["quality"] = quality- if model != "gpt-image-1":- kwargs["response_format"] = response_format- if model == "gpt-image-1":- response_format = "b64_json"-- try:- import openai-- client = openai.OpenAI(- api_key=api_key,- )- response = client.images.generate(- **kwargs,- )- image_blocks: list = []- if response_format == "url":- image_urls = [_.url for _ in response.data]- for image_url in image_urls:- image_blocks.append(- ImageBlock(- type="image",- source=URLSource(- type="url",- url=image_url,- ),- ),- )- else:- image_datas = [_.b64_json for _ in response.data]- for image_data in image_datas:- image_blocks.append(- ImageBlock(- type="image",- source=Base64Source(- type="base64",- media_type="image/png",- data=image_data,- ),- ),- )- return ToolResponse(- content=image_blocks,- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Failed to generate image: {str(e)}",- ),- ],- )---def openai_edit_image(- image_url: str,- prompt: str,- api_key: str,- model: Literal["dall-e-2", "gpt-image-1"] = "dall-e-2",- mask_url: str | None = None,- n: int = 1,- size: Literal[- "256x256",- "512x512",- "1024x1024",- ] = "256x256",- response_format: Literal["url", "b64_json"] = "url",-) -> ToolResponse:- """- Edit an image based on the provided mask and prompt, and return the edited- image URL(s) or base64 data.-- Args:- image_url (`str`):- The file path or URL to the image that needs editing.- prompt (`str`):- The text prompt describing the edits to be made to the image.- api_key (`str`):- The API key for the OpenAI API.- model (`Literal["dall-e-2", "gpt-image-1"]`, defaults to `"dall-e-2"`):- The model to use for image generation.- mask_url (`str | None`, defaults to `None`):- The file path or URL to the mask image that specifies the regions- to be edited.- n (`int`, defaults to `1`):- The number of edited images to generate.- size (`Literal["256x256", "512x512", "1024x1024"]`, defaults to \- `"256x256"`):- The size of the edited images.- response_format (`Literal["url", "b64_json"]`, defaults to `"url"`):- The format in which generated images are returned.-- - Must be one of "url" or "b64_json".- - URLs are only valid for 60 minutes after generation.- - This parameter isn't supported for gpt-image-1 which will- always return base64-encoded images.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.-- """-- try:- import openai-- client = openai.OpenAI(- api_key=api_key,- )-- def prepare_image(url_or_path: str) -> BytesIO:- from PIL import Image-- if url_or_path.startswith(("http://", "https://")):- response = requests.get(url_or_path)- response.raise_for_status()- img = Image.open(BytesIO(response.content))- else:- img = Image.open(url_or_path)-- if img.mode != "RGBA":- img = img.convert("RGBA")- img_buffer = BytesIO()- img.save(img_buffer, format="PNG")- img_buffer.seek(0)- img_buffer.name = "image.png"- return img_buffer-- image_file = prepare_image(image_url)-- kwargs = {- "model": model,- "image": image_file,- "prompt": prompt,- "n": n,- "size": size,- }-- if mask_url:- kwargs["mask"] = prepare_image(mask_url)-- if model == "dall-e-2":- kwargs["response_format"] = response_format- else:- response_format = "b64_json"-- response = client.images.edit(**kwargs)-- if response_format == "url":- urls = [_.url for _ in response.data]- image_blocks: list = []- for url in urls:- image_blocks.append(- ImageBlock(- type="image",- source=URLSource(- type="url",- url=url,- ),- ),- )- return ToolResponse(- content=image_blocks,- )- elif response_format == "b64_json":- image_datas = [_.b64_json for _ in response.data]- image_blocks = []- for image_data in image_datas:- image_blocks.append(- ImageBlock(- type="image",- source=Base64Source(- type="base64",- media_type="image/png",- data=image_data,- ),- ),- )- return ToolResponse(- content=image_blocks,- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Failed to generate image: {str(e)}",- ),- ],- )---def openai_create_image_variation(- image_url: str,- api_key: str,- n: int = 1,- model: Literal["dall-e-2"] = "dall-e-2",- size: Literal[- "256x256",- "512x512",- "1024x1024",- ] = "256x256",- response_format: Literal["url", "b64_json"] = "url",-) -> ToolResponse:- """- Create variations of an image and return the image URL(s) or base64 data.-- Args:- image_url (`str`):- The file path or URL to the image from which variations will be- generated.- api_key (`str`):- The API key for the OpenAI API.- n (`int`, defaults to `1`):- The number of image variations to generate.- model (` Literal["dall-e-2"]`, default to `dall-e-2`):- The model to use for image variation.- size (`Literal["256x256", "512x512", "1024x1024"]`, defaults to \- `"256x256"`):- The size of the generated image variations.- response_format (`Literal["url", "b64_json"]`, defaults to `"url"`):- The format in which generated images are returned.-- - Must be one of url or b64_json.- - URLs are only valid for 60 minutes after the image has been- generated.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.- """-- # _parse_url handles both local and web URLs and returns BytesIO- image = _parse_url(image_url)- try:- import openai-- client = openai.OpenAI(- api_key=api_key,- )- response = client.images.create_variation(- model=model,- image=image,- n=n,- size=size,- )- image_blocks: list = []- if response_format == "url":- urls = [_.url for _ in response.data]- for url in urls:- image_blocks.append(- ImageBlock(- type="image",- source=URLSource(- type="url",- url=url,- ),- ),- )- else:- image_datas = [_.b64_json for _ in response.data]- for image_data in image_datas:- image_blocks.append(- ImageBlock(- type="image",- source=Base64Source(- type="base64",- media_type="image/png",- data=image_data,- ),- ),- )- return ToolResponse(- content=image_blocks,- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Failed to generate image: {str(e)}",- ),- ],- )---def openai_image_to_text(- image_urls: str | list[str],- api_key: str,- prompt: str = "Describe the image",- model: str = "gpt-4o",-) -> ToolResponse:- """- Generate descriptive text for given image(s) using a specified model, and- return the generated text.-- Args:- image_urls (`str | list[str]`):- The URL or list of URLs pointing to the images that need to be- described.- api_key (`str`):- The API key for the OpenAI API.- prompt (`str`, defaults to `"Describe the image"`):- The prompt that instructs the model on how to describe- the image(s).- model (`str`, defaults to `"gpt-4o"`):- The model to use for generating the text descriptions.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.-- """- if isinstance(image_urls, str):- image_urls = [image_urls]-- content = []- for url in image_urls:- content.append(- {- "type": "image_url",- "image_url": {- "url": _to_openai_image_url(url),- },- },- )- content.append(- {- "type": "text",- "text": prompt,- },- )- messages = [- {- "role": "user",- "content": content,- },- ]-- try:- import openai-- client = openai.OpenAI(- api_key=api_key,- )- response = client.chat.completions.create(- messages=messages,- model=model,- )- return ToolResponse(- [- TextBlock(- type="text",- text=response.choices[0].message.content,- ),- ],- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Failed to generate text: {str(e)}",- ),- ],- )---def openai_text_to_audio(- text: str,- api_key: str,- model: Literal["tts-1", "tts-1-hd", "gpt-4o-mini-tts"] = "tts-1",- voice: Literal[- "alloy",- "ash",- "ballad",- "coral",- "echo",- "fable",- "nova",- "onyx",- "sage",- "shimmer",- ] = "alloy",- speed: float = 1.0,- res_format: Literal[- "mp3",- "opus",- "aac",- "flac",- "wav",- "pcm",- ] = "mp3",-) -> ToolResponse:- """- Convert text to an audio file using a specified model and voice.-- Args:- text (`str`):- The text to convert to audio.- api_key (`str`):- The API key for the OpenAI API.- model (`Literal["tts-1", "tts-1-hd"]`, defaults to `"tts-1"`):- The model to use for text-to-speech conversion.- voice (`Literal["alloy", "echo", "fable", "onyx", "nova", \- "shimmer"]`, defaults to `"alloy"`):- The voice to use for the audio output.- speed (`float`, defaults to `1.0`):- The speed of the audio playback. A value of 1.0 is normal speed.- res_format (`Literal["mp3", "wav", "opus", "aac", "flac", \- "wav", "pcm"]`, defaults to `"mp3"`):- The format of the audio file.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.- """-- try:- import openai-- client = openai.OpenAI(- api_key=api_key,- )- response = client.audio.speech.create(- model=model,- voice=voice,- speed=speed,- input=text,- response_format=res_format,- )- audio_bytes = response.content- audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")-- return ToolResponse(- [- AudioBlock(- type="audio",- source=Base64Source(- type="base64",- media_type=f"audio/{res_format}",- data=audio_base64,- ),- ),- ],- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Error: Failed to generate audio. {str(e)}",- ),- ],- )---def openai_audio_to_text(- audio_file_url: str,- api_key: str,- language: str = "en",- temperature: float = 0.2,-) -> ToolResponse:- """- Convert an audio file to text using OpenAI's transcription service.-- Args:- audio_file_url (`str`):- The file path or URL to the audio file that needs to be- transcribed.- api_key (`str`):- The API key for the OpenAI API.- language (`str`, defaults to `"en"`):- The language of the input audio in- `ISO-639-1 format \- <https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes>`_- (e.g., "en", "zh", "fr"). Improves accuracy and latency.- temperature (`float`, defaults to `0.2`):- The temperature for the transcription, which affects the- randomness of the output.-- Returns:- `ToolResponse`:- A ToolResponse containing the generated content- (ImageBlock/TextBlock/AudioBlock) or error information if the- operation failed.- """-- try:- import openai-- client = openai.OpenAI(- api_key=api_key,- )-- if audio_file_url.startswith(("http://", "https://")):- response = requests.get(audio_file_url)- response.raise_for_status()- audio_buffer = BytesIO(response.content)- import urllib.parse- from pathlib import Path-- parsed_url = urllib.parse.urlparse(audio_file_url)- filename = Path(parsed_url.path).name or "audio.mp3"- audio_buffer.name = filename- audio_file = audio_buffer- transcription = client.audio.transcriptions.create(- model="whisper-1",- file=audio_file,- language=language,- temperature=temperature,- )- else:- if not os.path.exists(audio_file_url):- raise FileNotFoundError(f"File not found: {audio_file_url}")- with open(audio_file_url, "rb") as audio_file:- transcription = client.audio.transcriptions.create(- model="whisper-1",- file=audio_file,- language=language,- temperature=temperature,- )- return ToolResponse(- [- TextBlock(- type="text",- text=transcription.text,- ),- ],- )- except Exception as e:- return ToolResponse(- [- TextBlock(- type="text",- text=f"Error: Failed to transcribe audio: {str(e)}",- ),- ],- )diff --git a/src/agentscope/tool/_response.py b/src/agentscope/tool/_response.pyindex bb939199b..77ab38188 100644--- a/src/agentscope/tool/_response.py+++ b/src/agentscope/tool/_response.py@@ -1,18 +1,17 @@# -*- coding: utf-8 -*-"""The tool response class."""-+import uuidfrom dataclasses import dataclass, fieldfrom typing import Optional, List-from .._utils._common import _get_timestamp-from ..message import AudioBlock, ImageBlock, TextBlock, VideoBlock+from ..message import DataBlock, TextBlock@dataclassclass ToolResponse:"""The result chunk of a tool call."""- content: List[TextBlock | ImageBlock | AudioBlock | VideoBlock]+ content: List[TextBlock | DataBlock]"""The execution output of the tool function."""metadata: Optional[dict] = None@@ -28,5 +27,5 @@ class ToolResponse:is_interrupted: bool = False"""Whether the tool execution is interrupted."""- id: str = field(default_factory=lambda: _get_timestamp(True))+ id: str = field(default_factory=lambda: uuid.uuid4().hex)"""The identity of the tool response."""diff --git a/src/agentscope/tool/_text_file/__init__.py b/src/agentscope/tool/_text_file/__init__.pydeleted file mode 100644index cd404b503..000000000--- a/src/agentscope/tool/_text_file/__init__.py+++ /dev/null@@ -1,13 +0,0 @@-# -*- coding: utf-8 -*--"""The text file tool module in agentscope."""-from ._view_text_file import view_text_file-from ._write_text_file import (- insert_text_file,- write_text_file,-)--__all__ = [- "insert_text_file",- "write_text_file",- "view_text_file",-]diff --git a/src/agentscope/tool/_text_file/_utils.py b/src/agentscope/tool/_text_file/_utils.pydeleted file mode 100644index 4f0877f6e..000000000--- a/src/agentscope/tool/_text_file/_utils.py+++ /dev/null@@ -1,88 +0,0 @@-# -*- coding: utf-8 -*--"""The utility functions for text file tools in agentscope."""-from ...exception import ToolInvalidArgumentsError---def _calculate_view_ranges(- old_n_lines: int,- new_n_lines: int,- start: int,- end: int,- extra_view_n_lines: int = 5,-) -> tuple[int, int]:- """Calculate after writing the new content, the view ranges of the file.-- Args:- old_n_lines (`int`):- The number of lines before writing the new content.- new_n_lines (`int`):- The number of lines after writing the new content.- start (`int`):- The start line of the writing range.- end (`int`):- The end line of the writing range.- extra_view_n_lines (`int`, optional):- The number of extra lines to view before and after the range.- """-- view_start = max(1, start - extra_view_n_lines)-- delta_lines = new_n_lines - old_n_lines- view_end = min(end + delta_lines + extra_view_n_lines, new_n_lines)-- return view_start, view_end---def _assert_ranges(- ranges: list[int],-) -> None:- """Check if the ranges are valid.-- Raises:- ToolInvalidArgumentsError: If the ranges are invalid.- """- if (- isinstance(ranges, list)- and len(ranges) == 2- and all(isinstance(i, int) for i in ranges)- ):- start, end = ranges- if start > end:- raise ToolInvalidArgumentsError(- f"InvalidArgumentError: The start line is greater than the "- f"end line in the given range {ranges}.",- )- else:- raise ToolInvalidArgumentsError(- f"InvalidArgumentError: Invalid range format. Expected a list of "- f"two integers, but got {ranges}.",- )---def _view_text_file(- file_path: str,- ranges: list[int] | None = None,-) -> str:- """Return the file content in the specified range with line numbers."""- with open(file_path, "r", encoding="utf-8") as file:- lines = file.readlines()-- if ranges:- _assert_ranges(ranges)- start, end = ranges-- if start > len(lines):- raise ToolInvalidArgumentsError(- f"InvalidArgumentError: The range '{ranges}' is out of bounds "- f"for the file '{file_path}', which has only {len(lines)} "- f"lines.",- )-- view_content = [- f"{index + start}: {line}"- for index, line in enumerate(lines[start - 1 : end])- ]-- return "".join(view_content)-- return "".join(f"{index + 1}: {line}" for index, line in enumerate(lines))diff --git a/src/agentscope/tool/_text_file/_view_text_file.py b/src/agentscope/tool/_text_file/_view_text_file.pydeleted file mode 100644index 5955a0a76..000000000--- a/src/agentscope/tool/_text_file/_view_text_file.py+++ /dev/null@@ -1,82 +0,0 @@-# -*- coding: utf-8 -*--# flake8: noqa: E501-# pylint: disable=line-too-long-"""The view text file tool in agentscope."""-import os--from ._write_text_file import _view_text_file-from .._response import ToolResponse-from ...exception import ToolInvalidArgumentsError-from ...message import TextBlock---async def view_text_file(- file_path: str,- ranges: list[int] | None = None,-) -> ToolResponse:- """View the file content in the specified range with line numbers. If `ranges` is not provided, the entire file will be returned.-- Args:- file_path (`str`):- The target file path.- ranges:- The range of lines to be viewed (e.g. lines 1 to 100: [1, 100]), inclusive. If not provided, the entire file will be returned. To view the last 100 lines, use [-100, -1].-- Returns:- `ToolResponse`:- The tool response containing the file content or an error message.- """- file_path = os.path.expanduser(file_path)- if not os.path.exists(file_path):- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Error: The file {file_path} does not exist.",- ),- ],- )- if not os.path.isfile(file_path):- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Error: The path {file_path} is not a file.",- ),- ],- )-- try:- content = _view_text_file(file_path, ranges)- except ToolInvalidArgumentsError as e:- return ToolResponse(- content=[- TextBlock(- type="text",- text=e.message,- ),- ],- )-- if ranges is None:- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"""The content of {file_path}:-```-{content}```""",- ),- ],- )- else:- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"""The content of {file_path} in {ranges} lines:-```-{content}```""",- ),- ],- )diff --git a/src/agentscope/tool/_text_file/_write_text_file.py b/src/agentscope/tool/_text_file/_write_text_file.pydeleted file mode 100644index e5e06ad9a..000000000--- a/src/agentscope/tool/_text_file/_write_text_file.py+++ /dev/null@@ -1,241 +0,0 @@-# -*- coding: utf-8 -*--# flake8: noqa: E501-# pylint: disable=line-too-long-"""The text file tools in agentscope."""-import os--from ._utils import _calculate_view_ranges, _view_text_file-from .._response import ToolResponse-from ...message import TextBlock---async def insert_text_file(- file_path: str,- content: str,- line_number: int,-) -> ToolResponse:- """Insert the content at the specified line number in a text file.-- Args:- file_path (`str`):- The target file path.- content (`str`):- The content to be inserted.- line_number (`int`):- The line number at which the content should be inserted, starting- from 1. If exceeds the number of lines in the file, it will be- appended to the end of the file.-- Returns:- `ToolResponse`:- The tool response containing the result of the insertion operation.- """- if line_number <= 0:- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"InvalidArgumentsError: "- f"The line number {line_number} is invalid. ",- ),- ],- )-- file_path = os.path.expanduser(file_path)- if not os.path.exists(file_path):- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"InvalidArgumentsError: The target file "- f"{file_path} does not exist. ",- ),- ],- )-- with open(file_path, "r", encoding="utf-8") as file:- original_lines = file.readlines()-- if line_number == len(original_lines) + 1:- new_lines = original_lines + ["\n" + content]- elif line_number < len(original_lines) + 1:- new_lines = (- original_lines[: line_number - 1]- + [content + "\n"]- + original_lines[line_number - 1 :]- )- else:- return ToolResponse(- content=[- TextBlock(- type="text",- text="InvalidArgumentsError: The given line_number "- f"({line_number}) is not in the valid range "- f"[1, {len(original_lines) + 1}].",- ),- ],- )-- with open(file_path, "w", encoding="utf-8") as file:- file.writelines(new_lines)-- with open(file_path, "r", encoding="utf-8") as file:- new_lines = file.readlines()-- start, end = _calculate_view_ranges(- len(original_lines),- len(new_lines),- line_number,- line_number,- extra_view_n_lines=5,- )-- show_content = _view_text_file(file_path, [start, end])-- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Insert content into {file_path} at line "- f"{line_number} successfully. The new content "- f"between lines {start}-{end} is:\n"- f"```\n{show_content}```",- ),- ],- )---async def write_text_file(- file_path: str,- content: str,- ranges: None | list[int] = None,-) -> ToolResponse:- """Create/Replace/Overwrite content in a text file. When `ranges` is provided, the content will be replaced in the specified range. Otherwise, the entire file (if exists) will be overwritten.-- Args:- file_path (`str`):- The target file path.- content (`str`):- The content to be written.- ranges (`list[int] | None`, defaults to `None`):- The range of lines to be replaced. If `None`, the entire file will- be overwritten.-- Returns:- `ToolResponse`:- The tool response containing the result of the writing operation.- """- file_path = os.path.expanduser(file_path)- if not os.path.exists(file_path):- with open(file_path, "w", encoding="utf-8") as file:- file.write(content)-- if ranges:- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Create and write {file_path} successfully. "- f"The ranges {ranges} is ignored because the "- f"file does not exist.",- ),- ],- )-- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Create and write {file_path} successfully.",- ),- ],- )-- with open(file_path, "r", encoding="utf-8") as file:- original_lines = file.readlines()-- if ranges is not None:- if (- isinstance(ranges, list)- and len(ranges) == 2- and all(isinstance(i, int) for i in ranges)- ):- # Replace content in the specified range- start, end = ranges- if start > len(original_lines):- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Error: The start line {start} is invalid. "- f"The file only has {len(original_lines)} "- f"lines.",- ),- ],- )-- new_content = (- original_lines[: start - 1]- + [- content,- ]- + original_lines[end:]- )-- with open(file_path, "w", encoding="utf-8") as file:- file.write("".join(new_content))-- # The written content may contain multiple "\n", to avoid mis- # counting the lines, we read the file again to get the new content- with open(file_path, "r", encoding="utf-8") as file:- new_lines = file.readlines()-- view_start, view_end = _calculate_view_ranges(- len(original_lines),- len(new_lines),- start,- end,- )-- content = "".join(- [- f"{index + view_start}: {line}"- for index, line in enumerate(- new_lines[view_start - 1 : view_end],- )- ],- )-- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"""Write {file_path} successfully. The new content snippet:-```-{content}```""",- ),- ],- )-- else:- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Error: Invalid range format. Expected a list "- f"of two integers, but got {ranges}.",- ),- ],- )-- with open(file_path, "w", encoding="utf-8") as file:- file.write(content)-- return ToolResponse(- content=[- TextBlock(- type="text",- text=f"Overwrite {file_path} successfully.",- ),- ],- )diff --git a/src/agentscope/tool/_toolkit.py b/src/agentscope/tool/_toolkit.pyindex d17257f65..87e2c6bdd 100644--- a/src/agentscope/tool/_toolkit.py+++ b/src/agentscope/tool/_toolkit.py@@ -42,10 +42,9 @@ from ..mcp import (StatefulClientBase,)from ..message import (- ToolUseBlock,+ ToolCallBlock,TextBlock,)-from ..module import StateModulefrom ..types import (JSONSerializableObject,ToolFunction,@@ -72,7 +71,7 @@ def _apply_middlewares(@wraps(func)async def wrapper(self: "Toolkit",- tool_call: ToolUseBlock,+ tool_call: ToolCallBlock,) -> AsyncGenerator[ToolResponse, None]:"""Wrapper that applies middleware chain."""middlewares = getattr(self, "_middlewares", [])@@ -114,7 +113,7 @@ def _apply_middlewares(return wrapper-class Toolkit(StateModule): # pylint: disable=too-many-public-methods+class Toolkit: # pylint: disable=too-many-public-methods"""Toolkit is the core module to register, manage and delete toolfunctions, MCP clients, Agent skills in AgentScope.@@ -284,11 +283,11 @@ Check "{dir}/SKILL.md" for how to use this skill"""include_var_keyword: bool = False,postprocess_func: (Callable[- [ToolUseBlock, ToolResponse],+ [ToolCallBlock, ToolResponse],ToolResponse | None,]| Callable[- [ToolUseBlock, ToolResponse],+ [ToolCallBlock, ToolResponse],Awaitable[ToolResponse | None],])@@ -338,8 +337,8 @@ Check "{dir}/SKILL.md" for how to use this skill"""include_var_keyword (`bool`, defaults to `False`):Whether to include the variable keyword arguments (`**kwargs`)in the function schema.- postprocess_func (`(Callable[[ToolUseBlock, ToolResponse], \- ToolResponse | None] | Callable[[ToolUseBlock, ToolResponse], \+ postprocess_func (`(Callable[[ToolCallBlock, ToolResponse], \+ ToolResponse | None] | Callable[[ToolCallBlock, ToolResponse], \Awaitable[ToolResponse | None]]) | None`, optional):A post-processing function that will be called after the toolfunction is executed, taking the tool call block and tool@@ -852,16 +851,16 @@ Check "{dir}/SKILL.md" for how to use this skill"""@_apply_middlewaresasync def call_tool_function(self,- tool_call: ToolUseBlock,+ tool_call: ToolCallBlock,) -> AsyncGenerator[ToolResponse, None]:- """Execute the tool function by the `ToolUseBlock` and return the+ """Execute the tool function by the `ToolCallBlock` and return thetool response chunk in unified streaming mode, i.e. an asyncgenerator of `ToolResponse` objects... note:: The tool response chunk is **accumulated**.Args:- tool_call (`ToolUseBlock`):+ tool_call (`ToolCallBlock`):A tool call block.Yields:@@ -917,7 +916,7 @@ Check "{dir}/SKILL.md" for how to use this skill"""# Prepare postprocess functionif tool_func.postprocess_func:# Type: partial wraps the postprocess_func with tool_call bound,- # reducing it from (ToolUseBlock, ToolResponse) to (ToolResponse)+ # reducing it from (ToolCallBlock, ToolResponse) to (ToolResponse)partial_postprocess_func: (Callable[[ToolResponse], ToolResponse | None]| Callable[[ToolResponse], Awaitable[ToolResponse | None]]@@ -1041,11 +1040,11 @@ Check "{dir}/SKILL.md" for how to use this skill"""preset_kwargs_mapping: dict[str, dict[str, Any]] | None = None,postprocess_func: (Callable[- [ToolUseBlock, ToolResponse],+ [ToolCallBlock, ToolResponse],ToolResponse | None,]| Callable[- [ToolUseBlock, ToolResponse],+ [ToolCallBlock, ToolResponse],Awaitable[ToolResponse | None],])@@ -1074,8 +1073,8 @@ Check "{dir}/SKILL.md" for how to use this skill"""defaults to `None`):The preset keyword arguments mapping, whose keys are the toolfunction names and values are the preset keyword arguments.- postprocess_func (`(Callable[[ToolUseBlock, ToolResponse], \- ToolResponse | None] | Callable[[ToolUseBlock, ToolResponse], \+ postprocess_func (`(Callable[[ToolCallBlock, ToolResponse], \+ ToolResponse | None] | Callable[[ToolCallBlock, ToolResponse], \Awaitable[ToolResponse | None]]) | None`, optional):A post-processing function that will be called after the toolfunction is executed, taking the tool call block and tool@@ -1459,7 +1458,7 @@ Check "{dir}/SKILL.md" for how to use this skill"""parameter and ``next_handler`` as the second parameter. The ``kwargs``dict currently contains:- - ``tool_call`` (`ToolUseBlock`): The tool call request+ - ``tool_call`` (`ToolCallBlock`): The tool call requestWhen calling ``next_handler``, pass ``**kwargs`` to unpack the dict.@@ -1521,7 +1520,7 @@ AsyncGenerator[ToolResponse, None]] | AsyncGenerator[ToolResponse, None]]`):The middleware function that accepts ``kwargs`` (dict) and``next_handler`` (Callable), and returns a coroutine thatyields AsyncGenerator of ToolResponse objects. The ``kwargs``- dict currently includes ``tool_call`` (ToolUseBlock), and may+ dict currently includes ``tool_call`` (ToolCallBlock), and mayinclude additional context in future versions... note:: The middleware chain is applied inside thediff --git a/src/agentscope/tool/_types.py b/src/agentscope/tool/_types.pyindex 3dbc8cfe2..386fea4c8 100644--- a/src/agentscope/tool/_types.py+++ b/src/agentscope/tool/_types.py@@ -8,7 +8,7 @@ from pydantic import BaseModelfrom . import ToolResponsefrom .._utils._common import _remove_title_field-from ..message import ToolUseBlock+from ..message import ToolCallBlockfrom ..types import ToolFunction, JSONSerializableObject@@ -40,11 +40,11 @@ class RegisteredToolFunction:"""The name of the MCP, if the tool function comes from an MCP server."""postprocess_func: (Callable[- [ToolUseBlock, ToolResponse],+ [ToolCallBlock, ToolResponse],ToolResponse | None,]| Callable[- [ToolUseBlock, ToolResponse],+ [ToolCallBlock, ToolResponse],Awaitable[ToolResponse | None],]) | None = Nonediff --git a/src/agentscope/tracing/_extractor.py b/src/agentscope/tracing/_extractor.pyindex 391073856..06179db09 100644--- a/src/agentscope/tracing/_extractor.py+++ b/src/agentscope/tracing/_extractor.py@@ -4,7 +4,7 @@ import inspectfrom typing import Any, Dict, Tuple, TYPE_CHECKINGfrom ..embedding import EmbeddingModelBase-from ..message import Msg, ToolUseBlock+from ..message import Msg, ToolCallBlockfrom ..model import ChatModelBasefrom ._attributes import (@@ -551,7 +551,7 @@ def _get_agent_response_attributes(def _get_tool_request_attributes(instance: "Toolkit",- tool_call: ToolUseBlock,+ tool_call: ToolCallBlock,) -> Dict[str, str]:"""Get tool request attributes for OpenTelemetry tracing.@@ -561,7 +561,7 @@ def _get_tool_request_attributes(instance (`Toolkit`):Toolkit instance with tool definitions. Used to extract tooldescription from the tool's JSON schema.- tool_call (`ToolUseBlock`):+ tool_call (`ToolCallBlock`):Tool use block with call information including id, name, and inputarguments.diff --git a/src/agentscope/tracing/_trace.py b/src/agentscope/tracing/_trace.pyindex c56bd9544..bc878f0eb 100644--- a/src/agentscope/tracing/_trace.py+++ b/src/agentscope/tracing/_trace.py@@ -17,7 +17,7 @@ import aioitertoolsfrom ..embedding import EmbeddingModelBase, EmbeddingResponsefrom .._logging import logger-from ..message import Msg, ToolUseBlock+from ..message import Msg, ToolCallBlockfrom ..model import ChatModelBase, ChatResponsefrom ._attributes import SpanAttributes, OperationNameValues@@ -326,7 +326,7 @@ def trace_toolkit(@wraps(func)async def wrapper(self: Toolkit,- tool_call: ToolUseBlock,+ tool_call: ToolCallBlock,) -> AsyncGenerator[ToolResponse, None]:"""The wrapper function for tracing the toolkit call_tool_functionmethod."""@@ -388,16 +388,16 @@ def trace_reply(if not _check_tracing_enabled():return await func(self, *args, **kwargs)- from ..agent import AgentBase-- if not isinstance(self, AgentBase):- logger.warning(- "Skipping tracing for %s as the first argument"- "is not an instance of AgentBase, but %s",- func.__name__,- type(self),- )- return await func(self, *args, **kwargs)+ # from ..agent import AgentBase+ #+ # if not isinstance(self, AgentBase):+ # logger.warning(+ # "Skipping tracing for %s as the first argument"+ # "is not an instance of AgentBase, but %s",+ # func.__name__,+ # type(self),+ # )+ # return await func(self, *args, **kwargs)tracer = _get_tracer()diff --git a/src/agentscope/types/__init__.py b/src/agentscope/types/__init__.pyindex a8e92a3f5..b8da5ac65 100644--- a/src/agentscope/types/__init__.py+++ b/src/agentscope/types/__init__.py@@ -10,7 +10,7 @@ from ._json import (JSONPrimitive,JSONSerializableObject,)-from ._tool import ToolFunction+from ._tool import ToolFunction, ToolChoice__all__ = ["AgentHookTypes",@@ -19,4 +19,5 @@ __all__ = ["JSONPrimitive","JSONSerializableObject","ToolFunction",+ "ToolChoice",]diff --git a/src/agentscope/types/_json.py b/src/agentscope/types/_json.pyindex 6ab7e230c..3a506a01c 100644--- a/src/agentscope/types/_json.py+++ b/src/agentscope/types/_json.py@@ -1,21 +1,14 @@# -*- coding: utf-8 -*-"""The JSON related types"""+from typing import TypeAlias-from typing import Union+JSONPrimitive: TypeAlias = str | int | float | bool | None-JSONPrimitive = Union[- str,- int,- float,- bool,- None,-]--JSONSerializableObject = Union[- JSONPrimitive,- list["JSONSerializableObject"],- dict[+JSONSerializableObject: TypeAlias = (+ JSONPrimitive+ | list["JSONSerializableObject"]+ | dict[str,"JSONSerializableObject",- ],-]+ ]+)diff --git a/src/agentscope/types/_tool.py b/src/agentscope/types/_tool.pyindex 588eb2ab9..f80ec483d 100644--- a/src/agentscope/types/_tool.py+++ b/src/agentscope/types/_tool.py@@ -10,6 +10,8 @@ from typing import (Coroutine,Any,TYPE_CHECKING,+ TypeAlias,+ Literal,)if TYPE_CHECKING:@@ -34,3 +36,6 @@ ToolFunction = Callable[Coroutine[Any, Any, Generator[ToolResponse, None, None]],],]+++ToolChoice: TypeAlias = Literal["auto", "none", "required"] | str