Skip to content

Build and run agents you can see, understand and trust.

refactor(msg): simplify core building blocks and Msg class structure (#1440)

DavdGaob158fc2e0ccd
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b26df494c..5f34d4102 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -99,6 +99,9 @@ repos:
--disable=W0632,
--disable=W0123,
--disable=C3001,
+ --max-branches=30,
+ --max-nested-blocks=7,
+ --max-statements=100,
]
- repo: https://github.com/regebro/pyroma
rev: "5.0"
diff --git a/src/agentscope/__init__.py b/src/agentscope/__init__.py
index d52e1385b..31cda97ce 100644
--- a/src/agentscope/__init__.py
+++ b/src/agentscope/__init__.py
@@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
"""The agentscope serialization module"""
-
import warnings
from . import exception
@@ -11,7 +10,6 @@ from . import formatter
from . import agent
from . import embedding
from . import tracing
-
from ._logging import (
logger,
setup_logger,
@@ -33,7 +31,6 @@ __all__ = [
"logger",
"embedding",
"tracing",
- "a2a",
# functions
"setup_logger",
"__version__",
diff --git a/src/agentscope/a2a/__init__.py b/src/agentscope/a2a/__init__.py
deleted file mode 100644
index 828f2621d..000000000
--- a/src/agentscope/a2a/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The A2A related modules."""
-from ._base import AgentCardResolverBase
-from ._file_resolver import FileAgentCardResolver
-from ._well_known_resolver import WellKnownAgentCardResolver
-from ._nacos_resolver import NacosAgentCardResolver
-
-
-__all__ = [
- "AgentCardResolverBase",
- "FileAgentCardResolver",
- "WellKnownAgentCardResolver",
- "NacosAgentCardResolver",
-]
diff --git a/src/agentscope/a2a/_base.py b/src/agentscope/a2a/_base.py
deleted file mode 100644
index 23ff1ea7e..000000000
--- a/src/agentscope/a2a/_base.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The A2A agent card resolver base class."""
-from abc import abstractmethod
-from typing import Any, TYPE_CHECKING
-
-if TYPE_CHECKING:
- from a2a.types import AgentCard
-else:
- AgentCard = "a2a.types.AgentCard"
-
-
-class AgentCardResolverBase:
- """Base class for A2A agent card resolvers, responsible for fetching
- agent cards from various sources. Implementations must provide the
- `get_agent_card` method to retrieve the agent card.
- """
-
- @abstractmethod
- async def get_agent_card(self, *args: Any, **kwargs: Any) -> AgentCard:
- """Get Agent Card from the configured source.
-
- Returns:
- `AgentCard`:
- The resolved agent card object.
- """
diff --git a/src/agentscope/a2a/_file_resolver.py b/src/agentscope/a2a/_file_resolver.py
deleted file mode 100644
index e47114096..000000000
--- a/src/agentscope/a2a/_file_resolver.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The JSON file based A2A agent card resolver."""
-import json
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-from ._base import AgentCardResolverBase
-
-if TYPE_CHECKING:
- from a2a.types import AgentCard
-else:
- AgentCard = "a2a.types.AgentCard"
-
-
-class FileAgentCardResolver(AgentCardResolverBase):
- """Agent card resolver that loads AgentCard from a JSON file.
-
- The JSON file should contain an AgentCard object with the following
- required fields:
-
- - name (str): The name of the agent
- - url (str): The URL of the agent
- - version (str): The version of the agent
- - capabilities (dict): The capabilities of the agent
- - default_input_modes (list[str]): Default input modes
- - default_output_modes (list[str]): Default output modes
- - skills (list): List of agent skills
-
- Example JSON file content:
-
- .. code-block:: json
-
- {
- "name": "RemoteAgent",
- "url": "http://localhost:8000",
- "description": "A remote A2A agent",
- "version": "1.0.0",
- "capabilities": {},
- "default_input_modes": ["text/plain"],
- "default_output_modes": ["text/plain"],
- "skills": []
- }
-
- """
-
- def __init__(
- self,
- file_path: str,
- ) -> None:
- """Initialize the FileAgentCardResolver with the path to the JSON file.
-
- Args:
- file_path (`str`):
- The path to the JSON file containing the agent card.
- """
- self._file_path = file_path
-
- async def get_agent_card(self) -> AgentCard:
- """Get the agent card from the JSON file.
-
- Returns:
- `AgentCard`:
- The agent card loaded from the file.
- """
- from a2a.types import AgentCard
-
- path = Path(self._file_path)
- if not path.exists():
- raise FileNotFoundError(
- f"Agent card file not found: {self._file_path}",
- )
-
- if not path.is_file():
- raise ValueError(f"Path is not a file: {self._file_path}")
-
- with path.open("r", encoding="utf-8") as f:
- agent_json_data = json.load(f)
- return AgentCard.model_validate(agent_json_data)
diff --git a/src/agentscope/a2a/_nacos_resolver.py b/src/agentscope/a2a/_nacos_resolver.py
deleted file mode 100644
index 5a5690234..000000000
--- a/src/agentscope/a2a/_nacos_resolver.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The Nacos-based A2A Agent Card resolver."""
-
-from typing import TYPE_CHECKING
-
-from ._base import AgentCardResolverBase
-from .._logging import logger
-
-if TYPE_CHECKING:
- from a2a.types import AgentCard
- from v2.nacos.common.client_config import ClientConfig
-else:
- AgentCard = "a2a.types.AgentCard"
- ClientConfig = "v2.nacos.common.client_config.ClientConfig"
-
-
-class NacosAgentCardResolver(AgentCardResolverBase):
- """Nacos-based A2A Agent Card resolver.
-
- Nacos is a dynamic service discovery, configuration and service
- management platform for building cloud native applications. This resolver
- fetches the agent card from a Nacos server and subscribes to updates.
- """
-
- def __init__(
- self,
- remote_agent_name: str,
- nacos_client_config: ClientConfig,
- version: str | None = None,
- ) -> None:
- """Initialize the nacos agent card resolver.
-
- Args:
- remote_agent_name (`str`):
- Name of the remote agent in Nacos.
- nacos_client_config (`ClientConfig | None`, optional):
- Nacos client configuration, where a `server_addresses`
- parameter is required.
- version (`str | None`, optional):
- Version of the agent card to fetch. If None, fetches the
- latest version. This version is also used when subscribing
- to agent card updates.
- Defaults to None (latest version).
- """
- if not remote_agent_name:
- raise ValueError(
- "The remote_agent_name cannot be empty.",
- )
-
- if not nacos_client_config:
- raise ValueError(
- "The nacos_client_config cannot be None.",
- )
-
- self._nacos_client_config = nacos_client_config
- self._remote_agent_name = remote_agent_name
- self._version = version
-
- async def get_agent_card(self) -> AgentCard:
- """Get agent card from Nacos with lazy initialization.
-
- Returns:
- `AgentCard`:
- The resolved agent card from Nacos.
- """
- try:
- from v2.nacos.ai.model.ai_param import GetAgentCardParam
- from v2.nacos.ai.nacos_ai_service import NacosAIService
- except ImportError as e:
- raise ImportError(
- "Please install the nacos sdk by running `pip install "
- "nacos-sdk-python>=3.0.0` first.",
- ) from e
-
- client = None
- try:
- client = await NacosAIService.create_ai_service(
- self._nacos_client_config,
- )
-
- await client.start()
- return await client.get_agent_card(
- GetAgentCardParam(
- agent_name=self._remote_agent_name,
- version=self._version,
- ),
- )
-
- finally:
- if client:
- # Close the Nacos client to free resources
- try:
- await client.shutdown()
- except Exception as e:
- logger.warning(
- "Failed to shutdown Nacos client: %s",
- str(e),
- )
diff --git a/src/agentscope/a2a/_well_known_resolver.py b/src/agentscope/a2a/_well_known_resolver.py
deleted file mode 100644
index 2f9445884..000000000
--- a/src/agentscope/a2a/_well_known_resolver.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The A2A well-known agent card resolver."""
-from typing import TYPE_CHECKING
-from urllib.parse import urlparse
-
-from ._base import AgentCardResolverBase
-from .._logging import logger
-
-if TYPE_CHECKING:
- from a2a.types import AgentCard
-else:
- AgentCard = "a2a.types.AgentCard"
-
-
-class WellKnownAgentCardResolver(AgentCardResolverBase):
- """Agent card resolver that loads AgentCard from a well-known URL."""
-
- def __init__(
- self,
- base_url: str,
- agent_card_path: str | None = None,
- ) -> None:
- """Initialize the WellKnownAgentCardResolver.
-
- Args:
- base_url (`str`):
- The base URL to resolve the agent card from.
- agent_card_path (`str | None`, optional):
- The path to the agent card relative to the base URL.
- Defaults to AGENT_CARD_WELL_KNOWN_PATH from a2a.utils.
- """
- self._base_url = base_url
- self._agent_card_path = agent_card_path
-
- async def get_agent_card(self) -> AgentCard:
- """Get the agent card from the well-known URL.
-
- Returns:
- `AgentCard`:
- The agent card loaded from the URL.
- """
- import httpx
- from a2a.client import A2ACardResolver
- from a2a.utils import AGENT_CARD_WELL_KNOWN_PATH
-
- try:
- parsed_url = urlparse(self._base_url)
- if not parsed_url.scheme or not parsed_url.netloc:
- logger.error(
- "[%s] Invalid URL format: %s",
- self.__class__.__name__,
- self._base_url,
- )
- raise ValueError(
- f"Invalid URL format: {self._base_url}",
- )
-
- base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
- relative_card_path = parsed_url.path
-
- # Use default path if not specified
- agent_card_path = (
- self._agent_card_path
- if self._agent_card_path is not None
- else AGENT_CARD_WELL_KNOWN_PATH
- )
-
- # Use async context manager to ensure proper cleanup
- async with httpx.AsyncClient(
- timeout=httpx.Timeout(timeout=600),
- ) as _http_client:
- resolver = A2ACardResolver(
- httpx_client=_http_client,
- base_url=base_url,
- agent_card_path=agent_card_path,
- )
- return await resolver.get_agent_card(
- relative_card_path=relative_card_path,
- )
- except Exception as e:
- logger.error(
- "[%s] Failed to resolve agent card from URL %s: %s",
- self.__class__.__name__,
- self._base_url,
- e,
- )
- raise RuntimeError(
- f"Failed to resolve AgentCard from URL "
- f"{self._base_url}: {e}",
- ) from e
diff --git a/src/agentscope/agent/__init__.py b/src/agentscope/agent/__init__.py
index e63c4c420..d65420dde 100644
--- a/src/agentscope/agent/__init__.py
+++ b/src/agentscope/agent/__init__.py
@@ -1,28 +1,5 @@
# -*- coding: utf-8 -*-
-"""The agent base class."""
-from ._agent_base import AgentBase
-from ._react_agent_base import ReActAgentBase
-from ._react_agent import ReActAgent
-from ._user_input import (
- UserInputBase,
- UserInputData,
- TerminalUserInput,
- StudioUserInput,
-)
-from ._user_agent import UserAgent
-from ._a2a_agent import A2AAgent
-from ._realtime_agent import RealtimeAgent
+"""Initialize the agent module."""
+from ._agent import Agent
-
-__all__ = [
- "AgentBase",
- "ReActAgentBase",
- "ReActAgent",
- "UserInputData",
- "UserInputBase",
- "TerminalUserInput",
- "StudioUserInput",
- "UserAgent",
- "A2AAgent",
- "RealtimeAgent",
-]
+__all__ = ["Agent"]
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.py
new file mode 100644
index 000000000..dae40adda
--- /dev/null
+++ b/src/agentscope/agent/_agent.py
@@ -0,0 +1,594 @@
+# -*- coding: utf-8 -*-
+"""The unified agent class in AgentScope library."""
+import uuid
+from dataclasses import dataclass
+from typing import AsyncGenerator, Literal
+
+from ..event import (
+ AgentEvent,
+ EventType,
+ ModelCallEndedEvent,
+ ModelCallStartedEvent,
+ RunFinishedEvent,
+ RunStartedEvent,
+ TextBlockDeltaEvent,
+ TextBlockEndEvent,
+ TextBlockStartEvent,
+ ThinkingBlockDeltaEvent,
+ ThinkingBlockEndEvent,
+ ThinkingBlockStartEvent,
+ ToolCallDeltaEvent,
+ ToolCallEndEvent,
+ ToolCallStartEvent,
+ ToolResultBinaryDeltaEvent,
+ ToolResultEndEvent,
+ ToolResultStartEvent,
+ ToolResultTextDeltaEvent,
+ RequireUserConfirmEvent,
+ RequireExternalExecutionEvent,
+ ExternalExecutionResultEvent,
+ UserConfirmResultEvent,
+)
+from ..message import Msg
+from ..model import ChatResponse
+from ..model import ChatUsage
+
+from ..message import (
+ TextBlock,
+ ThinkingBlock,
+ ToolCallBlock,
+ ToolResultBlock,
+ DataBlock,
+ Base64Source,
+ URLSource,
+)
+
+from ..model import ChatModelBase
+
+
+DEFAULT_COMPRESSION_PROMPT = (
+ "<system-hint>You have been working on the task described above but have "
+ "not yet completed it. "
+ "Now write a continuation summary that will allow you to resume work "
+ "efficiently in a future context window "
+ "where the conversation history will be replaced with this summary. "
+ "Your summary should be structured, concise, and actionable.</system-hint>"
+)
+
+
+@dataclass
+class AgentState:
+ """The agent state that should be saved and loaded from storage."""
+
+ context: list[Msg]
+ """The uncompressed conversation context, that will be feed into the LLM"""
+
+ reply_id: str
+ """The id of the current reply, which is also used as the id of the
+ final message of the reply."""
+
+ cur_iter: int = 0
+ """The current iteration of the agent's reasoning-acting loop."""
+
+ cur_summary: str = ""
+ """The current compressed summary of the context, which will be prepended
+ to the context when feed into the LLM."""
+
+
+class Agent:
+ """The unified agent class in AgentScope library."""
+
+ def __init__(
+ self,
+ name: str,
+ sys_prompt: str,
+ model: "ChatModelBase",
+ max_iters: int = 20,
+ ) -> None:
+ """Initialize an agent instance."""
+ if max_iters <= 0:
+ raise ValueError("max_iters must be greater than 0")
+
+ self.name = name
+ self._sys_prompt = sys_prompt
+ self.model = model
+ self.max_iters = max_iters
+
+ # TODO: storage - these should be loaded from storage
+ self._loaded = False
+ self.context: list[Msg] = []
+ self.reply_id: str = ""
+ self.cur_iter: int = 0
+ self.confirmed_tool_call_ids: list[str] = []
+ self.cur_summary: str = ""
+
+ @property
+ def sys_prompt(self) -> str:
+ """Get the system prompt of the agent."""
+ # TODO: Add toolkit skills prompt when toolkit is implemented
+ return self._sys_prompt
+
+ async def load_state(self) -> None:
+ """Load the state from storage if available."""
+ # TODO: Implement storage loading
+
+ async def save_state(self) -> None:
+ """Save the state to storage if available."""
+ # TODO: Implement storage saving
+
+ async def reply_stream(
+ self,
+ msgs: Msg | list[Msg] | None = None,
+ event: AgentEvent | None = None,
+ ) -> AsyncGenerator[AgentEvent, None]:
+ """Reply to the given message and stream agent events."""
+ await self.load_state()
+ try:
+ async for chunk in self._reply(msgs=msgs, event=event):
+ if not isinstance(chunk, Msg):
+ yield chunk
+ finally:
+ await self.save_state()
+
+ async def reply(
+ self,
+ msgs: Msg | list[Msg] | None = None,
+ event: AgentEvent | None = None,
+ ) -> Msg:
+ """Reply to the given message, consuming all streamed events."""
+ await self.load_state()
+ try:
+ final_msg: Msg | None = None
+ async for chunk in self._reply(msgs=msgs, event=event):
+ if chunk.is_last:
+ final_msg = chunk.msg
+ if final_msg is None:
+ raise RuntimeError("Agent did not produce a final message.")
+ return final_msg
+ finally:
+ await self.save_state()
+
+ def _save_to_context(
+ self,
+ blocks: list,
+ _usage: ChatUsage | None = None,
+ ) -> None:
+ """Save content blocks into the context."""
+ if len(self.context) == 0:
+ self.context.append(
+ Msg(name=self.name, content=list(blocks), role="assistant"),
+ )
+ else:
+ last_msg = self.context[-1]
+ if last_msg.role == "assistant" and last_msg.name == self.name:
+ if isinstance(last_msg.content, str):
+ last_msg.content = [TextBlock(text=last_msg.content)]
+ last_msg.content.extend(blocks)
+ # TODO: Merge usage if needed
+ else:
+ self.context.append(
+ Msg(
+ name=self.name,
+ content=list(blocks),
+ role="assistant",
+ ),
+ )
+
+ def _get_pending_tool_calls(self) -> list[ToolCallBlock]:
+ """Get tool calls from the last assistant message that have no result
+ yet."""
+ if len(self.context) == 0:
+ return []
+ last_msg = self.context[-1]
+ if last_msg.role != "assistant":
+ return []
+ content = (
+ last_msg.content if isinstance(last_msg.content, list) else []
+ )
+ tool_calls = [b for b in content if isinstance(b, ToolCallBlock)]
+ result_ids = {b.id for b in content if isinstance(b, ToolResultBlock)}
+ return [tc for tc in tool_calls if tc.id not in result_ids]
+
+ def _get_awaiting_tool_calls(self) -> dict:
+ """Get the awaiting tool calls that need user confirmation or external
+ execution."""
+ pending_tool_calls = self._get_pending_tool_calls()
+ pre_tool_calls: list = []
+
+ for tool_call in enumerate(pending_tool_calls):
+ # TODO: toolkit checks for user confirmation and external execution
+ pre_tool_calls.append(tool_call)
+
+ return {
+ "awaiting_type": None,
+ "expected_event_type": None,
+ "awaiting_tool_calls": [],
+ "pre_tool_calls": pre_tool_calls,
+ }
+
+ async def _reply(
+ self,
+ msgs: Msg | list[Msg] | None = None,
+ event: AgentEvent | None = None,
+ ) -> AsyncGenerator[AgentEvent | Msg, None]:
+ """Core reply logic. Yields chunks with is_last flag."""
+ awaiting_info = self._get_awaiting_tool_calls()
+ expected_event_type = awaiting_info["expected_event_type"]
+
+ if expected_event_type is not None:
+ if event is None or event.type != expected_event_type:
+ raise ValueError(
+ f"Agent is awaiting '{expected_event_type}' but received "
+ f"event of type '{event.type if event else 'none'}'.",
+ )
+
+ if isinstance(event, ExternalExecutionResultEvent):
+ self._save_to_context(list(event.execution_results))
+
+ elif isinstance(event, UserConfirmResultEvent):
+ for result in event.confirm_results:
+ confirmed = result.confirmed
+ tool_call = result.tool_call
+ if confirmed:
+ self.confirmed_tool_call_ids.append(tool_call["id"])
+ else:
+ rejection_text = (
+ f"<system-info>**Note** the user rejected the "
+ f"execution of tool "
+ f'"{tool_call["name"]}"!</system-info>'
+ )
+ yield ToolResultStartEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call["id"],
+ tool_call_name=tool_call["name"],
+ )
+ yield ToolResultTextDeltaEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call["id"],
+ delta=rejection_text,
+ )
+ yield ToolResultEndEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call["id"],
+ state="interrupted",
+ )
+ self._save_to_context(
+ [
+ ToolResultBlock(
+ id=tool_call["id"],
+ name=tool_call["name"],
+ output=[TextBlock(text=rejection_text)],
+ state="interrupted",
+ ),
+ ],
+ )
+
+ processed_ids = {
+ r.get("tool_call", {}).get("id")
+ for r in event.confirm_results
+ }
+ if self.context:
+ last_content = self.context[-1].content
+ if isinstance(last_content, list):
+ for block in last_content:
+ if (
+ isinstance(block, ToolCallBlock)
+ and block.id in processed_ids
+ ):
+ block.await_user_confirmation = False
+
+ else:
+ self.cur_iter = 0
+ self.reply_id = str(uuid.uuid4())
+ self.confirmed_tool_call_ids = []
+
+ yield RunStartedEvent(
+ session_id="",
+ reply_id=self.reply_id,
+ name=self.name,
+ role="assistant",
+ )
+
+ if isinstance(msgs, list):
+ self.context.extend(msgs)
+ elif msgs is not None:
+ self.context.append(msgs)
+
+ while self.cur_iter < self.max_iters:
+ pending_tool_calls = self._get_pending_tool_calls()
+ if len(pending_tool_calls) == 0:
+ await self._compress_memory_if_needed()
+ async for evt in self._reasoning(tool_choice="auto"):
+ yield evt
+
+ awaiting_info = self._get_awaiting_tool_calls()
+ awaiting_type = awaiting_info["awaiting_type"]
+ awaiting_tool_calls = awaiting_info["awaiting_tool_calls"]
+ pre_tool_calls = awaiting_info["pre_tool_calls"]
+
+ for tool_call in pre_tool_calls:
+ async for evt in self._acting(tool_call=tool_call):
+ yield evt
+ self.confirmed_tool_call_ids = [
+ cid
+ for cid in self.confirmed_tool_call_ids
+ if cid != tool_call.id
+ ]
+
+ if awaiting_type is not None:
+ if awaiting_type == EventType.REQUIRE_USER_CONFIRM:
+ yield RequireUserConfirmEvent(
+ reply_id=self.reply_id,
+ tool_calls=awaiting_tool_calls,
+ )
+ else:
+ yield RequireExternalExecutionEvent(
+ reply_id=self.reply_id,
+ tool_calls=awaiting_tool_calls,
+ )
+ waiting_text = (
+ "Waiting for user confirmation ..."
+ if awaiting_type == EventType.REQUIRE_USER_CONFIRM
+ else "Waiting for external execution ..."
+ )
+ yield Msg(
+ name=self.name,
+ content=[TextBlock(text=waiting_text)],
+ role="assistant",
+ )
+ return
+
+ if len(pre_tool_calls) == 0:
+ break
+
+ self.cur_iter += 1
+
+ last_block = None
+ if self.context:
+ last_content = self.context[-1].content
+ if isinstance(last_content, list) and last_content:
+ last_block = last_content[-1]
+ if last_block is None or not isinstance(last_block, TextBlock):
+ async for evt in self._reasoning(tool_choice="none"):
+ yield evt
+
+ yield RunFinishedEvent(
+ session_id="",
+ reply_id=self.reply_id,
+ )
+
+ final_block = None
+ if self.context:
+ last_content = self.context[-1].content
+ if isinstance(last_content, list) and last_content:
+ final_block = last_content[-1]
+ yield Msg(
+ id=self.reply_id,
+ name=self.name,
+ content=[final_block] if final_block else [],
+ role="assistant",
+ )
+
+ async def _reasoning(
+ self,
+ tool_choice: Literal["auto", "none", "required"] = "auto",
+ ) -> AsyncGenerator:
+ """Core reasoning logic. Yields chunks with is_last flag."""
+ # TODO: Pass tool schemas from toolkit when toolkit is implemented
+
+ yield ModelCallStartedEvent(
+ reply_id=self.reply_id,
+ model_name=self.model.model_name,
+ )
+
+ system_msg = Msg(
+ name="system",
+ content=[TextBlock(text=self.sys_prompt)],
+ role="system",
+ )
+ messages = [system_msg]
+ if self.cur_summary:
+ messages.append(
+ Msg(
+ name="user",
+ content=[TextBlock(text=self.cur_summary)],
+ role="user",
+ ),
+ )
+ messages.extend(self.context)
+
+ # TODO: Pass tools and tool_choice when toolkit is implemented
+ res = await self.model(
+ messages=messages,
+ tool_choice=tool_choice,
+ )
+
+ block_ids: dict = {
+ "text_block_id": None,
+ "thinking_block_id": None,
+ "tool_call_ids": [],
+ }
+ completed_response: ChatResponse | None = None
+
+ if self.model.stream:
+ async for chunk in res:
+ if chunk.is_last:
+ completed_response = chunk
+ break
+ async for evt in self._convert_chat_response_to_event(
+ block_ids,
+ chunk,
+ ):
+ yield evt
+ elif isinstance(res, ChatResponse):
+ completed_response = res
+ async for evt in self._convert_chat_response_to_event(
+ block_ids,
+ res,
+ ):
+ yield evt
+
+ if block_ids["text_block_id"] is not None:
+ yield TextBlockEndEvent(
+ reply_id=self.reply_id,
+ block_id=block_ids["text_block_id"],
+ )
+ if block_ids["thinking_block_id"] is not None:
+ yield ThinkingBlockEndEvent(
+ reply_id=self.reply_id,
+ block_id=block_ids["thinking_block_id"],
+ )
+ for tool_call_id in block_ids["tool_call_ids"]:
+ yield ToolCallEndEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call_id,
+ )
+
+ yield ModelCallEndedEvent(
+ reply_id=self.reply_id,
+ input_tokens=completed_response.usage.input_tokens
+ if completed_response.usage
+ else 0,
+ output_tokens=completed_response.usage.output_tokens
+ if completed_response.usage
+ else 0,
+ )
+
+ self._save_to_context(
+ list(completed_response.content),
+ completed_response.usage,
+ )
+
+ async def _acting(
+ self,
+ tool_call: ToolCallBlock,
+ ) -> AsyncGenerator:
+ """Core acting logic. Yields chunks with is_last flag."""
+ yield ToolResultStartEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call.id,
+ tool_call_name=tool_call.name,
+ )
+
+ # TODO: Implement toolkit execution when toolkit is available
+ # res = self.toolkit.call_tool_function(tool_call)
+ # async for tool_res in res:
+ # if tool_res.is_last:
+ # result_block = {
+ # "type": "tool_result",
+ # "id": tool_call["id"],
+ # "name": tool_call["name"],
+ # "output": tool_res.content,
+ # "state": tool_res.state,
+ # }
+ # self._save_to_context([result_block])
+ # return
+ # async for evt in self._convert_tool_response_to_event(
+ # tool_call, tool_res
+ # ):
+ # yield evt
+ raise NotImplementedError("Toolkit not yet implemented.")
+
+ async def _observe(self, msgs: Msg | list[Msg] | None = None) -> None:
+ """Receive external observation message(s) and save them into
+ context."""
+ await self.load_state()
+ if isinstance(msgs, list):
+ self.context.extend(msgs)
+ elif msgs is not None:
+ self.context.append(msgs)
+
+ async def _convert_chat_response_to_event(
+ self,
+ block_ids: dict,
+ chunk: ChatResponse,
+ ) -> AsyncGenerator:
+ """Convert a ChatResponse chunk into a sequence of agent events."""
+ for block in chunk.content:
+ if isinstance(block, TextBlock):
+ if block_ids["text_block_id"] is None:
+ block_ids["text_block_id"] = str(uuid.uuid4())
+ yield TextBlockStartEvent(
+ reply_id=self.reply_id,
+ block_id=block_ids["text_block_id"],
+ )
+ yield TextBlockDeltaEvent(
+ reply_id=self.reply_id,
+ block_id=block_ids["text_block_id"],
+ delta=block.text,
+ )
+
+ elif isinstance(block, ThinkingBlock):
+ if block_ids["thinking_block_id"] is None:
+ block_ids["thinking_block_id"] = str(uuid.uuid4())
+ yield ThinkingBlockStartEvent(
+ reply_id=self.reply_id,
+ block_id=block_ids["thinking_block_id"],
+ )
+ yield ThinkingBlockDeltaEvent(
+ reply_id=self.reply_id,
+ block_id=block_ids["thinking_block_id"],
+ delta=block.thinking,
+ )
+
+ elif isinstance(block, ToolCallBlock):
+ if block.id not in block_ids["tool_call_ids"]:
+ block_ids["tool_call_ids"].append(block.id)
+ yield ToolCallStartEvent(
+ reply_id=self.reply_id,
+ tool_call_id=block.id,
+ tool_call_name=block.name,
+ )
+ yield ToolCallDeltaEvent(
+ reply_id=self.reply_id,
+ tool_call_id=block.id,
+ delta=block.input,
+ )
+
+ async def _convert_tool_response_to_event(
+ self,
+ tool_call: ToolCallBlock,
+ tool_res: object,
+ ) -> AsyncGenerator:
+ """Convert a ToolResponse into a sequence of agent events."""
+ # TODO: Properly typed once toolkit is implemented
+ for block in tool_res.content: # type: ignore[attr-defined]
+ if isinstance(block, TextBlock):
+ yield ToolResultTextDeltaEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call.id,
+ delta=block.text,
+ )
+
+ elif isinstance(block, DataBlock):
+ if isinstance(block.source, Base64Source):
+ yield ToolResultBinaryDeltaEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call.id,
+ media_type=block.source.media_type,
+ data=block.source.data,
+ )
+ elif isinstance(block.source, URLSource):
+ yield ToolResultBinaryDeltaEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call.id,
+ media_type=block.source.media_type,
+ url=block.source.url,
+ )
+
+ yield ToolResultEndEvent(
+ reply_id=self.reply_id,
+ tool_call_id=tool_call.id,
+ state=tool_res.state, # type: ignore[attr-defined]
+ )
+
+ def _split_context_for_compression(self) -> tuple[list[Msg], list[Msg]]:
+ """Split context into parts to compress and parts to keep recent."""
+ # TODO: Implement full splitting logic when compression_config is
+ # available
+ return [], list(self.context)
+
+ async def _compress_memory_if_needed(self) -> None:
+ """Compress the agent's memory if the token count exceeds the
+ threshold."""
+ # TODO: Implement when compression_config is available
diff --git a/src/agentscope/embedding/_dashscope_multimodal_embedding.py b/src/agentscope/embedding/_dashscope_multimodal_embedding.py
index f8a50a156..e63bfc612 100644
--- a/src/agentscope/embedding/_dashscope_multimodal_embedding.py
+++ b/src/agentscope/embedding/_dashscope_multimodal_embedding.py
@@ -8,8 +8,7 @@ from ._embedding_response import EmbeddingResponse
from ._embedding_usage import EmbeddingUsage
from ._embedding_base import EmbeddingModelBase
from ..message import (
- VideoBlock,
- ImageBlock,
+ DataBlock,
TextBlock,
)
@@ -90,14 +89,14 @@ class DashScopeMultiModalEmbedding(EmbeddingModelBase):
async def __call__(
self,
- inputs: list[TextBlock | ImageBlock | VideoBlock],
+ inputs: list[TextBlock | DataBlock],
**kwargs: Any,
) -> EmbeddingResponse:
"""Call the DashScope multimodal embedding API, which accepts text,
image, and video data.
Args:
- inputs (`list[TextBlock | ImageBlock | VideoBlock]`):
+ inputs (`list[TextBlock | DataBlock]`):
The input data to be embedded. It can be a list of text,
image, and video blocks.
diff --git a/src/agentscope/event/__init__.py b/src/agentscope/event/__init__.py
new file mode 100644
index 000000000..4d045dc8f
--- /dev/null
+++ b/src/agentscope/event/__init__.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""The event module of agentscope."""
+
+from ._event import (
+ EventType,
+ EventBase,
+ RunStartedEvent,
+ RunFinishedEvent,
+ ModelCallStartedEvent,
+ ModelCallEndedEvent,
+ TextBlockStartEvent,
+ TextBlockDeltaEvent,
+ TextBlockEndEvent,
+ BinaryBlockStartEvent,
+ BinaryBlockDeltaEvent,
+ BinaryBlockEndEvent,
+ ThinkingBlockStartEvent,
+ ThinkingBlockDeltaEvent,
+ ThinkingBlockEndEvent,
+ ToolCallStartEvent,
+ ToolCallDeltaEvent,
+ ToolCallEndEvent,
+ ToolResultStartEvent,
+ ToolResultTextDeltaEvent,
+ ToolResultBinaryDeltaEvent,
+ ToolResultEndEvent,
+ ExceedMaxItersEvent,
+ RequireUserConfirmEvent,
+ RequireExternalExecutionEvent,
+ UserConfirmResultEvent,
+ ExternalExecutionResultEvent,
+ AgentEvent,
+)
+
+
+__all__ = [
+ "EventType",
+ "EventBase",
+ "RunStartedEvent",
+ "RunFinishedEvent",
+ "ModelCallStartedEvent",
+ "ModelCallEndedEvent",
+ "TextBlockStartEvent",
+ "TextBlockDeltaEvent",
+ "TextBlockEndEvent",
+ "BinaryBlockStartEvent",
+ "BinaryBlockDeltaEvent",
+ "BinaryBlockEndEvent",
+ "ThinkingBlockStartEvent",
+ "ThinkingBlockDeltaEvent",
+ "ThinkingBlockEndEvent",
+ "ToolCallStartEvent",
+ "ToolCallDeltaEvent",
+ "ToolCallEndEvent",
+ "ToolResultStartEvent",
+ "ToolResultTextDeltaEvent",
+ "ToolResultBinaryDeltaEvent",
+ "ToolResultEndEvent",
+ "ExceedMaxItersEvent",
+ "RequireUserConfirmEvent",
+ "RequireExternalExecutionEvent",
+ "UserConfirmResultEvent",
+ "ExternalExecutionResultEvent",
+ "AgentEvent",
+]
diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.py
new file mode 100644
index 000000000..fcb8460f2
--- /dev/null
+++ b/src/agentscope/event/_event.py
@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+"""Event types for agent execution."""
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Literal, List, TypeAlias
+
+from pydantic import BaseModel, Field
+
+from ..message import ToolCallBlock, ToolResultBlock
+
+
+class EventType(str, Enum):
+ """Event type enumeration."""
+
+ RUN_STARTED = "RUN_STARTED"
+ RUN_FINISHED = "RUN_FINISHED"
+
+ MODEL_CALL_STARTED = "MODEL_CALL_STARTED"
+ MODEL_CALL_ENDED = "MODEL_CALL_ENDED"
+
+ TEXT_BLOCK_START = "TEXT_BLOCK_START"
+ TEXT_BLOCK_DELTA = "TEXT_BLOCK_DELTA"
+ TEXT_BLOCK_END = "TEXT_BLOCK_END"
+
+ BINARY_BLOCK_START = "BINARY_BLOCK_START"
+ BINARY_BLOCK_DELTA = "BINARY_BLOCK_DELTA"
+ BINARY_BLOCK_END = "BINARY_BLOCK_END"
+
+ THINKING_BLOCK_START = "THINKING_BLOCK_START"
+ THINKING_BLOCK_DELTA = "THINKING_BLOCK_DELTA"
+ THINKING_BLOCK_END = "THINKING_BLOCK_END"
+
+ TOOL_CALL_START = "TOOL_CALL_START"
+ TOOL_CALL_DELTA = "TOOL_CALL_DELTA"
+ TOOL_CALL_END = "TOOL_CALL_END"
+
+ TOOL_RESULT_START = "TOOL_RESULT_START"
+ TOOL_RESULT_TEXT_DELTA = "TOOL_RESULT_TEXT_DELTA"
+ TOOL_RESULT_BINARY_DELTA = "TOOL_RESULT_BINARY_DELTA"
+ TOOL_RESULT_END = "TOOL_RESULT_END"
+
+ EXCEED_MAX_ITERS = "EXCEED_MAX_ITERS"
+
+ REQUIRE_USER_CONFIRM = "REQUIRE_USER_CONFIRM"
+ REQUIRE_EXTERNAL_EXECUTION = "REQUIRE_EXTERNAL_EXECUTION"
+
+ USER_CONFIRM_RESULT = "USER_CONFIRM_RESULT"
+ EXTERNAL_EXECUTION_RESULT = "EXTERNAL_EXECUTION_RESULT"
+
+
+class EventBase(BaseModel):
+ """Base event class."""
+
+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+ """Unique event identifier."""
+ created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
+ """ISO 8601 timestamp of when the event was created."""
+
+
+class RunStartedEvent(EventBase):
+ """Run started event."""
+
+ type: Literal[EventType.RUN_STARTED] = EventType.RUN_STARTED
+ """Event type."""
+ session_id: str
+ """ID of the session this run belongs to."""
+ reply_id: str
+ """ID of the reply message produced by this run."""
+ name: str
+ """Name of the agent."""
+ role: Literal["user", "assistant", "system"] = "assistant"
+ """Role of the agent."""
+
+
+class RunFinishedEvent(EventBase):
+ """Run finished event."""
+
+ type: Literal[EventType.RUN_FINISHED] = EventType.RUN_FINISHED
+ """Event type."""
+ session_id: str
+ """ID of the session this run belongs to."""
+ reply_id: str
+ """ID of the reply message produced by this run."""
+
+
+class ModelCallStartedEvent(EventBase):
+ """Model call started event."""
+
+ type: Literal[EventType.MODEL_CALL_STARTED] = EventType.MODEL_CALL_STARTED
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this model call belongs to."""
+ model_name: str
+ """Name of the model being called."""
+
+
+class ModelCallEndedEvent(EventBase):
+ """Model call ended event."""
+
+ type: Literal[EventType.MODEL_CALL_ENDED] = EventType.MODEL_CALL_ENDED
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this model call belongs to."""
+ input_tokens: int
+ """Number of input tokens consumed."""
+ output_tokens: int
+ """Number of output tokens generated."""
+
+
+class TextBlockStartEvent(EventBase):
+ """Text block start event."""
+
+ type: Literal[EventType.TEXT_BLOCK_START] = EventType.TEXT_BLOCK_START
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the text block."""
+
+
+class TextBlockDeltaEvent(EventBase):
+ """Text block delta event."""
+
+ type: Literal[EventType.TEXT_BLOCK_DELTA] = EventType.TEXT_BLOCK_DELTA
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the text block."""
+ delta: str
+ """Incremental text content."""
+
+
+class TextBlockEndEvent(EventBase):
+ """Text block end event."""
+
+ type: Literal[EventType.TEXT_BLOCK_END] = EventType.TEXT_BLOCK_END
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the text block."""
+
+
+class BinaryBlockStartEvent(EventBase):
+ """Binary block start event."""
+
+ type: Literal[EventType.BINARY_BLOCK_START] = EventType.BINARY_BLOCK_START
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the binary block."""
+ media_type: str
+ """MIME type of the binary content (e.g. "image/png")."""
+
+
+class BinaryBlockDeltaEvent(EventBase):
+ """Binary block delta event."""
+
+ type: Literal[EventType.BINARY_BLOCK_DELTA] = EventType.BINARY_BLOCK_DELTA
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the binary block."""
+ data: str
+ """Incremental base64-encoded binary data."""
+ media_type: str
+ """MIME type of the binary content."""
+
+
+class BinaryBlockEndEvent(EventBase):
+ """Binary block end event."""
+
+ type: Literal[EventType.BINARY_BLOCK_END] = EventType.BINARY_BLOCK_END
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the binary block."""
+
+
+class ThinkingBlockStartEvent(EventBase):
+ """Thinking block start event."""
+
+ type: Literal[
+ EventType.THINKING_BLOCK_START
+ ] = EventType.THINKING_BLOCK_START
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the thinking block."""
+
+
+class ThinkingBlockDeltaEvent(EventBase):
+ """Thinking block delta event."""
+
+ type: Literal[
+ EventType.THINKING_BLOCK_DELTA
+ ] = EventType.THINKING_BLOCK_DELTA
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the thinking block."""
+ delta: str
+ """Incremental thinking text content."""
+
+
+class ThinkingBlockEndEvent(EventBase):
+ """Thinking block end event."""
+
+ type: Literal[EventType.THINKING_BLOCK_END] = EventType.THINKING_BLOCK_END
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this block belongs to."""
+ block_id: str
+ """Unique identifier of the thinking block."""
+
+
+class ToolCallStartEvent(EventBase):
+ """Tool call start event."""
+
+ type: Literal[EventType.TOOL_CALL_START] = EventType.TOOL_CALL_START
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this tool call belongs to."""
+ tool_call_id: str
+ """Unique identifier of the tool call."""
+ tool_call_name: str
+ """Name of the tool being called."""
+
+
+class ToolCallDeltaEvent(EventBase):
+ """Tool call delta event."""
+
+ type: Literal[EventType.TOOL_CALL_DELTA] = EventType.TOOL_CALL_DELTA
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this tool call belongs to."""
+ tool_call_id: str
+ """Unique identifier of the tool call."""
+ delta: str
+ """Incremental tool call arguments (JSON fragment)."""
+
+
+class ToolCallEndEvent(EventBase):
+ """Tool call end event."""
+
+ type: Literal[EventType.TOOL_CALL_END] = EventType.TOOL_CALL_END
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this tool call belongs to."""
+ tool_call_id: str
+ """Unique identifier of the tool call."""
+
+
+class ToolResultStartEvent(EventBase):
+ """Tool result start event."""
+
+ type: Literal[EventType.TOOL_RESULT_START] = EventType.TOOL_RESULT_START
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this tool result belongs to."""
+ tool_call_id: str
+ """ID of the corresponding tool call."""
+ tool_call_name: str
+ """Name of the tool that was called."""
+
+
+class ToolResultTextDeltaEvent(EventBase):
+ """Tool result text delta event."""
+
+ type: Literal[
+ EventType.TOOL_RESULT_TEXT_DELTA
+ ] = EventType.TOOL_RESULT_TEXT_DELTA
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this tool result belongs to."""
+ tool_call_id: str
+ """ID of the corresponding tool call."""
+ delta: str
+ """Incremental text content of the tool result."""
+
+
+class ToolResultBinaryDeltaEvent(EventBase):
+ """Tool result binary delta event."""
+
+ type: Literal[
+ EventType.TOOL_RESULT_BINARY_DELTA
+ ] = EventType.TOOL_RESULT_BINARY_DELTA
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this tool result belongs to."""
+ tool_call_id: str
+ """ID of the corresponding tool call."""
+ media_type: str
+ """MIME type of the binary content."""
+ data: str | None = None
+ """Base64-encoded binary data, mutually exclusive with `url`."""
+ url: str | None = None
+ """URL pointing to the binary content, mutually exclusive with `data`."""
+
+
+class ToolResultEndEvent(EventBase):
+ """Tool result end event."""
+
+ type: Literal[EventType.TOOL_RESULT_END] = EventType.TOOL_RESULT_END
+ """Event type."""
+ reply_id: str
+ """ID of the reply message this tool result belongs to."""
+ tool_call_id: str
+ """ID of the corresponding tool call."""
+ state: Literal["success", "error", "interrupted", "running"]
+ """Final execution state of the tool call."""
+
+
+class ExceedMaxItersEvent(EventBase):
+ """Exceed max iterations event."""
+
+ type: Literal[EventType.EXCEED_MAX_ITERS] = EventType.EXCEED_MAX_ITERS
+ """Event type."""
+ reply_id: str
+ """ID of the reply message associated with this run."""
+ agent_id: str
+ """ID of the agent that exceeded the iteration limit."""
+ name: str
+ """Name of the agent."""
+
+
+class RequireUserConfirmEvent(EventBase):
+ """Require user confirm event."""
+
+ type: Literal[
+ EventType.REQUIRE_USER_CONFIRM
+ ] = EventType.REQUIRE_USER_CONFIRM
+ """Event type."""
+ reply_id: str
+ """ID of the reply message associated with this run."""
+ tool_calls: List[ToolCallBlock]
+ """Tool calls pending user confirmation."""
+
+
+class RequireExternalExecutionEvent(EventBase):
+ """Require external execution event."""
+
+ type: Literal[
+ EventType.REQUIRE_EXTERNAL_EXECUTION
+ ] = EventType.REQUIRE_EXTERNAL_EXECUTION
+ """Event type."""
+ reply_id: str
+ """ID of the reply message associated with this run."""
+ tool_calls: List[ToolCallBlock]
+ """Tool calls to be executed externally."""
+
+
+class ConfirmResult(BaseModel):
+ """Confirm result for a tool call."""
+
+ confirmed: bool
+ """Whether the user confirmed the tool call."""
+ tool_call: ToolCallBlock
+ """The tool call that was confirmed or rejected."""
+
+
+class UserConfirmResultEvent(EventBase):
+ """User confirm result event."""
+
+ type: Literal[
+ EventType.USER_CONFIRM_RESULT
+ ] = EventType.USER_CONFIRM_RESULT
+ """Event type."""
+ reply_id: str
+ """ID of the reply message associated with this run."""
+ confirm_results: list[ConfirmResult]
+ """Confirmation results for each pending tool call."""
+
+
+class ExternalExecutionResultEvent(EventBase):
+ """External execution result event."""
+
+ type: Literal[
+ EventType.EXTERNAL_EXECUTION_RESULT
+ ] = EventType.EXTERNAL_EXECUTION_RESULT
+ """Event type."""
+ reply_id: str
+ """ID of the reply message associated with this run."""
+ execution_results: List[ToolResultBlock]
+ """Results returned by the external executor."""
+
+
+AgentEvent: TypeAlias = (
+ RunStartedEvent
+ | RunFinishedEvent
+ | ExceedMaxItersEvent
+ | RequireUserConfirmEvent
+ | RequireExternalExecutionEvent
+ | ModelCallStartedEvent
+ | ModelCallEndedEvent
+ | TextBlockStartEvent
+ | TextBlockDeltaEvent
+ | TextBlockEndEvent
+ | BinaryBlockStartEvent
+ | BinaryBlockDeltaEvent
+ | BinaryBlockEndEvent
+ | ThinkingBlockStartEvent
+ | ThinkingBlockDeltaEvent
+ | ThinkingBlockEndEvent
+ | ToolCallStartEvent
+ | ToolCallDeltaEvent
+ | ToolCallEndEvent
+ | ToolResultStartEvent
+ | ToolResultTextDeltaEvent
+ | ToolResultBinaryDeltaEvent
+ | ToolResultEndEvent
+ | UserConfirmResultEvent
+ | ExternalExecutionResultEvent
+)
diff --git a/src/agentscope/formatter/__init__.py b/src/agentscope/formatter/__init__.py
index 320229813..974880ce8 100644
--- a/src/agentscope/formatter/__init__.py
+++ b/src/agentscope/formatter/__init__.py
@@ -2,7 +2,6 @@
"""The formatter module in agentscope."""
from ._formatter_base import FormatterBase
-from ._truncated_formatter_base import TruncatedFormatterBase
from ._dashscope_formatter import (
DashScopeChatFormatter,
DashScopeMultiAgentFormatter,
@@ -27,11 +26,9 @@ from ._deepseek_formatter import (
DeepSeekChatFormatter,
DeepSeekMultiAgentFormatter,
)
-from ._a2a_formatter import A2AChatFormatter
__all__ = [
"FormatterBase",
- "TruncatedFormatterBase",
"DashScopeChatFormatter",
"DashScopeMultiAgentFormatter",
"OpenAIChatFormatter",
@@ -44,5 +41,4 @@ __all__ = [
"OllamaMultiAgentFormatter",
"DeepSeekChatFormatter",
"DeepSeekMultiAgentFormatter",
- "A2AChatFormatter",
]
diff --git a/src/agentscope/formatter/_a2a_formatter.py b/src/agentscope/formatter/_a2a_formatter.py
deleted file mode 100644
index 4382dd89d..000000000
--- a/src/agentscope/formatter/_a2a_formatter.py
+++ /dev/null
@@ -1,364 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The A2A message formatter class."""
-import mimetypes
-import uuid
-from typing import Literal, TYPE_CHECKING
-
-
-from .._logging import logger
-from ._formatter_base import FormatterBase
-from ..message import (
- Msg,
- TextBlock,
- URLSource,
- Base64Source,
- ContentBlock,
-)
-
-
-if TYPE_CHECKING:
- from a2a.types import (
- Message,
- Task,
- Part,
- )
-else:
- Message = "a2a.types.Message"
- Task = "a2a.types.Task"
- Part = "a2a.types.Part"
-
-
-class A2AChatFormatter(FormatterBase):
- """A2A message formatter class, which convert AgentScope messages into
- A2A message format."""
-
- async def format(self, msgs: list[Msg]) -> Message:
- """Convert AgentScope messages into a A2A message object. Note that
- A2A server only supports single request message, so the input msgs
- list will be merged into a single A2A Message.
-
- .. note:: Note the A2A protocol receives a single message per request,
- so multi-message inputs will be merged into one A2A Message with role
- 'user'.
-
- Args:
- msgs (`list[Msg]`):
- List of AgentScope Msg objects to be converted.
-
- Returns:
- `Message`:
- The converted A2A Message object.
- """
-
- from a2a.types import (
- Part,
- TextPart,
- FilePart,
- FileWithUri,
- FileWithBytes,
- DataPart,
- Role,
- Message,
- )
-
- self.assert_list_of_msgs(msgs)
-
- parts = []
- for msg in msgs:
- for block in msg.get_content_blocks():
- block_type = block.get("type")
- if block_type == "text" and block.get("text"):
- parts.append(
- Part(
- root=TextPart(
- text=block.get("text"),
- ),
- ),
- )
-
- elif block_type == "thinking" and block.get("thinking"):
- parts.append(
- Part(
- root=TextPart(
- text=block.get("thinking"),
- ),
- ),
- )
-
- elif block_type in [
- "image",
- "video",
- "audio",
- ] and block.get("source"):
- source = block.get("source", {})
- source_type = source.get("type")
-
- if source_type == "url":
- parts.append(
- Part(
- root=FilePart(
- file=FileWithUri(
- uri=source.get("url"),
- ),
- ),
- ),
- )
-
- elif source_type == "base64":
- parts.append(
- Part(
- root=FilePart(
- file=FileWithBytes(
- bytes=source.get("data"),
- mime_type=source.get("media_type"),
- ),
- ),
- ),
- )
-
- else:
- raise ValueError(
- f"Unsupported source type: {source_type}",
- )
-
- elif block_type in ["tool_use", "tool_result"]:
- parts.append(
- Part(
- root=DataPart(
- data=block,
- ),
- ),
- )
-
- else:
- logger.error(
- "Unsupported block type %s in A2AFormatter.",
- block_type,
- )
-
- a2a_message = Message(
- message_id=str(uuid.uuid4()),
- role=Role.user,
- parts=parts,
- )
-
- return a2a_message
-
- async def format_a2a_message(self, name: str, message: Message) -> Msg:
- """Convert A2A Message object back to AgentScope Msg format.
-
- Args:
- name (`str`):
- The name of the message sender.
- message (`Message`):
- The A2A Message object to be converted.
-
- Returns:
- `list[Msg]`:
- List of converted AgentScope Msg objects.
- """
-
- from a2a.types import Role
-
- content = []
- metadata = None
- for part in message.parts:
- content.append(
- await self._format_a2a_part(part),
- )
-
- if message.role == Role.user:
- role: Literal["user", "assistant"] = "user"
- elif message.role == Role.agent:
- role = "assistant"
- else:
- raise ValueError(
- f"Unsupported role: {message.role} in A2A message.",
- )
-
- return Msg(
- name=name,
- role=role,
- content=content,
- metadata=metadata,
- )
-
- @staticmethod
- def _guess_type(
- uri: str | None = None,
- mime_type: str | None = None,
- ) -> Literal["image", "video", "audio", "unknown"]:
- """Guess the content type from the uri or mime type.
-
- Args:
- uri (`str | None`, optional):
- The uri of the content.
- mime_type (`str | None`, optional):
- The mime type of the content.
-
- Returns:
- `Literal["image", "video", "audio", "unknown"]`:
- The guessed content type.
- """
- if mime_type is None and uri is None:
- raise ValueError(
- "Either uri or mime_type must be provided to guess the"
- " content type.",
- )
-
- if mime_type is None:
- mime_type, _encoding = mimetypes.guess_type(uri or "")
-
- if isinstance(mime_type, str):
- if mime_type.startswith("image/"):
- return "image"
-
- if mime_type.startswith("video/"):
- return "video"
-
- if mime_type.startswith("audio/"):
- return "audio"
-
- return "unknown"
-
- async def format_a2a_task(self, name: str, task: Task) -> list[Msg]:
- """Convert A2A Task object back to AgentScope Msg format.
-
- Args:
- name (`str`):
- The name of the message sender.
- task (`Task`):
- The A2A Task object to be converted.
-
- Returns:
- `list[Msg]`:
- Converted AgentScope Msg objects.
- """
- msgs = []
- if task.status and task.status.message:
- msgs.append(
- await self.format_a2a_message(name, task.status.message),
- )
-
- merged_msgs = []
- for msg in msgs:
- if merged_msgs and merged_msgs[-1].role == msg.role:
- merged_msgs[-1].content.extend(msg.content)
-
- else:
- merged_msgs.append(msg)
-
- if task.artifacts:
- for artifact in task.artifacts:
- artifact_content = [
- await self._format_a2a_part(_) for _ in artifact.parts
- ]
-
- if merged_msgs and merged_msgs[-1].role == "assistant":
- merged_msgs[-1].content.extend(artifact_content)
- merged_msgs[-1].metadata = artifact.metadata
-
- else:
- merged_msgs.append(
- Msg(
- name=name,
- role="assistant",
- content=artifact_content,
- metadata=artifact.metadata,
- ),
- )
-
- return merged_msgs
-
- async def _format_a2a_part(self, part: Part) -> ContentBlock:
- """Convert a single A2A Part object into AgentScope ContentBlock.
-
- .. note:: We will try to convert the `DataPart` into tool use and tool
- result blocks if possible.
-
- Args:
- part (`Part`):
- The A2A Part object to be converted.
-
- Returns:
- `ContentBlock`:
- The converted AgentScope ContentBlock.
- """
-
- from a2a.types import (
- TextPart,
- FilePart,
- FileWithUri,
- FileWithBytes,
- DataPart,
- )
-
- if isinstance(part.root, TextPart):
- return TextBlock(
- type="text",
- text=part.root.text,
- )
-
- if isinstance(part.root, FilePart):
- if isinstance(part.root.file, FileWithUri):
- return { # type: ignore[return-value, misc]
- "type": self._guess_type(
- part.root.file.uri,
- part.root.file.mime_type,
- ),
- "source": URLSource(
- type="url",
- url=part.root.file.uri,
- ),
- }
-
- if isinstance(part.root.file, FileWithBytes):
- return { # type: ignore[return-value, misc]
- "type": self._guess_type(
- mime_type=part.root.file.mime_type,
- ),
- "source": Base64Source(
- type="base64",
- media_type=part.root.file.mime_type
- or "application/octet-stream",
- data=part.root.file.bytes,
- ),
- }
-
- raise ValueError(
- f"Unsupported File type: {type(part.root.file)} in A2A"
- "message.",
- )
-
- if isinstance(part.root, DataPart):
- # Maybe the tool use and tool result blocks
- if {
- "type",
- "name",
- "input",
- "id",
- } <= part.root.data.keys() and part.root.data[
- "type"
- ] == "tool_use":
- return part.root.data
-
- if {
- "type",
- "name",
- "output",
- "id",
- } <= part.root.data.keys() and part.root.data[
- "type"
- ] == "tool_result":
- return part.root.data
-
- # TODO: what about the other data parts?
- return TextBlock(
- type="text",
- text=str(part.root.data),
- )
-
- raise ValueError(
- f"Unsupported Part type: {type(part.root)} in A2A message"
- f": {part.root}",
- )
diff --git a/src/agentscope/formatter/_anthropic_formatter.py b/src/agentscope/formatter/_anthropic_formatter.py
index 3a1818325..0d2061d29 100644
--- a/src/agentscope/formatter/_anthropic_formatter.py
+++ b/src/agentscope/formatter/_anthropic_formatter.py
@@ -1,41 +1,34 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches
"""The Anthropic formatter module."""
-
+import base64
+import fnmatch
+from abc import ABC
from typing import Any
-from ._truncated_formatter_base import TruncatedFormatterBase
-from .._logging import logger
-from ..message import Msg, TextBlock, ImageBlock, ToolUseBlock, ToolResultBlock
-from ..token import TokenCounterBase
-
-
-class AnthropicChatFormatter(TruncatedFormatterBase):
- """The Anthropic formatter class for chatbot scenario, where only a user
- and an agent are involved. We use the `role` field to identify different
- entities in the conversation.
- """
-
- support_tools_api: bool = True
- """Whether support tools API"""
+import requests
- support_multiagent: bool = False
- """Whether support multi-agent conversations"""
-
- support_vision: bool = True
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Multimodal
- ImageBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
-
- async def _format(
+from ._formatter_base import FormatterBase
+from .._logging import logger
+from ..message import (
+ Msg,
+ TextBlock,
+ ThinkingBlock,
+ HintBlock,
+ DataBlock,
+ ToolCallBlock,
+ ToolResultBlock,
+ URLSource,
+ Base64Source,
+)
+
+
+class _AnthropicFormatterBase(FormatterBase, ABC):
+ """Mixin for formatting Anthropic formatters to avoid duplication between
+ AnthropicChatFormatter and AnthropicMultiAgentFormatter."""
+
+ supported_input_media_types: list[str]
+
+ async def _format_messages(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
@@ -58,92 +51,263 @@ class AnthropicChatFormatter(TruncatedFormatterBase):
self.assert_list_of_msgs(msgs)
messages: list[dict] = []
- for index, msg in enumerate(msgs):
- content_blocks = []
+ for msg in msgs:
+ content_blocks: list = []
for block in msg.get_content_blocks():
- typ = block.get("type")
- if typ in ["thinking", "text", "image"]:
- content_blocks.append({**block})
+ if isinstance(block, TextBlock):
+ content_blocks.append(
+ {"type": "text", "text": block.text},
+ )
- elif typ == "tool_use":
+ elif isinstance(block, ThinkingBlock):
+ content_blocks.append(
+ {"type": "thinking", "thinking": block.thinking},
+ )
+
+ elif isinstance(block, HintBlock):
+ pass # Anthropic does not support hint blocks
+
+ elif isinstance(block, DataBlock):
+ formatted_block = self._format_anthropic_data_block(block)
+ if formatted_block:
+ content_blocks.append(formatted_block)
+
+ elif isinstance(block, ToolCallBlock):
content_blocks.append(
{
- "id": block.get("id"),
"type": "tool_use",
- "name": block.get("name"),
- "input": block.get("input", {}),
+ "id": block.id,
+ "name": block.name,
+ "input": block.input,
},
)
- elif typ == "tool_result":
- output = block.get("output")
- if output is None:
- content_value = [{"type": "text", "text": None}]
- elif isinstance(output, list):
- content_value = output
- else:
- content_value = [{"type": "text", "text": str(output)}]
- messages.append(
- {
- "role": "user",
- "content": [
+ elif isinstance(block, ToolResultBlock):
+ (
+ textual_output,
+ multimodal_data,
+ ) = self.convert_tool_result_to_string(block.output)
+
+ tool_result_content = []
+ if textual_output:
+ tool_result_content.append(
+ {"type": "text", "text": textual_output},
+ )
+
+ for data_block in multimodal_data:
+ formatted_block = self._format_anthropic_data_block(
+ data_block,
+ )
+ if formatted_block:
+ tool_result_content.append(formatted_block)
+
+ # If there's multimodal data, promote to UserMsg
+ if multimodal_data:
+ # Add tool result first
+ content_blocks.append(
+ {
+ "type": "tool_result",
+ "tool_use_id": block.id,
+ "content": tool_result_content,
+ },
+ )
+
+ # Flush current message
+ if content_blocks:
+ messages.append(
{
- "type": "tool_result",
- "tool_use_id": block.get("id"),
- "content": content_value,
+ "role": msg.role,
+ "content": content_blocks,
},
- ],
- },
- )
+ )
+ content_blocks = []
+
+ # Insert UserMsg with multimodal data
+ user_content = []
+ for data_block in multimodal_data:
+ formatted_block = (
+ self._format_anthropic_data_block(data_block)
+ )
+ if formatted_block:
+ user_content.append(formatted_block)
+
+ if user_content:
+ messages.append(
+ {
+ "role": "user",
+ "content": user_content,
+ },
+ )
+ else:
+ # No multimodal data, just add tool result
+ content_blocks.append(
+ {
+ "type": "tool_result",
+ "tool_use_id": block.id,
+ "content": tool_result_content,
+ },
+ )
+
else:
logger.warning(
"Unsupported block type %s in the message, skipped.",
- typ,
+ type(block),
)
- # Claude only allow the first message to be system message
- if msg.role == "system" and index != 0:
- role = "user"
- else:
- role = msg.role
+ if content_blocks:
+ messages.append(
+ {
+ "role": msg.role,
+ "content": content_blocks,
+ },
+ )
- msg_anthropic = {
- "role": role,
- "content": content_blocks or None,
- }
+ return messages
+
+ def _format_anthropic_data_block(
+ self,
+ block: DataBlock,
+ ) -> dict[str, Any] | None:
+ """Format a DataBlock into Anthropic API format.
- # When both content and tool_calls are None, skipped
- if msg_anthropic["content"] or msg_anthropic.get("tool_calls"):
- messages.append(msg_anthropic)
+ Args:
+ block (`DataBlock`):
+ The data block to format.
- return messages
+ Returns:
+ `dict[str, Any] | None`:
+ The formatted data block, or None if the media type is not
+ supported.
+ """
+ source = block.source
+ media_type = source.media_type
+
+ # Check if media type is supported
+ if not any(
+ fnmatch.fnmatch(media_type, pattern)
+ for pattern in self.supported_input_media_types
+ ):
+ logger.warning(
+ "Media type %s is not supported, skipped.",
+ media_type,
+ )
+ return None
+ # Anthropic only supports images
+ if not media_type.startswith("image/"):
+ logger.warning(
+ "Anthropic only supports image data, got %s, skipped.",
+ media_type,
+ )
+ return None
-class AnthropicMultiAgentFormatter(TruncatedFormatterBase):
- """
- Anthropic formatter for multi-agent conversations, where more than
- a user and an agent are involved.
+ return self._format_image_source(source)
+
+ @staticmethod
+ def _format_image_source(
+ source: URLSource | Base64Source,
+ ) -> dict[str, Any]:
+ """Format an image source into Anthropic API format.
+
+ Args:
+ source (`URLSource | Base64Source`):
+ The image source to format.
+
+ Returns:
+ `dict[str, Any]`:
+ The formatted image source.
+ """
+ if isinstance(source, Base64Source):
+ return {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": source.media_type,
+ "data": source.data,
+ },
+ }
+ elif isinstance(source, URLSource):
+ url = source.url
+ if url.startswith("file://"):
+ # Local file - read and convert to base64
+ file_path = url.removeprefix("file://")
+ with open(file_path, "rb") as f:
+ data = base64.b64encode(f.read()).decode("utf-8")
+ return {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": source.media_type,
+ "data": data,
+ },
+ }
+ else:
+ # Remote URL - download and convert to base64
+ response = requests.get(url, timeout=30)
+ response.raise_for_status()
+ data = base64.b64encode(response.content).decode("utf-8")
+ return {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": source.media_type,
+ "data": data,
+ },
+ }
+ else:
+ raise ValueError(f"Unsupported source type: {type(source)}")
+
+
+class AnthropicChatFormatter(_AnthropicFormatterBase):
+ """The Anthropic formatter class for chatbot scenario, where only a user
+ and an agent are involved. We use the `role` field to identify different
+ entities in the conversation.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
+ def __init__(
+ self,
+ supported_input_media_types: list[str] | None = None,
+ ) -> None:
+ """Initialize the Anthropic chat formatter.
+
+ Args:
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types. Defaults to
+ ``["image/*"]``.
+ """
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*"],
+ )
+
+ async def format(
+ self,
+ msgs: list[Msg],
+ ) -> list[dict[str, Any]]:
+ """Format message objects into Anthropic API format.
+
+ Args:
+ msgs (`list[Msg]`):
+ The list of message objects to format.
- support_multiagent: bool = True
- """Whether support multi-agent conversations"""
+ Returns:
+ `list[dict[str, Any]]`:
+ The formatted messages as a list of dictionaries.
+
+ .. note:: Anthropic suggests always passing all previous thinking
+ blocks back to the API in subsequent calls to maintain reasoning
+ continuity. For more details, please refer to
+ `Anthropic's documentation
+ <https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#preserving-thinking-blocks>`_.
+ """
+ return await self._format_messages(msgs)
- support_vision: bool = True
- """Whether support vision data"""
- supported_blocks: list[type] = [
- TextBlock,
- # Multimodal
- ImageBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
+class AnthropicMultiAgentFormatter(_AnthropicFormatterBase):
+ """Anthropic formatter for multi-agent conversations, where more than
+ a user and an agent are involved.
+ """
def __init__(
self,
@@ -152,102 +316,143 @@ class AnthropicMultiAgentFormatter(TruncatedFormatterBase):
"The content between <history></history> tags contains "
"your conversation history\n"
),
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
- """Initialize the DashScope multi-agent formatter.
+ """Initialize the Anthropic multi-agent formatter.
Args:
conversation_history_prompt (`str`):
The prompt to use for the conversation history section.
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types. Defaults to
+ ``["image/*"]``.
"""
- super().__init__(token_counter=token_counter, max_tokens=max_tokens)
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*"],
+ )
self.conversation_history_prompt = conversation_history_prompt
- async def _format_tool_sequence(
- self,
- msgs: list[Msg],
- ) -> list[dict[str, Any]]:
- """Given a sequence of tool call/result messages, format them into
- the required format for the Anthropic API."""
- return await AnthropicChatFormatter().format(msgs)
+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
+ """Format input messages into the structure required by the Anthropic
+ API for multi-agent conversations."""
+ self.assert_list_of_msgs(msgs)
+
+ formatted_msgs = []
+ start_index = 0
+ if len(msgs) > 0 and msgs[0].role == "system":
+ formatted_msgs.append(
+ await self._format_system_message(msgs[0]),
+ )
+ start_index = 1
+
+ is_first_agent_message = True
+ async for typ, group in self._group_messages(msgs[start_index:]):
+ match typ:
+ case "tool_sequence":
+ formatted_msgs.extend(
+ await self._format_messages(group),
+ )
+ case "agent_message":
+ formatted_msgs.extend(
+ await self._format_agent_message(
+ group,
+ is_first_agent_message,
+ ),
+ )
+ is_first_agent_message = False
+
+ return formatted_msgs
async def _format_agent_message(
self,
msgs: list[Msg],
- is_first: bool = True,
+ is_first: bool,
) -> list[dict[str, Any]]:
- """Given a sequence of messages without tool calls/results, format
- them into the required format for the Anthropic API."""
-
- if is_first:
- conversation_history_prompt = self.conversation_history_prompt
- else:
- conversation_history_prompt = ""
-
- # Format into required Anthropic format
- formatted_msgs: list[dict] = []
-
- # Collect the multimodal files
- conversation_blocks: list = []
+ """Format agent messages into conversation history."""
+ conversation_blocks = []
accumulated_text = []
+
for msg in msgs:
- for block in msg.get_content_blocks():
- if block["type"] == "text":
- accumulated_text.append(f"{msg.name}: {block['text']}")
+ agent_name = msg.name or "Agent"
+ agent_text_parts = []
- elif block["type"] == "image":
- # Handle the accumulated text as a single block
- if accumulated_text:
- conversation_blocks.append(
- {
- "text": "\n".join(accumulated_text),
- "type": "text",
- },
- )
- accumulated_text.clear()
+ for block in msg.get_content_blocks():
+ if isinstance(block, TextBlock):
+ agent_text_parts.append(block.text)
+ elif isinstance(block, DataBlock):
+ formatted_block = self._format_anthropic_data_block(block)
+ if formatted_block:
+ if accumulated_text:
+ conversation_blocks.append(
+ {
+ "type": "text",
+ "text": "\n".join(accumulated_text),
+ },
+ )
+ accumulated_text = []
+ conversation_blocks.append(formatted_block)
- conversation_blocks.append({**block})
+ if agent_text_parts:
+ agent_message = f"{agent_name}: {' '.join(agent_text_parts)}"
+ accumulated_text.append(agent_message)
if accumulated_text:
conversation_blocks.append(
{
- "text": "\n".join(accumulated_text),
"type": "text",
+ "text": "\n".join(accumulated_text),
},
)
- if conversation_blocks:
+ if conversation_blocks and is_first:
if conversation_blocks[0].get("text"):
conversation_blocks[0]["text"] = (
- conversation_history_prompt
+ self.conversation_history_prompt
+ "<history>\n"
+ conversation_blocks[0]["text"]
)
-
else:
conversation_blocks.insert(
0,
{
"type": "text",
- "text": conversation_history_prompt + "<history>\n",
+ "text": self.conversation_history_prompt
+ + "<history>\n",
},
)
if conversation_blocks[-1].get("text"):
conversation_blocks[-1]["text"] += "\n</history>"
-
else:
conversation_blocks.append(
{"type": "text", "text": "</history>"},
)
if conversation_blocks:
- formatted_msgs.append(
+ return [
{
"role": "user",
"content": conversation_blocks,
},
- )
+ ]
- return formatted_msgs
+ return []
+
+ @staticmethod
+ async def _format_system_message(msg: Msg) -> dict[str, Any]:
+ """Format a system message."""
+ text_parts = []
+ for block in msg.get_content_blocks():
+ if isinstance(block, TextBlock):
+ text_parts.append(block.text)
+
+ return {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "\n".join(text_parts),
+ },
+ ],
+ }
diff --git a/src/agentscope/formatter/_dashscope_formatter.py b/src/agentscope/formatter/_dashscope_formatter.py
index 766876b99..56913d23e 100644
--- a/src/agentscope/formatter/_dashscope_formatter.py
+++ b/src/agentscope/formatter/_dashscope_formatter.py
@@ -1,150 +1,73 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches
"""The dashscope formatter module."""
-import json
-import os.path
from typing import Any
+from fnmatch import fnmatch
+from abc import ABC
-from ._truncated_formatter_base import TruncatedFormatterBase
+from . import FormatterBase
from .._logging import logger
-from .._utils._common import _is_accessible_local_file
from ..message import (
Msg,
TextBlock,
- ImageBlock,
- AudioBlock,
- VideoBlock,
- ToolUseBlock,
ToolResultBlock,
URLSource,
+ DataBlock,
+ ToolCallBlock,
+ Base64Source,
+ UserMsg,
+ HintBlock,
)
-from ..token import TokenCounterBase
-def _format_dashscope_media_block(
- block: ImageBlock | AudioBlock,
-) -> dict[str, str]:
- """Format an image or audio block for DashScope API.
+class _DashScopeFormatterBase(FormatterBase, ABC):
+ """Base class for DashScope formatters, providing shared data block
+ formatting logic."""
- Args:
- block (`ImageBlock` | `AudioBlock`):
- The image or audio block to format.
-
- Returns:
- `dict[str, str]`:
- A dictionary with "image" or "audio" key and the formatted URL or
- data URI as value.
-
- Raises:
- `NotImplementedError`:
- If the source type is not supported.
- """
- typ = block["type"]
- source = block["source"]
- if source["type"] == "url":
- url = source["url"]
- if _is_accessible_local_file(url):
- return {typ: "file://" + os.path.abspath(url)}
- else:
- # treat as web url
- return {typ: url}
-
- elif source["type"] == "base64":
- media_type = source["media_type"]
- base64_data = source["data"]
- return {
- typ: f"data:{media_type};base64,{base64_data}",
- }
-
- else:
- raise NotImplementedError(
- f"Unsupported source type '{source.get('type')}' "
- f"for {typ} block.",
- )
+ supported_input_media_types: list[str]
+ def _format_dashscope_data_block(
+ self,
+ block: DataBlock,
+ ) -> dict[str, Any] | None:
+ """Format a DataBlock into the required format for DashScope API.
-def _reformat_messages(
- messages: list[dict[str, Any]],
-) -> list[dict[str, Any]]:
- """Reformat the content to be compatible with HuggingFaceTokenCounter.
-
- This function processes a list of messages and converts multi-part
- text content into single string content when all parts are plain text.
- This is necessary for compatibility with HuggingFaceTokenCounter which
- expects simple string content rather than structured content with
- multiple parts.
-
- Args:
- messages (list[dict[str, Any]]):
- A list of message dictionaries where each message may contain a
- "content" field. The content can be either:
- - A string (unchanged)
- - A list of content items, where each item is a dict that may
- contain "text", "type", and other fields
-
- Returns:
- list[dict[str, Any]]:
- A list of reformatted messages. For messages where all content
- items are plain text (have "text" field and either no "type"
- field or "type" == "text"), the content list is converted to a
- single newline-joined string. Other messages remain unchanged.
-
- Example:
- .. code-block:: python
-
- # Case 1: All text content - will be converted
- messages = [
- {
- "role": "user",
- "content": [
- {"text": "Hello", "type": "text"},
- {"text": "World", "type": "text"}
- ]
- }
- ]
- result = _reformat_messages(messages)
- print(result[0]["content"])
- # Output: "Hello\nWorld"
-
- # Case 2: Mixed content - will remain unchanged
- messages = [
- {
- "role": "user",
- "content": [
- {"text": "Hello", "type": "text"},
- {"image_url": "...", "type": "image"}
- ]
- }
- ]
+ Args:
+ block (`DataBlock`):
+ The DataBlock to format.
- result = _reformat_messages(messages) # remain unchanged
- print(type(result[0]["content"]))
- # Output: <class 'list'>
+ Returns:
+ `dict[str, Any] | None`:
+ A dictionary representing the formatted DataBlock for
+ DashScope API.
+ """
+ if not any(
+ fnmatch(block.source.media_type, pattern)
+ for pattern in self.supported_input_media_types
+ ):
+ logger.warning(
+ "Unsupported media type %s for DashScope API. Supported "
+ "types: %s. This block will be skipped.",
+ block.source.media_type,
+ ", ".join(self.supported_input_media_types),
+ )
+ return None
- """
- for message in messages:
- content = message.get("content", [])
+ main_type = block.source.media_type.split("/")[0]
- is_all_text = True
- texts = []
- for item in content:
- if not isinstance(item, dict) or "text" not in item:
- is_all_text = False
- break
- if "type" in item and item["type"] != "text":
- is_all_text = False
- break
- if item["text"]:
- texts.append(item["text"])
+ if isinstance(block.source, URLSource):
+ return {main_type: block.source.url}
- if is_all_text and texts:
- message["content"] = "\n".join(texts)
+ if isinstance(block.source, Base64Source):
+ return {
+ main_type: f"data:{block.source.media_type};"
+ f"base64,{block.source.data}",
+ }
- return messages
+ return None
-class DashScopeChatFormatter(TruncatedFormatterBase):
+class DashScopeChatFormatter(_DashScopeFormatterBase):
"""The DashScope formatter class for chatbot scenario, where only a user
and an agent are involved. We use the `role` field to identify different
entities in the conversation.
@@ -165,71 +88,25 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
To avoid these issues, this formatter assigns content as an empty
list ``[]`` for messages without valid content blocks.
-
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = False
- """Whether support multi-agent conversations"""
-
- support_vision: bool = True
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- ImageBlock,
- AudioBlock,
- VideoBlock,
- ToolUseBlock,
- ToolResultBlock,
- ]
-
def __init__(
self,
- promote_tool_result_images: bool = False,
- promote_tool_result_audios: bool = False,
- promote_tool_result_videos: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_type: list[str] | None = None,
) -> None:
"""Initialize the DashScope chat formatter.
Args:
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- promote_tool_result_audios (`bool`, defaults to `False`):
- Whether to promote audios from tool results to user messages.
- Most LLM APIs don't support audios in tool result blocks, but
- do support them in user message blocks. When `True`, audios are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- promote_tool_result_videos (`bool`, defaults to `False`):
- Whether to promote videos from tool results to user messages.
- Most LLM APIs don't support videos in tool result blocks, but
- do support them in user message blocks. When `True`, videos are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- A token counter instance used to count tokens in the messages.
- If not provided, the formatter will format the messages
- without considering token limits.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If not provided, the formatter will not truncate
- the messages.
+ supported_input_media_type (`list[str] | None`, optional):
+ The list of supported input media types. Defaults to
+ ["image/*", "audio/*", "video/*"].
"""
- super().__init__(token_counter, max_tokens)
- self.promote_tool_result_images = promote_tool_result_images
- self.promote_tool_result_audios = promote_tool_result_audios
- self.promote_tool_result_videos = promote_tool_result_videos
+ super().__init__(
+ supported_input_media_types=supported_input_media_type
+ or ["image/*", "audio/*", "video/*"],
+ )
- async def _format(
+ async def format(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
@@ -246,153 +123,79 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
self.assert_list_of_msgs(msgs)
formatted_msgs: list[dict] = []
-
i = 0
while i < len(msgs):
msg = msgs[i]
- content_blocks: list[dict[str, Any]] = []
+ content_blocks: list[dict] = []
tool_calls = []
for block in msg.get_content_blocks():
- typ = block.get("type")
-
- if typ == "text":
- content_blocks.append(
- {
- "text": block.get("text"),
- },
- )
-
- elif typ in ["image", "audio", "video"]:
- content_blocks.append(
- _format_dashscope_media_block(
- block, # type: ignore[arg-type]
- ),
- )
+ if isinstance(block, TextBlock):
+ content_blocks.append({"text": block.text})
+
+ elif isinstance(block, DataBlock):
+ formatted_block = self._format_dashscope_data_block(block)
+ if formatted_block:
+ content_blocks.append(formatted_block)
+
+ elif isinstance(block, HintBlock):
+ # Insert a new user message with the hint content right
+ # after the current message, and go on processing the
+ # rest of the blocks in the current message
+ if content_blocks or tool_calls:
+ formatted_msgs.append(
+ {
+ "role": "user",
+ "content": content_blocks,
+ "tool_calls": tool_calls
+ if tool_calls
+ else None,
+ },
+ )
+ # Refresh content_blocks and tool_calls for the last
+ # blocks
+ content_blocks = []
+ tool_calls = []
- elif typ == "tool_use":
+ elif isinstance(block, ToolCallBlock):
tool_calls.append(
{
- "id": block.get("id"),
+ "id": block.id,
"type": "function",
"function": {
- "name": block.get("name"),
- "arguments": json.dumps(
- block.get("input", {}),
- ensure_ascii=False,
- ),
+ "name": block.name,
+ "arguments": block.input,
},
},
)
- elif typ == "tool_result":
+ elif isinstance(block, ToolResultBlock):
(
textual_output,
multimodal_data,
- ) = self.convert_tool_result_to_string(block["output"])
+ ) = self.convert_tool_result_to_string(block.output)
# First add the tool result message in DashScope API format
formatted_msgs.append(
{
"role": "tool",
- "tool_call_id": block.get("id"),
+ "tool_call_id": block.id,
"content": textual_output,
- "name": block.get("name"),
+ "name": block.name,
},
)
- # Then, handle the multimodal data if any
- promoted_blocks: list = []
- for url, multimodal_block in multimodal_data:
- if (
- multimodal_block["type"] == "image"
- and self.promote_tool_result_images
- ):
- promoted_blocks.extend(
- [
- TextBlock(
- type="text",
- text=f"\n- The image from '{url}': ",
- ),
- ImageBlock(
- type="image",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- ],
- )
- elif (
- multimodal_block["type"] == "audio"
- and self.promote_tool_result_audios
- ):
- promoted_blocks.extend(
- [
- TextBlock(
- type="text",
- text=f"\n- The audio from '{url}': ",
- ),
- AudioBlock(
- type="audio",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- ],
- )
- elif (
- multimodal_block["type"] == "video"
- and self.promote_tool_result_videos
- ):
- promoted_blocks.extend(
- [
- TextBlock(
- type="text",
- text=f"\n- The video from '{url}': ",
- ),
- VideoBlock(
- type="video",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- ],
- )
-
- if promoted_blocks:
- # Insert promoted blocks as new user message(s)
- promoted_blocks = [
- TextBlock(
- type="text",
- text="<system-info>The following are "
- f"the media contents from the tool "
- f"result of '{block['name']}':",
- ),
- *promoted_blocks,
- TextBlock(
- type="text",
- text="</system-info>",
- ),
- ]
-
+ # If we have multimodal data that needs to be prompted to
+ # the out-tool-result message
+ if multimodal_data:
msgs.insert(
i + 1,
- Msg(
- name="user",
- content=promoted_blocks,
- role="user",
+ UserMsg(
+ name="system-reminder",
+ content=multimodal_data,
),
)
- else:
- logger.warning(
- "Unsupported block type %s in the message, skipped.",
- typ,
- )
-
msg_dashscope = {
"role": msg.role,
"content": content_blocks,
@@ -407,10 +210,26 @@ class DashScopeChatFormatter(TruncatedFormatterBase):
# Move to next message
i += 1
- return _reformat_messages(formatted_msgs)
+ # Merge adjacent text block into one block to avoid API issues
+ cleaned_msgs: list = []
+ for msg in formatted_msgs:
+ new_content = []
+ for block in msg["content"]:
+ if (
+ block.get("text")
+ and new_content
+ and new_content[-1].get("text")
+ ):
+ new_content[-1]["text"] += "\n" + block["text"]
+ else:
+ new_content.append(block)
+ msg["content"] = new_content
+ cleaned_msgs.append(msg)
+
+ return cleaned_msgs
-class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
+class DashScopeMultiAgentFormatter(_DashScopeFormatterBase):
"""DashScope formatter for multi-agent conversations, where more than
a user and an agent are involved.
@@ -429,27 +248,6 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = True
- """Whether support multi-agent conversations"""
-
- support_vision: bool = True
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Multimodal
- ImageBlock,
- AudioBlock,
- VideoBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
-
def __init__(
self,
conversation_history_prompt: str = (
@@ -457,46 +255,67 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
"The content between <history></history> tags contains "
"your conversation history\n"
),
- promote_tool_result_images: bool = False,
- promote_tool_result_audios: bool = False,
- promote_tool_result_videos: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
"""Initialize the DashScope multi-agent formatter.
Args:
conversation_history_prompt (`str`):
The prompt to use for the conversation history section.
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- promote_tool_result_audios (`bool`, defaults to `False`):
- Whether to promote audios from tool results to user messages.
- Most LLM APIs don't support audios in tool result blocks, but
- do support them in user message blocks. When `True`, audios are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- promote_tool_result_videos (`bool`, defaults to `False`):
- Whether to promote videos from tool results to user messages.
- Most LLM APIs don't support videos in tool result blocks, but
- do support them in user message blocks. When `True`, videos are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- The token counter used for truncation.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If `None`, no truncation will be applied.
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types. Defaults to
+ ["image/*", "audio/*", "video/*"].
"""
- super().__init__(token_counter=token_counter, max_tokens=max_tokens)
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*", "audio/*", "video/*"],
+ )
self.conversation_history_prompt = conversation_history_prompt
- self.promote_tool_result_images = promote_tool_result_images
- self.promote_tool_result_audios = promote_tool_result_audios
- self.promote_tool_result_videos = promote_tool_result_videos
+
+ async def format(self, msgs: list[Msg]) -> list[dict]:
+ """Format input messages into the structure required by the DashScope
+ API.
+
+ To support multi-agent conversations, this formatter processes messages
+ as follows:
+
+ - Prepends an instruction before the first conversation history
+ section.
+ - Combines conversation turns into a history section, where each entry
+ is formatted as `{name}: {content}`.
+ - Wraps the conversation history with `<history>` and `</history>`
+ tags.
+
+ Returns:
+ `list[dict[str, Any]]`:
+ A list of dictionaries formatted for the DashScope API.
+ """
+
+ formatted_msgs = []
+ start_index = 0
+ if len(msgs) > 0 and msgs[0].role == "system":
+ formatted_msgs.append(
+ await self._format_system_message(msgs[0]),
+ )
+ start_index = 1
+
+ is_first_agent_message = True
+ async for typ, group in self._group_messages(msgs[start_index:]):
+ match typ:
+ case "tool_sequence":
+ formatted_msgs.extend(
+ await self._format_tool_sequence(group),
+ )
+ case "agent_message":
+ formatted_msgs.extend(
+ await self._format_agent_message(
+ group,
+ is_first_agent_message,
+ ),
+ )
+ is_first_agent_message = False
+
+ return formatted_msgs
async def _format_tool_sequence(
self,
@@ -514,9 +333,7 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
A list of dictionaries formatted for the DashScope API.
"""
return await DashScopeChatFormatter(
- promote_tool_result_images=self.promote_tool_result_images,
- promote_tool_result_audios=self.promote_tool_result_audios,
- promote_tool_result_videos=self.promote_tool_result_videos,
+ supported_input_media_type=self.supported_input_media_types,
).format(msgs)
async def _format_agent_message(
@@ -539,7 +356,6 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
`list[dict[str, Any]]`:
A list of dictionaries formatted for the DashScope API.
"""
-
if is_first:
conversation_history_prompt = self.conversation_history_prompt
else:
@@ -553,10 +369,10 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
accumulated_text = []
for msg in msgs:
for block in msg.get_content_blocks():
- if block["type"] == "text":
+ if isinstance(block, TextBlock):
accumulated_text.append(f"{msg.name}: {block['text']}")
- elif block["type"] in ["image", "audio", "video"]:
+ elif isinstance(block, DataBlock):
# Handle the accumulated text as a single block
if accumulated_text:
conversation_blocks.append(
@@ -564,35 +380,9 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
)
accumulated_text.clear()
- if block["source"]["type"] == "url":
- url = block["source"]["url"]
- if _is_accessible_local_file(url):
- conversation_blocks.append(
- {
- block["type"]: "file://"
- + os.path.abspath(url),
- },
- )
- else:
- conversation_blocks.append({block["type"]: url})
-
- elif block["source"]["type"] == "base64":
- media_type = block["source"]["media_type"]
- base64_data = block["source"]["data"]
- conversation_blocks.append(
- {
- block[
- "type"
- ]: f"data:{media_type};base64,{base64_data}",
- },
- )
-
- else:
- logger.warning(
- "Unsupported block type %s in the message, "
- "skipped.",
- block["type"],
- )
+ formatted_block = self._format_dashscope_data_block(block)
+ if formatted_block is not None:
+ conversation_blocks.append(formatted_block)
if accumulated_text:
conversation_blocks.append({"text": "\n".join(accumulated_text)})
@@ -613,23 +403,32 @@ class DashScopeMultiAgentFormatter(TruncatedFormatterBase):
},
)
- if conversation_blocks[-1].get("text"):
- conversation_blocks[-1]["text"] += "\n</history>"
-
- else:
- conversation_blocks.append({"text": "</history>"})
+ conversation_blocks.append({"text": "</history>"})
+
+ # Merge the adjacent text blocks into one text blocks to avoid API
+ # issues
+ new_content = []
+ for block in conversation_blocks:
+ if (
+ block.get("text")
+ and new_content
+ and new_content[-1].get("text")
+ ):
+ new_content[-1]["text"] += "\n" + block["text"]
+ else:
+ new_content.append(block)
formatted_msgs.append(
{
"role": "user",
- "content": conversation_blocks,
+ "content": new_content,
},
)
- return _reformat_messages(formatted_msgs)
+ return formatted_msgs
+ @staticmethod
async def _format_system_message(
- self,
msg: Msg,
) -> dict[str, Any]:
"""Format system message for DashScope API."""
diff --git a/src/agentscope/formatter/_deepseek_formatter.py b/src/agentscope/formatter/_deepseek_formatter.py
index f9dce3953..9328961ee 100644
--- a/src/agentscope/formatter/_deepseek_formatter.py
+++ b/src/agentscope/formatter/_deepseek_formatter.py
@@ -1,39 +1,34 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches
"""The DeepSeek formatter module."""
-import json
from typing import Any
-from ._truncated_formatter_base import TruncatedFormatterBase
+from ._formatter_base import FormatterBase
from .._logging import logger
-from ..message import Msg, TextBlock, ToolUseBlock, ToolResultBlock
-from ..token import TokenCounterBase
+from ..message import (
+ Msg,
+ TextBlock,
+ ThinkingBlock,
+ HintBlock,
+ ToolCallBlock,
+ ToolResultBlock,
+)
-class DeepSeekChatFormatter(TruncatedFormatterBase):
+class DeepSeekChatFormatter(FormatterBase):
"""The DeepSeek formatter class for chatbot scenario, where only a user
and an agent are involved. We use the `role` field to identify different
entities in the conversation.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = False
- """Whether support multi-agent conversations"""
-
- support_vision: bool = False
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
+ def __init__(
+ self,
+ supported_input_media_types: list[str] | None = None,
+ ) -> None:
+ super().__init__(
+ supported_input_media_types=supported_input_media_types or [],
+ )
- async def _format(
+ async def format(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
@@ -56,60 +51,57 @@ class DeepSeekChatFormatter(TruncatedFormatterBase):
tool_calls = []
for block in msg.get_content_blocks():
- typ = block.get("type")
- if typ == "text":
- content_blocks.append({**block})
- elif typ == "thinking":
- reasoning_content_blocks.append({**block})
+ if isinstance(block, TextBlock):
+ content_blocks.append({"type": "text", "text": block.text})
- elif typ == "tool_use":
+ elif isinstance(block, ThinkingBlock):
+ reasoning_content_blocks.append(block.thinking)
+
+ elif isinstance(block, HintBlock):
+ pass # DeepSeek does not support hint blocks
+
+ elif isinstance(block, ToolCallBlock):
tool_calls.append(
{
- "id": block.get("id"),
+ "id": block.id,
"type": "function",
"function": {
- "name": block.get("name"),
- "arguments": json.dumps(
- block.get("input", {}),
- ensure_ascii=False,
- ),
+ "name": block.name,
+ "arguments": block.input,
},
},
)
- elif typ == "tool_result":
+ elif isinstance(block, ToolResultBlock):
textual_output, _ = self.convert_tool_result_to_string(
- block.get("output"), # type: ignore[arg-type]
+ block.output,
)
messages.append(
{
"role": "tool",
- "tool_call_id": block.get("id"),
+ "tool_call_id": block.id,
"content": textual_output,
- "name": block.get("name"),
+ "name": block.name,
},
)
else:
logger.warning(
"Unsupported block type %s in the message, skipped.",
- typ,
+ type(block),
)
- content_msg = "\n".join(
- content.get("text", "") for content in content_blocks
- )
- reasoning_msg = "\n".join(
- reasoning.get("thinking", "")
- for reasoning in reasoning_content_blocks
- )
- msg_deepseek = {
+ content_msg = "\n".join(b.get("text", "") for b in content_blocks)
+
+ msg_deepseek: dict[str, Any] = {
"role": msg.role,
"content": content_msg or None,
}
- if reasoning_msg:
- msg_deepseek["reasoning_content"] = reasoning_msg
+ if reasoning_content_blocks:
+ msg_deepseek["reasoning_content"] = "\n".join(
+ reasoning_content_blocks,
+ )
if tool_calls:
msg_deepseek["tool_calls"] = tool_calls
@@ -120,29 +112,12 @@ class DeepSeekChatFormatter(TruncatedFormatterBase):
return messages
-class DeepSeekMultiAgentFormatter(TruncatedFormatterBase):
+class DeepSeekMultiAgentFormatter(FormatterBase):
"""
DeepSeek formatter for multi-agent conversations, where more than
a user and an agent are involved.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = True
- """Whether support multi-agent conversations"""
-
- support_vision: bool = False
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
-
def __init__(
self,
conversation_history_prompt: str = (
@@ -150,42 +125,62 @@ class DeepSeekMultiAgentFormatter(TruncatedFormatterBase):
"The content between <history></history> tags contains "
"your conversation history\n"
),
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
"""Initialize the DeepSeek multi-agent formatter.
Args:
conversation_history_prompt (`str`):
The prompt to use for the conversation history section.
- token_counter (`TokenCounterBase | None`, optional):
- A token counter instance used to count tokens in the messages.
- If not provided, the formatter will format the messages
- without considering token limits.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If not provided, the formatter will not truncate
- the messages.
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types. Defaults to ``[]``
+ (DeepSeek does not support multimodal input).
"""
- super().__init__(token_counter=token_counter, max_tokens=max_tokens)
+ super().__init__(
+ supported_input_media_types=supported_input_media_types or [],
+ )
self.conversation_history_prompt = conversation_history_prompt
+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
+ """Format input messages into the structure required by the DeepSeek
+ API for multi-agent conversations."""
+ self.assert_list_of_msgs(msgs)
+
+ formatted_msgs = []
+ start_index = 0
+ if len(msgs) > 0 and msgs[0].role == "system":
+ formatted_msgs.append(
+ await self._format_system_message(msgs[0]),
+ )
+ start_index = 1
+
+ is_first_agent_message = True
+ async for typ, group in self._group_messages(msgs[start_index:]):
+ match typ:
+ case "tool_sequence":
+ formatted_msgs.extend(
+ await self._format_tool_sequence(group),
+ )
+ case "agent_message":
+ formatted_msgs.extend(
+ await self._format_agent_message(
+ group,
+ is_first_agent_message,
+ ),
+ )
+ is_first_agent_message = False
+
+ return formatted_msgs
+
async def _format_tool_sequence(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
"""Given a sequence of tool call/result messages, format them into
- the required format for the DeepSeek API.
-
- Args:
- msgs (`list[Msg]`):
- The list of messages containing tool calls/results to format.
-
- Returns:
- `list[dict[str, Any]]`:
- A list of dictionaries formatted for the DeepSeek API.
- """
- return await DeepSeekChatFormatter().format(msgs)
+ the required format for the DeepSeek API."""
+ return await DeepSeekChatFormatter(
+ supported_input_media_types=self.supported_input_media_types,
+ ).format(msgs)
async def _format_agent_message(
self,
@@ -193,73 +188,46 @@ class DeepSeekMultiAgentFormatter(TruncatedFormatterBase):
is_first: bool = True,
) -> list[dict[str, Any]]:
"""Given a sequence of messages without tool calls/results, format
- them into the required format for the DeepSeek API.
-
- Args:
- msgs (`list[Msg]`):
- A list of Msg objects to be formatted.
- is_first (`bool`, defaults to `True`):
- Whether this is the first agent message in the conversation.
- If `True`, the conversation history prompt will be included.
-
- Returns:
- `list[dict[str, Any]]`:
- A list of dictionaries formatted for the DeepSeek API.
- """
+ them into the required format for the DeepSeek API."""
if is_first:
conversation_history_prompt = self.conversation_history_prompt
else:
conversation_history_prompt = ""
- # Format into required DeepSeek format
formatted_msgs: list[dict] = []
-
- conversation_blocks: list = []
accumulated_text = []
+
for msg in msgs:
for block in msg.get_content_blocks():
- if block["type"] == "text":
- accumulated_text.append(f"{msg.name}: {block['text']}")
+ if isinstance(block, TextBlock):
+ accumulated_text.append(f"{msg.name}: {block.text}")
+ conversation_blocks_text = ""
if accumulated_text:
- conversation_blocks.append(
- {"text": "\n".join(accumulated_text)},
+ conversation_blocks_text = (
+ conversation_history_prompt
+ + "<history>\n"
+ + "\n".join(accumulated_text)
+ + "\n</history>"
)
- if conversation_blocks:
- if conversation_blocks[0].get("text"):
- conversation_blocks[0]["text"] = (
- conversation_history_prompt
- + "<history>\n"
- + conversation_blocks[0]["text"]
- )
-
- else:
- conversation_blocks.insert(
- 0,
- {
- "text": conversation_history_prompt + "<history>\n",
- },
- )
-
- if conversation_blocks[-1].get("text"):
- conversation_blocks[-1]["text"] += "\n</history>"
-
- else:
- conversation_blocks.append({"text": "</history>"})
+ if conversation_blocks_text:
+ formatted_msgs.append(
+ {
+ "role": "user",
+ "content": conversation_blocks_text,
+ },
+ )
- conversation_blocks_text = "\n".join(
- conversation_block.get("text", "")
- for conversation_block in conversation_blocks
- )
+ return formatted_msgs
- user_message = {
- "role": "user",
- "content": conversation_blocks_text,
+ @staticmethod
+ async def _format_system_message(
+ msg: Msg,
+ ) -> dict[str, Any]:
+ """Format system message for DeepSeek API."""
+ return {
+ "role": "system",
+ "content": msg.get_text_content(),
}
-
- if conversation_blocks:
- formatted_msgs.append(user_message)
-
- return formatted_msgs
diff --git a/src/agentscope/formatter/_formatter_base.py b/src/agentscope/formatter/_formatter_base.py
index 6abb05a99..3ac02590e 100644
--- a/src/agentscope/formatter/_formatter_base.py
+++ b/src/agentscope/formatter/_formatter_base.py
@@ -1,16 +1,42 @@
# -*- coding: utf-8 -*-
"""The formatter module."""
-
+import base64
+import mimetypes
+import tempfile
from abc import abstractmethod
-from typing import Any, List, Tuple, Sequence
+from fnmatch import fnmatch
+from typing import Any, List, AsyncGenerator
+
+import shortuuid
-from .._utils._common import _save_base64_data
-from ..message import Msg, AudioBlock, ImageBlock, TextBlock, VideoBlock
+from ..message import (
+ Msg,
+ DataBlock,
+ TextBlock,
+ URLSource,
+ Base64Source,
+)
class FormatterBase:
"""The base class for formatters."""
+ supported_input_media_types: list[str]
+ """The supported media types for multimodal data in the input messages,
+ supporting mime types like "image/*", "image/png", "audio/*", "video/*",
+ etc."""
+
+ def __init__(self, supported_input_media_types: list[str]) -> None:
+ """Initialize the formatter base class.
+
+ Args:
+ supported_input_media_types (`list[str]`):
+ The supported media types for multimodal data in the input
+ messages, supporting mime types like "image/*", "image/png",
+ "audio/*", "video/*", etc.
+ """
+ self.supported_input_media_types = supported_input_media_types
+
@abstractmethod
async def format(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
"""Format the Msg objects to a list of dictionaries that satisfy the
@@ -33,18 +59,10 @@ class FormatterBase:
f"Expected Msg object, got {type(msg)} instead.",
)
- @staticmethod
def convert_tool_result_to_string(
- output: str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock],
- ) -> tuple[
- str,
- Sequence[
- Tuple[
- str,
- ImageBlock | AudioBlock | TextBlock | VideoBlock,
- ]
- ],
- ]:
+ self,
+ output: str | List[TextBlock | DataBlock],
+ ) -> tuple[str, list[TextBlock | DataBlock]]:
"""Turn the tool result list into a textual output to be compatible
with the LLM API that doesn't support multimodal data in the tool
result.
@@ -54,76 +72,139 @@ class FormatterBase:
is included in the returned list.
Args:
- output (`str | List[TextBlock | ImageBlock | AudioBlock | \
- VideoBlock]`):
+ output (`str | List[TextBlock | DataBlock]`):
The output of the tool response, including text and multimodal
data like images and audio.
Returns:
- `tuple[str, list[Tuple[str, ImageBlock | AudioBlock | VideoBlock \
- TextBlock]]]`:
+ `tuple[str, list[TextBlock | DataBlock]]`:
A tuple containing the textual representation of the tool
- result and a list of tuples. The first element of each tuple
- is the local file path or URL of the multimodal data, and the
- second element is the corresponding block.
+ result and a list of blocks to be promoted as a user message.
"""
if isinstance(output, str):
return output, []
textual_output = []
- multimodal_data = []
+ multimodal_data: list = []
+
for block in output:
- assert isinstance(block, dict) and "type" in block, (
- f"Invalid block: {block}, a TextBlock, ImageBlock, "
- f"AudioBlock, or VideoBlock is expected."
- )
- if block["type"] == "text":
- textual_output.append(block["text"])
-
- elif block["type"] in ["image", "audio", "video"]:
- assert "source" in block, (
- f"Invalid {block['type']} block: {block}, 'source' key "
- "is required."
- )
- source = block["source"]
- # Save the image locally and return the file path
- if source["type"] == "url":
- textual_output.append(
- f"The returned {block['type']} can be found "
- f"at: {source['url']}",
- )
+ if isinstance(block, TextBlock):
+ textual_output.append(block.text)
+
+ elif isinstance(block, DataBlock):
+ main_type = block.source.media_type.split("/")[0]
+
+ if any(
+ fnmatch(block.source.media_type, _)
+ for _ in self.supported_input_media_types
+ ):
+ # If supported, promote the block
- path_multimodal_file = source["url"]
+ # Create an identifier for such multimodal data for
+ # accurate reference (in terms of order, position, etc.)
+ identifier = shortuuid.uuid()
- elif source["type"] == "base64":
- path_multimodal_file = _save_base64_data(
- source["media_type"],
- source["data"],
+ textual_output.append(
+ f"<system-reminder>A(n) {main_type} file is returned "
+ f"and will be presented to you with the identifier "
+ f"[{identifier}].</system-reminder>",
+ )
+ multimodal_data.extend(
+ [
+ TextBlock(
+ text=f"- {identifier} ({main_type} file): ",
+ ),
+ block,
+ ],
)
+ multimodal_data.append(block)
+
+ # For unsupported media types, if it's a URL, include it in
+ # the textual output; if it's base64 data, save it locally
+ # and include the file path in the textual output.
+ # Note if you don't want to save the local file, you should
+ # transform the base64 data in the tool execution hook
+ # rather than changing the formatter.
+ elif isinstance(block.source, URLSource):
textual_output.append(
- f"The returned {block['type']} can be found "
- f"at: {path_multimodal_file}",
+ f"<system-reminder>A(n) {main_type} file is returned "
+ f"and can be accessed at the URL: {block.source.url}."
+ f"</system-reminder>",
)
- else:
- raise ValueError(
- f"Invalid image source: {block['source']}, "
- "expected 'url' or 'base64'.",
+ elif isinstance(block.source, Base64Source):
+ # Have to save the base64 data locally
+ extension = mimetypes.guess_extension(
+ block.source.media_type,
)
+ with tempfile.NamedTemporaryFile(
+ suffix=extension,
+ delete=False,
+ ) as temp_file:
+ decoded_data = base64.b64decode(block.source.data)
+ temp_file.write(decoded_data)
+ textual_output.append(
+ f"<system-reminder>A(n) {main_type} file is "
+ f"returned and saved locally at: {temp_file.name}."
+ f"</system-reminder>",
+ )
+
+ # Add system reminder tags if there is multimodal data to be promoted
+ if multimodal_data:
+ multimodal_data = [
+ TextBlock(
+ text="<system-reminder>The multimodal data and their "
+ "identifiers are listed as follows:",
+ ),
+ *multimodal_data,
+ TextBlock(
+ text="</system-reminder>",
+ ),
+ ]
- multimodal_data.append(
- (path_multimodal_file, block),
- )
+ return "\n".join(textual_output), multimodal_data
- else:
- raise ValueError(
- f"Unsupported block type: {block['type']}, "
- "expected 'text', 'image', 'audio', or 'video'.",
- )
+ @staticmethod
+ async def _group_messages(msgs: list[Msg]) -> AsyncGenerator:
+ """Group messages into tool sequences and agent messages.
- if len(textual_output) == 1:
- return textual_output[0], multimodal_data
+ Args:
+ msgs (`list[Msg]`):
+ A list of Msg objects to be grouped.
+ """
+ group_type = None
+ group = []
+ for msg in msgs:
+ if group_type is None:
+ if msg.get_content_blocks(
+ "tool_call",
+ ) or msg.get_content_blocks("tool_result"):
+ group_type = "tool_sequence"
+ else:
+ group_type = "agent_message"
+ group.append(msg)
+ continue
+
+ if group_type == "tool_sequence":
+ if msg.has_content_blocks(
+ "tool_call",
+ ) or msg.has_content_blocks("tool_result"):
+ group.append(msg)
+ else:
+ yield group_type, group
+ group = [msg]
+ group_type = "agent_message"
+
+ elif group_type == "agent_message":
+ if msg.has_content_blocks(
+ "tool_call",
+ ) or msg.has_content_blocks("tool_result"):
+ yield group_type, group
+ group = [msg]
+ group_type = "tool_sequence"
+ else:
+ group.append(msg)
- else:
- return "\n".join("- " + _ for _ in textual_output), multimodal_data
+ if group_type:
+ yield group_type, group
diff --git a/src/agentscope/formatter/_gemini_formatter.py b/src/agentscope/formatter/_gemini_formatter.py
index 81d2dfbd2..d31b2f17c 100644
--- a/src/agentscope/formatter/_gemini_formatter.py
+++ b/src/agentscope/formatter/_gemini_formatter.py
@@ -1,229 +1,192 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches
-"""Google gemini API formatter in agentscope."""
+"""Google Gemini API formatter in agentscope."""
import base64
-import os
+import fnmatch
+from abc import ABC
from typing import Any
-from urllib.parse import urlparse
-from ._truncated_formatter_base import TruncatedFormatterBase
-from .._utils._common import _get_bytes_from_web_url
+import requests
+
+from ._formatter_base import FormatterBase
+from .._logging import logger
from ..message import (
Msg,
TextBlock,
- ImageBlock,
- AudioBlock,
- ToolUseBlock,
+ ThinkingBlock,
+ HintBlock,
+ DataBlock,
+ ToolCallBlock,
ToolResultBlock,
- VideoBlock,
+ UserMsg,
URLSource,
+ Base64Source,
)
-from .._logging import logger
-from ..token import TokenCounterBase
-
-def _format_gemini_media_block(
- media_block: ImageBlock | AudioBlock | VideoBlock,
-) -> dict[str, Any]:
- """Format an image/audio/video block for Gemini API.
- Args:
- media_block (`ImageBlock | AudioBlock | VideoBlock`):
- The media block to format.
+class _GeminiFormatterBase(FormatterBase, ABC):
+ """Base class for Gemini formatters, providing shared data block
+ formatting logic."""
- Returns:
- `dict[str, Any]`:
- A dictionary with "inline_data" key in Gemini format.
+ supported_input_media_types: list[str]
- Raises:
- `ValueError`:
- If the source type is not supported.
- """
- source = media_block["source"]
- if source["type"] == "base64":
- return {
- "inline_data": {
- "data": source["data"],
- "mime_type": source["media_type"],
- },
- }
- elif source["type"] == "url":
- return {
- "inline_data": _to_gemini_inline_data(source["url"]),
- }
- else:
- raise ValueError(
- f"Unsupported source type: {source['type']}",
- )
+ def _format_gemini_data_block(
+ self,
+ block: DataBlock,
+ ) -> dict[str, Any] | None:
+ """Format a DataBlock into Gemini API format.
+ Args:
+ block (`DataBlock`):
+ The data block to format.
-def _to_gemini_inline_data(url: str) -> dict:
- """Convert url into the Gemini API required format."""
- raw_url = url.removeprefix("file://")
- parsed_url = urlparse(raw_url)
- extension = raw_url.split(".")[-1].lower()
-
- # Pre-calculate media type from extension (image/audio/video).
- typ = None
- for k, v in GeminiChatFormatter.supported_extensions.items():
- if extension in v:
- typ = k
- break
-
- if os.path.exists(raw_url):
- # Local file
- if typ is None:
- raise TypeError(
- f"Unsupported file extension: {extension}, expected "
- f"{GeminiChatFormatter.supported_extensions}",
+ Returns:
+ `dict[str, Any] | None`:
+ The formatted data block in Gemini ``inline_data`` format,
+ or None if the media type is not supported.
+ """
+ source = block.source
+ media_type = source.media_type
+
+ # Check if media type is supported
+ if not any(
+ fnmatch.fnmatch(media_type, pattern)
+ for pattern in self.supported_input_media_types
+ ):
+ logger.warning(
+ "Media type %s is not supported, skipped.",
+ media_type,
)
+ return None
- with open(raw_url, "rb") as f:
- data = base64.b64encode(f.read()).decode("utf-8")
+ return self._format_media_source(source)
- return {
- "data": data,
- "mime_type": f"{typ}/{extension}",
- }
-
- elif parsed_url.scheme not in ["", "file"]:
- # Web url
- if typ is None:
- raise TypeError(
- f"Unsupported file extension: {extension}, expected "
- f"{GeminiChatFormatter.supported_extensions}",
- )
+ @staticmethod
+ def _format_media_source(
+ source: URLSource | Base64Source,
+ ) -> dict[str, Any]:
+ """Format a media source into Gemini API ``inline_data`` format.
- data = _get_bytes_from_web_url(url)
- return {
- "data": data,
- "mime_type": f"{typ}/{extension}",
- }
+ Args:
+ source (`URLSource | Base64Source`):
+ The media source to format.
- raise ValueError(
- f"The URL `{url}` is not a valid image URL or local file.",
- )
+ Returns:
+ `dict[str, Any]`:
+ The formatted media source.
+ """
+ if isinstance(source, Base64Source):
+ return {
+ "inline_data": {
+ "data": source.data,
+ "mime_type": source.media_type,
+ },
+ }
+ elif isinstance(source, URLSource):
+ url = source.url
+ if url.startswith("file://"):
+ # Local file - read and convert to base64
+ file_path = url.removeprefix("file://")
+ with open(file_path, "rb") as f:
+ data = base64.b64encode(f.read()).decode("utf-8")
+ return {
+ "inline_data": {
+ "data": data,
+ "mime_type": source.media_type,
+ },
+ }
+ else:
+ # Remote URL - download and convert to base64
+ response = requests.get(url, timeout=30)
+ response.raise_for_status()
+ data = base64.b64encode(response.content).decode("utf-8")
+ return {
+ "inline_data": {
+ "data": data,
+ "mime_type": source.media_type,
+ },
+ }
+ else:
+ raise ValueError(f"Unsupported source type: {type(source)}")
-class GeminiChatFormatter(TruncatedFormatterBase):
+class GeminiChatFormatter(_GeminiFormatterBase):
"""The Gemini formatter class for chatbot scenario, where only a user
and an agent are involved. We use the `role` field to identify different
entities in the conversation.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = False
- """Whether support multi-agent conversations"""
-
- support_vision: bool = True
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Multimodal
- ImageBlock,
- VideoBlock,
- AudioBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
-
- supported_extensions: dict[str, list[str]] = {
- "image": ["png", "jpeg", "webp", "heic", "heif"],
- "video": [
- "mp4",
- "mpeg",
- "mov",
- "avi",
- "x-flv",
- "mpg",
- "webm",
- "wmv",
- "3gpp",
- ],
- "audio": ["mp3", "wav", "aiff", "aac", "ogg", "flac"],
- }
-
def __init__(
self,
- promote_tool_result_images: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
- """Initialize the Gemini chat formatter.
-
- Args:
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- A token counter instance used to count tokens in the messages.
- If not provided, the formatter will format the messages
- without considering token limits.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If not provided, the formatter will not truncate
- the messages.
- """
- super().__init__(token_counter, max_tokens)
- self.promote_tool_result_images = promote_tool_result_images
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*", "audio/*", "video/*"],
+ )
- async def _format(
+ async def format(
self,
msgs: list[Msg],
- ) -> list[dict]:
- """Format message objects into Gemini API required format."""
+ ) -> list[dict[str, Any]]:
+ """Format message objects into Gemini API required format.
+
+ Args:
+ msgs (`list[Msg]`):
+ The list of message objects to format.
+
+ Returns:
+ `list[dict[str, Any]]`:
+ The formatted messages as a list of dictionaries.
+ """
self.assert_list_of_msgs(msgs)
- messages: list = []
+ messages: list[dict] = []
i = 0
while i < len(msgs):
msg = msgs[i]
- parts = []
+ parts: list = []
for block in msg.get_content_blocks():
- typ = block.get("type")
- if typ == "text":
- parts.append(
- {
- "text": block.get("text"),
- },
- )
+ if isinstance(block, TextBlock):
+ parts.append({"text": block.text})
+
+ elif isinstance(block, ThinkingBlock):
+ parts.append({"text": block.thinking})
- elif typ == "tool_use":
+ elif isinstance(block, HintBlock):
+ pass # Gemini does not support hint blocks
+
+ elif isinstance(block, DataBlock):
+ formatted = self._format_gemini_data_block(block)
+ if formatted:
+ parts.append(formatted)
+
+ elif isinstance(block, ToolCallBlock):
parts.append(
{
"function_call": {
"id": None,
- "name": block["name"],
- "args": block["input"],
+ "name": block.name,
+ "args": block.input,
},
- "thought_signature": block.get("id", None),
+ "thought_signature": block.id,
},
)
- elif typ == "tool_result":
+ elif isinstance(block, ToolResultBlock):
(
textual_output,
multimodal_data,
- ) = self.convert_tool_result_to_string(block["output"])
+ ) = self.convert_tool_result_to_string(block.output)
- # First add the tool result message in DashScope API format
messages.append(
{
"role": "user",
"parts": [
{
"function_response": {
- "id": block["id"],
- "name": block["name"],
+ "id": block.id,
+ "name": block.name,
"response": {
"output": textual_output,
},
@@ -233,66 +196,22 @@ class GeminiChatFormatter(TruncatedFormatterBase):
},
)
- promoted_blocks: list = []
- for url, multimodal_block in multimodal_data:
- if (
- multimodal_block["type"] == "image"
- and self.promote_tool_result_images
- ):
- promoted_blocks.extend(
- [
- TextBlock(
- type="text",
- text=f"\n- The image from '{url}': ",
- ),
- ImageBlock(
- type="image",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- ],
- )
-
- if promoted_blocks:
- # Insert promoted blocks as new user message(s)
- promoted_blocks = [
- TextBlock(
- type="text",
- text="<system-info>The following are "
- "the image contents from the tool "
- f"result of '{block['name']}':",
- ),
- *promoted_blocks,
- TextBlock(
- type="text",
- text="</system-info>",
- ),
- ]
-
+ if multimodal_data:
msgs.insert(
i + 1,
- Msg(
- name="user",
- content=promoted_blocks,
- role="user",
+ UserMsg(
+ name="system-reminder",
+ content=multimodal_data,
),
)
- elif typ in ["image", "audio", "video"]:
- parts.append(
- _format_gemini_media_block(
- block, # type: ignore[arg-type]
- ),
- )
-
else:
logger.warning(
- "Unsupported block type: %s in the message, skipped. ",
- typ,
+ "Unsupported block type %s in the message, skipped.",
+ type(block),
)
+ # Gemini uses "model" instead of "assistant"
role = "model" if msg.role == "assistant" else "user"
if parts:
@@ -303,14 +222,12 @@ class GeminiChatFormatter(TruncatedFormatterBase):
},
)
- # Move to next message (including inserted messages, which will
- # be processed in subsequent iterations)
i += 1
return messages
-class GeminiMultiAgentFormatter(TruncatedFormatterBase):
+class GeminiMultiAgentFormatter(_GeminiFormatterBase):
"""The multi-agent formatter for Google Gemini API, where more than a
user and an agent are involved.
@@ -329,27 +246,6 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = True
- """Whether support multi-agent conversations"""
-
- support_vision: bool = True
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Multimodal
- ImageBlock,
- VideoBlock,
- AudioBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
-
def __init__(
self,
conversation_history_prompt: str = (
@@ -357,62 +253,62 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):
"The content between <history></history> tags contains "
"your conversation history\n"
),
- promote_tool_result_images: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
"""Initialize the Gemini multi-agent formatter.
Args:
conversation_history_prompt (`str`):
The prompt to be used for the conversation history section.
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- The token counter used for truncation.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If `None`, no truncation will be applied.
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types. Defaults to
+ ``["image/*", "audio/*", "video/*"]``.
"""
- super().__init__(token_counter=token_counter, max_tokens=max_tokens)
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*", "audio/*", "video/*"],
+ )
self.conversation_history_prompt = conversation_history_prompt
- self.promote_tool_result_images = promote_tool_result_images
- async def _format_system_message(
- self,
- msg: Msg,
- ) -> dict[str, Any]:
- """Format system message for the Gemini API."""
- return {
- "role": "user",
- "parts": [
- {
- "text": msg.get_text_content(),
- },
- ],
- }
+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
+ """Format input messages into the structure required by the Gemini
+ API for multi-agent conversations."""
+ self.assert_list_of_msgs(msgs)
+
+ formatted_msgs = []
+ start_index = 0
+ if len(msgs) > 0 and msgs[0].role == "system":
+ formatted_msgs.append(
+ await self._format_system_message(msgs[0]),
+ )
+ start_index = 1
+
+ is_first_agent_message = True
+ async for typ, group in self._group_messages(msgs[start_index:]):
+ match typ:
+ case "tool_sequence":
+ formatted_msgs.extend(
+ await self._format_tool_sequence(group),
+ )
+ case "agent_message":
+ formatted_msgs.extend(
+ await self._format_agent_message(
+ group,
+ is_first_agent_message,
+ ),
+ )
+ is_first_agent_message = False
+
+ return formatted_msgs
async def _format_tool_sequence(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
"""Given a sequence of tool call/result messages, format them into
- the required format for the Gemini API.
-
- Args:
- msgs (`list[Msg]`):
- The list of messages containing tool calls/results to format.
-
- Returns:
- `list[dict[str, Any]]`:
- A list of dictionaries formatted for the Gemini API.
- """
+ the required format for the Gemini API."""
return await GeminiChatFormatter(
- promote_tool_result_images=self.promote_tool_result_images,
+ supported_input_media_types=self.supported_input_media_types,
).format(msgs)
async def _format_agent_message(
@@ -421,58 +317,37 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):
is_first: bool = True,
) -> list[dict[str, Any]]:
"""Given a sequence of messages without tool calls/results, format
- them into the required format for the Gemini API.
-
- Args:
- msgs (`list[Msg]`):
- A list of Msg objects to be formatted.
- is_first (`bool`, defaults to `True`):
- Whether this is the first agent message in the conversation.
- If `True`, the conversation history prompt will be included.
-
- Returns:
- `list[dict[str, Any]]`:
- A list of dictionaries formatted for the Gemini API.
- """
+ them into the required format for the Gemini API."""
if is_first:
conversation_history_prompt = self.conversation_history_prompt
else:
conversation_history_prompt = ""
- # Format into Gemini API required format
- formatted_msgs: list = []
+ formatted_msgs: list[dict] = []
+ conversation_parts: list[dict] = []
+ accumulated_text: list[str] = []
- # Collect the multimodal files
- conversation_parts: list = []
- accumulated_text = []
for msg in msgs:
for block in msg.get_content_blocks():
- if block["type"] == "text":
- accumulated_text.append(f"{msg.name}: {block['text']}")
+ if isinstance(block, TextBlock):
+ accumulated_text.append(f"{msg.name}: {block.text}")
- elif block["type"] in ["image", "video", "audio"]:
- # handle the accumulated text as a single part if exists
+ elif isinstance(block, DataBlock):
+ # Flush accumulated text first
if accumulated_text:
conversation_parts.append(
- {
- "text": "\n".join(accumulated_text),
- },
+ {"text": "\n".join(accumulated_text)},
)
- accumulated_text.clear()
+ accumulated_text = []
- # handle the multimodal data
- conversation_parts.append(
- _format_gemini_media_block(
- block, # type: ignore[arg-type]
- ),
- )
+ formatted = self._format_gemini_data_block(block)
+ if formatted:
+ conversation_parts.append(formatted)
if accumulated_text:
conversation_parts.append(
- {
- "text": "\n".join(accumulated_text),
- },
+ {"text": "\n".join(accumulated_text)},
)
# Add prompt and <history></history> tags around conversation history
@@ -480,23 +355,19 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):
if conversation_parts[0].get("text"):
conversation_parts[0]["text"] = (
conversation_history_prompt
- + "<history>"
+ + "<history>\n"
+ conversation_parts[0]["text"]
)
-
else:
conversation_parts.insert(
0,
- {"text": conversation_history_prompt + "<history>"},
+ {"text": conversation_history_prompt + "<history>\n"},
)
if conversation_parts[-1].get("text"):
conversation_parts[-1]["text"] += "\n</history>"
-
else:
- conversation_parts.append(
- {"text": "</history>"},
- )
+ conversation_parts.append({"text": "</history>"})
formatted_msgs.append(
{
@@ -506,3 +377,15 @@ class GeminiMultiAgentFormatter(TruncatedFormatterBase):
)
return formatted_msgs
+
+ @staticmethod
+ async def _format_system_message(msg: Msg) -> dict[str, Any]:
+ """Format system message for the Gemini API."""
+ return {
+ "role": "user",
+ "parts": [
+ {
+ "text": msg.get_text_content(),
+ },
+ ],
+ }
diff --git a/src/agentscope/formatter/_ollama_formatter.py b/src/agentscope/formatter/_ollama_formatter.py
index 1b0c31502..43f51196d 100644
--- a/src/agentscope/formatter/_ollama_formatter.py
+++ b/src/agentscope/formatter/_ollama_formatter.py
@@ -1,128 +1,119 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches
"""The Ollama formatter module."""
import base64
-import os
+import fnmatch
+from abc import ABC
from typing import Any
-from urllib.parse import urlparse
-from ._truncated_formatter_base import TruncatedFormatterBase
+import requests
+
+from ._formatter_base import FormatterBase
from .._logging import logger
-from .._utils._common import _get_bytes_from_web_url
from ..message import (
Msg,
TextBlock,
- ImageBlock,
- ToolUseBlock,
+ HintBlock,
+ DataBlock,
+ ToolCallBlock,
ToolResultBlock,
URLSource,
+ Base64Source,
)
-from ..token import TokenCounterBase
-def _format_ollama_image_block(
- image_block: ImageBlock,
-) -> str:
- """Format an image block for Ollama API.
+class _OllamaFormatterBase(FormatterBase, ABC):
+ """Base class for Ollama formatters, providing shared data block
+ formatting logic."""
- Args:
- image_block (`ImageBlock`):
- The image block to format.
+ supported_input_media_types: list[str]
- Returns:
- `str`:
- Base64 encoded image data as a string.
+ def _format_ollama_data_block(
+ self,
+ block: DataBlock,
+ ) -> str | None:
+ """Format a DataBlock into Ollama API format (base64 string).
- Raises:
- `ValueError`:
- If the source type is not supported.
- """
- source = image_block["source"]
- if source["type"] == "url":
- return _convert_ollama_image_url_to_base64_data(source["url"])
- elif source["type"] == "base64":
- return source["data"]
- else:
- raise ValueError(
- f"Unsupported image source type: {source['type']}",
- )
+ Args:
+ block (`DataBlock`):
+ The data block to format.
+ Returns:
+ `str | None`:
+ Base64 encoded data as a string, or None if the media type
+ is not supported.
+ """
+ source = block.source
+ media_type = source.media_type
+
+ # Check if media type is supported
+ if not any(
+ fnmatch.fnmatch(media_type, pattern)
+ for pattern in self.supported_input_media_types
+ ):
+ logger.warning(
+ "Media type %s is not supported, skipped.",
+ media_type,
+ )
+ return None
-def _convert_ollama_image_url_to_base64_data(url: str) -> str:
- """Convert image url to base64."""
- raw_url = url.removeprefix("file://")
- parsed_url = urlparse(raw_url)
+ # Ollama only supports images
+ if not media_type.startswith("image/"):
+ logger.warning(
+ "Ollama only supports image data, got %s, skipped.",
+ media_type,
+ )
+ return None
- if os.path.exists(raw_url):
- # Local file
- with open(raw_url, "rb") as f:
- data = base64.b64encode(f.read()).decode("utf-8")
+ return self._format_image_source(source)
- return data
+ @staticmethod
+ def _format_image_source(source: URLSource | Base64Source) -> str:
+ """Format an image source into Ollama API format (base64 string).
- if parsed_url.scheme not in ["", "file"]:
- # Web url
- data = _get_bytes_from_web_url(url)
- return data
+ Args:
+ source (`URLSource | Base64Source`):
+ The image source to format.
- raise ValueError(
- f"The URL `{url}` is not a valid image URL or local file.",
- )
+ Returns:
+ `str`:
+ Base64 encoded image data.
+ """
+ if isinstance(source, Base64Source):
+ return source.data
+ elif isinstance(source, URLSource):
+ url = source.url
+ if url.startswith("file://"):
+ # Local file - read and convert to base64
+ file_path = url.removeprefix("file://")
+ with open(file_path, "rb") as f:
+ data = base64.b64encode(f.read()).decode("utf-8")
+ return data
+ else:
+ # Remote URL - download and convert to base64
+ response = requests.get(url, timeout=30)
+ response.raise_for_status()
+ data = base64.b64encode(response.content).decode("utf-8")
+ return data
+ else:
+ raise ValueError(f"Unsupported source type: {type(source)}")
-class OllamaChatFormatter(TruncatedFormatterBase):
+class OllamaChatFormatter(_OllamaFormatterBase):
"""The Ollama formatter class for chatbot scenario, where only a user
and an agent are involved. We use the `role` field to identify different
participants in the conversation.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = False
- """Whether support multi-agent conversations"""
-
- support_vision: bool = True
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Multimodal
- ImageBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
-
def __init__(
self,
- promote_tool_result_images: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
- """Initialize the Ollama chat formatter.
-
- Args:
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- A token counter instance used to count tokens in the messages.
- If not provided, the formatter will format the messages
- without considering token limits.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If not provided, the formatter will not truncate
- the messages.
- """
- super().__init__(token_counter, max_tokens)
- self.promote_tool_result_images = promote_tool_result_images
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*"],
+ )
- async def _format(
+ async def format(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
@@ -138,158 +129,105 @@ class OllamaChatFormatter(TruncatedFormatterBase):
"""
self.assert_list_of_msgs(msgs)
- messages: list = []
- i = 0
- while i < len(msgs):
- msg = msgs[i]
- content_blocks: list = []
- tool_calls = []
+ messages: list[dict] = []
+ for msg in msgs:
+ content_parts = []
images = []
for block in msg.get_content_blocks():
- typ = block.get("type")
- if typ == "text":
- content_blocks.append({**block})
+ if isinstance(block, TextBlock):
+ content_parts.append(block.text)
- elif typ == "tool_use":
- tool_calls.append(
- {
- "id": block.get("id"),
- "type": "function",
- "function": {
- "name": block.get("name"),
- "arguments": block.get("input", {}),
- },
- },
- )
+ elif isinstance(block, HintBlock):
+ pass # Ollama does not support hint blocks
- elif typ == "tool_result":
- (
- textual_output,
- multimodal_data,
- ) = self.convert_tool_result_to_string(block["output"])
+ elif isinstance(block, DataBlock):
+ formatted_image = self._format_ollama_data_block(block)
+ if formatted_image:
+ images.append(formatted_image)
+ elif isinstance(block, ToolCallBlock):
messages.append(
{
- "role": "tool",
- "tool_call_id": block.get("id"),
- "content": textual_output,
- "name": block.get("name"),
+ "role": msg.role,
+ "content": "\n".join(content_parts)
+ if content_parts
+ else "",
+ "tool_calls": [
+ {
+ "function": {
+ "name": block.name,
+ "arguments": block.input,
+ },
+ },
+ ],
},
)
+ content_parts = []
+ images = []
- # Then, handle the multimodal data if any
- promoted_blocks: list = []
- for url, multimodal_block in multimodal_data:
- if (
- multimodal_block["type"] == "image"
- and self.promote_tool_result_images
- ):
- promoted_blocks.extend(
- [
- TextBlock(
- type="text",
- text=f"\n- The image from '{url}': ",
- ),
- ImageBlock(
- type="image",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- ],
- )
-
- if promoted_blocks:
- # Insert promoted blocks as new user message(s)
- promoted_blocks = [
- TextBlock(
- type="text",
- text="<system-info>The following are "
- "the image contents from the tool "
- f"result of '{block['name']}':",
- ),
- *promoted_blocks,
- TextBlock(
- type="text",
- text="</system-info>",
- ),
- ]
-
- msgs.insert(
- i + 1,
- Msg(
- name="user",
- content=promoted_blocks,
- role="user",
- ),
- )
-
- elif typ == "image":
- images.append(
- _format_ollama_image_block(
- block, # type: ignore[arg-type]
- ),
- )
+ elif isinstance(block, ToolResultBlock):
+ (
+ textual_output,
+ multimodal_data,
+ ) = self.convert_tool_result_to_string(block.output)
+
+ # If there's multimodal data, insert a user message
+ if multimodal_data:
+ user_images = []
+ user_content_parts = []
+ for data_block in multimodal_data:
+ if isinstance(data_block, DataBlock):
+ formatted_image = (
+ self._format_ollama_data_block(
+ data_block,
+ )
+ )
+ if formatted_image:
+ user_images.append(formatted_image)
+ elif isinstance(data_block, TextBlock):
+ user_content_parts.append(data_block.text)
+
+ user_msg = {
+ "role": "user",
+ "content": "\n".join(user_content_parts)
+ if user_content_parts
+ else textual_output,
+ }
+ if user_images:
+ user_msg["images"] = user_images
+ messages.append(user_msg)
+ else:
+ # No multimodal data, just add text to content
+ content_parts.append(textual_output)
else:
logger.warning(
"Unsupported block type %s in the message, skipped.",
- typ,
+ type(block),
)
- content_msg = "\n".join(
- content.get("text", "") for content in content_blocks
- )
- msg_ollama = {
- "role": msg.role,
- "content": content_msg or None,
- }
-
- if tool_calls:
- msg_ollama["tool_calls"] = tool_calls
-
- if images:
- msg_ollama["images"] = images
-
- if (
- msg_ollama["content"]
- or msg_ollama.get("images")
- or msg_ollama.get("tool_calls")
- ):
- messages.append(msg_ollama)
- # Move to next message
- i += 1
+ # Add the message if there's content or images
+ if content_parts or images:
+ msg_ollama: dict[str, Any] = {
+ "role": msg.role,
+ "content": "\n".join(content_parts)
+ if content_parts
+ else "",
+ }
+ if images:
+ msg_ollama["images"] = images
+ messages.append(msg_ollama)
return messages
-class OllamaMultiAgentFormatter(TruncatedFormatterBase):
+class OllamaMultiAgentFormatter(_OllamaFormatterBase):
"""
Ollama formatter for multi-agent conversations, where more than
a user and an agent are involved.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = True
- """Whether support multi-agent conversations"""
-
- support_vision: bool = True
- """Whether support vision data"""
-
- supported_blocks: list[type] = [
- TextBlock,
- # Multimodal
- ImageBlock,
- # Tool use
- ToolUseBlock,
- ToolResultBlock,
- ]
- """The list of supported message blocks"""
-
def __init__(
self,
conversation_history_prompt: str = (
@@ -297,117 +235,105 @@ class OllamaMultiAgentFormatter(TruncatedFormatterBase):
"The content between <history></history> tags contains "
"your conversation history\n"
),
- promote_tool_result_images: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
"""Initialize the Ollama multi-agent formatter.
Args:
conversation_history_prompt (`str`):
The prompt to use for the conversation history section.
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- The token counter used for truncation.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If `None`, no truncation will be applied.
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types. Defaults to
+ ``["image/*"]``.
"""
- super().__init__(token_counter=token_counter, max_tokens=max_tokens)
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*"],
+ )
self.conversation_history_prompt = conversation_history_prompt
- self.promote_tool_result_images = promote_tool_result_images
- async def _format_system_message(
- self,
- msg: Msg,
- ) -> dict[str, Any]:
- """Format system message for the Ollama API."""
- return {
- "role": "system",
- "content": msg.get_text_content(),
- }
+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
+ """Format input messages into the structure required by the Ollama
+ API for multi-agent conversations."""
+ self.assert_list_of_msgs(msgs)
+
+ formatted_msgs = []
+ start_index = 0
+ if len(msgs) > 0 and msgs[0].role == "system":
+ formatted_msgs.append(
+ await self._format_system_message(msgs[0]),
+ )
+ start_index = 1
+
+ is_first_agent_message = True
+ async for typ, group in self._group_messages(msgs[start_index:]):
+ match typ:
+ case "tool_sequence":
+ formatted_msgs.extend(
+ await self._format_tool_sequence(group),
+ )
+ case "agent_message":
+ formatted_msgs.extend(
+ await self._format_agent_message(
+ group,
+ is_first_agent_message,
+ ),
+ )
+ is_first_agent_message = False
+
+ return formatted_msgs
async def _format_tool_sequence(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
- """Given a sequence of tool call/result messages, format them into
- the required format for the Ollama API.
-
- Args:
- msgs (`list[Msg]`):
- The list of messages containing tool calls/results to format.
-
- Returns:
- `list[dict[str, Any]]`:
- A list of dictionaries formatted for the Ollama API.
- """
+ """Format a sequence of tool-related messages."""
return await OllamaChatFormatter(
- promote_tool_result_images=self.promote_tool_result_images,
+ supported_input_media_types=self.supported_input_media_types,
).format(msgs)
async def _format_agent_message(
self,
msgs: list[Msg],
- is_first: bool = True,
+ is_first: bool,
) -> list[dict[str, Any]]:
- """Given a sequence of messages without tool calls/results, format
- them into the required format for the Ollama API.
-
- Args:
- msgs (`list[Msg]`):
- A list of Msg objects to be formatted.
- is_first (`bool`, defaults to `True`):
- Whether this is the first agent message in the conversation.
- If `True`, the conversation history prompt will be included.
-
- Returns:
- `list[dict[str, Any]]`:
- A list of dictionaries formatted for the ollama API.
- """
+ """Format agent messages into conversation history format."""
+ conversation_blocks: list[dict] = []
+ accumulated_text: list[str] = []
+ images: list[str] = []
- if is_first:
- conversation_history_prompt = self.conversation_history_prompt
- else:
- conversation_history_prompt = ""
-
- # Format into required Ollama format
- formatted_msgs: list[dict] = []
-
- # Collect the multimodal files
- conversation_blocks: list = []
- accumulated_text = []
- images = []
for msg in msgs:
- for block in msg.get_content_blocks():
- if block["type"] == "text":
- accumulated_text.append(f"{msg.name}: {block['text']}")
+ msg_text_parts = []
+ if msg.name:
+ msg_text_parts.append(f"{msg.name}:")
- elif block["type"] == "image":
- # Handle the accumulated text as a single block
- if accumulated_text:
- conversation_blocks.append(
- {"text": "\n".join(accumulated_text)},
- )
- accumulated_text.clear()
+ for block in msg.get_content_blocks():
+ if isinstance(block, TextBlock):
+ msg_text_parts.append(block.text)
+ elif isinstance(block, DataBlock):
+ formatted_image = self._format_ollama_data_block(block)
+ if formatted_image:
+ images.append(formatted_image)
+ elif isinstance(block, HintBlock):
+ pass # Ollama does not support hint blocks
+ else:
+ logger.warning(
+ "Unsupported block type %s in agent message, skipped.",
+ type(block),
+ )
- images.append(_format_ollama_image_block(block))
- conversation_blocks.append({**block})
+ if msg_text_parts:
+ accumulated_text.append("\n".join(msg_text_parts))
if accumulated_text:
conversation_blocks.append(
{"text": "\n".join(accumulated_text)},
)
- if conversation_blocks:
+ if conversation_blocks and is_first:
if conversation_blocks[0].get("text"):
conversation_blocks[0]["text"] = (
- conversation_history_prompt
+ self.conversation_history_prompt
+ "<history>\n"
+ conversation_blocks[0]["text"]
)
@@ -416,7 +342,8 @@ class OllamaMultiAgentFormatter(TruncatedFormatterBase):
conversation_blocks.insert(
0,
{
- "text": conversation_history_prompt + "<history>\n",
+ "text": self.conversation_history_prompt
+ + "<history>\n",
},
)
@@ -431,13 +358,32 @@ class OllamaMultiAgentFormatter(TruncatedFormatterBase):
for conversation_block in conversation_blocks
)
- user_message = {
+ user_message: dict[str, Any] = {
"role": "user",
"content": conversation_blocks_text,
}
if images:
user_message["images"] = images
+
+ formatted_msgs = []
if conversation_blocks:
formatted_msgs.append(user_message)
return formatted_msgs
+
+ @staticmethod
+ async def _format_system_message(msg: Msg) -> dict[str, Any]:
+ """Format a system message."""
+ text_parts = []
+ for block in msg.get_content_blocks():
+ if isinstance(block, TextBlock):
+ text_parts.append(block.text)
+ else:
+ logger.warning(
+ "Unsupported block type %s in system message, skipped.",
+ type(block),
+ )
+ return {
+ "role": "system",
+ "content": "\n".join(text_parts),
+ }
diff --git a/src/agentscope/formatter/_openai_formatter.py b/src/agentscope/formatter/_openai_formatter.py
index fef61e60a..490023cab 100644
--- a/src/agentscope/formatter/_openai_formatter.py
+++ b/src/agentscope/formatter/_openai_formatter.py
@@ -1,222 +1,220 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches, too-many-nested-blocks
"""The OpenAI formatter for agentscope."""
import base64
-import json
-import os
+from abc import ABC
+from fnmatch import fnmatch
from typing import Any
from urllib.parse import urlparse
import requests
-from ._truncated_formatter_base import TruncatedFormatterBase
+from . import FormatterBase
from .._logging import logger
from ..message import (
Msg,
URLSource,
TextBlock,
- ImageBlock,
- AudioBlock,
+ DataBlock,
Base64Source,
- ToolUseBlock,
+ ToolCallBlock,
ToolResultBlock,
+ HintBlock,
+ UserMsg,
)
-from ..token import TokenCounterBase
-def _format_openai_image_block(
- image_block: ImageBlock,
-) -> dict[str, Any]:
- """Format an image block for OpenAI API.
+class _OpenAIFormatterBase(FormatterBase, ABC):
+ """Base class for OpenAI formatters, providing shared data block
+ formatting logic."""
- Args:
- image_block (`ImageBlock`):
- The image block to format.
+ supported_input_media_types: list[str]
- Returns:
- `dict[str, Any]`:
- A dictionary with "type" and "image_url" keys in OpenAI format.
+ def _format_openai_data_block(
+ self,
+ block: DataBlock,
+ role: str = "user",
+ ) -> dict[str, Any] | None:
+ """Format a DataBlock into the required format for OpenAI API.
- Raises:
- `ValueError`:
- If the source type is not supported.
- """
- source = image_block["source"]
- if source["type"] == "url":
- url = _to_openai_image_url(source["url"])
- elif source["type"] == "base64":
- data = source["data"]
- media_type = source["media_type"]
- url = f"data:{media_type};base64,{data}"
- else:
- raise ValueError(
- f"Unsupported image source type: {source['type']}",
- )
+ For image blocks, URLs are returned as-is (or converted to base64 for
+ local ``file://`` paths). For audio blocks, data is always converted
+ to base64 as required by the OpenAI input_audio format.
- return {
- "type": "image_url",
- "image_url": {
- "url": url,
- },
- }
+ Args:
+ block (`DataBlock`):
+ The DataBlock to format.
+ role (`str`, defaults to ``"user"``):
+ The role of the message that contains this block. Audio blocks
+ in assistant messages are skipped to avoid errors in subsequent
+ model calls.
+ Returns:
+ `dict[str, Any] | None`:
+ A dictionary in OpenAI API format, or ``None`` if the block
+ should be skipped.
+ """
+ if not any(
+ fnmatch(block.source.media_type, pattern)
+ for pattern in self.supported_input_media_types
+ ):
+ logger.warning(
+ "Unsupported media type %s for OpenAI API. "
+ "Supported types: %s. This block will be skipped.",
+ block.source.media_type,
+ ", ".join(self.supported_input_media_types),
+ )
+ return None
-def _to_openai_image_url(url: str) -> str:
- """Convert an image url to openai format. If the given url is a local
- file, it will be converted to base64 format. Otherwise, it will be
- returned directly.
+ main_type = block.source.media_type.split("/")[0]
- Args:
- url (`str`):
- The local or public url of the image.
- """
- import filetype
-
- # See https://platform.openai.com/docs/guides/vision for details of
- # support image extensions.
- support_image_extensions = (
- ".png",
- ".jpg",
- ".jpeg",
- ".gif",
- ".webp",
- )
-
- raw_url = url.removeprefix("file://")
- # For local files
- if os.path.exists(raw_url) and os.path.isfile(raw_url):
- if any(raw_url.endswith(_) for _ in support_image_extensions):
- with open(raw_url, "rb") as image_file:
- base64_image = base64.b64encode(image_file.read()).decode(
- "utf-8",
- )
- extension = raw_url.lower().split(".")[-1]
- mime_type = f"image/{extension}"
- return f"data:{mime_type};base64,{base64_image}"
-
- # No extension - detect file type using filetype
- kind = filetype.guess(raw_url)
- if kind is not None and kind.mime.startswith("image/"):
- with open(raw_url, "rb") as image_file:
- base64_image = base64.b64encode(image_file.read()).decode(
- "utf-8",
- )
- return f"data:{kind.mime};base64,{base64_image}"
-
- # For web urls
- parsed_url = urlparse(raw_url)
- if parsed_url.scheme not in ["", "file"]:
- return url
-
- raise ValueError(
- f'Invalid image URL: "{url}". It should be a local file or a web URL.',
- )
-
-
-def _to_openai_audio_data(source: URLSource | Base64Source) -> dict:
- """Covert an audio source to OpenAI format."""
- if source["type"] == "url":
- raw_url = source["url"].removeprefix("file://")
- extension = raw_url.split(".")[-1].lower()
- if extension not in ["wav", "mp3"]:
- raise TypeError(
- f"Unsupported audio file extension: {extension}, "
- "wav and mp3 are supported.",
- )
+ if main_type == "image":
+ return self._format_image_source(block.source)
+
+ if main_type == "audio":
+ # Filter out audio content when the multimodal model outputs both
+ # text and audio, to prevent errors in subsequent model calls
+ if role == "assistant":
+ return None
+ return self._format_audio_source(block.source)
+
+ logger.warning(
+ "Unsupported main media type %s for OpenAI API. "
+ "This block will be skipped.",
+ main_type,
+ )
+ return None
+
+ @staticmethod
+ def _format_image_source(
+ source: URLSource | Base64Source,
+ ) -> dict[str, Any]:
+ """Convert an image source to OpenAI image_url format.
- parsed_url = urlparse(raw_url)
+ Local ``file://`` URLs are read from disk and converted to base64
+ data URIs. Remote URLs are passed through unchanged.
- if os.path.exists(raw_url):
- with open(raw_url, "rb") as audio_file:
- data = base64.b64encode(audio_file.read()).decode("utf-8")
+ Args:
+ source (`URLSource | Base64Source`):
+ The image source to convert.
- # web url
- elif parsed_url.scheme not in ["", "file"]:
- response = requests.get(source["url"])
- response.raise_for_status()
- data = base64.b64encode(response.content).decode("utf-8")
+ Returns:
+ `dict[str, Any]`:
+ A dictionary with ``"type": "image_url"`` in OpenAI format.
+ """
+ if isinstance(source, Base64Source):
+ url = f"data:{source.media_type};base64,{source.data}"
+
+ elif isinstance(source, URLSource):
+ if source.url.startswith("file://"):
+ # Local file — read and encode as base64 data URI
+ local_path = source.url.removeprefix("file://")
+ with open(local_path, "rb") as f:
+ encoded = base64.b64encode(f.read()).decode("utf-8")
+ url = f"data:{source.media_type};base64,{encoded}"
+ else:
+ # Remote URL — pass through as-is
+ url = source.url
else:
- raise ValueError(
- f"Unsupported audio source: {source['url']}, "
- "it should be a local file or a web URL.",
- )
+ raise ValueError(f"Unsupported image source type: {type(source)}")
return {
- "data": data,
- "format": extension,
+ "type": "image_url",
+ "image_url": {"url": url},
}
- if source["type"] == "base64":
- data = source["data"]
- media_type = source["media_type"]
+ @staticmethod
+ def _format_audio_source(
+ source: URLSource | Base64Source,
+ ) -> dict[str, Any]:
+ """Convert an audio source to OpenAI input_audio format.
- if media_type not in ["audio/wav", "audio/mp3"]:
- raise TypeError(
- f"Unsupported audio media type: {media_type}, "
- "only audio/wav and audio/mp3 are supported.",
- )
+ Local ``file://`` URLs are read from disk. Remote URLs are downloaded.
+ Only ``wav`` and ``mp3`` formats are supported by the OpenAI API.
- return {
- "data": data,
- "format": media_type.split("/")[-1],
- }
+ Args:
+ source (`URLSource | Base64Source`):
+ The audio source to convert.
+
+ Returns:
+ `dict[str, Any]`:
+ A dictionary with ``"type": "input_audio"`` in OpenAI format.
+ """
+ if isinstance(source, Base64Source):
+ media_type = source.media_type
+ if media_type not in ["audio/wav", "audio/mp3"]:
+ raise TypeError(
+ f"Unsupported audio media type: {media_type}, "
+ "only audio/wav and audio/mp3 are supported.",
+ )
+ return {
+ "type": "input_audio",
+ "input_audio": {
+ "data": source.data,
+ "format": media_type.split("/")[-1],
+ },
+ }
- raise TypeError(f"Unsupported audio source: {source['type']}.")
+ if isinstance(source, URLSource):
+ if source.url.startswith("file://"):
+ # Local file
+ local_path = source.url.removeprefix("file://")
+ extension = local_path.rsplit(".", 1)[-1].lower()
+ if extension not in ["wav", "mp3"]:
+ raise TypeError(
+ f"Unsupported audio file extension: {extension}, "
+ "wav and mp3 are supported.",
+ )
+ with open(local_path, "rb") as f:
+ data = base64.b64encode(f.read()).decode("utf-8")
+ else:
+ # Remote URL — download and encode
+ parsed = urlparse(source.url)
+ extension = parsed.path.rsplit(".", 1)[-1].lower()
+ if extension not in ["wav", "mp3"]:
+ raise TypeError(
+ f"Unsupported audio file extension: {extension}, "
+ "wav and mp3 are supported.",
+ )
+ response = requests.get(source.url)
+ response.raise_for_status()
+ data = base64.b64encode(response.content).decode("utf-8")
+
+ return {
+ "type": "input_audio",
+ "input_audio": {
+ "data": data,
+ "format": extension,
+ },
+ }
+ raise TypeError(f"Unsupported audio source type: {type(source)}.")
-class OpenAIChatFormatter(TruncatedFormatterBase):
+
+class OpenAIChatFormatter(_OpenAIFormatterBase):
"""The OpenAI formatter class for chatbot scenario, where only a user
and an agent are involved. We use the `name` field in OpenAI API to
identify different entities in the conversation.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = True
- """Whether support multi-agent conversation"""
-
- support_vision: bool = True
- """Whether support vision models"""
-
- supported_blocks: list[type] = [
- TextBlock,
- ImageBlock,
- AudioBlock,
- ToolUseBlock,
- ToolResultBlock,
- ]
- """Supported message blocks for OpenAI API"""
-
def __init__(
self,
- promote_tool_result_images: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
"""Initialize the OpenAI chat formatter.
Args:
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- A token counter instance used to count tokens in the messages.
- If not provided, the formatter will format the messages
- without considering token limits.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If not provided, the formatter will not truncate
- the messages.
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types, using glob-style
+ patterns (e.g. ``"image/*"``, ``"audio/mp3"``). Defaults to
+ ``["image/*", "audio/*"]``.
"""
- super().__init__(token_counter=token_counter, max_tokens=max_tokens)
- self.promote_tool_result_images = promote_tool_result_images
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*", "audio/*"],
+ )
- async def _format(
+ async def format(
self,
msgs: list[Msg],
) -> list[dict[str, Any]]:
@@ -241,115 +239,73 @@ class OpenAIChatFormatter(TruncatedFormatterBase):
tool_calls = []
for block in msg.get_content_blocks():
- typ = block.get("type")
- if typ == "text":
- content_blocks.append({**block})
+ if isinstance(block, TextBlock):
+ content_blocks.append({"type": "text", "text": block.text})
- elif typ == "tool_use":
+ elif isinstance(block, DataBlock):
+ formatted = self._format_openai_data_block(
+ block,
+ role=msg.role,
+ )
+ if formatted is not None:
+ content_blocks.append(formatted)
+
+ elif isinstance(block, HintBlock):
+ # Insert a new user message with the hint content right
+ # after the current message, and go on processing the
+ # rest of the blocks in the current message
+ if content_blocks or tool_calls:
+ msg_openai = {
+ "role": msg.role,
+ "name": msg.name,
+ "content": content_blocks or None,
+ }
+ if tool_calls:
+ msg_openai["tool_calls"] = tool_calls
+ messages.append(msg_openai)
+ content_blocks = []
+ tool_calls = []
+
+ elif isinstance(block, ToolCallBlock):
tool_calls.append(
{
- "id": block.get("id"),
+ "id": block.id,
"type": "function",
"function": {
- "name": block.get("name"),
- "arguments": json.dumps(
- block.get("input", {}),
- ensure_ascii=False,
- ),
+ "name": block.name,
+ "arguments": block.input,
},
},
)
- elif typ == "tool_result":
+ elif isinstance(block, ToolResultBlock):
(
textual_output,
multimodal_data,
- ) = self.convert_tool_result_to_string(block["output"])
+ ) = self.convert_tool_result_to_string(block.output)
messages.append(
{
"role": "tool",
- "tool_call_id": block.get("id"),
- "content": ( # type: ignore[arg-type]
- textual_output
- ),
- "name": block.get("name"),
+ "tool_call_id": block.id,
+ "content": textual_output,
+ "name": block.name,
},
)
- # Then, handle the multimodal data if any
- promoted_blocks: list = []
- for url, multimodal_block in multimodal_data:
- if (
- multimodal_block["type"] == "image"
- and self.promote_tool_result_images
- ):
- promoted_blocks.extend(
- [
- TextBlock(
- type="text",
- text=f"\n- The image from '{url}': ",
- ),
- ImageBlock(
- type="image",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- ],
- )
-
- if promoted_blocks:
- # Insert promoted blocks as new user message(s)
- promoted_blocks = [
- TextBlock(
- type="text",
- text="<system-info>The following are "
- "the image contents from the tool "
- f"result of '{block['name']}':",
- ),
- *promoted_blocks,
- TextBlock(
- type="text",
- text="</system-info>",
- ),
- ]
-
+ if multimodal_data:
msgs.insert(
i + 1,
- Msg(
- name="user",
- content=promoted_blocks,
- role="user",
+ UserMsg(
+ name="system-reminder",
+ content=multimodal_data,
),
)
- elif typ == "image":
- content_blocks.append(
- _format_openai_image_block(
- block, # type: ignore[arg-type]
- ),
- )
-
- elif typ == "audio":
- # Filter out audio content when the multimodal model
- # outputs both text and audio, to prevent errors in
- # subsequent model calls
- if msg.role == "assistant":
- continue
- input_audio = _to_openai_audio_data(block["source"])
- content_blocks.append(
- {
- "type": "input_audio",
- "input_audio": input_audio,
- },
- )
-
else:
logger.warning(
"Unsupported block type %s in the message, skipped.",
- typ,
+ type(block),
)
msg_openai = {
@@ -371,32 +327,15 @@ class OpenAIChatFormatter(TruncatedFormatterBase):
return messages
-class OpenAIMultiAgentFormatter(TruncatedFormatterBase):
+class OpenAIMultiAgentFormatter(_OpenAIFormatterBase):
"""
OpenAI formatter for multi-agent conversations, where more than
a user and an agent are involved.
+
.. tip:: This formatter is compatible with OpenAI API and
- OpenAI-compatible services like vLLM, Azure OpenAI, and others.
+ OpenAI-compatible services like vLLM, Azure OpenAI, and others.
"""
- support_tools_api: bool = True
- """Whether support tools API"""
-
- support_multiagent: bool = True
- """Whether support multi-agent conversation"""
-
- support_vision: bool = True
- """Whether support vision models"""
-
- supported_blocks: list[type] = [
- TextBlock,
- ImageBlock,
- AudioBlock,
- ToolUseBlock,
- ToolResultBlock,
- ]
- """Supported message blocks for OpenAI API"""
-
def __init__(
self,
conversation_history_prompt: str = (
@@ -404,33 +343,54 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):
"The content between <history></history> tags contains "
"your conversation history\n"
),
- promote_tool_result_images: bool = False,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
+ supported_input_media_types: list[str] | None = None,
) -> None:
"""Initialize the OpenAI multi-agent formatter.
Args:
conversation_history_prompt (`str`):
The prompt to use for the conversation history section.
- promote_tool_result_images (`bool`, defaults to `False`):
- Whether to promote images from tool results to user messages.
- Most LLM APIs don't support images in tool result blocks, but
- do support them in user message blocks. When `True`, images are
- extracted and appended as a separate user message with
- explanatory text indicating their source.
- token_counter (`TokenCounterBase | None`, optional):
- A token counter instance used to count tokens in the messages.
- If not provided, the formatter will format the messages
- without considering token limits.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If not provided, the formatter will not truncate
- the messages.
+ supported_input_media_types (`list[str] | None`, optional):
+ The list of supported input media types, using glob-style
+ patterns (e.g. ``"image/*"``, ``"audio/mp3"``). Defaults to
+ ``["image/*", "audio/*"]``.
"""
- super().__init__(token_counter=token_counter, max_tokens=max_tokens)
+ super().__init__(
+ supported_input_media_types=supported_input_media_types
+ or ["image/*", "audio/*"],
+ )
self.conversation_history_prompt = conversation_history_prompt
- self.promote_tool_result_images = promote_tool_result_images
+
+ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
+ """Format input messages into the structure required by the OpenAI API
+ for multi-agent conversations."""
+ self.assert_list_of_msgs(msgs)
+
+ formatted_msgs = []
+ start_index = 0
+ if len(msgs) > 0 and msgs[0].role == "system":
+ formatted_msgs.append(
+ await self._format_system_message(msgs[0]),
+ )
+ start_index = 1
+
+ is_first_agent_message = True
+ async for typ, group in self._group_messages(msgs[start_index:]):
+ match typ:
+ case "tool_sequence":
+ formatted_msgs.extend(
+ await self._format_tool_sequence(group),
+ )
+ case "agent_message":
+ formatted_msgs.extend(
+ await self._format_agent_message(
+ group,
+ is_first_agent_message,
+ ),
+ )
+ is_first_agent_message = False
+
+ return formatted_msgs
async def _format_tool_sequence(
self,
@@ -439,7 +399,7 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):
"""Given a sequence of tool call/result messages, format them into
the required format for the OpenAI API."""
return await OpenAIChatFormatter(
- promote_tool_result_images=self.promote_tool_result_images,
+ supported_input_media_types=self.supported_input_media_types,
).format(msgs)
async def _format_agent_message(
@@ -455,34 +415,23 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):
else:
conversation_history_prompt = ""
- # Format into required OpenAI format
formatted_msgs: list[dict] = []
-
conversation_blocks: list = []
accumulated_text = []
- images = []
- audios = []
+ media_blocks: list[dict] = []
for msg in msgs:
for block in msg.get_content_blocks():
- if block["type"] == "text":
- accumulated_text.append(f"{msg.name}: {block['text']}")
-
- elif block["type"] == "image":
- images.append(_format_openai_image_block(block))
- elif block["type"] == "audio":
- # Filter out audio content when the multimodal model
- # outputs both text and audio, to prevent errors in
- # subsequent model calls
- if msg.role == "assistant":
- continue
- input_audio = _to_openai_audio_data(block["source"])
- audios.append(
- {
- "type": "input_audio",
- "input_audio": input_audio,
- },
+ if isinstance(block, TextBlock):
+ accumulated_text.append(f"{msg.name}: {block.text}")
+
+ elif isinstance(block, DataBlock):
+ formatted = self._format_openai_data_block(
+ block,
+ role=msg.role,
)
+ if formatted is not None:
+ media_blocks.append(formatted)
if accumulated_text:
conversation_blocks.append(
@@ -496,45 +445,39 @@ class OpenAIMultiAgentFormatter(TruncatedFormatterBase):
+ "<history>\n"
+ conversation_blocks[0]["text"]
)
-
else:
conversation_blocks.insert(
0,
- {
- "text": conversation_history_prompt + "<history>\n",
- },
+ {"text": conversation_history_prompt + "<history>\n"},
)
if conversation_blocks[-1].get("text"):
conversation_blocks[-1]["text"] += "\n</history>"
-
else:
conversation_blocks.append({"text": "</history>"})
conversation_blocks_text = "\n".join(
- conversation_block.get("text", "")
- for conversation_block in conversation_blocks
+ b.get("text", "") for b in conversation_blocks
)
content_list: list[dict[str, Any]] = []
if conversation_blocks_text:
content_list.append(
- {
- "type": "text",
- "text": conversation_blocks_text,
- },
+ {"type": "text", "text": conversation_blocks_text},
)
- if images:
- content_list.extend(images)
- if audios:
- content_list.extend(audios)
-
- user_message = {
- "role": "user",
- "content": content_list,
- }
+ content_list.extend(media_blocks)
if content_list:
- formatted_msgs.append(user_message)
+ formatted_msgs.append({"role": "user", "content": content_list})
return formatted_msgs
+
+ @staticmethod
+ async def _format_system_message(
+ msg: Msg,
+ ) -> dict[str, Any]:
+ """Format system message for OpenAI API."""
+ return {
+ "role": "system",
+ "content": msg.get_text_content(),
+ }
diff --git a/src/agentscope/formatter/_truncated_formatter_base.py b/src/agentscope/formatter/_truncated_formatter_base.py
deleted file mode 100644
index 86d42ad41..000000000
--- a/src/agentscope/formatter/_truncated_formatter_base.py
+++ /dev/null
@@ -1,297 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The truncated formatter base class, which allows to truncate the input
-messages."""
-from abc import ABC
-from copy import deepcopy
-from typing import (
- Any,
- Tuple,
- Literal,
- AsyncGenerator,
-)
-
-from ._formatter_base import FormatterBase
-from ..message import Msg
-from ..token import TokenCounterBase
-from ..tracing import trace_format
-
-
-class TruncatedFormatterBase(FormatterBase, ABC):
- """Base class for truncated formatters, which formats input messages into
- required formats with tokens under a specified limit."""
-
- def __init__(
- self,
- token_counter: TokenCounterBase | None = None,
- max_tokens: int | None = None,
- ) -> None:
- """Initialize the TruncatedFormatterBase.
-
- Args:
- token_counter (`TokenCounterBase | None`, optional):
- A token counter instance used to count tokens in the messages.
- If not provided, the formatter will format the messages
- without considering token limits.
- max_tokens (`int | None`, optional):
- The maximum number of tokens allowed in the formatted
- messages. If not provided, the formatter will not truncate
- the messages.
- """
- self.token_counter = token_counter
-
- assert (
- max_tokens is None or 0 < max_tokens
- ), "max_tokens must be greater than 0"
- self.max_tokens = max_tokens
-
- @trace_format
- async def format(
- self,
- msgs: list[Msg],
- **kwargs: Any,
- ) -> list[dict[str, Any]]:
- """Format the input messages into the required format. If token
- counter and max token limit are provided, the messages will be
- truncated to fit the limit.
-
- Args:
- msgs (`list[Msg]`):
- The input messages to be formatted.
-
- Returns:
- `list[dict[str, Any]]`:
- The formatted messages in the required format.
- """
-
- # Check if the input messages are valid
- self.assert_list_of_msgs(msgs)
-
- msgs = deepcopy(msgs)
-
- while True:
- formatted_msgs = await self._format(msgs)
- n_tokens = await self._count(formatted_msgs)
-
- if (
- n_tokens is None
- or self.max_tokens is None
- or n_tokens <= self.max_tokens
- ):
- return formatted_msgs
-
- # truncate the input messages
- msgs = await self._truncate(msgs)
-
- async def _format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
- """Format the input messages into the required format. This method
- should be implemented by the subclasses."""
-
- formatted_msgs = []
- start_index = 0
- if len(msgs) > 0 and msgs[0].role == "system":
- formatted_msgs.append(
- await self._format_system_message(msgs[0]),
- )
- start_index = 1
-
- is_first_agent_message = True
- async for typ, group in self._group_messages(msgs[start_index:]):
- match typ:
- case "tool_sequence":
- formatted_msgs.extend(
- await self._format_tool_sequence(group),
- )
- case "agent_message":
- formatted_msgs.extend(
- await self._format_agent_message(
- group,
- is_first_agent_message,
- ),
- )
- is_first_agent_message = False
-
- return formatted_msgs
-
- async def _format_system_message(
- self,
- msg: Msg,
- ) -> dict[str, Any]:
- """Format system message for the LLM API.
-
- .. note:: This is the default implementation. For certain LLM APIs
- with specific requirements, you may need to implement a custom
- formatting function to accommodate those particular needs.
- """
- return {
- "role": "system",
- "content": msg.get_content_blocks("text"),
- }
-
- async def _format_tool_sequence(
- self,
- msgs: list[Msg],
- ) -> list[dict[str, Any]]:
- """Given a sequence of tool call/result messages, format them into
- the required format for the LLM API."""
- raise NotImplementedError(
- "_format_tool_sequence is not implemented",
- )
-
- async def _format_agent_message(
- self,
- msgs: list[Msg],
- is_first: bool = True,
- ) -> list[dict[str, Any]]:
- """Given a sequence of messages without tool calls/results, format
- them into the required format for the LLM API."""
- raise NotImplementedError(
- "_format_agent_message is not implemented",
- )
-
- async def _truncate(self, msgs: list[Msg]) -> list[Msg]:
- """Truncate the input messages, so that it can fit the token limit.
- This function is called only when
-
- - both `token_counter` and `max_tokens` are provided,
- - the formatted output of the input messages exceeds the token limit.
-
- .. tip:: This function only provides a simple strategy, and developers
- can override this method to implement more sophisticated
- truncation strategies.
-
- .. note:: The tool call message should be truncated together with
- its corresponding tool result message to satisfy the LLM API
- requirements.
-
- Args:
- msgs (`list[Msg]`):
- The input messages to be truncated.
-
- Raises:
- `ValueError`:
- If the system prompt message already exceeds the token limit,
- or if there are tool calls without corresponding tool results.
-
- Returns:
- `list[Msg]`:
- The truncated messages.
- """
- start_index = 0
- if len(msgs) > 0 and msgs[0].role == "system":
- if len(msgs) == 1:
- # If the system prompt already exceeds the token limit, we
- # raise an error.
- raise ValueError(
- f"The system prompt message already exceeds the token "
- f"limit ({self.max_tokens} tokens).",
- )
-
- start_index = 1
-
- # Create a tool call IDs queues to delete the corresponding tool
- # result message
- tool_call_ids = set()
- for i in range(start_index, len(msgs)):
- msg = msgs[i]
- for block in msg.get_content_blocks("tool_use"):
- tool_call_ids.add(block["id"])
-
- for block in msg.get_content_blocks("tool_result"):
- try:
- tool_call_ids.remove(block["id"])
- except KeyError:
- pass
-
- # We can stop truncating if the queue is empty
- if len(tool_call_ids) == 0:
- return msgs[:start_index] + msgs[i + 1 :]
-
- if len(tool_call_ids) > 0:
- raise ValueError(
- "The input messages contains tool call(s) that do not have "
- f"the corresponding tool result(s): {tool_call_ids}. ",
- )
-
- return msgs[:start_index]
-
- async def _count(self, msgs: list[dict[str, Any]]) -> int | None:
- """Count the number of tokens in the input messages. If token counter
- is not provided, `None` will be returned.
-
- Args:
- msgs (`list[Msg]`):
- The input messages to count tokens for.
- """
- if self.token_counter is None:
- return None
-
- return await self.token_counter.count(msgs)
-
- @staticmethod
- async def _group_messages(
- msgs: list[Msg],
- ) -> AsyncGenerator[
- Tuple[Literal["tool_sequence", "agent_message"], list[Msg]],
- None,
- ]:
- """Group the input messages into two types and yield them as a
- generator. The two types are:
-
- - agent message that doesn't contain tool calls/results, and
- - tool sequence that consisted of a sequence of tool calls/results
-
- .. note:: The group operation is used in multi-agent scenario, where
- multiple entities are involved in the input messages. So that to be
- compatible with tools API, we have to group the messages and format
- them with different strategies.
-
- Args:
- msgs (`list[Msg]`):
- The input messages to be grouped, where the system prompt
- message shouldn't be included.
-
- Yields:
- `AsyncGenerator[Tuple[str, list[Msg]], None]`:
- A generator that yields tuples of group type and the list of
- messages in that group. The group type can be either
- "tool_sequence" or "agent_message".
- """
-
- group_type: Literal["tool_sequence", "agent_message"] | None = None
- group = []
- for msg in msgs:
- if group_type is None:
- if msg.has_content_blocks(
- "tool_use",
- ) or msg.has_content_blocks("tool_result"):
- group_type = "tool_sequence"
- else:
- group_type = "agent_message"
-
- group.append(msg)
- continue
-
- # determine if this msg has the same type as the current group
- if group_type == "tool_sequence":
- if msg.has_content_blocks(
- "tool_use",
- ) or msg.has_content_blocks("tool_result"):
- group.append(msg)
-
- else:
- yield group_type, group
- group = [msg]
- group_type = "agent_message"
-
- elif group_type == "agent_message":
- if msg.has_content_blocks(
- "tool_use",
- ) or msg.has_content_blocks("tool_result"):
- yield group_type, group
- group = [msg]
- group_type = "tool_sequence"
-
- else:
- group.append(msg)
- if group_type:
- yield group_type, group
diff --git a/src/agentscope/mcp/_client_base.py b/src/agentscope/mcp/_client_base.py
index 368887034..c9a70a627 100644
--- a/src/agentscope/mcp/_client_base.py
+++ b/src/agentscope/mcp/_client_base.py
@@ -7,11 +7,9 @@ import mcp.types
from .._logging import logger
from ..message import (
- ImageBlock,
Base64Source,
- AudioBlock,
TextBlock,
- VideoBlock,
+ DataBlock,
)
@@ -39,33 +37,19 @@ class MCPClientBase:
@staticmethod
def _convert_mcp_content_to_as_blocks(
mcp_content_blocks: list,
- ) -> List[TextBlock | ImageBlock | AudioBlock | VideoBlock]:
+ ) -> List[TextBlock | DataBlock]:
"""Convert MCP content to AgentScope blocks."""
as_content: list = []
for content in mcp_content_blocks:
if isinstance(content, mcp.types.TextContent):
+ as_content.append(TextBlock(text=content.text))
+ elif isinstance(
+ content,
+ (mcp.types.ImageContent, mcp.types.AudioContent),
+ ):
as_content.append(
- TextBlock(
- type="text",
- text=content.text,
- ),
- )
- elif isinstance(content, mcp.types.ImageContent):
- as_content.append(
- ImageBlock(
- type="image",
- source=Base64Source(
- type="base64",
- media_type=content.mimeType,
- data=content.data,
- ),
- ),
- )
- elif isinstance(content, mcp.types.AudioContent):
- as_content.append(
- AudioBlock(
- type="audio",
+ DataBlock(
source=Base64Source(
type="base64",
media_type=content.mimeType,
@@ -80,7 +64,6 @@ class MCPClientBase:
):
as_content.append(
TextBlock(
- type="text",
text=content.resource.model_dump_json(indent=2),
),
)
@@ -93,6 +76,7 @@ class MCPClientBase:
"Skipping this content.",
type(content.resource),
)
+
else:
logger.warning(
"Unsupported content type: %s. Skipping this content.",
diff --git a/src/agentscope/mcp/_http_stateful_client.py b/src/agentscope/mcp/_http_stateful_client.py
index 9c13d8c84..610471e53 100644
--- a/src/agentscope/mcp/_http_stateful_client.py
+++ b/src/agentscope/mcp/_http_stateful_client.py
@@ -3,7 +3,7 @@
from typing import Any, Literal
from mcp.client.sse import sse_client
-from mcp.client.streamable_http import streamablehttp_client
+from mcp.client.streamable_http import streamable_http_client
from ._stateful_client_base import StatefulClientBase
@@ -67,7 +67,7 @@ class HttpStatefulClient(StatefulClientBase):
self.transport = transport
if self.transport == "streamable_http":
- self.client = streamablehttp_client(
+ self.client = streamable_http_client(
url=url,
headers=headers,
timeout=timeout,
diff --git a/src/agentscope/mcp/_http_stateless_client.py b/src/agentscope/mcp/_http_stateless_client.py
index c90f965df..dbeab5686 100644
--- a/src/agentscope/mcp/_http_stateless_client.py
+++ b/src/agentscope/mcp/_http_stateless_client.py
@@ -6,7 +6,7 @@ from typing import Any, Callable, Awaitable, Literal, List
import mcp.types
from mcp import ClientSession
from mcp.client.sse import sse_client
-from mcp.client.streamable_http import streamablehttp_client
+from mcp.client.streamable_http import streamable_http_client
from . import MCPToolFunction
from ._client_base import MCPClientBase
@@ -82,7 +82,7 @@ class HttpStatelessClient(MCPClientBase):
return sse_client(**self.client_config)
if self.transport == "streamable_http":
- return streamablehttp_client(**self.client_config)
+ return streamable_http_client(**self.client_config)
raise ValueError(
f"Unsupported transport type: {self.transport}. "
diff --git a/src/agentscope/mcp/_mcp_function.py b/src/agentscope/mcp/_mcp_function.py
index 44bfe6d62..d205a1e5b 100644
--- a/src/agentscope/mcp/_mcp_function.py
+++ b/src/agentscope/mcp/_mcp_function.py
@@ -5,13 +5,35 @@ from datetime import timedelta
from typing import Any, Callable
import mcp
-from mcp import ClientSession
+from mcp import ClientSession, Tool
from ._client_base import MCPClientBase
-from .._utils._common import _extract_json_schema_from_mcp_tool
from ..tool import ToolResponse
+def _extract_json_schema_from_mcp_tool(tool: Tool) -> dict[str, Any]:
+ """Extract JSON schema from MCP tool."""
+
+ return {
+ "type": "function",
+ "function": {
+ "name": tool.name,
+ "description": tool.description,
+ "parameters": {
+ "type": "object",
+ "properties": tool.inputSchema.get(
+ "properties",
+ {},
+ ),
+ "required": tool.inputSchema.get(
+ "required",
+ [],
+ ),
+ },
+ },
+ }
+
+
class MCPToolFunction:
"""An MCP tool function class that can be called directly."""
diff --git a/src/agentscope/message/__init__.py b/src/agentscope/message/__init__.py
index 1dfe69f5b..178e08417 100644
--- a/src/agentscope/message/__init__.py
+++ b/src/agentscope/message/__init__.py
@@ -1,31 +1,34 @@
# -*- coding: utf-8 -*-
"""The message module in agentscope."""
-from ._message_block import (
+from ._block import (
ContentBlock,
+ ContentBlockTypes,
TextBlock,
ThinkingBlock,
- ToolUseBlock,
+ HintBlock,
+ ToolCallBlock,
ToolResultBlock,
- ImageBlock,
- AudioBlock,
- VideoBlock,
+ DataBlock,
Base64Source,
URLSource,
)
-from ._message_base import Msg
+from ._base import Msg, UserMsg, AssistantMsg, SystemMsg
__all__ = [
"TextBlock",
"ThinkingBlock",
+ "HintBlock",
+ "ToolCallBlock",
+ "ToolResultBlock",
+ "DataBlock",
"Base64Source",
"URLSource",
- "ImageBlock",
- "AudioBlock",
- "VideoBlock",
- "ToolUseBlock",
- "ToolResultBlock",
"ContentBlock",
+ "ContentBlockTypes",
"Msg",
+ "UserMsg",
+ "AssistantMsg",
+ "SystemMsg",
]
diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.py
new file mode 100644
index 000000000..26e859499
--- /dev/null
+++ b/src/agentscope/message/_base.py
@@ -0,0 +1,275 @@
+# -*- coding: utf-8 -*-
+"""The message class in agentscope."""
+from datetime import datetime
+from typing import Literal, List, overload, Sequence
+
+import shortuuid
+from pydantic import BaseModel, Field
+
+from ._block import (
+ TextBlock,
+ ToolCallBlock,
+ DataBlock,
+ ContentBlock,
+ ToolResultBlock,
+ ContentBlockTypes,
+ HintBlock,
+ ThinkingBlock,
+)
+
+
+def _assert_user_content_blocks(content: str | Sequence[ContentBlock]) -> None:
+ """Assert that the content blocks in user message are valid."""
+ if isinstance(content, str):
+ return
+
+ for block in content:
+ if block.type not in ["text", "data"]:
+ raise ValueError(
+ "User message can only contain text blocks or data blocks.",
+ )
+
+
+def _assert_system_content_blocks(
+ content: str | Sequence[ContentBlock],
+) -> None:
+ """Assert that the content blocks in system message are valid."""
+ if isinstance(content, str):
+ return
+
+ for block in content:
+ if block.type not in ["text"]:
+ raise ValueError("System message can only contain text blocks.")
+
+
+class Msg(BaseModel):
+ """The message class in agentscope."""
+
+ name: str
+ content: str | list[ContentBlock]
+ role: Literal["user", "assistant", "system"]
+
+ id: str = Field(default_factory=shortuuid.uuid)
+ metadata: dict = Field(default_factory=dict)
+ created_at: str = Field(default_factory=datetime.now().isoformat)
+
+ def has_content_blocks(
+ self,
+ block_type: ContentBlockTypes | None = None,
+ ) -> bool:
+ """Check if the message has content blocks of the given type.
+
+ Args:
+ block_type (ContentBlockTypes | None, defaults to None):
+ The type of the block to be checked. If `None`, it will
+ check if there are any content blocks.
+ """
+ return len(self.get_content_blocks(block_type)) > 0
+
+ def get_text_content(self, separator: str = "\n") -> str | None:
+ """Get the pure text blocks from the message content.
+
+ Args:
+ separator (`str`, defaults to `\n`):
+ The separator to use when concatenating multiple text blocks.
+ Defaults to newline character.
+
+ Returns:
+ `str | None`:
+ The concatenated text content, or `None` if there is no text
+ content.
+ """
+ if isinstance(self.content, str):
+ return self.content
+
+ gathered_text = []
+ for block in self.content:
+ if block.type == "text":
+ gathered_text.append(block.text)
+
+ if gathered_text:
+ return separator.join(gathered_text)
+
+ return None
+
+ @overload
+ def get_content_blocks(
+ self,
+ block_type: Literal["text"],
+ ) -> list[TextBlock]:
+ ...
+
+ @overload
+ def get_content_blocks(
+ self,
+ block_type: Literal["thinking"],
+ ) -> list[ThinkingBlock]:
+ ...
+
+ @overload
+ def get_content_blocks(
+ self,
+ block_type: Literal["tool_call"],
+ ) -> list[ToolCallBlock]:
+ ...
+
+ @overload
+ def get_content_blocks(
+ self,
+ block_type: Literal["tool_result"],
+ ) -> list[ToolResultBlock]:
+ ...
+
+ @overload
+ def get_content_blocks(
+ self,
+ block_type: Literal["data"],
+ ) -> list[DataBlock]:
+ ...
+
+ @overload
+ def get_content_blocks(
+ self,
+ block_type: None = None,
+ ) -> list[ContentBlock]:
+ ...
+
+ @overload
+ def get_content_blocks(
+ self,
+ block_type: Literal["hint"],
+ ) -> list[HintBlock]:
+ ...
+
+ def get_content_blocks(
+ self,
+ block_type: ContentBlockTypes | List[ContentBlockTypes] | None = None,
+ ) -> Sequence[ContentBlock]:
+ """Get the content in block format. If the content is a string,
+ it will be converted to a text block.
+
+ Args:
+ block_type (`ContentBlockTypes | List[ContentBlockTypes] | None`, \
+ optional):
+ The type of the block to be extracted. If `None`, all blocks
+ will be returned.
+
+ Returns:
+ `List[ContentBlock]`:
+ The content blocks.
+ """
+ blocks = []
+ if isinstance(self.content, str):
+ blocks.append(
+ TextBlock(text=self.content),
+ )
+ else:
+ blocks = self.content or []
+
+ if isinstance(block_type, str):
+ blocks = [_ for _ in blocks if _.type == block_type]
+
+ elif isinstance(block_type, list):
+ blocks = [_ for _ in blocks if _.type in block_type]
+
+ return blocks
+
+
+def UserMsg(
+ name: str,
+ content: str | list[TextBlock | DataBlock],
+ metadata: dict | None = None,
+ created_at: str | None = None,
+) -> Msg:
+ """Create a user message with role "user".
+
+ Args:
+ name (`str`):
+ The name of the message sender.
+ content (`str | list[TextBlock | DataBlock]`):
+ The content of the message. It can be a string or a list of
+ TextBlock or DataBlock.
+ metadata (`dict | None`, optional):
+ The metadata of the message. Defaults to `None`.
+ created_at (`str | None`, optional):
+ The creation time of the message in ISO format. Defaults to `None`.
+
+ Returns:
+ `Msg`:
+ The created user message.
+ """
+ _assert_user_content_blocks(content)
+
+ return Msg(
+ name=name,
+ content=content,
+ role="user",
+ metadata=metadata or {},
+ created_at=created_at or datetime.now().isoformat(),
+ )
+
+
+def AssistantMsg(
+ name: str,
+ content: str | list[ContentBlock],
+ metadata: dict | None = None,
+ created_at: str | None = None,
+) -> Msg:
+ """Create an assistant message with role "assistant".
+
+ Args:
+ name (`str`):
+ The name of the message sender.
+ content (`str | list[ContentBlock]`):
+ The content of the message. It can be a string or a list of
+ ContentBlock.
+ metadata (`dict | None`, optional):
+ The metadata of the message. Defaults to `None`.
+ created_at (`str | None`, optional):
+ The creation time of the message in ISO format. Defaults to `None`.
+
+ Returns:
+ `Msg`:
+ The created assistant message.
+ """
+ return Msg(
+ name=name,
+ content=content,
+ role="assistant",
+ metadata=metadata or {},
+ created_at=created_at or datetime.now().isoformat(),
+ )
+
+
+def SystemMsg(
+ name: str,
+ content: str | list[TextBlock],
+ metadata: dict | None = None,
+ created_at: str | None = None,
+) -> Msg:
+ """Create a system message with role "system".
+
+ Args:
+ name (`str`):
+ The name of the message sender.
+ content (`str | list[TextBlock]`):
+ The content of the message. It can be a string or a list of
+ TextBlock.
+ metadata (`dict | None`, optional):
+ The metadata of the message. Defaults to `None`.
+ created_at (`str | None`, optional):
+ The creation time of the message in ISO format. Defaults to `None`.
+
+ Returns:
+ `Msg`:
+ The created system message.
+ """
+ _assert_system_content_blocks(content)
+
+ return Msg(
+ name=name,
+ content=content,
+ role="system",
+ metadata=metadata or {},
+ created_at=created_at or datetime.now().isoformat(),
+ )
diff --git a/src/agentscope/message/_block.py b/src/agentscope/message/_block.py
new file mode 100644
index 000000000..366091491
--- /dev/null
+++ b/src/agentscope/message/_block.py
@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""The content blocks of messages."""
+import uuid
+from typing import Literal, List, TypeAlias
+from pydantic import BaseModel, Field
+
+
+class TextBlock(BaseModel):
+ """The text block."""
+
+ type: Literal["text"] = "text"
+ text: str
+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+
+
+class ThinkingBlock(BaseModel):
+ """The thinking block."""
+
+ type: Literal["thinking"] = "thinking"
+ thinking: str
+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+
+
+class HintBlock(BaseModel):
+ """The hint block."""
+
+ type: Literal["hint"] = "hint"
+ hint: str
+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+
+
+class Base64Source(BaseModel):
+ """The base64 source."""
+
+ type: Literal["base64"] = "base64"
+ data: str
+ media_type: str
+
+
+class URLSource(BaseModel):
+ """The URL source."""
+
+ type: Literal["url"] = "url"
+ url: str
+ media_type: str
+
+
+class DataBlock(BaseModel):
+ """The data block for binary content (images, audio, video, etc.)."""
+
+ type: Literal["data"] = "data"
+ id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+ source: Base64Source | URLSource
+ name: str | None = None
+
+
+class ToolCallBlock(BaseModel):
+ """The tool call block."""
+
+ type: Literal["tool_call"] = "tool_call"
+ id: str
+ name: str
+ input: str
+ """The raw JSON string input of the tool, accumulated during streaming."""
+ await_user_confirmation: bool = False
+
+
+class ToolResultBlock(BaseModel):
+ """The tool result block."""
+
+ type: Literal["tool_result"] = "tool_result"
+ id: str
+ name: str
+ output: str | List[TextBlock | DataBlock]
+ state: Literal["success", "error", "interrupted", "running"]
+
+
+ContentBlock: TypeAlias = (
+ TextBlock
+ | ThinkingBlock
+ | HintBlock
+ | ToolCallBlock
+ | ToolResultBlock
+ | DataBlock
+)
+
+ContentBlockTypes: TypeAlias = Literal[
+ "text",
+ "thinking",
+ "hint",
+ "tool_call",
+ "tool_result",
+ "data",
+]
diff --git a/src/agentscope/message/_message_base.py b/src/agentscope/message/_message_base.py
deleted file mode 100644
index 6a69a9c58..000000000
--- a/src/agentscope/message/_message_base.py
+++ /dev/null
@@ -1,241 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The message class in agentscope."""
-from datetime import datetime
-from typing import Literal, List, overload, Sequence
-
-import shortuuid
-
-from ._message_block import (
- TextBlock,
- ToolUseBlock,
- ImageBlock,
- AudioBlock,
- ContentBlock,
- VideoBlock,
- ToolResultBlock,
- ContentBlockTypes,
-)
-from ..types import JSONSerializableObject
-
-
-class Msg:
- """The message class in agentscope."""
-
- def __init__(
- self,
- name: str,
- content: str | Sequence[ContentBlock],
- role: Literal["user", "assistant", "system"],
- metadata: dict[str, JSONSerializableObject] | None = None,
- timestamp: str | None = None,
- invocation_id: str | None = None,
- ) -> None:
- """Initialize the Msg object.
-
- Args:
- name (`str`):
- The name of the message sender.
- content (`str | list[ContentBlock]`):
- The content of the message.
- role (`Literal["user", "assistant", "system"]`):
- The role of the message sender.
- metadata (`dict[str, JSONSerializableObject] | None`, optional):
- The metadata of the message, e.g. structured output.
- timestamp (`str | None`, optional):
- The created timestamp of the message. If not given, the
- timestamp will be set automatically.
- invocation_id (`str | None`, optional):
- The related API invocation id, if any. This is useful for
- tracking the message in the context of an API call.
- """
-
- self.name = name
-
- assert isinstance(
- content,
- (list, str),
- ), "The content must be a string or a list of content blocks."
-
- self.content = content
-
- assert role in ["user", "assistant", "system"]
- self.role = role
-
- self.metadata = metadata or {}
-
- self.id = shortuuid.uuid()
- self.timestamp = (
- timestamp
- or datetime.now().strftime(
- "%Y-%m-%d %H:%M:%S.%f",
- )[:-3]
- )
- self.invocation_id = invocation_id
-
- def to_dict(self) -> dict:
- """Convert the message into JSON dict data."""
- return {
- "id": self.id,
- "name": self.name,
- "role": self.role,
- "content": self.content,
- "metadata": self.metadata,
- "timestamp": self.timestamp,
- }
-
- @classmethod
- def from_dict(cls, json_data: dict) -> "Msg":
- """Load a message object from the given JSON data."""
- new_obj = cls(
- name=json_data["name"],
- content=json_data["content"],
- role=json_data["role"],
- metadata=json_data.get("metadata", None),
- timestamp=json_data.get("timestamp", None),
- invocation_id=json_data.get("invocation_id", None),
- )
-
- new_obj.id = json_data.get("id", new_obj.id)
- return new_obj
-
- def has_content_blocks(
- self,
- block_type: Literal[
- "text",
- "tool_use",
- "tool_result",
- "image",
- "audio",
- "video",
- ]
- | None = None,
- ) -> bool:
- """Check if the message has content blocks of the given type.
-
- Args:
- block_type (Literal["text", "tool_use", "tool_result", "image", \
- "audio", "video"] | None, defaults to None):
- The type of the block to be checked. If `None`, it will
- check if there are any content blocks.
- """
- return len(self.get_content_blocks(block_type)) > 0
-
- def get_text_content(self, separator: str = "\n") -> str | None:
- """Get the pure text blocks from the message content.
-
- Args:
- separator (`str`, defaults to `\n`):
- The separator to use when concatenating multiple text blocks.
- Defaults to newline character.
-
- Returns:
- `str | None`:
- The concatenated text content, or `None` if there is no text
- content.
- """
- if isinstance(self.content, str):
- return self.content
-
- gathered_text = []
- for block in self.content:
- if block.get("type") == "text":
- gathered_text.append(block["text"])
-
- if gathered_text:
- return separator.join(gathered_text)
-
- return None
-
- @overload
- def get_content_blocks(
- self,
- block_type: Literal["text"],
- ) -> Sequence[TextBlock]:
- ...
-
- @overload
- def get_content_blocks(
- self,
- block_type: Literal["tool_use"],
- ) -> Sequence[ToolUseBlock]:
- ...
-
- @overload
- def get_content_blocks(
- self,
- block_type: Literal["tool_result"],
- ) -> Sequence[ToolResultBlock]:
- ...
-
- @overload
- def get_content_blocks(
- self,
- block_type: Literal["image"],
- ) -> Sequence[ImageBlock]:
- ...
-
- @overload
- def get_content_blocks(
- self,
- block_type: Literal["audio"],
- ) -> Sequence[AudioBlock]:
- ...
-
- @overload
- def get_content_blocks(
- self,
- block_type: Literal["video"],
- ) -> Sequence[VideoBlock]:
- ...
-
- @overload
- def get_content_blocks(
- self,
- block_type: None = None,
- ) -> Sequence[ContentBlock]:
- ...
-
- def get_content_blocks(
- self,
- block_type: ContentBlockTypes | List[ContentBlockTypes] | None = None,
- ) -> Sequence[ContentBlock]:
- """Get the content in block format. If the content is a string,
- it will be converted to a text block.
-
- Args:
- block_type (`ContentBlockTypes | List[ContentBlockTypes] | None`, \
- optional):
- The type of the block to be extracted. If `None`, all blocks
- will be returned.
-
- Returns:
- `List[ContentBlock]`:
- The content blocks.
- """
- blocks = []
- if isinstance(self.content, str):
- blocks.append(
- TextBlock(type="text", text=self.content),
- )
- else:
- blocks = self.content or []
-
- if isinstance(block_type, str):
- blocks = [_ for _ in blocks if _["type"] == block_type]
-
- elif isinstance(block_type, list):
- blocks = [_ for _ in blocks if _["type"] in block_type]
-
- return blocks
-
- def __repr__(self) -> str:
- """Get the string representation of the message."""
- return (
- f"Msg(id='{self.id}', "
- f"name='{self.name}', "
- f"content={repr(self.content)}, "
- f"role='{self.role}', "
- f"metadata={repr(self.metadata)}, "
- f"timestamp='{self.timestamp}', "
- f"invocation_id='{self.invocation_id}')"
- )
diff --git a/src/agentscope/message/_message_block.py b/src/agentscope/message/_message_block.py
deleted file mode 100644
index 2adb683ec..000000000
--- a/src/agentscope/message/_message_block.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# -*- coding: utf-8 -*-
-# pylint: disable=R0901
-"""The content blocks of messages"""
-
-from typing import Literal, List
-from typing_extensions import TypedDict, Required
-
-
-class TextBlock(TypedDict, total=False):
- """The text block."""
-
- type: Required[Literal["text"]]
- """The type of the block"""
- text: str
- """The text content"""
-
-
-class ThinkingBlock(TypedDict, total=False):
- """The thinking block."""
-
- type: Required[Literal["thinking"]]
- """The type of the block"""
- thinking: str
-
-
-class Base64Source(TypedDict, total=False):
- """The base64 source"""
-
- type: Required[Literal["base64"]]
- """The type of the src, must be `base64`"""
-
- media_type: Required[str]
- """The media type of the data, e.g. `image/jpeg` or `audio/mpeg`"""
-
- data: Required[str]
- """The base64 data, in format of RFC 2397"""
-
-
-class URLSource(TypedDict, total=False):
- """The URL source"""
-
- type: Required[Literal["url"]]
- """The type of the src, must be `url`"""
-
- url: Required[str]
- """The URL of the image or audio"""
-
-
-class ImageBlock(TypedDict, total=False):
- """The image block"""
-
- type: Required[Literal["image"]]
- """The type of the block"""
-
- source: Required[Base64Source | URLSource]
- """The src of the image"""
-
-
-class AudioBlock(TypedDict, total=False):
- """The audio block"""
-
- type: Required[Literal["audio"]]
- """The type of the block"""
-
- source: Required[Base64Source | URLSource]
- """The src of the audio"""
-
-
-class VideoBlock(TypedDict, total=False):
- """The video block"""
-
- type: Required[Literal["video"]]
- """The type of the block"""
-
- source: Required[Base64Source | URLSource]
- """The src of the audio"""
-
-
-class ToolUseBlock(TypedDict, total=False):
- """The tool use block."""
-
- type: Required[Literal["tool_use"]]
- """The type of the block, must be `tool_use`"""
- id: Required[str]
- """The identity of the tool call"""
- name: Required[str]
- """The name of the tool"""
- input: Required[dict[str, object]]
- """The input of the tool"""
- raw_input: str
- """The raw string input of the tool from the model API"""
-
-
-class ToolResultBlock(TypedDict, total=False):
- """The tool result block."""
-
- type: Required[Literal["tool_result"]]
- """The type of the block"""
- id: Required[str]
- """The identity of the tool call result"""
- output: Required[
- str | List[TextBlock | ImageBlock | AudioBlock | VideoBlock]
- ]
- """The output of the tool function"""
- name: Required[str]
- """The name of the tool function"""
-
-
-# The content block
-ContentBlock = (
- ToolUseBlock
- | ToolResultBlock
- | TextBlock
- | ThinkingBlock
- | ImageBlock
- | AudioBlock
- | VideoBlock
-)
-
-ContentBlockTypes = Literal[
- "text",
- "thinking",
- "tool_use",
- "tool_result",
- "image",
- "audio",
- "video",
-]
diff --git a/src/agentscope/model/__init__.py b/src/agentscope/model/__init__.py
index 7cd0b7a8f..221272767 100644
--- a/src/agentscope/model/__init__.py
+++ b/src/agentscope/model/__init__.py
@@ -9,8 +9,10 @@ from ._anthropic_model import AnthropicChatModel
from ._ollama_model import OllamaChatModel
from ._gemini_model import GeminiChatModel
from ._trinity_model import TrinityChatModel
+from ._model_usage import ChatUsage
__all__ = [
+ "ChatUsage",
"ChatModelBase",
"ChatResponse",
"DashScopeChatModel",
diff --git a/src/agentscope/model/_anthropic_model.py b/src/agentscope/model/_anthropic_model.py
index 1c62fbe0c..744c7526e 100644
--- a/src/agentscope/model/_anthropic_model.py
+++ b/src/agentscope/model/_anthropic_model.py
@@ -1,8 +1,5 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches, too-many-statements
"""The Anthropic API model classes."""
-import copy
-import warnings
from datetime import datetime
from typing import (
Any,
@@ -10,7 +7,6 @@ from typing import (
TYPE_CHECKING,
List,
Literal,
- Type,
)
from collections import OrderedDict
@@ -19,15 +15,10 @@ from pydantic import BaseModel
from ._model_base import ChatModelBase
from ._model_response import ChatResponse
from ._model_usage import ChatUsage
-from .._logging import logger
-from .._utils._common import (
- _json_loads_with_repair,
- _parse_streaming_json_dict,
- _create_tool_from_base_model,
-)
-from ..message import TextBlock, ToolUseBlock, ThinkingBlock
+from ..formatter import FormatterBase, AnthropicChatFormatter
+from ..message import TextBlock, ToolCallBlock, ThinkingBlock
from ..tracing import trace_llm
-from ..types._json import JSONSerializableObject
+from ..types import JSONSerializableObject, ToolChoice
if TYPE_CHECKING:
from anthropic.types.message import Message
@@ -40,17 +31,25 @@ else:
class AnthropicChatModel(ChatModelBase):
"""The Anthropic model wrapper for AgentScope."""
+ class ThinkingConfig(BaseModel):
+ """Configuration for Claude's internal reasoning process."""
+
+ enable_thinking: bool
+ budget_tokens: int = 1024
+
def __init__(
self,
model_name: str,
api_key: str | None = None,
max_tokens: int = 2048,
stream: bool = True,
- thinking: dict | None = None,
+ max_retries: int = 0,
+ fallback_model_name: str | None = None,
+ formatter: FormatterBase | None = None,
+ thinking_config: ThinkingConfig | None = None,
stream_tool_parsing: bool = True,
client_kwargs: dict[str, JSONSerializableObject] | None = None,
generate_kwargs: dict[str, JSONSerializableObject] | None = None,
- **kwargs: Any,
) -> None:
"""Initialize the Anthropic chat model.
@@ -59,21 +58,18 @@ class AnthropicChatModel(ChatModelBase):
The model names.
api_key (`str`):
The anthropic API key.
- stream (`bool`):
- The streaming output or not
max_tokens (`int`):
Limit the maximum token count the model can generate.
- thinking (`dict | None`, default `None`):
+ stream (`bool`):
+ The streaming output or not
+ max_retries (`int`, optional):
+ Maximum number of retries on failure. Defaults to 0.
+ fallback_model_name (`str | None`, optional):
+ Fallback model name to use after all retries fail.
+ formatter (`FormatterBase | None`, optional):
+ Formatter for message preprocessing.
+ thinking_config (`ThinkingConfig | None`, default `None`):
Configuration for Claude's internal reasoning process.
-
- .. code-block:: python
- :caption: Example of thinking
-
- {
- "type": "enabled" | "disabled",
- "budget_tokens": 1024
- }
-
stream_tool_parsing (`bool`, default to `True`):
Whether to parse incomplete tool use JSON during streaming
with auto-repair. If True, partial JSON (e.g., `'{"a": "x'`)
@@ -87,33 +83,8 @@ class AnthropicChatModel(ChatModelBase):
optional):
The extra keyword arguments used in Anthropic API generation,
e.g. `temperature`, `seed`.
- **kwargs (`Any`):
- Additional keyword arguments.
"""
- # Handle deprecated client_args parameter from kwargs
- client_args = kwargs.pop("client_args", None)
- if client_args is not None and client_kwargs is not None:
- raise ValueError(
- "Cannot specify both 'client_args' and 'client_kwargs'. "
- "Please use only 'client_kwargs' (client_args is deprecated).",
- )
-
- if client_args is not None:
- logger.warning(
- "The parameter 'client_args' is deprecated and will be "
- "removed in a future version. Please use 'client_kwargs' "
- "instead. Automatically converting 'client_args' to "
- "'client_kwargs'.",
- )
- client_kwargs = client_args
-
- if kwargs:
- logger.warning(
- "Unknown keyword arguments: %s. These will be ignored.",
- list(kwargs.keys()),
- )
-
try:
import anthropic
except ImportError as e:
@@ -122,127 +93,75 @@ class AnthropicChatModel(ChatModelBase):
"`pip install anthropic`.",
) from e
- super().__init__(model_name, stream)
+ super().__init__(
+ model_name=model_name,
+ stream=stream,
+ max_retries=max_retries,
+ fallback_model_name=fallback_model_name,
+ formatter=formatter or AnthropicChatFormatter(),
+ )
self.client = anthropic.AsyncAnthropic(
api_key=api_key,
**(client_kwargs or {}),
)
self.max_tokens = max_tokens
- self.thinking = thinking
+ self.thinking_config = thinking_config
self.stream_tool_parsing = stream_tool_parsing
self.generate_kwargs = generate_kwargs or {}
@trace_llm
- async def __call__(
+ async def _call_api(
self,
+ model_name: str,
messages: list[dict[str, Any]],
tools: list[dict] | None = None,
tool_choice: Literal["auto", "none", "required"] | str | None = None,
- structured_model: Type[BaseModel] | None = None,
**generate_kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
"""Get the response from Anthropic chat completions API by the given
arguments.
Args:
+ model_name (`str`):
+ The model name to use for this call.
messages (`list[dict]`):
A list of dictionaries, where `role` and `content` fields are
required, and `name` field is optional.
tools (`list[dict]`, default `None`):
- The tools JSON schemas that in format of:
-
- .. code-block:: python
- :caption: Example of tools JSON schemas
-
- [
- {
- "type": "function",
- "function": {
- "name": "xxx",
- "description": "xxx",
- "parameters": {
- "type": "object",
- "properties": {
- "param1": {
- "type": "string",
- "description": "..."
- },
- # Add more parameters as needed
- },
- "required": ["param1"]
- }
- },
- # More schemas here
- ]
-
+ The tools JSON schemas.
tool_choice (`Literal["auto", "none", "required"] | str \
| None`, default `None`):
Controls which (if any) tool is called by the model.
- Can be "auto", "none", "required", or specific tool
- name. For more details, please refer to
- https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/implement-tool-use
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output. When provided, the model will be forced
- to return data that conforms to this schema by automatically
- converting the BaseModel to a tool function and setting
- `tool_choice` to enforce its usage. This enables structured
- output generation.
-
- .. note:: When `structured_model` is specified,
- both `tools` and `tool_choice` parameters are ignored,
- and the model will only perform structured output
- generation without calling any other tools.
-
**generate_kwargs (`Any`):
- The keyword arguments for Anthropic chat completions API,
- e.g. `temperature`, `top_p`, etc. Please
- refer to the Anthropic API documentation for more details.
+ The keyword arguments for Anthropic chat completions API.
Returns:
`ChatResponse | AsyncGenerator[ChatResponse, None]`:
The response from the Anthropic chat completions API."""
kwargs: dict[str, Any] = {
- "model": self.model_name,
+ "model": model_name,
"max_tokens": self.max_tokens,
"stream": self.stream,
**self.generate_kwargs,
**generate_kwargs,
}
- if self.thinking and "thinking" not in kwargs:
- kwargs["thinking"] = self.thinking
+ if self.thinking_config and "thinking" not in kwargs:
+ kwargs["thinking"] = {
+ "type": "enabled"
+ if self.thinking_config.enable_thinking
+ else "disabled",
+ "budget_tokens": self.thinking_config.budget_tokens,
+ }
if tools:
kwargs["tools"] = self._format_tools_json_schemas(tools)
if tool_choice:
- # Handle deprecated "any" option with warning
- if tool_choice == "any":
- warnings.warn(
- '"any" is deprecated and will be removed in a future '
- "version.",
- DeprecationWarning,
- )
- tool_choice = "required"
- self._validate_tool_choice(tool_choice, tools)
- kwargs["tool_choice"] = self._format_tool_choice(tool_choice)
-
- if structured_model:
- if tools or tool_choice:
- logger.warning(
- "structured_model is provided. Both 'tools' and "
- "'tool_choice' parameters will be overridden and "
- "ignored. The model will only perform structured output "
- "generation without calling any other tools.",
- )
- format_tool = _create_tool_from_base_model(structured_model)
- kwargs["tools"] = self._format_tools_json_schemas(
- [format_tool],
- )
kwargs["tool_choice"] = self._format_tool_choice(
- format_tool["function"]["name"],
+ tool_choice,
+ tools,
)
# Extract the system message
@@ -260,14 +179,12 @@ class AnthropicChatModel(ChatModelBase):
return self._parse_anthropic_stream_completion_response(
start_datetime,
response,
- structured_model,
)
# Non-streaming response
parsed_response = await self._parse_anthropic_completion_response(
start_datetime,
response,
- structured_model,
)
return parsed_response
@@ -276,7 +193,6 @@ class AnthropicChatModel(ChatModelBase):
self,
start_datetime: datetime,
response: Message,
- structured_model: Type[BaseModel] | None = None,
) -> ChatResponse:
"""Given an Anthropic Message object, extract the content blocks and
usages from it.
@@ -286,20 +202,12 @@ class AnthropicChatModel(ChatModelBase):
The start datetime of the response generation.
response (`Message`):
Anthropic Message object to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
ChatResponse (`ChatResponse`):
A ChatResponse object containing the content blocks and usage.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
- content_blocks: List[ThinkingBlock | TextBlock | ToolUseBlock] = []
- metadata = None
+ content_blocks: List[ThinkingBlock | TextBlock | ToolCallBlock] = []
if hasattr(response, "content") and response.content:
for content_block in response.content:
@@ -308,7 +216,6 @@ class AnthropicChatModel(ChatModelBase):
and content_block.type == "thinking"
):
thinking_block = ThinkingBlock(
- type="thinking",
thinking=content_block.thinking,
)
thinking_block["signature"] = content_block.signature
@@ -319,26 +226,22 @@ class AnthropicChatModel(ChatModelBase):
and content_block.type == "text"
):
content_blocks.append(
- TextBlock(
- type="text",
- text=content_block.text,
- ),
+ TextBlock(text=content_block.text),
)
elif (
hasattr(content_block, "type")
and content_block.type == "tool_use"
):
+ import json
+
content_blocks.append(
- ToolUseBlock(
- type="tool_use",
+ ToolCallBlock(
id=content_block.id,
name=content_block.name,
- input=content_block.input,
+ input=json.dumps(content_block.input),
),
)
- if structured_model:
- metadata = content_block.input
usage = None
if response.usage:
@@ -350,8 +253,8 @@ class AnthropicChatModel(ChatModelBase):
resp_kwargs: dict[str, Any] = {
"content": content_blocks,
+ "is_last": True,
"usage": usage,
- "metadata": metadata,
}
response_id = getattr(response, "id", None)
if response_id:
@@ -363,7 +266,6 @@ class AnthropicChatModel(ChatModelBase):
self,
start_datetime: datetime,
response: AsyncStream,
- structured_model: Type[BaseModel] | None = None,
) -> AsyncGenerator[ChatResponse, None]:
"""Given an Anthropic streaming response, extract the content blocks
and usages from it and yield ChatResponse objects.
@@ -373,38 +275,26 @@ class AnthropicChatModel(ChatModelBase):
The start datetime of the response generation.
response (`AsyncStream`):
Anthropic AsyncStream object to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
`AsyncGenerator[ChatResponse, None]`:
An async generator that yields ChatResponse objects containing
the content blocks and usage information for each chunk in
the streaming response.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
usage = None
response_id: str | None = None
- text_buffer = ""
- thinking_buffer = ""
+ # Accumulated state
+ acc_text = ""
+ acc_thinking = ""
thinking_signature = ""
- tool_calls = OrderedDict()
- tool_call_buffers = {}
- last_input_objs = {} # Store last input_obj for each tool_call
- res = None
- metadata = None
-
- # Record the last yielded content to parse the tools' input
- last_content = None
+ acc_tool_calls: OrderedDict = (
+ OrderedDict()
+ ) # index -> {id, name, input}
async for event in response:
- content_changed = False
- thinking_changed = False
+ delta_content: list = []
if event.type == "message_start":
message = event.message
@@ -425,117 +315,80 @@ class AnthropicChatModel(ChatModelBase):
if event.content_block.type == "tool_use":
block_index = event.index
tool_block = event.content_block
- tool_calls[block_index] = {
- "type": "tool_use",
+ acc_tool_calls[block_index] = {
"id": tool_block.id,
"name": tool_block.name,
"input": "",
}
- tool_call_buffers[block_index] = ""
- content_changed = True
elif event.type == "content_block_delta":
block_index = event.index
delta = event.delta
if delta.type == "text_delta":
- text_buffer += delta.text
- content_changed = True
+ acc_text += delta.text
+ delta_content.append(TextBlock(text=delta.text))
elif delta.type == "thinking_delta":
- thinking_buffer += delta.thinking
- thinking_changed = True
+ acc_thinking += delta.thinking
+ delta_content.append(
+ ThinkingBlock(thinking=delta.thinking),
+ )
elif delta.type == "signature_delta":
thinking_signature = delta.signature
elif (
delta.type == "input_json_delta"
- and block_index in tool_calls
+ and block_index in acc_tool_calls
):
- tool_call_buffers[block_index] += delta.partial_json or ""
- tool_calls[block_index]["input"] = tool_call_buffers[
- block_index
- ]
- content_changed = True
+ fragment = delta.partial_json or ""
+ acc_tool_calls[block_index]["input"] += fragment
+ tc = acc_tool_calls[block_index]
+ delta_content.append(
+ ToolCallBlock(
+ id=tc["id"],
+ name=tc["name"],
+ input=fragment,
+ ),
+ )
elif event.type == "message_delta":
if event.usage and usage:
usage.output_tokens = event.usage.output_tokens
- if (thinking_changed or content_changed) and usage:
- contents: list = []
- if thinking_buffer:
- thinking_block = ThinkingBlock(
- type="thinking",
- thinking=thinking_buffer,
- )
- thinking_block["signature"] = thinking_signature
- contents.append(thinking_block)
- if text_buffer:
- contents.append(
- TextBlock(
- type="text",
- text=text_buffer,
- ),
- )
- for block_index, tool_call in tool_calls.items():
- input_str = tool_call["input"]
- tool_id = tool_call["id"]
-
- # If parsing the tool input in streaming mode
- if self.stream_tool_parsing:
- repaired_input = _parse_streaming_json_dict(
- input_str,
- last_input_objs.get(tool_id),
- )
- last_input_objs[tool_id] = repaired_input
-
- else:
- repaired_input = {}
-
- contents.append(
- ToolUseBlock(
- type=tool_call["type"],
- id=tool_call["id"],
- name=tool_call["name"],
- input=repaired_input,
- raw_input=input_str,
- ),
- )
-
- if structured_model:
- metadata = repaired_input
-
- if contents:
- _kwargs: dict[str, Any] = {
- "content": contents,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _kwargs["id"] = response_id
- res = ChatResponse(**_kwargs)
- yield res
- last_content = copy.deepcopy(contents)
-
- # If stream_tool_parsing is False, yield last contents
- if not self.stream_tool_parsing and last_content and tool_calls:
- metadata = None
- # Update tool use blocks in last_contents inplace
- for block in last_content:
- if block.get("type") == "tool_use":
- block["input"] = input_obj = _json_loads_with_repair(
- block.get("raw_input") or "{}",
- )
-
- if structured_model:
- metadata = input_obj
+ if delta_content:
+ _kwargs: dict[str, Any] = {
+ "content": delta_content,
+ "is_last": False,
+ "usage": usage,
+ }
+ if response_id:
+ _kwargs["id"] = response_id
+ yield ChatResponse(**_kwargs)
+
+ # Build final accumulated content
+ final_content: list = []
+ if acc_thinking:
+ thinking_block = ThinkingBlock(thinking=acc_thinking)
+ thinking_block["signature"] = thinking_signature
+ final_content.append(thinking_block)
+ if acc_text:
+ final_content.append(TextBlock(text=acc_text))
+ for tc in acc_tool_calls.values():
+ input_str = tc["input"]
+ final_content.append(
+ ToolCallBlock(
+ id=tc["id"],
+ name=tc["name"],
+ input=input_str,
+ ),
+ )
- _final_kwargs: dict[str, Any] = {
- "content": last_content,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _final_kwargs["id"] = response_id
- yield ChatResponse(**_final_kwargs)
+ _final_kwargs: dict[str, Any] = {
+ "content": final_content,
+ "is_last": True,
+ "usage": usage,
+ }
+ if response_id:
+ _final_kwargs["id"] = response_id
+ yield ChatResponse(**_final_kwargs)
def _format_tools_json_schemas(
self,
@@ -566,22 +419,26 @@ class AnthropicChatModel(ChatModelBase):
def _format_tool_choice(
self,
- tool_choice: Literal["auto", "none", "required"] | str | None,
+ tool_choice: ToolChoice | None,
+ tools: list[dict] | None,
) -> dict | None:
"""Format tool_choice parameter for API compatibility.
Args:
- tool_choice (`Literal["auto", "none", "required"] | str \
- | None`, default `None`):
- Controls which (if any) tool is called by the model.
- Can be "auto", "none", "required", or specific tool
- name. For more details, please refer to
- https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/implement-tool-use
+ tool_choice (`ToolChoice | None`):
+ The unified tool choice parameter which can be a mode ("auto",
+ "none", "required") or a specific function name.
+ tools (`list[dict] | None`):
+ The list of available tools, used for validation if
+ tool_choice is a specific function name.
+
Returns:
`dict | None`:
The formatted tool choice configuration dict, or None if
tool_choice is None.
"""
+ self._validate_tool_choice(tool_choice, tools)
+
if tool_choice is None:
return None
diff --git a/src/agentscope/model/_dashscope_model.py b/src/agentscope/model/_dashscope_model.py
index c37f1dc6f..255448e76 100644
--- a/src/agentscope/model/_dashscope_model.py
+++ b/src/agentscope/model/_dashscope_model.py
@@ -1,9 +1,6 @@
# -*- coding: utf-8 -*-
"""The dashscope API model classes."""
-import copy
-import collections
-import json
-import os
+import uuid
import warnings
from datetime import datetime
from http import HTTPStatus
@@ -14,25 +11,21 @@ from typing import (
Union,
TYPE_CHECKING,
List,
- Literal,
- Type,
)
+
from pydantic import BaseModel
from aioitertools import iter as giter
from ._model_base import ChatModelBase
from ._model_response import ChatResponse
from ._model_usage import ChatUsage
-from .._utils._common import (
- _json_loads_with_repair,
- _parse_streaming_json_dict,
- _create_tool_from_base_model,
-)
-from ..message import TextBlock, ToolUseBlock, ThinkingBlock
+from ..formatter import FormatterBase, DashScopeChatFormatter
+from ..message import TextBlock, ToolCallBlock, ThinkingBlock, Msg
from ..tracing import trace_llm
-from ..types import JSONSerializableObject
+from ..types import JSONSerializableObject, ToolChoice
from .._logging import logger
+
if TYPE_CHECKING:
from dashscope.api_entities.dashscope_response import GenerationResponse
from dashscope.api_entities.dashscope_response import (
@@ -70,16 +63,25 @@ class DashScopeChatModel(ChatModelBase):
DashScope's diverse model offerings.
"""
+ class ThinkingConfig(BaseModel):
+ """The configuration for the thinking process in DashScope API."""
+
+ enable_thinking: bool
+ thinking_budget: int = 2000
+ preserve_thinking: bool = False
+
def __init__(
self,
model_name: str,
api_key: str,
stream: bool = True,
- enable_thinking: bool | None = None,
- multimodality: bool | None = None,
+ max_retries: int = 0,
+ fallback_model_name: str | None = None,
+ formatter: FormatterBase | None = None,
+ thinking_config: ThinkingConfig | None = None,
+ multimodality: bool = False,
generate_kwargs: dict[str, JSONSerializableObject] | None = None,
base_http_api_url: str | None = None,
- stream_tool_parsing: bool = True,
**_kwargs: Any,
) -> None:
"""Initialize the DashScope chat model.
@@ -120,65 +122,58 @@ class DashScopeChatModel(ChatModelBase):
**_kwargs (`Any`):
Additional keyword arguments.
"""
- if enable_thinking and not stream:
+
+ self.thinking_config = (
+ thinking_config
+ or DashScopeChatModel.ThinkingConfig(
+ enable_thinking=False,
+ )
+ )
+
+ if self.thinking_config.enable_thinking and not stream:
logger.info(
"In DashScope API, `stream` must be True when "
"`enable_thinking` is True. ",
)
stream = True
- super().__init__(model_name, stream)
+ super().__init__(
+ model_name,
+ stream,
+ max_retries,
+ fallback_model_name,
+ formatter or DashScopeChatFormatter(),
+ )
self.api_key = api_key
- self.enable_thinking = enable_thinking
self.multimodality = multimodality
self.generate_kwargs = generate_kwargs or {}
- self.stream_tool_parsing = stream_tool_parsing
if base_http_api_url is not None:
import dashscope
dashscope.base_http_api_url = base_http_api_url
- # Load headers from environment variable if exists
- headers = os.getenv("DASHSCOPE_API_HEADERS")
- if headers:
- try:
- headers = json.loads(str(headers))
- if not isinstance(headers, dict):
- raise json.JSONDecodeError("", "", 0)
-
- if self.generate_kwargs.get("headers"):
- headers.update(self.generate_kwargs["headers"])
-
- self.generate_kwargs["headers"] = headers
-
- except json.JSONDecodeError:
- logger.warning(
- "Failed to parse DASHSCOPE_API_HEADERS environment "
- "variable as JSON. It should be a JSON object.",
- )
-
@trace_llm
- async def __call__(
+ async def _call_api(
self,
- messages: list[dict[str, Any]],
+ model_name: str,
+ messages: list[Msg],
tools: list[dict] | None = None,
- tool_choice: Literal["auto", "none", "required"] | str | None = None,
- structured_model: Type[BaseModel] | None = None,
+ tool_choice: ToolChoice | None = None,
**kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
"""Get the response from the dashscope
Generation/MultimodalConversation API by the given arguments.
- .. note:: We unify the dashscope generation and multimodal conversation
+ .. note:: We unify the DashScope generation and multimodal conversation
APIs into one method, since they support similar arguments and share
the same functionality.
Args:
- messages (`list[dict[str, Any]]`):
- A list of dictionaries, where `role` and `content` fields are
- required.
+ messages (`list[Msg]`):
+ The Msg object that will be formatted and sent to the LLM API
+ as the conversation messages.
tools (`list[dict] | None`, default `None`):
The tools JSON schemas that the model can use.
tool_choice (`Literal["auto", "none", "required"] | str \
@@ -189,19 +184,6 @@ class DashScopeChatModel(ChatModelBase):
"required" will be converted to "auto".
For more details, please refer to
https://help.aliyun.com/zh/model-studio/qwen-function-calling
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output. When provided, the model will be forced
- to return data that conforms to this schema by automatically
- converting the BaseModel to a tool function and setting
- `tool_choice` to enforce its usage. This enables structured
- output generation.
-
- .. note:: When `structured_model` is specified,
- both `tools` and `tool_choice` parameters are ignored,
- and the model will only perform structured output
- generation without calling any other tools.
-
**kwargs (`Any`):
The keyword arguments for DashScope chat completions API,
e.g. `temperature`, `max_tokens`, `top_p`, etc. Please
@@ -211,9 +193,13 @@ class DashScopeChatModel(ChatModelBase):
"""
import dashscope
+ # 1. Format the messages to the format required by DashScope API
+ formatted_msg = await self.formatter.format(messages)
+
+ # 2. Prepare the generation keyword arguments
kwargs = {
- "messages": messages,
- "model": self.model_name,
+ "messages": formatted_msg,
+ "model": model_name,
"stream": self.stream,
"result_format": "message",
# In agentscope, the `incremental_output` must be `True` when
@@ -223,53 +209,27 @@ class DashScopeChatModel(ChatModelBase):
**kwargs,
}
+ # tools
if tools:
kwargs["tools"] = self._format_tools_json_schemas(tools)
+ # tool choice options
if tool_choice:
- # Handle deprecated "any" option with warning
- if tool_choice in ["any", "required"]:
- warnings.warn(
- f"'{tool_choice}' is not supported by DashScope API. "
- "It will be converted to 'auto'.",
- DeprecationWarning,
- )
- tool_choice = "auto"
-
- self._validate_tool_choice(tool_choice, tools)
- kwargs["tool_choice"] = self._format_tool_choice(tool_choice)
-
- if (
- self.enable_thinking is not None
- and "enable_thinking" not in kwargs
- ):
- kwargs["enable_thinking"] = self.enable_thinking
-
- if structured_model:
- if tools or tool_choice:
- logger.warning(
- "structured_model is provided. Both 'tools' and "
- "'tool_choice' parameters will be overridden and "
- "ignored. The model will only perform structured output "
- "generation without calling any other tools.",
- )
- format_tool = _create_tool_from_base_model(structured_model)
- kwargs["tools"] = self._format_tools_json_schemas(
- [format_tool],
- )
kwargs["tool_choice"] = self._format_tool_choice(
- format_tool["function"]["name"],
+ tool_choice,
+ tools,
)
+ # thinking related options
+ kwargs = {**kwargs, **self.thinking_config.model_dump()}
+
+ # 3. Call the API and parse the response
start_datetime = datetime.now()
+ # Use MultiModalConversation API if multimodality is True, or if the
+ # model is multimodal based on the model name.
if self.multimodality or (
self.multimodality is None
- and (
- self.model_name.startswith(
- "qvq",
- )
- or "-vl" in self.model_name
- )
+ and ("qvq" in self.model_name or "-vl" in self.model_name)
):
response = await dashscope.AioMultiModalConversation.call(
api_key=self.api_key,
@@ -286,27 +246,21 @@ class DashScopeChatModel(ChatModelBase):
return self._parse_dashscope_stream_response(
start_datetime,
response,
- structured_model,
)
parsed_response = await self._parse_dashscope_generation_response(
start_datetime,
response,
- structured_model,
)
return parsed_response
- # pylint: disable=too-many-branches, too-many-statements
async def _parse_dashscope_stream_response(
self,
start_datetime: datetime,
- response: Union[
- AsyncGenerator[GenerationResponse, None],
- AsyncGenerator[MultiModalConversationResponse, None],
- Generator[MultiModalConversationResponse, None, None],
- ],
- structured_model: Type[BaseModel] | None = None,
+ response: AsyncGenerator[GenerationResponse, None]
+ | AsyncGenerator[MultiModalConversationResponse, None]
+ | Generator[MultiModalConversationResponse, None, None],
) -> AsyncGenerator[ChatResponse, Any]:
"""Given a DashScope streaming response generator, extract the content
blocks and usages from it and yield ChatResponse objects.
@@ -315,132 +269,117 @@ class DashScopeChatModel(ChatModelBase):
start_datetime (`datetime`):
The start datetime of the response generation.
response (
- `Union[AsyncGenerator[GenerationResponse, None], \
- AsyncGenerator[MultiModalConversationResponse, None], \
- Generator[MultiModalConversationResponse, None, None]]`
+ `AsyncGenerator[GenerationResponse, None] | \
+ AsyncGenerator[MultiModalConversationResponse, None] | \
+ Generator[MultiModalConversationResponse, None, None]`
):
DashScope streaming response (async) generator
(GenerationResponse or MultiModalConversationResponse).
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
- AsyncGenerator[ChatResponse, Any]:
+ `AsyncGenerator[ChatResponse, Any]`:
An async generator that yields ChatResponse objects containing
the content blocks and usage information for each chunk in the
streaming response.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
- acc_content, acc_thinking_content = "", ""
- acc_tool_calls = collections.defaultdict(dict)
- last_input_objs = {} # Store last input_obj for each tool_call
- metadata = None
- last_content = None
+ # All delta should have the same block identifier
+ acc_text = TextBlock(text="")
+ acc_thinking = ThinkingBlock(thinking="")
+ acc_tool_calls = {}
usage = None
response_id: str | None = None
async for chunk in giter(response):
+ # The yield delta content blocks in the current chunk
+ delta_content: list = []
+
if chunk.status_code != HTTPStatus.OK:
raise RuntimeError(
- f"Failed to get response from _ API: {chunk}",
+ f"Failed to get response from DashScope API: {chunk}",
)
- if response_id is None:
- response_id = getattr(chunk, "request_id", None)
+ # Capture response_id from the first chunk if not already set
+ response_id = response_id or chunk.request_id
+ # The message field in the chunk
message = chunk.output.choices[0].message
# Update reasoning content
if isinstance(message.get("reasoning_content"), str):
- acc_thinking_content += message["reasoning_content"]
+ acc_thinking.thinking += message["reasoning_content"]
+ delta_content.append(
+ ThinkingBlock(
+ id=acc_thinking.id,
+ thinking=message["reasoning_content"],
+ ),
+ )
# Update text content
if isinstance(message.content, str):
- acc_content += message.content
+ acc_text.text += message.content
+ delta_content.append(
+ TextBlock(
+ id=acc_text.id,
+ text=message.content,
+ ),
+ )
elif isinstance(message.content, list):
- for item in message.content:
- if isinstance(item, dict) and "text" in item:
- acc_content += item["text"]
+ delta_text = "".join(
+ [
+ _["text"]
+ for _ in message.content
+ if isinstance(_, dict)
+ and isinstance(_.get("text"), str)
+ ],
+ )
+ acc_text.text += delta_text
+ delta_content.append(
+ ThinkingBlock(id=acc_text.id, thinking=delta_text),
+ )
# Update tool calls
- for tool_call in message.get("tool_calls", []):
+ for tool_call in message.get("tool_calls") or []:
+ # Must be a dictionary
+ if not isinstance(tool_call, dict):
+ continue
+
+ # Use the index to identify different tool calls
index = tool_call.get("index", 0)
+ # Avoid appending duplicate id
if "id" in tool_call and tool_call["id"] != acc_tool_calls[
index
].get("id"):
- acc_tool_calls[index]["id"] = (
- acc_tool_calls[index].get("id", "") + tool_call["id"]
- )
+ acc_tool_calls[index]["id"] = tool_call["id"]
- if "function" in tool_call:
+ # Arguments
+ if isinstance(tool_call.get("function"), dict):
func = tool_call["function"]
+
+ # Function name
if "name" in func:
- acc_tool_calls[index]["name"] = (
- acc_tool_calls[index].get("name", "")
- + func["name"]
- )
+ acc_tool_calls[index]["name"] = func["name"]
- if "arguments" in func:
+ # Function input arguments
+ if isinstance(func.get("arguments"), str):
acc_tool_calls[index]["arguments"] = (
acc_tool_calls[index].get("arguments", "")
+ func["arguments"]
)
- # Build content blocks (always include thinking and text)
- content_blocks: list[TextBlock | ToolUseBlock | ThinkingBlock] = []
-
- if acc_thinking_content:
- content_blocks.append(
- ThinkingBlock(
- type="thinking",
- thinking=acc_thinking_content,
- ),
- )
-
- if acc_content:
- content_blocks.append(
- TextBlock(
- type="text",
- text=acc_content,
- ),
- )
-
- for tool_call in acc_tool_calls.values():
- # Only add intermediate tool use blocks if
- # stream_tool_parsing is True
- tool_id = tool_call.get("id", "")
- input_str = tool_call.get("arguments")
-
- # If parsing the tool input in streaming mode
- if self.stream_tool_parsing:
- repaired_input = _parse_streaming_json_dict(
- input_str,
- last_input_objs.get(tool_id),
+ # Append the tool call block for this chunk
+ delta_content.append(
+ ToolCallBlock(
+ id=acc_tool_calls[index].get(
+ "id",
+ uuid.uuid4().hex,
+ ),
+ name=func.get("name", ""),
+ input=func.get("arguments", ""),
+ ),
)
- last_input_objs[tool_id] = repaired_input
-
- else:
- # Otherwise, keep input as empty dict until the final chunk
- repaired_input = {}
-
- content_blocks.append(
- ToolUseBlock(
- type="tool_use",
- id=tool_id,
- name=tool_call.get("name", ""),
- input=repaired_input,
- raw_input=input_str,
- ),
- )
-
- if structured_model:
- metadata = repaired_input
+ # The chunk usage
if chunk.usage:
usage = ChatUsage(
input_tokens=chunk.usage.input_tokens,
@@ -449,40 +388,32 @@ class DashScopeChatModel(ChatModelBase):
metadata=chunk.usage,
)
- if content_blocks:
- _kwargs: dict[str, Any] = {
- "content": content_blocks,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _kwargs["id"] = response_id
- parsed_chunk = ChatResponse(**_kwargs)
- yield parsed_chunk
- last_content = copy.deepcopy(content_blocks)
-
- # If stream_tool_parsing is False, we need to parse the final tool
- # use inputs here
- if not self.stream_tool_parsing and last_content and acc_tool_calls:
- metadata = None
- # Update tool use blocks in last_contents inplace
- for block in last_content:
- if block.get("type") == "tool_use":
- block["input"] = input_obj = _json_loads_with_repair(
- str(block.get("raw_input") or "{}"),
- )
+ # Yield the chat response
+ yield ChatResponse(
+ id=response_id,
+ content=delta_content,
+ is_last=False,
+ usage=usage,
+ )
+
+ # Yield a final complete response with the accumulated content and
+ # final usage
+ content = []
+ if acc_text.text:
+ content.append(acc_text)
- if structured_model:
- metadata = input_obj
+ if acc_thinking.thinking:
+ content.append(acc_thinking)
- _final_kwargs: dict[str, Any] = {
- "content": last_content,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _final_kwargs["id"] = response_id
- yield ChatResponse(**_final_kwargs)
+ if acc_tool_calls:
+ content.extend(acc_tool_calls.values())
+
+ yield ChatResponse(
+ id=response_id or uuid.uuid4().hex,
+ content=content,
+ is_last=True,
+ usage=usage,
+ )
async def _parse_dashscope_generation_response(
self,
@@ -491,7 +422,6 @@ class DashScopeChatModel(ChatModelBase):
GenerationResponse,
MultiModalConversationResponse,
],
- structured_model: Type[BaseModel] | None = None,
) -> ChatResponse:
"""Given a DashScope GenerationResponse object, extract the content
blocks and usages from it.
@@ -504,71 +434,55 @@ class DashScopeChatModel(ChatModelBase):
):
Dashscope GenerationResponse | MultiModalConversationResponse
object to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
- ChatResponse (`ChatResponse`):
+ `ChatResponse`:
A ChatResponse object containing the content blocks and usage.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
# Collect the content blocks from the response.
- if response.status_code != 200:
- raise RuntimeError(response)
+ if response.status_code != HTTPStatus.OK:
+ raise RuntimeError(
+ f"Failed to get response from DashScope API: {response}",
+ )
- content_blocks: List[TextBlock | ToolUseBlock] = []
- metadata: dict | None = None
+ content_blocks: List[TextBlock | ToolCallBlock] = []
message = response.output.choices[0].message
- content = message.get("content")
-
- if response.output.choices[0].message.get("content") not in [
- None,
- "",
- [],
- ]:
- if isinstance(content, list):
- for item in content:
- if isinstance(item, dict) and "text" in item:
- content_blocks.append(
- TextBlock(
- type="text",
- text=item["text"],
- ),
- )
- else:
- content_blocks.append(
- TextBlock(
- type="text",
- text=content,
+
+ # The content and tool calls in the message
+ content, tool_calls = message.get("content"), message.get("tool_calls")
+
+ if isinstance(content, str):
+ content_blocks.append(TextBlock(text=content))
+
+ elif isinstance(content, list):
+ content_blocks.append(
+ TextBlock(
+ text="".join(
+ [
+ _["text"]
+ for _ in content
+ if isinstance(_, dict)
+ and isinstance(_.get("text"), str)
+ ],
),
- )
+ ),
+ )
+
+ if isinstance(tool_calls, list):
+ for tool_call in tool_calls:
+ function = tool_call.get("function") or {}
+ name = function.get("name") or ""
+ input_ = function.get("arguments") or "{}"
- if message.get("tool_calls"):
- for tool_call in message["tool_calls"]:
- input_ = _json_loads_with_repair(
- tool_call["function"].get(
- "arguments",
- "{}",
- )
- or "{}",
- )
content_blocks.append(
- ToolUseBlock(
- type="tool_use",
- name=tool_call["function"]["name"],
+ ToolCallBlock(
+ id=tool_call.get("id", uuid.uuid4().hex),
+ name=name,
input=input_,
- id=tool_call["id"],
),
)
- if structured_model:
- metadata = input_
-
# Usage information
usage = None
if response.usage:
@@ -579,19 +493,15 @@ class DashScopeChatModel(ChatModelBase):
metadata=response.usage,
)
- resp_kwargs: dict[str, Any] = {
- "content": content_blocks,
- "usage": usage,
- "metadata": metadata,
- }
- response_id = getattr(response, "request_id", None)
- if response_id:
- resp_kwargs["id"] = response_id
-
- return ChatResponse(**resp_kwargs)
+ return ChatResponse(
+ id=getattr(response, "request_id", uuid.uuid4().hex),
+ content=content_blocks,
+ is_last=True,
+ usage=usage,
+ )
+ @staticmethod
def _format_tools_json_schemas(
- self,
schemas: list[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Format the tools JSON schema into required format for DashScope API.
@@ -617,26 +527,37 @@ class DashScopeChatModel(ChatModelBase):
def _format_tool_choice(
self,
- tool_choice: Literal["auto", "none", "required"] | str | None,
+ tool_choice: ToolChoice | None,
+ tools: list[dict] | None,
) -> str | dict | None:
"""Format tool_choice parameter for API compatibility.
Args:
- tool_choice (`Literal["auto", "none", "required"] | str \
- | None`, default `None`):
- Controls which (if any) tool is called by the model. For more
- details, please refer to
- https://help.aliyun.com/zh/model-studio/qwen-function-calling
+ tool_choice (`ToolChoice | None`):
+ The unified tool choice parameter which can be a mode ("auto",
+ "none", "required") or a specific function name.
+ tools (`list[dict] | None`):
+ The list of available tools, used for validation if
+ tool_choice is a specific function name.
Returns:
`dict | None`:
The formatted tool choice configuration dict, or None if
tool_choice is None.
"""
- if tool_choice is None:
- return None
+ # Validate the tool_choice value
+ self._validate_tool_choice(tool_choice, tools)
+
+ # DashScope API specific validation and formatting
if tool_choice in ["auto", "none"]:
return tool_choice
+
if tool_choice == "required":
+ warnings.warn(
+ f"'{tool_choice}' is not supported by DashScope API. "
+ "It will be converted to 'auto'.",
+ DeprecationWarning,
+ )
return "auto"
+
return {"type": "function", "function": {"name": tool_choice}}
diff --git a/src/agentscope/model/_gemini_model.py b/src/agentscope/model/_gemini_model.py
index 0a393b69d..e5d2bd33e 100644
--- a/src/agentscope/model/_gemini_model.py
+++ b/src/agentscope/model/_gemini_model.py
@@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
-# mypy: disable-error-code="dict-item"
"""The Google Gemini model in agentscope."""
import base64
import copy
-import warnings
from datetime import datetime
import json
from typing import (
@@ -12,20 +10,19 @@ from typing import (
TYPE_CHECKING,
AsyncIterator,
Literal,
- Type,
List,
)
from pydantic import BaseModel
from .._logging import logger
-from .._utils._common import _json_loads_with_repair
-from ..message import ToolUseBlock, TextBlock, ThinkingBlock
+from ..formatter import FormatterBase
+from ..message import ToolCallBlock, TextBlock, ThinkingBlock
from ._model_usage import ChatUsage
from ._model_base import ChatModelBase
from ._model_response import ChatResponse
from ..tracing import trace_llm
-from ..types import JSONSerializableObject
+from ..types import JSONSerializableObject, ToolChoice
if TYPE_CHECKING:
from google.genai.types import GenerateContentResponse
@@ -115,15 +112,23 @@ def _flatten_json_schema(schema: dict) -> dict:
class GeminiChatModel(ChatModelBase):
"""The Google Gemini chat model class in agentscope."""
+ class ThinkingConfig(BaseModel):
+ """Configuration for enabling thinking blocks in Gemini responses."""
+
+ enable_thinking: bool
+ thinking_budget: int = 1024
+
def __init__(
self,
model_name: str,
api_key: str,
stream: bool = True,
- thinking_config: dict | None = None,
+ max_retries: int = 0,
+ fallback_model_name: str | None = None,
+ formatter: FormatterBase | None = None,
+ thinking_config: ThinkingConfig | None = None,
client_kwargs: dict[str, JSONSerializableObject] | None = None,
generate_kwargs: dict[str, JSONSerializableObject] | None = None,
- **kwargs: Any,
) -> None:
"""Initialize the Gemini chat model.
@@ -134,19 +139,14 @@ class GeminiChatModel(ChatModelBase):
The API key for Google Gemini.
stream (`bool`, default `True`):
Whether to use streaming output or not.
- thinking_config (`dict | None`, optional):
+ max_retries (`int`, optional):
+ Maximum number of retries on failure. Defaults to 0.
+ fallback_model_name (`str | None`, optional):
+ Fallback model name to use after all retries fail.
+ formatter (`FormatterBase | None`, optional):
+ Formatter for message preprocessing.
+ thinking_config (`ThinkingConfig | None`, optional):
Thinking config, supported models are 2.5 Pro, 2.5 Flash, etc.
- Refer to https://ai.google.dev/gemini-api/docs/thinking for
- more details.
-
- .. code-block:: python
- :caption: Example of thinking_config
-
- {
- "include_thoughts": True, # enable thoughts or not
- "thinking_budget": 1024 # Max tokens for reasoning
- }
-
client_kwargs (`dict[str, JSONSerializableObject] | None`, \
optional):
The extra keyword arguments to initialize the Gemini client.
@@ -154,33 +154,8 @@ class GeminiChatModel(ChatModelBase):
optional):
The extra keyword arguments used in Gemini API generation,
e.g. `temperature`, `seed`.
- **kwargs (`Any`):
- Additional keyword arguments.
"""
- # Handle deprecated client_args parameter from kwargs
- client_args = kwargs.pop("client_args", None)
- if client_args is not None and client_kwargs is not None:
- raise ValueError(
- "Cannot specify both 'client_args' and 'client_kwargs'. "
- "Please use only 'client_kwargs' (client_args is deprecated).",
- )
-
- if client_args is not None:
- logger.warning(
- "The parameter 'client_args' is deprecated and will be "
- "removed in a future version. Please use 'client_kwargs' "
- "instead. Automatically converting 'client_args' to "
- "'client_kwargs'.",
- )
- client_kwargs = client_args
-
- if kwargs:
- logger.warning(
- "Unknown keyword arguments: %s. These will be ignored.",
- list(kwargs.keys()),
- )
-
try:
from google import genai
except ImportError as e:
@@ -189,7 +164,13 @@ class GeminiChatModel(ChatModelBase):
"`pip install -q -U google-genai`",
) from e
- super().__init__(model_name, stream)
+ super().__init__(
+ model_name=model_name,
+ stream=stream,
+ max_retries=max_retries,
+ fallback_model_name=fallback_model_name,
+ formatter=formatter,
+ )
self.client = genai.Client(
api_key=api_key,
@@ -199,12 +180,12 @@ class GeminiChatModel(ChatModelBase):
self.generate_kwargs = generate_kwargs or {}
@trace_llm
- async def __call__(
+ async def _call_api(
self,
+ model_name: str,
messages: list[dict],
tools: list[dict] | None = None,
tool_choice: Literal["auto", "none", "required"] | str | None = None,
- structured_model: Type[BaseModel] | None = None,
**config_kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
"""Call the Gemini model with the provided arguments.
@@ -221,59 +202,32 @@ class GeminiChatModel(ChatModelBase):
Can be "auto", "none", "required", or specific tool name.
For more details, please refer to
https://ai.google.dev/gemini-api/docs/function-calling?hl=en&example=meeting#function_calling_modes
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
-
- .. note:: When `structured_model` is specified,
- both `tools` and `tool_choice` parameters are ignored,
- and the model will only perform structured output
- generation without calling any other tools.
-
- For more details, please refer to
- https://ai.google.dev/gemini-api/docs/structured-output
-
**config_kwargs (`Any`):
The keyword arguments for Gemini chat completions API.
"""
config: dict = {
- "thinking_config": self.thinking_config,
**self.generate_kwargs,
**config_kwargs,
}
+ if self.thinking_config:
+ config["thinking_config"] = {
+ "include_thoughts": self.thinking_config.enable_thinking,
+ "thinking_budget": self.thinking_config.thinking_budget,
+ }
if tools:
config["tools"] = self._format_tools_json_schemas(tools)
if tool_choice:
- # Handle deprecated "any" option with warning
- if tool_choice == "any":
- warnings.warn(
- '"any" is deprecated and will be removed in a future '
- "version.",
- DeprecationWarning,
- )
- tool_choice = "required"
- self._validate_tool_choice(tool_choice, tools)
- config["tool_config"] = self._format_tool_choice(tool_choice)
-
- if structured_model:
- if tools or tool_choice:
- logger.warning(
- "structured_model is provided. Both 'tools' and "
- "'tool_choice' parameters will be overridden and "
- "ignored. The model will only perform structured output "
- "generation without calling any other tools.",
- )
- config.pop("tools", None)
- config.pop("tool_config", None)
- config["response_mime_type"] = "application/json"
- config["response_schema"] = structured_model
+ config["tool_config"] = self._format_tool_choice(
+ tool_choice,
+ tools,
+ )
# Prepare the arguments for the Gemini API call
- kwargs: dict[str, JSONSerializableObject] = {
- "model": self.model_name,
+ kwargs: dict = {
+ "model": model_name,
"contents": messages,
"config": config,
}
@@ -287,7 +241,6 @@ class GeminiChatModel(ChatModelBase):
return self._parse_gemini_stream_generation_response(
start_datetime,
response,
- structured_model,
)
# non-streaming
@@ -298,7 +251,6 @@ class GeminiChatModel(ChatModelBase):
parsed_response = self._parse_gemini_generation_response(
start_datetime,
response,
- structured_model,
)
return parsed_response
@@ -338,7 +290,6 @@ class GeminiChatModel(ChatModelBase):
self,
start_datetime: datetime,
response: AsyncIterator[GenerateContentResponse],
- structured_model: Type[BaseModel] | None = None,
) -> AsyncGenerator[ChatResponse, None]:
"""Given a Gemini streaming generation response, extract the
content blocks and usages from it and yield ChatResponse objects.
@@ -348,27 +299,23 @@ class GeminiChatModel(ChatModelBase):
The start datetime of the response generation.
response (`AsyncIterator[GenerateContentResponse]`):
Gemini GenerateContentResponse async iterator to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
`AsyncGenerator[ChatResponse, None]`:
An async generator that yields ChatResponse objects containing
the content blocks and usage information for each chunk in the
streaming response.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
- text = ""
- thinking = ""
- tool_calls: list[ToolUseBlock] = []
- metadata: dict | None = None
- response_id: str | None = None
+ # Accumulated state
+ acc_text = ""
+ acc_thinking = ""
+ acc_tool_calls: dict = {} # call_id -> {name, input}
+ usage = None
+
async for chunk in response:
+ delta_content: list = []
+
if (
chunk.candidates
and chunk.candidates[0].content
@@ -377,20 +324,16 @@ class GeminiChatModel(ChatModelBase):
for part in chunk.candidates[0].content.parts:
if part.text:
if part.thought:
- thinking += part.text
+ acc_thinking += part.text
+ delta_content.append(
+ ThinkingBlock(thinking=part.text),
+ )
else:
- text += part.text
+ acc_text += part.text
+ delta_content.append(TextBlock(text=part.text))
if part.function_call:
keyword_args = part.function_call.args or {}
- # .. note:: Gemini API always returns None for
- # function_call.id, so we use thought_signature
- # as the unique identifier for tool
- # calls when available. That maybe
- # infeasible someday, but Gemini
- # requires the thought_signature for some
- # llms like gemini-3-pro
-
if part.thought_signature:
call_id = base64.b64encode(
part.thought_signature,
@@ -398,61 +341,56 @@ class GeminiChatModel(ChatModelBase):
else:
call_id = part.function_call.id
- tool_calls.append(
- ToolUseBlock(
- type="tool_use",
+ input_str = json.dumps(
+ keyword_args,
+ ensure_ascii=False,
+ )
+ acc_tool_calls[call_id] = {
+ "name": part.function_call.name,
+ "input": input_str,
+ }
+ delta_content.append(
+ ToolCallBlock(
id=call_id,
name=part.function_call.name,
- input=keyword_args,
- raw_input=json.dumps(
- keyword_args,
- ensure_ascii=False,
- ),
+ input=input_str,
),
)
- # Text parts
- if text and structured_model:
- metadata = _json_loads_with_repair(text)
-
usage = self._extract_usage(chunk.usage_metadata, start_datetime)
- # The content blocks for the current chunk
- content_blocks: list = []
-
- if thinking:
- content_blocks.append(
- ThinkingBlock(
- type="thinking",
- thinking=thinking,
- ),
- )
-
- if text:
- content_blocks.append(
- TextBlock(
- type="text",
- text=text,
- ),
+ if delta_content:
+ yield ChatResponse(
+ content=delta_content,
+ is_last=False,
+ usage=usage,
)
- if response_id is None:
- response_id = getattr(chunk, "response_id", None)
+ # Build final accumulated content
+ final_content: list = []
+ if acc_thinking:
+ final_content.append(ThinkingBlock(thinking=acc_thinking))
+ if acc_text:
+ final_content.append(TextBlock(text=acc_text))
+ for call_id, tc in acc_tool_calls.items():
+ final_content.append(
+ ToolCallBlock(
+ id=call_id,
+ name=tc["name"],
+ input=tc["input"],
+ ),
+ )
- _kwargs: dict[str, Any] = {
- "content": content_blocks + tool_calls,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _kwargs["id"] = response_id
- yield ChatResponse(**_kwargs)
+ yield ChatResponse(
+ content=final_content,
+ is_last=True,
+ usage=usage,
+ )
def _parse_gemini_generation_response(
self,
start_datetime: datetime,
response: GenerateContentResponse,
- structured_model: Type[BaseModel] | None = None,
) -> ChatResponse:
"""Given a Gemini chat completion response object, extract the content
blocks and usages from it.
@@ -462,21 +400,12 @@ class GeminiChatModel(ChatModelBase):
The start datetime of the response generation.
response (`GenerateContentResponse`):
The Gemini generation response object to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
ChatResponse (`ChatResponse`):
A ChatResponse object containing the content blocks and usage.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
- content_blocks: List[TextBlock | ToolUseBlock | ThinkingBlock] = []
- metadata: dict | None = None
- tool_calls: list = []
+ content_blocks: List[TextBlock | ToolCallBlock | ThinkingBlock] = []
if (
response.candidates
@@ -487,28 +416,15 @@ class GeminiChatModel(ChatModelBase):
if part.text:
if part.thought:
content_blocks.append(
- ThinkingBlock(
- type="thinking",
- thinking=part.text,
- ),
+ ThinkingBlock(thinking=part.text),
)
else:
content_blocks.append(
- TextBlock(
- type="text",
- text=part.text,
- ),
+ TextBlock(text=part.text),
)
if part.function_call:
keyword_args = part.function_call.args or {}
- # .. note:: Gemini API always returns None for
- # function_call.id, so we use thought_signature
- # as the unique identifier for tool
- # calls when available. That maybe infeasible
- # someday, but Gemini requires the thought_signature
- # for some llms like gemini-3-pro
-
if part.thought_signature:
call_id = base64.b64encode(
part.thought_signature,
@@ -516,35 +432,21 @@ class GeminiChatModel(ChatModelBase):
else:
call_id = part.function_call.id
- tool_calls.append(
- ToolUseBlock(
- type="tool_use",
+ content_blocks.append(
+ ToolCallBlock(
id=call_id,
name=part.function_call.name,
- input=keyword_args,
- raw_input=json.dumps(
- keyword_args,
- ensure_ascii=False,
- ),
+ input=json.dumps(keyword_args, ensure_ascii=False),
),
)
- # For the structured output case
- if response.text and structured_model:
- metadata = _json_loads_with_repair(response.text)
-
usage = self._extract_usage(response.usage_metadata, start_datetime)
- resp_kwargs: dict[str, Any] = {
- "content": content_blocks + tool_calls,
- "usage": usage,
- "metadata": metadata,
- }
- response_id = getattr(response, "response_id", None)
- if response_id:
- resp_kwargs["id"] = response_id
-
- return ChatResponse(**resp_kwargs)
+ return ChatResponse(
+ content=content_blocks,
+ is_last=True,
+ usage=usage,
+ )
def _format_tools_json_schemas(
self,
@@ -637,23 +539,26 @@ class GeminiChatModel(ChatModelBase):
def _format_tool_choice(
self,
- tool_choice: Literal["auto", "none", "required"] | str | None,
+ tool_choice: ToolChoice | None,
+ tools: list[dict] | None,
) -> dict | None:
"""Format tool_choice parameter for API compatibility.
Args:
- tool_choice (`Literal["auto", "none", "required"] | str | None`, \
- default `None`):
- Controls which (if any) tool is called by the model.
- Can be "auto", "none", "required", or specific tool name.
- For more details, please refer to
- https://ai.google.dev/gemini-api/docs/function-calling?hl=en&example=meeting#function_calling_modes
+ tool_choice (`ToolChoice | None`):
+ The unified tool choice parameter which can be a mode ("auto",
+ "none", "required") or a specific function name.
+ tools (`list[dict] | None`):
+ The list of available tools, used for validation if
+ tool_choice is a specific function name.
Returns:
`dict | None`:
The formatted tool choice configuration dict, or None if
tool_choice is None.
"""
+ self._validate_tool_choice(tool_choice, tools)
+
if tool_choice is None:
return None
diff --git a/src/agentscope/model/_model_base.py b/src/agentscope/model/_model_base.py
index ff2503cad..7553e2057 100644
--- a/src/agentscope/model/_model_base.py
+++ b/src/agentscope/model/_model_base.py
@@ -5,7 +5,10 @@ from abc import abstractmethod
from typing import AsyncGenerator, Any
from ._model_response import ChatResponse
-
+from .._logging import logger
+from ..formatter import FormatterBase
+from ..message import Msg
+from ..types import ToolChoice
_TOOL_CHOICE_MODES = ["auto", "none", "required"]
@@ -19,10 +22,23 @@ class ChatModelBase:
stream: bool
"""Is the model output streaming or not"""
+ max_retries: int
+ """Maximum number of retries on failure"""
+
+ fallback_model_name: str | None
+ """Fallback model name to use after all retries fail"""
+
+ formatter: Any | None
+ """The API formatter that format the messages into the required format for
+ the underlying API."""
+
def __init__(
self,
model_name: str,
stream: bool,
+ max_retries: int = 0,
+ fallback_model_name: str | None = None,
+ formatter: FormatterBase | None = None,
) -> None:
"""Initialize the chat model base class.
@@ -31,37 +47,130 @@ class ChatModelBase:
The name of the model
stream (`bool`):
Whether the model output is streaming or not
+ max_retries (`int`, optional):
+ Maximum number of retries on failure. Defaults to 0.
+ fallback_model_name (`str | None`, optional):
+ Fallback model name to use after all retries fail.
+ formatter (`FormatterBase | None`, optional):
+ Formatter for message preprocessing.
"""
self.model_name = model_name
self.stream = stream
+ self.max_retries = max_retries
+ self.fallback_model_name = fallback_model_name
+ self.formatter = formatter
- @abstractmethod
async def __call__(
self,
- *args: Any,
+ messages: list[Msg],
+ tools: list[dict] | None = None,
+ tool_choice: ToolChoice | None = None,
+ **kwargs: Any,
+ ) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
+ """Call the model with retry and fallback logic.
+
+ Formats messages using formatter if available, then attempts
+ to call the model up to max_retries + 1 times. If all attempts
+ fail and a fallback model is configured, retries with that model.
+
+ Args:
+ messages (`list[Msg]`):
+ The messages to send to the model.
+ tools (`list[dict] | None`, optional):
+ The tools available to the model.
+ tool_choice (`ToolChoice | None`, optional):
+ The tool choice mode or function name.
+ **kwargs:
+ Additional keyword arguments passed to the underlying API.
+ """
+ if self.formatter is not None:
+ messages = await self.formatter.format(messages)
+
+ last_error: Exception | None = None
+
+ for model_name in self._models_to_try():
+ for attempt in range(self.max_retries + 1):
+ try:
+ return await self._call_api(
+ model_name,
+ messages=messages,
+ tools=tools,
+ tool_choice=tool_choice,
+ **kwargs,
+ )
+ except Exception as e:
+ last_error = e
+ if attempt < self.max_retries:
+ logger.warning(
+ "Attempt %d failed for model %s: %s. Retrying...",
+ attempt + 1,
+ model_name,
+ str(e),
+ )
+ else:
+ logger.warning(
+ "All %d attempt(s) failed for model %s.",
+ self.max_retries + 1,
+ model_name,
+ )
+
+ if last_error is not None:
+ raise last_error
+ raise RuntimeError("No models to try")
+
+ def _models_to_try(self) -> list[str]:
+ """Return the ordered list of model names to try."""
+ models = [self.model_name]
+ if self.fallback_model_name:
+ models.append(self.fallback_model_name)
+ return models
+
+ @abstractmethod
+ async def _call_api(
+ self,
+ model_name: str,
+ messages: list[Msg],
+ tools: list[dict] | None = None,
+ tool_choice: ToolChoice | None = None,
**kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
- pass
+ """Call the underlying API. Subclasses must implement this method.
+
+ Args:
+ model_name (`str`):
+ The model name to use for this call.
+ messages (`list[Msg]`):
+ The messages to send to the model.
+ tools (`list[dict] | None`, optional):
+ The tools available to the model.
+ tool_choice (`ToolChoice | None`, optional):
+ The tool choice mode or function name.
+ **kwargs:
+ Additional keyword arguments for the underlying API.
+ """
def _validate_tool_choice(
self,
- tool_choice: str,
+ tool_choice: ToolChoice | None,
tools: list[dict] | None,
) -> None:
- """
- Validate tool_choice parameter.
+ """Validate tool_choice parameter.
Args:
- tool_choice (`str`):
+ tool_choice (`ToolChoice | None`):
Tool choice mode or function name
tools (`list[dict] | None`):
Available tools list
+
Raises:
- TypeError: If tool_choice is not string
- ValueError: If tool_choice is invalid
+ `ValueError`:
+ If tool_choice is not a valid mode or function name.
"""
+ if tool_choice is None:
+ return
+
if not isinstance(tool_choice, str):
- raise TypeError(
+ raise ValueError(
f"tool_choice must be str, got {type(tool_choice)}",
)
if tool_choice in _TOOL_CHOICE_MODES:
diff --git a/src/agentscope/model/_model_response.py b/src/agentscope/model/_model_response.py
index 6f8e52f61..985091b55 100644
--- a/src/agentscope/model/_model_response.py
+++ b/src/agentscope/model/_model_response.py
@@ -1,17 +1,17 @@
# -*- coding: utf-8 -*-
"""The model response module."""
-
+import uuid
from dataclasses import dataclass, field
+from datetime import datetime
from typing import Literal, Sequence
from ._model_usage import ChatUsage
-from .._utils._common import _get_timestamp
from .._utils._mixin import DictMixin
from ..message import (
TextBlock,
- ToolUseBlock,
+ ToolCallBlock,
ThinkingBlock,
- AudioBlock,
+ DataBlock,
)
from ..types import JSONSerializableObject
@@ -20,14 +20,18 @@ from ..types import JSONSerializableObject
class ChatResponse(DictMixin):
"""The response of chat models."""
- content: Sequence[TextBlock | ToolUseBlock | ThinkingBlock | AudioBlock]
+ content: Sequence[TextBlock | ToolCallBlock | ThinkingBlock | DataBlock]
"""The content of the chat response, which can include text blocks,
tool use blocks, or thinking blocks."""
- id: str = field(default_factory=lambda: _get_timestamp(True))
+ is_last: bool
+ """Whether this response is the last response, if `Ture`, the content will
+ be the complete response, otherwise the content is a partial response"""
+
+ id: str = field(default_factory=lambda: uuid.uuid4().hex)
"""The unique identifier formatter """
- created_at: str = field(default_factory=_get_timestamp)
+ created_at: str = field(default_factory=datetime.now().isoformat)
"""When the response was created"""
type: Literal["chat"] = field(default_factory=lambda: "chat")
diff --git a/src/agentscope/model/_ollama_model.py b/src/agentscope/model/_ollama_model.py
index 6a86fa1a1..09d3190bb 100644
--- a/src/agentscope/model/_ollama_model.py
+++ b/src/agentscope/model/_ollama_model.py
@@ -9,9 +9,7 @@ from typing import (
AsyncGenerator,
AsyncIterator,
Literal,
- Type,
)
-from collections import OrderedDict
from pydantic import BaseModel
@@ -19,8 +17,8 @@ from . import ChatResponse
from ._model_base import ChatModelBase
from ._model_usage import ChatUsage
from .._logging import logger
-from .._utils._common import _json_loads_with_repair
-from ..message import ToolUseBlock, TextBlock, ThinkingBlock
+from ..formatter import FormatterBase, OllamaChatFormatter
+from ..message import ToolCallBlock, TextBlock, ThinkingBlock
from ..tracing import trace_llm
from ..types import JSONSerializableObject
@@ -33,17 +31,24 @@ else:
class OllamaChatModel(ChatModelBase):
"""The Ollama chat model class in agentscope."""
+ class ThinkingConfig(BaseModel):
+ """Configuration for thinking in Ollama chat models."""
+
+ enable_thinking: bool
+
def __init__(
self,
model_name: str,
stream: bool = False,
+ max_retries: int = 0,
+ fallback_model_name: str | None = None,
+ formatter: FormatterBase | None = None,
+ thinking_config: ThinkingConfig | None = None,
options: dict = None,
keep_alive: str = "5m",
- enable_thinking: bool | None = None,
host: str | None = None,
client_kwargs: dict[str, JSONSerializableObject] | None = None,
generate_kwargs: dict[str, JSONSerializableObject] | None = None,
- **kwargs: Any,
) -> None:
"""Initialize the Ollama chat model.
@@ -52,6 +57,15 @@ class OllamaChatModel(ChatModelBase):
The name of the model.
stream (`bool`, default `True`):
Streaming mode or not.
+ max_retries (`int`, optional):
+ Maximum number of retries on failure. Defaults to 0.
+ fallback_model_name (`str | None`, optional):
+ Fallback model name to use after all retries fail.
+ formatter (`FormatterBase | None`, optional):
+ Formatter for message preprocessing.
+ thinking_config (`ThinkingConfig | None`, default `None`)
+ Configuration for thinking, only for models such as qwen3,
+ deepseek-r1, etc.
options (`dict`, default `None`):
Additional parameters to pass to the Ollama API. These can
include temperature etc.
@@ -59,10 +73,6 @@ class OllamaChatModel(ChatModelBase):
Duration to keep the model loaded in memory. The format is a
number followed by a unit suffix (s for seconds, m for minutes
, h for hours).
- enable_thinking (`bool | None`, default `None`)
- Whether enable thinking or not, only for models such as qwen3,
- deepseek-r1, etc. For more details, please refer to
- https://ollama.com/search?c=thinking
host (`str | None`, default `None`):
The host address of the Ollama server. If None, uses the
default address (typically http://localhost:11434).
@@ -72,9 +82,6 @@ class OllamaChatModel(ChatModelBase):
generate_kwargs (`dict[str, JSONSerializableObject] | None`, \
optional):
The extra keyword arguments used in Ollama API generation.
- **kwargs (`Any`):
- Additional keyword arguments to pass to the base chat model
- class.
"""
try:
@@ -85,31 +92,38 @@ class OllamaChatModel(ChatModelBase):
'running command `pip install "ollama>=0.1.7"`',
) from e
- super().__init__(model_name, stream)
+ super().__init__(
+ model_name=model_name,
+ stream=stream,
+ max_retries=max_retries,
+ fallback_model_name=fallback_model_name,
+ formatter=formatter or OllamaChatFormatter(),
+ )
self.client = ollama.AsyncClient(
host=host,
**(client_kwargs or {}),
- **kwargs,
)
self.options = options
self.keep_alive = keep_alive
- self.think = enable_thinking
+ self.thinking_config = thinking_config
self.generate_kwargs = generate_kwargs or {}
@trace_llm
- async def __call__(
+ async def _call_api(
self,
+ model_name: str,
messages: list[dict[str, Any]],
tools: list[dict] | None = None,
tool_choice: Literal["auto", "none", "required"] | str | None = None,
- structured_model: Type[BaseModel] | None = None,
**kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
"""Get the response from Ollama chat completions API by the given
arguments.
Args:
+ model_name (`str`):
+ The model name to use for this call.
messages (`list[dict]`):
A list of dictionaries, where `role` and `content` fields are
required, and `name` field is optional.
@@ -118,9 +132,6 @@ class OllamaChatModel(ChatModelBase):
tool_choice (`Literal["auto", "none", "required"] | str \
| None`, default `None`):
Ollama doesn't support `tool_choice` argument yet.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
**kwargs (`Any`):
The keyword arguments for Ollama chat completions API,
e.g. `think`etc. Please refer to the Ollama API
@@ -132,7 +143,7 @@ class OllamaChatModel(ChatModelBase):
"""
kwargs = {
- "model": self.model_name,
+ "model": model_name,
"messages": messages,
"stream": self.stream,
"options": self.options,
@@ -141,8 +152,8 @@ class OllamaChatModel(ChatModelBase):
**kwargs,
}
- if self.think is not None and "think" not in kwargs:
- kwargs["think"] = self.think
+ if self.thinking_config and "think" not in kwargs:
+ kwargs["think"] = self.thinking_config.enable_thinking
if tools:
kwargs["tools"] = self._format_tools_json_schemas(tools)
@@ -150,9 +161,6 @@ class OllamaChatModel(ChatModelBase):
if tool_choice:
logger.warning("Ollama does not support tool_choice yet, ignored.")
- if structured_model:
- kwargs["format"] = structured_model.model_json_schema()
-
start_datetime = datetime.now()
response = await self.client.chat(**kwargs)
@@ -160,13 +168,11 @@ class OllamaChatModel(ChatModelBase):
return self._parse_ollama_stream_completion_response(
start_datetime,
response,
- structured_model,
)
parsed_response = await self._parse_ollama_completion_response(
start_datetime,
response,
- structured_model,
)
return parsed_response
@@ -175,7 +181,6 @@ class OllamaChatModel(ChatModelBase):
self,
start_datetime: datetime,
response: AsyncIterator[OllamaChatResponse],
- structured_model: Type[BaseModel] | None = None,
) -> AsyncGenerator[ChatResponse, None]:
"""Given an Ollama streaming completion response, extract the
content blocks and usages from it and yield ChatResponse objects.
@@ -185,9 +190,6 @@ class OllamaChatModel(ChatModelBase):
The start datetime of the response generation.
response (`AsyncIterator[OllamaChatResponse]`):
Ollama streaming response async iterator to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
AsyncGenerator[ChatResponse, None]:
@@ -195,94 +197,94 @@ class OllamaChatModel(ChatModelBase):
the content blocks and usage information for each chunk in the
streaming response.
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
-
"""
- accumulated_text = ""
- acc_thinking_content = ""
- tool_calls = OrderedDict() # Store tool calls
- metadata: dict | None = None
+ # Accumulated state
+ acc_text = ""
+ acc_thinking = ""
+ acc_tool_calls: dict = {} # tool_id -> {name, input}
response_id: str | None = None
+ usage = None
async for chunk in response:
- # Handle text content
+ delta_content: list = []
msg = chunk.message
- acc_thinking_content += msg.thinking or ""
- accumulated_text += msg.content or ""
+
+ # Handle thinking delta
+ if msg.thinking:
+ acc_thinking += msg.thinking
+ delta_content.append(ThinkingBlock(thinking=msg.thinking))
+
+ # Handle text delta
+ if msg.content:
+ acc_text += msg.content
+ delta_content.append(TextBlock(text=msg.content))
# Handle tool calls
for idx, tool_call in enumerate(msg.tool_calls or []):
function = tool_call.function
tool_id = f"{idx}_{function.name}"
- tool_calls[tool_id] = {
- "type": "tool_use",
- "id": tool_id,
+ input_str = json.dumps(function.arguments)
+ acc_tool_calls[tool_id] = {
"name": function.name,
- "input": function.arguments,
- "raw_input": json.dumps(function.arguments),
+ "input": input_str,
}
- # Calculate usage statistics
+ delta_content.append(
+ ToolCallBlock(
+ id=tool_id,
+ name=function.name,
+ input=input_str,
+ ),
+ )
+
+ if response_id is None:
+ response_id = getattr(chunk, "id", None)
+
+ # Calculate usage
current_time = (datetime.now() - start_datetime).total_seconds()
usage = ChatUsage(
input_tokens=getattr(chunk, "prompt_eval_count", 0) or 0,
output_tokens=getattr(chunk, "eval_count", 0) or 0,
time=current_time,
)
- # Create content blocks
- contents: list = []
-
- if acc_thinking_content:
- contents.append(
- ThinkingBlock(
- type="thinking",
- thinking=acc_thinking_content,
- ),
- )
- if accumulated_text:
- contents.append(TextBlock(type="text", text=accumulated_text))
- if structured_model:
- metadata = _json_loads_with_repair(accumulated_text)
-
- # Add tool call blocks
- for tool_call in tool_calls.values():
- try:
- input_data = tool_call["input"]
- if isinstance(input_data, str):
- input_data = _json_loads_with_repair(input_data)
- contents.append(
- ToolUseBlock(
- type=tool_call["type"],
- id=tool_call["id"],
- name=tool_call["name"],
- input=input_data,
- ),
- )
- except Exception as e:
- print(f"Error parsing tool call input: {e}")
-
- if response_id is None:
- response_id = getattr(chunk, "id", None)
-
- # Generate response when there's new content or at final chunk
- if chunk.done or contents:
+ if delta_content:
_kwargs: dict[str, Any] = {
- "content": contents,
+ "content": delta_content,
+ "is_last": False,
"usage": usage,
- "metadata": metadata,
}
if response_id:
_kwargs["id"] = response_id
- res = ChatResponse(**_kwargs)
- yield res
+ yield ChatResponse(**_kwargs)
+
+ # Build final accumulated content
+ final_content: list = []
+ if acc_thinking:
+ final_content.append(ThinkingBlock(thinking=acc_thinking))
+ if acc_text:
+ final_content.append(TextBlock(text=acc_text))
+ for tool_id, tc in acc_tool_calls.items():
+ final_content.append(
+ ToolCallBlock(
+ id=tool_id,
+ name=tc["name"],
+ input=tc["input"],
+ ),
+ )
+
+ _final_kwargs: dict[str, Any] = {
+ "content": final_content,
+ "is_last": True,
+ "usage": usage,
+ }
+ if response_id:
+ _final_kwargs["id"] = response_id
+ yield ChatResponse(**_final_kwargs)
async def _parse_ollama_completion_response(
self,
start_datetime: datetime,
response: OllamaChatResponse,
- structured_model: Type[BaseModel] | None = None,
) -> ChatResponse:
"""Given an Ollama chat completion response object, extract the content
blocks and usages from it.
@@ -292,49 +294,29 @@ class OllamaChatModel(ChatModelBase):
The start datetime of the response generation.
response (`OllamaChatResponse`):
Ollama OllamaChatResponse object to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
`ChatResponse`:
A ChatResponse object containing the content blocks and usage.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
- content_blocks: List[TextBlock | ToolUseBlock | ThinkingBlock] = []
- metadata: dict | None = None
+ content_blocks: List[TextBlock | ToolCallBlock | ThinkingBlock] = []
if response.message.thinking:
content_blocks.append(
- ThinkingBlock(
- type="thinking",
- thinking=response.message.thinking,
- ),
+ ThinkingBlock(thinking=response.message.thinking),
)
if response.message.content:
content_blocks.append(
- TextBlock(
- type="text",
- text=response.message.content,
- ),
+ TextBlock(text=response.message.content),
)
- if structured_model:
- metadata = _json_loads_with_repair(
- response.message.content,
- )
for idx, tool_call in enumerate(response.message.tool_calls or []):
content_blocks.append(
- ToolUseBlock(
- type="tool_use",
+ ToolCallBlock(
id=f"{idx}_{tool_call.function.name}",
name=tool_call.function.name,
- input=tool_call.function.arguments,
- raw_input=json.dumps(tool_call.function.arguments),
+ input=json.dumps(tool_call.function.arguments),
),
)
@@ -348,8 +330,8 @@ class OllamaChatModel(ChatModelBase):
resp_kwargs: dict[str, Any] = {
"content": content_blocks,
+ "is_last": True,
"usage": usage,
- "metadata": metadata,
}
response_id = getattr(response, "id", None)
if response_id:
diff --git a/src/agentscope/model/_openai_model.py b/src/agentscope/model/_openai_model.py
index 19752c1fa..1b6da8fb7 100644
--- a/src/agentscope/model/_openai_model.py
+++ b/src/agentscope/model/_openai_model.py
@@ -1,7 +1,5 @@
# -*- coding: utf-8 -*-
-# pylint: disable=too-many-branches
"""OpenAI Chat model class."""
-import copy
import warnings
from datetime import datetime
from typing import (
@@ -10,29 +8,25 @@ from typing import (
List,
AsyncGenerator,
Literal,
- Type,
)
from collections import OrderedDict
+from openai.types import ReasoningEffort
from pydantic import BaseModel
from . import ChatResponse
from ._model_base import ChatModelBase
from ._model_usage import ChatUsage
-from .._logging import logger
-from .._utils._common import (
- _json_loads_with_repair,
- _parse_streaming_json_dict,
-)
+from ..formatter import FormatterBase
from ..message import (
- ToolUseBlock,
TextBlock,
ThinkingBlock,
- AudioBlock,
Base64Source,
+ DataBlock,
+ ToolCallBlock,
)
from ..tracing import trace_llm
-from ..types import JSONSerializableObject
+from ..types import JSONSerializableObject, ToolChoice
if TYPE_CHECKING:
from openai.types.chat import ChatCompletion
@@ -70,18 +64,25 @@ def _format_audio_data_for_qwen_omni(messages: list[dict]) -> None:
class OpenAIChatModel(ChatModelBase):
"""The OpenAI chat model class."""
+ class ThinkingConfig(BaseModel):
+ """Configuration for reasoning effort levels."""
+
+ enable_thinking: bool
+ reasoning_effect: ReasoningEffort | None = None
+
def __init__(
self,
model_name: str,
api_key: str | None = None,
stream: bool = True,
- reasoning_effort: Literal["low", "medium", "high"] | None = None,
+ max_retries: int = 0,
+ fallback_model_name: str | None = None,
+ formatter: FormatterBase | None = None,
+ thinking_config: ThinkingConfig | None = None,
organization: str = None,
- stream_tool_parsing: bool = True,
client_type: Literal["openai", "azure"] = "openai",
client_kwargs: dict[str, JSONSerializableObject] | None = None,
generate_kwargs: dict[str, JSONSerializableObject] | None = None,
- **kwargs: Any,
) -> None:
"""Initialize the openai client.
@@ -93,21 +94,16 @@ class OpenAIChatModel(ChatModelBase):
be read from the environment variable `OPENAI_API_KEY`.
stream (`bool`, default `True`):
Whether to use streaming output or not.
- reasoning_effort (`Literal["low", "medium", "high"] | None`, \
- optional):
- Reasoning effort, supported for o3, o4, etc. Please refer to
- `OpenAI documentation
- <https://platform.openai.com/docs/guides/reasoning?api-mode=chat>`_
- for more details.
+ max_retries (`int`, optional):
+ Maximum number of retries on failure. Defaults to 0.
+ fallback_model_name (`str | None`, optional):
+ Fallback model name to use after all retries fail.
+ formatter (`FormatterBase | None`, optional):
+ Formatter for message preprocessing.
+ thinking_config (`ThinkingConfig | None`, optional):
+ Configuration for reasoning effort levels.
organization (`str`, default `None`):
- The organization ID for OpenAI API. If not specified, it will
- be read from the environment variable `OPENAI_ORGANIZATION`.
- stream_tool_parsing (`bool`, default to `True`):
- Whether to parse incomplete tool use JSON during streaming
- with auto-repair. If True, partial JSON (e.g., `'{"a": "x'`)
- is repaired to valid dicts ({"a": "x"}) in real-time for
- immediate tool function input. Otherwise, the input field
- remains {} until the final chunk arrives.
+ The organization ID for OpenAI API.
client_type (`Literal["openai", "azure"]`, default `openai`):
Selects which OpenAI-compatible client to initialize.
client_kwargs (`dict[str, JSONSerializableObject] | None`, \
@@ -117,34 +113,19 @@ class OpenAIChatModel(ChatModelBase):
optional):
The extra keyword arguments used in OpenAI API generation,
e.g. `temperature`, `seed`.
- **kwargs (`Any`):
- Additional keyword arguments.
"""
+ self.thinking_config = (
+ thinking_config
+ or OpenAIChatModel.ThinkingConfig(enable_thinking=False)
+ )
- # Handle deprecated client_args parameter from kwargs
- client_args = kwargs.pop("client_args", None)
- if client_args is not None and client_kwargs is not None:
- raise ValueError(
- "Cannot specify both 'client_args' and 'client_kwargs'. "
- "Please use only 'client_kwargs' (client_args is deprecated).",
- )
-
- if client_args is not None:
- logger.warning(
- "The parameter 'client_args' is deprecated and will be "
- "removed in a future version. Please use 'client_kwargs' "
- "instead. Automatically converting 'client_args' to "
- "'client_kwargs'.",
- )
- client_kwargs = client_args
-
- if kwargs:
- logger.warning(
- "Unknown keyword arguments: %s. These will be ignored.",
- list(kwargs.keys()),
- )
-
- super().__init__(model_name, stream)
+ super().__init__(
+ model_name=model_name,
+ stream=stream,
+ max_retries=max_retries,
+ fallback_model_name=fallback_model_name,
+ formatter=formatter,
+ )
import openai
@@ -166,23 +147,23 @@ class OpenAIChatModel(ChatModelBase):
**(client_kwargs or {}),
)
- self.reasoning_effort = reasoning_effort
- self.stream_tool_parsing = stream_tool_parsing
self.generate_kwargs = generate_kwargs or {}
@trace_llm
- async def __call__(
+ async def _call_api(
self,
+ model_name: str,
messages: list[dict],
tools: list[dict] | None = None,
tool_choice: Literal["auto", "none", "required"] | str | None = None,
- structured_model: Type[BaseModel] | None = None,
**kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
"""Get the response from OpenAI chat completions API by the given
arguments.
Args:
+ model_name (`str`):
+ The model name to use for this call.
messages (`list[dict]`):
A list of dictionaries, where `role` and `content` fields are
required, and `name` field is optional.
@@ -191,29 +172,8 @@ class OpenAIChatModel(ChatModelBase):
tool_choice (`Literal["auto", "none", "required"] | str \
| None`, default `None`):
Controls which (if any) tool is called by the model.
- Can be "auto", "none", "required", or specific tool
- name. For more details, please refer to
- https://platform.openai.com/docs/api-reference/responses/create#responses_create-tool_choice
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output. When provided, the model will be forced
- to return data that conforms to this schema by automatically
- converting the BaseModel to a tool function and setting
- `tool_choice` to enforce its usage. This enables structured
- output generation.
-
- .. note:: When `structured_model` is specified,
- both `tools` and `tool_choice` parameters are ignored,
- and the model will only perform structured output
- generation without calling any other tools.
-
- For more details, please refer to the `official document
- <https://platform.openai.com/docs/guides/structured-outputs>`_
-
**kwargs (`Any`):
- The keyword arguments for OpenAI chat completions API,
- e.g. `temperature`, `max_tokens`, `top_p`, etc. Please
- refer to the OpenAI API documentation for more details.
+ The keyword arguments for OpenAI chat completions API.
Returns:
`ChatResponse | AsyncGenerator[ChatResponse, None]`:
@@ -233,18 +193,21 @@ class OpenAIChatModel(ChatModelBase):
)
# Qwen-omni requires different base64 audio format from openai
- if "omni" in self.model_name.lower():
+ if "omni" in model_name.lower():
_format_audio_data_for_qwen_omni(messages)
kwargs = {
- "model": self.model_name,
+ "model": model_name,
"messages": messages,
"stream": self.stream,
**self.generate_kwargs,
**kwargs,
}
- if self.reasoning_effort and "reasoning_effort" not in kwargs:
- kwargs["reasoning_effort"] = self.reasoning_effort
+ if (
+ self.thinking_config.enable_thinking
+ and self.thinking_config.reasoning_effect
+ ):
+ kwargs["reasoning_effort"] = self.thinking_config.reasoning_effect
if tools:
kwargs["tools"] = self._format_tools_json_schemas(tools)
@@ -258,60 +221,36 @@ class OpenAIChatModel(ChatModelBase):
DeprecationWarning,
)
tool_choice = "required"
- self._validate_tool_choice(tool_choice, tools)
- kwargs["tool_choice"] = self._format_tool_choice(tool_choice)
+ kwargs["tool_choice"] = self._format_tool_choice(
+ tool_choice,
+ tools,
+ )
if self.stream:
kwargs["stream_options"] = {"include_usage": True}
start_datetime = datetime.now()
- if structured_model:
- if tools or tool_choice:
- logger.warning(
- "structured_model is provided. Both 'tools' and "
- "'tool_choice' parameters will be overridden and "
- "ignored. The model will only perform structured output "
- "generation without calling any other tools.",
- )
- kwargs.pop("stream", None)
- kwargs.pop("tools", None)
- kwargs.pop("tool_choice", None)
- kwargs["response_format"] = structured_model
- if not self.stream:
- response = await self.client.chat.completions.parse(**kwargs)
- else:
- response = self.client.chat.completions.stream(**kwargs)
- return self._parse_openai_stream_response(
- start_datetime,
- response,
- structured_model,
- )
- else:
- response = await self.client.chat.completions.create(**kwargs)
+ response = await self.client.chat.completions.create(**kwargs)
if self.stream:
return self._parse_openai_stream_response(
start_datetime,
response,
- structured_model,
)
# Non-streaming response
parsed_response = self._parse_openai_completion_response(
start_datetime,
response,
- structured_model,
)
return parsed_response
- # pylint: disable=too-many-statements
async def _parse_openai_stream_response(
self,
start_datetime: datetime,
response: AsyncStream,
- structured_model: Type[BaseModel] | None = None,
) -> AsyncGenerator[ChatResponse, None]:
"""Given an OpenAI streaming completion response, extract the content
blocks and usages from it and yield ChatResponse objects.
@@ -321,45 +260,23 @@ class OpenAIChatModel(ChatModelBase):
The start datetime of the response generation.
response (`AsyncStream`):
OpenAI AsyncStream object to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
`AsyncGenerator[ChatResponse, None]`:
An async generator that yields ChatResponse objects containing
the content blocks and usage information for each chunk in
the streaming response.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
- usage, res = None, None
- response_id: str | None = None
- text = ""
- thinking = ""
- audio = ""
- tool_calls = OrderedDict()
- last_input_objs = {} # Store last input_obj for each tool_call
- metadata: dict | None = None
- contents: List[
- TextBlock | ToolUseBlock | ThinkingBlock | AudioBlock
- ] = []
- last_contents = None
-
- async with response as stream:
- async for item in stream:
- if structured_model:
- if item.type != "chunk":
- continue
- chunk = item.chunk
- else:
- chunk = item
+ usage = None
- if response_id is None:
- response_id = getattr(chunk, "id", None)
+ # Accumulators
+ acc_text = ""
+ acc_thinking = ""
+ acc_audio = ""
+ acc_tool_calls = OrderedDict()
+ async with response as stream:
+ async for chunk in stream:
if chunk.usage:
usage = ChatUsage(
input_tokens=chunk.usage.prompt_tokens,
@@ -369,164 +286,133 @@ class OpenAIChatModel(ChatModelBase):
)
if not chunk.choices:
- if usage and contents:
- _kwargs: dict[str, Any] = {
- "content": contents,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _kwargs["id"] = response_id
- res = ChatResponse(**_kwargs)
- yield res
continue
choice = chunk.choices[0]
- delta_reasoning = getattr(
+ # Extract delta content
+ delta_thinking = getattr(
choice.delta,
"reasoning_content",
None,
)
- if not isinstance(delta_reasoning, str):
- delta_reasoning = getattr(choice.delta, "reasoning", None)
- if not isinstance(delta_reasoning, str):
- delta_reasoning = ""
-
- thinking += delta_reasoning
- text += getattr(choice.delta, "content", None) or ""
-
- if (
- hasattr(choice.delta, "audio")
- and "data" in choice.delta.audio
- ):
- audio += choice.delta.audio["data"]
- if (
- hasattr(choice.delta, "audio")
- and "transcript" in choice.delta.audio
- ):
- text += choice.delta.audio["transcript"]
-
+ if not isinstance(delta_thinking, str):
+ delta_thinking = getattr(choice.delta, "reasoning", None)
+ if not isinstance(delta_thinking, str):
+ delta_thinking = ""
+
+ delta_text = getattr(choice.delta, "content", None) or ""
+ delta_audio = ""
+
+ if isinstance(getattr(choice.delta, "audio", None), dict):
+ if "data" in choice.delta.audio:
+ delta_audio = choice.delta.audio["data"]
+ if "transcript" in choice.delta.audio:
+ delta_text += choice.delta.audio["transcript"]
+
+ # Accumulate
+ acc_thinking += delta_thinking
+ acc_text += delta_text
+ acc_audio += delta_audio
+
+ # Handle tool calls
for tool_call in (
getattr(choice.delta, "tool_calls", None) or []
):
- if tool_call.index in tool_calls:
+ if tool_call.index in acc_tool_calls:
if tool_call.function.arguments is not None:
- tool_calls[tool_call.index][
+ acc_tool_calls[tool_call.index][
"input"
] += tool_call.function.arguments
-
else:
- tool_calls[tool_call.index] = {
- "type": "tool_use",
+ acc_tool_calls[tool_call.index] = {
"id": tool_call.id,
"name": tool_call.function.name,
"input": tool_call.function.arguments or "",
}
- contents = []
+ # Build delta content blocks
+ delta_contents: List[
+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock
+ ] = []
- if thinking:
- contents.append(
- ThinkingBlock(
- type="thinking",
- thinking=thinking,
- ),
+ if delta_thinking:
+ delta_contents.append(
+ ThinkingBlock(thinking=delta_thinking),
)
- if audio:
+ if delta_audio:
media_type = self.generate_kwargs.get("audio", {}).get(
"format",
"wav",
)
- contents.append(
- AudioBlock(
- type="audio",
+ delta_contents.append(
+ DataBlock(
source=Base64Source(
- data=audio,
+ data=delta_audio,
media_type=f"audio/{media_type}",
- type="base64",
),
),
)
- if text:
- contents.append(
- TextBlock(
- type="text",
- text=text,
- ),
+ if delta_text:
+ delta_contents.append(
+ TextBlock(text=delta_text),
)
- if structured_model:
- metadata = _json_loads_with_repair(text)
+ # Yield delta response
+ if delta_contents:
+ yield ChatResponse(
+ content=delta_contents,
+ usage=usage,
+ is_last=False,
+ )
- for tool_call in tool_calls.values():
- input_str = tool_call["input"]
- tool_id = tool_call["id"]
+ # Build final accumulated content blocks
+ final_contents: List[
+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock
+ ] = []
- # If parsing the tool input in streaming mode
- if self.stream_tool_parsing:
- repaired_input = _parse_streaming_json_dict(
- input_str,
- last_input_objs.get(tool_id),
- )
- last_input_objs[tool_id] = repaired_input
+ if acc_thinking:
+ final_contents.append(ThinkingBlock(thinking=acc_thinking))
- else:
- # Otherwise, keep input as empty dict until the final
- # chunk
- repaired_input = {}
-
- contents.append(
- ToolUseBlock(
- type=tool_call["type"],
- id=tool_id,
- name=tool_call["name"],
- input=repaired_input,
- raw_input=input_str,
- ),
- )
+ if acc_audio:
+ media_type = self.generate_kwargs.get("audio", {}).get(
+ "format",
+ "wav",
+ )
+ final_contents.append(
+ DataBlock(
+ source=Base64Source(
+ data=acc_audio,
+ media_type=f"audio/{media_type}",
+ ),
+ ),
+ )
- if contents:
- _kwargs = {
- "content": contents,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _kwargs["id"] = response_id
- res = ChatResponse(**_kwargs)
- yield res
- last_contents = copy.deepcopy(contents)
-
- # If stream_tool_parsing is False, yield last contents
- if not self.stream_tool_parsing and tool_calls and last_contents:
- metadata = None
- # Update tool use blocks in last_contents inplace
- for block in last_contents:
- if block.get("type") == "tool_use":
- block["input"] = input_obj = _json_loads_with_repair(
- str(block.get("raw_input") or "{}"),
- )
+ if acc_text:
+ final_contents.append(TextBlock(text=acc_text))
- if structured_model:
- metadata = input_obj
+ for tool_call in acc_tool_calls.values():
+ final_contents.append(
+ ToolCallBlock(
+ id=tool_call["id"],
+ name=tool_call["name"],
+ input=tool_call["input"],
+ ),
+ )
- _kwargs = {
- "content": last_contents,
- "usage": usage,
- "metadata": metadata,
- }
- if response_id:
- _kwargs["id"] = response_id
- yield ChatResponse(**_kwargs)
+ # Yield final accumulated response
+ yield ChatResponse(
+ content=final_contents,
+ usage=usage,
+ is_last=True,
+ )
def _parse_openai_completion_response(
self,
start_datetime: datetime,
response: ChatCompletion,
- structured_model: Type[BaseModel] | None = None,
) -> ChatResponse:
"""Given an OpenAI chat completion response object, extract the content
blocks and usages from it.
@@ -536,22 +422,14 @@ class OpenAIChatModel(ChatModelBase):
The start datetime of the response generation.
response (`ChatCompletion`):
OpenAI ChatCompletion object to parse.
- structured_model (`Type[BaseModel] | None`, default `None`):
- A Pydantic BaseModel class that defines the expected structure
- for the model's output.
Returns:
ChatResponse (`ChatResponse`):
A ChatResponse object containing the content blocks and usage.
-
- .. note::
- If `structured_model` is not `None`, the expected structured output
- will be stored in the metadata of the `ChatResponse`.
"""
content_blocks: List[
- TextBlock | ToolUseBlock | ThinkingBlock | AudioBlock
+ TextBlock | ToolCallBlock | ThinkingBlock | DataBlock
] = []
- metadata: dict | None = None
if response.choices:
choice = response.choices[0]
@@ -563,18 +441,12 @@ class OpenAIChatModel(ChatModelBase):
if reasoning is not None:
content_blocks.append(
- ThinkingBlock(
- type="thinking",
- thinking=reasoning,
- ),
+ ThinkingBlock(thinking=reasoning),
)
if choice.message.content:
content_blocks.append(
- TextBlock(
- type="text",
- text=response.choices[0].message.content,
- ),
+ TextBlock(text=response.choices[0].message.content),
)
if choice.message.audio:
media_type = self.generate_kwargs.get("audio", {}).get(
@@ -582,39 +454,28 @@ class OpenAIChatModel(ChatModelBase):
"mp3",
)
content_blocks.append(
- AudioBlock(
- type="audio",
+ DataBlock(
source=Base64Source(
data=choice.message.audio.data,
media_type=f"audio/{media_type}",
- type="base64",
),
),
)
if choice.message.audio.transcript:
content_blocks.append(
- TextBlock(
- type="text",
- text=choice.message.audio.transcript,
- ),
+ TextBlock(text=choice.message.audio.transcript),
)
for tool_call in choice.message.tool_calls or []:
content_blocks.append(
- ToolUseBlock(
- type="tool_use",
+ ToolCallBlock(
id=tool_call.id,
name=tool_call.function.name,
- input=_json_loads_with_repair(
- tool_call.function.arguments,
- ),
+ input=tool_call.function.arguments,
),
)
- if structured_model:
- metadata = choice.message.parsed.model_dump()
-
usage = None
if response.usage:
usage = ChatUsage(
@@ -626,8 +487,8 @@ class OpenAIChatModel(ChatModelBase):
resp_kwargs: dict[str, Any] = {
"content": content_blocks,
+ "is_last": True,
"usage": usage,
- "metadata": metadata,
}
response_id = getattr(response, "id", None)
if response_id:
@@ -644,30 +505,30 @@ class OpenAIChatModel(ChatModelBase):
def _format_tool_choice(
self,
- tool_choice: Literal["auto", "none", "required"] | str | None,
+ tool_choice: ToolChoice | None,
+ tools: list[dict] | None,
) -> str | dict | None:
"""Format tool_choice parameter for API compatibility.
Args:
- tool_choice (`Literal["auto", "none", "required"] | str \
- | None`, default `None`):
- Controls which (if any) tool is called by the model.
- Can be "auto", "none", "required", or specific tool name.
- For more details, please refer to
- https://platform.openai.com/docs/api-reference/responses/create#responses_create-tool_choice
+ tool_choice (`ToolChoice | None`):
+ The unified tool choice parameter which can be a mode ("auto",
+ "none", "required") or a specific function name.
+ tools (`list[dict] | None`):
+ The list of available tools, used for validation if
+ tool_choice is a specific function name.
+
Returns:
`dict | None`:
The formatted tool choice configuration dict, or None if
tool_choice is None.
"""
+ super()._validate_tool_choice(tool_choice, tools)
+
if tool_choice is None:
return None
- mode_mapping = {
- "auto": "auto",
- "none": "none",
- "required": "required",
- }
- if tool_choice in mode_mapping:
- return mode_mapping[tool_choice]
+ if tool_choice in ["auto", "none", "required"]:
+ return tool_choice
+
return {"type": "function", "function": {"name": tool_choice}}
diff --git a/src/agentscope/tool/_coding/__init__.py b/src/agentscope/tool/_coding/__init__.py
deleted file mode 100644
index acad79d70..000000000
--- a/src/agentscope/tool/_coding/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The coding-related tools module in agentscope."""
-
-from ._python import execute_python_code
-from ._shell import execute_shell_command
-
-__all__ = [
- "execute_python_code",
- "execute_shell_command",
-]
diff --git a/src/agentscope/tool/_coding/_python.py b/src/agentscope/tool/_coding/_python.py
deleted file mode 100644
index d9f58976c..000000000
--- a/src/agentscope/tool/_coding/_python.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# -*- coding: utf-8 -*-
-# pylint: disable=unused-argument
-"""The Python code execution tool in agentscope."""
-
-import asyncio
-import os
-import sys
-import tempfile
-from typing import Any
-
-import shortuuid
-
-from ...message import TextBlock
-from .._response import ToolResponse
-
-
-async def execute_python_code(
- code: str,
- timeout: float = 300,
- **kwargs: Any,
-) -> ToolResponse:
- """Execute the given python code in a temp file and capture the return
- code, standard output and error. Note you must `print` the output to get
- the result, and the tmp file will be removed right after the execution.
-
- Args:
- code (`str`):
- The Python code to be executed.
- timeout (`float`, defaults to `300`):
- The maximum time (in seconds) allowed for the code to run.
-
- Returns:
- `ToolResponse`:
- The response containing the return code, standard output, and
- standard error of the executed code.
- """
-
- with tempfile.TemporaryDirectory() as temp_dir:
- temp_file = os.path.join(temp_dir, f"tmp_{shortuuid.uuid()}.py")
- with open(temp_file, "w", encoding="utf-8") as f:
- f.write(code)
-
- env = os.environ.copy()
- env["PYTHONUTF8"] = "1"
- env["PYTHONIOENCODING"] = "utf-8"
- proc = await asyncio.create_subprocess_exec(
- sys.executable,
- "-u",
- temp_file,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- env=env,
- )
-
- try:
- await asyncio.wait_for(proc.wait(), timeout=timeout)
- stdout, stderr = await proc.communicate()
- stdout_str = stdout.decode("utf-8")
- stderr_str = stderr.decode("utf-8")
- returncode = proc.returncode
-
- except asyncio.TimeoutError:
- stderr_suffix = (
- f"TimeoutError: The code execution exceeded "
- f"the timeout of {timeout} seconds."
- )
- returncode = -1
- try:
- proc.terminate()
- stdout, stderr = await proc.communicate()
- stdout_str = stdout.decode("utf-8")
- stderr_str = stderr.decode("utf-8")
- if stderr_str:
- stderr_str += f"\n{stderr_suffix}"
- else:
- stderr_str = stderr_suffix
- except ProcessLookupError:
- stdout_str = ""
- stderr_str = stderr_suffix
-
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"<returncode>{returncode}</returncode>"
- f"<stdout>{stdout_str}</stdout>"
- f"<stderr>{stderr_str}</stderr>",
- ),
- ],
- )
diff --git a/src/agentscope/tool/_coding/_shell.py b/src/agentscope/tool/_coding/_shell.py
deleted file mode 100644
index 653eeca14..000000000
--- a/src/agentscope/tool/_coding/_shell.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# -*- coding: utf-8 -*-
-# pylint: disable=unused-argument
-"""The shell command tool in agentscope."""
-
-import asyncio
-from typing import Any
-
-from .._response import ToolResponse
-from ...message import TextBlock
-
-
-async def execute_shell_command(
- command: str,
- timeout: int = 300,
- **kwargs: Any,
-) -> ToolResponse:
- """Execute given command and return the return code, standard output and
- error within <returncode></returncode>, <stdout></stdout> and
- <stderr></stderr> tags.
-
- Args:
- command (`str`):
- The shell command to execute.
- timeout (`float`, defaults to `300`):
- The maximum time (in seconds) allowed for the command to run.
-
- Returns:
- `ToolResponse`:
- The tool response containing the return code, standard output, and
- standard error of the executed command.
- """
-
- proc = await asyncio.create_subprocess_shell(
- command,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- bufsize=0,
- )
-
- try:
- await asyncio.wait_for(proc.wait(), timeout=timeout)
- stdout, stderr = await proc.communicate()
- stdout_str = stdout.decode("utf-8")
- stderr_str = stderr.decode("utf-8")
- returncode = proc.returncode
-
- except asyncio.TimeoutError:
- stderr_suffix = (
- f"TimeoutError: The command execution exceeded "
- f"the timeout of {timeout} seconds."
- )
- returncode = -1
- try:
- proc.terminate()
- stdout, stderr = await proc.communicate()
- stdout_str = stdout.decode("utf-8")
- stderr_str = stderr.decode("utf-8")
- if stderr_str:
- stderr_str += f"\n{stderr_suffix}"
- else:
- stderr_str = stderr_suffix
- except ProcessLookupError:
- stdout_str = ""
- stderr_str = stderr_suffix
-
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=(
- f"<returncode>{returncode}</returncode>"
- f"<stdout>{stdout_str}</stdout>"
- f"<stderr>{stderr_str}</stderr>"
- ),
- ),
- ],
- )
diff --git a/src/agentscope/tool/_multi_modality/__init__.py b/src/agentscope/tool/_multi_modality/__init__.py
deleted file mode 100644
index 766c169b9..000000000
--- a/src/agentscope/tool/_multi_modality/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The multi-modal-related tools module in agentscope."""
-from ._dashscope_tools import (
- dashscope_image_to_text,
- dashscope_text_to_audio,
- dashscope_text_to_image,
-)
-from ._openai_tools import (
- openai_text_to_image,
- openai_edit_image,
- openai_text_to_audio,
- openai_create_image_variation,
- openai_image_to_text,
- openai_audio_to_text,
-)
-
-__all__ = [
- "dashscope_image_to_text",
- "dashscope_text_to_audio",
- "dashscope_text_to_image",
- "openai_text_to_image",
- "openai_text_to_audio",
- "openai_edit_image",
- "openai_create_image_variation",
- "openai_image_to_text",
- "openai_audio_to_text",
-]
diff --git a/src/agentscope/tool/_multi_modality/_dashscope_tools.py b/src/agentscope/tool/_multi_modality/_dashscope_tools.py
deleted file mode 100644
index cc8861599..000000000
--- a/src/agentscope/tool/_multi_modality/_dashscope_tools.py
+++ /dev/null
@@ -1,302 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Use DashScope API to generate images,
-convert text to audio, and convert images to text.
-Please refer to the `official documentation <https://dashscope.aliyun.com/>`_
- for more details.
-"""
-import base64
-from typing import Literal, Sequence
-
-import os
-
-
-from ..._utils._common import _get_bytes_from_web_url
-from ...message import ImageBlock, TextBlock, AudioBlock
-from ...tool import ToolResponse
-
-
-def dashscope_text_to_image(
- prompt: str,
- api_key: str,
- n: int = 1,
- size: Literal["1024*1024", "720*1280", "1280*720"] = "1024*1024",
- model: str = "wanx-v1",
- use_base64: bool = False,
-) -> ToolResponse:
- """Generate image(s) based on the given prompt, and return image url(s)
- or base64 data.
-
- Args:
- prompt (`str`):
- The text prompt to generate image.
- api_key (`str`):
- The api key for the dashscope api.
- n (`int`, defaults to `1`):
- The number of images to generate.
- size (`Literal["1024*1024", "720*1280", "1280*720"]`, defaults to \
- `"1024*1024"`):
- Size of the image.
- model (`str`, defaults to '"wanx-v1"'):
- The model to use, such as "wanx-v1", "qwen-image",
- "wan2.2-t2i-flash", etc.
- use_base64 (`bool`, defaults to 'False'):
- Whether to use base64 data for images.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
- """
- try:
- import dashscope
-
- response = dashscope.ImageSynthesis.call(
- model=model,
- prompt=prompt,
- api_key=api_key,
- n=n,
- size=size,
- )
- images = response.output["results"]
- urls = [_["url"] for _ in images]
-
- image_blocks: list = []
-
- if urls is not None:
- for url in urls:
- if use_base64:
- extension = url.split(".")[-1].lower()
-
- image_data = _get_bytes_from_web_url(url)
- image_blocks.append(
- ImageBlock(
- type="image",
- source={
- "type": "base64",
- "media_type": f"image/{extension}",
- "data": image_data,
- },
- ),
- )
- else:
- image_blocks.append(
- ImageBlock(
- type="image",
- source={
- "type": "url",
- "url": url,
- },
- ),
- )
-
- return ToolResponse(
- content=image_blocks,
- )
-
- else:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text="Error: Failed to generate images",
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Failed to generate images: {str(e)}",
- ),
- ],
- )
-
-
-def dashscope_image_to_text(
- image_urls: str | Sequence[str],
- api_key: str,
- prompt: str = "Describe the image",
- model: str = "qwen-vl-plus",
-) -> ToolResponse:
- """Generate text based on the given images.
-
- Args:
- image_urls (`str | Sequence[str]`):
- The url of single or multiple images.
- api_key (`str`):
- The api key for the dashscope api.
- prompt (`str`, defaults to 'Describe the image' ):
- The text prompt.
- model (`str`, defaults to 'qwen-vl-plus'):
- The model to use in DashScope MultiModal API.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
- """
-
- if isinstance(image_urls, str):
- image_urls = [image_urls]
-
- # Check if the local url is valid
- img_abs_urls = []
- for url in image_urls:
- if os.path.exists(url):
- if os.path.isfile(url):
- img_abs_urls.append(os.path.abspath(url))
- else:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f'Error: The input image url "{url}" is '
- f"not a file.",
- ),
- ],
- )
- else:
- # Maybe a web url or an invalid url, we leave it to the API
- # to handle
- img_abs_urls.append(url)
-
- # Convert image paths according to the model requirements
- contents = []
- for url in img_abs_urls:
- contents.append(
- {
- "image": url,
- },
- )
-
- contents.append({"text": prompt})
-
- # currently only support one round of conversation
- # if multiple rounds of conversation are needed,
- # it would be better to implement an Agent class
- sys_message = {
- "role": "system",
- "content": [{"text": "You are a helpful assistant."}],
- }
- user_message = {
- "role": "user",
- "content": contents,
- }
- messages = [sys_message, user_message]
- try:
- import dashscope
-
- response = dashscope.MultiModalConversation.call(
- model=model,
- messages=messages,
- api_key=api_key,
- )
- content = response.output["choices"][0]["message"]["content"]
- if isinstance(content, list):
- content = content[0]["text"]
- if content is not None:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=content,
- ),
- ],
- )
- else:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text="Error: Failed to generate text",
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Failed to generate text: {str(e)}",
- ),
- ],
- )
-
-
-def dashscope_text_to_audio(
- text: str,
- api_key: str,
- model: str = "sambert-zhichu-v1",
- sample_rate: int = 48000,
-) -> ToolResponse:
- """Convert the given text to audio.
-
- Args:
- text (`str`):
- The text to be converted into audio.
- api_key (`str`):
- The api key for the dashscope API.
- model (`str`, defaults to 'sambert-zhichu-v1'):
- The model to use. Full model list can be found in the
- `official document
- <https://help.aliyun.com/zh/model-studio/sambert-python-sdk>`_.
- sample_rate (`int`, defaults to 48000):
- Sample rate of the audio.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
-
- """
- try:
- import dashscope
-
- dashscope.api_key = api_key
-
- res = dashscope.audio.tts.SpeechSynthesizer.call(
- model=model,
- text=text,
- sample_rate=sample_rate,
- format="wav",
- )
-
- audio_data = res.get_audio_data()
-
- if audio_data is not None:
- audio_base64 = base64.b64encode(audio_data).decode("utf-8")
-
- return ToolResponse(
- [
- AudioBlock(
- type="audio",
- source={
- "type": "base64",
- "media_type": "audio/wav",
- "data": audio_base64,
- },
- ),
- ],
- )
- else:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text="Error: Failed to generate audio",
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Failed to generate audio: {str(e)}",
- ),
- ],
- )
diff --git a/src/agentscope/tool/_multi_modality/_openai_tools.py b/src/agentscope/tool/_multi_modality/_openai_tools.py
deleted file mode 100644
index 2c77c4238..000000000
--- a/src/agentscope/tool/_multi_modality/_openai_tools.py
+++ /dev/null
@@ -1,672 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Wrap OpenAI API calls as tools. Refer the official
-`OpenAI API documentation <https://platform.openai.com/docs/overview>`_ for
-more details.
-"""
-import base64
-from io import BytesIO
-import os
-from typing import Literal, IO
-import requests
-
-from .. import ToolResponse
-from ...formatter._openai_formatter import _to_openai_image_url
-from ...message import (
- ImageBlock,
- TextBlock,
- Base64Source,
- URLSource,
- AudioBlock,
-)
-
-
-def _parse_url(url: str) -> BytesIO | IO[bytes]:
- """
- If url is a local file path, return a BytesIO of the file content.
- If url is a web URL, fetch the content and return as BytesIO.
- """
- if url.startswith(("http://", "https://")):
- response = requests.get(url)
- response.raise_for_status() # Raise an exception for HTTP errors
- return BytesIO(response.content)
- else:
- if not os.path.exists(url):
- raise FileNotFoundError(f"File not found: {url}")
- return open(os.path.abspath(url), "rb")
-
-
-def openai_text_to_image(
- prompt: str,
- api_key: str,
- n: int = 1,
- model: Literal["dall-e-2", "dall-e-3", "gpt-image-1"] = "dall-e-2",
- size: Literal[
- "256x256",
- "512x512",
- "1024x1024",
- "1792x1024",
- "1024x1792",
- ] = "256x256",
- quality: Literal[
- "auto",
- "standard",
- "hd",
- "high",
- "medium",
- "low",
- ] = "auto",
- style: Literal["vivid", "natural"] = "vivid",
- response_format: Literal["url", "b64_json"] = "url",
-) -> ToolResponse:
- """
- Generate image(s) based on the given prompt, and return image URL(s) or
- base64 data.
-
- Args:
- prompt (`str`):
- The text prompt to generate images.
- api_key (`str`):
- The API key for the OpenAI API.
- n (`int`, defaults to `1`):
- The number of images to generate.
- model (`Literal["dall-e-2", "dall-e-3"]`, defaults to `"dall-e-2"`):
- The model to use for image generation.
- size (`Literal["256x256", "512x512", "1024x1024", "1792x1024", \
- "1024x1792"]`, defaults to `"256x256"`):
- The size of the generated images.
- Must be one of 1024x1024, 1536x1024 (landscape), 1024x1536 (
- portrait), or auto (default value) for gpt-image-1,
- one of 256x256, 512x512, or 1024x1024 for dall-e-2,
- and one of 1024x1024, 1792x1024, or 1024x1792 for dall-e-3.
- quality (`Literal["auto", "standard", "hd", "high", "medium", \
- "low"]`, defaults to `"auto"`):
- The quality of the image that will be generated.
-
- - `auto` (default value) will automatically select the best
- quality for the given model.
- - `high`, `medium` and `low` are supported for gpt-image-1.
- - `hd` and `standard` are supported for dall-e-3.
- - `standard` is the only option for dall-e-2.
- style (`Literal["vivid", "natural"]`, defaults to `"vivid"`):
- The style of the generated images.
- This parameter is only supported for dall-e-3.
- Must be one of `vivid` or `natural`.
-
- - `Vivid` causes the model to lean towards generating hyper-real
- and dramatic images.
- - `Natural` causes the model to produce more natural,
- less hyper-real looking images.
- response_format (`Literal["url", "b64_json"]`, defaults to `"url"`):
- The format in which generated images with dall-e-2 and dall-e-3
- are returned.
-
- - Must be one of "url" or "b64_json".
- - URLs are only valid for 60 minutes after the image has been
- generated.
- - This parameter isn't supported for gpt-image-1 which will always
- return base64-encoded images.
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
- """
-
- kwargs = {
- "model": model,
- "prompt": prompt,
- "n": n,
- "size": size,
- }
- if model == "dall-e-3":
- kwargs["style"] = style
- if model != "dall-e-2":
- kwargs["quality"] = quality
- if model != "gpt-image-1":
- kwargs["response_format"] = response_format
- if model == "gpt-image-1":
- response_format = "b64_json"
-
- try:
- import openai
-
- client = openai.OpenAI(
- api_key=api_key,
- )
- response = client.images.generate(
- **kwargs,
- )
- image_blocks: list = []
- if response_format == "url":
- image_urls = [_.url for _ in response.data]
- for image_url in image_urls:
- image_blocks.append(
- ImageBlock(
- type="image",
- source=URLSource(
- type="url",
- url=image_url,
- ),
- ),
- )
- else:
- image_datas = [_.b64_json for _ in response.data]
- for image_data in image_datas:
- image_blocks.append(
- ImageBlock(
- type="image",
- source=Base64Source(
- type="base64",
- media_type="image/png",
- data=image_data,
- ),
- ),
- )
- return ToolResponse(
- content=image_blocks,
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Failed to generate image: {str(e)}",
- ),
- ],
- )
-
-
-def openai_edit_image(
- image_url: str,
- prompt: str,
- api_key: str,
- model: Literal["dall-e-2", "gpt-image-1"] = "dall-e-2",
- mask_url: str | None = None,
- n: int = 1,
- size: Literal[
- "256x256",
- "512x512",
- "1024x1024",
- ] = "256x256",
- response_format: Literal["url", "b64_json"] = "url",
-) -> ToolResponse:
- """
- Edit an image based on the provided mask and prompt, and return the edited
- image URL(s) or base64 data.
-
- Args:
- image_url (`str`):
- The file path or URL to the image that needs editing.
- prompt (`str`):
- The text prompt describing the edits to be made to the image.
- api_key (`str`):
- The API key for the OpenAI API.
- model (`Literal["dall-e-2", "gpt-image-1"]`, defaults to `"dall-e-2"`):
- The model to use for image generation.
- mask_url (`str | None`, defaults to `None`):
- The file path or URL to the mask image that specifies the regions
- to be edited.
- n (`int`, defaults to `1`):
- The number of edited images to generate.
- size (`Literal["256x256", "512x512", "1024x1024"]`, defaults to \
- `"256x256"`):
- The size of the edited images.
- response_format (`Literal["url", "b64_json"]`, defaults to `"url"`):
- The format in which generated images are returned.
-
- - Must be one of "url" or "b64_json".
- - URLs are only valid for 60 minutes after generation.
- - This parameter isn't supported for gpt-image-1 which will
- always return base64-encoded images.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
-
- """
-
- try:
- import openai
-
- client = openai.OpenAI(
- api_key=api_key,
- )
-
- def prepare_image(url_or_path: str) -> BytesIO:
- from PIL import Image
-
- if url_or_path.startswith(("http://", "https://")):
- response = requests.get(url_or_path)
- response.raise_for_status()
- img = Image.open(BytesIO(response.content))
- else:
- img = Image.open(url_or_path)
-
- if img.mode != "RGBA":
- img = img.convert("RGBA")
- img_buffer = BytesIO()
- img.save(img_buffer, format="PNG")
- img_buffer.seek(0)
- img_buffer.name = "image.png"
- return img_buffer
-
- image_file = prepare_image(image_url)
-
- kwargs = {
- "model": model,
- "image": image_file,
- "prompt": prompt,
- "n": n,
- "size": size,
- }
-
- if mask_url:
- kwargs["mask"] = prepare_image(mask_url)
-
- if model == "dall-e-2":
- kwargs["response_format"] = response_format
- else:
- response_format = "b64_json"
-
- response = client.images.edit(**kwargs)
-
- if response_format == "url":
- urls = [_.url for _ in response.data]
- image_blocks: list = []
- for url in urls:
- image_blocks.append(
- ImageBlock(
- type="image",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- )
- return ToolResponse(
- content=image_blocks,
- )
- elif response_format == "b64_json":
- image_datas = [_.b64_json for _ in response.data]
- image_blocks = []
- for image_data in image_datas:
- image_blocks.append(
- ImageBlock(
- type="image",
- source=Base64Source(
- type="base64",
- media_type="image/png",
- data=image_data,
- ),
- ),
- )
- return ToolResponse(
- content=image_blocks,
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Failed to generate image: {str(e)}",
- ),
- ],
- )
-
-
-def openai_create_image_variation(
- image_url: str,
- api_key: str,
- n: int = 1,
- model: Literal["dall-e-2"] = "dall-e-2",
- size: Literal[
- "256x256",
- "512x512",
- "1024x1024",
- ] = "256x256",
- response_format: Literal["url", "b64_json"] = "url",
-) -> ToolResponse:
- """
- Create variations of an image and return the image URL(s) or base64 data.
-
- Args:
- image_url (`str`):
- The file path or URL to the image from which variations will be
- generated.
- api_key (`str`):
- The API key for the OpenAI API.
- n (`int`, defaults to `1`):
- The number of image variations to generate.
- model (` Literal["dall-e-2"]`, default to `dall-e-2`):
- The model to use for image variation.
- size (`Literal["256x256", "512x512", "1024x1024"]`, defaults to \
- `"256x256"`):
- The size of the generated image variations.
- response_format (`Literal["url", "b64_json"]`, defaults to `"url"`):
- The format in which generated images are returned.
-
- - Must be one of url or b64_json.
- - URLs are only valid for 60 minutes after the image has been
- generated.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
- """
-
- # _parse_url handles both local and web URLs and returns BytesIO
- image = _parse_url(image_url)
- try:
- import openai
-
- client = openai.OpenAI(
- api_key=api_key,
- )
- response = client.images.create_variation(
- model=model,
- image=image,
- n=n,
- size=size,
- )
- image_blocks: list = []
- if response_format == "url":
- urls = [_.url for _ in response.data]
- for url in urls:
- image_blocks.append(
- ImageBlock(
- type="image",
- source=URLSource(
- type="url",
- url=url,
- ),
- ),
- )
- else:
- image_datas = [_.b64_json for _ in response.data]
- for image_data in image_datas:
- image_blocks.append(
- ImageBlock(
- type="image",
- source=Base64Source(
- type="base64",
- media_type="image/png",
- data=image_data,
- ),
- ),
- )
- return ToolResponse(
- content=image_blocks,
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Failed to generate image: {str(e)}",
- ),
- ],
- )
-
-
-def openai_image_to_text(
- image_urls: str | list[str],
- api_key: str,
- prompt: str = "Describe the image",
- model: str = "gpt-4o",
-) -> ToolResponse:
- """
- Generate descriptive text for given image(s) using a specified model, and
- return the generated text.
-
- Args:
- image_urls (`str | list[str]`):
- The URL or list of URLs pointing to the images that need to be
- described.
- api_key (`str`):
- The API key for the OpenAI API.
- prompt (`str`, defaults to `"Describe the image"`):
- The prompt that instructs the model on how to describe
- the image(s).
- model (`str`, defaults to `"gpt-4o"`):
- The model to use for generating the text descriptions.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
-
- """
- if isinstance(image_urls, str):
- image_urls = [image_urls]
-
- content = []
- for url in image_urls:
- content.append(
- {
- "type": "image_url",
- "image_url": {
- "url": _to_openai_image_url(url),
- },
- },
- )
- content.append(
- {
- "type": "text",
- "text": prompt,
- },
- )
- messages = [
- {
- "role": "user",
- "content": content,
- },
- ]
-
- try:
- import openai
-
- client = openai.OpenAI(
- api_key=api_key,
- )
- response = client.chat.completions.create(
- messages=messages,
- model=model,
- )
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=response.choices[0].message.content,
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Failed to generate text: {str(e)}",
- ),
- ],
- )
-
-
-def openai_text_to_audio(
- text: str,
- api_key: str,
- model: Literal["tts-1", "tts-1-hd", "gpt-4o-mini-tts"] = "tts-1",
- voice: Literal[
- "alloy",
- "ash",
- "ballad",
- "coral",
- "echo",
- "fable",
- "nova",
- "onyx",
- "sage",
- "shimmer",
- ] = "alloy",
- speed: float = 1.0,
- res_format: Literal[
- "mp3",
- "opus",
- "aac",
- "flac",
- "wav",
- "pcm",
- ] = "mp3",
-) -> ToolResponse:
- """
- Convert text to an audio file using a specified model and voice.
-
- Args:
- text (`str`):
- The text to convert to audio.
- api_key (`str`):
- The API key for the OpenAI API.
- model (`Literal["tts-1", "tts-1-hd"]`, defaults to `"tts-1"`):
- The model to use for text-to-speech conversion.
- voice (`Literal["alloy", "echo", "fable", "onyx", "nova", \
- "shimmer"]`, defaults to `"alloy"`):
- The voice to use for the audio output.
- speed (`float`, defaults to `1.0`):
- The speed of the audio playback. A value of 1.0 is normal speed.
- res_format (`Literal["mp3", "wav", "opus", "aac", "flac", \
- "wav", "pcm"]`, defaults to `"mp3"`):
- The format of the audio file.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
- """
-
- try:
- import openai
-
- client = openai.OpenAI(
- api_key=api_key,
- )
- response = client.audio.speech.create(
- model=model,
- voice=voice,
- speed=speed,
- input=text,
- response_format=res_format,
- )
- audio_bytes = response.content
- audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")
-
- return ToolResponse(
- [
- AudioBlock(
- type="audio",
- source=Base64Source(
- type="base64",
- media_type=f"audio/{res_format}",
- data=audio_base64,
- ),
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Error: Failed to generate audio. {str(e)}",
- ),
- ],
- )
-
-
-def openai_audio_to_text(
- audio_file_url: str,
- api_key: str,
- language: str = "en",
- temperature: float = 0.2,
-) -> ToolResponse:
- """
- Convert an audio file to text using OpenAI's transcription service.
-
- Args:
- audio_file_url (`str`):
- The file path or URL to the audio file that needs to be
- transcribed.
- api_key (`str`):
- The API key for the OpenAI API.
- language (`str`, defaults to `"en"`):
- The language of the input audio in
- `ISO-639-1 format \
- <https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes>`_
- (e.g., "en", "zh", "fr"). Improves accuracy and latency.
- temperature (`float`, defaults to `0.2`):
- The temperature for the transcription, which affects the
- randomness of the output.
-
- Returns:
- `ToolResponse`:
- A ToolResponse containing the generated content
- (ImageBlock/TextBlock/AudioBlock) or error information if the
- operation failed.
- """
-
- try:
- import openai
-
- client = openai.OpenAI(
- api_key=api_key,
- )
-
- if audio_file_url.startswith(("http://", "https://")):
- response = requests.get(audio_file_url)
- response.raise_for_status()
- audio_buffer = BytesIO(response.content)
- import urllib.parse
- from pathlib import Path
-
- parsed_url = urllib.parse.urlparse(audio_file_url)
- filename = Path(parsed_url.path).name or "audio.mp3"
- audio_buffer.name = filename
- audio_file = audio_buffer
- transcription = client.audio.transcriptions.create(
- model="whisper-1",
- file=audio_file,
- language=language,
- temperature=temperature,
- )
- else:
- if not os.path.exists(audio_file_url):
- raise FileNotFoundError(f"File not found: {audio_file_url}")
- with open(audio_file_url, "rb") as audio_file:
- transcription = client.audio.transcriptions.create(
- model="whisper-1",
- file=audio_file,
- language=language,
- temperature=temperature,
- )
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=transcription.text,
- ),
- ],
- )
- except Exception as e:
- return ToolResponse(
- [
- TextBlock(
- type="text",
- text=f"Error: Failed to transcribe audio: {str(e)}",
- ),
- ],
- )
diff --git a/src/agentscope/tool/_response.py b/src/agentscope/tool/_response.py
index bb939199b..77ab38188 100644
--- a/src/agentscope/tool/_response.py
+++ b/src/agentscope/tool/_response.py
@@ -1,18 +1,17 @@
# -*- coding: utf-8 -*-
"""The tool response class."""
-
+import uuid
from dataclasses import dataclass, field
from typing import Optional, List
-from .._utils._common import _get_timestamp
-from ..message import AudioBlock, ImageBlock, TextBlock, VideoBlock
+from ..message import DataBlock, TextBlock
@dataclass
class ToolResponse:
"""The result chunk of a tool call."""
- content: List[TextBlock | ImageBlock | AudioBlock | VideoBlock]
+ content: List[TextBlock | DataBlock]
"""The execution output of the tool function."""
metadata: Optional[dict] = None
@@ -28,5 +27,5 @@ class ToolResponse:
is_interrupted: bool = False
"""Whether the tool execution is interrupted."""
- id: str = field(default_factory=lambda: _get_timestamp(True))
+ id: str = field(default_factory=lambda: uuid.uuid4().hex)
"""The identity of the tool response."""
diff --git a/src/agentscope/tool/_text_file/__init__.py b/src/agentscope/tool/_text_file/__init__.py
deleted file mode 100644
index cd404b503..000000000
--- a/src/agentscope/tool/_text_file/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The text file tool module in agentscope."""
-from ._view_text_file import view_text_file
-from ._write_text_file import (
- insert_text_file,
- write_text_file,
-)
-
-__all__ = [
- "insert_text_file",
- "write_text_file",
- "view_text_file",
-]
diff --git a/src/agentscope/tool/_text_file/_utils.py b/src/agentscope/tool/_text_file/_utils.py
deleted file mode 100644
index 4f0877f6e..000000000
--- a/src/agentscope/tool/_text_file/_utils.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# -*- coding: utf-8 -*-
-"""The utility functions for text file tools in agentscope."""
-from ...exception import ToolInvalidArgumentsError
-
-
-def _calculate_view_ranges(
- old_n_lines: int,
- new_n_lines: int,
- start: int,
- end: int,
- extra_view_n_lines: int = 5,
-) -> tuple[int, int]:
- """Calculate after writing the new content, the view ranges of the file.
-
- Args:
- old_n_lines (`int`):
- The number of lines before writing the new content.
- new_n_lines (`int`):
- The number of lines after writing the new content.
- start (`int`):
- The start line of the writing range.
- end (`int`):
- The end line of the writing range.
- extra_view_n_lines (`int`, optional):
- The number of extra lines to view before and after the range.
- """
-
- view_start = max(1, start - extra_view_n_lines)
-
- delta_lines = new_n_lines - old_n_lines
- view_end = min(end + delta_lines + extra_view_n_lines, new_n_lines)
-
- return view_start, view_end
-
-
-def _assert_ranges(
- ranges: list[int],
-) -> None:
- """Check if the ranges are valid.
-
- Raises:
- ToolInvalidArgumentsError: If the ranges are invalid.
- """
- if (
- isinstance(ranges, list)
- and len(ranges) == 2
- and all(isinstance(i, int) for i in ranges)
- ):
- start, end = ranges
- if start > end:
- raise ToolInvalidArgumentsError(
- f"InvalidArgumentError: The start line is greater than the "
- f"end line in the given range {ranges}.",
- )
- else:
- raise ToolInvalidArgumentsError(
- f"InvalidArgumentError: Invalid range format. Expected a list of "
- f"two integers, but got {ranges}.",
- )
-
-
-def _view_text_file(
- file_path: str,
- ranges: list[int] | None = None,
-) -> str:
- """Return the file content in the specified range with line numbers."""
- with open(file_path, "r", encoding="utf-8") as file:
- lines = file.readlines()
-
- if ranges:
- _assert_ranges(ranges)
- start, end = ranges
-
- if start > len(lines):
- raise ToolInvalidArgumentsError(
- f"InvalidArgumentError: The range '{ranges}' is out of bounds "
- f"for the file '{file_path}', which has only {len(lines)} "
- f"lines.",
- )
-
- view_content = [
- f"{index + start}: {line}"
- for index, line in enumerate(lines[start - 1 : end])
- ]
-
- return "".join(view_content)
-
- return "".join(f"{index + 1}: {line}" for index, line in enumerate(lines))
diff --git a/src/agentscope/tool/_text_file/_view_text_file.py b/src/agentscope/tool/_text_file/_view_text_file.py
deleted file mode 100644
index 5955a0a76..000000000
--- a/src/agentscope/tool/_text_file/_view_text_file.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# -*- coding: utf-8 -*-
-# flake8: noqa: E501
-# pylint: disable=line-too-long
-"""The view text file tool in agentscope."""
-import os
-
-from ._write_text_file import _view_text_file
-from .._response import ToolResponse
-from ...exception import ToolInvalidArgumentsError
-from ...message import TextBlock
-
-
-async def view_text_file(
- file_path: str,
- ranges: list[int] | None = None,
-) -> ToolResponse:
- """View the file content in the specified range with line numbers. If `ranges` is not provided, the entire file will be returned.
-
- Args:
- file_path (`str`):
- The target file path.
- ranges:
- The range of lines to be viewed (e.g. lines 1 to 100: [1, 100]), inclusive. If not provided, the entire file will be returned. To view the last 100 lines, use [-100, -1].
-
- Returns:
- `ToolResponse`:
- The tool response containing the file content or an error message.
- """
- file_path = os.path.expanduser(file_path)
- if not os.path.exists(file_path):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The file {file_path} does not exist.",
- ),
- ],
- )
- if not os.path.isfile(file_path):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The path {file_path} is not a file.",
- ),
- ],
- )
-
- try:
- content = _view_text_file(file_path, ranges)
- except ToolInvalidArgumentsError as e:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=e.message,
- ),
- ],
- )
-
- if ranges is None:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"""The content of {file_path}:
-```
-{content}```""",
- ),
- ],
- )
- else:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"""The content of {file_path} in {ranges} lines:
-```
-{content}```""",
- ),
- ],
- )
diff --git a/src/agentscope/tool/_text_file/_write_text_file.py b/src/agentscope/tool/_text_file/_write_text_file.py
deleted file mode 100644
index e5e06ad9a..000000000
--- a/src/agentscope/tool/_text_file/_write_text_file.py
+++ /dev/null
@@ -1,241 +0,0 @@
-# -*- coding: utf-8 -*-
-# flake8: noqa: E501
-# pylint: disable=line-too-long
-"""The text file tools in agentscope."""
-import os
-
-from ._utils import _calculate_view_ranges, _view_text_file
-from .._response import ToolResponse
-from ...message import TextBlock
-
-
-async def insert_text_file(
- file_path: str,
- content: str,
- line_number: int,
-) -> ToolResponse:
- """Insert the content at the specified line number in a text file.
-
- Args:
- file_path (`str`):
- The target file path.
- content (`str`):
- The content to be inserted.
- line_number (`int`):
- The line number at which the content should be inserted, starting
- from 1. If exceeds the number of lines in the file, it will be
- appended to the end of the file.
-
- Returns:
- `ToolResponse`:
- The tool response containing the result of the insertion operation.
- """
- if line_number <= 0:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"InvalidArgumentsError: "
- f"The line number {line_number} is invalid. ",
- ),
- ],
- )
-
- file_path = os.path.expanduser(file_path)
- if not os.path.exists(file_path):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"InvalidArgumentsError: The target file "
- f"{file_path} does not exist. ",
- ),
- ],
- )
-
- with open(file_path, "r", encoding="utf-8") as file:
- original_lines = file.readlines()
-
- if line_number == len(original_lines) + 1:
- new_lines = original_lines + ["\n" + content]
- elif line_number < len(original_lines) + 1:
- new_lines = (
- original_lines[: line_number - 1]
- + [content + "\n"]
- + original_lines[line_number - 1 :]
- )
- else:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text="InvalidArgumentsError: The given line_number "
- f"({line_number}) is not in the valid range "
- f"[1, {len(original_lines) + 1}].",
- ),
- ],
- )
-
- with open(file_path, "w", encoding="utf-8") as file:
- file.writelines(new_lines)
-
- with open(file_path, "r", encoding="utf-8") as file:
- new_lines = file.readlines()
-
- start, end = _calculate_view_ranges(
- len(original_lines),
- len(new_lines),
- line_number,
- line_number,
- extra_view_n_lines=5,
- )
-
- show_content = _view_text_file(file_path, [start, end])
-
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Insert content into {file_path} at line "
- f"{line_number} successfully. The new content "
- f"between lines {start}-{end} is:\n"
- f"```\n{show_content}```",
- ),
- ],
- )
-
-
-async def write_text_file(
- file_path: str,
- content: str,
- ranges: None | list[int] = None,
-) -> ToolResponse:
- """Create/Replace/Overwrite content in a text file. When `ranges` is provided, the content will be replaced in the specified range. Otherwise, the entire file (if exists) will be overwritten.
-
- Args:
- file_path (`str`):
- The target file path.
- content (`str`):
- The content to be written.
- ranges (`list[int] | None`, defaults to `None`):
- The range of lines to be replaced. If `None`, the entire file will
- be overwritten.
-
- Returns:
- `ToolResponse`:
- The tool response containing the result of the writing operation.
- """
- file_path = os.path.expanduser(file_path)
- if not os.path.exists(file_path):
- with open(file_path, "w", encoding="utf-8") as file:
- file.write(content)
-
- if ranges:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Create and write {file_path} successfully. "
- f"The ranges {ranges} is ignored because the "
- f"file does not exist.",
- ),
- ],
- )
-
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Create and write {file_path} successfully.",
- ),
- ],
- )
-
- with open(file_path, "r", encoding="utf-8") as file:
- original_lines = file.readlines()
-
- if ranges is not None:
- if (
- isinstance(ranges, list)
- and len(ranges) == 2
- and all(isinstance(i, int) for i in ranges)
- ):
- # Replace content in the specified range
- start, end = ranges
- if start > len(original_lines):
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: The start line {start} is invalid. "
- f"The file only has {len(original_lines)} "
- f"lines.",
- ),
- ],
- )
-
- new_content = (
- original_lines[: start - 1]
- + [
- content,
- ]
- + original_lines[end:]
- )
-
- with open(file_path, "w", encoding="utf-8") as file:
- file.write("".join(new_content))
-
- # The written content may contain multiple "\n", to avoid mis
- # counting the lines, we read the file again to get the new content
- with open(file_path, "r", encoding="utf-8") as file:
- new_lines = file.readlines()
-
- view_start, view_end = _calculate_view_ranges(
- len(original_lines),
- len(new_lines),
- start,
- end,
- )
-
- content = "".join(
- [
- f"{index + view_start}: {line}"
- for index, line in enumerate(
- new_lines[view_start - 1 : view_end],
- )
- ],
- )
-
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"""Write {file_path} successfully. The new content snippet:
-```
-{content}```""",
- ),
- ],
- )
-
- else:
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Error: Invalid range format. Expected a list "
- f"of two integers, but got {ranges}.",
- ),
- ],
- )
-
- with open(file_path, "w", encoding="utf-8") as file:
- file.write(content)
-
- return ToolResponse(
- content=[
- TextBlock(
- type="text",
- text=f"Overwrite {file_path} successfully.",
- ),
- ],
- )
diff --git a/src/agentscope/tool/_toolkit.py b/src/agentscope/tool/_toolkit.py
index d17257f65..87e2c6bdd 100644
--- a/src/agentscope/tool/_toolkit.py
+++ b/src/agentscope/tool/_toolkit.py
@@ -42,10 +42,9 @@ from ..mcp import (
StatefulClientBase,
)
from ..message import (
- ToolUseBlock,
+ ToolCallBlock,
TextBlock,
)
-from ..module import StateModule
from ..types import (
JSONSerializableObject,
ToolFunction,
@@ -72,7 +71,7 @@ def _apply_middlewares(
@wraps(func)
async def wrapper(
self: "Toolkit",
- tool_call: ToolUseBlock,
+ tool_call: ToolCallBlock,
) -> AsyncGenerator[ToolResponse, None]:
"""Wrapper that applies middleware chain."""
middlewares = getattr(self, "_middlewares", [])
@@ -114,7 +113,7 @@ def _apply_middlewares(
return wrapper
-class Toolkit(StateModule): # pylint: disable=too-many-public-methods
+class Toolkit: # pylint: disable=too-many-public-methods
"""Toolkit is the core module to register, manage and delete tool
functions, MCP clients, Agent skills in AgentScope.
@@ -284,11 +283,11 @@ Check "{dir}/SKILL.md" for how to use this skill"""
include_var_keyword: bool = False,
postprocess_func: (
Callable[
- [ToolUseBlock, ToolResponse],
+ [ToolCallBlock, ToolResponse],
ToolResponse | None,
]
| Callable[
- [ToolUseBlock, ToolResponse],
+ [ToolCallBlock, ToolResponse],
Awaitable[ToolResponse | None],
]
)
@@ -338,8 +337,8 @@ Check "{dir}/SKILL.md" for how to use this skill"""
include_var_keyword (`bool`, defaults to `False`):
Whether to include the variable keyword arguments (`**kwargs`)
in the function schema.
- postprocess_func (`(Callable[[ToolUseBlock, ToolResponse], \
- ToolResponse | None] | Callable[[ToolUseBlock, ToolResponse], \
+ postprocess_func (`(Callable[[ToolCallBlock, ToolResponse], \
+ ToolResponse | None] | Callable[[ToolCallBlock, ToolResponse], \
Awaitable[ToolResponse | None]]) | None`, optional):
A post-processing function that will be called after the tool
function is executed, taking the tool call block and tool
@@ -852,16 +851,16 @@ Check "{dir}/SKILL.md" for how to use this skill"""
@_apply_middlewares
async def call_tool_function(
self,
- tool_call: ToolUseBlock,
+ tool_call: ToolCallBlock,
) -> AsyncGenerator[ToolResponse, None]:
- """Execute the tool function by the `ToolUseBlock` and return the
+ """Execute the tool function by the `ToolCallBlock` and return the
tool response chunk in unified streaming mode, i.e. an async
generator of `ToolResponse` objects.
.. note:: The tool response chunk is **accumulated**.
Args:
- tool_call (`ToolUseBlock`):
+ tool_call (`ToolCallBlock`):
A tool call block.
Yields:
@@ -917,7 +916,7 @@ Check "{dir}/SKILL.md" for how to use this skill"""
# Prepare postprocess function
if tool_func.postprocess_func:
# Type: partial wraps the postprocess_func with tool_call bound,
- # reducing it from (ToolUseBlock, ToolResponse) to (ToolResponse)
+ # reducing it from (ToolCallBlock, ToolResponse) to (ToolResponse)
partial_postprocess_func: (
Callable[[ToolResponse], ToolResponse | None]
| Callable[[ToolResponse], Awaitable[ToolResponse | None]]
@@ -1041,11 +1040,11 @@ Check "{dir}/SKILL.md" for how to use this skill"""
preset_kwargs_mapping: dict[str, dict[str, Any]] | None = None,
postprocess_func: (
Callable[
- [ToolUseBlock, ToolResponse],
+ [ToolCallBlock, ToolResponse],
ToolResponse | None,
]
| Callable[
- [ToolUseBlock, ToolResponse],
+ [ToolCallBlock, ToolResponse],
Awaitable[ToolResponse | None],
]
)
@@ -1074,8 +1073,8 @@ Check "{dir}/SKILL.md" for how to use this skill"""
defaults to `None`):
The preset keyword arguments mapping, whose keys are the tool
function names and values are the preset keyword arguments.
- postprocess_func (`(Callable[[ToolUseBlock, ToolResponse], \
- ToolResponse | None] | Callable[[ToolUseBlock, ToolResponse], \
+ postprocess_func (`(Callable[[ToolCallBlock, ToolResponse], \
+ ToolResponse | None] | Callable[[ToolCallBlock, ToolResponse], \
Awaitable[ToolResponse | None]]) | None`, optional):
A post-processing function that will be called after the tool
function is executed, taking the tool call block and tool
@@ -1459,7 +1458,7 @@ Check "{dir}/SKILL.md" for how to use this skill"""
parameter and ``next_handler`` as the second parameter. The ``kwargs``
dict currently contains:
- - ``tool_call`` (`ToolUseBlock`): The tool call request
+ - ``tool_call`` (`ToolCallBlock`): The tool call request
When calling ``next_handler``, pass ``**kwargs`` to unpack the dict.
@@ -1521,7 +1520,7 @@ AsyncGenerator[ToolResponse, None]] | AsyncGenerator[ToolResponse, None]]`):
The middleware function that accepts ``kwargs`` (dict) and
``next_handler`` (Callable), and returns a coroutine that
yields AsyncGenerator of ToolResponse objects. The ``kwargs``
- dict currently includes ``tool_call`` (ToolUseBlock), and may
+ dict currently includes ``tool_call`` (ToolCallBlock), and may
include additional context in future versions.
.. note:: The middleware chain is applied inside the
diff --git a/src/agentscope/tool/_types.py b/src/agentscope/tool/_types.py
index 3dbc8cfe2..386fea4c8 100644
--- a/src/agentscope/tool/_types.py
+++ b/src/agentscope/tool/_types.py
@@ -8,7 +8,7 @@ from pydantic import BaseModel
from . import ToolResponse
from .._utils._common import _remove_title_field
-from ..message import ToolUseBlock
+from ..message import ToolCallBlock
from ..types import ToolFunction, JSONSerializableObject
@@ -40,11 +40,11 @@ class RegisteredToolFunction:
"""The name of the MCP, if the tool function comes from an MCP server."""
postprocess_func: (
Callable[
- [ToolUseBlock, ToolResponse],
+ [ToolCallBlock, ToolResponse],
ToolResponse | None,
]
| Callable[
- [ToolUseBlock, ToolResponse],
+ [ToolCallBlock, ToolResponse],
Awaitable[ToolResponse | None],
]
) | None = None
diff --git a/src/agentscope/tracing/_extractor.py b/src/agentscope/tracing/_extractor.py
index 391073856..06179db09 100644
--- a/src/agentscope/tracing/_extractor.py
+++ b/src/agentscope/tracing/_extractor.py
@@ -4,7 +4,7 @@ import inspect
from typing import Any, Dict, Tuple, TYPE_CHECKING
from ..embedding import EmbeddingModelBase
-from ..message import Msg, ToolUseBlock
+from ..message import Msg, ToolCallBlock
from ..model import ChatModelBase
from ._attributes import (
@@ -551,7 +551,7 @@ def _get_agent_response_attributes(
def _get_tool_request_attributes(
instance: "Toolkit",
- tool_call: ToolUseBlock,
+ tool_call: ToolCallBlock,
) -> Dict[str, str]:
"""Get tool request attributes for OpenTelemetry tracing.
@@ -561,7 +561,7 @@ def _get_tool_request_attributes(
instance (`Toolkit`):
Toolkit instance with tool definitions. Used to extract tool
description from the tool's JSON schema.
- tool_call (`ToolUseBlock`):
+ tool_call (`ToolCallBlock`):
Tool use block with call information including id, name, and input
arguments.
diff --git a/src/agentscope/tracing/_trace.py b/src/agentscope/tracing/_trace.py
index c56bd9544..bc878f0eb 100644
--- a/src/agentscope/tracing/_trace.py
+++ b/src/agentscope/tracing/_trace.py
@@ -17,7 +17,7 @@ import aioitertools
from ..embedding import EmbeddingModelBase, EmbeddingResponse
from .._logging import logger
-from ..message import Msg, ToolUseBlock
+from ..message import Msg, ToolCallBlock
from ..model import ChatModelBase, ChatResponse
from ._attributes import SpanAttributes, OperationNameValues
@@ -326,7 +326,7 @@ def trace_toolkit(
@wraps(func)
async def wrapper(
self: Toolkit,
- tool_call: ToolUseBlock,
+ tool_call: ToolCallBlock,
) -> AsyncGenerator[ToolResponse, None]:
"""The wrapper function for tracing the toolkit call_tool_function
method."""
@@ -388,16 +388,16 @@ def trace_reply(
if not _check_tracing_enabled():
return await func(self, *args, **kwargs)
- from ..agent import AgentBase
-
- if not isinstance(self, AgentBase):
- logger.warning(
- "Skipping tracing for %s as the first argument"
- "is not an instance of AgentBase, but %s",
- func.__name__,
- type(self),
- )
- return await func(self, *args, **kwargs)
+ # from ..agent import AgentBase
+ #
+ # if not isinstance(self, AgentBase):
+ # logger.warning(
+ # "Skipping tracing for %s as the first argument"
+ # "is not an instance of AgentBase, but %s",
+ # func.__name__,
+ # type(self),
+ # )
+ # return await func(self, *args, **kwargs)
tracer = _get_tracer()
diff --git a/src/agentscope/types/__init__.py b/src/agentscope/types/__init__.py
index a8e92a3f5..b8da5ac65 100644
--- a/src/agentscope/types/__init__.py
+++ b/src/agentscope/types/__init__.py
@@ -10,7 +10,7 @@ from ._json import (
JSONPrimitive,
JSONSerializableObject,
)
-from ._tool import ToolFunction
+from ._tool import ToolFunction, ToolChoice
__all__ = [
"AgentHookTypes",
@@ -19,4 +19,5 @@ __all__ = [
"JSONPrimitive",
"JSONSerializableObject",
"ToolFunction",
+ "ToolChoice",
]
diff --git a/src/agentscope/types/_json.py b/src/agentscope/types/_json.py
index 6ab7e230c..3a506a01c 100644
--- a/src/agentscope/types/_json.py
+++ b/src/agentscope/types/_json.py
@@ -1,21 +1,14 @@
# -*- coding: utf-8 -*-
"""The JSON related types"""
+from typing import TypeAlias
-from typing import Union
+JSONPrimitive: TypeAlias = str | int | float | bool | None
-JSONPrimitive = Union[
- str,
- int,
- float,
- bool,
- None,
-]
-
-JSONSerializableObject = Union[
- JSONPrimitive,
- list["JSONSerializableObject"],
- dict[
+JSONSerializableObject: TypeAlias = (
+ JSONPrimitive
+ | list["JSONSerializableObject"]
+ | dict[
str,
"JSONSerializableObject",
- ],
-]
+ ]
+)
diff --git a/src/agentscope/types/_tool.py b/src/agentscope/types/_tool.py
index 588eb2ab9..f80ec483d 100644
--- a/src/agentscope/types/_tool.py
+++ b/src/agentscope/types/_tool.py
@@ -10,6 +10,8 @@ from typing import (
Coroutine,
Any,
TYPE_CHECKING,
+ TypeAlias,
+ Literal,
)
if TYPE_CHECKING:
@@ -34,3 +36,6 @@ ToolFunction = Callable[
Coroutine[Any, Any, Generator[ToolResponse, None, None]],
],
]
+
+
+ToolChoice: TypeAlias = Literal["auto", "none", "required"] | str