Skip to content

Build and run agents you can see, understand and trust.

fix(realtime): format the tools argument for the `OpenAIRealtimeModel` (#1216)

qbcd6df246a3967
diff --git a/src/agentscope/_version.py b/src/agentscope/_version.py
index 05c528ac8..17cb2915a 100644
--- a/src/agentscope/_version.py
+++ b/src/agentscope/_version.py
@@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
"""The version of agentscope."""
-__version__ = "1.0.15"
+__version__ = "1.0.16dev"
diff --git a/src/agentscope/realtime/_openai_realtime_model.py b/src/agentscope/realtime/_openai_realtime_model.py
index 355727566..9209adc27 100644
--- a/src/agentscope/realtime/_openai_realtime_model.py
+++ b/src/agentscope/realtime/_openai_realtime_model.py
@@ -19,7 +19,7 @@ from ..message import (
class OpenAIRealtimeModel(RealtimeModelBase):
"""The OpenAI realtime model class."""
- support_input_modalities: list[str] = ["audio", "text"]
+ support_input_modalities: list[str] = ["audio", "text", "tool_result"]
"""The supported input modalities of the OpenAI realtime model."""
support_tools: bool = True
@@ -117,14 +117,37 @@ class OpenAIRealtimeModel(RealtimeModelBase):
# Tools configuration
if tools:
- # TODO: format the tools
- session_config["tools"] = tools
+ session_config["tools"] = self._format_toolkit_schema(tools)
return {
"type": "session.update",
"session": session_config,
}
+ def _format_toolkit_schema(
+ self,
+ schemas: list[dict[str, Any]],
+ ) -> list[dict[str, Any]]:
+ """Format the tools JSON schema into OpenAI realtime model format.
+
+ Args:
+ schemas (`list[dict[str, Any]]`):
+ The tool schemas.
+
+ Returns:
+ `list[dict[str, Any]]`:
+ The formatted tools for OpenAI realtime model.
+
+ .. note::
+ The OpenAI Realtime API uses a different tool format compared to
+ the regular Chat Completions API. While the Chat API expects tools
+ to be wrapped in ``{"type": "function", "function": {...}}``, the
+ Realtime API expects a flattened structure where the function
+ definition is directly at the top level with an added ``"type":
+ "function"`` field.
+ """
+ return [{"type": "function", **tool["function"]} for tool in schemas]
+
async def send(
self,
data: AudioBlock | TextBlock | ImageBlock | ToolResultBlock,
@@ -228,7 +251,6 @@ class OpenAIRealtimeModel(RealtimeModelBase):
# ================ Response related events ================
case "response.created":
- # TODO: @qbc we create one shortuuid here?
self._response_id = data.get("response", {}).get("id", "")
model_event = ModelEvents.ModelResponseCreatedEvent(
response_id=self._response_id,
diff --git a/src/agentscope/tts/_openai_tts_model.py b/src/agentscope/tts/_openai_tts_model.py
index 46978a48c..cbb7b801a 100644
--- a/src/agentscope/tts/_openai_tts_model.py
+++ b/src/agentscope/tts/_openai_tts_model.py
@@ -103,7 +103,7 @@ class OpenAITTSModel(TTSModelBase):
model=self.model_name,
voice=self.voice,
input=text,
- response_format="mp3",
+ response_format="pcm",
**self.generate_kwargs,
**kwargs,
)