Skip to content

Build and run agents you can see, understand and trust.

ci(realtime): add unittests for `OpenAIRealtimeModel` and `GeminiRealtimeModel` (#1235)

qbcd04505718824
diff --git a/tests/realtime_gemini_test.py b/tests/realtime_gemini_test.py
new file mode 100644
index 000000000..bbea6b539
--- /dev/null
+++ b/tests/realtime_gemini_test.py
@@ -0,0 +1,297 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access
+"""Unit tests for Gemini Realtime Model class."""
+import json
+from unittest.async_case import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock
+
+from agentscope.realtime import GeminiRealtimeModel, ModelEvents
+from agentscope.message import (
+ AudioBlock,
+ TextBlock,
+ ImageBlock,
+ ToolResultBlock,
+ Base64Source,
+)
+
+
+class TestGeminiRealtimeModelParseAPIMessage(IsolatedAsyncioTestCase):
+ """Test parsing API messages from Gemini realtime model."""
+
+ def setUp(self) -> None:
+ """Set up test fixtures."""
+ self.model = GeminiRealtimeModel(
+ model_name="gemini-2.5-flash-native-audio-preview",
+ api_key="test_api_key",
+ voice="Puck",
+ )
+
+ async def test_parse_setup_complete_event(self) -> None:
+ """Test parsing setupComplete event."""
+ message = json.dumps({"setupComplete": {}})
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelSessionCreatedEvent)
+ self.assertEqual(event.session_id, "gemini_session")
+ self.assertEqual(event.type, "model_session_created")
+
+ async def test_parse_audio_delta_event(self) -> None:
+ """Test parsing serverContent with audio data."""
+ message = json.dumps(
+ {
+ "serverContent": {
+ "modelTurn": {
+ "parts": [
+ {
+ "inlineData": {
+ "mimeType": "audio/pcm;rate=24000",
+ "data": "base64_audio_chunk",
+ },
+ },
+ ],
+ },
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelResponseAudioDeltaEvent)
+ self.assertEqual(event.delta, "base64_audio_chunk")
+ self.assertEqual(event.format.type, "audio/pcm")
+ self.assertEqual(event.format.rate, 24000)
+ self.assertEqual(event.type, "model_response_audio_delta")
+
+ async def test_parse_output_transcription_event(self) -> None:
+ """Test parsing serverContent with output transcription."""
+ message = json.dumps(
+ {
+ "serverContent": {
+ "outputTranscription": {
+ "text": "Hello there",
+ },
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelResponseAudioTranscriptDeltaEvent,
+ )
+ self.assertEqual(event.delta, "Hello there")
+
+ async def test_parse_input_transcription_event(self) -> None:
+ """Test parsing serverContent with input transcription."""
+ message = json.dumps(
+ {
+ "serverContent": {
+ "inputTranscription": {
+ "text": "How are you?",
+ },
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelInputTranscriptionDoneEvent,
+ )
+ self.assertEqual(event.transcript, "How are you?")
+
+ async def test_parse_generation_complete_event(self) -> None:
+ """Test parsing serverContent with generationComplete."""
+ self.model._response_id = "resp_123"
+
+ message = json.dumps(
+ {
+ "serverContent": {
+ "generationComplete": True,
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelResponseDoneEvent)
+ self.assertEqual(event.response_id, "resp_123")
+ self.assertIsNone(self.model._response_id)
+
+ async def test_parse_turn_complete_event(self) -> None:
+ """Test parsing serverContent with turnComplete."""
+ self.model._response_id = "resp_456"
+
+ message = json.dumps(
+ {
+ "serverContent": {
+ "turnComplete": True,
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelResponseDoneEvent)
+ self.assertEqual(event.response_id, "resp_456")
+ self.assertIsNone(self.model._response_id)
+
+ async def test_parse_tool_call_event(self) -> None:
+ """Test parsing toolCall event."""
+ message = json.dumps(
+ {
+ "toolCall": {
+ "functionCalls": [
+ {
+ "name": "get_weather",
+ "id": "call_789",
+ "args": {"location": "San Francisco"},
+ },
+ ],
+ },
+ },
+ )
+
+ events = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(events, list)
+ self.assertEqual(len(events), 1)
+ event = events[0]
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelResponseToolUseDoneEvent,
+ )
+ self.assertEqual(event.tool_use["name"], "get_weather")
+ self.assertEqual(event.tool_use["id"], "call_789")
+ self.assertEqual(
+ event.tool_use["input"],
+ {"location": "San Francisco"},
+ )
+
+ async def test_parse_error_event(self) -> None:
+ """Test parsing error event."""
+ message = json.dumps(
+ {
+ "error": {
+ "status": "INVALID_ARGUMENT",
+ "code": 400,
+ "message": "Invalid request",
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelErrorEvent)
+ self.assertEqual(event.error_type, "INVALID_ARGUMENT")
+ self.assertEqual(event.code, "400")
+ self.assertEqual(event.message, "Invalid request")
+ self.assertEqual(event.type, "model_error")
+
+
+class TestGeminiRealtimeModelSend(IsolatedAsyncioTestCase):
+ """Test sending data to Gemini realtime model."""
+
+ def setUp(self) -> None:
+ """Set up test fixtures."""
+ from websockets import State
+
+ self.model = GeminiRealtimeModel(
+ model_name="gemini-2.5-flash-native-audio-preview",
+ api_key="test_api_key",
+ voice="Puck",
+ )
+ self.mock_websocket = AsyncMock()
+ self.mock_websocket.state = State.OPEN
+ self.model._websocket = self.mock_websocket
+
+ async def test_send_audio_base64(self) -> None:
+ """Test sending audio data with base64 source."""
+ audio_data = AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ media_type="audio/wav",
+ data="base64_encoded_audio_data",
+ ),
+ )
+
+ await self.model.send(audio_data)
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertIn("realtimeInput", sent_data)
+ self.assertEqual(
+ sent_data["realtimeInput"]["audio"]["data"],
+ "base64_encoded_audio_data",
+ )
+
+ async def test_send_image_base64(self) -> None:
+ """Test sending image data with base64 source."""
+ image_data = ImageBlock(
+ type="image",
+ source=Base64Source(
+ type="base64",
+ media_type="image/jpeg",
+ data="base64_encoded_image_data",
+ ),
+ )
+
+ await self.model.send(image_data)
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertIn("realtimeInput", sent_data)
+ self.assertEqual(
+ sent_data["realtimeInput"]["video"]["data"],
+ "base64_encoded_image_data",
+ )
+
+ async def test_send_text(self) -> None:
+ """Test sending text data."""
+ text_data = TextBlock(
+ type="text",
+ text="Hello, how are you?",
+ )
+
+ await self.model.send(text_data)
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertIn("clientContent", sent_data)
+ turns = sent_data["clientContent"]["turns"]
+ self.assertEqual(turns[0]["parts"][0]["text"], "Hello, how are you?")
+
+ async def test_send_tool_result(self) -> None:
+ """Test sending tool result data."""
+ tool_result = ToolResultBlock(
+ type="tool_result",
+ id="call_123",
+ output="The weather is sunny",
+ name="get_weather",
+ )
+
+ await self.model.send(tool_result)
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertIn("toolResponse", sent_data)
+ func_responses = sent_data["toolResponse"]["functionResponses"]
+ self.assertEqual(len(func_responses), 1)
+ self.assertEqual(func_responses[0]["id"], "call_123")
+ self.assertEqual(func_responses[0]["name"], "get_weather")
diff --git a/tests/realtime_openai_test.py b/tests/realtime_openai_test.py
new file mode 100644
index 000000000..82eb62cd7
--- /dev/null
+++ b/tests/realtime_openai_test.py
@@ -0,0 +1,430 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access
+"""Unit tests for OpenAI Realtime Model class."""
+import json
+from unittest.async_case import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, patch
+
+from agentscope.realtime import OpenAIRealtimeModel, ModelEvents
+from agentscope.message import (
+ AudioBlock,
+ TextBlock,
+ ToolResultBlock,
+ Base64Source,
+ URLSource,
+)
+
+
+class TestOpenAIRealtimeModelParseAPIMessage(IsolatedAsyncioTestCase):
+ """Test parsing API messages from OpenAI realtime model."""
+
+ def setUp(self) -> None:
+ """Set up test fixtures."""
+ self.model = OpenAIRealtimeModel(
+ model_name="gpt-4o-realtime-preview",
+ api_key="test_api_key",
+ voice="alloy",
+ )
+
+ async def test_parse_session_created_event(self) -> None:
+ """Test parsing session.created event."""
+ message = json.dumps(
+ {
+ "type": "session.created",
+ "session": {
+ "id": "session_123",
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelSessionCreatedEvent)
+ self.assertEqual(event.session_id, "session_123")
+ self.assertEqual(event.type, "model_session_created")
+
+ async def test_parse_response_created_event(self) -> None:
+ """Test parsing response.created event."""
+ message = json.dumps(
+ {
+ "type": "response.created",
+ "response": {
+ "id": "resp_456",
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelResponseCreatedEvent)
+ self.assertEqual(event.response_id, "resp_456")
+ self.assertEqual(event.type, "model_response_created")
+ self.assertEqual(self.model._response_id, "resp_456")
+
+ async def test_parse_response_done_event(self) -> None:
+ """Test parsing response.done event."""
+ self.model._response_id = "resp_789"
+
+ message = json.dumps(
+ {
+ "type": "response.done",
+ "response": {
+ "id": "resp_789",
+ "usage": {
+ "input_tokens": 100,
+ "output_tokens": 50,
+ },
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelResponseDoneEvent)
+ self.assertEqual(event.response_id, "resp_789")
+ self.assertEqual(event.input_tokens, 100)
+ self.assertEqual(event.output_tokens, 50)
+ self.assertEqual(event.type, "model_response_done")
+ self.assertEqual(self.model._response_id, "")
+
+ async def test_parse_response_audio_delta_event(self) -> None:
+ """Test parsing response.output_audio.delta event."""
+ self.model._response_id = "resp_audio_1"
+
+ message = json.dumps(
+ {
+ "type": "response.output_audio.delta",
+ "item_id": "item_audio_1",
+ "delta": "base64_audio_data_chunk",
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelResponseAudioDeltaEvent)
+ self.assertEqual(event.response_id, "resp_audio_1")
+ self.assertEqual(event.item_id, "item_audio_1")
+ self.assertEqual(event.delta, "base64_audio_data_chunk")
+ self.assertEqual(event.format.type, "audio/pcm")
+ self.assertEqual(event.format.rate, 24000)
+ self.assertEqual(event.type, "model_response_audio_delta")
+
+ async def test_parse_response_audio_done_event(self) -> None:
+ """Test parsing response.output_audio.done event."""
+ self.model._response_id = "resp_audio_2"
+
+ message = json.dumps(
+ {
+ "type": "response.output_audio.done",
+ "item_id": "item_audio_2",
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelResponseAudioDoneEvent)
+ self.assertEqual(event.response_id, "resp_audio_2")
+ self.assertEqual(event.item_id, "item_audio_2")
+ self.assertEqual(event.type, "model_response_audio_done")
+
+ async def test_parse_response_audio_transcript_delta_event(self) -> None:
+ """Test parsing response.output_audio_transcript.delta event."""
+ self.model._response_id = "resp_transcript_1"
+
+ message = json.dumps(
+ {
+ "type": "response.output_audio_transcript.delta",
+ "item_id": "item_transcript_1",
+ "delta": "Hello ",
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelResponseAudioTranscriptDeltaEvent,
+ )
+ self.assertEqual(event.response_id, "resp_transcript_1")
+ self.assertEqual(event.item_id, "item_transcript_1")
+ self.assertEqual(event.delta, "Hello ")
+ self.assertEqual(
+ event.type,
+ "model_response_audio_transcript_delta",
+ )
+
+ async def test_parse_response_audio_transcript_done_event(self) -> None:
+ """Test parsing response.output_audio_transcript.done event."""
+ self.model._response_id = "resp_transcript_2"
+
+ message = json.dumps(
+ {
+ "type": "response.output_audio_transcript.done",
+ "item_id": "item_transcript_2",
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelResponseAudioTranscriptDoneEvent,
+ )
+ self.assertEqual(event.response_id, "resp_transcript_2")
+ self.assertEqual(event.item_id, "item_transcript_2")
+ self.assertEqual(event.type, "model_response_audio_transcript_done")
+
+ async def test_parse_function_call_arguments_delta_event(self) -> None:
+ """Test parsing response.function_call_arguments.delta event."""
+ self.model._response_id = "resp_tool_1"
+
+ message = json.dumps(
+ {
+ "type": "response.function_call_arguments.delta",
+ "call_id": "call_123",
+ "item_id": "item_tool_1",
+ "name": "get_weather",
+ "delta": '{"location": "San',
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelResponseToolUseDeltaEvent,
+ )
+ self.assertEqual(event.response_id, "resp_tool_1")
+ self.assertEqual(event.item_id, "item_tool_1")
+ self.assertEqual(event.tool_use["id"], "call_123")
+ self.assertEqual(event.tool_use["name"], "get_weather")
+
+ async def test_parse_function_call_arguments_done_event(self) -> None:
+ """Test parsing response.function_call_arguments.done event."""
+ self.model._response_id = "resp_tool_2"
+ self.model._tool_args_accumulator[
+ "call_456"
+ ] = '{"location": "San Francisco"}'
+
+ message = json.dumps(
+ {
+ "type": "response.function_call_arguments.done",
+ "call_id": "call_456",
+ "item_id": "item_tool_2",
+ "name": "get_weather",
+ "arguments": '{"location": "San Francisco"}',
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelResponseToolUseDoneEvent,
+ )
+ self.assertEqual(event.response_id, "resp_tool_2")
+ self.assertEqual(event.item_id, "item_tool_2")
+ self.assertEqual(event.tool_use["id"], "call_456")
+ self.assertEqual(event.tool_use["name"], "get_weather")
+ self.assertEqual(
+ event.tool_use["input"],
+ {"location": "San Francisco"},
+ )
+ self.assertNotIn("call_456", self.model._tool_args_accumulator)
+
+ async def test_parse_input_audio_transcription_completed_event(
+ self,
+ ) -> None:
+ """Test parsing
+ conversation.item.input_audio_transcription.completed event."""
+ message = json.dumps(
+ {
+ "type": "conversation.item.input_audio_transcription."
+ "completed",
+ "item_id": "item_input_1",
+ "transcript": "Hello world",
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(
+ event,
+ ModelEvents.ModelInputTranscriptionDoneEvent,
+ )
+ self.assertEqual(event.item_id, "item_input_1")
+ self.assertEqual(event.transcript, "Hello world")
+ self.assertEqual(event.type, "model_input_transcription_done")
+
+ async def test_parse_input_audio_buffer_speech_started_event(
+ self,
+ ) -> None:
+ """Test parsing input_audio_buffer.speech_started event."""
+ message = json.dumps(
+ {
+ "type": "input_audio_buffer.speech_started",
+ "item_id": "item_vad_1",
+ "audio_start_ms": 1000,
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelInputStartedEvent)
+ self.assertEqual(event.item_id, "item_vad_1")
+ self.assertEqual(event.audio_start_ms, 1000)
+ self.assertEqual(event.type, "model_input_started")
+
+ async def test_parse_input_audio_buffer_speech_stopped_event(
+ self,
+ ) -> None:
+ """Test parsing input_audio_buffer.speech_stopped event."""
+ message = json.dumps(
+ {
+ "type": "input_audio_buffer.speech_stopped",
+ "item_id": "item_vad_2",
+ "audio_end_ms": 5000,
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelInputDoneEvent)
+ self.assertEqual(event.item_id, "item_vad_2")
+ self.assertEqual(event.audio_end_ms, 5000)
+ self.assertEqual(event.type, "model_input_done")
+
+ async def test_parse_error_event(self) -> None:
+ """Test parsing error event."""
+ message = json.dumps(
+ {
+ "type": "error",
+ "error": {
+ "type": "invalid_request",
+ "code": "400",
+ "message": "Invalid request format",
+ },
+ },
+ )
+
+ event = await self.model.parse_api_message(message)
+
+ self.assertIsInstance(event, ModelEvents.ModelErrorEvent)
+ self.assertEqual(event.error_type, "invalid_request")
+ self.assertEqual(event.code, "400")
+ self.assertEqual(event.message, "Invalid request format")
+ self.assertEqual(event.type, "model_error")
+
+
+class TestOpenAIRealtimeModelSend(IsolatedAsyncioTestCase):
+ """Test sending data to OpenAI realtime model."""
+
+ def setUp(self) -> None:
+ """Set up test fixtures."""
+ from websockets import State
+
+ self.model = OpenAIRealtimeModel(
+ model_name="gpt-4o-realtime-preview",
+ api_key="test_api_key",
+ voice="alloy",
+ )
+ self.mock_websocket = AsyncMock()
+ self.mock_websocket.state = State.OPEN
+ self.model._websocket = self.mock_websocket
+
+ async def test_send_audio_base64(self) -> None:
+ """Test sending audio data with base64 source."""
+ audio_data = AudioBlock(
+ type="audio",
+ source=Base64Source(
+ type="base64",
+ media_type="audio/wav",
+ data="base64_encoded_audio_data",
+ ),
+ )
+
+ await self.model.send(audio_data)
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertEqual(sent_data["type"], "input_audio_buffer.append")
+ self.assertEqual(sent_data["audio"], "base64_encoded_audio_data")
+
+ async def test_send_text(self) -> None:
+ """Test sending text data."""
+ text_data = TextBlock(
+ type="text",
+ text="Hello, how are you?",
+ )
+
+ await self.model.send(text_data)
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertEqual(sent_data["type"], "conversation.item.create")
+ self.assertEqual(sent_data["item"]["type"], "message")
+ self.assertEqual(sent_data["item"]["role"], "user")
+ self.assertEqual(
+ sent_data["item"]["content"][0]["text"],
+ "Hello, how are you?",
+ )
+
+ async def test_send_tool_result(self) -> None:
+ """Test sending tool result data."""
+ tool_result = ToolResultBlock(
+ type="tool_result",
+ id="call_123",
+ output="The weather is sunny, 25°C",
+ name="get_weather",
+ )
+
+ await self.model.send(tool_result)
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertEqual(sent_data["type"], "conversation.item.create")
+ self.assertEqual(sent_data["item"]["type"], "function_call_output")
+ self.assertEqual(sent_data["item"]["call_id"], "call_123")
+ self.assertEqual(
+ sent_data["item"]["output"],
+ "The weather is sunny, 25°C",
+ )
+
+ async def test_send_audio_url(self) -> None:
+ """Test sending audio data with URL source."""
+ audio_data = AudioBlock(
+ type="audio",
+ source=URLSource(
+ type="url",
+ url="https://example.com/audio.wav",
+ ),
+ )
+
+ with patch(
+ "agentscope.realtime._openai_realtime_model."
+ "_get_bytes_from_web_url",
+ ) as mock_get_bytes:
+ mock_get_bytes.return_value = "fetched_audio_bytes"
+
+ await self.model.send(audio_data)
+
+ mock_get_bytes.assert_called_once_with(
+ "https://example.com/audio.wav",
+ )
+
+ self.mock_websocket.send.assert_called_once()
+
+ sent_message = self.mock_websocket.send.call_args[0][0]
+ sent_data = json.loads(sent_message)
+
+ self.assertEqual(sent_data["type"], "input_audio_buffer.append")
+ self.assertEqual(sent_data["audio"], "fetched_audio_bytes")