Skip to content

Build and run agents you can see, understand and trust.

feat(realtime): support xAI Grok voice API model (#2553)

DavdGao467d60806947
diff --git a/src/agentscope/credential/_xai.py b/src/agentscope/credential/_xai.py
index ec0df6447..a2f02c26d 100644
--- a/src/agentscope/credential/_xai.py
+++ b/src/agentscope/credential/_xai.py
@@ -8,6 +8,7 @@ from ._base import CredentialBase
if TYPE_CHECKING:
from ..model import ChatModelBase
+ from ..realtime import RealtimeModelBase
class XAICredential(CredentialBase):
@@ -41,3 +42,12 @@ class XAICredential(CredentialBase):
from ..model import XAIChatModel
return XAIChatModel
+
+ @classmethod
+ def get_realtime_model_classes(
+ cls,
+ ) -> list[Type["RealtimeModelBase"]]:
+ """Return the xAI realtime model classes."""
+ from ..realtime import XAIRealtimeModel
+
+ return [XAIRealtimeModel]
diff --git a/src/agentscope/realtime/__init__.py b/src/agentscope/realtime/__init__.py
index 97e1d3def..85ee2b267 100644
--- a/src/agentscope/realtime/__init__.py
+++ b/src/agentscope/realtime/__init__.py
@@ -34,12 +34,14 @@ from ._transport import (
TransportFrame,
)
from ._vad import SpeechTransition, VADBase
+from ._xai import XAIRealtimeModel
__all__ = [
# Model
"RealtimeModelBase",
"DashScopeRealtimeModel",
"DashScopeAudioRealtimeModel",
+ "XAIRealtimeModel",
"RealtimeModelCard",
"TruncationSupport",
"ModelDisconnectedError",
diff --git a/src/agentscope/realtime/_xai/__init__.py b/src/agentscope/realtime/_xai/__init__.py
new file mode 100644
index 000000000..a390edbe7
--- /dev/null
+++ b/src/agentscope/realtime/_xai/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+"""The xAI realtime models."""
+from ._model import XAIRealtimeModel
+
+__all__ = ["XAIRealtimeModel"]
diff --git a/src/agentscope/realtime/_xai/_model.py b/src/agentscope/realtime/_xai/_model.py
new file mode 100644
index 000000000..e5a5131cc
--- /dev/null
+++ b/src/agentscope/realtime/_xai/_model.py
@@ -0,0 +1,380 @@
+# -*- coding: utf-8 -*-
+"""The xAI Grok voice realtime model."""
+import asyncio
+import base64
+import json
+from typing import Any, AsyncIterator, Literal
+
+from pydantic import Field
+
+from .. import _events as me
+from .._base import (
+ ModelDisconnectedError,
+ RealtimeModelBase,
+ TruncationSupport,
+)
+from .._model_card import RealtimeModelCard
+from ..._logging import logger
+from ...credential import XAICredential
+from ...message import TextBlock, ToolCallBlock, ToolResultBlock
+
+
+class XAIRealtimeModel(RealtimeModelBase):
+ """The Grok voice speech-to-speech API over WebSocket.
+
+ OpenAI-shaped, with three xAI departures: the audio format lives in a
+ nested ``session.audio`` block, the reply-side frames are the GA names
+ (``response.output_audio.delta``), and one response spans several
+ items, so events are grouped by ``response_id``. The docs list no
+ ``conversation.item.truncate``, so an interrupted reply is not
+ corrected on the provider side.
+ """
+
+ class Parameters(RealtimeModelBase.Parameters):
+ """Tuneables surfaced to the UI."""
+
+ voice: str = Field(
+ default="eve",
+ title="Voice",
+ description="A built-in voice name or a custom voice ID.",
+ )
+ turn_detection: Literal["server_vad", "none"] = Field(
+ default="server_vad",
+ title="Turn Detection",
+ description="``none`` hands endpointing to the caller.",
+ )
+ vad_threshold: float = Field(default=0.85, ge=0.1, le=0.9)
+ vad_silence_duration_ms: int = Field(default=500, ge=0, le=10000)
+ reasoning_effort: Literal["high", "none"] = Field(
+ default="high",
+ title="Reasoning Effort",
+ description="``none`` answers faster and more shallowly.",
+ )
+ language_hint: str | None = Field(
+ default=None,
+ title="Language Hint",
+ description="BCP-47 code biasing transcription, e.g. ``es-MX``.",
+ )
+
+ type = "xai_realtime"
+ truncation = TruncationSupport.NONE
+ supports_text_input = True
+
+ def __init__(
+ self,
+ model_name: str,
+ credential: XAICredential,
+ parameters: "XAIRealtimeModel.Parameters | None" = None,
+ model_card: RealtimeModelCard | None = None,
+ ) -> None:
+ """Initialize the xAI realtime model.
+
+ Args:
+ model_name (`str`):
+ The model name, e.g. ``"grok-voice-latest"``.
+ credential (`XAICredential`):
+ The xAI credential.
+ parameters (`XAIRealtimeModel.Parameters | None`, optional):
+ Provider parameters; defaults are used if omitted.
+ model_card (`RealtimeModelCard | None`, optional):
+ The model card, looked up by name if omitted.
+ """
+ super().__init__(model_name, credential, parameters, model_card)
+ self.parameters: XAIRealtimeModel.Parameters
+ self._ws: Any = None
+ self._reader: asyncio.Task | None = None
+ self._queue: asyncio.Queue[me.ModelEvent | None] = asyncio.Queue()
+ self._response_id = ""
+ self._tool_args: dict[str, str] = {}
+
+ # ------------------------------------------------------------------
+ # Lifecycle
+ # ------------------------------------------------------------------
+
+ async def connect(
+ self,
+ instructions: str,
+ tools: list[dict] | None = None,
+ **kwargs: Any,
+ ) -> None:
+ """Open the WebSocket and send the session config."""
+ import websockets
+
+ if kwargs.get("turn_detection_disabled"):
+ self.parameters = self.parameters.model_copy(
+ update={"turn_detection": "none"},
+ )
+
+ credential: XAICredential = self.credential # type: ignore
+ self._ws = await websockets.connect(
+ f"wss://{credential.api_host}/v1/realtime"
+ f"?model={self.model_name}",
+ additional_headers={
+ "Authorization": f"Bearer "
+ f"{credential.api_key.get_secret_value()}",
+ },
+ )
+ self._reader = asyncio.create_task(self._read(), name="xai-rt")
+ await self._send(self._session_update(instructions, tools))
+
+ async def close(self) -> None:
+ """Stop reading and close the WebSocket."""
+ if self._reader is not None:
+ self._reader.cancel()
+ await asyncio.gather(self._reader, return_exceptions=True)
+ self._reader = None
+ if self._ws is not None:
+ await self._ws.close()
+ self._ws = None
+
+ async def events(self) -> AsyncIterator[me.ModelEvent]:
+ """Yield events until the session ends."""
+ while True:
+ event = await self._queue.get()
+ if event is None:
+ return
+ yield event
+
+ # ------------------------------------------------------------------
+ # Input
+ # ------------------------------------------------------------------
+
+ async def push_audio(self, pcm: bytes) -> None:
+ """Append PCM16 mono audio at :attr:`input_sample_rate`."""
+ await self._send(
+ {
+ "type": "input_audio_buffer.append",
+ "audio": base64.b64encode(pcm).decode("ascii"),
+ },
+ )
+
+ async def push_text(self, text: str) -> None:
+ """Send a text user turn and ask for the reply."""
+ await self._send(
+ {
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{"type": "input_text", "text": text}],
+ },
+ },
+ )
+ await self.request_response()
+
+ async def push_tool_result(self, block: ToolResultBlock) -> None:
+ """Send a ``function_call_output`` item."""
+ output = (
+ block.output
+ if isinstance(block.output, str)
+ else "".join(
+ _.text for _ in block.output if isinstance(_, TextBlock)
+ )
+ )
+ await self._send(
+ {
+ "type": "conversation.item.create",
+ "item": {
+ "type": "function_call_output",
+ "call_id": block.id,
+ "output": output,
+ },
+ },
+ )
+
+ async def commit_turn(self) -> None:
+ """Commit the input buffer as one user turn."""
+ await self._send({"type": "input_audio_buffer.commit"})
+
+ # ------------------------------------------------------------------
+ # Response control
+ # ------------------------------------------------------------------
+
+ async def request_response(self) -> None:
+ """Ask for a reply, cancelling one already in flight first."""
+ if self._response_id:
+ await self.cancel_response()
+ await self._send({"type": "response.create"})
+
+ async def cancel_response(self) -> None:
+ """Cancel the reply in flight, if any."""
+ if self._response_id:
+ await self._send({"type": "response.cancel"})
+
+ async def truncate(
+ self,
+ item_id: str,
+ played_ms: int,
+ played_text: str,
+ ) -> None:
+ """No-op: the protocol documents no truncate frame."""
+
+ # ------------------------------------------------------------------
+ # Wire
+ # ------------------------------------------------------------------
+
+ def _session_update(
+ self,
+ instructions: str,
+ tools: list[dict] | None,
+ ) -> dict:
+ """Build the ``session.update`` message."""
+ p = self.parameters
+ audio_input: dict[str, Any] = {
+ "format": {"type": "audio/pcm", "rate": self.input_sample_rate},
+ }
+ if p.language_hint:
+ audio_input["transcription"] = {"language_hint": p.language_hint}
+ session: dict[str, Any] = {
+ "instructions": instructions,
+ "voice": p.voice,
+ "reasoning": {"effort": p.reasoning_effort},
+ # Manual turns are ``type: null``, not a null block.
+ "turn_detection": {"type": None}
+ if p.turn_detection == "none"
+ else {
+ "type": "server_vad",
+ "threshold": p.vad_threshold,
+ "silence_duration_ms": p.vad_silence_duration_ms,
+ },
+ "audio": {
+ "input": audio_input,
+ "output": {
+ "format": {
+ "type": "audio/pcm",
+ "rate": self.output_sample_rate,
+ },
+ },
+ },
+ }
+ if tools and self.card.supports_tools:
+ session["tools"] = tools
+ return {"type": "session.update", "session": session}
+
+ async def _send(self, payload: dict) -> None:
+ """Send one JSON frame.
+
+ Raises:
+ `ModelDisconnectedError`: If the provider has closed the
+ session, e.g. after its idle timeout.
+ """
+ from websockets.exceptions import ConnectionClosed
+
+ if self._ws is None:
+ raise ModelDisconnectedError("Not connected.")
+ try:
+ await self._ws.send(json.dumps(payload, ensure_ascii=False))
+ except ConnectionClosed as exc:
+ self._ws = None
+ raise ModelDisconnectedError(str(exc.rcvd or exc)) from exc
+
+ async def _read(self) -> None:
+ """Drain the WebSocket into the event queue until it closes."""
+ try:
+ async for raw in self._ws:
+ if isinstance(raw, bytes):
+ raw = raw.decode("utf-8")
+ try:
+ event = self._parse(json.loads(raw))
+ except Exception: # noqa: BLE001
+ logger.exception("XAIRealtimeModel: bad frame")
+ continue
+ if event is not None:
+ self._queue.put_nowait(event)
+ except Exception as exc: # noqa: BLE001
+ logger.error("XAIRealtimeModel: connection lost: %s", exc)
+ finally:
+ self._queue.put_nowait(me.SessionEndedEvent(reason="closed"))
+ self._queue.put_nowait(None)
+
+ # pylint: disable=too-many-return-statements
+ def _parse(self, data: dict) -> me.ModelEvent | None:
+ """Translate one server frame into a model event."""
+ kind = data.get("type", "")
+ item_id = data.get("item_id", "")
+
+ match kind:
+ case "response.created":
+ self._response_id = data.get("response", {}).get("id", "")
+ return me.ResponseCreatedEvent(item_id=self._response_id)
+
+ case "response.done":
+ response = data.get("response", {})
+ usage = response.get("usage") or {}
+ event = me.ResponseDoneEvent(
+ item_id=response.get("id") or self._response_id,
+ input_tokens=usage.get("input_tokens", 0),
+ output_tokens=usage.get("output_tokens", 0),
+ )
+ self._response_id = ""
+ return event
+
+ case "response.output_audio.delta":
+ delta = data.get("delta")
+ if not delta:
+ return None
+ return me.AudioDeltaEvent(
+ item_id=self._response_id,
+ pcm=base64.b64decode(delta),
+ sample_rate=self.output_sample_rate,
+ )
+
+ case "response.output_audio_transcript.delta":
+ delta = data.get("delta")
+ if not delta:
+ return None
+ return me.TranscriptDeltaEvent(
+ item_id=self._response_id,
+ delta=delta,
+ )
+
+ case "conversation.item.input_audio_transcription.completed":
+ return me.InputTranscriptionEvent(
+ item_id=item_id,
+ text=data.get("transcript", ""),
+ )
+
+ case "input_audio_buffer.speech_started":
+ return me.SpeechStartedEvent(
+ item_id=item_id,
+ at_ms=data.get("audio_start_ms", 0),
+ )
+
+ case "input_audio_buffer.speech_stopped":
+ return me.SpeechEndedEvent(
+ item_id=item_id,
+ at_ms=data.get("audio_end_ms", 0),
+ )
+
+ case "response.function_call_arguments.delta":
+ call_id = data.get("call_id", "")
+ self._tool_args[call_id] = self._tool_args.get(
+ call_id,
+ "",
+ ) + data.get("delta", "")
+ return None
+
+ case "response.function_call_arguments.done":
+ # The done frame is authoritative; accumulated deltas are
+ # only the fallback for a frame that omits ``arguments``.
+ call_id = data.get("call_id", "")
+ accumulated = self._tool_args.pop(call_id, "")
+ return me.ToolCallEvent(
+ item_id=self._response_id,
+ tool_call=ToolCallBlock(
+ id=call_id,
+ name=data.get("name", ""),
+ input=data.get("arguments") or accumulated,
+ ),
+ )
+
+ case "error":
+ err = data.get("error", {})
+ return me.ModelErrorEvent(
+ code=err.get("code", ""),
+ message=err.get("message", ""),
+ )
+
+ case _:
+ logger.debug("XAIRealtimeModel: ignoring %s", kind)
+ return None
diff --git a/src/agentscope/realtime/_xai/_models/grok-voice-latest.yaml b/src/agentscope/realtime/_xai/_models/grok-voice-latest.yaml
new file mode 100644
index 000000000..d376efdd9
--- /dev/null
+++ b/src/agentscope/realtime/_xai/_models/grok-voice-latest.yaml
@@ -0,0 +1,20 @@
+type: realtime_model
+name: grok-voice-latest
+label: Grok Voice (Latest)
+status: active
+
+input_types:
+ - audio/pcm
+ - text/plain
+output_types:
+ - audio/pcm
+ - text/plain
+
+# PCM is accepted at 8-48 kHz; 24 kHz is the documented default on both
+# directions.
+input_sample_rate: 24000
+output_sample_rate: 24000
+
+supports_tools: true
+
+# The API documents no turn, token or session-duration ceiling.
diff --git a/src/agentscope/realtime/_xai/_models/grok-voice-think-fast-2.0.yaml b/src/agentscope/realtime/_xai/_models/grok-voice-think-fast-2.0.yaml
new file mode 100644
index 000000000..2be32523a
--- /dev/null
+++ b/src/agentscope/realtime/_xai/_models/grok-voice-think-fast-2.0.yaml
@@ -0,0 +1,20 @@
+type: realtime_model
+name: grok-voice-think-fast-2.0
+label: Grok Voice Think Fast 2.0
+status: active
+
+input_types:
+ - audio/pcm
+ - text/plain
+output_types:
+ - audio/pcm
+ - text/plain
+
+# PCM is accepted at 8-48 kHz; 24 kHz is the documented default on both
+# directions.
+input_sample_rate: 24000
+output_sample_rate: 24000
+
+supports_tools: true
+
+# The API documents no turn, token or session-duration ceiling.
diff --git a/tests/realtime_xai_test.py b/tests/realtime_xai_test.py
new file mode 100644
index 000000000..f4c6cc2ab
--- /dev/null
+++ b/tests/realtime_xai_test.py
@@ -0,0 +1,416 @@
+# -*- coding: utf-8 -*-
+"""Unit tests for the xAI Grok voice realtime adapter: cards, session
+config and frame parsing. Nothing here opens a connection."""
+# pylint: disable=protected-access
+import base64
+import unittest
+from unittest.async_case import IsolatedAsyncioTestCase
+
+from utils import AnyString
+
+from agentscope.credential import XAICredential
+from agentscope.message import TextBlock, ToolResultBlock
+from agentscope.realtime import (
+ ModelDisconnectedError,
+ TruncationSupport,
+ XAIRealtimeModel,
+)
+from agentscope.realtime import _events as me
+
+CRED = XAICredential(api_key="sk-x")
+TRANSCRIPTION_DONE = "conversation.item.input_audio_transcription.completed"
+
+
+class XAICardsTest(unittest.TestCase):
+ """The adapter lists its own cards, tagged with its type."""
+
+ def test_cards(self) -> None:
+ """Only the two active Grok voice models ship a card."""
+ self.assertListEqual(
+ [
+ (
+ c.name,
+ c.model_type,
+ c.status,
+ c.supports_tools,
+ c.input_sample_rate,
+ c.output_sample_rate,
+ )
+ for c in XAIRealtimeModel.list_models()
+ ],
+ [
+ (
+ "grok-voice-latest",
+ "xai_realtime",
+ "active",
+ True,
+ 24000,
+ 24000,
+ ),
+ (
+ "grok-voice-think-fast-2.0",
+ "xai_realtime",
+ "active",
+ True,
+ 24000,
+ 24000,
+ ),
+ ],
+ )
+
+ def test_credential_maps_card_back_to_class(self) -> None:
+ """The service-layer lookup: card.model_type -> class, no scan."""
+ classes = {c.type: c for c in CRED.get_realtime_model_classes()}
+ self.assertDictEqual(
+ {
+ card.name: classes[card.model_type].__name__
+ for card in CRED.list_realtime_models()
+ },
+ {
+ "grok-voice-latest": "XAIRealtimeModel",
+ "grok-voice-think-fast-2.0": "XAIRealtimeModel",
+ },
+ )
+
+ def test_unknown_model_name_is_rejected(self) -> None:
+ """A name with no card fails at construction."""
+ with self.assertRaises(ValueError):
+ XAIRealtimeModel("no-such-model", CRED)
+
+ def test_adapter_facts(self) -> None:
+ """Grok takes text turns but documents no truncate frame."""
+ model = XAIRealtimeModel("grok-voice-latest", CRED)
+ self.assertListEqual(
+ [model.type, model.truncation, model.supports_text_input],
+ ["xai_realtime", TruncationSupport.NONE, True],
+ )
+
+
+class XAISessionUpdateTest(unittest.TestCase):
+ """The session.update payload sent on connect."""
+
+ def test_server_vad_payload(self) -> None:
+ """Defaults, tools and a language hint, VAD left to the server."""
+ model = XAIRealtimeModel(
+ "grok-voice-latest",
+ CRED,
+ parameters=XAIRealtimeModel.Parameters(language_hint="es-MX"),
+ )
+ self.assertDictEqual(
+ model._session_update("be nice", [{"type": "function"}]),
+ {
+ "type": "session.update",
+ "session": {
+ "instructions": "be nice",
+ "voice": "eve",
+ "reasoning": {"effort": "high"},
+ "turn_detection": {
+ "type": "server_vad",
+ "threshold": 0.85,
+ "silence_duration_ms": 500,
+ },
+ "audio": {
+ "input": {
+ "format": {"type": "audio/pcm", "rate": 24000},
+ "transcription": {"language_hint": "es-MX"},
+ },
+ "output": {
+ "format": {"type": "audio/pcm", "rate": 24000},
+ },
+ },
+ "tools": [{"type": "function"}],
+ },
+ },
+ )
+
+ def test_turn_detection_none_hands_endpointing_to_caller(self) -> None:
+ """``none`` sends a null detection type and no tools without any."""
+ model = XAIRealtimeModel(
+ "grok-voice-think-fast-2.0",
+ CRED,
+ parameters=XAIRealtimeModel.Parameters(
+ turn_detection="none",
+ voice="ara",
+ reasoning_effort="none",
+ ),
+ )
+ self.assertDictEqual(
+ model._session_update("be brief", None),
+ {
+ "type": "session.update",
+ "session": {
+ "instructions": "be brief",
+ "voice": "ara",
+ "reasoning": {"effort": "none"},
+ "turn_detection": {"type": None},
+ "audio": {
+ "input": {
+ "format": {"type": "audio/pcm", "rate": 24000},
+ },
+ "output": {
+ "format": {"type": "audio/pcm", "rate": 24000},
+ },
+ },
+ },
+ },
+ )
+
+
+class XAIParseTest(unittest.TestCase):
+ """Server frames -> model events."""
+
+ def setUp(self) -> None:
+ """Open a response so deltas have an item to attach to."""
+ self.model = XAIRealtimeModel("grok-voice-latest", CRED)
+ self.model._parse(
+ {"type": "response.created", "response": {"id": "resp_001"}},
+ )
+
+ def test_frames(self) -> None:
+ """Every frame the adapter knows maps to one model event."""
+ frames = [
+ {
+ "type": "input_audio_buffer.speech_started",
+ "item_id": "msg_003",
+ "audio_start_ms": 120,
+ },
+ {
+ "type": "input_audio_buffer.speech_stopped",
+ "item_id": "msg_003",
+ "audio_end_ms": 980,
+ },
+ {
+ "type": TRANSCRIPTION_DONE,
+ "item_id": "msg_003",
+ "transcript": "Hello, how are you?",
+ },
+ {
+ "type": "response.output_audio_transcript.delta",
+ "item_id": "msg_008",
+ "delta": "Hello! I'm doing",
+ },
+ {
+ "type": "response.output_audio.delta",
+ "item_id": "msg_008",
+ "delta": base64.b64encode(b"\x01\x00").decode(),
+ },
+ {
+ "type": "response.done",
+ "response": {
+ "id": "resp_001",
+ "usage": {"input_tokens": 10, "output_tokens": 5},
+ },
+ },
+ {
+ "type": "error",
+ "error": {"code": "invalid_audio_format", "message": "boom"},
+ },
+ {"type": "session.updated"},
+ ]
+ self.assertListEqual(
+ [self.model._parse(f) for f in frames],
+ [
+ me.SpeechStartedEvent(item_id="msg_003", at_ms=120),
+ me.SpeechEndedEvent(item_id="msg_003", at_ms=980),
+ me.InputTranscriptionEvent(
+ item_id="msg_003",
+ text="Hello, how are you?",
+ ),
+ me.TranscriptDeltaEvent(
+ item_id="resp_001",
+ delta="Hello! I'm doing",
+ ),
+ me.AudioDeltaEvent(
+ item_id="resp_001",
+ pcm=b"\x01\x00",
+ sample_rate=24000,
+ ),
+ me.ResponseDoneEvent(
+ item_id="resp_001",
+ input_tokens=10,
+ output_tokens=5,
+ ),
+ me.ModelErrorEvent(
+ code="invalid_audio_format",
+ message="boom",
+ ),
+ None,
+ ],
+ )
+
+ def test_response_created_frame(self) -> None:
+ """The response id groups every event of the turn."""
+ self.assertEqual(
+ self.model._parse(
+ {"type": "response.created", "response": {"id": "resp_002"}},
+ ),
+ me.ResponseCreatedEvent(item_id="resp_002"),
+ )
+
+ def test_tool_call_done_frame_is_authoritative(self) -> None:
+ """Accumulated deltas are only a fallback for a done frame that
+ omits ``arguments``."""
+ self.model._parse(
+ {
+ "type": "response.function_call_arguments.delta",
+ "call_id": "call_001",
+ "delta": '{"location":',
+ },
+ )
+ with_args = self.model._parse(
+ {
+ "type": "response.function_call_arguments.done",
+ "item_id": "msg_009",
+ "call_id": "call_001",
+ "name": "get_weather",
+ "arguments": '{"location": "San Francisco"}',
+ },
+ )
+ self.model._parse(
+ {
+ "type": "response.function_call_arguments.delta",
+ "call_id": "call_002",
+ "delta": '{"a": 1}',
+ },
+ )
+ without_args = self.model._parse(
+ {
+ "type": "response.function_call_arguments.done",
+ "call_id": "call_002",
+ "name": "f",
+ },
+ )
+ self.assertListEqual(
+ [with_args.model_dump(), without_args.model_dump()],
+ [
+ {
+ "item_id": "resp_001",
+ "tool_call": {
+ "type": "tool_call",
+ "id": "call_001",
+ "name": "get_weather",
+ "input": '{"location": "San Francisco"}',
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ },
+ {
+ "item_id": "resp_001",
+ "tool_call": {
+ "type": "tool_call",
+ "id": "call_002",
+ "name": "f",
+ "input": '{"a": 1}',
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ },
+ ],
+ )
+
+
+class XAIClientFramesTest(IsolatedAsyncioTestCase):
+ """The frames the adapter puts on the wire."""
+
+ def setUp(self) -> None:
+ """Capture sends instead of opening a socket."""
+ self.model = XAIRealtimeModel("grok-voice-latest", CRED)
+ self.sent: list[dict] = []
+
+ async def capture(payload: dict) -> None:
+ """Record one frame."""
+ self.sent.append(payload)
+
+ self.model._send = capture # type: ignore[method-assign]
+
+ async def test_text_turn(self) -> None:
+ """A text turn is one item plus a response request."""
+ await self.model.push_text("hi there")
+ self.assertListEqual(
+ self.sent,
+ [
+ {
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": "hi there"},
+ ],
+ },
+ },
+ {"type": "response.create"},
+ ],
+ )
+
+ async def test_tool_result_and_commit(self) -> None:
+ """A tool result is a ``function_call_output`` item."""
+ await self.model.push_tool_result(
+ ToolResultBlock(
+ id="call_001",
+ name="get_weather",
+ output=[TextBlock(text="sunny")],
+ ),
+ )
+ await self.model.commit_turn()
+ self.assertListEqual(
+ self.sent,
+ [
+ {
+ "type": "conversation.item.create",
+ "item": {
+ "type": "function_call_output",
+ "call_id": "call_001",
+ "output": "sunny",
+ },
+ },
+ {"type": "input_audio_buffer.commit"},
+ ],
+ )
+
+ async def test_barge_in_cancels_without_truncating(self) -> None:
+ """A barge-in cancels the response; there is no truncate frame."""
+ self.model._parse(
+ {"type": "response.created", "response": {"id": "resp_001"}},
+ )
+ await self.model.cancel_response()
+ await self.model.truncate("resp_001", 1500, "Hello! I'm")
+ self.assertListEqual(self.sent, [{"type": "response.cancel"}])
+
+
+class XAIDisconnectTest(IsolatedAsyncioTestCase):
+ """A closed WebSocket surfaces as ModelDisconnectedError."""
+
+ async def test_send_on_closed_socket(self) -> None:
+ """websockets' ConnectionClosed becomes the realtime-level error
+ and the socket reference is dropped."""
+ from websockets.exceptions import ConnectionClosedError
+ from websockets.frames import Close
+
+ class ClosedSocket:
+ """Raises like a socket the provider already closed."""
+
+ async def send(self, _payload: str) -> None:
+ """Fail with the provider's close frame."""
+ close = Close(1007, "idle 180s")
+ raise ConnectionClosedError(close, close, True)
+
+ model = XAIRealtimeModel("grok-voice-latest", CRED)
+ model._ws = ClosedSocket()
+
+ with self.assertRaises(ModelDisconnectedError) as ctx:
+ await model.push_audio(b"\x00\x00")
+ self.assertEqual(
+ (str(ctx.exception), model._ws),
+ ("1007 (invalid frame payload data) idle 180s", None),
+ )
+
+ async def test_send_before_connect(self) -> None:
+ """No socket at all is the same condition."""
+ model = XAIRealtimeModel("grok-voice-latest", CRED)
+ with self.assertRaises(ModelDisconnectedError):
+ await model.commit_turn()