Skip to content

Build and run agents you can see, understand and trust.

feat(tui): add launch_realtime_ui for voice sessions (#2594)

DavdGao166c3314b3d0
diff --git a/examples/realtime/README.md b/examples/realtime/README.md
index e72932adf..1f8076081 100644
--- a/examples/realtime/README.md
+++ b/examples/realtime/README.md
@@ -13,13 +13,18 @@ Talk to a DashScope speech-to-speech model through your microphone.
The agent owns the model session, you own the transport, and one
`agent.reply_stream(transport)` borrows both until the transport ends.
+The conversation is shown by `agentscope.tui`: `launch_realtime_ui()` renders
+the events of that stream — what you said, what the agent replied, the tools
+it called — and hands what you confirm or type back to `agent.send()`. The
+audio itself never reaches the view; the transport plays it.
+
## Prerequisites
- Python 3.11 or newer
-- AgentScope with the realtime extra:
+- AgentScope with the realtime and TUI extras:
```bash
- pip install "agentscope[realtime]"
+ pip install "agentscope[realtime,tui]"
```
`sounddevice` needs PortAudio: it is bundled on macOS and Windows; on
@@ -33,7 +38,7 @@ export DASHSCOPE_API_KEY=sk-...
python examples/realtime/local_mic.py
```
-Speak, hear the reply, and speak over it to interrupt. `Ctrl-C` quits.
+Speak, hear the reply, and speak over it to interrupt. `Ctrl-Q` quits.
The default model is `qwen-audio-3.0-realtime-plus`. Any card listed for the
credential works; the class is looked up from the card, the same way the
@@ -45,23 +50,13 @@ REALTIME_MODEL=qwen3.5-omni-flash-realtime python examples/realtime/local_mic.py
An unknown name prints the available ones.
-Every reply ends with a diagnostic line: how it finished, the model's time to
-first audio byte (`ttfb`), the user's end-to-end wait (`e2e`), and the
-assistant message as it now stands in the context — after a barge-in that is
-only the part you actually heard.
-
### Tools
-The example registers `Bash`, `Edit`, `Write` and `Read`. When the model
-calls one that needs permission, the terminal asks:
-
-```
- [permission] Bash({"command": "ls"})
- allow? [y/N]
-```
-
-The prompt runs in a thread, so audio keeps flowing while you decide. The
-agent gives up on a prompt after five minutes.
+The example registers `Bash`, `Edit`, `Write` and `Read`. When the model calls
+one that needs permission, the composer is replaced by an approval card
+showing the call and its arguments; pick an answer with the arrow keys and
+`Enter`. Audio keeps flowing while the card is up, and the agent gives up on
+an unanswered one after five minutes.
## Tips
@@ -90,4 +85,5 @@ agent gives up on a prompt after five minutes.
dropped silently on the provider side. The cards under
`agentscope/realtime/_dashscope/` list each model's limits.
- **Typed input** only works on models that accept text
- (`qwen-audio-3.0-realtime-*`); the Omni models are audio-in only.
+ (`qwen-audio-3.0-realtime-*`); the Omni models are audio-in only, and the
+ composer is disabled for them.
diff --git a/examples/realtime/local_mic.py b/examples/realtime/local_mic.py
index a25115110..b02bde636 100644
--- a/examples/realtime/local_mic.py
+++ b/examples/realtime/local_mic.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Talk to a DashScope realtime model through the local microphone, with
-tools the model may call and a terminal prompt for the ones that need
-your permission.
+tools the model may call and a terminal UI that shows the conversation
+and asks for permission where it is needed.
export DASHSCOPE_API_KEY=sk-...
python examples/realtime/local_mic.py
@@ -16,29 +16,20 @@ are wrong, e.g. a Bluetooth headset used for both directions:
REALTIME_INPUT_DEVICE=3 REALTIME_OUTPUT_DEVICE=2 python ...
-Speak, hear the reply, and speak over it to interrupt. Ctrl-C to quit.
+Speak, hear the reply, and speak over it to interrupt. Ctrl-Q to quit.
"""
import asyncio
import os
from agentscope.agent import RealtimeAgent
from agentscope.credential import DashScopeCredential
-from agentscope.event import (
- ConfirmResult,
- ReplyEndEvent,
- ReplyStartEvent,
- RequireUserConfirmEvent,
- TextBlockDeltaEvent,
- ToolResultEndEvent,
- ToolResultStartEvent,
- UserConfirmResultEvent,
-)
from agentscope.realtime import LocalAudioTransport
from agentscope.tool import Bash, Edit, Read, Toolkit, Write
+from agentscope.tui import launch_realtime_ui
async def main() -> None:
- """Run one voice session until interrupted."""
+ """Run one voice session until the UI is closed."""
api_key = os.environ.get("DASHSCOPE_API_KEY")
if not api_key:
raise SystemExit("Set DASHSCOPE_API_KEY first.")
@@ -68,71 +59,11 @@ async def main() -> None:
output_device=_device("REALTIME_OUTPUT_DEVICE"),
)
- print(f"[{name}] listening... (Ctrl-C to quit)")
- # The agent owns the model session, we own the transport, and one
- # reply_stream() borrows both until the transport ends.
- # The user's speech is reported as a reply too, with role "user":
- # its transcript arrives as text block events once it has settled.
- user_turns: set[str] = set()
+ # The agent owns the model session, we own the transport, and the UI
+ # borrows both: it renders the events of one reply_stream() call and
+ # hands what you type or confirm back through agent.send().
async with agent, transport:
- async for event in agent.reply_stream(transport):
- match event:
- case ReplyStartEvent(role="user"):
- user_turns.add(event.reply_id)
- print("\n[you] ...", end="", flush=True)
- case ReplyStartEvent():
- print(f"\n[{agent.name}] ", end="", flush=True)
- case TextBlockDeltaEvent() if event.reply_id in user_turns:
- print(f"\r[you] {event.delta}")
- case TextBlockDeltaEvent():
- print(event.delta, end="", flush=True)
- case RequireUserConfirmEvent():
- await confirm(agent, event)
- case ToolResultStartEvent():
- print(f"\n [tool] {event.tool_call_name} running...")
- case ToolResultEndEvent():
- print(f" [tool] {event.state}")
- case ReplyEndEvent() if event.reply_id not in user_turns:
- m = agent.last_turn_metrics
- print(
- f"\n ({event.finished_reason}"
- f" | ttfb={_ms(m.backend_ttfb)}"
- f" | e2e={_ms(m.e2e_latency)})",
- )
- if agent.state.context:
- tail = agent.state.context[-1]
- print(
- f" context[-1] = {tail.role}: "
- f"{tail.get_text_content()!r}",
- )
-
-
-async def confirm(
- agent: RealtimeAgent,
- event: RequireUserConfirmEvent,
-) -> None:
- """Ask on the terminal whether each pending tool call may run.
-
- The prompt runs in a thread so the audio pumps keep going while we
- wait; the agent itself gives up after five minutes.
- """
- loop = asyncio.get_running_loop()
- results = []
- for call in event.tool_calls:
- print(f"\n [permission] {call.name}({call.input})")
- answer = await loop.run_in_executor(None, input, " allow? [y/N] ")
- results.append(
- ConfirmResult(
- tool_call=call,
- confirmed=answer.strip().lower() in ("y", "yes"),
- ),
- )
- await agent.send(
- UserConfirmResultEvent(
- reply_id=event.reply_id,
- confirm_results=results,
- ),
- )
+ await launch_realtime_ui(agent, transport)
def _device(env: str) -> int | str | None:
@@ -143,11 +74,6 @@ def _device(env: str) -> int | str | None:
return int(value) if value.isdigit() else value
-def _ms(seconds: float | None) -> str:
- """Format a latency for the console."""
- return "n/a" if seconds is None else f"{seconds * 1000:.0f}ms"
-
-
if __name__ == "__main__":
try:
asyncio.run(main())
diff --git a/examples/tui/README.md b/examples/tui/README.md
index d16f67b41..caf3638ab 100644
--- a/examples/tui/README.md
+++ b/examples/tui/README.md
@@ -149,6 +149,22 @@ reply queue. It retains only unfinished replies in a dictionary keyed by
reply ID, including replies waiting for HITL. After displaying `ReplyEndEvent`,
it releases that reply; the UI keeps its own historical display copy.
+A `RealtimeAgent` is driven the other way round: audio flows continuously, so
+the stream belongs to the transport rather than to a submission, and discrete
+input goes back through `send()`. That is a launcher of its own, over the same
+display:
+
+```python
+from agentscope.tui import launch_realtime_ui
+
+async with agent, transport:
+ await launch_realtime_ui(agent, transport)
+```
+
+It shows the transcripts of both sides, the tools and their approval cards,
+and drops the audio blocks, which the transport plays. The composer is
+disabled for a model that accepts no text turn mid-session.
+
## Live CSS editing
Textual can reload external CSS while an app is running. Install its
diff --git a/src/agentscope/tui/__init__.py b/src/agentscope/tui/__init__.py
index db2c6f8d8..b71536166 100644
--- a/src/agentscope/tui/__init__.py
+++ b/src/agentscope/tui/__init__.py
@@ -3,7 +3,7 @@
try:
from ._chat import ChatUI
- from ._launcher import launch_tui
+ from ._launcher import launch_realtime_ui, launch_tui
from ._messages import MessagesUI
except ImportError as error:
if error.name == "textual":
@@ -15,5 +15,6 @@ except ImportError as error:
__all__ = [
"ChatUI",
"MessagesUI",
+ "launch_realtime_ui",
"launch_tui",
]
diff --git a/src/agentscope/tui/_launcher.py b/src/agentscope/tui/_launcher.py
index 9607b178b..6f6286c57 100644
--- a/src/agentscope/tui/_launcher.py
+++ b/src/agentscope/tui/_launcher.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-"""Standalone Textual application for chatting with an Agent or pipeline."""
+"""Standalone Textual applications for chatting with an agent."""
# Textual lifecycle and message handlers inherit their intent from the App.
# pylint: disable=missing-function-docstring
@@ -7,23 +7,29 @@
from __future__ import annotations
import asyncio
-from typing import Sequence, TypeAlias
+from typing import Any, Coroutine, Sequence, TypeAlias
from textual import on
from textual.app import App, ComposeResult
from .._logging import logger
-from ..agent import Agent
+from ..agent import Agent, RealtimeAgent
from ..event import (
+ AgentEvent,
+ DataBlockDeltaEvent,
+ DataBlockEndEvent,
+ DataBlockStartEvent,
ExternalExecutionResultEvent,
ReplyStartEvent,
ReplyEndEvent,
UserConfirmResultEvent,
UserInterruptEvent,
)
-from ..message import AssistantMsg, Msg
+from ..message import AssistantMsg, Msg, UserMsg
from ..pipeline import PipelineProtocol
+from ..realtime import TransportBase
from ._chat import ChatUI
+from ._messages import MessagesUI
_TUIInput: TypeAlias = (
Msg
@@ -32,12 +38,17 @@ _TUIInput: TypeAlias = (
| UserInterruptEvent
)
+_RealtimeInput: TypeAlias = Msg | UserConfirmResultEvent | UserInterruptEvent
-class _AgentScopeTUI(App[None]):
- """The private application used by :func:`launch_tui`."""
+
+class _ChatAppBase(App[None]):
+ """The half both applications share: one :class:`ChatUI`, the replies
+ still open, and the folding of a stream into them. What drives the
+ target — a reply per submission, or one continuous session — is left
+ to the subclass.
+ """
TITLE = "AgentScope"
- SUB_TITLE = "Interactive agent chat"
BINDINGS = []
CSS = """
#agentscope-chat {
@@ -48,14 +59,13 @@ class _AgentScopeTUI(App[None]):
def __init__(
self,
- target: Agent | PipelineProtocol,
messages: Sequence[Msg],
user_name: str,
+ input_enabled: bool = True,
) -> None:
# Render ANSI default colors so transparent widgets inherit the
# user's terminal background rather than Textual's dark theme.
super().__init__(ansi_color=True)
- self.target = target
self._initial_messages = messages
self._replies = {
msg.id: msg.model_copy(deep=True)
@@ -63,29 +73,110 @@ class _AgentScopeTUI(App[None]):
if msg.role == "assistant" and msg.finished_at is None
}
self.user_name = user_name
+ self.input_enabled = input_enabled
self._tasks: set[asyncio.Task[None]] = set()
- self._reply_tasks: dict[str, asyncio.Task[None]] = {}
- self._reply_lock = asyncio.Lock()
def compose(self) -> ComposeResult:
yield ChatUI(
self._initial_messages,
user_name=self.user_name,
+ input_enabled=self.input_enabled,
id="agentscope-chat",
)
+ def _spawn(self, coro: Coroutine[Any, Any, None]) -> None:
+ """Run *coro* until it finishes or the application unmounts."""
+ task = asyncio.create_task(coro)
+ self._tasks.add(task)
+ task.add_done_callback(self._tasks.discard)
+
+ async def _publish(self, item: Msg | AgentEvent) -> None:
+ """Fold one item of a stream into the open replies and render it."""
+ if isinstance(item, Msg):
+ message = item.model_copy(deep=True)
+ if message.role == "assistant" and message.finished_at is None:
+ self._replies[message.id] = message
+ else:
+ reply_id = getattr(item, "reply_id", None)
+ if reply_id is None:
+ return
+ # An ended reply can still receive events, e.g. a spoken turn's
+ # transcript trails its end, so continue from its shown copy.
+ message = self._replies.get(reply_id) or next(
+ (
+ msg.model_copy(deep=True)
+ for msg in reversed(
+ self.query_one(MessagesUI).current_messages(),
+ )
+ if msg.id == reply_id
+ ),
+ None,
+ )
+ if message is None:
+ # A reply resumed after a confirmation gets no
+ # ReplyStartEvent, so open one here as well.
+ start = item if isinstance(item, ReplyStartEvent) else None
+ if start is not None and start.role == "user":
+ message = UserMsg(name=start.name, content=[], id=reply_id)
+ else:
+ message = AssistantMsg(
+ name=start.name if start else "agent",
+ content=[],
+ id=reply_id,
+ )
+ self._replies[reply_id] = message
+ if isinstance(item, ReplyStartEvent):
+ message.name = item.name
+ else:
+ message.append_event(item)
+ await self.query_one(ChatUI).update_message(message)
+ if isinstance(item, ReplyEndEvent) or message.finished_at is not None:
+ self._replies.pop(message.id, None)
+
+ def _submit(self, msg: Msg) -> None:
+ """Hand a message composed by the user to the target."""
+ raise NotImplementedError
+
+ @on(ChatUI.Submitted)
+ def _on_submitted(self, event: ChatUI.Submitted) -> None:
+ if event.msg.get_text_content().strip().casefold() == "/exit":
+ self.exit()
+ return
+ self._submit(event.msg)
+
+ async def on_unmount(self) -> None:
+ tasks = list(self._tasks)
+ for task in tasks:
+ task.cancel()
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+
+class _AgentScopeTUI(_ChatAppBase):
+ """The private application used by :func:`launch_tui`."""
+
+ SUB_TITLE = "Interactive agent chat"
+
+ def __init__(
+ self,
+ target: Agent | PipelineProtocol,
+ messages: Sequence[Msg],
+ user_name: str,
+ ) -> None:
+ super().__init__(messages, user_name)
+ self.target = target
+ self._reply_tasks: dict[str, asyncio.Task[None]] = {}
+ self._reply_lock = asyncio.Lock()
+
def _start_stream(self, inputs: _TUIInput) -> None:
# The standalone application accepts input while a reply is running,
# but queues each reply_stream call so one target context is never
# mutated by concurrent replies.
- task = asyncio.create_task(self._consume(inputs))
- self._tasks.add(task)
- task.add_done_callback(self._tasks.discard)
+ self._spawn(self._consume(inputs))
async def _consume(self, inputs: _TUIInput) -> None:
- chat = self.query_one(ChatUI)
if isinstance(inputs, Msg):
- await chat.update_message(inputs)
+ await self._publish(inputs)
async with self._reply_lock:
task = asyncio.current_task()
owned_reply_ids: set[str] = set()
@@ -94,48 +185,12 @@ class _AgentScopeTUI(App[None]):
owned_reply_ids.add(inputs.reply_id)
try:
if isinstance(inputs, UserConfirmResultEvent):
- message = self._replies.get(inputs.reply_id)
- if message is not None:
- message.append_event(inputs)
- await chat.update_message(message)
+ await self._publish(inputs)
async for item in self.target.reply_stream(inputs):
if isinstance(item, ReplyStartEvent) and task is not None:
self._reply_tasks[item.reply_id] = task
owned_reply_ids.add(item.reply_id)
- if isinstance(item, Msg):
- message = item.model_copy(deep=True)
- if (
- message.role == "assistant"
- and message.finished_at is None
- ):
- self._replies[message.id] = message
- else:
- reply_id = getattr(item, "reply_id", None)
- if reply_id is None:
- continue
- message = self._replies.get(reply_id)
- if message is None:
- name = (
- item.name
- if isinstance(item, ReplyStartEvent)
- else "agent"
- )
- message = AssistantMsg(
- name=name,
- content=[],
- id=reply_id,
- )
- self._replies[reply_id] = message
- if isinstance(item, ReplyStartEvent):
- message.name = item.name
- else:
- message.append_event(item)
- await chat.update_message(message)
- if (
- isinstance(item, ReplyEndEvent)
- or message.finished_at is not None
- ):
- self._replies.pop(message.id, None)
+ await self._publish(item)
# The standalone UI must keep running if its target fails.
# pylint: disable-next=broad-exception-caught
except Exception as error:
@@ -146,12 +201,8 @@ class _AgentScopeTUI(App[None]):
if self._reply_tasks.get(reply_id) is task:
self._reply_tasks.pop(reply_id, None)
- @on(ChatUI.Submitted)
- def _on_submitted(self, event: ChatUI.Submitted) -> None:
- if event.msg.get_text_content().strip().casefold() == "/exit":
- self.exit()
- return
- self._start_stream(event.msg)
+ def _submit(self, msg: Msg) -> None:
+ self._start_stream(msg)
@on(ChatUI.Confirmed)
def _on_confirmed(self, event: ChatUI.Confirmed) -> None:
@@ -174,12 +225,83 @@ class _AgentScopeTUI(App[None]):
if task is not None:
task.cancel()
- async def on_unmount(self) -> None:
- tasks = list(self._tasks)
- for task in tasks:
- task.cancel()
- if tasks:
- await asyncio.gather(*tasks, return_exceptions=True)
+
+class _RealtimeTUI(_ChatAppBase):
+ """The private application used by :func:`launch_realtime_ui`."""
+
+ SUB_TITLE = "Realtime voice chat"
+
+ def __init__(
+ self,
+ agent: RealtimeAgent,
+ transport: TransportBase,
+ messages: Sequence[Msg],
+ user_name: str,
+ ) -> None:
+ # Speaking is the way in; the composer is only good for a model
+ # that also takes a text turn mid-session.
+ super().__init__(
+ messages,
+ user_name,
+ input_enabled=agent.model.supports_text_input,
+ )
+ self.agent = agent
+ self.transport = transport
+
+ def on_mount(self) -> None:
+ # Audio flows for as long as the transport lives, so the session is
+ # one stream, started here, rather than one per submission.
+ self._spawn(self._consume())
+
+ async def _consume(self) -> None:
+ try:
+ async for event in self.agent.reply_stream(self.transport):
+ # The transport plays the reply; the view shows what was
+ # said, so the audio blocks are of no use here.
+ if isinstance(
+ event,
+ (
+ DataBlockStartEvent,
+ DataBlockDeltaEvent,
+ DataBlockEndEvent,
+ ),
+ ):
+ continue
+ await self._publish(event)
+ # The stream ends with the transport, and there is no second
+ # way in: nothing is left to show.
+ self.exit()
+ # The standalone UI must keep running if its agent fails.
+ # pylint: disable-next=broad-exception-caught
+ except Exception as error:
+ logger.exception("Realtime TUI session failed")
+ self.notify(str(error), title="Agent error", severity="error")
+
+ async def _send(self, inputs: _RealtimeInput) -> None:
+ """Hand the agent everything that is not audio."""
+ # An interrupt leaves its mark on the reply it cuts off; the rest
+ # is shown as it is submitted, since the agent does not echo it.
+ if not isinstance(inputs, UserInterruptEvent):
+ await self._publish(inputs)
+ try:
+ await self.agent.send(inputs)
+ # pylint: disable-next=broad-exception-caught
+ except Exception as error:
+ logger.exception("Realtime TUI input failed")
+ self.notify(str(error), title="Agent error", severity="error")
+
+ def _submit(self, msg: Msg) -> None:
+ self._spawn(self._send(msg))
+
+ @on(ChatUI.Confirmed)
+ def _on_confirmed(self, event: ChatUI.Confirmed) -> None:
+ self._spawn(self._send(event.value))
+
+ @on(ChatUI.InterruptRequested)
+ def _on_interrupt(self, event: ChatUI.InterruptRequested) -> None:
+ # Interrupting means barging in on the reply being spoken; there is
+ # no per-reply task to cancel.
+ self._spawn(self._send(UserInterruptEvent(reply_id=event.reply_id)))
async def launch_tui(
@@ -200,3 +322,32 @@ async def launch_tui(
Name assigned to messages submitted from the composer.
"""
await _AgentScopeTUI(target, messages, user_name).run_async()
+
+
+async def launch_realtime_ui(
+ agent: RealtimeAgent,
+ transport: TransportBase,
+ *,
+ messages: Sequence[Msg] = (),
+ user_name: str = "user",
+) -> None:
+ """Launch a full-screen terminal view of a voice session.
+
+ Both the agent and the transport are borrowed: they must already be
+ started, and neither is closed here.
+
+ Args:
+ agent (`RealtimeAgent`):
+ A connected realtime agent. Its ``reply_stream`` runs for the
+ whole session, while a confirmation, an interrupt or a typed
+ turn goes back in through ``send``.
+ transport (`TransportBase`):
+ The started transport carrying the audio. It plays the reply
+ itself, so the view shows the transcripts only.
+ messages (`Sequence[Msg]`, optional):
+ Historical messages displayed before live interaction starts.
+ user_name (`str`, defaults to ``"user"``):
+ Name assigned to messages submitted from the composer, which is
+ disabled for a model that takes no text input.
+ """
+ await _RealtimeTUI(agent, transport, messages, user_name).run_async()
diff --git a/tests/tui_test.py b/tests/tui_test.py
index 0b464aeed..75ac20051 100644
--- a/tests/tui_test.py
+++ b/tests/tui_test.py
@@ -9,6 +9,7 @@
import asyncio
from collections.abc import AsyncGenerator
import json
+from types import SimpleNamespace
from typing import Any
import unittest
from unittest.mock import patch
@@ -18,7 +19,13 @@ from textual.app import App, ComposeResult
from textual.message import Message as TextualMessage
from textual.widgets import Collapsible, Input, OptionList, Static
+from utils import AnyString
+
from agentscope.event import (
+ ConfirmResult,
+ DataBlockDeltaEvent,
+ DataBlockEndEvent,
+ DataBlockStartEvent,
ExternalExecutionResultEvent,
ReplyEndEvent,
ReplyStartEvent,
@@ -52,7 +59,7 @@ from agentscope.tool import AskUser
from agentscope.tui import ChatUI, MessagesUI
from agentscope.tui._ask_user import AskUserUI
from agentscope.tui._chat import ComposerUI, HitlUI, _ComposerTextArea
-from agentscope.tui._launcher import _AgentScopeTUI
+from agentscope.tui._launcher import _AgentScopeTUI, _RealtimeTUI
from agentscope.tui._messages import (
MessageUI,
TextBlockUI,
@@ -1411,5 +1418,356 @@ class LauncherTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(app.BINDINGS, [])
+class _FakeRealtimeAgent:
+ def __init__(
+ self,
+ events: list[Any] | None = None,
+ supports_text_input: bool = True,
+ ) -> None:
+ self.model = SimpleNamespace(supports_text_input=supports_text_input)
+ self.events = events or []
+ self.sent: list[Any] = []
+ self.streamed = asyncio.Event()
+ self.ended = asyncio.Event()
+
+ async def reply_stream(self, _: Any) -> AsyncGenerator[Any, None]:
+ for event in self.events:
+ yield event
+ self.streamed.set()
+ # A voice session outlives its replies: it ends with the transport.
+ await self.ended.wait()
+
+ async def send(self, inputs: Any) -> None:
+ self.sent.append(inputs)
+
+
+class RealtimeLauncherTest(unittest.IsolatedAsyncioTestCase):
+ async def test_voice_turns_render_without_the_audio(self) -> None:
+ agent = _FakeRealtimeAgent(
+ [
+ ReplyStartEvent(
+ session_id="s",
+ reply_id="turn",
+ name="user",
+ role="user",
+ ),
+ # The user's turn ends with the speech; its transcript
+ # settles afterwards.
+ ReplyEndEvent(session_id="s", reply_id="turn"),
+ TextBlockStartEvent(reply_id="turn", block_id="heard"),
+ TextBlockDeltaEvent(
+ reply_id="turn",
+ block_id="heard",
+ delta="hello",
+ ),
+ TextBlockEndEvent(reply_id="turn", block_id="heard"),
+ ReplyStartEvent(
+ session_id="s",
+ reply_id="reply",
+ name="Friday",
+ ),
+ DataBlockStartEvent(
+ reply_id="reply",
+ block_id="audio",
+ media_type="audio/pcm;rate=24000",
+ ),
+ DataBlockDeltaEvent(
+ reply_id="reply",
+ block_id="audio",
+ media_type="audio/pcm;rate=24000",
+ data="AAAA",
+ ),
+ TextBlockStartEvent(reply_id="reply", block_id="spoken"),
+ TextBlockDeltaEvent(
+ reply_id="reply",
+ block_id="spoken",
+ delta="hi there",
+ ),
+ TextBlockEndEvent(reply_id="reply", block_id="spoken"),
+ DataBlockEndEvent(reply_id="reply", block_id="audio"),
+ ReplyEndEvent(session_id="s", reply_id="reply"),
+ ],
+ )
+ app = _RealtimeTUI(agent, object(), [], "user")
+ async with app.run_test(size=(80, 24)) as pilot:
+ await asyncio.wait_for(agent.streamed.wait(), timeout=1)
+ await pilot.pause()
+
+ self.assertListEqual(
+ [msg.model_dump() for msg in app.query_one(ChatUI).messages],
+ [
+ {
+ "name": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "hello",
+ "id": "heard",
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ },
+ ],
+ "role": "user",
+ "id": "turn",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": "completed",
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "Friday",
+ "content": [
+ {
+ "type": "text",
+ "text": "hi there",
+ "id": "spoken",
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ },
+ ],
+ "role": "assistant",
+ "id": "reply",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": "completed",
+ "structured_output": None,
+ "error": None,
+ },
+ ],
+ )
+ self.assertDictEqual(app._replies, {})
+
+ async def test_typed_turn_is_shown_and_sent(self) -> None:
+ agent = _FakeRealtimeAgent()
+ app = _RealtimeTUI(agent, object(), [], "user")
+ async with app.run_test(size=(80, 24)) as pilot:
+ with patch.object(app, "_spawn") as spawn:
+ editor = app.query_one(_ComposerTextArea)
+ editor.focus()
+ await pilot.press("h", "i", "enter")
+ await spawn.call_args.args[0]
+
+ expected = [
+ {
+ "name": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "hi",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": AnyString(),
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ ]
+ self.assertListEqual(
+ [msg.model_dump() for msg in app.query_one(ChatUI).messages],
+ expected,
+ )
+ self.assertListEqual(
+ [inputs.model_dump() for inputs in agent.sent],
+ expected,
+ )
+
+ async def test_confirmation_and_interrupt_reach_the_agent(self) -> None:
+ agent = _FakeRealtimeAgent()
+ app = _RealtimeTUI(
+ agent,
+ object(),
+ [
+ AssistantMsg(
+ name="Friday",
+ id="reply",
+ content=[
+ ToolCallBlock(
+ id="call",
+ name="Bash",
+ input='{"command": "ls"}',
+ state=ToolCallState.ASKING,
+ ),
+ ],
+ ),
+ ],
+ "user",
+ )
+ confirmed = UserConfirmResultEvent(
+ reply_id="reply",
+ confirm_results=[
+ ConfirmResult(
+ tool_call=ToolCallBlock(
+ id="call",
+ name="Bash",
+ input='{"command": "ls"}',
+ ),
+ confirmed=True,
+ ),
+ ],
+ )
+ async with app.run_test(size=(80, 24)) as pilot:
+ with patch.object(app, "_spawn") as spawn:
+ app._on_confirmed(ChatUI.Confirmed(confirmed))
+ app._on_interrupt(ChatUI.InterruptRequested("reply"))
+ for call in spawn.call_args_list:
+ await call.args[0]
+ await pilot.pause()
+
+ # The agent does not echo a confirmation, so the view applies it.
+ self.assertListEqual(
+ [
+ block.model_dump()
+ for block in app.query_one(ChatUI).messages[0].content
+ ],
+ [
+ {
+ "type": "tool_call",
+ "id": "call",
+ "name": "Bash",
+ "input": '{"command": "ls"}',
+ "state": "allowed",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ )
+ self.assertListEqual(
+ [inputs.model_dump() for inputs in agent.sent],
+ [
+ {
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "metadata": {},
+ "type": "USER_CONFIRM_RESULT",
+ "reply_id": "reply",
+ "confirm_results": [
+ {
+ "confirmed": True,
+ "tool_call": {
+ "type": "tool_call",
+ "id": "call",
+ "name": "Bash",
+ "input": '{"command": "ls"}',
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "rules": None,
+ },
+ ],
+ },
+ {
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "metadata": {},
+ "type": "USER_INTERRUPT",
+ "reply_id": "reply",
+ },
+ ],
+ )
+
+ async def test_confirmation_after_the_reply_ended(self) -> None:
+ # A barge-in ends the reply while its tool still waits for an answer.
+ agent = _FakeRealtimeAgent(
+ [
+ ReplyEndEvent(
+ session_id="s",
+ reply_id="reply",
+ finished_reason="interrupted",
+ ),
+ ],
+ )
+ app = _RealtimeTUI(
+ agent,
+ object(),
+ [
+ AssistantMsg(
+ name="Friday",
+ id="reply",
+ content=[
+ ToolCallBlock(
+ id="call",
+ name="Bash",
+ input='{"command": "ls"}',
+ state=ToolCallState.ASKING,
+ ),
+ ],
+ ),
+ ],
+ "user",
+ )
+ async with app.run_test(size=(80, 24)) as pilot:
+ await asyncio.wait_for(agent.streamed.wait(), timeout=1)
+ await app._send(
+ UserConfirmResultEvent(
+ reply_id="reply",
+ confirm_results=[
+ ConfirmResult(
+ tool_call=ToolCallBlock(
+ id="call",
+ name="Bash",
+ input='{"command": "ls"}',
+ ),
+ confirmed=True,
+ ),
+ ],
+ ),
+ )
+ await pilot.pause()
+
+ self.assertListEqual(
+ [msg.model_dump() for msg in app.query_one(ChatUI).messages],
+ [
+ {
+ "name": "Friday",
+ "content": [
+ {
+ "type": "tool_call",
+ "id": "call",
+ "name": "Bash",
+ "input": '{"command": "ls"}',
+ "state": "allowed",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "assistant",
+ "id": "reply",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": "interrupted",
+ "structured_output": None,
+ "error": None,
+ },
+ ],
+ )
+ self.assertDictEqual(app._replies, {})
+
+ async def test_composer_is_disabled_without_text_input(self) -> None:
+ agent = _FakeRealtimeAgent(supports_text_input=False)
+ app = _RealtimeTUI(agent, object(), [], "user")
+ async with app.run_test(size=(80, 24)):
+ self.assertFalse(app.query_one(ChatUI).input_enabled)
+ self.assertTrue(app.query_one(_ComposerTextArea).disabled)
+
+
if __name__ == "__main__":
unittest.main()