Skip to content

Build and run agents you can see, understand and trust.

refactor(realtime): align RealtimeAgent naming with Agent (#2550)

DavdGao1aa0417f2c3b
diff --git a/examples/realtime/README.md b/examples/realtime/README.md
index 33cf864ce..e72932adf 100644
--- a/examples/realtime/README.md
+++ b/examples/realtime/README.md
@@ -11,7 +11,7 @@ Talk to a DashScope speech-to-speech model through your microphone.
tool calls with permission prompts, per-turn latency metrics.
The agent owns the model session, you own the transport, and one
-`agent.run(transport)` borrows both until the transport ends.
+`agent.reply_stream(transport)` borrows both until the transport ends.
## Prerequisites
@@ -82,9 +82,9 @@ agent gives up on a prompt after five minutes.
- **Silence is fine.** DashScope closes a session after about three minutes
without a response. The example logs that it happened and keeps the
- microphone open; the next thing you say reconnects the model. The
- conversation history stays in `agent.state`, but the provider starts fresh —
- it cannot be seeded with prior turns.
+ microphone open; the next thing you say reconnects the model, with the
+ transcript so far appended to the system prompt so the model keeps the
+ thread.
- **Model limits are per turn, not per token.** `qwen3-omni-flash-realtime`
remembers eight turns, `qwen-audio-3.0-realtime-*` fifty; older turns are
dropped silently on the provider side. The cards under
diff --git a/examples/realtime/local_mic.py b/examples/realtime/local_mic.py
index d4d91f7d3..a25115110 100644
--- a/examples/realtime/local_mic.py
+++ b/examples/realtime/local_mic.py
@@ -32,8 +32,6 @@ from agentscope.event import (
ToolResultEndEvent,
ToolResultStartEvent,
UserConfirmResultEvent,
- UserInputAudioStartEvent,
- UserInputTranscriptionEvent,
)
from agentscope.realtime import LocalAudioTransport
from agentscope.tool import Bash, Edit, Read, Toolkit, Write
@@ -59,7 +57,7 @@ async def main() -> None:
agent = RealtimeAgent(
name="Friday",
- sys_prompt="你是一个中文语音助手,回答尽量简短。",
+ system_prompt="你是一个中文语音助手,回答尽量简短。",
model=model,
toolkit=Toolkit(tools=[Bash(), Edit(), Write(), Read()]),
)
@@ -72,16 +70,20 @@ async def main() -> None:
print(f"[{name}] listening... (Ctrl-C to quit)")
# The agent owns the model session, we own the transport, and one
- # run() borrows both until the transport ends.
+ # reply_stream() borrows both until the transport ends.
+ # The user's speech is reported as a reply too, with role "user":
+ # its transcript arrives as text block events once it has settled.
+ user_turns: set[str] = set()
async with agent, transport:
- async for event in agent.run(transport):
+ async for event in agent.reply_stream(transport):
match event:
- case UserInputAudioStartEvent():
+ case ReplyStartEvent(role="user"):
+ user_turns.add(event.reply_id)
print("\n[you] ...", end="", flush=True)
- case UserInputTranscriptionEvent():
- print(f"\r[you] {event.transcript}")
case ReplyStartEvent():
- print(f"[{agent.name}] ", end="", flush=True)
+ print(f"\n[{agent.name}] ", end="", flush=True)
+ case TextBlockDeltaEvent() if event.reply_id in user_turns:
+ print(f"\r[you] {event.delta}")
case TextBlockDeltaEvent():
print(event.delta, end="", flush=True)
case RequireUserConfirmEvent():
@@ -90,7 +92,7 @@ async def main() -> None:
print(f"\n [tool] {event.tool_call_name} running...")
case ToolResultEndEvent():
print(f" [tool] {event.state}")
- case ReplyEndEvent():
+ case ReplyEndEvent() if event.reply_id not in user_turns:
m = agent.last_turn_metrics
print(
f"\n ({event.finished_reason}"
diff --git a/src/agentscope/agent/_realtime/_agent.py b/src/agentscope/agent/_realtime/_agent.py
index ba3cb228b..6d8ddc54d 100644
--- a/src/agentscope/agent/_realtime/_agent.py
+++ b/src/agentscope/agent/_realtime/_agent.py
@@ -26,7 +26,6 @@ from ...event import (
DataBlockDeltaEvent,
DataBlockEndEvent,
DataBlockStartEvent,
- ExternalExecutionResultEvent,
ModelCallEndEvent,
ModelCallStartEvent,
ReplyEndEvent,
@@ -41,8 +40,6 @@ from ...event import (
ToolResultStartEvent,
ToolResultTextDeltaEvent,
UserConfirmResultEvent,
- UserInputAudioStartEvent,
- UserInputTranscriptionEvent,
UserInterruptEvent,
)
from ...message import (
@@ -51,6 +48,7 @@ from ...message import (
ToolCallBlock,
ToolResultBlock,
ToolResultState,
+ Usage,
UserMsg,
)
from ...permission import PermissionBehavior, PermissionEngine
@@ -68,9 +66,16 @@ class _Reply:
to work out what the user actually heard."""
item_id: str
+ """The provider's response item; what playout and truncation refer to."""
+ reply_id: str
+ """The agent's reply this response belongs to, as seen in events and
+ context. A reply spans every response up to the next user turn, so
+ one that calls tools has several responses."""
text_block_id: str = field(default_factory=lambda: uuid.uuid4().hex)
audio_block_id: str = field(default_factory=lambda: uuid.uuid4().hex)
text: str = ""
+ final_text: str | None = None
+ """Set on a barge-in: the text block is truncated to this."""
audio_ms: float = 0.0
text_started: bool = False
audio_started: bool = False
@@ -105,14 +110,14 @@ class RealtimeAgent:
Unlike :class:`~agentscope.agent.Agent` it is bidirectional and has no
request/reply boundary — audio flows in continuously while events flow
- out of :meth:`run`.
+ out of :meth:`reply_stream`.
Three lifetimes are kept apart on purpose. The agent owns the model
session and the state; a transport is owned by whoever created it; a
- :meth:`run` borrows both for as long as both are alive. So a client
- can drop and reconnect without losing the model session, and the model
- session can time out during a long silence and be re-established on
- the next word without touching the transport.
+ :meth:`reply_stream` borrows both for as long as both are alive. So a
+ client can drop and reconnect without losing the model session, and
+ the model session can time out during a long silence and be
+ re-established on the next word without touching the transport.
The public methods are the only entry point for discrete input. A
transport carrying a browser's ``ControlFrame`` calls exactly those
@@ -126,14 +131,14 @@ class RealtimeAgent:
agent = RealtimeAgent("Friday", "Be brief.", model)
async with agent: # model session
async with LocalAudioTransport() as t: # sound card
- async for event in agent.run(t): # this call
+ async for event in agent.reply_stream(t): # this call
print(event)
"""
def __init__(
self,
name: str,
- sys_prompt: str,
+ system_prompt: str,
model: RealtimeModelBase,
toolkit: Toolkit | None = None,
state: AgentState | None = None,
@@ -145,7 +150,7 @@ class RealtimeAgent:
Args:
name (`str`):
Display name stamped on assistant messages and events.
- sys_prompt (`str`):
+ system_prompt (`str`):
System instructions sent to the model on connect, with the
toolkit's skill instructions appended.
model (`RealtimeModelBase`):
@@ -175,7 +180,7 @@ class RealtimeAgent:
omitted.
"""
self.name = name
- self.sys_prompt = sys_prompt
+ self.system_prompt = system_prompt
self.model = model
self.toolkit = toolkit
self.state = state or AgentState()
@@ -187,6 +192,13 @@ class RealtimeAgent:
self._out: asyncio.Queue = asyncio.Queue()
self._reply: _Reply | None = None
self._finished_item = ""
+ # The agent's open reply, and whether the next response continues
+ # it (after tool results) rather than starting a new one.
+ self._reply_id = ""
+ self._continuing = False
+ # The user's turn in flight, reported as a reply of its own.
+ self._user_turn = ""
+ self._user_turn_open = False
self._metrics = TurnMetrics()
self._pending_tools: dict[str, ToolCallBlock] = {}
self._confirmations: dict[str, asyncio.Future[ConfirmResult]] = {}
@@ -221,7 +233,7 @@ class RealtimeAgent:
if self._connected:
return
- instructions = self.sys_prompt
+ instructions = self.system_prompt
tools = None
if self.toolkit is not None:
groups = self.state.tool_context.activated_groups
@@ -244,8 +256,19 @@ class RealtimeAgent:
# context. No provider documents whether the update applies
# retroactively, so treat it as affecting future turns only. Do
# not let it change `voice`: OpenAI locks it after first audio.
+ # Providers differ in whether prior turns can be seeded, so the
+ # transcript so far rides along in the instructions, which every
+ # provider takes. Only matters on reconnect; first connect is empty.
+ history = "\n".join(
+ f"{m.name}: {text}"
+ for m in self.state.context
+ if (text := m.get_text_content())
+ )
+ if history:
+ instructions = (
+ f"{instructions}\n\n## Conversation so far\n{history}"
+ )
await self.model.connect(
- context=self.state.context,
instructions=instructions,
tools=tools,
turn_detection_disabled=self.vad is not None,
@@ -313,7 +336,10 @@ class RealtimeAgent:
# Lifecycle: one transport
# ------------------------------------------------------------------
- async def run(self, transport: TransportBase) -> AsyncIterator[AgentEvent]:
+ async def reply_stream(
+ self,
+ transport: TransportBase,
+ ) -> AsyncIterator[AgentEvent]:
"""Pump *transport* and yield agent events until it ends.
The transport is borrowed, not owned: it must already be started
@@ -327,7 +353,7 @@ class RealtimeAgent:
matters.
"""
if self._transport is not None:
- raise RuntimeError("RealtimeAgent.run is already active.")
+ raise RuntimeError("RealtimeAgent.reply_stream is already active.")
# Audio is forwarded as-is in both directions, so the rates must
# already agree; resampling belongs to the transport, not here.
pairs = {
@@ -387,24 +413,17 @@ class RealtimeAgent:
async def send(
self,
- inputs: (
- str
- | Msg
- | UserConfirmResultEvent
- | UserInterruptEvent
- | ExternalExecutionResultEvent
- ),
+ inputs: (str | Msg | UserConfirmResultEvent | UserInterruptEvent),
) -> None:
"""Feed the agent anything that is not audio.
Mirrors :meth:`Agent.reply`'s ``inputs``: a text turn (which cuts
off any reply in progress first), the outcome of a permission
- prompt, an interrupt, or the result of a tool run elsewhere.
+ prompt, or an interrupt.
Raises:
`NotImplementedError`: For a text turn when the provider takes
- no text input, or for external execution results, which
- are not supported yet.
+ no text input.
"""
match inputs:
case UserInterruptEvent():
@@ -416,11 +435,6 @@ class RealtimeAgent:
if future and not future.done():
future.set_result(result)
- case ExternalExecutionResultEvent():
- raise NotImplementedError(
- "External tool execution is not supported yet.",
- )
-
case str() | Msg():
if not self.model.supports_text_input:
raise NotImplementedError(
@@ -497,8 +511,10 @@ class RealtimeAgent:
pushed = True
if speech is SpeechTransition.STARTED:
+ self._start_user_turn()
await self._barge_in()
elif speech is SpeechTransition.ENDED:
+ self._end_user_turn()
now = time.monotonic()
self._metrics.user_speech_end_at = now
await self.model.commit_turn()
@@ -516,8 +532,6 @@ class RealtimeAgent:
await self.send(UserConfirmResultEvent(**frame.data))
case ControlFrameType.INTERRUPT:
await self.send(UserInterruptEvent())
- case ControlFrameType.EXTERNAL_EXECUTION_RESULT:
- await self.send(ExternalExecutionResultEvent(**frame.data))
case _:
logger.debug("RealtimeAgent: ignoring %s frame", frame.type)
@@ -533,8 +547,8 @@ class RealtimeAgent:
detection, by the VAD's own debounce otherwise.
Reached from the uplink pump, the downlink pump, :meth:`send` and
- :meth:`run`'s exit, so it is serialised; the losers find the reply
- already closed and return.
+ the exit of :meth:`reply_stream`, so it is serialised; the losers
+ find the reply already closed and return.
"""
async with self._barge_lock:
await self._barge_in_locked()
@@ -543,6 +557,8 @@ class RealtimeAgent:
"""Body of :meth:`_barge_in`, run under the lock."""
reply = self._reply
if reply is None:
+ # Nothing playing, but a reply may be waiting on its tools.
+ self._finish_reply(ReplyFinishedReason.INTERRUPTED)
return
spoken, played_ms = (
@@ -563,6 +579,7 @@ class RealtimeAgent:
spoken = reply.spoken_prefix(played_ms)
self._truncate_reply(spoken)
+ reply.final_text = spoken
if self._connected:
await self.model.truncate(reply.item_id, played_ms, spoken)
await self.model.cancel_response()
@@ -619,15 +636,11 @@ class RealtimeAgent:
rate = self.model.output_sample_rate
match event:
case me.SpeechStartedEvent():
- self._emit(
- UserInputAudioStartEvent(
- session_id=self.state.session_id,
- item_id=event.item_id,
- ),
- )
+ self._start_user_turn(event.item_id)
await self._barge_in()
case me.SpeechEndedEvent():
+ self._end_user_turn()
# With provider turn detection this is also its commit.
now = time.monotonic()
self._metrics.user_speech_end_at = now
@@ -664,12 +677,27 @@ class RealtimeAgent:
case me.ToolCallEvent():
if self._start_reply(event.item_id) is not None:
self._pending_tools[event.tool_call.id] = event.tool_call
+ self.state.append_context(self.name, [event.tool_call])
case me.ResponseDoneEvent():
self._metrics.input_tokens = event.input_tokens
self._metrics.output_tokens = event.output_tokens
- self._finish_reply(ReplyFinishedReason.COMPLETED)
- self._schedule_tools()
+ tail = self.state.context[-1] if self.state.context else None
+ if tail is not None and tail.id == self._reply_id:
+ tail.append_usage(
+ Usage(
+ input_tokens=event.input_tokens,
+ output_tokens=event.output_tokens,
+ ),
+ )
+ if self._pending_tools and self.toolkit is not None:
+ # The reply goes on: tools run, then the next response
+ # answers with their results.
+ self._finish_response()
+ self._continuing = True
+ self._schedule_tools()
+ else:
+ self._finish_reply(ReplyFinishedReason.COMPLETED)
case me.ModelErrorEvent():
logger.error(
@@ -680,28 +708,72 @@ class RealtimeAgent:
self._finish_reply(ReplyFinishedReason.ERROR)
case me.SessionEndedEvent():
- pass # the events() iterator ends right after this
+ # The events() iterator ends right after this.
+ logger.info(
+ "RealtimeAgent: model closed the session (%s)",
+ event.reason,
+ )
+
+ def _start_user_turn(self, item_id: str = "") -> None:
+ """Open the user's turn as a reply of its own. A local VAD sees it
+ before the provider creates an item, hence the generated id."""
+ if self._user_turn_open:
+ return # providers repeat speech_started; one turn, one reply
+ self._user_turn = item_id or uuid.uuid4().hex
+ self._user_turn_open = True
+ self._emit(
+ ReplyStartEvent(
+ session_id=self.state.session_id,
+ reply_id=self._user_turn,
+ name="user",
+ role="user",
+ ),
+ )
+
+ def _end_user_turn(self) -> None:
+ """Close the user's turn; the transcript may still be on its way."""
+ if not self._user_turn_open:
+ return
+ self._user_turn_open = False
+ self._emit(
+ ReplyEndEvent(
+ session_id=self.state.session_id,
+ reply_id=self._user_turn,
+ finished_reason=ReplyFinishedReason.COMPLETED,
+ ),
+ )
def _on_transcription(self, event: me.InputTranscriptionEvent) -> None:
- """Record a settled user turn, merging a split one back together."""
+ """Record a settled user turn, merging a split one back together,
+ and report its text on the user's reply."""
turn = self.aggregator.take(event.text)
if turn is None:
logger.debug("RealtimeAgent: dropping %r", event.text)
return
- if self.aggregator.merges_with_previous() and self._merge_user(turn):
- transcript = self.state.context[-1].get_text_content() or turn
- else:
- self.state.context.append(UserMsg(name="user", content=turn))
- transcript = turn
-
+ # A transcript with no detected speech, e.g. after a reconnect,
+ # gets a turn of its own.
+ if not self._user_turn:
+ self._start_user_turn(event.item_id)
+ reply_id, block_id = self._user_turn, uuid.uuid4().hex
+ self._emit(TextBlockStartEvent(reply_id=reply_id, block_id=block_id))
self._emit(
- UserInputTranscriptionEvent(
- session_id=self.state.session_id,
- item_id=event.item_id,
- transcript=transcript,
+ TextBlockDeltaEvent(
+ reply_id=reply_id,
+ block_id=block_id,
+ delta=turn,
),
)
+ self._emit(TextBlockEndEvent(reply_id=reply_id, block_id=block_id))
+ self._end_user_turn()
+ self._user_turn = ""
+
+ if not (
+ self.aggregator.merges_with_previous() and self._merge_user(turn)
+ ):
+ self.state.context.append(
+ UserMsg(name="user", content=turn, id=reply_id),
+ )
def _merge_user(self, text: str) -> bool:
"""Append *text* to the previous user turn that endpointing split.
@@ -712,7 +784,10 @@ class RealtimeAgent:
"""
context = self.state.context
if context and context[-1].role == "assistant":
- if not (context[-1].get_text_content() or "").strip():
+ tail = context[-1]
+ if not (tail.get_text_content() or "").strip() and all(
+ isinstance(b, TextBlock) for b in tail.content
+ ):
context.pop()
if not context or context[-1].role != "user":
return False
@@ -721,7 +796,8 @@ class RealtimeAgent:
return True
def _start_reply(self, item_id: str) -> _Reply | None:
- """Open a reply for *item_id*, emitting its start events once.
+ """Open the response *item_id*, and the reply it belongs to unless
+ one is waiting for it after tool results.
Returns ``None`` for an item already closed — deltas still in
flight after a barge-in must not reopen it.
@@ -730,30 +806,37 @@ class RealtimeAgent:
return self._reply
if item_id == self._finished_item:
return None
-
- self._reply = _Reply(item_id=item_id)
- self.state.reply_id = item_id
- self._metrics = TurnMetrics(
- user_speech_end_at=self._metrics.user_speech_end_at,
- turn_committed_at=self._metrics.turn_committed_at,
- )
- self._emit(
- ReplyStartEvent(
- session_id=self.state.session_id,
- reply_id=item_id,
- name=self.name,
- ),
- )
+ if self._reply is not None:
+ self._finish_response()
+
+ if not self._continuing:
+ # The agent takes the turn; the reply is named by its first
+ # response so that every event of the turn shares one id.
+ self._reply_id = item_id
+ self.state.reply_id = item_id
+ self._metrics = TurnMetrics(
+ user_speech_end_at=self._metrics.user_speech_end_at,
+ turn_committed_at=self._metrics.turn_committed_at,
+ )
+ self._emit(
+ ReplyStartEvent(
+ session_id=self.state.session_id,
+ reply_id=item_id,
+ name=self.name,
+ ),
+ )
+ self._continuing = False
+ self._reply = _Reply(item_id=item_id, reply_id=self._reply_id)
self._emit(
ModelCallStartEvent(
- reply_id=item_id,
+ reply_id=self._reply_id,
model_name=self.model.model_name,
),
)
return self._reply
- def _finish_reply(self, reason: ReplyFinishedReason) -> None:
- """Close the open reply, if any."""
+ def _finish_response(self) -> None:
+ """Close the open response: its blocks and the model call."""
reply = self._reply
if reply is None:
return
@@ -764,33 +847,42 @@ class RealtimeAgent:
if reply.text_started:
self._emit(
TextBlockEndEvent(
- reply_id=reply.item_id,
+ reply_id=reply.reply_id,
block_id=reply.text_block_id,
+ text=reply.final_text,
),
)
if reply.audio_started:
self._emit(
DataBlockEndEvent(
- reply_id=reply.item_id,
+ reply_id=reply.reply_id,
block_id=reply.audio_block_id,
),
)
self._emit(
ModelCallEndEvent(
- reply_id=reply.item_id,
+ reply_id=reply.reply_id,
input_tokens=self._metrics.input_tokens,
output_tokens=self._metrics.output_tokens,
),
)
+ self._finished_item = reply.item_id
+ self._reply = None
+
+ def _finish_reply(self, reason: ReplyFinishedReason) -> None:
+ """Close the open reply, if any, response included."""
+ self._finish_response()
+ if not self._reply_id:
+ return
self._emit(
ReplyEndEvent(
session_id=self.state.session_id,
- reply_id=reply.item_id,
+ reply_id=self._reply_id,
finished_reason=reason,
),
)
- self._finished_item = reply.item_id
- self._reply = None
+ self._reply_id = ""
+ self._continuing = False
def _emit_text(self, reply: _Reply, delta: str) -> None:
"""Emit a transcript delta, opening the block on first use."""
@@ -798,7 +890,7 @@ class RealtimeAgent:
reply.text_started = True
self._emit(
TextBlockStartEvent(
- reply_id=reply.item_id,
+ reply_id=reply.reply_id,
block_id=reply.text_block_id,
),
)
@@ -807,7 +899,7 @@ class RealtimeAgent:
tail = self.state.context[-1] if self.state.context else None
blocks = (
tail.content
- if tail is not None and tail.id == reply.item_id
+ if tail is not None and tail.id == reply.reply_id
else None
)
if (
@@ -820,7 +912,7 @@ class RealtimeAgent:
self.state.append_context(self.name, [TextBlock(text=delta)])
self._emit(
TextBlockDeltaEvent(
- reply_id=reply.item_id,
+ reply_id=reply.reply_id,
block_id=reply.text_block_id,
delta=delta,
),
@@ -833,14 +925,14 @@ class RealtimeAgent:
reply.audio_started = True
self._emit(
DataBlockStartEvent(
- reply_id=reply.item_id,
+ reply_id=reply.reply_id,
block_id=reply.audio_block_id,
media_type=media_type,
),
)
self._emit(
DataBlockDeltaEvent(
- reply_id=reply.item_id,
+ reply_id=reply.reply_id,
block_id=reply.audio_block_id,
data=base64.b64encode(pcm).decode("ascii"),
media_type=media_type,
@@ -848,7 +940,7 @@ class RealtimeAgent:
)
def _emit(self, event: AgentEvent) -> None:
- """Queue one event for :meth:`run`."""
+ """Queue one event for :meth:`reply_stream`."""
self._out.put_nowait(event)
# ------------------------------------------------------------------
@@ -867,13 +959,17 @@ class RealtimeAgent:
async def _run_tools(self, calls: list[ToolCallBlock]) -> None:
"""Execute *calls* in order, then trigger the follow-up response."""
- reply_id = self.state.reply_id
+ reply_id = self._reply_id
try:
for call in calls:
await self._run_tool(reply_id, call)
- await self.model.request_response()
+ # Unless the user cut the reply short while the tools ran.
+ if self._reply_id == reply_id:
+ await self.model.request_response()
except Exception: # noqa: BLE001
logger.exception("RealtimeAgent: tool execution failed")
+ if self._reply_id == reply_id:
+ self._finish_reply(ReplyFinishedReason.ERROR)
async def _run_tool(self, reply_id: str, call: ToolCallBlock) -> None:
"""Check permission for one call, run it, and report the result."""
@@ -1029,6 +1125,18 @@ class RealtimeAgent:
state=state,
),
)
- await self.model.push_tool_result(
- ToolResultBlock(id=call.id, name=call.name, output=output),
+ block = ToolResultBlock(
+ id=call.id,
+ name=call.name,
+ output=output,
+ state=state,
)
+ # The result belongs to the reply that made the call, which may no
+ # longer be the current one if the user interrupted a slow tool.
+ for msg in reversed(self.state.context):
+ if msg.id == reply_id:
+ msg.content.append(block)
+ break
+ else:
+ self.state.append_context(self.name, [block])
+ await self.model.push_tool_result(block)
diff --git a/src/agentscope/event/__init__.py b/src/agentscope/event/__init__.py
index dd9e40ac8..51632a731 100644
--- a/src/agentscope/event/__init__.py
+++ b/src/agentscope/event/__init__.py
@@ -35,8 +35,6 @@ from ._event import (
ExternalExecutionResultEvent,
CustomEvent,
AgentEvent,
- UserInputAudioStartEvent,
- UserInputTranscriptionEvent,
ConfirmResult,
)
@@ -75,7 +73,5 @@ __all__ = [
"ExternalExecutionResultEvent",
"CustomEvent",
"AgentEvent",
- "UserInputAudioStartEvent",
- "UserInputTranscriptionEvent",
"ConfirmResult",
]
diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.py
index 8bce51cbd..40db34c7d 100644
--- a/src/agentscope/event/_event.py
+++ b/src/agentscope/event/_event.py
@@ -64,9 +64,6 @@ class EventType(StrEnum):
USER_INTERRUPT = "USER_INTERRUPT"
EXTERNAL_EXECUTION_RESULT = "EXTERNAL_EXECUTION_RESULT"
- USER_INPUT_AUDIO_START = "USER_INPUT_AUDIO_START"
- USER_INPUT_TRANSCRIPTION = "USER_INPUT_TRANSCRIPTION"
-
CUSTOM = "CUSTOM"
@@ -193,6 +190,9 @@ class TextBlockEndEvent(EventBase):
"""ID of the reply message this block belongs to."""
block_id: str
"""Unique identifier of the text block."""
+ text: str | None = None
+ """The block's final text, when it is not the concatenation of the
+ deltas: a voice reply cut short is truncated to what the user heard."""
class DataBlockStartEvent(EventBase):
@@ -565,34 +565,6 @@ class CustomEvent(EventBase):
"""Arbitrary payload."""
-class UserInputAudioStartEvent(EventBase):
- """The user started speaking, as detected by VAD."""
-
- type: Literal[
- EventType.USER_INPUT_AUDIO_START
- ] = EventType.USER_INPUT_AUDIO_START
- """Event type."""
- session_id: str
- """ID of the session this input belongs to."""
- item_id: str
- """ID of the user input item the provider assigned."""
-
-
-class UserInputTranscriptionEvent(EventBase):
- """The settled transcript of one spoken user turn."""
-
- type: Literal[
- EventType.USER_INPUT_TRANSCRIPTION
- ] = EventType.USER_INPUT_TRANSCRIPTION
- """Event type."""
- session_id: str
- """ID of the session this input belongs to."""
- item_id: str
- """ID of the user input item the provider assigned."""
- transcript: str
- """What the user said."""
-
-
AgentEvent: TypeAlias = (
ReplyStartEvent
| ReplyEndEvent
@@ -621,7 +593,5 @@ AgentEvent: TypeAlias = (
| UserConfirmResultEvent
| UserInterruptEvent
| ExternalExecutionResultEvent
- | UserInputAudioStartEvent
- | UserInputTranscriptionEvent
| CustomEvent
)
diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.py
index 930c304bf..149f34d06 100644
--- a/src/agentscope/message/_base.py
+++ b/src/agentscope/message/_base.py
@@ -307,6 +307,8 @@ class Msg(BaseModel):
elif event.type == EventType.TEXT_BLOCK_DELTA:
block.text += event.delta
else:
+ if event.text is not None:
+ block.text = event.text
block.finished_at = event.created_at
case EventType.DATA_BLOCK_START:
diff --git a/src/agentscope/realtime/DECISIONS.md b/src/agentscope/realtime/DECISIONS.md
deleted file mode 100644
index d8b734a2b..000000000
--- a/src/agentscope/realtime/DECISIONS.md
+++ /dev/null
@@ -1,57 +0,0 @@
-# Realtime 设计决策记录
-
-已定的事项不因单个提问重开;改变需要说明新事实。
-
-## 已定
-
-| 决策 | 依据 |
-|---|---|
-| 三层事实划分:类属性 = adapter 实现;card = 模型是什么;property = 各家归属不同的(采样率、turn detection) | 四家 API 调研:xAI 采样率可调、DashScope 固定 |
-| `TruncationSupport` 三态 NONE / SERVER / EXPLICIT | OpenAI(WS)=EXPLICIT、Gemini=SERVER、DashScope/xAI=NONE |
-| card 四个上下文限制字段并存,全部可选 | DashScope 按轮数+秒数、OpenAI/Gemini 按 token、同时生效 |
-| 公开方法是离散输入的唯一入口,`ControlFrame` 是其线上形态 | livekit / openai SDK / pipecat / vocode 四家一致 |
-| `_barge_in` 串行化,三个调用点共用一把锁 | pipecat CHANGELOG:4712 记录的双队列插队 bug |
-| 复用 `AgentState`,不另建 `VoiceContext` | `append_context` 已覆盖进行中回合的追加语义 |
-| 打断截断留在 `RealtimeAgent._truncate_reply`,不进 `AgentState` | 用户决定,不扩散到 `state/` |
-| `SessionConfig` 二分:`chat`(Agent 驱动)/ `voice`(RealtimeAgent 驱动),内层再按 backend 判别 | 分类依据是驱动的运行时类,不是字段形状 |
-| `TurnAggregator` 在 M1 做上下文合并那一半 | SERVER 模式无提交权,但脏历史会进持久化/重连/压缩 |
-| M1 只做 DashScope,不做 OpenAI | 打断闭环除 provider RPC 外整条链 DashScope 都会跑到 |
-| 不复用 `Agent` 实现级联 | 用户决定 |
-| VAD 归 agent(`RealtimeAgent(vad=)`),不归 transport;无客户端上报的 speech 字段 | livekit 放 session 层、pipecat 主动从 transport 搬出、无一家采信浏览器 speech |
-| 给了 `vad=` 就关 provider 的 turn detection,一个来源判回合;`TurnMode` 枚举删除,由 `vad is None` 推导 | livekit `_resolve_rt_turn_detection_enabled`;pipecat 警告双源发重复帧 |
-| transport 由创建者拥有,`run(transport)` 只借用,不 start/close | livekit `start(room=)` 不关 room;三者平级:agent / transport / run |
-| 下行泵归 agent(`connect()` 起),上行泵归 `run()`;两次 run 之间模型队列照常消费 | 否则空档里工具结果、模型帧无人处理 |
-| 模型会话被 provider 关掉后**不主动重连**,下一帧用户音频触发 `connect()`(可重入,带退避,攒帧) | 静默时挂着 API 连接没必要 |
-| 离散输入统一为 `send(inputs)`,镜像 `Agent.reply(inputs=)`;`interrupt()` 为快捷方式 | `send_text` / `send_confirm` 合并;`ExternalExecutionResultEvent` 入口留着,M1 抛 NotImplementedError |
-| 流式接口叫 `run`,async generator,起动即迭代 | 与 `reply_stream` 结束条件不同(连接断才结束),不复用其名 |
-| `TurnAggregator` 作为实体注入(`aggregator=`),不经 config;pydantic `TurnConfig` 只存在于 app 层 | 它不依赖 agent 内部件,可子类化(pipecat 的 turn strategy 家族) |
-| `fade_ms` 归 transport 构造参数 | 淡出由 transport 执行 |
-| provider 连接异常由 adapter 翻译成 `ModelDisconnectedError`,agent 只认它、不 import websockets;发送失败 = 断开,攒帧等下一帧重连,上行泵不死 | 真机复现:DashScope 180s 空闲关闭后 `push_audio` 撞上已关 socket,比下行泵察觉更早 |
-| `run()` 入口校验 transport / VAD 与 model 的采样率,不一致直接拒绝;agent 不做重采样 | Copilot review;重采样归 transport |
-| 工具结果只取 `ToolResponse.content`,`ToolChunk` 仅用于展示 | `ToolResponse` 是完整结果(与 `Agent` 一致),否则流式工具的文本会重复 |
-| `RealtimeAgent` / `TurnAggregator` / `TurnMetrics` 放 `agent/_realtime/`;`realtime/` 只放模型侧(model、card、事件、transport、VAD) | 与 `Agent` 同目录;`agent/` 单向依赖 `realtime/` |
-| 模型事件统一 `Event` 后缀(`AudioDeltaEvent` 等),基类 `ModelEvent` | 对齐 `agentscope.event` 的命名 |
-| 命名保留 `realtime`,不改 `live`;agent 层将来挂级联后端时再评估 `VoiceAgent` | 六家里五家叫 Realtime,M1 的 provider 产品名就是 Qwen-Omni-Realtime |
-| DashScope 的 Omni 和 Audio-3.0 两个 S2S API 用两个类,不在一个类里按模型名路由 | `supports_text_input` 等是 per-API 事实;`tts/_dashscope/` 一个 provider 三个类的先例 |
-| 反向查找靠 card 上的 `model_type`(产出它的 class 的 `type` tag),服务层建 `{cls.type: cls}` dict,不扫名字 | 同名模型可出现在两个 class 的 card 里(OpenAI chat/responses)|
-| `TurnMetrics` 是 agent 的**产出**(四个时刻),不是配置,不进构造函数;`last_turn_metrics` 读取 | 每回合的指标事件(livekit `metrics_collected`)待前端需要时再加 |
-| `PlayoutPosition.first_played_at` 由 transport 在音频线程记录,agent 在回合结束时读取 | e2e_latency 的唯一真实来源;随 `LocalAudioTransport` 落地 |
-| 用户打字先打断当前回复 | livekit / pipecat 默认;不打断需要额外的待处理状态 |
-
-## 已推迟(保留概念,届时形态如下)
-
-| 事项 | 阶段 | 预期形态 | 为什么现在不做 |
-|---|---|---|---|
-| filler(工具执行期间的填充语) | M2 | 本地 TTS 播放,不走 provider | DashScope 不收文本输入 |
-| 独立的打断判定(叠在 VAD 之上) | M2 | 词数门槛(pipecat)或独立模型(livekit) | SERVER 模式下 provider 已决定;M1 无本地 VAD |
-| `update_session(instructions, tools)` | M2 | OpenAI/DashScope/xAI 发 `session.update`;Gemini 重连+resumption handle;不改 `voice` | 见 `_agent.py` connect() 处 TODO |
-| `ModelEvent.SessionResumption` | Gemini 接入时 | 随 Gemini adapter 一起回来 | Gemini 唯一的改 instructions 路径 |
-| 语义端点检测 | M3 之后 | 先接 pipecat smart-turn ONNX | OpenAI/DashScope 已有 `semantic_vad` |
-| 上下文压缩 | 重连功能时 | `summarize(messages) -> str` 灌进新 session | S2S 下压缩本地历史不影响 provider |
-| WebSocket transport 兜底 | Safari 支持面确认后 | 同 9 字节头 + 二进制 Opus | 优先 WebTransport |
-
-## 待定
-
-- `SessionConfig` 二分改造单独 PR 的时机
-- 模型重连后 UI 提示事件(M1 只记日志)
-- `AudioDelta` 事件的 base64 惰性编码(local 场景全部白做)
diff --git a/src/agentscope/realtime/_base.py b/src/agentscope/realtime/_base.py
index f3d82b2db..dd7030d8f 100644
--- a/src/agentscope/realtime/_base.py
+++ b/src/agentscope/realtime/_base.py
@@ -11,7 +11,7 @@ from pydantic import BaseModel
from ._events import ModelEvent
from ._model_card import RealtimeModelCard
from ..credential import CredentialBase
-from ..message import Msg, ToolResultBlock
+from ..message import ToolResultBlock
class ModelDisconnectedError(ConnectionError):
@@ -138,12 +138,6 @@ class RealtimeModelBase(ABC):
"""The PCM rate of the audio in :class:`AudioDelta`, in Hz."""
return self.card.output_sample_rate
- @property
- @abstractmethod
- def turn_detection_enabled(self) -> bool:
- """Whether the provider decides turn boundaries. When ``False``
- the caller owns endpointing and must call :meth:`commit_turn`."""
-
# ------------------------------------------------------------------
# Lifecycle
# ------------------------------------------------------------------
@@ -151,16 +145,13 @@ class RealtimeModelBase(ABC):
@abstractmethod
async def connect(
self,
- context: list[Msg],
instructions: str,
tools: list[dict] | None = None,
**kwargs: Any,
) -> None:
- """Open the session, seeding it with prior *context*.
+ """Open the session.
Args:
- context (`list[Msg]`):
- Prior conversation, e.g. from ``AgentState.context``.
instructions (`str`):
System instructions.
tools (`list[dict] | None`, optional):
@@ -205,10 +196,8 @@ class RealtimeModelBase(ABC):
@abstractmethod
async def commit_turn(self) -> None:
- """Declare the user turn finished.
-
- Only called when :attr:`turn_detection_enabled` is ``False``.
- """
+ """Declare the user turn finished. Only called when the caller owns
+ endpointing, i.e. the session was opened with turn detection off."""
# ------------------------------------------------------------------
# Response control
diff --git a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yaml b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yaml
index 5582971fd..cabece70f 100644
--- a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yaml
+++ b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-flash.yaml
@@ -14,7 +14,6 @@ input_sample_rate: 16000
output_sample_rate: 24000
supports_tools: true
-supports_session_resumption: false
# Turns and cumulative audio seconds both apply; the provider discards
# the oldest turn once either is exceeded.
diff --git a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yaml b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yaml
index c02378a69..ef6df29fb 100644
--- a/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yaml
+++ b/src/agentscope/realtime/_dashscope/_audio_models/qwen-audio-3.0-realtime-plus.yaml
@@ -14,7 +14,6 @@ input_sample_rate: 16000
output_sample_rate: 24000
supports_tools: true
-supports_session_resumption: false
# Turns and cumulative audio seconds both apply; the provider discards
# the oldest turn once either is exceeded.
diff --git a/src/agentscope/realtime/_dashscope/_model.py b/src/agentscope/realtime/_dashscope/_model.py
index 9afd6461f..f350510c6 100644
--- a/src/agentscope/realtime/_dashscope/_model.py
+++ b/src/agentscope/realtime/_dashscope/_model.py
@@ -16,7 +16,7 @@ from .._base import (
from .._model_card import RealtimeModelCard
from ..._logging import logger
from ...credential import DashScopeCredential
-from ...message import Msg, TextBlock, ToolCallBlock, ToolResultBlock
+from ...message import TextBlock, ToolCallBlock, ToolResultBlock
_REALTIME_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime"
@@ -79,36 +79,19 @@ class DashScopeRealtimeModel(RealtimeModelBase):
self._tool_args: dict[str, str] = {}
self._tool_names: dict[str, str] = {}
- @property
- def turn_detection_enabled(self) -> bool:
- """Whether the provider decides turn boundaries."""
- return self.parameters.turn_detection != "none"
-
# ------------------------------------------------------------------
# Lifecycle
# ------------------------------------------------------------------
async def connect(
self,
- context: list[Msg],
instructions: str,
tools: list[dict] | None = None,
**kwargs: Any,
) -> None:
- """Open the WebSocket and send the session config.
-
- DashScope offers no way to replay *context* into a new session, so
- a non-empty one is logged and dropped rather than silently lost.
- """
+ """Open the WebSocket and send the session config."""
import websockets
- if context:
- logger.warning(
- "DashScopeRealtimeModel: %d prior messages cannot be "
- "seeded into the session; the model starts fresh.",
- len(context),
- )
-
if kwargs.get("turn_detection_disabled"):
self.parameters = self.parameters.model_copy(
update={"turn_detection": "none"},
diff --git a/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yaml
index 3aad9e615..f77c8be5b 100644
--- a/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yaml
+++ b/src/agentscope/realtime/_dashscope/_models/qwen-omni-turbo-realtime.yaml
@@ -15,7 +15,6 @@ output_sample_rate: 24000
# The model page lists "Function Calling: 不支持".
supports_tools: false
-supports_session_resumption: false
# The model page states the context in tokens; the realtime guide's
# turn/duration table does not cover this model.
diff --git a/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yaml
index 8e3296584..0ad882b73 100644
--- a/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yaml
+++ b/src/agentscope/realtime/_dashscope/_models/qwen3-omni-flash-realtime.yaml
@@ -15,7 +15,6 @@ output_sample_rate: 24000
# Function calling is only offered on the qwen3.5 series.
supports_tools: false
-supports_session_resumption: false
# History is bounded by dialog turns, not tokens; the provider discards
# the oldest turn once exceeded.
diff --git a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yaml
index 617b97472..df48a83c5 100644
--- a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yaml
+++ b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-flash-realtime.yaml
@@ -14,7 +14,6 @@ input_sample_rate: 16000
output_sample_rate: 24000
supports_tools: true
-supports_session_resumption: false
# Turns and cumulative audio seconds both apply; the provider discards
# the oldest turn once either is exceeded.
diff --git a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yaml
index 8235b8999..e54af5d75 100644
--- a/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yaml
+++ b/src/agentscope/realtime/_dashscope/_models/qwen3.5-omni-plus-realtime.yaml
@@ -14,7 +14,6 @@ input_sample_rate: 16000
output_sample_rate: 24000
supports_tools: true
-supports_session_resumption: false
# Turns and cumulative audio seconds both apply; the provider discards
# the oldest turn once either is exceeded.
diff --git a/src/agentscope/realtime/_model_card.py b/src/agentscope/realtime/_model_card.py
index 77eeed6fa..575356658 100644
--- a/src/agentscope/realtime/_model_card.py
+++ b/src/agentscope/realtime/_model_card.py
@@ -60,10 +60,6 @@ class RealtimeModelCard(BaseModel):
default=False,
description="Whether this model accepts tool schemas.",
)
- supports_session_resumption: bool = Field(
- default=False,
- description="Whether a previous session can be resumed by handle.",
- )
# Context limits. Providers use different units and several can apply
# at once, so all are optional; ``None`` means unlimited or unknown.
diff --git a/src/agentscope/realtime/_transport/_base.py b/src/agentscope/realtime/_transport/_base.py
index a430daa69..8aba5f698 100644
--- a/src/agentscope/realtime/_transport/_base.py
+++ b/src/agentscope/realtime/_transport/_base.py
@@ -34,9 +34,6 @@ class ControlFrameType(StrEnum):
INTERRUPT = "interrupt"
"""The user pressed stop, as opposed to speaking over the reply."""
- EXTERNAL_EXECUTION_RESULT = "external_execution_result"
- """The result of a tool the client ran itself."""
-
class ControlFrame(BaseModel):
"""One upstream control event."""
@@ -99,10 +96,6 @@ class TransportBase(ABC):
position can be attributed to it.
"""
- @abstractmethod
- async def send_event(self, event: dict) -> None:
- """Send one control event downstream."""
-
@abstractmethod
async def clear_audio(self) -> PlayoutPosition:
"""Cut playback short and report how much was actually heard.
diff --git a/src/agentscope/realtime/_transport/_local.py b/src/agentscope/realtime/_transport/_local.py
index 52339232a..e4f2003cd 100644
--- a/src/agentscope/realtime/_transport/_local.py
+++ b/src/agentscope/realtime/_transport/_local.py
@@ -151,9 +151,6 @@ class LocalAudioTransport(TransportBase):
self._first_played_at = None
self._pending.extend(pcm)
- async def send_event(self, event: dict) -> None:
- """No peer to send to; events are consumed via the agent."""
-
def _on_output(
self,
outdata: Any,
diff --git a/tests/a2a_agent_test.py b/tests/a2a_agent_test.py
index e87c80a9f..2198e6c41 100644
--- a/tests/a2a_agent_test.py
+++ b/tests/a2a_agent_test.py
@@ -209,6 +209,7 @@ class A2AAgentReplyTest(IsolatedAsyncioTestCase):
"type": "TEXT_BLOCK_END",
"reply_id": AnyString(),
"block_id": AnyString(),
+ "text": None,
},
{
"id": AnyString(),
@@ -405,6 +406,7 @@ class A2AAgentReplyTest(IsolatedAsyncioTestCase):
"type": "TEXT_BLOCK_END",
"reply_id": AnyString(),
"block_id": AnyString(),
+ "text": None,
},
{
"id": AnyString(),
@@ -430,6 +432,7 @@ class A2AAgentReplyTest(IsolatedAsyncioTestCase):
"type": "TEXT_BLOCK_END",
"reply_id": AnyString(),
"block_id": AnyString(),
+ "text": None,
},
{
"id": AnyString(),
diff --git a/tests/agent_basic_test.py b/tests/agent_basic_test.py
index 31870f724..b00ed6fe3 100644
--- a/tests/agent_basic_test.py
+++ b/tests/agent_basic_test.py
@@ -293,6 +293,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
{
"type": "TEXT_BLOCK_END",
"block_id": AnyString(),
+ "text": None,
},
{
"type": "MODEL_CALL_END",
@@ -507,6 +508,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
{
"type": "TEXT_BLOCK_END",
"block_id": AnyString(),
+ "text": None,
},
{
"type": "MODEL_CALL_END",
@@ -1232,7 +1234,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"tool_call_id": tool_call_id_1,
"delta": '{"input": ',
},
- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},
+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},
{
"type": "TOOL_CALL_DELTA",
"tool_call_id": tool_call_id_1,
@@ -1295,7 +1297,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"block_id": AnyString(),
"delta": "ended",
},
- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},
+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},
{
"type": "MODEL_CALL_END",
"input_tokens": 0,
@@ -1553,7 +1555,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"block_id": AnyString(),
"delta": "All done",
},
- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},
+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},
{
"type": "MODEL_CALL_END",
"input_tokens": 0,
@@ -1850,7 +1852,7 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"block_id": AnyString(),
"delta": "All done",
},
- {"type": "TEXT_BLOCK_END", "block_id": AnyString()},
+ {"type": "TEXT_BLOCK_END", "block_id": AnyString(), "text": None},
{
"type": "MODEL_CALL_END",
"input_tokens": 0,
diff --git a/tests/hitl_external_execution_test.py b/tests/hitl_external_execution_test.py
index d28c0048b..3fd98d8fe 100644
--- a/tests/hitl_external_execution_test.py
+++ b/tests/hitl_external_execution_test.py
@@ -145,6 +145,7 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):
{
"type": "TEXT_BLOCK_END",
"block_id": AnyString(),
+ "text": None,
},
{
"type": "MODEL_CALL_END",
diff --git a/tests/hitl_mixed_test.py b/tests/hitl_mixed_test.py
index f4a2a8c80..f72164281 100644
--- a/tests/hitl_mixed_test.py
+++ b/tests/hitl_mixed_test.py
@@ -149,6 +149,7 @@ class AgentMixTest(IsolatedAsyncioTestCase):
{
"type": "TEXT_BLOCK_END",
"block_id": AnyString(),
+ "text": None,
},
{
"type": "MODEL_CALL_END",
diff --git a/tests/hitl_user_confirmation_test.py b/tests/hitl_user_confirmation_test.py
index c7d2bce93..060d388c0 100644
--- a/tests/hitl_user_confirmation_test.py
+++ b/tests/hitl_user_confirmation_test.py
@@ -239,6 +239,7 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):
{
"type": "TEXT_BLOCK_END",
"block_id": AnyString(),
+ "text": None,
},
{
"type": "MODEL_CALL_END",
diff --git a/tests/realtime_agent_test.py b/tests/realtime_agent_test.py
index b01b6806f..a92fd5d33 100644
--- a/tests/realtime_agent_test.py
+++ b/tests/realtime_agent_test.py
@@ -11,8 +11,10 @@ from agentscope.agent import RealtimeAgent, TurnAggregator
from agentscope.credential import DashScopeCredential
from agentscope.event import (
ReplyEndEvent,
- UserInputTranscriptionEvent,
+ ReplyStartEvent,
+ TextBlockDeltaEvent,
)
+from agentscope.message import Msg
from agentscope.realtime import (
AudioFrame,
ModelDisconnectedError,
@@ -34,7 +36,7 @@ from agentscope.event import (
ToolResultTextDeltaEvent,
UserConfirmResultEvent,
)
-from agentscope.message import Msg, TextBlock, ToolCallBlock, ToolResultBlock
+from agentscope.message import TextBlock, ToolCallBlock, ToolResultBlock
from agentscope.permission import (
PermissionBehavior,
PermissionContext,
@@ -86,26 +88,23 @@ class ScriptedModel(RealtimeModelBase):
self.scripts = scripts
self.calls: list[str] = []
self.sessions = 0
+ self.instructions = ""
self._open = asyncio.Event()
self._requested = asyncio.Event()
- @property
- def turn_detection_enabled(self) -> bool:
- """The provider owns turn boundaries unless a VAD is given."""
- return True
-
async def connect(
self,
- context: list[Msg],
instructions: str,
tools: list[dict] | None = None,
**kwargs: Any,
) -> None:
- """Record a session open and the turn-detection request."""
+ """Record a session open, its instructions and the turn-detection
+ request."""
self.sessions += 1
self._open.clear()
+ self.instructions = instructions
self.calls.append(
- f"connect(session={self.sessions},ctx={len(context)},"
+ f"connect(session={self.sessions},"
f"td_off={kwargs.get('turn_detection_disabled')})",
)
@@ -193,9 +192,6 @@ class FakeTransport(TransportBase):
"""Remember which item is playing."""
self.item = item_id
- async def send_event(self, event: dict) -> None:
- """No peer to send to."""
-
async def clear_audio(self) -> PlayoutPosition:
"""Count cuts and report the fixed playout position."""
self.cleared += 1
@@ -211,7 +207,8 @@ class FakeTransport(TransportBase):
class EndOnSecondFrameVAD(VADBase):
- """Reports the user stopping on the second chunk."""
+ """Reports the user starting on the first chunk and stopping on the
+ second."""
sample_rate = 16000
@@ -219,8 +216,10 @@ class EndOnSecondFrameVAD(VADBase):
self.seen = 0
def push(self, pcm: bytes) -> SpeechTransition | None:
- """ENDED on the second chunk, otherwise nothing."""
+ """STARTED on the first chunk, ENDED on the second, else nothing."""
self.seen += 1
+ if self.seen == 1:
+ return SpeechTransition.STARTED
return SpeechTransition.ENDED if self.seen == 2 else None
def reset(self) -> None:
@@ -236,14 +235,20 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):
agent: RealtimeAgent,
transport: FakeTransport,
) -> list[tuple[str, Any]]:
- """Run the agent over *transport* and summarise the events."""
+ """Run the agent over *transport* and summarise the events: the
+ user's transcripts and how each of the agent's replies ended."""
summary: list[tuple[str, Any]] = []
+ user_turns: set[str] = set()
async with transport:
- async for event in agent.run(transport):
- if isinstance(event, ReplyEndEvent):
- summary.append(("reply_end", event.finished_reason))
- elif isinstance(event, UserInputTranscriptionEvent):
- summary.append(("user", event.transcript))
+ async for event in agent.reply_stream(transport):
+ if isinstance(event, ReplyStartEvent) and event.role == "user":
+ user_turns.add(event.reply_id)
+ elif isinstance(event, TextBlockDeltaEvent):
+ if event.reply_id in user_turns:
+ summary.append(("user", event.delta))
+ elif isinstance(event, ReplyEndEvent):
+ if event.reply_id not in user_turns:
+ summary.append(("reply_end", event.finished_reason))
return summary
async def test_barge_in_truncates_to_what_was_heard(self) -> None:
@@ -257,20 +262,47 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):
agent = RealtimeAgent("Friday", "be brief", model)
transport = FakeTransport(frames=3)
- async with agent:
- summary = await self._collect(agent, transport)
+ # Rebuild the agent's message from its events the way a client
+ # does; the text deltas ran ahead of what was heard.
+ summary = []
+ rebuilt = Msg(id="r1", role="assistant", name="Friday", content=[])
+ async with agent, transport:
+ async for event in agent.reply_stream(transport):
+ if getattr(event, "reply_id", None) == "r1":
+ rebuilt.append_event(event)
+ if isinstance(event, ReplyStartEvent) and event.role == "user":
+ summary.append(("user_start", event.reply_id))
+ elif isinstance(event, TextBlockDeltaEvent):
+ summary.append(event.delta)
+ elif isinstance(event, ReplyEndEvent):
+ summary.append(("reply_end", event.finished_reason))
+ # The duplicate speech_started opens the user's turn only once.
self.assertListEqual(
summary,
- [("user", "讲个故事"), ("reply_end", "interrupted")],
+ [
+ ("user_start", "u1"),
+ "讲个故事",
+ ("reply_end", "completed"), # the user's turn
+ "从前",
+ "有座山",
+ "山里",
+ "有座庙",
+ "庙里",
+ "有个",
+ "老和尚",
+ ("user_start", "u2"),
+ ("reply_end", "interrupted"),
+ ],
)
+ self.assertEqual(rebuilt.get_text_content(), "从前有座山山里有座庙")
self.assertEqual(transport.cleared, 1)
# Audio frames interleave with model events on the transport's
# clock, so they are counted rather than positioned.
self.assertListEqual(
[c for c in model.calls if c != "push_audio"],
[
- "connect(session=1,ctx=0,td_off=False)",
+ "connect(session=1,td_off=False)",
"truncate(r1,320ms,'从前有座山山里有座庙')",
"cancel",
"close",
@@ -309,8 +341,13 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):
summary = await self._collect(agent, FakeTransport(frames=2))
self.assertEqual(model.sessions, 2)
- # The orphaned reply was dropped, so one message carried over.
- self.assertIn("connect(session=2,ctx=1,td_off=False)", model.calls)
+ # The orphaned reply was dropped, so one message carried over,
+ # riding along in the instructions of the new session.
+ self.assertIn("connect(session=2,td_off=False)", model.calls)
+ self.assertEqual(
+ model.instructions,
+ "be brief\n\n## Conversation so far\nuser: 讲个故事",
+ )
# Events produced while no run was active are delivered first;
# a reply streamed with nobody listening is cut off exactly once.
self.assertListEqual(
@@ -328,8 +365,9 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):
)
async def test_local_vad_owns_turns(self) -> None:
- """Passing a VAD disables provider turn detection and commits the
- turn when the VAD reports the user stopped."""
+ """Passing a VAD disables provider turn detection, reports the
+ user's speech as events and commits the turn when the VAD reports
+ the user stopped."""
model = ScriptedModel([[]])
agent = RealtimeAgent(
"Friday",
@@ -337,13 +375,36 @@ class RealtimeAgentTest(IsolatedAsyncioTestCase):
model,
vad=EndOnSecondFrameVAD(),
)
+ speech = []
async with agent:
- await self._collect(agent, FakeTransport(frames=3))
-
+ transport = FakeTransport(frames=3)
+ async with transport:
+ async for event in agent.reply_stream(transport):
+ if isinstance(event, ReplyStartEvent):
+ speech.append((event.type, event.role, event.reply_id))
+ elif isinstance(event, ReplyEndEvent):
+ speech.append(
+ (
+ event.type,
+ event.finished_reason,
+ event.reply_id,
+ ),
+ )
+
+ # The user's turn is a reply of its own, with a locally generated id
+ # since no provider item exists yet.
+ self.assertListEqual(
+ speech,
+ [
+ ("REPLY_START", "user", AnyString()),
+ ("REPLY_END", "completed", AnyString()),
+ ],
+ )
+ self.assertEqual(speech[0][2], speech[1][2])
self.assertListEqual(
model.calls,
[
- "connect(session=1,ctx=0,td_off=True)",
+ "connect(session=1,td_off=True)",
"push_audio",
"commit_turn",
"push_audio",
@@ -488,7 +549,7 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):
async with agent:
transport = FakeTransport(frames=4)
async with transport:
- async for event in agent.run(transport):
+ async for event in agent.reply_stream(transport):
match event:
case ToolCallStartEvent():
events.append(("call_start", event.tool_call_name))
@@ -538,7 +599,7 @@ class RealtimeAgentToolTest(IsolatedAsyncioTestCase):
self.assertListEqual(
[c for c in model.calls if not c.startswith("push_audio")],
[
- "connect(session=1,ctx=0,td_off=False)",
+ "connect(session=1,td_off=False)",
"tool_result(c1,'x-final')",
"request_response",
"close",
@@ -659,7 +720,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
async with agent:
transport = FakeTransport(frames=6)
async with transport:
- async for event in agent.run(transport):
+ async for event in agent.reply_stream(transport):
events.append(event.model_dump(mode="json"))
self.assertListEqual(
@@ -669,10 +730,47 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"id": AnyString(),
"created_at": AnyString(),
"metadata": {},
- "type": "USER_INPUT_TRANSCRIPTION",
+ "type": "REPLY_START",
"session_id": AnyString(),
- "item_id": "u1",
- "transcript": "查天气",
+ "reply_id": "u1",
+ "name": "user",
+ "role": "user",
+ },
+ {
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "metadata": {},
+ "type": "TEXT_BLOCK_START",
+ "reply_id": "u1",
+ "block_id": AnyString(),
+ },
+ {
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "metadata": {},
+ "type": "TEXT_BLOCK_DELTA",
+ "reply_id": "u1",
+ "block_id": AnyString(),
+ "delta": "查天气",
+ },
+ {
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "metadata": {},
+ "type": "TEXT_BLOCK_END",
+ "reply_id": "u1",
+ "block_id": AnyString(),
+ "text": None,
+ },
+ {
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "metadata": {},
+ "type": "REPLY_END",
+ "session_id": AnyString(),
+ "reply_id": "u1",
+ "finished_reason": "completed",
+ "error": None,
},
{
"id": AnyString(),
@@ -737,6 +835,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"type": "TEXT_BLOCK_END",
"reply_id": "r1",
"block_id": AnyString(),
+ "text": None,
},
{
"id": AnyString(),
@@ -758,16 +857,6 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
- {
- "id": AnyString(),
- "created_at": AnyString(),
- "metadata": {},
- "type": "REPLY_END",
- "session_id": AnyString(),
- "reply_id": "r1",
- "finished_reason": "completed",
- "error": None,
- },
{
"id": AnyString(),
"created_at": AnyString(),
@@ -821,22 +910,12 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"tool_call_id": "c1",
"state": "success",
},
- {
- "id": AnyString(),
- "created_at": AnyString(),
- "metadata": {},
- "type": "REPLY_START",
- "session_id": AnyString(),
- "reply_id": "r2",
- "name": "Friday",
- "role": "assistant",
- },
{
"id": AnyString(),
"created_at": AnyString(),
"metadata": {},
"type": "MODEL_CALL_START",
- "reply_id": "r2",
+ "reply_id": "r1",
"model_name": "scripted",
},
{
@@ -844,7 +923,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"created_at": AnyString(),
"metadata": {},
"type": "TEXT_BLOCK_START",
- "reply_id": "r2",
+ "reply_id": "r1",
"block_id": AnyString(),
},
{
@@ -852,7 +931,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"created_at": AnyString(),
"metadata": {},
"type": "TEXT_BLOCK_DELTA",
- "reply_id": "r2",
+ "reply_id": "r1",
"block_id": AnyString(),
"delta": "今天晴",
},
@@ -861,7 +940,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"created_at": AnyString(),
"metadata": {},
"type": "DATA_BLOCK_START",
- "reply_id": "r2",
+ "reply_id": "r1",
"block_id": AnyString(),
"media_type": "audio/pcm;rate=24000",
"name": None,
@@ -871,7 +950,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"created_at": AnyString(),
"metadata": {},
"type": "DATA_BLOCK_DELTA",
- "reply_id": "r2",
+ "reply_id": "r1",
"block_id": AnyString(),
"media_type": "audio/pcm;rate=24000",
"data": "AQA=",
@@ -882,15 +961,16 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"created_at": AnyString(),
"metadata": {},
"type": "TEXT_BLOCK_END",
- "reply_id": "r2",
+ "reply_id": "r1",
"block_id": AnyString(),
+ "text": None,
},
{
"id": AnyString(),
"created_at": AnyString(),
"metadata": {},
"type": "DATA_BLOCK_END",
- "reply_id": "r2",
+ "reply_id": "r1",
"block_id": AnyString(),
},
{
@@ -898,7 +978,7 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"created_at": AnyString(),
"metadata": {},
"type": "MODEL_CALL_END",
- "reply_id": "r2",
+ "reply_id": "r1",
"input_tokens": 9,
"output_tokens": 3,
"cache_input_tokens": 0,
@@ -911,24 +991,98 @@ class RealtimeAgentFullStreamTest(IsolatedAsyncioTestCase):
"metadata": {},
"type": "REPLY_END",
"session_id": AnyString(),
- "reply_id": "r2",
+ "reply_id": "r1",
"finished_reason": "completed",
"error": None,
},
],
)
+ # The context records the whole turn as one assistant message: the
+ # first words, the tool call and its result, then the spoken answer.
self.assertListEqual(
- [(m.role, m.get_text_content()) for m in agent.state.context],
+ [m.model_dump() for m in agent.state.context],
[
- ("user", "查天气"),
- ("assistant", "我查一下"),
- ("assistant", "今天晴"),
+ {
+ "name": "user",
+ "role": "user",
+ "id": "u1",
+ "content": [
+ {
+ "type": "text",
+ "text": "查天气",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "Friday",
+ "role": "assistant",
+ "id": "r1",
+ "content": [
+ {
+ "type": "text",
+ "text": "我查一下",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "tool_call",
+ "id": "c1",
+ "name": "stream_tool",
+ "input": '{"q": "x"}',
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "tool_result",
+ "id": "c1",
+ "name": "stream_tool",
+ "output": "x-final",
+ "state": "success",
+ "metadata": {},
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": "今天晴",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "metadata": {},
+ "created_at": AnyString(),
+ # Both model calls of the reply, summed.
+ "usage": {
+ "input_tokens": 14,
+ "output_tokens": 5,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ "finished_at": None,
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
],
)
self.assertListEqual(
[c for c in model.calls if c != "push_audio"],
[
- "connect(session=1,ctx=0,td_off=False)",
+ "connect(session=1,td_off=False)",
"tool_result(c1,'x-final')",
"request_response",
"close",
@@ -966,15 +1120,15 @@ class RealtimeAgentDisconnectTest(IsolatedAsyncioTestCase):
transport = FakeTransport(frames=3)
with self.assertLogs("as", level="INFO") as logs:
async with transport:
- async for _ in agent.run(transport):
+ async for _ in agent.reply_stream(transport):
pass
self.assertEqual(model.sessions, 2)
self.assertListEqual(
[c for c in model.calls if c != "push_audio"],
[
- "connect(session=1,ctx=0,td_off=False)",
- "connect(session=2,ctx=0,td_off=False)",
+ "connect(session=1,td_off=False)",
+ "connect(session=2,td_off=False)",
"close",
],
)
diff --git a/tests/realtime_dashscope_test.py b/tests/realtime_dashscope_test.py
index 33c1eb012..4f7bf6d03 100644
--- a/tests/realtime_dashscope_test.py
+++ b/tests/realtime_dashscope_test.py
@@ -186,7 +186,6 @@ class DashScopeSessionUpdateTest(unittest.TestCase):
)
session = model._session_update("x", None)["session"]
self.assertIsNone(session["turn_detection"])
- self.assertFalse(model.turn_detection_enabled)
def test_adapter_facts(self) -> None:
"""Protocol facts differ per adapter, not per model."""