Skip to content

Build and run agents you can see, understand and trust.

fix(model): emit the dashscope audio transcript as text (#2925)

ANIRUDDHA ADAK80591b6eb184
diff --git a/src/agentscope/model/_dashscope/_model.py b/src/agentscope/model/_dashscope/_model.py
index 6d1e512e8..b1a828ca9 100644
--- a/src/agentscope/model/_dashscope/_model.py
+++ b/src/agentscope/model/_dashscope/_model.py
@@ -386,8 +386,25 @@ class DashScopeChatModel(ChatModelBase):
delta_audio = getattr(delta, "audio", None)
if isinstance(delta_audio, dict):
audio_chunk = delta_audio.get("data", "")
+ transcript_chunk = (
+ delta_audio.get("transcript", "") or ""
+ )
else:
audio_chunk = getattr(delta_audio, "data", "")
+ transcript_chunk = (
+ getattr(delta_audio, "transcript", "") or ""
+ )
+
+ # Omni models deliver the spoken text in
+ # ``delta.audio.transcript`` rather than in
+ # ``delta.content``. Append it to the same text block id
+ # so the agent receives the text as well as the audio,
+ # mirroring the OpenAI Chat model.
+ if transcript_chunk:
+ delta_res.append_text(
+ block_id=text_id,
+ text=transcript_chunk,
+ )
if audio_chunk:
pcm_bytes = base64.b64decode(audio_chunk)
diff --git a/tests/model_dashscope_test.py b/tests/model_dashscope_test.py
index 13803b5e8..89c31de24 100644
--- a/tests/model_dashscope_test.py
+++ b/tests/model_dashscope_test.py
@@ -581,6 +581,56 @@ class TestDashScopeStream(IsolatedAsyncioTestCase):
frames = wav.readframes(wav.getnframes())
self.assertEqual(frames, pcm_full)
+ async def test_stream_audio_transcript_yields_text(
+ self,
+ ) -> None:
+ """An omni audio transcript is emitted as text, not dropped.
+
+ Omni models send the spoken text in ``delta.audio.transcript``
+ instead of ``delta.content``, so the parser has to pick it up for
+ the agent to receive what the model said.
+ """
+ pcm = bytes([1, 2, 3, 4])
+ chunks = [
+ _make_stream_chunk(
+ delta_audio={
+ "data": base64.b64encode(pcm).decode(),
+ "transcript": "Hello",
+ },
+ ),
+ _make_stream_chunk(
+ delta_audio={
+ "data": base64.b64encode(pcm).decode(),
+ "transcript": "Hello",
+ },
+ ),
+ _make_stream_chunk(
+ has_choices=False,
+ usage={"prompt_tokens": 5, "completion_tokens": 3},
+ ),
+ ]
+ mock_create = AsyncMock(return_value=_MockAsyncStream(chunks))
+ self.mock_client.chat.completions.create = mock_create
+
+ gen = await self.model([])
+ responses = [r async for r in gen]
+
+ # The transcript is folded into the same text block, so the final
+ # response carries it concatenated.
+ final = responses[-1]
+ self.assertTrue(final.is_last)
+ texts = [b.text for b in final.content if isinstance(b, TextBlock)]
+ self.assertEqual("".join(texts), "HelloHello")
+
+ # The audio stream itself is untouched: one stable id across chunks.
+ audio_ids = {
+ b.id
+ for r in responses
+ for b in r.content
+ if isinstance(b, DataBlock)
+ }
+ self.assertEqual(len(audio_ids), 1)
+
# ---------------------------------------------------------------------------
# _format_tools tests