Skip to content

Build and run agents you can see, understand and trust.

feat(realtime): add Qwen3.8 Omni Realtime support (#3196)

qbce4c1e074dee3
diff --git a/examples/realtime/README.md b/examples/realtime/README.md
index 9aacf3a99..eac9be9f8 100644
--- a/examples/realtime/README.md
+++ b/examples/realtime/README.md
@@ -50,6 +50,28 @@ REALTIME_MODEL=qwen3.5-omni-flash-realtime python examples/realtime/local_mic.py
An unknown name prints the available ones.
+`qwen3.8-omni-flash-realtime` requires the endpoint bound to the same
+DashScope workspace as the API key. Set `DASHSCOPE_BASE_URL` to the HTTPS
+Chat Completions base URL (not the WebSocket URL):
+
+```bash
+export DASHSCOPE_BASE_URL=https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/compatible-mode/v1
+REALTIME_MODEL=qwen3.8-omni-flash-realtime python examples/realtime/local_mic.py
+```
+
+The credential uses that URL for Chat Completions. For realtime sessions,
+the adapter keeps the same workspace host and derives this WebSocket URL:
+
+```text
+wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime?model=qwen3.8-omni-flash-realtime
+```
+
+For the Singapore region, use the corresponding workspace URL:
+
+```bash
+export DASHSCOPE_BASE_URL=https://{WorkspaceId}.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1
+```
+
### Tools
The example registers `Bash`, `Edit`, `Write` and `Read`. When the model calls
@@ -81,8 +103,9 @@ an unanswered one after five minutes.
transcript so far appended to the system prompt so the model keeps the
thread.
- **Model limits are per turn, not per token.** `qwen3-omni-flash-realtime`
- remembers eight turns, `qwen-audio-3.0-realtime-*` fifty; older turns are
- dropped silently on the provider side. The cards under
+ remembers eight turns, `qwen3.8-omni-flash-realtime` one hundred turns or
+ six hundred seconds of audio, and `qwen-audio-3.0-realtime-*` fifty turns;
+ older turns are dropped silently on the provider side. The cards under
`agentscope/realtime/_dashscope/` list each model's limits.
- **Typed input** only works on models that accept text
(`qwen-audio-3.0-realtime-*`); the Omni models are audio-in only, and the
diff --git a/examples/realtime/local_mic.py b/examples/realtime/local_mic.py
index b02bde636..952189576 100644
--- a/examples/realtime/local_mic.py
+++ b/examples/realtime/local_mic.py
@@ -33,7 +33,13 @@ async def main() -> None:
api_key = os.environ.get("DASHSCOPE_API_KEY")
if not api_key:
raise SystemExit("Set DASHSCOPE_API_KEY first.")
- credential = DashScopeCredential(api_key=api_key)
+ credential = DashScopeCredential(
+ api_key=api_key,
+ base_url=os.environ.get(
+ "DASHSCOPE_BASE_URL",
+ "https://dashscope.aliyuncs.com/compatible-mode/v1",
+ ),
+ )
# Resolve the model class from its card rather than hard-coding it:
# a credential can serve several realtime APIs, and each card carries
@@ -48,7 +54,7 @@ async def main() -> None:
agent = RealtimeAgent(
name="Friday",
- system_prompt="你是一个中文语音助手,回答尽量简短。",
+ system_prompt="You are a concise voice assistant.",
model=model,
toolkit=Toolkit(tools=[Bash(), Edit(), Write(), Read()]),
)
diff --git a/src/agentscope/realtime/_dashscope/_audio_model.py b/src/agentscope/realtime/_dashscope/_audio_model.py
index 903be67c6..68c0445c9 100644
--- a/src/agentscope/realtime/_dashscope/_audio_model.py
+++ b/src/agentscope/realtime/_dashscope/_audio_model.py
@@ -180,6 +180,17 @@ class DashScopeAudioRealtimeModel(DashScopeRealtimeModel):
},
)
+ async def request_response(self) -> None:
+ """Ask for a reply with the Qwen-Audio modality override."""
+ if self._item_id:
+ await self.cancel_response()
+ await self._send(
+ {
+ "type": "response.create",
+ "response": {"modalities": ["text", "audio"]},
+ },
+ )
+
def _session_update(
self,
instructions: str,
diff --git a/src/agentscope/realtime/_dashscope/_model.py b/src/agentscope/realtime/_dashscope/_model.py
index e57836c5b..54f757cb3 100644
--- a/src/agentscope/realtime/_dashscope/_model.py
+++ b/src/agentscope/realtime/_dashscope/_model.py
@@ -228,12 +228,7 @@ class DashScopeRealtimeModel(RealtimeModelBase):
"""Ask for a reply, cancelling one already in flight first."""
if self._item_id:
await self.cancel_response()
- await self._send(
- {
- "type": "response.create",
- "response": {"modalities": ["text", "audio"]},
- },
- )
+ await self._send({"type": "response.create"})
async def cancel_response(self) -> None:
"""Cancel the reply in flight, if any."""
@@ -251,23 +246,26 @@ class DashScopeRealtimeModel(RealtimeModelBase):
) -> dict:
"""Build the ``session.update`` message."""
p = self.parameters
- session: dict[str, Any] = {
- "instructions": instructions,
- "modalities": ["audio", "text"],
- "voice": p.voice,
- "input_audio_format": f"pcm{self.input_sample_rate // 1000}",
- "output_audio_format": f"pcm{self.output_sample_rate // 1000}",
- "turn_detection": None
+ turn_detection = (
+ None
if p.turn_detection == "none"
else {
"type": p.turn_detection,
"threshold": p.vad_threshold,
"silence_duration_ms": p.vad_silence_duration_ms,
- },
+ }
+ )
+ session: dict[str, Any] = {
+ "instructions": instructions,
+ "modalities": ["audio", "text"],
+ "voice": p.voice,
+ "input_audio_format": "pcm",
+ "output_audio_format": "pcm",
+ "turn_detection": turn_detection,
}
if p.input_audio_transcription:
session["input_audio_transcription"] = {
- "model": "gummy-realtime-v1",
+ "model": "qwen3-asr-flash-realtime",
}
if tools and self.card.supports_tools:
session["tools"] = tools
diff --git a/src/agentscope/realtime/_dashscope/_models/qwen3.8-omni-flash-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3.8-omni-flash-realtime.yaml
new file mode 100644
index 000000000..ef2b2a63b
--- /dev/null
+++ b/src/agentscope/realtime/_dashscope/_models/qwen3.8-omni-flash-realtime.yaml
@@ -0,0 +1,26 @@
+type: realtime_model
+name: qwen3.8-omni-flash-realtime
+label: Qwen3.8-Omni-Flash Realtime
+status: active
+
+input_types:
+ - audio/pcm
+ - image/jpeg
+output_types:
+ - audio/pcm
+ - text/plain
+
+input_sample_rate: 16000
+output_sample_rate: 24000
+
+supports_tools: true
+
+max_context_tokens: 196608
+max_audio_turns: 100
+max_audio_duration_s: 600
+max_session_duration_s: 7200
+
+parameter_overrides:
+ voice:
+ default: "Tina"
+ enum: ["Tina", "Serena", "Ryan", "Katerina", "longanlingxin"]
diff --git a/tests/realtime_dashscope_test.py b/tests/realtime_dashscope_test.py
index 3d46d548a..a56cf1a1b 100644
--- a/tests/realtime_dashscope_test.py
+++ b/tests/realtime_dashscope_test.py
@@ -27,6 +27,12 @@ from agentscope.message import (
)
CRED = DashScopeCredential(api_key="sk-x")
+WORKSPACE_HOST = "https://llm-workspace.cn-beijing.maas.aliyuncs.com"
+WORKSPACE_BASE_URL = f"{WORKSPACE_HOST}/compatible-mode/v1"
+WORKSPACE_CRED = DashScopeCredential(
+ api_key="sk-workspace",
+ base_url=WORKSPACE_BASE_URL,
+)
TRANSCRIPTION_DONE = "conversation.item.input_audio_transcription.completed"
AMBIENT_DELTA = "conversation.item.ambient_audio_transcription.delta"
@@ -63,10 +69,15 @@ class DashScopeCardsTest(unittest.TestCase):
"""Each adapter lists only its own cards, tagged with its type."""
def test_omni_cards(self) -> None:
- """The Omni adapter lists the four Omni cards."""
+ """The Omni adapter lists the five Omni cards."""
self.assertListEqual(
[
- (c.name, c.model_type, c.supports_tools, c.max_audio_turns)
+ (
+ c.name,
+ c.model_type,
+ c.supports_tools,
+ c.max_audio_turns,
+ )
for c in DashScopeRealtimeModel.list_models()
],
[
@@ -94,9 +105,52 @@ class DashScopeCardsTest(unittest.TestCase):
True,
100,
),
+ (
+ "qwen3.8-omni-flash-realtime",
+ "dashscope_omni_realtime",
+ True,
+ 100,
+ ),
],
)
+ def test_qwen38_card_limits(self) -> None:
+ """The Qwen3.8 card exposes the documented audio limits."""
+ card = next(
+ card
+ for card in DashScopeRealtimeModel.list_models()
+ if card.name == "qwen3.8-omni-flash-realtime"
+ )
+
+ self.assertDictEqual(
+ card.model_dump(
+ include={
+ "name",
+ "input_types",
+ "output_types",
+ "input_sample_rate",
+ "output_sample_rate",
+ "supports_tools",
+ "max_context_tokens",
+ "max_audio_turns",
+ "max_audio_duration_s",
+ "max_session_duration_s",
+ },
+ ),
+ {
+ "name": "qwen3.8-omni-flash-realtime",
+ "input_types": ["audio/pcm", "image/jpeg"],
+ "output_types": ["audio/pcm", "text/plain"],
+ "input_sample_rate": 16000,
+ "output_sample_rate": 24000,
+ "supports_tools": True,
+ "max_context_tokens": 196608,
+ "max_audio_turns": 100,
+ "max_audio_duration_s": 600,
+ "max_session_duration_s": 7200,
+ },
+ )
+
def test_audio_cards(self) -> None:
"""The Audio adapter lists the three Audio cards."""
self.assertListEqual(
@@ -144,7 +198,7 @@ class DashScopeCardsTest(unittest.TestCase):
],
"runtime_default": DashScopeRealtimeModel(
name,
- CRED,
+ WORKSPACE_CRED,
).parameters.voice,
}
for name in cards
@@ -186,6 +240,21 @@ class DashScopeCardsTest(unittest.TestCase):
},
"runtime_default": "Tina",
},
+ "qwen3.8-omni-flash-realtime": {
+ "schema": {
+ "default": "Tina",
+ "enum": [
+ "Tina",
+ "Serena",
+ "Ryan",
+ "Katerina",
+ "longanlingxin",
+ ],
+ "title": "Voice",
+ "type": "string",
+ },
+ "runtime_default": "Tina",
+ },
},
)
@@ -298,6 +367,10 @@ class DashScopeCardsTest(unittest.TestCase):
"http://localhost:8080/custom/dashscope/path",
"model-x",
),
+ _realtime_url(
+ WORKSPACE_BASE_URL,
+ "qwen3.8-omni-flash-realtime",
+ ),
],
[
"wss://dashscope.aliyuncs.com/api-ws/v1/realtime"
@@ -307,6 +380,9 @@ class DashScopeCardsTest(unittest.TestCase):
"wss://llm-beijing.cn-beijing.maas.aliyuncs.com"
"/api-ws/v1/realtime?model=model-x",
"ws://localhost:8080/api-ws/v1/realtime?model=model-x",
+ "wss://llm-workspace.cn-beijing.maas.aliyuncs.com"
+ "/api-ws/v1/realtime"
+ "?model=qwen3.8-omni-flash-realtime",
],
)
@@ -329,6 +405,7 @@ class DashScopeCardsTest(unittest.TestCase):
"qwen3-omni-flash-realtime": "DashScopeRealtimeModel",
"qwen3.5-omni-flash-realtime": "DashScopeRealtimeModel",
"qwen3.5-omni-plus-realtime": "DashScopeRealtimeModel",
+ "qwen3.8-omni-flash-realtime": "DashScopeRealtimeModel",
},
)
@@ -352,15 +429,15 @@ class DashScopeSessionUpdateTest(unittest.TestCase):
"instructions": "be nice",
"modalities": ["audio", "text"],
"voice": "Tina",
- "input_audio_format": "pcm16",
- "output_audio_format": "pcm24",
+ "input_audio_format": "pcm",
+ "output_audio_format": "pcm",
"turn_detection": {
"type": "server_vad",
"threshold": 0.5,
"silence_duration_ms": 800,
},
"input_audio_transcription": {
- "model": "gummy-realtime-v1",
+ "model": "qwen3-asr-flash-realtime",
},
"tools": [{"type": "function"}],
},
@@ -373,6 +450,36 @@ class DashScopeSessionUpdateTest(unittest.TestCase):
session = model._session_update("x", [{"type": "function"}])["session"]
self.assertNotIn("tools", session)
+ def test_qwen38_payload(self) -> None:
+ """Qwen3.8 uses the shared Qwen-Omni wire format."""
+ model = DashScopeRealtimeModel(
+ "qwen3.8-omni-flash-realtime",
+ WORKSPACE_CRED,
+ )
+
+ self.assertDictEqual(
+ model._session_update("be nice", [{"type": "function"}]),
+ {
+ "type": "session.update",
+ "session": {
+ "instructions": "be nice",
+ "modalities": ["audio", "text"],
+ "voice": "Tina",
+ "input_audio_format": "pcm",
+ "output_audio_format": "pcm",
+ "turn_detection": {
+ "type": "server_vad",
+ "threshold": 0.5,
+ "silence_duration_ms": 800,
+ },
+ "input_audio_transcription": {
+ "model": "qwen3-asr-flash-realtime",
+ },
+ "tools": [{"type": "function"}],
+ },
+ },
+ )
+
def test_audio_payload_with_smart_turn(self) -> None:
"""Audio session.update with smart_turn and a voiceprint."""
model = DashScopeAudioRealtimeModel(
@@ -655,6 +762,33 @@ class DashScopeParseTest(unittest.TestCase):
)
+class DashScopeResponseControlTest(IsolatedAsyncioTestCase):
+ """Response requests follow each adapter's wire shape."""
+
+ async def test_omni_response_request_has_no_config(self) -> None:
+ """Qwen-Omni reads modalities from the session configuration."""
+ sent: list[dict] = []
+
+ async def capture(payload: dict) -> None:
+ sent.append(payload)
+
+ for card in DashScopeRealtimeModel.list_models():
+ with self.subTest(model=card.name):
+ model = DashScopeRealtimeModel(
+ card.name,
+ CRED,
+ model_card=card,
+ )
+ model._send = capture # type: ignore[method-assign]
+ await model.request_response()
+
+ self.assertListEqual(
+ sent,
+ [{"type": "response.create"}],
+ )
+ sent.clear()
+
+
class DashScopeAudioTextInputTest(IsolatedAsyncioTestCase):
"""push_text sends a text item and asks for the reply."""
@@ -940,15 +1074,15 @@ class DashScopeDisconnectTest(IsolatedAsyncioTestCase):
"instructions": "test",
"modalities": ["audio", "text"],
"voice": "Tina",
- "input_audio_format": "pcm16",
- "output_audio_format": "pcm24",
+ "input_audio_format": "pcm",
+ "output_audio_format": "pcm",
"turn_detection": {
"type": "server_vad",
"threshold": 0.5,
"silence_duration_ms": 800,
},
"input_audio_transcription": {
- "model": "gummy-realtime-v1",
+ "model": "qwen3-asr-flash-realtime",
},
},
},