Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(realtime): add Qwen3.8 Omni Realtime support (#3196)
qbce4c1e074dee3
diff --git a/examples/realtime/README.md b/examples/realtime/README.mdindex 9aacf3a99..eac9be9f8 100644--- a/examples/realtime/README.md+++ b/examples/realtime/README.md@@ -50,6 +50,28 @@ REALTIME_MODEL=qwen3.5-omni-flash-realtime python examples/realtime/local_mic.pyAn unknown name prints the available ones.+`qwen3.8-omni-flash-realtime` requires the endpoint bound to the same+DashScope workspace as the API key. Set `DASHSCOPE_BASE_URL` to the HTTPS+Chat Completions base URL (not the WebSocket URL):++```bash+export DASHSCOPE_BASE_URL=https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/compatible-mode/v1+REALTIME_MODEL=qwen3.8-omni-flash-realtime python examples/realtime/local_mic.py+```++The credential uses that URL for Chat Completions. For realtime sessions,+the adapter keeps the same workspace host and derives this WebSocket URL:++```text+wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime?model=qwen3.8-omni-flash-realtime+```++For the Singapore region, use the corresponding workspace URL:++```bash+export DASHSCOPE_BASE_URL=https://{WorkspaceId}.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1+```+### ToolsThe example registers `Bash`, `Edit`, `Write` and `Read`. When the model calls@@ -81,8 +103,9 @@ an unanswered one after five minutes.transcript so far appended to the system prompt so the model keeps thethread.- **Model limits are per turn, not per token.** `qwen3-omni-flash-realtime`- remembers eight turns, `qwen-audio-3.0-realtime-*` fifty; older turns are- dropped silently on the provider side. The cards under+ remembers eight turns, `qwen3.8-omni-flash-realtime` one hundred turns or+ six hundred seconds of audio, and `qwen-audio-3.0-realtime-*` fifty turns;+ older turns are dropped silently on the provider side. The cards under`agentscope/realtime/_dashscope/` list each model's limits.- **Typed input** only works on models that accept text(`qwen-audio-3.0-realtime-*`); the Omni models are audio-in only, and thediff --git a/examples/realtime/local_mic.py b/examples/realtime/local_mic.pyindex b02bde636..952189576 100644--- a/examples/realtime/local_mic.py+++ b/examples/realtime/local_mic.py@@ -33,7 +33,13 @@ async def main() -> None:api_key = os.environ.get("DASHSCOPE_API_KEY")if not api_key:raise SystemExit("Set DASHSCOPE_API_KEY first.")- credential = DashScopeCredential(api_key=api_key)+ credential = DashScopeCredential(+ api_key=api_key,+ base_url=os.environ.get(+ "DASHSCOPE_BASE_URL",+ "https://dashscope.aliyuncs.com/compatible-mode/v1",+ ),+ )# Resolve the model class from its card rather than hard-coding it:# a credential can serve several realtime APIs, and each card carries@@ -48,7 +54,7 @@ async def main() -> None:agent = RealtimeAgent(name="Friday",- system_prompt="你是一个中文语音助手,回答尽量简短。",+ system_prompt="You are a concise voice assistant.",model=model,toolkit=Toolkit(tools=[Bash(), Edit(), Write(), Read()]),)diff --git a/src/agentscope/realtime/_dashscope/_audio_model.py b/src/agentscope/realtime/_dashscope/_audio_model.pyindex 903be67c6..68c0445c9 100644--- a/src/agentscope/realtime/_dashscope/_audio_model.py+++ b/src/agentscope/realtime/_dashscope/_audio_model.py@@ -180,6 +180,17 @@ class DashScopeAudioRealtimeModel(DashScopeRealtimeModel):},)+ async def request_response(self) -> None:+ """Ask for a reply with the Qwen-Audio modality override."""+ if self._item_id:+ await self.cancel_response()+ await self._send(+ {+ "type": "response.create",+ "response": {"modalities": ["text", "audio"]},+ },+ )+def _session_update(self,instructions: str,diff --git a/src/agentscope/realtime/_dashscope/_model.py b/src/agentscope/realtime/_dashscope/_model.pyindex e57836c5b..54f757cb3 100644--- a/src/agentscope/realtime/_dashscope/_model.py+++ b/src/agentscope/realtime/_dashscope/_model.py@@ -228,12 +228,7 @@ class DashScopeRealtimeModel(RealtimeModelBase):"""Ask for a reply, cancelling one already in flight first."""if self._item_id:await self.cancel_response()- await self._send(- {- "type": "response.create",- "response": {"modalities": ["text", "audio"]},- },- )+ await self._send({"type": "response.create"})async def cancel_response(self) -> None:"""Cancel the reply in flight, if any."""@@ -251,23 +246,26 @@ class DashScopeRealtimeModel(RealtimeModelBase):) -> dict:"""Build the ``session.update`` message."""p = self.parameters- session: dict[str, Any] = {- "instructions": instructions,- "modalities": ["audio", "text"],- "voice": p.voice,- "input_audio_format": f"pcm{self.input_sample_rate // 1000}",- "output_audio_format": f"pcm{self.output_sample_rate // 1000}",- "turn_detection": None+ turn_detection = (+ Noneif p.turn_detection == "none"else {"type": p.turn_detection,"threshold": p.vad_threshold,"silence_duration_ms": p.vad_silence_duration_ms,- },+ }+ )+ session: dict[str, Any] = {+ "instructions": instructions,+ "modalities": ["audio", "text"],+ "voice": p.voice,+ "input_audio_format": "pcm",+ "output_audio_format": "pcm",+ "turn_detection": turn_detection,}if p.input_audio_transcription:session["input_audio_transcription"] = {- "model": "gummy-realtime-v1",+ "model": "qwen3-asr-flash-realtime",}if tools and self.card.supports_tools:session["tools"] = toolsdiff --git a/src/agentscope/realtime/_dashscope/_models/qwen3.8-omni-flash-realtime.yaml b/src/agentscope/realtime/_dashscope/_models/qwen3.8-omni-flash-realtime.yamlnew file mode 100644index 000000000..ef2b2a63b--- /dev/null+++ b/src/agentscope/realtime/_dashscope/_models/qwen3.8-omni-flash-realtime.yaml@@ -0,0 +1,26 @@+type: realtime_model+name: qwen3.8-omni-flash-realtime+label: Qwen3.8-Omni-Flash Realtime+status: active++input_types:+ - audio/pcm+ - image/jpeg+output_types:+ - audio/pcm+ - text/plain++input_sample_rate: 16000+output_sample_rate: 24000++supports_tools: true++max_context_tokens: 196608+max_audio_turns: 100+max_audio_duration_s: 600+max_session_duration_s: 7200++parameter_overrides:+ voice:+ default: "Tina"+ enum: ["Tina", "Serena", "Ryan", "Katerina", "longanlingxin"]diff --git a/tests/realtime_dashscope_test.py b/tests/realtime_dashscope_test.pyindex 3d46d548a..a56cf1a1b 100644--- a/tests/realtime_dashscope_test.py+++ b/tests/realtime_dashscope_test.py@@ -27,6 +27,12 @@ from agentscope.message import ()CRED = DashScopeCredential(api_key="sk-x")+WORKSPACE_HOST = "https://llm-workspace.cn-beijing.maas.aliyuncs.com"+WORKSPACE_BASE_URL = f"{WORKSPACE_HOST}/compatible-mode/v1"+WORKSPACE_CRED = DashScopeCredential(+ api_key="sk-workspace",+ base_url=WORKSPACE_BASE_URL,+)TRANSCRIPTION_DONE = "conversation.item.input_audio_transcription.completed"AMBIENT_DELTA = "conversation.item.ambient_audio_transcription.delta"@@ -63,10 +69,15 @@ class DashScopeCardsTest(unittest.TestCase):"""Each adapter lists only its own cards, tagged with its type."""def test_omni_cards(self) -> None:- """The Omni adapter lists the four Omni cards."""+ """The Omni adapter lists the five Omni cards."""self.assertListEqual([- (c.name, c.model_type, c.supports_tools, c.max_audio_turns)+ (+ c.name,+ c.model_type,+ c.supports_tools,+ c.max_audio_turns,+ )for c in DashScopeRealtimeModel.list_models()],[@@ -94,9 +105,52 @@ class DashScopeCardsTest(unittest.TestCase):True,100,),+ (+ "qwen3.8-omni-flash-realtime",+ "dashscope_omni_realtime",+ True,+ 100,+ ),],)+ def test_qwen38_card_limits(self) -> None:+ """The Qwen3.8 card exposes the documented audio limits."""+ card = next(+ card+ for card in DashScopeRealtimeModel.list_models()+ if card.name == "qwen3.8-omni-flash-realtime"+ )++ self.assertDictEqual(+ card.model_dump(+ include={+ "name",+ "input_types",+ "output_types",+ "input_sample_rate",+ "output_sample_rate",+ "supports_tools",+ "max_context_tokens",+ "max_audio_turns",+ "max_audio_duration_s",+ "max_session_duration_s",+ },+ ),+ {+ "name": "qwen3.8-omni-flash-realtime",+ "input_types": ["audio/pcm", "image/jpeg"],+ "output_types": ["audio/pcm", "text/plain"],+ "input_sample_rate": 16000,+ "output_sample_rate": 24000,+ "supports_tools": True,+ "max_context_tokens": 196608,+ "max_audio_turns": 100,+ "max_audio_duration_s": 600,+ "max_session_duration_s": 7200,+ },+ )+def test_audio_cards(self) -> None:"""The Audio adapter lists the three Audio cards."""self.assertListEqual(@@ -144,7 +198,7 @@ class DashScopeCardsTest(unittest.TestCase):],"runtime_default": DashScopeRealtimeModel(name,- CRED,+ WORKSPACE_CRED,).parameters.voice,}for name in cards@@ -186,6 +240,21 @@ class DashScopeCardsTest(unittest.TestCase):},"runtime_default": "Tina",},+ "qwen3.8-omni-flash-realtime": {+ "schema": {+ "default": "Tina",+ "enum": [+ "Tina",+ "Serena",+ "Ryan",+ "Katerina",+ "longanlingxin",+ ],+ "title": "Voice",+ "type": "string",+ },+ "runtime_default": "Tina",+ },},)@@ -298,6 +367,10 @@ class DashScopeCardsTest(unittest.TestCase):"http://localhost:8080/custom/dashscope/path","model-x",),+ _realtime_url(+ WORKSPACE_BASE_URL,+ "qwen3.8-omni-flash-realtime",+ ),],["wss://dashscope.aliyuncs.com/api-ws/v1/realtime"@@ -307,6 +380,9 @@ class DashScopeCardsTest(unittest.TestCase):"wss://llm-beijing.cn-beijing.maas.aliyuncs.com""/api-ws/v1/realtime?model=model-x","ws://localhost:8080/api-ws/v1/realtime?model=model-x",+ "wss://llm-workspace.cn-beijing.maas.aliyuncs.com"+ "/api-ws/v1/realtime"+ "?model=qwen3.8-omni-flash-realtime",],)@@ -329,6 +405,7 @@ class DashScopeCardsTest(unittest.TestCase):"qwen3-omni-flash-realtime": "DashScopeRealtimeModel","qwen3.5-omni-flash-realtime": "DashScopeRealtimeModel","qwen3.5-omni-plus-realtime": "DashScopeRealtimeModel",+ "qwen3.8-omni-flash-realtime": "DashScopeRealtimeModel",},)@@ -352,15 +429,15 @@ class DashScopeSessionUpdateTest(unittest.TestCase):"instructions": "be nice","modalities": ["audio", "text"],"voice": "Tina",- "input_audio_format": "pcm16",- "output_audio_format": "pcm24",+ "input_audio_format": "pcm",+ "output_audio_format": "pcm","turn_detection": {"type": "server_vad","threshold": 0.5,"silence_duration_ms": 800,},"input_audio_transcription": {- "model": "gummy-realtime-v1",+ "model": "qwen3-asr-flash-realtime",},"tools": [{"type": "function"}],},@@ -373,6 +450,36 @@ class DashScopeSessionUpdateTest(unittest.TestCase):session = model._session_update("x", [{"type": "function"}])["session"]self.assertNotIn("tools", session)+ def test_qwen38_payload(self) -> None:+ """Qwen3.8 uses the shared Qwen-Omni wire format."""+ model = DashScopeRealtimeModel(+ "qwen3.8-omni-flash-realtime",+ WORKSPACE_CRED,+ )++ self.assertDictEqual(+ model._session_update("be nice", [{"type": "function"}]),+ {+ "type": "session.update",+ "session": {+ "instructions": "be nice",+ "modalities": ["audio", "text"],+ "voice": "Tina",+ "input_audio_format": "pcm",+ "output_audio_format": "pcm",+ "turn_detection": {+ "type": "server_vad",+ "threshold": 0.5,+ "silence_duration_ms": 800,+ },+ "input_audio_transcription": {+ "model": "qwen3-asr-flash-realtime",+ },+ "tools": [{"type": "function"}],+ },+ },+ )+def test_audio_payload_with_smart_turn(self) -> None:"""Audio session.update with smart_turn and a voiceprint."""model = DashScopeAudioRealtimeModel(@@ -655,6 +762,33 @@ class DashScopeParseTest(unittest.TestCase):)+class DashScopeResponseControlTest(IsolatedAsyncioTestCase):+ """Response requests follow each adapter's wire shape."""++ async def test_omni_response_request_has_no_config(self) -> None:+ """Qwen-Omni reads modalities from the session configuration."""+ sent: list[dict] = []++ async def capture(payload: dict) -> None:+ sent.append(payload)++ for card in DashScopeRealtimeModel.list_models():+ with self.subTest(model=card.name):+ model = DashScopeRealtimeModel(+ card.name,+ CRED,+ model_card=card,+ )+ model._send = capture # type: ignore[method-assign]+ await model.request_response()++ self.assertListEqual(+ sent,+ [{"type": "response.create"}],+ )+ sent.clear()++class DashScopeAudioTextInputTest(IsolatedAsyncioTestCase):"""push_text sends a text item and asks for the reply."""@@ -940,15 +1074,15 @@ class DashScopeDisconnectTest(IsolatedAsyncioTestCase):"instructions": "test","modalities": ["audio", "text"],"voice": "Tina",- "input_audio_format": "pcm16",- "output_audio_format": "pcm24",+ "input_audio_format": "pcm",+ "output_audio_format": "pcm","turn_detection": {"type": "server_vad","threshold": 0.5,"silence_duration_ms": 800,},"input_audio_transcription": {- "model": "gummy-realtime-v1",+ "model": "qwen3-asr-flash-realtime",},},},