SOTA-Class TTS at Compact Scale
Python
Add streaming audio output to SGLang Omni
linux295650-ux06bea5bb48d9
diff --git a/README.md b/README.mdindex daa90e5..2bbc1ed 100644--- a/README.md+++ b/README.md@@ -286,11 +286,21 @@ curl -sS --fail-with-body \```The reference path must be visible inside the service environment. The current-adapter supports TP=1, one reference per request, and non-streaming WAV output.+adapter supports TP=1 and one reference per request. Set `"stream": true` to+receive SSE audio chunks as they are generated. For the lowest overhead, use+`"response_format": "pcm"`; each event contains Base64-encoded audio in+`audio.data`. Streaming defaults to 12 codec frames per chunk, with 128 frames+of decoder context and a one-frame boundary guard. Override these values with+`AUDIO8_TTS_STREAM_CHUNK_FRAMES`, `AUDIO8_TTS_STREAM_CONTEXT_FRAMES`, and+`AUDIO8_TTS_STREAM_GUARD_FRAMES`.+Run the smoke test to verify a deployment:```bashBASE_URL=http://127.0.0.1:8010 ./sglang_omni/scripts/smoke_test.sh+python3 ./sglang_omni/scripts/stream_smoke_test.py \+ --base-url http://127.0.0.1:8010 \+ --output /tmp/audio8_stream.wav```To build the adapter into an existing image, appenddiff --git a/README_zh.md b/README_zh.mdindex 7f98872..1f42c34 100644--- a/README_zh.md+++ b/README_zh.md@@ -266,11 +266,20 @@ curl -sS --fail-with-body \-o audio8_clone.wav```-参考音频路径必须在服务运行环境中可见。当前适配器支持 TP=1、每个请求一个 reference 和-非流式 WAV 输出。运行以下 smoke test 可以验证部署:+参考音频路径必须在服务运行环境中可见。当前适配器支持 TP=1 和每个请求一个 reference。+设置 `"stream": true` 可在生成过程中接收 SSE 音频分块。使用+`"response_format": "pcm"` 开销最低,每个事件的 `audio.data` 包含 Base64 编码音频。+流式输出默认每 12 个 codec frames 生成一块,使用 128 帧 decoder context 和 1 帧边界+guard。可通过 `AUDIO8_TTS_STREAM_CHUNK_FRAMES`、+`AUDIO8_TTS_STREAM_CONTEXT_FRAMES` 和 `AUDIO8_TTS_STREAM_GUARD_FRAMES` 调整。++运行以下 smoke test 可以验证部署:```bashBASE_URL=http://127.0.0.1:8010 ./sglang_omni/scripts/smoke_test.sh+python3 ./sglang_omni/scripts/stream_smoke_test.py \+ --base-url http://127.0.0.1:8010 \+ --output /tmp/audio8_stream.wav```构建镜像时,请在安装 SGLang Omni 包及其 Python 依赖后追加diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.pyindex 5078e63..20e20fe 100644--- a/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.py+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.py@@ -3,7 +3,13 @@ from __future__ import annotationsfrom typing import ClassVar-from sglang_omni.config import ExecutorConfig, PipelineConfig, RelayConfig, StageConfig+from sglang_omni.config import (+ ExecutorConfig,+ PipelineConfig,+ RelayConfig,+ StageConfig,+)+from sglang_omni.config.schema import StreamTargetConfig_PKG = "sglang_omni.models.audio8_tts.pipeline"@@ -31,6 +37,7 @@ class Audio8TTSPipelineConfig(PipelineConfig):),get_next=f"{_PKG}.next_stage.tts_engine_next",relay=RelayConfig(device="cuda"),+ stream_to=[StreamTargetConfig(to_stage="vocoder")],),StageConfig(name="vocoder",diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.pyindex 415262d..24689a6 100644--- a/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.py+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.py@@ -2,6 +2,7 @@from __future__ import annotationsimport os+from collections.abc import Callablefrom typing import Anyfrom sglang_omni.engines.omni.engine import OmniEngine@@ -19,6 +20,7 @@ def create_audio8_engine(gpu_id: int,eos_token_id: int,max_new_tokens: int,+ stream_fn: Callable[[str, Any], None] | None = None,) -> OmniEngine:from sglang.srt.models.registry import ModelRegistry@@ -76,6 +78,14 @@ def create_audio8_engine(on_retract=lambda request: prefill_manager.add_one_request(request),)planner = SGLangBatchPlanner(prefill_manager, decode_manager, server_args)+ def stream_adapter(request: Any, output: Any) -> Any:+ if output.data is None:+ return None+ codes = output.data.codes.clone()+ if stream_fn is not None:+ stream_fn(request.request_id, codes)+ return codes+scheduler = Scheduler(batch_planner=planner,resource_manager=Audio8ResourceManager(@@ -88,9 +98,7 @@ def create_audio8_engine(eos_token_id=eos_token_id,max_new_tokens=max_new_tokens,),- stream_adapter=lambda request, output: (- None if output.data is None else output.data.codes- ),+ stream_adapter=stream_adapter,)return OmniEngine(scheduler=scheduler,diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.pyindex c69c496..dcb7551 100644--- a/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.py+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.py@@ -5,6 +5,7 @@ import base64import importlib.utilimport jsonimport logging+import osimport sysfrom pathlib import Pathfrom types import SimpleNamespace@@ -21,6 +22,9 @@ from sglang_omni.models.audio8_tts.pipeline.engine_io import (build_tts_request,)from sglang_omni.models.audio8_tts.pipeline.state_io import load_state, store_state+from sglang_omni.models.audio8_tts.pipeline.streaming_vocoder import (+ Audio8StreamingVocoderExecutor,+)from sglang_omni.models.audio8_tts.tokenizer import Audio8TokenizerAdapter, Referencefrom sglang_omni.proto import StagePayload@@ -195,11 +199,19 @@ def create_sglang_tts_engine_executor()server_args = make_server_args(model_path)gpu_id = int(device.split(":")[-1]) if ":" in device else 0+ enqueue_fn_holder: dict[str, Any] = {}++ def enqueue_stream(request_id: str, codes: torch.Tensor) -> None:+ enqueue_fn = enqueue_fn_holder.get("fn")+ if enqueue_fn is not None:+ enqueue_fn(request_id, codes, target_stage="vocoder")+engine = create_audio8_engine(server_args,gpu_id=gpu_id,eos_token_id=config.eos_token_id,max_new_tokens=max_new_tokens,+ stream_fn=enqueue_stream,)def request_builder(payload: StagePayload):@@ -210,18 +222,24 @@ def create_sglang_tts_engine_executor(apply_tts_result(state, result)return store_state(payload, state)- return EngineExecutor(+ executor = EngineExecutor(engine=engine,request_builder=request_builder,result_builder=result_builder,)+ def set_stream_fn(stream_fn: Any) -> None:+ enqueue_fn_holder["fn"] = stream_fn++ executor.set_stream_fn = set_stream_fn+ return executor+def create_vocoder_executor(model_path: str,*,device: str = "cuda:0",-) -> PreprocessingExecutor:+) -> Audio8StreamingVocoderExecutor:codec = _load_codec(model_path, device)config = _load_config(model_path)warmup_codes = torch.zeros(@@ -233,29 +251,13 @@ def create_vocoder_executor()with torch.inference_mode():codec.decode(warmup_codes)-- def vocode(payload: StagePayload) -> StagePayload:- state = load_state(payload)- if state.output_codes is None:- raise ValueError("Audio8 generation produced no codec frames")- codes = state.output_codes.to(device=device, dtype=torch.long)- with torch.inference_mode():- audio = codec.decode(codes.unsqueeze(0))[0, 0].float().cpu()- state.audio_samples = audio- state.sample_rate = codec.sample_rate- payload = store_state(payload, state)- payload.data.update(- {- "audio_data": audio.tolist(),- "sample_rate": codec.sample_rate,- "modality": "audio",- "usage": {- "prompt_tokens": state.prompt_tokens,- "completion_tokens": state.completion_tokens,- "total_tokens": state.prompt_tokens + state.completion_tokens,- },- }- )- return payload-- return PreprocessingExecutor(vocode)+ return Audio8StreamingVocoderExecutor(+ codec,+ device=device,+ eos_token_id=config.eos_token_id,+ num_codebooks=config.num_codebooks,+ chunk_frames=int(os.getenv("AUDIO8_TTS_STREAM_CHUNK_FRAMES", "12")),+ context_frames=int(os.getenv("AUDIO8_TTS_STREAM_CONTEXT_FRAMES", "128")),+ guard_frames=int(os.getenv("AUDIO8_TTS_STREAM_GUARD_FRAMES", "1")),+ hop_length=config.codec_frame_size,+ )diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/streaming_vocoder.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/streaming_vocoder.pynew file mode 100644index 0000000..e6cf7f1--- /dev/null+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/streaming_vocoder.py@@ -0,0 +1,191 @@+# SPDX-License-Identifier: Apache-2.0+from __future__ import annotations++import asyncio+from typing import Any++import numpy as np+import torch++from sglang_omni.executors import Executor+from sglang_omni.models.audio8_tts.pipeline.state_io import load_state+from sglang_omni.proto import StagePayload+++class Audio8StreamingVocoderExecutor(Executor):+ def __init__(+ self,+ codec: Any,+ *,+ device: str,+ eos_token_id: int,+ num_codebooks: int,+ chunk_frames: int = 12,+ context_frames: int = 128,+ guard_frames: int = 1,+ hop_length: int = 2048,+ ) -> None:+ self._codec = codec+ self._device = torch.device(device)+ self._eos_token_id = int(eos_token_id)+ self._num_codebooks = int(num_codebooks)+ self._chunk_frames = max(int(chunk_frames), 1)+ self._context_frames = max(int(context_frames), 0)+ self._guard_samples = max(int(guard_frames), 0) * int(hop_length)+ self._hop_length = int(hop_length)+ self._sample_rate = int(codec.sample_rate)+ self._stream_queue: Any | None = None+ self._done: asyncio.Queue[str] = asyncio.Queue()+ self._tasks: dict[str, asyncio.Task[StagePayload]] = {}+ self._output_queues: dict[str, asyncio.Queue[dict[str, Any] | None]] = {}+ self._aborted: set[str] = set()+ self._gpu_lock = asyncio.Lock()++ async def add_request(self, payload: StagePayload) -> None:+ request_id = payload.request_id+ if request_id in self._aborted:+ return+ self._output_queues[request_id] = asyncio.Queue()+ task = asyncio.create_task(self._run_request(payload))+ self._tasks[request_id] = task+ task.add_done_callback(lambda _task: self._done.put_nowait(request_id))++ async def get_result(self) -> StagePayload:+ while True:+ request_id = await self._done.get()+ task = self._tasks.pop(request_id, None)+ if task is None:+ continue+ if request_id in self._aborted:+ self._output_queues.pop(request_id, None)+ continue+ try:+ return await task+ except Exception as exc:+ exc.request_id = request_id+ raise++ async def abort(self, request_id: str) -> None:+ self._aborted.add(request_id)+ task = self._tasks.pop(request_id, None)+ if task is not None:+ task.cancel()+ queue = self._output_queues.pop(request_id, None)+ if queue is not None:+ queue.put_nowait(None)++ async def stream(self, request_id: str):+ queue = self._output_queues.get(request_id)+ if queue is None:+ return+ while True:+ item = await queue.get()+ if item is None:+ return+ yield item++ async def _run_request(self, payload: StagePayload) -> StagePayload:+ if self._stream_queue is None:+ raise RuntimeError("Audio8 streaming vocoder requires a stream queue")++ request_id = payload.request_id+ output_queue = self._output_queues[request_id]+ state = load_state(payload)+ frames: list[torch.Tensor] = []+ emitted_samples = 0++ try:+ while True:+ item = await self._stream_queue.get(request_id)+ if item is None:+ break+ codes = torch.as_tensor(item.data, device=self._device, dtype=torch.long)+ if codes.ndim == 1:+ codes = codes[:, None]+ semantic = int(codes[0, -1].item())+ if semantic == self._eos_token_id:+ continue+ frame = codes[1 : self._num_codebooks + 1, -1].clone()+ if frame.numel() != self._num_codebooks:+ raise ValueError(+ f"Audio8 stream frame has {frame.numel()} codebooks, "+ f"expected {self._num_codebooks}"+ )+ frames.append(frame)+ if len(frames) % self._chunk_frames:+ continue+ audio, absolute_start = await self._decode_stream_window(frames)+ stable_end = max(0, audio.size - self._guard_samples)+ begin = max(0, emitted_samples - absolute_start)+ if stable_end > begin:+ chunk = np.ascontiguousarray(audio[begin:stable_end])+ emitted_samples = absolute_start + stable_end+ await output_queue.put(self._audio_payload(chunk))++ if not frames:+ raise ValueError("Audio8 generation produced no codec frames")++ tail, absolute_start = await self._decode_stream_window(frames)+ begin = max(0, emitted_samples - absolute_start)+ if begin < tail.size:+ chunk = np.ascontiguousarray(tail[begin:])+ emitted_samples = absolute_start + tail.size+ await output_queue.put(self._audio_payload(chunk))+ await output_queue.put(None)++ full_audio = await self._decode_frames(frames, 0, len(frames))+ payload.data = self._audio_payload(full_audio)+ prompt_tokens = len(state.input_ids) if state.input_ids is not None else 0+ payload.data["usage"] = {+ "prompt_tokens": prompt_tokens,+ "completion_tokens": len(frames),+ "total_tokens": prompt_tokens + len(frames),+ }+ return payload+ finally:+ self._output_queues.pop(request_id, None)++ async def _decode_stream_window(+ self, frames: list[torch.Tensor]+ ) -> tuple[np.ndarray, int]:+ end = len(frames)+ start = max(0, end - self._context_frames - self._chunk_frames)+ return await self._decode_frames(frames, start, end), start * self._hop_length++ async def _decode_frames(+ self,+ frames: list[torch.Tensor],+ start: int,+ end: int,+ ) -> np.ndarray:+ loop = asyncio.get_running_loop()+ async with self._gpu_lock:+ return await loop.run_in_executor(+ None,+ self._decode_frames_sync,+ frames,+ start,+ end,+ )++ def _decode_frames_sync(+ self,+ frames: list[torch.Tensor],+ start: int,+ end: int,+ ) -> np.ndarray:+ torch.cuda.set_device(self._device)+ codes = torch.stack(frames[start:end], dim=1).unsqueeze(0)+ with torch.inference_mode():+ audio = self._codec.decode(codes)[0, 0]+ return audio.detach().float().cpu().numpy().copy()++ def _audio_payload(self, audio: np.ndarray) -> dict[str, Any]:+ value = audio.astype(np.float32, copy=False)+ return {+ "audio_waveform": value.tobytes(),+ "audio_waveform_shape": list(value.shape),+ "audio_waveform_dtype": "float32",+ "sample_rate": self._sample_rate,+ "modality": "audio",+ }diff --git a/sglang_omni/configs/audio8_tts_0_6b.yaml b/sglang_omni/configs/audio8_tts_0_6b.yamlindex 6d67b19..7fe38be 100644--- a/sglang_omni/configs/audio8_tts_0_6b.yaml+++ b/sglang_omni/configs/audio8_tts_0_6b.yaml@@ -24,6 +24,8 @@ stages:get_next: sglang_omni.models.audio8_tts.pipeline.next_stage.tts_engine_nextrelay:device: cuda+ stream_to:+ - to_stage: vocoder- name: vocodernum_workers: 1executor:diff --git a/sglang_omni/scripts/stream_smoke_test.py b/sglang_omni/scripts/stream_smoke_test.pynew file mode 100755index 0000000..9bcf91b--- /dev/null+++ b/sglang_omni/scripts/stream_smoke_test.py@@ -0,0 +1,88 @@+#!/usr/bin/env python3+# SPDX-License-Identifier: Apache-2.0+from __future__ import annotations++import argparse+import base64+import json+import time+import urllib.request+import wave+from pathlib import Path+++def main() -> None:+ parser = argparse.ArgumentParser()+ parser.add_argument("--base-url", default="http://127.0.0.1:8010")+ parser.add_argument("--output", type=Path, default=Path("/tmp/audio8_stream.wav"))+ parser.add_argument(+ "--text",+ default="Streaming speech should begin playing before the complete sentence is ready.",+ )+ args = parser.parse_args()++ body = json.dumps(+ {+ "model": "audio8/tts-0.6b",+ "input": args.text,+ "response_format": "pcm",+ "stream": True,+ "max_new_tokens": 256,+ "temperature": 0.7,+ "top_p": 0.8,+ "top_k": 50,+ }+ ).encode("utf-8")+ request = urllib.request.Request(+ f"{args.base_url}/v1/audio/speech",+ data=body,+ headers={"Content-Type": "application/json"},+ method="POST",+ )++ started = time.perf_counter()+ first_chunk_seconds: float | None = None+ sample_rate: int | None = None+ chunks: list[bytes] = []+ with urllib.request.urlopen(request, timeout=300) as response:+ for raw_line in response:+ line = raw_line.decode("utf-8").strip()+ if not line.startswith("data: "):+ continue+ value = line[6:]+ if value == "[DONE]":+ break+ event = json.loads(value)+ audio = event.get("audio")+ if not audio:+ continue+ if audio.get("format") != "pcm":+ raise RuntimeError(f"Expected PCM stream, got {audio.get('format')}")+ if first_chunk_seconds is None:+ first_chunk_seconds = time.perf_counter() - started+ sample_rate = int(audio["sample_rate"])+ chunks.append(base64.b64decode(audio["data"]))++ total_seconds = time.perf_counter() - started+ if not chunks or sample_rate is None or first_chunk_seconds is None:+ raise RuntimeError("The server returned no streaming audio chunks")+ pcm = b"".join(chunks)+ if len(pcm) % 2:+ raise RuntimeError("PCM stream has an odd byte count")+ args.output.parent.mkdir(parents=True, exist_ok=True)+ with wave.open(str(args.output), "wb") as output:+ output.setnchannels(1)+ output.setsampwidth(2)+ output.setframerate(sample_rate)+ output.writeframes(pcm)++ print(f"chunks={len(chunks)}")+ print(f"first_chunk_seconds={first_chunk_seconds:.3f}")+ print(f"total_seconds={total_seconds:.3f}")+ print(f"sample_rate={sample_rate}")+ print(f"audio_seconds={len(pcm) / 2 / sample_rate:.3f}")+ print(f"output={args.output}")+++if __name__ == "__main__":+ main()