Skip to content

SOTA-Class TTS at Compact Scale

Python

Add streaming audio output to SGLang Omni

linux295650-ux06bea5bb48d9
diff --git a/README.md b/README.md
index daa90e5..2bbc1ed 100644
--- a/README.md
+++ b/README.md
@@ -286,11 +286,21 @@ curl -sS --fail-with-body \
```
The reference path must be visible inside the service environment. The current
-adapter supports TP=1, one reference per request, and non-streaming WAV output.
+adapter supports TP=1 and one reference per request. Set `"stream": true` to
+receive SSE audio chunks as they are generated. For the lowest overhead, use
+`"response_format": "pcm"`; each event contains Base64-encoded audio in
+`audio.data`. Streaming defaults to 12 codec frames per chunk, with 128 frames
+of decoder context and a one-frame boundary guard. Override these values with
+`AUDIO8_TTS_STREAM_CHUNK_FRAMES`, `AUDIO8_TTS_STREAM_CONTEXT_FRAMES`, and
+`AUDIO8_TTS_STREAM_GUARD_FRAMES`.
+
Run the smoke test to verify a deployment:
```bash
BASE_URL=http://127.0.0.1:8010 ./sglang_omni/scripts/smoke_test.sh
+python3 ./sglang_omni/scripts/stream_smoke_test.py \
+ --base-url http://127.0.0.1:8010 \
+ --output /tmp/audio8_stream.wav
```
To build the adapter into an existing image, append
diff --git a/README_zh.md b/README_zh.md
index 7f98872..1f42c34 100644
--- a/README_zh.md
+++ b/README_zh.md
@@ -266,11 +266,20 @@ curl -sS --fail-with-body \
-o audio8_clone.wav
```
-参考音频路径必须在服务运行环境中可见。当前适配器支持 TP=1、每个请求一个 reference 和
-非流式 WAV 输出。运行以下 smoke test 可以验证部署:
+参考音频路径必须在服务运行环境中可见。当前适配器支持 TP=1 和每个请求一个 reference。
+设置 `"stream": true` 可在生成过程中接收 SSE 音频分块。使用
+`"response_format": "pcm"` 开销最低,每个事件的 `audio.data` 包含 Base64 编码音频。
+流式输出默认每 12 个 codec frames 生成一块,使用 128 帧 decoder context 和 1 帧边界
+guard。可通过 `AUDIO8_TTS_STREAM_CHUNK_FRAMES`、
+`AUDIO8_TTS_STREAM_CONTEXT_FRAMES` 和 `AUDIO8_TTS_STREAM_GUARD_FRAMES` 调整。
+
+运行以下 smoke test 可以验证部署:
```bash
BASE_URL=http://127.0.0.1:8010 ./sglang_omni/scripts/smoke_test.sh
+python3 ./sglang_omni/scripts/stream_smoke_test.py \
+ --base-url http://127.0.0.1:8010 \
+ --output /tmp/audio8_stream.wav
```
构建镜像时,请在安装 SGLang Omni 包及其 Python 依赖后追加
diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.py
index 5078e63..20e20fe 100644
--- a/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.py
+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/config.py
@@ -3,7 +3,13 @@ from __future__ import annotations
from typing import ClassVar
-from sglang_omni.config import ExecutorConfig, PipelineConfig, RelayConfig, StageConfig
+from sglang_omni.config import (
+ ExecutorConfig,
+ PipelineConfig,
+ RelayConfig,
+ StageConfig,
+)
+from sglang_omni.config.schema import StreamTargetConfig
_PKG = "sglang_omni.models.audio8_tts.pipeline"
@@ -31,6 +37,7 @@ class Audio8TTSPipelineConfig(PipelineConfig):
),
get_next=f"{_PKG}.next_stage.tts_engine_next",
relay=RelayConfig(device="cuda"),
+ stream_to=[StreamTargetConfig(to_stage="vocoder")],
),
StageConfig(
name="vocoder",
diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.py
index 415262d..24689a6 100644
--- a/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.py
+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/factory.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import os
+from collections.abc import Callable
from typing import Any
from sglang_omni.engines.omni.engine import OmniEngine
@@ -19,6 +20,7 @@ def create_audio8_engine(
gpu_id: int,
eos_token_id: int,
max_new_tokens: int,
+ stream_fn: Callable[[str, Any], None] | None = None,
) -> OmniEngine:
from sglang.srt.models.registry import ModelRegistry
@@ -76,6 +78,14 @@ def create_audio8_engine(
on_retract=lambda request: prefill_manager.add_one_request(request),
)
planner = SGLangBatchPlanner(prefill_manager, decode_manager, server_args)
+ def stream_adapter(request: Any, output: Any) -> Any:
+ if output.data is None:
+ return None
+ codes = output.data.codes.clone()
+ if stream_fn is not None:
+ stream_fn(request.request_id, codes)
+ return codes
+
scheduler = Scheduler(
batch_planner=planner,
resource_manager=Audio8ResourceManager(
@@ -88,9 +98,7 @@ def create_audio8_engine(
eos_token_id=eos_token_id,
max_new_tokens=max_new_tokens,
),
- stream_adapter=lambda request, output: (
- None if output.data is None else output.data.codes
- ),
+ stream_adapter=stream_adapter,
)
return OmniEngine(
scheduler=scheduler,
diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.py
index c69c496..dcb7551 100644
--- a/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.py
+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/stages.py
@@ -5,6 +5,7 @@ import base64
import importlib.util
import json
import logging
+import os
import sys
from pathlib import Path
from types import SimpleNamespace
@@ -21,6 +22,9 @@ from sglang_omni.models.audio8_tts.pipeline.engine_io import (
build_tts_request,
)
from sglang_omni.models.audio8_tts.pipeline.state_io import load_state, store_state
+from sglang_omni.models.audio8_tts.pipeline.streaming_vocoder import (
+ Audio8StreamingVocoderExecutor,
+)
from sglang_omni.models.audio8_tts.tokenizer import Audio8TokenizerAdapter, Reference
from sglang_omni.proto import StagePayload
@@ -195,11 +199,19 @@ def create_sglang_tts_engine_executor(
)
server_args = make_server_args(model_path)
gpu_id = int(device.split(":")[-1]) if ":" in device else 0
+ enqueue_fn_holder: dict[str, Any] = {}
+
+ def enqueue_stream(request_id: str, codes: torch.Tensor) -> None:
+ enqueue_fn = enqueue_fn_holder.get("fn")
+ if enqueue_fn is not None:
+ enqueue_fn(request_id, codes, target_stage="vocoder")
+
engine = create_audio8_engine(
server_args,
gpu_id=gpu_id,
eos_token_id=config.eos_token_id,
max_new_tokens=max_new_tokens,
+ stream_fn=enqueue_stream,
)
def request_builder(payload: StagePayload):
@@ -210,18 +222,24 @@ def create_sglang_tts_engine_executor(
apply_tts_result(state, result)
return store_state(payload, state)
- return EngineExecutor(
+ executor = EngineExecutor(
engine=engine,
request_builder=request_builder,
result_builder=result_builder,
)
+ def set_stream_fn(stream_fn: Any) -> None:
+ enqueue_fn_holder["fn"] = stream_fn
+
+ executor.set_stream_fn = set_stream_fn
+ return executor
+
def create_vocoder_executor(
model_path: str,
*,
device: str = "cuda:0",
-) -> PreprocessingExecutor:
+) -> Audio8StreamingVocoderExecutor:
codec = _load_codec(model_path, device)
config = _load_config(model_path)
warmup_codes = torch.zeros(
@@ -233,29 +251,13 @@ def create_vocoder_executor(
)
with torch.inference_mode():
codec.decode(warmup_codes)
-
- def vocode(payload: StagePayload) -> StagePayload:
- state = load_state(payload)
- if state.output_codes is None:
- raise ValueError("Audio8 generation produced no codec frames")
- codes = state.output_codes.to(device=device, dtype=torch.long)
- with torch.inference_mode():
- audio = codec.decode(codes.unsqueeze(0))[0, 0].float().cpu()
- state.audio_samples = audio
- state.sample_rate = codec.sample_rate
- payload = store_state(payload, state)
- payload.data.update(
- {
- "audio_data": audio.tolist(),
- "sample_rate": codec.sample_rate,
- "modality": "audio",
- "usage": {
- "prompt_tokens": state.prompt_tokens,
- "completion_tokens": state.completion_tokens,
- "total_tokens": state.prompt_tokens + state.completion_tokens,
- },
- }
- )
- return payload
-
- return PreprocessingExecutor(vocode)
+ return Audio8StreamingVocoderExecutor(
+ codec,
+ device=device,
+ eos_token_id=config.eos_token_id,
+ num_codebooks=config.num_codebooks,
+ chunk_frames=int(os.getenv("AUDIO8_TTS_STREAM_CHUNK_FRAMES", "12")),
+ context_frames=int(os.getenv("AUDIO8_TTS_STREAM_CONTEXT_FRAMES", "128")),
+ guard_frames=int(os.getenv("AUDIO8_TTS_STREAM_GUARD_FRAMES", "1")),
+ hop_length=config.codec_frame_size,
+ )
diff --git a/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/streaming_vocoder.py b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/streaming_vocoder.py
new file mode 100644
index 0000000..e6cf7f1
--- /dev/null
+++ b/sglang_omni/adapter/sglang_omni/models/audio8_tts/pipeline/streaming_vocoder.py
@@ -0,0 +1,191 @@
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import numpy as np
+import torch
+
+from sglang_omni.executors import Executor
+from sglang_omni.models.audio8_tts.pipeline.state_io import load_state
+from sglang_omni.proto import StagePayload
+
+
+class Audio8StreamingVocoderExecutor(Executor):
+ def __init__(
+ self,
+ codec: Any,
+ *,
+ device: str,
+ eos_token_id: int,
+ num_codebooks: int,
+ chunk_frames: int = 12,
+ context_frames: int = 128,
+ guard_frames: int = 1,
+ hop_length: int = 2048,
+ ) -> None:
+ self._codec = codec
+ self._device = torch.device(device)
+ self._eos_token_id = int(eos_token_id)
+ self._num_codebooks = int(num_codebooks)
+ self._chunk_frames = max(int(chunk_frames), 1)
+ self._context_frames = max(int(context_frames), 0)
+ self._guard_samples = max(int(guard_frames), 0) * int(hop_length)
+ self._hop_length = int(hop_length)
+ self._sample_rate = int(codec.sample_rate)
+ self._stream_queue: Any | None = None
+ self._done: asyncio.Queue[str] = asyncio.Queue()
+ self._tasks: dict[str, asyncio.Task[StagePayload]] = {}
+ self._output_queues: dict[str, asyncio.Queue[dict[str, Any] | None]] = {}
+ self._aborted: set[str] = set()
+ self._gpu_lock = asyncio.Lock()
+
+ async def add_request(self, payload: StagePayload) -> None:
+ request_id = payload.request_id
+ if request_id in self._aborted:
+ return
+ self._output_queues[request_id] = asyncio.Queue()
+ task = asyncio.create_task(self._run_request(payload))
+ self._tasks[request_id] = task
+ task.add_done_callback(lambda _task: self._done.put_nowait(request_id))
+
+ async def get_result(self) -> StagePayload:
+ while True:
+ request_id = await self._done.get()
+ task = self._tasks.pop(request_id, None)
+ if task is None:
+ continue
+ if request_id in self._aborted:
+ self._output_queues.pop(request_id, None)
+ continue
+ try:
+ return await task
+ except Exception as exc:
+ exc.request_id = request_id
+ raise
+
+ async def abort(self, request_id: str) -> None:
+ self._aborted.add(request_id)
+ task = self._tasks.pop(request_id, None)
+ if task is not None:
+ task.cancel()
+ queue = self._output_queues.pop(request_id, None)
+ if queue is not None:
+ queue.put_nowait(None)
+
+ async def stream(self, request_id: str):
+ queue = self._output_queues.get(request_id)
+ if queue is None:
+ return
+ while True:
+ item = await queue.get()
+ if item is None:
+ return
+ yield item
+
+ async def _run_request(self, payload: StagePayload) -> StagePayload:
+ if self._stream_queue is None:
+ raise RuntimeError("Audio8 streaming vocoder requires a stream queue")
+
+ request_id = payload.request_id
+ output_queue = self._output_queues[request_id]
+ state = load_state(payload)
+ frames: list[torch.Tensor] = []
+ emitted_samples = 0
+
+ try:
+ while True:
+ item = await self._stream_queue.get(request_id)
+ if item is None:
+ break
+ codes = torch.as_tensor(item.data, device=self._device, dtype=torch.long)
+ if codes.ndim == 1:
+ codes = codes[:, None]
+ semantic = int(codes[0, -1].item())
+ if semantic == self._eos_token_id:
+ continue
+ frame = codes[1 : self._num_codebooks + 1, -1].clone()
+ if frame.numel() != self._num_codebooks:
+ raise ValueError(
+ f"Audio8 stream frame has {frame.numel()} codebooks, "
+ f"expected {self._num_codebooks}"
+ )
+ frames.append(frame)
+ if len(frames) % self._chunk_frames:
+ continue
+ audio, absolute_start = await self._decode_stream_window(frames)
+ stable_end = max(0, audio.size - self._guard_samples)
+ begin = max(0, emitted_samples - absolute_start)
+ if stable_end > begin:
+ chunk = np.ascontiguousarray(audio[begin:stable_end])
+ emitted_samples = absolute_start + stable_end
+ await output_queue.put(self._audio_payload(chunk))
+
+ if not frames:
+ raise ValueError("Audio8 generation produced no codec frames")
+
+ tail, absolute_start = await self._decode_stream_window(frames)
+ begin = max(0, emitted_samples - absolute_start)
+ if begin < tail.size:
+ chunk = np.ascontiguousarray(tail[begin:])
+ emitted_samples = absolute_start + tail.size
+ await output_queue.put(self._audio_payload(chunk))
+ await output_queue.put(None)
+
+ full_audio = await self._decode_frames(frames, 0, len(frames))
+ payload.data = self._audio_payload(full_audio)
+ prompt_tokens = len(state.input_ids) if state.input_ids is not None else 0
+ payload.data["usage"] = {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": len(frames),
+ "total_tokens": prompt_tokens + len(frames),
+ }
+ return payload
+ finally:
+ self._output_queues.pop(request_id, None)
+
+ async def _decode_stream_window(
+ self, frames: list[torch.Tensor]
+ ) -> tuple[np.ndarray, int]:
+ end = len(frames)
+ start = max(0, end - self._context_frames - self._chunk_frames)
+ return await self._decode_frames(frames, start, end), start * self._hop_length
+
+ async def _decode_frames(
+ self,
+ frames: list[torch.Tensor],
+ start: int,
+ end: int,
+ ) -> np.ndarray:
+ loop = asyncio.get_running_loop()
+ async with self._gpu_lock:
+ return await loop.run_in_executor(
+ None,
+ self._decode_frames_sync,
+ frames,
+ start,
+ end,
+ )
+
+ def _decode_frames_sync(
+ self,
+ frames: list[torch.Tensor],
+ start: int,
+ end: int,
+ ) -> np.ndarray:
+ torch.cuda.set_device(self._device)
+ codes = torch.stack(frames[start:end], dim=1).unsqueeze(0)
+ with torch.inference_mode():
+ audio = self._codec.decode(codes)[0, 0]
+ return audio.detach().float().cpu().numpy().copy()
+
+ def _audio_payload(self, audio: np.ndarray) -> dict[str, Any]:
+ value = audio.astype(np.float32, copy=False)
+ return {
+ "audio_waveform": value.tobytes(),
+ "audio_waveform_shape": list(value.shape),
+ "audio_waveform_dtype": "float32",
+ "sample_rate": self._sample_rate,
+ "modality": "audio",
+ }
diff --git a/sglang_omni/configs/audio8_tts_0_6b.yaml b/sglang_omni/configs/audio8_tts_0_6b.yaml
index 6d67b19..7fe38be 100644
--- a/sglang_omni/configs/audio8_tts_0_6b.yaml
+++ b/sglang_omni/configs/audio8_tts_0_6b.yaml
@@ -24,6 +24,8 @@ stages:
get_next: sglang_omni.models.audio8_tts.pipeline.next_stage.tts_engine_next
relay:
device: cuda
+ stream_to:
+ - to_stage: vocoder
- name: vocoder
num_workers: 1
executor:
diff --git a/sglang_omni/scripts/stream_smoke_test.py b/sglang_omni/scripts/stream_smoke_test.py
new file mode 100755
index 0000000..9bcf91b
--- /dev/null
+++ b/sglang_omni/scripts/stream_smoke_test.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import time
+import urllib.request
+import wave
+from pathlib import Path
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--base-url", default="http://127.0.0.1:8010")
+ parser.add_argument("--output", type=Path, default=Path("/tmp/audio8_stream.wav"))
+ parser.add_argument(
+ "--text",
+ default="Streaming speech should begin playing before the complete sentence is ready.",
+ )
+ args = parser.parse_args()
+
+ body = json.dumps(
+ {
+ "model": "audio8/tts-0.6b",
+ "input": args.text,
+ "response_format": "pcm",
+ "stream": True,
+ "max_new_tokens": 256,
+ "temperature": 0.7,
+ "top_p": 0.8,
+ "top_k": 50,
+ }
+ ).encode("utf-8")
+ request = urllib.request.Request(
+ f"{args.base_url}/v1/audio/speech",
+ data=body,
+ headers={"Content-Type": "application/json"},
+ method="POST",
+ )
+
+ started = time.perf_counter()
+ first_chunk_seconds: float | None = None
+ sample_rate: int | None = None
+ chunks: list[bytes] = []
+ with urllib.request.urlopen(request, timeout=300) as response:
+ for raw_line in response:
+ line = raw_line.decode("utf-8").strip()
+ if not line.startswith("data: "):
+ continue
+ value = line[6:]
+ if value == "[DONE]":
+ break
+ event = json.loads(value)
+ audio = event.get("audio")
+ if not audio:
+ continue
+ if audio.get("format") != "pcm":
+ raise RuntimeError(f"Expected PCM stream, got {audio.get('format')}")
+ if first_chunk_seconds is None:
+ first_chunk_seconds = time.perf_counter() - started
+ sample_rate = int(audio["sample_rate"])
+ chunks.append(base64.b64decode(audio["data"]))
+
+ total_seconds = time.perf_counter() - started
+ if not chunks or sample_rate is None or first_chunk_seconds is None:
+ raise RuntimeError("The server returned no streaming audio chunks")
+ pcm = b"".join(chunks)
+ if len(pcm) % 2:
+ raise RuntimeError("PCM stream has an odd byte count")
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ with wave.open(str(args.output), "wb") as output:
+ output.setnchannels(1)
+ output.setsampwidth(2)
+ output.setframerate(sample_rate)
+ output.writeframes(pcm)
+
+ print(f"chunks={len(chunks)}")
+ print(f"first_chunk_seconds={first_chunk_seconds:.3f}")
+ print(f"total_seconds={total_seconds:.3f}")
+ print(f"sample_rate={sample_rate}")
+ print(f"audio_seconds={len(pcm) / 2 / sample_rate:.3f}")
+ print(f"output={args.output}")
+
+
+if __name__ == "__main__":
+ main()