Skip to content

The open-source AI voice studio. Clone, dictate, create.

fix(api): pcm at 24 kHz, reject unavailable formats before synthesis, declare openai test dep

jamiepine38416b89684f
diff --git a/backend/routes/openai_compat.py b/backend/routes/openai_compat.py
index 449fbbb2..f8e69c9c 100644
--- a/backend/routes/openai_compat.py
+++ b/backend/routes/openai_compat.py
@@ -20,7 +20,7 @@ Request field mapping
The text to speak, in the profile's language.
``response_format``
``wav`` (default), ``mp3``, ``flac``, ``opus``, ``pcm`` (raw 16-bit
- little-endian mono at the engine's sample rate), ``aac`` (needs ffmpeg).
+ little-endian mono at 24 kHz, as OpenAI defines it), ``aac`` (needs ffmpeg).
``speed``
0.25-4.0, applied as a pitch-preserving time stretch after synthesis.
``instructions``
@@ -66,6 +66,7 @@ _FFMPEG_FORMATS: dict[str, tuple[list[str], str]] = {
"aac": (["-c:a", "aac", "-b:a", "128k", "-f", "adts"], "audio/aac"),
}
RESPONSE_FORMATS = (*_SOUNDFILE_FORMATS, "pcm", *_FFMPEG_FORMATS)
+PCM_SAMPLE_RATE = 24000
# Voicebox launched long after the OpenAI `created` epoch convention; a fixed
# value keeps the field deterministic for clients that display it.
@@ -115,7 +116,9 @@ class SpeechRequest(BaseModel):
def _available_formats() -> list[str]:
- formats = [f for f in RESPONSE_FORMATS if f not in _FFMPEG_FORMATS]
+ """Formats this server can actually produce, given its libsndfile build and PATH."""
+ formats = ["pcm"]
+ formats.extend(f for f, (fmt, _, _) in _SOUNDFILE_FORMATS.items() if fmt in sf.available_formats())
if shutil.which("ffmpeg"):
formats.extend(_FFMPEG_FORMATS)
return formats
@@ -195,6 +198,12 @@ def encode_audio(audio: np.ndarray, sample_rate: int, response_format: str) -> t
import io
if response_format == "pcm":
+ # OpenAI defines pcm as headerless 16-bit mono at 24 kHz; the body
+ # carries no rate, so engines at another rate (LuxTTS: 48 kHz) must be resampled.
+ if sample_rate != PCM_SAMPLE_RATE:
+ import librosa
+
+ audio = librosa.resample(audio.astype(np.float32), orig_sr=sample_rate, target_sr=PCM_SAMPLE_RATE)
pcm = np.clip(audio, -1.0, 1.0)
return (pcm * 32767).astype("<i2").tobytes(), "audio/pcm"
@@ -300,10 +309,11 @@ async def _create_speech(request: Request, db: Session) -> Response:
param = ".".join(str(p) for p in first.get("loc", ())) or None
raise OpenAIError(400, f"Invalid value for '{param}': {first['msg']}", param=param) from exc
- if data.response_format not in RESPONSE_FORMATS:
+ if data.response_format not in _available_formats():
raise OpenAIError(
400,
- f"Invalid response_format '{data.response_format}'. Supported: {', '.join(RESPONSE_FORMATS)}.",
+ f"Invalid response_format '{data.response_format}'. Supported on this server: "
+ f"{', '.join(_available_formats())}.",
param="response_format",
code="unsupported_format",
)
diff --git a/backend/tests/test_openai_compat.py b/backend/tests/test_openai_compat.py
index 96896568..847a529c 100644
--- a/backend/tests/test_openai_compat.py
+++ b/backend/tests/test_openai_compat.py
@@ -15,7 +15,6 @@ import numpy as np
import pytest
import soundfile as sf
from fastapi import FastAPI
-from openai import AsyncOpenAI, BadRequestError, NotFoundError
import backend.backends as backends
import backend.mcp_server.resolve as resolve_mod
@@ -25,6 +24,9 @@ import backend.utils.chunked_tts as chunked_tts
from backend.database import get_db
from backend.routes import openai_compat
+openai = pytest.importorskip("openai", reason="the openai SDK is a test-only dependency (installed by `just setup`)")
+AsyncOpenAI, BadRequestError, NotFoundError = openai.AsyncOpenAI, openai.BadRequestError, openai.NotFoundError
+
SAMPLE_RATE = 24000
DURATION_S = 1.0
@@ -298,6 +300,23 @@ async def test_preset_profile_rejects_other_engine(raw_client, monkeypatch):
# ─── encoder unit coverage ────────────────────────────────────────────────
+def test_encode_audio_pcm_resamples_to_24k():
+ content, media_type = openai_compat.encode_audio(np.zeros(48000, dtype=np.float32), 48000, "pcm")
+ assert media_type == "audio/pcm"
+ assert len(content) == 24000 * 2
+
+
+async def test_aac_without_ffmpeg_is_rejected_before_synthesis(raw_client, app, monkeypatch):
+ monkeypatch.setattr(openai_compat.shutil, "which", lambda name: None)
+ response = await raw_client.post(
+ "/v1/audio/speech",
+ json={"model": "tts-1", "voice": "Morgan", "input": "Hi", "response_format": "aac"},
+ )
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "unsupported_format"
+ assert "text" not in app.state.calls
+
+
def test_encode_audio_wav_roundtrip():
audio = np.zeros(2400, dtype=np.float32)
content, media_type = openai_compat.encode_audio(audio, 24000, "wav")
diff --git a/docs/content/docs/overview/mcp-server.mdx b/docs/content/docs/overview/mcp-server.mdx
index 12d07a68..4acdeee0 100644
--- a/docs/content/docs/overview/mcp-server.mdx
+++ b/docs/content/docs/overview/mcp-server.mdx
@@ -267,7 +267,7 @@ How the OpenAI fields map onto Voicebox:
| `voice` | A voice profile, by name (case-insensitive) or id. Omit it and the [voice resolution](#voice-resolution) chain applies: the `X-Voicebox-Client-Id` binding, then the default playback voice. There are no built-in `alloy`/`nova` voices — create a profile (a Kokoro preset profile named "Alloy" works fine). |
| `model` | A TTS engine (`qwen`, `qwen_custom_voice`, `luxtts`, `chatterbox`, `chatterbox_turbo`, `tada`, `kokoro`) or a model variant such as `qwen-tts-0.6B`. OpenAI's own ids `tts-1`, `tts-1-hd` and `gpt-4o-mini-tts` are accepted as aliases for "the engine configured on the profile", so unchanged client code works. `GET /v1/models` lists every accepted id. The model must already be downloaded (Voicebox → Models). |
| `input` | The text to speak, up to 10,000 characters, in the profile's language. Long text is chunked at sentence boundaries like the rest of Voicebox. |
-| `response_format` | `wav` (default), `mp3`, `flac`, `opus`, `pcm` (raw 16-bit little-endian mono at the engine's sample rate, 24 kHz for most engines) and `aac` (requires `ffmpeg` on the server's PATH). |
+| `response_format` | `wav` (default), `mp3`, `flac`, `opus`, `pcm` (raw 16-bit little-endian mono at 24 kHz, as OpenAI defines it; engines running at another rate are resampled) and `aac` (requires `ffmpeg` on the server's PATH). |
| `speed` | `0.25`–`4.0`. Applied as a pitch-preserving time stretch after synthesis. |
| `instructions` | Forwarded as the engine's instruct prompt where supported (Qwen). |
diff --git a/justfile b/justfile
index 6c87aa84..cadf100f 100644
--- a/justfile
+++ b/justfile
@@ -134,7 +134,7 @@ setup-python:
"{{ python }}" -m pip install --no-deps mlx-audio==0.4.1
fi
"{{ python }}" -m pip install git+https://github.com/QwenLM/Qwen3-TTS.git
- "{{ python }}" -m pip install pyinstaller ruff pytest pytest-asyncio -q
+ "{{ python }}" -m pip install pyinstaller ruff pytest pytest-asyncio openai -q
echo "Python environment ready."
[windows]
@@ -208,7 +208,7 @@ setup-python:
Invoke-Pip install --no-deps chatterbox-tts; \
Invoke-Pip install --no-deps hume-tada; \
Invoke-Pip install git+https://github.com/QwenLM/Qwen3-TTS.git; \
- Invoke-Pip install pyinstaller ruff pytest pytest-asyncio -q; \
+ Invoke-Pip install pyinstaller ruff pytest pytest-asyncio openai -q; \
Write-Host "Python environment ready."
# Install JavaScript dependencies