Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: speak through a saved connection on the Audio page (#14288)

Nilayb4f3e06940b8
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 7a4290dcc3f..6a0be53280c 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -6303,6 +6303,14 @@ class AudioRunRequest(BaseModel):
pattern = r"^[A-Za-z0-9_-]+$",
description = "Your id for this run; POST it to /audio/cancel to stop the run",
)
+ provider_id: Optional[str] = Field(
+ None,
+ description = "[x-unsloth] Saved connection ID. When set, Speak sends the text to that "
+ "connection's /audio/speech as `model` and the clip is saved to history.",
+ )
+ voice: Optional[str] = Field(
+ None, max_length = 256, description = "Voice on the connection named by `provider_id`."
+ )
@model_validator(mode = "after")
def _take_fields(self):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 944e0e8c055..afb768f4de7 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -25465,6 +25465,8 @@ def _discard_audio_clips(outputs: list[str], sources: list[str]) -> None:
async def _dispatch_audio_workflow(
body: AudioRunRequest, request: Request, current_subject: str
) -> AudioRunResponse:
+ if body.provider_id:
+ return await _speak_on_connection(body, request)
if body.workflow == "separate":
return await _run_separation(body, request, current_subject)
if body.workflow == "convert":
@@ -25958,6 +25960,64 @@ async def _external_tts_speech(body: AudioSpeechRequest, request: Request) -> Re
return Response(content = audio_bytes, media_type = media_type)
+async def _speak_on_connection(body: AudioRunRequest, request: Request) -> AudioRunResponse:
+ import wave
+
+ if (
+ body.workflow != "speak"
+ or body.inputs != AudioRunInputs()
+ or body.language
+ or body.options
+ or body.seed is not None
+ or body.max_tokens is not None
+ or any(
+ name in body.model_fields_set and getattr(body, name) not in (None, False, 1)
+ for name in _MUSIC_ONLY_FIELDS
+ )
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = "A saved connection only speaks text, in a voice, with instructions and speed.",
+ )
+ speech = await _external_tts_speech(
+ AudioSpeechRequest(
+ input = body.text,
+ model = body.model,
+ voice = body.voice,
+ speed = body.speed,
+ instructions = body.instructions,
+ provider_id = body.provider_id,
+ ),
+ request,
+ )
+ try:
+ with wave.open(io.BytesIO(speech.body)) as wav:
+ params = wav.getparams()
+ frames = wav.readframes(wav.getnframes())
+ except (wave.Error, EOFError):
+ raise HTTPException(
+ status_code = 502, detail = "The TTS endpoint did not return WAV audio."
+ ) from None
+ # OpenAI streams WAV with 0xFFFFFFFF sizes; rewrite the header from the frames received.
+ out = io.BytesIO()
+ with wave.open(out, "wb") as wav:
+ wav.setparams(params._replace(nframes = 0))
+ wav.writeframes(frames)
+ wav_bytes, sample_rate = out.getvalue(), params.framerate
+ settings = {"voice": body.voice, "instructions": body.instructions, "speed": body.speed}
+ record = await asyncio.to_thread(
+ _persist_tts_clip,
+ wav_bytes,
+ sample_rate,
+ body.text,
+ body.model,
+ "connection",
+ {"settings": settings},
+ "speak",
+ )
+ return _audio_run_response(record, wav_bytes, sample_rate, body.model)
+
+
# openai-compatible speech api: the router's dual mount also answers /v1/audio/speech
@router.post("/audio/speech")
async def openai_audio_speech(
diff --git a/studio/backend/tests/test_audio_run_route.py b/studio/backend/tests/test_audio_run_route.py
index 14da336bf27..ab6243de7c5 100644
--- a/studio/backend/tests/test_audio_run_route.py
+++ b/studio/backend/tests/test_audio_run_route.py
@@ -23,7 +23,7 @@ from fastapi.testclient import TestClient
from auth.authentication import authenticated_via_api_key, get_current_subject
from core.inference import audio_gallery, audio_inputs, audio_voices
from routes import inference
-from utils.account_context import bind_account, reset_account, run_as
+from utils.account_context import OWNER, bind_account, reset_account, run_as
_TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
@@ -2206,3 +2206,133 @@ def test_take_or_group_rejected_outside_single_variation_music(stub, body):
with _client(ALICE) as client:
response = client.post("/api/inference/audio/run", json = {"text": "x", **body})
assert response.status_code == 422, response.text
+
+
+def _connection(monkeypatch, handler):
+ import httpx
+
+ import core.inference.external_provider as external_provider
+
+ monkeypatch.setattr(
+ external_provider,
+ "_http_client",
+ httpx.AsyncClient(transport = httpx.MockTransport(handler)),
+ )
+ monkeypatch.setattr(
+ inference.providers_db,
+ "get_provider",
+ lambda pid: (
+ {
+ "provider_type": "custom",
+ "display_name": "Kokoro Box",
+ "base_url": "http://127.0.0.1:8880/v1",
+ "is_enabled": True,
+ }
+ if pid == "conn-1"
+ else None
+ ),
+ )
+ monkeypatch.setattr(inference, "resolve_provider_api_key_or_400", lambda *a, **k: "sk-test")
+
+
+def _speak_on_connection(client, **body):
+ payload = {
+ "workflow": "speak",
+ "text": "Hello there.",
+ "provider_id": "conn-1",
+ "model": "kokoro",
+ "voice": "af_heart",
+ **body,
+ }
+ return client.post("/api/inference/audio/run", json = payload)
+
+
+def test_speak_through_a_connection_lands_in_history(stub, monkeypatch):
+ import httpx
+
+ sent = []
+
+ def handler(req):
+ sent.append((str(req.url), req.headers.get("authorization"), json.loads(req.content)))
+ return httpx.Response(200, content = _wav(rate = 22050), headers = {"content-type": "audio/wav"})
+
+ _connection(monkeypatch, handler)
+ with _client(OWNER) as client:
+ response = _speak_on_connection(client, instructions = "Warmly.", speed = 1.25)
+ assert response.status_code == 200, response.text
+ body = response.json()
+ (clip,) = body["clips"]
+ assert client.get(clip["url"]).status_code == 200
+ listed = client.get("/api/inference/audio/gallery").json()["audio"]
+ assert stub["backend"].calls == []
+ assert sent == [
+ (
+ "http://127.0.0.1:8880/v1/audio/speech",
+ "Bearer sk-test",
+ {
+ "model": "kokoro",
+ "input": "Hello there.",
+ "response_format": "wav",
+ "voice": "af_heart",
+ "speed": 1.25,
+ "instructions": "Warmly.",
+ },
+ )
+ ]
+ assert (body["model"], body["audio"]) == ("kokoro", None)
+ assert (clip["role"], clip["workflow"], clip["sample_rate"]) == ("output", "speak", 22050)
+ assert [(c["id"], c["prompt"], c["model"], c["workflow"]) for c in listed] == [
+ (clip["id"], "Hello there.", "kokoro", "speak")
+ ]
+ sidecar = run_as(OWNER, audio_gallery.gallery_dir) / f"{clip['id']}.json"
+ meta = json.loads(sidecar.read_text(encoding = "utf-8"))
+ assert meta["settings"] == {"voice": "af_heart", "instructions": "Warmly.", "speed": 1.25}
+
+
+ "extra",
+ [
+ {"seed": 7},
+ {"options": {"temperature": 0.5}},
+ {"inputs": {"reference": {"voice_id": "abc"}}},
+ {"workflow": "clone"},
+ ],
+)
+def test_speak_through_a_connection_refuses_settings_it_cannot_send(stub, monkeypatch, extra):
+ sent = []
+ _connection(monkeypatch, lambda req: sent.append(req))
+ with _client(OWNER) as client:
+ response = _speak_on_connection(client, **extra)
+ assert response.status_code == 400, response.text
+ assert sent == [] and stub["backend"].calls == []
+
+
+def test_speak_through_a_connection_reports_the_upstream_error(stub, monkeypatch):
+ import httpx
+
+ _connection(monkeypatch, lambda req: httpx.Response(404, text = "model 'kokoro' not found"))
+ with _client(OWNER) as client:
+ response = _speak_on_connection(client)
+ listed = client.get("/api/inference/audio/gallery").json()["audio"]
+ assert response.status_code == 502
+ assert "model 'kokoro' not found" in response.text
+ assert listed == []
+
+
+def test_speak_through_a_connection_saves_the_real_length_of_a_streamed_wav(stub, monkeypatch):
+ import httpx
+
+ wav = bytearray(_wav(rate = 24000, frames = 24000))
+ wav[4:8] = wav[40:44] = b"\xff\xff\xff\xff"
+ _connection(
+ monkeypatch,
+ lambda req: httpx.Response(200, content = bytes(wav), headers = {"content-type": "audio/wav"}),
+ )
+ with _client(OWNER) as client:
+ response = _speak_on_connection(client)
+ assert response.status_code == 200, response.text
+ (clip,) = response.json()["clips"]
+ listed = client.get("/api/inference/audio/gallery").json()["audio"]
+ assert [c["duration_s"] for c in listed] == [1.0]
+ sidecar = run_as(OWNER, audio_gallery.gallery_dir) / f"{clip['id']}.json"
+ assert json.loads(sidecar.read_text(encoding = "utf-8"))["duration_s"] == 1.0
diff --git a/studio/frontend/src/features/audio/api.ts b/studio/frontend/src/features/audio/api.ts
index 8b26a42100b..58d52f28c34 100644
--- a/studio/frontend/src/features/audio/api.ts
+++ b/studio/frontend/src/features/audio/api.ts
@@ -543,6 +543,34 @@ export async function runAudio(
return parseAudioJson<AudioRunResponse>(response);
}
+export async function runSpeechOnConnection(
+ request: {
+ text: string;
+ providerId: string;
+ model: string;
+ voice: string;
+ instructions: string | null;
+ speed: number;
+ },
+ signal?: AbortSignal,
+): Promise<AudioRunResponse> {
+ const response = await authFetch("/api/inference/audio/run", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ workflow: "speak",
+ text: request.text,
+ provider_id: request.providerId,
+ model: request.model,
+ voice: request.voice,
+ speed: request.speed,
+ ...(request.instructions ? { instructions: request.instructions } : {}),
+ }),
+ signal,
+ });
+ return parseAudioJson<AudioRunResponse>(response);
+}
+
export interface AudioVoice {
id: string;
name: string;
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index 4ff62cc7fb0..484d08220aa 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -87,6 +87,7 @@ import { useCloneGeneration } from "./hooks/use-clone-generation";
import { useConvertGeneration } from "./hooks/use-convert-generation";
import { adoptEditSource, useEditGeneration } from "./hooks/use-edit-generation";
import { useSeparateGeneration } from "./hooks/use-separate-generation";
+import { useSpeakConnection } from "./hooks/use-speak-connection";
import { useSpeechGeneration } from "./hooks/use-speech-generation";
import { useSttSidecar } from "./hooks/use-stt-sidecar";
import { useTranscription } from "./hooks/use-transcription";
@@ -105,7 +106,11 @@ import {
SeparateOutput,
SeparateRail,
} from "./pages/separate-page";
-import { SpeakOutput, SpeakRail } from "./pages/speak-page";
+import {
+ SpeakConnectionRail,
+ SpeakOutput,
+ SpeakRail,
+} from "./pages/speak-page";
import { type GenerateBlocker, TtsFooter } from "./pages/tts-workspace";
import { TranscribeFooter, TranscribeOutput, TranscribeRail } from "./pages/transcribe-page";
import type { SendTarget } from "./components/stem-mixer-types";
@@ -512,6 +517,7 @@ export function AudioPage({
reloadOnCpu,
setLoadFailureHandlers,
handleEject,
+ invalidatePendingTtsSelection,
trainedTtsModels,
} = useAudioModelSlot({
active,
@@ -563,6 +569,12 @@ export function AudioPage({
if (busy === "generating") setAnnouncement("");
}, [busy]);
+ const speakConnection = useSpeakConnection();
+ const connection =
+ mode === "speak" && ttsWorkflow === "speak"
+ ? speakConnection.connection
+ : null;
+
const {
prompt,
setPrompt,
@@ -617,6 +629,7 @@ export function AudioPage({
setSelectedId,
pendingTranscribeRelease,
replayQueuedTtsPick,
+ connection,
});
const music = useMusicGeneration({
@@ -1028,12 +1041,13 @@ export function AudioPage({
);
const pageModelLoaded =
- ttsLoaded &&
- loadedModelRunsWorkflow({
- workflow: ttsWorkflow,
- audioWorkflows: status?.audio_workflows,
- music: musicGeneration,
- });
+ connection !== null ||
+ (ttsLoaded &&
+ loadedModelRunsWorkflow({
+ workflow: ttsWorkflow,
+ audioWorkflows: status?.audio_workflows,
+ music: musicGeneration,
+ }));
const rememberedModels = useAudioWorkspaceStore(
(state) => state.lastModelByWorkflow,
);
@@ -1165,6 +1179,10 @@ export function AudioPage({
? { reason: "Waiting for the model to finish loading." }
: busy !== null
? null
+ : connection
+ ? prompt.trim()
+ ? null
+ : { reason: "Type the text to speak." }
: !ttsLoaded
? {
reason:
@@ -1639,10 +1657,25 @@ export function AudioPage({
(row: PageModelRow) => rowOnPage(pageWorkflow, row, pagePrefs),
[pageWorkflow, pagePrefs],
);
+ const setSpeakConnectionId = speakConnection.setModelId;
const selectModel = useCallback(
- (id: string, meta: Parameters<typeof handleModelSelect>[1]) =>
- handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta)),
- [audioCatalog.byId, handleModelSelect],
+ (id: string, meta: Parameters<typeof handleModelSelect>[1]) => {
+ if (meta.source === "external") {
+ if (busyRef.current === null) {
+ invalidatePendingTtsSelection();
+ setSpeakConnectionId(id);
+ }
+ return;
+ }
+ if (modeRef.current === "speak") setSpeakConnectionId(null);
+ handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta));
+ },
+ [
+ audioCatalog.byId,
+ handleModelSelect,
+ invalidatePendingTtsSelection,
+ setSpeakConnectionId,
+ ],
);
const sttOnDeviceModels = downloadedSttArtifacts.map((artifact) => {
const catalogModel = MODELS_BY_MODE.transcribe.find(
@@ -1682,7 +1715,9 @@ export function AudioPage({
const capabilityLine =
mode === "speak"
- ? showLastPageModel
+ ? connection
+ ? `Speaks through ${connection.providerName}, a saved connection.`
+ : showLastPageModel
? `${lastPageModel.split("/").pop()} is not loaded. Pick it above to load it.`
: ttsLoaded
? pageModelLoaded
@@ -1794,6 +1829,11 @@ export function AudioPage({
(ttsWorkflow === "music"),
)
}
+ externalModels={
+ mode === "speak" && ttsWorkflow === "speak"
+ ? speakConnection.models
+ : undefined
+ }
rowFilter={selectorRowFilter}
moreModels={selectorMoreModels}
loadedModelIdOverride={
@@ -1813,9 +1853,10 @@ export function AudioPage({
? false
: undefined
}
- value={selectorValue}
+ value={(connection && speakConnection.modelId) || selectorValue}
onValueChange={selectModel}
onEject={
+ !connection &&
audioEjectAvailable({
mode,
busy,
@@ -1884,10 +1925,12 @@ export function AudioPage({
</div>
<div className="pointer-events-none col-start-3 flex min-w-0 items-start justify-end pr-2 pt-[var(--studio-chat-header-padding-top,11px)]">
<div className="pointer-events-auto flex min-w-0 items-center gap-2">
- <UseViaApiButton
- workflow={pageWorkflow}
- model={selectorValue ?? null}
- />
+ {connection ? null : (
+ <UseViaApiButton
+ workflow={pageWorkflow}
+ model={selectorValue ?? null}
+ />
+ )}
<LibraryPageLink
tab="audio"
labelClassName="hidden @[50rem]:inline"
@@ -2020,6 +2063,17 @@ export function AudioPage({
historyClips={clips}
/>
</AudioActiveProvider>
+ ) : connection ? (
+ <SpeakConnectionRail
+ connection={connection}
+ prompt={prompt}
+ setPrompt={setPrompt}
+ instructions={audioInstructions}
+ setInstructions={setAudioInstructions}
+ setVoice={speakConnection.setVoice}
+ setSpeed={speakConnection.setSpeed}
+ disabled={busy === "generating"}
+ />
) : (
<SpeakRail {...railProps} />
);
@@ -2110,7 +2164,9 @@ export function AudioPage({
ttsLoaded={pageModelLoaded}
prompt={prompt}
lyricsOptional={musicStudio || lyricsOptional}
- musicNeedsDescription={!musicStudio && musicNeedsDescription}
+ musicNeedsDescription={
+ !musicStudio && !connection && musicNeedsDescription
+ }
audioInstructions={audioInstructions}
blocker={generateBlocker}
shortcutLabel={shortcutLabel}
diff --git a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts
index 80851a39774..a3395d4ce0d 100644
--- a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts
+++ b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts
@@ -1282,6 +1282,7 @@ export function useAudioModelSlot({
reloadOnCpu,
setLoadFailureHandlers,
handleEject,
+ invalidatePendingTtsSelection,
trainedTtsModels,
};
}
diff --git a/studio/frontend/src/features/audio/hooks/use-speak-connection.ts b/studio/frontend/src/features/audio/hooks/use-speak-connection.ts
new file mode 100644
index 00000000000..972d5486c6b
--- /dev/null
+++ b/studio/frontend/src/features/audio/hooks/use-speak-connection.ts
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useMemo, useState } from "react";
+import {
+ buildExternalModelId,
+ isDecisionConnection,
+ useExternalProvidersStore,
+} from "@/features/chat";
+import type { ExternalModelOption } from "@/features/model-picker/components/model-selector/types";
+import { connectionSpeechModels } from "../speak-connection-models";
+
+export interface SpeakConnection {
+ providerId: string;
+ model: string;
+ providerName: string;
+ voice: string;
+ speed: number;
+}
+
+export function useSpeakConnection() {
+ const connections = useExternalProvidersStore((s) => s.providers);
+ const connectionsEnabled = useExternalProvidersStore(
+ (s) => s.connectionsEnabled,
+ );
+ const models = useMemo<ExternalModelOption[]>(
+ () =>
+ connectionsEnabled
+ ? connections
+ .filter((c) => !isDecisionConnection(c))
+ .flatMap((c) =>
+ connectionSpeechModels(c).map((model) => ({
+ id: buildExternalModelId(c.id, model),
+ name: model,
+ providerId: c.id,
+ providerName: c.name,
+ providerType: c.providerType,
+ })),
+ )
+ : [],
+ [connections, connectionsEnabled],
+ );
+ const [modelId, setModelId] = useState<string | null>(null);
+ const [voice, setVoice] = useState("");
+ const [speed, setSpeed] = useState(1);
+ const connection = useMemo<SpeakConnection | null>(() => {
+ const option = models.find((model) => model.id === modelId);
+ return option
+ ? {
+ providerId: option.providerId,
+ model: option.name,
+ providerName: option.providerName,
+ voice,
+ speed,
+ }
+ : null;
+ }, [models, modelId, voice, speed]);
+ return { models, modelId, setModelId, connection, setVoice, setSpeed };
+}
diff --git a/studio/frontend/src/features/audio/hooks/use-speech-generation.ts b/studio/frontend/src/features/audio/hooks/use-speech-generation.ts
index fdf0fa6c9cd..10931c1b1d7 100644
--- a/studio/frontend/src/features/audio/hooks/use-speech-generation.ts
+++ b/studio/frontend/src/features/audio/hooks/use-speech-generation.ts
@@ -4,7 +4,7 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import { readLastPrompt, saveLastPrompt } from "@/lib/last-prompt";
import { toast } from "@/lib/toast";
-import { generateAudio, runAudio } from "../api";
+import { generateAudio, runAudio, runSpeechOnConnection } from "../api";
import { TTS_MAX_TOKENS } from "../audio-workspace-constants";
import {
audioOptionLabel,
@@ -58,6 +58,7 @@ import type { AudioWorkflowId } from "../workflows";
import { reportAudioFailure } from "../audio-failure-actions";
import type { AudioGallery } from "./use-audio-gallery";
import type { AudioModelSlot } from "./use-audio-model-slot";
+import type { SpeakConnection } from "./use-speak-connection";
type TtsWorkflow = "speak" | "music";
type TtsDraftField = "prompt" | "instructions";
@@ -90,7 +91,8 @@ export function useSpeechGeneration({
setSelectedId,
pendingTranscribeRelease,
replayQueuedTtsPick,
-}: { workflow: TtsWorkflow } & Pick<
+ connection,
+}: { workflow: TtsWorkflow; connection: SpeakConnection | null } & Pick<
AudioHostState,
| "status"
| "busyRef"
@@ -339,7 +341,7 @@ export function useSpeechGeneration({
return;
}
const instructions = audioInstructions.trim();
- if (musicNeedsDescription && !instructions) {
+ if (!connection && musicNeedsDescription && !instructions) {
updateGenerationPhase(null);
busyRef.current = null;
setBusy(null);
@@ -348,17 +350,16 @@ export function useSpeechGeneration({
);
return;
}
- if (toolRequest.error) {
+ if (!connection && toolRequest.error) {
updateGenerationPhase(null);
busyRef.current = null;
setBusy(null);
toast.error(toolRequest.error);
return;
}
- const missingOptions = missingRequiredAudioOptions(
- advancedOptionSpecs,
- audioOptionValues,
- );
+ const missingOptions = connection
+ ? []
+ : missingRequiredAudioOptions(advancedOptionSpecs, audioOptionValues);
if (missingOptions.length > 0) {
updateGenerationPhase(null);
busyRef.current = null;
@@ -380,24 +381,38 @@ export function useSpeechGeneration({
toolRequest.patch.language ?? (mossLocalGeneration ? audioLanguage : "")
).trim();
saveLastPrompt(ttsDraftKey("prompt", workflow), prompt);
- if (savedVoice) {
+ if (savedVoice || connection) {
const voiceController = new AbortController();
generateAbort.current = voiceController;
updateGenerationPhase("generating");
try {
- const response = await runAudio(
- {
- workflow: "speak",
- text,
- instructions:
- instructionsKind !== null && instructions ? instructions : null,
- language: language || null,
- inputs: { reference: savedVoice },
- options: requestOptions,
- speed: toolRequest.patch.speed ?? null,
- },
- voiceController.signal,
- );
+ const response = connection
+ ? await runSpeechOnConnection(
+ {
+ text,
+ providerId: connection.providerId,
+ model: connection.model,
+ voice: connection.voice.trim() || "alloy",
+ instructions: instructions || null,
+ speed: connection.speed,
+ },
+ voiceController.signal,
+ )
+ : await runAudio(
+ {
+ workflow: "speak",
+ text,
+ instructions:
+ instructionsKind !== null && instructions
+ ? instructions
+ : null,
+ language: language || null,
+ inputs: { reference: savedVoice },
+ options: requestOptions,
+ speed: toolRequest.patch.speed ?? null,
+ },
+ voiceController.signal,
+ );
updateGenerationPhase("finishing");
await showRunResult({
response,
@@ -533,6 +548,7 @@ export function useSpeechGeneration({
refreshStatus,
replayQueuedTtsPick,
selectClip,
+ connection,
]);
// Only unmount aborts: RootLayout keeps this page mounted so leaving the tab does not cancel synthesis.
diff --git a/studio/frontend/src/features/audio/pages/speak-page.tsx b/studio/frontend/src/features/audio/pages/speak-page.tsx
index 78d28cbee17..f071fa80419 100644
--- a/studio/frontend/src/features/audio/pages/speak-page.tsx
+++ b/studio/frontend/src/features/audio/pages/speak-page.tsx
@@ -2,6 +2,11 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import type { ComponentProps } from "react";
+import { Input } from "@/components/ui/input";
+import { Textarea } from "@/components/ui/textarea";
+import { ParamSlider } from "@/features/chat";
+import { Field } from "../components/field";
+import type { SpeakConnection } from "../hooks/use-speak-connection";
import { TtsOutput, TtsRailFields } from "./tts-workspace";
export function SpeakRail(
@@ -10,6 +15,79 @@ export function SpeakRail(
return <TtsRailFields {...props} musicGeneration={false} />;
}
+export function SpeakConnectionRail({
+ connection,
+ prompt,
+ setPrompt,
+ instructions,
+ setInstructions,
+ setVoice,
+ setSpeed,
+ disabled,
+}: {
+ connection: SpeakConnection;
+ prompt: string;
+ setPrompt: (value: string) => void;
+ instructions: string;
+ setInstructions: (value: string) => void;
+ setVoice: (value: string) => void;
+ setSpeed: (value: number) => void;
+ disabled: boolean;
+}) {
+ return (
+ <>
+ <Field
+ label="Text"
+ htmlFor="audio-prompt"
+ hint={`What ${connection.providerName} should say. The clip lands in the gallery.`}
+ >
+ <Textarea
+ data-type-to-activate="prompt"
+ id="audio-prompt"
+ value={prompt}
+ onChange={(event) => setPrompt(event.target.value)}
+ placeholder="Type the sentence to speak…"
+ className="min-h-28"
+ />
+ </Field>
+ <Field
+ label="Voice"
+ htmlFor="audio-connection-voice"
+ hint="A voice the connection offers. Empty uses alloy."
+ >
+ <Input
+ id="audio-connection-voice"
+ value={connection.voice}
+ onChange={(event) => setVoice(event.target.value)}
+ placeholder="alloy"
+ />
+ </Field>
+ <Field
+ label="Style instructions"
+ htmlFor="audio-instructions"
+ hint="Optional. Sent to models that take instructions; others ignore them."
+ >
+ <Textarea
+ id="audio-instructions"
+ value={instructions}
+ onChange={(event) => setInstructions(event.target.value)}
+ placeholder="Warm, measured delivery with a calm conversational tone…"
+ className="min-h-24"
+ />
+ </Field>
+ <ParamSlider
+ label="Speed"
+ value={connection.speed}
+ min={0.25}
+ max={4}
+ step={0.05}
+ onChange={setSpeed}
+ disabled={disabled}
+ />
+ </>
+ );
+}
+
export function SpeakOutput(
{
modelReady,
diff --git a/studio/frontend/src/features/audio/speak-connection-models.ts b/studio/frontend/src/features/audio/speak-connection-models.ts
new file mode 100644
index 00000000000..d723b72756f
--- /dev/null
+++ b/studio/frontend/src/features/audio/speak-connection-models.ts
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+const SELF_HOSTED_TYPES = new Set(["custom", "vllm", "llama_cpp", "lemonade"]);
+const OPENAI_SPEECH_MODELS = ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"];
+const SPEECH_MODEL_ID =
+ /kokoro|tts|speech|orpheus|piper|chatterbox|parler|zonos|cosyvoice|(?:^|[\W_])(?:f5|csm|dia|bark|melo|sesame)(?:[\W_\d]|$)/i;
+// A self-hosted server such as speaches lists its speech-to-text models beside its voices.
+const TRANSCRIBE_MODEL_ID =
+ /whisper|transcri|parakeet|(?:^|[\W_])(?:asr|stt)(?:[\W_]|$)/i;
+
+function isOpenAIApi(baseUrl: string | undefined): boolean {
+ if (!baseUrl) return true;
+ try {
+ return new URL(baseUrl).hostname.toLowerCase() === "api.openai.com";
+ } catch {
+ return false;
+ }
+}
+
+export function connectionSpeechModels(connection: {
+ providerType: string;
+ baseUrl?: string;
+ models: readonly string[];
+}): string[] {
+ if (connection.providerType === "openai" && isOpenAIApi(connection.baseUrl)) {
+ return OPENAI_SPEECH_MODELS;
+ }
+ // Azure and other OpenAI-typed endpoints name models by deployment.
+ return connection.providerType === "openai" ||
+ SELF_HOSTED_TYPES.has(connection.providerType)
+ ? connection.models.filter(
+ (id) => SPEECH_MODEL_ID.test(id) && !TRANSCRIBE_MODEL_ID.test(id),
+ )
+ : [];
+}
diff --git a/studio/frontend/tests/audio-model-eject.test.ts b/studio/frontend/tests/audio-model-eject.test.ts
index 08c94c73baf..17e4a9679d9 100644
--- a/studio/frontend/tests/audio-model-eject.test.ts
+++ b/studio/frontend/tests/audio-model-eject.test.ts
@@ -14,7 +14,7 @@ test("Audio exposes the shared picker eject action only while idle", () => {
// The selector keeps showing the last Transcribe model once it is unloaded; Eject there did nothing.
assert.match(
source,
- /onEject=\{\s*audioEjectAvailable\(\{\s*mode,\s*busy,\s*hasValue: Boolean\(selectorValue\),\s*showLastPageModel,\s*sttReady,\s*\}\)\s*\? handleEject\s*: undefined\s*\}/,
+ /onEject=\{\s*!connection &&\s*audioEjectAvailable\(\{\s*mode,\s*busy,\s*hasValue: Boolean\(selectorValue\),\s*showLastPageModel,\s*sttReady,\s*\}\)\s*\? handleEject\s*: undefined\s*\}/,
);
assert.match(source, /const handleEject = useCallback\(\(\) => \{\s*if \(busy !== null\) \{/);
assert.match(
@@ -285,3 +285,18 @@ test("leaving Audio during the residency read skips the transcription load", ()
/await readSttResidency\(engine\)[\s\S]{0,400}?\|\| !activeRef\.current\) return null;[\s\S]*?await ensureSttLoaded\(repo, model, engine\)/,
);
});
+
+test("Use via API is hidden while Speak runs through a connection", () => {
+ assert.match(source, /\{connection \? null : \(\s*<UseViaApiButton\s+workflow=\{pageWorkflow\}/);
+});
+
+test("picking a connection retires a local Speak pick still downloading or planning", () => {
+ assert.match(
+ source,
+ /if \(meta\.source === "external"\) \{\s*if \(busyRef\.current === null\) \{\s*invalidatePendingTtsSelection\(\);\s*setSpeakConnectionId\(id\);/,
+ );
+ assert.match(
+ source,
+ /const invalidatePendingTtsSelection = useCallback\(\(\) => \{\s*ttsPickGeneration\.current \+= 1;\s*pendingRoutedTtsPick\.current = null;\s*invalidatePendingStagedTts\(\);/,
+ );
+});
diff --git a/studio/frontend/tests/speak-connection-models.test.ts b/studio/frontend/tests/speak-connection-models.test.ts
new file mode 100644
index 00000000000..b12f9bbe0db
--- /dev/null
+++ b/studio/frontend/tests/speak-connection-models.test.ts
@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { connectionSpeechModels } from "../src/features/audio/speak-connection-models.ts";
+
+test("an OpenAI connection offers its speech models, not its chat models", () => {
+ assert.deepEqual(
+ connectionSpeechModels({ providerType: "openai", models: ["gpt-5.5"] }),
+ ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],
+ );
+});
+
+test("a self-hosted connection offers the models it lists; hosted chat providers none", () => {
+ assert.deepEqual(
+ connectionSpeechModels({ providerType: "custom", models: ["kokoro"] }),
+ ["kokoro"],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "custom",
+ models: [
+ "speaches-ai/Kokoro-82M-v1.0-ONNX",
+ "Systran/faster-whisper-tiny",
+ "nvidia/parakeet-tdt-0.6b-v3",
+ "Qwen/Qwen3-ASR-0.6B",
+ ],
+ }),
+ ["speaches-ai/Kokoro-82M-v1.0-ONNX"],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "mistral",
+ models: ["mistral-large"],
+ }),
+ [],
+ );
+});
+
+test("an OpenAI-typed Azure connection offers its deployment names", () => {
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "openai",
+ baseUrl: "https://studio.openai.azure.com/openai/v1",
+ models: ["speech-prod"],
+ }),
+ ["speech-prod"],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "openai",
+ baseUrl: "https://api.openai.com/v1",
+ models: ["gpt-5.5"],
+ }),
+ ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],
+ );
+});
+
+test("a connection offers only the models that look like speech models", () => {
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "ollama",
+ models: ["qwen3:8b", "llama3.2:3b"],
+ }),
+ [],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "custom",
+ baseUrl: "https://openrouter.ai/api/v1",
+ models: ["qwen3:8b", "gpt-4o", "kokoro"],
+ }),
+ ["kokoro"],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "openai",
+ baseUrl: "https://openrouter.ai/api/v1",
+ models: [
+ "openai/gpt-4o",
+ "meta-llama/llama-3.3-70b-instruct",
+ "media-7b",
+ ],
+ }),
+ [],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "vllm",
+ models: ["hexgrad/Kokoro-82M", "Qwen/Qwen3-8B"],
+ }),
+ ["hexgrad/Kokoro-82M"],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "custom",
+ models: [
+ "canopylabs/orpheus-3b-0.1-ft",
+ "nari-labs/Dia-1.6B",
+ "sesame/csm-1b",
+ "suno/bark-small",
+ "SWivid/F5-TTS",
+ "coqui/XTTS-v2",
+ "claudia-7b",
+ ],
+ }),
+ [
+ "canopylabs/orpheus-3b-0.1-ft",
+ "nari-labs/Dia-1.6B",
+ "sesame/csm-1b",
+ "suno/bark-small",
+ "SWivid/F5-TTS",
+ "coqui/XTTS-v2",
+ ],
+ );
+ assert.deepEqual(
+ connectionSpeechModels({
+ providerType: "openai",
+ baseUrl: "https://api.openai.com/v1",
+ models: ["gpt-4o", "qwen3:8b"],
+ }),
+ ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],
+ );
+});