Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: speak through a saved connection on the Audio page (#14288)
Nilayb4f3e06940b8
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 7a4290dcc3f..6a0be53280c 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -6303,6 +6303,14 @@ class AudioRunRequest(BaseModel):pattern = r"^[A-Za-z0-9_-]+$",description = "Your id for this run; POST it to /audio/cancel to stop the run",)+ provider_id: Optional[str] = Field(+ None,+ description = "[x-unsloth] Saved connection ID. When set, Speak sends the text to that "+ "connection's /audio/speech as `model` and the clip is saved to history.",+ )+ voice: Optional[str] = Field(+ None, max_length = 256, description = "Voice on the connection named by `provider_id`."+ )@model_validator(mode = "after")def _take_fields(self):diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 944e0e8c055..afb768f4de7 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -25465,6 +25465,8 @@ def _discard_audio_clips(outputs: list[str], sources: list[str]) -> None:async def _dispatch_audio_workflow(body: AudioRunRequest, request: Request, current_subject: str) -> AudioRunResponse:+ if body.provider_id:+ return await _speak_on_connection(body, request)if body.workflow == "separate":return await _run_separation(body, request, current_subject)if body.workflow == "convert":@@ -25958,6 +25960,64 @@ async def _external_tts_speech(body: AudioSpeechRequest, request: Request) -> Rereturn Response(content = audio_bytes, media_type = media_type)+async def _speak_on_connection(body: AudioRunRequest, request: Request) -> AudioRunResponse:+ import wave++ if (+ body.workflow != "speak"+ or body.inputs != AudioRunInputs()+ or body.language+ or body.options+ or body.seed is not None+ or body.max_tokens is not None+ or any(+ name in body.model_fields_set and getattr(body, name) not in (None, False, 1)+ for name in _MUSIC_ONLY_FIELDS+ )+ ):+ raise HTTPException(+ status_code = 400,+ detail = "A saved connection only speaks text, in a voice, with instructions and speed.",+ )+ speech = await _external_tts_speech(+ AudioSpeechRequest(+ input = body.text,+ model = body.model,+ voice = body.voice,+ speed = body.speed,+ instructions = body.instructions,+ provider_id = body.provider_id,+ ),+ request,+ )+ try:+ with wave.open(io.BytesIO(speech.body)) as wav:+ params = wav.getparams()+ frames = wav.readframes(wav.getnframes())+ except (wave.Error, EOFError):+ raise HTTPException(+ status_code = 502, detail = "The TTS endpoint did not return WAV audio."+ ) from None+ # OpenAI streams WAV with 0xFFFFFFFF sizes; rewrite the header from the frames received.+ out = io.BytesIO()+ with wave.open(out, "wb") as wav:+ wav.setparams(params._replace(nframes = 0))+ wav.writeframes(frames)+ wav_bytes, sample_rate = out.getvalue(), params.framerate+ settings = {"voice": body.voice, "instructions": body.instructions, "speed": body.speed}+ record = await asyncio.to_thread(+ _persist_tts_clip,+ wav_bytes,+ sample_rate,+ body.text,+ body.model,+ "connection",+ {"settings": settings},+ "speak",+ )+ return _audio_run_response(record, wav_bytes, sample_rate, body.model)++# openai-compatible speech api: the router's dual mount also answers /v1/audio/speech@router.post("/audio/speech")async def openai_audio_speech(diff --git a/studio/backend/tests/test_audio_run_route.py b/studio/backend/tests/test_audio_run_route.pyindex 14da336bf27..ab6243de7c5 100644--- a/studio/backend/tests/test_audio_run_route.py+++ b/studio/backend/tests/test_audio_run_route.py@@ -23,7 +23,7 @@ from fastapi.testclient import TestClientfrom auth.authentication import authenticated_via_api_key, get_current_subjectfrom core.inference import audio_gallery, audio_inputs, audio_voicesfrom routes import inference-from utils.account_context import bind_account, reset_account, run_as+from utils.account_context import OWNER, bind_account, reset_account, run_as_TESTS_DIR = str(Path(__file__).resolve().parent)if _TESTS_DIR not in sys.path:@@ -2206,3 +2206,133 @@ def test_take_or_group_rejected_outside_single_variation_music(stub, body):with _client(ALICE) as client:response = client.post("/api/inference/audio/run", json = {"text": "x", **body})assert response.status_code == 422, response.text+++def _connection(monkeypatch, handler):+ import httpx++ import core.inference.external_provider as external_provider++ monkeypatch.setattr(+ external_provider,+ "_http_client",+ httpx.AsyncClient(transport = httpx.MockTransport(handler)),+ )+ monkeypatch.setattr(+ inference.providers_db,+ "get_provider",+ lambda pid: (+ {+ "provider_type": "custom",+ "display_name": "Kokoro Box",+ "base_url": "http://127.0.0.1:8880/v1",+ "is_enabled": True,+ }+ if pid == "conn-1"+ else None+ ),+ )+ monkeypatch.setattr(inference, "resolve_provider_api_key_or_400", lambda *a, **k: "sk-test")+++def _speak_on_connection(client, **body):+ payload = {+ "workflow": "speak",+ "text": "Hello there.",+ "provider_id": "conn-1",+ "model": "kokoro",+ "voice": "af_heart",+ **body,+ }+ return client.post("/api/inference/audio/run", json = payload)+++def test_speak_through_a_connection_lands_in_history(stub, monkeypatch):+ import httpx++ sent = []++ def handler(req):+ sent.append((str(req.url), req.headers.get("authorization"), json.loads(req.content)))+ return httpx.Response(200, content = _wav(rate = 22050), headers = {"content-type": "audio/wav"})++ _connection(monkeypatch, handler)+ with _client(OWNER) as client:+ response = _speak_on_connection(client, instructions = "Warmly.", speed = 1.25)+ assert response.status_code == 200, response.text+ body = response.json()+ (clip,) = body["clips"]+ assert client.get(clip["url"]).status_code == 200+ listed = client.get("/api/inference/audio/gallery").json()["audio"]+ assert stub["backend"].calls == []+ assert sent == [+ (+ "http://127.0.0.1:8880/v1/audio/speech",+ "Bearer sk-test",+ {+ "model": "kokoro",+ "input": "Hello there.",+ "response_format": "wav",+ "voice": "af_heart",+ "speed": 1.25,+ "instructions": "Warmly.",+ },+ )+ ]+ assert (body["model"], body["audio"]) == ("kokoro", None)+ assert (clip["role"], clip["workflow"], clip["sample_rate"]) == ("output", "speak", 22050)+ assert [(c["id"], c["prompt"], c["model"], c["workflow"]) for c in listed] == [+ (clip["id"], "Hello there.", "kokoro", "speak")+ ]+ sidecar = run_as(OWNER, audio_gallery.gallery_dir) / f"{clip['id']}.json"+ meta = json.loads(sidecar.read_text(encoding = "utf-8"))+ assert meta["settings"] == {"voice": "af_heart", "instructions": "Warmly.", "speed": 1.25}+++ "extra",+ [+ {"seed": 7},+ {"options": {"temperature": 0.5}},+ {"inputs": {"reference": {"voice_id": "abc"}}},+ {"workflow": "clone"},+ ],+)+def test_speak_through_a_connection_refuses_settings_it_cannot_send(stub, monkeypatch, extra):+ sent = []+ _connection(monkeypatch, lambda req: sent.append(req))+ with _client(OWNER) as client:+ response = _speak_on_connection(client, **extra)+ assert response.status_code == 400, response.text+ assert sent == [] and stub["backend"].calls == []+++def test_speak_through_a_connection_reports_the_upstream_error(stub, monkeypatch):+ import httpx++ _connection(monkeypatch, lambda req: httpx.Response(404, text = "model 'kokoro' not found"))+ with _client(OWNER) as client:+ response = _speak_on_connection(client)+ listed = client.get("/api/inference/audio/gallery").json()["audio"]+ assert response.status_code == 502+ assert "model 'kokoro' not found" in response.text+ assert listed == []+++def test_speak_through_a_connection_saves_the_real_length_of_a_streamed_wav(stub, monkeypatch):+ import httpx++ wav = bytearray(_wav(rate = 24000, frames = 24000))+ wav[4:8] = wav[40:44] = b"\xff\xff\xff\xff"+ _connection(+ monkeypatch,+ lambda req: httpx.Response(200, content = bytes(wav), headers = {"content-type": "audio/wav"}),+ )+ with _client(OWNER) as client:+ response = _speak_on_connection(client)+ assert response.status_code == 200, response.text+ (clip,) = response.json()["clips"]+ listed = client.get("/api/inference/audio/gallery").json()["audio"]+ assert [c["duration_s"] for c in listed] == [1.0]+ sidecar = run_as(OWNER, audio_gallery.gallery_dir) / f"{clip['id']}.json"+ assert json.loads(sidecar.read_text(encoding = "utf-8"))["duration_s"] == 1.0diff --git a/studio/frontend/src/features/audio/api.ts b/studio/frontend/src/features/audio/api.tsindex 8b26a42100b..58d52f28c34 100644--- a/studio/frontend/src/features/audio/api.ts+++ b/studio/frontend/src/features/audio/api.ts@@ -543,6 +543,34 @@ export async function runAudio(return parseAudioJson<AudioRunResponse>(response);}+export async function runSpeechOnConnection(+ request: {+ text: string;+ providerId: string;+ model: string;+ voice: string;+ instructions: string | null;+ speed: number;+ },+ signal?: AbortSignal,+): Promise<AudioRunResponse> {+ const response = await authFetch("/api/inference/audio/run", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ workflow: "speak",+ text: request.text,+ provider_id: request.providerId,+ model: request.model,+ voice: request.voice,+ speed: request.speed,+ ...(request.instructions ? { instructions: request.instructions } : {}),+ }),+ signal,+ });+ return parseAudioJson<AudioRunResponse>(response);+}+export interface AudioVoice {id: string;name: string;diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsxindex 4ff62cc7fb0..484d08220aa 100644--- a/studio/frontend/src/features/audio/audio-page.tsx+++ b/studio/frontend/src/features/audio/audio-page.tsx@@ -87,6 +87,7 @@ import { useCloneGeneration } from "./hooks/use-clone-generation";import { useConvertGeneration } from "./hooks/use-convert-generation";import { adoptEditSource, useEditGeneration } from "./hooks/use-edit-generation";import { useSeparateGeneration } from "./hooks/use-separate-generation";+import { useSpeakConnection } from "./hooks/use-speak-connection";import { useSpeechGeneration } from "./hooks/use-speech-generation";import { useSttSidecar } from "./hooks/use-stt-sidecar";import { useTranscription } from "./hooks/use-transcription";@@ -105,7 +106,11 @@ import {SeparateOutput,SeparateRail,} from "./pages/separate-page";-import { SpeakOutput, SpeakRail } from "./pages/speak-page";+import {+ SpeakConnectionRail,+ SpeakOutput,+ SpeakRail,+} from "./pages/speak-page";import { type GenerateBlocker, TtsFooter } from "./pages/tts-workspace";import { TranscribeFooter, TranscribeOutput, TranscribeRail } from "./pages/transcribe-page";import type { SendTarget } from "./components/stem-mixer-types";@@ -512,6 +517,7 @@ export function AudioPage({reloadOnCpu,setLoadFailureHandlers,handleEject,+ invalidatePendingTtsSelection,trainedTtsModels,} = useAudioModelSlot({active,@@ -563,6 +569,12 @@ export function AudioPage({if (busy === "generating") setAnnouncement("");}, [busy]);+ const speakConnection = useSpeakConnection();+ const connection =+ mode === "speak" && ttsWorkflow === "speak"+ ? speakConnection.connection+ : null;+const {prompt,setPrompt,@@ -617,6 +629,7 @@ export function AudioPage({setSelectedId,pendingTranscribeRelease,replayQueuedTtsPick,+ connection,});const music = useMusicGeneration({@@ -1028,12 +1041,13 @@ export function AudioPage({);const pageModelLoaded =- ttsLoaded &&- loadedModelRunsWorkflow({- workflow: ttsWorkflow,- audioWorkflows: status?.audio_workflows,- music: musicGeneration,- });+ connection !== null ||+ (ttsLoaded &&+ loadedModelRunsWorkflow({+ workflow: ttsWorkflow,+ audioWorkflows: status?.audio_workflows,+ music: musicGeneration,+ }));const rememberedModels = useAudioWorkspaceStore((state) => state.lastModelByWorkflow,);@@ -1165,6 +1179,10 @@ export function AudioPage({? { reason: "Waiting for the model to finish loading." }: busy !== null? null+ : connection+ ? prompt.trim()+ ? null+ : { reason: "Type the text to speak." }: !ttsLoaded? {reason:@@ -1639,10 +1657,25 @@ export function AudioPage({(row: PageModelRow) => rowOnPage(pageWorkflow, row, pagePrefs),[pageWorkflow, pagePrefs],);+ const setSpeakConnectionId = speakConnection.setModelId;const selectModel = useCallback(- (id: string, meta: Parameters<typeof handleModelSelect>[1]) =>- handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta)),- [audioCatalog.byId, handleModelSelect],+ (id: string, meta: Parameters<typeof handleModelSelect>[1]) => {+ if (meta.source === "external") {+ if (busyRef.current === null) {+ invalidatePendingTtsSelection();+ setSpeakConnectionId(id);+ }+ return;+ }+ if (modeRef.current === "speak") setSpeakConnectionId(null);+ handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta));+ },+ [+ audioCatalog.byId,+ handleModelSelect,+ invalidatePendingTtsSelection,+ setSpeakConnectionId,+ ],);const sttOnDeviceModels = downloadedSttArtifacts.map((artifact) => {const catalogModel = MODELS_BY_MODE.transcribe.find(@@ -1682,7 +1715,9 @@ export function AudioPage({const capabilityLine =mode === "speak"- ? showLastPageModel+ ? connection+ ? `Speaks through ${connection.providerName}, a saved connection.`+ : showLastPageModel? `${lastPageModel.split("/").pop()} is not loaded. Pick it above to load it.`: ttsLoaded? pageModelLoaded@@ -1794,6 +1829,11 @@ export function AudioPage({(ttsWorkflow === "music"),)}+ externalModels={+ mode === "speak" && ttsWorkflow === "speak"+ ? speakConnection.models+ : undefined+ }rowFilter={selectorRowFilter}moreModels={selectorMoreModels}loadedModelIdOverride={@@ -1813,9 +1853,10 @@ export function AudioPage({? false: undefined}- value={selectorValue}+ value={(connection && speakConnection.modelId) || selectorValue}onValueChange={selectModel}onEject={+ !connection &&audioEjectAvailable({mode,busy,@@ -1884,10 +1925,12 @@ export function AudioPage({</div><div className="pointer-events-none col-start-3 flex min-w-0 items-start justify-end pr-2 pt-[var(--studio-chat-header-padding-top,11px)]"><div className="pointer-events-auto flex min-w-0 items-center gap-2">- <UseViaApiButton- workflow={pageWorkflow}- model={selectorValue ?? null}- />+ {connection ? null : (+ <UseViaApiButton+ workflow={pageWorkflow}+ model={selectorValue ?? null}+ />+ )}<LibraryPageLinktab="audio"labelClassName="hidden @[50rem]:inline"@@ -2020,6 +2063,17 @@ export function AudioPage({historyClips={clips}/></AudioActiveProvider>+ ) : connection ? (+ <SpeakConnectionRail+ connection={connection}+ prompt={prompt}+ setPrompt={setPrompt}+ instructions={audioInstructions}+ setInstructions={setAudioInstructions}+ setVoice={speakConnection.setVoice}+ setSpeed={speakConnection.setSpeed}+ disabled={busy === "generating"}+ />) : (<SpeakRail {...railProps} />);@@ -2110,7 +2164,9 @@ export function AudioPage({ttsLoaded={pageModelLoaded}prompt={prompt}lyricsOptional={musicStudio || lyricsOptional}- musicNeedsDescription={!musicStudio && musicNeedsDescription}+ musicNeedsDescription={+ !musicStudio && !connection && musicNeedsDescription+ }audioInstructions={audioInstructions}blocker={generateBlocker}shortcutLabel={shortcutLabel}diff --git a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.tsindex 80851a39774..a3395d4ce0d 100644--- a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts+++ b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts@@ -1282,6 +1282,7 @@ export function useAudioModelSlot({reloadOnCpu,setLoadFailureHandlers,handleEject,+ invalidatePendingTtsSelection,trainedTtsModels,};}diff --git a/studio/frontend/src/features/audio/hooks/use-speak-connection.ts b/studio/frontend/src/features/audio/hooks/use-speak-connection.tsnew file mode 100644index 00000000000..972d5486c6b--- /dev/null+++ b/studio/frontend/src/features/audio/hooks/use-speak-connection.ts@@ -0,0 +1,59 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { useMemo, useState } from "react";+import {+ buildExternalModelId,+ isDecisionConnection,+ useExternalProvidersStore,+} from "@/features/chat";+import type { ExternalModelOption } from "@/features/model-picker/components/model-selector/types";+import { connectionSpeechModels } from "../speak-connection-models";++export interface SpeakConnection {+ providerId: string;+ model: string;+ providerName: string;+ voice: string;+ speed: number;+}++export function useSpeakConnection() {+ const connections = useExternalProvidersStore((s) => s.providers);+ const connectionsEnabled = useExternalProvidersStore(+ (s) => s.connectionsEnabled,+ );+ const models = useMemo<ExternalModelOption[]>(+ () =>+ connectionsEnabled+ ? connections+ .filter((c) => !isDecisionConnection(c))+ .flatMap((c) =>+ connectionSpeechModels(c).map((model) => ({+ id: buildExternalModelId(c.id, model),+ name: model,+ providerId: c.id,+ providerName: c.name,+ providerType: c.providerType,+ })),+ )+ : [],+ [connections, connectionsEnabled],+ );+ const [modelId, setModelId] = useState<string | null>(null);+ const [voice, setVoice] = useState("");+ const [speed, setSpeed] = useState(1);+ const connection = useMemo<SpeakConnection | null>(() => {+ const option = models.find((model) => model.id === modelId);+ return option+ ? {+ providerId: option.providerId,+ model: option.name,+ providerName: option.providerName,+ voice,+ speed,+ }+ : null;+ }, [models, modelId, voice, speed]);+ return { models, modelId, setModelId, connection, setVoice, setSpeed };+}diff --git a/studio/frontend/src/features/audio/hooks/use-speech-generation.ts b/studio/frontend/src/features/audio/hooks/use-speech-generation.tsindex fdf0fa6c9cd..10931c1b1d7 100644--- a/studio/frontend/src/features/audio/hooks/use-speech-generation.ts+++ b/studio/frontend/src/features/audio/hooks/use-speech-generation.ts@@ -4,7 +4,7 @@import { useCallback, useEffect, useMemo, useState } from "react";import { readLastPrompt, saveLastPrompt } from "@/lib/last-prompt";import { toast } from "@/lib/toast";-import { generateAudio, runAudio } from "../api";+import { generateAudio, runAudio, runSpeechOnConnection } from "../api";import { TTS_MAX_TOKENS } from "../audio-workspace-constants";import {audioOptionLabel,@@ -58,6 +58,7 @@ import type { AudioWorkflowId } from "../workflows";import { reportAudioFailure } from "../audio-failure-actions";import type { AudioGallery } from "./use-audio-gallery";import type { AudioModelSlot } from "./use-audio-model-slot";+import type { SpeakConnection } from "./use-speak-connection";type TtsWorkflow = "speak" | "music";type TtsDraftField = "prompt" | "instructions";@@ -90,7 +91,8 @@ export function useSpeechGeneration({setSelectedId,pendingTranscribeRelease,replayQueuedTtsPick,-}: { workflow: TtsWorkflow } & Pick<+ connection,+}: { workflow: TtsWorkflow; connection: SpeakConnection | null } & Pick<AudioHostState,| "status"| "busyRef"@@ -339,7 +341,7 @@ export function useSpeechGeneration({return;}const instructions = audioInstructions.trim();- if (musicNeedsDescription && !instructions) {+ if (!connection && musicNeedsDescription && !instructions) {updateGenerationPhase(null);busyRef.current = null;setBusy(null);@@ -348,17 +350,16 @@ export function useSpeechGeneration({);return;}- if (toolRequest.error) {+ if (!connection && toolRequest.error) {updateGenerationPhase(null);busyRef.current = null;setBusy(null);toast.error(toolRequest.error);return;}- const missingOptions = missingRequiredAudioOptions(- advancedOptionSpecs,- audioOptionValues,- );+ const missingOptions = connection+ ? []+ : missingRequiredAudioOptions(advancedOptionSpecs, audioOptionValues);if (missingOptions.length > 0) {updateGenerationPhase(null);busyRef.current = null;@@ -380,24 +381,38 @@ export function useSpeechGeneration({toolRequest.patch.language ?? (mossLocalGeneration ? audioLanguage : "")).trim();saveLastPrompt(ttsDraftKey("prompt", workflow), prompt);- if (savedVoice) {+ if (savedVoice || connection) {const voiceController = new AbortController();generateAbort.current = voiceController;updateGenerationPhase("generating");try {- const response = await runAudio(- {- workflow: "speak",- text,- instructions:- instructionsKind !== null && instructions ? instructions : null,- language: language || null,- inputs: { reference: savedVoice },- options: requestOptions,- speed: toolRequest.patch.speed ?? null,- },- voiceController.signal,- );+ const response = connection+ ? await runSpeechOnConnection(+ {+ text,+ providerId: connection.providerId,+ model: connection.model,+ voice: connection.voice.trim() || "alloy",+ instructions: instructions || null,+ speed: connection.speed,+ },+ voiceController.signal,+ )+ : await runAudio(+ {+ workflow: "speak",+ text,+ instructions:+ instructionsKind !== null && instructions+ ? instructions+ : null,+ language: language || null,+ inputs: { reference: savedVoice },+ options: requestOptions,+ speed: toolRequest.patch.speed ?? null,+ },+ voiceController.signal,+ );updateGenerationPhase("finishing");await showRunResult({response,@@ -533,6 +548,7 @@ export function useSpeechGeneration({refreshStatus,replayQueuedTtsPick,selectClip,+ connection,]);// Only unmount aborts: RootLayout keeps this page mounted so leaving the tab does not cancel synthesis.diff --git a/studio/frontend/src/features/audio/pages/speak-page.tsx b/studio/frontend/src/features/audio/pages/speak-page.tsxindex 78d28cbee17..f071fa80419 100644--- a/studio/frontend/src/features/audio/pages/speak-page.tsx+++ b/studio/frontend/src/features/audio/pages/speak-page.tsx@@ -2,6 +2,11 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import type { ComponentProps } from "react";+import { Input } from "@/components/ui/input";+import { Textarea } from "@/components/ui/textarea";+import { ParamSlider } from "@/features/chat";+import { Field } from "../components/field";+import type { SpeakConnection } from "../hooks/use-speak-connection";import { TtsOutput, TtsRailFields } from "./tts-workspace";export function SpeakRail(@@ -10,6 +15,79 @@ export function SpeakRail(return <TtsRailFields {...props} musicGeneration={false} />;}+export function SpeakConnectionRail({+ connection,+ prompt,+ setPrompt,+ instructions,+ setInstructions,+ setVoice,+ setSpeed,+ disabled,+}: {+ connection: SpeakConnection;+ prompt: string;+ setPrompt: (value: string) => void;+ instructions: string;+ setInstructions: (value: string) => void;+ setVoice: (value: string) => void;+ setSpeed: (value: number) => void;+ disabled: boolean;+}) {+ return (+ <>+ <Field+ label="Text"+ htmlFor="audio-prompt"+ hint={`What ${connection.providerName} should say. The clip lands in the gallery.`}+ >+ <Textarea+ data-type-to-activate="prompt"+ id="audio-prompt"+ value={prompt}+ onChange={(event) => setPrompt(event.target.value)}+ placeholder="Type the sentence to speak…"+ className="min-h-28"+ />+ </Field>+ <Field+ label="Voice"+ htmlFor="audio-connection-voice"+ hint="A voice the connection offers. Empty uses alloy."+ >+ <Input+ id="audio-connection-voice"+ value={connection.voice}+ onChange={(event) => setVoice(event.target.value)}+ placeholder="alloy"+ />+ </Field>+ <Field+ label="Style instructions"+ htmlFor="audio-instructions"+ hint="Optional. Sent to models that take instructions; others ignore them."+ >+ <Textarea+ id="audio-instructions"+ value={instructions}+ onChange={(event) => setInstructions(event.target.value)}+ placeholder="Warm, measured delivery with a calm conversational tone…"+ className="min-h-24"+ />+ </Field>+ <ParamSlider+ label="Speed"+ value={connection.speed}+ min={0.25}+ max={4}+ step={0.05}+ onChange={setSpeed}+ disabled={disabled}+ />+ </>+ );+}+export function SpeakOutput({modelReady,diff --git a/studio/frontend/src/features/audio/speak-connection-models.ts b/studio/frontend/src/features/audio/speak-connection-models.tsnew file mode 100644index 00000000000..d723b72756f--- /dev/null+++ b/studio/frontend/src/features/audio/speak-connection-models.ts@@ -0,0 +1,36 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++const SELF_HOSTED_TYPES = new Set(["custom", "vllm", "llama_cpp", "lemonade"]);+const OPENAI_SPEECH_MODELS = ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"];+const SPEECH_MODEL_ID =+ /kokoro|tts|speech|orpheus|piper|chatterbox|parler|zonos|cosyvoice|(?:^|[\W_])(?:f5|csm|dia|bark|melo|sesame)(?:[\W_\d]|$)/i;+// A self-hosted server such as speaches lists its speech-to-text models beside its voices.+const TRANSCRIBE_MODEL_ID =+ /whisper|transcri|parakeet|(?:^|[\W_])(?:asr|stt)(?:[\W_]|$)/i;++function isOpenAIApi(baseUrl: string | undefined): boolean {+ if (!baseUrl) return true;+ try {+ return new URL(baseUrl).hostname.toLowerCase() === "api.openai.com";+ } catch {+ return false;+ }+}++export function connectionSpeechModels(connection: {+ providerType: string;+ baseUrl?: string;+ models: readonly string[];+}): string[] {+ if (connection.providerType === "openai" && isOpenAIApi(connection.baseUrl)) {+ return OPENAI_SPEECH_MODELS;+ }+ // Azure and other OpenAI-typed endpoints name models by deployment.+ return connection.providerType === "openai" ||+ SELF_HOSTED_TYPES.has(connection.providerType)+ ? connection.models.filter(+ (id) => SPEECH_MODEL_ID.test(id) && !TRANSCRIBE_MODEL_ID.test(id),+ )+ : [];+}diff --git a/studio/frontend/tests/audio-model-eject.test.ts b/studio/frontend/tests/audio-model-eject.test.tsindex 08c94c73baf..17e4a9679d9 100644--- a/studio/frontend/tests/audio-model-eject.test.ts+++ b/studio/frontend/tests/audio-model-eject.test.ts@@ -14,7 +14,7 @@ test("Audio exposes the shared picker eject action only while idle", () => {// The selector keeps showing the last Transcribe model once it is unloaded; Eject there did nothing.assert.match(source,- /onEject=\{\s*audioEjectAvailable\(\{\s*mode,\s*busy,\s*hasValue: Boolean\(selectorValue\),\s*showLastPageModel,\s*sttReady,\s*\}\)\s*\? handleEject\s*: undefined\s*\}/,+ /onEject=\{\s*!connection &&\s*audioEjectAvailable\(\{\s*mode,\s*busy,\s*hasValue: Boolean\(selectorValue\),\s*showLastPageModel,\s*sttReady,\s*\}\)\s*\? handleEject\s*: undefined\s*\}/,);assert.match(source, /const handleEject = useCallback\(\(\) => \{\s*if \(busy !== null\) \{/);assert.match(@@ -285,3 +285,18 @@ test("leaving Audio during the residency read skips the transcription load", ()/await readSttResidency\(engine\)[\s\S]{0,400}?\|\| !activeRef\.current\) return null;[\s\S]*?await ensureSttLoaded\(repo, model, engine\)/,);});++test("Use via API is hidden while Speak runs through a connection", () => {+ assert.match(source, /\{connection \? null : \(\s*<UseViaApiButton\s+workflow=\{pageWorkflow\}/);+});++test("picking a connection retires a local Speak pick still downloading or planning", () => {+ assert.match(+ source,+ /if \(meta\.source === "external"\) \{\s*if \(busyRef\.current === null\) \{\s*invalidatePendingTtsSelection\(\);\s*setSpeakConnectionId\(id\);/,+ );+ assert.match(+ source,+ /const invalidatePendingTtsSelection = useCallback\(\(\) => \{\s*ttsPickGeneration\.current \+= 1;\s*pendingRoutedTtsPick\.current = null;\s*invalidatePendingStagedTts\(\);/,+ );+});diff --git a/studio/frontend/tests/speak-connection-models.test.ts b/studio/frontend/tests/speak-connection-models.test.tsnew file mode 100644index 00000000000..b12f9bbe0db--- /dev/null+++ b/studio/frontend/tests/speak-connection-models.test.ts@@ -0,0 +1,126 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { connectionSpeechModels } from "../src/features/audio/speak-connection-models.ts";++test("an OpenAI connection offers its speech models, not its chat models", () => {+ assert.deepEqual(+ connectionSpeechModels({ providerType: "openai", models: ["gpt-5.5"] }),+ ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],+ );+});++test("a self-hosted connection offers the models it lists; hosted chat providers none", () => {+ assert.deepEqual(+ connectionSpeechModels({ providerType: "custom", models: ["kokoro"] }),+ ["kokoro"],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "custom",+ models: [+ "speaches-ai/Kokoro-82M-v1.0-ONNX",+ "Systran/faster-whisper-tiny",+ "nvidia/parakeet-tdt-0.6b-v3",+ "Qwen/Qwen3-ASR-0.6B",+ ],+ }),+ ["speaches-ai/Kokoro-82M-v1.0-ONNX"],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "mistral",+ models: ["mistral-large"],+ }),+ [],+ );+});++test("an OpenAI-typed Azure connection offers its deployment names", () => {+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "openai",+ baseUrl: "https://studio.openai.azure.com/openai/v1",+ models: ["speech-prod"],+ }),+ ["speech-prod"],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "openai",+ baseUrl: "https://api.openai.com/v1",+ models: ["gpt-5.5"],+ }),+ ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],+ );+});++test("a connection offers only the models that look like speech models", () => {+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "ollama",+ models: ["qwen3:8b", "llama3.2:3b"],+ }),+ [],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "custom",+ baseUrl: "https://openrouter.ai/api/v1",+ models: ["qwen3:8b", "gpt-4o", "kokoro"],+ }),+ ["kokoro"],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "openai",+ baseUrl: "https://openrouter.ai/api/v1",+ models: [+ "openai/gpt-4o",+ "meta-llama/llama-3.3-70b-instruct",+ "media-7b",+ ],+ }),+ [],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "vllm",+ models: ["hexgrad/Kokoro-82M", "Qwen/Qwen3-8B"],+ }),+ ["hexgrad/Kokoro-82M"],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "custom",+ models: [+ "canopylabs/orpheus-3b-0.1-ft",+ "nari-labs/Dia-1.6B",+ "sesame/csm-1b",+ "suno/bark-small",+ "SWivid/F5-TTS",+ "coqui/XTTS-v2",+ "claudia-7b",+ ],+ }),+ [+ "canopylabs/orpheus-3b-0.1-ft",+ "nari-labs/Dia-1.6B",+ "sesame/csm-1b",+ "suno/bark-small",+ "SWivid/F5-TTS",+ "coqui/XTTS-v2",+ ],+ );+ assert.deepEqual(+ connectionSpeechModels({+ providerType: "openai",+ baseUrl: "https://api.openai.com/v1",+ models: ["gpt-4o", "qwen3:8b"],+ }),+ ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],+ );+});