Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: transcribe audio through a saved connection on the Audio page (#14289)

Nilay2cfa64fda7a3
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 077da213c52..afceaa2f9ef 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -6486,6 +6486,12 @@ class TranscribeSourceRequest(BaseModel):
timestamps: bool = Field(False, description = "Ask for segment and word timestamps")
speakers: bool = Field(False, description = "Keep who spoke each segment, when the model tells")
title: Optional[str] = Field(None, max_length = 255, description = "Defaults to the source's name")
+ provider_id: Optional[str] = Field(
+ None,
+ max_length = 128,
+ description = "Saved connection ID. When set, the audio goes to that connection's "
+ "/audio/transcriptions as `model`; engine and device are ignored.",
+ )
class AudioAnalyzeRequest(TranscribeSourceRef):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 4e38a6d0522..0867209d61a 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -27630,14 +27630,93 @@ def _source_transcript(result: dict, source, speakers: bool) -> dict:
return out
+async def _external_source_transcript(
+ body: TranscribeSourceRequest, source, request: Request
+) -> dict:
+ from core.inference import audio_inputs, audio_metrics
+
+ def _encode():
+ path = audio_inputs.prepared_path(source, 16000)
+ flac, _ = audio_inputs.encode_wav(path.read_bytes(), "flac")
+ return audio_metrics.wav_seconds(path), flac
+
+ try:
+ duration, audio = await asyncio.to_thread(_encode)
+ except audio_inputs.AudioInputError as exc:
+ raise _audio_source_error(exc) from None
+ if len(audio) > _MAX_AUDIO_RAW_BYTES:
+ raise HTTPException(
+ status_code = 413,
+ detail = (
+ f"This audio comes to {len(audio) / 2**20:.0f} MB, past the "
+ f"{_MAX_AUDIO_RAW_BYTES // 2**20} MB a connection takes. Trim it, or pick an "
+ "on-device model."
+ ),
+ )
+ response = await _external_stt_transcription(
+ provider_id = body.provider_id,
+ raw = audio,
+ filename = "audio.flac",
+ content_type = "audio/flac",
+ model = body.model,
+ language = body.language,
+ response_format = "verbose_json" if body.timestamps else "json",
+ timestamp_granularities = ["segment", "word"] if body.timestamps else None,
+ encrypted_api_key = None,
+ request = request,
+ )
+ data = json.loads(response.body) if "json" in response.media_type else {}
+ if not isinstance(data, dict) or not isinstance(data.get("text"), str):
+ raise HTTPException(status_code = 502, detail = "The connection returned no transcript.")
+
+ def spans(key: str, text_key: str) -> list[dict]:
+ return [
+ {
+ "start": s.get("start"),
+ "end": s.get("end"),
+ text_key: str(s.get(text_key, "")).strip(),
+ }
+ for s in data.get(key) or ()
+ if isinstance(s, dict)
+ ]
+
+ result = {
+ "text": data["text"].strip(),
+ "model": body.model,
+ "language": data.get("language") or body.language,
+ "duration": duration,
+ }
+ if body.timestamps:
+ result.update(segments = spans("segments", "text"), words = spans("words", "word"))
+ return _source_transcript(result, source, False)
+
+
@studio_router.post("/audio/transcribe/source")
async def transcribe_audio_source(
- body: TranscribeSourceRequest, current_subject: str = Depends(get_current_subject)
+ body: TranscribeSourceRequest,
+ request: Request,
+ current_subject: str = Depends(get_current_subject),
):
"""Transcribe Audio page audio named by id, streamed as NDJSON and saved to history."""
from core.inference import audio_inputs, stt_capabilities, stt_details
from core.inference.transcript_stream import stream_transcript
+ if body.provider_id is not None:
+ try:
+ source = await asyncio.to_thread(
+ audio_inputs.resolve_source, body.source.model_dump(exclude_none = True)
+ )
+ except audio_inputs.AudioInputError as exc:
+ raise _audio_source_error(exc) from None
+ return StreamingResponse(
+ stream_transcript(
+ lambda _on_progress: _external_source_transcript(body, source, request),
+ body.title or source.name,
+ ),
+ media_type = "application/x-ndjson",
+ headers = {"Cache-Control": "no-store", "X-Accel-Buffering": "no"},
+ )
+
engine = body.engine or await asyncio.to_thread(_stt_engine_for_model, body.model)
serving_engine = _resolve_serving_stt_engine(engine)
caps = await asyncio.to_thread(stt_capabilities.capabilities_for, body.model, serving_engine)
diff --git a/studio/backend/tests/test_transcribe_source_route.py b/studio/backend/tests/test_transcribe_source_route.py
index 1a7fc7681aa..569c1ab3054 100644
--- a/studio/backend/tests/test_transcribe_source_route.py
+++ b/studio/backend/tests/test_transcribe_source_route.py
@@ -224,3 +224,82 @@ def test_transcript_routes_get_rename_and_archive(stub):
cleared = client.patch(url, json = {"speaker_names": {"S01": None}, "archived": True}).json()
assert "speaker_names" not in cleared and cleared["archived"] is True
assert client.get("/api/inference/audio/transcripts/" + "e" * 32).status_code == 404
+
+
+def connection(monkeypatch):
+ import httpx
+
+ from core.inference import external_provider, providers
+
+ config = {
+ "provider_type": "custom",
+ "display_name": "Whisper Box",
+ "base_url": "http://stt.local:8000/v1",
+ "is_enabled": True,
+ }
+ monkeypatch.setattr(
+ inference.providers_db,
+ "get_provider",
+ lambda pid: dict(config) if pid == "conn-1" else None,
+ )
+ for module in (inference, providers):
+ monkeypatch.setattr(module, "validate_provider_base_url", lambda url: url)
+ monkeypatch.setattr(inference, "resolve_provider_api_key_or_400", lambda *a, **k: "sk-test")
+ calls = []
+ reply = {
+ "task": "transcribe",
+ "language": "english",
+ "duration": 1.0,
+ "text": " Hello there.",
+ "segments": [{"id": 0, "seek": 0, "start": 0.0, "end": 1.0, "text": " Hello there."}],
+ "words": [{"word": " Hello", "start": 0.0, "end": 0.4}],
+ }
+
+ class _Http:
+ async def post(self, url, **kwargs):
+ calls.append({"url": url, **kwargs})
+ return httpx.Response(200, json = reply, request = httpx.Request("POST", url))
+
+ monkeypatch.setattr(external_provider, "_client", lambda: _Http())
+ return calls
+
+
+def test_a_connection_transcribes_the_source_and_saves_it(stub, connection):
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ response, events = _post(
+ client,
+ provider_id = "conn-1",
+ model = "whisper-1",
+ source = {"input_id": input_id},
+ timestamps = True,
+ )
+ assert response.status_code == 200, response.text
+ assert not (stub.paths or stub.bytes or stub.loads)
+ (call,) = connection
+ assert call["url"] == "http://stt.local:8000/v1/audio/transcriptions"
+ name, audio, content_type = call["files"]["file"]
+ assert (name, content_type, audio[:4]) == ("audio.flac", "audio/flac", b"fLaC")
+ assert call["data"]["model"] == "whisper-1"
+ assert call["data"]["response_format"] == "verbose_json"
+ assert call["data"]["timestamp_granularities[]"] == ["segment", "word"]
+ complete = events[-1]
+ assert complete["type"] == "complete" and complete["text"] == "Hello there."
+ assert complete["segments"] == [{"start": 0.0, "end": 1.0, "text": "Hello there."}]
+ assert complete["words"] == [{"start": 0.0, "end": 0.4, "word": "Hello"}]
+ assert complete["timestamps"] is True and complete["duration"] == pytest.approx(1.0, abs = 0.05)
+ record = complete["record"]
+ assert (record["model"], record["language"]) == ("whisper-1", "english")
+ assert record["source"] == {"kind": "input", "id": input_id, "name": "meeting.webm"}
+
+
+def test_a_source_past_the_connection_limit_is_refused_before_upload(stub, connection, monkeypatch):
+ monkeypatch.setattr(inference, "_MAX_AUDIO_RAW_BYTES", 1024)
+ with _client(ALICE) as client:
+ _, events = _post(
+ client, provider_id = "conn-1", model = "whisper-1", source = {"input_id": _input(ALICE)}
+ )
+ assert events[-1]["type"] == "error"
+ assert "a connection takes" in events[-1]["message"]
+ assert not connection and not stub.bytes
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index 484d08220aa..ee0c14e52e4 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -17,8 +17,11 @@ import { useSidebar } from "@/components/ui/sidebar";
import { usePlatformStore } from "@/config/env";
import {
type InferenceStatusResponse,
+ buildExternalModelId,
getInferenceStatus,
+ isDecisionConnection,
syncKeepModelsLoaded,
+ useExternalProvidersStore,
} from "@/features/chat";
import { useHubInventory } from "@/features/hub";
import { useInventoryVersion } from "@/features/hub/stores/inventory-events";
@@ -26,7 +29,11 @@ import { ModelSelector } from "@/features/model-picker/components/model-selector
import { ggufQuantLabel } from "@/features/model-picker/model-config/model-identity";
import { AUDIO_CATALOG } from "@/features/model-picker/components/model-selector/model-catalog";
import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";
-import type { ModelOption } from "@/features/model-picker/components/model-selector/types";
+import type {
+ ExternalModelOption,
+ ModelOption,
+} from "@/features/model-picker/components/model-selector/types";
+import { useVoiceSettingsStore } from "@/features/settings";
import { sttModelSize } from "@/features/settings/stores/stt-model-catalog";
import { usePersistedChoice } from "@/hooks/use-persisted-choice";
import { usePersistedToggle } from "@/hooks/use-persisted-toggle";
@@ -121,7 +128,12 @@ import type { AudioSourceStatus } from "./hooks/audio-source-state";
import { useAlignerDownload } from "./hooks/use-aligner-download";
import { useTranscribeCapabilities } from "./hooks/use-transcribe-capabilities";
import { useAudioTranscribeStore } from "./stores/audio-transcribe-store";
-import { SPEAKERS_MODEL_NAME, transcribeSwitches } from "./transcribe-capabilities";
+import {
+ SPEAKERS_MODEL_NAME,
+ connectionTranscribeModels,
+ connectionTranscribeSwitches,
+ transcribeSwitches,
+} from "./transcribe-capabilities";
import {
transcribeLanguageFor,
transcribeLanguagesFor,
@@ -1431,18 +1443,52 @@ export function AudioPage({
string | null
>(null);
const transcribeRepo = (selectedSttRepo ?? lastSttRepo) || null;
+ const connections = useExternalProvidersStore((s) => s.providers);
+ const connectionsEnabled = useExternalProvidersStore(
+ (s) => s.connectionsEnabled,
+ );
+ const dictationProviderId = useVoiceSettingsStore((s) => s.sttProviderId);
+ const dictationModel = useVoiceSettingsStore((s) => s.sttProviderModel);
+ const transcribeExternalModels = useMemo<ExternalModelOption[]>(
+ () =>
+ connectionsEnabled
+ ? connections
+ .filter((c) => !isDecisionConnection(c))
+ .flatMap((c) =>
+ connectionTranscribeModels(
+ c,
+ c.id === dictationProviderId ? dictationModel.trim() : undefined,
+ ).map((model) => ({
+ id: buildExternalModelId(c.id, model),
+ name: model,
+ providerId: c.id,
+ providerName: c.name,
+ providerType: c.providerType,
+ })),
+ )
+ : [],
+ [connections, connectionsEnabled, dictationProviderId, dictationModel],
+ );
+ const connectionModel = useAudioTranscribeStore(
+ (state) => state.connectionModel,
+ );
+ const transcribeConnection =
+ transcribeExternalModels.find((model) => model.id === connectionModel) ??
+ null;
const transcribeCaps = useTranscribeCapabilities(
- transcribeRepo,
+ transcribeConnection ? null : transcribeRepo,
transcribeRepo ? sttEngineFor(transcribeRepo) : null,
);
- const transcribeOptions = transcribeSwitches(
- transcribeCaps.caps,
- transcribePrefs,
- { loading: transcribeCaps.loading, hasModel: transcribeRepo !== null },
- );
- const transcribeModelName = transcribeRepo?.split("/").pop() ?? "";
+ const transcribeOptions = transcribeConnection
+ ? connectionTranscribeSwitches(transcribePrefs, transcribeConnection.name)
+ : transcribeSwitches(transcribeCaps.caps, transcribePrefs, {
+ loading: transcribeCaps.loading,
+ hasModel: transcribeRepo !== null,
+ });
+ const transcribeModelName =
+ transcribeConnection?.name ?? transcribeRepo?.split("/").pop() ?? "";
const transcribeLoadsFirst = Boolean(
- transcribeRepo && !(sttSelected && sttReady),
+ !transcribeConnection && transcribeRepo && !(sttSelected && sttReady),
);
const transcribeNotice = !transcribeLoadsFirst
? transcribeOptions.notice
@@ -1453,7 +1499,7 @@ export function AudioPage({
.filter(Boolean)
.join(" ");
const transcribeLanguages = transcribeLanguagesFor(
- audioCppModelFor(transcribeRepo)?.languages,
+ transcribeConnection ? null : audioCppModelFor(transcribeRepo)?.languages,
);
const transcribeLanguage = transcribeLanguageFor(
transcribePrefs.language,
@@ -1482,15 +1528,18 @@ export function AudioPage({
transcribeCaps.refresh,
transcribeOptions.request.timestamps,
);
+ // A connection never uses the local aligner, so its download does not hold Transcribe.
const alignerBlocks =
- alignerDownload.staging && transcribeOptions.request.timestamps;
+ !transcribeConnection &&
+ alignerDownload.staging &&
+ transcribeOptions.request.timestamps;
const transcribeBlocker: GenerateBlocker | null = alignerBlocks
? { reason: "Downloading the timing aligner…" }
: busy === "loading"
? { reason: "Waiting for the model to finish loading." }
: busy !== null
? null
- : !transcribeRepo
+ : !transcribeRepo && !transcribeConnection
? {
reason: "Pick a speech-to-text model to transcribe.",
actions: [chooseModelAction, ...recommendedSttActions],
@@ -1529,7 +1578,13 @@ export function AudioPage({
const canTranscribe =
mode === "transcribe" && busy === null && transcribeBlocker === null;
const startTranscription = (source: AudioSourceSelection) => {
- const { request } = transcribeOptions;
+ const request = {
+ ...transcribeOptions.request,
+ connection: transcribeConnection && {
+ providerId: transcribeConnection.providerId,
+ modelId: transcribeConnection.name,
+ },
+ };
void runTranscription(
source,
{ language: transcribeLanguage, ...request },
@@ -1543,18 +1598,24 @@ export function AudioPage({
});
};
const startTranscriptionRef = useRef(startTranscription);
+ const transcribeTarget = transcribeConnection?.id ?? transcribeRepo;
+ const transcribeTargetRef = useRef(transcribeTarget);
useEffect(() => {
startTranscriptionRef.current = startTranscription;
+ transcribeTargetRef.current = transcribeTarget;
});
const handleTranscribe = () => {
const source = useAudioTranscribeStore.getState().source;
if (!source) return;
+ const target = transcribeTarget;
alignerDownload.runWithAligner(transcribeOptions.request.timestamps, () => {
- // The download can outlast the click: run only while Transcribe still shows the same audio.
+ // The download can outlast the click: run only while Transcribe still shows the same audio
+ // and model, so a connection picked meanwhile never receives audio nobody sent it.
if (
!activeRef.current ||
modeRef.current !== "transcribe" ||
- useAudioTranscribeStore.getState().source?.id !== source.id
+ useAudioTranscribeStore.getState().source?.id !== source.id ||
+ transcribeTargetRef.current !== target
)
return;
// A run would be refused while a model loads or another task runs: say so instead of dropping it.
@@ -1662,13 +1723,17 @@ export function AudioPage({
(id: string, meta: Parameters<typeof handleModelSelect>[1]) => {
if (meta.source === "external") {
if (busyRef.current === null) {
- invalidatePendingTtsSelection();
- setSpeakConnectionId(id);
+ if (modeRef.current === "transcribe") {
+ useAudioTranscribeStore.setState({ connectionModel: id });
+ } else {
+ invalidatePendingTtsSelection();
+ setSpeakConnectionId(id);
+ }
}
return;
}
if (modeRef.current === "speak") setSpeakConnectionId(null);
- handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta));
+ return handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta));
},
[
audioCatalog.byId,
@@ -1761,11 +1826,13 @@ export function AudioPage({
: ttsWorkflow === "convert"
? "No voice conversion model loaded."
: "No TTS model loaded."
- : sttSelected
- ? audioCapabilityLine("stt", sttReady ? "ready" : "loading")
- : transcribeRepo
- ? `${transcribeModelName} is not loaded. It loads when you transcribe.`
- : "No transcription model selected.";
+ : transcribeConnection
+ ? `Transcribes through ${transcribeConnection.providerName}.`
+ : sttSelected
+ ? audioCapabilityLine("stt", sttReady ? "ready" : "loading")
+ : transcribeRepo
+ ? `${transcribeModelName} is not loaded. It loads when you transcribe.`
+ : "No transcription model selected.";
return (
<div
@@ -1830,9 +1897,11 @@ export function AudioPage({
)
}
externalModels={
- mode === "speak" && ttsWorkflow === "speak"
- ? speakConnection.models
- : undefined
+ mode === "transcribe"
+ ? transcribeExternalModels
+ : mode === "speak" && ttsWorkflow === "speak"
+ ? speakConnection.models
+ : undefined
}
rowFilter={selectorRowFilter}
moreModels={selectorMoreModels}
@@ -1853,7 +1922,11 @@ export function AudioPage({
? false
: undefined
}
- value={(connection && speakConnection.modelId) || selectorValue}
+ value={
+ mode === "transcribe" && transcribeConnection
+ ? transcribeConnection.id
+ : (connection && speakConnection.modelId) || selectorValue
+ }
onValueChange={selectModel}
onEject={
!connection &&
@@ -1925,7 +1998,7 @@ export function AudioPage({
</div>
<div className="pointer-events-none col-start-3 flex min-w-0 items-start justify-end pr-2 pt-[var(--studio-chat-header-padding-top,11px)]">
<div className="pointer-events-auto flex min-w-0 items-center gap-2">
- {connection ? null : (
+ {connection || (mode === "transcribe" && transcribeConnection) ? null : (
<UseViaApiButton
workflow={pageWorkflow}
model={selectorValue ?? null}
diff --git a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts
index a3395d4ce0d..81cff464e22 100644
--- a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts
+++ b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts
@@ -76,6 +76,7 @@ import {
sttSidecarKeyFor,
usesNativeAudioRuntime,
} from "../catalog";
+import { useAudioTranscribeStore } from "../stores/audio-transcribe-store";
import { useAudioWorkspaceStore } from "../stores/audio-workspace-store";
import {
type AudioWorkflowId,
@@ -951,6 +952,7 @@ export function useAudioModelSlot({
sttGgufVariants.current.delete(id.toLowerCase());
}
deferredSttLoad.current = null;
+ useAudioTranscribeStore.setState({ connectionModel: null });
selectedSttRepoRef.current = id;
setSelectedSttRepo(id);
void ensureSttLoaded(id, sidecarKey, engine);
diff --git a/studio/frontend/src/features/audio/hooks/use-transcription.ts b/studio/frontend/src/features/audio/hooks/use-transcription.ts
index d9a0518af35..b2ce1c840bc 100644
--- a/studio/frontend/src/features/audio/hooks/use-transcription.ts
+++ b/studio/frontend/src/features/audio/hooks/use-transcription.ts
@@ -237,7 +237,15 @@ export function useTranscription({
const runTranscription = useCallback(
async (
source: AudioSourceSelection,
- options: { language: string; timestamps: boolean; speakers: boolean },
+ {
+ connection,
+ ...options
+ }: {
+ language: string;
+ timestamps: boolean;
+ speakers: boolean;
+ connection?: { providerId: string; modelId: string } | null;
+ },
onSourceExpired?: () => void,
confirmedVersion?: number,
) => {
@@ -251,7 +259,9 @@ export function useTranscription({
const controller = new AbortController();
transcriptionAbort.current = controller;
try {
- const target = await prepareTranscriptionModel();
+ const target = connection
+ ? { model: connection.modelId, engine: "" }
+ : await prepareTranscriptionModel();
if (!target || controller.signal.aborted || !activeRef.current) return;
setBusy("transcribing");
clearTranscript();
@@ -267,7 +277,8 @@ export function useTranscription({
source.name,
{
...target,
- device: useVoiceSettingsStore.getState().sttDevice,
+ device: connection ? "" : useVoiceSettingsStore.getState().sttDevice,
+ providerId: connection?.providerId,
...options,
signal: controller.signal,
},
diff --git a/studio/frontend/src/features/audio/speak-connection-models.ts b/studio/frontend/src/features/audio/speak-connection-models.ts
index d723b72756f..abf3a4daa2d 100644
--- a/studio/frontend/src/features/audio/speak-connection-models.ts
+++ b/studio/frontend/src/features/audio/speak-connection-models.ts
@@ -22,9 +22,13 @@ export function connectionSpeechModels(connection: {
providerType: string;
baseUrl?: string;
models: readonly string[];
+ sharedByOwner?: boolean;
}): string[] {
if (connection.providerType === "openai" && isOpenAIApi(connection.baseUrl)) {
- return OPENAI_SPEECH_MODELS;
+ // An owner-shared connection refuses any model outside the list the owner allowed (403).
+ return connection.sharedByOwner
+ ? OPENAI_SPEECH_MODELS.filter((id) => connection.models.includes(id))
+ : OPENAI_SPEECH_MODELS;
}
// Azure and other OpenAI-typed endpoints name models by deployment.
return connection.providerType === "openai" ||
diff --git a/studio/frontend/src/features/audio/stores/audio-transcribe-store.ts b/studio/frontend/src/features/audio/stores/audio-transcribe-store.ts
index d6241d34e55..ff97d6e5957 100644
--- a/studio/frontend/src/features/audio/stores/audio-transcribe-store.ts
+++ b/studio/frontend/src/features/audio/stores/audio-transcribe-store.ts
@@ -16,6 +16,7 @@ interface AudioTranscribeState {
timestamps: boolean;
speakers: boolean;
view: TranscriptView;
+ connectionModel: string | null;
}
export const useAudioTranscribeStore = create<AudioTranscribeState>()(
@@ -26,6 +27,7 @@ export const useAudioTranscribeStore = create<AudioTranscribeState>()(
timestamps: false,
speakers: true,
view: "text",
+ connectionModel: null,
}),
{ name: AUDIO_TRANSCRIBE_STORAGE_KEY, version: 1 },
),
diff --git a/studio/frontend/src/features/audio/transcribe-api.ts b/studio/frontend/src/features/audio/transcribe-api.ts
index 8fe0c8a8d79..7f4f72fef34 100644
--- a/studio/frontend/src/features/audio/transcribe-api.ts
+++ b/studio/frontend/src/features/audio/transcribe-api.ts
@@ -34,6 +34,7 @@ export async function transcribeSourceWithProgress(
language: string;
timestamps: boolean;
speakers: boolean;
+ providerId?: string;
signal: AbortSignal;
},
onProgress: (progress: TranscriptProgress) => void,
@@ -50,6 +51,7 @@ export async function transcribeSourceWithProgress(
timestamps: options.timestamps,
speakers: options.speakers,
title: title.slice(0, 255),
+ ...(options.providerId && { provider_id: options.providerId }),
}),
signal: options.signal,
});
diff --git a/studio/frontend/src/features/audio/transcribe-capabilities.ts b/studio/frontend/src/features/audio/transcribe-capabilities.ts
index 0cdb6f69e7e..bdbd24a90a3 100644
--- a/studio/frontend/src/features/audio/transcribe-capabilities.ts
+++ b/studio/frontend/src/features/audio/transcribe-capabilities.ts
@@ -62,6 +62,79 @@ function formatSize(bytes: number): string {
return `${Math.max(1, Math.round(bytes / 1e6))} MB`;
}
+const TRANSCRIBE_MODEL_ID =
+ /whisper|transcri|parakeet|voxtral-mini|(?:^|[\W_])(?:asr|stt)(?:[\W_]|$)/i;
+// OpenAI's catalog in Studio keeps only chat models.
+const OPENAI_TRANSCRIBE_MODELS = [
+ "whisper-1",
+ "gpt-4o-transcribe",
+ "gpt-4o-mini-transcribe",
+];
+
+// Azure and other OpenAI-typed endpoints name models by deployment (as speak-connection-models.ts).
+function isOpenAIApi(connection: { providerType: string; baseUrl?: string }): boolean {
+ if (connection.providerType !== "openai") return false;
+ if (!connection.baseUrl) return true;
+ try {
+ return new URL(connection.baseUrl).hostname.toLowerCase() === "api.openai.com";
+ } catch {
+ return false;
+ }
+}
+
+export function connectionTranscribeModels(
+ connection: {
+ providerType: string;
+ baseUrl?: string;
+ models: readonly string[];
+ availableModels?: readonly string[];
+ sharedByOwner?: boolean;
+ },
+ dictationModel?: string,
+): string[] {
+ const ids = [
+ ...(isOpenAIApi(connection) ? OPENAI_TRANSCRIBE_MODELS : []),
+ ...connection.models,
+ ...(connection.availableModels ?? []),
+ ].filter((id) => TRANSCRIBE_MODEL_ID.test(id) && !/tts/i.test(id));
+ if (dictationModel) ids.push(dictationModel);
+ // An owner-shared connection refuses any model outside the list the owner allowed (403).
+ const allowed = connection.sharedByOwner ? new Set(connection.models) : null;
+ return [...new Set(ids)].filter((id) => !allowed || allowed.has(id));
+}
+
+const OPENAI_UNTIMED_MODELS = new Set([
+ "gpt-4o-transcribe",
+ "gpt-4o-mini-transcribe",
+]);
+
+export function connectionTranscribeSwitches(
+ prefs: { timestamps: boolean },
+ model: string,
+): TranscribeSwitches {
+ const untimed = OPENAI_UNTIMED_MODELS.has(model);
+ return {
+ timestamps: untimed
+ ? {
+ checked: false,
+ disabled: true,
+ hint: "This model returns text only. Use whisper-1 for timestamps.",
+ }
+ : {
+ checked: prefs.timestamps,
+ disabled: false,
+ hint: "Asks the connection for the time of each line. Not every model returns them.",
+ },
+ speakers: {
+ checked: false,
+ disabled: true,
+ hint: "A connection returns no speaker labels.",
+ },
+ notice: null,
+ request: { timestamps: !untimed && prefs.timestamps, speakers: false },
+ };
+}
+
export function transcribeSwitches(
caps: SttCapabilities | null,
prefs: { timestamps: boolean; speakers: boolean },
diff --git a/studio/frontend/tests/audio-model-eject.test.ts b/studio/frontend/tests/audio-model-eject.test.ts
index 17e4a9679d9..e687ad4821d 100644
--- a/studio/frontend/tests/audio-model-eject.test.ts
+++ b/studio/frontend/tests/audio-model-eject.test.ts
@@ -287,16 +287,28 @@ test("leaving Audio during the residency read skips the transcription load", ()
});
test("Use via API is hidden while Speak runs through a connection", () => {
- assert.match(source, /\{connection \? null : \(\s*<UseViaApiButton\s+workflow=\{pageWorkflow\}/);
+ assert.match(source, /\{connection \|\|[^?]*\? null : \(\s*<UseViaApiButton\s+workflow=\{pageWorkflow\}/);
});
test("picking a connection retires a local Speak pick still downloading or planning", () => {
assert.match(
source,
- /if \(meta\.source === "external"\) \{\s*if \(busyRef\.current === null\) \{\s*invalidatePendingTtsSelection\(\);\s*setSpeakConnectionId\(id\);/,
+ /if \(meta\.source === "external"\) \{\s*if \(busyRef\.current === null\) \{\s*if \(modeRef\.current === "transcribe"\) \{\s*useAudioTranscribeStore\.setState\(\{ connectionModel: id \}\);\s*\} else \{\s*invalidatePendingTtsSelection\(\);\s*setSpeakConnectionId\(id\);/,
);
assert.match(
source,
/const invalidatePendingTtsSelection = useCallback\(\(\) => \{\s*ttsPickGeneration\.current \+= 1;\s*pendingRoutedTtsPick\.current = null;\s*invalidatePendingStagedTts\(\);/,
);
});
+
+test("a transcription waiting on the aligner runs only on the model picked at click time", () => {
+ assert.match(source, /const target = transcribeTarget;/);
+ assert.match(source, /\|\|\s*transcribeTargetRef\.current !== target\s*\)\s*return;/);
+});
+
+test("Use via API is hidden while Transcribe runs through a connection", () => {
+ assert.match(
+ source,
+ /\{connection \|\| \(mode === "transcribe" && transcribeConnection\) \? null : \(\s*<UseViaApiButton/,
+ );
+});
diff --git a/studio/frontend/tests/audio-transcribe-aligner-download.test.ts b/studio/frontend/tests/audio-transcribe-aligner-download.test.ts
index 0bea74f2aa7..1c22a823d57 100644
--- a/studio/frontend/tests/audio-transcribe-aligner-download.test.ts
+++ b/studio/frontend/tests/audio-transcribe-aligner-download.test.ts
@@ -85,16 +85,16 @@ test("only a timestamped Transcribe waits for the aligner download", () => {
);
assert.match(
page,
- /const alignerBlocks =\s*alignerDownload\.staging && transcribeOptions\.request\.timestamps;\s*const transcribeBlocker: GenerateBlocker \| null = alignerBlocks\s*\?\s*\{ reason: "Downloading the timing aligner…" \}/,
+ /const alignerBlocks =\s*!transcribeConnection &&\s*alignerDownload\.staging &&\s*transcribeOptions\.request\.timestamps;\s*const transcribeBlocker: GenerateBlocker \| null = alignerBlocks\s*\?\s*\{ reason: "Downloading the timing aligner…" \}/,
);
// The run reads the page as it is when the download ends, and only while it is still showing.
assert.match(
page,
- /const startTranscriptionRef = useRef\(startTranscription\);\s*useEffect\(\(\) => \{\s*startTranscriptionRef\.current = startTranscription;\s*\}\);/,
+ /const startTranscriptionRef = useRef\(startTranscription\);[\s\S]*?useEffect\(\(\) => \{\s*startTranscriptionRef\.current = startTranscription;\s*transcribeTargetRef\.current = transcribeTarget;\s*\}\);/,
);
assert.match(
page,
- /alignerDownload\.runWithAligner\(transcribeOptions\.request\.timestamps, \(\) => \{[\s\S]*?!activeRef\.current \|\|\s*modeRef\.current !== "transcribe" \|\|\s*useAudioTranscribeStore\.getState\(\)\.source\?\.id !== source\.id\s*\)\s*return;[\s\S]*?startTranscriptionRef\.current\(source\);/,
+ /alignerDownload\.runWithAligner\(transcribeOptions\.request\.timestamps, \(\) => \{[\s\S]*?!activeRef\.current \|\|\s*modeRef\.current !== "transcribe" \|\|\s*useAudioTranscribeStore\.getState\(\)\.source\?\.id !== source\.id \|\|\s*transcribeTargetRef\.current !== target\s*\)\s*return;[\s\S]*?startTranscriptionRef\.current\(source\);/,
);
// A download ending while a model loads cannot start the run; the user is told, not left waiting.
assert.match(
diff --git a/studio/frontend/tests/audio-transcribe-store.test.ts b/studio/frontend/tests/audio-transcribe-store.test.ts
index 669e7d69568..3e960417a1e 100644
--- a/studio/frontend/tests/audio-transcribe-store.test.ts
+++ b/studio/frontend/tests/audio-transcribe-store.test.ts
@@ -26,8 +26,8 @@ const { AUDIO_TRANSCRIBE_STORAGE_KEY, useAudioTranscribeStore } = await import(
);
test("Transcribe's settings come back after a reload and are written back under the versioned key", () => {
- assert.deepEqual(useAudioTranscribeStore.getState(), saved);
- const next = { ...saved, source: null, view: "text" as const };
+ assert.deepEqual(useAudioTranscribeStore.getState(), { ...saved, connectionModel: null });
+ const next = { ...saved, connectionModel: null, source: null, view: "text" as const };
useAudioTranscribeStore.setState(next);
assert.deepEqual(
JSON.parse(store.get(AUDIO_TRANSCRIBE_STORAGE_KEY) ?? "{}"),
diff --git a/studio/frontend/tests/speak-connection-models.test.ts b/studio/frontend/tests/speak-connection-models.test.ts
index b12f9bbe0db..c3879af52ab 100644
--- a/studio/frontend/tests/speak-connection-models.test.ts
+++ b/studio/frontend/tests/speak-connection-models.test.ts
@@ -124,3 +124,9 @@ test("a connection offers only the models that look like speech models", () => {
["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],
);
});
+
+test("an owner-shared OpenAI connection offers only the speech models the owner allowed", () => {
+ const shared = { providerType: "openai", models: ["gpt-5.5", "tts-1"], sharedByOwner: true };
+ assert.deepEqual(connectionSpeechModels(shared), ["tts-1"]);
+ assert.deepEqual(connectionSpeechModels({ ...shared, models: ["gpt-5.5"] }), []);
+});
diff --git a/studio/frontend/tests/transcribe-capabilities.test.ts b/studio/frontend/tests/transcribe-capabilities.test.ts
index 68e506dcc92..40352b2a3a3 100644
--- a/studio/frontend/tests/transcribe-capabilities.test.ts
+++ b/studio/frontend/tests/transcribe-capabilities.test.ts
@@ -7,6 +7,8 @@ import test from "node:test";
import {
type SttCapabilities,
type TranscribeSwitch,
+ connectionTranscribeModels,
+ connectionTranscribeSwitches,
transcribeSwitches,
} from "../src/features/audio/transcribe-capabilities.ts";
import {
@@ -116,3 +118,80 @@ test("languages are ISO codes with Auto first; a saved one the model lacks is se
assert.equal(transcribeLanguageFor("de", canary), "de");
assert.equal(transcribeLanguageFor("de", transcribeLanguagesFor(["en"])), "");
});
+
+test("a connection lists its speech-to-text models and the one dictation uses", () => {
+ const custom = {
+ providerType: "custom",
+ models: ["llama-3.1-8b", "whisper-1"],
+ availableModels: ["Systran/faster-whisper-small", "unsloth/Qwen3-ASR-0.6B-GGUF", "whisper-1"],
+ };
+ assert.deepEqual(connectionTranscribeModels(custom), [
+ "whisper-1",
+ "Systran/faster-whisper-small",
+ "unsloth/Qwen3-ASR-0.6B-GGUF",
+ ]);
+ assert.deepEqual(connectionTranscribeModels({ ...custom, models: [], availableModels: [] }, "large-v3"), [
+ "large-v3",
+ ]);
+ assert.deepEqual(connectionTranscribeModels({ providerType: "openai", models: ["gpt-5.5"] }), [
+ "whisper-1",
+ "gpt-4o-transcribe",
+ "gpt-4o-mini-transcribe",
+ ]);
+});
+
+test("on a connection, timestamps follow the switch and speakers stay off", () => {
+ const result = connectionTranscribeSwitches({ timestamps: true }, "whisper-1");
+ assert.deepEqual(result.request, { timestamps: true, speakers: false });
+ assert.equal(result.timestamps.disabled, false);
+ assert.equal(result.speakers.disabled, true);
+ assert.equal(result.speakers.suggestSpeakersModel, undefined);
+ assert.equal(
+ connectionTranscribeSwitches({ timestamps: false }, "whisper-1").request.timestamps,
+ false,
+ );
+});
+
+test("OpenAI's gpt-4o transcribe models never ask for timestamps", () => {
+ for (const model of ["gpt-4o-transcribe", "gpt-4o-mini-transcribe"]) {
+ const result = connectionTranscribeSwitches({ timestamps: true }, model);
+ assert.equal(result.timestamps.disabled, true);
+ assert.equal(result.timestamps.checked, false);
+ assert.equal(result.request.timestamps, false);
+ }
+});
+
+test("an owner-shared connection lists only the models the owner allowed", () => {
+ const shared = { providerType: "openai", models: ["gpt-5.5", "whisper-1"], sharedByOwner: true };
+ assert.deepEqual(connectionTranscribeModels(shared, "gpt-4o-transcribe"), ["whisper-1"]);
+ assert.deepEqual(connectionTranscribeModels({ ...shared, models: ["gpt-5.5"] }), []);
+});
+
+test("an OpenAI-typed Azure connection offers its deployments, not OpenAI's model IDs", () => {
+ const azure = {
+ providerType: "openai",
+ baseUrl: "https://contoso.openai.azure.com/openai/v1",
+ models: ["gpt-5.5", "whisper-deploy"],
+ };
+ assert.deepEqual(connectionTranscribeModels(azure), ["whisper-deploy"]);
+ assert.deepEqual(
+ connectionTranscribeModels({ ...azure, baseUrl: "https://api.openai.com/v1", models: [] }),
+ ["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe"],
+ );
+});
+
+test("a connection's Voxtral TTS model stays out of Transcribe; its realtime ASR does not", () => {
+ const custom = {
+ providerType: "custom",
+ models: [
+ "voxtral-mini-tts-latest",
+ "voxtral-small-latest",
+ "voxtral-mini-latest",
+ "voxtral-mini-transcribe-realtime",
+ ],
+ };
+ assert.deepEqual(connectionTranscribeModels(custom), [
+ "voxtral-mini-latest",
+ "voxtral-mini-transcribe-realtime",
+ ]);
+});