Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: transcribe audio through a saved connection on the Audio page (#14289)
Nilay2cfa64fda7a3
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 077da213c52..afceaa2f9ef 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -6486,6 +6486,12 @@ class TranscribeSourceRequest(BaseModel):timestamps: bool = Field(False, description = "Ask for segment and word timestamps")speakers: bool = Field(False, description = "Keep who spoke each segment, when the model tells")title: Optional[str] = Field(None, max_length = 255, description = "Defaults to the source's name")+ provider_id: Optional[str] = Field(+ None,+ max_length = 128,+ description = "Saved connection ID. When set, the audio goes to that connection's "+ "/audio/transcriptions as `model`; engine and device are ignored.",+ )class AudioAnalyzeRequest(TranscribeSourceRef):diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 4e38a6d0522..0867209d61a 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -27630,14 +27630,93 @@ def _source_transcript(result: dict, source, speakers: bool) -> dict:return out+async def _external_source_transcript(+ body: TranscribeSourceRequest, source, request: Request+) -> dict:+ from core.inference import audio_inputs, audio_metrics++ def _encode():+ path = audio_inputs.prepared_path(source, 16000)+ flac, _ = audio_inputs.encode_wav(path.read_bytes(), "flac")+ return audio_metrics.wav_seconds(path), flac++ try:+ duration, audio = await asyncio.to_thread(_encode)+ except audio_inputs.AudioInputError as exc:+ raise _audio_source_error(exc) from None+ if len(audio) > _MAX_AUDIO_RAW_BYTES:+ raise HTTPException(+ status_code = 413,+ detail = (+ f"This audio comes to {len(audio) / 2**20:.0f} MB, past the "+ f"{_MAX_AUDIO_RAW_BYTES // 2**20} MB a connection takes. Trim it, or pick an "+ "on-device model."+ ),+ )+ response = await _external_stt_transcription(+ provider_id = body.provider_id,+ raw = audio,+ filename = "audio.flac",+ content_type = "audio/flac",+ model = body.model,+ language = body.language,+ response_format = "verbose_json" if body.timestamps else "json",+ timestamp_granularities = ["segment", "word"] if body.timestamps else None,+ encrypted_api_key = None,+ request = request,+ )+ data = json.loads(response.body) if "json" in response.media_type else {}+ if not isinstance(data, dict) or not isinstance(data.get("text"), str):+ raise HTTPException(status_code = 502, detail = "The connection returned no transcript.")++ def spans(key: str, text_key: str) -> list[dict]:+ return [+ {+ "start": s.get("start"),+ "end": s.get("end"),+ text_key: str(s.get(text_key, "")).strip(),+ }+ for s in data.get(key) or ()+ if isinstance(s, dict)+ ]++ result = {+ "text": data["text"].strip(),+ "model": body.model,+ "language": data.get("language") or body.language,+ "duration": duration,+ }+ if body.timestamps:+ result.update(segments = spans("segments", "text"), words = spans("words", "word"))+ return _source_transcript(result, source, False)++@studio_router.post("/audio/transcribe/source")async def transcribe_audio_source(- body: TranscribeSourceRequest, current_subject: str = Depends(get_current_subject)+ body: TranscribeSourceRequest,+ request: Request,+ current_subject: str = Depends(get_current_subject),):"""Transcribe Audio page audio named by id, streamed as NDJSON and saved to history."""from core.inference import audio_inputs, stt_capabilities, stt_detailsfrom core.inference.transcript_stream import stream_transcript+ if body.provider_id is not None:+ try:+ source = await asyncio.to_thread(+ audio_inputs.resolve_source, body.source.model_dump(exclude_none = True)+ )+ except audio_inputs.AudioInputError as exc:+ raise _audio_source_error(exc) from None+ return StreamingResponse(+ stream_transcript(+ lambda _on_progress: _external_source_transcript(body, source, request),+ body.title or source.name,+ ),+ media_type = "application/x-ndjson",+ headers = {"Cache-Control": "no-store", "X-Accel-Buffering": "no"},+ )+engine = body.engine or await asyncio.to_thread(_stt_engine_for_model, body.model)serving_engine = _resolve_serving_stt_engine(engine)caps = await asyncio.to_thread(stt_capabilities.capabilities_for, body.model, serving_engine)diff --git a/studio/backend/tests/test_transcribe_source_route.py b/studio/backend/tests/test_transcribe_source_route.pyindex 1a7fc7681aa..569c1ab3054 100644--- a/studio/backend/tests/test_transcribe_source_route.py+++ b/studio/backend/tests/test_transcribe_source_route.py@@ -224,3 +224,82 @@ def test_transcript_routes_get_rename_and_archive(stub):cleared = client.patch(url, json = {"speaker_names": {"S01": None}, "archived": True}).json()assert "speaker_names" not in cleared and cleared["archived"] is Trueassert client.get("/api/inference/audio/transcripts/" + "e" * 32).status_code == 404+++def connection(monkeypatch):+ import httpx++ from core.inference import external_provider, providers++ config = {+ "provider_type": "custom",+ "display_name": "Whisper Box",+ "base_url": "http://stt.local:8000/v1",+ "is_enabled": True,+ }+ monkeypatch.setattr(+ inference.providers_db,+ "get_provider",+ lambda pid: dict(config) if pid == "conn-1" else None,+ )+ for module in (inference, providers):+ monkeypatch.setattr(module, "validate_provider_base_url", lambda url: url)+ monkeypatch.setattr(inference, "resolve_provider_api_key_or_400", lambda *a, **k: "sk-test")+ calls = []+ reply = {+ "task": "transcribe",+ "language": "english",+ "duration": 1.0,+ "text": " Hello there.",+ "segments": [{"id": 0, "seek": 0, "start": 0.0, "end": 1.0, "text": " Hello there."}],+ "words": [{"word": " Hello", "start": 0.0, "end": 0.4}],+ }++ class _Http:+ async def post(self, url, **kwargs):+ calls.append({"url": url, **kwargs})+ return httpx.Response(200, json = reply, request = httpx.Request("POST", url))++ monkeypatch.setattr(external_provider, "_client", lambda: _Http())+ return calls+++def test_a_connection_transcribes_the_source_and_saves_it(stub, connection):+ input_id = _input(ALICE)+ with _client(ALICE) as client:+ response, events = _post(+ client,+ provider_id = "conn-1",+ model = "whisper-1",+ source = {"input_id": input_id},+ timestamps = True,+ )+ assert response.status_code == 200, response.text+ assert not (stub.paths or stub.bytes or stub.loads)+ (call,) = connection+ assert call["url"] == "http://stt.local:8000/v1/audio/transcriptions"+ name, audio, content_type = call["files"]["file"]+ assert (name, content_type, audio[:4]) == ("audio.flac", "audio/flac", b"fLaC")+ assert call["data"]["model"] == "whisper-1"+ assert call["data"]["response_format"] == "verbose_json"+ assert call["data"]["timestamp_granularities[]"] == ["segment", "word"]+ complete = events[-1]+ assert complete["type"] == "complete" and complete["text"] == "Hello there."+ assert complete["segments"] == [{"start": 0.0, "end": 1.0, "text": "Hello there."}]+ assert complete["words"] == [{"start": 0.0, "end": 0.4, "word": "Hello"}]+ assert complete["timestamps"] is True and complete["duration"] == pytest.approx(1.0, abs = 0.05)+ record = complete["record"]+ assert (record["model"], record["language"]) == ("whisper-1", "english")+ assert record["source"] == {"kind": "input", "id": input_id, "name": "meeting.webm"}+++def test_a_source_past_the_connection_limit_is_refused_before_upload(stub, connection, monkeypatch):+ monkeypatch.setattr(inference, "_MAX_AUDIO_RAW_BYTES", 1024)+ with _client(ALICE) as client:+ _, events = _post(+ client, provider_id = "conn-1", model = "whisper-1", source = {"input_id": _input(ALICE)}+ )+ assert events[-1]["type"] == "error"+ assert "a connection takes" in events[-1]["message"]+ assert not connection and not stub.bytesdiff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsxindex 484d08220aa..ee0c14e52e4 100644--- a/studio/frontend/src/features/audio/audio-page.tsx+++ b/studio/frontend/src/features/audio/audio-page.tsx@@ -17,8 +17,11 @@ import { useSidebar } from "@/components/ui/sidebar";import { usePlatformStore } from "@/config/env";import {type InferenceStatusResponse,+ buildExternalModelId,getInferenceStatus,+ isDecisionConnection,syncKeepModelsLoaded,+ useExternalProvidersStore,} from "@/features/chat";import { useHubInventory } from "@/features/hub";import { useInventoryVersion } from "@/features/hub/stores/inventory-events";@@ -26,7 +29,11 @@ import { ModelSelector } from "@/features/model-picker/components/model-selectorimport { ggufQuantLabel } from "@/features/model-picker/model-config/model-identity";import { AUDIO_CATALOG } from "@/features/model-picker/components/model-selector/model-catalog";import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";-import type { ModelOption } from "@/features/model-picker/components/model-selector/types";+import type {+ ExternalModelOption,+ ModelOption,+} from "@/features/model-picker/components/model-selector/types";+import { useVoiceSettingsStore } from "@/features/settings";import { sttModelSize } from "@/features/settings/stores/stt-model-catalog";import { usePersistedChoice } from "@/hooks/use-persisted-choice";import { usePersistedToggle } from "@/hooks/use-persisted-toggle";@@ -121,7 +128,12 @@ import type { AudioSourceStatus } from "./hooks/audio-source-state";import { useAlignerDownload } from "./hooks/use-aligner-download";import { useTranscribeCapabilities } from "./hooks/use-transcribe-capabilities";import { useAudioTranscribeStore } from "./stores/audio-transcribe-store";-import { SPEAKERS_MODEL_NAME, transcribeSwitches } from "./transcribe-capabilities";+import {+ SPEAKERS_MODEL_NAME,+ connectionTranscribeModels,+ connectionTranscribeSwitches,+ transcribeSwitches,+} from "./transcribe-capabilities";import {transcribeLanguageFor,transcribeLanguagesFor,@@ -1431,18 +1443,52 @@ export function AudioPage({string | null>(null);const transcribeRepo = (selectedSttRepo ?? lastSttRepo) || null;+ const connections = useExternalProvidersStore((s) => s.providers);+ const connectionsEnabled = useExternalProvidersStore(+ (s) => s.connectionsEnabled,+ );+ const dictationProviderId = useVoiceSettingsStore((s) => s.sttProviderId);+ const dictationModel = useVoiceSettingsStore((s) => s.sttProviderModel);+ const transcribeExternalModels = useMemo<ExternalModelOption[]>(+ () =>+ connectionsEnabled+ ? connections+ .filter((c) => !isDecisionConnection(c))+ .flatMap((c) =>+ connectionTranscribeModels(+ c,+ c.id === dictationProviderId ? dictationModel.trim() : undefined,+ ).map((model) => ({+ id: buildExternalModelId(c.id, model),+ name: model,+ providerId: c.id,+ providerName: c.name,+ providerType: c.providerType,+ })),+ )+ : [],+ [connections, connectionsEnabled, dictationProviderId, dictationModel],+ );+ const connectionModel = useAudioTranscribeStore(+ (state) => state.connectionModel,+ );+ const transcribeConnection =+ transcribeExternalModels.find((model) => model.id === connectionModel) ??+ null;const transcribeCaps = useTranscribeCapabilities(- transcribeRepo,+ transcribeConnection ? null : transcribeRepo,transcribeRepo ? sttEngineFor(transcribeRepo) : null,);- const transcribeOptions = transcribeSwitches(- transcribeCaps.caps,- transcribePrefs,- { loading: transcribeCaps.loading, hasModel: transcribeRepo !== null },- );- const transcribeModelName = transcribeRepo?.split("/").pop() ?? "";+ const transcribeOptions = transcribeConnection+ ? connectionTranscribeSwitches(transcribePrefs, transcribeConnection.name)+ : transcribeSwitches(transcribeCaps.caps, transcribePrefs, {+ loading: transcribeCaps.loading,+ hasModel: transcribeRepo !== null,+ });+ const transcribeModelName =+ transcribeConnection?.name ?? transcribeRepo?.split("/").pop() ?? "";const transcribeLoadsFirst = Boolean(- transcribeRepo && !(sttSelected && sttReady),+ !transcribeConnection && transcribeRepo && !(sttSelected && sttReady),);const transcribeNotice = !transcribeLoadsFirst? transcribeOptions.notice@@ -1453,7 +1499,7 @@ export function AudioPage({.filter(Boolean).join(" ");const transcribeLanguages = transcribeLanguagesFor(- audioCppModelFor(transcribeRepo)?.languages,+ transcribeConnection ? null : audioCppModelFor(transcribeRepo)?.languages,);const transcribeLanguage = transcribeLanguageFor(transcribePrefs.language,@@ -1482,15 +1528,18 @@ export function AudioPage({transcribeCaps.refresh,transcribeOptions.request.timestamps,);+ // A connection never uses the local aligner, so its download does not hold Transcribe.const alignerBlocks =- alignerDownload.staging && transcribeOptions.request.timestamps;+ !transcribeConnection &&+ alignerDownload.staging &&+ transcribeOptions.request.timestamps;const transcribeBlocker: GenerateBlocker | null = alignerBlocks? { reason: "Downloading the timing aligner…" }: busy === "loading"? { reason: "Waiting for the model to finish loading." }: busy !== null? null- : !transcribeRepo+ : !transcribeRepo && !transcribeConnection? {reason: "Pick a speech-to-text model to transcribe.",actions: [chooseModelAction, ...recommendedSttActions],@@ -1529,7 +1578,13 @@ export function AudioPage({const canTranscribe =mode === "transcribe" && busy === null && transcribeBlocker === null;const startTranscription = (source: AudioSourceSelection) => {- const { request } = transcribeOptions;+ const request = {+ ...transcribeOptions.request,+ connection: transcribeConnection && {+ providerId: transcribeConnection.providerId,+ modelId: transcribeConnection.name,+ },+ };void runTranscription(source,{ language: transcribeLanguage, ...request },@@ -1543,18 +1598,24 @@ export function AudioPage({});};const startTranscriptionRef = useRef(startTranscription);+ const transcribeTarget = transcribeConnection?.id ?? transcribeRepo;+ const transcribeTargetRef = useRef(transcribeTarget);useEffect(() => {startTranscriptionRef.current = startTranscription;+ transcribeTargetRef.current = transcribeTarget;});const handleTranscribe = () => {const source = useAudioTranscribeStore.getState().source;if (!source) return;+ const target = transcribeTarget;alignerDownload.runWithAligner(transcribeOptions.request.timestamps, () => {- // The download can outlast the click: run only while Transcribe still shows the same audio.+ // The download can outlast the click: run only while Transcribe still shows the same audio+ // and model, so a connection picked meanwhile never receives audio nobody sent it.if (!activeRef.current ||modeRef.current !== "transcribe" ||- useAudioTranscribeStore.getState().source?.id !== source.id+ useAudioTranscribeStore.getState().source?.id !== source.id ||+ transcribeTargetRef.current !== target)return;// A run would be refused while a model loads or another task runs: say so instead of dropping it.@@ -1662,13 +1723,17 @@ export function AudioPage({(id: string, meta: Parameters<typeof handleModelSelect>[1]) => {if (meta.source === "external") {if (busyRef.current === null) {- invalidatePendingTtsSelection();- setSpeakConnectionId(id);+ if (modeRef.current === "transcribe") {+ useAudioTranscribeStore.setState({ connectionModel: id });+ } else {+ invalidatePendingTtsSelection();+ setSpeakConnectionId(id);+ }}return;}if (modeRef.current === "speak") setSpeakConnectionId(null);- handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta));+ return handleModelSelect(id, withCatalogPipelineTag(audioCatalog.byId, id, meta));},[audioCatalog.byId,@@ -1761,11 +1826,13 @@ export function AudioPage({: ttsWorkflow === "convert"? "No voice conversion model loaded.": "No TTS model loaded."- : sttSelected- ? audioCapabilityLine("stt", sttReady ? "ready" : "loading")- : transcribeRepo- ? `${transcribeModelName} is not loaded. It loads when you transcribe.`- : "No transcription model selected.";+ : transcribeConnection+ ? `Transcribes through ${transcribeConnection.providerName}.`+ : sttSelected+ ? audioCapabilityLine("stt", sttReady ? "ready" : "loading")+ : transcribeRepo+ ? `${transcribeModelName} is not loaded. It loads when you transcribe.`+ : "No transcription model selected.";return (<div@@ -1830,9 +1897,11 @@ export function AudioPage({)}externalModels={- mode === "speak" && ttsWorkflow === "speak"- ? speakConnection.models- : undefined+ mode === "transcribe"+ ? transcribeExternalModels+ : mode === "speak" && ttsWorkflow === "speak"+ ? speakConnection.models+ : undefined}rowFilter={selectorRowFilter}moreModels={selectorMoreModels}@@ -1853,7 +1922,11 @@ export function AudioPage({? false: undefined}- value={(connection && speakConnection.modelId) || selectorValue}+ value={+ mode === "transcribe" && transcribeConnection+ ? transcribeConnection.id+ : (connection && speakConnection.modelId) || selectorValue+ }onValueChange={selectModel}onEject={!connection &&@@ -1925,7 +1998,7 @@ export function AudioPage({</div><div className="pointer-events-none col-start-3 flex min-w-0 items-start justify-end pr-2 pt-[var(--studio-chat-header-padding-top,11px)]"><div className="pointer-events-auto flex min-w-0 items-center gap-2">- {connection ? null : (+ {connection || (mode === "transcribe" && transcribeConnection) ? null : (<UseViaApiButtonworkflow={pageWorkflow}model={selectorValue ?? null}diff --git a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.tsindex a3395d4ce0d..81cff464e22 100644--- a/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts+++ b/studio/frontend/src/features/audio/hooks/use-audio-model-slot.ts@@ -76,6 +76,7 @@ import {sttSidecarKeyFor,usesNativeAudioRuntime,} from "../catalog";+import { useAudioTranscribeStore } from "../stores/audio-transcribe-store";import { useAudioWorkspaceStore } from "../stores/audio-workspace-store";import {type AudioWorkflowId,@@ -951,6 +952,7 @@ export function useAudioModelSlot({sttGgufVariants.current.delete(id.toLowerCase());}deferredSttLoad.current = null;+ useAudioTranscribeStore.setState({ connectionModel: null });selectedSttRepoRef.current = id;setSelectedSttRepo(id);void ensureSttLoaded(id, sidecarKey, engine);diff --git a/studio/frontend/src/features/audio/hooks/use-transcription.ts b/studio/frontend/src/features/audio/hooks/use-transcription.tsindex d9a0518af35..b2ce1c840bc 100644--- a/studio/frontend/src/features/audio/hooks/use-transcription.ts+++ b/studio/frontend/src/features/audio/hooks/use-transcription.ts@@ -237,7 +237,15 @@ export function useTranscription({const runTranscription = useCallback(async (source: AudioSourceSelection,- options: { language: string; timestamps: boolean; speakers: boolean },+ {+ connection,+ ...options+ }: {+ language: string;+ timestamps: boolean;+ speakers: boolean;+ connection?: { providerId: string; modelId: string } | null;+ },onSourceExpired?: () => void,confirmedVersion?: number,) => {@@ -251,7 +259,9 @@ export function useTranscription({const controller = new AbortController();transcriptionAbort.current = controller;try {- const target = await prepareTranscriptionModel();+ const target = connection+ ? { model: connection.modelId, engine: "" }+ : await prepareTranscriptionModel();if (!target || controller.signal.aborted || !activeRef.current) return;setBusy("transcribing");clearTranscript();@@ -267,7 +277,8 @@ export function useTranscription({source.name,{...target,- device: useVoiceSettingsStore.getState().sttDevice,+ device: connection ? "" : useVoiceSettingsStore.getState().sttDevice,+ providerId: connection?.providerId,...options,signal: controller.signal,},diff --git a/studio/frontend/src/features/audio/speak-connection-models.ts b/studio/frontend/src/features/audio/speak-connection-models.tsindex d723b72756f..abf3a4daa2d 100644--- a/studio/frontend/src/features/audio/speak-connection-models.ts+++ b/studio/frontend/src/features/audio/speak-connection-models.ts@@ -22,9 +22,13 @@ export function connectionSpeechModels(connection: {providerType: string;baseUrl?: string;models: readonly string[];+ sharedByOwner?: boolean;}): string[] {if (connection.providerType === "openai" && isOpenAIApi(connection.baseUrl)) {- return OPENAI_SPEECH_MODELS;+ // An owner-shared connection refuses any model outside the list the owner allowed (403).+ return connection.sharedByOwner+ ? OPENAI_SPEECH_MODELS.filter((id) => connection.models.includes(id))+ : OPENAI_SPEECH_MODELS;}// Azure and other OpenAI-typed endpoints name models by deployment.return connection.providerType === "openai" ||diff --git a/studio/frontend/src/features/audio/stores/audio-transcribe-store.ts b/studio/frontend/src/features/audio/stores/audio-transcribe-store.tsindex d6241d34e55..ff97d6e5957 100644--- a/studio/frontend/src/features/audio/stores/audio-transcribe-store.ts+++ b/studio/frontend/src/features/audio/stores/audio-transcribe-store.ts@@ -16,6 +16,7 @@ interface AudioTranscribeState {timestamps: boolean;speakers: boolean;view: TranscriptView;+ connectionModel: string | null;}export const useAudioTranscribeStore = create<AudioTranscribeState>()(@@ -26,6 +27,7 @@ export const useAudioTranscribeStore = create<AudioTranscribeState>()(timestamps: false,speakers: true,view: "text",+ connectionModel: null,}),{ name: AUDIO_TRANSCRIBE_STORAGE_KEY, version: 1 },),diff --git a/studio/frontend/src/features/audio/transcribe-api.ts b/studio/frontend/src/features/audio/transcribe-api.tsindex 8fe0c8a8d79..7f4f72fef34 100644--- a/studio/frontend/src/features/audio/transcribe-api.ts+++ b/studio/frontend/src/features/audio/transcribe-api.ts@@ -34,6 +34,7 @@ export async function transcribeSourceWithProgress(language: string;timestamps: boolean;speakers: boolean;+ providerId?: string;signal: AbortSignal;},onProgress: (progress: TranscriptProgress) => void,@@ -50,6 +51,7 @@ export async function transcribeSourceWithProgress(timestamps: options.timestamps,speakers: options.speakers,title: title.slice(0, 255),+ ...(options.providerId && { provider_id: options.providerId }),}),signal: options.signal,});diff --git a/studio/frontend/src/features/audio/transcribe-capabilities.ts b/studio/frontend/src/features/audio/transcribe-capabilities.tsindex 0cdb6f69e7e..bdbd24a90a3 100644--- a/studio/frontend/src/features/audio/transcribe-capabilities.ts+++ b/studio/frontend/src/features/audio/transcribe-capabilities.ts@@ -62,6 +62,79 @@ function formatSize(bytes: number): string {return `${Math.max(1, Math.round(bytes / 1e6))} MB`;}+const TRANSCRIBE_MODEL_ID =+ /whisper|transcri|parakeet|voxtral-mini|(?:^|[\W_])(?:asr|stt)(?:[\W_]|$)/i;+// OpenAI's catalog in Studio keeps only chat models.+const OPENAI_TRANSCRIBE_MODELS = [+ "whisper-1",+ "gpt-4o-transcribe",+ "gpt-4o-mini-transcribe",+];++// Azure and other OpenAI-typed endpoints name models by deployment (as speak-connection-models.ts).+function isOpenAIApi(connection: { providerType: string; baseUrl?: string }): boolean {+ if (connection.providerType !== "openai") return false;+ if (!connection.baseUrl) return true;+ try {+ return new URL(connection.baseUrl).hostname.toLowerCase() === "api.openai.com";+ } catch {+ return false;+ }+}++export function connectionTranscribeModels(+ connection: {+ providerType: string;+ baseUrl?: string;+ models: readonly string[];+ availableModels?: readonly string[];+ sharedByOwner?: boolean;+ },+ dictationModel?: string,+): string[] {+ const ids = [+ ...(isOpenAIApi(connection) ? OPENAI_TRANSCRIBE_MODELS : []),+ ...connection.models,+ ...(connection.availableModels ?? []),+ ].filter((id) => TRANSCRIBE_MODEL_ID.test(id) && !/tts/i.test(id));+ if (dictationModel) ids.push(dictationModel);+ // An owner-shared connection refuses any model outside the list the owner allowed (403).+ const allowed = connection.sharedByOwner ? new Set(connection.models) : null;+ return [...new Set(ids)].filter((id) => !allowed || allowed.has(id));+}++const OPENAI_UNTIMED_MODELS = new Set([+ "gpt-4o-transcribe",+ "gpt-4o-mini-transcribe",+]);++export function connectionTranscribeSwitches(+ prefs: { timestamps: boolean },+ model: string,+): TranscribeSwitches {+ const untimed = OPENAI_UNTIMED_MODELS.has(model);+ return {+ timestamps: untimed+ ? {+ checked: false,+ disabled: true,+ hint: "This model returns text only. Use whisper-1 for timestamps.",+ }+ : {+ checked: prefs.timestamps,+ disabled: false,+ hint: "Asks the connection for the time of each line. Not every model returns them.",+ },+ speakers: {+ checked: false,+ disabled: true,+ hint: "A connection returns no speaker labels.",+ },+ notice: null,+ request: { timestamps: !untimed && prefs.timestamps, speakers: false },+ };+}+export function transcribeSwitches(caps: SttCapabilities | null,prefs: { timestamps: boolean; speakers: boolean },diff --git a/studio/frontend/tests/audio-model-eject.test.ts b/studio/frontend/tests/audio-model-eject.test.tsindex 17e4a9679d9..e687ad4821d 100644--- a/studio/frontend/tests/audio-model-eject.test.ts+++ b/studio/frontend/tests/audio-model-eject.test.ts@@ -287,16 +287,28 @@ test("leaving Audio during the residency read skips the transcription load", ()});test("Use via API is hidden while Speak runs through a connection", () => {- assert.match(source, /\{connection \? null : \(\s*<UseViaApiButton\s+workflow=\{pageWorkflow\}/);+ assert.match(source, /\{connection \|\|[^?]*\? null : \(\s*<UseViaApiButton\s+workflow=\{pageWorkflow\}/);});test("picking a connection retires a local Speak pick still downloading or planning", () => {assert.match(source,- /if \(meta\.source === "external"\) \{\s*if \(busyRef\.current === null\) \{\s*invalidatePendingTtsSelection\(\);\s*setSpeakConnectionId\(id\);/,+ /if \(meta\.source === "external"\) \{\s*if \(busyRef\.current === null\) \{\s*if \(modeRef\.current === "transcribe"\) \{\s*useAudioTranscribeStore\.setState\(\{ connectionModel: id \}\);\s*\} else \{\s*invalidatePendingTtsSelection\(\);\s*setSpeakConnectionId\(id\);/,);assert.match(source,/const invalidatePendingTtsSelection = useCallback\(\(\) => \{\s*ttsPickGeneration\.current \+= 1;\s*pendingRoutedTtsPick\.current = null;\s*invalidatePendingStagedTts\(\);/,);});++test("a transcription waiting on the aligner runs only on the model picked at click time", () => {+ assert.match(source, /const target = transcribeTarget;/);+ assert.match(source, /\|\|\s*transcribeTargetRef\.current !== target\s*\)\s*return;/);+});++test("Use via API is hidden while Transcribe runs through a connection", () => {+ assert.match(+ source,+ /\{connection \|\| \(mode === "transcribe" && transcribeConnection\) \? null : \(\s*<UseViaApiButton/,+ );+});diff --git a/studio/frontend/tests/audio-transcribe-aligner-download.test.ts b/studio/frontend/tests/audio-transcribe-aligner-download.test.tsindex 0bea74f2aa7..1c22a823d57 100644--- a/studio/frontend/tests/audio-transcribe-aligner-download.test.ts+++ b/studio/frontend/tests/audio-transcribe-aligner-download.test.ts@@ -85,16 +85,16 @@ test("only a timestamped Transcribe waits for the aligner download", () => {);assert.match(page,- /const alignerBlocks =\s*alignerDownload\.staging && transcribeOptions\.request\.timestamps;\s*const transcribeBlocker: GenerateBlocker \| null = alignerBlocks\s*\?\s*\{ reason: "Downloading the timing aligner…" \}/,+ /const alignerBlocks =\s*!transcribeConnection &&\s*alignerDownload\.staging &&\s*transcribeOptions\.request\.timestamps;\s*const transcribeBlocker: GenerateBlocker \| null = alignerBlocks\s*\?\s*\{ reason: "Downloading the timing aligner…" \}/,);// The run reads the page as it is when the download ends, and only while it is still showing.assert.match(page,- /const startTranscriptionRef = useRef\(startTranscription\);\s*useEffect\(\(\) => \{\s*startTranscriptionRef\.current = startTranscription;\s*\}\);/,+ /const startTranscriptionRef = useRef\(startTranscription\);[\s\S]*?useEffect\(\(\) => \{\s*startTranscriptionRef\.current = startTranscription;\s*transcribeTargetRef\.current = transcribeTarget;\s*\}\);/,);assert.match(page,- /alignerDownload\.runWithAligner\(transcribeOptions\.request\.timestamps, \(\) => \{[\s\S]*?!activeRef\.current \|\|\s*modeRef\.current !== "transcribe" \|\|\s*useAudioTranscribeStore\.getState\(\)\.source\?\.id !== source\.id\s*\)\s*return;[\s\S]*?startTranscriptionRef\.current\(source\);/,+ /alignerDownload\.runWithAligner\(transcribeOptions\.request\.timestamps, \(\) => \{[\s\S]*?!activeRef\.current \|\|\s*modeRef\.current !== "transcribe" \|\|\s*useAudioTranscribeStore\.getState\(\)\.source\?\.id !== source\.id \|\|\s*transcribeTargetRef\.current !== target\s*\)\s*return;[\s\S]*?startTranscriptionRef\.current\(source\);/,);// A download ending while a model loads cannot start the run; the user is told, not left waiting.assert.match(diff --git a/studio/frontend/tests/audio-transcribe-store.test.ts b/studio/frontend/tests/audio-transcribe-store.test.tsindex 669e7d69568..3e960417a1e 100644--- a/studio/frontend/tests/audio-transcribe-store.test.ts+++ b/studio/frontend/tests/audio-transcribe-store.test.ts@@ -26,8 +26,8 @@ const { AUDIO_TRANSCRIBE_STORAGE_KEY, useAudioTranscribeStore } = await import();test("Transcribe's settings come back after a reload and are written back under the versioned key", () => {- assert.deepEqual(useAudioTranscribeStore.getState(), saved);- const next = { ...saved, source: null, view: "text" as const };+ assert.deepEqual(useAudioTranscribeStore.getState(), { ...saved, connectionModel: null });+ const next = { ...saved, connectionModel: null, source: null, view: "text" as const };useAudioTranscribeStore.setState(next);assert.deepEqual(JSON.parse(store.get(AUDIO_TRANSCRIBE_STORAGE_KEY) ?? "{}"),diff --git a/studio/frontend/tests/speak-connection-models.test.ts b/studio/frontend/tests/speak-connection-models.test.tsindex b12f9bbe0db..c3879af52ab 100644--- a/studio/frontend/tests/speak-connection-models.test.ts+++ b/studio/frontend/tests/speak-connection-models.test.ts@@ -124,3 +124,9 @@ test("a connection offers only the models that look like speech models", () => {["gpt-4o-mini-tts", "tts-1", "tts-1-hd"],);});++test("an owner-shared OpenAI connection offers only the speech models the owner allowed", () => {+ const shared = { providerType: "openai", models: ["gpt-5.5", "tts-1"], sharedByOwner: true };+ assert.deepEqual(connectionSpeechModels(shared), ["tts-1"]);+ assert.deepEqual(connectionSpeechModels({ ...shared, models: ["gpt-5.5"] }), []);+});diff --git a/studio/frontend/tests/transcribe-capabilities.test.ts b/studio/frontend/tests/transcribe-capabilities.test.tsindex 68e506dcc92..40352b2a3a3 100644--- a/studio/frontend/tests/transcribe-capabilities.test.ts+++ b/studio/frontend/tests/transcribe-capabilities.test.ts@@ -7,6 +7,8 @@ import test from "node:test";import {type SttCapabilities,type TranscribeSwitch,+ connectionTranscribeModels,+ connectionTranscribeSwitches,transcribeSwitches,} from "../src/features/audio/transcribe-capabilities.ts";import {@@ -116,3 +118,80 @@ test("languages are ISO codes with Auto first; a saved one the model lacks is seassert.equal(transcribeLanguageFor("de", canary), "de");assert.equal(transcribeLanguageFor("de", transcribeLanguagesFor(["en"])), "");});++test("a connection lists its speech-to-text models and the one dictation uses", () => {+ const custom = {+ providerType: "custom",+ models: ["llama-3.1-8b", "whisper-1"],+ availableModels: ["Systran/faster-whisper-small", "unsloth/Qwen3-ASR-0.6B-GGUF", "whisper-1"],+ };+ assert.deepEqual(connectionTranscribeModels(custom), [+ "whisper-1",+ "Systran/faster-whisper-small",+ "unsloth/Qwen3-ASR-0.6B-GGUF",+ ]);+ assert.deepEqual(connectionTranscribeModels({ ...custom, models: [], availableModels: [] }, "large-v3"), [+ "large-v3",+ ]);+ assert.deepEqual(connectionTranscribeModels({ providerType: "openai", models: ["gpt-5.5"] }), [+ "whisper-1",+ "gpt-4o-transcribe",+ "gpt-4o-mini-transcribe",+ ]);+});++test("on a connection, timestamps follow the switch and speakers stay off", () => {+ const result = connectionTranscribeSwitches({ timestamps: true }, "whisper-1");+ assert.deepEqual(result.request, { timestamps: true, speakers: false });+ assert.equal(result.timestamps.disabled, false);+ assert.equal(result.speakers.disabled, true);+ assert.equal(result.speakers.suggestSpeakersModel, undefined);+ assert.equal(+ connectionTranscribeSwitches({ timestamps: false }, "whisper-1").request.timestamps,+ false,+ );+});++test("OpenAI's gpt-4o transcribe models never ask for timestamps", () => {+ for (const model of ["gpt-4o-transcribe", "gpt-4o-mini-transcribe"]) {+ const result = connectionTranscribeSwitches({ timestamps: true }, model);+ assert.equal(result.timestamps.disabled, true);+ assert.equal(result.timestamps.checked, false);+ assert.equal(result.request.timestamps, false);+ }+});++test("an owner-shared connection lists only the models the owner allowed", () => {+ const shared = { providerType: "openai", models: ["gpt-5.5", "whisper-1"], sharedByOwner: true };+ assert.deepEqual(connectionTranscribeModels(shared, "gpt-4o-transcribe"), ["whisper-1"]);+ assert.deepEqual(connectionTranscribeModels({ ...shared, models: ["gpt-5.5"] }), []);+});++test("an OpenAI-typed Azure connection offers its deployments, not OpenAI's model IDs", () => {+ const azure = {+ providerType: "openai",+ baseUrl: "https://contoso.openai.azure.com/openai/v1",+ models: ["gpt-5.5", "whisper-deploy"],+ };+ assert.deepEqual(connectionTranscribeModels(azure), ["whisper-deploy"]);+ assert.deepEqual(+ connectionTranscribeModels({ ...azure, baseUrl: "https://api.openai.com/v1", models: [] }),+ ["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe"],+ );+});++test("a connection's Voxtral TTS model stays out of Transcribe; its realtime ASR does not", () => {+ const custom = {+ providerType: "custom",+ models: [+ "voxtral-mini-tts-latest",+ "voxtral-small-latest",+ "voxtral-mini-latest",+ "voxtral-mini-transcribe-realtime",+ ],+ };+ assert.deepEqual(connectionTranscribeModels(custom), [+ "voxtral-mini-latest",+ "voxtral-mini-transcribe-realtime",+ ]);+});