TypeScript
fix(web): first-run voice setup offers a model for the user's language (#32035)
Julius Marminge15b72755956c
diff --git a/apps/desktop/src/app/DesktopAppIdentity.test.ts b/apps/desktop/src/app/DesktopAppIdentity.test.tsindex 32e7b3b3118..74d134c0b10 100644--- a/apps/desktop/src/app/DesktopAppIdentity.test.ts+++ b/apps/desktop/src/app/DesktopAppIdentity.test.ts@@ -42,6 +42,7 @@ const layerElectronApp = (calls: ElectronAppCalls) =>metadata: Effect.die("unexpected metadata read"),name: Effect.succeed("T3 Code"),systemLocale: Effect.succeed("en-US"),+ preferredSystemLanguages: Effect.succeed(["en-US"]),whenReady: Effect.void,requestSingleInstanceLock: Effect.succeed(true),quit: Effect.void,diff --git a/apps/desktop/src/app/DesktopLifecycle.test.ts b/apps/desktop/src/app/DesktopLifecycle.test.tsindex fa1bcd8bbf0..30df96c58ad 100644--- a/apps/desktop/src/app/DesktopLifecycle.test.ts+++ b/apps/desktop/src/app/DesktopLifecycle.test.ts@@ -39,6 +39,7 @@ function layerElectronApp(metadata: Effect.die("unexpected metadata read"),name: Effect.succeed("T3 Code"),systemLocale: Effect.succeed("en-US"),+ preferredSystemLanguages: Effect.succeed(["en-US"]),whenReady: Effect.void,requestSingleInstanceLock: Effect.succeed(true),quit,diff --git a/apps/desktop/src/electron/ElectronApp.ts b/apps/desktop/src/electron/ElectronApp.tsindex d677baa56dc..188f53c4be8 100644--- a/apps/desktop/src/electron/ElectronApp.ts+++ b/apps/desktop/src/electron/ElectronApp.ts@@ -46,6 +46,7 @@ export class ElectronApp extends Context.Service<readonly name: Effect.Effect<string>;/** Packaged builds ship only en-US; app.getLocale() and Intl defaults do not reflect the OS locale. */readonly systemLocale: Effect.Effect<string>;+ readonly preferredSystemLanguages: Effect.Effect<ReadonlyArray<string>>;readonly whenReady: Effect.Effect<void, ElectronAppWhenReadyError>;readonly requestSingleInstanceLock: Effect.Effect<boolean>;readonly quit: Effect.Effect<void>;@@ -155,6 +156,7 @@ export const make = ElectronApp.of({}return toSystemLocaleTag(identifier, hourCycle);}),+ preferredSystemLanguages: Effect.sync(() => Electron.app.getPreferredSystemLanguages()),whenReady: Effect.gen(function* () {const isPackaged = Electron.app.isPackaged;yield* Effect.tryPromise({diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.tsindex a46dbafab0f..66c558e86fc 100644--- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts+++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts@@ -38,6 +38,7 @@ import {getLocalEnvironmentBearerToken,installRelayTelemetryEnabled,getSystemLocale,+ getPreferredSystemLanguages,getWindowFullscreenState,openExternal,readMedia,@@ -94,6 +95,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"yield* ipc.handleSync(getAppBranding);yield* installRelayTelemetryEnabled();yield* ipc.handleSync(getSystemLocale);+ yield* ipc.handleSync(getPreferredSystemLanguages);yield* ipc.handleSync(getWindowFullscreenState);yield* ipc.handleSync(getLocalEnvironmentBootstraps);yield* ipc.handleSync(getLocalEnvironmentEnabled);diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.tsindex 1d7edaf6832..840e5f11680 100644--- a/apps/desktop/src/ipc/channels.ts+++ b/apps/desktop/src/ipc/channels.ts@@ -34,6 +34,7 @@ export const CLI_COMMAND_UNINSTALL_CHANNEL = "desktop:cli-command-uninstall";export const GET_APP_BRANDING_CHANNEL = "desktop:get-app-branding";export const GET_RELAY_TELEMETRY_ENABLED_CHANNEL = "desktop:get-relay-telemetry-enabled";export const GET_SYSTEM_LOCALE_CHANNEL = "desktop:get-system-locale";+export const GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL = "desktop:get-preferred-system-languages";export const GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL = "desktop:get-local-environment-bootstraps";export const GET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:get-local-environment-enabled";export const SET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:set-local-environment-enabled";diff --git a/apps/desktop/src/ipc/methods/window.ts b/apps/desktop/src/ipc/methods/window.tsindex 2833abca438..9580b29b94b 100644--- a/apps/desktop/src/ipc/methods/window.ts+++ b/apps/desktop/src/ipc/methods/window.ts@@ -106,6 +106,15 @@ export const getSystemLocale = DesktopIpc.makeSyncIpcMethod({}),});+export const getPreferredSystemLanguages = DesktopIpc.makeSyncIpcMethod({+ channel: IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,+ result: Schema.Array(Schema.String),+ handler: Effect.fn("desktop.ipc.window.getPreferredSystemLanguages")(function* () {+ const electronApp = yield* ElectronApp.ElectronApp;+ return yield* electronApp.preferredSystemLanguages;+ }),+});+export const getWindowFullscreenState = DesktopIpc.makeSyncIpcMethod({channel: IpcChannels.GET_WINDOW_FULLSCREEN_STATE_CHANNEL,result: Schema.Boolean,diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.tsindex 800f1e744c9..2d2bd4f5164 100644--- a/apps/desktop/src/preload.ts+++ b/apps/desktop/src/preload.ts@@ -107,6 +107,12 @@ contextBridge.exposeInMainWorld("desktopBridge", {const result = ipcRenderer.sendSync(IpcChannels.GET_SYSTEM_LOCALE_CHANNEL);return typeof result === "string" ? result : null;},+ getPreferredSystemLanguages: () => {+ const result: unknown = ipcRenderer.sendSync(+ IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,+ );+ return Array.isArray(result) ? result.filter((tag) => typeof tag === "string") : [];+ },getLocalEnvironmentBootstraps: () => {const result = ipcRenderer.sendSync(IpcChannels.GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL);if (!Array.isArray(result)) {diff --git a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.tsindex 1180c0fa709..9201bc01110 100644--- a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts+++ b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts@@ -37,6 +37,7 @@ function layerElectronApp(metadata: Effect.die("unexpected metadata read"),name: Effect.succeed("T3 Code"),systemLocale: Effect.succeed("en-US"),+ preferredSystemLanguages: Effect.succeed(["en-US"]),whenReady: Effect.void,requestSingleInstanceLock: Effect.succeed(true),quit: Effect.void,diff --git a/apps/desktop/src/window/DesktopApplicationMenu.test.ts b/apps/desktop/src/window/DesktopApplicationMenu.test.tsindex 9d8efd05666..c1bb89b759e 100644--- a/apps/desktop/src/window/DesktopApplicationMenu.test.ts+++ b/apps/desktop/src/window/DesktopApplicationMenu.test.ts@@ -31,6 +31,7 @@ const layerElectronApp = Layer.succeed(ElectronApp.ElectronApp, {metadata: Effect.die("unexpected metadata read"),name: Effect.succeed("T3 Code"),systemLocale: Effect.succeed("en-US"),+ preferredSystemLanguages: Effect.succeed(["en-US"]),whenReady: Effect.void,requestSingleInstanceLock: Effect.succeed(true),quit: Effect.void,diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.tsindex 972c401ce30..7b347728370 100644--- a/apps/server/src/speech/SpeechService.test.ts+++ b/apps/server/src/speech/SpeechService.test.ts@@ -25,6 +25,7 @@ const native = vi.hoisted(() => ({supportsLongForm: false,supportsTranslation: true,supportsVocabulary: false,+ languages: ["en", "fr"] as readonly string[],begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),feed: vi.fn(async (_pcm: Float32Array) => ({revision: 1,@@ -119,6 +120,7 @@ beforeEach(() => {native.backend = "Vulkan0";native.supportsVocabulary = false;native.supportsStreaming = true;+ native.languages = ["en", "fr"];readyModels.clear();readyModels.add("test-model");readyModels.add("fallback-model");@@ -171,6 +173,60 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr}).pipe(Effect.provide(layer)),);+it.effect("sends the chosen language in the spelling the loaded model advertises", () =>+ Effect.gen(function* () {+ const speech = yield* SpeechService.SpeechService;+ // Nemotron Streaming 3.5 advertises regional tags while the catalog lists bare codes.+ native.languages = ["en-US", "en-GB", "fr-FR"];+ yield* Effect.gen(function* () {+ const stream = yield* speech.startStream({+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,+ speechLanguage: "fr",+ });+ expect(native.begin).toHaveBeenLastCalledWith(+ { timestamps: "none", language: "fr-FR" },+ true,+ "test.gguf",+ );+ expect(yield* stream.finish).toBe("hello");+ }).pipe(Effect.scoped);+ native.supportsStreaming = false;+ expect(+ yield* speech.transcribe(pcm(), {+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,+ speechLanguage: "fr",+ }),+ ).toBe("hello");+ expect(native.transcribe).toHaveBeenLastCalledWith(+ expect.any(Float32Array),+ { timestamps: "none", language: "fr-FR" },+ true,+ "test.gguf",+ );+ // A language the model does not advertise reaches it unchanged, so the model reports it.+ yield* speech.updateAcceleration("cpu");+ native.languages = ["en-US"];+ native.transcribe.mockRejectedValueOnce(new Error("transcribe_run: unsupported language"));+ const rejected = yield* Effect.result(+ speech.transcribe(pcm(), { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }),+ );+ expect(Result.isFailure(rejected) && rejected.failure).toMatchObject({+ _tag: "SpeechOperationError",+ operation: "inference",+ });+ expect(native.transcribe).toHaveBeenLastCalledWith(+ expect.any(Float32Array),+ { timestamps: "none", language: "fr" },+ true,+ "test.gguf",+ );+ expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual([+ ["test.gguf", "auto"],+ ["test.gguf", "cpu"],+ ]);+ }).pipe(Effect.provide(layer)),+);+it.effect.each(["cancellation", "timeout"] as const)("releases a stalled model preparation on %s",(reason) =>diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.tsindex c1f762f2608..24eb6ef408d 100644--- a/apps/server/src/speech/SpeechService.ts+++ b/apps/server/src/speech/SpeechService.ts@@ -32,6 +32,7 @@ import {type SpeechModel,} from "./model.ts";import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";+import { nativeSpeechLanguage } from "./nativeLanguage.ts";import { SILERO_VAD } from "./silero.ts";import { isSpeechPlatformSupported } from "./platform.ts";import * as SpeechModelStore from "./SpeechModelStore.ts";@@ -613,11 +614,12 @@ const make = Effect.gen(function* () {try: (signal) => {// Interruption kills the isolated process; the next request reloads the model.signal.addEventListener("abort", () => dropModel(inference), { once: true });+ const hint = nativeSpeechLanguage(inference.languages, language);return inference.transcribe(pcm,{timestamps: "none",- ...(language === "auto" ? {} : { language }),+ ...(hint ? { language: hint } : {}),...(translateToEnglish? { task: "translate" as const, targetLanguage: "en" as const }: {}),@@ -749,10 +751,11 @@ const make = Effect.gen(function* () {streamModel.supportsVocabulary && !translateToEnglish && customWords.length? customWords: undefined;+ const hint = nativeSpeechLanguage(streamModel.languages, language);await streamModel.begin({timestamps: "none",- ...(language === "auto" ? {} : { language }),+ ...(hint ? { language: hint } : {}),...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),...(vocabulary ? { vocabulary } : {}),},diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.tsindex 97aa1b56602..8712c63ed34 100644--- a/apps/server/src/speech/native.test.ts+++ b/apps/server/src/speech/native.test.ts@@ -4,7 +4,7 @@ import { expect, it } from "vite-plus/test";import { loadNativeSpeechModel } from "./native.ts";const fixture = (transcribe: string, maxAudioMs = 0, supportsVocabulary = false) =>- `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs} }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;+ `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs}, languages: ["en-US", "de-DE"] }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;const fixtureWithUnrelatedIpcMessage = () =>`data:text/javascript,${encodeURIComponent(`@@ -38,6 +38,7 @@ it("returns transcription from an isolated process and disposes it", async () =>fixture("async (pcm) => ({ text: String(pcm[0]) })"),);try {+ expect(model.languages).toEqual(["en-US", "de-DE"]);expect(await model.transcribe(new Float32Array([0.25]),diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.tsindex fae308333b2..8331e1b5e0e 100644--- a/apps/server/src/speech/native.ts+++ b/apps/server/src/speech/native.ts@@ -147,6 +147,7 @@ process.on("message", async (message) => {throw new Error("The selected speech GPU is unavailable.");model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,+ languages: model.capabilities.languages,supportsStreaming: model.capabilities.supportsStreaming,supportsLongForm: model.supports("long_form"),supportsTranslation: model.capabilities.supportsTranslate,@@ -194,6 +195,7 @@ type Reply = {readonly supportsTranslation?: boolean;readonly supportsVocabulary?: boolean;readonly backend?: string;+ readonly languages?: unknown;readonly revision?: number;readonly preview?: SpeechStreamText | null;};@@ -391,6 +393,9 @@ async function loadNativeSpeechModelOnce(}return {backend: loaded.backend ?? "unknown",+ languages: Array.isArray(loaded.languages)+ ? loaded.languages.filter((entry): entry is string => typeof entry === "string")+ : [],supportsStreaming: loaded.supportsStreaming === true,supportsLongForm: loaded.supportsLongForm === true,supportsTranslation: loaded.supportsTranslation === true,diff --git a/apps/server/src/speech/nativeLanguage.test.ts b/apps/server/src/speech/nativeLanguage.test.tsnew file mode 100644index 00000000000..3632878285d--- /dev/null+++ b/apps/server/src/speech/nativeLanguage.test.ts@@ -0,0 +1,18 @@+import { expect, it } from "vite-plus/test";++import { nativeSpeechLanguage } from "./nativeLanguage.ts";++const nemotron = ["en-US", "en-GB", "es-US", "es-ES", "de-DE"];++it.each([+ ["auto omits the hint", nemotron, "auto", undefined],+ ["exact entry", ["en", "de"], "de", "de"],+ ["exact entry before a regional one", ["de-DE", "de"], "de", "de"],+ ["case and underscore normalize to the advertised spelling", nemotron, "DE_de", "de-DE"],+ ["bare code takes the first regional entry", nemotron, "en", "en-US"],+ ["regional request with another region passes through", nemotron, "de-AT", "de-AT"],+ ["unadvertised language passes through", nemotron, "fr", "fr"],+ ["model without an advertised list passes through", [], "de", "de"],+])("%s", (_name, advertised, language, expected) => {+ expect(nativeSpeechLanguage(advertised, language)).toBe(expected);+});diff --git a/apps/server/src/speech/nativeLanguage.ts b/apps/server/src/speech/nativeLanguage.tsnew file mode 100644index 00000000000..0ff013de970--- /dev/null+++ b/apps/server/src/speech/nativeLanguage.ts@@ -0,0 +1,21 @@+/**+ * transcribe.cpp accepts a language only when it equals an entry of the GGUF's+ * `general.languages` byte for byte. Nemotron Streaming 3.5 lists `de-DE` where the catalog+ * says `de`, so the request carries the loaded model's own spelling. "auto" omits the hint.+ */+export function nativeSpeechLanguage(+ advertised: readonly string[],+ language: string,+): string | undefined {+ if (language === "auto") return undefined;+ const tag = (value: string) => value.toLowerCase().replaceAll("_", "-");+ const wanted = tag(language);+ return (+ advertised.find((entry) => entry === language) ??+ advertised.find((entry) => tag(entry) === wanted) ??+ (wanted.includes("-")+ ? undefined+ : advertised.find((entry) => tag(entry).split("-", 1)[0] === wanted)) ??+ language+ );+}diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsxindex 69022a93fbe..be106a25159 100644--- a/apps/web/src/components/chat/ChatComposer.tsx+++ b/apps/web/src/components/chat/ChatComposer.tsx@@ -3709,9 +3709,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)step={speechInput.setup.step}status={speechInput.status}model={speechInput.setup.model}+ language={speechInput.setup.language}downloading={speechInput.setup.downloading}error={speechInput.setup.error}onOpenChange={speechInput.setup.setOpen}+ onRetryCatalog={speechInput.setup.retryCatalog}onDownload={() => void speechInput.setup.download()}onCancelDownload={() => void speechInput.setup.cancelDownload()}onStartRecording={() => void speechInput.setup.startRecording()}diff --git a/apps/web/src/components/chat/VoiceInputSetup.tsx b/apps/web/src/components/chat/VoiceInputSetup.tsxindex 7b30783cf21..cf697979fc2 100644--- a/apps/web/src/components/chat/VoiceInputSetup.tsx+++ b/apps/web/src/components/chat/VoiceInputSetup.tsx@@ -13,6 +13,7 @@ import { useClientSettings, updateClientSettings } from "../../hooks/useSettingsimport { useEnvironment } from "../../state/environments";import { AuthSettingsWriteScope } from "@t3tools/contracts";import { useEnvironmentScope } from "../../state/session";+import { languageLabel } from "../settings/SpeechLanguageSelect";import { Button } from "../ui/button";import { Dialog } from "../ui/dialog";import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";@@ -24,9 +25,12 @@ export function VoiceInputSetup(props: {step: number;status: EnvironmentSpeechStatus | null;model: EnvironmentSpeechModel | null;+ /** The language the offered model was chosen for. */+ language: string | null;downloading: boolean;error: string | null;onOpenChange(open: boolean): void;+ onRetryCatalog: (() => void) | null;onDownload(): void;onCancelDownload(): void;onStartRecording(): void;@@ -85,49 +89,55 @@ export function VoiceInputSetup(props: {<WizardPanel>{props.step === 0 ? (<section className="space-y-3 text-sm">- <div className="rounded-lg border border-border/70 bg-card/60 p-4">- <div className="flex items-center justify-between gap-2">- <h3 className="font-medium">{status?.model ?? "English transcription model"}</h3>- {status?.modelId === "handy-computer/parakeet-unified-en-0.6b-gguf" ? (- <span className="text-xs font-medium text-primary">- Recommended for English- </span>+ {model ? (+ <div className="rounded-lg border border-border/70 bg-card/60 p-4">+ <div className="flex items-center justify-between gap-2">+ <h3 className="font-medium">{model.name}</h3>+ {model?.recommended && props.language ? (+ <span className="text-xs font-medium text-primary">+ Recommended for {languageLabel(props.language)}+ </span>+ ) : null}+ </div>+ <p className="mt-2 text-muted-foreground">+ Download the {Math.round(model.size / 1024 / 1024)} MB model to{" "}+ {environment?.label ?? "your transcription environment"}. Microphone audio is+ sent there for transcription, as chosen in Voice settings.+ </p>+ <p className="mt-2 text-muted-foreground">+ Silero (1.2 MB), used for silence filtering and pause detection, downloads+ alongside the speech model. An installed copy is reused.+ </p>+ {props.downloading ? (+ <div role="status" className="mt-4 space-y-1 text-muted-foreground">+ <div+ role="progressbar"+ aria-label="Speech model download progress"+ aria-valuemin={0}+ aria-valuemax={100}+ aria-valuenow={progress ?? undefined}+ className="h-1 overflow-hidden rounded-full bg-muted"+ >+ <div+ className="h-full bg-primary transition-[width] duration-200"+ style={{ width: `${progress ?? 0}%` }}+ />+ </div>+ <p className="text-xs">+ {model?.state === "verifying"+ ? "Verifying download…"+ : progress === null+ ? "Starting download…"+ : `${progress}% downloaded`}+ </p>+ </div>) : null}</div>- <p className="mt-2 text-muted-foreground">- Download the {status ? Math.round(status.size / 1024 / 1024) : 697} MB model to{" "}- {environment?.label ?? "your transcription environment"}. Microphone audio is sent- there for transcription, as chosen in Voice settings.+ ) : !props.error ? (+ <p role="status" className="text-muted-foreground">+ Loading transcription models…</p>- <p className="mt-2 text-muted-foreground">- Silero (1.2 MB), used for silence filtering and pause detection, downloads- alongside the speech model. An installed copy is reused.- </p>- {props.downloading ? (- <div role="status" className="mt-4 space-y-1 text-muted-foreground">- <div- role="progressbar"- aria-label="Speech model download progress"- aria-valuemin={0}- aria-valuemax={100}- aria-valuenow={progress ?? undefined}- className="h-1 overflow-hidden rounded-full bg-muted"- >- <div- className="h-full bg-primary transition-[width] duration-200"- style={{ width: `${progress ?? 0}%` }}- />- </div>- <p className="text-xs">- {model?.state === "verifying"- ? "Verifying download…"- : progress === null- ? "Starting download…"- : `${progress}% downloaded`}- </p>- </div>- ) : null}- </div>+ ) : null}{props.error ? (<p role="alert" className="text-destructive">{props.error}@@ -208,25 +218,30 @@ export function VoiceInputSetup(props: {{props.step === 0 ? (<><Button variant="outline" onClick={() => openSettings("speech-models")}>- Choose another model or language+ {model ? "Choose another model or language" : "Choose a model or language"}</Button>+ {props.onRetryCatalog ? (+ <Button onClick={props.onRetryCatalog}>Retry loading models</Button>+ ) : null}{props.downloading ? (<Button variant="outline" disabled={!canManage} onClick={props.onCancelDownload}>Cancel download</Button>) : null}- <Button- disabled={!canManage || props.downloading || !status}- onClick={props.onDownload}- >- {props.downloading- ? "Downloading…"- : props.error- ? "Retry download"- : canManage- ? "Download model"- : "Ask an environment admin to download"}- </Button>+ {model ? (+ <Button+ disabled={!canManage || props.downloading || !status}+ onClick={props.onDownload}+ >+ {props.downloading+ ? "Downloading…"+ : props.error+ ? "Try again"+ : canManage+ ? "Download model"+ : "Ask an environment admin to download"}+ </Button>+ ) : null}</>) : (<>diff --git a/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx b/apps/web/src/components/preview/PreviewAnnotationSpeech.tsxindex 639d11dd0fd..8d52066cd5f 100644--- a/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx+++ b/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx@@ -184,9 +184,11 @@ export function PreviewAnnotationSpeech({step={speech.setup.step}status={speech.status}model={speech.setup.model}+ language={speech.setup.language}downloading={speech.setup.downloading}error={speech.setup.error}onOpenChange={speech.setup.setOpen}+ onRetryCatalog={speech.setup.retryCatalog}onDownload={() => void speech.setup.download()}onCancelDownload={() => void speech.setup.cancelDownload()}onStartRecording={() => void speech.setup.startRecording()}diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsxindex 479fb73c7f3..969420ff2e0 100644--- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx+++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx@@ -161,7 +161,10 @@ it("tests the selected model after download and preserves its transcript acrossopen: false,step: 0,model: null,+ language: null,downloading: false,+ catalogFailed: false,+ retryCatalog: null,error: null,setOpen: () => {},download: async () => {},diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsxindex 5d2a8fbed22..4a3df08f40f 100644--- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx+++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx@@ -47,6 +47,7 @@ import { useEffect, useRef, useState, type ReactNode } from "react";import { useStableCallback } from "~/hooks/useStableCallback";import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";+import { preferredSpeechLanguages } from "../../speech/preferredSpeechLanguages";import {chooseAudioDestination,chooseTranscriptCleanup,@@ -760,7 +761,7 @@ function SpeechModelBrowser(props: {const [downloadedOnly, setDownloadedOnly] = useState(false);const [languageSearch, setLanguageSearch] = useState("");const [selection, setSelection] = useState<string[] | null>(null);- const preferred = navigator.languages.map((language) => language.toLowerCase().split("-")[0]!);+ const [preferred] = useState(preferredSpeechLanguages);const supportedLanguages = [...new Set(models.flatMap((model) => model.languages))];const defaultLanguage = preferred.find((code) => supportedLanguages.includes(code));const selectedLanguages = selection ?? (defaultLanguage ? [defaultLanguage] : []);@@ -1077,9 +1078,11 @@ function TranscriptionTest(props: {step={speech.setup.step}status={speech.status}model={speech.setup.model}+ language={speech.setup.language}downloading={speech.setup.downloading}error={speech.setup.error}onOpenChange={speech.setup.setOpen}+ onRetryCatalog={speech.setup.retryCatalog}onDownload={() => void speech.setup.download()}onCancelDownload={() => void speech.setup.cancelDownload()}onStartRecording={() => void speech.setup.startRecording()}diff --git a/apps/web/src/speech/preferredSpeechLanguages.ts b/apps/web/src/speech/preferredSpeechLanguages.tsnew file mode 100644index 00000000000..548e7955ff0--- /dev/null+++ b/apps/web/src/speech/preferredSpeechLanguages.ts@@ -0,0 +1,10 @@+// Packaged desktop pins `navigator.languages` to en-US, so the OS's own list leads.+export function preferredSpeechLanguages() {+ const bridge = window.desktopBridge;+ let system = bridge?.getPreferredSystemLanguages?.() ?? [];+ if (!system.length) {+ const locale = bridge?.getSystemLocale?.();+ if (locale) system = [locale];+ }+ return [...system, ...navigator.languages].map((tag) => tag.toLowerCase().split(/[-_]/)[0]!);+}diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsxindex 4586f1f5d5c..508ef9aa7b6 100644--- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx+++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx@@ -1,19 +1,24 @@import { useLayoutEffect } from "react";import { expect, it, vi } from "vite-plus/test";+import { page, userEvent } from "vite-plus/test/browser";+import * as Deferred from "effect/Deferred";import * as Effect from "effect/Effect";import * as Option from "effect/Option";import type { PreparedConnection } from "@t3tools/client-runtime/connection";+import { getEnvironmentSpeechModels } from "@t3tools/client-runtime/voice-input";import {DEFAULT_CLIENT_SETTINGS,DEFAULT_SERVER_SETTINGS,EnvironmentId,+ type EnvironmentSpeechModel,type ProjectId,type SpeechCustomWords,type SpeechPostProcessingOptions,type SpeechTranscriptionOptions,} from "@t3tools/contracts";-import { act, render } from "../../test/render";+import { act, render, stubGlobal } from "../../test/render";+import { VoiceInputSetup } from "../components/chat/VoiceInputSetup";import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";import {@@ -36,7 +41,12 @@ const mocks = vi.hoisted(() => ({statusRequests: [] as string[],statusUnavailable: false,modelState: "ready" as "ready" | "missing-model",- download: vi.fn<() => Promise<void>>(),+ statusModelId: "test-model",+ models: [] as EnvironmentSpeechModel[],+ catalogOffline: false,+ download: vi.fn<(modelId: string) => Promise<void>>(),+ selected: [] as string[],+ selectionFails: false,platforms: [] as string[],microphoneRequests: [] as string[],transcribeCalls: [] as string[],@@ -80,6 +90,7 @@ vi.mock("../state/environments", () => ({: null,}));vi.mock("../hooks/useSettings", () => ({+ updateClientSettings: vi.fn(),useClientSettingsHydrated: () => true,useClientSettings: (selector?: (settings: Record<string, unknown>) => unknown) => {const settings = { ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings };@@ -109,7 +120,7 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({return {supported: true,state: mocks.modelState,- modelId: "test-model",+ modelId: mocks.statusModelId,model: "Test",size: 1,supportsStreaming: false,@@ -119,8 +130,24 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({gpuDevices: [],};}),- downloadEnvironmentSpeechModel: () => Effect.promise(mocks.download),- getEnvironmentSpeechModels: () => Effect.succeed({ models: [] }),+ downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>+ Effect.promise(async () => {+ await mocks.download(modelId);+ return { models: mocks.models };+ }),+ selectEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>+ Effect.sync(() => {+ mocks.selected.push(modelId);+ if (mocks.selectionFails) throw new Error("selection failed");+ mocks.statusModelId = modelId;+ return { models: mocks.models };+ }),+ getEnvironmentSpeechModels: vi.fn(() =>+ Effect.sync(() => {+ if (mocks.catalogOffline) throw new Error("Catalog offline");+ return { models: mocks.models };+ }),+ ),postProcessEnvironmentTranscript: (prepared: PreparedConnection,transcript: string,@@ -189,7 +216,12 @@ function setup(options: { readonly destination?: string | null } = {}) {mocks.statusRequests = [];mocks.statusUnavailable = false;mocks.modelState = "ready";+ mocks.statusModelId = "test-model";+ mocks.models = [];+ mocks.catalogOffline = false;mocks.download.mockReset();+ mocks.selected = [];+ mocks.selectionFails = false;mocks.platforms = [];mocks.microphoneRequests = [];mocks.transcribeCalls = [];@@ -213,6 +245,7 @@ function Probe(props: {readonly projectId?: string;readonly projectName?: string;readonly cleanup?: boolean;+ readonly showWizard?: boolean;}) {const value = useEnvironmentSpeechInput({environmentId: THREAD_ENV as EnvironmentId,@@ -230,7 +263,18 @@ function Probe(props: {useLayoutEffect(() => {mocks.voice = value;});- return null;+ return props.showWizard ? (+ <VoiceInputSetup+ {...value.setup}+ environmentId={value.transcriptionEnvironmentId}+ status={value.status}+ onOpenChange={value.setup.setOpen}+ onRetryCatalog={value.setup.retryCatalog}+ onDownload={value.setup.download}+ onCancelDownload={value.setup.cancelDownload}+ onStartRecording={value.setup.startRecording}+ />+ ) : null;}function expectNoAudioSent() {@@ -333,6 +377,8 @@ it("retries a failed transcription on the same chosen environment", async () =>it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {setup({ destination: "env-a" });mocks.modelState = "missing-model";+ mocks.settings = { speechLanguage: "en" };+ mocks.models = [catalogModel(mocks.statusModelId, ["en"], true)];let finishDownload!: () => void;mocks.download.mockReturnValue(new Promise<void>((resolve) => {@@ -360,6 +406,171 @@ it("reopens setup after reconnecting during a download and ignores its stale comexpect(mocks.voice.setup.error).toBeNull();});+function catalogModel(+ id: string,+ languages: readonly string[],+ recommended: boolean,+): EnvironmentSpeechModel {+ return {+ id,+ name: id,+ description: "",+ size: 1,+ languages,+ accuracy: 0,+ speed: 0,+ recommended,+ supportsStreaming: false,+ supportsLanguageDetection: languages.length > 1,+ supportsTranslation: false,+ active: false,+ state: "downloadable",+ };+}++it.each([+ ["the browser prefers German", ["de-DE"], undefined, "auto"],+ ["the desktop exposes no locale methods", ["de-DE"], {}, "auto"],+ [+ "an older preload exposes only the OS locale",+ ["en-US"],+ { getSystemLocale: () => "de-DE" },+ "auto",+ ],+ [+ "the desktop OS prefers German",+ ["en-US"],+ { getPreferredSystemLanguages: () => ["de-DE"] },+ "auto",+ ],+ ["German is the chosen Voice language", ["en-US"], undefined, "de"],+])(+ "offers and downloads a German-capable model when %s",+ async (_, languages, bridge, speechLanguage) => {+ setup({ destination: "env-a" });+ mocks.settings = { speechLanguage };+ using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);+ using _bridge = stubGlobal("desktopBridge", bridge);+ const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";+ const german = "handy-computer/nemotron-3.5-asr-streaming-0.6b-gguf";+ mocks.modelState = "missing-model";+ mocks.statusModelId = englishDefault;+ mocks.models = [+ { ...catalogModel(englishDefault, ["en"], true), active: true },+ catalogModel(german, ["en", "de", "fr"], true),+ catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),+ ];+ mocks.download.mockResolvedValue(undefined);+ await using _screen = await render(<Probe />);+ await act(() => mocks.voice.start());+ expect(mocks.voice.setup.open).toBe(true);+ await act(() => mocks.voice.setup.download());+ expect(mocks.download).toHaveBeenCalledExactlyOnceWith(german);+ expect(mocks.voice.setup.model?.id).toBe(german);+ },+);++it.each(["cancel", "failed restart"] as const)(+ "closes pending catalog setup on %s",+ async (action) => {+ setup({ destination: "env-a" });+ mocks.modelState = "missing-model";+ mocks.settings = { speechLanguage: "en" };+ mocks.models = [catalogModel("current", ["en"], true)];+ const response = Deferred.makeUnsafe<{ models: EnvironmentSpeechModel[] }>();+ vi.mocked(getEnvironmentSpeechModels).mockReturnValueOnce(Deferred.await(response));+ await using _screen = await render(<Probe showWizard />);+ let firstStart!: Promise<void>;+ await act(() => {+ firstStart = mocks.voice.start();+ });+ await expect.element(page.getByText("Loading transcription models…")).toBeVisible();+ if (action === "cancel") {+ await act(() => mocks.voice.cancel());+ } else {+ mocks.statusUnavailable = true;+ await act(() => mocks.voice.start());+ mocks.statusUnavailable = false;+ }+ expect(mocks.voice.setup.open).toBe(false);++ await act(() => mocks.voice.start());+ Deferred.doneUnsafe(+ response,+ Effect.succeed({ models: [catalogModel("stale", ["en"], true)] }),+ );+ await act(() => firstStart);+ await expect.element(page.getByRole("heading", { name: "current" })).toBeVisible();+ expect(mocks.microphoneRequests).toEqual([]);+ },+);++it.each([true, false])(+ "never downloads the English default when no German recommendation is available (catalog offline: %s)",+ async (offline) => {+ setup({ destination: "env-a" });+ mocks.settings = { speechLanguage: "de" };+ mocks.modelState = "missing-model";+ mocks.statusModelId = "english";+ mocks.models = [catalogModel("english", ["en"], true)];+ mocks.catalogOffline = offline;+ await using _screen = await render(<Probe showWizard />);+ await act(() => mocks.voice.start());+ const wizard = page.getByRole("dialog", { name: "Set up voice input" });+ const retry = wizard.getByRole("button", { name: "Retry loading models" });+ const download = wizard.getByRole("button", { name: "Download model" });+ await expect.element(wizard.getByRole("alert")).toBeVisible();+ await expect.element(download).not.toBeInTheDocument();+ if (offline) {+ mocks.catalogOffline = false;+ mocks.models.push(catalogModel("german", ["de"], true));+ // Another client can finish setup while the catalog is offline.+ mocks.modelState = "ready";+ await userEvent.click(retry);+ await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();+ await expect.element(download).toBeEnabled();+ }+ await expect.element(retry).not.toBeInTheDocument();+ expect(mocks.download).not.toHaveBeenCalled();+ expect(mocks.microphoneRequests).toEqual([]);+ },+);++it.each([+ ["selects the offered model once its download installed it", "installed", ["german"], null],+ [+ "does not advance a cancelled download when another model is ready",+ "downloadable",+ [],+ "The model is not ready. Try downloading it again.",+ ],+ [+ "reports a failed selection apart from the download",+ "installed",+ ["german"],+ "The model downloaded, but this environment could not select it. Try again.",+ ],+] as const)("%s", async (_, landedState, selected, error) => {+ setup({ destination: "env-a" });+ using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(["de-DE"]);+ mocks.selectionFails = error !== null && landedState === "installed";+ mocks.modelState = "missing-model";+ mocks.statusModelId = "english";+ mocks.models = [catalogModel("english", ["en"], true), catalogModel("german", ["de"], true)];+ mocks.download.mockImplementation(async () => {+ mocks.models = [mocks.models[0]!, { ...mocks.models[1]!, state: landedState }];+ mocks.modelState = "ready";+ });+ await using _screen = await render(<Probe />);+ await act(() => mocks.voice.start());+ expect(mocks.voice.setup.language).toBe("de");+ await act(() => mocks.voice.setup.download());+ expect(mocks.selected).toEqual(selected);+ expect(mocks.voice.setup.error).toBe(error);+ expect(mocks.voice.setup.downloading).toBe(false);+ expect(mocks.voice.setup.step).toBe(landedState === "installed" && !error ? 1 : 0);+});+it("does not resend a failed recording after consent is revoked", async () => {setup({ destination: "env-a" });mocks.transcripts = [new Error("inference failed")];diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.tsindex 0cb468dae50..c569c14500b 100644--- a/apps/web/src/speech/useEnvironmentSpeechInput.ts+++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts@@ -4,6 +4,7 @@ import {getEnvironmentSpeechModels,getEnvironmentSpeechStatus,postProcessEnvironmentTranscript,+ selectEnvironmentSpeechModel,VoiceInputController,voiceInputBlocksSubmission,type VoiceDraftSnapshot,@@ -22,7 +23,7 @@ import type {SpeechTranscriptionOptions,} from "@t3tools/contracts";import { resolveProjectSettings } from "@t3tools/shared/projectSettings";-import { resolveSpeechVocabulary } from "@t3tools/shared/speech";+import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";import { useNavigate } from "@tanstack/react-router";import * as Option from "effect/Option";import { useEffect, useRef, useState } from "react";@@ -37,6 +38,7 @@ import {import { useEnvironment } from "../state/environments";import { useEnvironmentScope, usePreparedConnection } from "../state/session";import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";+import { preferredSpeechLanguages } from "./preferredSpeechLanguages";import { runEnvironmentSpeech } from "./runEnvironmentSpeech";import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";@@ -46,7 +48,9 @@ const INITIAL_SETUP = {open: false,step: 0,model: null as EnvironmentSpeechModel | null,+ language: null as string | null,downloading: false,+ catalogFailed: false,error: null as string | null,};@@ -387,11 +391,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {};}, [prepared, cloudProvider]);const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;+ const setupModelId = setup.model?.id;// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model downloaduseEffect(() => {- if (!setup.open || !setup.downloading || !prepared || !currentStatus?.supported) return;- const modelId = currentStatus.modelId;+ if (!setup.open || !setup.downloading || !prepared || !setupModelId) return;let disposed = false;let refreshing = false;const timer = window.setInterval(() => {@@ -400,8 +404,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared)).then((result) => {if (disposed) return;- const model = result.models.find((candidate) => candidate.id === modelId) ?? null;- setSetup((current) => ({ ...current, model }));+ const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;+ if (model) setSetup((current) => ({ ...current, model }));}).catch(() => undefined).finally(() => {@@ -412,7 +416,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {disposed = true;window.clearInterval(timer);};- }, [currentStatus, prepared, setup.downloading, setup.open]);+ }, [prepared, setup.downloading, setup.open, setupModelId]);const previousOwnerRef = useRef(input.ownerKey);// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes@@ -445,8 +449,34 @@ export function useEnvironmentSpeechInput(input: HookInput) {if (controller.currentState.phase === "recording") void controller.stop();});+ const loadSetupCatalog = useStableCallback(+ async (connection: PreparedConnection, selectedModelId: string, request: number) => {+ setSetup({ ...INITIAL_SETUP, open: true });+ const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(+ () => null,+ );+ if (request !== startRequestRef.current) return;+ const choice = chooseSetupSpeechModel(catalog?.models ?? [], {+ selectedModelId,+ intent: latestRef.current.clientSettings.speechLanguage,+ preferred: preferredSpeechLanguages(),+ });+ setSetup((current) => ({+ ...current,+ model: choice?.model ?? null,+ language: choice?.language ?? null,+ catalogFailed: catalog === null,+ error: choice+ ? null+ : catalog+ ? "No model matches your language. Choose a model or change the language in Voice settings."+ : "Could not load transcription models. Check the connection and try again.",+ }));+ },+ );+const start = useStableCallback(async () => {- if (latestRef.current.input.disabled) return;+ if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return;if (!grantId) {if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false)await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });@@ -466,6 +496,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {controllerRef.current === controller &&latestRef.current.input.ownerKey === owner &&!latestRef.current.input.disabled;+ setSetup(INITIAL_SETUP);setQueuedStart(prepared);let latestStatus: EnvironmentSpeechStatus;try {@@ -481,8 +512,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {return;}if (!stillCurrent()) return;- setQueuedStart(null);setStatus({ prepared, value: latestStatus });+ setQueuedStart(null);+ if (latestStatus.supported && latestStatus.state === "missing-model") {+ return loadSetupCatalog(prepared, latestStatus.modelId, request);+ }if (!latestStatus.supported) {toastManager.add({type: "error",@@ -491,32 +525,46 @@ export function useEnvironmentSpeechInput(input: HookInput) {});return;}- if (latestStatus.state === "missing-model") {- setSetup({ ...INITIAL_SETUP, open: true });- return;- }setLevel(0);await controller.start();});const downloadSetupModel = useStableCallback(async () => {- if (!prepared || !currentStatus?.supported || setup.downloading) return;+ if (!prepared || !currentStatus?.supported || !setupModelId || setup.downloading) return;+ const modelId = setupModelId;const request = ++startRequestRef.current;setSetup((current) => ({ ...current, downloading: true, error: null }));try {- await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));+ const { models } = await runEnvironmentSpeech(+ downloadEnvironmentSpeechModel(prepared, modelId),+ );+ if (request !== startRequestRef.current) return;+ // A download cancelled from elsewhere still resolves, so select the model only once it landed.+ const installed = models.some(({ id, state }) => id === modelId && state === "installed");+ if (installed && modelId !== currentStatus.modelId) {+ try {+ await runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, modelId));+ } catch {+ if (request !== startRequestRef.current) return;+ setSetup((current) => ({+ ...current,+ downloading: false,+ error: "The model downloaded, but this environment could not select it. Try again.",+ }));+ return;+ }+ }if (request !== startRequestRef.current) return;const next = await probeSpeechStatus(prepared, true);if (request !== startRequestRef.current) return;setStatus({ prepared, value: next });+ const ready =+ installed && next.supported && next.modelId === modelId && next.state !== "missing-model";setSetup((current) => ({...current,downloading: false,- step: next.supported && next.state !== "missing-model" ? 1 : current.step,- error:- next.supported && next.state !== "missing-model"- ? null- : "The model is not ready. Try downloading it again.",+ step: ready ? 1 : current.step,+ error: ready ? null : "The model is not ready. Try downloading it again.",}));} catch {if (request !== startRequestRef.current) return;@@ -529,13 +577,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {});const cancelSetupDownload = useStableCallback(async () => {- if (!prepared || !currentStatus?.supported) return;+ if (!prepared || !setupModelId) return;startRequestRef.current += 1;setSetup((current) => ({ ...current, open: false, downloading: false }));try {- await runEnvironmentSpeech(- cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),- );+ await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, setupModelId));} catch (error) {setSetup((current) => ({...current,@@ -562,6 +608,10 @@ export function useEnvironmentSpeechInput(input: HookInput) {setup: {...setup,setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),+ retryCatalog:+ setup.open && setup.catalogFailed && prepared && currentStatus?.supported+ ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)+ : null,download: downloadSetupModel,cancelDownload: cancelSetupDownload,startRecording: async () => {@@ -582,6 +632,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {pendingSubmissionRef.current = null;startRequestRef.current += 1;setQueuedStart(null);+ setSetup(INITIAL_SETUP);controllerRef.current?.cancel();}),skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),diff --git a/apps/web/src/speech/useTextFieldSpeech.tsx b/apps/web/src/speech/useTextFieldSpeech.tsxindex 9d80d7366b5..a6c99535501 100644--- a/apps/web/src/speech/useTextFieldSpeech.tsx+++ b/apps/web/src/speech/useTextFieldSpeech.tsx@@ -110,9 +110,11 @@ export function TextFieldSpeechControls({step={speech.setup.step}status={speech.status}model={speech.setup.model}+ language={speech.setup.language}downloading={speech.setup.downloading}error={speech.setup.error}onOpenChange={speech.setup.setOpen}+ onRetryCatalog={speech.setup.retryCatalog}onDownload={() => void speech.setup.download()}onCancelDownload={() => void speech.setup.cancelDownload()}onStartRecording={() => void speech.setup.startRecording()}diff --git a/docs/user/voice-input.md b/docs/user/voice-input.mdindex 65fef2cce87..c116c9e67a3 100644--- a/docs/user/voice-input.md+++ b/docs/user/voice-input.md@@ -52,6 +52,7 @@ until you choose another.In **Settings → Voice → Transcription**, download a model and choose **Use model** if itisn’t already selected. **Test model** lets you try it without sending a message.If you use the composer microphone before a model is installed, it offers to download one.+The recommendation uses your Voice language, or your system’s preferred languages with **Auto-detect**.Select the languages you dictate in to compare models that support all of them.Accuracy shows how often the model got words or characters wrong in eachdiff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.tsindex 6f1354dcdd9..182b238d155 100644--- a/packages/contracts/src/ipc.ts+++ b/packages/contracts/src/ipc.ts@@ -1048,6 +1048,8 @@ export interface DesktopBridge {* regardless of OS settings.*/getSystemLocale: () => string | null;+ /** The OS's preferred languages, most preferred first; `navigator.languages` is pinned like the locale. */+ getPreferredSystemLanguages: () => readonly string[];getLocalEnvironmentBootstraps: () => readonly DesktopEnvironmentBootstrap[];getLocalEnvironmentEnabled: () => boolean;setLocalEnvironmentEnabled: (enabled: boolean) => Promise<void>;diff --git a/packages/shared/src/speech.test.ts b/packages/shared/src/speech.test.tsindex 8ef4e7fd20a..6d1829fd67a 100644--- a/packages/shared/src/speech.test.ts+++ b/packages/shared/src/speech.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test";import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "@t3tools/contracts";import {+ chooseSetupSpeechModel,decodeSpeechPcmRequest,encodeSpeechPcmRequest,resolveSpeechVocabulary,@@ -71,3 +72,81 @@ describe("resolveSpeechVocabulary", () => {expect(resolved.at(-1)?.term).toBe("word97");});});++describe("chooseSetupSpeechModel", () => {+ const model = (+ id: string,+ languages: string[],+ supportsLanguageDetection: boolean,+ recommended: boolean,+ ) => ({ id, languages, supportsLanguageDetection, recommended });+ const catalog = [+ model("unrecommended", ["en", "de", "es"], true, false),+ model("english", ["en"], false, true),+ model("fixed", ["en", "de"], false, true),+ model("detecting", ["en", "de", "fr"], true, true),+ ];++ it.each([+ [+ "keeps the selected English model for an English speaker",+ "english",+ "auto",+ ["en"],+ { id: "english", language: "en" },+ ],+ [+ "skips models that would transcribe English instead of German",+ "english",+ "auto",+ ["de", "en"],+ { id: "detecting", language: "de" },+ ],+ [+ "accepts a model without detection for a chosen language",+ "english",+ "de",+ ["en"],+ { id: "fixed", language: "de" },+ ],+ [+ "keeps a selected model that transcribes the language",+ "unrecommended",+ "auto",+ ["de"],+ { id: "unrecommended", language: "de" },+ ],+ [+ "moves on to the next preferred language a model transcribes",+ "english",+ "auto",+ ["ja", "fr"],+ { id: "detecting", language: "fr" },+ ],+ [+ "falls back to a model that is not recommended",+ "english",+ "auto",+ ["es"],+ { id: "unrecommended", language: "es" },+ ],+ ["offers nothing when no model transcribes the language", "english", "auto", ["ja"], undefined],+ [+ "offers the selected model without a language recommendation when Auto has no preferences",+ "fixed",+ "auto",+ [],+ { id: "fixed", language: null },+ ],+ [+ "does not guess another model when Auto has no preferences or selection",+ "gone",+ "auto",+ [],+ undefined,+ ],+ ] as const)("%s", (_, selectedModelId, intent, preferred, expected) => {+ const choice = chooseSetupSpeechModel(catalog, { selectedModelId, intent, preferred });+ expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected);+ });+});diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.tsindex 0fb077452bb..c5ed9ab14ea 100644--- a/packages/shared/src/speech.ts+++ b/packages/shared/src/speech.ts@@ -36,6 +36,44 @@ export function effectiveSpeechLanguage(return model.languages.find((language) => language === "en") ?? model.languages[0] ?? "auto";}+/**+ * The model first-run setup offers: the selected one, else a recommended one, else any that the+ * server would really transcribe the person's language with. Without a language hint,+ * offer only the selected model, without a recommendation.+ */+export function chooseSetupSpeechModel<+ Model extends {+ readonly id: string;+ readonly languages: readonly string[];+ readonly supportsLanguageDetection: boolean;+ readonly recommended: boolean;+ },+>(+ models: readonly Model[],+ input: {+ readonly selectedModelId: string;+ readonly intent: string;+ readonly preferred: readonly string[];+ },+) {+ if (input.intent === "auto" && input.preferred.length === 0) {+ const model = models.find(({ id }) => id === input.selectedModelId);+ return model && { model, language: null };+ }+ for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {+ const capable = models.filter((model) => {+ const effective = effectiveSpeechLanguage(model, input.intent);+ return effective === language || (effective === "auto" && model.languages.includes(language));+ });+ const model =+ capable.find(({ id }) => id === input.selectedModelId) ??+ capable.find(({ recommended }) => recommended) ??+ capable[0];+ if (model) return { model, language };+ }+ return undefined;+}+const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleLowerCase();/**