Skip to content
TypeScript

fix(web): first-run voice setup offers a model for the user's language (#32035)

Julius Marminge15b72755956c
diff --git a/apps/desktop/src/app/DesktopAppIdentity.test.ts b/apps/desktop/src/app/DesktopAppIdentity.test.ts
index 32e7b3b3118..74d134c0b10 100644
--- a/apps/desktop/src/app/DesktopAppIdentity.test.ts
+++ b/apps/desktop/src/app/DesktopAppIdentity.test.ts
@@ -42,6 +42,7 @@ const layerElectronApp = (calls: ElectronAppCalls) =>
metadata: Effect.die("unexpected metadata read"),
name: Effect.succeed("T3 Code"),
systemLocale: Effect.succeed("en-US"),
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
whenReady: Effect.void,
requestSingleInstanceLock: Effect.succeed(true),
quit: Effect.void,
diff --git a/apps/desktop/src/app/DesktopLifecycle.test.ts b/apps/desktop/src/app/DesktopLifecycle.test.ts
index fa1bcd8bbf0..30df96c58ad 100644
--- a/apps/desktop/src/app/DesktopLifecycle.test.ts
+++ b/apps/desktop/src/app/DesktopLifecycle.test.ts
@@ -39,6 +39,7 @@ function layerElectronApp(
metadata: Effect.die("unexpected metadata read"),
name: Effect.succeed("T3 Code"),
systemLocale: Effect.succeed("en-US"),
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
whenReady: Effect.void,
requestSingleInstanceLock: Effect.succeed(true),
quit,
diff --git a/apps/desktop/src/electron/ElectronApp.ts b/apps/desktop/src/electron/ElectronApp.ts
index d677baa56dc..188f53c4be8 100644
--- a/apps/desktop/src/electron/ElectronApp.ts
+++ b/apps/desktop/src/electron/ElectronApp.ts
@@ -46,6 +46,7 @@ export class ElectronApp extends Context.Service<
readonly name: Effect.Effect<string>;
/** Packaged builds ship only en-US; app.getLocale() and Intl defaults do not reflect the OS locale. */
readonly systemLocale: Effect.Effect<string>;
+ readonly preferredSystemLanguages: Effect.Effect<ReadonlyArray<string>>;
readonly whenReady: Effect.Effect<void, ElectronAppWhenReadyError>;
readonly requestSingleInstanceLock: Effect.Effect<boolean>;
readonly quit: Effect.Effect<void>;
@@ -155,6 +156,7 @@ export const make = ElectronApp.of({
}
return toSystemLocaleTag(identifier, hourCycle);
}),
+ preferredSystemLanguages: Effect.sync(() => Electron.app.getPreferredSystemLanguages()),
whenReady: Effect.gen(function* () {
const isPackaged = Electron.app.isPackaged;
yield* Effect.tryPromise({
diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts
index a46dbafab0f..66c558e86fc 100644
--- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts
+++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts
@@ -38,6 +38,7 @@ import {
getLocalEnvironmentBearerToken,
installRelayTelemetryEnabled,
getSystemLocale,
+ getPreferredSystemLanguages,
getWindowFullscreenState,
openExternal,
readMedia,
@@ -94,6 +95,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
yield* ipc.handleSync(getAppBranding);
yield* installRelayTelemetryEnabled();
yield* ipc.handleSync(getSystemLocale);
+ yield* ipc.handleSync(getPreferredSystemLanguages);
yield* ipc.handleSync(getWindowFullscreenState);
yield* ipc.handleSync(getLocalEnvironmentBootstraps);
yield* ipc.handleSync(getLocalEnvironmentEnabled);
diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts
index 1d7edaf6832..840e5f11680 100644
--- a/apps/desktop/src/ipc/channels.ts
+++ b/apps/desktop/src/ipc/channels.ts
@@ -34,6 +34,7 @@ export const CLI_COMMAND_UNINSTALL_CHANNEL = "desktop:cli-command-uninstall";
export const GET_APP_BRANDING_CHANNEL = "desktop:get-app-branding";
export const GET_RELAY_TELEMETRY_ENABLED_CHANNEL = "desktop:get-relay-telemetry-enabled";
export const GET_SYSTEM_LOCALE_CHANNEL = "desktop:get-system-locale";
+export const GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL = "desktop:get-preferred-system-languages";
export const GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL = "desktop:get-local-environment-bootstraps";
export const GET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:get-local-environment-enabled";
export const SET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:set-local-environment-enabled";
diff --git a/apps/desktop/src/ipc/methods/window.ts b/apps/desktop/src/ipc/methods/window.ts
index 2833abca438..9580b29b94b 100644
--- a/apps/desktop/src/ipc/methods/window.ts
+++ b/apps/desktop/src/ipc/methods/window.ts
@@ -106,6 +106,15 @@ export const getSystemLocale = DesktopIpc.makeSyncIpcMethod({
}),
});
+export const getPreferredSystemLanguages = DesktopIpc.makeSyncIpcMethod({
+ channel: IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,
+ result: Schema.Array(Schema.String),
+ handler: Effect.fn("desktop.ipc.window.getPreferredSystemLanguages")(function* () {
+ const electronApp = yield* ElectronApp.ElectronApp;
+ return yield* electronApp.preferredSystemLanguages;
+ }),
+});
+
export const getWindowFullscreenState = DesktopIpc.makeSyncIpcMethod({
channel: IpcChannels.GET_WINDOW_FULLSCREEN_STATE_CHANNEL,
result: Schema.Boolean,
diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts
index 800f1e744c9..2d2bd4f5164 100644
--- a/apps/desktop/src/preload.ts
+++ b/apps/desktop/src/preload.ts
@@ -107,6 +107,12 @@ contextBridge.exposeInMainWorld("desktopBridge", {
const result = ipcRenderer.sendSync(IpcChannels.GET_SYSTEM_LOCALE_CHANNEL);
return typeof result === "string" ? result : null;
},
+ getPreferredSystemLanguages: () => {
+ const result: unknown = ipcRenderer.sendSync(
+ IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,
+ );
+ return Array.isArray(result) ? result.filter((tag) => typeof tag === "string") : [];
+ },
getLocalEnvironmentBootstraps: () => {
const result = ipcRenderer.sendSync(IpcChannels.GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL);
if (!Array.isArray(result)) {
diff --git a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts
index 1180c0fa709..9201bc01110 100644
--- a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts
+++ b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts
@@ -37,6 +37,7 @@ function layerElectronApp(
metadata: Effect.die("unexpected metadata read"),
name: Effect.succeed("T3 Code"),
systemLocale: Effect.succeed("en-US"),
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
whenReady: Effect.void,
requestSingleInstanceLock: Effect.succeed(true),
quit: Effect.void,
diff --git a/apps/desktop/src/window/DesktopApplicationMenu.test.ts b/apps/desktop/src/window/DesktopApplicationMenu.test.ts
index 9d8efd05666..c1bb89b759e 100644
--- a/apps/desktop/src/window/DesktopApplicationMenu.test.ts
+++ b/apps/desktop/src/window/DesktopApplicationMenu.test.ts
@@ -31,6 +31,7 @@ const layerElectronApp = Layer.succeed(ElectronApp.ElectronApp, {
metadata: Effect.die("unexpected metadata read"),
name: Effect.succeed("T3 Code"),
systemLocale: Effect.succeed("en-US"),
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
whenReady: Effect.void,
requestSingleInstanceLock: Effect.succeed(true),
quit: Effect.void,
diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts
index 972c401ce30..7b347728370 100644
--- a/apps/server/src/speech/SpeechService.test.ts
+++ b/apps/server/src/speech/SpeechService.test.ts
@@ -25,6 +25,7 @@ const native = vi.hoisted(() => ({
supportsLongForm: false,
supportsTranslation: true,
supportsVocabulary: false,
+ languages: ["en", "fr"] as readonly string[],
begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
feed: vi.fn(async (_pcm: Float32Array) => ({
revision: 1,
@@ -119,6 +120,7 @@ beforeEach(() => {
native.backend = "Vulkan0";
native.supportsVocabulary = false;
native.supportsStreaming = true;
+ native.languages = ["en", "fr"];
readyModels.clear();
readyModels.add("test-model");
readyModels.add("fallback-model");
@@ -171,6 +173,60 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr
}).pipe(Effect.provide(layer)),
);
+it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
+ Effect.gen(function* () {
+ const speech = yield* SpeechService.SpeechService;
+ // Nemotron Streaming 3.5 advertises regional tags while the catalog lists bare codes.
+ native.languages = ["en-US", "en-GB", "fr-FR"];
+ yield* Effect.gen(function* () {
+ const stream = yield* speech.startStream({
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "fr",
+ });
+ expect(native.begin).toHaveBeenLastCalledWith(
+ { timestamps: "none", language: "fr-FR" },
+ true,
+ "test.gguf",
+ );
+ expect(yield* stream.finish).toBe("hello");
+ }).pipe(Effect.scoped);
+ native.supportsStreaming = false;
+ expect(
+ yield* speech.transcribe(pcm(), {
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "fr",
+ }),
+ ).toBe("hello");
+ expect(native.transcribe).toHaveBeenLastCalledWith(
+ expect.any(Float32Array),
+ { timestamps: "none", language: "fr-FR" },
+ true,
+ "test.gguf",
+ );
+ // A language the model does not advertise reaches it unchanged, so the model reports it.
+ yield* speech.updateAcceleration("cpu");
+ native.languages = ["en-US"];
+ native.transcribe.mockRejectedValueOnce(new Error("transcribe_run: unsupported language"));
+ const rejected = yield* Effect.result(
+ speech.transcribe(pcm(), { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }),
+ );
+ expect(Result.isFailure(rejected) && rejected.failure).toMatchObject({
+ _tag: "SpeechOperationError",
+ operation: "inference",
+ });
+ expect(native.transcribe).toHaveBeenLastCalledWith(
+ expect.any(Float32Array),
+ { timestamps: "none", language: "fr" },
+ true,
+ "test.gguf",
+ );
+ expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual([
+ ["test.gguf", "auto"],
+ ["test.gguf", "cpu"],
+ ]);
+ }).pipe(Effect.provide(layer)),
+);
+
it.effect.each(["cancellation", "timeout"] as const)(
"releases a stalled model preparation on %s",
(reason) =>
diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts
index c1f762f2608..24eb6ef408d 100644
--- a/apps/server/src/speech/SpeechService.ts
+++ b/apps/server/src/speech/SpeechService.ts
@@ -32,6 +32,7 @@ import {
type SpeechModel,
} from "./model.ts";
import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";
+import { nativeSpeechLanguage } from "./nativeLanguage.ts";
import { SILERO_VAD } from "./silero.ts";
import { isSpeechPlatformSupported } from "./platform.ts";
import * as SpeechModelStore from "./SpeechModelStore.ts";
@@ -613,11 +614,12 @@ const make = Effect.gen(function* () {
try: (signal) => {
// Interruption kills the isolated process; the next request reloads the model.
signal.addEventListener("abort", () => dropModel(inference), { once: true });
+ const hint = nativeSpeechLanguage(inference.languages, language);
return inference.transcribe(
pcm,
{
timestamps: "none",
- ...(language === "auto" ? {} : { language }),
+ ...(hint ? { language: hint } : {}),
...(translateToEnglish
? { task: "translate" as const, targetLanguage: "en" as const }
: {}),
@@ -749,10 +751,11 @@ const make = Effect.gen(function* () {
streamModel.supportsVocabulary && !translateToEnglish && customWords.length
? customWords
: undefined;
+ const hint = nativeSpeechLanguage(streamModel.languages, language);
await streamModel.begin(
{
timestamps: "none",
- ...(language === "auto" ? {} : { language }),
+ ...(hint ? { language: hint } : {}),
...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),
...(vocabulary ? { vocabulary } : {}),
},
diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts
index 97aa1b56602..8712c63ed34 100644
--- a/apps/server/src/speech/native.test.ts
+++ b/apps/server/src/speech/native.test.ts
@@ -4,7 +4,7 @@ import { expect, it } from "vite-plus/test";
import { loadNativeSpeechModel } from "./native.ts";
const fixture = (transcribe: string, maxAudioMs = 0, supportsVocabulary = false) =>
- `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs} }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;
+ `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs}, languages: ["en-US", "de-DE"] }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;
const fixtureWithUnrelatedIpcMessage = () =>
`data:text/javascript,${encodeURIComponent(`
@@ -38,6 +38,7 @@ it("returns transcription from an isolated process and disposes it", async () =>
fixture("async (pcm) => ({ text: String(pcm[0]) })"),
);
try {
+ expect(model.languages).toEqual(["en-US", "de-DE"]);
expect(
await model.transcribe(
new Float32Array([0.25]),
diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts
index fae308333b2..8331e1b5e0e 100644
--- a/apps/server/src/speech/native.ts
+++ b/apps/server/src/speech/native.ts
@@ -147,6 +147,7 @@ process.on("message", async (message) => {
throw new Error("The selected speech GPU is unavailable.");
model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });
process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
+ languages: model.capabilities.languages,
supportsStreaming: model.capabilities.supportsStreaming,
supportsLongForm: model.supports("long_form"),
supportsTranslation: model.capabilities.supportsTranslate,
@@ -194,6 +195,7 @@ type Reply = {
readonly supportsTranslation?: boolean;
readonly supportsVocabulary?: boolean;
readonly backend?: string;
+ readonly languages?: unknown;
readonly revision?: number;
readonly preview?: SpeechStreamText | null;
};
@@ -391,6 +393,9 @@ async function loadNativeSpeechModelOnce(
}
return {
backend: loaded.backend ?? "unknown",
+ languages: Array.isArray(loaded.languages)
+ ? loaded.languages.filter((entry): entry is string => typeof entry === "string")
+ : [],
supportsStreaming: loaded.supportsStreaming === true,
supportsLongForm: loaded.supportsLongForm === true,
supportsTranslation: loaded.supportsTranslation === true,
diff --git a/apps/server/src/speech/nativeLanguage.test.ts b/apps/server/src/speech/nativeLanguage.test.ts
new file mode 100644
index 00000000000..3632878285d
--- /dev/null
+++ b/apps/server/src/speech/nativeLanguage.test.ts
@@ -0,0 +1,18 @@
+import { expect, it } from "vite-plus/test";
+
+import { nativeSpeechLanguage } from "./nativeLanguage.ts";
+
+const nemotron = ["en-US", "en-GB", "es-US", "es-ES", "de-DE"];
+
+it.each([
+ ["auto omits the hint", nemotron, "auto", undefined],
+ ["exact entry", ["en", "de"], "de", "de"],
+ ["exact entry before a regional one", ["de-DE", "de"], "de", "de"],
+ ["case and underscore normalize to the advertised spelling", nemotron, "DE_de", "de-DE"],
+ ["bare code takes the first regional entry", nemotron, "en", "en-US"],
+ ["regional request with another region passes through", nemotron, "de-AT", "de-AT"],
+ ["unadvertised language passes through", nemotron, "fr", "fr"],
+ ["model without an advertised list passes through", [], "de", "de"],
+])("%s", (_name, advertised, language, expected) => {
+ expect(nativeSpeechLanguage(advertised, language)).toBe(expected);
+});
diff --git a/apps/server/src/speech/nativeLanguage.ts b/apps/server/src/speech/nativeLanguage.ts
new file mode 100644
index 00000000000..0ff013de970
--- /dev/null
+++ b/apps/server/src/speech/nativeLanguage.ts
@@ -0,0 +1,21 @@
+/**
+ * transcribe.cpp accepts a language only when it equals an entry of the GGUF's
+ * `general.languages` byte for byte. Nemotron Streaming 3.5 lists `de-DE` where the catalog
+ * says `de`, so the request carries the loaded model's own spelling. "auto" omits the hint.
+ */
+export function nativeSpeechLanguage(
+ advertised: readonly string[],
+ language: string,
+): string | undefined {
+ if (language === "auto") return undefined;
+ const tag = (value: string) => value.toLowerCase().replaceAll("_", "-");
+ const wanted = tag(language);
+ return (
+ advertised.find((entry) => entry === language) ??
+ advertised.find((entry) => tag(entry) === wanted) ??
+ (wanted.includes("-")
+ ? undefined
+ : advertised.find((entry) => tag(entry).split("-", 1)[0] === wanted)) ??
+ language
+ );
+}
diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx
index 69022a93fbe..be106a25159 100644
--- a/apps/web/src/components/chat/ChatComposer.tsx
+++ b/apps/web/src/components/chat/ChatComposer.tsx
@@ -3709,9 +3709,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
step={speechInput.setup.step}
status={speechInput.status}
model={speechInput.setup.model}
+ language={speechInput.setup.language}
downloading={speechInput.setup.downloading}
error={speechInput.setup.error}
onOpenChange={speechInput.setup.setOpen}
+ onRetryCatalog={speechInput.setup.retryCatalog}
onDownload={() => void speechInput.setup.download()}
onCancelDownload={() => void speechInput.setup.cancelDownload()}
onStartRecording={() => void speechInput.setup.startRecording()}
diff --git a/apps/web/src/components/chat/VoiceInputSetup.tsx b/apps/web/src/components/chat/VoiceInputSetup.tsx
index 7b30783cf21..cf697979fc2 100644
--- a/apps/web/src/components/chat/VoiceInputSetup.tsx
+++ b/apps/web/src/components/chat/VoiceInputSetup.tsx
@@ -13,6 +13,7 @@ import { useClientSettings, updateClientSettings } from "../../hooks/useSettings
import { useEnvironment } from "../../state/environments";
import { AuthSettingsWriteScope } from "@t3tools/contracts";
import { useEnvironmentScope } from "../../state/session";
+import { languageLabel } from "../settings/SpeechLanguageSelect";
import { Button } from "../ui/button";
import { Dialog } from "../ui/dialog";
import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
@@ -24,9 +25,12 @@ export function VoiceInputSetup(props: {
step: number;
status: EnvironmentSpeechStatus | null;
model: EnvironmentSpeechModel | null;
+ /** The language the offered model was chosen for. */
+ language: string | null;
downloading: boolean;
error: string | null;
onOpenChange(open: boolean): void;
+ onRetryCatalog: (() => void) | null;
onDownload(): void;
onCancelDownload(): void;
onStartRecording(): void;
@@ -85,49 +89,55 @@ export function VoiceInputSetup(props: {
<WizardPanel>
{props.step === 0 ? (
<section className="space-y-3 text-sm">
- <div className="rounded-lg border border-border/70 bg-card/60 p-4">
- <div className="flex items-center justify-between gap-2">
- <h3 className="font-medium">{status?.model ?? "English transcription model"}</h3>
- {status?.modelId === "handy-computer/parakeet-unified-en-0.6b-gguf" ? (
- <span className="text-xs font-medium text-primary">
- Recommended for English
- </span>
+ {model ? (
+ <div className="rounded-lg border border-border/70 bg-card/60 p-4">
+ <div className="flex items-center justify-between gap-2">
+ <h3 className="font-medium">{model.name}</h3>
+ {model?.recommended && props.language ? (
+ <span className="text-xs font-medium text-primary">
+ Recommended for {languageLabel(props.language)}
+ </span>
+ ) : null}
+ </div>
+ <p className="mt-2 text-muted-foreground">
+ Download the {Math.round(model.size / 1024 / 1024)} MB model to{" "}
+ {environment?.label ?? "your transcription environment"}. Microphone audio is
+ sent there for transcription, as chosen in Voice settings.
+ </p>
+ <p className="mt-2 text-muted-foreground">
+ Silero (1.2 MB), used for silence filtering and pause detection, downloads
+ alongside the speech model. An installed copy is reused.
+ </p>
+ {props.downloading ? (
+ <div role="status" className="mt-4 space-y-1 text-muted-foreground">
+ <div
+ role="progressbar"
+ aria-label="Speech model download progress"
+ aria-valuemin={0}
+ aria-valuemax={100}
+ aria-valuenow={progress ?? undefined}
+ className="h-1 overflow-hidden rounded-full bg-muted"
+ >
+ <div
+ className="h-full bg-primary transition-[width] duration-200"
+ style={{ width: `${progress ?? 0}%` }}
+ />
+ </div>
+ <p className="text-xs">
+ {model?.state === "verifying"
+ ? "Verifying download…"
+ : progress === null
+ ? "Starting download…"
+ : `${progress}% downloaded`}
+ </p>
+ </div>
) : null}
</div>
- <p className="mt-2 text-muted-foreground">
- Download the {status ? Math.round(status.size / 1024 / 1024) : 697} MB model to{" "}
- {environment?.label ?? "your transcription environment"}. Microphone audio is sent
- there for transcription, as chosen in Voice settings.
+ ) : !props.error ? (
+ <p role="status" className="text-muted-foreground">
+ Loading transcription models…
</p>
- <p className="mt-2 text-muted-foreground">
- Silero (1.2 MB), used for silence filtering and pause detection, downloads
- alongside the speech model. An installed copy is reused.
- </p>
- {props.downloading ? (
- <div role="status" className="mt-4 space-y-1 text-muted-foreground">
- <div
- role="progressbar"
- aria-label="Speech model download progress"
- aria-valuemin={0}
- aria-valuemax={100}
- aria-valuenow={progress ?? undefined}
- className="h-1 overflow-hidden rounded-full bg-muted"
- >
- <div
- className="h-full bg-primary transition-[width] duration-200"
- style={{ width: `${progress ?? 0}%` }}
- />
- </div>
- <p className="text-xs">
- {model?.state === "verifying"
- ? "Verifying download…"
- : progress === null
- ? "Starting download…"
- : `${progress}% downloaded`}
- </p>
- </div>
- ) : null}
- </div>
+ ) : null}
{props.error ? (
<p role="alert" className="text-destructive">
{props.error}
@@ -208,25 +218,30 @@ export function VoiceInputSetup(props: {
{props.step === 0 ? (
<>
<Button variant="outline" onClick={() => openSettings("speech-models")}>
- Choose another model or language
+ {model ? "Choose another model or language" : "Choose a model or language"}
</Button>
+ {props.onRetryCatalog ? (
+ <Button onClick={props.onRetryCatalog}>Retry loading models</Button>
+ ) : null}
{props.downloading ? (
<Button variant="outline" disabled={!canManage} onClick={props.onCancelDownload}>
Cancel download
</Button>
) : null}
- <Button
- disabled={!canManage || props.downloading || !status}
- onClick={props.onDownload}
- >
- {props.downloading
- ? "Downloading…"
- : props.error
- ? "Retry download"
- : canManage
- ? "Download model"
- : "Ask an environment admin to download"}
- </Button>
+ {model ? (
+ <Button
+ disabled={!canManage || props.downloading || !status}
+ onClick={props.onDownload}
+ >
+ {props.downloading
+ ? "Downloading…"
+ : props.error
+ ? "Try again"
+ : canManage
+ ? "Download model"
+ : "Ask an environment admin to download"}
+ </Button>
+ ) : null}
</>
) : (
<>
diff --git a/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx b/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx
index 639d11dd0fd..8d52066cd5f 100644
--- a/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx
+++ b/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx
@@ -184,9 +184,11 @@ export function PreviewAnnotationSpeech({
step={speech.setup.step}
status={speech.status}
model={speech.setup.model}
+ language={speech.setup.language}
downloading={speech.setup.downloading}
error={speech.setup.error}
onOpenChange={speech.setup.setOpen}
+ onRetryCatalog={speech.setup.retryCatalog}
onDownload={() => void speech.setup.download()}
onCancelDownload={() => void speech.setup.cancelDownload()}
onStartRecording={() => void speech.setup.startRecording()}
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
index 479fb73c7f3..969420ff2e0 100644
--- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
@@ -161,7 +161,10 @@ it("tests the selected model after download and preserves its transcript across
open: false,
step: 0,
model: null,
+ language: null,
downloading: false,
+ catalogFailed: false,
+ retryCatalog: null,
error: null,
setOpen: () => {},
download: async () => {},
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
index 5d2a8fbed22..4a3df08f40f 100644
--- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
@@ -47,6 +47,7 @@ import { useEffect, useRef, useState, type ReactNode } from "react";
import { useStableCallback } from "~/hooks/useStableCallback";
import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
+import { preferredSpeechLanguages } from "../../speech/preferredSpeechLanguages";
import {
chooseAudioDestination,
chooseTranscriptCleanup,
@@ -760,7 +761,7 @@ function SpeechModelBrowser(props: {
const [downloadedOnly, setDownloadedOnly] = useState(false);
const [languageSearch, setLanguageSearch] = useState("");
const [selection, setSelection] = useState<string[] | null>(null);
- const preferred = navigator.languages.map((language) => language.toLowerCase().split("-")[0]!);
+ const [preferred] = useState(preferredSpeechLanguages);
const supportedLanguages = [...new Set(models.flatMap((model) => model.languages))];
const defaultLanguage = preferred.find((code) => supportedLanguages.includes(code));
const selectedLanguages = selection ?? (defaultLanguage ? [defaultLanguage] : []);
@@ -1077,9 +1078,11 @@ function TranscriptionTest(props: {
step={speech.setup.step}
status={speech.status}
model={speech.setup.model}
+ language={speech.setup.language}
downloading={speech.setup.downloading}
error={speech.setup.error}
onOpenChange={speech.setup.setOpen}
+ onRetryCatalog={speech.setup.retryCatalog}
onDownload={() => void speech.setup.download()}
onCancelDownload={() => void speech.setup.cancelDownload()}
onStartRecording={() => void speech.setup.startRecording()}
diff --git a/apps/web/src/speech/preferredSpeechLanguages.ts b/apps/web/src/speech/preferredSpeechLanguages.ts
new file mode 100644
index 00000000000..548e7955ff0
--- /dev/null
+++ b/apps/web/src/speech/preferredSpeechLanguages.ts
@@ -0,0 +1,10 @@
+// Packaged desktop pins `navigator.languages` to en-US, so the OS's own list leads.
+export function preferredSpeechLanguages() {
+ const bridge = window.desktopBridge;
+ let system = bridge?.getPreferredSystemLanguages?.() ?? [];
+ if (!system.length) {
+ const locale = bridge?.getSystemLocale?.();
+ if (locale) system = [locale];
+ }
+ return [...system, ...navigator.languages].map((tag) => tag.toLowerCase().split(/[-_]/)[0]!);
+}
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
index 4586f1f5d5c..508ef9aa7b6 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
@@ -1,19 +1,24 @@
import { useLayoutEffect } from "react";
import { expect, it, vi } from "vite-plus/test";
+import { page, userEvent } from "vite-plus/test/browser";
+import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import { getEnvironmentSpeechModels } from "@t3tools/client-runtime/voice-input";
import {
DEFAULT_CLIENT_SETTINGS,
DEFAULT_SERVER_SETTINGS,
EnvironmentId,
+ type EnvironmentSpeechModel,
type ProjectId,
type SpeechCustomWords,
type SpeechPostProcessingOptions,
type SpeechTranscriptionOptions,
} from "@t3tools/contracts";
-import { act, render } from "../../test/render";
+import { act, render, stubGlobal } from "../../test/render";
+import { VoiceInputSetup } from "../components/chat/VoiceInputSetup";
import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
import {
@@ -36,7 +41,12 @@ const mocks = vi.hoisted(() => ({
statusRequests: [] as string[],
statusUnavailable: false,
modelState: "ready" as "ready" | "missing-model",
- download: vi.fn<() => Promise<void>>(),
+ statusModelId: "test-model",
+ models: [] as EnvironmentSpeechModel[],
+ catalogOffline: false,
+ download: vi.fn<(modelId: string) => Promise<void>>(),
+ selected: [] as string[],
+ selectionFails: false,
platforms: [] as string[],
microphoneRequests: [] as string[],
transcribeCalls: [] as string[],
@@ -80,6 +90,7 @@ vi.mock("../state/environments", () => ({
: null,
}));
vi.mock("../hooks/useSettings", () => ({
+ updateClientSettings: vi.fn(),
useClientSettingsHydrated: () => true,
useClientSettings: (selector?: (settings: Record<string, unknown>) => unknown) => {
const settings = { ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings };
@@ -109,7 +120,7 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
return {
supported: true,
state: mocks.modelState,
- modelId: "test-model",
+ modelId: mocks.statusModelId,
model: "Test",
size: 1,
supportsStreaming: false,
@@ -119,8 +130,24 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
gpuDevices: [],
};
}),
- downloadEnvironmentSpeechModel: () => Effect.promise(mocks.download),
- getEnvironmentSpeechModels: () => Effect.succeed({ models: [] }),
+ downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
+ Effect.promise(async () => {
+ await mocks.download(modelId);
+ return { models: mocks.models };
+ }),
+ selectEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
+ Effect.sync(() => {
+ mocks.selected.push(modelId);
+ if (mocks.selectionFails) throw new Error("selection failed");
+ mocks.statusModelId = modelId;
+ return { models: mocks.models };
+ }),
+ getEnvironmentSpeechModels: vi.fn(() =>
+ Effect.sync(() => {
+ if (mocks.catalogOffline) throw new Error("Catalog offline");
+ return { models: mocks.models };
+ }),
+ ),
postProcessEnvironmentTranscript: (
prepared: PreparedConnection,
transcript: string,
@@ -189,7 +216,12 @@ function setup(options: { readonly destination?: string | null } = {}) {
mocks.statusRequests = [];
mocks.statusUnavailable = false;
mocks.modelState = "ready";
+ mocks.statusModelId = "test-model";
+ mocks.models = [];
+ mocks.catalogOffline = false;
mocks.download.mockReset();
+ mocks.selected = [];
+ mocks.selectionFails = false;
mocks.platforms = [];
mocks.microphoneRequests = [];
mocks.transcribeCalls = [];
@@ -213,6 +245,7 @@ function Probe(props: {
readonly projectId?: string;
readonly projectName?: string;
readonly cleanup?: boolean;
+ readonly showWizard?: boolean;
}) {
const value = useEnvironmentSpeechInput({
environmentId: THREAD_ENV as EnvironmentId,
@@ -230,7 +263,18 @@ function Probe(props: {
useLayoutEffect(() => {
mocks.voice = value;
});
- return null;
+ return props.showWizard ? (
+ <VoiceInputSetup
+ {...value.setup}
+ environmentId={value.transcriptionEnvironmentId}
+ status={value.status}
+ onOpenChange={value.setup.setOpen}
+ onRetryCatalog={value.setup.retryCatalog}
+ onDownload={value.setup.download}
+ onCancelDownload={value.setup.cancelDownload}
+ onStartRecording={value.setup.startRecording}
+ />
+ ) : null;
}
function expectNoAudioSent() {
@@ -333,6 +377,8 @@ it("retries a failed transcription on the same chosen environment", async () =>
it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
setup({ destination: "env-a" });
mocks.modelState = "missing-model";
+ mocks.settings = { speechLanguage: "en" };
+ mocks.models = [catalogModel(mocks.statusModelId, ["en"], true)];
let finishDownload!: () => void;
mocks.download.mockReturnValue(
new Promise<void>((resolve) => {
@@ -360,6 +406,171 @@ it("reopens setup after reconnecting during a download and ignores its stale com
expect(mocks.voice.setup.error).toBeNull();
});
+function catalogModel(
+ id: string,
+ languages: readonly string[],
+ recommended: boolean,
+): EnvironmentSpeechModel {
+ return {
+ id,
+ name: id,
+ description: "",
+ size: 1,
+ languages,
+ accuracy: 0,
+ speed: 0,
+ recommended,
+ supportsStreaming: false,
+ supportsLanguageDetection: languages.length > 1,
+ supportsTranslation: false,
+ active: false,
+ state: "downloadable",
+ };
+}
+
+it.each([
+ ["the browser prefers German", ["de-DE"], undefined, "auto"],
+ ["the desktop exposes no locale methods", ["de-DE"], {}, "auto"],
+ [
+ "an older preload exposes only the OS locale",
+ ["en-US"],
+ { getSystemLocale: () => "de-DE" },
+ "auto",
+ ],
+ [
+ "the desktop OS prefers German",
+ ["en-US"],
+ { getPreferredSystemLanguages: () => ["de-DE"] },
+ "auto",
+ ],
+ ["German is the chosen Voice language", ["en-US"], undefined, "de"],
+])(
+ "offers and downloads a German-capable model when %s",
+ async (_, languages, bridge, speechLanguage) => {
+ setup({ destination: "env-a" });
+ mocks.settings = { speechLanguage };
+ using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);
+ using _bridge = stubGlobal("desktopBridge", bridge);
+ const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";
+ const german = "handy-computer/nemotron-3.5-asr-streaming-0.6b-gguf";
+ mocks.modelState = "missing-model";
+ mocks.statusModelId = englishDefault;
+ mocks.models = [
+ { ...catalogModel(englishDefault, ["en"], true), active: true },
+ catalogModel(german, ["en", "de", "fr"], true),
+ catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),
+ ];
+ mocks.download.mockResolvedValue(undefined);
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ expect(mocks.voice.setup.open).toBe(true);
+ await act(() => mocks.voice.setup.download());
+ expect(mocks.download).toHaveBeenCalledExactlyOnceWith(german);
+ expect(mocks.voice.setup.model?.id).toBe(german);
+ },
+);
+
+it.each(["cancel", "failed restart"] as const)(
+ "closes pending catalog setup on %s",
+ async (action) => {
+ setup({ destination: "env-a" });
+ mocks.modelState = "missing-model";
+ mocks.settings = { speechLanguage: "en" };
+ mocks.models = [catalogModel("current", ["en"], true)];
+ const response = Deferred.makeUnsafe<{ models: EnvironmentSpeechModel[] }>();
+ vi.mocked(getEnvironmentSpeechModels).mockReturnValueOnce(Deferred.await(response));
+ await using _screen = await render(<Probe showWizard />);
+ let firstStart!: Promise<void>;
+ await act(() => {
+ firstStart = mocks.voice.start();
+ });
+ await expect.element(page.getByText("Loading transcription models…")).toBeVisible();
+ if (action === "cancel") {
+ await act(() => mocks.voice.cancel());
+ } else {
+ mocks.statusUnavailable = true;
+ await act(() => mocks.voice.start());
+ mocks.statusUnavailable = false;
+ }
+ expect(mocks.voice.setup.open).toBe(false);
+
+ await act(() => mocks.voice.start());
+ Deferred.doneUnsafe(
+ response,
+ Effect.succeed({ models: [catalogModel("stale", ["en"], true)] }),
+ );
+ await act(() => firstStart);
+ await expect.element(page.getByRole("heading", { name: "current" })).toBeVisible();
+ expect(mocks.microphoneRequests).toEqual([]);
+ },
+);
+
+it.each([true, false])(
+ "never downloads the English default when no German recommendation is available (catalog offline: %s)",
+ async (offline) => {
+ setup({ destination: "env-a" });
+ mocks.settings = { speechLanguage: "de" };
+ mocks.modelState = "missing-model";
+ mocks.statusModelId = "english";
+ mocks.models = [catalogModel("english", ["en"], true)];
+ mocks.catalogOffline = offline;
+ await using _screen = await render(<Probe showWizard />);
+ await act(() => mocks.voice.start());
+ const wizard = page.getByRole("dialog", { name: "Set up voice input" });
+ const retry = wizard.getByRole("button", { name: "Retry loading models" });
+ const download = wizard.getByRole("button", { name: "Download model" });
+ await expect.element(wizard.getByRole("alert")).toBeVisible();
+ await expect.element(download).not.toBeInTheDocument();
+ if (offline) {
+ mocks.catalogOffline = false;
+ mocks.models.push(catalogModel("german", ["de"], true));
+ // Another client can finish setup while the catalog is offline.
+ mocks.modelState = "ready";
+ await userEvent.click(retry);
+ await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();
+ await expect.element(download).toBeEnabled();
+ }
+ await expect.element(retry).not.toBeInTheDocument();
+ expect(mocks.download).not.toHaveBeenCalled();
+ expect(mocks.microphoneRequests).toEqual([]);
+ },
+);
+
+it.each([
+ ["selects the offered model once its download installed it", "installed", ["german"], null],
+ [
+ "does not advance a cancelled download when another model is ready",
+ "downloadable",
+ [],
+ "The model is not ready. Try downloading it again.",
+ ],
+ [
+ "reports a failed selection apart from the download",
+ "installed",
+ ["german"],
+ "The model downloaded, but this environment could not select it. Try again.",
+ ],
+] as const)("%s", async (_, landedState, selected, error) => {
+ setup({ destination: "env-a" });
+ using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(["de-DE"]);
+ mocks.selectionFails = error !== null && landedState === "installed";
+ mocks.modelState = "missing-model";
+ mocks.statusModelId = "english";
+ mocks.models = [catalogModel("english", ["en"], true), catalogModel("german", ["de"], true)];
+ mocks.download.mockImplementation(async () => {
+ mocks.models = [mocks.models[0]!, { ...mocks.models[1]!, state: landedState }];
+ mocks.modelState = "ready";
+ });
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ expect(mocks.voice.setup.language).toBe("de");
+ await act(() => mocks.voice.setup.download());
+ expect(mocks.selected).toEqual(selected);
+ expect(mocks.voice.setup.error).toBe(error);
+ expect(mocks.voice.setup.downloading).toBe(false);
+ expect(mocks.voice.setup.step).toBe(landedState === "installed" && !error ? 1 : 0);
+});
+
it("does not resend a failed recording after consent is revoked", async () => {
setup({ destination: "env-a" });
mocks.transcripts = [new Error("inference failed")];
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts
index 0cb468dae50..c569c14500b 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.ts
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts
@@ -4,6 +4,7 @@ import {
getEnvironmentSpeechModels,
getEnvironmentSpeechStatus,
postProcessEnvironmentTranscript,
+ selectEnvironmentSpeechModel,
VoiceInputController,
voiceInputBlocksSubmission,
type VoiceDraftSnapshot,
@@ -22,7 +23,7 @@ import type {
SpeechTranscriptionOptions,
} from "@t3tools/contracts";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
-import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
+import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";
import { useNavigate } from "@tanstack/react-router";
import * as Option from "effect/Option";
import { useEffect, useRef, useState } from "react";
@@ -37,6 +38,7 @@ import {
import { useEnvironment } from "../state/environments";
import { useEnvironmentScope, usePreparedConnection } from "../state/session";
import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
+import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";
@@ -46,7 +48,9 @@ const INITIAL_SETUP = {
open: false,
step: 0,
model: null as EnvironmentSpeechModel | null,
+ language: null as string | null,
downloading: false,
+ catalogFailed: false,
error: null as string | null,
};
@@ -387,11 +391,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
};
}, [prepared, cloudProvider]);
const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;
+ const setupModelId = setup.model?.id;
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
useEffect(() => {
- if (!setup.open || !setup.downloading || !prepared || !currentStatus?.supported) return;
- const modelId = currentStatus.modelId;
+ if (!setup.open || !setup.downloading || !prepared || !setupModelId) return;
let disposed = false;
let refreshing = false;
const timer = window.setInterval(() => {
@@ -400,8 +404,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared))
.then((result) => {
if (disposed) return;
- const model = result.models.find((candidate) => candidate.id === modelId) ?? null;
- setSetup((current) => ({ ...current, model }));
+ const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
+ if (model) setSetup((current) => ({ ...current, model }));
})
.catch(() => undefined)
.finally(() => {
@@ -412,7 +416,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
disposed = true;
window.clearInterval(timer);
};
- }, [currentStatus, prepared, setup.downloading, setup.open]);
+ }, [prepared, setup.downloading, setup.open, setupModelId]);
const previousOwnerRef = useRef(input.ownerKey);
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
@@ -445,8 +449,34 @@ export function useEnvironmentSpeechInput(input: HookInput) {
if (controller.currentState.phase === "recording") void controller.stop();
});
+ const loadSetupCatalog = useStableCallback(
+ async (connection: PreparedConnection, selectedModelId: string, request: number) => {
+ setSetup({ ...INITIAL_SETUP, open: true });
+ const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(
+ () => null,
+ );
+ if (request !== startRequestRef.current) return;
+ const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
+ selectedModelId,
+ intent: latestRef.current.clientSettings.speechLanguage,
+ preferred: preferredSpeechLanguages(),
+ });
+ setSetup((current) => ({
+ ...current,
+ model: choice?.model ?? null,
+ language: choice?.language ?? null,
+ catalogFailed: catalog === null,
+ error: choice
+ ? null
+ : catalog
+ ? "No model matches your language. Choose a model or change the language in Voice settings."
+ : "Could not load transcription models. Check the connection and try again.",
+ }));
+ },
+ );
+
const start = useStableCallback(async () => {
- if (latestRef.current.input.disabled) return;
+ if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return;
if (!grantId) {
if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false)
await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
@@ -466,6 +496,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
controllerRef.current === controller &&
latestRef.current.input.ownerKey === owner &&
!latestRef.current.input.disabled;
+ setSetup(INITIAL_SETUP);
setQueuedStart(prepared);
let latestStatus: EnvironmentSpeechStatus;
try {
@@ -481,8 +512,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
return;
}
if (!stillCurrent()) return;
- setQueuedStart(null);
setStatus({ prepared, value: latestStatus });
+ setQueuedStart(null);
+ if (latestStatus.supported && latestStatus.state === "missing-model") {
+ return loadSetupCatalog(prepared, latestStatus.modelId, request);
+ }
if (!latestStatus.supported) {
toastManager.add({
type: "error",
@@ -491,32 +525,46 @@ export function useEnvironmentSpeechInput(input: HookInput) {
});
return;
}
- if (latestStatus.state === "missing-model") {
- setSetup({ ...INITIAL_SETUP, open: true });
- return;
- }
setLevel(0);
await controller.start();
});
const downloadSetupModel = useStableCallback(async () => {
- if (!prepared || !currentStatus?.supported || setup.downloading) return;
+ if (!prepared || !currentStatus?.supported || !setupModelId || setup.downloading) return;
+ const modelId = setupModelId;
const request = ++startRequestRef.current;
setSetup((current) => ({ ...current, downloading: true, error: null }));
try {
- await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
+ const { models } = await runEnvironmentSpeech(
+ downloadEnvironmentSpeechModel(prepared, modelId),
+ );
+ if (request !== startRequestRef.current) return;
+ // A download cancelled from elsewhere still resolves, so select the model only once it landed.
+ const installed = models.some(({ id, state }) => id === modelId && state === "installed");
+ if (installed && modelId !== currentStatus.modelId) {
+ try {
+ await runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, modelId));
+ } catch {
+ if (request !== startRequestRef.current) return;
+ setSetup((current) => ({
+ ...current,
+ downloading: false,
+ error: "The model downloaded, but this environment could not select it. Try again.",
+ }));
+ return;
+ }
+ }
if (request !== startRequestRef.current) return;
const next = await probeSpeechStatus(prepared, true);
if (request !== startRequestRef.current) return;
setStatus({ prepared, value: next });
+ const ready =
+ installed && next.supported && next.modelId === modelId && next.state !== "missing-model";
setSetup((current) => ({
...current,
downloading: false,
- step: next.supported && next.state !== "missing-model" ? 1 : current.step,
- error:
- next.supported && next.state !== "missing-model"
- ? null
- : "The model is not ready. Try downloading it again.",
+ step: ready ? 1 : current.step,
+ error: ready ? null : "The model is not ready. Try downloading it again.",
}));
} catch {
if (request !== startRequestRef.current) return;
@@ -529,13 +577,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
});
const cancelSetupDownload = useStableCallback(async () => {
- if (!prepared || !currentStatus?.supported) return;
+ if (!prepared || !setupModelId) return;
startRequestRef.current += 1;
setSetup((current) => ({ ...current, open: false, downloading: false }));
try {
- await runEnvironmentSpeech(
- cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),
- );
+ await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, setupModelId));
} catch (error) {
setSetup((current) => ({
...current,
@@ -562,6 +608,10 @@ export function useEnvironmentSpeechInput(input: HookInput) {
setup: {
...setup,
setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
+ retryCatalog:
+ setup.open && setup.catalogFailed && prepared && currentStatus?.supported
+ ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)
+ : null,
download: downloadSetupModel,
cancelDownload: cancelSetupDownload,
startRecording: async () => {
@@ -582,6 +632,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
pendingSubmissionRef.current = null;
startRequestRef.current += 1;
setQueuedStart(null);
+ setSetup(INITIAL_SETUP);
controllerRef.current?.cancel();
}),
skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
diff --git a/apps/web/src/speech/useTextFieldSpeech.tsx b/apps/web/src/speech/useTextFieldSpeech.tsx
index 9d80d7366b5..a6c99535501 100644
--- a/apps/web/src/speech/useTextFieldSpeech.tsx
+++ b/apps/web/src/speech/useTextFieldSpeech.tsx
@@ -110,9 +110,11 @@ export function TextFieldSpeechControls({
step={speech.setup.step}
status={speech.status}
model={speech.setup.model}
+ language={speech.setup.language}
downloading={speech.setup.downloading}
error={speech.setup.error}
onOpenChange={speech.setup.setOpen}
+ onRetryCatalog={speech.setup.retryCatalog}
onDownload={() => void speech.setup.download()}
onCancelDownload={() => void speech.setup.cancelDownload()}
onStartRecording={() => void speech.setup.startRecording()}
diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md
index 65fef2cce87..c116c9e67a3 100644
--- a/docs/user/voice-input.md
+++ b/docs/user/voice-input.md
@@ -52,6 +52,7 @@ until you choose another.
In **Settings → Voice → Transcription**, download a model and choose **Use model** if it
isn’t already selected. **Test model** lets you try it without sending a message.
If you use the composer microphone before a model is installed, it offers to download one.
+The recommendation uses your Voice language, or your system’s preferred languages with **Auto-detect**.
Select the languages you dictate in to compare models that support all of them.
Accuracy shows how often the model got words or characters wrong in each
diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts
index 6f1354dcdd9..182b238d155 100644
--- a/packages/contracts/src/ipc.ts
+++ b/packages/contracts/src/ipc.ts
@@ -1048,6 +1048,8 @@ export interface DesktopBridge {
* regardless of OS settings.
*/
getSystemLocale: () => string | null;
+ /** The OS's preferred languages, most preferred first; `navigator.languages` is pinned like the locale. */
+ getPreferredSystemLanguages: () => readonly string[];
getLocalEnvironmentBootstraps: () => readonly DesktopEnvironmentBootstrap[];
getLocalEnvironmentEnabled: () => boolean;
setLocalEnvironmentEnabled: (enabled: boolean) => Promise<void>;
diff --git a/packages/shared/src/speech.test.ts b/packages/shared/src/speech.test.ts
index 8ef4e7fd20a..6d1829fd67a 100644
--- a/packages/shared/src/speech.test.ts
+++ b/packages/shared/src/speech.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test";
import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "@t3tools/contracts";
import {
+ chooseSetupSpeechModel,
decodeSpeechPcmRequest,
encodeSpeechPcmRequest,
resolveSpeechVocabulary,
@@ -71,3 +72,81 @@ describe("resolveSpeechVocabulary", () => {
expect(resolved.at(-1)?.term).toBe("word97");
});
});
+
+describe("chooseSetupSpeechModel", () => {
+ const model = (
+ id: string,
+ languages: string[],
+ supportsLanguageDetection: boolean,
+ recommended: boolean,
+ ) => ({ id, languages, supportsLanguageDetection, recommended });
+ const catalog = [
+ model("unrecommended", ["en", "de", "es"], true, false),
+ model("english", ["en"], false, true),
+ model("fixed", ["en", "de"], false, true),
+ model("detecting", ["en", "de", "fr"], true, true),
+ ];
+
+ it.each([
+ [
+ "keeps the selected English model for an English speaker",
+ "english",
+ "auto",
+ ["en"],
+ { id: "english", language: "en" },
+ ],
+ [
+ "skips models that would transcribe English instead of German",
+ "english",
+ "auto",
+ ["de", "en"],
+ { id: "detecting", language: "de" },
+ ],
+ [
+ "accepts a model without detection for a chosen language",
+ "english",
+ "de",
+ ["en"],
+ { id: "fixed", language: "de" },
+ ],
+ [
+ "keeps a selected model that transcribes the language",
+ "unrecommended",
+ "auto",
+ ["de"],
+ { id: "unrecommended", language: "de" },
+ ],
+ [
+ "moves on to the next preferred language a model transcribes",
+ "english",
+ "auto",
+ ["ja", "fr"],
+ { id: "detecting", language: "fr" },
+ ],
+ [
+ "falls back to a model that is not recommended",
+ "english",
+ "auto",
+ ["es"],
+ { id: "unrecommended", language: "es" },
+ ],
+ ["offers nothing when no model transcribes the language", "english", "auto", ["ja"], undefined],
+ [
+ "offers the selected model without a language recommendation when Auto has no preferences",
+ "fixed",
+ "auto",
+ [],
+ { id: "fixed", language: null },
+ ],
+ [
+ "does not guess another model when Auto has no preferences or selection",
+ "gone",
+ "auto",
+ [],
+ undefined,
+ ],
+ ] as const)("%s", (_, selectedModelId, intent, preferred, expected) => {
+ const choice = chooseSetupSpeechModel(catalog, { selectedModelId, intent, preferred });
+ expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected);
+ });
+});
diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts
index 0fb077452bb..c5ed9ab14ea 100644
--- a/packages/shared/src/speech.ts
+++ b/packages/shared/src/speech.ts
@@ -36,6 +36,44 @@ export function effectiveSpeechLanguage(
return model.languages.find((language) => language === "en") ?? model.languages[0] ?? "auto";
}
+/**
+ * The model first-run setup offers: the selected one, else a recommended one, else any that the
+ * server would really transcribe the person's language with. Without a language hint,
+ * offer only the selected model, without a recommendation.
+ */
+export function chooseSetupSpeechModel<
+ Model extends {
+ readonly id: string;
+ readonly languages: readonly string[];
+ readonly supportsLanguageDetection: boolean;
+ readonly recommended: boolean;
+ },
+>(
+ models: readonly Model[],
+ input: {
+ readonly selectedModelId: string;
+ readonly intent: string;
+ readonly preferred: readonly string[];
+ },
+) {
+ if (input.intent === "auto" && input.preferred.length === 0) {
+ const model = models.find(({ id }) => id === input.selectedModelId);
+ return model && { model, language: null };
+ }
+ for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {
+ const capable = models.filter((model) => {
+ const effective = effectiveSpeechLanguage(model, input.intent);
+ return effective === language || (effective === "auto" && model.languages.includes(language));
+ });
+ const model =
+ capable.find(({ id }) => id === input.selectedModelId) ??
+ capable.find(({ recommended }) => recommended) ??
+ capable[0];
+ if (model) return { model, language };
+ }
+ return undefined;
+}
+
const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleLowerCase();
/**