Skip to content
TypeScript

feat(voice): opt into cloud dictation with OpenAI, Grok, or OpenRouter (#31769)

Julius Marmingefe901ac7ba95
diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts
index 784668082bb..97eb9b0cadc 100644
--- a/apps/mobile/app.config.ts
+++ b/apps/mobile/app.config.ts
@@ -458,6 +458,9 @@ const config: ExpoConfig = {
],
extra: {
appVariant: APP_VARIANT,
+ secretsFeatureEnabled: ["1", "true"].includes(
+ process.env.T3CODE_FEATURE_SECRETS?.trim().toLowerCase() ?? "",
+ ),
iosPersonalTeamBuild: isIosPersonalTeamBuild,
relay: {
url: repoEnv.T3CODE_RELAY_URL ?? null,
diff --git a/apps/mobile/src/components/ProviderIcon.tsx b/apps/mobile/src/components/ProviderIcon.tsx
index cd4574b6f84..c5d9acacba2 100644
--- a/apps/mobile/src/components/ProviderIcon.tsx
+++ b/apps/mobile/src/components/ProviderIcon.tsx
@@ -11,7 +11,7 @@ import { resolveOfficialAcpRegistryIconUrl } from "@t3tools/contracts";
import { useAppearancePreferences } from "../features/settings/appearance/AppearancePreferencesProvider";
import { AppText as Text } from "./ui/AppText";
import { cn } from "../lib/cn";
-import { getProviderClient } from "@t3tools/client-runtime/provider-clients";
+import { getProviderClient, openRouterIcon } from "@t3tools/client-runtime/provider-clients";
type ProviderIconProps = {
readonly provider: string | null | undefined;
@@ -99,7 +99,8 @@ export function ProviderIcon(props: ProviderIconProps) {
);
}
- const packageIcon = getProviderClient(props.provider)?.icon;
+ const packageIcon =
+ props.provider === "openrouter" ? openRouterIcon : getProviderClient(props.provider)?.icon;
if (packageIcon) {
const themeFill = (fill: { readonly light: string; readonly dark: string }) =>
isDarkMode ? fill.dark : fill.light;
diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx
new file mode 100644
index 00000000000..684f8dbc837
--- /dev/null
+++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx
@@ -0,0 +1,473 @@
+import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
+import { cloudSpeechDisclosure } from "@t3tools/shared/speech";
+import { setCloudSpeechKey } from "@t3tools/client-runtime/voice-input";
+import {
+ configuredSecretSources,
+ secretRequestReference,
+} from "@t3tools/client-runtime/secret-request";
+import {
+ AuthSettingsWriteScope,
+ CLOUD_SPEECH_LABELS,
+ CLOUD_SPEECH_PROVIDER_NAMES,
+ CloudSpeechProvider,
+ CloudSpeechApiKey,
+ CloudSpeechModelId,
+ isSecretReference,
+ type CloudSpeechModel,
+ type SecretProviderId,
+ type SettingsSecretValue,
+ type VoiceTranscriptionDestination,
+} from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+import Constants from "expo-constants";
+import { AsyncResult } from "effect/reactivity";
+import { useState } from "react";
+import { ScrollView, TextInput, View } from "react-native";
+import { Sheet } from "../../components/ui/Sheet";
+import * as Dialog from "../../components/ui/Dialog";
+import { ProviderIcon } from "../../components/ProviderIcon";
+import { AppText as Text } from "../../components/ui/AppText";
+import { ListActionRow } from "../../components/ui/ListActionRow";
+import { ListChoiceRow } from "../../components/ui/ListChoiceRow";
+import { Chip } from "../../components/ui/Chip";
+import type { EnvironmentPresentation } from "../../state/environments";
+import { useEnvironmentScope } from "../../state/session";
+import {
+ chooseVoiceDestination,
+ pendingCloudVoiceChoice,
+ cloudVoiceConnectionAtom,
+ cloudVoiceKeysAtom,
+ cloudVoiceModelsAtom,
+ runSpeech,
+} from "../voice-input/voiceDestination";
+import { SettingsSection } from "./components/SettingsSection";
+
+type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
+type Props = { environment: EnvironmentPresentation; active: CloudDestination | null };
+const isApiKey = Schema.is(CloudSpeechApiKey);
+const isModelId = Schema.is(CloudSpeechModelId);
+const MODEL_ROWS = 6;
+
+export function CloudVoiceSection({ environment, active }: Props) {
+ const [open, setOpen] = useState(false);
+ return (
+ <SettingsSection title={`Cloud via ${environment.label}`}>
+ <ListActionRow
+ label="Cloud providers"
+ description={
+ active
+ ? `Using ${CLOUD_SPEECH_LABELS[active.provider]}`
+ : "Optional · use your own API key"
+ }
+ onPress={() => setOpen(true)}
+ />
+ {open && (
+ <CloudChoices environment={environment} active={active} onClose={() => setOpen(false)} />
+ )}
+ </SettingsSection>
+ );
+}
+
+function ModelPicker({
+ models,
+ value,
+ current,
+ disabled,
+ onChange,
+}: {
+ models: ReadonlyArray<CloudSpeechModel>;
+ value: string;
+ /** The grant's model, flagged when the catalog no longer lists it. */
+ current: string | undefined;
+ disabled: boolean;
+ onChange: (model: CloudSpeechModelId) => void;
+}) {
+ const [query, setQuery] = useState("");
+ const [showAll, setShowAll] = useState(false);
+ const needle = query.trim().toLowerCase();
+ const selected = models.find((model) => model.id === value);
+ const matches = models.filter((model) =>
+ `${model.name} ${model.id}`.toLowerCase().includes(needle),
+ );
+ const shown = showAll ? matches : matches.slice(0, MODEL_ROWS);
+ const delisted = current !== undefined && !models.some((model) => model.id === current);
+ return (
+ <>
+ <TextInput
+ accessibilityLabel="Search OpenRouter models"
+ placeholder="Search transcription models"
+ autoCapitalize="none"
+ autoCorrect={false}
+ editable={!disabled}
+ value={query}
+ onChangeText={setQuery}
+ className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground"
+ />
+ {delisted ? (
+ <Text className="mx-4 text-sm text-destructive">
+ OpenRouter no longer lists {current}. Dictation stays off until you choose another model.
+ </Text>
+ ) : null}
+ {selected && !shown.some((model) => model.id === selected.id) ? (
+ <ListChoiceRow
+ label={selected.name}
+ description={selected.id}
+ selected
+ disabled={disabled}
+ onPress={() => {}}
+ />
+ ) : null}
+ {shown.map((model, index) => (
+ <ListChoiceRow
+ key={model.id}
+ label={model.name}
+ description={model.id}
+ selected={model.id === value}
+ separated={index > 0}
+ disabled={disabled}
+ onPress={() => onChange(model.id)}
+ />
+ ))}
+ {matches.length === 0 ? (
+ <Text className="mx-4 text-sm text-foreground-muted">No transcription model matches.</Text>
+ ) : matches.length > shown.length ? (
+ <ListActionRow
+ label={`Show all ${matches.length} models`}
+ onPress={() => setShowAll(true)}
+ />
+ ) : null}
+ </>
+ );
+}
+
+function CloudChoices({ environment, active, onClose }: Props & { onClose: () => void }) {
+ const id = environment.environmentId;
+ const connection = useAtomValue(cloudVoiceConnectionAtom(id));
+ const keys = useAtomValue(cloudVoiceKeysAtom(id));
+ const reload = useAtomRefresh(cloudVoiceKeysAtom(id));
+ const canManage = useEnvironmentScope(id, AuthSettingsWriteScope);
+ const [editing, setEditing] = useState<{
+ provider: CloudSpeechProvider;
+ enable: boolean;
+ } | null>(null);
+ const catalogAtom = cloudVoiceModelsAtom(editing?.provider === "openrouter" ? id : null);
+ const models = useAtomValue(catalogAtom);
+ const reloadModels = useAtomRefresh(catalogAtom);
+ const [draft, setDraft] = useState("");
+ const [model, setModel] = useState("");
+ const [sourceId, setSourceId] = useState<SecretProviderId | null>(null);
+ const sources =
+ Constants.expoConfig?.extra?.secretsFeatureEnabled === true &&
+ environment.serverConfig?.secretReferences
+ ? configuredSecretSources(environment.serverConfig.settings.secretProviders)
+ : [];
+ const source = sources.find((candidate) => candidate.id === sourceId);
+ const credential =
+ sourceId !== null
+ ? source && secretRequestReference(source, draft)
+ : isApiKey(draft.trim())
+ ? draft.trim()
+ : undefined;
+ const [saving, setSaving] = useState(false);
+ const activeModel = active?.provider === "openrouter" ? active.model : undefined;
+ const enable = (commit: () => void) => {
+ try {
+ commit();
+ } catch {
+ Dialog.alert(
+ "Could not enable cloud dictation",
+ "Check device storage and choose the provider again.",
+ );
+ }
+ };
+ const edit = (provider: CloudSpeechProvider, enable: boolean) => {
+ const stored = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider];
+ const ref =
+ isSecretReference(stored) && sources.some((candidate) => candidate.id === stored.provider)
+ ? stored
+ : null;
+ setSourceId(ref?.provider ?? null);
+ setDraft(ref?.reference ?? "");
+ setModel(provider === "openrouter" ? (activeModel ?? "") : "");
+ setEditing({ provider, enable });
+ };
+ const reset = () => {
+ setDraft("");
+ setModel("");
+ setEditing(null);
+ };
+ const changeKey = async (
+ provider: CloudSpeechProvider,
+ value: SettingsSecretValue | null,
+ chosen?: CloudSpeechModelId,
+ ) => {
+ if (!connection || !canManage || saving) return;
+ const commit =
+ value !== null && (editing?.enable || (chosen !== undefined && chosen !== activeModel))
+ ? pendingCloudVoiceChoice(id, provider, chosen)
+ : null;
+ setSaving(true);
+ try {
+ await runSpeech(setCloudSpeechKey(connection, provider, value));
+ } catch {
+ setSaving(false);
+ Dialog.alert(
+ "Could not update the API key",
+ "Check the environment connection and your settings permissions.",
+ );
+ return;
+ }
+ setSaving(false);
+ if (commit) enable(commit);
+ reload();
+ onClose();
+ };
+ const catalog = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null;
+ const content =
+ !AsyncResult.isSuccess(keys) || !keys.value || !connection ? (
+ <ListActionRow
+ label={
+ !connection
+ ? "Environment not connected"
+ : keys._tag === "Failure"
+ ? "Could not load keys · Retry"
+ : "Checking API keys…"
+ }
+ disabled={!connection || keys._tag !== "Failure"}
+ onPress={reload}
+ />
+ ) : (
+ CloudSpeechProvider.literals.map((provider) => {
+ if (editing && provider !== editing.provider) return null;
+ const label = CLOUD_SPEECH_LABELS[provider];
+ const company = CLOUD_SPEECH_PROVIDER_NAMES[provider];
+ const routed = provider === "openrouter";
+ const disclosure = cloudSpeechDisclosure(provider, environment.label);
+ const saved = keys.value![provider];
+ const isActive = active?.provider === provider;
+ const isEditing = editing?.provider === provider;
+ const modelChosen =
+ isModelId(model) && (catalog?.some((entry) => entry.id === model) ?? false);
+ const modelChanged = model !== (activeModel ?? "");
+ const original = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider];
+ const keyChanged =
+ credential !== undefined &&
+ !(
+ isSecretReference(credential) &&
+ isSecretReference(original) &&
+ original.provider === credential.provider &&
+ original.reference === credential.reference
+ );
+ const routedReady =
+ modelChosen &&
+ (draft.trim() ? credential !== undefined : Boolean(saved)) &&
+ (!isActive || modelChanged || keyChanged);
+ const saveRouted = () => {
+ if (!routedReady || !isModelId(model)) return;
+ const grantChanges = !isActive || modelChanged;
+ if (credential !== undefined && keyChanged)
+ return void changeKey(provider, credential, model);
+ // Model-only changes never write to the environment.
+ if (grantChanges) enable(() => chooseVoiceDestination(id, provider, model));
+ onClose();
+ };
+ return (
+ <View key={provider}>
+ {!isEditing && (
+ <ListChoiceRow
+ label={label}
+ leading={
+ <ProviderIcon
+ provider={
+ provider === "openai" ? "codex" : provider === "xai" ? "grok" : provider
+ }
+ size={20}
+ />
+ }
+ selected={isActive}
+ disabled={saved === undefined || (!saved && !canManage) || saving}
+ description={
+ saved === undefined
+ ? "Update the environment to use this provider"
+ : isActive && routed
+ ? `${activeModel} · Tap to change model${canManage ? " or key" : ""}`
+ : isActive && saved && canManage
+ ? "Tap to manage key"
+ : saved
+ ? routed
+ ? "API key saved · choose a model to use it"
+ : "API key saved on this environment"
+ : canManage
+ ? "Add an API key to use this provider"
+ : "Ask an environment administrator to add an API key"
+ }
+ onPress={() => {
+ if (routed) return edit(provider, !isActive);
+ if (!saved) return edit(provider, true);
+ if (isActive) {
+ if (canManage) edit(provider, false);
+ return;
+ }
+ Dialog.confirm({
+ title: `Use ${label} for dictation?`,
+ message: `${disclosure} Select This device to stop sharing.`,
+ confirmText: `Use ${label}`,
+ onConfirm: () => enable(pendingCloudVoiceChoice(id, provider)),
+ });
+ }}
+ />
+ )}
+ {isEditing && routed && (
+ <>
+ <Text className="mx-4 text-sm text-foreground-muted">{disclosure}</Text>
+ {catalog ? (
+ <ModelPicker
+ models={catalog}
+ value={model}
+ current={activeModel}
+ disabled={saving}
+ onChange={setModel}
+ />
+ ) : (
+ <ListActionRow
+ label={
+ AsyncResult.isFailure(models)
+ ? "Could not load models · Retry"
+ : "Loading transcription models…"
+ }
+ disabled={!AsyncResult.isFailure(models)}
+ onPress={reloadModels}
+ />
+ )}
+ </>
+ )}
+ {isEditing && (canManage || routed) && (
+ <>
+ {canManage && sources.length > 0 && (
+ <View className="mx-4 flex-row flex-wrap gap-2">
+ <Chip
+ label="Value"
+ size="small"
+ selected={sourceId === null}
+ disabled={saving}
+ onPress={() => {
+ setDraft("");
+ setSourceId(null);
+ }}
+ />
+ {sources.map((candidate) => (
+ <Chip
+ key={candidate.id}
+ label={candidate.label}
+ size="small"
+ selected={sourceId === candidate.id}
+ disabled={saving}
+ onPress={() => {
+ setDraft("");
+ setSourceId(candidate.id);
+ }}
+ />
+ ))}
+ </View>
+ )}
+ {canManage && (
+ <TextInput
+ key={sourceId ?? "literal"}
+ accessibilityLabel={
+ sourceId ? `${label} API key reference` : `${label} API key`
+ }
+ placeholder={
+ source?.referencePlaceholder ??
+ (routed && saved ? "Saved key · enter a new one to replace" : "API key")
+ }
+ secureTextEntry={sourceId === null}
+ autoCapitalize="none"
+ autoCorrect={false}
+ autoComplete="off"
+ importantForAutofill="no"
+ maxLength={4096}
+ editable={!saving}
+ returnKeyType="done"
+ onSubmitEditing={() => {
+ if (routed) saveRouted();
+ else if (credential) void changeKey(provider, credential);
+ }}
+ value={draft}
+ onChangeText={setDraft}
+ className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground"
+ />
+ )}
+ {routed && !saved && !canManage ? (
+ <ListActionRow
+ label="Ask an administrator to add an OpenRouter key"
+ disabled
+ onPress={() => {}}
+ />
+ ) : (
+ <ListActionRow
+ label={
+ routed
+ ? isActive
+ ? "Save"
+ : "Save and enable"
+ : editing.enable
+ ? "Save and enable"
+ : "Save key"
+ }
+ description={
+ draft && !credential
+ ? (source?.invalidReferenceMessage ?? "Enter a valid API key.")
+ : routed
+ ? saved && !credential
+ ? "Uses the API key saved on this environment. The model choice applies only to this device."
+ : `Anyone with dictation access to ${environment.label} can bill this key.`
+ : editing.enable
+ ? `${disclosure} Anyone with dictation access to ${environment.label} can bill this key.`
+ : `Updates the ${company} key on ${environment.label}. Your dictation destination stays the same.`
+ }
+ disabled={(routed ? !routedReady : !credential) || saving}
+ onPress={() => {
+ if (routed) saveRouted();
+ else if (credential) void changeKey(provider, credential);
+ }}
+ />
+ )}
+ {saved && canManage && (
+ <ListActionRow
+ tone="danger"
+ label="Remove key from environment"
+ disabled={saving}
+ onPress={() =>
+ Dialog.confirm({
+ title: `Remove the ${label} key from ${environment.label}?`,
+ message: "Everyone using it for dictation will lose access.",
+ confirmText: "Remove key",
+ destructive: true,
+ onConfirm: () => void changeKey(provider, null),
+ })
+ }
+ />
+ )}
+ <ListActionRow label="Back to providers" disabled={saving} onPress={reset} />
+ </>
+ )}
+ </View>
+ );
+ })
+ );
+ return (
+ <Sheet
+ title={editing ? CLOUD_SPEECH_LABELS[editing.provider] : "Cloud providers"}
+ onClose={onClose}
+ dismissDisabled={saving}
+ >
+ <ScrollView
+ contentInsetAdjustmentBehavior="automatic"
+ keyboardShouldPersistTaps="handled"
+ contentContainerClassName="gap-2 pb-8"
+ >
+ {content}
+ </ScrollView>
+ </Sheet>
+ );
+}
diff --git a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx
index 57ca2cfe5dd..5f7328214cb 100644
--- a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx
+++ b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx
@@ -30,6 +30,7 @@ import {
hasEnvironmentVoiceCapture,
} from "../voice-input/voiceDestination";
import { VoiceModelSection } from "./VoiceModelSection";
+import { CloudVoiceSection } from "./CloudVoiceSection";
import type { EnvironmentId } from "@t3tools/contracts";
import { ThreadSwipeable } from "../home/thread-swipe-actions";
import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow";
@@ -241,7 +242,10 @@ export function SettingsVoiceRouteScreen() {
<EnvironmentChoice
key={environment.environmentId}
environment={environment}
- selected={grant?.destination.environmentId === environment.environmentId}
+ selected={
+ grant?.destination.kind === "environment" &&
+ grant.destination.environmentId === environment.environmentId
+ }
/>
))}
{isReady && grant && !grantEnvironment ? (
@@ -255,7 +259,27 @@ export function SettingsVoiceRouteScreen() {
/>
) : null}
</SettingsSection>
- {grantEnvironment ? (
+ {hasEnvironmentVoiceCapture &&
+ environments
+ .filter(
+ (env) =>
+ env.serverConfig?.environment.capabilities.cloudVoiceTranscription ||
+ (grant?.destination.kind === "cloud" &&
+ grant.destination.environmentId === env.environmentId),
+ )
+ .map((environment) => (
+ <CloudVoiceSection
+ key={environment.environmentId}
+ environment={environment}
+ active={
+ grant?.destination.kind === "cloud" &&
+ grant.destination.environmentId === environment.environmentId
+ ? grant.destination
+ : null
+ }
+ />
+ ))}
+ {grantEnvironment && grant?.destination.kind === "environment" ? (
<VoiceModelSection
environmentId={grantEnvironment.environmentId}
label={grantEnvironment.label}
diff --git a/apps/mobile/src/features/voice-input/voiceDestination.test.ts b/apps/mobile/src/features/voice-input/voiceDestination.test.ts
index 8d2f5acae08..dca713c3975 100644
--- a/apps/mobile/src/features/voice-input/voiceDestination.test.ts
+++ b/apps/mobile/src/features/voice-input/voiceDestination.test.ts
@@ -2,7 +2,12 @@ import { EnvironmentId } from "@t3tools/contracts";
import { Atom } from "effect/reactivity";
import { expect, it, vi } from "vite-plus/test";
import { appAtomRegistry } from "../../state/atom-registry";
-import { chooseVoiceDestination, voiceGrantAtom, voiceRouteAtom } from "./voiceDestination";
+import {
+ chooseVoiceDestination,
+ pendingCloudVoiceChoice,
+ voiceGrantAtom,
+ voiceRouteAtom,
+} from "./voiceDestination";
import { readVoiceGrant } from "./voiceGrantStorage";
const mocks = vi.hoisted(() => ({
@@ -46,7 +51,11 @@ vi.mock("../../state/presentation", () => ({
mocks.probe(id);
return Atom.make({
connection: { phase: id === "offline" ? "offline" : "connected" },
- serverConfig: { environment: { capabilities: { voiceTranscription: true } } },
+ serverConfig: {
+ environment: {
+ capabilities: { voiceTranscription: true, cloudVoiceTranscription: true },
+ },
+ },
});
}),
},
@@ -159,6 +168,58 @@ it("keeps local as the default and never reuses consent for an offline or resele
["https://my-server.test/api/voice/status", "error"],
["https://my-server.test/api/voice/transcribe", "error"],
]);
+ chooseVoiceDestination(host, "openai");
+ const cloud = appAtomRegistry.get(voiceRouteAtom)!.transcriber!;
+ mocks.fetch.mockResolvedValueOnce(Response.json({ openai: false, xai: true }));
+ await expect(cloud.prepare(options)).rejects.toMatchObject({
+ code: "unavailable",
+ message: expect.stringContaining("Add your OpenAI API key"),
+ });
+ mocks.fetch.mockResolvedValueOnce(Response.json({ openai: true, xai: true }));
+ const recorded = await cloud.prepare(options);
+ chooseVoiceDestination(host, "xai");
+ await expect(recorded.transcribe("file:///voice.m4a", options)).rejects.toThrow(
+ "Your transcription choice changed.",
+ );
+ mocks.fetch
+ .mockResolvedValueOnce(Response.json({ openai: true, xai: true }))
+ .mockResolvedValueOnce(Response.json({ text: "cloud" }));
+ const next = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options);
+ await expect(next.transcribe("file:///voice.m4a", options)).resolves.toBe("cloud");
+ mocks.fetch.mockResolvedValueOnce(
+ Response.json(
+ {
+ _tag: "CloudSpeechError",
+ provider: "xai",
+ reason: "rejected-key",
+ },
+ { status: 422 },
+ ),
+ );
+ await expect(next.transcribe("file:///voice.m4a", options)).rejects.toMatchObject({
+ _tag: "VoiceTranscriptionError",
+ code: "transcription-failed",
+ message: expect.stringContaining("rejected this API key"),
+ });
+ expect(mocks.fetch.mock.calls.slice(2).map(([url]) => new URL(String(url)).pathname)).toEqual([
+ "/api/voice/cloud/keys",
+ "/api/voice/cloud/keys",
+ "/api/voice/cloud/keys",
+ "/api/voice/cloud/xai/transcribe",
+ "/api/voice/cloud/xai/transcribe",
+ ]);
+ // OpenRouter without a model is no destination; with one, the exact id travels with the audio.
+ chooseVoiceDestination(host, "openrouter");
+ expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
+ chooseVoiceDestination(host, "openrouter", "acme/transcribe-1");
+ mocks.fetch
+ .mockResolvedValueOnce(Response.json({ openai: true, xai: true, openrouter: true }))
+ .mockResolvedValueOnce(Response.json({ text: "routed" }));
+ const routed = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options);
+ await expect(routed.transcribe("file:///voice.m4a", options)).resolves.toBe("routed");
+ const upload = new URL(String(mocks.fetch.mock.lastCall![0]));
+ expect(upload.pathname).toBe("/api/voice/cloud/openrouter/transcribe");
+ expect(upload.searchParams.get("model")).toBe("acme/transcribe-1");
const failWrite = () => {
throw new Error("Device storage is unavailable");
};
@@ -172,3 +233,27 @@ it("keeps local as the default and never reuses consent for an offline or resele
appAtomRegistry.reset();
}
});
+
+it("a late cloud key save cannot override a newer destination choice", () => {
+ const host = EnvironmentId.make("my-server");
+ try {
+ chooseVoiceDestination(null);
+ const first = pendingCloudVoiceChoice(host, "openai");
+ chooseVoiceDestination(null);
+ first();
+ expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
+ expect(JSON.parse(mocks.write.mock.lastCall![1])).toBeNull();
+
+ const older = pendingCloudVoiceChoice(host, "openai");
+ const newer = pendingCloudVoiceChoice(host, "xai");
+ older();
+ expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
+ newer();
+ const grant = appAtomRegistry.get(voiceGrantAtom);
+ older();
+ expect(appAtomRegistry.get(voiceGrantAtom)).toBe(grant);
+ expect(grant?.destination).toEqual({ kind: "cloud", environmentId: host, provider: "xai" });
+ } finally {
+ appAtomRegistry.reset();
+ }
+});
diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts
index 6f30769c2d3..dd27cca6748 100644
--- a/apps/mobile/src/features/voice-input/voiceDestination.ts
+++ b/apps/mobile/src/features/voice-input/voiceDestination.ts
@@ -1,10 +1,17 @@
import {
AuthOrchestrationOperateScope,
+ type CloudSpeechModelId,
+ type CloudSpeechProvider,
type EnvironmentId,
DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ type VoiceTranscriptionDestination,
} from "@t3tools/contracts";
import {
getEnvironmentSpeechModels,
+ getCloudSpeechKeys,
+ getCloudSpeechModels,
+ requireCloudSpeechKey,
+ transcribeCloudPcm,
getEnvironmentSpeechStatus,
transcribeEnvironmentPcm,
VoiceTranscriptionError,
@@ -34,38 +41,74 @@ export const voiceGrantAtom = Atom.make((get) => {
return AsyncResult.getOrElse(get(storedGrant), () => null);
});
-export function chooseVoiceDestination(environmentId: EnvironmentId | null) {
+let choiceRevision = 0;
+
+type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
+/** OpenRouter has no default model, so a cloud choice without one is no choice at all. */
+function cloudVoiceDestination(
+ environmentId: EnvironmentId,
+ provider: CloudSpeechProvider,
+ model?: CloudSpeechModelId,
+): CloudDestination | null {
+ if (provider === "openrouter")
+ return model ? { kind: "cloud", environmentId, provider, model } : null;
+ return { kind: "cloud", environmentId, provider };
+}
+
+export function chooseVoiceDestination(
+ environmentId: EnvironmentId | null,
+ provider?: CloudSpeechProvider,
+ model?: CloudSpeechModelId,
+) {
+ choiceRevision++;
// A failed change must not leave the old destination armed in this session.
appAtomRegistry.set(choice, null);
- const grant: VoiceGrant | null =
+ const destination: VoiceTranscriptionDestination | null =
environmentId === null
? null
- : {
- id: uuidv4(),
- destination: { kind: "environment", environmentId },
- };
+ : provider
+ ? cloudVoiceDestination(environmentId, provider, model)
+ : { kind: "environment", environmentId };
+ const grant: VoiceGrant | null = destination && { id: uuidv4(), destination };
writeVoiceGrant(grant);
appAtomRegistry.set(choice, grant);
}
+export function pendingCloudVoiceChoice(
+ id: EnvironmentId,
+ provider: CloudSpeechProvider,
+ model?: CloudSpeechModelId,
+) {
+ const revision = ++choiceRevision;
+ return () => {
+ if (revision === choiceRevision) chooseVoiceDestination(id, provider, model);
+ };
+}
+
const native = requireOptionalNativeModule<{
decodeRecording?: (uri: string) => Promise<Uint8Array>;
}>("T3VoiceTranscription");
export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function";
-export const voiceConnectionAtom = Atom.family((id: EnvironmentId) =>
- Atom.make((get) => {
- const environment = get(environmentPresentations.presentationAtom(id));
- if (
- !hasEnvironmentVoiceCapture ||
- environment?.connection.phase !== "connected" ||
- !environment.serverConfig?.environment.capabilities.voiceTranscription ||
- !sessionHasScope(get(environmentSession.sessionStateAtom(id)), AuthOrchestrationOperateScope)
- )
- return null;
- return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
- }),
-);
+const voiceConnections = (capability: "voiceTranscription" | "cloudVoiceTranscription") =>
+ Atom.family((id: EnvironmentId) =>
+ Atom.make((get) => {
+ const environment = get(environmentPresentations.presentationAtom(id));
+ if (
+ !hasEnvironmentVoiceCapture ||
+ environment?.connection.phase !== "connected" ||
+ !environment.serverConfig?.environment.capabilities[capability] ||
+ !sessionHasScope(
+ get(environmentSession.sessionStateAtom(id)),
+ AuthOrchestrationOperateScope,
+ )
+ )
+ return null;
+ return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
+ }),
+ );
+export const voiceConnectionAtom = voiceConnections("voiceTranscription");
+export const cloudVoiceConnectionAtom = voiceConnections("cloudVoiceTranscription");
// React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error".
export function runSpeech<A, E>(
@@ -92,10 +135,20 @@ export function runSpeech<A, E>(
{ signal },
);
}
-function environmentTranscriber(prepared: PreparedConnection, canSend: () => boolean) {
+function environmentTranscriber(
+ prepared: PreparedConnection,
+ canSend: () => boolean,
+ cloud?: CloudDestination,
+) {
+ const provider = cloud?.provider;
+ const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
return createEnvironmentTranscriber({
canSend,
prepare: async (signal) => {
+ if (provider) {
+ await runSpeech(requireCloudSpeechKey(prepared, provider), signal, canSend);
+ return "auto";
+ }
const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend);
if (!status.supported)
throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
@@ -126,7 +179,21 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo
upload: async (pcm, signal) =>
(
await runSpeech(
- transcribeEnvironmentPcm(prepared, pcm, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, canSend),
+ provider
+ ? transcribeCloudPcm(
+ prepared,
+ provider,
+ pcm,
+ DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ canSend,
+ model,
+ )
+ : transcribeEnvironmentPcm(
+ prepared,
+ pcm,
+ DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ canSend,
+ ),
signal,
canSend,
)
@@ -137,9 +204,10 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo
const speechAtom = <A, E>(
id: EnvironmentId,
read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
+ connectionAtom = voiceConnectionAtom,
) =>
Atom.make((get) => {
- const prepared = get(voiceConnectionAtom(id));
+ const prepared = get(connectionAtom(id));
return prepared
? Effect.tryPromise({
try: (signal) => runSpeech(read(prepared), signal),
@@ -155,6 +223,14 @@ const speechAtom = <A, E>(
export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) =>
speechAtom(id, getEnvironmentSpeechStatus),
);
+export const cloudVoiceKeysAtom = Atom.family((id: EnvironmentId) =>
+ speechAtom(id, getCloudSpeechKeys, cloudVoiceConnectionAtom),
+);
+export const cloudVoiceModelsAtom = Atom.family((id: EnvironmentId | null) =>
+ id === null
+ ? Atom.make(Effect.succeed(null))
+ : speechAtom(id, getCloudSpeechModels, cloudVoiceConnectionAtom),
+);
export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) =>
speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)),
);
@@ -167,7 +243,10 @@ export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) =>
export const voiceRouteAtom = Atom.make((get) => {
const grant = get(voiceGrantAtom);
if (!grant) return null;
- const connectionAtom = voiceConnectionAtom(grant.destination.environmentId);
+ const cloud = grant.destination.kind === "cloud" ? grant.destination : undefined;
+ const connectionAtom = (cloud ? cloudVoiceConnectionAtom : voiceConnectionAtom)(
+ grant.destination.environmentId,
+ );
const connection = get(connectionAtom);
const registry = get.registry;
return {
@@ -176,6 +255,7 @@ export const voiceRouteAtom = Atom.make((get) => {
environmentTranscriber(
connection,
() => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection,
+ cloud,
),
};
});
diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts
index 3e780298d70..6d2c30db161 100644
--- a/apps/server/src/environment/ServerEnvironment.ts
+++ b/apps/server/src/environment/ServerEnvironment.ts
@@ -274,6 +274,7 @@ export const make = Effect.gen(function* () {
scheduledTaskThreadModes: true,
npmReleaseLookup: true,
voiceTranscription: isSpeechPlatformSupported(hostPlatform, hostArchitecture, hostLibc),
+ cloudVoiceTranscription: true,
...(serverSelfUpdate === null ? {} : { serverSelfUpdate }),
...(serverInstallation === null ? {} : { serverInstallation }),
...(serverSelfUpdate === "boot-service" || desktopAppUpdate
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index 5c9ab9c433f..e7baa0a9116 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -43,6 +43,7 @@ import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts";
import * as PullRequestHttp from "./pullRequest/http.ts";
import * as SpeechModelStore from "./speech/SpeechModelStore.ts";
import * as SpeechService from "./speech/SpeechService.ts";
+import * as CloudSpeech from "./speech/CloudSpeech.ts";
import { speechHttpApiLayer } from "./speech/http.ts";
import { speechStreamRouteLayer } from "./speech/stream.ts";
import * as PullRequestProviderRegistry from "./pullRequest/PullRequestProviderRegistry.ts";
@@ -823,6 +824,7 @@ const layerMakeRoutes = Layer.mergeAll(
Layer.provide(SpeechModelStore.layer),
),
),
+ Layer.provide(CloudSpeech.layer.pipe(Layer.provide(SecretResolver.layer))),
Layer.provide(layerServerSettings),
Layer.provide(PreviewAutomationBroker.layer),
Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))),
diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts
index caba93a788b..6c470d1f83f 100644
--- a/apps/server/src/serverSettings.test.ts
+++ b/apps/server/src/serverSettings.test.ts
@@ -1642,6 +1642,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
const openCode = ProviderInstanceId.make("opencode");
const hub = UsageLimitSourceId.make("hub");
const withUsage = yield* serverSettings.updateSettings({
+ cloudSpeechCredentials: { openai: usageReference, xai: "literal-fixture" },
usageLimitSources: {
[hub]: {
kind: "cliproxy",
@@ -1687,6 +1688,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
secretReferences: false,
});
assert.notInclude(JSON.stringify(legacy), "op://");
+ assert.deepEqual(legacy.cloudSpeechCredentials, {
+ openai: SECRET_REDACTED,
+ xai: SECRET_REDACTED,
+ openrouter: "",
+ });
assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED);
assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED);
assert.equal(legacy.easDeviceHosts[0]?.accessToken, SECRET_REDACTED);
@@ -1696,6 +1702,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
assert.equal(configOf(legacy, antigravity)?.apiKey, SECRET_REDACTED);
assert.equal(configOf(legacy, openCode)?.serverPassword, SECRET_REDACTED);
const echoed = yield* serverSettings.updateSettings({
+ cloudSpeechCredentials: legacy.cloudSpeechCredentials,
sourceControlHosts: legacy.sourceControlHosts,
usageLimitSources: legacy.usageLimitSources,
easDeviceHosts: legacy.easDeviceHosts,
@@ -1708,6 +1715,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
"ghe.acme.test": "ghp_ghe",
});
assert.deepEqual(echoed.usageLimitSources[hub]?.managementKey, usageReference);
+ assert.deepEqual(echoed.cloudSpeechCredentials, {
+ openai: usageReference,
+ xai: "literal-fixture",
+ openrouter: "",
+ });
assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference);
assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference);
assert.deepEqual(configOf(echoed, antigravity)?.apiKey, antigravityReference);
@@ -1747,6 +1759,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
bitbucket: { accessToken: personal("Bitbucket/token") },
github: { tokens: { "github.com": personal("GitHub/token"), "ghe.acme.test": work } },
},
+ cloudSpeechCredentials: { openai: personal("Voice/key"), xai: work },
usageLimitSources: {
[hub]: {
kind: "cliproxy",
@@ -1791,6 +1804,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
"ghe.acme.test": work,
});
assert.equal(removed.usageLimitSources[hub]?.managementKey, "");
+ assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" });
assert.equal(removed.easDeviceHosts[0]?.accessToken, "");
assert.deepEqual(Object.keys(removed.secretProviders), ["work"]);
}).pipe(Effect.provide(layerServerSettingsWithSecrets())),
@@ -1800,6 +1814,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
Effect.sync(() => {
// Shipped clients decode these as strings. Each must be covered by the older-client case above.
assert.deepEqual(strictSecretReferencePaths(ServerSettings.ast), [
+ "cloudSpeechCredentials.openai",
+ "cloudSpeechCredentials.xai",
+ "cloudSpeechCredentials.openrouter",
"easDeviceHosts.[].accessToken",
"simlockDeviceHosts.[].agentToken",
"usageLimitSources.*.managementKey",
diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts
index ce93b51a8eb..ae69a61f1f5 100644
--- a/apps/server/src/settingsSecrets.ts
+++ b/apps/server/src/settingsSecrets.ts
@@ -1,6 +1,7 @@
import {
AntigravitySettings,
ClaudeSettings,
+ CloudSpeechCredentials,
CodexSettings,
EasDeviceHostConfig,
type ProviderInstanceConfig,
@@ -45,6 +46,7 @@ const providerSecretFields = new Map([
),
]);
const usageLimitSecretFields = secretSettingsFields(UsageLimitSourceConfig);
+const voiceSecretFields = secretSettingsFields(CloudSpeechCredentials);
const easDeviceHostSecretFields = secretSettingsFields(EasDeviceHostConfig);
const simlockDeviceHostSecretFields = secretSettingsFields(SimlockDeviceHostConfig);
const encode = (value: string) => Buffer.from(value, "utf8").toString("base64url");
@@ -232,6 +234,11 @@ function mapSecretFieldGroups(
return {
...settings,
+ cloudSpeechCredentials: mapFields(
+ settings.cloudSpeechCredentials,
+ voiceSecretFields,
+ (provider) => `voice-cloud-${provider}`,
+ ) as ServerSettings["cloudSpeechCredentials"],
providerInstances,
sourceControlHosts,
usageLimitSources,
diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts
index 10a37b631ec..58c84365dfc 100644
--- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts
+++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts
@@ -164,6 +164,7 @@ it.effect("reads a collected reference again once it is invalidated", () =>
},
];
cache.set("cliproxy#key", "stale");
+ cache.set("voice#key", "stale");
cache.set("expo#token", "stale");
cache.set("simlock#token", "stale");
assert.deepEqual((yield* read()).bitbucket, {
@@ -175,12 +176,14 @@ it.effect("reads a collected reference again once it is invalidated", () =>
resolver,
Effect.succeed({
sourceControlHosts: hosts,
+ cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" },
usageLimitSources,
easDeviceHosts,
simlockDeviceHosts,
}),
);
assert.isFalse(cache.has("cliproxy#key"));
+ assert.isFalse(cache.has("voice#key"));
assert.isFalse(cache.has("expo#token"));
assert.isFalse(cache.has("simlock#token"));
assert.deepEqual((yield* read()).bitbucket, {
diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.ts
index ba5a868e079..94eec45d7c9 100644
--- a/apps/server/src/sourceControl/sourceControlHostSecrets.ts
+++ b/apps/server/src/sourceControl/sourceControlHostSecrets.ts
@@ -118,7 +118,11 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate")
settings: Effect.Effect<
Pick<
ServerSettings,
- "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts" | "simlockDeviceHosts"
+ | "sourceControlHosts"
+ | "usageLimitSources"
+ | "easDeviceHosts"
+ | "simlockDeviceHosts"
+ | "cloudSpeechCredentials"
>,
E
>,
@@ -128,6 +132,7 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate")
...collectSourceControlHostSecretReferences(current.sourceControlHosts),
...collectUsageLimitSecretReferences(current.usageLimitSources),
...collectDeviceHostSecretReferences(current),
+ ...Object.values(current.cloudSpeechCredentials).filter(isSecretReference),
]),
Effect.orElseSucceed(() => undefined),
);
diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts
new file mode 100644
index 00000000000..112c7a88d9b
--- /dev/null
+++ b/apps/server/src/speech/CloudSpeech.test.ts
@@ -0,0 +1,200 @@
+import { it } from "@effect/vitest";
+import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SecretProviderId } from "@t3tools/contracts";
+import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import { expect, vi } from "vite-plus/test";
+import * as ServerSettings from "../serverSettings.ts";
+import * as CloudSpeech from "./CloudSpeech.ts";
+
+it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved key", (provider) =>
+ Effect.gen(function* () {
+ const reference = {
+ provider: SecretProviderId.make("work"),
+ reference: "op://Work/Voice/key",
+ };
+ let resolved: string | undefined;
+ let cached: string | undefined;
+ const read = vi.fn(() => Effect.sync(() => [(cached ??= resolved)]));
+ const upstream = vi.fn<typeof fetch>(async () => Response.json({ text: "hello" }));
+ const service = yield* CloudSpeech.CloudSpeech.pipe(
+ Effect.provide(
+ CloudSpeech.layer.pipe(
+ Layer.provide(FetchHttpClient.layer),
+ Layer.provide(ServerSettings.ServerSettingsService.layerTest()),
+ Layer.provide(
+ Layer.mock(SecretResolver.SecretResolver)({
+ read,
+ label: () => Effect.succeed("Work"),
+ invalidate: () =>
+ Effect.sync(() => {
+ cached = undefined;
+ }),
+ }),
+ ),
+ ),
+ ),
+ );
+ const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) =>
+ service
+ .transcribe(provider, pcm, {
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "en",
+ speechCustomWords: [{ term: "T3 Code", aliases: [] }],
+ })
+ .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
+ yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other"));
+ expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError");
+ yield* service.setKey(provider, Redacted.make("chosen"));
+ expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false });
+ yield* Effect.flip(transcribe(new Uint8Array(3)));
+ expect(upstream).not.toHaveBeenCalled();
+ expect(yield* transcribe()).toBe("hello");
+ const [url, init] = upstream.mock.calls[0]!;
+ expect(String(url)).toBe(
+ {
+ openai: "https://api.openai.com/v1/audio/transcriptions",
+ xai: "https://api.x.ai/v1/stt",
+ }[provider],
+ );
+ expect(init?.redirect).toBe("error");
+ expect(new Headers(init?.headers).get("authorization")).toBe("Bearer chosen");
+ const form = init?.body as FormData;
+ expect(form.get("model")).toBe(
+ provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0",
+ );
+ expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]);
+ expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en");
+ expect([...form.keys()].at(-1)).toBe("file");
+ const wav = new DataView(yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer()));
+ expect(wav.getUint32(24, true)).toBe(16_000);
+ expect(wav.getInt16(46, true)).toBe(16384);
+ upstream.mockResolvedValueOnce(new Response("sensitive upstream diagnostic", { status: 401 }));
+ expect(yield* Effect.flip(transcribe())).toMatchObject({ provider, reason: "rejected-key" });
+ yield* service.setKey(provider, null);
+ expect((yield* service.configured)[provider]).toBe(false);
+ yield* Effect.flip(transcribe());
+ expect(upstream).toHaveBeenCalledTimes(2);
+ yield* service.setKey(provider, reference);
+ expect((yield* service.configured)[provider]).toBe(true);
+ expect(read).not.toHaveBeenCalled();
+ resolved = "resolved-fixture";
+ expect(yield* transcribe()).toBe("hello");
+ expect(read).toHaveBeenCalledWith([reference]);
+ expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get("authorization")).toBe(
+ "Bearer resolved-fixture",
+ );
+ resolved = undefined;
+ expect(yield* Effect.flip(transcribe())).toMatchObject({
+ provider,
+ reason: "unreadable-key",
+ });
+ expect(upstream).toHaveBeenCalledTimes(3);
+ }),
+);
+
+it.effect("uploads to OpenRouter only for a listed exact model, as zero-retention JSON WAV", () =>
+ Effect.gen(function* () {
+ const upstream = vi.fn<typeof fetch>(async (input) =>
+ new URL(String(input)).pathname.endsWith("/models")
+ ? Response.json({ data: [{ id: "acme/transcribe-1" }, { id: "acme/transcribe-2" }] })
+ : Response.json({ text: "hello" }),
+ );
+ const service = yield* CloudSpeech.CloudSpeech.pipe(
+ Effect.provide(
+ CloudSpeech.layer.pipe(
+ Layer.provide(FetchHttpClient.layer),
+ Layer.provide(ServerSettings.ServerSettingsService.layerTest()),
+ Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})),
+ ),
+ ),
+ );
+ const uploads = () =>
+ upstream.mock.calls.filter(([url]) => String(url).endsWith("/audio/transcriptions"));
+ const transcribe = (model: string | undefined) =>
+ service
+ .transcribe(
+ "openrouter",
+ new Uint8Array(new Float32Array([0, 0.5]).buffer),
+ { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" },
+ model,
+ )
+ .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
+ yield* service.setKey("openrouter", Redacted.make("router-key"));
+ expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ provider: "openrouter" });
+ upstream.mockResolvedValueOnce(new Response(null, { status: 503 }));
+ expect(yield* Effect.flip(transcribe("acme/transcribe-2"))).toMatchObject({
+ reason: "unavailable",
+ });
+ expect(yield* Effect.flip(transcribe("acme/delisted"))).toMatchObject({
+ provider: "openrouter",
+ });
+ expect(uploads()).toEqual([]);
+ expect(yield* transcribe("acme/transcribe-2")).toBe("hello");
+ const [[url, init]] = uploads() as [[URL, RequestInit]];
+ expect(new URL(String(upstream.mock.calls[0]![0])).searchParams.get("zdr")).toBe("true");
+ expect(String(url)).toBe("https://openrouter.ai/api/v1/audio/transcriptions");
+ expect(init.redirect).toBe("error");
+ const headers = new Headers(init.headers);
+ expect(headers.get("authorization")).toBe("Bearer router-key");
+ expect(headers.get("content-type")).toContain("application/json");
+ const body = JSON.parse(yield* Effect.promise(() => new Response(init.body).text())) as {
+ model: string;
+ models?: unknown;
+ provider: { zdr: boolean };
+ input_audio: { format: string; data: string };
+ };
+ expect(body).toMatchObject({
+ model: "acme/transcribe-2",
+ provider: { zdr: true },
+ input_audio: { format: "wav" },
+ });
+ expect(body.models).toBeUndefined();
+ const wav = Buffer.from(body.input_audio.data, "base64");
+ expect(wav.toString("ascii", 0, 4)).toBe("RIFF");
+ expect(wav.readInt16LE(46)).toBe(16384);
+ }),
+);
+
+it.effect("removing a key while a catalog read is pending prevents the upload", () =>
+ Effect.gen(function* () {
+ const service = yield* CloudSpeech.CloudSpeech.pipe(
+ Effect.provide(
+ CloudSpeech.layer.pipe(
+ Layer.provide(FetchHttpClient.layer),
+ Layer.provide(ServerSettings.ServerSettingsService.layerTest()),
+ Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})),
+ ),
+ ),
+ );
+ const catalogStarted = Promise.withResolvers<void>();
+ const catalog = Promise.withResolvers<Response>();
+ const uploads: string[] = [];
+ const upstream: typeof fetch = async (input) => {
+ if (new URL(String(input)).pathname.endsWith("/models")) {
+ catalogStarted.resolve();
+ return catalog.promise;
+ }
+ uploads.push(String(input));
+ return Response.json({ text: "unexpected" });
+ };
+ yield* service.setKey("openrouter", Redacted.make("old-key"));
+ const job = yield* service
+ .transcribe(
+ "openrouter",
+ new Uint8Array(4),
+ DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ "acme/voice",
+ )
+ .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream), Effect.forkChild);
+ yield* Effect.promise(() => catalogStarted.promise);
+ yield* service.setKey("openrouter", null);
+ catalog.resolve(Response.json({ data: [{ id: "acme/voice" }] }));
+ const exit = yield* Fiber.await(job);
+ expect(uploads).toEqual([]);
+ expect(exit._tag).toBe("Failure");
+ }),
+);
diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts
new file mode 100644
index 00000000000..3b7eab6b524
--- /dev/null
+++ b/apps/server/src/speech/CloudSpeech.ts
@@ -0,0 +1,258 @@
+import {
+ CloudSpeechApiKey,
+ CloudSpeechError,
+ CloudSpeechModel,
+ CloudSpeechModelId,
+ SettingsSecretValue,
+ isSecretReference,
+ type SecretReference,
+ type ServerSettingsError,
+ type CloudSpeechKeys,
+ type CloudSpeechProvider,
+ EnvironmentSpeechTranscriptionResult,
+ type SpeechTranscriptionOptions,
+} from "@t3tools/contracts";
+import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
+import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import * as Schema from "effect/Schema";
+import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
+import * as ServerSettings from "../serverSettings.ts";
+import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
+import { applySpeechAliases } from "./customWords.ts";
+import { removeSpeechFillerWords } from "./fillerWords.ts";
+import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts";
+
+const endpoints = {
+ openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" },
+ xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" },
+ openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" },
+};
+const OPENROUTER_CATALOG_URL =
+ "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true";
+const isApiKey = Schema.is(CloudSpeechApiKey);
+const isModelId = Schema.is(CloudSpeechModelId);
+const sameCredential = Schema.toEquivalence(SettingsSecretValue);
+const OpenRouterCatalog = Schema.Struct({
+ data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })),
+});
+
+function wave(samples: Float32Array) {
+ const buffer = new ArrayBuffer(44 + samples.length * 2);
+ const view = new DataView(buffer);
+ const tag = (at: number, value: string) =>
+ new Uint8Array(buffer, at, 4).set(new TextEncoder().encode(value));
+ tag(0, "RIFF");
+ view.setUint32(4, buffer.byteLength - 8, true);
+ tag(8, "WAVE");
+ tag(12, "fmt ");
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, 1, true);
+ view.setUint32(24, 16_000, true);
+ view.setUint32(28, 32_000, true);
+ view.setUint16(32, 2, true);
+ view.setUint16(34, 16, true);
+ tag(36, "data");
+ view.setUint32(40, buffer.byteLength - 44, true);
+ for (let index = 0; index < samples.length; index++) {
+ const sample = samples[index]!;
+ view.setInt16(44 + index * 2, Math.round(sample * (sample < 0 ? 32768 : 32767)), true);
+ }
+ return buffer;
+}
+
+function audioBody(
+ provider: "openai" | "xai",
+ audio: ArrayBuffer,
+ language: string,
+ options: SpeechTranscriptionOptions,
+) {
+ const form = new FormData();
+ form.set("model", endpoints[provider].model);
+ if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language);
+ for (const { term } of options.speechCustomWords) {
+ if (provider === "openai" && /[<>\r\n]/.test(term)) continue;
+ form.append(provider === "openai" ? "keywords[]" : "keyterm", term);
+ }
+ if (provider === "xai") {
+ if (language !== "auto") form.set("format", "true");
+ form.set(
+ "filler_words",
+ String(!options.speechRemoveFillerWords || Boolean(options.speechCorrectionWord)),
+ );
+ if (!options.speechSilenceFiltering) form.set("vad_threshold", "0");
+ }
+ // xAI ignores options that follow the file part.
+ form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav");
+ return HttpClientRequest.bodyFormData(form);
+}
+
+function openRouterBody(model: string, audio: ArrayBuffer, language: string) {
+ return HttpClientRequest.bodyJsonUnsafe({
+ model,
+ input_audio: { data: Buffer.from(audio).toString("base64"), format: "wav" },
+ ...(language !== "auto" ? { language } : {}),
+ provider: { zdr: true },
+ });
+}
+
+export class CloudSpeech extends Context.Service<
+ CloudSpeech,
+ {
+ readonly configured: Effect.Effect<CloudSpeechKeys, ServerSettingsError>;
+ /** OpenRouter's public transcription catalog, cached briefly on this server. */
+ readonly models: Effect.Effect<ReadonlyArray<CloudSpeechModel>, CloudSpeechError>;
+ readonly setKey: (
+ provider: CloudSpeechProvider,
+ key: Redacted.Redacted<string> | SecretReference | null,
+ ) => Effect.Effect<void, ServerSettingsError>;
+ readonly transcribe: (
+ provider: CloudSpeechProvider,
+ pcm: Uint8Array,
+ options: SpeechTranscriptionOptions,
+ model?: string | undefined,
+ ) => Effect.Effect<string, CloudSpeechError | SpeechInvalidAudioError | ServerSettingsError>;
+ }
+>()("t3/speech/CloudSpeech") {}
+
+export const layer = Layer.effect(
+ CloudSpeech,
+ Effect.gen(function* () {
+ const settings = yield* ServerSettings.ServerSettingsService;
+ const resolver = yield* SecretResolver.SecretResolver;
+ const http = yield* HttpClient.HttpClient;
+ const credentials = settings.getSettings.pipe(
+ Effect.map((value) => value.cloudSpeechCredentials),
+ );
+ const configured = credentials.pipe(
+ Effect.map(({ openai, xai, openrouter }) => ({
+ openai: Boolean(openai),
+ xai: Boolean(xai),
+ openrouter: Boolean(openrouter),
+ })),
+ );
+ const execute = (request: HttpClientRequest.HttpClientRequest) =>
+ http
+ .execute(request)
+ .pipe(
+ Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
+ Effect.provideService(HttpClient.TracerPropagationEnabled, false),
+ );
+ const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () {
+ const unavailable = () =>
+ new CloudSpeechError({ provider: "openrouter", reason: "unavailable" });
+ const response = yield* execute(HttpClientRequest.get(OPENROUTER_CATALOG_URL)).pipe(
+ Effect.mapError(unavailable),
+ );
+ if (response.status < 200 || response.status >= 300) return yield* unavailable();
+ const { data } = yield* HttpClientResponse.schemaBodyJson(OpenRouterCatalog)(response).pipe(
+ Effect.mapError(unavailable),
+ );
+ return data.flatMap(({ id, name }) =>
+ isModelId(id) ? [{ id, name: name?.trim() || id }] : [],
+ );
+ });
+ // The catalog is public, so the cache holds no credential or consent state.
+ const models = yield* Effect.cachedWithTTL(
+ fetchCatalog().pipe(
+ Effect.timeoutOrElse({
+ duration: "10 seconds",
+ orElse: () =>
+ Effect.fail(new CloudSpeechError({ provider: "openrouter", reason: "unavailable" })),
+ }),
+ ),
+ (exit) => (Exit.isSuccess(exit) ? "10 minutes" : 0),
+ );
+ const setKey = Effect.fn("CloudSpeech.setKey")(function* (
+ provider: CloudSpeechProvider,
+ key: Redacted.Redacted<string> | SecretReference | null,
+ ) {
+ yield* settings.updateSettings({
+ cloudSpeechCredentials: {
+ [provider]: Redacted.isRedacted(key) ? Redacted.value(key) : (key ?? ""),
+ },
+ });
+ });
+ const transcribe = Effect.fn("CloudSpeech.transcribe")(
+ function* (
+ provider: CloudSpeechProvider,
+ pcm: Uint8Array,
+ options: SpeechTranscriptionOptions,
+ model?: string,
+ ) {
+ const source = (yield* credentials)[provider];
+ if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" });
+ if (provider === "openrouter" && (!model || !isModelId(model)))
+ return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
+ const samples = yield* Effect.try({
+ try: () => decodeSpeechPcm(pcm),
+ catch: () =>
+ new SpeechInvalidAudioError({
+ byteLength: pcm.byteLength,
+ message: "Invalid PCM audio.",
+ }),
+ });
+ const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage);
+ if (provider === "openrouter" && !(yield* models).some((entry) => entry.id === model))
+ return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
+ // A vault that locked since the last recording must not authorize a new upload.
+ if (isSecretReference(source)) yield* resolver.invalidate([source]);
+ const unreadable = () => new CloudSpeechError({ provider, reason: "unreadable-key" });
+ const key = yield* requireSettingsSecret(resolver, source).pipe(
+ Effect.mapError(unreadable),
+ );
+ if (!isApiKey(key)) return yield* unreadable();
+ const audio = wave(samples);
+ const request = HttpClientRequest.post(endpoints[provider].url).pipe(
+ HttpClientRequest.bearerToken(Redacted.make(key)),
+ provider === "openrouter"
+ ? openRouterBody(model!, audio, language)
+ : audioBody(provider, audio, language, options),
+ );
+ const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" });
+ // A catalog fetch or vault lookup may have outlived the credential it started with.
+ const current = (yield* credentials)[provider];
+ if (!current || !sameCredential(source, current))
+ return yield* new CloudSpeechError({
+ provider,
+ reason: current ? "key-changed" : "missing-key",
+ });
+ const response = yield* execute(request).pipe(Effect.mapError(unavailable));
+ if (response.status < 200 || response.status >= 300)
+ return yield* new CloudSpeechError({
+ provider,
+ reason:
+ response.status === 429
+ ? "rate-limit"
+ : [401, 402, 403].includes(response.status)
+ ? "rejected-key"
+ : response.status === 404 && provider === "openrouter"
+ ? "model-unavailable"
+ : "unavailable",
+ });
+ const { text } = yield* HttpClientResponse.schemaBodyJson(
+ EnvironmentSpeechTranscriptionResult,
+ )(response).pipe(Effect.mapError(unavailable));
+ const corrected = applySpeechAliases(text, options.speechCustomWords);
+ return (
+ options.speechRemoveFillerWords
+ ? removeSpeechFillerWords(corrected, language, options.speechCorrectionWord)
+ : corrected
+ ).trim();
+ },
+ (effect, provider) =>
+ effect.pipe(
+ Effect.timeoutOrElse({
+ duration: "2 minutes",
+ orElse: () => Effect.fail(new CloudSpeechError({ provider, reason: "unavailable" })),
+ }),
+ ),
+ );
+ return CloudSpeech.of({ configured, models, setKey, transcribe });
+ }),
+);
diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts
index 169a1e893ce..f67129be604 100644
--- a/apps/server/src/speech/http.test.ts
+++ b/apps/server/src/speech/http.test.ts
@@ -7,6 +7,9 @@ import { expect, vi } from "vite-plus/test";
import {
AuthSessionId,
AuthOrchestrationOperateScope,
+ AuthSettingsWriteScope,
+ type AuthEnvironmentScope,
+ CloudSpeechError,
DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
SPEECH_MAX_OPTIONS_BYTES,
DEFAULT_SERVER_SETTINGS,
@@ -16,6 +19,7 @@ import {
} from "@t3tools/contracts";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
import * as Schema from "effect/Schema";
import * as HttpRouter from "effect/http/HttpRouter";
import * as HttpServer from "effect/http/HttpServer";
@@ -23,6 +27,7 @@ import * as HttpApi from "effect/http-api/HttpApi";
import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
import type * as NetAddress from "effect/net/NetAddress";
import * as SpeechService from "./SpeechService.ts";
+import * as CloudSpeech from "./CloudSpeech.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as TextGeneration from "../textGeneration/TextGeneration.ts";
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
@@ -30,7 +35,20 @@ import { speechHttpApiLayer } from "./http.ts";
const decodeSessionId = Schema.decodeUnknownSync(AuthSessionId);
+const principal = (...scopes: AuthEnvironmentScope[]) =>
+ Layer.succeed(EnvironmentAuthenticatedAuth, (effect) =>
+ effect.pipe(
+ Effect.provideService(EnvironmentAuthenticatedPrincipal, {
+ sessionId: decodeSessionId("voice-test"),
+ subject: "test",
+ method: "bearer-access-token",
+ scopes: new Set(scopes),
+ }),
+ ),
+ );
+
const dependencies = Layer.mergeAll(
+ Layer.mock(CloudSpeech.CloudSpeech)({}),
Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
Layer.mock(ServerSettings.ServerSettingsService)({
getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS),
@@ -51,21 +69,11 @@ const dependencies = Layer.mergeAll(
effectIt.live("rejects oversized chunked audio without invoking transcription", () =>
Effect.gen(function* () {
const transcribe = vi.fn(() => Effect.succeed("unexpected"));
- const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) =>
- effect.pipe(
- Effect.provideService(EnvironmentAuthenticatedPrincipal, {
- sessionId: decodeSessionId("voice-test"),
- subject: "test",
- method: "bearer-access-token",
- scopes: new Set([AuthOrchestrationOperateScope]),
- }),
- ),
- );
const routes = HttpApiBuilder.layer(
HttpApi.make("environment").add(EnvironmentVoiceHttpApi),
).pipe(
Layer.provide(speechHttpApiLayer),
- Layer.provide(auth),
+ Layer.provide(principal(AuthOrchestrationOperateScope)),
Layer.provide(
Layer.succeed(SpeechService.SpeechService, {
status: Effect.succeed({ supported: false as const, reason: "test" }),
@@ -97,6 +105,14 @@ effectIt.live("rejects oversized chunked audio without invoking transcription",
}),
);
expect(select.status).toBe(403);
+ const changeKey = yield* Effect.promise(() =>
+ fetch(new URL("cloud/xai/key", url), {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ key: "fixture" }),
+ }),
+ );
+ expect(changeKey.status).toBe(403);
yield* Effect.promise(() =>
expect(
fetch(url, {
@@ -120,6 +136,101 @@ effectIt.live("rejects oversized chunked audio without invoking transcription",
}),
);
+effectIt.effect(
+ "keeps keys write-only and returns provider failures without an auth challenge",
+ () =>
+ Effect.gen(function* () {
+ let stored = false;
+ const reference = { provider: "work", reference: "op://Work/Voice/key" };
+ const save = vi.fn();
+ const transcribe = vi.fn();
+ const cloud = Layer.mock(CloudSpeech.CloudSpeech)({
+ configured: Effect.sync(() => ({ openai: stored, xai: false })),
+ models: Effect.succeed([{ id: "acme/transcribe-1", name: "Acme Transcribe" }]),
+ setKey: (provider, key) =>
+ Effect.sync(() => {
+ save(provider, key);
+ stored =
+ provider === "openai" &&
+ Redacted.isRedacted(key) &&
+ Redacted.value(key) === "fixture";
+ }),
+ transcribe: (provider, _pcm, _options, model) => {
+ transcribe(provider, model);
+ return Effect.fail(new CloudSpeechError({ provider, reason: "rejected-key" }));
+ },
+ });
+ const { handler, dispose } = HttpRouter.toWebHandler(
+ HttpApiBuilder.layer(HttpApi.make("environment").add(EnvironmentVoiceHttpApi)).pipe(
+ Layer.provide(speechHttpApiLayer),
+ Layer.provide(principal(AuthOrchestrationOperateScope, AuthSettingsWriteScope)),
+ Layer.provide(Layer.mock(SpeechService.SpeechService)({})),
+ Layer.provide(cloud),
+ Layer.provide(dependencies),
+ Layer.provide(HttpServer.layerServices),
+ Layer.provide(NodeServices.layer),
+ ),
+ { disableLogger: true },
+ );
+ yield* Effect.addFinalizer(() => Effect.promise(dispose));
+ const context = yield* Layer.build(NodeServices.layer);
+ const call = (path: string, init?: RequestInit) =>
+ Effect.promise(() =>
+ handler(new Request(`http://localhost/api/voice/cloud/${path}`, init), context),
+ );
+ const saved = yield* call("openai/key", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ key: "fixture" }),
+ });
+ expect(saved.status).toBe(204);
+ expect(yield* Effect.promise(() => saved.text())).toBe("");
+ const keys = yield* call("keys");
+ expect(yield* Effect.promise(() => keys.json())).toEqual({
+ openai: true,
+ xai: false,
+ });
+ const setReference = (key: unknown) =>
+ call("xai/key", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ key }),
+ });
+ expect((yield* setReference(reference)).status).toBe(204);
+ expect(save).toHaveBeenLastCalledWith("xai", reference);
+ expect((yield* setReference(reference.reference)).status).toBe(400);
+ expect(save).toHaveBeenCalledTimes(2);
+ const failed = yield* call("openai/transcribe", {
+ method: "POST",
+ headers: { "content-type": "application/octet-stream" },
+ body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS),
+ });
+ expect(failed.status).toBe(422);
+ expect(yield* Effect.promise(() => failed.json())).toEqual({
+ _tag: "CloudSpeechError",
+ provider: "openai",
+ reason: "rejected-key",
+ });
+ expect(transcribe).toHaveBeenLastCalledWith("openai", undefined);
+ const catalog = yield* call("openrouter/models");
+ expect(yield* Effect.promise(() => catalog.json())).toEqual({
+ models: [{ id: "acme/transcribe-1", name: "Acme Transcribe" }],
+ });
+ const audio = {
+ method: "POST",
+ headers: { "content-type": "application/octet-stream" },
+ body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS),
+ };
+ expect((yield* call("openrouter/transcribe?model=openrouter%2Fauto", audio)).status).toBe(
+ 400,
+ );
+ expect((yield* call("openrouter/transcribe?model=acme%2Ftranscribe-1", audio)).status).toBe(
+ 422,
+ );
+ expect(transcribe).toHaveBeenLastCalledWith("openrouter", "acme/transcribe-1");
+ }).pipe(Effect.scoped),
+);
+
effectIt.effect.each([
{
error: new SpeechService.SpeechInvalidAudioError({ byteLength: 3, message: "invalid PCM" }),
@@ -151,16 +262,6 @@ effectIt.effect.each([
({ error, reason, ...options }) =>
Effect.gen(function* () {
// Only authentication and the native service are replaced; exercise the actual HTTP handlers and codecs.
- const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) =>
- effect.pipe(
- Effect.provideService(EnvironmentAuthenticatedPrincipal, {
- sessionId: decodeSessionId("voice-test"),
- subject: "test",
- method: "bearer-access-token",
- scopes: new Set([AuthOrchestrationOperateScope]),
- }),
- ),
- );
const transcribe = vi.fn(() => Effect.fail(error));
const service = Layer.succeed(SpeechService.SpeechService, {
status: Effect.succeed({ supported: false as const, reason: "test" }),
@@ -179,7 +280,7 @@ effectIt.effect.each([
const { handler, dispose } = HttpRouter.toWebHandler(
HttpApiBuilder.layer(api).pipe(
Layer.provide(speechHttpApiLayer),
- Layer.provide(auth),
+ Layer.provide(principal(AuthOrchestrationOperateScope)),
Layer.provide(service),
Layer.provide(HttpServer.layerServices),
Layer.provide(dependencies),
diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts
index 2e1d1ae71fc..eb4a7a01b88 100644
--- a/apps/server/src/speech/http.ts
+++ b/apps/server/src/speech/http.ts
@@ -21,6 +21,7 @@ import {
requireEnvironmentScope,
} from "../auth/http.ts";
import * as SpeechPostProcessing from "./postProcessing.ts";
+import * as CloudSpeech from "./CloudSpeech.ts";
import * as SpeechService from "./SpeechService.ts";
const MAX_TRANSCRIBE_BODY_BYTES = SpeechService.MAX_SPEECH_BYTES + SPEECH_MAX_OPTIONS_BYTES + 4;
@@ -57,6 +58,7 @@ export const speechHttpApiLayer = HttpApiBuilder.group(
"voice",
Effect.fnUntraced(function* (handlers) {
const speech = yield* SpeechService.SpeechService;
+ const cloud = yield* CloudSpeech.CloudSpeech;
const postProcessing = yield* SpeechPostProcessing.SpeechPostProcessing;
const guard = Effect.fnUntraced(function* (
endpoint: string,
@@ -66,6 +68,53 @@ export const speechHttpApiLayer = HttpApiBuilder.group(
yield* requireEnvironmentScope(scope);
});
return handlers
+ .handle(
+ "cloudKeys",
+ Effect.fn("environment.voice.cloudKeys")(function* (args) {
+ yield* guard(args.endpoint.name);
+ return yield* cloud.configured.pipe(
+ Effect.catch((error) => failEnvironmentInternal("internal_error", error)),
+ );
+ }),
+ )
+ .handle(
+ "setCloudKey",
+ Effect.fn("environment.voice.setCloudKey")(function* (args) {
+ yield* guard(args.endpoint.name, AuthSettingsWriteScope);
+ yield* cloud
+ .setKey(args.params.provider, args.payload.key)
+ .pipe(Effect.catch((error) => failEnvironmentInternal("internal_error", error)));
+ }),
+ )
+ .handle(
+ "cloudModels",
+ Effect.fn("environment.voice.cloudModels")(function* (args) {
+ yield* guard(args.endpoint.name);
+ return { models: yield* cloud.models };
+ }),
+ )
+ .handle(
+ "transcribeCloud",
+ Effect.fn("environment.voice.transcribeCloud")(function* (args) {
+ yield* guard(args.endpoint.name);
+ const decoded = parseAudio(args.payload);
+ if (Option.isNone(decoded)) return yield* failEnvironmentInvalidRequest("invalid_audio");
+ const text = yield* cloud
+ .transcribe(
+ args.params.provider,
+ decoded.value.pcm,
+ decoded.value.options,
+ args.query.model,
+ )
+ .pipe(
+ Effect.catchTags({
+ ServerSettingsError: (error) => failEnvironmentInternal("internal_error", error),
+ SpeechInvalidAudioError: () => failEnvironmentInvalidRequest("invalid_audio"),
+ }),
+ );
+ return { text };
+ }),
+ )
.handle(
"status",
Effect.fn("environment.voice.status")(function* (args) {
diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx
new file mode 100644
index 00000000000..0295603ed20
--- /dev/null
+++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx
@@ -0,0 +1,227 @@
+import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection";
+import { EnvironmentId } from "@t3tools/contracts";
+import * as Option from "effect/Option";
+import { expect, it, vi } from "vite-plus/test";
+import { page, userEvent } from "vite-plus/test/browser";
+import { render } from "../../../test/render";
+import "~/index.css";
+import { chooseAudioDestination } from "../../speech/voiceConsent";
+import { useEnvironmentScope, usePreparedConnection } from "../../state/session";
+import { VoiceSettingsPanel } from "./VoiceSettingsPanel";
+import { SettingsScopeProvider } from "./SettingsScopeContext";
+
+vi.mock(import("../../state/session"), { spy: true });
+vi.mock("../../env", async (original) => ({
+ ...(await original<typeof import("../../env")>()),
+ isSecretsFeatureEnabled: true,
+}));
+vi.mock("../../state/environments", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../../state/environments")>()),
+ useEnvironments: () => ({
+ environments: [
+ {
+ environmentId: "voice-host",
+ label: "Laptop",
+ connection: { phase: "available" },
+ serverConfig: {
+ secretReferences: true,
+ settings: {
+ secretProviders: {
+ work: { kind: "one_password", label: "Work vault", config: { account: "work" } },
+ },
+ },
+ environment: {
+ platform: { os: "linux", arch: "x64" },
+ capabilities: { voiceTranscription: true, cloudVoiceTranscription: true },
+ },
+ },
+ },
+ ],
+ }),
+}));
+
+it("enables the chosen provider only after a successful setup and keeps its grant when editing the key", async () => {
+ const id = EnvironmentId.make("voice-host");
+ const target = new PrimaryConnectionTarget({
+ environmentId: id,
+ label: "Laptop",
+ httpBaseUrl: window.location.origin,
+ wsBaseUrl: "ws://localhost",
+ });
+ const prepared = Option.some({
+ ...target,
+ target,
+ socketUrl: target.wsBaseUrl,
+ httpAuthorization: null,
+ });
+ using cleanup = new DisposableStack();
+ cleanup.defer(() => chooseAudioDestination(null));
+ chooseAudioDestination({ kind: "environment", environmentId: id });
+ const writes: unknown[] = [];
+ let rejectNextWrite = true;
+ const fetch = globalThis.fetch;
+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => {
+ const url = new URL(request instanceof Request ? request.url : request, window.location.href);
+ if (url.pathname === "/api/voice/cloud/keys")
+ return Promise.resolve(Response.json({ openai: writes.length > 0, xai: true }));
+ if (url.pathname === "/api/voice/cloud/openai/key") {
+ if (rejectNextWrite) {
+ rejectNextWrite = false;
+ return Promise.resolve(new Response(null, { status: 403 }));
+ }
+ return new Request(request, init).json().then((body) => {
+ writes.push(body);
+ return new Response(null, { status: 204 });
+ });
+ }
+ return fetch(request, init);
+ });
+ await vi.mocked(useEnvironmentScope).withImplementation(
+ () => true,
+ async () => {
+ await vi.mocked(usePreparedConnection).withImplementation(
+ () => prepared,
+ async () => {
+ await using screen = await render(
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel />
+ </SettingsScopeProvider>,
+ );
+ const destination = screen.getByRole("combobox", { name: "Transcribe with" });
+ await expect.element(destination).toHaveTextContent("Laptop");
+ await destination.click();
+ await page.getByRole("option", { name: "OpenAI via Laptop" }).click();
+ const dialog = page.getByRole("dialog");
+ await dialog.getByRole("button", { name: "Cancel" }).click();
+ await expect.element(destination).toHaveTextContent("Laptop");
+ expect(writes).toEqual([]);
+ await destination.click();
+ await page.getByRole("option", { name: "OpenAI via Laptop" }).click();
+ await dialog.getByRole("combobox", { name: "OpenAI API key source" }).click();
+ await page.getByRole("option", { name: "Work vault" }).click();
+ await dialog.getByLabelText("OpenAI API key reference").fill("op://Work/Voice/key");
+ await userEvent.keyboard("{Enter}");
+ await expect.element(dialog.getByRole("alert")).toBeVisible();
+ expect(writes).toEqual([]);
+ await expect
+ .element(dialog.getByRole("button", { name: "Save and enable" }))
+ .toBeEnabled();
+ await dialog.getByLabelText("OpenAI API key reference").click();
+ await userEvent.keyboard("{Enter}");
+ await expect.element(dialog).not.toBeInTheDocument();
+ await expect.element(destination).toHaveTextContent("OpenAI via Laptop");
+ expect(writes).toEqual([{ key: { provider: "work", reference: "op://Work/Voice/key" } }]);
+ const grant = window.localStorage.getItem("t3code:voice-audio-grant:v1");
+ await screen.getByRole("button", { name: "Manage key" }).click();
+ await dialog.getByLabelText("OpenAI API key", { exact: true }).fill("replacement");
+ await userEvent.keyboard("{Enter}");
+ await expect.element(dialog).not.toBeInTheDocument();
+ expect(writes.at(-1)).toEqual({ key: "replacement" });
+ expect(window.localStorage.getItem("t3code:voice-audio-grant:v1")).toBe(grant);
+ await destination.click();
+ await page.getByRole("option", { name: "Grok Voice Transcribe via Laptop" }).click();
+ await dialog.getByRole("button", { name: "Use Grok Voice Transcribe" }).click();
+ await expect.element(destination).toHaveTextContent("Grok Voice Transcribe via Laptop");
+ await destination.click();
+ await page.getByRole("option", { name: "Off", exact: true }).click();
+ await expect.element(destination).toHaveTextContent("Off");
+ },
+ );
+ },
+ );
+});
+
+it("OpenRouter needs an exact model, and changing it writes no credential", async () => {
+ const id = EnvironmentId.make("voice-host");
+ const target = new PrimaryConnectionTarget({
+ environmentId: id,
+ label: "Laptop",
+ httpBaseUrl: window.location.origin,
+ wsBaseUrl: "ws://localhost",
+ });
+ const prepared = Option.some({
+ ...target,
+ target,
+ socketUrl: target.wsBaseUrl,
+ httpAuthorization: null,
+ });
+ using cleanup = new DisposableStack();
+ cleanup.defer(() => chooseAudioDestination(null));
+ chooseAudioDestination(null);
+ const writes: unknown[] = [];
+ const fetch = globalThis.fetch;
+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => {
+ const url = new URL(request instanceof Request ? request.url : request, window.location.href);
+ if (url.pathname === "/api/voice/cloud/keys")
+ return Promise.resolve(Response.json({ openai: false, xai: false, openrouter: true }));
+ if (url.pathname === "/api/voice/cloud/openrouter/models")
+ return Promise.resolve(
+ Response.json({
+ models: [
+ { id: "acme/transcribe-1", name: "Acme: Transcribe 1" },
+ { id: "other/listen-2", name: "Other: Listen 2" },
+ ],
+ }),
+ );
+ if (url.pathname === "/api/voice/cloud/openrouter/key")
+ return new Request(request, init).json().then((body) => {
+ writes.push(body);
+ return new Response(null, { status: 204 });
+ });
+ return fetch(request, init);
+ });
+ const grantKey = "t3code:voice-audio-grant:v1";
+ const grantModel = () => {
+ const raw = window.localStorage.getItem(grantKey);
+ const parsed = raw ? (JSON.parse(raw) as { destination?: { model?: string } }) : null;
+ return parsed?.destination?.model ?? null;
+ };
+ await vi.mocked(useEnvironmentScope).withImplementation(
+ () => true,
+ async () => {
+ await vi.mocked(usePreparedConnection).withImplementation(
+ () => prepared,
+ async () => {
+ await using screen = await render(
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel />
+ </SettingsScopeProvider>,
+ );
+ const destination = screen.getByRole("combobox", { name: "Transcribe with" });
+ await destination.click();
+ await page.getByRole("option", { name: "OpenRouter via Laptop" }).click();
+ const dialog = page.getByRole("dialog");
+ const save = dialog.getByRole("button", { name: "Save and enable" });
+ await expect.element(save).toBeDisabled();
+ const picker = dialog.getByRole("combobox", { name: "OpenRouter model" });
+ await expect.element(picker).toBeVisible();
+ await picker.click();
+ await userEvent.keyboard("listen");
+ await expect.element(page.getByRole("option", { name: /Other: Listen 2/ })).toBeVisible();
+ await page.getByRole("option", { name: /Other: Listen 2/ }).click();
+ await expect.element(save).toBeEnabled();
+ const key = dialog.getByLabelText("OpenRouter API key", { exact: true });
+ await key.fill("op://not-a-literal-key");
+ await expect.element(save).toBeDisabled();
+ await key.fill("");
+ await save.click();
+ await expect.element(dialog).not.toBeInTheDocument();
+ await expect.element(destination).toHaveTextContent("OpenRouter via Laptop");
+ expect(grantModel()).toBe("other/listen-2");
+ expect(writes).toEqual([]);
+ await screen.getByRole("button", { name: "Manage OpenRouter" }).click();
+ const current = dialog.getByRole("combobox", { name: "OpenRouter model" });
+ await expect.element(current).toHaveValue("Other: Listen 2");
+ await userEvent.clear(current);
+ await userEvent.type(current, "acme");
+ await page.getByRole("option", { name: /Acme: Transcribe 1/ }).click();
+ await dialog.getByRole("button", { name: "Save" }).click();
+ await expect.element(dialog).not.toBeInTheDocument();
+ await expect.element(destination).toHaveTextContent("OpenRouter via Laptop");
+ expect(grantModel()).toBe("acme/transcribe-1");
+ expect(writes).toEqual([]);
+ },
+ );
+ },
+ );
+});
diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx
new file mode 100644
index 00000000000..eafde3d65bb
--- /dev/null
+++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx
@@ -0,0 +1,471 @@
+import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
+import { cloudSpeechDisclosure } from "@t3tools/shared/speech";
+import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import {
+ configuredSecretSources,
+ secretRequestReference,
+} from "@t3tools/client-runtime/secret-request";
+import {
+ getCloudSpeechKeys,
+ getCloudSpeechModels,
+ setCloudSpeechKey,
+} from "@t3tools/client-runtime/voice-input";
+import {
+ AuthOrchestrationOperateScope,
+ AuthSettingsWriteScope,
+ CloudSpeechApiKey,
+ CloudSpeechModelId,
+ CLOUD_SPEECH_LABELS,
+ CLOUD_SPEECH_PROVIDER_NAMES,
+ isSecretReference,
+ type CloudSpeechModel,
+ type CloudSpeechProvider,
+ type SecretProviderId,
+ type SettingsSecretValue,
+ type VoiceTranscriptionDestination,
+} from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import * as Schema from "effect/Schema";
+import type { HttpClient } from "effect/http";
+import { Atom, AsyncResult } from "effect/reactivity";
+import { useId, useState } from "react";
+import { isSecretsFeatureEnabled } from "../../env";
+import {
+ inEnvironmentSpeechRuntime,
+ runEnvironmentSpeech,
+} from "../../speech/runEnvironmentSpeech";
+import {
+ chooseAudioDestination,
+ pendingCloudAudioChoice,
+ useAudioGrant,
+} from "../../speech/voiceConsent";
+import type { EnvironmentPresentation } from "../../state/environments";
+import { useEnvironmentScope, usePreparedConnection } from "../../state/session";
+import { Button } from "../ui/button";
+import {
+ Combobox,
+ ComboboxEmpty,
+ ComboboxInput,
+ ComboboxItem,
+ ComboboxList,
+ ComboboxPopup,
+ useComboboxFilter,
+} from "../ui/combobox";
+import {
+ Dialog,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogPanel,
+ DialogPopup,
+ DialogTitle,
+} from "../ui/dialog";
+import { Input } from "../ui/input";
+import { Label } from "../ui/label";
+import { SecretInput } from "./SecretInput";
+import { SecretSourceSelect } from "./SecretSourceInput";
+
+// Per dialog instance, not an Atom.family: key presence must be read fresh each time it opens.
+function useCloudRead<A, E>(
+ prepared: PreparedConnection | null,
+ read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
+) {
+ const make = (connection: PreparedConnection | null) => ({
+ prepared: connection,
+ atom: Atom.make(
+ connection ? inEnvironmentSpeechRuntime(read(connection)) : Effect.succeed(null),
+ ),
+ });
+ const [current, setCurrent] = useState(() => make(prepared));
+ if (current.prepared !== prepared) setCurrent(make(prepared));
+ return [useAtomValue(current.atom), useAtomRefresh(current.atom)] as const;
+}
+const isApiKey = Schema.is(CloudSpeechApiKey);
+const isModelId = Schema.is(CloudSpeechModelId);
+type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
+
+function ModelPicker({
+ models,
+ value,
+ onChange,
+ disabled,
+ current,
+}: {
+ models: ReadonlyArray<CloudSpeechModel>;
+ value: string;
+ onChange: (next: string) => void;
+ disabled: boolean;
+ /** The grant's model, flagged when the catalog no longer lists it. */
+ current: string | undefined;
+}) {
+ const { contains } = useComboboxFilter();
+ const selected = models.find((model) => model.id === value) ?? null;
+ const delisted = current !== undefined && !models.some((model) => model.id === current);
+ return (
+ <div className="grid gap-1">
+ <Combobox
+ items={models}
+ itemToStringLabel={(model: CloudSpeechModel) => model.name}
+ itemToStringValue={(model: CloudSpeechModel) => model.id}
+ filter={(model: CloudSpeechModel, query) =>
+ contains(model.name, query) || contains(model.id, query)
+ }
+ value={selected}
+ onValueChange={(model) => onChange(model?.id ?? "")}
+ autoHighlight
+ disabled={disabled}
+ >
+ <ComboboxInput
+ id="voice-cloud-model"
+ aria-label="OpenRouter model"
+ placeholder="Search transcription models"
+ autoFocus
+ size="sm"
+ />
+ <ComboboxPopup>
+ <ComboboxEmpty>No transcription model matches.</ComboboxEmpty>
+ <ComboboxList className="max-h-64">
+ {(model: CloudSpeechModel) => (
+ <ComboboxItem key={model.id} value={model}>
+ <span className="flex min-w-0 flex-col">
+ <span className="truncate">{model.name}</span>
+ <span className="truncate text-xs text-muted-foreground">{model.id}</span>
+ </span>
+ </ComboboxItem>
+ )}
+ </ComboboxList>
+ </ComboboxPopup>
+ </Combobox>
+ {delisted ? (
+ <p className="text-sm text-destructive" role="alert">
+ OpenRouter no longer lists {current}. Dictation stays off until you choose another model.
+ </p>
+ ) : selected ? (
+ <p className="truncate text-xs text-muted-foreground">{selected.id}</p>
+ ) : null}
+ </div>
+ );
+}
+
+export function CloudVoiceConfiguration({
+ environment,
+ provider,
+ active,
+ close,
+}: {
+ environment: Pick<EnvironmentPresentation, "environmentId" | "label" | "serverConfig">;
+ provider: CloudSpeechProvider;
+ active: boolean;
+ close: () => void;
+}) {
+ const id = environment.environmentId;
+ const name = CLOUD_SPEECH_LABELS[provider];
+ const company = CLOUD_SPEECH_PROVIDER_NAMES[provider];
+ const routed = provider === "openrouter";
+ const formId = useId();
+ const prepared = Option.getOrNull(usePreparedConnection(id));
+ const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope);
+ const canManage = useEnvironmentScope(id, AuthSettingsWriteScope);
+ const [presence, refresh] = useCloudRead(canUse ? prepared : null, getCloudSpeechKeys);
+ const [models, refreshModels] = useCloudRead(
+ routed && canUse ? prepared : null,
+ getCloudSpeechModels,
+ );
+ const grant = useAudioGrant();
+ const currentModel =
+ active && grant?.destination.kind === "cloud" && grant.destination.provider === "openrouter"
+ ? grant.destination.model
+ : undefined;
+ const [remove, setRemove] = useState(false);
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState("");
+ const [model, setModel] = useState(currentModel ?? "");
+ const saved = AsyncResult.isSuccess(presence) ? presence.value?.[provider] : undefined;
+ const existing = environment.serverConfig?.settings?.cloudSpeechCredentials?.[provider];
+ const sources =
+ isSecretsFeatureEnabled && environment.serverConfig?.secretReferences
+ ? configuredSecretSources(environment.serverConfig.settings?.secretProviders ?? {})
+ : [];
+ const reference =
+ isSecretReference(existing) && sources.some((s) => s.id === existing.provider)
+ ? existing
+ : null;
+ const [sourceId, setSourceId] = useState<SecretProviderId | null>(reference?.provider ?? null);
+ const [draft, setDraft] = useState(reference?.reference ?? "");
+ const source = sources.find((s) => s.id === sourceId);
+ const credential =
+ sourceId !== null
+ ? source && secretRequestReference(source, draft)
+ : isApiKey(draft.trim())
+ ? draft.trim()
+ : undefined;
+ const keyChanged =
+ credential !== undefined &&
+ !(
+ isSecretReference(credential) &&
+ reference?.provider === credential.provider &&
+ reference.reference === credential.reference
+ );
+ const connected = canUse && prepared !== null;
+ // Fixed providers edit the key only when nothing is saved or the device already uses them.
+ const editingKey = canManage && saved !== undefined && (routed || active || !saved);
+ const listedModels = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null;
+ const modelChosen = isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false);
+ const keyReady = draft.trim() ? credential !== undefined : Boolean(saved);
+ const modelChanged = model !== (currentModel ?? "");
+ const destination = (): CloudDestination =>
+ routed && isModelId(model)
+ ? { kind: "cloud", provider: "openrouter", environmentId: id, model }
+ : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id };
+ const enable = () => chooseAudioDestination(destination());
+ const canSubmit = routed
+ ? modelChosen && keyReady && (!active || modelChanged || keyChanged)
+ : credential !== undefined;
+
+ const saveKey = async (value: SettingsSecretValue | null, thenEnable: boolean) => {
+ if (!connected || !canManage || busy) return;
+ setError("");
+ setBusy(true);
+ try {
+ const commit = thenEnable ? pendingCloudAudioChoice(destination()) : null;
+ await runEnvironmentSpeech(setCloudSpeechKey(prepared, provider, value));
+ refresh();
+ commit?.();
+ close();
+ } catch {
+ setError("Couldn’t save this change. Check your connection and try again.");
+ }
+ setBusy(false);
+ };
+ const submit = () => {
+ if (!canSubmit || busy) return;
+ const grantChanges = !active || modelChanged;
+ if (credential !== undefined && (!routed || keyChanged))
+ return void saveKey(credential, grantChanges);
+ // Model-only changes never write to the environment.
+ if (grantChanges) enable();
+ close();
+ };
+
+ const keyField = (
+ <>
+ <Label htmlFor="voice-cloud-key">{routed && saved ? "Replace API key" : "API key"}</Label>
+ <div className="flex min-w-0 items-start gap-2">
+ {sources.length > 0 && (
+ <SecretSourceSelect
+ value={sourceId}
+ sources={sources}
+ label={`${company} API key`}
+ disabled={busy}
+ onValueChange={(next) => {
+ setSourceId(next);
+ setDraft("");
+ setError("");
+ }}
+ />
+ )}
+ <div className="min-w-0 flex-1">
+ {sourceId !== null ? (
+ <Input
+ key={sourceId}
+ id="voice-cloud-key"
+ name="reference"
+ aria-label={`${company} API key reference`}
+ autoFocus={!routed}
+ placeholder={source?.referencePlaceholder}
+ value={draft}
+ disabled={busy}
+ autoComplete="off"
+ spellCheck={false}
+ onChange={(event) => setDraft(event.target.value)}
+ />
+ ) : (
+ <SecretInput
+ id="voice-cloud-key"
+ name="key"
+ aria-label={`${company} API key`}
+ autoFocus={!routed}
+ placeholder="Enter API key"
+ value={draft}
+ configured={saved === true}
+ disabled={busy}
+ maxLength={4096}
+ onChange={(event) => setDraft(event.target.value)}
+ />
+ )}
+ </div>
+ </div>
+ <p className="text-sm text-muted-foreground">
+ {draft.trim() && !credential
+ ? (source?.invalidReferenceMessage ?? "Enter an API key, not a URL or secret reference.")
+ : active && !routed
+ ? "Updating the key keeps dictation enabled."
+ : routed && saved
+ ? "Leave blank to keep the saved key."
+ : `Anyone with dictation access to ${environment.label} can bill this key.`}
+ </p>
+ </>
+ );
+
+ return (
+ <Dialog
+ open
+ onOpenChange={(open) => {
+ if (!open && !busy) close();
+ }}
+ >
+ <DialogPopup className="max-w-md" showCloseButton={!busy}>
+ <DialogHeader>
+ <DialogTitle>
+ {remove
+ ? `Remove the ${company} key?`
+ : active
+ ? routed
+ ? "Manage OpenRouter dictation"
+ : `Manage ${company} key`
+ : `Use ${name} for dictation?`}
+ </DialogTitle>
+ <DialogDescription>
+ {remove
+ ? `Everyone using ${name} for dictation on ${environment.label} will lose access.`
+ : active && !routed
+ ? `Anyone with dictation access to ${environment.label} can use this API key.`
+ : cloudSpeechDisclosure(provider, environment.label)}
+ </DialogDescription>
+ </DialogHeader>
+ {!remove && (
+ <DialogPanel>
+ {!connected ? (
+ <p>Connect to {environment.label} with dictation access to continue.</p>
+ ) : AsyncResult.isFailure(presence) && saved === undefined ? (
+ <Button size="sm" variant="outline" onClick={refresh}>
+ Retry loading API key
+ </Button>
+ ) : routed &&
+ AsyncResult.isSuccess(presence) &&
+ presence.value &&
+ saved === undefined ? (
+ <p>Update {environment.label} to use OpenRouter dictation.</p>
+ ) : saved === undefined ? (
+ <p role="status">Checking for an API key…</p>
+ ) : routed ? (
+ <form
+ id={formId}
+ className="grid gap-3"
+ onSubmit={(event) => {
+ event.preventDefault();
+ submit();
+ }}
+ >
+ <div className="grid gap-2">
+ <Label htmlFor="voice-cloud-model">Model</Label>
+ {listedModels ? (
+ <ModelPicker
+ models={listedModels}
+ value={model}
+ onChange={setModel}
+ disabled={busy}
+ current={currentModel}
+ />
+ ) : AsyncResult.isFailure(models) ? (
+ <Button size="sm" variant="outline" type="button" onClick={refreshModels}>
+ Retry loading models
+ </Button>
+ ) : (
+ <p role="status" className="text-sm text-muted-foreground">
+ Loading transcription models…
+ </p>
+ )}
+ </div>
+ {editingKey ? (
+ <div className="grid gap-2">{keyField}</div>
+ ) : !saved ? (
+ <p>Ask an administrator on {environment.label} to add an API key.</p>
+ ) : (
+ <p className="text-sm text-muted-foreground">
+ Uses the API key saved on {environment.label}. This choice applies only to this
+ device.
+ </p>
+ )}
+ </form>
+ ) : editingKey ? (
+ <form
+ id={formId}
+ className="grid gap-2"
+ onSubmit={(event) => {
+ event.preventDefault();
+ submit();
+ }}
+ >
+ {keyField}
+ </form>
+ ) : !saved ? (
+ <p>Ask an administrator on {environment.label} to add an API key.</p>
+ ) : (
+ <p className="text-sm text-muted-foreground">
+ This choice applies only to this device. Choose Off to stop sharing.
+ </p>
+ )}
+ </DialogPanel>
+ )}
+ {error && (
+ <p className="px-6 text-sm text-destructive" role="alert">
+ {error}
+ </p>
+ )}
+ <DialogFooter>
+ {connected && canManage && saved && (active || routed) && !remove && (
+ <div className="sm:mr-auto">
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ disabled={busy}
+ onClick={() => setRemove(true)}
+ >
+ Remove key…
+ </Button>
+ </div>
+ )}
+ <Button type="button" size="sm" variant="outline" disabled={busy} onClick={close}>
+ Cancel
+ </Button>
+ {connected &&
+ (remove ? (
+ <Button
+ type="button"
+ size="sm"
+ variant="destructive"
+ disabled={busy}
+ onClick={() => void saveKey(null, false)}
+ >
+ Remove key
+ </Button>
+ ) : routed ? (
+ saved !== undefined && (saved || canManage) ? (
+ <Button type="submit" form={formId} size="sm" disabled={busy || !canSubmit}>
+ {active ? "Save" : "Save and enable"}
+ </Button>
+ ) : null
+ ) : editingKey ? (
+ <Button type="submit" form={formId} size="sm" disabled={busy || !credential}>
+ {active ? "Save key" : "Save and enable"}
+ </Button>
+ ) : saved && !active ? (
+ <Button
+ type="button"
+ size="sm"
+ onClick={() => {
+ enable();
+ close();
+ }}
+ >
+ Use {name}
+ </Button>
+ ) : null)}
+ </DialogFooter>
+ </DialogPopup>
+ </Dialog>
+ );
+}
diff --git a/apps/web/src/components/settings/SecretsSettings.logic.test.ts b/apps/web/src/components/settings/SecretsSettings.logic.test.ts
index 1a0ff29e11c..1480844f58b 100644
--- a/apps/web/src/components/settings/SecretsSettings.logic.test.ts
+++ b/apps/web/src/components/settings/SecretsSettings.logic.test.ts
@@ -31,6 +31,7 @@ const settings = Schema.decodeSync(ServerSettings)({
easDeviceHosts: [
{ kind: "eas", id: "eas", label: "EAS Simulator", accessToken: personal("op://expo") },
],
+ cloudSpeechCredentials: { openai: personal("op://voice"), xai: "" },
});
it.each([
@@ -42,6 +43,7 @@ it.each([
"GitHub · github.com",
"https://hub.test · Management Key",
"EAS Simulator · Access Token",
+ "Voice · OpenAI",
],
],
["work", ["Antigravity Work · OTHER_TOKEN", "GitHub · ghe.acme.test"]],
diff --git a/apps/web/src/components/settings/SecretsSettings.logic.ts b/apps/web/src/components/settings/SecretsSettings.logic.ts
index 3ae528aed16..a85bc9e3301 100644
--- a/apps/web/src/components/settings/SecretsSettings.logic.ts
+++ b/apps/web/src/components/settings/SecretsSettings.logic.ts
@@ -1,4 +1,6 @@
import {
+ CLOUD_SPEECH_LABELS,
+ CloudSpeechProvider,
EasDeviceHostConfig,
isSecretReference,
secretReferencesInSettings,
@@ -12,7 +14,11 @@ import { deriveProviderSettingsFields, type SettingsFormDefinition } from "./Pro
type SecretUseSettings = Pick<
ServerSettings,
- "providerInstances" | "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts"
+ | "providerInstances"
+ | "sourceControlHosts"
+ | "usageLimitSources"
+ | "easDeviceHosts"
+ | "cloudSpeechCredentials"
>;
interface SecretProviderUse {
@@ -86,5 +92,11 @@ export function secretProviderUses(
...settings.easDeviceHosts.flatMap((host) =>
uses({ settingsSchema: EasDeviceHostConfig }, `eas:${host.id}`, host.label, host),
),
+ ...CloudSpeechProvider.literals.flatMap((provider) => {
+ const credential = settings.cloudSpeechCredentials[provider];
+ return isSecretReference(credential) && credential.provider === providerId
+ ? [{ id: `voice:${provider}`, label: `Voice · ${CLOUD_SPEECH_LABELS[provider]}` }]
+ : [];
+ }),
];
}
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
index e62e7180fe0..c8e81874ce4 100644
--- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
@@ -375,7 +375,7 @@ it.each([
);
await vi.waitFor(() => expect(document.activeElement?.id).toBe(prerequisite));
await expect
- .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
+ .element(screen.getByRole("combobox", { name: "Transcribe with" }))
.toHaveTextContent("Off");
await expect
.element(screen.getByRole("switch", { name: "Clean up transcripts with a provider" }))
@@ -394,7 +394,7 @@ it("keeps a missing destination selected without claiming it cannot transcribe",
</SettingsScopeProvider>,
);
await expect
- .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
+ .element(screen.getByRole("combobox", { name: "Transcribe with" }))
.toHaveTextContent("Chosen environment (unavailable)");
await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument();
});
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
index fc351783126..5d2a8fbed22 100644
--- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
@@ -9,13 +9,17 @@ import {
updateEnvironmentSpeechModelUnloadTimeout,
} from "@t3tools/client-runtime/voice-input";
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import { openRouterIcon } from "@t3tools/client-runtime/provider-clients";
import {
AuthSettingsWriteScope,
+ CLOUD_SPEECH_LABELS,
+ CloudSpeechProvider,
DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
type EnvironmentId,
type EnvironmentSpeechModel,
type EnvironmentSpeechStatus,
type ProjectId,
+ ProviderDriverKind,
type SpeechAcceleration,
type SpeechCustomWords,
type SpeechModelUnloadTimeout,
@@ -23,11 +27,15 @@ import {
} from "@t3tools/contracts";
import { createModelSelection } from "@t3tools/shared/model";
import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings";
+import { cloudSpeechLanguages } from "@t3tools/shared/speech";
import {
AudioLinesIcon,
CheckIcon,
DownloadIcon,
LanguagesIcon,
+ KeyRoundIcon,
+ MonitorIcon,
+ MicOffIcon,
MicIcon,
RefreshCwIcon,
StarIcon,
@@ -60,6 +68,8 @@ import {
} from "../../speech/useEnvironmentSpeechInput";
import { ComposerSpeechRecordingPill } from "../chat/ComposerSpeechButton";
import { ProviderModelPicker } from "../chat/ProviderModelPicker";
+import { ProviderInstanceIcon } from "../chat/ProviderInstanceIcon";
+import { ProviderPackageIcon } from "../chat/ProviderPackageIcon";
import { TraitsPicker } from "../chat/TraitsPicker";
import { VoiceInputSetup } from "../chat/VoiceInputSetup";
import { Badge } from "../ui/badge";
@@ -79,11 +89,20 @@ import {
MenuSeparator,
MenuTrigger,
} from "../ui/menu";
-import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
+import {
+ Select,
+ SelectGroup,
+ SelectGroupLabel,
+ SelectItem,
+ SelectPopup,
+ SelectTrigger,
+ SelectValue,
+} from "../ui/select";
import { Switch } from "../ui/switch";
import { Textarea } from "../ui/textarea";
import { toastManager } from "../ui/toast";
import { MicrophoneTest } from "./MicrophoneTest";
+import { CloudVoiceConfiguration } from "./CloudVoiceConfiguration";
import { LanguageRow, ModelBenchmarks, orderSpeechLanguages } from "./SpeechModelComparison";
import { languageLabel, SpeechLanguageSelect } from "./SpeechLanguageSelect";
import { filterSpeechModels, type SpeechModelSort } from "./speechModelBrowser";
@@ -102,6 +121,17 @@ import { useScopedModelDisabledReason } from "./useScopedModelAvailability";
import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings";
const OFF = "off";
+function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) {
+ if (provider === "openrouter")
+ return <ProviderPackageIcon icon={openRouterIcon} aria-hidden className="size-4 shrink-0" />;
+ return (
+ <ProviderInstanceIcon
+ driverKind={ProviderDriverKind.make(provider === "openai" ? "codex" : "grok")}
+ displayName={CLOUD_SPEECH_LABELS[provider]}
+ iconClassName="size-4"
+ />
+ );
+}
const SYSTEM_DEFAULT = "system-default";
const formatSize = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`;
const SHORTCUT_MODE_LABELS: Record<VoiceShortcutMode, string> = {
@@ -1063,9 +1093,14 @@ export function VoiceSettingsPanel() {
const settings = useClientSettings();
const settingsScope = useOptionalSettingsScope();
const destination = useVoiceTranscriptionDestination();
+ const canManageKey = useEnvironmentScope(destination.environmentId, AuthSettingsWriteScope);
+ const [pendingCloud, setPendingCloud] = useState<{
+ provider: CloudSpeechProvider;
+ environmentId: EnvironmentId;
+ } | null>(null);
const { environments } = useEnvironments();
const [microphones, setMicrophones] = useState<BrowserMicrophone[]>([]);
- const prepared = destination.prepared;
+ const prepared = destination.cloudProvider ? null : destination.prepared;
const catalog = useVoiceModels(prepared);
const refreshMicrophones = useStableCallback(async () => {
@@ -1098,11 +1133,39 @@ export function VoiceSettingsPanel() {
? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ??
"Selected microphone (unavailable)")
: "System default";
- const destinationValue = destination.environmentId ?? OFF;
+ const environmentChoices = environments.map((environment) => ({
+ environment,
+ value: JSON.stringify(["environment", environment.environmentId]),
+ }));
+ const cloudGroups = environments
+ .filter(
+ (environment) => environment.serverConfig?.environment.capabilities.cloudVoiceTranscription,
+ )
+ .map((environment) => ({
+ environment,
+ choices: CloudSpeechProvider.literals.map((provider) => ({
+ environment,
+ provider,
+ value: JSON.stringify(["cloud", environment.environmentId, provider]),
+ })),
+ }));
+ const cloudChoices = cloudGroups.flatMap((group) => group.choices);
+ const destinationValue = destination.cloudProvider
+ ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider])
+ : destination.environmentId
+ ? JSON.stringify(["environment", destination.environmentId])
+ : OFF;
+ const cloudEnvironment = environments.find(
+ (env) => env.environmentId === pendingCloud?.environmentId,
+ );
+ // OpenRouter users without settings access still manage their own model choice.
+ const manageLabel =
+ destination.cloudProvider === "openrouter" ? "Manage OpenRouter" : "Manage key";
const chosenEnvironment = environments.find(
(environment) => environment.environmentId === destination.environmentId,
);
const chosenUnsupported =
+ !destination.cloudProvider &&
chosenEnvironment?.serverConfig != null &&
chosenEnvironment.serverConfig.environment.capabilities.voiceTranscription !== true;
const model = prepared && modelStatus?.supported ? modelStatus : null;
@@ -1110,6 +1173,8 @@ export function VoiceSettingsPanel() {
return (
<SettingsPageContainer
searchTargetFallbacks={{
+ "speech-language":
+ model || destination.cloudProvider ? undefined : "voice-transcription-destination",
"speech-models": prepared ? undefined : "voice-transcription-destination",
"speech-hardware": prepared && model ? undefined : "voice-transcription-destination",
"transcription-test": !prepared
@@ -1196,68 +1261,181 @@ export function VoiceSettingsPanel() {
</SettingsSection>
<SettingsSection id="voice-transcription" title="Transcription">
- <SettingsRow
+ <SettingsSearchTarget
{...searchableSetting("voice-transcription-destination")}
- title="Send microphone audio to"
- description={
- <>
- Microphone audio is sent to this environment to transcribe, then discarded. Choose Off
- to turn voice input off on this device. This choice is saved on this device only.
- </>
- }
- control={
- <Select
- value={destinationValue}
- onValueChange={(value) => {
- if (!value) return;
- chooseAudioDestination(
- value === OFF
- ? null
- : { kind: "environment", environmentId: value as EnvironmentId },
- );
- }}
- >
- <SelectTrigger size="sm" aria-label="Send microphone audio to" className="max-w-72">
- <SelectValue>
- {destinationValue === OFF
- ? "Off"
- : (chosenEnvironment?.label ?? "Chosen environment (unavailable)")}
- </SelectValue>
- </SelectTrigger>
- <SelectPopup align="end" alignItemWithTrigger={false}>
- <SelectItem value={OFF}>Off</SelectItem>
- {environments.map((environment) => (
- <SelectItem key={environment.environmentId} value={environment.environmentId}>
- {environment.label}
- </SelectItem>
- ))}
- {destination.environmentId && !chosenEnvironment ? (
- <SelectItem value={destination.environmentId}>
- Chosen environment (unavailable)
- </SelectItem>
- ) : null}
- </SelectPopup>
- </Select>
- }
+ className="@container/destination px-3 py-3 sm:px-4"
>
+ <div className="flex items-center justify-between gap-2">
+ <h3 className="shrink-0 text-sm font-medium">
+ <span className="@max-[18rem]/destination:hidden">Transcribe with</span>
+ <span className="@min-[18rem]/destination:hidden">Use</span>
+ </h3>
+ <div className="flex min-w-0 items-center justify-end gap-1">
+ <Select
+ value={destinationValue}
+ onValueChange={(value) => {
+ if (!value) return;
+ const cloud = cloudChoices.find((choice) => choice.value === value);
+ if (cloud) {
+ setPendingCloud({
+ environmentId: cloud.environment.environmentId,
+ provider: cloud.provider,
+ });
+ return;
+ }
+ setPendingCloud(null);
+ const chosen = environmentChoices.find((choice) => choice.value === value);
+ if (value === OFF) chooseAudioDestination(null);
+ else if (chosen)
+ chooseAudioDestination({
+ kind: "environment",
+ environmentId: chosen.environment.environmentId,
+ });
+ }}
+ >
+ <SelectTrigger
+ size="sm"
+ aria-label="Transcribe with"
+ className="w-auto min-w-0 max-w-72"
+ >
+ <SelectValue>
+ <span className="flex min-w-0 items-center gap-2">
+ {destination.cloudProvider ? (
+ <CloudSpeechLogo provider={destination.cloudProvider} />
+ ) : destination.environmentId ? (
+ <MonitorIcon aria-hidden className="size-4" />
+ ) : (
+ <MicOffIcon aria-hidden className="size-4" />
+ )}
+ <span className="truncate">
+ {destination.cloudProvider
+ ? CLOUD_SPEECH_LABELS[destination.cloudProvider]
+ : destination.environmentId
+ ? (chosenEnvironment?.label ?? "Chosen environment (unavailable)")
+ : "Off"}
+ {destination.cloudProvider && (
+ <span className="text-muted-foreground">
+ {" via "}
+ {chosenEnvironment?.label ?? "unavailable environment"}
+ </span>
+ )}
+ </span>
+ </span>
+ </SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value={OFF}>
+ <span className="flex items-center gap-2">
+ <MicOffIcon aria-hidden className="size-4" />
+ Off
+ </span>
+ </SelectItem>
+ {environmentChoices.length > 0 && (
+ <SelectGroup>
+ <SelectGroupLabel>On an environment</SelectGroupLabel>
+ {environmentChoices.map(({ environment, value }) => (
+ <SelectItem key={value} value={value}>
+ <span className="flex items-center gap-2">
+ <MonitorIcon aria-hidden className="size-4" />
+ {environment.label}
+ </span>
+ </SelectItem>
+ ))}
+ </SelectGroup>
+ )}
+ {cloudGroups.map(({ environment, choices }) => (
+ <SelectGroup key={environment.environmentId}>
+ <SelectGroupLabel>Cloud via {environment.label}</SelectGroupLabel>
+ {choices.map(({ value, provider }) => (
+ <SelectItem
+ key={value}
+ value={value}
+ aria-label={`${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}`}
+ >
+ <span className="flex items-center gap-2">
+ <CloudSpeechLogo provider={provider} />
+ {CLOUD_SPEECH_LABELS[provider]}
+ </span>
+ </SelectItem>
+ ))}
+ </SelectGroup>
+ ))}
+ {destination.environmentId &&
+ (!chosenEnvironment ||
+ (destination.cloudProvider &&
+ !cloudChoices.some((option) => option.value === destinationValue))) ? (
+ <SelectItem value={destinationValue}>
+ Chosen environment (unavailable)
+ </SelectItem>
+ ) : null}
+ </SelectPopup>
+ </Select>
+ {destination.cloudProvider &&
+ chosenEnvironment &&
+ (canManageKey || destination.cloudProvider === "openrouter") && (
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ size="icon-sm"
+ type="button"
+ variant="ghost"
+ aria-label={manageLabel}
+ onClick={() => {
+ if (destination.cloudProvider)
+ setPendingCloud({
+ provider: destination.cloudProvider,
+ environmentId: chosenEnvironment.environmentId,
+ });
+ }}
+ >
+ <KeyRoundIcon aria-hidden className="size-4" />
+ </Button>
+ }
+ />
+ <TooltipPopup>{manageLabel}</TooltipPopup>
+ </Tooltip>
+ )}
+ </div>
+ </div>
{destination.environmentId && chosenUnsupported ? (
<p className="mt-2 text-xs text-destructive">
- {chosenEnvironment.label} can’t transcribe audio. Pick an environment that can to use
- voice input.
+ {chosenEnvironment.label} can’t transcribe audio locally. Choose a different
+ destination.
</p>
) : null}
- </SettingsRow>
+ </SettingsSearchTarget>
- <SettingsRow
- {...searchableSetting("speech-language")}
- title="Language"
- description={
- model
- ? `${model.model} ${model.supportsLanguageDetection ? "detects the language automatically" : "uses one fixed language"}.`
- : "Choose an environment above to see the languages its transcription model supports."
- }
- control={<SpeechLanguageSelect model={model} />}
- />
+ {pendingCloud && cloudEnvironment && (
+ <CloudVoiceConfiguration
+ key={JSON.stringify(pendingCloud)}
+ provider={pendingCloud.provider}
+ environment={cloudEnvironment}
+ active={
+ destination.environmentId === pendingCloud.environmentId &&
+ destination.cloudProvider === pendingCloud.provider
+ }
+ close={() => setPendingCloud(null)}
+ />
+ )}
+ {(model || destination.cloudProvider) && (
+ <SettingsRow
+ {...searchableSetting("speech-language")}
+ title="Language"
+ description={
+ destination.cloudProvider
+ ? "The cloud provider can detect your language automatically."
+ : model
+ ? `${model.model} ${model.supportsLanguageDetection ? "detects the language automatically" : "uses one fixed language"}.`
+ : "Choose an environment above to see the languages its transcription model supports."
+ }
+ control={
+ <SpeechLanguageSelect
+ model={destination.cloudProvider ? cloudSpeechLanguages : model}
+ />
+ }
+ />
+ )}
{model?.supportsTranslation ? (
<SettingsRow
title="Translate to English"
@@ -1291,10 +1469,15 @@ export function VoiceSettingsPanel() {
<SettingsRow
{...searchableSetting("speech-silence-filtering")}
title="Silence filtering"
- description="Skip silence and other non-speech audio before transcription, keeping a short buffer so words aren’t cut off."
+ description={
+ destination.cloudProvider === "openai"
+ ? "OpenAI handles silence automatically."
+ : "Skip silence and other non-speech audio before transcription, keeping a short buffer so words aren’t cut off."
+ }
control={
<Switch
aria-label="Silence filtering"
+ disabled={destination.cloudProvider === "openai"}
checked={settings.speechSilenceFiltering}
onCheckedChange={(checked) =>
void updateClientSettings({ speechSilenceFiltering: checked })
@@ -1341,7 +1524,7 @@ export function VoiceSettingsPanel() {
<SettingsSection id={searchableSetting("speech-post-processing").id} title="Cleanup">
<SettingsRow
title="Clean up transcripts with a provider"
- description="Sends each transcript and the draft text around your cursor to the draft environment’s text generation provider to fix punctuation, numbers, and spoken corrections. Your audio is never sent."
+ description="Sends the transcript and nearby draft text to the text generation provider on the draft’s environment to fix punctuation, numbers, and spoken corrections. Sends text only; off by default."
control={
<Switch
aria-label="Clean up transcripts with a provider"
diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts
index 4c890c05aa4..e50bbace016 100644
--- a/apps/web/src/components/settings/settingsSearch.ts
+++ b/apps/web/src/components/settings/settingsSearch.ts
@@ -499,9 +499,11 @@ export const SETTINGS_SEARCH_ITEMS = [
},
{
id: "voice-transcription-destination",
- title: "Send microphone audio to",
+ title: "Transcribe with",
to: "/settings/voice",
- searchTerms: ["speech voice dictation stt transcription environment machine consent privacy"],
+ searchTerms: [
+ "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter",
+ ],
},
{
id: "voice-microphone",
diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts
index 5690773a610..345ebf78f2b 100644
--- a/apps/web/src/speech/browserVoiceInput.ts
+++ b/apps/web/src/speech/browserVoiceInput.ts
@@ -1,5 +1,7 @@
-import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
+import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
import {
+ requireCloudSpeechKey,
+ transcribeCloudPcm,
getEnvironmentSpeechStatus,
getEnvironmentSpeechStreamUrl,
openSpeechStream,
@@ -10,12 +12,19 @@ import {
} from "@t3tools/client-runtime/voice-input";
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
+import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
-import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
import { createBrowserPcmCapture } from "./browserPcmCapture";
+import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
+
+const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
export function createBrowserVoiceInputPlatform(input: {
readonly prepared: PreparedConnection;
+ readonly cloudProvider?: CloudSpeechProvider | undefined;
+ /** The exact OpenRouter model the grant names; required with that provider. */
+ readonly cloudModel?: CloudSpeechModelId | undefined;
readonly hasAudioConsent: () => boolean;
readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
readonly getMicrophoneId: () => string;
@@ -26,25 +35,6 @@ export function createBrowserVoiceInputPlatform(input: {
}) {
return createBrowserPcmCapture(input, async (options) => {
const recordingOptions = input.getTranscriptionOptions();
- const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
- throwIfVoiceTranscriptionAborted(options.signal);
- if (!status.supported)
- throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
- const language = status.languages
- ? effectiveSpeechLanguage(
- {
- languages: status.languages,
- supportsLanguageDetection: status.supportsLanguageDetection ?? false,
- },
- recordingOptions.speechLanguage,
- )
- : "auto";
- const locale =
- !status.supportsStreaming &&
- status.supportsTranslation &&
- recordingOptions.speechTranslateToEnglish
- ? "en"
- : language;
const transcribeRecording = async (
uri: string,
{ signal: transcriptionSignal }: VoiceTranscriptionOptions,
@@ -53,22 +43,70 @@ export function createBrowserVoiceInputPlatform(input: {
const response = await fetch(uri, { signal: transcriptionSignal });
const pcm = new Uint8Array(await response.arrayBuffer());
const result = await runEnvironmentSpeech(
- transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent),
- {
- signal: transcriptionSignal,
- },
+ input.cloudProvider
+ ? transcribeCloudPcm(
+ input.prepared,
+ input.cloudProvider,
+ pcm,
+ recordingOptions,
+ input.hasAudioConsent,
+ input.cloudModel,
+ )
+ : transcribeEnvironmentPcm(
+ input.prepared,
+ pcm,
+ recordingOptions,
+ input.hasAudioConsent,
+ ),
+ { signal: transcriptionSignal },
);
throwIfVoiceTranscriptionAborted(transcriptionSignal);
return result.text;
} catch (cause) {
throwIfVoiceTranscriptionAborted(transcriptionSignal);
+ if (isVoiceTranscriptionError(cause)) throw cause;
throw new VoiceTranscriptionError({
code: "transcription-failed",
- message: "Voice transcription on this environment failed.",
+ message: "Voice transcription failed. Check the destination in Settings → Voice.",
cause,
});
}
};
+ if (input.cloudProvider) {
+ if (input.cloudProvider === "openrouter" && !input.cloudModel)
+ throw new VoiceTranscriptionError({
+ code: "unavailable",
+ message: "Choose an OpenRouter model in Settings → Voice.",
+ });
+ await runEnvironmentSpeech(
+ requireCloudSpeechKey(input.prepared, input.cloudProvider),
+ options,
+ );
+ throwIfVoiceTranscriptionAborted(options.signal);
+ return {
+ locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage),
+ transcribe: transcribeRecording,
+ };
+ }
+ const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
+ throwIfVoiceTranscriptionAborted(options.signal);
+ if (!status.supported)
+ throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
+ const language = status.languages
+ ? effectiveSpeechLanguage(
+ {
+ languages: status.languages,
+ supportsLanguageDetection: status.supportsLanguageDetection ?? false,
+ },
+ recordingOptions.speechLanguage,
+ )
+ : "auto";
+ const locale =
+ !status.supportsStreaming &&
+ status.supportsTranslation &&
+ recordingOptions.speechTranslateToEnglish
+ ? "en"
+ : language;
if (!status.supportsStreaming && !status.supportsBufferedStreaming) {
return { locale, transcribe: transcribeRecording };
}
diff --git a/apps/web/src/speech/runEnvironmentSpeech.ts b/apps/web/src/speech/runEnvironmentSpeech.ts
index bc6d1db747c..f9635774d03 100644
--- a/apps/web/src/speech/runEnvironmentSpeech.ts
+++ b/apps/web/src/speech/runEnvironmentSpeech.ts
@@ -6,14 +6,17 @@ import { connectionAtomRuntime } from "../connection/runtime";
import { appAtomRegistry } from "../rpc/atomRegistry";
// Relay (DPoP) environments authorize through services only the connection runtime provides.
+export function inEnvironmentSpeechRuntime<A, E>(
+ effect: Effect.Effect<A, E, HttpClient.HttpClient>,
+) {
+ return AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
+ Effect.flatMap((context) => Effect.provide(effect, context)),
+ );
+}
+
export function runEnvironmentSpeech<A, E>(
effect: Effect.Effect<A, E, HttpClient.HttpClient>,
options?: Effect.RunOptions,
): Promise<A> {
- return Effect.runPromise(
- AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
- Effect.flatMap((context) => Effect.provide(effect, context)),
- ),
- options,
- );
+ return Effect.runPromise(inEnvironmentSpeechRuntime(effect), options);
}
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
index 04534c3e0cf..8fb47222827 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
@@ -6,7 +6,7 @@ import type { PreparedConnection } from "@t3tools/client-runtime/connection";
import {
DEFAULT_CLIENT_SETTINGS,
DEFAULT_SERVER_SETTINGS,
- type EnvironmentId,
+ EnvironmentId,
type ProjectId,
type SpeechCustomWords,
type SpeechPostProcessingOptions,
@@ -14,8 +14,14 @@ import {
} from "@t3tools/contracts";
import { act, render } from "../../test/render";
+import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
-import { chooseAudioDestination, chooseTranscriptCleanup, bindAudioGrant } from "./voiceConsent";
+import {
+ chooseAudioDestination,
+ chooseTranscriptCleanup,
+ bindAudioGrant,
+ pendingCloudAudioChoice,
+} from "./voiceConsent";
import { writeBrowserClientSettings } from "../clientPersistenceStorage";
const mocks = vi.hoisted(() => ({
@@ -23,6 +29,7 @@ const mocks = vi.hoisted(() => ({
settings: {} as Record<string, unknown>,
connected: new Set<string>(),
voiceCapable: new Set<string>(),
+ cloudVoiceCapable: new Set<string>(),
canDictate: new Set<string>(),
prepared: new Map<string, unknown>(),
primaryEnvironmentId: "primary-env",
@@ -63,7 +70,10 @@ vi.mock("../state/environments", () => ({
environmentId,
serverConfig: {
environment: {
- capabilities: { voiceTranscription: mocks.voiceCapable.has(environmentId) },
+ capabilities: {
+ voiceTranscription: mocks.voiceCapable.has(environmentId),
+ cloudVoiceTranscription: mocks.cloudVoiceCapable.has(environmentId),
+ },
},
},
}
@@ -124,12 +134,10 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
return { text: `Cleaned ${transcript}` };
}),
}));
-// Capture and the wire live behind the platform; its `prepared` is where audio would go.
vi.mock("./browserVoiceInput", () => ({
- createBrowserVoiceInputPlatform: (input: {
- prepared: PreparedConnection;
- getTranscriptionOptions?: () => SpeechTranscriptionOptions;
- }) => {
+ createBrowserVoiceInputPlatform: (
+ input: Parameters<typeof createBrowserVoiceInputPlatform>[0],
+ ) => {
const environmentId = input.prepared.environmentId;
mocks.platforms.push(environmentId);
return {
@@ -143,12 +151,15 @@ vi.mock("./browserVoiceInput", () => ({
},
transcriber: {
prepare: async () => {
- const options = input.getTranscriptionOptions?.();
- if (options) mocks.recordingOptions.push(options);
+ mocks.recordingOptions.push(input.getTranscriptionOptions());
return {
locale: "en",
transcribe: async () => {
- mocks.transcribeCalls.push(environmentId);
+ mocks.transcribeCalls.push(
+ input.cloudProvider
+ ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}`
+ : environmentId,
+ );
const next = mocks.transcripts.shift() ?? "dictated";
if (next instanceof Error) throw next;
return next;
@@ -171,6 +182,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
mocks.primaryEnvironmentId = "primary-env";
mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
+ mocks.cloudVoiceCapable.clear();
mocks.canDictate = new Set(mocks.voiceCapable);
mocks.prepared.clear();
mocks.statusRequests = [];
@@ -367,7 +379,15 @@ it("revokes across windows even before the storage event and survives stale pref
const grant = JSON.parse(storage.getItem(key)!) as { id: string };
const canSend = bindAudioGrant(grant.id);
expect(canSend()).toBe(true);
+ const saved = pendingCloudAudioChoice({
+ kind: "cloud",
+ provider: "openai",
+ environmentId: EnvironmentId.make("env-a"),
+ });
+ expect(canSend()).toBe(true);
storage.removeItem(key);
+ saved();
+ expect(storage.getItem(key)).toBeNull();
expect(canSend()).toBe(false);
writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false });
await vi.waitFor(() => expect(mocks.voice.available).toBe(false));
@@ -381,15 +401,89 @@ it("revokes across windows even before the storage event and survives stale pref
}
});
-it("cleans the transcript only after cleanup is separately enabled", async () => {
+it("recovers from a corrupt choice without enabling cloud dictation before the save completes", async () => {
+ setup();
+ mocks.cloudVoiceCapable.add("env-b");
+ mocks.modelState = "missing-model";
+ window.localStorage.setItem("t3code:voice-audio-grant:v1", "not-json");
+ const save = pendingCloudAudioChoice({
+ kind: "cloud",
+ environmentId: EnvironmentId.make("env-b"),
+ provider: "xai",
+ });
+ await using _screen = await render(<Probe />);
+ expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
+ await act(() => mocks.voice.start());
+ expectNoAudioSent();
+ await act(save);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.transcribeCalls).toEqual(["xai via env-b"]);
+ expect(mocks.statusRequests).toEqual([]);
+});
+
+it("sends audio to OpenRouter only under a device grant naming the exact model", async () => {
+ setup();
+ mocks.cloudVoiceCapable.add("env-b");
+ const key = "t3code:voice-audio-grant:v1";
+ const destination = { kind: "cloud", environmentId: "env-b", provider: "openrouter" };
+ window.localStorage.setItem(key, JSON.stringify({ id: "no-model", destination }));
+ {
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ expectNoAudioSent();
+ }
+ const model = "acme/transcribe-1";
+ window.localStorage.setItem(
+ key,
+ JSON.stringify({ id: "model", destination: { ...destination, model } }),
+ );
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.transcribeCalls).toEqual([`openrouter:${model} via env-b`]);
+ expect(mocks.statusRequests).toEqual([]);
+});
+
+it("uses neither destination when the cloud save marker cannot be read", async () => {
+ setup({ destination: "env-a" });
+ const key = "t3code:voice-audio-grant:v1";
+ const previous = JSON.parse(window.localStorage.getItem(key)!) as { id: string };
+ const canSendToPrevious = bindAudioGrant(previous.id);
+ const save = pendingCloudAudioChoice({
+ kind: "cloud",
+ environmentId: EnvironmentId.make("env-b"),
+ provider: "openai",
+ });
+ save();
+ const getItem = Storage.prototype.getItem;
+ using _blocked = vi.spyOn(Storage.prototype, "getItem").mockImplementation(function (
+ this: Storage,
+ k,
+ ) {
+ if (k.startsWith(`${key}:saved:`))
+ throw new DOMException("Storage is blocked", "SecurityError");
+ return getItem.call(this, k);
+ });
+ await using _screen = await render(<Probe />);
+ expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
+ expect(canSendToPrevious()).toBe(false);
+ await act(() => mocks.voice.start());
+ expectNoAudioSent();
+});
+
+it("cleans text on a cloud-only draft environment only while cleanup is separately enabled", async () => {
setup({ destination: "env-a" });
+ mocks.voiceCapable.delete(THREAD_ENV);
+ mocks.cloudVoiceCapable.add(THREAD_ENV);
chooseTranscriptCleanup(true);
const nearby = "x".repeat(1_024);
mocks.draft = "unrelated private paragraph " + nearby;
await using screen = await render(<Probe />);
await act(() => mocks.voice.start());
await act(() => mocks.voice.stop());
- expect(mocks.cleanupCalls).toHaveLength(1);
+ expect(mocks.transcribeCalls).toEqual(["env-a"]);
+ expect(mocks.cleanupCalls).toEqual([THREAD_ENV]);
expect(mocks.cleanupDraft).toHaveBeenCalledExactlyOnceWith({
text: nearby,
selection: { start: nearby.length, end: nearby.length },
@@ -414,6 +508,7 @@ it.each([
"keeps a recording without authorized cleanup off the provider: $cleanup / $authorized",
async ({ cleanup, authorized }) => {
setup({ destination: "env-a" });
+ mocks.cloudVoiceCapable.add(THREAD_ENV);
chooseTranscriptCleanup(true);
mocks.settings.speechCorrectionWord = "sorry";
if (!authorized) mocks.canDictate.delete(THREAD_ENV);
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts
index 0d1853a5dcc..279bfcfa8e8 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.ts
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts
@@ -11,7 +11,7 @@ import {
type VoiceInputState,
} from "@t3tools/client-runtime/voice-input";
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
-import { AuthOrchestrationOperateScope } from "@t3tools/contracts";
+import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts";
import type {
EnvironmentId,
EnvironmentSpeechModel,
@@ -109,11 +109,22 @@ export function useVoiceTranscriptionDestination() {
const environment = useEnvironment(environmentId);
const connection = Option.getOrNull(usePreparedConnection(environmentId));
const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope);
- const supported = environment?.serverConfig?.environment.capabilities.voiceTranscription === true;
+ const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined;
+ const provider = cloud?.provider;
+ const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
+ const supported =
+ environment?.serverConfig?.environment.capabilities[
+ provider ? "cloudVoiceTranscription" : "voiceTranscription"
+ ] === true;
return {
environmentId,
+ ...(provider ? { cloudProvider: provider } : {}),
+ ...(model ? { cloudModel: model } : {}),
grantId: grant?.id ?? null,
- label: environment?.label ?? null,
+ label:
+ provider && environment
+ ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}`
+ : (environment?.label ?? null),
prepared: supported && canDictate ? connection : null,
};
}
@@ -122,12 +133,15 @@ export function useEnvironmentSpeechInput(input: HookInput) {
const destination = useVoiceTranscriptionDestination();
const prepared = destination.prepared;
const grantId = destination.grantId;
- const originatingEnvironment = useEnvironment(input.environmentId);
+ const cloudProvider = destination.cloudProvider;
+ const cloudModel = destination.cloudModel;
+ const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment
+ .capabilities;
const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId));
const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope);
const cleanupPrepared =
canCleanUp &&
- originatingEnvironment?.serverConfig?.environment.capabilities.voiceTranscription === true
+ (draftCapabilities?.voiceTranscription || draftCapabilities?.cloudVoiceTranscription)
? originatingConnection
: null;
const clientSettings = useClientSettings();
@@ -231,6 +245,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
};
const platform = createBrowserVoiceInputPlatform({
prepared,
+ cloudProvider,
+ cloudModel,
hasAudioConsent,
getTranscriptionOptions: (): SpeechTranscriptionOptions => {
const {
@@ -344,11 +360,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
controller.dispose();
if (controllerRef.current === controller) controllerRef.current = null;
};
- }, [prepared, grantId]);
+ }, [prepared, grantId, cloudProvider, cloudModel]);
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
useEffect(() => {
- if (!prepared) return;
+ if (!prepared || cloudProvider) return;
let disposed = false;
void probeSpeechStatus(prepared)
.then((value) => {
@@ -360,8 +376,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
return () => {
disposed = true;
};
- }, [prepared]);
- const currentStatus = status?.prepared === prepared ? status.value : null;
+ }, [prepared, cloudProvider]);
+ const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
useEffect(() => {
@@ -425,6 +441,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
const controller = controllerRef.current;
const owner = latestRef.current.input.ownerKey;
if (latestRef.current.input.disabled || !controller || !prepared) return;
+ if (cloudProvider) {
+ setLevel(0);
+ await controller.start();
+ return;
+ }
const stillCurrent = () =>
request === startRequestRef.current &&
controllerRef.current === controller &&
@@ -511,8 +532,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
return {
available:
prepared !== null &&
- status?.prepared === prepared &&
- status.value?.supported !== false &&
+ (cloudProvider !== undefined ||
+ (status?.prepared === prepared && status.value?.supported !== false)) &&
typeof navigator !== "undefined" &&
Boolean(navigator.mediaDevices?.getUserMedia) &&
typeof AudioWorkletNode !== "undefined",
diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts
index 81da7cd6adb..726717a583e 100644
--- a/apps/web/src/speech/voiceConsent.ts
+++ b/apps/web/src/speech/voiceConsent.ts
@@ -1,8 +1,11 @@
import { VoiceTranscriptionDestination } from "@t3tools/contracts";
import * as Schema from "effect/Schema";
+import { useSyncExternalStore } from "react";
import { randomUUID } from "../lib/utils";
import {
+ getLocalStorageItem,
readLocalStorageValue,
+ subscribeLocalStorageKey,
useLocalStorage,
writeLocalStorageValue,
} from "../hooks/useLocalStorage";
@@ -10,33 +13,91 @@ import {
// Separate keys: a stale window saving ordinary preferences must not restore either grant.
const AUDIO_KEY = "t3code:voice-audio-grant:v1";
const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1";
-const AudioGrant = Schema.NullOr(
- Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }),
+const Grant = Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination });
+const AudioGrant = Schema.NullOr(Grant);
+const sameGrant = Schema.toEquivalence(AudioGrant);
+const AudioChoice = Schema.NullOr(
+ Schema.Union([Grant, Schema.Struct({ pending: Grant, previous: AudioGrant })]),
);
+type AudioChoice = typeof AudioChoice.Type;
+const savedKey = (id: string) => `${AUDIO_KEY}:saved:${id}`;
+const pendingGrant = (choice: AudioChoice) =>
+ choice && "pending" in choice ? choice.pending : null;
const CleanupGrant = Schema.NullOr(Schema.String);
-const decodeAudioGrant = Schema.decodeSync(Schema.fromJsonString(AudioGrant));
+const decodeChoice = Schema.decodeSync(Schema.fromJsonString(AudioChoice));
+const readChoice = () => readLocalStorageValue(AUDIO_KEY, null, AudioChoice);
+
+function readSaveMarker(id: string | undefined) {
+ try {
+ return id ? window.localStorage.getItem(savedKey(id)) : null;
+ } catch {
+ return undefined;
+ }
+}
+
+function activeGrant(choice: AudioChoice, marker: string | null | undefined) {
+ if (!choice || !("pending" in choice)) return choice;
+ // Only a confirmed missing marker means the previous grant still applies.
+ if (marker === null) return choice.previous;
+ return marker === "true" ? choice.pending : null;
+}
export function useAudioGrant() {
- return useLocalStorage(AUDIO_KEY, null, AudioGrant)[0];
+ const [choice] = useLocalStorage(AUDIO_KEY, null, AudioChoice);
+ const id = pendingGrant(choice)?.id;
+ const marker = useSyncExternalStore(
+ (notify) => (id ? subscribeLocalStorageKey(savedKey(id), notify) : () => {}),
+ () => readSaveMarker(id),
+ () => undefined,
+ );
+ return activeGrant(choice, marker);
+}
+
+function replaceChoice(choice: AudioChoice, previous = readChoice()) {
+ const old = pendingGrant(previous);
+ writeLocalStorageValue(AUDIO_KEY, choice, null, AudioChoice);
+ if (old && pendingGrant(getLocalStorageItem(AUDIO_KEY, AudioChoice))?.id !== old.id)
+ window.localStorage.removeItem(savedKey(old.id));
}
export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) {
- writeLocalStorageValue(
- AUDIO_KEY,
- destination && { id: randomUUID(), destination },
- null,
- AudioGrant,
+ replaceChoice(destination && { id: randomUUID(), destination });
+}
+
+export function pendingCloudAudioChoice(
+ destination: Extract<VoiceTranscriptionDestination, { kind: "cloud" }>,
+) {
+ const choice = readChoice();
+ const pending = { id: randomUUID(), destination };
+ replaceChoice(
+ { pending, previous: activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id)) },
+ choice,
);
+ // A late response can acknowledge only its own choice, never write the destination another tab picked.
+ return () => {
+ if (pendingGrant(readChoice())?.id === pending.id)
+ writeLocalStorageValue(savedKey(pending.id), true, false, Schema.Boolean);
+ };
}
// Bind this recording to this act of consent, even if the user later reselects the same destination.
export function bindAudioGrant(id: string) {
try {
- const raw = window.localStorage.getItem(AUDIO_KEY);
- if (!raw || decodeAudioGrant(raw)?.id !== id) return () => false;
+ let cachedRaw: string | null = null;
+ let choice: AudioChoice = null;
+ const current = () => {
+ const raw = window.localStorage.getItem(AUDIO_KEY);
+ if (raw !== cachedRaw) {
+ choice = raw ? decodeChoice(raw) : null;
+ cachedRaw = raw;
+ }
+ return activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id));
+ };
+ const bound = current();
+ if (bound?.id !== id) return () => false;
return () => {
try {
- return window.localStorage.getItem(AUDIO_KEY) === raw;
+ return sameGrant(current(), bound);
} catch {
return false;
}
diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md
index fb84458a03a..2bf73af0ab3 100644
--- a/docs/internals/voice-input.md
+++ b/docs/internals/voice-input.md
@@ -2,8 +2,9 @@
Transcription edits a composer draft. It does not submit an agent turn. Audio is
temporary client input, and only normal message submission sends the resulting
-text. Mobile defaults to on-device transcription; any client that sends audio off
-the device needs a grant for one environment.
+text to the agent. Mobile defaults to on-device transcription. Sending audio off
+the device requires an explicit grant naming one environment and, for cloud
+transcription, one provider.
The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
owns the operation while the client supplies capture and transcription. Preparation
@@ -15,28 +16,42 @@ capture and `finish` replaces the file-based `transcribe`.
## Where audio may go
-Only an explicit audio grant authorizes off-device transcription. On mobile the
-grant is stored in device-bound secure storage, outside preference backups. The
-recorder is the same for both destinations; only a finished recording can be
-uploaded, and its grant is checked again after native decoding and before dispatch.
+On mobile, on-device transcription needs no audio grant. Environment and cloud
+transcription share a recorder and upload only finished recordings. The client
+keeps the grant in device-bound secure storage, outside preference backups, and
+rechecks it after native decoding and before upload.
+
The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another
-window saving unrelated settings cannot resurrect a revoked grant. Each new
-choice gets a distinct grant, even if the destination was previously selected. The
-[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a
-recording controller to the prepared connection of that destination and to
-nothing else. When the destination changes, disconnects, or stops advertising the
-`voiceTranscription` capability, the controller is disposed: a recording in
-progress is discarded, a failed recording kept for retry is deleted, and nothing
-falls back to another environment. Every sender in the web app, including the
-settings page's transcription test, goes through that hook. The microphone loopback
+window saving unrelated settings cannot resurrect a revoked grant. Each new choice
+gets a distinct grant, even if the destination was previously selected. The
+[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds the
+controller to that grant and the selected environment's connection. It requires
+`voiceTranscription` to use the environment's model or `cloudVoiceTranscription`
+to forward audio to the selected provider. Changing the grant or losing the
+connection, permission, or required capability discards both the active recording
+and any recording retained for retry. It never chooses a fallback destination.
+The settings page's transcription test uses the same hook. The microphone loopback
test captures separately for local playback and sends nothing. The transport
refuses redirects so credentials and audio cannot follow one to another host.
Transcript cleanup requires its own grant, off by default.
It sends text, never audio, to the text generation provider of the draft's
-own environment, because that is where the provider credentials live. A later
-backend adds its own destination kind, disclosure, and transcriber; capture and
-the controller stay as they are.
+own environment, because that is where the provider credentials live.
+
+A cloud destination binds both the forwarding environment and the named provider.
+It uses a separate, batch-only endpoint: changing a server key cannot upgrade
+an environment-local grant, and a cloud recording never uses the streaming
+socket or falls back to another provider or the environment's native model.
+
+OpenRouter routes to many upstreams, so its grant also names the exact model and
+the model travels with every upload as a query parameter. The server accepts only
+ids present in OpenRouter's public ZDR-eligible transcription catalog,
+which it caches for ten minutes after a successful read;
+router aliases such as `openrouter/auto` or `:nitro` fail schema validation before
+that. Requests set `provider.zdr` and carry no `models` fallback list. OpenRouter
+applies the account's safety-retention policy when choosing an endpoint. The catalog
+holds no credential or consent state, which is why it may be cached while key
+presence is read fresh on every request.
## Server side
diff --git a/docs/user/composer.md b/docs/user/composer.md
index 13f451e2caa..22b499ebe7b 100644
--- a/docs/user/composer.md
+++ b/docs/user/composer.md
@@ -203,6 +203,8 @@ choose the environment, and confirm. Finished recordings go only to that environ
even when a draft belongs to another one. This also works on phones without local
speech recognition. If the chosen environment disconnects, dictation stops; it never
switches hosts. Choose **This device** to revoke access and discard pending audio.
+For [cloud dictation](./voice-input.md#choose-where-audio-goes), open **Cloud providers**
+under the environment in **Settings → Voice** and confirm which provider may receive recordings.
On Android, keep T3 Code open while recording and transcribing.
On-device dictation may need a network connection to download a speech model the
diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md
index e204b286606..877a1a38a20 100644
--- a/docs/user/voice-input.md
+++ b/docs/user/voice-input.md
@@ -12,9 +12,9 @@ to a named environment on that phone, open **Settings → Voice → Transcribe o
## Choose where audio goes
-On web and desktop, transcription runs on one of your T3 Code environments.
+On web and desktop, you choose where transcription runs.
Nothing is sent until you choose one: open **Settings → Voice** and set
-**Send microphone audio to** the environment that should transcribe for you.
+**Transcribe with** to the environment that should transcribe for you.
Microphone audio recorded on this device is sent to that environment over your
authenticated connection, transcribed there, and discarded. The choice is saved
on this device only, so every computer and browser you use asks for it once.
@@ -23,10 +23,33 @@ Set it back to **Off** at any time; a recording in progress is discarded.
If the chosen environment goes offline, voice input waits for it. T3 Code never
substitutes another environment.
+For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment
+you want to use. If it needs a key, enter one and choose **Save and enable**.
+You can also choose a [secret provider](secrets.md) instead of **Value** and enter a
+reference. Unlock the vault on that environment before dictating. Anyone with
+dictation access to the environment can use the key, but only this device opts in.
+If a key is already saved, just confirm the provider. After
+you stop, the environment forwards your recording and any vocabulary hints to
+the selected provider, which bills its API account. No local model download or chat
+subscription is needed. Switch back to **This device** on mobile or **Off** on
+web and desktop to stop sharing. Removing the key stops that provider for everyone
+on the environment.
+
+**OpenRouter** works the same way but has no default model: pick one from its live
+speech-to-text catalog when you enable it, or later under **Manage OpenRouter**.
+The model choice is yours alone and needs no settings access, so anyone with
+dictation access can pick a different model without touching the saved key.
+OpenRouter chooses who runs the model. T3 requests zero data retention; your
+OpenRouter account can also allow safety retention. If no endpoint qualifies,
+dictation fails instead of switching models.
+Vocabulary hints are not sent to OpenRouter; dictionary spellings are applied
+after transcription. If OpenRouter stops listing your model, dictation stops
+until you choose another.
+
## Set up a model
-The first time you dictate, T3 Code offers to download the recommended English
-model to the chosen environment. In **Settings → Voice → Transcription**, select
+If you choose environment transcription, T3 Code offers to download the recommended
+English model to that environment. In **Settings → Voice → Transcription**, select
the languages you dictate in to compare models that support all of them.
Accuracy shows how often the model got words or characters wrong in each
language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
diff --git a/packages/client-runtime/src/providerClients.ts b/packages/client-runtime/src/providerClients.ts
index e23200653b9..3759cecbc7c 100644
--- a/packages/client-runtime/src/providerClients.ts
+++ b/packages/client-runtime/src/providerClients.ts
@@ -6,7 +6,7 @@ import {
ProviderDriverKind,
} from "@t3tools/contracts";
import { acpRegistryClient } from "@t3tools/provider-acp-registry/client";
-import { makeProviderClientRegistry } from "@t3tools/provider-core/client";
+import { makeProviderClientRegistry, type ProviderIcon } from "@t3tools/provider-core/client";
import { modelShortName } from "@t3tools/provider-core/modelShortName";
import { cursorClient } from "@t3tools/provider-cursor/client";
import { fxClient } from "@t3tools/provider-fx/client";
@@ -16,6 +16,17 @@ import { museClient } from "@t3tools/provider-muse/client";
import { openCodeClient } from "@t3tools/provider-opencode/client";
import { piClient } from "@t3tools/provider-pi/client";
+// OpenRouter's Ink and Cloud glyphs from https://openrouter.ai/brand.
+export const openRouterIcon: ProviderIcon = {
+ viewBox: "0 0 1024 730",
+ fill: { light: "#03080A", dark: "#FCFCFE" },
+ paths: [
+ {
+ d: "M795.893 0C915.776 0 1012.95 97.9963 1012.95 218.88C1012.95 339.764 915.776 437.76 795.893 437.76L1011.2 654.869C1038.55 682.447 1019.18 729.6 980.504 729.6H361.77C161.97 729.6 0 566.273 0 364.8C0 163.327 161.97 0 361.77 0L795.893 0ZM361.77 145.92C241.89 145.92 144.708 243.916 144.708 364.8C144.708 485.684 241.89 583.68 361.77 583.68C481.649 583.68 578.831 485.684 578.831 364.8C578.831 243.916 481.649 145.92 361.77 145.92Z",
+ },
+ ],
+};
+
export const providerClients = makeProviderClientRegistry([
{
driverKind: ProviderDriverKind.make("codex"),
diff --git a/packages/client-runtime/src/secretRequest.ts b/packages/client-runtime/src/secretRequest.ts
index d9612411db0..26a32e735bc 100644
--- a/packages/client-runtime/src/secretRequest.ts
+++ b/packages/client-runtime/src/secretRequest.ts
@@ -95,6 +95,12 @@ export function secretRequestSources(
providers: ServerSettings["secretProviders"],
): ReadonlyArray<SecretRequestSource> {
if (config?.secretRequestReferences !== true) return NO_SOURCES;
+ return configuredSecretSources(providers);
+}
+
+export function configuredSecretSources(
+ providers: ServerSettings["secretProviders"],
+): ReadonlyArray<SecretRequestSource> {
return (Object.keys(providers) as Array<SecretProviderId>).flatMap((id) => {
const provider = providers[id];
const definition =
diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts
index 21166d8b9a0..52fa5a0d1fe 100644
--- a/packages/client-runtime/src/voice-input/environment.ts
+++ b/packages/client-runtime/src/voice-input/environment.ts
@@ -1,8 +1,12 @@
import { encodeSpeechPcmRequest } from "@t3tools/shared/speech";
import * as Effect from "effect/Effect";
import * as Redacted from "effect/Redacted";
-import { SPEECH_STREAM_PATH } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts";
import type {
+ CloudSpeechModelId,
+ CloudSpeechProvider,
+ SettingsSecretValue,
SpeechAcceleration,
SpeechModelUnloadTimeout,
SpeechTranscriptionOptions,
@@ -17,8 +21,10 @@ import {
makeEnvironmentHttpApiUrlBuilder,
} from "../rpc/http.ts";
import { executeAuthenticatedEnvironmentHttpRequest } from "../state/environmentHttpAuth.ts";
+import { VoiceTranscriptionError } from "./transcription.ts";
const INFERENCE_OR_DOWNLOAD_TIMEOUT_MS = 30 * 60_000;
+const isCloudSpeechError = Schema.is(CloudSpeechError);
type EnvironmentApiClient<G extends "voice" | "auth"> = Effect.Success<
ReturnType<typeof makeEnvironmentHttpApiGroupClient<G>>
>;
@@ -65,6 +71,94 @@ export const getEnvironmentSpeechStatus = (prepared: PreparedConnection) =>
run: ({ client, headers }) => client.status({ headers }),
});
+export const getCloudSpeechKeys = (prepared: PreparedConnection) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "GET",
+ path: (urls) => urls.cloudKeys(),
+ run: ({ client, headers }) => client.cloudKeys({ headers }),
+ });
+
+export const getCloudSpeechModels = (prepared: PreparedConnection) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "GET",
+ path: (urls) => urls.cloudModels(),
+ run: ({ client, headers }) => client.cloudModels({ headers }),
+ });
+
+export const requireCloudSpeechKey = (
+ prepared: PreparedConnection,
+ provider: CloudSpeechProvider,
+) =>
+ getCloudSpeechKeys(prepared).pipe(
+ Effect.flatMap((keys) =>
+ keys[provider]
+ ? Effect.void
+ : Effect.fail(
+ new VoiceTranscriptionError({
+ code: "unavailable",
+ message: new CloudSpeechError({ provider, reason: "missing-key" }).message,
+ }),
+ ),
+ ),
+ );
+
+export const setCloudSpeechKey = (
+ prepared: PreparedConnection,
+ provider: CloudSpeechProvider,
+ key: SettingsSecretValue | null,
+) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.setCloudKey({ params: { provider } }),
+ run: ({ client, headers }) =>
+ client.setCloudKey({
+ headers,
+ params: { provider },
+ payload: { key: typeof key === "string" ? Redacted.make(key) : key },
+ }),
+ });
+
+export const transcribeCloudPcm = (
+ prepared: PreparedConnection,
+ provider: CloudSpeechProvider,
+ pcm: Uint8Array,
+ options: SpeechTranscriptionOptions,
+ canSend: () => boolean,
+ model?: CloudSpeechModelId,
+) => {
+ const query = model === undefined ? {} : { model };
+ return request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ canSend,
+ timeoutMs: 130_000,
+ path: (urls) => urls.transcribeCloud({ params: { provider }, query }),
+ run: ({ client, headers }) =>
+ client.transcribeCloud({
+ headers,
+ params: { provider },
+ query,
+ payload: encodeSpeechPcmRequest(pcm, options),
+ }),
+ }).pipe(
+ Effect.mapError((error) =>
+ error._tag === "RemoteEnvironmentAuthFetchError" && isCloudSpeechError(error.cause)
+ ? new VoiceTranscriptionError({
+ code: "transcription-failed",
+ message: error.cause.message,
+ })
+ : error,
+ ),
+ );
+};
+
export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => {
let endpoint = prepared.httpBaseUrl;
return request({
diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts
index 06a5399d681..5843267800f 100644
--- a/packages/client-runtime/src/voice-input/index.ts
+++ b/packages/client-runtime/src/voice-input/index.ts
@@ -14,6 +14,11 @@ export {
cancelEnvironmentSpeechModelDownload,
downloadEnvironmentSpeechModel,
getEnvironmentSpeechModels,
+ getCloudSpeechKeys,
+ getCloudSpeechModels,
+ requireCloudSpeechKey,
+ setCloudSpeechKey,
+ transcribeCloudPcm,
getEnvironmentSpeechStatus,
getEnvironmentSpeechStreamUrl,
postProcessEnvironmentTranscript,
diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts
index 627b9529a6c..96b4537478e 100644
--- a/packages/contracts/src/environment.ts
+++ b/packages/contracts/src/environment.ts
@@ -174,6 +174,7 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({
scheduledTaskThreadModes: Schema.optionalKey(Schema.Boolean),
/** Server can transcribe 16 kHz mono PCM audio on the environment machine. */
voiceTranscription: Schema.optionalKey(Schema.Boolean),
+ cloudVoiceTranscription: Schema.optionalKey(Schema.Boolean),
});
export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type;
diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts
index 8dd67640a0b..46dfcde197a 100644
--- a/packages/contracts/src/environmentHttp.ts
+++ b/packages/contracts/src/environmentHttp.ts
@@ -49,6 +49,12 @@ import {
TrimmedNonEmptyString,
} from "./baseSchemas.ts";
import {
+ CloudSpeechApiKey,
+ CloudSpeechError,
+ CloudSpeechKeys,
+ CloudSpeechModelId,
+ CloudSpeechModels,
+ CloudSpeechProvider,
EnvironmentSpeechAccelerationRequest,
EnvironmentSpeechModelRequest,
EnvironmentSpeechModels,
@@ -58,6 +64,7 @@ import {
EnvironmentSpeechStatus,
EnvironmentSpeechTranscriptionResult,
} from "./speech.ts";
+import { SecretReference } from "./secretReference.ts";
import {
OrchestrationV2ShellSnapshot,
OrchestrationV2ThreadBoundedSnapshot,
@@ -775,6 +782,50 @@ export class EnvironmentVoiceBodyLimit extends HttpApiMiddleware.Service<Environ
) {}
export class EnvironmentVoiceHttpApi extends HttpApiGroup.make("voice")
+ .add(
+ HttpApiEndpoint.get("cloudKeys", "/api/voice/cloud/keys", {
+ headers: OptionalBearerHeaders,
+ success: CloudSpeechKeys,
+ error: EnvironmentScopedOperationErrors,
+ }).middleware(EnvironmentAuthenticatedAuth),
+ )
+ .add(
+ HttpApiEndpoint.post("setCloudKey", "/api/voice/cloud/:provider/key", {
+ headers: OptionalBearerHeaders,
+ params: Schema.Struct({ provider: CloudSpeechProvider }),
+ payload: Schema.Struct({
+ key: Schema.NullOr(
+ Schema.Union([Schema.RedactedFromValue(CloudSpeechApiKey), SecretReference]),
+ ),
+ }),
+ success: HttpApiSchema.NoContent,
+ error: EnvironmentScopedOperationErrors,
+ }).middleware(EnvironmentAuthenticatedAuth),
+ )
+ .add(
+ // Public catalog; carries no credential and no audio.
+ HttpApiEndpoint.get("cloudModels", "/api/voice/cloud/openrouter/models", {
+ headers: OptionalBearerHeaders,
+ success: CloudSpeechModels,
+ error: [...EnvironmentScopedOperationErrors, CloudSpeechError],
+ }).middleware(EnvironmentAuthenticatedAuth),
+ )
+ .add(
+ HttpApiEndpoint.post("transcribeCloud", "/api/voice/cloud/:provider/transcribe", {
+ headers: OptionalBearerHeaders,
+ params: Schema.Struct({ provider: CloudSpeechProvider }),
+ query: Schema.Struct({ model: Schema.optionalKey(CloudSpeechModelId) }),
+ payload: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array()),
+ success: EnvironmentSpeechTranscriptionResult,
+ error: [
+ ...EnvironmentScopedOperationErrors,
+ EnvironmentRequestInvalidError,
+ CloudSpeechError,
+ ],
+ })
+ .middleware(EnvironmentVoiceBodyLimit)
+ .middleware(EnvironmentAuthenticatedAuth),
+ )
.add(
HttpApiEndpoint.get("status", "/api/voice/status", {
headers: OptionalBearerHeaders,
diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts
index 8a4b2558912..4d1aed1d2b2 100644
--- a/packages/contracts/src/settings.ts
+++ b/packages/contracts/src/settings.ts
@@ -53,6 +53,7 @@ import {
} from "./providerInstance.ts";
import { PullRequestMergeMethod } from "./pullRequest.ts";
import {
+ CloudSpeechModelId,
SpeechAcceleration,
SpeechCorrectionCue,
SpeechCustomWords,
@@ -365,11 +366,24 @@ export type ModelPreferences = {
export const VoiceShortcutMode = Schema.Literals(["auto", "hold", "toggle"]);
export type VoiceShortcutMode = typeof VoiceShortcutMode.Type;
-/** Where this device sends microphone audio. A later backend adds its own kind here. */
-export const VoiceTranscriptionDestination = Schema.Struct({
- kind: Schema.Literal("environment"),
- environmentId: EnvironmentId,
-});
+/**
+ * Where this device sends microphone audio. A cloud grant names the forwarding environment and
+ * the provider; an OpenRouter grant also names the exact model, since OpenRouter has no default.
+ */
+export const VoiceTranscriptionDestination = Schema.Union([
+ Schema.Struct({ kind: Schema.Literal("environment"), environmentId: EnvironmentId }),
+ Schema.Struct({
+ kind: Schema.Literal("cloud"),
+ environmentId: EnvironmentId,
+ provider: Schema.Literals(["openai", "xai"]),
+ }),
+ Schema.Struct({
+ kind: Schema.Literal("cloud"),
+ environmentId: EnvironmentId,
+ provider: Schema.Literal("openrouter"),
+ model: CloudSpeechModelId,
+ }),
+]);
export type VoiceTranscriptionDestination = typeof VoiceTranscriptionDestination.Type;
export const ClientSettingsSchema = Schema.Struct({
@@ -1224,6 +1238,15 @@ export const StorageCleanupSettings = Schema.Struct({
});
export type StorageCleanupSettings = typeof StorageCleanupSettings.Type;
+const CloudSpeechCredential = SettingsSecretValue.pipe(
+ Schema.withDecodingDefault(Effect.succeed("")),
+);
+export const CloudSpeechCredentials = Schema.Struct({
+ openai: CloudSpeechCredential,
+ xai: CloudSpeechCredential,
+ openrouter: CloudSpeechCredential,
+});
+
export const ServerSettings = Schema.Struct({
// Null means unmigrated; empty is authoritative and must not resurrect cleared preferences.
favorites: Schema.NullOr(ModelFavorites).pipe(Schema.withDecodingDefault(Effect.succeed(null))),
@@ -1249,6 +1272,9 @@ export const ServerSettings = Schema.Struct({
),
enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
+ cloudSpeechCredentials: CloudSpeechCredentials.pipe(
+ Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })),
+ ),
speechModelId: Schema.String.pipe(
Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")),
),
@@ -1668,6 +1694,13 @@ export const ServerSettingsPatch = Schema.Struct({
enableProviderUpdateChecks: Schema.optionalKey(Schema.Boolean),
continueThreadsAfterServerUpdate: Schema.optionalKey(Schema.Boolean),
enableAgentBrowserAccess: Schema.optionalKey(Schema.Boolean),
+ cloudSpeechCredentials: Schema.optionalKey(
+ Schema.Struct({
+ openai: Schema.optionalKey(SettingsSecretValue),
+ xai: Schema.optionalKey(SettingsSecretValue),
+ openrouter: Schema.optionalKey(SettingsSecretValue),
+ }),
+ ),
speechModelId: Schema.optionalKey(Schema.String),
speechAcceleration: Schema.optionalKey(SpeechAcceleration),
speechModelUnloadTimeout: Schema.optionalKey(SpeechModelUnloadTimeout),
diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts
index 5983b6951e0..e6e4d3ddbc5 100644
--- a/packages/contracts/src/speech.ts
+++ b/packages/contracts/src/speech.ts
@@ -1,5 +1,80 @@
import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";
+import { TrimmedNonEmptyString } from "./baseSchemas.ts";
+
+export const CloudSpeechApiKey = TrimmedNonEmptyString.check(
+ Schema.isMaxLength(4096),
+ Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i),
+);
+
+export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]);
+export type CloudSpeechProvider = typeof CloudSpeechProvider.Type;
+export const CLOUD_SPEECH_LABELS = {
+ openai: "OpenAI",
+ xai: "Grok Voice Transcribe",
+ openrouter: "OpenRouter",
+} as const;
+export const CLOUD_SPEECH_PROVIDER_NAMES = {
+ openai: "OpenAI",
+ xai: "xAI",
+ openrouter: "OpenRouter",
+} as const;
+/** Servers that predate OpenRouter report only the first two providers. */
+export const CloudSpeechKeys = Schema.Struct({
+ openai: Schema.Boolean,
+ xai: Schema.Boolean,
+ openrouter: Schema.optionalKey(Schema.Boolean),
+});
+export type CloudSpeechKeys = typeof CloudSpeechKeys.Type;
+
+/** An exact `author/model` slug. Router shortcuts such as `:nitro` or `openrouter/auto` are not models. */
+export const CloudSpeechModelId = Schema.String.check(
+ Schema.isMinLength(3),
+ Schema.isMaxLength(160),
+ Schema.isPattern(/^(?!openrouter\/)[a-z\d][a-z\d.-]*\/[a-z\d][a-z\d._-]*$/i),
+);
+export type CloudSpeechModelId = typeof CloudSpeechModelId.Type;
+export const CloudSpeechModel = Schema.Struct({ id: CloudSpeechModelId, name: Schema.String });
+export type CloudSpeechModel = typeof CloudSpeechModel.Type;
+export const CloudSpeechModels = Schema.Struct({ models: Schema.Array(CloudSpeechModel) });
+export type CloudSpeechModels = typeof CloudSpeechModels.Type;
+
+export class CloudSpeechError extends Schema.TaggedError<CloudSpeechError>()(
+ "CloudSpeechError",
+ {
+ provider: CloudSpeechProvider,
+ reason: Schema.Literals([
+ "missing-key",
+ "key-changed",
+ "unreadable-key",
+ "rejected-key",
+ "rate-limit",
+ "model-unavailable",
+ "unavailable",
+ ]),
+ },
+ { httpApiStatus: 422 },
+) {
+ override get message() {
+ const name = CLOUD_SPEECH_PROVIDER_NAMES[this.provider];
+ switch (this.reason) {
+ case "missing-key":
+ return `Add your ${name} API key in Settings → Voice.`;
+ case "key-changed":
+ return `The ${name} key changed before your recording was sent. Try again.`;
+ case "unreadable-key":
+ return `Couldn’t read the ${name} API key. Unlock the vault on the selected environment or update the key in Settings → Voice. No audio was sent to ${name}.`;
+ case "rejected-key":
+ return `${name} rejected this API key. Check its permissions and billing in Settings → Voice.`;
+ case "rate-limit":
+ return `${name} reached a usage limit. Check the account's quota or try again later.`;
+ case "model-unavailable":
+ return `The chosen model isn’t available on ${name}. Choose another model in Settings → Voice.`;
+ case "unavailable":
+ return `${name} couldn’t transcribe this recording. Try again later.`;
+ }
+ }
+}
export const SpeechModelId = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(160));
export type SpeechModelId = typeof SpeechModelId.Type;
diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts
index fcf74f10187..0fb077452bb 100644
--- a/packages/shared/src/speech.ts
+++ b/packages/shared/src/speech.ts
@@ -1,4 +1,6 @@
import {
+ CLOUD_SPEECH_PROVIDER_NAMES,
+ type CloudSpeechProvider,
SPEECH_MAX_OPTIONS_BYTES,
type SpeechCustomWords,
SpeechTranscriptionOptions,
@@ -8,6 +10,21 @@ import * as Schema from "effect/Schema";
const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions));
const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions));
+export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) {
+ return provider === "openrouter"
+ ? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.`
+ : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`;
+}
+
+// Only offer explicit hints understood by all cloud APIs; other languages can still be detected.
+export const cloudSpeechLanguages = {
+ supportsLanguageDetection: true,
+ languages:
+ "ar bg ca cs da de el en es fa fi fr hi hr hu id it ja ko mk ms nl pl pt ro ru sk sv th tr uk ur vi zh".split(
+ " ",
+ ),
+};
+
/** Resolve the language the model can produce from the requested language. */
export function effectiveSpeechLanguage(
model: { readonly languages: readonly string[]; readonly supportsLanguageDetection: boolean },