Skip to content
TypeScript

feat(mobile): opt in to dictation on a paired environment (#31059)

Julius Marminge79ee39fdcbb8
diff --git a/apps/mobile/.swiftlint.yml b/apps/mobile/.swiftlint.yml
index 4450e0485c5..532a13e37ed 100644
--- a/apps/mobile/.swiftlint.yml
+++ b/apps/mobile/.swiftlint.yml
@@ -5,6 +5,7 @@ included:
- modules/t3-native-controls/ios
- modules/t3-terminal/ios
- modules/t3-review-diff/ios
+ - modules/t3-voice-transcription/ios
excluded:
- ios/Pods
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
index f193926f1ce..6884b4d54ed 100644
--- a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
@@ -29,6 +29,8 @@ class T3VoiceTranscriptionModule : Module() {
override fun definition() = ModuleDefinition {
Name("T3VoiceTranscription")
+ defineDecodeRecording()
+
AsyncFunction("checkAvailability").Coroutine<Boolean> {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
false
@@ -104,6 +106,21 @@ class T3VoiceTranscriptionModule : Module() {
}
}
+ private fun ModuleDefinitionBuilder.defineDecodeRecording() {
+ AsyncFunction("decodeRecording").Coroutine { uri: String ->
+ val context = requireNotNull(appContext.reactContext)
+ withContext(Dispatchers.IO) {
+ require(android.net.Uri.parse(uri).scheme == "file") { "Expected a local recording." }
+ withRecordedPcm(context, uri) { pcm ->
+ requireCompleteAudio(pcm.length() <= 16000L * 2 * 301) {
+ "Recording exceeds five minutes."
+ }
+ pcm.readBytes()
+ }
+ }
+ }
+ }
+
private fun ModuleDefinitionBuilder.defineLiveRecorder() {
Class(LiveVoiceRecorder::class) {
Constructor {
diff --git a/apps/mobile/modules/t3-voice-transcription/expo-module.config.json b/apps/mobile/modules/t3-voice-transcription/expo-module.config.json
index 5867f41ffe7..9626dea6a1a 100644
--- a/apps/mobile/modules/t3-voice-transcription/expo-module.config.json
+++ b/apps/mobile/modules/t3-voice-transcription/expo-module.config.json
@@ -1,5 +1,9 @@
{
- "platforms": ["android"],
+ "platforms": ["apple", "android"],
+ "apple": {
+ "modules": ["T3VoiceTranscriptionModule"],
+ "podspecPath": "ios/T3VoiceTranscription.podspec"
+ },
"android": {
"modules": ["expo.modules.t3voicetranscription.T3VoiceTranscriptionModule"]
}
diff --git a/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscription.podspec b/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscription.podspec
new file mode 100644
index 00000000000..946a2c70de0
--- /dev/null
+++ b/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscription.podspec
@@ -0,0 +1,13 @@
+Pod::Spec.new do |s|
+ s.name = 'T3VoiceTranscription'
+ s.version = '1.0.0'
+ s.summary = 'Local recording conversion for T3 Code dictation.'
+ s.author = 'T3 Tools'
+ s.homepage = 'https://t3tools.com'
+ s.platform = :ios, '18.0'
+ s.source = { :path => '.' }
+ s.static_framework = true
+ s.dependency 'ExpoModulesCore'
+ s.source_files = '*.swift'
+ s.frameworks = 'AudioToolbox'
+end
diff --git a/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift b/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift
new file mode 100644
index 00000000000..8c02fbd84a9
--- /dev/null
+++ b/apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift
@@ -0,0 +1,44 @@
+import AudioToolbox
+import ExpoModulesCore
+import Foundation
+
+public class T3VoiceTranscriptionModule: Module {
+ public func definition() -> ModuleDefinition {
+ Name("T3VoiceTranscription")
+
+ AsyncFunction("decodeRecording") { (uri: URL) throws -> Data in
+ guard uri.isFileURL else { throw CocoaError(.fileReadUnsupportedScheme) }
+ var opened: ExtAudioFileRef?
+ try check(ExtAudioFileOpenURL(uri as CFURL, &opened))
+ guard let file = opened else { throw CocoaError(.fileReadCorruptFile) }
+ defer { ExtAudioFileDispose(file) }
+ var format = AudioStreamBasicDescription(
+ mSampleRate: 16_000, mFormatID: kAudioFormatLinearPCM,
+ mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked,
+ mBytesPerPacket: 2, mFramesPerPacket: 1, mBytesPerFrame: 2,
+ mChannelsPerFrame: 1, mBitsPerChannel: 16, mReserved: 0)
+ let formatSize = UInt32(MemoryLayout<AudioStreamBasicDescription>.size)
+ try check(ExtAudioFileSetProperty(file, kExtAudioFileProperty_ClientDataFormat, formatSize, &format))
+ var pcm = Data()
+ var chunk = [Int16](repeating: 0, count: 8192)
+ try chunk.withUnsafeMutableBytes { bytes in
+ while true {
+ var frames = UInt32(bytes.count / 2)
+ var buffers = AudioBufferList(mNumberBuffers: 1, mBuffers: AudioBuffer(
+ mNumberChannels: 1, mDataByteSize: UInt32(bytes.count), mData: bytes.baseAddress))
+ try check(ExtAudioFileRead(file, &frames, &buffers))
+ if frames == 0 { break }
+ let count = Int(frames) * 2
+ guard pcm.count + count <= 16_000 * 2 * 301 else { throw CocoaError(.fileReadTooLarge) }
+ pcm.append(contentsOf: bytes.prefix(count))
+ }
+ }
+ guard !pcm.isEmpty else { throw CocoaError(.fileReadCorruptFile) }
+ return pcm
+ }
+ }
+}
+
+private func check(_ status: OSStatus) throws {
+ guard status == noErr else { throw NSError(domain: NSOSStatusErrorDomain, code: Int(status)) }
+}
diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx
index dfe81a7d308..ef79ef182cc 100644
--- a/apps/mobile/src/Stack.tsx
+++ b/apps/mobile/src/Stack.tsx
@@ -113,7 +113,8 @@ import {
} from "./features/settings/ScheduledTaskPickerScreens";
import { SettingsKeyboardRouteScreen } from "./features/settings/SettingsKeyboardRouteScreen";
import { SettingsLegalRouteScreen } from "./features/settings/SettingsLegalRouteScreen";
-import { SettingsMicrophoneRouteScreen } from "./features/settings/SettingsMicrophoneRouteScreen";
+import { SettingsVoiceRouteScreen } from "./features/settings/SettingsVoiceRouteScreen";
+import { SettingsVoiceModelsRouteScreen } from "./features/settings/SettingsVoiceModelsRouteScreen";
import {
SettingsOpenSourceLicenseRouteScreen,
SettingsOpenSourceLicensesRouteScreen,
@@ -265,10 +266,14 @@ const SETTINGS_SCREENS = {
linking: "settings/keyboard",
options: settingsPage("Keyboard"),
}),
- SettingsMicrophone: createNativeStackScreen({
- screen: SettingsMicrophoneRouteScreen,
- linking: "settings/microphone",
- options: settingsPage("Microphone"),
+ SettingsVoice: createNativeStackScreen({
+ screen: SettingsVoiceRouteScreen,
+ linking: { path: "settings/voice", alias: ["settings/microphone"] },
+ options: settingsPage("Voice"),
+ }),
+ SettingsVoiceModels: createNativeStackScreen({
+ screen: SettingsVoiceModelsRouteScreen,
+ options: SEARCH_SHEET_ROUTE,
}),
SettingsFollowUp: createNativeStackScreen({
screen: SettingsFollowUpRouteScreen,
diff --git a/apps/mobile/src/components/ui/AppSymbol.tsx b/apps/mobile/src/components/ui/AppSymbol.tsx
index 4b17bddd3c3..3ca8f61aaec 100644
--- a/apps/mobile/src/components/ui/AppSymbol.tsx
+++ b/apps/mobile/src/components/ui/AppSymbol.tsx
@@ -123,6 +123,7 @@ import IconTextIncrease from "@tabler/icons-react-native/IconTextIncrease";
import IconTool from "@tabler/icons-react-native/IconTool";
import IconTrash from "@tabler/icons-react-native/IconTrash";
import IconTypography from "@tabler/icons-react-native/IconTypography";
+import IconDownload from "@tabler/icons-react-native/IconDownload";
import IconUpload from "@tabler/icons-react-native/IconUpload";
import IconUserCircle from "@tabler/icons-react-native/IconUserCircle";
import IconUserCheck from "@tabler/icons-react-native/IconUserCheck";
@@ -277,6 +278,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
"textformat.size": IconTypography,
"textformat.size.larger": IconTextIncrease,
"textformat.size.smaller": IconTextDecrease,
+ "tray.and.arrow.down": IconDownload,
"tray.and.arrow.up": IconUpload,
trash: IconTrash,
wifi: IconWifi,
diff --git a/apps/mobile/src/components/ui/ListChoiceRow.tsx b/apps/mobile/src/components/ui/ListChoiceRow.tsx
index 9e231b7c25b..0b320296bfe 100644
--- a/apps/mobile/src/components/ui/ListChoiceRow.tsx
+++ b/apps/mobile/src/components/ui/ListChoiceRow.tsx
@@ -1,3 +1,4 @@
+import type { ReactNode } from "react";
import { Platform, View } from "react-native";
import { useResolveClassNames } from "uniwind";
@@ -9,7 +10,9 @@ import { RowPressable } from "./RowPressable";
export function ListChoiceRow(props: {
readonly label: string;
+ readonly leading?: ReactNode;
readonly description?: string;
+ readonly meta?: ReactNode;
readonly selected: boolean;
readonly separated?: boolean;
readonly disabled?: boolean;
@@ -31,7 +34,9 @@ export function ListChoiceRow(props: {
>
<ListRowLayout
label={props.label}
+ leading={props.leading}
description={props.description}
+ meta={props.meta}
disabled={props.disabled}
accessory={
<View
diff --git a/apps/mobile/src/components/ui/ListRowLayout.tsx b/apps/mobile/src/components/ui/ListRowLayout.tsx
index 8564e7d13ac..8e39884faf2 100644
--- a/apps/mobile/src/components/ui/ListRowLayout.tsx
+++ b/apps/mobile/src/components/ui/ListRowLayout.tsx
@@ -7,17 +7,23 @@ import { SymbolView, type AppSymbolName } from "./AppSymbol";
export interface ListRowContentProps {
readonly icon?: AppSymbolName;
+ /** Drawn in the icon column for marks a single symbol can't express. */
+ readonly leading?: ReactNode;
readonly label: string;
readonly description?: string;
readonly detail?: string;
+ /** Rich secondary content under the description, such as scores with emphasized values. */
+ readonly meta?: ReactNode;
readonly disabled?: boolean;
}
export function ListRowLayout({
icon,
+ leading,
label,
description,
detail,
+ meta,
disabled,
accessory,
tone = "default",
@@ -42,6 +48,8 @@ export function ListRowLayout({
weight="regular"
/>
</View>
+ ) : leading ? (
+ <View className="size-6 items-center justify-center">{leading}</View>
) : null}
<View className="min-w-0 flex-1 gap-1">
<AppText
@@ -55,6 +63,7 @@ export function ListRowLayout({
{description}
</AppText>
) : null}
+ {meta}
{detail && stackDetail ? (
<AppText textRole="footnote" className="text-foreground-muted">
{detail}
diff --git a/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.test.ts b/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.test.ts
index 167cebafb86..3f217fd70c8 100644
--- a/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.test.ts
+++ b/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.test.ts
@@ -101,6 +101,11 @@ describe("opening a workspace tab", () => {
"Settings",
"SettingsEnvironments",
]);
+ expect(
+ names(
+ workspaceTabNavigationState(state, "settings", { settings: "SettingsMicrophone" }, columns),
+ ),
+ ).toEqual(["Home", "Settings", "SettingsVoice"]);
});
it("keeps the route that is already open, so re-selecting a tab doesn't rebuild it", () => {
diff --git a/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.ts b/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.ts
index 900cfa90056..6a44e617e53 100644
--- a/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.ts
+++ b/apps/mobile/src/features/layout/workspace/workspace-tab-navigation.ts
@@ -32,6 +32,7 @@ const REMEMBERED_SETTINGS_PAGES: ReadonlySet<string> = new Set(
);
export function rememberedSettingsPage(selections: WorkspaceSelections): SettingsPageTarget {
+ if (selections.settings === "SettingsMicrophone") return "SettingsVoice";
return selections.settings && REMEMBERED_SETTINGS_PAGES.has(selections.settings)
? (selections.settings as SettingsPageTarget)
: DEFAULT_SETTINGS_PAGE;
diff --git a/apps/mobile/src/features/pull-requests/PullRequestSearchSheet.tsx b/apps/mobile/src/features/pull-requests/PullRequestSearchSheet.tsx
index 860de90f275..d51b88258c2 100644
--- a/apps/mobile/src/features/pull-requests/PullRequestSearchSheet.tsx
+++ b/apps/mobile/src/features/pull-requests/PullRequestSearchSheet.tsx
@@ -1,6 +1,7 @@
import { useNavigation } from "@react-navigation/native";
import type { ReactNode } from "react";
-import { View } from "react-native";
+import { Platform, View } from "react-native";
+import { useSafeAreaInsets } from "react-native-safe-area-context";
import { AppBar } from "../../components/ui/AppBar";
import { useNativeMailSearchToolbar } from "../../native/use-native-mail-search-toolbar";
@@ -20,6 +21,7 @@ export function PullRequestSearchSheet(props: {
const navigation = useNavigation();
// The compact search toolbar floats over the end of the list.
const bottomSearch = useNativeMailSearchToolbar();
+ const insets = useSafeAreaInsets();
return (
<View className="flex-1 bg-sheet" collapsable={false}>
<AppBar
@@ -37,7 +39,13 @@ export function PullRequestSearchSheet(props: {
}}
/>
{props.children({
- paddingBottom: bottomSearch ? NATIVE_MAIL_SEARCH_TOOLBAR_CONTENT_INSET + 16 : 16,
+ // Android's bottom inset also covers the floating workspace tabs while Settings is open.
+ paddingBottom:
+ Platform.OS === "android"
+ ? insets.bottom + 16
+ : bottomSearch
+ ? NATIVE_MAIL_SEARCH_TOOLBAR_CONTENT_INSET + 16
+ : 16,
})}
</View>
);
diff --git a/apps/mobile/src/features/settings/SettingsClientStorageRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsClientStorageRouteScreen.tsx
index 48dc425badb..c0f93d2ee32 100644
--- a/apps/mobile/src/features/settings/SettingsClientStorageRouteScreen.tsx
+++ b/apps/mobile/src/features/settings/SettingsClientStorageRouteScreen.tsx
@@ -21,6 +21,7 @@ import { useSavedRemoteConnections } from "../../state/use-remote-environment-re
import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow";
import { SettingsSection } from "./components/SettingsSection";
import { SettingsScreen } from "./components/SettingsScreen";
+import { formatBytes } from "../../lib/formatBytes";
export function SettingsClientStorageRouteScreen() {
const insets = useSafeAreaInsets();
@@ -179,9 +180,3 @@ function CacheEnvironmentRow(props: {
</View>
);
}
-
-function formatBytes(bytes: number): string {
- if (bytes < 1024) return `${bytes} B`;
- if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(bytes < 10 * 1024 ? 1 : 0)} KB`;
- return `${(bytes / (1024 * 1024)).toFixed(bytes < 10 * 1024 * 1024 ? 1 : 0)} MB`;
-}
diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx
index 05048e71074..a0e24692e1e 100644
--- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx
+++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx
@@ -247,7 +247,7 @@ function useSettingsIndexSections(): SettingsListSection[] {
? [page("SettingsNavigation", "line.3.horizontal", "Navigation tabs")]
: []),
...(Platform.OS === "ios" ? [page("SettingsKeyboard", "keyboard", "Keyboard")] : []),
- page("SettingsMicrophone", "mic", "Microphone"),
+ page("SettingsVoice", "mic", "Voice"),
],
},
{
diff --git a/apps/mobile/src/features/settings/SettingsVoiceModelsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceModelsRouteScreen.tsx
new file mode 100644
index 00000000000..56493a60a5f
--- /dev/null
+++ b/apps/mobile/src/features/settings/SettingsVoiceModelsRouteScreen.tsx
@@ -0,0 +1,102 @@
+import { useAtomValue } from "@effect/atom-react";
+import type { StaticScreenProps } from "@react-navigation/native";
+import type { EnvironmentId } from "@t3tools/contracts";
+import { AsyncResult } from "effect/reactivity";
+import { useState } from "react";
+import { FlatList, Platform, View } from "react-native";
+
+import { AppActivityIndicator } from "../../components/ui/AppActivityIndicator";
+import { AppText as Text } from "../../components/ui/AppText";
+import { EmptyState } from "../../components/ui/EmptyState";
+import { cn } from "../../lib/cn";
+import { useEnvironments } from "../../state/environments";
+import { voiceSpeechBenchmarksAtom } from "../voice-input/voiceDestination";
+import { PullRequestSearchSheet } from "../pull-requests/PullRequestSearchSheet";
+import { useVoiceModels, VoiceModelRow } from "./VoiceModelSection";
+
+export function SettingsVoiceModelsRouteScreen({
+ route,
+}: StaticScreenProps<{ readonly environmentId: EnvironmentId }>) {
+ const { environments } = useEnvironments();
+ const label =
+ environments.find((environment) => environment.environmentId === route.params.environmentId)
+ ?.label ?? "This environment";
+ const voice = useVoiceModels(route.params.environmentId, label);
+ const benchmarks = new Map(
+ (
+ AsyncResult.getOrElse(
+ useAtomValue(voiceSpeechBenchmarksAtom(route.params.environmentId)),
+ () => null,
+ )?.models ?? []
+ ).map((model) => [model.id, model.benchmark]),
+ );
+ const language = new Intl.DateTimeFormat().resolvedOptions().locale.split("-")[0]!.toLowerCase();
+ const [query, setQuery] = useState("");
+ const needle = query.trim().toLowerCase();
+ // Catalog order, not state order: re-sorting when a download finishes would move rows under a tap.
+ const models = voice.models
+ .map((model) => {
+ const benchmark = benchmarks.get(model.id);
+ return benchmark ? { ...model, benchmark } : model;
+ })
+ .filter((model) =>
+ `${model.name} ${model.description} ${model.languages.join(" ")}`
+ .toLowerCase()
+ .includes(needle),
+ );
+ return (
+ <PullRequestSearchSheet
+ title="Transcription models"
+ placeholder="Search models or languages"
+ closeAccessibilityLabel="Close transcription models"
+ clearAccessibilityLabel="Clear model search"
+ query={query}
+ onQueryChange={setQuery}
+ >
+ {({ paddingBottom }) => (
+ <FlatList
+ data={models}
+ keyExtractor={(model) => model.id}
+ className="flex-1"
+ contentInsetAdjustmentBehavior="automatic"
+ automaticallyAdjustKeyboardInsets={Platform.OS === "ios"}
+ keyboardDismissMode={Platform.OS === "ios" ? "interactive" : "on-drag"}
+ keyboardShouldPersistTaps="handled"
+ showsVerticalScrollIndicator={false}
+ contentContainerStyle={{ paddingHorizontal: 16, paddingTop: 16, paddingBottom }}
+ ListHeaderComponent={
+ voice.canManage ? undefined : (
+ <Text className="pb-3 text-xs text-foreground-muted">
+ This connection can’t change settings on {label}.
+ </Text>
+ )
+ }
+ renderItem={({ item, index }) => (
+ <View
+ className={cn(
+ "overflow-hidden",
+ index === 0 && "rounded-t-2xl",
+ index === models.length - 1 && "rounded-b-2xl",
+ )}
+ >
+ <VoiceModelRow model={item} voice={voice} separated={index > 0} language={language} />
+ </View>
+ )}
+ ListEmptyComponent={
+ voice.loading ? (
+ <AppActivityIndicator className="my-8" />
+ ) : (
+ <EmptyState
+ title={needle ? "No models match" : "No transcription models"}
+ detail={
+ needle ? "Try another name or language." : `${label} has no models to list.`
+ }
+ variant="plain"
+ />
+ )
+ }
+ />
+ )}
+ </PullRequestSearchSheet>
+ );
+}
diff --git a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx
similarity index 69%
rename from apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx
rename to apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx
index e911ca7b43d..57ca2cfe5dd 100644
--- a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx
+++ b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx
@@ -19,6 +19,18 @@ import {
} from "../../lib/microphonePriority";
import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
import { ListRowLayout } from "../../components/ui/ListRowLayout";
+import { ListChoiceRow } from "../../components/ui/ListChoiceRow";
+import * as Dialog from "../../components/ui/Dialog";
+import { useEnvironments, type EnvironmentPresentation } from "../../state/environments";
+import {
+ chooseVoiceDestination,
+ voiceConnectionAtom,
+ voiceGrantAtom,
+ voiceSpeechStatusAtom,
+ hasEnvironmentVoiceCapture,
+} from "../voice-input/voiceDestination";
+import { VoiceModelSection } from "./VoiceModelSection";
+import type { EnvironmentId } from "@t3tools/contracts";
import { ThreadSwipeable } from "../home/thread-swipe-actions";
import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow";
import { SettingsDragHandle } from "./components/SettingsDragHandle";
@@ -69,7 +81,7 @@ const KINDS: Record<
wired: {
label: "Wired",
row: "Other wired",
- rowDescription: "Headsets and USB microphones not listed",
+ rowDescription: "Headsets and USB microphones not listed here",
icon: "cable.connector",
},
bluetooth: {
@@ -77,8 +89,8 @@ const KINDS: Record<
row: "Other Bluetooth",
rowDescription:
Platform.OS === "android"
- ? "Earbuds, headsets, and cars, including Android Auto, not listed"
- : "AirPods, headsets, and cars not listed",
+ ? "Earbuds, headsets, and cars, including Android Auto, not listed here"
+ : "AirPods, headsets, and cars not listed here",
icon: "headphones",
},
builtIn: {
@@ -97,37 +109,71 @@ const KINDS: Record<
const UNAVAILABLE_EXPLANATION =
Platform.OS === "android"
- ? "T3 Code's voice input uses Google's on-device speech recognition, which needs Android 12 or later with compatible Google speech services for the device language. Without it, the composer has no microphone button, and allowing microphone access won't add one. Keyboard voice typing is separate and isn't affected."
- : "T3 Code's voice input uses Apple's on-device speech transcription, which needs iOS 26 or later on a device that supports it. Without it, the composer has no microphone button, and allowing microphone access won't add one. Apple keyboard dictation is separate and isn't affected.";
+ ? "Dictation on this device needs Android 12 or later and Google speech services that support your language. Your keyboard’s voice typing is separate."
+ : "Dictation on this device needs iOS 26 or later, a supported device, and a supported language. Your keyboard’s dictation is separate.";
-export function SettingsMicrophoneRouteScreen() {
- const transcriber = useAtomValue(localVoiceTranscriberAtom);
- // Android's probe is asynchronous; an empty sheet avoids flashing "unavailable" first.
- if (transcriber === "checking") return <SettingsScreen title="Microphone">{null}</SettingsScreen>;
- return transcriber === null ? <VoiceInputUnavailable /> : <MicrophoneSettings />;
+function saveDestination(id: EnvironmentId | null) {
+ try {
+ chooseVoiceDestination(id);
+ } catch {
+ Dialog.alert(
+ "Could not save transcription choice",
+ "Dictation on an environment is off for this session. Save again before closing the app, or your previous choice may return when it restarts.",
+ );
+ }
}
-function VoiceInputUnavailable() {
- const insets = useSafeAreaInsets();
+function EnvironmentChoice({
+ environment,
+ selected,
+}: {
+ environment: EnvironmentPresentation;
+ selected: boolean;
+}) {
+ const usable = useAtomValue(voiceConnectionAtom(environment.environmentId)) !== null;
+ const status = AsyncResult.getOrElse(
+ useAtomValue(voiceSpeechStatusAtom(environment.environmentId)),
+ () => null,
+ );
return (
- <SettingsScreen title="Microphone">
- <ScrollView
- contentInsetAdjustmentBehavior="automatic"
- showsVerticalScrollIndicator={false}
- className="flex-1"
- contentContainerClassName="gap-3 px-5 pt-4"
- contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }}
- >
- <SettingsSection title="Voice input">
- <ListRowLayout icon="mic" label="Not available on this device" />
- </SettingsSection>
- <Text className="px-2 text-sm text-foreground-muted">{UNAVAILABLE_EXPLANATION}</Text>
- </ScrollView>
- </SettingsScreen>
+ <ListChoiceRow
+ label={environment.label}
+ selected={selected}
+ disabled={!usable}
+ description={
+ !usable
+ ? environment.connection.phase !== "connected"
+ ? "Not connected"
+ : "Transcription unavailable"
+ : status?.supported === false
+ ? status.reason
+ : status?.state === "missing-model"
+ ? "No transcription model installed"
+ : (environment.displayUrl ?? "Send finished recordings to this environment.")
+ }
+ separated
+ onPress={() => {
+ if (!selected)
+ Dialog.confirm({
+ title: `Send microphone audio to ${environment.label}?`,
+ message: `Finished recordings will be sent to ${environment.displayUrl ?? environment.label} for transcription. You can switch back to This device at any time.`,
+ confirmText: "Use environment",
+ onConfirm: () => saveDestination(environment.environmentId),
+ });
+ }}
+ />
);
}
-function MicrophoneSettings() {
+export function SettingsVoiceRouteScreen() {
+ const transcriber = useAtomValue(localVoiceTranscriberAtom);
+ const grant = useAtomValue(voiceGrantAtom);
+ const { environments, isReady } = useEnvironments();
+ const grantEnvironment = grant
+ ? environments.find(
+ (environment) => environment.environmentId === grant.destination.environmentId,
+ )
+ : undefined;
const insets = useSafeAreaInsets();
const preferencesResult = useAtomValue(mobilePreferencesAtom);
const savePreferences = useAtomSet(updateMobilePreferencesAtom);
@@ -175,7 +221,7 @@ function MicrophoneSettings() {
const dragTo = drag === null ? -1 : dragFrom + dropSteps(dragFrom, drag.translation);
return (
- <SettingsScreen title="Microphone">
+ <SettingsScreen title="Voice">
<ScrollView
contentInsetAdjustmentBehavior="automatic"
showsVerticalScrollIndicator={false}
@@ -183,6 +229,41 @@ function MicrophoneSettings() {
contentContainerClassName="gap-3 px-5 pt-4"
contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }}
>
+ <SettingsSection title="Transcribe on">
+ <ListChoiceRow
+ label="This device"
+ description="Audio stays on this device and is transcribed here."
+ selected={!grant}
+ onPress={() => saveDestination(null)}
+ />
+ {hasEnvironmentVoiceCapture &&
+ environments.map((environment) => (
+ <EnvironmentChoice
+ key={environment.environmentId}
+ environment={environment}
+ selected={grant?.destination.environmentId === environment.environmentId}
+ />
+ ))}
+ {isReady && grant && !grantEnvironment ? (
+ <ListChoiceRow
+ label="Previously selected environment"
+ description="No longer paired"
+ separated
+ selected
+ disabled
+ onPress={() => {}}
+ />
+ ) : null}
+ </SettingsSection>
+ {grantEnvironment ? (
+ <VoiceModelSection
+ environmentId={grantEnvironment.environmentId}
+ label={grantEnvironment.label}
+ />
+ ) : null}
+ {!grant && transcriber === null ? (
+ <Text className="px-2 text-sm text-foreground-muted">{UNAVAILABLE_EXPLANATION}</Text>
+ ) : null}
{permission === "ask" || permission === "denied" ? (
<>
<SettingsSection title="Microphone access">
@@ -196,8 +277,8 @@ function MicrophoneSettings() {
</SettingsSection>
<Text className="px-2 text-sm text-foreground-muted">
{permission === "ask"
- ? "Voice input needs microphone access before it can record or list your microphones."
- : "Microphone access is off for T3 Code. Turn it on in Settings to use voice input."}
+ ? "Allow microphone access to record and to see your microphones."
+ : "Turn on microphone access for T3 Code in Settings to use voice input."}
</Text>
</>
) : null}
@@ -236,11 +317,11 @@ function MicrophoneSettings() {
})}
</SettingsSection>
<Text className="px-2 text-sm text-foreground-muted">
- Voice input prefers the first connected microphone in this list. Wired and Bluetooth
- devices are added by name after you use voice input with them connected; until then, the
- Other row ranks them. Swipe a device left to forget it.
+ Voice input uses the first connected microphone in this list. A wired or Bluetooth
+ microphone is added by name the first time you dictate with it connected; until then it
+ falls under Other. Swipe left on a named microphone to forget it.
{Platform.OS === "android"
- ? " Android may use another input when the preferred microphone cannot be routed."
+ ? " Android may still use a different microphone if the preferred one can’t be routed."
: null}
</Text>
</ScrollView>
diff --git a/apps/mobile/src/features/settings/VoiceModelSection.tsx b/apps/mobile/src/features/settings/VoiceModelSection.tsx
new file mode 100644
index 00000000000..a00674c4901
--- /dev/null
+++ b/apps/mobile/src/features/settings/VoiceModelSection.tsx
@@ -0,0 +1,319 @@
+import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
+import { useNavigation } from "@react-navigation/native";
+import {
+ cancelEnvironmentSpeechModelDownload,
+ downloadEnvironmentSpeechModel,
+ selectEnvironmentSpeechModel,
+} from "@t3tools/client-runtime/voice-input";
+import {
+ AuthSettingsWriteScope,
+ type EnvironmentId,
+ type EnvironmentSpeechModel,
+} from "@t3tools/contracts";
+import { AsyncResult } from "effect/reactivity";
+import { type ReactNode, useEffect, useState } from "react";
+import { Platform, Text as NativeText, View } from "react-native";
+
+import { SymbolView } from "../../components/ui/AppSymbol";
+import { AppText as Text } from "../../components/ui/AppText";
+import * as Dialog from "../../components/ui/Dialog";
+import { ListActionRow } from "../../components/ui/ListActionRow";
+import { ListChoiceRow } from "../../components/ui/ListChoiceRow";
+import { ListRow } from "../../components/ui/ListRow";
+import { Progress } from "../../components/ui/Progress";
+import { formatBytes } from "../../lib/formatBytes";
+import { useEnvironmentScope } from "../../state/session";
+import {
+ runSpeech,
+ voiceConnectionAtom,
+ voiceSpeechModelsAtom,
+ voiceSpeechStatusAtom,
+} from "../voice-input/voiceDestination";
+import { SettingsSection } from "./components/SettingsSection";
+
+const isDownloading = (model: EnvironmentSpeechModel) =>
+ model.state === "downloading" || model.state === "verifying";
+
+export function useVoiceModels(environmentId: EnvironmentId, label: string) {
+ const connection = useAtomValue(voiceConnectionAtom(environmentId));
+ const statusAtom = voiceSpeechStatusAtom(environmentId);
+ const modelsAtom = voiceSpeechModelsAtom(environmentId);
+ const status = AsyncResult.getOrElse(useAtomValue(statusAtom), () => null);
+ const modelsResult = useAtomValue(modelsAtom);
+ const refreshStatus = useAtomRefresh(statusAtom);
+ const refreshModels = useAtomRefresh(modelsAtom);
+ const canManage = useEnvironmentScope(environmentId, AuthSettingsWriteScope);
+ const [requested, setRequested] = useState<string | null>(null);
+ const [busy, setBusy] = useState(false);
+
+ const models = AsyncResult.getOrElse(modelsResult, () => null)?.models ?? [];
+ const downloading = requested !== null || models.some(isDownloading);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls the environment's download progress while one runs
+ useEffect(() => {
+ if (!downloading) return;
+ const timer = setInterval(refreshModels, 1_000);
+ return () => clearInterval(timer);
+ }, [downloading, refreshModels]);
+
+ const run = async (title: string, work: () => Promise<unknown>, downloadId?: string) => {
+ if (downloadId) setRequested(downloadId);
+ else setBusy(true);
+ try {
+ await work();
+ } catch (error) {
+ Dialog.alert(title, error instanceof Error ? error.message : String(error));
+ }
+ if (downloadId) setRequested(null);
+ else setBusy(false);
+ refreshModels();
+ refreshStatus();
+ };
+ const ready = connection !== null && status?.supported === true;
+ return {
+ connection,
+ status,
+ models,
+ loading: modelsResult._tag === "Initial",
+ canManage,
+ requested,
+ busy,
+ downloading,
+ label,
+ download: (model: EnvironmentSpeechModel) => {
+ if (!ready) return;
+ Dialog.confirm({
+ title: `Download ${model.name}?`,
+ message: `${label} will download ${formatBytes(model.size)} and use it as its transcription model.`,
+ confirmText: "Download",
+ onConfirm: () =>
+ void run(
+ "Could not download the model",
+ async () => {
+ await runSpeech(downloadEnvironmentSpeechModel(connection, model.id));
+ if (model.id !== status.modelId)
+ await runSpeech(selectEnvironmentSpeechModel(connection, model.id));
+ },
+ model.id,
+ ),
+ });
+ },
+ select: (model: EnvironmentSpeechModel) => {
+ if (!ready || model.active) return;
+ void run("Could not select the model", () =>
+ runSpeech(selectEnvironmentSpeechModel(connection, model.id)),
+ );
+ },
+ cancel: (model: EnvironmentSpeechModel) => {
+ if (!ready) return;
+ void run("Could not cancel the download", () =>
+ runSpeech(cancelEnvironmentSpeechModelDownload(connection, model.id)),
+ );
+ },
+ };
+}
+
+export type VoiceModels = ReturnType<typeof useVoiceModels>;
+
+const languageNames =
+ typeof Intl.DisplayNames === "function"
+ ? new Intl.DisplayNames(undefined, { type: "language" })
+ : null;
+const languageLabel = (code: string) => languageNames?.of(code) ?? code.toUpperCase();
+
+function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly language: string }) {
+ const { model, language } = props;
+ const data = model.benchmark;
+ const code = !data
+ ? undefined
+ : data.accuracy[language]
+ ? language
+ : (model.languages.find((candidate) => data.accuracy[candidate]) ?? model.languages[0]);
+ const score = code ? data?.accuracy[code] : undefined;
+ const parts: ReadonlyArray<readonly [string, ReactNode]> = [
+ ...(code && score
+ ? ([
+ [
+ "score",
+ <>
+ {languageLabel(code)}{" "}
+ <Text className="font-t3-medium text-foreground">{score.grade}</Text> (
+ {score.errorPercent}% error)
+ </>,
+ ],
+ ] as const)
+ : []),
+ ...(data && data.secondsPer30s !== null
+ ? ([
+ [
+ "speed",
+ <>
+ <Text className="font-t3-medium text-foreground">{data.secondsPer30s} s</Text> / 30 s
+ </>,
+ ],
+ ] as const)
+ : []),
+ // A single-language model's score already names its language.
+ ...(model.languages.length > 1
+ ? ([
+ [
+ "languages",
+ <>
+ <SymbolView
+ name="globe"
+ size={13}
+ tintColorClassName="accent-icon-muted"
+ type="monochrome"
+ weight="regular"
+ />{" "}
+ {model.languages.length}
+ </>,
+ ],
+ ] as const)
+ : score
+ ? []
+ : ([["languages", languageLabel(model.languages[0] ?? "")]] as const)),
+ ["size", formatBytes(model.size)],
+ ];
+ return (
+ <Text textRole="footnote" className="text-foreground-muted tabular-nums">
+ {parts.map(([key, part], index) => (
+ // Plain Text inherits the muted footnote style; AppText would reset it.
+ <NativeText key={key}>
+ {index > 0 ? " · " : null}
+ {part}
+ </NativeText>
+ ))}
+ </Text>
+ );
+}
+
+function DownloadedGlyph() {
+ return (
+ <View className="size-6">
+ <SymbolView
+ name="tray.and.arrow.down"
+ size={Platform.OS === "android" ? 24 : 22}
+ tintColorClassName="accent-icon"
+ type="monochrome"
+ weight="regular"
+ />
+ <View className="absolute -right-1.5 -bottom-1 size-3.5 items-center justify-center rounded-full bg-primary ring-2 ring-grouped-card">
+ <SymbolView
+ name="checkmark"
+ size={9}
+ tintColorClassName="accent-primary-foreground"
+ type="monochrome"
+ weight="bold"
+ />
+ </View>
+ </View>
+ );
+}
+
+export function VoiceModelRow(props: {
+ readonly model: EnvironmentSpeechModel;
+ readonly voice: VoiceModels;
+ readonly separated: boolean;
+ /** The language whose benchmark score to show, when the model has one. */
+ readonly language: string;
+}) {
+ const { model, voice, separated } = props;
+ const meta = <ModelStats model={model} language={props.language} />;
+ if (model.state === "installed")
+ return (
+ <ListChoiceRow
+ leading={<DownloadedGlyph />}
+ label={model.name}
+ description={model.description}
+ meta={meta}
+ selected={model.active}
+ separated={separated}
+ disabled={!voice.canManage || voice.busy || voice.downloading}
+ onPress={() => voice.select(model)}
+ />
+ );
+ const progress = model.downloaded === undefined ? undefined : model.downloaded / model.size;
+ if (isDownloading(model) || voice.requested === model.id)
+ return (
+ <View className={separated ? "border-t border-border-subtle" : ""}>
+ <ListActionRow
+ icon="xmark"
+ label={model.name}
+ description={
+ model.state === "verifying"
+ ? "Verifying download…"
+ : `${Math.round((progress ?? 0) * 100)}% downloaded · Tap to cancel`
+ }
+ disabled={!voice.canManage || voice.busy}
+ onPress={() => voice.cancel(model)}
+ />
+ <Progress
+ className="bg-grouped-card px-4 pb-3"
+ value={model.state === "verifying" ? undefined : progress}
+ label={`${model.name} download progress`}
+ />
+ </View>
+ );
+ return (
+ <View className={separated ? "border-t border-border-subtle" : ""}>
+ <ListActionRow
+ icon="tray.and.arrow.down"
+ label={model.name}
+ description={model.description}
+ meta={meta}
+ disabled={!voice.canManage || voice.busy || voice.downloading}
+ onPress={() => voice.download(model)}
+ />
+ </View>
+ );
+}
+
+export function VoiceModelSection(props: {
+ readonly environmentId: EnvironmentId;
+ readonly label: string;
+}) {
+ const navigation = useNavigation();
+ const voice = useVoiceModels(props.environmentId, props.label);
+ const { status } = voice;
+ if (!voice.connection || !status) return null;
+ const title = `Transcription model on ${props.label}`;
+ if (!status.supported) {
+ return (
+ <SettingsSection title={title} footer={status.reason}>
+ {null}
+ </SettingsSection>
+ );
+ }
+
+ const active = status.state === "missing-model" ? null : status;
+ const pending = voice.models.find(isDownloading);
+ const progress =
+ pending?.downloaded === undefined ? undefined : pending.downloaded / pending.size;
+ return (
+ <SettingsSection
+ title={title}
+ footer={
+ voice.canManage
+ ? undefined
+ : `This connection can’t change settings on ${props.label}. Add a transcription model using T3 Code on that machine.`
+ }
+ >
+ <ListRow
+ label={active?.model ?? "No model selected"}
+ description={
+ pending
+ ? pending.state === "verifying"
+ ? `Verifying ${pending.name}…`
+ : `Downloading ${pending.name} · ${Math.round((progress ?? 0) * 100)}%`
+ : active
+ ? formatBytes(active.size)
+ : "Choose a transcription model to dictate on this environment."
+ }
+ onPress={() =>
+ navigation.navigate("SettingsVoiceModels", { environmentId: props.environmentId })
+ }
+ />
+ </SettingsSection>
+ );
+}
diff --git a/apps/mobile/src/features/settings/components/settings-page-targets.ts b/apps/mobile/src/features/settings/components/settings-page-targets.ts
index 6a98023f2ed..eadd9c46fff 100644
--- a/apps/mobile/src/features/settings/components/settings-page-targets.ts
+++ b/apps/mobile/src/features/settings/components/settings-page-targets.ts
@@ -15,7 +15,7 @@ export const SETTINGS_PAGE_TARGETS = [
"SettingsEnvironmentMaintenance",
"SettingsProviderAccounts",
"SettingsKeyboard",
- "SettingsMicrophone",
+ "SettingsVoice",
"SettingsFollowUp",
"SettingsScheduledTasks",
"SettingsProjectGrouping",
diff --git a/apps/mobile/src/features/settings/settings-search.test.ts b/apps/mobile/src/features/settings/settings-search.test.ts
index db51f64f838..4aa686dfa43 100644
--- a/apps/mobile/src/features/settings/settings-search.test.ts
+++ b/apps/mobile/src/features/settings/settings-search.test.ts
@@ -16,6 +16,7 @@ const SECTIONS = [
rows: [
{ id: "SettingsAppearance", label: "Appearance" },
{ id: "SettingsKeyboard", label: "Keyboard" },
+ { id: "SettingsVoice", label: "Voice" },
],
},
{
@@ -47,6 +48,7 @@ describe("filterSettingsSections", () => {
{ query: "appear", expected: [["Interface", ["Appearance"]]] },
{ query: "KEYBOARD", expected: [["Interface", ["Keyboard"]]] },
{ query: "dark mode", expected: [["Interface", ["Appearance"]]] },
+ { query: "microphone", expected: [["Interface", ["Voice"]]] },
{ query: "agent credits", expected: [["Server settings", ["Source control"]]] },
{
query: "server settings",
diff --git a/apps/mobile/src/features/settings/settings-search.ts b/apps/mobile/src/features/settings/settings-search.ts
index 9dcaf7b6bf0..342b2a1ade4 100644
--- a/apps/mobile/src/features/settings/settings-search.ts
+++ b/apps/mobile/src/features/settings/settings-search.ts
@@ -29,7 +29,20 @@ const SETTINGS_SEARCH_TERMS = {
],
SettingsNavigation: ["tabs", "tab order", "reorder", "hide tabs"],
SettingsKeyboard: ["return key", "send message", "new line", "hardware keyboard"],
- SettingsMicrophone: ["voice", "dictation", "audio input", "bluetooth"],
+ SettingsVoice: [
+ "microphone",
+ "dictation",
+ "transcription",
+ "transcribe on",
+ "transcription model",
+ "this device",
+ "environment",
+ "on-device",
+ "audio input",
+ "preferred microphone",
+ "microphone order",
+ "bluetooth",
+ ],
SettingsScheduledTasks: ["automations", "schedule", "cron", "recurring", "webhook"],
SettingsProjectOverview: ["project", "overview", "checkouts", "delete project"],
SettingsOrganization: [
diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
index ace4f2594a7..6e891dd04e7 100644
--- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
+++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
@@ -1,5 +1,5 @@
import { useAtomValue } from "@effect/atom-react";
-import { RecordingPresets, type RecordingStatus } from "expo-audio";
+import { AudioModule, RecordingPresets, type RecordingStatus } from "expo-audio";
import { File } from "expo-file-system";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
import { useSyncExternalStore } from "react";
@@ -25,6 +25,7 @@ import { createLazyVoiceRecorder } from "./lazyVoiceRecorder";
import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
import { VoiceInputSession } from "./voiceInputSession";
import { createVoiceRecordingPermission } from "./voiceRecordingPermission";
+import { voiceRouteAtom } from "./voiceDestination";
import { makePermissionStatusAtom } from "../../lib/permissionStatusAtom";
import {
acquireAudioSession,
@@ -63,8 +64,9 @@ export const localVoiceTranscriberAtom = makePermissionStatusAtom<
read: () => getLocalVoiceTranscriber().catch(() => null),
}).pipe(Atom.keepAlive);
-// Each foreground re-probe yields a new transcriber; composers re-render only when availability flips.
-const localVoiceInputAvailableAtom = Atom.make((get) => {
+const voiceInputAvailableAtom = Atom.make((get) => {
+ const route = get(voiceRouteAtom);
+ if (route) return route.transcriber !== null;
const transcriber = get(localVoiceTranscriberAtom);
return transcriber !== null && transcriber !== "checking";
});
@@ -78,7 +80,6 @@ const voiceInputAtom = Atom.make((get) => {
// Mounted so the recorder can read and remember microphones synchronously.
get.mount(mobilePreferencesAtom);
get.mount(updateMobilePreferencesAtom);
- get.mount(localVoiceTranscriberAtom);
const audioLevels = makeMutable<ReadonlyArray<number>>(
Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0),
);
@@ -88,7 +89,11 @@ const voiceInputAtom = Atom.make((get) => {
void session.controller.appMovedToBackground();
});
const recorder = createLazyVoiceRecorder({
- create: () => createVoiceRecorder(VOICE_RECORDING_OPTIONS),
+ getDestination: () => (registry.get(voiceRouteAtom) === null ? "local" : "environment"),
+ create: (destination) =>
+ destination === "local"
+ ? createVoiceRecorder(VOICE_RECORDING_OPTIONS)
+ : new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
onStatus: (status: RecordingStatus) => {
session.controller.handleRecorderStatus({
isFinished: status.isFinished,
@@ -116,6 +121,8 @@ const voiceInputAtom = Atom.make((get) => {
const session: VoiceInputSession = new VoiceInputSession({
recorder,
getTranscriber: () => {
+ const route = registry.get(voiceRouteAtom);
+ if (route) return route.transcriber;
const transcriber = registry.get(localVoiceTranscriberAtom);
return transcriber === "checking" ? null : transcriber;
},
@@ -144,6 +151,7 @@ const voiceInputAtom = Atom.make((get) => {
publishAudioLevels: (levels) => audioLevels.set(levels),
keepAwake: () => keepScreenAwake(`voice-input:${++keepAwakeSequence}`),
});
+ get.subscribe(voiceRouteAtom, () => session.controller.cancel());
get.addFinalizer(() => {
permission.dispose();
session.dispose();
@@ -154,7 +162,7 @@ const voiceInputAtom = Atom.make((get) => {
export function useGlobalVoiceInput() {
const { session, audioLevels } = useAtomValue(voiceInputAtom);
- const isTranscriberAvailable = useAtomValue(localVoiceInputAvailableAtom);
+ const isTranscriberAvailable = useAtomValue(voiceInputAvailableAtom);
const snapshot = useSyncExternalStore(session.subscribe, session.getSnapshot);
return {
...snapshot,
diff --git a/apps/mobile/src/features/voice-input/environmentTranscriber.test.ts b/apps/mobile/src/features/voice-input/environmentTranscriber.test.ts
new file mode 100644
index 00000000000..fae6b03f85d
--- /dev/null
+++ b/apps/mobile/src/features/voice-input/environmentTranscriber.test.ts
@@ -0,0 +1,69 @@
+import { VOICE_RECORDING_LIMIT_SECONDS } from "@t3tools/client-runtime/voice-input";
+import { expect, it, vi } from "vite-plus/test";
+import { createEnvironmentTranscriber } from "./environmentTranscriber";
+
+it("never uploads a recording whose grant changed while the phone decoded it", async () => {
+ let grant = 1;
+ const decoded = Promise.withResolvers<Uint8Array>();
+ const upload = vi.fn(async () => "hello");
+ const transcriber = createEnvironmentTranscriber({
+ canSend: () => grant === 1,
+ prepare: async () => "en",
+ decode: () => decoded.promise,
+ upload,
+ });
+ const options = { signal: new AbortController().signal };
+ const recording = await transcriber.prepare(options);
+ const transcription = recording.transcribe("file:///draft.m4a", options);
+ grant++;
+ decoded.resolve(new Uint8Array([0, 128, 0, 64]));
+ await expect(transcription).rejects.toMatchObject({ code: "cancelled" });
+ expect(upload).not.toHaveBeenCalled();
+});
+
+it("uploads normalized mono samples only after the recording is finished", async () => {
+ const upload = vi.fn(async (bytes: Uint8Array) => {
+ expect(Array.from(new Float32Array(Uint8Array.from(bytes).buffer))).toEqual([-1, 0.5]);
+ return "hello";
+ });
+ const transcriber = createEnvironmentTranscriber({
+ canSend: () => true,
+ prepare: async () => "en",
+ decode: async () => new Uint8Array([255, 0, 128, 0, 64, 255]).subarray(1, 5),
+ upload,
+ });
+ const options = { signal: new AbortController().signal };
+ const recording = await transcriber.prepare(options);
+ expect(upload).not.toHaveBeenCalled();
+ await expect(recording.transcribe("file:///draft.m4a", options)).resolves.toBe("hello");
+});
+
+it("trims at most one second of recorder padding and rejects longer recordings", async () => {
+ const limit = 16_000 * VOICE_RECORDING_LIMIT_SECONDS;
+ const padded = new Uint8Array((limit + 16_000 + 1) * 2);
+ padded[(limit - 1) * 2 + 1] = 64;
+ let bytes = padded.subarray(0, -2);
+ const upload = vi.fn(async (body: Uint8Array) => {
+ expect(body.byteLength).toBe(limit * 4);
+ expect(
+ new DataView(body.buffer, body.byteOffset, body.byteLength).getFloat32(
+ body.byteLength - 4,
+ true,
+ ),
+ ).toBe(0.5);
+ return "hello";
+ });
+ const options = { signal: new AbortController().signal };
+ const recording = await createEnvironmentTranscriber({
+ canSend: () => true,
+ prepare: async () => "en",
+ decode: async () => bytes,
+ upload,
+ }).prepare(options);
+ await expect(recording.transcribe("file:///padded.m4a", options)).resolves.toBe("hello");
+ bytes = padded;
+ await expect(recording.transcribe("file:///too-long.m4a", options)).rejects.toThrow(
+ "This recording couldn’t be read. Try recording again.",
+ );
+ expect(upload).toHaveBeenCalledOnce();
+});
diff --git a/apps/mobile/src/features/voice-input/environmentTranscriber.ts b/apps/mobile/src/features/voice-input/environmentTranscriber.ts
new file mode 100644
index 00000000000..6f4f171afe6
--- /dev/null
+++ b/apps/mobile/src/features/voice-input/environmentTranscriber.ts
@@ -0,0 +1,62 @@
+import {
+ VoiceTranscriptionError,
+ VOICE_RECORDING_LIMIT_SECONDS,
+ throwIfVoiceTranscriptionAborted,
+ type VoiceTranscriber,
+} from "@t3tools/client-runtime/voice-input";
+
+const pcmBytesPerSecond = 16_000 * 2;
+const maxPcmBytes = pcmBytesPerSecond * VOICE_RECORDING_LIMIT_SECONDS;
+
+export function createEnvironmentTranscriber(input: {
+ readonly canSend: () => boolean;
+ readonly prepare: (signal: AbortSignal) => Promise<string>;
+ readonly decode: (uri: string) => Promise<Uint8Array>;
+ readonly upload: (pcm: Uint8Array, signal: AbortSignal) => Promise<string>;
+}): VoiceTranscriber {
+ function check(signal: AbortSignal) {
+ throwIfVoiceTranscriptionAborted(signal);
+ if (!input.canSend()) {
+ throw new VoiceTranscriptionError({
+ code: "cancelled",
+ message: "Your transcription choice changed.",
+ });
+ }
+ }
+ return {
+ async prepare({ signal }) {
+ check(signal);
+ const locale = await input.prepare(signal);
+ check(signal);
+ return {
+ locale,
+ async transcribe(uri, { signal }) {
+ check(signal);
+ const decoded = await input.decode(uri);
+ check(signal);
+ if (
+ !decoded.length ||
+ decoded.length % 2 ||
+ decoded.length > maxPcmBytes + pcmBytesPerSecond
+ ) {
+ throw new VoiceTranscriptionError({
+ code: "transcription-failed",
+ message: "This recording couldn’t be read. Try recording again.",
+ });
+ }
+ // A timed native stop can include a final encoder packet beyond the recording limit.
+ const bytes = decoded.subarray(0, maxPcmBytes);
+ const samples =
+ bytes.byteOffset % 2 === 0
+ ? new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2)
+ : new Int16Array(Uint8Array.from(bytes).buffer);
+ const pcm = new Float32Array(samples.length);
+ for (let i = 0; i < samples.length; i++) pcm[i] = samples[i]! / 32768;
+ const text = await input.upload(new Uint8Array(pcm.buffer), signal);
+ check(signal);
+ return text;
+ },
+ };
+ },
+ };
+}
diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts
index fbb6059099a..c16519c0ec7 100644
--- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts
+++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts
@@ -46,6 +46,7 @@ function createHarness(
const created: Array<ReturnType<typeof createNativeRecorder>> = [];
const statuses: Status[] = [];
const recorder = createLazyVoiceRecorder({
+ getDestination: () => "local",
create: () => {
const native = createNativeRecorder(`file:///voice-${created.length}.m4a`);
configure?.(native);
@@ -59,6 +60,32 @@ function createHarness(
}
describe("createLazyVoiceRecorder", () => {
+ it("switches between local live capture and an encoded file only for the next recording", async () => {
+ let destination: "local" | "environment" = "local";
+ const local = createNativeRecorder("file:///local.pcm");
+ const nextLocal = createNativeRecorder("file:///next-local.pcm");
+ const encoded = createNativeRecorder("file:///environment.m4a");
+ const localRecorders = [local, nextLocal];
+ const recorder = createLazyVoiceRecorder({
+ getDestination: () => destination,
+ create: (mode) => (mode === "local" ? localRecorders.shift()! : encoded),
+ onStatus: () => {},
+ });
+ await recorder.prepareToRecordAsync();
+ destination = "environment";
+ expect(recorder.uri).toBe(local.prepared);
+ await recorder.stop();
+ await recorder.prepareToRecordAsync();
+ expect(recorder.uri).toBe(encoded.prepared);
+ expect(local.release).toHaveBeenCalledOnce();
+ destination = "local";
+ await recorder.stop();
+ await recorder.prepareToRecordAsync();
+ expect(recorder.uri).toBe(nextLocal.prepared);
+ expect(encoded.release).toHaveBeenCalledOnce();
+ recorder.release();
+ });
+
it("keeps the system input when none is preferred or the preferred one disconnects", async () => {
const unpreferred = createHarness(() => null);
await unpreferred.recorder.prepareToRecordAsync();
diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts
index 6100a869fa4..511b051fa56 100644
--- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts
+++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts
@@ -4,6 +4,7 @@ import {
} from "@t3tools/client-runtime/voice-input";
type Subscription = { remove(): void };
+type RecordingDestination = "local" | "environment";
export interface VoiceRecordingInput {
readonly uid: string;
@@ -34,11 +35,15 @@ export interface LazyVoiceRecorder<TRecorderState> extends VoiceRecorder {
/** Construct lazily: cold startup blocks JS for tens of milliseconds and most composers never record. */
export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
- readonly create: () => NativeVoiceRecorder<TStatus, TRecorderState>;
+ readonly getDestination: () => RecordingDestination;
+ readonly create: (
+ destination: RecordingDestination,
+ ) => NativeVoiceRecorder<TStatus, TRecorderState>;
readonly onStatus: (status: TStatus) => void;
readonly selectInput?: (inputs: ReadonlyArray<VoiceRecordingInput>) => VoiceRecordingInput | null;
}): LazyVoiceRecorder<TRecorderState> {
let recorder: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
+ let destination: RecordingDestination | null = null;
let subscription: Subscription | null = null;
let retired: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
let retiredUri: string | null = null;
@@ -50,14 +55,34 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
return promise;
};
+ const release = () => {
+ const released = recorder;
+ if (!released) return;
+ abandonInputSelection?.();
+ recorder = null;
+ subscription?.remove();
+ subscription = null;
+ retired = released;
+ void pending.then(() => {
+ if (retired === released) {
+ retiredUri = released.uri;
+ retired = null;
+ }
+ released.release();
+ });
+ };
+
return {
get uri() {
if (recorder) return recorder.uri;
return retired ? retired.uri : retiredUri;
},
prepareToRecordAsync(signal) {
+ const next = input.getDestination();
+ if (next !== destination) release();
if (!recorder) {
- recorder = input.create();
+ recorder = input.create(next);
+ destination = next;
subscription = recorder.addListener("recordingStatusUpdate", input.onStatus);
}
const prepared = recorder;
@@ -95,7 +120,7 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
);
},
record(options) {
- if (!recorder) throw new Error("Voice recorder is not prepared.");
+ if (!recorder) throw new Error("The voice recorder isn’t ready. Try again.");
recorder.record(options);
},
stop() {
@@ -104,21 +129,6 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
getStatus() {
return recorder ? recorder.getStatus() : null;
},
- release() {
- const released = recorder;
- if (!released) return;
- abandonInputSelection?.();
- recorder = null;
- subscription?.remove();
- subscription = null;
- retired = released;
- void pending.then(() => {
- if (retired === released) {
- retiredUri = released.uri;
- retired = null;
- }
- released.release();
- });
- },
+ release,
};
}
diff --git a/apps/mobile/src/features/voice-input/voiceDestination.test.ts b/apps/mobile/src/features/voice-input/voiceDestination.test.ts
new file mode 100644
index 00000000000..8d2f5acae08
--- /dev/null
+++ b/apps/mobile/src/features/voice-input/voiceDestination.test.ts
@@ -0,0 +1,174 @@
+import { EnvironmentId } from "@t3tools/contracts";
+import { Atom } from "effect/reactivity";
+import { expect, it, vi } from "vite-plus/test";
+import { appAtomRegistry } from "../../state/atom-registry";
+import { chooseVoiceDestination, voiceGrantAtom, voiceRouteAtom } from "./voiceDestination";
+import { readVoiceGrant } from "./voiceGrantStorage";
+
+const mocks = vi.hoisted(() => ({
+ read: vi.fn<() => Promise<string | null>>(async () => null),
+ write: vi.fn(),
+ markerRead: vi.fn<() => string>(),
+ markerWrite: vi.fn<(id: string) => void>(),
+ probe: vi.fn(),
+ fetch: vi.fn<typeof fetch>(),
+}));
+vi.mock("expo", () => ({
+ requireOptionalNativeModule: () => ({ decodeRecording: async () => new Uint8Array([0, 0]) }),
+}));
+vi.mock("expo/fetch", () => ({ fetch: mocks.fetch }));
+vi.mock("expo-file-system", () => ({
+ Paths: { document: "file:///app/" },
+ File: class {
+ textSync = mocks.markerRead;
+ writeSync = mocks.markerWrite;
+ write(id: string) {
+ return Promise.resolve().then(() => mocks.markerWrite(id));
+ }
+ },
+}));
+vi.mock("expo-secure-store", () => ({
+ getItemAsync: mocks.read,
+ setItem: mocks.write,
+ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 6,
+}));
+vi.mock("../../lib/uuid", () => {
+ let nextGrant = 0;
+ return { uuidv4: () => `test-grant-${++nextGrant}` };
+});
+vi.mock("../../lib/protectedData", () => ({ whenProtectedDataAvailable: async () => {} }));
+vi.mock("../../connection/runtime", async () => ({
+ connectionAtomRuntime: Atom.runtime((await import("effect/http")).FetchHttpClient.layer),
+}));
+vi.mock("../../state/presentation", () => ({
+ environmentPresentations: {
+ presentationAtom: Atom.family((id: string) => {
+ mocks.probe(id);
+ return Atom.make({
+ connection: { phase: id === "offline" ? "offline" : "connected" },
+ serverConfig: { environment: { capabilities: { voiceTranscription: true } } },
+ });
+ }),
+ },
+}));
+vi.mock("../../state/session", async () => {
+ const Option = await import("effect/Option");
+ return {
+ environmentSession: {
+ preparedConnectionValueAtom: Atom.family((environmentId: string) =>
+ Atom.make(
+ Option.some({
+ environmentId,
+ httpBaseUrl: `https://${environmentId}.test`,
+ httpAuthorization: { _tag: "Bearer", token: "fixture" },
+ }),
+ ),
+ ),
+ sessionStateAtom: Atom.family((_id: string) => Atom.make(true)),
+ },
+ sessionHasScope: () => true,
+ };
+});
+
+it.each(["secure store", "marker"])(
+ "revokes across restarts when writing the %s fails",
+ async (failedStore) => {
+ let saved: string | null = null;
+ let marker = "";
+ mocks.read.mockImplementation(async () => saved);
+ mocks.write.mockImplementation((_key: string, value: string) => {
+ saved = value;
+ });
+ mocks.markerRead.mockImplementation(() => marker);
+ mocks.markerWrite.mockImplementation((id) => {
+ marker = id;
+ });
+ const host = EnvironmentId.make("my-server");
+ try {
+ chooseVoiceDestination(host);
+ expect(marker).toBe(appAtomRegistry.get(voiceGrantAtom)?.id);
+ expect((await readVoiceGrant())?.destination.environmentId).toBe(host);
+ mocks.markerRead.mockReturnValueOnce("");
+ expect(await readVoiceGrant()).toBeNull();
+ const old = { saved, marker };
+ const failedWrite = failedStore === "secure store" ? mocks.write : mocks.markerWrite;
+ failedWrite.mockImplementationOnce(() => {
+ throw new Error("storage unavailable");
+ });
+ chooseVoiceDestination(null);
+ expect(failedStore === "secure store" ? saved : marker).toBe(
+ failedStore === "secure store" ? old.saved : old.marker,
+ );
+ expect(await readVoiceGrant()).toBeNull();
+ chooseVoiceDestination(host);
+ failedWrite.mockImplementationOnce(() => {
+ throw new Error("storage unavailable");
+ });
+ expect(() => chooseVoiceDestination(EnvironmentId.make("other-host"))).toThrow();
+ expect(appAtomRegistry.get(voiceGrantAtom)).toBeNull();
+ expect(await readVoiceGrant()).toBeNull();
+ } finally {
+ mocks.read.mockReset();
+ mocks.write.mockReset();
+ mocks.markerRead.mockReset();
+ mocks.markerWrite.mockReset();
+ appAtomRegistry.reset();
+ }
+ },
+);
+
+it("keeps local as the default and never reuses consent for an offline or reselected host", async () => {
+ const unmount = appAtomRegistry.mount(voiceRouteAtom);
+ try {
+ expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
+ expect(mocks.probe).not.toHaveBeenCalled();
+ chooseVoiceDestination(EnvironmentId.make("offline"));
+ expect(appAtomRegistry.get(voiceRouteAtom)).toEqual({ transcriber: null });
+ const host = EnvironmentId.make("my-server");
+ chooseVoiceDestination(host);
+ const first = appAtomRegistry.get(voiceRouteAtom)?.transcriber;
+ expect(first).toBeTruthy();
+ chooseVoiceDestination(null);
+ expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
+ chooseVoiceDestination(host);
+ const current = appAtomRegistry.get(voiceRouteAtom)?.transcriber;
+ await expect(first!.prepare({ signal: new AbortController().signal })).rejects.toThrow(
+ "Your transcription choice changed.",
+ );
+ expect(mocks.fetch).not.toHaveBeenCalled();
+ mocks.fetch
+ .mockResolvedValueOnce(
+ Response.json({
+ supported: true,
+ state: "ready",
+ modelId: "fixture",
+ model: "Fixture",
+ size: 1,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ acceleration: "cpu",
+ modelUnloadTimeout: "never",
+ gpuDevices: [],
+ }),
+ )
+ .mockResolvedValueOnce(Response.json({ text: "hello" }));
+ const options = { signal: new AbortController().signal };
+ const recording = await current!.prepare(options);
+ await expect(recording.transcribe("file:///voice.m4a", options)).resolves.toBe("hello");
+ expect(mocks.fetch.mock.calls.map(([url, init]) => [String(url), init?.redirect])).toEqual([
+ ["https://my-server.test/api/voice/status", "error"],
+ ["https://my-server.test/api/voice/transcribe", "error"],
+ ]);
+ const failWrite = () => {
+ throw new Error("Device storage is unavailable");
+ };
+ mocks.markerWrite.mockImplementationOnce(failWrite);
+ mocks.write.mockImplementationOnce(failWrite);
+ expect(() => chooseVoiceDestination(null)).toThrow("Device storage is unavailable");
+ expect(appAtomRegistry.get(voiceGrantAtom)).toBeNull();
+ await expect(current!.prepare(options)).rejects.toThrow("Your transcription choice changed.");
+ } finally {
+ unmount();
+ appAtomRegistry.reset();
+ }
+});
diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts
new file mode 100644
index 00000000000..6f30769c2d3
--- /dev/null
+++ b/apps/mobile/src/features/voice-input/voiceDestination.ts
@@ -0,0 +1,181 @@
+import {
+ AuthOrchestrationOperateScope,
+ type EnvironmentId,
+ DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+} from "@t3tools/contracts";
+import {
+ getEnvironmentSpeechModels,
+ getEnvironmentSpeechStatus,
+ transcribeEnvironmentPcm,
+ VoiceTranscriptionError,
+} from "@t3tools/client-runtime/voice-input";
+import { FetchHttpClient, type HttpClient } from "effect/http";
+import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import { Atom, AsyncResult, AtomRegistry } from "effect/reactivity";
+import { requireOptionalNativeModule } from "expo";
+import { fetch as expoFetch } from "expo/fetch";
+
+import { appAtomRegistry } from "../../state/atom-registry";
+import { environmentPresentations } from "../../state/presentation";
+import { environmentSession, sessionHasScope } from "../../state/session";
+import { connectionAtomRuntime } from "../../connection/runtime";
+import { uuidv4 } from "../../lib/uuid";
+import { createEnvironmentTranscriber } from "./environmentTranscriber";
+import { readVoiceGrant, writeVoiceGrant, type VoiceGrant } from "./voiceGrantStorage";
+
+const storedGrant = Atom.make(Effect.promise(readVoiceGrant)).pipe(Atom.keepAlive);
+const choice = Atom.make<VoiceGrant | null | undefined>(undefined).pipe(Atom.keepAlive);
+export const voiceGrantAtom = Atom.make((get) => {
+ const chosen = get(choice);
+ if (chosen !== undefined) return chosen;
+ return AsyncResult.getOrElse(get(storedGrant), () => null);
+});
+
+export function chooseVoiceDestination(environmentId: EnvironmentId | null) {
+ // A failed change must not leave the old destination armed in this session.
+ appAtomRegistry.set(choice, null);
+ const grant: VoiceGrant | null =
+ environmentId === null
+ ? null
+ : {
+ id: uuidv4(),
+ destination: { kind: "environment", environmentId },
+ };
+ writeVoiceGrant(grant);
+ appAtomRegistry.set(choice, grant);
+}
+
+const native = requireOptionalNativeModule<{
+ decodeRecording?: (uri: string) => Promise<Uint8Array>;
+}>("T3VoiceTranscription");
+export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function";
+
+export const voiceConnectionAtom = Atom.family((id: EnvironmentId) =>
+ Atom.make((get) => {
+ const environment = get(environmentPresentations.presentationAtom(id));
+ if (
+ !hasEnvironmentVoiceCapture ||
+ environment?.connection.phase !== "connected" ||
+ !environment.serverConfig?.environment.capabilities.voiceTranscription ||
+ !sessionHasScope(get(environmentSession.sessionStateAtom(id)), AuthOrchestrationOperateScope)
+ )
+ return null;
+ return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
+ }),
+);
+
+// React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error".
+export function runSpeech<A, E>(
+ effect: Effect.Effect<A, E, HttpClient.HttpClient>,
+ signal?: AbortSignal,
+ canSend: () => boolean = () => true,
+) {
+ return Effect.runPromise(
+ AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
+ Effect.flatMap((context) =>
+ effect.pipe(
+ Effect.provideService(FetchHttpClient.Fetch, async (url, options) => {
+ if (signal?.aborted || !canSend())
+ throw new VoiceTranscriptionError({
+ code: "cancelled",
+ message: "Your transcription choice changed.",
+ });
+ return expoFetch(url, { ...options, redirect: "error" });
+ }),
+ Effect.provide(context),
+ ),
+ ),
+ ),
+ { signal },
+ );
+}
+function environmentTranscriber(prepared: PreparedConnection, canSend: () => boolean) {
+ return createEnvironmentTranscriber({
+ canSend,
+ prepare: async (signal) => {
+ const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend);
+ if (!status.supported)
+ throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
+ if (status.state === "missing-model")
+ throw new VoiceTranscriptionError({
+ code: "unavailable",
+ message:
+ "Install a transcription model for this environment in Settings → Voice, then try again.",
+ });
+ return status.languages
+ ? effectiveSpeechLanguage(
+ {
+ languages: status.languages,
+ supportsLanguageDetection: status.supportsLanguageDetection ?? false,
+ },
+ "auto",
+ )
+ : "auto";
+ },
+ decode: (uri) => {
+ if (!native?.decodeRecording)
+ throw new VoiceTranscriptionError({
+ code: "unavailable",
+ message: "Update T3 Code to transcribe on an environment.",
+ });
+ return native.decodeRecording(uri);
+ },
+ upload: async (pcm, signal) =>
+ (
+ await runSpeech(
+ transcribeEnvironmentPcm(prepared, pcm, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, canSend),
+ signal,
+ canSend,
+ )
+ ).text,
+ });
+}
+
+const speechAtom = <A, E>(
+ id: EnvironmentId,
+ read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
+) =>
+ Atom.make((get) => {
+ const prepared = get(voiceConnectionAtom(id));
+ return prepared
+ ? Effect.tryPromise({
+ try: (signal) => runSpeech(read(prepared), signal),
+ catch: (cause) =>
+ new VoiceTranscriptionError({
+ code: "unavailable",
+ message: cause instanceof Error ? cause.message : String(cause),
+ }),
+ })
+ : Effect.succeed(null);
+ });
+
+export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) =>
+ speechAtom(id, getEnvironmentSpeechStatus),
+);
+export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) =>
+ speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)),
+);
+// Fetched once for the model sheet; download progress polls the lighter list above.
+export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) =>
+ speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared, true)),
+);
+
+// null selects on-device; a grant with no usable transcriber must never fall back.
+export const voiceRouteAtom = Atom.make((get) => {
+ const grant = get(voiceGrantAtom);
+ if (!grant) return null;
+ const connectionAtom = voiceConnectionAtom(grant.destination.environmentId);
+ const connection = get(connectionAtom);
+ const registry = get.registry;
+ return {
+ transcriber:
+ connection &&
+ environmentTranscriber(
+ connection,
+ () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection,
+ ),
+ };
+});
diff --git a/apps/mobile/src/features/voice-input/voiceGrantStorage.ts b/apps/mobile/src/features/voice-input/voiceGrantStorage.ts
new file mode 100644
index 00000000000..a3580d8f7aa
--- /dev/null
+++ b/apps/mobile/src/features/voice-input/voiceGrantStorage.ts
@@ -0,0 +1,42 @@
+import { VoiceTranscriptionDestination } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+import { File, Paths } from "expo-file-system";
+import * as SecureStore from "expo-secure-store";
+import { whenProtectedDataAvailable } from "../../lib/protectedData";
+
+const key = "t3code.voice.audio-grant.v1";
+const deviceOnly = { keychainAccessible: SecureStore.WHEN_UNLOCKED_THIS_DEVICE_ONLY };
+// Both stores must name the same grant; clearing either one revokes it on the next launch.
+const markerFile = () => new File(Paths.document, "voice-audio-grant-id");
+const Grant = Schema.Struct({
+ id: Schema.NonEmptyString,
+ destination: VoiceTranscriptionDestination,
+});
+export type VoiceGrant = typeof Grant.Type;
+const decodeGrant = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.NullOr(Grant)));
+
+export async function readVoiceGrant() {
+ await whenProtectedDataAvailable();
+ try {
+ const raw = await SecureStore.getItemAsync(key, deviceOnly);
+ const grant = raw ? decodeGrant(raw) : null;
+ return grant && markerFile().textSync() === grant.id ? grant : null;
+ } catch {
+ return null;
+ }
+}
+
+export function writeVoiceGrant(grant: VoiceGrant | null) {
+ try {
+ markerFile().writeSync(grant?.id ?? "");
+ } catch (cause) {
+ SecureStore.setItem(key, "null", deviceOnly);
+ if (grant) throw cause;
+ return;
+ }
+ try {
+ SecureStore.setItem(key, JSON.stringify(grant), deviceOnly);
+ } catch (cause) {
+ if (grant) throw cause;
+ }
+}
diff --git a/apps/mobile/src/lib/formatBytes.ts b/apps/mobile/src/lib/formatBytes.ts
new file mode 100644
index 00000000000..709e73f8fb8
--- /dev/null
+++ b/apps/mobile/src/lib/formatBytes.ts
@@ -0,0 +1,7 @@
+export function formatBytes(bytes: number): string {
+ if (bytes < 1024) return `${bytes} B`;
+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(bytes < 10 * 1024 ? 1 : 0)} KB`;
+ if (bytes < 1024 ** 3)
+ return `${(bytes / (1024 * 1024)).toFixed(bytes < 10 * 1024 * 1024 ? 1 : 0)} MB`;
+ return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
+}
diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md
index 84cfeaea373..fb84458a03a 100644
--- a/docs/internals/voice-input.md
+++ b/docs/internals/voice-input.md
@@ -2,8 +2,8 @@
Transcription edits a composer draft. It does not submit an agent turn. Audio is
temporary client input, and only normal message submission sends the resulting
-text. Mobile transcribes on the device; web and desktop send audio to one
-environment the person chose.
+text. Mobile defaults to on-device transcription; any client that sends audio off
+the device needs a grant for one environment.
The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
owns the operation while the client supplies capture and transcription. Preparation
@@ -15,8 +15,11 @@ capture and `finish` replaces the file-based `transcribe`.
## Where audio may go
-Only an explicit [audio grant](../../apps/web/src/speech/voiceConsent.ts) authorizes
-off-device transcription. It lives outside the preferences snapshot so another
+Only an explicit audio grant authorizes off-device transcription. On mobile the
+grant is stored in device-bound secure storage, outside preference backups. The
+recorder is the same for both destinations; only a finished recording can be
+uploaded, and its grant is checked again after native decoding and before dispatch.
+The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another
window saving unrelated settings cannot resurrect a revoked grant. Each new
choice gets a distinct grant, even if the destination was previously selected. The
[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a
diff --git a/docs/user/composer.md b/docs/user/composer.md
index 6ab88afb3bd..84929ef861b 100644
--- a/docs/user/composer.md
+++ b/docs/user/composer.md
@@ -192,26 +192,28 @@ sent until you choose one. See [Voice input](./voice-input.md).
## Voice input on mobile
-On supported iPhones with iOS 26 or later and Android devices with Android 12 or
-later, use the composer's microphone to record, then confirm to transcribe. Text is
+Use the composer's microphone to record, then confirm to transcribe. Text is
inserted where your selection was when recording started, ready for you to review
and edit before sending.
-Voice input uses on-device speech transcription: Apple's on iOS, and compatible
-Google speech services on Android. When it is unavailable for the device or its
-language, the composer has no microphone button and **Settings → Microphone** says
-voice input isn't available. On Android, keep T3 Code open while recording and
-transcribing.
-
-The first use may download a speech model and needs a network connection.
-Later transcription works offline for that language. Android processes audio while
-you record, then finishes the transcript after you stop. Recordings can be up to five
+By default, audio stays on the phone. Local transcription needs a supported iPhone
+with iOS 26 or later, or Android 12 or later with compatible Google speech services.
+To use a paired environment instead, open **Settings → Voice → Transcribe on**,
+choose the environment, and confirm. Finished recordings go only to that environment,
+even when a draft belongs to another one. This also works on phones without local
+speech recognition. If the chosen environment disconnects, dictation stops; it never
+switches hosts. Choose **This device** to revoke access and discard pending audio.
+On Android, keep T3 Code open while recording and transcribing.
+
+On-device dictation may need a network connection to download a speech model the
+first time. After that it works offline for that language. Android transcribes on
+the device while you record and finishes after you stop. Recordings can be up to five
minutes long. Canceling or an audio interruption discards the recording and
preserves your existing draft. Navigating to another screen keeps dictation active;
use the floating controls to finish or cancel. While recording, the screen stays
awake; it can sleep normally once recording stops.
-Choose a preferred order in **Settings → Microphone**. Voice input requests the first
+Choose a preferred order in **Settings → Voice**. Voice input requests the first
connected microphone in the list. The list mixes kinds (built-in, other wired, other
Bluetooth, and CarPlay on iOS) with wired and Bluetooth devices you have dictated
with, listed by name. Drag to rank them together:
@@ -221,8 +223,8 @@ microphone on iOS. Swipe a device left to forget it. Android may use another inp
when the preferred microphone cannot be routed. Devices with the same name may share
a saved preference, but simultaneous same-name inputs remain separately selectable.
-Transcription runs on your device. T3 Code deletes the temporary audio after
-transcription or cancellation; only the message text is sent when you submit.
+T3 Code deletes the temporary recording from your phone after transcription or
+cancellation. The transcript stays in your draft until you send the message to the agent.
## Queued messages
diff --git a/docs/user/faq.md b/docs/user/faq.md
index 6f4a02e485b..accde144843 100644
--- a/docs/user/faq.md
+++ b/docs/user/faq.md
@@ -77,7 +77,7 @@ Open **Settings > Storage** and turn on automatic worktree cleanup. You can also
## Is there voice input?
-On iPhone, yes. You need iOS 26 or later. Tap the mic in the composer, talk, check the text, then send. Web, desktop, and Android do not have it yet. [Voice input on iPhone](./composer.md#voice-input-on-iphone)
+Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first. The transcript goes into your draft for review. See [Voice input](./voice-input.md).
## Does it work on Windows, Mac, and Linux?
diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md
index 0f43c1022e5..e204b286606 100644
--- a/docs/user/voice-input.md
+++ b/docs/user/voice-input.md
@@ -6,7 +6,8 @@ also works in citation comments, diff and file annotations, pull request
comments, replies, and review summaries, and in the desktop app's browser
annotation editor.
-On iPhone and Android, voice input transcribes on the device. See
+On iPhone and Android, voice input transcribes on the device by default. To opt in
+to a named environment on that phone, open **Settings → Voice → Transcribe on**. See
[Voice input on mobile](./composer.md#voice-input-on-mobile).
## Choose where audio goes