TypeScript
fix(mobile): use preferred languages for on-device iPhone dictation (#31800)
Julius Marminge2785e55fe855
diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.tsindex d83c371bc2a..4703fd6071f 100644--- a/apps/mobile/src/native/voiceTranscription.ios.test.ts+++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts@@ -8,6 +8,12 @@ const mocks = vi.hoisted(() => ({prepare: vi.fn<(locale: string) => Promise<string>>(),transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise<TranscriptionResult>>(),readAudio: vi.fn<() => Promise<ArrayBuffer>>(),+ preferredLanguages: [] as unknown,+}));++// iOS exposes the user's ordered Language & Region list through NSUserDefaults.+vi.mock("react-native", () => ({+ Settings: { get: (key: string) => (key === "AppleLanguages" ? mocks.preferredLanguages : null) },}));vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({@@ -35,16 +41,33 @@ const nativeTranscript: TranscriptionResult = {],};-function deferred<T>() {- let resolve!: (value: T) => void;- const promise = new Promise<T>((next) => {- resolve = next;+const unsupportedLocale = () =>+ Object.assign(new Error("unsupported"), { code: "AppleTranscriptionUnsupportedLocale" });++function useLanguages(preferred: ReadonlyArray<string>, appLocale: string) {+ mocks.preferredLanguages = preferred;+ const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();+ vi.spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions").mockReturnValue({+ ...resolvedOptions,+ locale: appLocale,});- return { promise, resolve };+}++/** Prepares and transcribes one recording, resolving with the error when either step fails. */+async function transcribeOnce(signal = new AbortController().signal) {+ const transcriber = (await getLocalVoiceTranscriber())!;+ return transcriber+ .prepare({ signal })+ .then(async (prepared) => {+ const text = await prepared.transcribe("file:///voice.m4a", { signal });+ return { text, locale: prepared.locale };+ })+ .catch((error: unknown) => error);}beforeEach(() => {vi.resetAllMocks();+ useLanguages([], "sv-SE");mocks.isAvailable.mockReturnValue(true);mocks.prepare.mockResolvedValue("sv-SE");mocks.readAudio.mockResolvedValue(audio);@@ -56,9 +79,77 @@ afterEach(() => {});describe("getLocalVoiceTranscriber", () => {+ it.each([+ {+ case: "a spoken language ahead of the English app locale",+ preferred: ["ko-KR", "en-US"],+ supported: ["ko-KR", "en-US"],+ tried: ["ko-KR"],+ },+ {+ case: "the next preferred language after an unsupported one",+ preferred: ["gsw-CH", "de-CH", "en-US"],+ supported: ["de-CH", "en-US"],+ tried: ["gsw-CH", "de-CH"],+ },+ {+ case: "the app locale when no preferred language is supported",+ preferred: ["gsw-CH", "yi"],+ supported: ["en-US"],+ tried: ["gsw-CH", "yi", "en-US"],+ },+ ])("prepares $case", async ({ preferred, supported, tried }) => {+ useLanguages(preferred, "en-US");+ mocks.prepare.mockImplementation(async (candidate) => {+ if (!supported.includes(candidate)) throw unsupportedLocale();+ return candidate;+ });++ const locale = tried.at(-1)!;+ expect(await transcribeOnce()).toEqual({ text: "Hej världen.", locale });+ expect(mocks.prepare.mock.calls.map(([candidate]) => candidate)).toEqual(tried);+ expect(mocks.transcribe).toHaveBeenCalledWith(audio, locale);+ });++ it("reports an unsupported language when neither preferred nor app locales are supported", async () => {+ useLanguages(["gsw-CH"], "yi");+ mocks.prepare.mockRejectedValue(unsupportedLocale());++ expect(await transcribeOnce()).toMatchObject({ code: "unsupported-locale" });+ expect(mocks.prepare.mock.calls.map(([candidate]) => candidate)).toEqual(["gsw-CH", "yi"]);+ });++ it.each(["cancellation", "a model failure"] as const)(+ "does not fall back to another language after %s",+ async (failure) => {+ useLanguages(["ko-KR", "ja-JP"], "en-US");+ const entered = Promise.withResolvers<void>();+ const nativePrepare = Promise.withResolvers<string>();+ mocks.prepare.mockImplementation(() => {+ entered.resolve();+ return nativePrepare.promise;+ });+ const controller = new AbortController();+ const result = transcribeOnce(controller.signal);++ await entered.promise;+ if (failure === "cancellation") {+ controller.abort();+ nativePrepare.reject(unsupportedLocale());+ } else {+ nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" }));+ }++ expect(await result).toMatchObject({+ code: failure === "cancellation" ? "cancelled" : "preparation-failed",+ });+ expect(mocks.prepare.mock.calls.map(([candidate]) => candidate)).toEqual(["ko-KR"]);+ },+ );+it("does not start native transcription after cancellation during a file read", async () => {- const enteredRead = deferred<void>();- const readResult = deferred<ArrayBuffer>();+ const enteredRead = Promise.withResolvers<void>();+ const readResult = Promise.withResolvers<ArrayBuffer>();mocks.readAudio.mockImplementation(() => {enteredRead.resolve();return readResult.promise;diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.tsindex 5fc6c9a0b8a..910a17c77fd 100644--- a/apps/mobile/src/native/voiceTranscription.ios.ts+++ b/apps/mobile/src/native/voiceTranscription.ios.ts@@ -1,6 +1,7 @@import * as Schema from "effect/Schema";import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription";import { File } from "expo-file-system";+import { Settings } from "react-native";import {VoiceTranscriptionError,@@ -12,8 +13,15 @@ import {const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);-function getDeviceLocale(): string {- return Intl.DateTimeFormat().resolvedOptions().locale;+/** The user's ordered Language & Region list, then the app locale; the app ships in English only. */+function getPreferredLocales(): string[] {+ const languages: unknown = Settings.get("AppleLanguages");+ const preferred = Array.isArray(languages)+ ? languages.filter(+ (language): language is string => typeof language === "string" && language.length > 0,+ )+ : [];+ return [...new Set([...preferred, Intl.DateTimeFormat().resolvedOptions().locale])];}function wrapError(@@ -37,23 +45,40 @@ function getNativeErrorCode(error: unknown): string | undefined {}export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | null> {- const locale = getDeviceLocale();- if (!AppleTranscription.isAvailable(locale)) return null;- return { prepare: (options) => prepareVoiceTranscription(locale, options) };+ const locales = getPreferredLocales();+ // Availability is a device and OS check; which languages have a model only `prepare` knows.+ if (!AppleTranscription.isAvailable(locales[0]!)) return null;+ return { prepare: (options) => prepareVoiceTranscription(locales, options) };}async function prepareVoiceTranscription(- locale: string,+ locales: readonly string[],{ signal }: VoiceTranscriptionOptions,): Promise<PreparedVoiceTranscription> {throwIfVoiceTranscriptionAborted(signal);- if (!AppleTranscription.isAvailable(locale)) {+ if (!AppleTranscription.isAvailable(locales[0]!)) {throw new VoiceTranscriptionError({code: "unavailable",message: "Voice transcription requires a supported device with iOS 26 or later.",});}+ let unsupported: VoiceTranscriptionError | undefined;+ for (const locale of locales) {+ try {+ return await prepareLocale(locale, signal);+ } catch (error) {+ if (!isVoiceTranscriptionError(error) || error.code !== "unsupported-locale") throw error;+ unsupported = error;+ }+ }+ throw unsupported!;+}++async function prepareLocale(+ locale: string,+ signal: AbortSignal,+): Promise<PreparedVoiceTranscription> {try {const supportedLocale = await AppleTranscription.prepare(locale);throwIfVoiceTranscriptionAborted(signal);@@ -66,7 +91,7 @@ async function prepareVoiceTranscription(if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") {throw new VoiceTranscriptionError({code: "unsupported-locale",- message: "Voice transcription does not support this device language.",+ message: "Voice transcription does not support any of your languages.",cause: error,});}diff --git a/docs/user/composer.md b/docs/user/composer.mdindex 84929ef861b..13f451e2caa 100644--- a/docs/user/composer.md+++ b/docs/user/composer.md@@ -206,8 +206,11 @@ switches hosts. Choose **This device** to revoke access and discard pending audiOn Android, keep T3 Code open while recording and transcribing.On-device dictation may need a network connection to download a speech model the-first time. After that it works offline for that language. Android transcribes on-the device while you record and finishes after you stop. Recordings can be up to five+first time. After that it works offline for that language. iPhone dictates in the+first of your preferred languages (**Settings → General → Language & Region**)+that Apple’s speech model supports. If none is supported, it tries the app’s+language. Android transcribes on the device while you+record and finishes after you stop. Recordings can be up to fiveminutes long. Canceling or an audio interruption discards the recording andpreserves your existing draft. Navigating to another screen keeps dictation active;use the floating controls to finish or cancel. While recording, the screen stays