TypeScript
fix(mobile): keep the opening words of local iPhone dictation (#31801)
Julius Marminge98b930f932a8
diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.tsindex e171cf9cb50..e69998f8b15 100644--- a/apps/mobile/src/native/voiceTranscription.android.test.ts+++ b/apps/mobile/src/native/voiceTranscription.android.test.ts@@ -78,10 +78,11 @@ it.each([selection: { start: 16, end: 16 },revision: 1,};+ const prepareToRecordAsync = vi.fn(async () => {});const controller = new VoiceInputController({recorder: {uri: "file:///acceptance.m4a",- prepareToRecordAsync: async () => {},+ prepareToRecordAsync,record: () => {},stop: async () => {},},@@ -101,4 +102,5 @@ it.each([expect(commit).not.toHaveBeenCalled();expect(draft.text).toBe("Preserved draft.");expect(controller.currentState.error).toMatch(expected);+ expect(prepareToRecordAsync).toHaveBeenCalledTimes(phase === "preparation" ? 0 : 1);});diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.tsindex 4703fd6071f..e23711626ac 100644--- a/apps/mobile/src/native/voiceTranscription.ios.test.ts+++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts@@ -1,7 +1,8 @@import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription";import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";-import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";+import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";+import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller";const mocks = vi.hoisted(() => ({isAvailable: vi.fn<(locale: string) => boolean>(),@@ -65,8 +66,43 @@ async function transcribeOnce(signal = new AbortController().signal) {.catch((error: unknown) => error);}+/** Drives one on-device dictation through the controller the composer uses. */+function createDictation() {+ const recorder = {+ uri: "file:///voice.m4a",+ prepareToRecordAsync: async () => {},+ record: vi.fn(),+ stop: vi.fn(async () => {}),+ };+ const commits: string[] = [];+ const deleted: string[] = [];+ const ended = Promise.withResolvers<void>();+ const controller = new VoiceInputController({+ recorder,+ getTranscriber: () => ({+ prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options),+ }),+ requestPermission: async () => ({ granted: true, canAskAgain: true }),+ configureRecording: async () => {},+ releaseRecording: async () => {},+ deleteRecording: (uri) => deleted.push(uri),+ readDraft: () => ({+ ownerKey: "environment:thread",+ text: "",+ selection: { start: 0, end: 0 },+ revision: 1,+ }),+ commitDraft: (text) => commits.push(text),+ onStateChange: (state) => {+ if (state.phase === "idle" || state.phase === "error") ended.resolve();+ },+ });+ return { controller, recorder, commits, deleted, ended: ended.promise };+}+beforeEach(() => {vi.resetAllMocks();+ resetVoiceInputGlobalsForTests();useLanguages([], "sv-SE");mocks.isAvailable.mockReturnValue(true);mocks.prepare.mockResolvedValue("sv-SE");@@ -147,6 +183,63 @@ describe("getLocalVoiceTranscriber", () => {},);+ it("records while Apple prepares its speech model and transcribes once it is ready", async () => {+ const nativePrepare = Promise.withResolvers<string>();+ mocks.prepare.mockReturnValue(nativePrepare.promise);+ const dictation = createDictation();++ await dictation.controller.start();+ expect(dictation.recorder.record).toHaveBeenCalledOnce();+ expect(dictation.controller.currentState.phase).toBe("recording");++ const stopping = dictation.controller.stop();+ nativePrepare.resolve("sv-SE");+ await stopping;++ expect(mocks.transcribe).toHaveBeenCalledOnce();+ expect(dictation.commits).toEqual(["Hej världen."]);+ expect(dictation.deleted).toEqual(["file:///voice.m4a"]);+ });++ it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)(+ "discards early audio without a late commit after %s",+ async (ending) => {+ const nativePrepare = Promise.withResolvers<string>();+ mocks.prepare.mockReturnValue(nativePrepare.promise);+ const dictation = createDictation();+ await dictation.controller.start();++ let finishing: Promise<void> | undefined;+ const cancelled = ending === "cancel" || ending === "cancel after stop";+ if (ending.endsWith("after stop")) finishing = dictation.controller.stop();+ if (cancelled) {+ dictation.controller.cancel();+ } else {+ nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" }));+ }+ await finishing;+ await dictation.ended;++ const next = createDictation();+ mocks.prepare.mockResolvedValue("sv-SE");+ await next.controller.start();+ expect(next.controller.currentState.phase).toBe("recording");+ expect(dictation.deleted).toEqual(["file:///voice.m4a"]);+ expect(mocks.transcribe).not.toHaveBeenCalled();+ await next.controller.stop();+ expect(next.commits).toEqual(["Hej världen."]);+ nativePrepare.resolve("sv-SE");++ expect(dictation.commits).toEqual([]);+ expect(dictation.recorder.stop).toHaveBeenCalled();+ expect(dictation.controller.currentState).toMatchObject(+ cancelled+ ? { phase: "idle" }+ : { phase: "error", error: "Voice transcription could not prepare this language." },+ );+ },+ );+it("does not start native transcription after cancellation during a file read", async () => {const enteredRead = Promise.withResolvers<void>();const readResult = Promise.withResolvers<ArrayBuffer>();diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.tsindex 910a17c77fd..8f5822afb01 100644--- a/apps/mobile/src/native/voiceTranscription.ios.ts+++ b/apps/mobile/src/native/voiceTranscription.ios.ts@@ -63,6 +63,41 @@ async function prepareVoiceTranscription(});}+ let locale = locales[0]!;+ const model = prepareLocales(locales, signal).then((supported) => {+ locale = supported;+ });+ // Recorder setup can fail before either consumer observes the model.+ void model.catch(() => {});++ async function ready(signal: AbortSignal) {+ throwIfVoiceTranscriptionAborted(signal);+ const cancelled = Promise.withResolvers<void>();+ const cancel = () => cancelled.resolve();+ signal.addEventListener("abort", cancel, { once: true });+ try {+ await Promise.race([model, cancelled.promise]);+ throwIfVoiceTranscriptionAborted(signal);+ return locale;+ } finally {+ signal.removeEventListener("abort", cancel);+ }+ }+ return {+ get locale() {+ return locale;+ },+ start: async (_uri, { signal }) => {+ await ready(signal);+ return null;+ },+ transcribe: async (uri, options) =>+ transcribeVoiceRecording(uri, await ready(options.signal), options),+ };+}++async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise<string> {+ throwIfVoiceTranscriptionAborted(signal);let unsupported: VoiceTranscriptionError | undefined;for (const locale of locales) {try {@@ -75,17 +110,11 @@ async function prepareVoiceTranscription(throw unsupported!;}-async function prepareLocale(- locale: string,- signal: AbortSignal,-): Promise<PreparedVoiceTranscription> {+async function prepareLocale(locale: string, signal: AbortSignal): Promise<string> {try {const supportedLocale = await AppleTranscription.prepare(locale);throwIfVoiceTranscriptionAborted(signal);- return {- locale: supportedLocale,- transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options),- };+ return supportedLocale;} catch (error) {throwIfVoiceTranscriptionAborted(signal);if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") {diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.tsindex 265a284e20f..41e56933ce1 100644--- a/packages/client-runtime/src/voice-input/controller.test.ts+++ b/packages/client-runtime/src/voice-input/controller.test.ts@@ -309,7 +309,9 @@ describe("VoiceInputController", () => {code: "unavailable",message: "This device requires iOS 26 or later.",});+ const configureRecording = vi.fn(async () => undefined);const unavailable = createHarness({+ configureRecording,getTranscriber: () => ({prepare: async () => {throw reason;@@ -318,6 +320,8 @@ describe("VoiceInputController", () => {});await unavailable.controller.start();expect(unavailable.controller.currentState.error).toBe(reason.message);+ // Environment and web transcribers gate capture on consent checked during preparation.+ expect(configureRecording).not.toHaveBeenCalled();expect(unavailable.recorder.record).not.toHaveBeenCalled();const denied = createHarness({diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.tsindex c67287dd74d..668705546a4 100644--- a/packages/client-runtime/src/voice-input/controller.ts+++ b/packages/client-runtime/src/voice-input/controller.ts@@ -236,7 +236,7 @@ export class VoiceInputController {private operationToken = 0;private sessionToken: VoiceInputSession | null = null;private transcription: PreparedVoiceTranscription | null = null;- private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;+ private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null;private transcriptionAbortController: AbortController | null = null;private postProcessingAbortController: AbortController | null = null;private skipPostProcessingRequested = false;@@ -504,17 +504,15 @@ export class VoiceInputController {const capturedDraft = this.capturedDraft;let transcript: string;try {- if (this.liveTranscription) {- const result = await this.liveTranscription;- if ("error" in result) throw result.error;- transcript = result.text;- } else {- transcript = await runTranscriptionOperation(() =>+ const live = this.liveTranscription ? await this.liveTranscription : null;+ if (live && "error" in live) throw live.error;+ transcript =+ live?.text ??+ (await runTranscriptionOperation(() =>transcription.streaming? transcription.streaming.finish({ signal }): transcription.transcribe(recordingUri!, { signal }),- );- }+ ));} catch (error) {if (this.isCurrent(operationToken)) {if (diff --git a/packages/client-runtime/src/voice-input/transcription.ts b/packages/client-runtime/src/voice-input/transcription.tsindex 7af1196c8c0..4c674ee9c1c 100644--- a/packages/client-runtime/src/voice-input/transcription.ts+++ b/packages/client-runtime/src/voice-input/transcription.ts@@ -11,7 +11,12 @@ export type VoiceStreamingSession = {export type PreparedVoiceTranscription = {readonly locale: string;- readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;+ /**+ * Runs alongside capture once recording starts. Resolves with the transcript when it+ * transcribed live, or null when `transcribe` still runs after the recording stops.+ * A rejection stops the recording.+ */+ readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string | null>;readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;/** Present when audio streams during capture; `finish` then replaces `transcribe`. */readonly streaming?: VoiceStreamingSession;