Skip to content
TypeScript

fix(mobile): keep the opening words of local iPhone dictation (#31801)

Julius Marminge98b930f932a8
diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.ts
index e171cf9cb50..e69998f8b15 100644
--- a/apps/mobile/src/native/voiceTranscription.android.test.ts
+++ b/apps/mobile/src/native/voiceTranscription.android.test.ts
@@ -78,10 +78,11 @@ it.each([
selection: { start: 16, end: 16 },
revision: 1,
};
+ const prepareToRecordAsync = vi.fn(async () => {});
const controller = new VoiceInputController({
recorder: {
uri: "file:///acceptance.m4a",
- prepareToRecordAsync: async () => {},
+ prepareToRecordAsync,
record: () => {},
stop: async () => {},
},
@@ -101,4 +102,5 @@ it.each([
expect(commit).not.toHaveBeenCalled();
expect(draft.text).toBe("Preserved draft.");
expect(controller.currentState.error).toMatch(expected);
+ expect(prepareToRecordAsync).toHaveBeenCalledTimes(phase === "preparation" ? 0 : 1);
});
diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts
index 4703fd6071f..e23711626ac 100644
--- a/apps/mobile/src/native/voiceTranscription.ios.test.ts
+++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts
@@ -1,7 +1,8 @@
import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription";
import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
-import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
+import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
+import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller";
const mocks = vi.hoisted(() => ({
isAvailable: vi.fn<(locale: string) => boolean>(),
@@ -65,8 +66,43 @@ async function transcribeOnce(signal = new AbortController().signal) {
.catch((error: unknown) => error);
}
+/** Drives one on-device dictation through the controller the composer uses. */
+function createDictation() {
+ const recorder = {
+ uri: "file:///voice.m4a",
+ prepareToRecordAsync: async () => {},
+ record: vi.fn(),
+ stop: vi.fn(async () => {}),
+ };
+ const commits: string[] = [];
+ const deleted: string[] = [];
+ const ended = Promise.withResolvers<void>();
+ const controller = new VoiceInputController({
+ recorder,
+ getTranscriber: () => ({
+ prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options),
+ }),
+ requestPermission: async () => ({ granted: true, canAskAgain: true }),
+ configureRecording: async () => {},
+ releaseRecording: async () => {},
+ deleteRecording: (uri) => deleted.push(uri),
+ readDraft: () => ({
+ ownerKey: "environment:thread",
+ text: "",
+ selection: { start: 0, end: 0 },
+ revision: 1,
+ }),
+ commitDraft: (text) => commits.push(text),
+ onStateChange: (state) => {
+ if (state.phase === "idle" || state.phase === "error") ended.resolve();
+ },
+ });
+ return { controller, recorder, commits, deleted, ended: ended.promise };
+}
+
beforeEach(() => {
vi.resetAllMocks();
+ resetVoiceInputGlobalsForTests();
useLanguages([], "sv-SE");
mocks.isAvailable.mockReturnValue(true);
mocks.prepare.mockResolvedValue("sv-SE");
@@ -147,6 +183,63 @@ describe("getLocalVoiceTranscriber", () => {
},
);
+ it("records while Apple prepares its speech model and transcribes once it is ready", async () => {
+ const nativePrepare = Promise.withResolvers<string>();
+ mocks.prepare.mockReturnValue(nativePrepare.promise);
+ const dictation = createDictation();
+
+ await dictation.controller.start();
+ expect(dictation.recorder.record).toHaveBeenCalledOnce();
+ expect(dictation.controller.currentState.phase).toBe("recording");
+
+ const stopping = dictation.controller.stop();
+ nativePrepare.resolve("sv-SE");
+ await stopping;
+
+ expect(mocks.transcribe).toHaveBeenCalledOnce();
+ expect(dictation.commits).toEqual(["Hej världen."]);
+ expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
+ });
+
+ it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)(
+ "discards early audio without a late commit after %s",
+ async (ending) => {
+ const nativePrepare = Promise.withResolvers<string>();
+ mocks.prepare.mockReturnValue(nativePrepare.promise);
+ const dictation = createDictation();
+ await dictation.controller.start();
+
+ let finishing: Promise<void> | undefined;
+ const cancelled = ending === "cancel" || ending === "cancel after stop";
+ if (ending.endsWith("after stop")) finishing = dictation.controller.stop();
+ if (cancelled) {
+ dictation.controller.cancel();
+ } else {
+ nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" }));
+ }
+ await finishing;
+ await dictation.ended;
+
+ const next = createDictation();
+ mocks.prepare.mockResolvedValue("sv-SE");
+ await next.controller.start();
+ expect(next.controller.currentState.phase).toBe("recording");
+ expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
+ expect(mocks.transcribe).not.toHaveBeenCalled();
+ await next.controller.stop();
+ expect(next.commits).toEqual(["Hej världen."]);
+ nativePrepare.resolve("sv-SE");
+
+ expect(dictation.commits).toEqual([]);
+ expect(dictation.recorder.stop).toHaveBeenCalled();
+ expect(dictation.controller.currentState).toMatchObject(
+ cancelled
+ ? { phase: "idle" }
+ : { phase: "error", error: "Voice transcription could not prepare this language." },
+ );
+ },
+ );
+
it("does not start native transcription after cancellation during a file read", async () => {
const enteredRead = Promise.withResolvers<void>();
const readResult = Promise.withResolvers<ArrayBuffer>();
diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts
index 910a17c77fd..8f5822afb01 100644
--- a/apps/mobile/src/native/voiceTranscription.ios.ts
+++ b/apps/mobile/src/native/voiceTranscription.ios.ts
@@ -63,6 +63,41 @@ async function prepareVoiceTranscription(
});
}
+ let locale = locales[0]!;
+ const model = prepareLocales(locales, signal).then((supported) => {
+ locale = supported;
+ });
+ // Recorder setup can fail before either consumer observes the model.
+ void model.catch(() => {});
+
+ async function ready(signal: AbortSignal) {
+ throwIfVoiceTranscriptionAborted(signal);
+ const cancelled = Promise.withResolvers<void>();
+ const cancel = () => cancelled.resolve();
+ signal.addEventListener("abort", cancel, { once: true });
+ try {
+ await Promise.race([model, cancelled.promise]);
+ throwIfVoiceTranscriptionAborted(signal);
+ return locale;
+ } finally {
+ signal.removeEventListener("abort", cancel);
+ }
+ }
+ return {
+ get locale() {
+ return locale;
+ },
+ start: async (_uri, { signal }) => {
+ await ready(signal);
+ return null;
+ },
+ transcribe: async (uri, options) =>
+ transcribeVoiceRecording(uri, await ready(options.signal), options),
+ };
+}
+
+async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise<string> {
+ throwIfVoiceTranscriptionAborted(signal);
let unsupported: VoiceTranscriptionError | undefined;
for (const locale of locales) {
try {
@@ -75,17 +110,11 @@ async function prepareVoiceTranscription(
throw unsupported!;
}
-async function prepareLocale(
- locale: string,
- signal: AbortSignal,
-): Promise<PreparedVoiceTranscription> {
+async function prepareLocale(locale: string, signal: AbortSignal): Promise<string> {
try {
const supportedLocale = await AppleTranscription.prepare(locale);
throwIfVoiceTranscriptionAborted(signal);
- return {
- locale: supportedLocale,
- transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options),
- };
+ return supportedLocale;
} catch (error) {
throwIfVoiceTranscriptionAborted(signal);
if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") {
diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts
index 265a284e20f..41e56933ce1 100644
--- a/packages/client-runtime/src/voice-input/controller.test.ts
+++ b/packages/client-runtime/src/voice-input/controller.test.ts
@@ -309,7 +309,9 @@ describe("VoiceInputController", () => {
code: "unavailable",
message: "This device requires iOS 26 or later.",
});
+ const configureRecording = vi.fn(async () => undefined);
const unavailable = createHarness({
+ configureRecording,
getTranscriber: () => ({
prepare: async () => {
throw reason;
@@ -318,6 +320,8 @@ describe("VoiceInputController", () => {
});
await unavailable.controller.start();
expect(unavailable.controller.currentState.error).toBe(reason.message);
+ // Environment and web transcribers gate capture on consent checked during preparation.
+ expect(configureRecording).not.toHaveBeenCalled();
expect(unavailable.recorder.record).not.toHaveBeenCalled();
const denied = createHarness({
diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts
index c67287dd74d..668705546a4 100644
--- a/packages/client-runtime/src/voice-input/controller.ts
+++ b/packages/client-runtime/src/voice-input/controller.ts
@@ -236,7 +236,7 @@ export class VoiceInputController {
private operationToken = 0;
private sessionToken: VoiceInputSession | null = null;
private transcription: PreparedVoiceTranscription | null = null;
- private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;
+ private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null;
private transcriptionAbortController: AbortController | null = null;
private postProcessingAbortController: AbortController | null = null;
private skipPostProcessingRequested = false;
@@ -504,17 +504,15 @@ export class VoiceInputController {
const capturedDraft = this.capturedDraft;
let transcript: string;
try {
- if (this.liveTranscription) {
- const result = await this.liveTranscription;
- if ("error" in result) throw result.error;
- transcript = result.text;
- } else {
- transcript = await runTranscriptionOperation(() =>
+ const live = this.liveTranscription ? await this.liveTranscription : null;
+ if (live && "error" in live) throw live.error;
+ transcript =
+ live?.text ??
+ (await runTranscriptionOperation(() =>
transcription.streaming
? transcription.streaming.finish({ signal })
: transcription.transcribe(recordingUri!, { signal }),
- );
- }
+ ));
} catch (error) {
if (this.isCurrent(operationToken)) {
if (
diff --git a/packages/client-runtime/src/voice-input/transcription.ts b/packages/client-runtime/src/voice-input/transcription.ts
index 7af1196c8c0..4c674ee9c1c 100644
--- a/packages/client-runtime/src/voice-input/transcription.ts
+++ b/packages/client-runtime/src/voice-input/transcription.ts
@@ -11,7 +11,12 @@ export type VoiceStreamingSession = {
export type PreparedVoiceTranscription = {
readonly locale: string;
- readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
+ /**
+ * Runs alongside capture once recording starts. Resolves with the transcript when it
+ * transcribed live, or null when `transcribe` still runs after the recording stops.
+ * A rejection stops the recording.
+ */
+ readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string | null>;
readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
/** Present when audio streams during capture; `finish` then replaces `transcribe`. */
readonly streaming?: VoiceStreamingSession;