Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(mobile): stop On-Device cleanup at its timeout and don't retry it (#2690)
Chad Piha2c84aaf612ea
diff --git a/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swift b/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swiftindex 612c6776c..93f1c8bda 100644--- a/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swift+++ b/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swift@@ -15,6 +15,32 @@ private struct AppleLLMGenerateRequest: Record {@Field var prompt: String = ""@Field var temperature: Double?@Field var maxTokens: Int?+ @Field var requestId: String?+}++// Generations JavaScript can stop by id. A timed-out cleanup must end the model+// run too, or it keeps the on-device model busy after JavaScript gave up on it.+private final class AppleLLMGenerations: @unchecked Sendable {+ private let lock = NSLock()+ private var tasks: [String: Task<Void, Never>] = [:]++ func start(_ requestId: String?, operation: @escaping @Sendable () async -> Void) {+ guard let requestId else {+ Task { await operation() }+ return+ }+ // Registered under the lock, so the task's own removal can't run before it.+ lock.withLock {+ tasks[requestId] = Task {+ await operation()+ self.lock.withLock { _ = self.tasks.removeValue(forKey: requestId) }+ }+ }+ }++ func cancel(_ requestId: String) {+ lock.withLock { tasks[requestId] }?.cancel()+ }}#if canImport(FoundationModels)@@ -37,6 +63,8 @@ private struct AppleGeneratedMeetingNotes {#endifpublic class AppleLLMModule: Module {+ private static let generations = AppleLLMGenerations()+public func definition() -> ModuleDefinition {Name("AppleLLM")@@ -83,7 +111,7 @@ public class AppleLLMModule: Module {return}- Task {+ Self.generations.start(request.requestId) {do {try Self.ensureAvailable()let session = LanguageModelSession(instructions: request.instructions)@@ -104,6 +132,10 @@ public class AppleLLMModule: Module {#endif}+ Function("cancelGeneration") { (requestId: String) in+ Self.generations.cancel(requestId)+ }+AsyncFunction("generateMeetingNotes") { (request: AppleLLMGenerateRequest, promise: Promise) in#if canImport(FoundationModels)guard #available(iOS 26.0, *) else {@@ -211,6 +243,11 @@ public class AppleLLMModule: Module {return}+ if error is CancellationError {+ promise.reject("APPLE_LLM_CANCELLED", "The Apple Foundation Models request was cancelled.")+ return+ }+if let generationError = error as? LanguageModelSession.GenerationError {switch generationError {case .guardrailViolation(_), .refusal(_, _):diff --git a/openwhispr-mobile/modules/apple-llm/src/__tests__/index.test.ts b/openwhispr-mobile/modules/apple-llm/src/__tests__/index.test.tsnew file mode 100644index 000000000..10203fb87--- /dev/null+++ b/openwhispr-mobile/modules/apple-llm/src/__tests__/index.test.ts@@ -0,0 +1,129 @@+const mockNative: {+ generateText: jest.Mock;+ cancelGeneration?: jest.Mock;+} = {+ generateText: jest.fn(),+ cancelGeneration: jest.fn(),+};++jest.mock('expo', () => ({+ requireNativeModule: jest.fn(() => mockNative),+}));+jest.mock('react-native', () => ({ Platform: { OS: 'ios' } }));++// Required after the mocks above: a hoisted import would read mockNative before it exists.+const { AppleLLM } = jest.requireActual<typeof import('../index')>('../index');++const request = { instructions: 'Clean this up.', prompt: 'um hello' };+const response = { text: 'Hello.', model: 'apple-foundation-models' as const };++function pendingNative(): { reject: (error: Error) => void } {+ let reject: (error: Error) => void = () => undefined;+ mockNative.generateText.mockReturnValue(+ new Promise((_resolve, rejectNative) => {+ reject = rejectNative;+ }),+ );+ return { reject: (error) => reject(error) };+}++beforeEach(() => {+ jest.clearAllMocks();+ mockNative.cancelGeneration = jest.fn();+});++it('sends the request unchanged when there is no signal', async () => {+ mockNative.generateText.mockResolvedValue(response);+ await expect(AppleLLM.generateText(request)).resolves.toEqual(response);+ expect(mockNative.generateText).toHaveBeenCalledWith(request);+});++it('cancels the native run by its id and rejects with an AbortError on abort', async () => {+ const native = pendingNative();+ const controller = new AbortController();+ const result = AppleLLM.generateText(request, { signal: controller.signal });++ controller.abort();++ await expect(result).rejects.toMatchObject({ name: 'AbortError' });+ const { requestId } = mockNative.generateText.mock.calls[0][0] as { requestId: string };+ expect(requestId).toEqual(expect.any(String));+ expect(mockNative.cancelGeneration).toHaveBeenCalledWith(requestId);+ // The native cancellation arriving afterwards is swallowed, not reported unhandled.+ native.reject(Object.assign(new Error('cancelled'), { code: 'APPLE_LLM_CANCELLED' }));+ await Promise.resolve();+});++it('gives each request its own id', async () => {+ mockNative.generateText.mockResolvedValue(response);+ const { signal } = new AbortController();+ await AppleLLM.generateText(request, { signal });+ await AppleLLM.generateText(request, { signal });+ const [first, second] = mockNative.generateText.mock.calls.map(+ ([sent]) => (sent as { requestId: string }).requestId,+ );+ expect(first).not.toBe(second);+});++// A JS reload restarts the counter, but native keeps the runs started before it.+it('keeps ids unique across a JS reload', async () => {+ mockNative.generateText.mockResolvedValue(response);+ const { signal } = new AbortController();+ const now = jest.spyOn(Date, 'now');+ for (const time of [1_000, 2_000]) {+ now.mockReturnValue(time);+ await jest.isolateModulesAsync(async () => {+ const reloaded = jest.requireActual<typeof import('../index')>('../index');+ await reloaded.AppleLLM.generateText(request, { signal });+ });+ }+ now.mockRestore();+ const [beforeReload, afterReload] = mockNative.generateText.mock.calls.map(+ ([sent]) => (sent as { requestId: string }).requestId,+ );+ expect(beforeReload).not.toBe(afterReload);+});++it('rejects an already-aborted signal without starting a native run', async () => {+ const controller = new AbortController();+ controller.abort();++ await expect(AppleLLM.generateText(request, { signal: controller.signal })).rejects.toMatchObject(+ { name: 'AbortError' },+ );+ expect(mockNative.generateText).not.toHaveBeenCalled();+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();+});++it('stops listening for abort once the native run settles', async () => {+ mockNative.generateText.mockResolvedValue(response);+ const controller = new AbortController();+ const removeEventListener = jest.spyOn(controller.signal, 'removeEventListener');++ await expect(AppleLLM.generateText(request, { signal: controller.signal })).resolves.toEqual(+ response,+ );+ await Promise.resolve();+ expect(removeEventListener).toHaveBeenCalledWith('abort', expect.any(Function));++ controller.abort();+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();+});++it('passes a native failure through', async () => {+ const failure = Object.assign(new Error('blocked'), { code: 'APPLE_LLM_GUARDRAIL' });+ mockNative.generateText.mockRejectedValue(failure);+ const { signal } = new AbortController();+ await expect(AppleLLM.generateText(request, { signal })).rejects.toBe(failure);+});++it('still rejects on abort when the binary predates cancelGeneration', async () => {+ pendingNative();+ delete mockNative.cancelGeneration;+ const controller = new AbortController();+ const result = AppleLLM.generateText(request, { signal: controller.signal });++ controller.abort();++ await expect(result).rejects.toMatchObject({ name: 'AbortError' });+});diff --git a/openwhispr-mobile/modules/apple-llm/src/index.ts b/openwhispr-mobile/modules/apple-llm/src/index.tsindex 9cb5a159d..ce9c92fa9 100644--- a/openwhispr-mobile/modules/apple-llm/src/index.ts+++ b/openwhispr-mobile/modules/apple-llm/src/index.ts@@ -11,7 +11,11 @@ import type {interface NativeAppleLLM {getAvailability(): Promise<AppleLLMAvailability>;countTokens(request: AppleLLMTokenCountRequest): Promise<number | null>;- generateText(request: AppleLLMGenerateTextRequest): Promise<AppleLLMGenerateTextResponse>;+ generateText(+ request: AppleLLMGenerateTextRequest & { requestId?: string },+ ): Promise<AppleLLMGenerateTextResponse>;+ // Missing on binaries built before it shipped; an OTA update can run this code there.+ cancelGeneration?(requestId: string): void;generateMeetingNotes(request: AppleLLMGenerateTextRequest,): Promise<AppleLLMGenerateMeetingNotesResponse>;@@ -28,6 +32,16 @@ const NativeModule: NativeAppleLLM | null = (() => {}})();+let requestSequence = 0;++// Hermes has no DOMException global, so cancellations use a plain Error that+// callers recognise by name, as they would a fetch AbortError.+function abortError(): Error {+ const error = new Error('Apple Foundation Models request cancelled.');+ error.name = 'AbortError';+ return error;+}+export const AppleLLM = {isAvailable(): boolean {return NativeModule !== null;@@ -47,11 +61,34 @@ export const AppleLLM = {return NativeModule ? NativeModule.countTokens(request) : null;},- async generateText(request: AppleLLMGenerateTextRequest): Promise<AppleLLMGenerateTextResponse> {+ async generateText(+ request: AppleLLMGenerateTextRequest,+ options: { signal?: AbortSignal } = {},+ ): Promise<AppleLLMGenerateTextResponse> {if (!NativeModule) {throw new Error('Apple Foundation Models are not linked in this build.');}- return NativeModule.generateText(request);+ const { signal } = options;+ if (!signal) return NativeModule.generateText(request);+ if (signal.aborted) throw abortError();++ requestSequence += 1;+ // The time keeps an id unique across JS reloads, which restart the counter but not native.+ const requestId = `apple-llm-${Date.now()}-${requestSequence}`;+ // Settle on abort without waiting for native: an older binary can't stop the+ // run, and a newer one reports the cancellation only once the model yields.+ return new Promise<AppleLLMGenerateTextResponse>((resolve, reject) => {+ const onAbort = (): void => {+ NativeModule.cancelGeneration?.(requestId);+ reject(abortError());+ };+ signal.addEventListener('abort', onAbort, { once: true });+ NativeModule.generateText({ ...request, requestId })+ .then(resolve, reject)+ .finally(() => {+ signal.removeEventListener('abort', onAbort);+ });+ });},async generateMeetingNotes(diff --git a/openwhispr-mobile/src/lib/__tests__/cleanupTranscript.onDevice.test.ts b/openwhispr-mobile/src/lib/__tests__/cleanupTranscript.onDevice.test.tsnew file mode 100644index 000000000..8c59207be--- /dev/null+++ b/openwhispr-mobile/src/lib/__tests__/cleanupTranscript.onDevice.test.ts@@ -0,0 +1,153 @@+/**+ * On-Device cleanup runs through the real retry policy, reasoning routing and+ * Apple Intelligence wrapper; only the native module is faked. A slow or failed+ * run must hand back the raw transcript after one attempt, since a retry would+ * meet the same refusal or time out again.+ */+import { CLEANUP_TIMEOUT_MS, cleanupTranscript } from '@/lib/cleanupTranscript';++interface MockNativeAppleLLM {+ getAvailability: jest.Mock;+ countTokens: jest.Mock;+ generateText: jest.Mock;+ cancelGeneration: jest.Mock;+}++jest.mock('expo', () => {+ const native: MockNativeAppleLLM = {+ getAvailability: jest.fn(),+ countTokens: jest.fn(),+ generateText: jest.fn(),+ cancelGeneration: jest.fn(),+ };+ return { requireNativeModule: (): MockNativeAppleLLM => native, mockNative: native };+});+// The Cloud client imports expo/fetch, which needs the real expo package; an+// On-Device cleanup never reaches it.+jest.mock('@/lib/apiClient', () => ({ api: { post: jest.fn() } }));+// accountAccess pulls in the router for its sign-in alert; stub it here.+jest.mock('expo-router', () => ({ router: { push: jest.fn() } }));+jest.mock('@/store/useSnippetsStore', () => ({+ useSnippetsStore: { getState: () => ({ isLoaded: true, entries: [] }) },+}));+jest.mock('@/store/useDictionaryStore', () => ({+ useDictionaryStore: { getState: () => ({ isLoaded: true, entries: [] }) },+}));+jest.mock('@/store/useCustomPromptsStore', () => ({+ getActiveCustomCleanupPrompt: () => undefined,+}));+jest.mock('@/store/useConfigStore', () => ({+ useConfigStore: {+ getState: () => ({ config: { cleanupEnabled: true, defaultMode: 'private' } }),+ },+}));+jest.mock('@/store/useProcessingModeStore', () => ({+ useProcessingModeStore: { getState: () => ({ activeMode: 'private' }) },+}));+jest.mock('@/store/useAuthStore', () => ({+ useAuthStore: { getState: () => ({ user: { id: 'u', isAnonymous: false } }) },+}));++const { mockNative } = jest.requireMock<{ mockNative: MockNativeAppleLLM }>('expo');+const onDeviceRoute = { mode: 'local', scope: 'cleanup' } as const;++beforeEach(() => {+ jest.clearAllMocks();+ jest.useFakeTimers();+ mockNative.getAvailability.mockResolvedValue({+ status: 'available',+ contextSize: 4096,+ tokenCounting: true,+ });+ mockNative.countTokens.mockResolvedValue(10);+ jest.spyOn(console, 'warn').mockImplementation(() => undefined);+});++afterEach(() => {+ jest.useRealTimers();+ jest.restoreAllMocks();+});++it('pastes the raw transcript at the timeout and stops the model run', async () => {+ mockNative.generateText.mockReturnValue(new Promise(() => undefined));+ const onSkipped = jest.fn();+ const result = cleanupTranscript('um hello there', { inferenceRoute: onDeviceRoute, onSkipped });++ await jest.advanceTimersByTimeAsync(CLEANUP_TIMEOUT_MS);+ // A retry would start a second run before the next timeout; checked before awaiting the+ // result, which a retry would hold back until the timers move again.+ await jest.advanceTimersByTimeAsync(CLEANUP_TIMEOUT_MS);+ expect(mockNative.generateText).toHaveBeenCalledTimes(1);++ await expect(result).resolves.toBe('um hello there');+ expect(onSkipped).toHaveBeenCalledWith(+ 'Apple Intelligence took too long. Your raw transcript is saved.',+ );+ const { requestId } = mockNative.generateText.mock.calls[0][0] as { requestId: string };+ expect(mockNative.cancelGeneration).toHaveBeenCalledWith(requestId);+});++// Apple Intelligence writes about as many tokens as the transcript has, so a long one gets+// longer than Cloud's 12 s.+describe('a long transcript', () => {+ beforeEach(() => {+ mockNative.countTokens.mockResolvedValue(600);+ });++ it('is cleaned when the model needs longer than 12 s', async () => {+ mockNative.generateText.mockReturnValue(+ new Promise((resolve) =>+ setTimeout(() => resolve({ text: 'Long text.', model: 'apple-foundation-models' }), 20_000),+ ),+ );+ const result = cleanupTranscript('um long text', { inferenceRoute: onDeviceRoute });++ await jest.advanceTimersByTimeAsync(20_000);++ await expect(result).resolves.toBe('Long text.');+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();+ });++ it('still stops a run that never finishes', async () => {+ mockNative.generateText.mockReturnValue(new Promise(() => undefined));+ const onSkipped = jest.fn();+ const result = cleanupTranscript('um long text', { inferenceRoute: onDeviceRoute, onSkipped });++ // 5 s to start, plus 600 tokens at 25 a second.+ await jest.advanceTimersByTimeAsync(28_999);+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();+ await jest.advanceTimersByTimeAsync(1);++ await expect(result).resolves.toBe('um long text');+ expect(mockNative.cancelGeneration).toHaveBeenCalledTimes(1);+ expect(onSkipped).toHaveBeenCalledWith(+ 'Apple Intelligence took too long. Your raw transcript is saved.',+ );+ });+});++it.each([+ ['APPLE_LLM_RATE_LIMITED', 'Apple Intelligence was busy.'],+ ['APPLE_LLM_GUARDRAIL', 'Cleanup failed.'],+ ['APPLE_LLM_FAILED', 'Cleanup failed.'],+])('pastes the raw transcript after one %s attempt', async (code, reason) => {+ mockNative.generateText.mockRejectedValue(Object.assign(new Error('refused'), { code }));+ const onSkipped = jest.fn();+ const result = cleanupTranscript('um hello there', { inferenceRoute: onDeviceRoute, onSkipped });++ await jest.advanceTimersByTimeAsync(CLEANUP_TIMEOUT_MS);+ expect(mockNative.generateText).toHaveBeenCalledTimes(1);++ await expect(result).resolves.toBe('um hello there');+ expect(onSkipped).toHaveBeenCalledWith(`${reason} Your raw transcript is saved.`);+});++it('returns the cleaned text when the model answers in time', async () => {+ mockNative.generateText.mockResolvedValue({+ text: 'Hello there.',+ model: 'apple-foundation-models',+ });+ await expect(+ cleanupTranscript('um hello there', { inferenceRoute: onDeviceRoute }),+ ).resolves.toBe('Hello there.');+});diff --git a/openwhispr-mobile/src/lib/__tests__/localReasoning.test.ts b/openwhispr-mobile/src/lib/__tests__/localReasoning.test.tsindex 741953aad..65a09ce2b 100644--- a/openwhispr-mobile/src/lib/__tests__/localReasoning.test.ts+++ b/openwhispr-mobile/src/lib/__tests__/localReasoning.test.ts@@ -30,6 +30,7 @@ import {getLocalReasoningUnavailableMessage,isLocalReasoningUnsupported,shouldUseLocalReasoning,+ toLocalReasoningError,} from '../localReasoning';import { AppleLLM } from '@/lib/appleLLM';@@ -149,3 +150,28 @@ it.each([] as const)('says what an unsupported iPhone needs (%s)', (status, message) => {expect(getLocalReasoningUnavailableMessage({ status, tokenCounting: false })).toBe(message);});++describe('toLocalReasoningError', () => {+ it('reads an aborted request as cancelled', () => {+ const aborted = Object.assign(new Error('cancelled'), { name: 'AbortError' });+ expect(toLocalReasoningError(aborted)).toMatchObject({+ code: 'APPLE_LLM_CANCELLED',+ retryable: false,+ });+ });++ it('keeps the native cancellation code', () => {+ const cancelled = Object.assign(new Error('cancelled'), { code: 'APPLE_LLM_CANCELLED' });+ expect(toLocalReasoningError(cancelled)).toMatchObject({ code: 'APPLE_LLM_CANCELLED' });+ });++ it.each(['APPLE_LLM_GUARDRAIL', 'APPLE_LLM_RATE_LIMITED', 'APPLE_LLM_CONTEXT_LIMIT'])(+ 'marks %s as not worth retrying',+ (code) => {+ expect(toLocalReasoningError(Object.assign(new Error('x'), { code }))).toMatchObject({+ code,+ retryable: false,+ });+ },+ );+});diff --git a/openwhispr-mobile/src/lib/__tests__/retry.test.ts b/openwhispr-mobile/src/lib/__tests__/retry.test.tsindex ccc018b8a..09d26bddb 100644--- a/openwhispr-mobile/src/lib/__tests__/retry.test.ts+++ b/openwhispr-mobile/src/lib/__tests__/retry.test.ts@@ -1,3 +1,4 @@+import { LocalReasoningError } from '../localReasoning';import { createApiRetryStrategy } from '../retry';it('does not retry an error marked final, even without an HTTP status', () => {@@ -7,3 +8,8 @@ it('does not retry an error marked final, even without an HTTP status', () => {);expect(shouldRetry?.(new Error('network down'))).toBe(true);});++it('does not retry an On-Device failure, which carries no HTTP status', () => {+ const { shouldRetry } = createApiRetryStrategy();+ expect(shouldRetry?.(new LocalReasoningError('APPLE_LLM_RATE_LIMITED', 'busy'))).toBe(false);+});diff --git a/openwhispr-mobile/src/lib/cleanupTranscript.ts b/openwhispr-mobile/src/lib/cleanupTranscript.tsindex abc012e0e..6c8550373 100644--- a/openwhispr-mobile/src/lib/cleanupTranscript.ts+++ b/openwhispr-mobile/src/lib/cleanupTranscript.ts@@ -18,6 +18,7 @@ import {estimateLocalTokens,getLocalReasoningReadiness,getLocalReasoningUnavailableMessage,+ LocalReasoningError,} from './localReasoning';import { withRetry, createApiRetryStrategy } from './retry';@@ -53,19 +54,46 @@ function createTimeoutSignal(timeoutMs: number): { signal: AbortSignal; cancel:// punctuation and capitals cleanup adds.const LOCAL_CLEANUP_MAX_TRANSCRIPT_TOKENS = Math.floor(LOCAL_OUTPUT_TOKEN_RESERVE * 0.8);-async function localSkipReason(+// Apple Intelligence writes a cleaned copy about as long as the transcript, at roughly 30 tokens a+// second on an iPhone 15 Pro (Apple's figure), so a long one can't finish in Cloud's 12 s. The rate+// here is lower so a slower or busier iPhone still finishes; the [reasoning] dev log reports the+// real one. The startup allowance covers the readiness and budget checks and loading the model.+const LOCAL_CLEANUP_TOKENS_PER_SECOND = 25;+const LOCAL_CLEANUP_STARTUP_MS = 5_000;++function localCleanupTimeoutMs(transcriptTokens: number): number {+ return Math.max(+ CLEANUP_TIMEOUT_MS,+ LOCAL_CLEANUP_STARTUP_MS + (transcriptTokens / LOCAL_CLEANUP_TOKENS_PER_SECOND) * 1000,+ );+}++// Why an on-device pass can't run, or, for cleanup, the transcript's length in tokens.+async function prepareLocalRun(text: string,scope: 'cleanup' | 'agent',-): Promise<string | undefined> {+): Promise<{ skipReason: string } | { tokens?: number }> {let readiness = await getLocalReasoningReadiness();// The cached answer outlives a fix in iOS Settings, so check again before refusing.if (readiness.status !== 'ready') readiness = await getLocalReasoningReadiness({ refresh: true });- if (readiness.status !== 'ready') return getLocalReasoningUnavailableMessage(readiness);- if (scope !== 'cleanup') return undefined;+ if (readiness.status !== 'ready') {+ return { skipReason: getLocalReasoningUnavailableMessage(readiness) };+ }+ if (scope !== 'cleanup') return {};const tokens = (await countLocalReasoningTokens({ prompt: text })) ?? estimateLocalTokens(text);return tokens > LOCAL_CLEANUP_MAX_TRANSCRIPT_TOKENS- ? 'This transcript is too long for On-Device cleanup.'- : undefined;+ ? { skipReason: 'This transcript is too long for On-Device cleanup.' }+ : { tokens };+}++// Apple Intelligence's two common failures say what happened. The only signal a cleanup run gets+// is its timeout, so a cancelled one took too long.+function cleanupFailureReason(error: unknown): string {+ if (error instanceof LocalReasoningError) {+ if (error.code === 'APPLE_LLM_CANCELLED') return 'Apple Intelligence took too long.';+ if (error.code === 'APPLE_LLM_RATE_LIMITED') return 'Apple Intelligence was busy.';+ }+ return 'Cleanup failed.';}function isDictationCleanupContext(context: CleanupContext | undefined): boolean {@@ -106,7 +134,7 @@ export async function cleanupTranscript(// Bump timeout only when the mention is detected — ordinary dictation shouldn't// wait longer on a slow network just because the feature is on.const mentionDetected = agentName !== undefined && detectAgentMention(rawText, agentName);- const timeoutMs = mentionDetected ? AGENT_ACTION_TIMEOUT_MS : CLEANUP_TIMEOUT_MS;+ let timeoutMs = mentionDetected ? AGENT_ACTION_TIMEOUT_MS : CLEANUP_TIMEOUT_MS;const scope = mentionDetected ? 'agent' : 'cleanup';const snapshottedRoute = mentionDetected? options.agentRoute@@ -145,11 +173,12 @@ export async function cleanupTranscript(const inferenceRoute = resolved.route;// Say why an on-device pass can't run, rather than retrying into a generic failure.if (inferenceRoute.mode === 'local') {- const skipReason = await localSkipReason(rawText, scope);- if (skipReason) {- options.onSkipped?.(`${skipReason} Your raw transcript is saved.`);+ const localRun = await prepareLocalRun(rawText, scope);+ if ('skipReason' in localRun) {+ options.onSkipped?.(`${localRun.skipReason} Your raw transcript is saved.`);return rawText;}+ if (localRun.tokens !== undefined) timeoutMs = localCleanupTimeoutMs(localRun.tokens);}// Only provider and on-device routes take the privacy hint. A Cloud route is// either the user's choice or a Cloud fallback they just consented to while@@ -228,7 +257,7 @@ export async function cleanupTranscript(}return result.text;} catch (err) {- options.onSkipped?.('Cleanup failed. Your raw transcript is saved.');+ options.onSkipped?.(`${cleanupFailureReason(err)} Your raw transcript is saved.`);if (configuredRoute?.mode !== 'providers')console.warn('Reasoning failed after retries, using original transcription:', err);return rawText;diff --git a/openwhispr-mobile/src/lib/localReasoning.ts b/openwhispr-mobile/src/lib/localReasoning.tsindex ff5dbf075..69f5087bc 100644--- a/openwhispr-mobile/src/lib/localReasoning.ts+++ b/openwhispr-mobile/src/lib/localReasoning.ts@@ -15,6 +15,9 @@ export const LOCAL_TOKEN_CHAR_RATIO = 3.5;let cachedReadiness: Promise<LocalReasoningReadiness> | null = null;export class LocalReasoningError extends Error {+ // On-device failures are refusals, limits, unavailability or a cancellation:+ // a retry gets the same answer, and a timed-out run would time out again.+ readonly retryable = false;code: LocalReasoningErrorCode;readiness?: LocalReasoningReadiness;@@ -215,6 +218,9 @@ export function toLocalReasoningError(error: unknown): LocalReasoningError {? String((error as { code?: unknown }).code): undefined;const message = error instanceof Error ? error.message : 'Local AI failed.';+ if (error instanceof Error && error.name === 'AbortError') {+ return new LocalReasoningError('APPLE_LLM_CANCELLED', message);+ }switch (code) {case 'APPLE_LLM_GUARDRAIL':@@ -222,6 +228,7 @@ export function toLocalReasoningError(error: unknown): LocalReasoningError {case 'APPLE_LLM_RATE_LIMITED':case 'APPLE_LLM_UNAVAILABLE':case 'APPLE_LLM_FAILED':+ case 'APPLE_LLM_CANCELLED':return new LocalReasoningError(code, message);default:return new LocalReasoningError('APPLE_LLM_FAILED', message);diff --git a/openwhispr-mobile/src/services/reasoning/LocalReasoningService.ts b/openwhispr-mobile/src/services/reasoning/LocalReasoningService.tsindex 3170c1fed..066b81374 100644--- a/openwhispr-mobile/src/services/reasoning/LocalReasoningService.ts+++ b/openwhispr-mobile/src/services/reasoning/LocalReasoningService.ts@@ -1,5 +1,10 @@import { AppleLLM } from '@/lib/appleLLM';-import { LOCAL_OUTPUT_TOKEN_RESERVE, toLocalReasoningError } from '@/lib/localReasoning';+import {+ LOCAL_OUTPUT_TOKEN_RESERVE,+ countLocalReasoningTokens,+ estimateLocalTokens,+ toLocalReasoningError,+} from '@/lib/localReasoning';import type { ReasoningRequest, ReasoningResponse } from '@/types';// Callers fold language, tone and dictionary into the system prompt first@@ -17,16 +22,26 @@ export class LocalReasoningService {try {const startedAt = Date.now();- const result = await AppleLLM.generateText({- instructions,- prompt: request.text,- temperature: request.temperature,- maxTokens: request.maxTokens ?? LOCAL_OUTPUT_TOKEN_RESERVE,- });+ const result = await AppleLLM.generateText(+ {+ instructions,+ prompt: request.text,+ temperature: request.temperature,+ maxTokens: request.maxTokens ?? LOCAL_OUTPUT_TOKEN_RESERVE,+ },+ { signal: request.signal },+ );if (__DEV__) {+ // The rate the On-Device cleanup timeout assumes (cleanupTranscript); it includes reading+ // the prompt, and is estimated from length where iOS can't count tokens (before 26.4).+ const elapsedMs = Date.now() - startedAt;+ const counted = await countLocalReasoningTokens({ prompt: result.text });+ const outputTokens = counted ?? estimateLocalTokens(result.text);console.log(- `[reasoning] provider=local model=apple-fm elapsedMs=${Date.now() - startedAt}`,+ `[reasoning] provider=local model=apple-fm elapsedMs=${elapsedMs} outputTokens=${outputTokens}${+ counted === null ? ' (estimated)' : ''+ } tokensPerSecond=${(outputTokens / Math.max(elapsedMs / 1000, 0.001)).toFixed(1)}`,);}diff --git a/openwhispr-mobile/src/services/reasoning/ReasoningService.ts b/openwhispr-mobile/src/services/reasoning/ReasoningService.tsindex 90cd3bff7..0fd8fda9f 100644--- a/openwhispr-mobile/src/services/reasoning/ReasoningService.ts+++ b/openwhispr-mobile/src/services/reasoning/ReasoningService.ts@@ -181,7 +181,8 @@ export class ReasoningService {try {return await LocalReasoningService.processText(localRequest);} catch (error) {- if (!allowCloudFallback) throw error;+ // A cancelled or timed-out request is over; it must not start again in the cloud.+ if (!allowCloudFallback || signal?.aborted) throw error;}}} else if (!allowCloudFallback) {diff --git a/openwhispr-mobile/src/services/reasoning/__tests__/LocalReasoningService.test.ts b/openwhispr-mobile/src/services/reasoning/__tests__/LocalReasoningService.test.tsnew file mode 100644index 000000000..1bd70ce8a--- /dev/null+++ b/openwhispr-mobile/src/services/reasoning/__tests__/LocalReasoningService.test.ts@@ -0,0 +1,67 @@+import { AppleLLM } from '@/lib/appleLLM';+import { LocalReasoningService } from '../LocalReasoningService';++jest.mock('@/lib/appleLLM', () => ({+ AppleLLM: { generateText: jest.fn(), countTokens: jest.fn(async () => null) },+}));++const mockGenerateText = AppleLLM.generateText as jest.Mock;+const mockCountTokens = AppleLLM.countTokens as jest.Mock;++beforeEach(() => {+ jest.clearAllMocks();+});++it('passes the caller signal to Apple Intelligence so a timeout stops the run', async () => {+ mockGenerateText.mockResolvedValue({ text: ' Hello. ', model: 'apple-foundation-models' });+ const { signal } = new AbortController();++ await expect(+ LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.', signal }),+ ).resolves.toEqual({ text: 'Hello.', model: 'apple-fm' });+ expect(mockGenerateText).toHaveBeenCalledWith(expect.objectContaining({ prompt: 'um hello' }), {+ signal,+ });+});++it('reports an aborted run as a cancelled local request', async () => {+ mockGenerateText.mockRejectedValue(Object.assign(new Error('cancelled'), { name: 'AbortError' }));++ await expect(+ LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.' }),+ ).rejects.toMatchObject({ name: 'LocalReasoningError', code: 'APPLE_LLM_CANCELLED' });+});++// The On-Device cleanup timeout assumes a generation rate; dev builds log the measured one.+describe('dev log', () => {+ let log: jest.SpyInstance;+ beforeEach(() => {+ log = jest.spyOn(console, 'log').mockImplementation(() => undefined);+ jest.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValueOnce(2_000);+ });+ afterEach(() => {+ jest.restoreAllMocks();+ });++ it('reports output tokens per second', async () => {+ mockGenerateText.mockResolvedValue({ text: 'Hello.', model: 'apple-foundation-models' });+ mockCountTokens.mockResolvedValueOnce(50);++ await LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.' });++ expect(mockCountTokens).toHaveBeenCalledWith({ prompt: 'Hello.' });+ expect(log).toHaveBeenCalledWith(+ '[reasoning] provider=local model=apple-fm elapsedMs=2000 outputTokens=50 tokensPerSecond=25.0',+ );+ });++ it('marks the count as estimated where iOS cannot count tokens', async () => {+ mockGenerateText.mockResolvedValue({ text: 'Hello there.', model: 'apple-foundation-models' });++ await LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.' });++ expect(log).toHaveBeenCalledWith(+ '[reasoning] provider=local model=apple-fm elapsedMs=2000 outputTokens=4 (estimated) tokensPerSecond=2.0',+ );+ });+});diff --git a/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.ts b/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.tsindex 96bbfa884..ee21c26c6 100644--- a/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.ts+++ b/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.ts@@ -102,6 +102,26 @@ describe('ReasoningService local routing', () => {expect(mockPost).toHaveBeenCalledTimes(1);});+ it('does not fall back to cloud once the request was cancelled', async () => {+ const controller = new AbortController();+ const cancelled = new Error('cancelled');+ mockLocalProcessText.mockImplementation(async () => {+ controller.abort();+ throw cancelled;+ });++ await expect(+ ReasoningService.processText({+ text: 'private note',+ systemPrompt: 'format this',+ signal: controller.signal,+ routing: { isPrivateNote: true, allowCloudFallback: true },+ }),+ ).rejects.toBe(cancelled);++ expect(mockPost).not.toHaveBeenCalled();+ });+it('keeps public signed-in requests on the cloud path', async () => {mockIsLocalRequired.mockReturnValue(false);diff --git a/openwhispr-mobile/src/types/index.ts b/openwhispr-mobile/src/types/index.tsindex 3d4a404d3..6a21050d6 100644--- a/openwhispr-mobile/src/types/index.ts+++ b/openwhispr-mobile/src/types/index.ts@@ -159,6 +159,7 @@ export type LocalReasoningErrorCode =| 'APPLE_LLM_RATE_LIMITED'| 'APPLE_LLM_UNAVAILABLE'| 'APPLE_LLM_FAILED'+ | 'APPLE_LLM_CANCELLED'| 'LOCAL_CONTEXT_LIMIT'| 'LOCAL_REASONING_UNAVAILABLE';