Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(mobile): stop On-Device cleanup at its timeout and don't retry it (#2690)

Chad Piha2c84aaf612ea
diff --git a/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swift b/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swift
index 612c6776c..93f1c8bda 100644
--- a/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swift
+++ b/openwhispr-mobile/modules/apple-llm/ios/AppleLLMModule.swift
@@ -15,6 +15,32 @@ private struct AppleLLMGenerateRequest: Record {
@Field var prompt: String = ""
@Field var temperature: Double?
@Field var maxTokens: Int?
+ @Field var requestId: String?
+}
+
+// Generations JavaScript can stop by id. A timed-out cleanup must end the model
+// run too, or it keeps the on-device model busy after JavaScript gave up on it.
+private final class AppleLLMGenerations: @unchecked Sendable {
+ private let lock = NSLock()
+ private var tasks: [String: Task<Void, Never>] = [:]
+
+ func start(_ requestId: String?, operation: @escaping @Sendable () async -> Void) {
+ guard let requestId else {
+ Task { await operation() }
+ return
+ }
+ // Registered under the lock, so the task's own removal can't run before it.
+ lock.withLock {
+ tasks[requestId] = Task {
+ await operation()
+ self.lock.withLock { _ = self.tasks.removeValue(forKey: requestId) }
+ }
+ }
+ }
+
+ func cancel(_ requestId: String) {
+ lock.withLock { tasks[requestId] }?.cancel()
+ }
}
#if canImport(FoundationModels)
@@ -37,6 +63,8 @@ private struct AppleGeneratedMeetingNotes {
#endif
public class AppleLLMModule: Module {
+ private static let generations = AppleLLMGenerations()
+
public func definition() -> ModuleDefinition {
Name("AppleLLM")
@@ -83,7 +111,7 @@ public class AppleLLMModule: Module {
return
}
- Task {
+ Self.generations.start(request.requestId) {
do {
try Self.ensureAvailable()
let session = LanguageModelSession(instructions: request.instructions)
@@ -104,6 +132,10 @@ public class AppleLLMModule: Module {
#endif
}
+ Function("cancelGeneration") { (requestId: String) in
+ Self.generations.cancel(requestId)
+ }
+
AsyncFunction("generateMeetingNotes") { (request: AppleLLMGenerateRequest, promise: Promise) in
#if canImport(FoundationModels)
guard #available(iOS 26.0, *) else {
@@ -211,6 +243,11 @@ public class AppleLLMModule: Module {
return
}
+ if error is CancellationError {
+ promise.reject("APPLE_LLM_CANCELLED", "The Apple Foundation Models request was cancelled.")
+ return
+ }
+
if let generationError = error as? LanguageModelSession.GenerationError {
switch generationError {
case .guardrailViolation(_), .refusal(_, _):
diff --git a/openwhispr-mobile/modules/apple-llm/src/__tests__/index.test.ts b/openwhispr-mobile/modules/apple-llm/src/__tests__/index.test.ts
new file mode 100644
index 000000000..10203fb87
--- /dev/null
+++ b/openwhispr-mobile/modules/apple-llm/src/__tests__/index.test.ts
@@ -0,0 +1,129 @@
+const mockNative: {
+ generateText: jest.Mock;
+ cancelGeneration?: jest.Mock;
+} = {
+ generateText: jest.fn(),
+ cancelGeneration: jest.fn(),
+};
+
+jest.mock('expo', () => ({
+ requireNativeModule: jest.fn(() => mockNative),
+}));
+jest.mock('react-native', () => ({ Platform: { OS: 'ios' } }));
+
+// Required after the mocks above: a hoisted import would read mockNative before it exists.
+const { AppleLLM } = jest.requireActual<typeof import('../index')>('../index');
+
+const request = { instructions: 'Clean this up.', prompt: 'um hello' };
+const response = { text: 'Hello.', model: 'apple-foundation-models' as const };
+
+function pendingNative(): { reject: (error: Error) => void } {
+ let reject: (error: Error) => void = () => undefined;
+ mockNative.generateText.mockReturnValue(
+ new Promise((_resolve, rejectNative) => {
+ reject = rejectNative;
+ }),
+ );
+ return { reject: (error) => reject(error) };
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ mockNative.cancelGeneration = jest.fn();
+});
+
+it('sends the request unchanged when there is no signal', async () => {
+ mockNative.generateText.mockResolvedValue(response);
+ await expect(AppleLLM.generateText(request)).resolves.toEqual(response);
+ expect(mockNative.generateText).toHaveBeenCalledWith(request);
+});
+
+it('cancels the native run by its id and rejects with an AbortError on abort', async () => {
+ const native = pendingNative();
+ const controller = new AbortController();
+ const result = AppleLLM.generateText(request, { signal: controller.signal });
+
+ controller.abort();
+
+ await expect(result).rejects.toMatchObject({ name: 'AbortError' });
+ const { requestId } = mockNative.generateText.mock.calls[0][0] as { requestId: string };
+ expect(requestId).toEqual(expect.any(String));
+ expect(mockNative.cancelGeneration).toHaveBeenCalledWith(requestId);
+ // The native cancellation arriving afterwards is swallowed, not reported unhandled.
+ native.reject(Object.assign(new Error('cancelled'), { code: 'APPLE_LLM_CANCELLED' }));
+ await Promise.resolve();
+});
+
+it('gives each request its own id', async () => {
+ mockNative.generateText.mockResolvedValue(response);
+ const { signal } = new AbortController();
+ await AppleLLM.generateText(request, { signal });
+ await AppleLLM.generateText(request, { signal });
+ const [first, second] = mockNative.generateText.mock.calls.map(
+ ([sent]) => (sent as { requestId: string }).requestId,
+ );
+ expect(first).not.toBe(second);
+});
+
+// A JS reload restarts the counter, but native keeps the runs started before it.
+it('keeps ids unique across a JS reload', async () => {
+ mockNative.generateText.mockResolvedValue(response);
+ const { signal } = new AbortController();
+ const now = jest.spyOn(Date, 'now');
+ for (const time of [1_000, 2_000]) {
+ now.mockReturnValue(time);
+ await jest.isolateModulesAsync(async () => {
+ const reloaded = jest.requireActual<typeof import('../index')>('../index');
+ await reloaded.AppleLLM.generateText(request, { signal });
+ });
+ }
+ now.mockRestore();
+ const [beforeReload, afterReload] = mockNative.generateText.mock.calls.map(
+ ([sent]) => (sent as { requestId: string }).requestId,
+ );
+ expect(beforeReload).not.toBe(afterReload);
+});
+
+it('rejects an already-aborted signal without starting a native run', async () => {
+ const controller = new AbortController();
+ controller.abort();
+
+ await expect(AppleLLM.generateText(request, { signal: controller.signal })).rejects.toMatchObject(
+ { name: 'AbortError' },
+ );
+ expect(mockNative.generateText).not.toHaveBeenCalled();
+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();
+});
+
+it('stops listening for abort once the native run settles', async () => {
+ mockNative.generateText.mockResolvedValue(response);
+ const controller = new AbortController();
+ const removeEventListener = jest.spyOn(controller.signal, 'removeEventListener');
+
+ await expect(AppleLLM.generateText(request, { signal: controller.signal })).resolves.toEqual(
+ response,
+ );
+ await Promise.resolve();
+ expect(removeEventListener).toHaveBeenCalledWith('abort', expect.any(Function));
+
+ controller.abort();
+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();
+});
+
+it('passes a native failure through', async () => {
+ const failure = Object.assign(new Error('blocked'), { code: 'APPLE_LLM_GUARDRAIL' });
+ mockNative.generateText.mockRejectedValue(failure);
+ const { signal } = new AbortController();
+ await expect(AppleLLM.generateText(request, { signal })).rejects.toBe(failure);
+});
+
+it('still rejects on abort when the binary predates cancelGeneration', async () => {
+ pendingNative();
+ delete mockNative.cancelGeneration;
+ const controller = new AbortController();
+ const result = AppleLLM.generateText(request, { signal: controller.signal });
+
+ controller.abort();
+
+ await expect(result).rejects.toMatchObject({ name: 'AbortError' });
+});
diff --git a/openwhispr-mobile/modules/apple-llm/src/index.ts b/openwhispr-mobile/modules/apple-llm/src/index.ts
index 9cb5a159d..ce9c92fa9 100644
--- a/openwhispr-mobile/modules/apple-llm/src/index.ts
+++ b/openwhispr-mobile/modules/apple-llm/src/index.ts
@@ -11,7 +11,11 @@ import type {
interface NativeAppleLLM {
getAvailability(): Promise<AppleLLMAvailability>;
countTokens(request: AppleLLMTokenCountRequest): Promise<number | null>;
- generateText(request: AppleLLMGenerateTextRequest): Promise<AppleLLMGenerateTextResponse>;
+ generateText(
+ request: AppleLLMGenerateTextRequest & { requestId?: string },
+ ): Promise<AppleLLMGenerateTextResponse>;
+ // Missing on binaries built before it shipped; an OTA update can run this code there.
+ cancelGeneration?(requestId: string): void;
generateMeetingNotes(
request: AppleLLMGenerateTextRequest,
): Promise<AppleLLMGenerateMeetingNotesResponse>;
@@ -28,6 +32,16 @@ const NativeModule: NativeAppleLLM | null = (() => {
}
})();
+let requestSequence = 0;
+
+// Hermes has no DOMException global, so cancellations use a plain Error that
+// callers recognise by name, as they would a fetch AbortError.
+function abortError(): Error {
+ const error = new Error('Apple Foundation Models request cancelled.');
+ error.name = 'AbortError';
+ return error;
+}
+
export const AppleLLM = {
isAvailable(): boolean {
return NativeModule !== null;
@@ -47,11 +61,34 @@ export const AppleLLM = {
return NativeModule ? NativeModule.countTokens(request) : null;
},
- async generateText(request: AppleLLMGenerateTextRequest): Promise<AppleLLMGenerateTextResponse> {
+ async generateText(
+ request: AppleLLMGenerateTextRequest,
+ options: { signal?: AbortSignal } = {},
+ ): Promise<AppleLLMGenerateTextResponse> {
if (!NativeModule) {
throw new Error('Apple Foundation Models are not linked in this build.');
}
- return NativeModule.generateText(request);
+ const { signal } = options;
+ if (!signal) return NativeModule.generateText(request);
+ if (signal.aborted) throw abortError();
+
+ requestSequence += 1;
+ // The time keeps an id unique across JS reloads, which restart the counter but not native.
+ const requestId = `apple-llm-${Date.now()}-${requestSequence}`;
+ // Settle on abort without waiting for native: an older binary can't stop the
+ // run, and a newer one reports the cancellation only once the model yields.
+ return new Promise<AppleLLMGenerateTextResponse>((resolve, reject) => {
+ const onAbort = (): void => {
+ NativeModule.cancelGeneration?.(requestId);
+ reject(abortError());
+ };
+ signal.addEventListener('abort', onAbort, { once: true });
+ NativeModule.generateText({ ...request, requestId })
+ .then(resolve, reject)
+ .finally(() => {
+ signal.removeEventListener('abort', onAbort);
+ });
+ });
},
async generateMeetingNotes(
diff --git a/openwhispr-mobile/src/lib/__tests__/cleanupTranscript.onDevice.test.ts b/openwhispr-mobile/src/lib/__tests__/cleanupTranscript.onDevice.test.ts
new file mode 100644
index 000000000..8c59207be
--- /dev/null
+++ b/openwhispr-mobile/src/lib/__tests__/cleanupTranscript.onDevice.test.ts
@@ -0,0 +1,153 @@
+/**
+ * On-Device cleanup runs through the real retry policy, reasoning routing and
+ * Apple Intelligence wrapper; only the native module is faked. A slow or failed
+ * run must hand back the raw transcript after one attempt, since a retry would
+ * meet the same refusal or time out again.
+ */
+import { CLEANUP_TIMEOUT_MS, cleanupTranscript } from '@/lib/cleanupTranscript';
+
+interface MockNativeAppleLLM {
+ getAvailability: jest.Mock;
+ countTokens: jest.Mock;
+ generateText: jest.Mock;
+ cancelGeneration: jest.Mock;
+}
+
+jest.mock('expo', () => {
+ const native: MockNativeAppleLLM = {
+ getAvailability: jest.fn(),
+ countTokens: jest.fn(),
+ generateText: jest.fn(),
+ cancelGeneration: jest.fn(),
+ };
+ return { requireNativeModule: (): MockNativeAppleLLM => native, mockNative: native };
+});
+// The Cloud client imports expo/fetch, which needs the real expo package; an
+// On-Device cleanup never reaches it.
+jest.mock('@/lib/apiClient', () => ({ api: { post: jest.fn() } }));
+// accountAccess pulls in the router for its sign-in alert; stub it here.
+jest.mock('expo-router', () => ({ router: { push: jest.fn() } }));
+jest.mock('@/store/useSnippetsStore', () => ({
+ useSnippetsStore: { getState: () => ({ isLoaded: true, entries: [] }) },
+}));
+jest.mock('@/store/useDictionaryStore', () => ({
+ useDictionaryStore: { getState: () => ({ isLoaded: true, entries: [] }) },
+}));
+jest.mock('@/store/useCustomPromptsStore', () => ({
+ getActiveCustomCleanupPrompt: () => undefined,
+}));
+jest.mock('@/store/useConfigStore', () => ({
+ useConfigStore: {
+ getState: () => ({ config: { cleanupEnabled: true, defaultMode: 'private' } }),
+ },
+}));
+jest.mock('@/store/useProcessingModeStore', () => ({
+ useProcessingModeStore: { getState: () => ({ activeMode: 'private' }) },
+}));
+jest.mock('@/store/useAuthStore', () => ({
+ useAuthStore: { getState: () => ({ user: { id: 'u', isAnonymous: false } }) },
+}));
+
+const { mockNative } = jest.requireMock<{ mockNative: MockNativeAppleLLM }>('expo');
+const onDeviceRoute = { mode: 'local', scope: 'cleanup' } as const;
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ jest.useFakeTimers();
+ mockNative.getAvailability.mockResolvedValue({
+ status: 'available',
+ contextSize: 4096,
+ tokenCounting: true,
+ });
+ mockNative.countTokens.mockResolvedValue(10);
+ jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+});
+
+afterEach(() => {
+ jest.useRealTimers();
+ jest.restoreAllMocks();
+});
+
+it('pastes the raw transcript at the timeout and stops the model run', async () => {
+ mockNative.generateText.mockReturnValue(new Promise(() => undefined));
+ const onSkipped = jest.fn();
+ const result = cleanupTranscript('um hello there', { inferenceRoute: onDeviceRoute, onSkipped });
+
+ await jest.advanceTimersByTimeAsync(CLEANUP_TIMEOUT_MS);
+ // A retry would start a second run before the next timeout; checked before awaiting the
+ // result, which a retry would hold back until the timers move again.
+ await jest.advanceTimersByTimeAsync(CLEANUP_TIMEOUT_MS);
+ expect(mockNative.generateText).toHaveBeenCalledTimes(1);
+
+ await expect(result).resolves.toBe('um hello there');
+ expect(onSkipped).toHaveBeenCalledWith(
+ 'Apple Intelligence took too long. Your raw transcript is saved.',
+ );
+ const { requestId } = mockNative.generateText.mock.calls[0][0] as { requestId: string };
+ expect(mockNative.cancelGeneration).toHaveBeenCalledWith(requestId);
+});
+
+// Apple Intelligence writes about as many tokens as the transcript has, so a long one gets
+// longer than Cloud's 12 s.
+describe('a long transcript', () => {
+ beforeEach(() => {
+ mockNative.countTokens.mockResolvedValue(600);
+ });
+
+ it('is cleaned when the model needs longer than 12 s', async () => {
+ mockNative.generateText.mockReturnValue(
+ new Promise((resolve) =>
+ setTimeout(() => resolve({ text: 'Long text.', model: 'apple-foundation-models' }), 20_000),
+ ),
+ );
+ const result = cleanupTranscript('um long text', { inferenceRoute: onDeviceRoute });
+
+ await jest.advanceTimersByTimeAsync(20_000);
+
+ await expect(result).resolves.toBe('Long text.');
+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();
+ });
+
+ it('still stops a run that never finishes', async () => {
+ mockNative.generateText.mockReturnValue(new Promise(() => undefined));
+ const onSkipped = jest.fn();
+ const result = cleanupTranscript('um long text', { inferenceRoute: onDeviceRoute, onSkipped });
+
+ // 5 s to start, plus 600 tokens at 25 a second.
+ await jest.advanceTimersByTimeAsync(28_999);
+ expect(mockNative.cancelGeneration).not.toHaveBeenCalled();
+ await jest.advanceTimersByTimeAsync(1);
+
+ await expect(result).resolves.toBe('um long text');
+ expect(mockNative.cancelGeneration).toHaveBeenCalledTimes(1);
+ expect(onSkipped).toHaveBeenCalledWith(
+ 'Apple Intelligence took too long. Your raw transcript is saved.',
+ );
+ });
+});
+
+it.each([
+ ['APPLE_LLM_RATE_LIMITED', 'Apple Intelligence was busy.'],
+ ['APPLE_LLM_GUARDRAIL', 'Cleanup failed.'],
+ ['APPLE_LLM_FAILED', 'Cleanup failed.'],
+])('pastes the raw transcript after one %s attempt', async (code, reason) => {
+ mockNative.generateText.mockRejectedValue(Object.assign(new Error('refused'), { code }));
+ const onSkipped = jest.fn();
+ const result = cleanupTranscript('um hello there', { inferenceRoute: onDeviceRoute, onSkipped });
+
+ await jest.advanceTimersByTimeAsync(CLEANUP_TIMEOUT_MS);
+ expect(mockNative.generateText).toHaveBeenCalledTimes(1);
+
+ await expect(result).resolves.toBe('um hello there');
+ expect(onSkipped).toHaveBeenCalledWith(`${reason} Your raw transcript is saved.`);
+});
+
+it('returns the cleaned text when the model answers in time', async () => {
+ mockNative.generateText.mockResolvedValue({
+ text: 'Hello there.',
+ model: 'apple-foundation-models',
+ });
+ await expect(
+ cleanupTranscript('um hello there', { inferenceRoute: onDeviceRoute }),
+ ).resolves.toBe('Hello there.');
+});
diff --git a/openwhispr-mobile/src/lib/__tests__/localReasoning.test.ts b/openwhispr-mobile/src/lib/__tests__/localReasoning.test.ts
index 741953aad..65a09ce2b 100644
--- a/openwhispr-mobile/src/lib/__tests__/localReasoning.test.ts
+++ b/openwhispr-mobile/src/lib/__tests__/localReasoning.test.ts
@@ -30,6 +30,7 @@ import {
getLocalReasoningUnavailableMessage,
isLocalReasoningUnsupported,
shouldUseLocalReasoning,
+ toLocalReasoningError,
} from '../localReasoning';
import { AppleLLM } from '@/lib/appleLLM';
@@ -149,3 +150,28 @@ it.each([
] as const)('says what an unsupported iPhone needs (%s)', (status, message) => {
expect(getLocalReasoningUnavailableMessage({ status, tokenCounting: false })).toBe(message);
});
+
+describe('toLocalReasoningError', () => {
+ it('reads an aborted request as cancelled', () => {
+ const aborted = Object.assign(new Error('cancelled'), { name: 'AbortError' });
+ expect(toLocalReasoningError(aborted)).toMatchObject({
+ code: 'APPLE_LLM_CANCELLED',
+ retryable: false,
+ });
+ });
+
+ it('keeps the native cancellation code', () => {
+ const cancelled = Object.assign(new Error('cancelled'), { code: 'APPLE_LLM_CANCELLED' });
+ expect(toLocalReasoningError(cancelled)).toMatchObject({ code: 'APPLE_LLM_CANCELLED' });
+ });
+
+ it.each(['APPLE_LLM_GUARDRAIL', 'APPLE_LLM_RATE_LIMITED', 'APPLE_LLM_CONTEXT_LIMIT'])(
+ 'marks %s as not worth retrying',
+ (code) => {
+ expect(toLocalReasoningError(Object.assign(new Error('x'), { code }))).toMatchObject({
+ code,
+ retryable: false,
+ });
+ },
+ );
+});
diff --git a/openwhispr-mobile/src/lib/__tests__/retry.test.ts b/openwhispr-mobile/src/lib/__tests__/retry.test.ts
index ccc018b8a..09d26bddb 100644
--- a/openwhispr-mobile/src/lib/__tests__/retry.test.ts
+++ b/openwhispr-mobile/src/lib/__tests__/retry.test.ts
@@ -1,3 +1,4 @@
+import { LocalReasoningError } from '../localReasoning';
import { createApiRetryStrategy } from '../retry';
it('does not retry an error marked final, even without an HTTP status', () => {
@@ -7,3 +8,8 @@ it('does not retry an error marked final, even without an HTTP status', () => {
);
expect(shouldRetry?.(new Error('network down'))).toBe(true);
});
+
+it('does not retry an On-Device failure, which carries no HTTP status', () => {
+ const { shouldRetry } = createApiRetryStrategy();
+ expect(shouldRetry?.(new LocalReasoningError('APPLE_LLM_RATE_LIMITED', 'busy'))).toBe(false);
+});
diff --git a/openwhispr-mobile/src/lib/cleanupTranscript.ts b/openwhispr-mobile/src/lib/cleanupTranscript.ts
index abc012e0e..6c8550373 100644
--- a/openwhispr-mobile/src/lib/cleanupTranscript.ts
+++ b/openwhispr-mobile/src/lib/cleanupTranscript.ts
@@ -18,6 +18,7 @@ import {
estimateLocalTokens,
getLocalReasoningReadiness,
getLocalReasoningUnavailableMessage,
+ LocalReasoningError,
} from './localReasoning';
import { withRetry, createApiRetryStrategy } from './retry';
@@ -53,19 +54,46 @@ function createTimeoutSignal(timeoutMs: number): { signal: AbortSignal; cancel:
// punctuation and capitals cleanup adds.
const LOCAL_CLEANUP_MAX_TRANSCRIPT_TOKENS = Math.floor(LOCAL_OUTPUT_TOKEN_RESERVE * 0.8);
-async function localSkipReason(
+// Apple Intelligence writes a cleaned copy about as long as the transcript, at roughly 30 tokens a
+// second on an iPhone 15 Pro (Apple's figure), so a long one can't finish in Cloud's 12 s. The rate
+// here is lower so a slower or busier iPhone still finishes; the [reasoning] dev log reports the
+// real one. The startup allowance covers the readiness and budget checks and loading the model.
+const LOCAL_CLEANUP_TOKENS_PER_SECOND = 25;
+const LOCAL_CLEANUP_STARTUP_MS = 5_000;
+
+function localCleanupTimeoutMs(transcriptTokens: number): number {
+ return Math.max(
+ CLEANUP_TIMEOUT_MS,
+ LOCAL_CLEANUP_STARTUP_MS + (transcriptTokens / LOCAL_CLEANUP_TOKENS_PER_SECOND) * 1000,
+ );
+}
+
+// Why an on-device pass can't run, or, for cleanup, the transcript's length in tokens.
+async function prepareLocalRun(
text: string,
scope: 'cleanup' | 'agent',
-): Promise<string | undefined> {
+): Promise<{ skipReason: string } | { tokens?: number }> {
let readiness = await getLocalReasoningReadiness();
// The cached answer outlives a fix in iOS Settings, so check again before refusing.
if (readiness.status !== 'ready') readiness = await getLocalReasoningReadiness({ refresh: true });
- if (readiness.status !== 'ready') return getLocalReasoningUnavailableMessage(readiness);
- if (scope !== 'cleanup') return undefined;
+ if (readiness.status !== 'ready') {
+ return { skipReason: getLocalReasoningUnavailableMessage(readiness) };
+ }
+ if (scope !== 'cleanup') return {};
const tokens = (await countLocalReasoningTokens({ prompt: text })) ?? estimateLocalTokens(text);
return tokens > LOCAL_CLEANUP_MAX_TRANSCRIPT_TOKENS
- ? 'This transcript is too long for On-Device cleanup.'
- : undefined;
+ ? { skipReason: 'This transcript is too long for On-Device cleanup.' }
+ : { tokens };
+}
+
+// Apple Intelligence's two common failures say what happened. The only signal a cleanup run gets
+// is its timeout, so a cancelled one took too long.
+function cleanupFailureReason(error: unknown): string {
+ if (error instanceof LocalReasoningError) {
+ if (error.code === 'APPLE_LLM_CANCELLED') return 'Apple Intelligence took too long.';
+ if (error.code === 'APPLE_LLM_RATE_LIMITED') return 'Apple Intelligence was busy.';
+ }
+ return 'Cleanup failed.';
}
function isDictationCleanupContext(context: CleanupContext | undefined): boolean {
@@ -106,7 +134,7 @@ export async function cleanupTranscript(
// Bump timeout only when the mention is detected — ordinary dictation shouldn't
// wait longer on a slow network just because the feature is on.
const mentionDetected = agentName !== undefined && detectAgentMention(rawText, agentName);
- const timeoutMs = mentionDetected ? AGENT_ACTION_TIMEOUT_MS : CLEANUP_TIMEOUT_MS;
+ let timeoutMs = mentionDetected ? AGENT_ACTION_TIMEOUT_MS : CLEANUP_TIMEOUT_MS;
const scope = mentionDetected ? 'agent' : 'cleanup';
const snapshottedRoute = mentionDetected
? options.agentRoute
@@ -145,11 +173,12 @@ export async function cleanupTranscript(
const inferenceRoute = resolved.route;
// Say why an on-device pass can't run, rather than retrying into a generic failure.
if (inferenceRoute.mode === 'local') {
- const skipReason = await localSkipReason(rawText, scope);
- if (skipReason) {
- options.onSkipped?.(`${skipReason} Your raw transcript is saved.`);
+ const localRun = await prepareLocalRun(rawText, scope);
+ if ('skipReason' in localRun) {
+ options.onSkipped?.(`${localRun.skipReason} Your raw transcript is saved.`);
return rawText;
}
+ if (localRun.tokens !== undefined) timeoutMs = localCleanupTimeoutMs(localRun.tokens);
}
// Only provider and on-device routes take the privacy hint. A Cloud route is
// either the user's choice or a Cloud fallback they just consented to while
@@ -228,7 +257,7 @@ export async function cleanupTranscript(
}
return result.text;
} catch (err) {
- options.onSkipped?.('Cleanup failed. Your raw transcript is saved.');
+ options.onSkipped?.(`${cleanupFailureReason(err)} Your raw transcript is saved.`);
if (configuredRoute?.mode !== 'providers')
console.warn('Reasoning failed after retries, using original transcription:', err);
return rawText;
diff --git a/openwhispr-mobile/src/lib/localReasoning.ts b/openwhispr-mobile/src/lib/localReasoning.ts
index ff5dbf075..69f5087bc 100644
--- a/openwhispr-mobile/src/lib/localReasoning.ts
+++ b/openwhispr-mobile/src/lib/localReasoning.ts
@@ -15,6 +15,9 @@ export const LOCAL_TOKEN_CHAR_RATIO = 3.5;
let cachedReadiness: Promise<LocalReasoningReadiness> | null = null;
export class LocalReasoningError extends Error {
+ // On-device failures are refusals, limits, unavailability or a cancellation:
+ // a retry gets the same answer, and a timed-out run would time out again.
+ readonly retryable = false;
code: LocalReasoningErrorCode;
readiness?: LocalReasoningReadiness;
@@ -215,6 +218,9 @@ export function toLocalReasoningError(error: unknown): LocalReasoningError {
? String((error as { code?: unknown }).code)
: undefined;
const message = error instanceof Error ? error.message : 'Local AI failed.';
+ if (error instanceof Error && error.name === 'AbortError') {
+ return new LocalReasoningError('APPLE_LLM_CANCELLED', message);
+ }
switch (code) {
case 'APPLE_LLM_GUARDRAIL':
@@ -222,6 +228,7 @@ export function toLocalReasoningError(error: unknown): LocalReasoningError {
case 'APPLE_LLM_RATE_LIMITED':
case 'APPLE_LLM_UNAVAILABLE':
case 'APPLE_LLM_FAILED':
+ case 'APPLE_LLM_CANCELLED':
return new LocalReasoningError(code, message);
default:
return new LocalReasoningError('APPLE_LLM_FAILED', message);
diff --git a/openwhispr-mobile/src/services/reasoning/LocalReasoningService.ts b/openwhispr-mobile/src/services/reasoning/LocalReasoningService.ts
index 3170c1fed..066b81374 100644
--- a/openwhispr-mobile/src/services/reasoning/LocalReasoningService.ts
+++ b/openwhispr-mobile/src/services/reasoning/LocalReasoningService.ts
@@ -1,5 +1,10 @@
import { AppleLLM } from '@/lib/appleLLM';
-import { LOCAL_OUTPUT_TOKEN_RESERVE, toLocalReasoningError } from '@/lib/localReasoning';
+import {
+ LOCAL_OUTPUT_TOKEN_RESERVE,
+ countLocalReasoningTokens,
+ estimateLocalTokens,
+ toLocalReasoningError,
+} from '@/lib/localReasoning';
import type { ReasoningRequest, ReasoningResponse } from '@/types';
// Callers fold language, tone and dictionary into the system prompt first
@@ -17,16 +22,26 @@ export class LocalReasoningService {
try {
const startedAt = Date.now();
- const result = await AppleLLM.generateText({
- instructions,
- prompt: request.text,
- temperature: request.temperature,
- maxTokens: request.maxTokens ?? LOCAL_OUTPUT_TOKEN_RESERVE,
- });
+ const result = await AppleLLM.generateText(
+ {
+ instructions,
+ prompt: request.text,
+ temperature: request.temperature,
+ maxTokens: request.maxTokens ?? LOCAL_OUTPUT_TOKEN_RESERVE,
+ },
+ { signal: request.signal },
+ );
if (__DEV__) {
+ // The rate the On-Device cleanup timeout assumes (cleanupTranscript); it includes reading
+ // the prompt, and is estimated from length where iOS can't count tokens (before 26.4).
+ const elapsedMs = Date.now() - startedAt;
+ const counted = await countLocalReasoningTokens({ prompt: result.text });
+ const outputTokens = counted ?? estimateLocalTokens(result.text);
console.log(
- `[reasoning] provider=local model=apple-fm elapsedMs=${Date.now() - startedAt}`,
+ `[reasoning] provider=local model=apple-fm elapsedMs=${elapsedMs} outputTokens=${outputTokens}${
+ counted === null ? ' (estimated)' : ''
+ } tokensPerSecond=${(outputTokens / Math.max(elapsedMs / 1000, 0.001)).toFixed(1)}`,
);
}
diff --git a/openwhispr-mobile/src/services/reasoning/ReasoningService.ts b/openwhispr-mobile/src/services/reasoning/ReasoningService.ts
index 90cd3bff7..0fd8fda9f 100644
--- a/openwhispr-mobile/src/services/reasoning/ReasoningService.ts
+++ b/openwhispr-mobile/src/services/reasoning/ReasoningService.ts
@@ -181,7 +181,8 @@ export class ReasoningService {
try {
return await LocalReasoningService.processText(localRequest);
} catch (error) {
- if (!allowCloudFallback) throw error;
+ // A cancelled or timed-out request is over; it must not start again in the cloud.
+ if (!allowCloudFallback || signal?.aborted) throw error;
}
}
} else if (!allowCloudFallback) {
diff --git a/openwhispr-mobile/src/services/reasoning/__tests__/LocalReasoningService.test.ts b/openwhispr-mobile/src/services/reasoning/__tests__/LocalReasoningService.test.ts
new file mode 100644
index 000000000..1bd70ce8a
--- /dev/null
+++ b/openwhispr-mobile/src/services/reasoning/__tests__/LocalReasoningService.test.ts
@@ -0,0 +1,67 @@
+import { AppleLLM } from '@/lib/appleLLM';
+import { LocalReasoningService } from '../LocalReasoningService';
+
+jest.mock('@/lib/appleLLM', () => ({
+ AppleLLM: { generateText: jest.fn(), countTokens: jest.fn(async () => null) },
+}));
+
+const mockGenerateText = AppleLLM.generateText as jest.Mock;
+const mockCountTokens = AppleLLM.countTokens as jest.Mock;
+
+beforeEach(() => {
+ jest.clearAllMocks();
+});
+
+it('passes the caller signal to Apple Intelligence so a timeout stops the run', async () => {
+ mockGenerateText.mockResolvedValue({ text: ' Hello. ', model: 'apple-foundation-models' });
+ const { signal } = new AbortController();
+
+ await expect(
+ LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.', signal }),
+ ).resolves.toEqual({ text: 'Hello.', model: 'apple-fm' });
+ expect(mockGenerateText).toHaveBeenCalledWith(expect.objectContaining({ prompt: 'um hello' }), {
+ signal,
+ });
+});
+
+it('reports an aborted run as a cancelled local request', async () => {
+ mockGenerateText.mockRejectedValue(Object.assign(new Error('cancelled'), { name: 'AbortError' }));
+
+ await expect(
+ LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.' }),
+ ).rejects.toMatchObject({ name: 'LocalReasoningError', code: 'APPLE_LLM_CANCELLED' });
+});
+
+// The On-Device cleanup timeout assumes a generation rate; dev builds log the measured one.
+describe('dev log', () => {
+ let log: jest.SpyInstance;
+ beforeEach(() => {
+ log = jest.spyOn(console, 'log').mockImplementation(() => undefined);
+ jest.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValueOnce(2_000);
+ });
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ it('reports output tokens per second', async () => {
+ mockGenerateText.mockResolvedValue({ text: 'Hello.', model: 'apple-foundation-models' });
+ mockCountTokens.mockResolvedValueOnce(50);
+
+ await LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.' });
+
+ expect(mockCountTokens).toHaveBeenCalledWith({ prompt: 'Hello.' });
+ expect(log).toHaveBeenCalledWith(
+ '[reasoning] provider=local model=apple-fm elapsedMs=2000 outputTokens=50 tokensPerSecond=25.0',
+ );
+ });
+
+ it('marks the count as estimated where iOS cannot count tokens', async () => {
+ mockGenerateText.mockResolvedValue({ text: 'Hello there.', model: 'apple-foundation-models' });
+
+ await LocalReasoningService.processText({ text: 'um hello', systemPrompt: 'Clean up.' });
+
+ expect(log).toHaveBeenCalledWith(
+ '[reasoning] provider=local model=apple-fm elapsedMs=2000 outputTokens=4 (estimated) tokensPerSecond=2.0',
+ );
+ });
+});
diff --git a/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.ts b/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.ts
index 96bbfa884..ee21c26c6 100644
--- a/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.ts
+++ b/openwhispr-mobile/src/services/reasoning/__tests__/ReasoningService.localRouting.test.ts
@@ -102,6 +102,26 @@ describe('ReasoningService local routing', () => {
expect(mockPost).toHaveBeenCalledTimes(1);
});
+ it('does not fall back to cloud once the request was cancelled', async () => {
+ const controller = new AbortController();
+ const cancelled = new Error('cancelled');
+ mockLocalProcessText.mockImplementation(async () => {
+ controller.abort();
+ throw cancelled;
+ });
+
+ await expect(
+ ReasoningService.processText({
+ text: 'private note',
+ systemPrompt: 'format this',
+ signal: controller.signal,
+ routing: { isPrivateNote: true, allowCloudFallback: true },
+ }),
+ ).rejects.toBe(cancelled);
+
+ expect(mockPost).not.toHaveBeenCalled();
+ });
+
it('keeps public signed-in requests on the cloud path', async () => {
mockIsLocalRequired.mockReturnValue(false);
diff --git a/openwhispr-mobile/src/types/index.ts b/openwhispr-mobile/src/types/index.ts
index 3d4a404d3..6a21050d6 100644
--- a/openwhispr-mobile/src/types/index.ts
+++ b/openwhispr-mobile/src/types/index.ts
@@ -159,6 +159,7 @@ export type LocalReasoningErrorCode =
| 'APPLE_LLM_RATE_LIMITED'
| 'APPLE_LLM_UNAVAILABLE'
| 'APPLE_LLM_FAILED'
+ | 'APPLE_LLM_CANCELLED'
| 'LOCAL_CONTEXT_LIMIT'
| 'LOCAL_REASONING_UNAVAILABLE';