Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(mobile): 自动重试语音空结果并补全失败诊断
Dasha343102606d6
diff --git a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.tsindex 23dfbccd4a..21a30a9a30 100644--- a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts+++ b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts@@ -1,6 +1,9 @@import { beforeAll, describe, expect, it, vi } from 'vitest';import { DEFAULT_MOBILE_VOICE_LITELLM_BASE_URL } from '@/config/env';import { i18n } from '@/i18n';+import { setMobileDebugSink } from '@/debug/mobileDebugLog';+import { serializeMobileDebugRecord } from '@/debug/mobileDebugRecord';+import { VoiceInputController, VoiceTimelineLogger } from '@cindy/voice-input-core';const GW_PROXY = `${DEFAULT_MOBILE_VOICE_LITELLM_BASE_URL}/proxy`;const GW_PROXY_WSS = GW_PROXY.replace(/^https/, 'wss');@@ -65,7 +68,7 @@ async function waitForFakeSocketCount(count: number): Promise<void> {}}-function volcengineTranscriptPacket(text: string, definite = false): ArrayBuffer {+function volcengineTranscriptPacket(text: string, definite = false, final = false): ArrayBuffer {const payload = gzip(JSON.stringify({result: {text,@@ -74,10 +77,11 @@ function volcengineTranscriptPacket(text: string, definite = false): ArrayBuffer}));const size = new Uint8Array(4);new DataView(size.buffer).setUint32(0, payload.length, false);- const packet = new Uint8Array(8 + payload.length);- packet.set([0x11, 0x90, 0x11, 0x00], 0);- packet.set(size, 4);- packet.set(payload, 8);+ const packet = new Uint8Array((final ? 12 : 8) + payload.length);+ packet.set([0x11, final ? 0x93 : 0x90, 0x11, 0x00], 0);+ if (final) new DataView(packet.buffer).setInt32(4, -1, false);+ packet.set(size, final ? 8 : 4);+ packet.set(payload, final ? 12 : 8);return packet.buffer.slice(packet.byteOffset, packet.byteOffset + packet.byteLength) as ArrayBuffer;}@@ -179,6 +183,107 @@ function volcengineProvider(options: VolcengineProviderOptions): MobileVolcengin}describe('mobileRealtimeAsrProvider', () => {+ it('replays the captured PCM after an empty final response and submits the recovered transcript once', async () => {+ vi.useFakeTimers();+ FakeSocket.instances = [];+ const asr = volcengineProvider({+ credential: credential({ asr: {+ provider: 'litellm-volcengine-sauc-asr', model: 'volcengine-sauc-asr',+ auth: 'api-key', mode: 'provider-native-websocket', pcmSampleRate: 16000,+ resourceId: 'volc.seedasr.sauc.duration',+ } }),+ websocketFactory: FakeSocket, flushTimeoutMs: 100,+ });+ const submitted = vi.fn(() => undefined);+ const onError = vi.fn();+ const controller = new VoiceInputController({+ asr, retryEmptyTranscript: true, stableWaitMs: 1,+ logger: new VoiceTimelineLogger(),+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },+ });+ try {+ const starting = controller.start();+ await waitForFakeSocketCount(1);+ const first = FakeSocket.instances[0];+ first.open();+ await starting;+ controller.appendAudio(new Int16Array(1600).fill(1000).buffer);+ const stopping = controller.stop();+ first.rawMessage(volcengineTranscriptPacket('', false, true));+ await vi.advanceTimersByTimeAsync(1);+ expect(FakeSocket.instances).toHaveLength(2);+ const retry = FakeSocket.instances[1];+ retry.open();+ await vi.advanceTimersByTimeAsync(0);+ expect(retry.sent).toHaveLength(3); // initial request, original PCM, final silence+ expect(retry.sent[1]).toEqual(first.sent[1]);+ retry.rawMessage(volcengineTranscriptPacket('找回这句话', true, true));+ await stopping;+ expect(submitted).toHaveBeenCalledExactlyOnceWith('找回这句话', expect.any(Object));+ expect(onError).not.toHaveBeenCalled();+ expect(controller.currentState).toBe('done');+ expect(retry.readyState).toBe(3);+ } finally {+ await controller.cancel();+ vi.useRealTimers();+ }+ });++ it.each(['final_response', 'transport_closed', 'transport_error', 'server_error', 'timeout'] as const)(+ 'records why Volcengine finalization settled: %s, without response content', async (reason) => {+ vi.useFakeTimers();+ const records: string[] = [];+ setMobileDebugSink((level, scope, args) => records.push(serializeMobileDebugRecord(level, scope, args)));+ const provider = volcengineProvider({+ credential: credential({ asr: {+ provider: 'litellm-volcengine-sauc-asr', model: 'volcengine-sauc-asr',+ auth: 'api-key', mode: 'provider-native-websocket', pcmSampleRate: 16000,+ resourceId: 'volc.seedasr.sauc.duration',+ } }),+ websocketFactory: FakeSocket, flushTimeoutMs: 100,+ });+ try {+ FakeSocket.instances = [];+ const starting = provider.start();+ await waitForFakeSocketCount(1);+ const socket = FakeSocket.instances[0];+ socket.open();+ await starting;+ provider.appendAudio(new Int16Array(1600).fill(1000).buffer);+ const flushing = provider.flushAudio();+ if (reason === 'final_response') {+ socket.rawMessage(volcengineTranscriptPacket('', false, true));+ } else if (reason === 'transport_closed') {+ socket.onclose?.({ code: 1006, reason: 'PRIVATE_RESPONSE_CONTENT' });+ } else if (reason === 'transport_error') {+ socket.onerror?.({ message: 'PRIVATE_RESPONSE_CONTENT' });+ } else if (reason === 'server_error') {+ const payload = new TextEncoder().encode('PRIVATE_RESPONSE_CONTENT');+ const packet = new Uint8Array(12 + payload.length);+ packet.set([0x11, 0xf0, 0, 0]);+ const view = new DataView(packet.buffer);+ view.setUint32(4, 45000081);+ view.setUint32(8, payload.length);+ packet.set(payload, 12);+ socket.rawMessage(packet.buffer);+ } else await vi.advanceTimersByTimeAsync(100);+ await flushing;+ const events = records.map((line) => JSON.parse(line));+ expect(events.find((event) => event.args[0] === 'volcengine sauc flush settled')?.args[1])+ .toMatchObject({ reason, hasStable: false, transcriptChars: 0 });+ if (reason === 'server_error') {+ expect(events.find((event) => event.args[0] === 'volcengine sauc server error')?.args[1])+ .toMatchObject({ code: 45000081 });+ }+ expect(records.join('\n')).not.toContain('PRIVATE_RESPONSE_CONTENT');+ } finally {+ await provider.stop();+ setMobileDebugSink(undefined);+ vi.useRealTimers();+ }+ },+ );+it('cancels an in-flight fallback candidate before it can open a managed socket', async () => {FakeSocket.instances = [];let resolveConnection: ((value: { websocketUrl: string; authorizationToken: string }) => void) | undefined;diff --git a/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.ts b/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.tsindex 94b8d0989c..b960081d32 100644--- a/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.ts+++ b/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.ts@@ -89,6 +89,7 @@ beforeEach(() => {afterEach(() => {setMobileDebugSink(undefined);vi.restoreAllMocks();+ vi.useRealTimers();});const messages = () =>@@ -97,6 +98,51 @@ const messages = () =>);describe('mobile voice diagnostics', () => {+ it.each(['text', 'empty', 'rate_limit'] as const)('recovers an empty mobile recording and diagnoses the result: %s', async (outcome) => {+ vi.useFakeTimers();+ let emit: (event: AsrEvent) => void = () => {};+ let recovered = false;+ const recover = vi.fn(async () => {+ if (outcome === 'rate_limit') throw new ApiError('RATE_LIMITED', 429, 'PRIVATE_SERVER_MESSAGE');+ recovered = true;+ });+ const onError = vi.fn();+ const onDraftChanged = vi.fn();+ const session = createMobileVoiceControllerSession({+ credential: credential(), initialDraft: '', startAudio: startAudibleAudio,+ refiner: null,+ asr: {+ onEvent(callback) { emit = callback; },+ async start() { emit({ type: 'connected', at: Date.now() }); },+ async stop() {}, appendAudio() {}, recover,+ async flushAudio() {+ if (recovered && outcome === 'text') emit({ type: 'stable', text: SPOKEN, at: Date.now() });+ },+ },+ onError, onDraftChanged,+ });+ await session.start();+ const stopping = session.stop().then(() => undefined, (error: unknown) => error);+ await vi.advanceTimersByTimeAsync(1100);+ const stopError = await stopping;+ expect(recover).toHaveBeenCalledTimes(1);+ const logged = messages();+ expect(logged.find((record) => record.args[0] === 'asr recovery attempted')?.args[1])+ .toMatchObject({ trigger: 'empty_transcript', attempt: 1 });+ if (outcome === 'text') {+ expect(stopError).toBeUndefined();+ expect(onError).not.toHaveBeenCalled();+ expect(JSON.stringify(onDraftChanged.mock.calls)).toContain(SPOKEN);+ } else {+ expect(stopError).toBeInstanceOf(Error);+ expect(onError).toHaveBeenCalledTimes(1);+ expect(logged.find((record) => record.args[0] === 'run failed')?.args[1])+ .toMatchObject({ reason: outcome === 'empty' ? 'empty_transcript' : 'rate_limited' });+ }+ expect(records.join('\n')).not.toContain(SPOKEN);+ expect(records.join('\n')).not.toContain('PRIVATE_SERVER_MESSAGE');+ });+it('records the run stages and summaries in the voice scope without any dictated text', async () => {const session = createMobileVoiceControllerSession({credential: credential(),diff --git a/apps/mobile/src/i18n/locales/en/composer.json b/apps/mobile/src/i18n/locales/en/composer.jsonindex a95532e346..1bb3cd2443 100644--- a/apps/mobile/src/i18n/locales/en/composer.json+++ b/apps/mobile/src/i18n/locales/en/composer.json@@ -98,7 +98,7 @@"sessionExpiredOrForbidden": "Your {{appName}} voice sign-in has expired or lacks permission. Check your sign-in status and try again (status {{status}}).","incomplete": "Voice input didn't finish. Try again.","missingRefinerTargetProvider": "Voice refinement is unavailable. Sign in to {{appName}} again and retry.",- "emptyTranscript": "Couldn't hear you clearly. Check your microphone or move closer and try again.",+ "emptyTranscript": "Speech recognition returned no text. Please record again.","connectionInterrupted": "The voice connection was interrupted. Check your network and try again.","recognitionStalled": "Voice recognition stopped returning results. Try again.","transcriptKept": "{{message}} The text recognized so far was kept — you can edit it directly.",diff --git a/apps/mobile/src/i18n/locales/ja/composer.json b/apps/mobile/src/i18n/locales/ja/composer.jsonindex 107eccc8ef..a0b88a824c 100644--- a/apps/mobile/src/i18n/locales/ja/composer.json+++ b/apps/mobile/src/i18n/locales/ja/composer.json@@ -98,7 +98,7 @@"sessionExpiredOrForbidden": "{{appName}} の音声サービスのログインが失効したか、権限がありません。ログイン状態を確認してからやり直してください(ステータス {{status}})。","incomplete": "音声入力が完了しませんでした。もう一度お試しください。","missingRefinerTargetProvider": "音声の推敲を利用できません。{{appName}} にもう一度ログインしてから再試行してください。",- "emptyTranscript": "聞き取れませんでした。マイクを確認するか、近づいてもう一度お試しください。",+ "emptyTranscript": "音声認識でテキストを取得できませんでした。もう一度録音してください。","connectionInterrupted": "音声接続が中断されました。ネットワークを確認してからやり直してください。","recognitionStalled": "音声認識の結果が返ってこなくなりました。もう一度お試しください。","transcriptKept": "{{message}}(ここまで認識したテキストは保持しており、そのまま編集できます)",diff --git a/apps/mobile/src/i18n/locales/ko/composer.json b/apps/mobile/src/i18n/locales/ko/composer.jsonindex 9df2bb43ad..3ed2cf3530 100644--- a/apps/mobile/src/i18n/locales/ko/composer.json+++ b/apps/mobile/src/i18n/locales/ko/composer.json@@ -98,7 +98,7 @@"sessionExpiredOrForbidden": "{{appName}} 음성 서비스 로그인이 만료되었거나 권한이 없습니다. 로그인 상태를 확인한 후 다시 시도하세요(상태 코드 {{status}}).","incomplete": "음성 입력이 완료되지 않았습니다. 다시 시도하세요.","missingRefinerTargetProvider": "음성 다듬기를 사용할 수 없습니다. {{appName}}에 다시 로그인한 후 다시 시도하세요.",- "emptyTranscript": "잘 듣지 못했습니다. 마이크를 확인하거나 더 가까이에서 다시 시도해 주세요.",+ "emptyTranscript": "음성 인식 결과가 비어 있습니다. 다시 녹음해 주세요.","connectionInterrupted": "음성 연결이 중단되었습니다. 네트워크를 확인한 뒤 다시 시도하세요.","recognitionStalled": "음성 인식 결과가 더 이상 오지 않습니다. 다시 시도하세요.","transcriptKept": "{{message}} 지금까지 인식된 텍스트는 유지했습니다. 바로 이어서 편집할 수 있습니다.",diff --git a/apps/mobile/src/i18n/locales/zh-CN/composer.json b/apps/mobile/src/i18n/locales/zh-CN/composer.jsonindex 5872b6957e..4867fbcab2 100644--- a/apps/mobile/src/i18n/locales/zh-CN/composer.json+++ b/apps/mobile/src/i18n/locales/zh-CN/composer.json@@ -98,7 +98,7 @@"sessionExpiredOrForbidden": "{{appName}} 语音服务的登录已失效或没有权限,请确认登录状态后重试(状态码 {{status}})。","incomplete": "语音输入未完成,请重试。","missingRefinerTargetProvider": "语音润色暂不可用,请重新登录 {{appName}} 后重试。",- "emptyTranscript": "没听清,请检查麦克风或靠近一点再试。",+ "emptyTranscript": "语音识别未返回文字,请重新录音。","connectionInterrupted": "语音连接中断了,请检查网络后重试。","recognitionStalled": "语音识别没有继续返回结果,请重试。","transcriptKept": "{{message}}(识别出的文字已经保留,可以直接编辑)",diff --git a/apps/mobile/src/i18n/locales/zh-TW/composer.json b/apps/mobile/src/i18n/locales/zh-TW/composer.jsonindex f6e4df3991..c10bad7fb0 100644--- a/apps/mobile/src/i18n/locales/zh-TW/composer.json+++ b/apps/mobile/src/i18n/locales/zh-TW/composer.json@@ -98,7 +98,7 @@"sessionExpiredOrForbidden": "{{appName}} 語音服務的登入已失效或沒有權限,請確認登入狀態後重試(狀態碼 {{status}})。","incomplete": "語音輸入未完成,請重試。","missingRefinerTargetProvider": "語音潤色暫時無法使用,請重新登入 {{appName}} 後重試。",- "emptyTranscript": "沒聽清楚,請檢查麥克風或靠近一點再試。",+ "emptyTranscript": "語音辨識未回傳文字,請重新錄音。","connectionInterrupted": "語音連線中斷了,請檢查網路後重試。","recognitionStalled": "語音識別沒有繼續返回結果,請重試。","transcriptKept": "{{message}}(識別出的文字已經保留,可以直接編輯)",diff --git a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts b/apps/mobile/src/session/mobileRealtimeAsrProvider.tsindex c3bc4482dd..33b2d10bb9 100644--- a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts+++ b/apps/mobile/src/session/mobileRealtimeAsrProvider.ts@@ -795,6 +795,8 @@ export class MobileRealtimeAsrProvider implements AsrProvider {}}+type VolcengineFlushReason = 'final_response' | 'transport_closed' | 'transport_error' | 'server_error' | 'timeout' | 'stopped' | 'reset';+export class MobileVolcengineSaucAsrProvider implements AsrProvider {private readonly credential: StoredMobileVoiceCredential;private readonly sourceLanguage: string;@@ -822,7 +824,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {private confirmedEndMs = 0;private readonly soundActivity: StopSoundActivity;private recovered = false;- private flushResolvers: Array<() => void> = [];+ private flushResolvers: Array<(reason: VolcengineFlushReason) => void> = [];private stopped = false;private recoveryPromise?: Promise<void>;private pendingConnect?: PendingConnect;@@ -933,14 +935,18 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {}} else {this.connected = false;- this.resolveFlushWaiters();+ this.resolveFlushWaiters('transport_error');this.callback({ type: 'error', message: redacted, at: Date.now() });}};socket.onclose = (event) => {if (this.socket !== socket) return;this.connected = false;- this.resolveFlushWaiters();+ this.resolveFlushWaiters('transport_closed');+ logMobileVoice('debug', 'volcengine sauc transport closed', {+ code: typeof event.code === 'number' ? event.code : undefined,+ finalRequested: this.finalRequested,+ });if (!settled) {fail(new Error(this.redactError(webSocketCloseBeforeReadyMessage(event,@@ -988,10 +994,12 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {logMobileVoice('debug', 'volcengine sauc stop finalization decision', decision);if (decision.fastFinish) return;const flushStartedAt = Date.now();- await this.waitForFlush();+ const reason = await this.waitForFlush();logMobileVoice('debug', 'volcengine sauc flush settled', {waitMs: Date.now() - flushStartedAt,hasStable: this.stableEmitted,+ transcriptChars: this.lastTranscript.length,+ reason,});if (this.lastTranscript && !this.stableEmitted) {this.finalizedTranscript = this.lastTranscript;@@ -1003,7 +1011,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {async stop(): Promise<void> {this.stopped = true;this.connected = false;- this.resolveFlushWaiters();+ this.resolveFlushWaiters('stopped');this.clearUnconfirmedAudio();this.sessionTranscriptPrefix = '';const socket = this.socket;@@ -1062,7 +1070,11 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {extractErrorMessage(message.payload) ?? message.payloadText ?? 'Volcengine SAUC ASR failed.',);this.callback({ type: 'error', message: errorMessage, at: Date.now() });- this.resolveFlushWaiters();+ this.resolveFlushWaiters('server_error');+ const code = (message.payload as { code?: unknown } | undefined)?.code;+ logMobileVoice('warn', 'volcengine sauc server error', {+ code: typeof code === 'number' ? code : undefined,+ });return;}if (message.messageType !== VOLC_MESSAGE_TYPE_FULL_SERVER_RESPONSE && message.messageType !== VOLC_MESSAGE_TYPE_SERVER_ACK) {@@ -1093,7 +1105,14 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {this.stableEmitted = isDefinite;this.callback({ type: isDefinite ? 'stable' : 'partial', text: transcript, at: Date.now() });}- if (this.finalRequested && isLastResponse) this.resolveFlushWaiters();+ if (isLastResponse) {+ logMobileVoice('debug', 'volcengine sauc final response', {+ transcriptChars: rawTranscript.length,+ finalRequested: this.finalRequested,+ messageType: message.messageType,+ });+ if (this.finalRequested) this.resolveFlushWaiters('final_response');+ }}/**@@ -1212,7 +1231,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {this.sequence = 1;this.finalRequested = false;this.stableEmitted = false;- this.resolveFlushWaiters();+ this.resolveFlushWaiters('reset');const socket = this.socket;this.socket = null;@@ -1242,18 +1261,20 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {return socket && socket.readyState === OPEN ? socket : null;}- private waitForFlush(): Promise<void> {+ private waitForFlush(): Promise<VolcengineFlushReason> {return new Promise((resolve) => {- const timer = setTimeout(resolve, this.flushTimeoutMs);- this.flushResolvers.push(() => {+ const finish = (reason: VolcengineFlushReason): void => {clearTimeout(timer);- resolve();- });+ this.flushResolvers = this.flushResolvers.filter((item) => item !== finish);+ resolve(reason);+ };+ const timer = setTimeout(() => finish('timeout'), this.flushTimeoutMs);+ this.flushResolvers.push(finish);});}- private resolveFlushWaiters(): void {- this.flushResolvers.splice(0).forEach((resolve) => resolve());+ private resolveFlushWaiters(reason: VolcengineFlushReason): void {+ this.flushResolvers.splice(0).forEach((resolve) => resolve(reason));}private resetState(): void {@@ -1270,7 +1291,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {this.clearUnconfirmedAudio();this.finalRequested = false;this.stableEmitted = false;- this.resolveFlushWaiters();+ this.resolveFlushWaiters('reset');}private redactError(message: string): string {diff --git a/apps/mobile/src/session/mobileVoiceController.ts b/apps/mobile/src/session/mobileVoiceController.tsindex 1e98bc850a..7f649b16b4 100644--- a/apps/mobile/src/session/mobileVoiceController.ts+++ b/apps/mobile/src/session/mobileVoiceController.ts@@ -395,6 +395,7 @@ export function createMobileVoiceControllerSession(// Refine during a speech pause and show the result right away. The shared// controller only acts on it when a refiner exists (refinement enabled).pauseRefinementEnabled: true,+ retryEmptyTranscript: true,// A reconnect allocates another managed session and can hit the account// limit; report that instead of the generic "stopped receiving" failure.recoveryErrorMessage: mobileVoiceRateLimitMessage,diff --git a/apps/mobile/src/session/mobileVoiceDiagnostics.ts b/apps/mobile/src/session/mobileVoiceDiagnostics.tsindex 2cfd0f68de..dc1b8fde33 100644--- a/apps/mobile/src/session/mobileVoiceDiagnostics.ts+++ b/apps/mobile/src/session/mobileVoiceDiagnostics.ts@@ -40,6 +40,7 @@ function rejectionReason(reason: string): string {}const FAILURE_PATTERNS: Array<[RegExp, string]> = [+ [/^empty_transcript$/, 'empty_transcript'],[/不可优化|次数已用完/, 'refine_quota_exhausted'],[/rate.?limit|too many|频繁|\b429\b/i, 'rate_limited'],[/timed? ?out|timeout|超时/i, 'timeout'],@@ -263,7 +264,7 @@ export function createMobileVoiceTimelineRecorder(context: {case 'error':logMobileVoice('error', 'run failed', {runId,- reason: classifyMobileVoiceFailure(event.message),+ reason: event.code ?? classifyMobileVoiceFailure(event.message),});return;}diff --git a/packages/voice-input-core/src/VoiceInputController.ts b/packages/voice-input-core/src/VoiceInputController.tsindex 0542fb1bfb..b7cf4ff61b 100644--- a/packages/voice-input-core/src/VoiceInputController.ts+++ b/packages/voice-input-core/src/VoiceInputController.ts@@ -44,6 +44,8 @@ type VoiceInputControllerOptions = {stableWaitMs?: number;/** Host opt-in: refine during a speech pause and publish a live draft. */pauseRefinementEnabled?: boolean;+ /** Replay retained audio once after an audible recording ends without text. */+ retryEmptyTranscript?: boolean;/** Host opt-in for known recovery failures; undefined keeps the generic error. */recoveryErrorMessage?: (error: unknown) => string | undefined;/**@@ -100,6 +102,7 @@ export class VoiceInputController {private readonly callbacks: VoiceInputCallbacks;private readonly stableWaitMs: number;private readonly pauseRefinementEnabled: boolean;+ private readonly retryEmptyTranscript: boolean;private readonly recoveryErrorMessage?: (error: unknown) => string | undefined;private readonly audioGate?: AudioGate;private lastSoundAt = 0;@@ -167,6 +170,7 @@ export class VoiceInputController {this.callbacks = options.callbacks;this.stableWaitMs = options.stableWaitMs ?? 500;this.pauseRefinementEnabled = options.pauseRefinementEnabled ?? false;+ this.retryEmptyTranscript = options.retryEmptyTranscript ?? false;this.recoveryErrorMessage = options.recoveryErrorMessage;this.audioGate = options.audioGate;@@ -274,16 +278,32 @@ export class VoiceInputController {if (this.audioGate) await this.audioGate.finish().catch(() => undefined);if (runId !== this.runId || this.leftSubmittingState()) return;const optimisticRefinement = this.startOptimisticRefinement(runId);+ const interrupted = (): boolean => {+ if (runId !== this.runId) return true;+ if (!this.currentRunCancelled) return false;+ this.discardRefinement(runId, optimisticRefinement, 'cancelled');+ return true;+ };+ let stable: string | undefined;try {await this.asr.flushAudio();+ if (interrupted()) return;+ if (!this.leftSubmittingState()) stable = await this.waitForStable(this.stableWaitMs);+ if (interrupted()) return;+ if (!this.leftSubmittingState() && !normalizeSubmittedText(stable || this.latestTranscript)+ && this.speechActivitySeen && this.retryEmptyTranscript && this.asr.recover+ && !this.networkRecoveryInFlight) {+ stable = await this.retryEmptyRecording(runId);+ }} catch (error) {- this.fail(error instanceof Error ? error.message : String(error));+ if (interrupted()) return;+ this.fail(this.recoveryErrorMessage?.(error) ?? (error instanceof Error ? error.message : String(error)));await this.asr.stop();return;}- const stable = await this.waitForStable(this.stableWaitMs);+ if (interrupted()) return;const text = normalizeSubmittedText(stable || this.latestTranscript);const source = stable ? 'stable' : this.latestTranscriptSource;@@ -299,6 +319,7 @@ export class VoiceInputController {// transcriptEmitted is checked too: it is the direct invariant ("this run has// already given the host its text"), so it holds even if a future failure// path leaves the state alone.+ if (runId !== this.runId) return;if (this.leftSubmittingState() || this.transcriptEmitted) {// cancel() also lands here (it moves the run to 'done'); logging that as// a failure would misread the timeline for an intentional abort.@@ -367,6 +388,41 @@ export class VoiceInputController {}}+ private async retryEmptyRecording(runId: string): Promise<string | undefined> {+ const startedAt = performance.now();+ this.logger.record({+ type: 'asr_recovery_attempted', runId, at: Date.now(), trigger: 'empty_transcript', attempt: 1,+ audioMsSinceLastSignal: this.audioMsSinceLastSignal,+ voicedAudioMsSinceLastSignal: this.voicedAudioMsSinceLastSignal,+ wallMsSinceLastSignal: startedAt - this.lastAsrSignalAt,+ everSawSignal: this.everSawAsrSignal,+ });+ try {+ // recover() owns replay of unconfirmed in-memory audio. stop() would+ // clear that audio, so only tear the provider down after this one attempt.+ await this.asr.recover!();+ if (runId !== this.runId || this.leftSubmittingState()) return;+ await this.asr.flushAudio();+ if (runId !== this.runId || this.leftSubmittingState()) return;+ const stable = await this.waitForStable(this.stableWaitMs);+ if (runId !== this.runId || this.leftSubmittingState()) return;+ const elapsedMs = performance.now() - startedAt;+ if (normalizeSubmittedText(stable || this.latestTranscript)) {+ this.logger.record({ type: 'asr_recovery_succeeded', runId, at: Date.now(), elapsedMs });+ } else {+ this.logger.record({ type: 'asr_recovery_failed', runId, at: Date.now(), elapsedMs, reason: 'empty_transcript' });+ }+ return stable;+ } catch (error) {+ if (runId !== this.runId || this.leftSubmittingState()) return;+ this.logger.record({+ type: 'asr_recovery_failed', runId, at: Date.now(), elapsedMs: performance.now() - startedAt,+ reason: error instanceof Error ? error.message : String(error),+ });+ throw error;+ }+ }+/*** True when something moved this run out of the state stop() set on entry —* fail() from a rejected recovery, or cancel(). Deliberately a method: inside@@ -748,7 +804,7 @@ export class VoiceInputController {if (this.state === 'error' || this.state === 'done') return;this.audioGate?.abort();if (this.runId) {- this.logger.record({ type: 'error', runId: this.runId, at: Date.now(), message });+ this.logger.record({ type: 'error', runId: this.runId, at: Date.now(), message, code });}this.stopStallWatchdog();this.pendingStableResolvers.splice(0).forEach((resolve) => resolve(undefined));diff --git a/packages/voice-input-core/src/__tests__/VoiceInputController.test.ts b/packages/voice-input-core/src/__tests__/VoiceInputController.test.tsindex d5e019f0ad..fee2fa064d 100644--- a/packages/voice-input-core/src/__tests__/VoiceInputController.test.ts+++ b/packages/voice-input-core/src/__tests__/VoiceInputController.test.ts@@ -1,4 +1,4 @@-import { describe, expect, it, vi } from 'vitest';+import { afterEach, describe, expect, it, vi } from 'vitest';import { VoiceInputController } from '../VoiceInputController.js';import { VoiceTimelineLogger } from '../VoiceTimelineLogger.js';@@ -71,6 +71,94 @@ function pcmChunk(amplitude: number): ArrayBuffer {}describe('VoiceInputController', () => {+ afterEach(() => vi.useRealTimers());++ it.each(['silent', 'partial', 'opted_out', 'provider_error'] as const)('does not replay a recording when %s', async (scenario) => {+ const asr = new FakeAsrProvider();+ asr.recover = vi.fn(async () => {});+ if (scenario === 'provider_error') {+ asr.flush = async () => asr.emit({ type: 'error', message: 'permission denied', at: Date.now() });+ }+ const submitted = vi.fn(acceptSubmission);+ const controller = new VoiceInputController({+ asr, retryEmptyTranscript: scenario !== 'opted_out', stableWaitMs: 0,+ logger: new VoiceTimelineLogger(),+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted },+ });+ await controller.start();+ controller.appendAudio(pcmChunk(scenario === 'silent' ? 0 : 1000));+ if (scenario === 'partial') asr.emit({ type: 'partial', text: 'Already recognized', at: Date.now() });+ await controller.stop();+ expect(asr.recover).not.toHaveBeenCalled();+ expect(submitted).toHaveBeenCalledTimes(scenario === 'partial' ? 1 : 0);+ expect(controller.currentState).toBe(['silent', 'partial'].includes(scenario) ? 'done' : 'error');+ });++ it.each([true, false])('replays an audible empty recording once (recovered text: %s)', async (returnsText) => {+ const asr = new FakeAsrProvider();+ const submitted = vi.fn(acceptSubmission);+ const onError = vi.fn();+ const events: VoiceTimelineEvent[] = [];+ asr.recover = vi.fn(async () => {+ if (returnsText) asr.emit({ type: 'stable', text: 'Recovered words', at: Date.now() });+ });+ const flush = vi.spyOn(asr, 'flushAudio');+ const controller = new VoiceInputController({+ asr, retryEmptyTranscript: true, stableWaitMs: 0,+ logger: new VoiceTimelineLogger((event) => events.push(event)),+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },+ });+ await controller.start();+ controller.appendAudio(pcmChunk(1000));+ await controller.stop();+ expect(asr.recover).toHaveBeenCalledTimes(1);+ expect(flush).toHaveBeenCalledTimes(2);+ expect(events).toContainEqual(expect.objectContaining({ type: 'asr_recovery_attempted', trigger: 'empty_transcript' }));+ if (returnsText) {+ expect(submitted).toHaveBeenCalledTimes(1);+ expect(submitted.mock.calls[0][0]).toBe('Recovered words');+ expect(onError).not.toHaveBeenCalled();+ expect(controller.currentState).toBe('done');+ } else {+ expect(submitted).not.toHaveBeenCalled();+ expect(onError).toHaveBeenCalledWith(expect.any(String), 'empty_transcript', { transcriptKept: false });+ expect(events).toContainEqual(expect.objectContaining({ type: 'error', code: 'empty_transcript' }));+ }+ });++ it.each([false, true])('ignores an empty-result retry settling after cancellation and restart (reject: %s)', async (rejects) => {+ vi.useFakeTimers();+ let settle!: () => void;+ const asr = new FakeAsrProvider();+ asr.recover = () => new Promise<void>((resolve, reject) => {+ settle = () => rejects ? reject(new Error('old recovery failed')) : resolve();+ });+ const flush = vi.spyOn(asr, 'flushAudio');+ const stop = vi.spyOn(asr, 'stop');+ const onError = vi.fn();+ const submitted = vi.fn(acceptSubmission);+ const controller = new VoiceInputController({+ asr, retryEmptyTranscript: true, stableWaitMs: 0,+ logger: new VoiceTimelineLogger(),+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },+ });+ await controller.start();+ controller.appendAudio(pcmChunk(1000));+ const stopping = controller.stop();+ await vi.advanceTimersByTimeAsync(0);+ expect(settle).toBeTypeOf('function');+ await controller.cancel();+ await controller.start();+ settle();+ await stopping;+ expect(flush).toHaveBeenCalledTimes(1);+ expect(stop).toHaveBeenCalledTimes(1);+ expect(onError).not.toHaveBeenCalled();+ expect(submitted).not.toHaveBeenCalled();+ expect(controller.currentState).toBe('listening');+ await controller.cancel();+ });+it('does not flush or stop a new run after cancellation during gate finish', async () => {let release!: () => void;const asr = new FakeAsrProvider();diff --git a/packages/voice-input-core/src/types.ts b/packages/voice-input-core/src/types.tsindex 7a8f8da7bc..7921dafc54 100644--- a/packages/voice-input-core/src/types.ts+++ b/packages/voice-input-core/src/types.ts@@ -88,7 +88,8 @@ export interface AsrProvider {flushAudio(): Promise<void>;onEvent(callback: (event: AsrEvent) => void): void;/**- * Best-effort reconnect for explicit transport failures. The controller does+ * Best-effort reconnect for explicit transport failures or, when opted in,+ * an audible recording that ends without any transcript. The controller does* not call this for ASR partial gaps: some realtime providers can legitimately* pause deltas and still return a complete transcript on flush. A provider* implementation should close the dead transport, re-establish the session,@@ -240,7 +241,7 @@ export type VoiceTimelineEvent =type: 'asr_recovery_attempted';runId: string;at: number;- trigger: 'error' | 'disconnected';+ trigger: 'error' | 'disconnected' | 'empty_transcript';attempt: number;audioMsSinceLastSignal: number;voicedAudioMsSinceLastSignal: number;@@ -264,7 +265,7 @@ export type VoiceTimelineEvent =source: 'stable' | 'partial';accepted: boolean;}- | { type: 'error'; runId: string; at: number; message: string };+ | { type: 'error'; runId: string; at: number; message: string; code?: VoiceInputErrorCode };export type VoiceInputCallbacks = {onStateChanged?: (state: VoiceInputState, outcome?: VoiceInputTerminalOutcome) => void;