Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(mobile): 自动重试语音空结果并补全失败诊断

Dasha343102606d6
diff --git a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts
index 23dfbccd4a..21a30a9a30 100644
--- a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts
+++ b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts
@@ -1,6 +1,9 @@
import { beforeAll, describe, expect, it, vi } from 'vitest';
import { DEFAULT_MOBILE_VOICE_LITELLM_BASE_URL } from '@/config/env';
import { i18n } from '@/i18n';
+import { setMobileDebugSink } from '@/debug/mobileDebugLog';
+import { serializeMobileDebugRecord } from '@/debug/mobileDebugRecord';
+import { VoiceInputController, VoiceTimelineLogger } from '@cindy/voice-input-core';
const GW_PROXY = `${DEFAULT_MOBILE_VOICE_LITELLM_BASE_URL}/proxy`;
const GW_PROXY_WSS = GW_PROXY.replace(/^https/, 'wss');
@@ -65,7 +68,7 @@ async function waitForFakeSocketCount(count: number): Promise<void> {
}
}
-function volcengineTranscriptPacket(text: string, definite = false): ArrayBuffer {
+function volcengineTranscriptPacket(text: string, definite = false, final = false): ArrayBuffer {
const payload = gzip(JSON.stringify({
result: {
text,
@@ -74,10 +77,11 @@ function volcengineTranscriptPacket(text: string, definite = false): ArrayBuffer
}));
const size = new Uint8Array(4);
new DataView(size.buffer).setUint32(0, payload.length, false);
- const packet = new Uint8Array(8 + payload.length);
- packet.set([0x11, 0x90, 0x11, 0x00], 0);
- packet.set(size, 4);
- packet.set(payload, 8);
+ const packet = new Uint8Array((final ? 12 : 8) + payload.length);
+ packet.set([0x11, final ? 0x93 : 0x90, 0x11, 0x00], 0);
+ if (final) new DataView(packet.buffer).setInt32(4, -1, false);
+ packet.set(size, final ? 8 : 4);
+ packet.set(payload, final ? 12 : 8);
return packet.buffer.slice(packet.byteOffset, packet.byteOffset + packet.byteLength) as ArrayBuffer;
}
@@ -179,6 +183,107 @@ function volcengineProvider(options: VolcengineProviderOptions): MobileVolcengin
}
describe('mobileRealtimeAsrProvider', () => {
+ it('replays the captured PCM after an empty final response and submits the recovered transcript once', async () => {
+ vi.useFakeTimers();
+ FakeSocket.instances = [];
+ const asr = volcengineProvider({
+ credential: credential({ asr: {
+ provider: 'litellm-volcengine-sauc-asr', model: 'volcengine-sauc-asr',
+ auth: 'api-key', mode: 'provider-native-websocket', pcmSampleRate: 16000,
+ resourceId: 'volc.seedasr.sauc.duration',
+ } }),
+ websocketFactory: FakeSocket, flushTimeoutMs: 100,
+ });
+ const submitted = vi.fn(() => undefined);
+ const onError = vi.fn();
+ const controller = new VoiceInputController({
+ asr, retryEmptyTranscript: true, stableWaitMs: 1,
+ logger: new VoiceTimelineLogger(),
+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },
+ });
+ try {
+ const starting = controller.start();
+ await waitForFakeSocketCount(1);
+ const first = FakeSocket.instances[0];
+ first.open();
+ await starting;
+ controller.appendAudio(new Int16Array(1600).fill(1000).buffer);
+ const stopping = controller.stop();
+ first.rawMessage(volcengineTranscriptPacket('', false, true));
+ await vi.advanceTimersByTimeAsync(1);
+ expect(FakeSocket.instances).toHaveLength(2);
+ const retry = FakeSocket.instances[1];
+ retry.open();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(retry.sent).toHaveLength(3); // initial request, original PCM, final silence
+ expect(retry.sent[1]).toEqual(first.sent[1]);
+ retry.rawMessage(volcengineTranscriptPacket('找回这句话', true, true));
+ await stopping;
+ expect(submitted).toHaveBeenCalledExactlyOnceWith('找回这句话', expect.any(Object));
+ expect(onError).not.toHaveBeenCalled();
+ expect(controller.currentState).toBe('done');
+ expect(retry.readyState).toBe(3);
+ } finally {
+ await controller.cancel();
+ vi.useRealTimers();
+ }
+ });
+
+ it.each(['final_response', 'transport_closed', 'transport_error', 'server_error', 'timeout'] as const)(
+ 'records why Volcengine finalization settled: %s, without response content', async (reason) => {
+ vi.useFakeTimers();
+ const records: string[] = [];
+ setMobileDebugSink((level, scope, args) => records.push(serializeMobileDebugRecord(level, scope, args)));
+ const provider = volcengineProvider({
+ credential: credential({ asr: {
+ provider: 'litellm-volcengine-sauc-asr', model: 'volcengine-sauc-asr',
+ auth: 'api-key', mode: 'provider-native-websocket', pcmSampleRate: 16000,
+ resourceId: 'volc.seedasr.sauc.duration',
+ } }),
+ websocketFactory: FakeSocket, flushTimeoutMs: 100,
+ });
+ try {
+ FakeSocket.instances = [];
+ const starting = provider.start();
+ await waitForFakeSocketCount(1);
+ const socket = FakeSocket.instances[0];
+ socket.open();
+ await starting;
+ provider.appendAudio(new Int16Array(1600).fill(1000).buffer);
+ const flushing = provider.flushAudio();
+ if (reason === 'final_response') {
+ socket.rawMessage(volcengineTranscriptPacket('', false, true));
+ } else if (reason === 'transport_closed') {
+ socket.onclose?.({ code: 1006, reason: 'PRIVATE_RESPONSE_CONTENT' });
+ } else if (reason === 'transport_error') {
+ socket.onerror?.({ message: 'PRIVATE_RESPONSE_CONTENT' });
+ } else if (reason === 'server_error') {
+ const payload = new TextEncoder().encode('PRIVATE_RESPONSE_CONTENT');
+ const packet = new Uint8Array(12 + payload.length);
+ packet.set([0x11, 0xf0, 0, 0]);
+ const view = new DataView(packet.buffer);
+ view.setUint32(4, 45000081);
+ view.setUint32(8, payload.length);
+ packet.set(payload, 12);
+ socket.rawMessage(packet.buffer);
+ } else await vi.advanceTimersByTimeAsync(100);
+ await flushing;
+ const events = records.map((line) => JSON.parse(line));
+ expect(events.find((event) => event.args[0] === 'volcengine sauc flush settled')?.args[1])
+ .toMatchObject({ reason, hasStable: false, transcriptChars: 0 });
+ if (reason === 'server_error') {
+ expect(events.find((event) => event.args[0] === 'volcengine sauc server error')?.args[1])
+ .toMatchObject({ code: 45000081 });
+ }
+ expect(records.join('\n')).not.toContain('PRIVATE_RESPONSE_CONTENT');
+ } finally {
+ await provider.stop();
+ setMobileDebugSink(undefined);
+ vi.useRealTimers();
+ }
+ },
+ );
+
it('cancels an in-flight fallback candidate before it can open a managed socket', async () => {
FakeSocket.instances = [];
let resolveConnection: ((value: { websocketUrl: string; authorizationToken: string }) => void) | undefined;
diff --git a/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.ts b/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.ts
index 94b8d0989c..b960081d32 100644
--- a/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.ts
+++ b/apps/mobile/src/__tests__/mobileVoiceDiagnostics.test.ts
@@ -89,6 +89,7 @@ beforeEach(() => {
afterEach(() => {
setMobileDebugSink(undefined);
vi.restoreAllMocks();
+ vi.useRealTimers();
});
const messages = () =>
@@ -97,6 +98,51 @@ const messages = () =>
);
describe('mobile voice diagnostics', () => {
+ it.each(['text', 'empty', 'rate_limit'] as const)('recovers an empty mobile recording and diagnoses the result: %s', async (outcome) => {
+ vi.useFakeTimers();
+ let emit: (event: AsrEvent) => void = () => {};
+ let recovered = false;
+ const recover = vi.fn(async () => {
+ if (outcome === 'rate_limit') throw new ApiError('RATE_LIMITED', 429, 'PRIVATE_SERVER_MESSAGE');
+ recovered = true;
+ });
+ const onError = vi.fn();
+ const onDraftChanged = vi.fn();
+ const session = createMobileVoiceControllerSession({
+ credential: credential(), initialDraft: '', startAudio: startAudibleAudio,
+ refiner: null,
+ asr: {
+ onEvent(callback) { emit = callback; },
+ async start() { emit({ type: 'connected', at: Date.now() }); },
+ async stop() {}, appendAudio() {}, recover,
+ async flushAudio() {
+ if (recovered && outcome === 'text') emit({ type: 'stable', text: SPOKEN, at: Date.now() });
+ },
+ },
+ onError, onDraftChanged,
+ });
+ await session.start();
+ const stopping = session.stop().then(() => undefined, (error: unknown) => error);
+ await vi.advanceTimersByTimeAsync(1100);
+ const stopError = await stopping;
+ expect(recover).toHaveBeenCalledTimes(1);
+ const logged = messages();
+ expect(logged.find((record) => record.args[0] === 'asr recovery attempted')?.args[1])
+ .toMatchObject({ trigger: 'empty_transcript', attempt: 1 });
+ if (outcome === 'text') {
+ expect(stopError).toBeUndefined();
+ expect(onError).not.toHaveBeenCalled();
+ expect(JSON.stringify(onDraftChanged.mock.calls)).toContain(SPOKEN);
+ } else {
+ expect(stopError).toBeInstanceOf(Error);
+ expect(onError).toHaveBeenCalledTimes(1);
+ expect(logged.find((record) => record.args[0] === 'run failed')?.args[1])
+ .toMatchObject({ reason: outcome === 'empty' ? 'empty_transcript' : 'rate_limited' });
+ }
+ expect(records.join('\n')).not.toContain(SPOKEN);
+ expect(records.join('\n')).not.toContain('PRIVATE_SERVER_MESSAGE');
+ });
+
it('records the run stages and summaries in the voice scope without any dictated text', async () => {
const session = createMobileVoiceControllerSession({
credential: credential(),
diff --git a/apps/mobile/src/i18n/locales/en/composer.json b/apps/mobile/src/i18n/locales/en/composer.json
index a95532e346..1bb3cd2443 100644
--- a/apps/mobile/src/i18n/locales/en/composer.json
+++ b/apps/mobile/src/i18n/locales/en/composer.json
@@ -98,7 +98,7 @@
"sessionExpiredOrForbidden": "Your {{appName}} voice sign-in has expired or lacks permission. Check your sign-in status and try again (status {{status}}).",
"incomplete": "Voice input didn't finish. Try again.",
"missingRefinerTargetProvider": "Voice refinement is unavailable. Sign in to {{appName}} again and retry.",
- "emptyTranscript": "Couldn't hear you clearly. Check your microphone or move closer and try again.",
+ "emptyTranscript": "Speech recognition returned no text. Please record again.",
"connectionInterrupted": "The voice connection was interrupted. Check your network and try again.",
"recognitionStalled": "Voice recognition stopped returning results. Try again.",
"transcriptKept": "{{message}} The text recognized so far was kept — you can edit it directly.",
diff --git a/apps/mobile/src/i18n/locales/ja/composer.json b/apps/mobile/src/i18n/locales/ja/composer.json
index 107eccc8ef..a0b88a824c 100644
--- a/apps/mobile/src/i18n/locales/ja/composer.json
+++ b/apps/mobile/src/i18n/locales/ja/composer.json
@@ -98,7 +98,7 @@
"sessionExpiredOrForbidden": "{{appName}} の音声サービスのログインが失効したか、権限がありません。ログイン状態を確認してからやり直してください(ステータス {{status}})。",
"incomplete": "音声入力が完了しませんでした。もう一度お試しください。",
"missingRefinerTargetProvider": "音声の推敲を利用できません。{{appName}} にもう一度ログインしてから再試行してください。",
- "emptyTranscript": "聞き取れませんでした。マイクを確認するか、近づいてもう一度お試しください。",
+ "emptyTranscript": "音声認識でテキストを取得できませんでした。もう一度録音してください。",
"connectionInterrupted": "音声接続が中断されました。ネットワークを確認してからやり直してください。",
"recognitionStalled": "音声認識の結果が返ってこなくなりました。もう一度お試しください。",
"transcriptKept": "{{message}}(ここまで認識したテキストは保持しており、そのまま編集できます)",
diff --git a/apps/mobile/src/i18n/locales/ko/composer.json b/apps/mobile/src/i18n/locales/ko/composer.json
index 9df2bb43ad..3ed2cf3530 100644
--- a/apps/mobile/src/i18n/locales/ko/composer.json
+++ b/apps/mobile/src/i18n/locales/ko/composer.json
@@ -98,7 +98,7 @@
"sessionExpiredOrForbidden": "{{appName}} 음성 서비스 로그인이 만료되었거나 권한이 없습니다. 로그인 상태를 확인한 후 다시 시도하세요(상태 코드 {{status}}).",
"incomplete": "음성 입력이 완료되지 않았습니다. 다시 시도하세요.",
"missingRefinerTargetProvider": "음성 다듬기를 사용할 수 없습니다. {{appName}}에 다시 로그인한 후 다시 시도하세요.",
- "emptyTranscript": "잘 듣지 못했습니다. 마이크를 확인하거나 더 가까이에서 다시 시도해 주세요.",
+ "emptyTranscript": "음성 인식 결과가 비어 있습니다. 다시 녹음해 주세요.",
"connectionInterrupted": "음성 연결이 중단되었습니다. 네트워크를 확인한 뒤 다시 시도하세요.",
"recognitionStalled": "음성 인식 결과가 더 이상 오지 않습니다. 다시 시도하세요.",
"transcriptKept": "{{message}} 지금까지 인식된 텍스트는 유지했습니다. 바로 이어서 편집할 수 있습니다.",
diff --git a/apps/mobile/src/i18n/locales/zh-CN/composer.json b/apps/mobile/src/i18n/locales/zh-CN/composer.json
index 5872b6957e..4867fbcab2 100644
--- a/apps/mobile/src/i18n/locales/zh-CN/composer.json
+++ b/apps/mobile/src/i18n/locales/zh-CN/composer.json
@@ -98,7 +98,7 @@
"sessionExpiredOrForbidden": "{{appName}} 语音服务的登录已失效或没有权限,请确认登录状态后重试(状态码 {{status}})。",
"incomplete": "语音输入未完成,请重试。",
"missingRefinerTargetProvider": "语音润色暂不可用,请重新登录 {{appName}} 后重试。",
- "emptyTranscript": "没听清,请检查麦克风或靠近一点再试。",
+ "emptyTranscript": "语音识别未返回文字,请重新录音。",
"connectionInterrupted": "语音连接中断了,请检查网络后重试。",
"recognitionStalled": "语音识别没有继续返回结果,请重试。",
"transcriptKept": "{{message}}(识别出的文字已经保留,可以直接编辑)",
diff --git a/apps/mobile/src/i18n/locales/zh-TW/composer.json b/apps/mobile/src/i18n/locales/zh-TW/composer.json
index f6e4df3991..c10bad7fb0 100644
--- a/apps/mobile/src/i18n/locales/zh-TW/composer.json
+++ b/apps/mobile/src/i18n/locales/zh-TW/composer.json
@@ -98,7 +98,7 @@
"sessionExpiredOrForbidden": "{{appName}} 語音服務的登入已失效或沒有權限,請確認登入狀態後重試(狀態碼 {{status}})。",
"incomplete": "語音輸入未完成,請重試。",
"missingRefinerTargetProvider": "語音潤色暫時無法使用,請重新登入 {{appName}} 後重試。",
- "emptyTranscript": "沒聽清楚,請檢查麥克風或靠近一點再試。",
+ "emptyTranscript": "語音辨識未回傳文字,請重新錄音。",
"connectionInterrupted": "語音連線中斷了,請檢查網路後重試。",
"recognitionStalled": "語音識別沒有繼續返回結果,請重試。",
"transcriptKept": "{{message}}(識別出的文字已經保留,可以直接編輯)",
diff --git a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts b/apps/mobile/src/session/mobileRealtimeAsrProvider.ts
index c3bc4482dd..33b2d10bb9 100644
--- a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts
+++ b/apps/mobile/src/session/mobileRealtimeAsrProvider.ts
@@ -795,6 +795,8 @@ export class MobileRealtimeAsrProvider implements AsrProvider {
}
}
+type VolcengineFlushReason = 'final_response' | 'transport_closed' | 'transport_error' | 'server_error' | 'timeout' | 'stopped' | 'reset';
+
export class MobileVolcengineSaucAsrProvider implements AsrProvider {
private readonly credential: StoredMobileVoiceCredential;
private readonly sourceLanguage: string;
@@ -822,7 +824,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
private confirmedEndMs = 0;
private readonly soundActivity: StopSoundActivity;
private recovered = false;
- private flushResolvers: Array<() => void> = [];
+ private flushResolvers: Array<(reason: VolcengineFlushReason) => void> = [];
private stopped = false;
private recoveryPromise?: Promise<void>;
private pendingConnect?: PendingConnect;
@@ -933,14 +935,18 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
}
} else {
this.connected = false;
- this.resolveFlushWaiters();
+ this.resolveFlushWaiters('transport_error');
this.callback({ type: 'error', message: redacted, at: Date.now() });
}
};
socket.onclose = (event) => {
if (this.socket !== socket) return;
this.connected = false;
- this.resolveFlushWaiters();
+ this.resolveFlushWaiters('transport_closed');
+ logMobileVoice('debug', 'volcengine sauc transport closed', {
+ code: typeof event.code === 'number' ? event.code : undefined,
+ finalRequested: this.finalRequested,
+ });
if (!settled) {
fail(new Error(this.redactError(webSocketCloseBeforeReadyMessage(
event,
@@ -988,10 +994,12 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
logMobileVoice('debug', 'volcengine sauc stop finalization decision', decision);
if (decision.fastFinish) return;
const flushStartedAt = Date.now();
- await this.waitForFlush();
+ const reason = await this.waitForFlush();
logMobileVoice('debug', 'volcengine sauc flush settled', {
waitMs: Date.now() - flushStartedAt,
hasStable: this.stableEmitted,
+ transcriptChars: this.lastTranscript.length,
+ reason,
});
if (this.lastTranscript && !this.stableEmitted) {
this.finalizedTranscript = this.lastTranscript;
@@ -1003,7 +1011,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
async stop(): Promise<void> {
this.stopped = true;
this.connected = false;
- this.resolveFlushWaiters();
+ this.resolveFlushWaiters('stopped');
this.clearUnconfirmedAudio();
this.sessionTranscriptPrefix = '';
const socket = this.socket;
@@ -1062,7 +1070,11 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
extractErrorMessage(message.payload) ?? message.payloadText ?? 'Volcengine SAUC ASR failed.',
);
this.callback({ type: 'error', message: errorMessage, at: Date.now() });
- this.resolveFlushWaiters();
+ this.resolveFlushWaiters('server_error');
+ const code = (message.payload as { code?: unknown } | undefined)?.code;
+ logMobileVoice('warn', 'volcengine sauc server error', {
+ code: typeof code === 'number' ? code : undefined,
+ });
return;
}
if (message.messageType !== VOLC_MESSAGE_TYPE_FULL_SERVER_RESPONSE && message.messageType !== VOLC_MESSAGE_TYPE_SERVER_ACK) {
@@ -1093,7 +1105,14 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
this.stableEmitted = isDefinite;
this.callback({ type: isDefinite ? 'stable' : 'partial', text: transcript, at: Date.now() });
}
- if (this.finalRequested && isLastResponse) this.resolveFlushWaiters();
+ if (isLastResponse) {
+ logMobileVoice('debug', 'volcengine sauc final response', {
+ transcriptChars: rawTranscript.length,
+ finalRequested: this.finalRequested,
+ messageType: message.messageType,
+ });
+ if (this.finalRequested) this.resolveFlushWaiters('final_response');
+ }
}
/**
@@ -1212,7 +1231,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
this.sequence = 1;
this.finalRequested = false;
this.stableEmitted = false;
- this.resolveFlushWaiters();
+ this.resolveFlushWaiters('reset');
const socket = this.socket;
this.socket = null;
@@ -1242,18 +1261,20 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
return socket && socket.readyState === OPEN ? socket : null;
}
- private waitForFlush(): Promise<void> {
+ private waitForFlush(): Promise<VolcengineFlushReason> {
return new Promise((resolve) => {
- const timer = setTimeout(resolve, this.flushTimeoutMs);
- this.flushResolvers.push(() => {
+ const finish = (reason: VolcengineFlushReason): void => {
clearTimeout(timer);
- resolve();
- });
+ this.flushResolvers = this.flushResolvers.filter((item) => item !== finish);
+ resolve(reason);
+ };
+ const timer = setTimeout(() => finish('timeout'), this.flushTimeoutMs);
+ this.flushResolvers.push(finish);
});
}
- private resolveFlushWaiters(): void {
- this.flushResolvers.splice(0).forEach((resolve) => resolve());
+ private resolveFlushWaiters(reason: VolcengineFlushReason): void {
+ this.flushResolvers.splice(0).forEach((resolve) => resolve(reason));
}
private resetState(): void {
@@ -1270,7 +1291,7 @@ export class MobileVolcengineSaucAsrProvider implements AsrProvider {
this.clearUnconfirmedAudio();
this.finalRequested = false;
this.stableEmitted = false;
- this.resolveFlushWaiters();
+ this.resolveFlushWaiters('reset');
}
private redactError(message: string): string {
diff --git a/apps/mobile/src/session/mobileVoiceController.ts b/apps/mobile/src/session/mobileVoiceController.ts
index 1e98bc850a..7f649b16b4 100644
--- a/apps/mobile/src/session/mobileVoiceController.ts
+++ b/apps/mobile/src/session/mobileVoiceController.ts
@@ -395,6 +395,7 @@ export function createMobileVoiceControllerSession(
// Refine during a speech pause and show the result right away. The shared
// controller only acts on it when a refiner exists (refinement enabled).
pauseRefinementEnabled: true,
+ retryEmptyTranscript: true,
// A reconnect allocates another managed session and can hit the account
// limit; report that instead of the generic "stopped receiving" failure.
recoveryErrorMessage: mobileVoiceRateLimitMessage,
diff --git a/apps/mobile/src/session/mobileVoiceDiagnostics.ts b/apps/mobile/src/session/mobileVoiceDiagnostics.ts
index 2cfd0f68de..dc1b8fde33 100644
--- a/apps/mobile/src/session/mobileVoiceDiagnostics.ts
+++ b/apps/mobile/src/session/mobileVoiceDiagnostics.ts
@@ -40,6 +40,7 @@ function rejectionReason(reason: string): string {
}
const FAILURE_PATTERNS: Array<[RegExp, string]> = [
+ [/^empty_transcript$/, 'empty_transcript'],
[/不可优化|次数已用完/, 'refine_quota_exhausted'],
[/rate.?limit|too many|频繁|\b429\b/i, 'rate_limited'],
[/timed? ?out|timeout|超时/i, 'timeout'],
@@ -263,7 +264,7 @@ export function createMobileVoiceTimelineRecorder(context: {
case 'error':
logMobileVoice('error', 'run failed', {
runId,
- reason: classifyMobileVoiceFailure(event.message),
+ reason: event.code ?? classifyMobileVoiceFailure(event.message),
});
return;
}
diff --git a/packages/voice-input-core/src/VoiceInputController.ts b/packages/voice-input-core/src/VoiceInputController.ts
index 0542fb1bfb..b7cf4ff61b 100644
--- a/packages/voice-input-core/src/VoiceInputController.ts
+++ b/packages/voice-input-core/src/VoiceInputController.ts
@@ -44,6 +44,8 @@ type VoiceInputControllerOptions = {
stableWaitMs?: number;
/** Host opt-in: refine during a speech pause and publish a live draft. */
pauseRefinementEnabled?: boolean;
+ /** Replay retained audio once after an audible recording ends without text. */
+ retryEmptyTranscript?: boolean;
/** Host opt-in for known recovery failures; undefined keeps the generic error. */
recoveryErrorMessage?: (error: unknown) => string | undefined;
/**
@@ -100,6 +102,7 @@ export class VoiceInputController {
private readonly callbacks: VoiceInputCallbacks;
private readonly stableWaitMs: number;
private readonly pauseRefinementEnabled: boolean;
+ private readonly retryEmptyTranscript: boolean;
private readonly recoveryErrorMessage?: (error: unknown) => string | undefined;
private readonly audioGate?: AudioGate;
private lastSoundAt = 0;
@@ -167,6 +170,7 @@ export class VoiceInputController {
this.callbacks = options.callbacks;
this.stableWaitMs = options.stableWaitMs ?? 500;
this.pauseRefinementEnabled = options.pauseRefinementEnabled ?? false;
+ this.retryEmptyTranscript = options.retryEmptyTranscript ?? false;
this.recoveryErrorMessage = options.recoveryErrorMessage;
this.audioGate = options.audioGate;
@@ -274,16 +278,32 @@ export class VoiceInputController {
if (this.audioGate) await this.audioGate.finish().catch(() => undefined);
if (runId !== this.runId || this.leftSubmittingState()) return;
const optimisticRefinement = this.startOptimisticRefinement(runId);
+ const interrupted = (): boolean => {
+ if (runId !== this.runId) return true;
+ if (!this.currentRunCancelled) return false;
+ this.discardRefinement(runId, optimisticRefinement, 'cancelled');
+ return true;
+ };
+ let stable: string | undefined;
try {
await this.asr.flushAudio();
+ if (interrupted()) return;
+ if (!this.leftSubmittingState()) stable = await this.waitForStable(this.stableWaitMs);
+ if (interrupted()) return;
+ if (!this.leftSubmittingState() && !normalizeSubmittedText(stable || this.latestTranscript)
+ && this.speechActivitySeen && this.retryEmptyTranscript && this.asr.recover
+ && !this.networkRecoveryInFlight) {
+ stable = await this.retryEmptyRecording(runId);
+ }
} catch (error) {
- this.fail(error instanceof Error ? error.message : String(error));
+ if (interrupted()) return;
+ this.fail(this.recoveryErrorMessage?.(error) ?? (error instanceof Error ? error.message : String(error)));
await this.asr.stop();
return;
}
- const stable = await this.waitForStable(this.stableWaitMs);
+ if (interrupted()) return;
const text = normalizeSubmittedText(stable || this.latestTranscript);
const source = stable ? 'stable' : this.latestTranscriptSource;
@@ -299,6 +319,7 @@ export class VoiceInputController {
// transcriptEmitted is checked too: it is the direct invariant ("this run has
// already given the host its text"), so it holds even if a future failure
// path leaves the state alone.
+ if (runId !== this.runId) return;
if (this.leftSubmittingState() || this.transcriptEmitted) {
// cancel() also lands here (it moves the run to 'done'); logging that as
// a failure would misread the timeline for an intentional abort.
@@ -367,6 +388,41 @@ export class VoiceInputController {
}
}
+ private async retryEmptyRecording(runId: string): Promise<string | undefined> {
+ const startedAt = performance.now();
+ this.logger.record({
+ type: 'asr_recovery_attempted', runId, at: Date.now(), trigger: 'empty_transcript', attempt: 1,
+ audioMsSinceLastSignal: this.audioMsSinceLastSignal,
+ voicedAudioMsSinceLastSignal: this.voicedAudioMsSinceLastSignal,
+ wallMsSinceLastSignal: startedAt - this.lastAsrSignalAt,
+ everSawSignal: this.everSawAsrSignal,
+ });
+ try {
+ // recover() owns replay of unconfirmed in-memory audio. stop() would
+ // clear that audio, so only tear the provider down after this one attempt.
+ await this.asr.recover!();
+ if (runId !== this.runId || this.leftSubmittingState()) return;
+ await this.asr.flushAudio();
+ if (runId !== this.runId || this.leftSubmittingState()) return;
+ const stable = await this.waitForStable(this.stableWaitMs);
+ if (runId !== this.runId || this.leftSubmittingState()) return;
+ const elapsedMs = performance.now() - startedAt;
+ if (normalizeSubmittedText(stable || this.latestTranscript)) {
+ this.logger.record({ type: 'asr_recovery_succeeded', runId, at: Date.now(), elapsedMs });
+ } else {
+ this.logger.record({ type: 'asr_recovery_failed', runId, at: Date.now(), elapsedMs, reason: 'empty_transcript' });
+ }
+ return stable;
+ } catch (error) {
+ if (runId !== this.runId || this.leftSubmittingState()) return;
+ this.logger.record({
+ type: 'asr_recovery_failed', runId, at: Date.now(), elapsedMs: performance.now() - startedAt,
+ reason: error instanceof Error ? error.message : String(error),
+ });
+ throw error;
+ }
+ }
+
/**
* True when something moved this run out of the state stop() set on entry —
* fail() from a rejected recovery, or cancel(). Deliberately a method: inside
@@ -748,7 +804,7 @@ export class VoiceInputController {
if (this.state === 'error' || this.state === 'done') return;
this.audioGate?.abort();
if (this.runId) {
- this.logger.record({ type: 'error', runId: this.runId, at: Date.now(), message });
+ this.logger.record({ type: 'error', runId: this.runId, at: Date.now(), message, code });
}
this.stopStallWatchdog();
this.pendingStableResolvers.splice(0).forEach((resolve) => resolve(undefined));
diff --git a/packages/voice-input-core/src/__tests__/VoiceInputController.test.ts b/packages/voice-input-core/src/__tests__/VoiceInputController.test.ts
index d5e019f0ad..fee2fa064d 100644
--- a/packages/voice-input-core/src/__tests__/VoiceInputController.test.ts
+++ b/packages/voice-input-core/src/__tests__/VoiceInputController.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
import { VoiceInputController } from '../VoiceInputController.js';
import { VoiceTimelineLogger } from '../VoiceTimelineLogger.js';
@@ -71,6 +71,94 @@ function pcmChunk(amplitude: number): ArrayBuffer {
}
describe('VoiceInputController', () => {
+ afterEach(() => vi.useRealTimers());
+
+ it.each(['silent', 'partial', 'opted_out', 'provider_error'] as const)('does not replay a recording when %s', async (scenario) => {
+ const asr = new FakeAsrProvider();
+ asr.recover = vi.fn(async () => {});
+ if (scenario === 'provider_error') {
+ asr.flush = async () => asr.emit({ type: 'error', message: 'permission denied', at: Date.now() });
+ }
+ const submitted = vi.fn(acceptSubmission);
+ const controller = new VoiceInputController({
+ asr, retryEmptyTranscript: scenario !== 'opted_out', stableWaitMs: 0,
+ logger: new VoiceTimelineLogger(),
+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted },
+ });
+ await controller.start();
+ controller.appendAudio(pcmChunk(scenario === 'silent' ? 0 : 1000));
+ if (scenario === 'partial') asr.emit({ type: 'partial', text: 'Already recognized', at: Date.now() });
+ await controller.stop();
+ expect(asr.recover).not.toHaveBeenCalled();
+ expect(submitted).toHaveBeenCalledTimes(scenario === 'partial' ? 1 : 0);
+ expect(controller.currentState).toBe(['silent', 'partial'].includes(scenario) ? 'done' : 'error');
+ });
+
+ it.each([true, false])('replays an audible empty recording once (recovered text: %s)', async (returnsText) => {
+ const asr = new FakeAsrProvider();
+ const submitted = vi.fn(acceptSubmission);
+ const onError = vi.fn();
+ const events: VoiceTimelineEvent[] = [];
+ asr.recover = vi.fn(async () => {
+ if (returnsText) asr.emit({ type: 'stable', text: 'Recovered words', at: Date.now() });
+ });
+ const flush = vi.spyOn(asr, 'flushAudio');
+ const controller = new VoiceInputController({
+ asr, retryEmptyTranscript: true, stableWaitMs: 0,
+ logger: new VoiceTimelineLogger((event) => events.push(event)),
+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },
+ });
+ await controller.start();
+ controller.appendAudio(pcmChunk(1000));
+ await controller.stop();
+ expect(asr.recover).toHaveBeenCalledTimes(1);
+ expect(flush).toHaveBeenCalledTimes(2);
+ expect(events).toContainEqual(expect.objectContaining({ type: 'asr_recovery_attempted', trigger: 'empty_transcript' }));
+ if (returnsText) {
+ expect(submitted).toHaveBeenCalledTimes(1);
+ expect(submitted.mock.calls[0][0]).toBe('Recovered words');
+ expect(onError).not.toHaveBeenCalled();
+ expect(controller.currentState).toBe('done');
+ } else {
+ expect(submitted).not.toHaveBeenCalled();
+ expect(onError).toHaveBeenCalledWith(expect.any(String), 'empty_transcript', { transcriptKept: false });
+ expect(events).toContainEqual(expect.objectContaining({ type: 'error', code: 'empty_transcript' }));
+ }
+ });
+
+ it.each([false, true])('ignores an empty-result retry settling after cancellation and restart (reject: %s)', async (rejects) => {
+ vi.useFakeTimers();
+ let settle!: () => void;
+ const asr = new FakeAsrProvider();
+ asr.recover = () => new Promise<void>((resolve, reject) => {
+ settle = () => rejects ? reject(new Error('old recovery failed')) : resolve();
+ });
+ const flush = vi.spyOn(asr, 'flushAudio');
+ const stop = vi.spyOn(asr, 'stop');
+ const onError = vi.fn();
+ const submitted = vi.fn(acceptSubmission);
+ const controller = new VoiceInputController({
+ asr, retryEmptyTranscript: true, stableWaitMs: 0,
+ logger: new VoiceTimelineLogger(),
+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },
+ });
+ await controller.start();
+ controller.appendAudio(pcmChunk(1000));
+ const stopping = controller.stop();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(settle).toBeTypeOf('function');
+ await controller.cancel();
+ await controller.start();
+ settle();
+ await stopping;
+ expect(flush).toHaveBeenCalledTimes(1);
+ expect(stop).toHaveBeenCalledTimes(1);
+ expect(onError).not.toHaveBeenCalled();
+ expect(submitted).not.toHaveBeenCalled();
+ expect(controller.currentState).toBe('listening');
+ await controller.cancel();
+ });
+
it('does not flush or stop a new run after cancellation during gate finish', async () => {
let release!: () => void;
const asr = new FakeAsrProvider();
diff --git a/packages/voice-input-core/src/types.ts b/packages/voice-input-core/src/types.ts
index 7a8f8da7bc..7921dafc54 100644
--- a/packages/voice-input-core/src/types.ts
+++ b/packages/voice-input-core/src/types.ts
@@ -88,7 +88,8 @@ export interface AsrProvider {
flushAudio(): Promise<void>;
onEvent(callback: (event: AsrEvent) => void): void;
/**
- * Best-effort reconnect for explicit transport failures. The controller does
+ * Best-effort reconnect for explicit transport failures or, when opted in,
+ * an audible recording that ends without any transcript. The controller does
* not call this for ASR partial gaps: some realtime providers can legitimately
* pause deltas and still return a complete transcript on flush. A provider
* implementation should close the dead transport, re-establish the session,
@@ -240,7 +241,7 @@ export type VoiceTimelineEvent =
type: 'asr_recovery_attempted';
runId: string;
at: number;
- trigger: 'error' | 'disconnected';
+ trigger: 'error' | 'disconnected' | 'empty_transcript';
attempt: number;
audioMsSinceLastSignal: number;
voicedAudioMsSinceLastSignal: number;
@@ -264,7 +265,7 @@ export type VoiceTimelineEvent =
source: 'stable' | 'partial';
accepted: boolean;
}
- | { type: 'error'; runId: string; at: number; message: string };
+ | { type: 'error'; runId: string; at: number; message: string; code?: VoiceInputErrorCode };
export type VoiceInputCallbacks = {
onStateChanged?: (state: VoiceInputState, outcome?: VoiceInputTerminalOutcome) => void;