Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(mobile): 保留备用识别空结果的重放音频
Dashdfcc80e454ae
diff --git a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.tsindex 21a30a9a30..9661ac2c68 100644--- a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts+++ b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts@@ -183,6 +183,76 @@ function volcengineProvider(options: VolcengineProviderOptions): MobileVolcengin}describe('mobileRealtimeAsrProvider', () => {+ // Empty completions used to discard PCM in both fallback protocols. Exercise+ // the controller/provider boundary, including Qwen's bound, young unbound,+ // and missing-commit paths, so reconnecting alone cannot pass this test.+ it.each(['openai', 'qwen_bound', 'qwen_unbound', 'qwen_no_commit'] as const)(+ 'replays audio after an empty fallback completion and recovers text: %s', async (scenario) => {+ vi.useFakeTimers();+ FakeSocket.instances = [];+ const qwen = scenario !== 'openai';+ const cred = qwen ? credential({ asr: {+ provider: 'litellm-qwen3-asr-flash-realtime', model: 'qwen3-asr-flash-realtime',+ auth: 'api-key', mode: 'realtime-websocket', pcmSampleRate: 16000,+ endpointPath: '/dashscope/api-ws/v1/realtime?model=qwen3-asr-flash-realtime',+ protocolProfile: 'qwen-asr-server-vad',+ } }) : credential();+ const asr = realtimeProvider({ credential: cred, websocketFactory: FakeSocket, flushTimeoutMs: 100 });+ const submitted = vi.fn(() => undefined);+ const onError = vi.fn();+ const controller = new VoiceInputController({+ asr, retryEmptyTranscript: true, stableWaitMs: 1,+ logger: new VoiceTimelineLogger(),+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },+ });+ try {+ const starting = controller.start();+ await waitForFakeSocketCount(1);+ const first = FakeSocket.instances[0];+ first.open();+ first.message({ type: 'session.updated' });+ await starting;+ controller.appendAudio(new Int16Array(qwen ? 3200 : 4800).fill(1000).buffer);+ if (scenario === 'qwen_bound' || scenario === 'qwen_no_commit') {+ await vi.advanceTimersByTimeAsync(2000);+ }+ const stopping = controller.stop();+ if (scenario !== 'qwen_no_commit') {+ first.message({ type: 'input_audio_buffer.committed', item_id: 'empty-item' });+ }+ first.message({+ type: 'conversation.item.input_audio_transcription.completed', item_id: 'empty-item',+ ...(qwen ? { text: ' \t ' } : { transcript: '' }),+ });+ await vi.advanceTimersByTimeAsync(1);+ expect(FakeSocket.instances).toHaveLength(2);+ const retry = FakeSocket.instances[1];+ retry.open();+ retry.message({ type: 'session.updated' });+ await vi.advanceTimersByTimeAsync(0);+ expect(retry.sent.slice(1)).toEqual([+ expect.objectContaining({+ type: 'input_audio_buffer.append',+ audio: (first.sent[1] as { audio: string }).audio,+ }),+ expect.objectContaining({ type: qwen ? 'session.finish' : 'input_audio_buffer.commit' }),+ ]);+ retry.message({+ type: 'conversation.item.input_audio_transcription.completed',+ item_id: 'recovered-item', transcript: '找回备用识别的文字',+ });+ await stopping;+ expect(submitted).toHaveBeenCalledExactlyOnceWith('找回备用识别的文字', expect.any(Object));+ expect(onError).not.toHaveBeenCalled();+ expect(controller.currentState).toBe('done');+ expect(retry.readyState).toBe(3);+ } finally {+ await controller.cancel();+ vi.useRealTimers();+ }+ },+ );+it('replays the captured PCM after an empty final response and submits the recovered transcript once', async () => {vi.useFakeTimers();FakeSocket.instances = [];diff --git a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts b/apps/mobile/src/session/mobileRealtimeAsrProvider.tsindex 33b2d10bb9..20f66cdb12 100644--- a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts+++ b/apps/mobile/src/session/mobileRealtimeAsrProvider.ts@@ -556,7 +556,7 @@ export class MobileRealtimeAsrProvider implements AsrProvider {if (this.pendingCommitCount > 0) this.pendingCommitCount -= 1;const aggregate = this.aggregateTranscript();if (aggregate) this.callback({ type: 'stable', text: aggregate, at: Date.now() });- this.clearConfirmedRealtimeAudio(event.item_id);+ this.finishRealtimeAudio(event.item_id, itemTranscript.trim().length > 0);if (this.pendingCommitCount === 0) this.resolveFlushWaiters();}@@ -686,25 +686,27 @@ export class MobileRealtimeAsrProvider implements AsrProvider {}}- private clearConfirmedRealtimeAudio(itemId: string): void {+ private finishRealtimeAudio(itemId: string, hasTranscript: boolean): void {+ // Completion ends the commit bookkeeping, but an empty result does not+ // confirm its audio. Keep those bytes for the one stop-time replay.let commitId = this.audioCommitIdsByItem.get(itemId);if (commitId === undefined) {const turnId = this.unboundAudioItemTurnIds.get(itemId);if (turnId !== undefined) {this.unboundAudioItemTurnIds.delete(itemId);- this.clearUnboundRealtimeAudioTurn(turnId);+ if (hasTranscript) this.clearUnboundRealtimeAudioTurn(turnId);return;}commitId = this.pendingAudioCommitIds.shift();}if (commitId === undefined) {- this.trimUnconfirmedAudioOlderThan(Date.now() - CONFIRMED_AUDIO_RETENTION_MS);+ if (hasTranscript) this.trimUnconfirmedAudioOlderThan(Date.now() - CONFIRMED_AUDIO_RETENTION_MS);return;}this.audioCommitIdsByItem.delete(itemId);this.unboundAudioItemTurnIds.delete(itemId);this.pendingAudioCommitIds = this.pendingAudioCommitIds.filter((id) => id !== commitId);- this.unconfirmedAudio = this.unconfirmedAudio.filter((entry) => entry.commitId !== commitId);+ if (hasTranscript) this.unconfirmedAudio = this.unconfirmedAudio.filter((entry) => entry.commitId !== commitId);}private clearUnboundRealtimeAudioTurn(turnId: number): void {