Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(mobile): 保留备用识别空结果的重放音频

Dashdfcc80e454ae
diff --git a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts
index 21a30a9a30..9661ac2c68 100644
--- a/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts
+++ b/apps/mobile/src/__tests__/mobileRealtimeAsrProvider.test.ts
@@ -183,6 +183,76 @@ function volcengineProvider(options: VolcengineProviderOptions): MobileVolcengin
}
describe('mobileRealtimeAsrProvider', () => {
+ // Empty completions used to discard PCM in both fallback protocols. Exercise
+ // the controller/provider boundary, including Qwen's bound, young unbound,
+ // and missing-commit paths, so reconnecting alone cannot pass this test.
+ it.each(['openai', 'qwen_bound', 'qwen_unbound', 'qwen_no_commit'] as const)(
+ 'replays audio after an empty fallback completion and recovers text: %s', async (scenario) => {
+ vi.useFakeTimers();
+ FakeSocket.instances = [];
+ const qwen = scenario !== 'openai';
+ const cred = qwen ? credential({ asr: {
+ provider: 'litellm-qwen3-asr-flash-realtime', model: 'qwen3-asr-flash-realtime',
+ auth: 'api-key', mode: 'realtime-websocket', pcmSampleRate: 16000,
+ endpointPath: '/dashscope/api-ws/v1/realtime?model=qwen3-asr-flash-realtime',
+ protocolProfile: 'qwen-asr-server-vad',
+ } }) : credential();
+ const asr = realtimeProvider({ credential: cred, websocketFactory: FakeSocket, flushTimeoutMs: 100 });
+ const submitted = vi.fn(() => undefined);
+ const onError = vi.fn();
+ const controller = new VoiceInputController({
+ asr, retryEmptyTranscript: true, stableWaitMs: 1,
+ logger: new VoiceTimelineLogger(),
+ callbacks: { onDraftChanged: () => {}, onSubmitted: submitted, onError },
+ });
+ try {
+ const starting = controller.start();
+ await waitForFakeSocketCount(1);
+ const first = FakeSocket.instances[0];
+ first.open();
+ first.message({ type: 'session.updated' });
+ await starting;
+ controller.appendAudio(new Int16Array(qwen ? 3200 : 4800).fill(1000).buffer);
+ if (scenario === 'qwen_bound' || scenario === 'qwen_no_commit') {
+ await vi.advanceTimersByTimeAsync(2000);
+ }
+ const stopping = controller.stop();
+ if (scenario !== 'qwen_no_commit') {
+ first.message({ type: 'input_audio_buffer.committed', item_id: 'empty-item' });
+ }
+ first.message({
+ type: 'conversation.item.input_audio_transcription.completed', item_id: 'empty-item',
+ ...(qwen ? { text: ' \t ' } : { transcript: '' }),
+ });
+ await vi.advanceTimersByTimeAsync(1);
+ expect(FakeSocket.instances).toHaveLength(2);
+ const retry = FakeSocket.instances[1];
+ retry.open();
+ retry.message({ type: 'session.updated' });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(retry.sent.slice(1)).toEqual([
+ expect.objectContaining({
+ type: 'input_audio_buffer.append',
+ audio: (first.sent[1] as { audio: string }).audio,
+ }),
+ expect.objectContaining({ type: qwen ? 'session.finish' : 'input_audio_buffer.commit' }),
+ ]);
+ retry.message({
+ type: 'conversation.item.input_audio_transcription.completed',
+ item_id: 'recovered-item', transcript: '找回备用识别的文字',
+ });
+ await stopping;
+ expect(submitted).toHaveBeenCalledExactlyOnceWith('找回备用识别的文字', expect.any(Object));
+ expect(onError).not.toHaveBeenCalled();
+ expect(controller.currentState).toBe('done');
+ expect(retry.readyState).toBe(3);
+ } finally {
+ await controller.cancel();
+ vi.useRealTimers();
+ }
+ },
+ );
+
it('replays the captured PCM after an empty final response and submits the recovered transcript once', async () => {
vi.useFakeTimers();
FakeSocket.instances = [];
diff --git a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts b/apps/mobile/src/session/mobileRealtimeAsrProvider.ts
index 33b2d10bb9..20f66cdb12 100644
--- a/apps/mobile/src/session/mobileRealtimeAsrProvider.ts
+++ b/apps/mobile/src/session/mobileRealtimeAsrProvider.ts
@@ -556,7 +556,7 @@ export class MobileRealtimeAsrProvider implements AsrProvider {
if (this.pendingCommitCount > 0) this.pendingCommitCount -= 1;
const aggregate = this.aggregateTranscript();
if (aggregate) this.callback({ type: 'stable', text: aggregate, at: Date.now() });
- this.clearConfirmedRealtimeAudio(event.item_id);
+ this.finishRealtimeAudio(event.item_id, itemTranscript.trim().length > 0);
if (this.pendingCommitCount === 0) this.resolveFlushWaiters();
}
@@ -686,25 +686,27 @@ export class MobileRealtimeAsrProvider implements AsrProvider {
}
}
- private clearConfirmedRealtimeAudio(itemId: string): void {
+ private finishRealtimeAudio(itemId: string, hasTranscript: boolean): void {
+ // Completion ends the commit bookkeeping, but an empty result does not
+ // confirm its audio. Keep those bytes for the one stop-time replay.
let commitId = this.audioCommitIdsByItem.get(itemId);
if (commitId === undefined) {
const turnId = this.unboundAudioItemTurnIds.get(itemId);
if (turnId !== undefined) {
this.unboundAudioItemTurnIds.delete(itemId);
- this.clearUnboundRealtimeAudioTurn(turnId);
+ if (hasTranscript) this.clearUnboundRealtimeAudioTurn(turnId);
return;
}
commitId = this.pendingAudioCommitIds.shift();
}
if (commitId === undefined) {
- this.trimUnconfirmedAudioOlderThan(Date.now() - CONFIRMED_AUDIO_RETENTION_MS);
+ if (hasTranscript) this.trimUnconfirmedAudioOlderThan(Date.now() - CONFIRMED_AUDIO_RETENTION_MS);
return;
}
this.audioCommitIdsByItem.delete(itemId);
this.unboundAudioItemTurnIds.delete(itemId);
this.pendingAudioCommitIds = this.pendingAudioCommitIds.filter((id) => id !== commitId);
- this.unconfirmedAudio = this.unconfirmedAudio.filter((entry) => entry.commitId !== commitId);
+ if (hasTranscript) this.unconfirmedAudio = this.unconfirmedAudio.filter((entry) => entry.commitId !== commitId);
}
private clearUnboundRealtimeAudioTurn(turnId: number): void {