Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
perf(voice): 提前启动收音并补齐停止前的音频
Dash238fc3853344
diff --git a/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.ts b/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.tsindex 5e7dfa5a31..20d8c3b7ab 100644--- a/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.ts+++ b/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.ts@@ -373,25 +373,6 @@ function readErrorString(error: unknown, key: 'name' | 'message' | 'reason'): streturn typeof value === 'string' ? value : undefined;}-async function assertSelectedMicrophoneAvailable(deviceId?: string): Promise<void> {- // Browser/Electron expose the system default microphone as the synthetic- // "default" device. Enumerating devices before every start adds startup- // latency and cannot prove that this moving target is unavailable, so let- // getUserMedia surface the platform error for automatic/default capture.- if (!isSpecificMicrophoneDeviceId(deviceId)) return;- const devices = await navigator.mediaDevices.enumerateDevices().catch(() => []);- const audioInputs = devices.filter((device) => device.kind === 'audioinput');- // Before permission is granted, browsers may expose opaque/empty IDs. Only- // fail fast when the list has comparable IDs and the selected one is- // definitely absent; otherwise let getUserMedia surface the platform error.- if (- audioInputs.some((device) => Boolean(device.deviceId)) &&- !audioInputs.some((device) => device.deviceId === deviceId)- ) {- throw new VoiceInputSelectedMicrophoneUnavailableError();- }-}-function keepAliveKey(options: KeepAliveSessionOptions): KeepAliveSessionKey {return JSON.stringify({workletUrl: options.workletUrl,@@ -527,12 +508,6 @@ class KeepAliveMicSession {private async startInternal(): Promise<void> {ensureKeepAliveDeviceChangeListener();ensureKeepAlivePowerReleaseListener();- // Also guarded: if a release lands while enumerating and the device then- // reads as absent, the raw VoiceInputSelectedMicrophoneUnavailableError- // would send prewarm/capture down their automatic-fallback paths and open- // the *default* microphone after the power event.- await this.awaitStartupStep(assertSelectedMicrophoneAvailable(this.options.deviceId));-// A release (suspend/lock/setting off) can land while this is pending, in// which case bailing out avoids opening a device nothing could then close.const sharedState = await this.awaitStartupStep(prewarmVoiceInputAudio(this.options.workletUrl));@@ -1222,13 +1197,8 @@ export class WebMicAudioEngine {const benchmarkFixturePromise = BENCHMARK_FIXTURE_ENABLED? prewarmVoiceInputBenchmarkFixture().catch(() => null): Promise.resolve(null);- await this.awaitDirectStartupStep(- assertSelectedMicrophoneAvailable(this.deviceId),- powerGenerationAtStart,- );- // The enumeration above passed its own check, but a release can land in the- // gap before we ask for the device. Without this the request would be made- // after the one-shot event and only be closed once it resolves.+ // exact deviceId makes getUserMedia authoritative. A separate enumeration+ // adds a serial device round trip and can become stale before opening it.this.assertNoPowerReleaseDuringStartup(powerGenerationAtStart);const streamPromise = this.awaitDirectStartupStep(navigator.mediaDevices.getUserMedia({diff --git a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.ts b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.tsindex 042c5a97b4..9fb8513ecb 100644--- a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.ts+++ b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.ts@@ -675,36 +675,18 @@ describe('keep-alive microphone idle window', () => {expect(track.stopped).toBe(false);});- it('does not open the microphone when enumeration outlives a power release', async () => {- let resolveEnumerate: (value: unknown) => void = () => undefined;- vi.stubGlobal('navigator', {- mediaDevices: {- enumerateDevices: vi.fn(() => new Promise((resolve) => {- resolveEnumerate = resolve;- })),- getUserMedia,- addEventListener: vi.fn(),- },- });-- const engine = new mod.WebMicAudioEngine({- workletUrl: WORKLET_URL,- keepAlive: false,- deviceId: 'specific-device',- onInterrupted: vi.fn(),- });- const starting = engine.start();- await vi.advanceTimersByTimeAsync(0);- expect(powerCallback).toBeDefined();-+ it('opens an exact device without waiting for enumeration and still releases it on lock', async () => {+ const enumerate = vi.fn(() => new Promise(() => undefined));+ vi.stubGlobal('navigator', { mediaDevices: { enumerateDevices: enumerate, getUserMedia, addEventListener: vi.fn() } });+ const engine = new mod.WebMicAudioEngine({ workletUrl: WORKLET_URL, deviceId: 'specific-device' });+ await engine.start();+ expect(enumerate).not.toHaveBeenCalled();+ expect(getUserMedia).toHaveBeenCalledOnce();+ expect(getUserMedia.mock.calls[0][0].audio.deviceId).toEqual({ exact: 'specific-device' });powerCallback?.({ reason: 'screen_locked' });await vi.advanceTimersByTimeAsync(0);- // Enumeration succeeds *after* the release, reporting the device as present.- resolveEnumerate([{ kind: 'audioinput', deviceId: 'specific-device' }]);-- await expect(starting).rejects.toThrow(/disposed/i);- // The one-shot event has passed; nothing may reopen the device now.- expect(getUserMedia).not.toHaveBeenCalled();+ expect(track.stopped).toBe(true);+ expect(getUserMedia).toHaveBeenCalledOnce();});it('does not report an error when a release hits a still-starting capture', async () => {diff --git a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.ts b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.tsindex 9e1293b9fe..89efec6139 100644--- a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.ts+++ b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.ts@@ -192,10 +192,11 @@ describe('prewarmVoiceInputMicrophoneWithAutomaticFallback', () => {});it('closes a temporary replacement device even when preparation could not open the selected one', async () => {+ getUserMedia.mockRejectedValueOnce(new DOMException('Device missing', 'NotFoundError'));const release = prepareVoiceInputMicrophone({workletUrl: 'https://app.local/pcm16k-worklet.js', deviceId: 'missing-device',});- await vi.waitFor(() => expect(enumerateDevices).toHaveBeenCalledOnce());+ await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledOnce());release?.();const engine = new WebMicAudioEngine({workletUrl: 'https://app.local/pcm16k-worklet.js', keepAlive: true, releaseAfterUse: true,@@ -261,6 +262,7 @@ describe('prewarmVoiceInputMicrophoneWithAutomaticFallback', () => {it('falls back to automatic microphone prewarm when the selected device is unavailable', async () => {const onFallback = vi.fn();+ getUserMedia.mockRejectedValueOnce(new DOMException('Device missing', 'OverconstrainedError'));await prewarmVoiceInputMicrophoneWithAutomaticFallback({workletUrl: 'https://app.local/pcm16k-worklet.js',@@ -268,8 +270,9 @@ describe('prewarmVoiceInputMicrophoneWithAutomaticFallback', () => {}, onFallback);expect(onFallback).toHaveBeenCalledTimes(1);- expect(enumerateDevices).toHaveBeenCalledTimes(1);- expect(getUserMedia).toHaveBeenCalledTimes(1);+ expect(enumerateDevices).not.toHaveBeenCalled();+ expect(getUserMedia).toHaveBeenCalledTimes(2);+ expect(getUserMedia.mock.calls[0][0].audio.deviceId).toEqual({ exact: 'missing-device' });expect(getUserMedia).toHaveBeenCalledWith({audio: {autoGainControl: true,diff --git a/apps/mobile/app/sessions/[sessionId].tsx b/apps/mobile/app/sessions/[sessionId].tsxindex de14aea296..16f44d54de 100644--- a/apps/mobile/app/sessions/[sessionId].tsx+++ b/apps/mobile/app/sessions/[sessionId].tsx@@ -548,7 +548,6 @@ import {} from '@/session/composerTextPlatformMetrics';import {isMobileRealtimeAudioAvailable,- prewarmMobileRealtimeAudio,shouldShowMobileVoiceUi,} from '@/session/mobileRealtimeAudio';import {@@ -5532,6 +5531,7 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: S}, [applyComposerDocument]);const startVoiceRecording = useCallback(async () => {+ const requestedAt = Date.now();if (voicePermissionRequestInFlightRef.current|| voiceStartupInFlightRef.current@@ -5650,7 +5650,7 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: Scredential.settings?.language,CINDY_MANAGED_REFINER_PROVIDER,);- const audioGate = await createMobileSpeakerGate();+ const audioGate = createMobileSpeakerGate();if (voiceStartupSeqRef.current !== startupSeq) {// This run never opened the microphone; release only its claimed ASR.void prewarmedVoice?.asr.stop().catch(() => undefined);@@ -5666,6 +5666,7 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: Scredential,// "Recognize my voice": undefined unless enabled, enrolled and the models are on disk.audioGate,+ requestedAt,...(prewarmedVoice ? { asr: prewarmedVoice.asr } : {}),connectionProvider: (providerId: string) => voiceContext.createAsrConnection(providerId),refinerTargetProvider: (providerId: string, options?: { refreshAccessToken?: boolean }) =>@@ -5944,9 +5945,6 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: Sif (voiceIsProcessing) return;if (voiceRecordingActiveRef.current || voiceState === 'listening') return;if (!deviceId || !isMobileRealtimeAudioAvailable()) return;- // Keep the native audio-session warmup on the synchronous press-down path- // (prewarmMobileVoiceStart re-runs it idempotently below).- prewarmMobileRealtimeAudio();warmMobileSpeakerFilter();// 托管预热:凭登录态提前拿 voice-server 票据并开 ASR WebSocket。prewarmMobileVoiceStart(deviceId, {diff --git a/apps/mobile/app/sessions/new.tsx b/apps/mobile/app/sessions/new.tsxindex a3e2b1dbcf..ff1496dd8f 100644--- a/apps/mobile/app/sessions/new.tsx+++ b/apps/mobile/app/sessions/new.tsx@@ -340,7 +340,6 @@ import {} from '@/session/composerTextPlatformMetrics';import {isMobileRealtimeAudioAvailable,- prewarmMobileRealtimeAudio,shouldShowMobileVoiceUi,} from '@/session/mobileRealtimeAudio';import {@@ -2988,6 +2987,7 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: bool};const startVoiceRecording = useCallback(async () => {+ const requestedAt = Date.now();if (voicePermissionRequestInFlightRef.current|| voiceStartupInFlightRef.current@@ -3099,7 +3099,7 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: boolcredential.settings?.language,CINDY_MANAGED_REFINER_PROVIDER,);- const audioGate = await createMobileSpeakerGate();+ const audioGate = createMobileSpeakerGate();if (voiceStartupSeqRef.current !== startupSeq) {// This run never opened the microphone; release only its claimed ASR.void prewarmedVoice?.asr.stop().catch(() => undefined);@@ -3114,6 +3114,7 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: boolcredential,// "Recognize my voice": undefined unless enabled, enrolled and the models are on disk.audioGate,+ requestedAt,...(prewarmedVoice ? { asr: prewarmedVoice.asr } : {}),connectionProvider: (providerId: string) => voiceContext.createAsrConnection(providerId),refinerTargetProvider: (providerId: string, options?: { refreshAccessToken?: boolean }) =>@@ -3233,9 +3234,6 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: boolif (creating || voiceIsProcessing) return;if (voiceRecordingActiveRef.current || voiceState === 'listening') return;if (!selectedDeviceId || !isMobileRealtimeAudioAvailable()) return;- // Keep the native audio-session warmup on the synchronous press-down path- // (prewarmMobileVoiceStart re-runs it idempotently below).- prewarmMobileRealtimeAudio();warmMobileSpeakerFilter();// 托管预热:凭登录态提前拿 voice-server 票据并开 ASR WebSocket。prewarmMobileVoiceStart(selectedDeviceId, {diff --git a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.h b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.hnew file mode 100644index 0000000000..e80f06963f--- /dev/null+++ b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.h@@ -0,0 +1,14 @@+#import <AVFoundation/AVFoundation.h>++NS_ASSUME_NONNULL_BEGIN+/// Single-producer audio ring. Only copy/downmix runs on the render thread;+/// resampling and bridging run on the caller's serialized non-realtime queue.+@interface XdtAudioCapture : NSObject+@property(nonatomic, readonly) AVAudioEngine *engine;+@property(nonatomic, readonly) double inputSampleRate;+@property(nonatomic, readonly) BOOL overflowed;+- (BOOL)startWithSampleRate:(double)sampleRate error:(NSError **)error;+- (NSArray<NSDictionary *> *)drain;+- (void)stop;+@end+NS_ASSUME_NONNULL_ENDdiff --git a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.m b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.mnew file mode 100644index 0000000000..b02bc41c35--- /dev/null+++ b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.m@@ -0,0 +1,168 @@+#import "XdtAudioCapture.h"+#import <stdatomic.h>+#import <mach/mach_time.h>+#import <math.h>++// 4 MiB, allocated before engine.start. At a 10ms I/O period the ring tolerates+// 2.56s of consumer starvation. Overflow is an explicit incomplete-capture error.+enum { SlotCount = 256, MaxFrames = 4096 };+typedef struct { float samples[MaxFrames]; UInt32 frames; uint64_t hostTime; } AudioSlot;+typedef struct {+ AudioSlot slots[SlotCount];+ _Atomic(uint32_t) written;+ _Atomic(uint32_t) read;+ _Atomic(bool) overflow;+} AudioRing;++static OSStatus Capture(AudioRing *ring, const AudioTimeStamp *time,+ AVAudioFrameCount frames, const AudioBufferList *input) {+ uint32_t written = atomic_load_explicit(&ring->written, memory_order_relaxed);+ uint32_t read = atomic_load_explicit(&ring->read, memory_order_acquire);+ if (frames > MaxFrames || written - read >= SlotCount) {+ atomic_store_explicit(&ring->overflow, true, memory_order_release);+ return noErr;+ }+ AudioSlot *slot = &ring->slots[written % SlotCount];+ UInt32 channels = 0;+ for (UInt32 b = 0; b < input->mNumberBuffers; b++) channels += input->mBuffers[b].mNumberChannels;+ if (channels == 0) return noErr;+ for (UInt32 f = 0; f < frames; f++) {+ float sample = 0;+ for (UInt32 b = 0; b < input->mNumberBuffers; b++) {+ const AudioBuffer *buffer = &input->mBuffers[b];+ const float *data = buffer->mData;+ for (UInt32 c = 0; c < buffer->mNumberChannels; c++) {+ sample += data ? data[f * buffer->mNumberChannels + c] : 0;+ }+ }+ slot->samples[f] = sample / channels;+ }+ slot->frames = frames;+ slot->hostTime = (time->mFlags & kAudioTimeStampHostTimeValid) ? time->mHostTime : mach_absolute_time();+ atomic_store_explicit(&ring->written, written + 1, memory_order_release);+ return noErr;+}++@implementation XdtAudioCapture {+ AudioRing *_ring;+ AVAudioSinkNode *_sink;+ double _targetRate;+ double _hostTicksToMs;+ double _wallAnchor;+ uint64_t _hostAnchor;+ uint64_t _sourceFrames;+ uint64_t _outputFrames;+ NSMutableData *_pending;+ double _pendingCapturedAt;+ NSInteger _chunkIndex;+}++- (instancetype)init {+ if ((self = [super init])) {+ _ring = calloc(1, sizeof(AudioRing));+ if (!_ring) return nil;+ _engine = [AVAudioEngine new];+ mach_timebase_info_data_t info;+ mach_timebase_info(&info);+ _hostTicksToMs = (double)info.numer / info.denom / 1e6;+ }+ return self;+}++- (void)dealloc {+ [self stop];+ free(_ring);+}++- (BOOL)overflowed { return atomic_load_explicit(&_ring->overflow, memory_order_acquire); }++- (BOOL)startWithSampleRate:(double)sampleRate error:(NSError **)error {+ [self stop];+ // A graph whose input unit survived session deactivation can accept start()+ // yet deliver no frames after a route/configuration change. Create it only+ // after the caller has activated the current session.+ _engine = [AVAudioEngine new];+ AVAudioInputNode *input = _engine.inputNode;+ AVAudioFormat *format = [input outputFormatForBus:0];+ if (format.commonFormat != AVAudioPCMFormatFloat32 || format.sampleRate <= 0 || format.channelCount == 0 ||+ !isfinite(sampleRate) || sampleRate < 8000 || sampleRate > 192000) {+ if (error) *error = [NSError errorWithDomain:@"XdtAudioCapture" code:1+ userInfo:@{NSLocalizedDescriptionKey: @"Microphone did not provide a supported PCM format."}];+ return NO;+ }+ atomic_store(&_ring->written, 0);+ atomic_store(&_ring->read, 0);+ atomic_store(&_ring->overflow, false);+ _inputSampleRate = format.sampleRate;+ _targetRate = sampleRate;+ _sourceFrames = _outputFrames = 0;+ _chunkIndex = 0;+ _pending = [NSMutableData new];+ _hostAnchor = mach_absolute_time();+ _wallAnchor = NSDate.date.timeIntervalSince1970 * 1000;+ AudioRing *ring = _ring;+ // No ObjC/Swift calls, allocation, dispatch or locks in this receiver.+ _sink = [[AVAudioSinkNode alloc] initWithReceiverBlock:^OSStatus(+ const AudioTimeStamp *time, AVAudioFrameCount frames, const AudioBufferList *data) {+ return Capture(ring, time, frames, data);+ }];+ [_engine attachNode:_sink];+ [_engine connect:input to:_sink format:format];+ [_engine prepare];+ if ([_engine startAndReturnError:error]) return YES;+ [self stop];+ return NO;+}++- (NSDictionary *)takePending {+ NSUInteger frames = _pending.length / sizeof(int16_t);+ NSDictionary *packet = @{+ @"base64Pcm16": [_pending base64EncodedStringWithOptions:0],+ @"capturedAt": @(_pendingCapturedAt), @"chunkIndex": @(_chunkIndex++),+ @"sampleRate": @(_targetRate), @"durationMs": @(frames / _targetRate * 1000)+ };+ [_pending setLength:0];+ return packet;+}++- (NSArray<NSDictionary *> *)drain {+ NSMutableArray *packets = [NSMutableArray new];+ uint32_t read = atomic_load_explicit(&_ring->read, memory_order_relaxed);+ const uint32_t written = atomic_load_explicit(&_ring->written, memory_order_acquire);+ while (read != written) {+ const AudioSlot *slot = &_ring->slots[read % SlotCount];+ uint64_t end = _sourceFrames + slot->frames;+ uint64_t outputEnd = (uint64_t)ceil(end * _targetRate / _inputSampleRate);+ for (; _outputFrames < outputEnd; _outputFrames++) {+ uint64_t source = (uint64_t)floor(_outputFrames * _inputSampleRate / _targetRate);+ UInt32 offset = (UInt32)(source - _sourceFrames);+ if (_pending.length == 0) {+ // Subtract as signed: input timestamps can predate engine.start.+ _pendingCapturedAt = _wallAnchor + ((int64_t)(slot->hostTime - _hostAnchor)) * _hostTicksToMs+ + offset / _inputSampleRate * 1000;+ }+ float value = fmaxf(-1, fminf(1, slot->samples[MIN(offset, slot->frames - 1)]));+ int16_t sample = (int16_t)(value * INT16_MAX);+ [_pending appendBytes:&sample length:sizeof(sample)];+ // First packet 10ms, subsequent packets 40ms: hardware latency is+ // independent of ASR network packet frequency.+ NSUInteger framesPerPacket = (NSUInteger)llround(_targetRate * (_chunkIndex == 0 ? .010 : .040));+ if (_pending.length / sizeof(int16_t) >= framesPerPacket) [packets addObject:[self takePending]];+ }+ _sourceFrames = end;+ read++;+ atomic_store_explicit(&_ring->read, read, memory_order_release);+ }+ if (!_engine.isRunning && _pending.length) [packets addObject:[self takePending]];+ return packets;+}++- (void)stop {+ [_engine stop];+ if (_sink) {+ [_engine disconnectNodeInput:_sink];+ [_engine detachNode:_sink];+ _sink = nil;+ }+}+@enddiff --git a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swift b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swiftindex caffbbe4ba..29fba90072 100644--- a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swift+++ b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swift@@ -5,11 +5,15 @@ private let onAudioChunk = "onAudioChunk"private let onAudioError = "onAudioError"public class XdtMobileRealtimeAudioModule: Module {- private let engine = AVAudioEngine()+ private let capture = XdtAudioCapture()+ private var drainTimer: DispatchSourceTimer?+ private var recentChunks: [[String: Any]] = []+ private var captureId = ""+ private var sessionActivatedAt = 0.0+ private var engineStartedAt = 0.0private var isCapturing = falseprivate var ownsAudioSession = falseprivate var targetSampleRate = 16_000.0- private var chunkIndex = 0private var interruptionObserver: NSObjectProtocol?private var routeChangeObserver: NSObjectProtocol?// Deferred AVAudioSession deactivation (keep-alive). Activating the session in@@ -44,7 +48,10 @@ public class XdtMobileRealtimeAudioModule: Module {return ["inputPortTypes": session.currentRoute.inputs.map { $0.portType.rawValue },"inputSampleRate": session.sampleRate,- "inputChannels": session.inputNumberOfChannels+ "inputChannels": session.inputNumberOfChannels,+ "ioBufferDurationMs": session.ioBufferDuration * 1000,+ "sessionActivatedAt": self.sessionActivatedAt,+ "engineStartedAt": self.engineStartedAt]}}@@ -53,6 +60,36 @@ public class XdtMobileRealtimeAudioModule: Module {self.stopCapture()}+ // Additive: old installed binaries still use stop(). Only audio not already+ // delivered to JS is returned; trim to the user's stop time, not bridge arrival.+ AsyncFunction("stopAndDrain") { (afterIndex: Int, stoppedAt: Double) in+ return try self.sessionStateQueue.sync { () -> [[String: Any]] in+ self.stopCaptureLocked(deactivateImmediately: false)+ defer { self.recentChunks.removeAll() }+ guard stoppedAt.isFinite else {+ throw Exception(name: "ERR_AUDIO_STOP_TIME", description: "Invalid microphone stop time.")+ }+ if self.capture.overflowed || (self.recentChunks.first?["chunkIndex"] as? Int ?? 0) > afterIndex + 1 {+ throw Exception(name: "ERR_AUDIO_OVERFLOW", description: "Realtime microphone delivery fell behind.")+ }+ return self.recentChunks.compactMap { chunk in+ guard let index = chunk["chunkIndex"] as? Int, index > afterIndex,+ let capturedAt = chunk["capturedAt"] as? Double,+ let sampleRate = chunk["sampleRate"] as? Double,+ let base64 = chunk["base64Pcm16"] as? String,+ var pcm = Data(base64Encoded: base64) else { return nil }+ let retainedFrames = min(Double(pcm.count / 2), max(0, ((stoppedAt - capturedAt) * sampleRate / 1000).rounded(.down)))+ let frames = Int(retainedFrames)+ guard frames > 0 else { return nil }+ pcm = pcm.prefix(frames * 2)+ var result = chunk+ result["base64Pcm16"] = pcm.base64EncodedString()+ result["durationMs"] = Double(frames) / sampleRate * 1000+ return result+ }+ }+ }+// Best-effort audio-session warm-up, called on touch-down of the mic button// (before the tap that actually starts recording). Configuring + activating// the session here means the ~100-150ms warm-up happens while the finger is@@ -99,9 +136,8 @@ public class XdtMobileRealtimeAudioModule: Module {return}do {- try session.setCategory(.playAndRecord, mode: .measurement, options: [.duckOthers, .defaultToSpeaker])- try session.setActive(true)- ownsAudioSession = true+ try activateAudioSessionLocked(session)+ installAudioSessionObservers()// Deliberately do NOT touch the engine here: engine.prepare() with no// attached nodes raises an uncatchable NSException (AVAudioEngineGraph// Initialize: inputNode != nullptr || outputNode != nullptr). Session@@ -125,6 +161,7 @@ public class XdtMobileRealtimeAudioModule: Module {return}self.deactivateOwnedSessionLocked()+ self.removeAudioSessionObservers()}deactivateWorkItem = workItemsessionStateQueue.asyncAfter(deadline: .now() + sessionKeepAliveSeconds, execute: workItem)@@ -158,7 +195,8 @@ public class XdtMobileRealtimeAudioModule: Module {if let sampleRate = numericOption(options, key: "sampleRate"), sampleRate > 0 {targetSampleRate = sampleRate}- let bufferSize = AVAudioFrameCount(integerOption(options, key: "bufferSize") ?? 2048)+ captureId = options?["captureId"] as? String ?? ""+ recentChunks.removeAll()let session = AVAudioSession.sharedInstance()if session.recordPermission == .denied {@@ -170,9 +208,7 @@ public class XdtMobileRealtimeAudioModule: Module {}do {- try session.setCategory(.playAndRecord, mode: .measurement, options: [.duckOthers, .defaultToSpeaker])- try session.setActive(true)- ownsAudioSession = true+ try activateAudioSessionLocked(session)} catch {scheduleDeactivateLocked()throw Exception(name: "ERR_AUDIO_SESSION", description: "Failed to configure audio session: \(error.localizedDescription)")@@ -180,26 +216,58 @@ public class XdtMobileRealtimeAudioModule: Module {installAudioSessionObservers()- let inputNode = engine.inputNode- let inputFormat = inputNode.inputFormat(forBus: 0)- chunkIndex = 0- inputNode.removeTap(onBus: 0)- inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: inputFormat) { [weak self] buffer, time in- self?.handleInputBuffer(buffer, time: time)- }-do {- engine.prepare()- try engine.start()+ _ = try capture.start(withSampleRate: targetSampleRate)+ engineStartedAt = Date().timeIntervalSince1970 * 1000isCapturing = true+ let timer = DispatchSource.makeTimerSource(queue: sessionStateQueue)+ timer.schedule(deadline: .now(), repeating: .milliseconds(5), leeway: .milliseconds(1))+ timer.setEventHandler { [weak self] in+ guard let self else { return }+ self.collectChunksLocked(emit: true)+ if self.capture.overflowed {+ self.emitError("Realtime microphone delivery fell behind.")+ self.stopCaptureLocked(deactivateImmediately: true)+ }+ }+ drainTimer = timer+ timer.resume()} catch {removeAudioSessionObservers()- inputNode.removeTap(onBus: 0)+ capture.stop()scheduleDeactivateLocked()throw Exception(name: "ERR_AUDIO_ENGINE_START", description: "Failed to start realtime audio engine: \(error.localizedDescription)")}}+ private func activateAudioSessionLocked(_ session: AVAudioSession) throws {+ // Reuse only our live, unchanged session. Interruptions and route loss clear+ // ownership; another playback feature's configuration never counts as warm.+ let matches = session.category == .playAndRecord && session.mode == .measurement+ && session.categoryOptions.subtracting(.mixWithOthers) == [.duckOthers, .defaultToSpeaker]+ if !ownsAudioSession || !matches {+ try session.setCategory(.playAndRecord, mode: .measurement, options: [.duckOthers, .defaultToSpeaker])+ // A preference, not a promise: report the actual period after activation.+ try? session.setPreferredIOBufferDuration(0.010)+ try session.setActive(true)+ ownsAudioSession = true+ }+ sessionActivatedAt = Date().timeIntervalSince1970 * 1000+ }++ private func collectChunksLocked(emit: Bool) {+ for raw in capture.drain() {+ var chunk = raw as! [String: Any]+ chunk["captureId"] = captureId+ recentChunks.append(chunk)+ if recentChunks.count > 128 { recentChunks.removeFirst() }+ if emit {+ let payload = chunk+ DispatchQueue.main.async { [weak self] in self?.sendEvent(onAudioChunk, payload) }+ }+ }+ }+private func stopCapture(deactivateImmediately: Bool = false) {sessionStateQueue.sync {self.stopCaptureLocked(deactivateImmediately: deactivateImmediately)@@ -208,11 +276,11 @@ public class XdtMobileRealtimeAudioModule: Module {// Must run on sessionStateQueue.private func stopCaptureLocked(deactivateImmediately: Bool) {- removeAudioSessionObservers()- if engine.isRunning {- engine.inputNode.removeTap(onBus: 0)- engine.stop()- }+ if deactivateImmediately { removeAudioSessionObservers() }+ drainTimer?.cancel()+ drainTimer = nil+ capture.stop()+ collectChunksLocked(emit: false)isCapturing = falseif deactivateImmediately {// System interruptions and teardown must release the session right away —@@ -225,56 +293,10 @@ public class XdtMobileRealtimeAudioModule: Module {}}- private func handleInputBuffer(_ buffer: AVAudioPCMBuffer, time: AVAudioTime) {- guard let channelData = buffer.floatChannelData else {- emitError("Realtime audio input did not provide float PCM samples.")- return- }- let frameCount = Int(buffer.frameLength)- guard frameCount > 0 else {- return- }-- let inputSampleRate = buffer.format.sampleRate- let channelCount = max(1, Int(buffer.format.channelCount))- let targetFrameCount = max(1, Int((Double(frameCount) * targetSampleRate / inputSampleRate).rounded(.down)))- var pcm = Data(capacity: targetFrameCount * 2)- let scale = inputSampleRate / targetSampleRate-- for targetIndex in 0..<targetFrameCount {- let sourceIndex = min(frameCount - 1, Int(Double(targetIndex) * scale))- var mono: Float = 0- for channelIndex in 0..<channelCount {- mono += channelData[channelIndex][sourceIndex]- }- mono /= Float(channelCount)- let clamped = max(-1, min(1, mono))- var sample = Int16(clamped * Float(Int16.max)).littleEndian- withUnsafeBytes(of: &sample) { bytes in- pcm.append(contentsOf: bytes)- }- }-- let capturedAt = Date().timeIntervalSince1970 * 1000- let index = chunkIndex- chunkIndex += 1- let durationMs = Double(targetFrameCount) / targetSampleRate * 1000- let payload: [String: Any] = [- "base64Pcm16": pcm.base64EncodedString(),- "capturedAt": capturedAt,- "chunkIndex": index,- "sampleRate": Int(targetSampleRate),- "durationMs": durationMs- ]-- DispatchQueue.main.async { [weak self] in- self?.sendEvent(onAudioChunk, payload)- }- }-private func emitError(_ message: String) {+ let id = captureIdDispatchQueue.main.async { [weak self] in- self?.sendEvent(onAudioError, ["message": message])+ self?.sendEvent(onAudioError, ["message": message, "captureId": id])}}@@ -311,27 +333,27 @@ public class XdtMobileRealtimeAudioModule: Module {}private func handleAudioSessionInterruption(_ notification: Notification) {- guard isCapturing else {- return- }let rawType = unsignedIntegerValue(notification.userInfo?[AVAudioSessionInterruptionTypeKey])guard rawType == AVAudioSession.InterruptionType.began.rawValue else {return}- emitError("Realtime voice input was interrupted by the system.")- stopCapture(deactivateImmediately: true)+ sessionStateQueue.async { [weak self] in+ guard let self else { return }+ if self.isCapturing { self.emitError("Realtime voice input was interrupted by the system.") }+ self.stopCaptureLocked(deactivateImmediately: true)+ }}private func handleAudioRouteChange(_ notification: Notification) {- guard isCapturing else {- return- }let rawReason = unsignedIntegerValue(notification.userInfo?[AVAudioSessionRouteChangeReasonKey])guard rawReason == AVAudioSession.RouteChangeReason.oldDeviceUnavailable.rawValue else {return}- emitError("Realtime voice input microphone route became unavailable.")- stopCapture(deactivateImmediately: true)+ sessionStateQueue.async { [weak self] in+ guard let self else { return }+ if self.isCapturing { self.emitError("Realtime voice input microphone route became unavailable.") }+ self.stopCaptureLocked(deactivateImmediately: true)+ }}}@@ -348,19 +370,6 @@ private func numericOption(_ options: [String: Any]?, key: String) -> Double? {return nil}-private func integerOption(_ options: [String: Any]?, key: String) -> Int? {- if let value = options?[key] as? Int {- return value- }- if let value = options?[key] as? Double {- return Int(value)- }- if let value = options?[key] as? NSNumber {- return value.intValue- }- return nil-}-private func unsignedIntegerValue(_ value: Any?) -> UInt? {if let value = value as? UInt {return valuediff --git a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.tsindex ba60be1041..9332743048 100644--- a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts+++ b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts@@ -33,6 +33,80 @@ afterEach(async () => {});describe('mobileRealtimeAudio', () => {+ it('replays a packet rejected after stop even when its event beat the native start receipt', async () => {+ let listener!: (event: unknown) => void;+ let finishStart!: () => void;+ const packet = { chunkIndex: 0, base64Pcm16: '6APoAw==', capturedAt: 10, sampleRate: 16000, durationMs: .125 };+ const native = {+ addListener: vi.fn((name: string, callback: (event: unknown) => void) => {+ if (name === 'onAudioChunk') listener = callback;+ return { remove: vi.fn() };+ }),+ start: vi.fn(() => new Promise<void>((resolve) => { finishStart = resolve; })),+ stopAndDrain: vi.fn(async () => [packet]),+ };+ vi.mocked(requireNativeModule).mockReturnValue(native);+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');+ const onChunk = vi.fn((chunk: { drained?: boolean }) => chunk.drained === true);+ const opening = startMobileRealtimeAudio({ onChunk });+ await vi.waitFor(() => expect(finishStart).toBeTypeOf('function'));+ listener(packet);+ finishStart();+ await (await opening)(20);+ expect(native.stopAndDrain).toHaveBeenCalledWith(-1, 20);+ expect(onChunk.mock.calls.map(([chunk]) => chunk.drained)).toEqual([undefined, true]);+ });++ it('drains only undelivered pre-stop packets, deduplicates the receipt, and ignores a previous capture', async () => {+ type Packet = { captureId: string; chunkIndex: number; base64Pcm16: string; capturedAt: number; sampleRate: number; durationMs: number };+ let listener!: (event: Packet) => void;+ let captureId = '';+ const packet = (index: number): Packet => ({+ captureId, chunkIndex: index, base64Pcm16: '6APoAw==',+ capturedAt: 10, sampleRate: 16000, durationMs: .125,+ });+ const native = {+ addListener: vi.fn((name: string, callback: (event: Packet) => void) => {+ if (name === 'onAudioChunk') listener = callback;+ return { remove: vi.fn() };+ }),+ start: vi.fn(async (options: { captureId: string }) => { captureId = options.captureId; }),+ stop: vi.fn(async () => undefined),+ stopAndDrain: vi.fn(async () => [packet(0), packet(1)]),+ };+ vi.mocked(requireNativeModule).mockReturnValue(native);+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');+ const onChunk = vi.fn();+ const stop = await startMobileRealtimeAudio({ onChunk });+ listener({ ...packet(0), captureId: 'previous' });+ listener(packet(0));+ await stop(20);+ expect(native.stopAndDrain).toHaveBeenCalledWith(0, 20);+ expect(native.stop).not.toHaveBeenCalled();+ expect(onChunk.mock.calls.map(([chunk]) => [chunk.trace.chunkIndex, chunk.drained])).toEqual([[0, undefined], [1, true]]);+ expect([...new Int16Array(onChunk.mock.calls[1][0].pcm16)]).toEqual([1000, 1000]);+ await stop();+ expect(native.stopAndDrain).toHaveBeenCalledOnce();+ });++ it('coalesces concurrent native prewarm calls and allows a later prewarm', async () => {+ let finish!: () => void;+ const native = { prewarm: vi.fn(() => new Promise<void>((resolve) => { finish = resolve; })) };+ vi.mocked(requireNativeModule).mockReturnValue(native);+ const { prewarmMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');+ prewarmMobileRealtimeAudio();+ prewarmMobileRealtimeAudio();+ expect(native.prewarm).toHaveBeenCalledOnce();+ finish();+ await Promise.resolve();+ await Promise.resolve();+ prewarmMobileRealtimeAudio();+ expect(native.prewarm).toHaveBeenCalledTimes(2);+ finish();+ await Promise.resolve();+ await Promise.resolve();+ });+it.each([undefined, { inputPortTypes: ['MicrophoneBuiltIn'], inputSampleRate: 48000, inputChannels: 1 }])('logs input metadata and supports installed modules without it: %j', async (input) => {const native = {diff --git a/apps/mobile/src/__tests__/mobileVoiceController.test.ts b/apps/mobile/src/__tests__/mobileVoiceController.test.tsindex 4230c63a6b..64c417a020 100644--- a/apps/mobile/src/__tests__/mobileVoiceController.test.ts+++ b/apps/mobile/src/__tests__/mobileVoiceController.test.ts@@ -10,6 +10,7 @@ import type {AsrEvent,AsrProvider,AudioTrace,+ AudioGate,RefinementResult,} from '@cindy/voice-input-core';import type { StoredMobileVoiceCredential } from '@/session/mobileVoiceCredentialStore';@@ -164,6 +165,103 @@ function credential(): StoredMobileVoiceCredential {}describe('mobileVoiceController', () => {+ it('fails capture if pending speaker settings reject instead of uploading unfiltered audio', async () => {+ let reject!: (error: Error) => void;+ const asr = new FinalOnlyAsrProvider();+ const append = vi.spyOn(asr, 'appendAudio');+ const error = vi.fn();+ const session = createMobileVoiceControllerSession({+ credential: credential(), asr, refiner: null, initialDraft: '', onDraftChanged: vi.fn(), onError: error,+ startAudio: startAudibleAudio, audioGate: new Promise((_resolve, fail) => { reject = fail; }),+ });+ await session.start();+ reject(new Error('speaker settings unavailable'));+ await Promise.resolve();+ await expect(session.stop()).rejects.toThrow('speaker settings unavailable');+ expect(append).not.toHaveBeenCalled();+ expect(error).toHaveBeenCalledOnce();+ });++ it('keeps a short utterance delivered only by the stop receipt, while rejecting ordinary late events', async () => {+ const asr = new FinalOnlyAsrProvider();+ const append = vi.spyOn(asr, 'appendAudio');+ const session = createMobileVoiceControllerSession({+ credential: credential(), asr, refiner: null, initialDraft: '', onDraftChanged: vi.fn(),+ startAudio: async ({ onChunk }) => async () => {+ const chunk = { pcm16: new Int16Array(160).fill(1000).buffer,+ trace: { capturedAt: 1, convertedAt: 2, chunkIndex: 0, sampleRate: 16000, durationMs: 10 } };+ onChunk(chunk);+ onChunk({ ...chunk, drained: true });+ },+ });+ await session.start();+ expect(await session.stop()).toBe('raw final');+ expect(append).toHaveBeenCalledOnce();+ });++ it('waits for a pending native start and its drain before treating an immediate stop as silence', async () => {+ let open!: () => void;+ const asr = new FinalOnlyAsrProvider();+ const append = vi.spyOn(asr, 'appendAudio');+ const session = createMobileVoiceControllerSession({+ credential: credential(), asr, refiner: null, initialDraft: '', onDraftChanged: vi.fn(),+ startAudio: ({ onChunk }) => new Promise((resolve) => {+ open = () => resolve(async () => {+ onChunk({ pcm16: new Int16Array(160).fill(1000).buffer, drained: true,+ trace: { capturedAt: 1, convertedAt: 2, chunkIndex: 0, sampleRate: 16000, durationMs: 10 } });+ });+ }),+ });+ const starting = session.start();+ const stopping = session.stop();+ open();+ expect(await stopping).toBe('raw final');+ await starting;+ expect(append).toHaveBeenCalledOnce();+ });++ it('captures while the speaker settings load, and filters every retained chunk before ASR', async () => {+ let resolveGate!: (gate: AudioGate | undefined) => void;+ const pendingGate = new Promise<AudioGate | undefined>((resolve) => { resolveGate = resolve; });+ const asr = new FinalOnlyAsrProvider();+ const append = vi.spyOn(asr, 'appendAudio');+ let emit!: (chunk: ArrayBuffer, trace?: AudioTrace) => void;+ const gate: AudioGate = {+ begin: (callback) => { emit = callback; },+ push: (chunk, trace) => emit(new ArrayBuffer(chunk.byteLength), trace),+ finish: async () => undefined,+ abort: vi.fn(),+ };+ const open = vi.fn(startAudibleAudio);+ const session = createMobileVoiceControllerSession({+ credential: credential(), asr, refiner: null, initialDraft: '',+ audioGate: pendingGate, startAudio: open, onDraftChanged: vi.fn(),+ });+ await session.start();+ expect(open).toHaveBeenCalledOnce();+ expect(append).not.toHaveBeenCalled();+ resolveGate(gate);+ await session.stop();+ expect(append).toHaveBeenCalledOnce();+ expect(new Int16Array(append.mock.calls[0][0]).every((sample) => sample === 0)).toBe(true);+ });++ it('discards pending speaker audio on cancel, including a late settings result', async () => {+ let resolveGate!: (gate: AudioGate | undefined) => void;+ const pendingGate = new Promise<AudioGate | undefined>((resolve) => { resolveGate = resolve; });+ const asr = new FinalOnlyAsrProvider();+ const append = vi.spyOn(asr, 'appendAudio');+ const session = createMobileVoiceControllerSession({+ credential: credential(), asr, refiner: null, initialDraft: '',+ audioGate: pendingGate, startAudio: startAudibleAudio, onDraftChanged: vi.fn(),+ });+ await session.start();+ await session.cancel();+ resolveGate(undefined);+ await Promise.resolve();+ expect(append).not.toHaveBeenCalled();+ });+it.each([false, true])('removes selected quotes even when the first identical transcript arrives at stop (partials=%s)', async (partials) => {const initialDocument: ComposerDocument = { version: 1, nodes: [{ type: 'quote', quote: { text: 'selected quote' } }, { type: 'text', text: 'raw final' },diff --git a/apps/mobile/src/session/askCardVoiceStart.ts b/apps/mobile/src/session/askCardVoiceStart.tsindex c00689153a..6a796dbe8c 100644--- a/apps/mobile/src/session/askCardVoiceStart.ts+++ b/apps/mobile/src/session/askCardVoiceStart.ts@@ -51,6 +51,7 @@ export async function startAskCardDictation(input: {onRunCreated?: (run: { stop: () => Promise<void>; cancel: () => Promise<void> }) => void;onError: (message: string) => void;}): Promise<{ stop: () => Promise<void>; cancel: () => Promise<void> }> {+ const requestedAt = Date.now();if (!isMobileRealtimeAudioAvailable()) {throw new Error(mobileVoiceRealtimeAudioUnavailableError());}@@ -89,16 +90,12 @@ export async function startAskCardDictation(input: {credential.settings?.language,CINDY_MANAGED_REFINER_PROVIDER,);- let audioGate: Awaited<ReturnType<typeof createMobileSpeakerGate>> | undefined;- try {- audioGate = await createMobileSpeakerGate();- } catch {- audioGate = undefined;- }+ const audioGate = createMobileSpeakerGate().catch(() => undefined);const initialDraft = input.getText();const session = createMobileVoiceControllerSession({credential,- ...(audioGate ? { audioGate } : {}),+ audioGate,+ requestedAt,connectionProvider: (providerId) => voiceContext.createAsrConnection(providerId),refinerTargetProvider: (providerId, options) => voiceContext.createRefinerTarget(providerId, options),warmRefiner: (request) => voiceContext.warmRefiner(request),diff --git a/apps/mobile/src/session/mobilePendingAudioGate.ts b/apps/mobile/src/session/mobilePendingAudioGate.tsnew file mode 100644index 0000000000..86e9540ad9--- /dev/null+++ b/apps/mobile/src/session/mobilePendingAudioGate.ts@@ -0,0 +1,60 @@+import type { AudioGate, AudioTrace } from '@cindy/voice-input-core';++/**+ * Opening the microphone must not wait for AsyncStorage / Keychain. Hold the+ * first PCM locally until createMobileSpeakerGate's bounded hydration finishes;+ * no unfiltered samples reach the ASR while the account's settings are unknown.+ */+export class MobilePendingAudioGate implements AudioGate {+ private gate?: AudioGate;+ private ready = false;+ private emit?: (chunk: ArrayBuffer, trace?: AudioTrace) => void;+ private pending: Array<{ chunk: ArrayBuffer; trace?: AudioTrace }> = [];+ private readonly resolved: Promise<void>;++ constructor(gate: Promise<AudioGate | undefined>, onError: (error: unknown) => void) {+ this.resolved = gate.then((value) => {+ this.gate = value;+ this.ready = true;+ if (!this.emit) return;+ value?.begin(this.emit);+ const pending = this.pending;+ this.pending = [];+ for (const { chunk, trace } of pending) this.push(chunk, trace);+ }).catch((error: unknown) => {+ this.abort();+ onError(error);+ throw error;+ });+ // finish() observes failure. Never leak an unhandled rejection if cancelled.+ void this.resolved.catch(() => undefined);+ }++ begin(emit: (chunk: ArrayBuffer, trace?: AudioTrace) => void): void {+ this.abort();+ this.emit = emit;+ if (this.ready) this.gate?.begin(emit);+ }++ push(chunk: ArrayBuffer, trace?: AudioTrace): void {+ if (!this.emit) return;+ if (!this.ready) {+ this.pending.push({ chunk, trace });+ } else if (this.gate) {+ this.gate.push(chunk, trace);+ } else {+ this.emit(chunk, trace);+ }+ }++ async finish(): Promise<void> {+ await this.resolved;+ if (this.emit) await this.gate?.finish();+ }++ abort(): void {+ this.emit = undefined;+ this.pending = [];+ this.gate?.abort();+ }+}diff --git a/apps/mobile/src/session/mobileRealtimeAudio.ts b/apps/mobile/src/session/mobileRealtimeAudio.tsindex 4426f89633..2c5e5bb3d5 100644--- a/apps/mobile/src/session/mobileRealtimeAudio.ts+++ b/apps/mobile/src/session/mobileRealtimeAudio.ts@@ -10,9 +10,14 @@ import { logMobileVoice } from '@/session/mobileVoiceDiagnostics';export type MobileRealtimeAudioChunk = {pcm16: ArrayBuffer;trace: AudioTrace;+ /** Older native modules and Expo Audio timestamp JS delivery instead. */+ timestampSource?: 'hardware' | 'delivery';+ /** Explicit native stop receipt; never inferred from a late event. */+ drained?: boolean;};type NativeAudioChunkEvent = {+ captureId?: string;base64Pcm16: string;capturedAt: number;chunkIndex: number;@@ -22,15 +27,20 @@ type NativeAudioChunkEvent = {type NativeAudioErrorEvent = {message: string;+ captureId?: string;};type NativeRealtimeAudioModule = {- start(options?: { sampleRate?: number; bufferSize?: number }): Promise<{+ start(options?: { sampleRate?: number; bufferSize?: number; captureId?: string }): Promise<{inputPortTypes: string[];inputSampleRate: number;inputChannels: number;+ ioBufferDurationMs?: number;+ sessionActivatedAt?: number;+ engineStartedAt?: number;} | void>;stop(): Promise<void>;+ stopAndDrain?(afterIndex: number, stoppedAt: number): Promise<NativeAudioChunkEvent[]>;prewarm(): Promise<void>;};@@ -105,6 +115,8 @@ let expoAudioNativeModule: ExpoAudioNativeModule | null | undefined;// The custom Apple module owns one native engine. A replacement must wait for// the previous caller's complete start/stop lifecycle, including late cleanup.let customNativeCaptureReleased: Promise<void> = Promise.resolve();+let nextCaptureId = 0;+let nativePrewarm: Promise<void> | undefined;export function isMobileRealtimeAudioAvailable(): boolean {if (isE2eMockRealtimeAudioEnabled()) return true;@@ -125,12 +137,12 @@ export function shouldShowMobileVoiceUi(export async function startMobileRealtimeAudio(options: {- onChunk: (chunk: MobileRealtimeAudioChunk) => void;+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;onError?: (error: Error) => void;sampleRate?: number;bufferSize?: number;},-): Promise<() => Promise<void>> {+): Promise<(stoppedAt?: number) => Promise<void>> {if (isE2eMockRealtimeAudioEnabled()) {return startE2eMockRealtimeAudio(options);}@@ -148,36 +160,50 @@ export async function startMobileRealtimeAudio(async function startCustomNativeRealtimeAudio(binding: RealtimeAudioNativeBinding,options: {- onChunk: (chunk: MobileRealtimeAudioChunk) => void;+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;onError?: (error: Error) => void;sampleRate?: number;bufferSize?: number;},-): Promise<() => Promise<void>> {+): Promise<(stoppedAt?: number) => Promise<void>> {const previousCapture = customNativeCaptureReleased;let releaseCapture!: () => void;customNativeCaptureReleased = new Promise<void>((resolve) => { releaseCapture = resolve; });await previousCapture;const subscriptions: EventSubscription[] = [];+ const captureId = String(++nextCaptureId);+ let lastChunkIndex = -1;+ const deliver = (event: NativeAudioChunkEvent, drained = false): void => {+ if (event.captureId !== undefined && event.captureId !== captureId) return;+ if (event.chunkIndex <= lastChunkIndex) return;+ const accepted = options.onChunk({+ pcm16: decodeBase64ToArrayBuffer(event.base64Pcm16),+ trace: {+ capturedAt: event.capturedAt,+ convertedAt: Date.now(),+ chunkIndex: event.chunkIndex,+ sampleRate: event.sampleRate,+ durationMs: event.durationMs,+ },+ timestampSource: binding.module.stopAndDrain ? 'hardware' : 'delivery',+ ...(drained ? { drained: true } : {}),+ });+ // A stop can beat the native start receipt. Audio rejected as an ordinary+ // late event must remain replayable through the explicit drain receipt.+ if (accepted !== false) lastChunkIndex = event.chunkIndex;+ };try {subscriptions.push(binding.module.addListener('onAudioChunk', (event) => {- options.onChunk({- pcm16: decodeBase64ToArrayBuffer(event.base64Pcm16),- trace: {- capturedAt: event.capturedAt,- convertedAt: Date.now(),- chunkIndex: event.chunkIndex,- sampleRate: event.sampleRate,- durationMs: event.durationMs,- },- });+ deliver(event);}));subscriptions.push(binding.module.addListener('onAudioError', (event) => {+ if (event.captureId !== undefined && event.captureId !== captureId) return;options.onError?.(new Error(event.message));}));const input = await binding.module.start({sampleRate: options.sampleRate ?? 16_000,bufferSize: options.bufferSize ?? DEFAULT_NATIVE_AUDIO_BUFFER_SIZE,+ captureId,});logMobileVoice('debug', 'capture input', {backend: 'ios-native',@@ -185,6 +211,11 @@ async function startCustomNativeRealtimeAudio(inputPortTypes: input?.inputPortTypes ?? ['unknown'],inputSampleRate: input?.inputSampleRate,inputChannels: input?.inputChannels,+ ...(input?.ioBufferDurationMs === undefined ? {} : {+ ioBufferDurationMs: input.ioBufferDurationMs,+ sessionActivatedAt: input.sessionActivatedAt,+ engineStartedAt: input.engineStartedAt,+ }),});} catch (error) {try {@@ -196,13 +227,19 @@ async function startCustomNativeRealtimeAudio(}let stopping: Promise<void> | undefined;- return () => {+ return (stopRequestedAt) => {stopping ??= (async () => {+ const stoppedAt = stopRequestedAt ?? Date.now();try {try {subscriptions.forEach((subscription) => subscription.remove());} finally {- await binding.module.stop();+ if (binding.module.stopAndDrain) {+ const chunks = await binding.module.stopAndDrain(lastChunkIndex, stoppedAt);+ for (const chunk of chunks) deliver(chunk, true);+ } else {+ await binding.module.stop();+ }}} finally {releaseCapture();@@ -221,7 +258,7 @@ async function startCustomNativeRealtimeAudio(async function startExpoAudioRealtimeAudio(module: ExpoAudioNativeModule,options: {- onChunk: (chunk: MobileRealtimeAudioChunk) => void;+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;onError?: (error: Error) => void;sampleRate?: number;},@@ -342,7 +379,9 @@ export function prewarmMobileRealtimeAudio(): void {if (isE2eMockRealtimeAudioEnabled()) return;const binding = getNativeBinding();if (!binding) return;- void binding.module.prewarm().catch(() => undefined);+ nativePrewarm ??= binding.module.prewarm().catch(() => undefined).finally(() => {+ nativePrewarm = undefined;+ });}export const __testing = {@@ -544,7 +583,7 @@ function isE2eMockRealtimeAudioEnabled(): boolean {}function startE2eMockRealtimeAudio(options: {- onChunk: (chunk: MobileRealtimeAudioChunk) => void;+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;sampleRate?: number;}): Promise<() => Promise<void>> {let stopped = false;diff --git a/apps/mobile/src/session/mobileSpeakerFilter.ts b/apps/mobile/src/session/mobileSpeakerFilter.tsindex 8a8e4da87b..a472a4c551 100644--- a/apps/mobile/src/session/mobileSpeakerFilter.ts+++ b/apps/mobile/src/session/mobileSpeakerFilter.ts@@ -37,8 +37,9 @@ import { logMobileVoice } from './mobileVoiceDiagnostics';* "Recognize my voice" on mobile. Same models, thresholds and gate as desktop;* inference runs through ONNX Runtime in the cindy-speaker-onnx module.*- * Dictation start waits for saved preferences and the voiceprint before- * choosing its gate. Model loading stays lazy; failures still pass through.+ * Audio forwarding waits for saved preferences and the voiceprint before+ * choosing its gate; microphone capture may buffer locally meanwhile.+ * Model loading stays lazy; failures still pass through.** The voiceprint is biometric data: it stays on this device, in the OS* keychain/keystore through SecureStore, one per account, and is neverdiff --git a/apps/mobile/src/session/mobileVoiceController.ts b/apps/mobile/src/session/mobileVoiceController.tsindex 7f649b16b4..1f8d4ffc26 100644--- a/apps/mobile/src/session/mobileVoiceController.ts+++ b/apps/mobile/src/session/mobileVoiceController.ts@@ -33,8 +33,9 @@ import {mobileVoiceTranscriptKeptError,type MobileVoiceDraftInsertion,} from '@/session/mobileVoiceInput';-import { startMobileRealtimeAudio } from '@/session/mobileRealtimeAudio';+import { startMobileRealtimeAudio, type MobileRealtimeAudioChunk } from '@/session/mobileRealtimeAudio';import { MobileVoiceAudioLevels } from '@/session/mobileVoiceAudioLevels';+import { MobilePendingAudioGate } from '@/session/mobilePendingAudioGate';import { readCachedMobileVoiceDictionary } from '@/session/mobileVoiceDictionaryCache';import {classifyMobileVoiceFailure,@@ -45,9 +46,9 @@ import {type StartRealtimeAudio = (options: {sampleRate: number;- onChunk: (chunk: { pcm16: ArrayBuffer; trace: AudioTrace }) => void;+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;onError?: (error: Error) => void;-}) => Promise<() => Promise<void>>;+}) => Promise<(stoppedAt?: number) => Promise<void>>;type MobileDictationRefiner = {refine(input: {@@ -113,7 +114,9 @@ type MobileVoiceControllerOptions = {* Optional filter between the microphone and the ASR ("recognize my voice").* Undefined keeps the unfiltered behavior.*/- audioGate?: AudioGate;+ audioGate?: AudioGate | Promise<AudioGate | undefined>;+ /** Gesture time, before permission / storage work, for end-to-end diagnostics. */+ requestedAt?: number;};type MobileVoiceInputHistorySource = readonly string[] | (() => readonly string[] | undefined);@@ -180,7 +183,10 @@ export function createMobileVoiceControllerSession(let lastPublishedDraft = baseDraft;let pendingDraftToPublish: string | null = null;let draftPublishThrottleTimer: ReturnType<typeof setTimeout> | null = null;- let stopAudio: (() => Promise<void>) | null = null;+ let stopAudio: ((stoppedAt?: number) => Promise<void>) | null = null;+ let audioOpening: Promise<void> | null = null;+ let audioStopping: Promise<void> | null = null;+ let captureStopRequestedAt: number | undefined;let state: VoiceInputState = 'idle';let controllerError: Error | null = null;// Run lifecycle as an explicit phase machine instead of a set of overlapping@@ -193,9 +199,9 @@ export function createMobileVoiceControllerSession(// any → running (a fresh start() begins a new run)// Every teardown-ish question is answered from the phase, so the answers can// never disagree with each other the way independent flags could:- // - captureTornDown(): should onChunk drop mic audio? (post-stop straggler- // chunks must never be buffered/flushed — a stop() that waits for a slow- // handshake would otherwise transcribe speech after the user stopped)+ // - captureTornDown(): drop ordinary post-stop events. Only an explicit+ // native drain receipt, clipped to the user's stop time, may still enter+ // the buffer while stopping; a slow ASR handshake cannot extend capture.// - phase === 'failed': did the run die (mic or connect failure)?// - phase === 'cancelled': was the run abandoned on purpose (not an error)?let runPhase: 'idle' | 'running' | 'stopping' | 'cancelled' | 'failed' = 'idle';@@ -316,10 +322,17 @@ export function createMobileVoiceControllerSession(// was deferred at start until the mic is actually capturing, so the UI's// "listening" cue matches real audio flow rather than the ~150-200ms-earlier// moment the audio session began warming up.- const markCaptureLive = (): void => {+ const markCaptureLive = (trace: AudioTrace, timestampSource?: 'hardware' | 'delivery'): void => {if (captureLive) return;captureLive = true;- logMobileVoice('debug', 'capture live', { runId: runTag(), elapsedMs: Date.now() - runStartedAt });+ logMobileVoice('debug', 'capture live', {+ runId: runTag(), elapsedMs: Date.now() - runStartedAt,+ timestampSource: timestampSource ?? 'delivery',+ ...(timestampSource === 'hardware' ? {+ firstSampleAfterRequestMs: trace.capturedAt - runStartedAt,+ deliveryDelayMs: Date.now() - trace.capturedAt,+ } : {}),+ });if (listeningPendingCaptureLive && state === 'listening') {listeningPendingCaptureLive = false;options.onStateChanged?.('listening');@@ -399,7 +412,11 @@ export function createMobileVoiceControllerSession(// A reconnect allocates another managed session and can hit the account// limit; report that instead of the generic "stopped receiving" failure.recoveryErrorMessage: mobileVoiceRateLimitMessage,- audioGate: options.audioGate,+ audioGate: options.audioGate instanceof Promise+ ? new MobilePendingAudioGate(options.audioGate, (error) => {+ void handleAudioFailure(error instanceof Error ? error : new Error(String(error)));+ })+ : options.audioGate,logger: new VoiceTimelineLogger((event) => {recordTimeline(event);options.onTimelineEvent?.(event);@@ -487,7 +504,9 @@ export function createMobileVoiceControllerSession(listeningPendingCaptureLive = false;soundDetected = false;audioLevels = new MobileVoiceAudioLevels();- runStartedAt = Date.now();+ audioStopping = null;+ captureStopRequestedAt = undefined;+ runStartedAt = options.requestedAt ?? Date.now();// A fresh start() begins a new run from ANY previous phase — in particular// 'failed': a mic interruption must not poison the next recording on the// same session, so the failure record is cleared here too.@@ -516,22 +535,23 @@ export function createMobileVoiceControllerSession(refinement: Boolean(refiner),});try {- stopAudio = await startAudio({+ audioOpening = startAudio({sampleRate: options.credential.asr.pcmSampleRate ?? 16_000,onChunk: (chunk) => {// Drop audio emitted after the run began tearing down capture, so an// early stop that waits for the in-flight handshake can't buffer and// later flush post-stop speech/noise.- if (captureTornDown()) return;+ if (captureTornDown() && !(runPhase === 'stopping' && chunk.drained)) return false;audioLevels.append(chunk.pcm16, chunk.trace.sampleRate ?? options.credential.asr.pcmSampleRate ?? 16_000);if (!soundDetected && hasPcmSound(chunk.pcm16)) soundDetected = true;- markCaptureLive();+ if (runPhase === 'running') markCaptureLive(chunk.trace, chunk.timestampSource);controller.appendAudio(chunk.pcm16, chunk.trace);},onError: (error) => {void handleAudioFailure(error);},- });+ }).then((stop) => { stopAudio = stop; });+ await audioOpening;} catch (error) {runPhase = 'failed';// Surface the mic failure immediately — do NOT wait for the in-flight@@ -549,9 +569,7 @@ export function createMobileVoiceControllerSession(// opening: its teardown saw stopAudio === null and could not close the// capture, so close the just-opened handle here — otherwise the mic and// its event subscriptions stay live even though the run is abandoned.- const stop = stopAudio;- stopAudio = null;- await stop().catch(() => undefined);+ await stopNativeCapture();}// Wait for the ASR connection to settle before declaring the start// successful, so a failed handshake still surfaces as a startup error (and@@ -601,6 +619,7 @@ export function createMobileVoiceControllerSession(// the connect settles, so short utterances are not lost.if (runPhase === 'running') runPhase = 'stopping';const stopRequestedAt = Date.now();+ captureStopRequestedAt = stopRequestedAt;logMobileVoice('debug', 'stop requested', {runId: runTag(),elapsedMs: stopRequestedAt - runStartedAt,@@ -609,11 +628,10 @@ export function createMobileVoiceControllerSession(...audioLevels.summary(),hasTranscript: Boolean(voiceInsertion),});- if (stopAudio) {- const stop = stopAudio;- stopAudio = null;- await stop().catch(() => undefined);- }+ // An immediate release can beat the native start receipt. Wait only for+ // capture (never ASR), then drain before deciding this was a silent run.+ if (!stopAudio && !audioStopping) await audioOpening?.catch(() => undefined);+ await stopNativeCapture();if (currentRunPhase() === 'stopping' && !soundDetected && !voiceInsertion) {logMobileVoice('info', 'silent recording ended without ASR finalization', {runId: runTag(),@@ -700,6 +718,17 @@ export function createMobileVoiceControllerSession(}}+ async function stopNativeCapture(): Promise<void> {+ if (audioStopping) return audioStopping;+ const stop = stopAudio;+ if (!stop) return;+ stopAudio = null;+ audioStopping = stop(captureStopRequestedAt).catch(async (error: unknown) => {+ await handleAudioFailure(error instanceof Error ? error : new Error(String(error)));+ });+ return audioStopping;+ }+function recordSubmittedHistory(text: string): void {if (!options.recordHistory) return;// 下面的 .catch 只兜得住异步拒绝;同步抛会顺着 onSubmitted 冒出 controller,