Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

perf(voice): 提前启动收音并补齐停止前的音频

Dash238fc3853344
diff --git a/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.ts b/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.ts
index 5e7dfa5a31..20d8c3b7ab 100644
--- a/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.ts
+++ b/apps/desktop/src/renderer/voice-input/WebMicAudioEngine.ts
@@ -373,25 +373,6 @@ function readErrorString(error: unknown, key: 'name' | 'message' | 'reason'): st
return typeof value === 'string' ? value : undefined;
}
-async function assertSelectedMicrophoneAvailable(deviceId?: string): Promise<void> {
- // Browser/Electron expose the system default microphone as the synthetic
- // "default" device. Enumerating devices before every start adds startup
- // latency and cannot prove that this moving target is unavailable, so let
- // getUserMedia surface the platform error for automatic/default capture.
- if (!isSpecificMicrophoneDeviceId(deviceId)) return;
- const devices = await navigator.mediaDevices.enumerateDevices().catch(() => []);
- const audioInputs = devices.filter((device) => device.kind === 'audioinput');
- // Before permission is granted, browsers may expose opaque/empty IDs. Only
- // fail fast when the list has comparable IDs and the selected one is
- // definitely absent; otherwise let getUserMedia surface the platform error.
- if (
- audioInputs.some((device) => Boolean(device.deviceId)) &&
- !audioInputs.some((device) => device.deviceId === deviceId)
- ) {
- throw new VoiceInputSelectedMicrophoneUnavailableError();
- }
-}
-
function keepAliveKey(options: KeepAliveSessionOptions): KeepAliveSessionKey {
return JSON.stringify({
workletUrl: options.workletUrl,
@@ -527,12 +508,6 @@ class KeepAliveMicSession {
private async startInternal(): Promise<void> {
ensureKeepAliveDeviceChangeListener();
ensureKeepAlivePowerReleaseListener();
- // Also guarded: if a release lands while enumerating and the device then
- // reads as absent, the raw VoiceInputSelectedMicrophoneUnavailableError
- // would send prewarm/capture down their automatic-fallback paths and open
- // the *default* microphone after the power event.
- await this.awaitStartupStep(assertSelectedMicrophoneAvailable(this.options.deviceId));
-
// A release (suspend/lock/setting off) can land while this is pending, in
// which case bailing out avoids opening a device nothing could then close.
const sharedState = await this.awaitStartupStep(prewarmVoiceInputAudio(this.options.workletUrl));
@@ -1222,13 +1197,8 @@ export class WebMicAudioEngine {
const benchmarkFixturePromise = BENCHMARK_FIXTURE_ENABLED
? prewarmVoiceInputBenchmarkFixture().catch(() => null)
: Promise.resolve(null);
- await this.awaitDirectStartupStep(
- assertSelectedMicrophoneAvailable(this.deviceId),
- powerGenerationAtStart,
- );
- // The enumeration above passed its own check, but a release can land in the
- // gap before we ask for the device. Without this the request would be made
- // after the one-shot event and only be closed once it resolves.
+ // exact deviceId makes getUserMedia authoritative. A separate enumeration
+ // adds a serial device round trip and can become stale before opening it.
this.assertNoPowerReleaseDuringStartup(powerGenerationAtStart);
const streamPromise = this.awaitDirectStartupStep(
navigator.mediaDevices.getUserMedia({
diff --git a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.ts b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.ts
index 042c5a97b4..9fb8513ecb 100644
--- a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.ts
+++ b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.keepAliveIdle.test.ts
@@ -675,36 +675,18 @@ describe('keep-alive microphone idle window', () => {
expect(track.stopped).toBe(false);
});
- it('does not open the microphone when enumeration outlives a power release', async () => {
- let resolveEnumerate: (value: unknown) => void = () => undefined;
- vi.stubGlobal('navigator', {
- mediaDevices: {
- enumerateDevices: vi.fn(() => new Promise((resolve) => {
- resolveEnumerate = resolve;
- })),
- getUserMedia,
- addEventListener: vi.fn(),
- },
- });
-
- const engine = new mod.WebMicAudioEngine({
- workletUrl: WORKLET_URL,
- keepAlive: false,
- deviceId: 'specific-device',
- onInterrupted: vi.fn(),
- });
- const starting = engine.start();
- await vi.advanceTimersByTimeAsync(0);
- expect(powerCallback).toBeDefined();
-
+ it('opens an exact device without waiting for enumeration and still releases it on lock', async () => {
+ const enumerate = vi.fn(() => new Promise(() => undefined));
+ vi.stubGlobal('navigator', { mediaDevices: { enumerateDevices: enumerate, getUserMedia, addEventListener: vi.fn() } });
+ const engine = new mod.WebMicAudioEngine({ workletUrl: WORKLET_URL, deviceId: 'specific-device' });
+ await engine.start();
+ expect(enumerate).not.toHaveBeenCalled();
+ expect(getUserMedia).toHaveBeenCalledOnce();
+ expect(getUserMedia.mock.calls[0][0].audio.deviceId).toEqual({ exact: 'specific-device' });
powerCallback?.({ reason: 'screen_locked' });
await vi.advanceTimersByTimeAsync(0);
- // Enumeration succeeds *after* the release, reporting the device as present.
- resolveEnumerate([{ kind: 'audioinput', deviceId: 'specific-device' }]);
-
- await expect(starting).rejects.toThrow(/disposed/i);
- // The one-shot event has passed; nothing may reopen the device now.
- expect(getUserMedia).not.toHaveBeenCalled();
+ expect(track.stopped).toBe(true);
+ expect(getUserMedia).toHaveBeenCalledOnce();
});
it('does not report an error when a release hits a still-starting capture', async () => {
diff --git a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.ts b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.ts
index 9e1293b9fe..89efec6139 100644
--- a/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.ts
+++ b/apps/desktop/src/renderer/voice-input/__tests__/WebMicAudioEngine.prewarm.test.ts
@@ -192,10 +192,11 @@ describe('prewarmVoiceInputMicrophoneWithAutomaticFallback', () => {
});
it('closes a temporary replacement device even when preparation could not open the selected one', async () => {
+ getUserMedia.mockRejectedValueOnce(new DOMException('Device missing', 'NotFoundError'));
const release = prepareVoiceInputMicrophone({
workletUrl: 'https://app.local/pcm16k-worklet.js', deviceId: 'missing-device',
});
- await vi.waitFor(() => expect(enumerateDevices).toHaveBeenCalledOnce());
+ await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledOnce());
release?.();
const engine = new WebMicAudioEngine({
workletUrl: 'https://app.local/pcm16k-worklet.js', keepAlive: true, releaseAfterUse: true,
@@ -261,6 +262,7 @@ describe('prewarmVoiceInputMicrophoneWithAutomaticFallback', () => {
it('falls back to automatic microphone prewarm when the selected device is unavailable', async () => {
const onFallback = vi.fn();
+ getUserMedia.mockRejectedValueOnce(new DOMException('Device missing', 'OverconstrainedError'));
await prewarmVoiceInputMicrophoneWithAutomaticFallback({
workletUrl: 'https://app.local/pcm16k-worklet.js',
@@ -268,8 +270,9 @@ describe('prewarmVoiceInputMicrophoneWithAutomaticFallback', () => {
}, onFallback);
expect(onFallback).toHaveBeenCalledTimes(1);
- expect(enumerateDevices).toHaveBeenCalledTimes(1);
- expect(getUserMedia).toHaveBeenCalledTimes(1);
+ expect(enumerateDevices).not.toHaveBeenCalled();
+ expect(getUserMedia).toHaveBeenCalledTimes(2);
+ expect(getUserMedia.mock.calls[0][0].audio.deviceId).toEqual({ exact: 'missing-device' });
expect(getUserMedia).toHaveBeenCalledWith({
audio: {
autoGainControl: true,
diff --git a/apps/mobile/app/sessions/[sessionId].tsx b/apps/mobile/app/sessions/[sessionId].tsx
index de14aea296..16f44d54de 100644
--- a/apps/mobile/app/sessions/[sessionId].tsx
+++ b/apps/mobile/app/sessions/[sessionId].tsx
@@ -548,7 +548,6 @@ import {
} from '@/session/composerTextPlatformMetrics';
import {
isMobileRealtimeAudioAvailable,
- prewarmMobileRealtimeAudio,
shouldShowMobileVoiceUi,
} from '@/session/mobileRealtimeAudio';
import {
@@ -5532,6 +5531,7 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: S
}, [applyComposerDocument]);
const startVoiceRecording = useCallback(async () => {
+ const requestedAt = Date.now();
if (
voicePermissionRequestInFlightRef.current
|| voiceStartupInFlightRef.current
@@ -5650,7 +5650,7 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: S
credential.settings?.language,
CINDY_MANAGED_REFINER_PROVIDER,
);
- const audioGate = await createMobileSpeakerGate();
+ const audioGate = createMobileSpeakerGate();
if (voiceStartupSeqRef.current !== startupSeq) {
// This run never opened the microphone; release only its claimed ASR.
void prewarmedVoice?.asr.stop().catch(() => undefined);
@@ -5666,6 +5666,7 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: S
credential,
// "Recognize my voice": undefined unless enabled, enrolled and the models are on disk.
audioGate,
+ requestedAt,
...(prewarmedVoice ? { asr: prewarmedVoice.asr } : {}),
connectionProvider: (providerId: string) => voiceContext.createAsrConnection(providerId),
refinerTargetProvider: (providerId: string, options?: { refreshAccessToken?: boolean }) =>
@@ -5944,9 +5945,6 @@ function SessionScreenContent({ beforeSessionSwitchRef, newSessionActionRef }: S
if (voiceIsProcessing) return;
if (voiceRecordingActiveRef.current || voiceState === 'listening') return;
if (!deviceId || !isMobileRealtimeAudioAvailable()) return;
- // Keep the native audio-session warmup on the synchronous press-down path
- // (prewarmMobileVoiceStart re-runs it idempotently below).
- prewarmMobileRealtimeAudio();
warmMobileSpeakerFilter();
// 托管预热:凭登录态提前拿 voice-server 票据并开 ASR WebSocket。
prewarmMobileVoiceStart(deviceId, {
diff --git a/apps/mobile/app/sessions/new.tsx b/apps/mobile/app/sessions/new.tsx
index a3e2b1dbcf..ff1496dd8f 100644
--- a/apps/mobile/app/sessions/new.tsx
+++ b/apps/mobile/app/sessions/new.tsx
@@ -340,7 +340,6 @@ import {
} from '@/session/composerTextPlatformMetrics';
import {
isMobileRealtimeAudioAvailable,
- prewarmMobileRealtimeAudio,
shouldShowMobileVoiceUi,
} from '@/session/mobileRealtimeAudio';
import {
@@ -2988,6 +2987,7 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: bool
};
const startVoiceRecording = useCallback(async () => {
+ const requestedAt = Date.now();
if (
voicePermissionRequestInFlightRef.current
|| voiceStartupInFlightRef.current
@@ -3099,7 +3099,7 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: bool
credential.settings?.language,
CINDY_MANAGED_REFINER_PROVIDER,
);
- const audioGate = await createMobileSpeakerGate();
+ const audioGate = createMobileSpeakerGate();
if (voiceStartupSeqRef.current !== startupSeq) {
// This run never opened the microphone; release only its claimed ASR.
void prewarmedVoice?.asr.stop().catch(() => undefined);
@@ -3114,6 +3114,7 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: bool
credential,
// "Recognize my voice": undefined unless enabled, enrolled and the models are on disk.
audioGate,
+ requestedAt,
...(prewarmedVoice ? { asr: prewarmedVoice.asr } : {}),
connectionProvider: (providerId: string) => voiceContext.createAsrConnection(providerId),
refinerTargetProvider: (providerId: string, options?: { refreshAccessToken?: boolean }) =>
@@ -3233,9 +3234,6 @@ function NewRemoteSessionContent({ creatingRef }: { creatingRef: { current: bool
if (creating || voiceIsProcessing) return;
if (voiceRecordingActiveRef.current || voiceState === 'listening') return;
if (!selectedDeviceId || !isMobileRealtimeAudioAvailable()) return;
- // Keep the native audio-session warmup on the synchronous press-down path
- // (prewarmMobileVoiceStart re-runs it idempotently below).
- prewarmMobileRealtimeAudio();
warmMobileSpeakerFilter();
// 托管预热:凭登录态提前拿 voice-server 票据并开 ASR WebSocket。
prewarmMobileVoiceStart(selectedDeviceId, {
diff --git a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.h b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.h
new file mode 100644
index 0000000000..e80f06963f
--- /dev/null
+++ b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.h
@@ -0,0 +1,14 @@
+#import <AVFoundation/AVFoundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+/// Single-producer audio ring. Only copy/downmix runs on the render thread;
+/// resampling and bridging run on the caller's serialized non-realtime queue.
+@interface XdtAudioCapture : NSObject
+@property(nonatomic, readonly) AVAudioEngine *engine;
+@property(nonatomic, readonly) double inputSampleRate;
+@property(nonatomic, readonly) BOOL overflowed;
+- (BOOL)startWithSampleRate:(double)sampleRate error:(NSError **)error;
+- (NSArray<NSDictionary *> *)drain;
+- (void)stop;
+@end
+NS_ASSUME_NONNULL_END
diff --git a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.m b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.m
new file mode 100644
index 0000000000..b02bc41c35
--- /dev/null
+++ b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtAudioCapture.m
@@ -0,0 +1,168 @@
+#import "XdtAudioCapture.h"
+#import <stdatomic.h>
+#import <mach/mach_time.h>
+#import <math.h>
+
+// 4 MiB, allocated before engine.start. At a 10ms I/O period the ring tolerates
+// 2.56s of consumer starvation. Overflow is an explicit incomplete-capture error.
+enum { SlotCount = 256, MaxFrames = 4096 };
+typedef struct { float samples[MaxFrames]; UInt32 frames; uint64_t hostTime; } AudioSlot;
+typedef struct {
+ AudioSlot slots[SlotCount];
+ _Atomic(uint32_t) written;
+ _Atomic(uint32_t) read;
+ _Atomic(bool) overflow;
+} AudioRing;
+
+static OSStatus Capture(AudioRing *ring, const AudioTimeStamp *time,
+ AVAudioFrameCount frames, const AudioBufferList *input) {
+ uint32_t written = atomic_load_explicit(&ring->written, memory_order_relaxed);
+ uint32_t read = atomic_load_explicit(&ring->read, memory_order_acquire);
+ if (frames > MaxFrames || written - read >= SlotCount) {
+ atomic_store_explicit(&ring->overflow, true, memory_order_release);
+ return noErr;
+ }
+ AudioSlot *slot = &ring->slots[written % SlotCount];
+ UInt32 channels = 0;
+ for (UInt32 b = 0; b < input->mNumberBuffers; b++) channels += input->mBuffers[b].mNumberChannels;
+ if (channels == 0) return noErr;
+ for (UInt32 f = 0; f < frames; f++) {
+ float sample = 0;
+ for (UInt32 b = 0; b < input->mNumberBuffers; b++) {
+ const AudioBuffer *buffer = &input->mBuffers[b];
+ const float *data = buffer->mData;
+ for (UInt32 c = 0; c < buffer->mNumberChannels; c++) {
+ sample += data ? data[f * buffer->mNumberChannels + c] : 0;
+ }
+ }
+ slot->samples[f] = sample / channels;
+ }
+ slot->frames = frames;
+ slot->hostTime = (time->mFlags & kAudioTimeStampHostTimeValid) ? time->mHostTime : mach_absolute_time();
+ atomic_store_explicit(&ring->written, written + 1, memory_order_release);
+ return noErr;
+}
+
+@implementation XdtAudioCapture {
+ AudioRing *_ring;
+ AVAudioSinkNode *_sink;
+ double _targetRate;
+ double _hostTicksToMs;
+ double _wallAnchor;
+ uint64_t _hostAnchor;
+ uint64_t _sourceFrames;
+ uint64_t _outputFrames;
+ NSMutableData *_pending;
+ double _pendingCapturedAt;
+ NSInteger _chunkIndex;
+}
+
+- (instancetype)init {
+ if ((self = [super init])) {
+ _ring = calloc(1, sizeof(AudioRing));
+ if (!_ring) return nil;
+ _engine = [AVAudioEngine new];
+ mach_timebase_info_data_t info;
+ mach_timebase_info(&info);
+ _hostTicksToMs = (double)info.numer / info.denom / 1e6;
+ }
+ return self;
+}
+
+- (void)dealloc {
+ [self stop];
+ free(_ring);
+}
+
+- (BOOL)overflowed { return atomic_load_explicit(&_ring->overflow, memory_order_acquire); }
+
+- (BOOL)startWithSampleRate:(double)sampleRate error:(NSError **)error {
+ [self stop];
+ // A graph whose input unit survived session deactivation can accept start()
+ // yet deliver no frames after a route/configuration change. Create it only
+ // after the caller has activated the current session.
+ _engine = [AVAudioEngine new];
+ AVAudioInputNode *input = _engine.inputNode;
+ AVAudioFormat *format = [input outputFormatForBus:0];
+ if (format.commonFormat != AVAudioPCMFormatFloat32 || format.sampleRate <= 0 || format.channelCount == 0 ||
+ !isfinite(sampleRate) || sampleRate < 8000 || sampleRate > 192000) {
+ if (error) *error = [NSError errorWithDomain:@"XdtAudioCapture" code:1
+ userInfo:@{NSLocalizedDescriptionKey: @"Microphone did not provide a supported PCM format."}];
+ return NO;
+ }
+ atomic_store(&_ring->written, 0);
+ atomic_store(&_ring->read, 0);
+ atomic_store(&_ring->overflow, false);
+ _inputSampleRate = format.sampleRate;
+ _targetRate = sampleRate;
+ _sourceFrames = _outputFrames = 0;
+ _chunkIndex = 0;
+ _pending = [NSMutableData new];
+ _hostAnchor = mach_absolute_time();
+ _wallAnchor = NSDate.date.timeIntervalSince1970 * 1000;
+ AudioRing *ring = _ring;
+ // No ObjC/Swift calls, allocation, dispatch or locks in this receiver.
+ _sink = [[AVAudioSinkNode alloc] initWithReceiverBlock:^OSStatus(
+ const AudioTimeStamp *time, AVAudioFrameCount frames, const AudioBufferList *data) {
+ return Capture(ring, time, frames, data);
+ }];
+ [_engine attachNode:_sink];
+ [_engine connect:input to:_sink format:format];
+ [_engine prepare];
+ if ([_engine startAndReturnError:error]) return YES;
+ [self stop];
+ return NO;
+}
+
+- (NSDictionary *)takePending {
+ NSUInteger frames = _pending.length / sizeof(int16_t);
+ NSDictionary *packet = @{
+ @"base64Pcm16": [_pending base64EncodedStringWithOptions:0],
+ @"capturedAt": @(_pendingCapturedAt), @"chunkIndex": @(_chunkIndex++),
+ @"sampleRate": @(_targetRate), @"durationMs": @(frames / _targetRate * 1000)
+ };
+ [_pending setLength:0];
+ return packet;
+}
+
+- (NSArray<NSDictionary *> *)drain {
+ NSMutableArray *packets = [NSMutableArray new];
+ uint32_t read = atomic_load_explicit(&_ring->read, memory_order_relaxed);
+ const uint32_t written = atomic_load_explicit(&_ring->written, memory_order_acquire);
+ while (read != written) {
+ const AudioSlot *slot = &_ring->slots[read % SlotCount];
+ uint64_t end = _sourceFrames + slot->frames;
+ uint64_t outputEnd = (uint64_t)ceil(end * _targetRate / _inputSampleRate);
+ for (; _outputFrames < outputEnd; _outputFrames++) {
+ uint64_t source = (uint64_t)floor(_outputFrames * _inputSampleRate / _targetRate);
+ UInt32 offset = (UInt32)(source - _sourceFrames);
+ if (_pending.length == 0) {
+ // Subtract as signed: input timestamps can predate engine.start.
+ _pendingCapturedAt = _wallAnchor + ((int64_t)(slot->hostTime - _hostAnchor)) * _hostTicksToMs
+ + offset / _inputSampleRate * 1000;
+ }
+ float value = fmaxf(-1, fminf(1, slot->samples[MIN(offset, slot->frames - 1)]));
+ int16_t sample = (int16_t)(value * INT16_MAX);
+ [_pending appendBytes:&sample length:sizeof(sample)];
+ // First packet 10ms, subsequent packets 40ms: hardware latency is
+ // independent of ASR network packet frequency.
+ NSUInteger framesPerPacket = (NSUInteger)llround(_targetRate * (_chunkIndex == 0 ? .010 : .040));
+ if (_pending.length / sizeof(int16_t) >= framesPerPacket) [packets addObject:[self takePending]];
+ }
+ _sourceFrames = end;
+ read++;
+ atomic_store_explicit(&_ring->read, read, memory_order_release);
+ }
+ if (!_engine.isRunning && _pending.length) [packets addObject:[self takePending]];
+ return packets;
+}
+
+- (void)stop {
+ [_engine stop];
+ if (_sink) {
+ [_engine disconnectNodeInput:_sink];
+ [_engine detachNode:_sink];
+ _sink = nil;
+ }
+}
+@end
diff --git a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swift b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swift
index caffbbe4ba..29fba90072 100644
--- a/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swift
+++ b/apps/mobile/modules/xdt-mobile-realtime-audio/ios/XdtMobileRealtimeAudioModule.swift
@@ -5,11 +5,15 @@ private let onAudioChunk = "onAudioChunk"
private let onAudioError = "onAudioError"
public class XdtMobileRealtimeAudioModule: Module {
- private let engine = AVAudioEngine()
+ private let capture = XdtAudioCapture()
+ private var drainTimer: DispatchSourceTimer?
+ private var recentChunks: [[String: Any]] = []
+ private var captureId = ""
+ private var sessionActivatedAt = 0.0
+ private var engineStartedAt = 0.0
private var isCapturing = false
private var ownsAudioSession = false
private var targetSampleRate = 16_000.0
- private var chunkIndex = 0
private var interruptionObserver: NSObjectProtocol?
private var routeChangeObserver: NSObjectProtocol?
// Deferred AVAudioSession deactivation (keep-alive). Activating the session in
@@ -44,7 +48,10 @@ public class XdtMobileRealtimeAudioModule: Module {
return [
"inputPortTypes": session.currentRoute.inputs.map { $0.portType.rawValue },
"inputSampleRate": session.sampleRate,
- "inputChannels": session.inputNumberOfChannels
+ "inputChannels": session.inputNumberOfChannels,
+ "ioBufferDurationMs": session.ioBufferDuration * 1000,
+ "sessionActivatedAt": self.sessionActivatedAt,
+ "engineStartedAt": self.engineStartedAt
]
}
}
@@ -53,6 +60,36 @@ public class XdtMobileRealtimeAudioModule: Module {
self.stopCapture()
}
+ // Additive: old installed binaries still use stop(). Only audio not already
+ // delivered to JS is returned; trim to the user's stop time, not bridge arrival.
+ AsyncFunction("stopAndDrain") { (afterIndex: Int, stoppedAt: Double) in
+ return try self.sessionStateQueue.sync { () -> [[String: Any]] in
+ self.stopCaptureLocked(deactivateImmediately: false)
+ defer { self.recentChunks.removeAll() }
+ guard stoppedAt.isFinite else {
+ throw Exception(name: "ERR_AUDIO_STOP_TIME", description: "Invalid microphone stop time.")
+ }
+ if self.capture.overflowed || (self.recentChunks.first?["chunkIndex"] as? Int ?? 0) > afterIndex + 1 {
+ throw Exception(name: "ERR_AUDIO_OVERFLOW", description: "Realtime microphone delivery fell behind.")
+ }
+ return self.recentChunks.compactMap { chunk in
+ guard let index = chunk["chunkIndex"] as? Int, index > afterIndex,
+ let capturedAt = chunk["capturedAt"] as? Double,
+ let sampleRate = chunk["sampleRate"] as? Double,
+ let base64 = chunk["base64Pcm16"] as? String,
+ var pcm = Data(base64Encoded: base64) else { return nil }
+ let retainedFrames = min(Double(pcm.count / 2), max(0, ((stoppedAt - capturedAt) * sampleRate / 1000).rounded(.down)))
+ let frames = Int(retainedFrames)
+ guard frames > 0 else { return nil }
+ pcm = pcm.prefix(frames * 2)
+ var result = chunk
+ result["base64Pcm16"] = pcm.base64EncodedString()
+ result["durationMs"] = Double(frames) / sampleRate * 1000
+ return result
+ }
+ }
+ }
+
// Best-effort audio-session warm-up, called on touch-down of the mic button
// (before the tap that actually starts recording). Configuring + activating
// the session here means the ~100-150ms warm-up happens while the finger is
@@ -99,9 +136,8 @@ public class XdtMobileRealtimeAudioModule: Module {
return
}
do {
- try session.setCategory(.playAndRecord, mode: .measurement, options: [.duckOthers, .defaultToSpeaker])
- try session.setActive(true)
- ownsAudioSession = true
+ try activateAudioSessionLocked(session)
+ installAudioSessionObservers()
// Deliberately do NOT touch the engine here: engine.prepare() with no
// attached nodes raises an uncatchable NSException (AVAudioEngineGraph
// Initialize: inputNode != nullptr || outputNode != nullptr). Session
@@ -125,6 +161,7 @@ public class XdtMobileRealtimeAudioModule: Module {
return
}
self.deactivateOwnedSessionLocked()
+ self.removeAudioSessionObservers()
}
deactivateWorkItem = workItem
sessionStateQueue.asyncAfter(deadline: .now() + sessionKeepAliveSeconds, execute: workItem)
@@ -158,7 +195,8 @@ public class XdtMobileRealtimeAudioModule: Module {
if let sampleRate = numericOption(options, key: "sampleRate"), sampleRate > 0 {
targetSampleRate = sampleRate
}
- let bufferSize = AVAudioFrameCount(integerOption(options, key: "bufferSize") ?? 2048)
+ captureId = options?["captureId"] as? String ?? ""
+ recentChunks.removeAll()
let session = AVAudioSession.sharedInstance()
if session.recordPermission == .denied {
@@ -170,9 +208,7 @@ public class XdtMobileRealtimeAudioModule: Module {
}
do {
- try session.setCategory(.playAndRecord, mode: .measurement, options: [.duckOthers, .defaultToSpeaker])
- try session.setActive(true)
- ownsAudioSession = true
+ try activateAudioSessionLocked(session)
} catch {
scheduleDeactivateLocked()
throw Exception(name: "ERR_AUDIO_SESSION", description: "Failed to configure audio session: \(error.localizedDescription)")
@@ -180,26 +216,58 @@ public class XdtMobileRealtimeAudioModule: Module {
installAudioSessionObservers()
- let inputNode = engine.inputNode
- let inputFormat = inputNode.inputFormat(forBus: 0)
- chunkIndex = 0
- inputNode.removeTap(onBus: 0)
- inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: inputFormat) { [weak self] buffer, time in
- self?.handleInputBuffer(buffer, time: time)
- }
-
do {
- engine.prepare()
- try engine.start()
+ _ = try capture.start(withSampleRate: targetSampleRate)
+ engineStartedAt = Date().timeIntervalSince1970 * 1000
isCapturing = true
+ let timer = DispatchSource.makeTimerSource(queue: sessionStateQueue)
+ timer.schedule(deadline: .now(), repeating: .milliseconds(5), leeway: .milliseconds(1))
+ timer.setEventHandler { [weak self] in
+ guard let self else { return }
+ self.collectChunksLocked(emit: true)
+ if self.capture.overflowed {
+ self.emitError("Realtime microphone delivery fell behind.")
+ self.stopCaptureLocked(deactivateImmediately: true)
+ }
+ }
+ drainTimer = timer
+ timer.resume()
} catch {
removeAudioSessionObservers()
- inputNode.removeTap(onBus: 0)
+ capture.stop()
scheduleDeactivateLocked()
throw Exception(name: "ERR_AUDIO_ENGINE_START", description: "Failed to start realtime audio engine: \(error.localizedDescription)")
}
}
+ private func activateAudioSessionLocked(_ session: AVAudioSession) throws {
+ // Reuse only our live, unchanged session. Interruptions and route loss clear
+ // ownership; another playback feature's configuration never counts as warm.
+ let matches = session.category == .playAndRecord && session.mode == .measurement
+ && session.categoryOptions.subtracting(.mixWithOthers) == [.duckOthers, .defaultToSpeaker]
+ if !ownsAudioSession || !matches {
+ try session.setCategory(.playAndRecord, mode: .measurement, options: [.duckOthers, .defaultToSpeaker])
+ // A preference, not a promise: report the actual period after activation.
+ try? session.setPreferredIOBufferDuration(0.010)
+ try session.setActive(true)
+ ownsAudioSession = true
+ }
+ sessionActivatedAt = Date().timeIntervalSince1970 * 1000
+ }
+
+ private func collectChunksLocked(emit: Bool) {
+ for raw in capture.drain() {
+ var chunk = raw as! [String: Any]
+ chunk["captureId"] = captureId
+ recentChunks.append(chunk)
+ if recentChunks.count > 128 { recentChunks.removeFirst() }
+ if emit {
+ let payload = chunk
+ DispatchQueue.main.async { [weak self] in self?.sendEvent(onAudioChunk, payload) }
+ }
+ }
+ }
+
private func stopCapture(deactivateImmediately: Bool = false) {
sessionStateQueue.sync {
self.stopCaptureLocked(deactivateImmediately: deactivateImmediately)
@@ -208,11 +276,11 @@ public class XdtMobileRealtimeAudioModule: Module {
// Must run on sessionStateQueue.
private func stopCaptureLocked(deactivateImmediately: Bool) {
- removeAudioSessionObservers()
- if engine.isRunning {
- engine.inputNode.removeTap(onBus: 0)
- engine.stop()
- }
+ if deactivateImmediately { removeAudioSessionObservers() }
+ drainTimer?.cancel()
+ drainTimer = nil
+ capture.stop()
+ collectChunksLocked(emit: false)
isCapturing = false
if deactivateImmediately {
// System interruptions and teardown must release the session right away —
@@ -225,56 +293,10 @@ public class XdtMobileRealtimeAudioModule: Module {
}
}
- private func handleInputBuffer(_ buffer: AVAudioPCMBuffer, time: AVAudioTime) {
- guard let channelData = buffer.floatChannelData else {
- emitError("Realtime audio input did not provide float PCM samples.")
- return
- }
- let frameCount = Int(buffer.frameLength)
- guard frameCount > 0 else {
- return
- }
-
- let inputSampleRate = buffer.format.sampleRate
- let channelCount = max(1, Int(buffer.format.channelCount))
- let targetFrameCount = max(1, Int((Double(frameCount) * targetSampleRate / inputSampleRate).rounded(.down)))
- var pcm = Data(capacity: targetFrameCount * 2)
- let scale = inputSampleRate / targetSampleRate
-
- for targetIndex in 0..<targetFrameCount {
- let sourceIndex = min(frameCount - 1, Int(Double(targetIndex) * scale))
- var mono: Float = 0
- for channelIndex in 0..<channelCount {
- mono += channelData[channelIndex][sourceIndex]
- }
- mono /= Float(channelCount)
- let clamped = max(-1, min(1, mono))
- var sample = Int16(clamped * Float(Int16.max)).littleEndian
- withUnsafeBytes(of: &sample) { bytes in
- pcm.append(contentsOf: bytes)
- }
- }
-
- let capturedAt = Date().timeIntervalSince1970 * 1000
- let index = chunkIndex
- chunkIndex += 1
- let durationMs = Double(targetFrameCount) / targetSampleRate * 1000
- let payload: [String: Any] = [
- "base64Pcm16": pcm.base64EncodedString(),
- "capturedAt": capturedAt,
- "chunkIndex": index,
- "sampleRate": Int(targetSampleRate),
- "durationMs": durationMs
- ]
-
- DispatchQueue.main.async { [weak self] in
- self?.sendEvent(onAudioChunk, payload)
- }
- }
-
private func emitError(_ message: String) {
+ let id = captureId
DispatchQueue.main.async { [weak self] in
- self?.sendEvent(onAudioError, ["message": message])
+ self?.sendEvent(onAudioError, ["message": message, "captureId": id])
}
}
@@ -311,27 +333,27 @@ public class XdtMobileRealtimeAudioModule: Module {
}
private func handleAudioSessionInterruption(_ notification: Notification) {
- guard isCapturing else {
- return
- }
let rawType = unsignedIntegerValue(notification.userInfo?[AVAudioSessionInterruptionTypeKey])
guard rawType == AVAudioSession.InterruptionType.began.rawValue else {
return
}
- emitError("Realtime voice input was interrupted by the system.")
- stopCapture(deactivateImmediately: true)
+ sessionStateQueue.async { [weak self] in
+ guard let self else { return }
+ if self.isCapturing { self.emitError("Realtime voice input was interrupted by the system.") }
+ self.stopCaptureLocked(deactivateImmediately: true)
+ }
}
private func handleAudioRouteChange(_ notification: Notification) {
- guard isCapturing else {
- return
- }
let rawReason = unsignedIntegerValue(notification.userInfo?[AVAudioSessionRouteChangeReasonKey])
guard rawReason == AVAudioSession.RouteChangeReason.oldDeviceUnavailable.rawValue else {
return
}
- emitError("Realtime voice input microphone route became unavailable.")
- stopCapture(deactivateImmediately: true)
+ sessionStateQueue.async { [weak self] in
+ guard let self else { return }
+ if self.isCapturing { self.emitError("Realtime voice input microphone route became unavailable.") }
+ self.stopCaptureLocked(deactivateImmediately: true)
+ }
}
}
@@ -348,19 +370,6 @@ private func numericOption(_ options: [String: Any]?, key: String) -> Double? {
return nil
}
-private func integerOption(_ options: [String: Any]?, key: String) -> Int? {
- if let value = options?[key] as? Int {
- return value
- }
- if let value = options?[key] as? Double {
- return Int(value)
- }
- if let value = options?[key] as? NSNumber {
- return value.intValue
- }
- return nil
-}
-
private func unsignedIntegerValue(_ value: Any?) -> UInt? {
if let value = value as? UInt {
return value
diff --git a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts
index ba60be1041..9332743048 100644
--- a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts
+++ b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts
@@ -33,6 +33,80 @@ afterEach(async () => {
});
describe('mobileRealtimeAudio', () => {
+ it('replays a packet rejected after stop even when its event beat the native start receipt', async () => {
+ let listener!: (event: unknown) => void;
+ let finishStart!: () => void;
+ const packet = { chunkIndex: 0, base64Pcm16: '6APoAw==', capturedAt: 10, sampleRate: 16000, durationMs: .125 };
+ const native = {
+ addListener: vi.fn((name: string, callback: (event: unknown) => void) => {
+ if (name === 'onAudioChunk') listener = callback;
+ return { remove: vi.fn() };
+ }),
+ start: vi.fn(() => new Promise<void>((resolve) => { finishStart = resolve; })),
+ stopAndDrain: vi.fn(async () => [packet]),
+ };
+ vi.mocked(requireNativeModule).mockReturnValue(native);
+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');
+ const onChunk = vi.fn((chunk: { drained?: boolean }) => chunk.drained === true);
+ const opening = startMobileRealtimeAudio({ onChunk });
+ await vi.waitFor(() => expect(finishStart).toBeTypeOf('function'));
+ listener(packet);
+ finishStart();
+ await (await opening)(20);
+ expect(native.stopAndDrain).toHaveBeenCalledWith(-1, 20);
+ expect(onChunk.mock.calls.map(([chunk]) => chunk.drained)).toEqual([undefined, true]);
+ });
+
+ it('drains only undelivered pre-stop packets, deduplicates the receipt, and ignores a previous capture', async () => {
+ type Packet = { captureId: string; chunkIndex: number; base64Pcm16: string; capturedAt: number; sampleRate: number; durationMs: number };
+ let listener!: (event: Packet) => void;
+ let captureId = '';
+ const packet = (index: number): Packet => ({
+ captureId, chunkIndex: index, base64Pcm16: '6APoAw==',
+ capturedAt: 10, sampleRate: 16000, durationMs: .125,
+ });
+ const native = {
+ addListener: vi.fn((name: string, callback: (event: Packet) => void) => {
+ if (name === 'onAudioChunk') listener = callback;
+ return { remove: vi.fn() };
+ }),
+ start: vi.fn(async (options: { captureId: string }) => { captureId = options.captureId; }),
+ stop: vi.fn(async () => undefined),
+ stopAndDrain: vi.fn(async () => [packet(0), packet(1)]),
+ };
+ vi.mocked(requireNativeModule).mockReturnValue(native);
+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');
+ const onChunk = vi.fn();
+ const stop = await startMobileRealtimeAudio({ onChunk });
+ listener({ ...packet(0), captureId: 'previous' });
+ listener(packet(0));
+ await stop(20);
+ expect(native.stopAndDrain).toHaveBeenCalledWith(0, 20);
+ expect(native.stop).not.toHaveBeenCalled();
+ expect(onChunk.mock.calls.map(([chunk]) => [chunk.trace.chunkIndex, chunk.drained])).toEqual([[0, undefined], [1, true]]);
+ expect([...new Int16Array(onChunk.mock.calls[1][0].pcm16)]).toEqual([1000, 1000]);
+ await stop();
+ expect(native.stopAndDrain).toHaveBeenCalledOnce();
+ });
+
+ it('coalesces concurrent native prewarm calls and allows a later prewarm', async () => {
+ let finish!: () => void;
+ const native = { prewarm: vi.fn(() => new Promise<void>((resolve) => { finish = resolve; })) };
+ vi.mocked(requireNativeModule).mockReturnValue(native);
+ const { prewarmMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');
+ prewarmMobileRealtimeAudio();
+ prewarmMobileRealtimeAudio();
+ expect(native.prewarm).toHaveBeenCalledOnce();
+ finish();
+ await Promise.resolve();
+ await Promise.resolve();
+ prewarmMobileRealtimeAudio();
+ expect(native.prewarm).toHaveBeenCalledTimes(2);
+ finish();
+ await Promise.resolve();
+ await Promise.resolve();
+ });
+
it.each([undefined, { inputPortTypes: ['MicrophoneBuiltIn'], inputSampleRate: 48000, inputChannels: 1 }])(
'logs input metadata and supports installed modules without it: %j', async (input) => {
const native = {
diff --git a/apps/mobile/src/__tests__/mobileVoiceController.test.ts b/apps/mobile/src/__tests__/mobileVoiceController.test.ts
index 4230c63a6b..64c417a020 100644
--- a/apps/mobile/src/__tests__/mobileVoiceController.test.ts
+++ b/apps/mobile/src/__tests__/mobileVoiceController.test.ts
@@ -10,6 +10,7 @@ import type {
AsrEvent,
AsrProvider,
AudioTrace,
+ AudioGate,
RefinementResult,
} from '@cindy/voice-input-core';
import type { StoredMobileVoiceCredential } from '@/session/mobileVoiceCredentialStore';
@@ -164,6 +165,103 @@ function credential(): StoredMobileVoiceCredential {
}
describe('mobileVoiceController', () => {
+ it('fails capture if pending speaker settings reject instead of uploading unfiltered audio', async () => {
+ let reject!: (error: Error) => void;
+ const asr = new FinalOnlyAsrProvider();
+ const append = vi.spyOn(asr, 'appendAudio');
+ const error = vi.fn();
+ const session = createMobileVoiceControllerSession({
+ credential: credential(), asr, refiner: null, initialDraft: '', onDraftChanged: vi.fn(), onError: error,
+ startAudio: startAudibleAudio, audioGate: new Promise((_resolve, fail) => { reject = fail; }),
+ });
+ await session.start();
+ reject(new Error('speaker settings unavailable'));
+ await Promise.resolve();
+ await expect(session.stop()).rejects.toThrow('speaker settings unavailable');
+ expect(append).not.toHaveBeenCalled();
+ expect(error).toHaveBeenCalledOnce();
+ });
+
+ it('keeps a short utterance delivered only by the stop receipt, while rejecting ordinary late events', async () => {
+ const asr = new FinalOnlyAsrProvider();
+ const append = vi.spyOn(asr, 'appendAudio');
+ const session = createMobileVoiceControllerSession({
+ credential: credential(), asr, refiner: null, initialDraft: '', onDraftChanged: vi.fn(),
+ startAudio: async ({ onChunk }) => async () => {
+ const chunk = { pcm16: new Int16Array(160).fill(1000).buffer,
+ trace: { capturedAt: 1, convertedAt: 2, chunkIndex: 0, sampleRate: 16000, durationMs: 10 } };
+ onChunk(chunk);
+ onChunk({ ...chunk, drained: true });
+ },
+ });
+ await session.start();
+ expect(await session.stop()).toBe('raw final');
+ expect(append).toHaveBeenCalledOnce();
+ });
+
+ it('waits for a pending native start and its drain before treating an immediate stop as silence', async () => {
+ let open!: () => void;
+ const asr = new FinalOnlyAsrProvider();
+ const append = vi.spyOn(asr, 'appendAudio');
+ const session = createMobileVoiceControllerSession({
+ credential: credential(), asr, refiner: null, initialDraft: '', onDraftChanged: vi.fn(),
+ startAudio: ({ onChunk }) => new Promise((resolve) => {
+ open = () => resolve(async () => {
+ onChunk({ pcm16: new Int16Array(160).fill(1000).buffer, drained: true,
+ trace: { capturedAt: 1, convertedAt: 2, chunkIndex: 0, sampleRate: 16000, durationMs: 10 } });
+ });
+ }),
+ });
+ const starting = session.start();
+ const stopping = session.stop();
+ open();
+ expect(await stopping).toBe('raw final');
+ await starting;
+ expect(append).toHaveBeenCalledOnce();
+ });
+
+ it('captures while the speaker settings load, and filters every retained chunk before ASR', async () => {
+ let resolveGate!: (gate: AudioGate | undefined) => void;
+ const pendingGate = new Promise<AudioGate | undefined>((resolve) => { resolveGate = resolve; });
+ const asr = new FinalOnlyAsrProvider();
+ const append = vi.spyOn(asr, 'appendAudio');
+ let emit!: (chunk: ArrayBuffer, trace?: AudioTrace) => void;
+ const gate: AudioGate = {
+ begin: (callback) => { emit = callback; },
+ push: (chunk, trace) => emit(new ArrayBuffer(chunk.byteLength), trace),
+ finish: async () => undefined,
+ abort: vi.fn(),
+ };
+ const open = vi.fn(startAudibleAudio);
+ const session = createMobileVoiceControllerSession({
+ credential: credential(), asr, refiner: null, initialDraft: '',
+ audioGate: pendingGate, startAudio: open, onDraftChanged: vi.fn(),
+ });
+ await session.start();
+ expect(open).toHaveBeenCalledOnce();
+ expect(append).not.toHaveBeenCalled();
+ resolveGate(gate);
+ await session.stop();
+ expect(append).toHaveBeenCalledOnce();
+ expect(new Int16Array(append.mock.calls[0][0]).every((sample) => sample === 0)).toBe(true);
+ });
+
+ it('discards pending speaker audio on cancel, including a late settings result', async () => {
+ let resolveGate!: (gate: AudioGate | undefined) => void;
+ const pendingGate = new Promise<AudioGate | undefined>((resolve) => { resolveGate = resolve; });
+ const asr = new FinalOnlyAsrProvider();
+ const append = vi.spyOn(asr, 'appendAudio');
+ const session = createMobileVoiceControllerSession({
+ credential: credential(), asr, refiner: null, initialDraft: '',
+ audioGate: pendingGate, startAudio: startAudibleAudio, onDraftChanged: vi.fn(),
+ });
+ await session.start();
+ await session.cancel();
+ resolveGate(undefined);
+ await Promise.resolve();
+ expect(append).not.toHaveBeenCalled();
+ });
+
it.each([false, true])('removes selected quotes even when the first identical transcript arrives at stop (partials=%s)', async (partials) => {
const initialDocument: ComposerDocument = { version: 1, nodes: [
{ type: 'quote', quote: { text: 'selected quote' } }, { type: 'text', text: 'raw final' },
diff --git a/apps/mobile/src/session/askCardVoiceStart.ts b/apps/mobile/src/session/askCardVoiceStart.ts
index c00689153a..6a796dbe8c 100644
--- a/apps/mobile/src/session/askCardVoiceStart.ts
+++ b/apps/mobile/src/session/askCardVoiceStart.ts
@@ -51,6 +51,7 @@ export async function startAskCardDictation(input: {
onRunCreated?: (run: { stop: () => Promise<void>; cancel: () => Promise<void> }) => void;
onError: (message: string) => void;
}): Promise<{ stop: () => Promise<void>; cancel: () => Promise<void> }> {
+ const requestedAt = Date.now();
if (!isMobileRealtimeAudioAvailable()) {
throw new Error(mobileVoiceRealtimeAudioUnavailableError());
}
@@ -89,16 +90,12 @@ export async function startAskCardDictation(input: {
credential.settings?.language,
CINDY_MANAGED_REFINER_PROVIDER,
);
- let audioGate: Awaited<ReturnType<typeof createMobileSpeakerGate>> | undefined;
- try {
- audioGate = await createMobileSpeakerGate();
- } catch {
- audioGate = undefined;
- }
+ const audioGate = createMobileSpeakerGate().catch(() => undefined);
const initialDraft = input.getText();
const session = createMobileVoiceControllerSession({
credential,
- ...(audioGate ? { audioGate } : {}),
+ audioGate,
+ requestedAt,
connectionProvider: (providerId) => voiceContext.createAsrConnection(providerId),
refinerTargetProvider: (providerId, options) => voiceContext.createRefinerTarget(providerId, options),
warmRefiner: (request) => voiceContext.warmRefiner(request),
diff --git a/apps/mobile/src/session/mobilePendingAudioGate.ts b/apps/mobile/src/session/mobilePendingAudioGate.ts
new file mode 100644
index 0000000000..86e9540ad9
--- /dev/null
+++ b/apps/mobile/src/session/mobilePendingAudioGate.ts
@@ -0,0 +1,60 @@
+import type { AudioGate, AudioTrace } from '@cindy/voice-input-core';
+
+/**
+ * Opening the microphone must not wait for AsyncStorage / Keychain. Hold the
+ * first PCM locally until createMobileSpeakerGate's bounded hydration finishes;
+ * no unfiltered samples reach the ASR while the account's settings are unknown.
+ */
+export class MobilePendingAudioGate implements AudioGate {
+ private gate?: AudioGate;
+ private ready = false;
+ private emit?: (chunk: ArrayBuffer, trace?: AudioTrace) => void;
+ private pending: Array<{ chunk: ArrayBuffer; trace?: AudioTrace }> = [];
+ private readonly resolved: Promise<void>;
+
+ constructor(gate: Promise<AudioGate | undefined>, onError: (error: unknown) => void) {
+ this.resolved = gate.then((value) => {
+ this.gate = value;
+ this.ready = true;
+ if (!this.emit) return;
+ value?.begin(this.emit);
+ const pending = this.pending;
+ this.pending = [];
+ for (const { chunk, trace } of pending) this.push(chunk, trace);
+ }).catch((error: unknown) => {
+ this.abort();
+ onError(error);
+ throw error;
+ });
+ // finish() observes failure. Never leak an unhandled rejection if cancelled.
+ void this.resolved.catch(() => undefined);
+ }
+
+ begin(emit: (chunk: ArrayBuffer, trace?: AudioTrace) => void): void {
+ this.abort();
+ this.emit = emit;
+ if (this.ready) this.gate?.begin(emit);
+ }
+
+ push(chunk: ArrayBuffer, trace?: AudioTrace): void {
+ if (!this.emit) return;
+ if (!this.ready) {
+ this.pending.push({ chunk, trace });
+ } else if (this.gate) {
+ this.gate.push(chunk, trace);
+ } else {
+ this.emit(chunk, trace);
+ }
+ }
+
+ async finish(): Promise<void> {
+ await this.resolved;
+ if (this.emit) await this.gate?.finish();
+ }
+
+ abort(): void {
+ this.emit = undefined;
+ this.pending = [];
+ this.gate?.abort();
+ }
+}
diff --git a/apps/mobile/src/session/mobileRealtimeAudio.ts b/apps/mobile/src/session/mobileRealtimeAudio.ts
index 4426f89633..2c5e5bb3d5 100644
--- a/apps/mobile/src/session/mobileRealtimeAudio.ts
+++ b/apps/mobile/src/session/mobileRealtimeAudio.ts
@@ -10,9 +10,14 @@ import { logMobileVoice } from '@/session/mobileVoiceDiagnostics';
export type MobileRealtimeAudioChunk = {
pcm16: ArrayBuffer;
trace: AudioTrace;
+ /** Older native modules and Expo Audio timestamp JS delivery instead. */
+ timestampSource?: 'hardware' | 'delivery';
+ /** Explicit native stop receipt; never inferred from a late event. */
+ drained?: boolean;
};
type NativeAudioChunkEvent = {
+ captureId?: string;
base64Pcm16: string;
capturedAt: number;
chunkIndex: number;
@@ -22,15 +27,20 @@ type NativeAudioChunkEvent = {
type NativeAudioErrorEvent = {
message: string;
+ captureId?: string;
};
type NativeRealtimeAudioModule = {
- start(options?: { sampleRate?: number; bufferSize?: number }): Promise<{
+ start(options?: { sampleRate?: number; bufferSize?: number; captureId?: string }): Promise<{
inputPortTypes: string[];
inputSampleRate: number;
inputChannels: number;
+ ioBufferDurationMs?: number;
+ sessionActivatedAt?: number;
+ engineStartedAt?: number;
} | void>;
stop(): Promise<void>;
+ stopAndDrain?(afterIndex: number, stoppedAt: number): Promise<NativeAudioChunkEvent[]>;
prewarm(): Promise<void>;
};
@@ -105,6 +115,8 @@ let expoAudioNativeModule: ExpoAudioNativeModule | null | undefined;
// The custom Apple module owns one native engine. A replacement must wait for
// the previous caller's complete start/stop lifecycle, including late cleanup.
let customNativeCaptureReleased: Promise<void> = Promise.resolve();
+let nextCaptureId = 0;
+let nativePrewarm: Promise<void> | undefined;
export function isMobileRealtimeAudioAvailable(): boolean {
if (isE2eMockRealtimeAudioEnabled()) return true;
@@ -125,12 +137,12 @@ export function shouldShowMobileVoiceUi(
export async function startMobileRealtimeAudio(
options: {
- onChunk: (chunk: MobileRealtimeAudioChunk) => void;
+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;
onError?: (error: Error) => void;
sampleRate?: number;
bufferSize?: number;
},
-): Promise<() => Promise<void>> {
+): Promise<(stoppedAt?: number) => Promise<void>> {
if (isE2eMockRealtimeAudioEnabled()) {
return startE2eMockRealtimeAudio(options);
}
@@ -148,36 +160,50 @@ export async function startMobileRealtimeAudio(
async function startCustomNativeRealtimeAudio(
binding: RealtimeAudioNativeBinding,
options: {
- onChunk: (chunk: MobileRealtimeAudioChunk) => void;
+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;
onError?: (error: Error) => void;
sampleRate?: number;
bufferSize?: number;
},
-): Promise<() => Promise<void>> {
+): Promise<(stoppedAt?: number) => Promise<void>> {
const previousCapture = customNativeCaptureReleased;
let releaseCapture!: () => void;
customNativeCaptureReleased = new Promise<void>((resolve) => { releaseCapture = resolve; });
await previousCapture;
const subscriptions: EventSubscription[] = [];
+ const captureId = String(++nextCaptureId);
+ let lastChunkIndex = -1;
+ const deliver = (event: NativeAudioChunkEvent, drained = false): void => {
+ if (event.captureId !== undefined && event.captureId !== captureId) return;
+ if (event.chunkIndex <= lastChunkIndex) return;
+ const accepted = options.onChunk({
+ pcm16: decodeBase64ToArrayBuffer(event.base64Pcm16),
+ trace: {
+ capturedAt: event.capturedAt,
+ convertedAt: Date.now(),
+ chunkIndex: event.chunkIndex,
+ sampleRate: event.sampleRate,
+ durationMs: event.durationMs,
+ },
+ timestampSource: binding.module.stopAndDrain ? 'hardware' : 'delivery',
+ ...(drained ? { drained: true } : {}),
+ });
+ // A stop can beat the native start receipt. Audio rejected as an ordinary
+ // late event must remain replayable through the explicit drain receipt.
+ if (accepted !== false) lastChunkIndex = event.chunkIndex;
+ };
try {
subscriptions.push(binding.module.addListener('onAudioChunk', (event) => {
- options.onChunk({
- pcm16: decodeBase64ToArrayBuffer(event.base64Pcm16),
- trace: {
- capturedAt: event.capturedAt,
- convertedAt: Date.now(),
- chunkIndex: event.chunkIndex,
- sampleRate: event.sampleRate,
- durationMs: event.durationMs,
- },
- });
+ deliver(event);
}));
subscriptions.push(binding.module.addListener('onAudioError', (event) => {
+ if (event.captureId !== undefined && event.captureId !== captureId) return;
options.onError?.(new Error(event.message));
}));
const input = await binding.module.start({
sampleRate: options.sampleRate ?? 16_000,
bufferSize: options.bufferSize ?? DEFAULT_NATIVE_AUDIO_BUFFER_SIZE,
+ captureId,
});
logMobileVoice('debug', 'capture input', {
backend: 'ios-native',
@@ -185,6 +211,11 @@ async function startCustomNativeRealtimeAudio(
inputPortTypes: input?.inputPortTypes ?? ['unknown'],
inputSampleRate: input?.inputSampleRate,
inputChannels: input?.inputChannels,
+ ...(input?.ioBufferDurationMs === undefined ? {} : {
+ ioBufferDurationMs: input.ioBufferDurationMs,
+ sessionActivatedAt: input.sessionActivatedAt,
+ engineStartedAt: input.engineStartedAt,
+ }),
});
} catch (error) {
try {
@@ -196,13 +227,19 @@ async function startCustomNativeRealtimeAudio(
}
let stopping: Promise<void> | undefined;
- return () => {
+ return (stopRequestedAt) => {
stopping ??= (async () => {
+ const stoppedAt = stopRequestedAt ?? Date.now();
try {
try {
subscriptions.forEach((subscription) => subscription.remove());
} finally {
- await binding.module.stop();
+ if (binding.module.stopAndDrain) {
+ const chunks = await binding.module.stopAndDrain(lastChunkIndex, stoppedAt);
+ for (const chunk of chunks) deliver(chunk, true);
+ } else {
+ await binding.module.stop();
+ }
}
} finally {
releaseCapture();
@@ -221,7 +258,7 @@ async function startCustomNativeRealtimeAudio(
async function startExpoAudioRealtimeAudio(
module: ExpoAudioNativeModule,
options: {
- onChunk: (chunk: MobileRealtimeAudioChunk) => void;
+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;
onError?: (error: Error) => void;
sampleRate?: number;
},
@@ -342,7 +379,9 @@ export function prewarmMobileRealtimeAudio(): void {
if (isE2eMockRealtimeAudioEnabled()) return;
const binding = getNativeBinding();
if (!binding) return;
- void binding.module.prewarm().catch(() => undefined);
+ nativePrewarm ??= binding.module.prewarm().catch(() => undefined).finally(() => {
+ nativePrewarm = undefined;
+ });
}
export const __testing = {
@@ -544,7 +583,7 @@ function isE2eMockRealtimeAudioEnabled(): boolean {
}
function startE2eMockRealtimeAudio(options: {
- onChunk: (chunk: MobileRealtimeAudioChunk) => void;
+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;
sampleRate?: number;
}): Promise<() => Promise<void>> {
let stopped = false;
diff --git a/apps/mobile/src/session/mobileSpeakerFilter.ts b/apps/mobile/src/session/mobileSpeakerFilter.ts
index 8a8e4da87b..a472a4c551 100644
--- a/apps/mobile/src/session/mobileSpeakerFilter.ts
+++ b/apps/mobile/src/session/mobileSpeakerFilter.ts
@@ -37,8 +37,9 @@ import { logMobileVoice } from './mobileVoiceDiagnostics';
* "Recognize my voice" on mobile. Same models, thresholds and gate as desktop;
* inference runs through ONNX Runtime in the cindy-speaker-onnx module.
*
- * Dictation start waits for saved preferences and the voiceprint before
- * choosing its gate. Model loading stays lazy; failures still pass through.
+ * Audio forwarding waits for saved preferences and the voiceprint before
+ * choosing its gate; microphone capture may buffer locally meanwhile.
+ * Model loading stays lazy; failures still pass through.
*
* The voiceprint is biometric data: it stays on this device, in the OS
* keychain/keystore through SecureStore, one per account, and is never
diff --git a/apps/mobile/src/session/mobileVoiceController.ts b/apps/mobile/src/session/mobileVoiceController.ts
index 7f649b16b4..1f8d4ffc26 100644
--- a/apps/mobile/src/session/mobileVoiceController.ts
+++ b/apps/mobile/src/session/mobileVoiceController.ts
@@ -33,8 +33,9 @@ import {
mobileVoiceTranscriptKeptError,
type MobileVoiceDraftInsertion,
} from '@/session/mobileVoiceInput';
-import { startMobileRealtimeAudio } from '@/session/mobileRealtimeAudio';
+import { startMobileRealtimeAudio, type MobileRealtimeAudioChunk } from '@/session/mobileRealtimeAudio';
import { MobileVoiceAudioLevels } from '@/session/mobileVoiceAudioLevels';
+import { MobilePendingAudioGate } from '@/session/mobilePendingAudioGate';
import { readCachedMobileVoiceDictionary } from '@/session/mobileVoiceDictionaryCache';
import {
classifyMobileVoiceFailure,
@@ -45,9 +46,9 @@ import {
type StartRealtimeAudio = (options: {
sampleRate: number;
- onChunk: (chunk: { pcm16: ArrayBuffer; trace: AudioTrace }) => void;
+ onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;
onError?: (error: Error) => void;
-}) => Promise<() => Promise<void>>;
+}) => Promise<(stoppedAt?: number) => Promise<void>>;
type MobileDictationRefiner = {
refine(input: {
@@ -113,7 +114,9 @@ type MobileVoiceControllerOptions = {
* Optional filter between the microphone and the ASR ("recognize my voice").
* Undefined keeps the unfiltered behavior.
*/
- audioGate?: AudioGate;
+ audioGate?: AudioGate | Promise<AudioGate | undefined>;
+ /** Gesture time, before permission / storage work, for end-to-end diagnostics. */
+ requestedAt?: number;
};
type MobileVoiceInputHistorySource = readonly string[] | (() => readonly string[] | undefined);
@@ -180,7 +183,10 @@ export function createMobileVoiceControllerSession(
let lastPublishedDraft = baseDraft;
let pendingDraftToPublish: string | null = null;
let draftPublishThrottleTimer: ReturnType<typeof setTimeout> | null = null;
- let stopAudio: (() => Promise<void>) | null = null;
+ let stopAudio: ((stoppedAt?: number) => Promise<void>) | null = null;
+ let audioOpening: Promise<void> | null = null;
+ let audioStopping: Promise<void> | null = null;
+ let captureStopRequestedAt: number | undefined;
let state: VoiceInputState = 'idle';
let controllerError: Error | null = null;
// Run lifecycle as an explicit phase machine instead of a set of overlapping
@@ -193,9 +199,9 @@ export function createMobileVoiceControllerSession(
// any → running (a fresh start() begins a new run)
// Every teardown-ish question is answered from the phase, so the answers can
// never disagree with each other the way independent flags could:
- // - captureTornDown(): should onChunk drop mic audio? (post-stop straggler
- // chunks must never be buffered/flushed — a stop() that waits for a slow
- // handshake would otherwise transcribe speech after the user stopped)
+ // - captureTornDown(): drop ordinary post-stop events. Only an explicit
+ // native drain receipt, clipped to the user's stop time, may still enter
+ // the buffer while stopping; a slow ASR handshake cannot extend capture.
// - phase === 'failed': did the run die (mic or connect failure)?
// - phase === 'cancelled': was the run abandoned on purpose (not an error)?
let runPhase: 'idle' | 'running' | 'stopping' | 'cancelled' | 'failed' = 'idle';
@@ -316,10 +322,17 @@ export function createMobileVoiceControllerSession(
// was deferred at start until the mic is actually capturing, so the UI's
// "listening" cue matches real audio flow rather than the ~150-200ms-earlier
// moment the audio session began warming up.
- const markCaptureLive = (): void => {
+ const markCaptureLive = (trace: AudioTrace, timestampSource?: 'hardware' | 'delivery'): void => {
if (captureLive) return;
captureLive = true;
- logMobileVoice('debug', 'capture live', { runId: runTag(), elapsedMs: Date.now() - runStartedAt });
+ logMobileVoice('debug', 'capture live', {
+ runId: runTag(), elapsedMs: Date.now() - runStartedAt,
+ timestampSource: timestampSource ?? 'delivery',
+ ...(timestampSource === 'hardware' ? {
+ firstSampleAfterRequestMs: trace.capturedAt - runStartedAt,
+ deliveryDelayMs: Date.now() - trace.capturedAt,
+ } : {}),
+ });
if (listeningPendingCaptureLive && state === 'listening') {
listeningPendingCaptureLive = false;
options.onStateChanged?.('listening');
@@ -399,7 +412,11 @@ export function createMobileVoiceControllerSession(
// A reconnect allocates another managed session and can hit the account
// limit; report that instead of the generic "stopped receiving" failure.
recoveryErrorMessage: mobileVoiceRateLimitMessage,
- audioGate: options.audioGate,
+ audioGate: options.audioGate instanceof Promise
+ ? new MobilePendingAudioGate(options.audioGate, (error) => {
+ void handleAudioFailure(error instanceof Error ? error : new Error(String(error)));
+ })
+ : options.audioGate,
logger: new VoiceTimelineLogger((event) => {
recordTimeline(event);
options.onTimelineEvent?.(event);
@@ -487,7 +504,9 @@ export function createMobileVoiceControllerSession(
listeningPendingCaptureLive = false;
soundDetected = false;
audioLevels = new MobileVoiceAudioLevels();
- runStartedAt = Date.now();
+ audioStopping = null;
+ captureStopRequestedAt = undefined;
+ runStartedAt = options.requestedAt ?? Date.now();
// A fresh start() begins a new run from ANY previous phase — in particular
// 'failed': a mic interruption must not poison the next recording on the
// same session, so the failure record is cleared here too.
@@ -516,22 +535,23 @@ export function createMobileVoiceControllerSession(
refinement: Boolean(refiner),
});
try {
- stopAudio = await startAudio({
+ audioOpening = startAudio({
sampleRate: options.credential.asr.pcmSampleRate ?? 16_000,
onChunk: (chunk) => {
// Drop audio emitted after the run began tearing down capture, so an
// early stop that waits for the in-flight handshake can't buffer and
// later flush post-stop speech/noise.
- if (captureTornDown()) return;
+ if (captureTornDown() && !(runPhase === 'stopping' && chunk.drained)) return false;
audioLevels.append(chunk.pcm16, chunk.trace.sampleRate ?? options.credential.asr.pcmSampleRate ?? 16_000);
if (!soundDetected && hasPcmSound(chunk.pcm16)) soundDetected = true;
- markCaptureLive();
+ if (runPhase === 'running') markCaptureLive(chunk.trace, chunk.timestampSource);
controller.appendAudio(chunk.pcm16, chunk.trace);
},
onError: (error) => {
void handleAudioFailure(error);
},
- });
+ }).then((stop) => { stopAudio = stop; });
+ await audioOpening;
} catch (error) {
runPhase = 'failed';
// Surface the mic failure immediately — do NOT wait for the in-flight
@@ -549,9 +569,7 @@ export function createMobileVoiceControllerSession(
// opening: its teardown saw stopAudio === null and could not close the
// capture, so close the just-opened handle here — otherwise the mic and
// its event subscriptions stay live even though the run is abandoned.
- const stop = stopAudio;
- stopAudio = null;
- await stop().catch(() => undefined);
+ await stopNativeCapture();
}
// Wait for the ASR connection to settle before declaring the start
// successful, so a failed handshake still surfaces as a startup error (and
@@ -601,6 +619,7 @@ export function createMobileVoiceControllerSession(
// the connect settles, so short utterances are not lost.
if (runPhase === 'running') runPhase = 'stopping';
const stopRequestedAt = Date.now();
+ captureStopRequestedAt = stopRequestedAt;
logMobileVoice('debug', 'stop requested', {
runId: runTag(),
elapsedMs: stopRequestedAt - runStartedAt,
@@ -609,11 +628,10 @@ export function createMobileVoiceControllerSession(
...audioLevels.summary(),
hasTranscript: Boolean(voiceInsertion),
});
- if (stopAudio) {
- const stop = stopAudio;
- stopAudio = null;
- await stop().catch(() => undefined);
- }
+ // An immediate release can beat the native start receipt. Wait only for
+ // capture (never ASR), then drain before deciding this was a silent run.
+ if (!stopAudio && !audioStopping) await audioOpening?.catch(() => undefined);
+ await stopNativeCapture();
if (currentRunPhase() === 'stopping' && !soundDetected && !voiceInsertion) {
logMobileVoice('info', 'silent recording ended without ASR finalization', {
runId: runTag(),
@@ -700,6 +718,17 @@ export function createMobileVoiceControllerSession(
}
}
+ async function stopNativeCapture(): Promise<void> {
+ if (audioStopping) return audioStopping;
+ const stop = stopAudio;
+ if (!stop) return;
+ stopAudio = null;
+ audioStopping = stop(captureStopRequestedAt).catch(async (error: unknown) => {
+ await handleAudioFailure(error instanceof Error ? error : new Error(String(error)));
+ });
+ return audioStopping;
+ }
+
function recordSubmittedHistory(text: string): void {
if (!options.recordHistory) return;
// 下面的 .catch 只兜得住异步拒绝;同步抛会顺着 onSubmitted 冒出 controller,