Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(voice): 修复手机声纹缓存恢复与录音并发

DavidShen28eb387db004
diff --git a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts
index 8a5bbf65d4..26108421b2 100644
--- a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts
+++ b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts
@@ -32,6 +32,86 @@ afterEach(async () => {
});
describe('mobileRealtimeAudio', () => {
+ it('waits for the previous singleton recorder to finish starting and stopping before reuse', async () => {
+ let finishStart!: () => void;
+ let finishStop!: () => void;
+ const native = {
+ addListener: vi.fn(() => ({ remove: vi.fn() })),
+ start: vi.fn(async (): Promise<void> => {})
+ .mockImplementationOnce(() => new Promise<void>((resolve) => { finishStart = resolve; })),
+ stop: vi.fn(async (): Promise<void> => {})
+ .mockImplementationOnce(() => new Promise<void>((resolve) => { finishStop = resolve; })),
+ };
+ vi.mocked(requireNativeModule).mockReturnValue(native);
+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');
+ const first = startMobileRealtimeAudio({ onChunk: vi.fn() });
+ await vi.waitFor(() => expect(finishStart).toBeTypeOf('function'));
+ const second = startMobileRealtimeAudio({ onChunk: vi.fn() });
+ for (let i = 0; i < 10; i += 1) await Promise.resolve();
+ const startsWhileStarting = native.start.mock.calls.length;
+ const listenersWhileStarting = native.addListener.mock.calls.length;
+ finishStart();
+ const stopFirst = await first;
+ const stopping = stopFirst();
+ await vi.waitFor(() => expect(finishStop).toBeTypeOf('function'));
+ const startsWhileStopping = native.start.mock.calls.length;
+ finishStop();
+ await stopping;
+ const stopSecond = await second;
+ await stopFirst();
+ const stopsAfterOldCleanup = native.stop.mock.calls.length;
+ await stopSecond();
+
+ expect(startsWhileStarting).toBe(1);
+ expect(listenersWhileStarting).toBe(2);
+ expect(startsWhileStopping).toBe(1);
+ expect(native.start).toHaveBeenCalledTimes(2);
+ expect(stopsAfterOldCleanup).toBe(1);
+ expect(native.stop).toHaveBeenCalledTimes(2);
+ });
+
+ it.each(['start', 'stop'] as const)('allows a new singleton recording after the previous %s rejects', async (failure) => {
+ const native = {
+ addListener: vi.fn(() => ({ remove: vi.fn() })),
+ start: vi.fn(async () => undefined),
+ stop: vi.fn(async () => undefined),
+ };
+ native[failure].mockRejectedValueOnce(new Error('native failure'));
+ vi.mocked(requireNativeModule).mockReturnValue(native);
+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');
+ const first = startMobileRealtimeAudio({ onChunk: vi.fn() });
+ if (failure === 'start') await expect(first).rejects.toThrow('native failure');
+ else await expect((await first)()).rejects.toThrow('native failure');
+ const stop = await startMobileRealtimeAudio({ onChunk: vi.fn() });
+ await stop();
+ expect(native.start).toHaveBeenCalledTimes(2);
+ });
+
+ it.each(['subscribe', 'unsubscribe'] as const)('releases the singleton recorder after %s throws', async (failure) => {
+ const remove = vi.fn();
+ const native = {
+ addListener: vi.fn(() => ({ remove })),
+ start: vi.fn(async () => undefined),
+ stop: vi.fn(async () => undefined),
+ };
+ if (failure === 'subscribe') {
+ native.addListener
+ .mockImplementationOnce(() => ({ remove }))
+ .mockImplementationOnce(() => { throw new Error('listener failure'); });
+ } else {
+ remove.mockImplementationOnce(() => { throw new Error('listener failure'); });
+ }
+ vi.mocked(requireNativeModule).mockReturnValue(native);
+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');
+ const first = startMobileRealtimeAudio({ onChunk: vi.fn() });
+ if (failure === 'subscribe') await expect(first).rejects.toThrow('listener failure');
+ else await expect((await first)()).rejects.toThrow('listener failure');
+ expect(remove).toHaveBeenCalled();
+ expect(native.stop).toHaveBeenCalledTimes(failure === 'subscribe' ? 0 : 1);
+ const stop = await startMobileRealtimeAudio({ onChunk: vi.fn() });
+ await stop();
+ });
+
it('decodes native PCM base64 payloads into ArrayBuffer chunks', async () => {
const { __testing } = await import('@/session/mobileRealtimeAudio');
diff --git a/apps/mobile/src/__tests__/mobileSpeakerFilter.test.ts b/apps/mobile/src/__tests__/mobileSpeakerFilter.test.ts
index c08af19bce..4ca21aeeb5 100644
--- a/apps/mobile/src/__tests__/mobileSpeakerFilter.test.ts
+++ b/apps/mobile/src/__tests__/mobileSpeakerFilter.test.ts
@@ -131,6 +131,133 @@ describe('mobile speaker filter', () => {
expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({ strictness: 'standard', strictnessCustomized: false });
});
+ it.each(['enabled', 'strictness', 'mixed'] as const)('restores committed preferences after overlapping %s writes fail', async (kind) => {
+ installModels();
+ const key = await signIn('user-a');
+ await service.setMobileSpeakerFilterStrictness('relaxed');
+ const saved = asyncStore.get(service.__testing.prefsKey(key));
+ let rejectFirst!: (error: Error) => void;
+ vi.mocked(AsyncStorage.setItem)
+ .mockImplementationOnce(() => new Promise((_resolve, reject) => { rejectFirst = reject; }))
+ .mockRejectedValueOnce(new Error('second write failed'));
+ const first = kind === 'strictness'
+ ? service.setMobileSpeakerFilterStrictness('strict')
+ : service.setMobileSpeakerFilterEnabled(true);
+ const firstFailure = expect(first).rejects.toThrow('first write failed');
+ await vi.waitFor(() => expect(rejectFirst).toBeTypeOf('function'));
+ const second = kind === 'enabled'
+ ? service.setMobileSpeakerFilterEnabled(false)
+ : service.setMobileSpeakerFilterStrictness(null);
+ const secondFailure = expect(second).rejects.toThrow('second write failed');
+ await vi.waitFor(() => expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject(
+ kind === 'enabled' ? { enabled: false } : { strictnessCustomized: false },
+ ));
+ rejectFirst(new Error('first write failed'));
+ await Promise.all([firstFailure, secondFailure]);
+ expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({
+ enabled: false, strictness: 'relaxed', strictnessCustomized: true,
+ });
+ expect(asyncStore.get(service.__testing.prefsKey(key))).toBe(saved);
+ });
+
+ it('rolls back to an overlapping successful write when the following write fails', async () => {
+ installModels();
+ const key = await signIn('user-a');
+ let finishFirst!: () => void;
+ vi.mocked(AsyncStorage.setItem)
+ .mockImplementationOnce(async (name, value) => {
+ await new Promise<void>((resolve) => { finishFirst = resolve; });
+ asyncStore.set(name, value);
+ })
+ .mockRejectedValueOnce(new Error('second write failed'));
+ const first = service.setMobileSpeakerFilterEnabled(true);
+ await vi.waitFor(() => expect(finishFirst).toBeTypeOf('function'));
+ const secondFailure = expect(service.setMobileSpeakerFilterStrictness('strict')).rejects.toThrow('second write failed');
+ await vi.waitFor(() => expect(service.getMobileSpeakerFilterSnapshot().strictness).toBe('strict'));
+ finishFirst();
+ await Promise.all([first, secondFailure]);
+ expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({
+ enabled: true, strictness: 'standard', strictnessCustomized: false,
+ });
+ expect(JSON.parse(asyncStore.get(service.__testing.prefsKey(key))!)).toEqual({ version: 1, enabled: true });
+ });
+
+ it('does not use a late write from the previous account as the new rollback baseline', async () => {
+ installModels();
+ await signIn('user-a');
+ let finishFirst!: () => void;
+ vi.mocked(AsyncStorage.setItem).mockImplementationOnce(async (name, value) => {
+ await new Promise<void>((resolve) => { finishFirst = resolve; });
+ asyncStore.set(name, value);
+ });
+ const first = service.setMobileSpeakerFilterStrictness('strict');
+ await vi.waitFor(() => expect(finishFirst).toBeTypeOf('function'));
+ await signIn('user-b');
+ await service.setMobileSpeakerFilterStrictness('relaxed');
+ finishFirst();
+ await first;
+ vi.mocked(AsyncStorage.setItem).mockRejectedValueOnce(new Error('write failed'));
+ await expect(service.setMobileSpeakerFilterEnabled(true)).rejects.toThrow('write failed');
+ expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({
+ enabled: false, strictness: 'relaxed', strictnessCustomized: true,
+ });
+ });
+
+ it('redownloads purged cache after a native load failure and can load again', async () => {
+ installModels();
+ const key = await signIn('user-a');
+ storeProfile(key);
+ service.__testing.reset();
+ await signIn('user-a');
+ await service.setMobileSpeakerFilterEnabled(true);
+ files.clear();
+ vi.mocked(loadSpeakerModels).mockRejectedValueOnce(new Error('model file missing'));
+ const states: string[] = [];
+ const unsubscribe = service.subscribeMobileSpeakerFilter(() => states.push(service.getMobileSpeakerFilterSnapshot().modelState));
+ try {
+ service.warmMobileSpeakerFilter();
+ await vi.waitFor(() => expect(FileSystem.createDownloadResumable).toHaveBeenCalledTimes(2));
+ await vi.waitFor(() => expect(service.getMobileSpeakerFilterSnapshot().modelState).toBe('ready'));
+ expect(states).toContain('missing');
+ expect(states).toContain('downloading');
+ for (const asset of SPEAKER_MODEL_ASSETS) {
+ expect(files.get('file:///cache/voice-models/' + speakerModelLocalName(asset))).toBe(asset.bytes);
+ }
+ service.warmMobileSpeakerFilter();
+ await vi.waitFor(() => expect(loadSpeakerModels).toHaveBeenCalledTimes(2));
+ expect(service.createMobileSpeakerGate()).toBeInstanceOf(SpeakerGate);
+ } finally {
+ unsubscribe();
+ }
+ });
+
+ it('retries a native load error without redownloading intact model files', async () => {
+ installModels();
+ await signIn('user-a');
+ vi.mocked(loadSpeakerModels).mockRejectedValueOnce(new Error('native load failed'));
+ await expect(service.enrollMobileSpeaker([new ArrayBuffer(32_000)])).resolves.toEqual({ ok: false, reason: 'failed' });
+ expect(service.getMobileSpeakerFilterSnapshot().modelState).toBe('ready');
+ expect(FileSystem.createDownloadResumable).not.toHaveBeenCalled();
+ await service.enrollMobileSpeaker([new ArrayBuffer(32_000)]);
+ expect(loadSpeakerModels).toHaveBeenCalledTimes(2);
+ });
+
+ it('does not overwrite a completed download with a stale cache check', async () => {
+ installModels();
+ await signIn('user-a');
+ vi.mocked(loadSpeakerModels).mockRejectedValueOnce(new Error('model file missing'));
+ let finishCheck!: (info: FileSystem.FileInfo) => void;
+ vi.mocked(FileSystem.getInfoAsync).mockImplementationOnce(() => new Promise((resolve) => { finishCheck = resolve; }));
+ const pending = service.enrollMobileSpeaker([new ArrayBuffer(32_000)]);
+ await vi.waitFor(() => expect(finishCheck).toBeTypeOf('function'));
+ files.clear();
+ await service.downloadMobileSpeakerModels();
+ expect(FileSystem.createDownloadResumable).toHaveBeenCalledTimes(2);
+ finishCheck({ exists: false, isDirectory: false, uri: 'file:///cache/voice-models/missing' });
+ await pending;
+ expect(service.getMobileSpeakerFilterSnapshot().modelState).toBe('ready');
+ });
+
it('loads the new owner even when the previous owner hydration is still pending', async () => {
let release!: (value: string | null) => void;
vi.mocked(getSecureItem).mockImplementationOnce(() => new Promise((resolve) => { release = resolve; }));
diff --git a/apps/mobile/src/session/mobileRealtimeAudio.ts b/apps/mobile/src/session/mobileRealtimeAudio.ts
index ef5a731a48..6f7dba7998 100644
--- a/apps/mobile/src/session/mobileRealtimeAudio.ts
+++ b/apps/mobile/src/session/mobileRealtimeAudio.ts
@@ -97,6 +97,9 @@ const EXPO_AUDIO_STREAM_WATCHDOG_INTERVAL_MS = 1_000;
let nativeBinding: RealtimeAudioNativeBinding | null | undefined;
let expoAudioNativeModule: ExpoAudioNativeModule | null | undefined;
+// The custom Apple module owns one native engine. A replacement must wait for
+// the previous caller's complete start/stop lifecycle, including late cleanup.
+let customNativeCaptureReleased: Promise<void> = Promise.resolve();
export function isMobileRealtimeAudioAvailable(): boolean {
if (isE2eMockRealtimeAudioEnabled()) return true;
@@ -146,8 +149,13 @@ async function startCustomNativeRealtimeAudio(
bufferSize?: number;
},
): Promise<() => Promise<void>> {
- const subscriptions: EventSubscription[] = [
- binding.module.addListener('onAudioChunk', (event) => {
+ const previousCapture = customNativeCaptureReleased;
+ let releaseCapture!: () => void;
+ customNativeCaptureReleased = new Promise<void>((resolve) => { releaseCapture = resolve; });
+ await previousCapture;
+ const subscriptions: EventSubscription[] = [];
+ try {
+ subscriptions.push(binding.module.addListener('onAudioChunk', (event) => {
options.onChunk({
pcm16: decodeBase64ToArrayBuffer(event.base64Pcm16),
trace: {
@@ -158,28 +166,37 @@ async function startCustomNativeRealtimeAudio(
durationMs: event.durationMs,
},
});
- }),
- binding.module.addListener('onAudioError', (event) => {
+ }));
+ subscriptions.push(binding.module.addListener('onAudioError', (event) => {
options.onError?.(new Error(event.message));
- }),
- ];
-
- try {
+ }));
await binding.module.start({
sampleRate: options.sampleRate ?? 16_000,
bufferSize: options.bufferSize ?? DEFAULT_NATIVE_AUDIO_BUFFER_SIZE,
});
} catch (error) {
- subscriptions.forEach((subscription) => subscription.remove());
+ try {
+ subscriptions.forEach((subscription) => subscription.remove());
+ } finally {
+ releaseCapture();
+ }
throw error;
}
- let stopped = false;
- return async () => {
- if (stopped) return;
- stopped = true;
- subscriptions.forEach((subscription) => subscription.remove());
- await binding.module.stop();
+ let stopping: Promise<void> | undefined;
+ return () => {
+ stopping ??= (async () => {
+ try {
+ try {
+ subscriptions.forEach((subscription) => subscription.remove());
+ } finally {
+ await binding.module.stop();
+ }
+ } finally {
+ releaseCapture();
+ }
+ })();
+ return stopping;
};
}
diff --git a/apps/mobile/src/session/mobileSpeakerFilter.ts b/apps/mobile/src/session/mobileSpeakerFilter.ts
index 595e73f792..e09310bcc7 100644
--- a/apps/mobile/src/session/mobileSpeakerFilter.ts
+++ b/apps/mobile/src/session/mobileSpeakerFilter.ts
@@ -81,7 +81,9 @@ let hydrating: Promise<void> | null = null;
let hydrationOwner: MobileAuthOwnerGeneration | null = null;
let profileRevision = 0;
let prefsRevision = 0;
+let savedPrefs: StoredPrefs = { version: 1, enabled: false };
let downloading: Promise<void> | null = null;
+let downloadRevision = 0;
let downloadAbort: AbortController | null = null;
let modelsLoaded: Promise<void> | null = null;
const listeners = new Set<() => void>();
@@ -140,6 +142,7 @@ export function hydrateMobileSpeakerFilter(): Promise<void> {
hydrating = null;
profile = null;
hydratedFor = null;
+ savedPrefs = { version: 1, enabled: false };
profileRevision += 1;
snapshot = {
...initialSnapshot(),
@@ -162,6 +165,7 @@ export function hydrateMobileSpeakerFilter(): Promise<void> {
if (!isMobileAuthOwnerCurrent(owner)) return;
profile = stored;
hydratedFor = owner.accountKey;
+ savedPrefs = { version: 1, ...prefs };
update({
supported: isSpeakerOnnxAvailable(),
hydrated: true,
@@ -207,7 +211,19 @@ async function writePrefs(): Promise<void> {
...(snapshot.strictnessCustomized ? { strictnessOverride: snapshot.strictness } : {}),
};
await serializeSpeakerFilterStorage(owner.accountKey, async () => {
- if (isMobileAuthOwnerCurrent(owner)) await AsyncStorage.setItem(prefsKey(owner.accountKey), JSON.stringify(stored));
+ if (!isMobileAuthOwnerCurrent(owner)) return;
+ await AsyncStorage.setItem(prefsKey(owner.accountKey), JSON.stringify(stored));
+ if (isMobileAuthOwnerCurrent(owner)) savedPrefs = stored;
+ });
+}
+
+function restoreSavedPrefs(): void {
+ // Overlapping optimistic edits can all fail. Only a completed write is a
+ // rollback baseline, including the other setting stored in the same record.
+ update({
+ enabled: savedPrefs.enabled,
+ strictness: savedPrefs.strictnessOverride ?? DEFAULT_SPEAKER_GATE_STRICTNESS,
+ strictnessCustomized: savedPrefs.strictnessOverride !== undefined,
});
}
@@ -231,18 +247,17 @@ export async function setMobileSpeakerFilterEnabled(enabled: boolean): Promise<v
const owner = getMobileAuthOwner();
await hydrateMobileSpeakerFilter();
if (!owner.accountKey || !isMobileAuthOwnerCurrent(owner)) return;
- const previous = snapshot.enabled;
const revision = ++prefsRevision;
update({ enabled });
try {
await writePrefs();
} catch (error) {
- if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) update({ enabled: previous });
+ if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) restoreSavedPrefs();
throw error;
}
if (!isMobileAuthOwnerCurrent(owner)) return;
// The models are fetched the first time the user turns the filter on.
- if (enabled && (snapshot.modelState === 'missing' || snapshot.modelState === 'failed')) {
+ if (snapshot.enabled && (snapshot.modelState === 'missing' || snapshot.modelState === 'failed')) {
void downloadMobileSpeakerModels();
}
}
@@ -252,7 +267,6 @@ export async function setMobileSpeakerFilterStrictness(strictness: SpeakerGateSt
const owner = getMobileAuthOwner();
await hydrateMobileSpeakerFilter();
if (!owner.accountKey || !isMobileAuthOwnerCurrent(owner)) return;
- const previous = { strictness: snapshot.strictness, strictnessCustomized: snapshot.strictnessCustomized };
const revision = ++prefsRevision;
update(
strictness === null
@@ -262,7 +276,7 @@ export async function setMobileSpeakerFilterStrictness(strictness: SpeakerGateSt
try {
await writePrefs();
} catch (error) {
- if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) update(previous);
+ if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) restoreSavedPrefs();
throw error;
}
}
@@ -285,6 +299,7 @@ async function modelsInstalled(): Promise<boolean> {
export function downloadMobileSpeakerModels(): Promise<void> {
if (!snapshot.supported) return Promise.resolve();
if (downloading) return downloading;
+ downloadRevision += 1;
const controller = new AbortController();
downloadAbort = controller;
update({ modelState: 'downloading', downloadedBytes: 0, downloadError: undefined });
@@ -301,6 +316,7 @@ export function downloadMobileSpeakerModels(): Promise<void> {
if (downloading === run) {
downloading = null;
downloadAbort = null;
+ downloadRevision += 1;
}
});
downloading = run;
@@ -440,12 +456,23 @@ function ensureModelsLoaded(): Promise<void> {
const vad = modelPath(speakerModelAsset('vad'));
const embedding = modelPath(speakerModelAsset('embedding'));
if (!vad || !embedding) return Promise.reject(new Error('speaker models are not installed'));
- const loading = withInferenceTimeout(loadSpeakerModels({ vad, embedding }));
- modelsLoaded = loading;
- // A failed load must not stick; the next run tries again.
- loading.catch(() => {
- if (modelsLoaded === loading) modelsLoaded = null;
+ const loading = withInferenceTimeout(loadSpeakerModels({ vad, embedding })).catch(async (error: unknown) => {
+ if (modelsLoaded === loading) {
+ // The OS can purge cache after hydration. Repair on load failure without
+ // adding filesystem I/O to every inference call or microphone startup.
+ const revision = downloadRevision;
+ const installed = await modelsInstalled();
+ if (modelsLoaded === loading) {
+ modelsLoaded = null;
+ if (!installed && !downloading && revision === downloadRevision) {
+ update({ modelState: 'missing', downloadedBytes: 0, downloadError: undefined });
+ if (snapshot.enabled) void downloadMobileSpeakerModels();
+ }
+ }
+ }
+ throw error;
});
+ modelsLoaded = loading;
return loading;
}
@@ -585,7 +612,9 @@ export const __testing = {
hydrationOwner = null;
profileRevision = 0;
prefsRevision = 0;
+ savedPrefs = { version: 1, enabled: false };
downloading = null;
+ downloadRevision = 0;
downloadAbort = null;
modelsLoaded = null;
ownerSubscription?.();