Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(voice): 修复手机声纹缓存恢复与录音并发
DavidShen28eb387db004
diff --git a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.tsindex 8a5bbf65d4..26108421b2 100644--- a/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts+++ b/apps/mobile/src/__tests__/mobileRealtimeAudio.test.ts@@ -32,6 +32,86 @@ afterEach(async () => {});describe('mobileRealtimeAudio', () => {+ it('waits for the previous singleton recorder to finish starting and stopping before reuse', async () => {+ let finishStart!: () => void;+ let finishStop!: () => void;+ const native = {+ addListener: vi.fn(() => ({ remove: vi.fn() })),+ start: vi.fn(async (): Promise<void> => {})+ .mockImplementationOnce(() => new Promise<void>((resolve) => { finishStart = resolve; })),+ stop: vi.fn(async (): Promise<void> => {})+ .mockImplementationOnce(() => new Promise<void>((resolve) => { finishStop = resolve; })),+ };+ vi.mocked(requireNativeModule).mockReturnValue(native);+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');+ const first = startMobileRealtimeAudio({ onChunk: vi.fn() });+ await vi.waitFor(() => expect(finishStart).toBeTypeOf('function'));+ const second = startMobileRealtimeAudio({ onChunk: vi.fn() });+ for (let i = 0; i < 10; i += 1) await Promise.resolve();+ const startsWhileStarting = native.start.mock.calls.length;+ const listenersWhileStarting = native.addListener.mock.calls.length;+ finishStart();+ const stopFirst = await first;+ const stopping = stopFirst();+ await vi.waitFor(() => expect(finishStop).toBeTypeOf('function'));+ const startsWhileStopping = native.start.mock.calls.length;+ finishStop();+ await stopping;+ const stopSecond = await second;+ await stopFirst();+ const stopsAfterOldCleanup = native.stop.mock.calls.length;+ await stopSecond();++ expect(startsWhileStarting).toBe(1);+ expect(listenersWhileStarting).toBe(2);+ expect(startsWhileStopping).toBe(1);+ expect(native.start).toHaveBeenCalledTimes(2);+ expect(stopsAfterOldCleanup).toBe(1);+ expect(native.stop).toHaveBeenCalledTimes(2);+ });++ it.each(['start', 'stop'] as const)('allows a new singleton recording after the previous %s rejects', async (failure) => {+ const native = {+ addListener: vi.fn(() => ({ remove: vi.fn() })),+ start: vi.fn(async () => undefined),+ stop: vi.fn(async () => undefined),+ };+ native[failure].mockRejectedValueOnce(new Error('native failure'));+ vi.mocked(requireNativeModule).mockReturnValue(native);+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');+ const first = startMobileRealtimeAudio({ onChunk: vi.fn() });+ if (failure === 'start') await expect(first).rejects.toThrow('native failure');+ else await expect((await first)()).rejects.toThrow('native failure');+ const stop = await startMobileRealtimeAudio({ onChunk: vi.fn() });+ await stop();+ expect(native.start).toHaveBeenCalledTimes(2);+ });++ it.each(['subscribe', 'unsubscribe'] as const)('releases the singleton recorder after %s throws', async (failure) => {+ const remove = vi.fn();+ const native = {+ addListener: vi.fn(() => ({ remove })),+ start: vi.fn(async () => undefined),+ stop: vi.fn(async () => undefined),+ };+ if (failure === 'subscribe') {+ native.addListener+ .mockImplementationOnce(() => ({ remove }))+ .mockImplementationOnce(() => { throw new Error('listener failure'); });+ } else {+ remove.mockImplementationOnce(() => { throw new Error('listener failure'); });+ }+ vi.mocked(requireNativeModule).mockReturnValue(native);+ const { startMobileRealtimeAudio } = await import('@/session/mobileRealtimeAudio');+ const first = startMobileRealtimeAudio({ onChunk: vi.fn() });+ if (failure === 'subscribe') await expect(first).rejects.toThrow('listener failure');+ else await expect((await first)()).rejects.toThrow('listener failure');+ expect(remove).toHaveBeenCalled();+ expect(native.stop).toHaveBeenCalledTimes(failure === 'subscribe' ? 0 : 1);+ const stop = await startMobileRealtimeAudio({ onChunk: vi.fn() });+ await stop();+ });+it('decodes native PCM base64 payloads into ArrayBuffer chunks', async () => {const { __testing } = await import('@/session/mobileRealtimeAudio');diff --git a/apps/mobile/src/__tests__/mobileSpeakerFilter.test.ts b/apps/mobile/src/__tests__/mobileSpeakerFilter.test.tsindex c08af19bce..4ca21aeeb5 100644--- a/apps/mobile/src/__tests__/mobileSpeakerFilter.test.ts+++ b/apps/mobile/src/__tests__/mobileSpeakerFilter.test.ts@@ -131,6 +131,133 @@ describe('mobile speaker filter', () => {expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({ strictness: 'standard', strictnessCustomized: false });});+ it.each(['enabled', 'strictness', 'mixed'] as const)('restores committed preferences after overlapping %s writes fail', async (kind) => {+ installModels();+ const key = await signIn('user-a');+ await service.setMobileSpeakerFilterStrictness('relaxed');+ const saved = asyncStore.get(service.__testing.prefsKey(key));+ let rejectFirst!: (error: Error) => void;+ vi.mocked(AsyncStorage.setItem)+ .mockImplementationOnce(() => new Promise((_resolve, reject) => { rejectFirst = reject; }))+ .mockRejectedValueOnce(new Error('second write failed'));+ const first = kind === 'strictness'+ ? service.setMobileSpeakerFilterStrictness('strict')+ : service.setMobileSpeakerFilterEnabled(true);+ const firstFailure = expect(first).rejects.toThrow('first write failed');+ await vi.waitFor(() => expect(rejectFirst).toBeTypeOf('function'));+ const second = kind === 'enabled'+ ? service.setMobileSpeakerFilterEnabled(false)+ : service.setMobileSpeakerFilterStrictness(null);+ const secondFailure = expect(second).rejects.toThrow('second write failed');+ await vi.waitFor(() => expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject(+ kind === 'enabled' ? { enabled: false } : { strictnessCustomized: false },+ ));+ rejectFirst(new Error('first write failed'));+ await Promise.all([firstFailure, secondFailure]);+ expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({+ enabled: false, strictness: 'relaxed', strictnessCustomized: true,+ });+ expect(asyncStore.get(service.__testing.prefsKey(key))).toBe(saved);+ });++ it('rolls back to an overlapping successful write when the following write fails', async () => {+ installModels();+ const key = await signIn('user-a');+ let finishFirst!: () => void;+ vi.mocked(AsyncStorage.setItem)+ .mockImplementationOnce(async (name, value) => {+ await new Promise<void>((resolve) => { finishFirst = resolve; });+ asyncStore.set(name, value);+ })+ .mockRejectedValueOnce(new Error('second write failed'));+ const first = service.setMobileSpeakerFilterEnabled(true);+ await vi.waitFor(() => expect(finishFirst).toBeTypeOf('function'));+ const secondFailure = expect(service.setMobileSpeakerFilterStrictness('strict')).rejects.toThrow('second write failed');+ await vi.waitFor(() => expect(service.getMobileSpeakerFilterSnapshot().strictness).toBe('strict'));+ finishFirst();+ await Promise.all([first, secondFailure]);+ expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({+ enabled: true, strictness: 'standard', strictnessCustomized: false,+ });+ expect(JSON.parse(asyncStore.get(service.__testing.prefsKey(key))!)).toEqual({ version: 1, enabled: true });+ });++ it('does not use a late write from the previous account as the new rollback baseline', async () => {+ installModels();+ await signIn('user-a');+ let finishFirst!: () => void;+ vi.mocked(AsyncStorage.setItem).mockImplementationOnce(async (name, value) => {+ await new Promise<void>((resolve) => { finishFirst = resolve; });+ asyncStore.set(name, value);+ });+ const first = service.setMobileSpeakerFilterStrictness('strict');+ await vi.waitFor(() => expect(finishFirst).toBeTypeOf('function'));+ await signIn('user-b');+ await service.setMobileSpeakerFilterStrictness('relaxed');+ finishFirst();+ await first;+ vi.mocked(AsyncStorage.setItem).mockRejectedValueOnce(new Error('write failed'));+ await expect(service.setMobileSpeakerFilterEnabled(true)).rejects.toThrow('write failed');+ expect(service.getMobileSpeakerFilterSnapshot()).toMatchObject({+ enabled: false, strictness: 'relaxed', strictnessCustomized: true,+ });+ });++ it('redownloads purged cache after a native load failure and can load again', async () => {+ installModels();+ const key = await signIn('user-a');+ storeProfile(key);+ service.__testing.reset();+ await signIn('user-a');+ await service.setMobileSpeakerFilterEnabled(true);+ files.clear();+ vi.mocked(loadSpeakerModels).mockRejectedValueOnce(new Error('model file missing'));+ const states: string[] = [];+ const unsubscribe = service.subscribeMobileSpeakerFilter(() => states.push(service.getMobileSpeakerFilterSnapshot().modelState));+ try {+ service.warmMobileSpeakerFilter();+ await vi.waitFor(() => expect(FileSystem.createDownloadResumable).toHaveBeenCalledTimes(2));+ await vi.waitFor(() => expect(service.getMobileSpeakerFilterSnapshot().modelState).toBe('ready'));+ expect(states).toContain('missing');+ expect(states).toContain('downloading');+ for (const asset of SPEAKER_MODEL_ASSETS) {+ expect(files.get('file:///cache/voice-models/' + speakerModelLocalName(asset))).toBe(asset.bytes);+ }+ service.warmMobileSpeakerFilter();+ await vi.waitFor(() => expect(loadSpeakerModels).toHaveBeenCalledTimes(2));+ expect(service.createMobileSpeakerGate()).toBeInstanceOf(SpeakerGate);+ } finally {+ unsubscribe();+ }+ });++ it('retries a native load error without redownloading intact model files', async () => {+ installModels();+ await signIn('user-a');+ vi.mocked(loadSpeakerModels).mockRejectedValueOnce(new Error('native load failed'));+ await expect(service.enrollMobileSpeaker([new ArrayBuffer(32_000)])).resolves.toEqual({ ok: false, reason: 'failed' });+ expect(service.getMobileSpeakerFilterSnapshot().modelState).toBe('ready');+ expect(FileSystem.createDownloadResumable).not.toHaveBeenCalled();+ await service.enrollMobileSpeaker([new ArrayBuffer(32_000)]);+ expect(loadSpeakerModels).toHaveBeenCalledTimes(2);+ });++ it('does not overwrite a completed download with a stale cache check', async () => {+ installModels();+ await signIn('user-a');+ vi.mocked(loadSpeakerModels).mockRejectedValueOnce(new Error('model file missing'));+ let finishCheck!: (info: FileSystem.FileInfo) => void;+ vi.mocked(FileSystem.getInfoAsync).mockImplementationOnce(() => new Promise((resolve) => { finishCheck = resolve; }));+ const pending = service.enrollMobileSpeaker([new ArrayBuffer(32_000)]);+ await vi.waitFor(() => expect(finishCheck).toBeTypeOf('function'));+ files.clear();+ await service.downloadMobileSpeakerModels();+ expect(FileSystem.createDownloadResumable).toHaveBeenCalledTimes(2);+ finishCheck({ exists: false, isDirectory: false, uri: 'file:///cache/voice-models/missing' });+ await pending;+ expect(service.getMobileSpeakerFilterSnapshot().modelState).toBe('ready');+ });+it('loads the new owner even when the previous owner hydration is still pending', async () => {let release!: (value: string | null) => void;vi.mocked(getSecureItem).mockImplementationOnce(() => new Promise((resolve) => { release = resolve; }));diff --git a/apps/mobile/src/session/mobileRealtimeAudio.ts b/apps/mobile/src/session/mobileRealtimeAudio.tsindex ef5a731a48..6f7dba7998 100644--- a/apps/mobile/src/session/mobileRealtimeAudio.ts+++ b/apps/mobile/src/session/mobileRealtimeAudio.ts@@ -97,6 +97,9 @@ const EXPO_AUDIO_STREAM_WATCHDOG_INTERVAL_MS = 1_000;let nativeBinding: RealtimeAudioNativeBinding | null | undefined;let expoAudioNativeModule: ExpoAudioNativeModule | null | undefined;+// The custom Apple module owns one native engine. A replacement must wait for+// the previous caller's complete start/stop lifecycle, including late cleanup.+let customNativeCaptureReleased: Promise<void> = Promise.resolve();export function isMobileRealtimeAudioAvailable(): boolean {if (isE2eMockRealtimeAudioEnabled()) return true;@@ -146,8 +149,13 @@ async function startCustomNativeRealtimeAudio(bufferSize?: number;},): Promise<() => Promise<void>> {- const subscriptions: EventSubscription[] = [- binding.module.addListener('onAudioChunk', (event) => {+ const previousCapture = customNativeCaptureReleased;+ let releaseCapture!: () => void;+ customNativeCaptureReleased = new Promise<void>((resolve) => { releaseCapture = resolve; });+ await previousCapture;+ const subscriptions: EventSubscription[] = [];+ try {+ subscriptions.push(binding.module.addListener('onAudioChunk', (event) => {options.onChunk({pcm16: decodeBase64ToArrayBuffer(event.base64Pcm16),trace: {@@ -158,28 +166,37 @@ async function startCustomNativeRealtimeAudio(durationMs: event.durationMs,},});- }),- binding.module.addListener('onAudioError', (event) => {+ }));+ subscriptions.push(binding.module.addListener('onAudioError', (event) => {options.onError?.(new Error(event.message));- }),- ];-- try {+ }));await binding.module.start({sampleRate: options.sampleRate ?? 16_000,bufferSize: options.bufferSize ?? DEFAULT_NATIVE_AUDIO_BUFFER_SIZE,});} catch (error) {- subscriptions.forEach((subscription) => subscription.remove());+ try {+ subscriptions.forEach((subscription) => subscription.remove());+ } finally {+ releaseCapture();+ }throw error;}- let stopped = false;- return async () => {- if (stopped) return;- stopped = true;- subscriptions.forEach((subscription) => subscription.remove());- await binding.module.stop();+ let stopping: Promise<void> | undefined;+ return () => {+ stopping ??= (async () => {+ try {+ try {+ subscriptions.forEach((subscription) => subscription.remove());+ } finally {+ await binding.module.stop();+ }+ } finally {+ releaseCapture();+ }+ })();+ return stopping;};}diff --git a/apps/mobile/src/session/mobileSpeakerFilter.ts b/apps/mobile/src/session/mobileSpeakerFilter.tsindex 595e73f792..e09310bcc7 100644--- a/apps/mobile/src/session/mobileSpeakerFilter.ts+++ b/apps/mobile/src/session/mobileSpeakerFilter.ts@@ -81,7 +81,9 @@ let hydrating: Promise<void> | null = null;let hydrationOwner: MobileAuthOwnerGeneration | null = null;let profileRevision = 0;let prefsRevision = 0;+let savedPrefs: StoredPrefs = { version: 1, enabled: false };let downloading: Promise<void> | null = null;+let downloadRevision = 0;let downloadAbort: AbortController | null = null;let modelsLoaded: Promise<void> | null = null;const listeners = new Set<() => void>();@@ -140,6 +142,7 @@ export function hydrateMobileSpeakerFilter(): Promise<void> {hydrating = null;profile = null;hydratedFor = null;+ savedPrefs = { version: 1, enabled: false };profileRevision += 1;snapshot = {...initialSnapshot(),@@ -162,6 +165,7 @@ export function hydrateMobileSpeakerFilter(): Promise<void> {if (!isMobileAuthOwnerCurrent(owner)) return;profile = stored;hydratedFor = owner.accountKey;+ savedPrefs = { version: 1, ...prefs };update({supported: isSpeakerOnnxAvailable(),hydrated: true,@@ -207,7 +211,19 @@ async function writePrefs(): Promise<void> {...(snapshot.strictnessCustomized ? { strictnessOverride: snapshot.strictness } : {}),};await serializeSpeakerFilterStorage(owner.accountKey, async () => {- if (isMobileAuthOwnerCurrent(owner)) await AsyncStorage.setItem(prefsKey(owner.accountKey), JSON.stringify(stored));+ if (!isMobileAuthOwnerCurrent(owner)) return;+ await AsyncStorage.setItem(prefsKey(owner.accountKey), JSON.stringify(stored));+ if (isMobileAuthOwnerCurrent(owner)) savedPrefs = stored;+ });+}++function restoreSavedPrefs(): void {+ // Overlapping optimistic edits can all fail. Only a completed write is a+ // rollback baseline, including the other setting stored in the same record.+ update({+ enabled: savedPrefs.enabled,+ strictness: savedPrefs.strictnessOverride ?? DEFAULT_SPEAKER_GATE_STRICTNESS,+ strictnessCustomized: savedPrefs.strictnessOverride !== undefined,});}@@ -231,18 +247,17 @@ export async function setMobileSpeakerFilterEnabled(enabled: boolean): Promise<vconst owner = getMobileAuthOwner();await hydrateMobileSpeakerFilter();if (!owner.accountKey || !isMobileAuthOwnerCurrent(owner)) return;- const previous = snapshot.enabled;const revision = ++prefsRevision;update({ enabled });try {await writePrefs();} catch (error) {- if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) update({ enabled: previous });+ if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) restoreSavedPrefs();throw error;}if (!isMobileAuthOwnerCurrent(owner)) return;// The models are fetched the first time the user turns the filter on.- if (enabled && (snapshot.modelState === 'missing' || snapshot.modelState === 'failed')) {+ if (snapshot.enabled && (snapshot.modelState === 'missing' || snapshot.modelState === 'failed')) {void downloadMobileSpeakerModels();}}@@ -252,7 +267,6 @@ export async function setMobileSpeakerFilterStrictness(strictness: SpeakerGateStconst owner = getMobileAuthOwner();await hydrateMobileSpeakerFilter();if (!owner.accountKey || !isMobileAuthOwnerCurrent(owner)) return;- const previous = { strictness: snapshot.strictness, strictnessCustomized: snapshot.strictnessCustomized };const revision = ++prefsRevision;update(strictness === null@@ -262,7 +276,7 @@ export async function setMobileSpeakerFilterStrictness(strictness: SpeakerGateSttry {await writePrefs();} catch (error) {- if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) update(previous);+ if (isMobileAuthOwnerCurrent(owner) && revision === prefsRevision) restoreSavedPrefs();throw error;}}@@ -285,6 +299,7 @@ async function modelsInstalled(): Promise<boolean> {export function downloadMobileSpeakerModels(): Promise<void> {if (!snapshot.supported) return Promise.resolve();if (downloading) return downloading;+ downloadRevision += 1;const controller = new AbortController();downloadAbort = controller;update({ modelState: 'downloading', downloadedBytes: 0, downloadError: undefined });@@ -301,6 +316,7 @@ export function downloadMobileSpeakerModels(): Promise<void> {if (downloading === run) {downloading = null;downloadAbort = null;+ downloadRevision += 1;}});downloading = run;@@ -440,12 +456,23 @@ function ensureModelsLoaded(): Promise<void> {const vad = modelPath(speakerModelAsset('vad'));const embedding = modelPath(speakerModelAsset('embedding'));if (!vad || !embedding) return Promise.reject(new Error('speaker models are not installed'));- const loading = withInferenceTimeout(loadSpeakerModels({ vad, embedding }));- modelsLoaded = loading;- // A failed load must not stick; the next run tries again.- loading.catch(() => {- if (modelsLoaded === loading) modelsLoaded = null;+ const loading = withInferenceTimeout(loadSpeakerModels({ vad, embedding })).catch(async (error: unknown) => {+ if (modelsLoaded === loading) {+ // The OS can purge cache after hydration. Repair on load failure without+ // adding filesystem I/O to every inference call or microphone startup.+ const revision = downloadRevision;+ const installed = await modelsInstalled();+ if (modelsLoaded === loading) {+ modelsLoaded = null;+ if (!installed && !downloading && revision === downloadRevision) {+ update({ modelState: 'missing', downloadedBytes: 0, downloadError: undefined });+ if (snapshot.enabled) void downloadMobileSpeakerModels();+ }+ }+ }+ throw error;});+ modelsLoaded = loading;return loading;}@@ -585,7 +612,9 @@ export const __testing = {hydrationOwner = null;profileRevision = 0;prefsRevision = 0;+ savedPrefs = { version: 1, enabled: false };downloading = null;+ downloadRevision = 0;downloadAbort = null;modelsLoaded = null;ownerSubscription?.();