Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(app): stop the Firefox microphone failure and self-mute storm (#1761)

Hampusb7de83f7fcdc
diff --git a/fluxer_app/src/features/devtools/commands/VoiceStateCommands.tsx b/fluxer_app/src/features/devtools/commands/VoiceStateCommands.tsx
index 929eee57e..173ddf820 100644
--- a/fluxer_app/src/features/devtools/commands/VoiceStateCommands.tsx
+++ b/fluxer_app/src/features/devtools/commands/VoiceStateCommands.tsx
@@ -125,7 +125,8 @@ export async function toggleSelfMute(_guildId: string | null = null): Promise<vo
}
const room = MediaEngine.room;
const connectedChannelId = MediaEngine.channelId;
- const currentMute = LocalVoiceState.getSelfMute();
+ const microphoneFailureLatched = MediaEngine.isMicrophoneFailureLatched();
+ const currentMute = LocalVoiceState.getSelfMute() || microphoneFailureLatched;
const currentDeaf = LocalVoiceState.getSelfDeaf();
const willUndeafen = currentDeaf;
const willUnmute = currentMute;
@@ -139,6 +140,7 @@ export async function toggleSelfMute(_guildId: string | null = null): Promise<vo
logger.info('toggleSelfMute', {
currentMute,
currentDeaf,
+ microphoneFailureLatched,
willUnmute,
willUndeafen,
willMute,
@@ -158,6 +160,7 @@ export async function toggleSelfMute(_guildId: string | null = null): Promise<vo
handleMicrophonePermissionBlocked();
return;
}
+ MediaEngine.clearMicrophoneFailureLatch();
if (!MediaPermission.isMicrophoneGranted()) {
logger.info('Microphone permission not granted, requesting permission');
const permissionGranted = room?.localParticipant
@@ -175,6 +178,11 @@ export async function toggleSelfMute(_guildId: string | null = null): Promise<vo
return;
}
}
+ if (!LocalVoiceState.getSelfMute() && !LocalVoiceState.getSelfDeaf()) {
+ logger.debug('Unmute intent only had to clear the microphone failure latch');
+ SoundCommands.playSound(SoundType.Unmute);
+ return;
+ }
}
LocalVoiceState.toggleSelfMute();
const newMute = LocalVoiceState.getSelfMute();
diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
index 02ca2f492..3a5b0938f 100644
--- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
+++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
@@ -130,6 +130,16 @@ import {
noteLocalVoiceActivity,
noteLocalVoiceActivityFromSnapshot,
} from '@app/features/voice/engine/VoiceIdleActivityBridge';
+import {
+ createVoiceLocalAudioReconcileCoalescerSnapshot,
+ selectVoiceLocalAudioEffectiveSelfMute,
+ selectVoiceLocalAudioReconcileCoalescedCount,
+ selectVoiceLocalAudioReconcileRunReasons,
+ shouldStartVoiceLocalAudioReconcileRun,
+ shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns,
+ transitionVoiceLocalAudioReconcileCoalescerSnapshot,
+ type VoiceLocalAudioReconcileCoalescerSnapshot,
+} from '@app/features/voice/engine/VoiceLocalAudioReconcilePolicy';
import {normalizeVoiceMediaGraphViewerStreamKeys} from '@app/features/voice/engine/VoiceMediaGraph';
import {startVoiceMediaGraphTimerScheduler} from '@app/features/voice/engine/VoiceMediaGraphTimerScheduler';
import {bindOutputDeviceSync} from '@app/features/voice/engine/VoiceOutputDeviceSync';
@@ -748,6 +758,9 @@ class MediaEngineFacade extends Store {
private videoCodecDecodeCapResyncDisposer: (() => void) | null = null;
private videoCodecPublishOverrideSyncDisposer: (() => void) | null = null;
private localStreamCodecReconcileScheduled = false;
+ private localAudioReconcileCoalescer: VoiceLocalAudioReconcileCoalescerSnapshot =
+ createVoiceLocalAudioReconcileCoalescerSnapshot();
+ private applyingGatewayVoiceStateEcho = false;
private previousWatchedStreamCodecGossip = new Map<
string,
{identity: string; source: VoiceEngineV2LocalStreamSource}
@@ -1349,9 +1362,40 @@ class MediaEngineFacade extends Store {
}
private reconcileLocalAudioStateInBackground(reason: string): void {
- void this.reconcileLocalAudioState(reason).catch((error) => {
- logger.warn('Local audio reconciliation failed', {reason, error});
- });
+ const previous = this.localAudioReconcileCoalescer;
+ const next = transitionVoiceLocalAudioReconcileCoalescerSnapshot(previous, {type: 'run.requested', reason});
+ this.localAudioReconcileCoalescer = next;
+ if (!shouldStartVoiceLocalAudioReconcileRun(previous, next)) return;
+ void this.drainLocalAudioReconcile();
+ }
+
+ private async drainLocalAudioReconcile(): Promise<void> {
+ try {
+ for (;;) {
+ const reasons = selectVoiceLocalAudioReconcileRunReasons(this.localAudioReconcileCoalescer);
+ try {
+ await this.reconcileLocalAudioState(reasons.join(' + '));
+ } catch (error) {
+ logger.warn('Local audio reconciliation failed', {reasons, error});
+ }
+ const previous = this.localAudioReconcileCoalescer;
+ const next = transitionVoiceLocalAudioReconcileCoalescerSnapshot(previous, {type: 'run.settled'});
+ this.localAudioReconcileCoalescer = next;
+ if (!shouldStartVoiceLocalAudioReconcileRun(previous, next)) return;
+ if (
+ shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(next) &&
+ !shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(previous)
+ ) {
+ logger.warn('Local audio reconciliation keeps re-queueing itself', {
+ reasons: selectVoiceLocalAudioReconcileRunReasons(next),
+ coalescedCount: selectVoiceLocalAudioReconcileCoalescedCount(previous),
+ });
+ }
+ }
+ } catch (error) {
+ logger.error('Local audio reconciliation loop aborted; resetting the coalescer', {error});
+ this.localAudioReconcileCoalescer = createVoiceLocalAudioReconcileCoalescerSnapshot();
+ }
}
private clearNativeVoiceTransportReconnect(): void {
@@ -1601,7 +1645,11 @@ class MediaEngineFacade extends Store {
previousState,
currentState,
});
+ const unmutedLocally = previousState.selfMute && !currentState.selfMute && !this.applyingGatewayVoiceStateEcho;
previousState = currentState;
+ if (unmutedLocally) {
+ voiceEngineV2AppMediaExecutionAdapter.noteUserMuteIntentChanged();
+ }
this.syncVoiceEngineV2AudioControlsFromAppState();
this.reconcileLocalAudioStateInBackground('local audio state change');
});
@@ -2110,6 +2158,7 @@ class MediaEngineFacade extends Store {
SoundCommands.playSound(SoundType.VoiceDisconnect);
VoiceCallLayout.reset();
voiceEngineV2AppMediaStateAdapter.resetLocalMediaState('disconnect');
+ voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
this.resetLocalMediaAndScreenShareTracking();
this.voiceEngineV2Participants.clear();
if (connectionId) {
@@ -2148,6 +2197,7 @@ class MediaEngineFacade extends Store {
}
VoiceCallLayout.reset();
voiceEngineV2AppMediaStateAdapter.resetLocalMediaState('disconnect');
+ voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
this.resetLocalMediaAndScreenShareTracking();
this.voiceEngineV2Participants.clear();
if (reason === 'user' && connectionId) {
@@ -2208,7 +2258,10 @@ class MediaEngineFacade extends Store {
private syncVoiceEngineV2AudioControlsFromAppState(): void {
this.voiceEngineV2Controller.setAudioControls({
mode: this.getVoiceEngineV2AudioMode(),
- locallyMuted: LocalVoiceState.getSelfMute(),
+ locallyMuted: selectVoiceLocalAudioEffectiveSelfMute({
+ localSelfMute: LocalVoiceState.getSelfMute(),
+ microphoneFailureLatched: voiceEngineV2AppMediaExecutionAdapter.isMicrophoneFailureLatched(),
+ }),
preferredLocallyMuted: LocalVoiceState.getSelfMute(),
locallyDeafened: LocalVoiceState.getSelfDeaf(),
mutedByPermission: LocalVoiceState.getMutedByPermission(),
@@ -2341,6 +2394,7 @@ class MediaEngineFacade extends Store {
}
void ScreenShareCodecNegotiation.publishLocalCapabilitiesNative('connected');
this.restoreNativeLocalMediaStateInBackground('native voice engine connected');
+ voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
this.reconcileLocalAudioStateInBackground('native voice engine connected');
}
@@ -3111,6 +3165,7 @@ class MediaEngineFacade extends Store {
this.statsHostAdapter.startStatsTracking();
void ScreenShareCodecNegotiation.publishLocalCapabilitiesNative('reconnected');
this.restoreNativeLocalMediaStateInBackground('native voice engine reconnected');
+ voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
this.reconcileLocalAudioStateInBackground('native voice engine reconnected');
break;
default:
@@ -3246,6 +3301,7 @@ class MediaEngineFacade extends Store {
if (shouldApplyPendingSessionRestore) {
await this.restorePendingSessionMedia();
}
+ voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
await this.reconcileLocalAudioState('voice room connected');
},
onDisconnected: () => {
@@ -3266,6 +3322,7 @@ class MediaEngineFacade extends Store {
this.statsHostAdapter.incrementReconnectionCount();
this.statsHostAdapter.startLatencyTracking();
this.statsHostAdapter.startStatsTracking();
+ voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
this.reconcileLocalAudioStateInBackground('voice room reconnected');
},
},
@@ -3320,6 +3377,7 @@ class MediaEngineFacade extends Store {
if (guildId && channelId) {
VoiceEngineV2AppPermissionAdapter.syncWithPermissionState(guildId, channelId, newRoom);
}
+ voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
this.reconcileLocalAudioStateInBackground('region hot-swap complete');
},
(_guildId, _channelId, _connectionId, _attemptId, error) => {
@@ -3471,13 +3529,18 @@ class MediaEngineFacade extends Store {
const pttActive = this.isPushToTalkActive();
const applyServerState = shouldApplyIncomingLocalState;
shouldApplyCurrentServerState = applyServerState;
- LocalVoiceState.syncConnectionState(voiceState.connection_id, {
- selfMute: ptmActive || pttActive || !applyServerState ? undefined : voiceState.self_mute,
- selfDeaf: applyServerState ? voiceState.self_deaf : undefined,
- selfVideo: applyServerState ? voiceState.self_video : undefined,
- selfStream: applyServerState ? voiceState.self_stream : undefined,
- viewerStreamKeys: applyServerState ? incomingViewerStreamKeys : undefined,
- });
+ this.applyingGatewayVoiceStateEcho = true;
+ try {
+ LocalVoiceState.syncConnectionState(voiceState.connection_id, {
+ selfMute: ptmActive || pttActive || !applyServerState ? undefined : voiceState.self_mute,
+ selfDeaf: applyServerState ? voiceState.self_deaf : undefined,
+ selfVideo: applyServerState ? voiceState.self_video : undefined,
+ selfStream: applyServerState ? voiceState.self_stream : undefined,
+ viewerStreamKeys: applyServerState ? incomingViewerStreamKeys : undefined,
+ });
+ } finally {
+ this.applyingGatewayVoiceStateEcho = false;
+ }
if (applyServerState) {
replaceWatchedStreamKeys([...incomingViewerStreamKeys], {sync: false});
}
@@ -3667,11 +3730,13 @@ class MediaEngineFacade extends Store {
}
syncLocalVoiceStateWithServer(partial?: VoiceStateSyncPartial): void {
- LocalVoiceState.ensurePermissionMute();
- const {guildId, channelId, connectionId} = voiceEngineV2AppConnectionHostAdapter.connectionState;
- if (!channelId || !connectionId) return;
const devicePermission = VoiceDevicePermissionState.getState().permissionStatus;
const micGranted = MediaPermission.isMicrophoneGranted() || devicePermission === 'granted';
+ if (!micGranted || LocalVoiceState.getMutedByPermission()) {
+ LocalVoiceState.ensurePermissionMute();
+ }
+ const {guildId, channelId, connectionId} = voiceEngineV2AppConnectionHostAdapter.connectionState;
+ if (!channelId || !connectionId) return;
const selfMute = resolveVoiceStateSelfMute({
guildId,
channelId,
@@ -4671,6 +4736,17 @@ class MediaEngineFacade extends Store {
return voiceEngineV2AppMediaExecutionAdapter.getMuteReason(voiceState, this.guildId, this.channelId);
}
+ isMicrophoneFailureLatched(): boolean {
+ return voiceEngineV2AppMediaExecutionAdapter.isMicrophoneFailureLatched();
+ }
+
+ clearMicrophoneFailureLatch(): void {
+ if (!voiceEngineV2AppMediaExecutionAdapter.isMicrophoneFailureLatched()) return;
+ voiceEngineV2AppMediaExecutionAdapter.noteUserMuteIntentChanged();
+ this.syncVoiceEngineV2AudioControlsFromAppState();
+ this.reconcileLocalAudioStateInBackground('microphone failure latch cleared');
+ }
+
async toggleCameraFromKeybind(): Promise<void> {
const current = LocalVoiceState.getSelfVideo();
await this.setCameraEnabled(!current, {deviceId: VoiceSettings.getVideoDeviceId()});
diff --git a/fluxer_app/src/features/voice/engine/VoiceLocalAudioReconcilePolicy.test.ts b/fluxer_app/src/features/voice/engine/VoiceLocalAudioReconcilePolicy.test.ts
new file mode 100644
index 000000000..94337e1f1
--- /dev/null
+++ b/fluxer_app/src/features/voice/engine/VoiceLocalAudioReconcilePolicy.test.ts
@@ -0,0 +1,407 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {describe, expect, it} from 'vitest';
+import {
+ createVoiceLocalAudioReconcileCoalescerSnapshot,
+ createVoiceMicrophoneFailureLatchSnapshot,
+ isVoiceMicrophoneFailureLatchActive,
+ selectVoiceLocalAudioEffectiveSelfMute,
+ selectVoiceLocalAudioReconcileCoalescedCount,
+ selectVoiceLocalAudioReconcileRunReasons,
+ selectVoiceLocalAudioReconcileStartedRuns,
+ selectVoiceMicrophoneFailureCount,
+ shouldStartVoiceLocalAudioReconcileRun,
+ shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns,
+ transitionVoiceLocalAudioReconcileCoalescerSnapshot,
+ transitionVoiceMicrophoneFailureLatchSnapshot,
+ VOICE_LOCAL_AUDIO_RECONCILE_FOLLOW_UP_WARN_LIMIT,
+ VOICE_LOCAL_AUDIO_RECONCILE_REASON_LIMIT,
+ type VoiceLocalAudioReconcileCoalescerEvent,
+ type VoiceLocalAudioReconcileCoalescerSnapshot,
+ type VoiceMicrophoneFailureLatchEvent,
+ type VoiceMicrophoneFailureLatchSnapshot,
+} from './VoiceLocalAudioReconcilePolicy';
+
+const RUNAWAY_RUN_LIMIT = 512;
+
+function sendCoalescer(
+ snapshot: VoiceLocalAudioReconcileCoalescerSnapshot,
+ event: VoiceLocalAudioReconcileCoalescerEvent,
+): VoiceLocalAudioReconcileCoalescerSnapshot {
+ return transitionVoiceLocalAudioReconcileCoalescerSnapshot(snapshot, event);
+}
+
+function sendLatch(
+ snapshot: VoiceMicrophoneFailureLatchSnapshot,
+ event: VoiceMicrophoneFailureLatchEvent,
+): VoiceMicrophoneFailureLatchSnapshot {
+ return transitionVoiceMicrophoneFailureLatchSnapshot(snapshot, event);
+}
+
+class ReconcileHarness {
+ snapshot = createVoiceLocalAudioReconcileCoalescerSnapshot();
+ startedRuns: Array<ReadonlyArray<string>> = [];
+ private runningDepth = 0;
+
+ constructor(private readonly body: (harness: ReconcileHarness) => void) {}
+
+ request(reason: string): void {
+ const previous = this.snapshot;
+ this.snapshot = sendCoalescer(previous, {type: 'run.requested', reason});
+ if (shouldStartVoiceLocalAudioReconcileRun(previous, this.snapshot)) {
+ this.drain();
+ }
+ }
+
+ private drain(): void {
+ expect(this.runningDepth).toBe(0);
+ this.runningDepth += 1;
+ for (;;) {
+ if (this.startedRuns.length >= RUNAWAY_RUN_LIMIT) {
+ throw new Error('runaway local audio reconciliation');
+ }
+ this.startedRuns.push(selectVoiceLocalAudioReconcileRunReasons(this.snapshot));
+ this.body(this);
+ const previous = this.snapshot;
+ this.snapshot = sendCoalescer(previous, {type: 'run.settled'});
+ if (!shouldStartVoiceLocalAudioReconcileRun(previous, this.snapshot)) break;
+ }
+ this.runningDepth -= 1;
+ }
+}
+
+describe('voice local audio reconcile coalescer', () => {
+ it('starts a run when a request arrives while idle', () => {
+ const initial = createVoiceLocalAudioReconcileCoalescerSnapshot();
+ const next = sendCoalescer(initial, {type: 'run.requested', reason: 'voice room connected'});
+ expect(shouldStartVoiceLocalAudioReconcileRun(initial, next)).toBe(true);
+ expect(next.value).toBe('running');
+ expect(selectVoiceLocalAudioReconcileStartedRuns(next)).toBe(1);
+ expect(selectVoiceLocalAudioReconcileRunReasons(next)).toEqual(['voice room connected']);
+ });
+
+ it('absorbs a request that arrives while a run is in flight instead of starting a second run', () => {
+ const running = sendCoalescer(createVoiceLocalAudioReconcileCoalescerSnapshot(), {
+ type: 'run.requested',
+ reason: 'voice room connected',
+ });
+ const dirty = sendCoalescer(running, {type: 'run.requested', reason: 'local audio state change'});
+ expect(shouldStartVoiceLocalAudioReconcileRun(running, dirty)).toBe(false);
+ expect(dirty.value).toBe('runningDirty');
+ expect(selectVoiceLocalAudioReconcileStartedRuns(dirty)).toBe(1);
+ expect(selectVoiceLocalAudioReconcileCoalescedCount(dirty)).toBe(1);
+ });
+
+ it('collapses K triggers arriving during one run into exactly one follow-up run', () => {
+ const triggerCount = 10;
+ let pendingTriggers = triggerCount;
+ const harness = new ReconcileHarness((self) => {
+ for (let index = 0; index < pendingTriggers; index++) {
+ self.request(`trigger ${index}`);
+ }
+ pendingTriggers = 0;
+ });
+
+ harness.request('voice room connected');
+
+ expect(harness.startedRuns).toHaveLength(2);
+ expect(selectVoiceLocalAudioReconcileStartedRuns(harness.snapshot)).toBe(2);
+ expect(harness.snapshot.value).toBe('idle');
+ });
+
+ it('records how many triggers were coalesced into the follow-up run', () => {
+ let snapshot = sendCoalescer(createVoiceLocalAudioReconcileCoalescerSnapshot(), {
+ type: 'run.requested',
+ reason: 'voice room connected',
+ });
+ for (let index = 0; index < 10; index++) {
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: `trigger ${index}`});
+ }
+ expect(selectVoiceLocalAudioReconcileCoalescedCount(snapshot)).toBe(10);
+ expect(selectVoiceLocalAudioReconcileStartedRuns(snapshot)).toBe(1);
+
+ const followUp = sendCoalescer(snapshot, {type: 'run.settled'});
+ expect(followUp.value).toBe('running');
+ expect(selectVoiceLocalAudioReconcileStartedRuns(followUp)).toBe(2);
+ expect(selectVoiceLocalAudioReconcileCoalescedCount(followUp)).toBe(0);
+ });
+
+ it('returns to idle when a clean run settles and does not start another run', () => {
+ const running = sendCoalescer(createVoiceLocalAudioReconcileCoalescerSnapshot(), {
+ type: 'run.requested',
+ reason: 'voice state update',
+ });
+ const settled = sendCoalescer(running, {type: 'run.settled'});
+ expect(settled.value).toBe('idle');
+ expect(shouldStartVoiceLocalAudioReconcileRun(running, settled)).toBe(false);
+ expect(selectVoiceLocalAudioReconcileRunReasons(settled)).toEqual([]);
+ });
+
+ it('promotes the pending reasons into the follow-up run exactly once', () => {
+ let snapshot = sendCoalescer(createVoiceLocalAudioReconcileCoalescerSnapshot(), {
+ type: 'run.requested',
+ reason: 'voice room connected',
+ });
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: 'local audio state change'});
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: 'voice state update'});
+ snapshot = sendCoalescer(snapshot, {type: 'run.settled'});
+ expect(selectVoiceLocalAudioReconcileRunReasons(snapshot)).toEqual([
+ 'local audio state change',
+ 'voice state update',
+ ]);
+ expect(selectVoiceLocalAudioReconcileStartedRuns(snapshot)).toBe(2);
+ const idle = sendCoalescer(snapshot, {type: 'run.settled'});
+ expect(idle.value).toBe('idle');
+ expect(selectVoiceLocalAudioReconcileStartedRuns(idle)).toBe(2);
+ });
+
+ it('ignores a settle that arrives while idle', () => {
+ const initial = createVoiceLocalAudioReconcileCoalescerSnapshot();
+ const settled = sendCoalescer(initial, {type: 'run.settled'});
+ expect(settled.value).toBe('idle');
+ expect(selectVoiceLocalAudioReconcileStartedRuns(settled)).toBe(0);
+ });
+
+ it('keeps the reconcile-storm ordering linear instead of doubling', () => {
+ const failingRuns = 8;
+ let remainingFailures = failingRuns;
+ const harness = new ReconcileHarness((self) => {
+ if (remainingFailures === 0) return;
+ remainingFailures -= 1;
+ self.request('local audio state change');
+ self.request('voice state update');
+ });
+
+ harness.request('voice room connected');
+
+ expect(harness.startedRuns).toHaveLength(failingRuns + 1);
+ expect(harness.startedRuns.length).toBeLessThan(2 ** failingRuns);
+ expect(harness.snapshot.value).toBe('idle');
+ });
+
+ it('settles after two runs once the failure latches self-mute and stops emitting triggers', () => {
+ let latched = false;
+ const harness = new ReconcileHarness((self) => {
+ if (latched) return;
+ latched = true;
+ self.request('local audio state change');
+ self.request('voice state update');
+ });
+
+ harness.request('voice room connected');
+
+ expect(harness.startedRuns).toEqual([['voice room connected'], ['local audio state change', 'voice state update']]);
+ expect(harness.snapshot.value).toBe('idle');
+ });
+
+ it('never drops a trigger: a run always starts after the last request in a burst', () => {
+ const observedAtRunStart: Array<number> = [];
+ let requestCounter = 0;
+ let burst = 3;
+ const harness = new ReconcileHarness((self) => {
+ observedAtRunStart.push(requestCounter);
+ for (let index = 0; index < burst; index++) {
+ requestCounter += 1;
+ self.request(`burst ${requestCounter}`);
+ }
+ burst = 0;
+ });
+
+ requestCounter += 1;
+ harness.request(`burst ${requestCounter}`);
+
+ expect(observedAtRunStart).toEqual([1, 4]);
+ expect(observedAtRunStart[observedAtRunStart.length - 1]).toBe(requestCounter);
+ });
+
+ it('caps the reason list while still counting every coalesced trigger', () => {
+ let snapshot = sendCoalescer(createVoiceLocalAudioReconcileCoalescerSnapshot(), {
+ type: 'run.requested',
+ reason: 'voice room connected',
+ });
+ const triggerCount = VOICE_LOCAL_AUDIO_RECONCILE_REASON_LIMIT + 5;
+ for (let index = 0; index < triggerCount; index++) {
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: `trigger ${index}`});
+ }
+ expect(selectVoiceLocalAudioReconcileCoalescedCount(snapshot)).toBe(triggerCount);
+ const followUp = sendCoalescer(snapshot, {type: 'run.settled'});
+ expect(selectVoiceLocalAudioReconcileRunReasons(followUp)).toHaveLength(VOICE_LOCAL_AUDIO_RECONCILE_REASON_LIMIT);
+ });
+
+ it('deduplicates repeated reasons so a storm stays readable', () => {
+ let snapshot = sendCoalescer(createVoiceLocalAudioReconcileCoalescerSnapshot(), {
+ type: 'run.requested',
+ reason: 'voice room connected',
+ });
+ for (let index = 0; index < 50; index++) {
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: 'local audio state change'});
+ }
+ const followUp = sendCoalescer(snapshot, {type: 'run.settled'});
+ expect(selectVoiceLocalAudioReconcileRunReasons(followUp)).toEqual(['local audio state change']);
+ expect(selectVoiceLocalAudioReconcileCoalescedCount(snapshot)).toBe(50);
+ });
+
+ it('flags a livelock once the follow-up runs never stop chaining', () => {
+ let snapshot = createVoiceLocalAudioReconcileCoalescerSnapshot();
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: 'voice room connected'});
+ for (let index = 0; index < VOICE_LOCAL_AUDIO_RECONCILE_FOLLOW_UP_WARN_LIMIT; index++) {
+ expect(shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(snapshot)).toBe(false);
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: 'local audio state change'});
+ snapshot = sendCoalescer(snapshot, {type: 'run.settled'});
+ }
+ expect(shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(snapshot)).toBe(true);
+
+ const drained = sendCoalescer(snapshot, {type: 'run.settled'});
+ expect(drained.value).toBe('idle');
+ expect(shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(drained)).toBe(false);
+ });
+
+ it('crosses the livelock threshold once so the drain can warn a single time', () => {
+ let snapshot = createVoiceLocalAudioReconcileCoalescerSnapshot();
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: 'voice room connected'});
+ let crossings = 0;
+ for (let index = 0; index < VOICE_LOCAL_AUDIO_RECONCILE_FOLLOW_UP_WARN_LIMIT * 4; index++) {
+ const before = snapshot;
+ snapshot = sendCoalescer(snapshot, {type: 'run.requested', reason: 'local audio state change'});
+ snapshot = sendCoalescer(snapshot, {type: 'run.settled'});
+ if (
+ shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(snapshot) &&
+ !shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(before)
+ ) {
+ crossings += 1;
+ }
+ }
+ expect(crossings).toBe(1);
+ });
+
+ it('is level-triggered: the reason never selects control flow', () => {
+ const script: Array<VoiceLocalAudioReconcileCoalescerEvent> = [
+ {type: 'run.requested', reason: 'first'},
+ {type: 'run.requested', reason: 'second'},
+ {type: 'run.settled'},
+ {type: 'run.settled'},
+ ];
+ const replay = (reasonFor: (index: number) => string) => {
+ let snapshot = createVoiceLocalAudioReconcileCoalescerSnapshot();
+ const trace: Array<string> = [];
+ script.forEach((event, index) => {
+ snapshot = sendCoalescer(
+ snapshot,
+ event.type === 'run.requested' ? {type: 'run.requested', reason: reasonFor(index)} : event,
+ );
+ trace.push(`${String(snapshot.value)}:${selectVoiceLocalAudioReconcileStartedRuns(snapshot)}`);
+ });
+ return trace;
+ };
+ expect(replay((index) => `reason-${index}`)).toEqual(replay(() => 'voice state update'));
+ });
+});
+
+describe('voice microphone failure latch', () => {
+ const scope = {channelId: 'channel-1', inputDeviceId: 'mic-1'};
+
+ function latchedSnapshot(): VoiceMicrophoneFailureLatchSnapshot {
+ return sendLatch(createVoiceMicrophoneFailureLatchSnapshot(), {type: 'microphone.enableFailed', ...scope});
+ }
+
+ it('starts clear', () => {
+ const initial = createVoiceMicrophoneFailureLatchSnapshot();
+ expect(isVoiceMicrophoneFailureLatchActive(initial)).toBe(false);
+ expect(selectVoiceMicrophoneFailureCount(initial)).toBe(0);
+ });
+
+ it('latches when enabling the microphone fails', () => {
+ const latched = latchedSnapshot();
+ expect(isVoiceMicrophoneFailureLatchActive(latched)).toBe(true);
+ expect(selectVoiceMicrophoneFailureCount(latched)).toBe(1);
+ });
+
+ it('keeps forcing self-mute when the server echoes a stale self_mute:false', () => {
+ const latched = latchedSnapshot();
+ expect(selectVoiceLocalAudioEffectiveSelfMute({localSelfMute: true, microphoneFailureLatched: true})).toBe(true);
+ const afterStaleEcho = sendLatch(latched, {type: 'scope.observed', ...scope});
+ expect(isVoiceMicrophoneFailureLatchActive(afterStaleEcho)).toBe(true);
+ expect(
+ selectVoiceLocalAudioEffectiveSelfMute({
+ localSelfMute: false,
+ microphoneFailureLatched: isVoiceMicrophoneFailureLatchActive(afterStaleEcho),
+ }),
+ ).toBe(true);
+ });
+
+ it('survives repeated recomputes of the audio-controls snapshot', () => {
+ let snapshot = latchedSnapshot();
+ for (let index = 0; index < 20; index++) {
+ snapshot = sendLatch(snapshot, {type: 'scope.observed', ...scope});
+ expect(
+ selectVoiceLocalAudioEffectiveSelfMute({
+ localSelfMute: false,
+ microphoneFailureLatched: isVoiceMicrophoneFailureLatchActive(snapshot),
+ }),
+ ).toBe(true);
+ }
+ expect(selectVoiceMicrophoneFailureCount(snapshot)).toBe(1);
+ });
+
+ it('clears when the microphone is enabled successfully', () => {
+ const cleared = sendLatch(latchedSnapshot(), {type: 'microphone.enableSucceeded'});
+ expect(isVoiceMicrophoneFailureLatchActive(cleared)).toBe(false);
+ expect(selectVoiceMicrophoneFailureCount(cleared)).toBe(0);
+ });
+
+ it('clears when the user changes their own mute intent', () => {
+ const cleared = sendLatch(latchedSnapshot(), {type: 'mute.userIntentChanged'});
+ expect(isVoiceMicrophoneFailureLatchActive(cleared)).toBe(false);
+ });
+
+ it('clears when the voice connection is torn down or re-established', () => {
+ const cleared = sendLatch(latchedSnapshot(), {type: 'latch.reset'});
+ expect(isVoiceMicrophoneFailureLatchActive(cleared)).toBe(false);
+ });
+
+ it('clears when the active channel changes', () => {
+ const cleared = sendLatch(latchedSnapshot(), {
+ type: 'scope.observed',
+ channelId: 'channel-2',
+ inputDeviceId: scope.inputDeviceId,
+ });
+ expect(isVoiceMicrophoneFailureLatchActive(cleared)).toBe(false);
+ });
+
+ it('clears when the input device changes', () => {
+ const cleared = sendLatch(latchedSnapshot(), {
+ type: 'scope.observed',
+ channelId: scope.channelId,
+ inputDeviceId: 'mic-2',
+ });
+ expect(isVoiceMicrophoneFailureLatchActive(cleared)).toBe(false);
+ });
+
+ it('re-latches after a cleared latch fails again and counts the failures', () => {
+ let snapshot = latchedSnapshot();
+ snapshot = sendLatch(snapshot, {type: 'mute.userIntentChanged'});
+ snapshot = sendLatch(snapshot, {type: 'microphone.enableFailed', ...scope});
+ expect(isVoiceMicrophoneFailureLatchActive(snapshot)).toBe(true);
+ expect(selectVoiceMicrophoneFailureCount(snapshot)).toBe(1);
+ snapshot = sendLatch(snapshot, {type: 'microphone.enableFailed', ...scope});
+ expect(selectVoiceMicrophoneFailureCount(snapshot)).toBe(2);
+ });
+
+ it('does not latch on recovery events observed while already clear', () => {
+ let snapshot = createVoiceMicrophoneFailureLatchSnapshot();
+ for (const event of [
+ {type: 'microphone.enableSucceeded'},
+ {type: 'mute.userIntentChanged'},
+ {type: 'latch.reset'},
+ {type: 'scope.observed', ...scope},
+ ] as Array<VoiceMicrophoneFailureLatchEvent>) {
+ snapshot = sendLatch(snapshot, event);
+ expect(isVoiceMicrophoneFailureLatchActive(snapshot)).toBe(false);
+ }
+ });
+
+ it('leaves self-mute untouched when nothing has failed', () => {
+ expect(selectVoiceLocalAudioEffectiveSelfMute({localSelfMute: false, microphoneFailureLatched: false})).toBe(false);
+ expect(selectVoiceLocalAudioEffectiveSelfMute({localSelfMute: true, microphoneFailureLatched: false})).toBe(true);
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/VoiceLocalAudioReconcilePolicy.ts b/fluxer_app/src/features/voice/engine/VoiceLocalAudioReconcilePolicy.ts
new file mode 100644
index 000000000..5f9a4703b
--- /dev/null
+++ b/fluxer_app/src/features/voice/engine/VoiceLocalAudioReconcilePolicy.ts
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {assign, getInitialSnapshot, type SnapshotFrom, setup, transition} from 'xstate';
+
+export const VOICE_LOCAL_AUDIO_RECONCILE_REASON_LIMIT = 8;
+export const VOICE_LOCAL_AUDIO_RECONCILE_FOLLOW_UP_WARN_LIMIT = 16;
+
+export type VoiceLocalAudioReconcileCoalescerEvent = {type: 'run.requested'; reason: string} | {type: 'run.settled'};
+
+interface VoiceLocalAudioReconcileCoalescerContext {
+ generation: number;
+ runningReasons: Array<string>;
+ pendingReasons: Array<string>;
+ coalescedCount: number;
+ consecutiveFollowUpRuns: number;
+}
+
+function createCoalescerContext(): VoiceLocalAudioReconcileCoalescerContext {
+ return {
+ generation: 0,
+ runningReasons: [],
+ pendingReasons: [],
+ coalescedCount: 0,
+ consecutiveFollowUpRuns: 0,
+ };
+}
+
+function appendReason(reasons: Array<string>, reason: string): Array<string> {
+ if (reasons.includes(reason)) return reasons;
+ if (reasons.length >= VOICE_LOCAL_AUDIO_RECONCILE_REASON_LIMIT) return reasons;
+ return [...reasons, reason];
+}
+
+const voiceLocalAudioReconcileCoalescerMachine = setup({
+ types: {} as {
+ context: VoiceLocalAudioReconcileCoalescerContext;
+ events: VoiceLocalAudioReconcileCoalescerEvent;
+ },
+ actions: {
+ beginRun: assign(({context, event}) =>
+ event.type === 'run.requested'
+ ? {
+ generation: context.generation + 1,
+ runningReasons: appendReason([], event.reason),
+ pendingReasons: [],
+ coalescedCount: 0,
+ consecutiveFollowUpRuns: 0,
+ }
+ : context,
+ ),
+ absorbRequest: assign(({context, event}) =>
+ event.type === 'run.requested'
+ ? {
+ pendingReasons: appendReason(context.pendingReasons, event.reason),
+ coalescedCount: context.coalescedCount + 1,
+ }
+ : context,
+ ),
+ finishRun: assign(() => ({
+ runningReasons: [],
+ pendingReasons: [],
+ coalescedCount: 0,
+ consecutiveFollowUpRuns: 0,
+ })),
+ beginFollowUpRun: assign(({context}) => ({
+ generation: context.generation + 1,
+ runningReasons: context.pendingReasons,
+ pendingReasons: [],
+ coalescedCount: 0,
+ consecutiveFollowUpRuns: context.consecutiveFollowUpRuns + 1,
+ })),
+ },
+}).createMachine({
+ id: 'voiceLocalAudioReconcileCoalescer',
+ context: () => createCoalescerContext(),
+ initial: 'idle',
+ states: {
+ idle: {
+ on: {
+ 'run.requested': {target: 'running', actions: 'beginRun'},
+ },
+ },
+ running: {
+ on: {
+ 'run.requested': {target: 'runningDirty', actions: 'absorbRequest'},
+ 'run.settled': {target: 'idle', actions: 'finishRun'},
+ },
+ },
+ runningDirty: {
+ on: {
+ 'run.requested': {actions: 'absorbRequest'},
+ 'run.settled': {target: 'running', actions: 'beginFollowUpRun'},
+ },
+ },
+ },
+});
+
+export type VoiceLocalAudioReconcileCoalescerSnapshot = SnapshotFrom<typeof voiceLocalAudioReconcileCoalescerMachine>;
+
+export function createVoiceLocalAudioReconcileCoalescerSnapshot(): VoiceLocalAudioReconcileCoalescerSnapshot {
+ return getInitialSnapshot(voiceLocalAudioReconcileCoalescerMachine);
+}
+
+export function transitionVoiceLocalAudioReconcileCoalescerSnapshot(
+ snapshot: VoiceLocalAudioReconcileCoalescerSnapshot,
+ event: VoiceLocalAudioReconcileCoalescerEvent,
+): VoiceLocalAudioReconcileCoalescerSnapshot {
+ return transition(
+ voiceLocalAudioReconcileCoalescerMachine,
+ snapshot,
+ event,
+ )[0] as VoiceLocalAudioReconcileCoalescerSnapshot;
+}
+
+export function shouldStartVoiceLocalAudioReconcileRun(
+ previous: VoiceLocalAudioReconcileCoalescerSnapshot,
+ next: VoiceLocalAudioReconcileCoalescerSnapshot,
+): boolean {
+ return next.context.generation !== previous.context.generation;
+}
+
+export function selectVoiceLocalAudioReconcileRunReasons(
+ snapshot: VoiceLocalAudioReconcileCoalescerSnapshot,
+): ReadonlyArray<string> {
+ return snapshot.context.runningReasons;
+}
+
+export function selectVoiceLocalAudioReconcileCoalescedCount(
+ snapshot: VoiceLocalAudioReconcileCoalescerSnapshot,
+): number {
+ return snapshot.context.coalescedCount;
+}
+
+export function selectVoiceLocalAudioReconcileStartedRuns(snapshot: VoiceLocalAudioReconcileCoalescerSnapshot): number {
+ return snapshot.context.generation;
+}
+
+export function shouldWarnAboutVoiceLocalAudioReconcileFollowUpRuns(
+ snapshot: VoiceLocalAudioReconcileCoalescerSnapshot,
+): boolean {
+ return snapshot.context.consecutiveFollowUpRuns >= VOICE_LOCAL_AUDIO_RECONCILE_FOLLOW_UP_WARN_LIMIT;
+}
+
+export type VoiceMicrophoneFailureLatchEvent =
+ | {type: 'microphone.enableFailed'; channelId: string | null; inputDeviceId: string | null}
+ | {type: 'microphone.enableSucceeded'}
+ | {type: 'mute.userIntentChanged'}
+ | {type: 'scope.observed'; channelId: string | null; inputDeviceId: string | null}
+ | {type: 'latch.reset'};
+
+interface VoiceMicrophoneFailureLatchContext {
+ channelId: string | null;
+ inputDeviceId: string | null;
+ failureCount: number;
+}
+
+function createLatchContext(): VoiceMicrophoneFailureLatchContext {
+ return {
+ channelId: null,
+ inputDeviceId: null,
+ failureCount: 0,
+ };
+}
+
+const voiceMicrophoneFailureLatchMachine = setup({
+ types: {} as {
+ context: VoiceMicrophoneFailureLatchContext;
+ events: VoiceMicrophoneFailureLatchEvent;
+ },
+ actions: {
+ recordFailure: assign(({context, event}) =>
+ event.type === 'microphone.enableFailed'
+ ? {
+ channelId: event.channelId,
+ inputDeviceId: event.inputDeviceId,
+ failureCount: context.failureCount + 1,
+ }
+ : context,
+ ),
+ clearLatch: assign(() => createLatchContext()),
+ },
+ guards: {
+ isOutOfScope: ({context, event}) =>
+ event.type === 'scope.observed' &&
+ (event.channelId !== context.channelId || event.inputDeviceId !== context.inputDeviceId),
+ },
+}).createMachine({
+ id: 'voiceMicrophoneFailureLatch',
+ context: () => createLatchContext(),
+ initial: 'clear',
+ states: {
+ clear: {
+ on: {
+ 'microphone.enableFailed': {target: 'latched', actions: 'recordFailure'},
+ },
+ },
+ latched: {
+ on: {
+ 'microphone.enableFailed': {actions: 'recordFailure'},
+ 'microphone.enableSucceeded': {target: 'clear', actions: 'clearLatch'},
+ 'mute.userIntentChanged': {target: 'clear', actions: 'clearLatch'},
+ 'latch.reset': {target: 'clear', actions: 'clearLatch'},
+ 'scope.observed': {guard: 'isOutOfScope', target: 'clear', actions: 'clearLatch'},
+ },
+ },
+ },
+});
+
+export type VoiceMicrophoneFailureLatchSnapshot = SnapshotFrom<typeof voiceMicrophoneFailureLatchMachine>;
+
+export function createVoiceMicrophoneFailureLatchSnapshot(): VoiceMicrophoneFailureLatchSnapshot {
+ return getInitialSnapshot(voiceMicrophoneFailureLatchMachine);
+}
+
+export function transitionVoiceMicrophoneFailureLatchSnapshot(
+ snapshot: VoiceMicrophoneFailureLatchSnapshot,
+ event: VoiceMicrophoneFailureLatchEvent,
+): VoiceMicrophoneFailureLatchSnapshot {
+ return transition(voiceMicrophoneFailureLatchMachine, snapshot, event)[0] as VoiceMicrophoneFailureLatchSnapshot;
+}
+
+export function isVoiceMicrophoneFailureLatchActive(snapshot: VoiceMicrophoneFailureLatchSnapshot): boolean {
+ return snapshot.value === 'latched';
+}
+
+export function selectVoiceMicrophoneFailureCount(snapshot: VoiceMicrophoneFailureLatchSnapshot): number {
+ return snapshot.context.failureCount;
+}
+
+export function selectVoiceLocalAudioEffectiveSelfMute(input: {
+ localSelfMute: boolean;
+ microphoneFailureLatched: boolean;
+}): boolean {
+ return input.localSelfMute || input.microphoneFailureLatched;
+}
diff --git a/fluxer_app/src/features/voice/engine/VoiceOutputDeviceSync.test.ts b/fluxer_app/src/features/voice/engine/VoiceOutputDeviceSync.test.ts
new file mode 100644
index 000000000..ac91899c5
--- /dev/null
+++ b/fluxer_app/src/features/voice/engine/VoiceOutputDeviceSync.test.ts
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {Room} from 'livekit-client';
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+const updateSettings = vi.fn();
+let outputDevices: Array<{deviceId: string}> = [];
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({
+ default: {
+ getOutputDeviceId: () => 'default',
+ updateSettings: (update: unknown) => updateSettings(update),
+ subscribe: () => () => undefined,
+ },
+}));
+
+vi.mock('@app/features/voice/utils/VoiceDeviceManager', () => ({
+ voiceDeviceManager: {
+ ensureDevices: async () => ({
+ inputDevices: [],
+ outputDevices,
+ videoDevices: [],
+ permissionStatus: 'granted',
+ }),
+ },
+}));
+
+const {applyOutputDeviceToRoom} = await import('./VoiceOutputDeviceSync');
+
+class FakeMediaElement {
+ setSinkId = vi.fn(async () => undefined);
+}
+
+Object.defineProperty(FakeMediaElement.prototype, 'setSinkId', {
+ configurable: true,
+ writable: true,
+ value: async (): Promise<void> => undefined,
+});
+
+vi.stubGlobal('HTMLMediaElement', FakeMediaElement);
+
+interface FakeRoomOptions {
+ webAudioMix?: boolean;
+ audioContextSupportsSetSinkId?: boolean;
+ switchActiveDevice?: () => Promise<boolean>;
+}
+
+function createRoom(options: FakeRoomOptions = {}) {
+ const {webAudioMix = true, audioContextSupportsSetSinkId = true} = options;
+ const switchActiveDevice = vi.fn(options.switchActiveDevice ?? (async () => true));
+ const mixerSetSinkId = vi.fn(async () => undefined);
+ const trackSetSinkId = vi.fn(async () => undefined);
+ const element = new FakeMediaElement();
+ const audioContext = audioContextSupportsSetSinkId ? {setSinkId: mixerSetSinkId} : {};
+ const room = {
+ options: {webAudioMix},
+ audioContext,
+ switchActiveDevice,
+ remoteParticipants: new Map([
+ [
+ 'participant-with-track-sink',
+ {
+ audioTrackPublications: new Map([
+ ['pub-track-sink', {kind: 'audio', track: {kind: 'audio', setSinkId: trackSetSinkId}}],
+ ]),
+ },
+ ],
+ [
+ 'participant-with-elements',
+ {
+ audioTrackPublications: new Map([
+ ['pub-elements', {kind: 'audio', track: {kind: 'audio', attachedElements: [element]}}],
+ ]),
+ },
+ ],
+ ]),
+ };
+ return {room: room as unknown as Room, switchActiveDevice, mixerSetSinkId, trackSetSinkId, element};
+}
+
+describe('applyOutputDeviceToRoom', () => {
+ beforeEach(() => {
+ updateSettings.mockClear();
+ outputDevices = [{deviceId: 'speaker-1'}];
+ });
+
+ it('hands the Web Audio mixer room the literal default id so LiveKit resolves it to a real device', async () => {
+ const {room, switchActiveDevice} = createRoom();
+
+ await applyOutputDeviceToRoom(room, 'default');
+
+ expect(switchActiveDevice).toHaveBeenCalledWith('audiooutput', 'default');
+ expect(switchActiveDevice).not.toHaveBeenCalledWith('audiooutput', '');
+ });
+
+ it('passes a concrete device id through verbatim', async () => {
+ const {room, switchActiveDevice} = createRoom();
+
+ await applyOutputDeviceToRoom(room, 'speaker-1');
+
+ expect(switchActiveDevice).toHaveBeenCalledWith('audiooutput', 'speaker-1');
+ });
+
+ it('applies the normalized sink id to the Web Audio mixer and the attached elements', async () => {
+ const {room, mixerSetSinkId, trackSetSinkId, element} = createRoom();
+
+ await applyOutputDeviceToRoom(room, 'default');
+
+ expect(mixerSetSinkId).toHaveBeenCalledWith('');
+ expect(trackSetSinkId).toHaveBeenCalledWith('');
+ expect(element.setSinkId).toHaveBeenCalledWith('');
+ });
+
+ it('applies a concrete sink id to the Web Audio mixer and the attached elements', async () => {
+ const {room, mixerSetSinkId, trackSetSinkId, element} = createRoom();
+
+ await applyOutputDeviceToRoom(room, 'speaker-1');
+
+ expect(mixerSetSinkId).toHaveBeenCalledWith('speaker-1');
+ expect(trackSetSinkId).toHaveBeenCalledWith('speaker-1');
+ expect(element.setSinkId).toHaveBeenCalledWith('speaker-1');
+ });
+
+ it('still runs the mixer and element paths when LiveKit rejects with NotFoundError', async () => {
+ const notFound = new Error('Requested device not found');
+ notFound.name = 'NotFoundError';
+ const {room, switchActiveDevice, mixerSetSinkId, trackSetSinkId, element} = createRoom({
+ switchActiveDevice: async () => {
+ throw notFound;
+ },
+ });
+
+ await expect(applyOutputDeviceToRoom(room, 'default')).resolves.toBeUndefined();
+
+ expect(switchActiveDevice).toHaveBeenCalledWith('audiooutput', 'default');
+ expect(mixerSetSinkId).toHaveBeenCalledWith('');
+ expect(trackSetSinkId).toHaveBeenCalledWith('');
+ expect(element.setSinkId).toHaveBeenCalledWith('');
+ });
+
+ it('skips the LiveKit switch when the Web Audio mixer context cannot switch sinks', async () => {
+ const {room, switchActiveDevice, trackSetSinkId, element} = createRoom({
+ audioContextSupportsSetSinkId: false,
+ });
+
+ await applyOutputDeviceToRoom(room, 'default');
+
+ expect(switchActiveDevice).not.toHaveBeenCalled();
+ expect(trackSetSinkId).toHaveBeenCalledWith('');
+ expect(element.setSinkId).toHaveBeenCalledWith('');
+ });
+
+ it('normalizes the sink id only when LiveKit forwards it straight to the browser', async () => {
+ const {room, switchActiveDevice} = createRoom({
+ webAudioMix: false,
+ audioContextSupportsSetSinkId: false,
+ });
+
+ await applyOutputDeviceToRoom(room, 'default');
+
+ expect(switchActiveDevice).toHaveBeenCalledWith('audiooutput', '');
+ });
+
+ it('falls back to the default device when the stored output device disappeared', async () => {
+ outputDevices = [{deviceId: 'speaker-2'}];
+ const {room, switchActiveDevice} = createRoom();
+
+ await applyOutputDeviceToRoom(room, 'speaker-1');
+
+ expect(updateSettings).toHaveBeenCalledWith({outputDeviceId: 'default'});
+ expect(switchActiveDevice).toHaveBeenCalledWith('audiooutput', 'default');
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/VoiceOutputDeviceSync.ts b/fluxer_app/src/features/voice/engine/VoiceOutputDeviceSync.ts
index a8fcca52e..1e485fe76 100644
--- a/fluxer_app/src/features/voice/engine/VoiceOutputDeviceSync.ts
+++ b/fluxer_app/src/features/voice/engine/VoiceOutputDeviceSync.ts
@@ -43,6 +43,29 @@ function getRoomAudioContext(room: Room): SinkableAudioContext | null {
return isSinkableAudioContext(audioContext) ? audioContext : null;
}
+function isRoomWebAudioMixEnabled(room: Room): boolean {
+ const options: unknown = Reflect.get(room, 'options');
+ if (typeof options !== 'object' || options === null) return false;
+ return Boolean((options as {webAudioMix?: unknown}).webAudioMix);
+}
+
+function supportsMediaElementSetSinkId(): boolean {
+ if (typeof HTMLMediaElement === 'undefined') return false;
+ return 'setSinkId' in HTMLMediaElement.prototype;
+}
+
+function canRoomSwitchAudioOutput(room: Room): boolean {
+ if (!isRoomWebAudioMixEnabled(room)) return supportsMediaElementSetSinkId();
+ const audioContext = getRoomAudioContext(room);
+ if (!audioContext) return true;
+ return typeof audioContext.setSinkId === 'function';
+}
+
+function resolveLiveKitSwitchDeviceId(room: Room, deviceId: string): string {
+ if (isRoomWebAudioMixEnabled(room)) return deviceId;
+ return normalizeSinkIdForBrowserApi(deviceId);
+}
+
function isSinkableMediaElement(value: unknown): value is SinkableMediaElement {
if (typeof HTMLMediaElement === 'undefined') return false;
return value instanceof HTMLMediaElement;
@@ -125,12 +148,19 @@ async function applyOutputDeviceToAttachedElements(room: Room, deviceId: string)
}
}
-async function applyOutputDeviceToRoom(room: Room, deviceId: string): Promise<void> {
+export async function applyOutputDeviceToRoom(room: Room, deviceId: string): Promise<void> {
const resolvedDeviceId = await resolveAvailableOutputDeviceId(deviceId);
- try {
- await room.switchActiveDevice('audiooutput', resolvedDeviceId);
- } catch (error) {
- logger.warn('LiveKit failed to apply audio output device', {deviceId: resolvedDeviceId, error});
+ if (canRoomSwitchAudioOutput(room)) {
+ try {
+ await room.switchActiveDevice('audiooutput', resolveLiveKitSwitchDeviceId(room, resolvedDeviceId));
+ } catch (error) {
+ logger.warn('LiveKit failed to apply audio output device', {deviceId: resolvedDeviceId, error});
+ }
+ } else {
+ logger.warn('Audio output selection is unavailable: this browser cannot move the voice playback sink', {
+ deviceId: resolvedDeviceId,
+ webAudioMix: isRoomWebAudioMixEnabled(room),
+ });
}
try {
await applyOutputDeviceToWebAudioMixer(room, resolvedDeviceId);
diff --git a/fluxer_app/src/features/voice/engine/VoiceSharedAudioContext.test.ts b/fluxer_app/src/features/voice/engine/VoiceSharedAudioContext.test.ts
new file mode 100644
index 000000000..b2c92614a
--- /dev/null
+++ b/fluxer_app/src/features/voice/engine/VoiceSharedAudioContext.test.ts
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {afterEach, describe, expect, it, vi} from 'vitest';
+
+interface FakeAudioContext {
+ state: AudioContextState;
+ sampleRate: number;
+ resumeCount: number;
+}
+
+interface FakeWindowHarness {
+ requestedOptions: Array<AudioContextOptions | undefined>;
+ contexts: Array<FakeAudioContext>;
+ bodyListeners: Array<string>;
+}
+
+const HARDWARE_SAMPLE_RATE = 44100;
+
+function installFakeWindow(
+ config: {
+ throwOn?: (options: AudioContextOptions | undefined) => boolean;
+ initialState?: AudioContextState;
+ honourSampleRate?: boolean;
+ } = {},
+): FakeWindowHarness {
+ const harness: FakeWindowHarness = {requestedOptions: [], contexts: [], bodyListeners: []};
+ const throwOn = config.throwOn ?? ((): boolean => false);
+ const honourSampleRate = config.honourSampleRate ?? true;
+
+ class FakeAudioContextImpl {
+ state: AudioContextState = config.initialState ?? 'running';
+ sampleRate: number;
+ resumeCount = 0;
+
+ constructor(options?: AudioContextOptions) {
+ harness.requestedOptions.push(options);
+ if (throwOn(options)) {
+ throw new DOMException('sample rate not supported', 'NotSupportedError');
+ }
+ this.sampleRate = (honourSampleRate ? options?.sampleRate : undefined) ?? HARDWARE_SAMPLE_RATE;
+ harness.contexts.push(this as unknown as FakeAudioContext);
+ }
+
+ async resume(): Promise<void> {
+ this.resumeCount += 1;
+ this.state = 'running';
+ }
+
+ addEventListener(): void {}
+ removeEventListener(): void {}
+ }
+
+ vi.stubGlobal('window', {
+ AudioContext: FakeAudioContextImpl,
+ document: {
+ body: {
+ addEventListener: (type: string): void => {
+ harness.bodyListeners.push(type);
+ },
+ removeEventListener: (): void => {},
+ },
+ },
+ });
+ return harness;
+}
+
+async function loadModule() {
+ vi.resetModules();
+ return await import('@app/features/voice/engine/VoiceSharedAudioContext');
+}
+
+describe('VoiceSharedAudioContext', () => {
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it('leaves the shared context on the hardware rate so it shares a graph with capture and playback', async () => {
+ const harness = installFakeWindow();
+ const {getSharedVoiceAudioContext} = await loadModule();
+
+ const context = getSharedVoiceAudioContext();
+
+ expect(harness.requestedOptions).toEqual([{latencyHint: 'interactive'}]);
+ expect(context).toBe(harness.contexts[0] as unknown as AudioContext);
+ expect(context?.sampleRate).toBe(HARDWARE_SAMPLE_RATE);
+ });
+
+ it('returns null without throwing when the construction fails', async () => {
+ const harness = installFakeWindow({throwOn: () => true});
+ const {getSharedVoiceAudioContext} = await loadModule();
+
+ let context: AudioContext | null | undefined;
+ expect(() => {
+ context = getSharedVoiceAudioContext();
+ }).not.toThrow();
+
+ expect(context).toBeNull();
+ expect(harness.contexts).toHaveLength(0);
+ expect(harness.requestedOptions).toHaveLength(1);
+ });
+
+ it('caches the context across calls and rebuilds it once it has closed', async () => {
+ const harness = installFakeWindow();
+ const {getSharedVoiceAudioContext} = await loadModule();
+
+ const first = getSharedVoiceAudioContext();
+ expect(getSharedVoiceAudioContext()).toBe(first);
+ expect(harness.requestedOptions).toHaveLength(1);
+
+ harness.contexts[0].state = 'closed';
+ const second = getSharedVoiceAudioContext();
+
+ expect(second).not.toBe(first);
+ expect(harness.contexts).toHaveLength(2);
+ expect(harness.requestedOptions).toEqual([{latencyHint: 'interactive'}, {latencyHint: 'interactive'}]);
+ });
+
+ it('resumes a suspended context and arms the user-gesture resume listener', async () => {
+ const harness = installFakeWindow({initialState: 'suspended'});
+ const {getSharedVoiceAudioContext} = await loadModule();
+
+ const context = getSharedVoiceAudioContext();
+
+ expect(context).not.toBeNull();
+ expect(harness.contexts[0].resumeCount).toBe(1);
+ expect(harness.bodyListeners).toEqual(['click']);
+ });
+
+ it('createVoiceAudioContext reports the rate the browser actually resolved, not the requested one', async () => {
+ installFakeWindow({honourSampleRate: false});
+ const {createVoiceAudioContext} = await loadModule();
+
+ const context = createVoiceAudioContext({latencyHint: 'interactive', sampleRate: 48000});
+
+ expect(context?.sampleRate).toBe(HARDWARE_SAMPLE_RATE);
+ });
+
+ it('createVoiceAudioContext returns null instead of throwing when the rate is rejected', async () => {
+ installFakeWindow({throwOn: (options) => options?.sampleRate !== undefined});
+ const {createVoiceAudioContext} = await loadModule();
+
+ expect(createVoiceAudioContext({latencyHint: 'interactive', sampleRate: 48000})).toBeNull();
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/VoiceSharedAudioContext.ts b/fluxer_app/src/features/voice/engine/VoiceSharedAudioContext.ts
index 269df7a31..95c76eecf 100644
--- a/fluxer_app/src/features/voice/engine/VoiceSharedAudioContext.ts
+++ b/fluxer_app/src/features/voice/engine/VoiceSharedAudioContext.ts
@@ -17,6 +17,17 @@ function resolveAudioContextConstructor(): AudioContextConstructor | null {
return ctor ?? null;
}
+export function createVoiceAudioContext(options: AudioContextOptions): AudioContext | null {
+ const ctor = resolveAudioContextConstructor();
+ if (!ctor) return null;
+ try {
+ return new ctor(options);
+ } catch (error) {
+ logger.warn('Failed to construct voice AudioContext', {options, error});
+ return null;
+ }
+}
+
function installResumeOnUserGesture(context: AudioContext): void {
if (typeof window === 'undefined') return;
if (!window.document?.body) return;
@@ -41,15 +52,8 @@ export function getSharedVoiceAudioContext(): AudioContext | null {
if (sharedAudioContext && sharedAudioContext.state !== 'closed') {
return sharedAudioContext;
}
- const ctor = resolveAudioContextConstructor();
- if (!ctor) return null;
- try {
- sharedAudioContext = new ctor({latencyHint: 'interactive'});
- } catch (error) {
- logger.warn('Failed to construct shared voice AudioContext', {error});
- sharedAudioContext = null;
- return null;
- }
+ sharedAudioContext = createVoiceAudioContext({latencyHint: 'interactive'});
+ if (!sharedAudioContext) return null;
assert.notEqual(sharedAudioContext.state, 'closed', 'shared voice AudioContext must not start closed');
if (sharedAudioContext.state === 'suspended') {
void sharedAudioContext.resume().catch((error) => {
diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx
index 84c191f1c..afed61a3b 100644
--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx
+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx
@@ -18,6 +18,14 @@ import {Store} from '@app/features/voice/engine/Store';
import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState';
import type {EffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState';
import {getEffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState';
+import {
+ createVoiceMicrophoneFailureLatchSnapshot,
+ isVoiceMicrophoneFailureLatchActive,
+ selectVoiceLocalAudioEffectiveSelfMute,
+ selectVoiceMicrophoneFailureCount,
+ transitionVoiceMicrophoneFailureLatchSnapshot,
+ type VoiceMicrophoneFailureLatchEvent,
+} from '@app/features/voice/engine/VoiceLocalAudioReconcilePolicy';
import {resolveLocalSpeakingOverrideState} from '@app/features/voice/engine/VoiceLocalSpeakingGate';
import {
getRoomFromMediaEngine,
@@ -151,10 +159,36 @@ function getVoiceEngineV2AudioModeFromAppState(): VoiceEngineV2AudioMode {
return 'voiceActivity';
}
+let microphoneFailureLatchSnapshot = createVoiceMicrophoneFailureLatchSnapshot();
+
+function applyMicrophoneFailureLatchEvent(event: VoiceMicrophoneFailureLatchEvent): boolean {
+ const wasLatched = isVoiceMicrophoneFailureLatchActive(microphoneFailureLatchSnapshot);
+ microphoneFailureLatchSnapshot = transitionVoiceMicrophoneFailureLatchSnapshot(microphoneFailureLatchSnapshot, event);
+ const isLatched = isVoiceMicrophoneFailureLatchActive(microphoneFailureLatchSnapshot);
+ if (wasLatched === isLatched) return false;
+ logger.info('Microphone failure self-mute latch changed', {
+ event: event.type,
+ latched: isLatched,
+ failureCount: selectVoiceMicrophoneFailureCount(microphoneFailureLatchSnapshot),
+ });
+ return true;
+}
+
+export function isMicrophoneEnableFailureLatched(): boolean {
+ return isVoiceMicrophoneFailureLatchActive(microphoneFailureLatchSnapshot);
+}
+
+function getLatchedLocalSelfMute(): boolean {
+ return selectVoiceLocalAudioEffectiveSelfMute({
+ localSelfMute: LocalVoiceState.getSelfMute(),
+ microphoneFailureLatched: isMicrophoneEnableFailureLatched(),
+ });
+}
+
function getVoiceEngineV2AudioControlsFromAppState(): VoiceEngineV2AudioControls {
return {
mode: getVoiceEngineV2AudioModeFromAppState(),
- locallyMuted: LocalVoiceState.getSelfMute(),
+ locallyMuted: getLatchedLocalSelfMute(),
preferredLocallyMuted: LocalVoiceState.getSelfMute(),
locallyDeafened: LocalVoiceState.getSelfDeaf(),
mutedByPermission: LocalVoiceState.getMutedByPermission(),
@@ -250,6 +284,31 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
this.cameraLifecycleBinding = null;
}
+ isMicrophoneFailureLatched(): boolean {
+ return isMicrophoneEnableFailureLatched();
+ }
+
+ resetMicrophoneFailureLatch(): void {
+ this.transitionMicrophoneFailureLatch({type: 'latch.reset'});
+ }
+
+ noteUserMuteIntentChanged(): void {
+ this.transitionMicrophoneFailureLatch({type: 'mute.userIntentChanged'});
+ }
+
+ private transitionMicrophoneFailureLatch(event: VoiceMicrophoneFailureLatchEvent): void {
+ if (!applyMicrophoneFailureLatchEvent(event)) return;
+ this.emitChange();
+ }
+
+ private observeMicrophoneFailureLatchScope(channelId: string | null): void {
+ this.transitionMicrophoneFailureLatch({
+ type: 'scope.observed',
+ channelId,
+ inputDeviceId: VoiceSettings.getInputDeviceId(),
+ });
+ }
+
private isSpeakPermissionDenied(channelId: string | null): boolean {
const connection = getVoiceConnectionContextFromMediaEngine();
return isVoiceSpeakPermissionDenied(connection?.guildId ?? null, channelId);
@@ -356,6 +415,7 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
async ensureMicrophone(room: Room, channelId: string): Promise<void> {
assertObjectLike<Room>(room, 'ensureMicrophone.room');
assertNonEmptyString(channelId, 'ensureMicrophone.channelId');
+ this.observeMicrophoneFailureLatchScope(channelId);
if (this.isSpeakPermissionDenied(channelId)) {
logger.debug('Skipping microphone: speak permission denied');
await this.enforceSpeakPermissionMute(room);
@@ -425,10 +485,12 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
logger.debug('Skipping audio-state reconciliation: no local participant');
return;
}
+ this.observeMicrophoneFailureLatchScope(params.channelId);
const permissionMuted = this.isSpeakPermissionDenied(params.channelId);
const audioState = this.getEffectiveAudioState({
serverMute: params.serverMute || permissionMuted,
serverDeaf: params.serverDeaf,
+ selfMute: getLatchedLocalSelfMute(),
});
logger.info('Reconciling local media after voice state update', {
channelId: params.channelId,
@@ -577,6 +639,7 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
async refreshMicrophone(room: Room | null, options: RefreshMicrophoneOptions = {}): Promise<void> {
assertNullableObjectLike<Room>(room, 'refreshMicrophone.room');
assertObjectLike<RefreshMicrophoneOptions>(options, 'refreshMicrophone.options');
+ this.observeMicrophoneFailureLatchScope(this.getActiveChannelId());
const refresh = async () => this.refreshMicrophoneNow(room, options);
const pendingRefresh = this.microphoneRefreshQueue.then(refresh, refresh);
this.microphoneRefreshQueue = pendingRefresh.catch(() => {});
@@ -762,6 +825,7 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
if (this.hasMicrophonePublication(room)) {
if (this.hasLiveMicrophonePublication(room)) {
logger.debug('Microphone track already published, skipping duplicate publish');
+ this.transitionMicrophoneFailureLatch({type: 'microphone.enableSucceeded'});
return;
}
logger.warn('Existing microphone publication has an ended track; unpublishing before reacquire');
@@ -781,8 +845,14 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
this.attachLocalSpeakingDetectorForPublish(ctx, state);
MediaPermission.updateMicrophonePermissionGranted();
this.transitionMediaState({type: 'microphone.enable.success'});
+ this.transitionMicrophoneFailureLatch({type: 'microphone.enableSucceeded'});
logger.info('Successfully enabled microphone');
} catch (e: unknown) {
+ this.transitionMicrophoneFailureLatch({
+ type: 'microphone.enableFailed',
+ channelId: ctx.channelId ?? this.getActiveChannelId(),
+ inputDeviceId: VoiceSettings.getInputDeviceId(),
+ });
await this.rollbackMicrophoneEnable(ctx, state, e);
throw e;
}
@@ -1411,7 +1481,11 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
return (micPublication?.track as LocalAudioTrack) ?? null;
}
- private getEffectiveAudioState(override?: {serverMute?: boolean; serverDeaf?: boolean}): EffectiveAudioState {
+ private getEffectiveAudioState(override?: {
+ serverMute?: boolean;
+ serverDeaf?: boolean;
+ selfMute?: boolean;
+ }): EffectiveAudioState {
return getEffectiveAudioState(override);
}
diff --git a/fluxer_app/src/features/voice/utils/DeepFilterNoiseProcessor.test.ts b/fluxer_app/src/features/voice/utils/DeepFilterNoiseProcessor.test.ts
new file mode 100644
index 000000000..f6c3a35dc
--- /dev/null
+++ b/fluxer_app/src/features/voice/utils/DeepFilterNoiseProcessor.test.ts
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+
+vi.mock('@app/features/app/state/RuntimeConfig', () => ({
+ default: {staticCdnEndpoint: 'https://cdn.test'},
+}));
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({
+ default: {
+ getDeepFilterNoiseSuppression: () => true,
+ getDeepFilterNoiseSuppressionLevel: () => 80,
+ },
+}));
+
+const deepFilterHarness = vi.hoisted(() => ({
+ instances: [] as Array<{
+ audioContext: AudioContext | null;
+ processedTrack: MediaStreamTrack | null;
+ sampleRateAtInit: number | null;
+ }>,
+}));
+
+vi.mock('deepfilternet3-noise-filter', () => ({
+ DeepFilterNoiseFilterProcessor: class {
+ audioContext: AudioContext | null = null;
+ processedTrack: MediaStreamTrack | null = null;
+ sampleRateAtInit: number | null = null;
+
+ constructor() {
+ deepFilterHarness.instances.push(this);
+ }
+
+ async init(): Promise<void> {
+ this.sampleRateAtInit = this.audioContext?.sampleRate ?? null;
+ this.processedTrack = {} as MediaStreamTrack;
+ }
+
+ async destroy(): Promise<void> {}
+ },
+}));
+
+const MODEL_SAMPLE_RATE = 48000;
+
+interface ContextHarness {
+ requestedRates: Array<number | undefined>;
+ closedRates: Array<number>;
+}
+
+function installAudioGraph(config: {hardwareSampleRate: number; canBridgeSampleRates: boolean}): ContextHarness {
+ const harness: ContextHarness = {requestedRates: [], closedRates: []};
+
+ class FakeAudioContext {
+ state: AudioContextState = 'running';
+ sampleRate: number;
+
+ constructor(options?: AudioContextOptions) {
+ harness.requestedRates.push(options?.sampleRate);
+ this.sampleRate = options?.sampleRate ?? config.hardwareSampleRate;
+ }
+
+ createMediaStreamSource(): {disconnect: () => void} {
+ if (!config.canBridgeSampleRates && this.sampleRate !== config.hardwareSampleRate) {
+ throw new DOMException(
+ 'Connecting AudioNodes from AudioContexts with different sample-rate is currently not supported.',
+ 'NotSupportedError',
+ );
+ }
+ return {disconnect: (): void => undefined};
+ }
+
+ async close(): Promise<void> {
+ harness.closedRates.push(this.sampleRate);
+ this.state = 'closed';
+ }
+ }
+
+ vi.stubGlobal('window', {AudioContext: FakeAudioContext});
+ vi.stubGlobal('MediaStream', class {});
+ return harness;
+}
+
+async function loadModule() {
+ vi.resetModules();
+ return await import('@app/features/voice/utils/DeepFilterNoiseProcessor');
+}
+
+function createSourceContext(sampleRate: number): AudioContext {
+ return {sampleRate} as unknown as AudioContext;
+}
+
+const feedTrack = {} as MediaStreamTrack;
+
+describe('resolveDeepFilterAudioContext', () => {
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it('runs the worklet at the model rate when the capture graph already runs there', async () => {
+ const harness = installAudioGraph({hardwareSampleRate: MODEL_SAMPLE_RATE, canBridgeSampleRates: false});
+ const {resolveDeepFilterAudioContext} = await loadModule();
+
+ const context = resolveDeepFilterAudioContext(createSourceContext(MODEL_SAMPLE_RATE), feedTrack);
+
+ expect(context?.sampleRate).toBe(MODEL_SAMPLE_RATE);
+ expect(harness.requestedRates).toEqual([MODEL_SAMPLE_RATE]);
+ expect(harness.closedRates).toEqual([]);
+ });
+
+ it('keeps the model rate on browsers that resample across graphs', async () => {
+ const harness = installAudioGraph({hardwareSampleRate: 44100, canBridgeSampleRates: true});
+ const {resolveDeepFilterAudioContext} = await loadModule();
+
+ const context = resolveDeepFilterAudioContext(createSourceContext(44100), feedTrack);
+
+ expect(context?.sampleRate).toBe(MODEL_SAMPLE_RATE);
+ expect(harness.requestedRates).toEqual([MODEL_SAMPLE_RATE]);
+ expect(harness.closedRates).toEqual([]);
+ });
+
+ it('falls back to the capture rate when the browser refuses to bridge sample rates', async () => {
+ const harness = installAudioGraph({hardwareSampleRate: 44100, canBridgeSampleRates: false});
+ const {resolveDeepFilterAudioContext} = await loadModule();
+
+ const context = resolveDeepFilterAudioContext(createSourceContext(44100), feedTrack);
+
+ expect(context?.sampleRate).toBe(44100);
+ expect(harness.requestedRates).toEqual([MODEL_SAMPLE_RATE, 44100]);
+ expect(harness.closedRates).toEqual([MODEL_SAMPLE_RATE]);
+ });
+
+ it('falls back to the capture rate for a bluetooth headset running well below the model rate', async () => {
+ installAudioGraph({hardwareSampleRate: 16000, canBridgeSampleRates: false});
+ const {resolveDeepFilterAudioContext} = await loadModule();
+
+ const context = resolveDeepFilterAudioContext(createSourceContext(16000), feedTrack);
+
+ expect(context?.sampleRate).toBe(16000);
+ });
+});
+
+function createCaptureContext(sampleRate: number): AudioContext {
+ const track = {stop: (): void => undefined} as unknown as MediaStreamTrack;
+ const node = {
+ connect: (next: unknown): unknown => next,
+ disconnect: (): void => undefined,
+ stream: {getAudioTracks: (): Array<MediaStreamTrack> => [track]},
+ type: '',
+ frequency: {value: 0},
+ Q: {value: 0},
+ threshold: {value: 0},
+ knee: {value: 0},
+ ratio: {value: 0},
+ attack: {value: 0},
+ release: {value: 0},
+ };
+ return {
+ sampleRate,
+ createMediaStreamDestination: () => node,
+ createMediaStreamSource: () => node,
+ createBiquadFilter: () => node,
+ createDynamicsCompressor: () => node,
+ } as unknown as AudioContext;
+}
+
+describe('buildDeepFilterAudioChain', () => {
+ beforeEach(() => {
+ deepFilterHarness.instances.length = 0;
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it('gives the processor its audio context before the processor builds its own graph', async () => {
+ installAudioGraph({hardwareSampleRate: MODEL_SAMPLE_RATE, canBridgeSampleRates: false});
+ const {buildDeepFilterAudioChain} = await loadModule();
+
+ await buildDeepFilterAudioChain({audioContext: createCaptureContext(MODEL_SAMPLE_RATE)});
+
+ expect(deepFilterHarness.instances[0]?.sampleRateAtInit).toBe(MODEL_SAMPLE_RATE);
+ });
+
+ it('gives the processor a capture-rate context when the browser refuses to bridge sample rates', async () => {
+ installAudioGraph({hardwareSampleRate: 44100, canBridgeSampleRates: false});
+ const {buildDeepFilterAudioChain} = await loadModule();
+
+ await buildDeepFilterAudioChain({audioContext: createCaptureContext(44100)});
+
+ expect(deepFilterHarness.instances[0]?.sampleRateAtInit).toBe(44100);
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/DeepFilterNoiseProcessor.ts b/fluxer_app/src/features/voice/utils/DeepFilterNoiseProcessor.ts
index 1fd2fd92c..7c06657fa 100644
--- a/fluxer_app/src/features/voice/utils/DeepFilterNoiseProcessor.ts
+++ b/fluxer_app/src/features/voice/utils/DeepFilterNoiseProcessor.ts
@@ -2,11 +2,13 @@
import RuntimeConfig from '@app/features/app/state/RuntimeConfig';
import {Logger} from '@app/features/platform/utils/AppLogger';
+import {createVoiceAudioContext} from '@app/features/voice/engine/VoiceSharedAudioContext';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {DeepFilterNoiseFilterProcessor} from 'deepfilternet3-noise-filter';
import type {LocalAudioTrack} from 'livekit-client';
const logger = new Logger('DeepFilterNoiseProcessor');
+const DEEP_FILTER_MODEL_SAMPLE_RATE = 48000;
const DEFAULT_SUPPRESSION_LEVEL = 80;
const HIGH_PASS_FREQUENCY_HZ = 60;
const HIGH_PASS_Q = Math.SQRT1_2;
@@ -24,7 +26,7 @@ export function createDeepFilterProcessor(
): DeepFilterNoiseFilterProcessor {
const clampedNoiseReductionLevel = Math.max(0, Math.min(100, noiseReductionLevel));
return new DeepFilterNoiseFilterProcessor({
- sampleRate: 48000,
+ sampleRate: DEEP_FILTER_MODEL_SAMPLE_RATE,
noiseReductionLevel: clampedNoiseReductionLevel,
enabled: true,
assetConfig: {
@@ -33,6 +35,32 @@ export function createDeepFilterProcessor(
});
}
+export function resolveDeepFilterAudioContext(
+ sourceContext: AudioContext,
+ feedTrack: MediaStreamTrack,
+): AudioContext | null {
+ const modelRateContext = createVoiceAudioContext({
+ latencyHint: 'interactive',
+ sampleRate: DEEP_FILTER_MODEL_SAMPLE_RATE,
+ });
+ if (!modelRateContext) return null;
+ if (modelRateContext.sampleRate === sourceContext.sampleRate) return modelRateContext;
+ try {
+ modelRateContext.createMediaStreamSource(new MediaStream([feedTrack])).disconnect();
+ return modelRateContext;
+ } catch (error) {
+ logger.info('DeepFilter cannot read the capture graph at the model rate; running it at the capture rate', {
+ modelSampleRate: modelRateContext.sampleRate,
+ captureSampleRate: sourceContext.sampleRate,
+ error,
+ });
+ }
+ void modelRateContext.close().catch((error) => {
+ logger.debug('Failed to close the rejected DeepFilter AudioContext', error);
+ });
+ return createVoiceAudioContext({latencyHint: 'interactive', sampleRate: sourceContext.sampleRate});
+}
+
export interface DeepFilterAudioChain {
processedTrack: MediaStreamTrack;
inputDestination: MediaStreamAudioDestinationNode;
@@ -85,6 +113,7 @@ export async function buildDeepFilterAudioChain(opts: {
throw new Error('buildDeepFilterAudioChain: missing DeepFilter feed track');
}
const processor = createDeepFilterProcessor(noiseReductionLevel);
+ processor.audioContext = resolveDeepFilterAudioContext(audioContext, deepFilterFeedTrack);
const disposeDeepFilterInputGraph = () => {
safeDisconnect(inputDestination);
safeDisconnect(hpfSource);