A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(voice): one-way audio and stream loss in p2p calls (#3405)
Hampusbcb30eea8842
diff --git a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.tsindex 73783591e..9a8931712 100644--- a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts+++ b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts@@ -468,9 +468,11 @@ export function useVoiceCallTracksAndLayout({channel, expandedUserIds}: UseVoiceuseEffect(() => {if (!pinnedParticipantIdentity) return;if (cameraTracksAll.length === 0) return;- const stillExists = cameraTracksAll.some((tr) => tr.participant.identity === pinnedParticipantIdentity);- if (!stillExists) VoiceCallLayoutCommands.setPinnedParticipant(null);- }, [cameraTracksAll, pinnedParticipantIdentity]);+ if (cameraTracksAll.some((tr) => tr.participant.identity === pinnedParticipantIdentity)) return;+ const {connectionId} = parseVoiceParticipantIdentity(pinnedParticipantIdentity);+ if (connectionVoiceStates[connectionId]?.channel_id === channel.id) return;+ VoiceCallLayoutCommands.setPinnedParticipant(null);+ }, [cameraTracksAll, channel.id, connectionVoiceStates, pinnedParticipantIdentity]);const {mainTrack: focusMainTrack,carouselTracks,diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.tsindex 6b94b3678..ded3d9171 100644--- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts+++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts@@ -801,6 +801,7 @@ class MediaEngineFacade extends Store {restoreStream: boolean;screenShareSnapshot: ScreenShareReconnectSnapshot | null;toastWhenStreamNotRestored: boolean;+ sendUpdateWhenStreamNotRestored: boolean;}): Promise<void> {assert.equal(typeof args.restoreCamera, 'boolean', 'restoreLocalMedia requires a camera flag');assert.equal(typeof args.restoreStream, 'boolean', 'restoreLocalMedia requires a stream flag');@@ -828,7 +829,9 @@ class MediaEngineFacade extends Store {}if (restored) return;if (streamRestoreFailed || args.screenShareSnapshot !== null) {- voiceEngineV2AppMediaStateAdapter.applyScreenShareState(false, {sendUpdate: false});+ voiceEngineV2AppMediaStateAdapter.applyScreenShareState(false, {+ sendUpdate: args.sendUpdateWhenStreamNotRestored,+ });}if (args.toastWhenStreamNotRestored && this.i18n) {ToastCommands.createToast({@@ -1690,7 +1693,7 @@ class MediaEngineFacade extends Store {}voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();this.reconcileLocalAudioStateInBackground('region hot-swap complete');- if (isConversion) void this.reapplyVideoQualityLimits(newRoom);+ if (isConversion) void this.finishMeshConversion(currentRoom, newRoom);},(_guildId, _channelId, _connectionId, _attemptId, error) => {logger.warn('LiveKit connect failed, clearing gateway voice state', {error});@@ -2715,6 +2718,19 @@ class MediaEngineFacade extends Store {}}+ private async finishMeshConversion(meshRoom: Room, room: Room): Promise<void> {+ await this.restoreLocalMedia({+ reason: 'mesh conversion',+ room,+ restoreCamera: false,+ restoreStream: false,+ screenShareSnapshot: voiceEngineV2AppScreenShareExecutionAdapter.prepareScreenShareReconnect(meshRoom),+ toastWhenStreamNotRestored: true,+ sendUpdateWhenStreamNotRestored: true,+ });+ await this.reapplyVideoQualityLimits(room);+ }+private async restoreLocalMediaState(roomOverride?: Room | null): Promise<void> {const room = roomOverride ?? voiceEngineV2AppConnectionHostAdapter.room;const participant = room?.localParticipant ?? null;@@ -2728,6 +2744,7 @@ class MediaEngineFacade extends Store {restoreStream: LocalVoiceState.getSelfStream() && !participant.isScreenShareEnabled,screenShareSnapshot: pendingScreenShareReconnect,toastWhenStreamNotRestored: false,+ sendUpdateWhenStreamNotRestored: false,});}@@ -2744,6 +2761,7 @@ class MediaEngineFacade extends Store {restoreStream: pendingRestore.restoreStream,screenShareSnapshot: null,toastWhenStreamNotRestored: true,+ sendUpdateWhenStreamNotRestored: false,});}diff --git a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.tsindex cf38b49cf..719bd9d5a 100644--- a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts+++ b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts@@ -284,6 +284,20 @@ export function bindRoomEvents(participantSpeakingDisposers.get(identity)?.dispose();participantSpeakingDisposers.delete(identity);};+ const observeRemoteScreenSharePublication = (participant: Participant, pub: RemoteTrackPublication): void => {+ reportScreenSharePublicationObserved(participant.identity, pub.trackSid ?? null);+ const streamKey = getStreamKeyForParticipantIdentity(guildId, channelId, participant.identity);+ if (streamKey) {+ cancelDeferredStopWatchingStreamKey(streamKey);+ }+ };+ const observeExistingRemoteScreenSharePublications = (): void => {+ room.remoteParticipants.forEach((participant) => {+ for (const pub of participant.trackPublications.values()) {+ if (pub.source === Track.Source.ScreenShare) observeRemoteScreenSharePublication(participant, pub);+ }+ });+ };const endScreenShareIfPublicationDidNotReturn = (): void => {const publication = screenSharePublicationAwaitingReconnect;screenSharePublicationAwaitingReconnect = null;@@ -305,6 +319,7 @@ export function bindRoomEvents(bindParticipantSpeakingEvents(room.localParticipant);room.remoteParticipants.forEach((participant) => bindParticipantSpeakingEvents(participant));dependencies.remoteSpeaking.hydrateFromRoom(room);+ observeExistingRemoteScreenSharePublications();bindScreenShareNegotiation();dependencies.permissions.applyDeafen(room, getEffectiveAudioState().effectiveDeaf);dependencies.connection.markConnected();@@ -596,11 +611,7 @@ export function bindRoomEvents(pub.setSubscribed(true);}if (pub.source === Track.Source.ScreenShare && !participant.isLocal) {- reportScreenSharePublicationObserved(participant.identity, pub.trackSid ?? null);- const streamKey = getStreamKeyForParticipantIdentity(guildId, channelId, participant.identity);- if (streamKey) {- cancelDeferredStopWatchingStreamKey(streamKey);- }+ observeRemoteScreenSharePublication(participant, pub);dependencies.subscriptions.reattachScreenShareAfterPublish(participant.identity, pub);dependencies.subscriptions.reconcileSubscriptions();}@@ -615,6 +626,7 @@ export function bindRoomEvents(}),);if (room.state === ConnectionState.Connected) {+ guard(attemptId, observeExistingRemoteScreenSharePublications)();guard(attemptId, bindScreenShareNegotiation)();}}diff --git a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.tsindex 66f3054d5..eb59103ca 100644--- a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts+++ b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts@@ -320,6 +320,7 @@ export class VoiceMeshEngine extends RTCEngine {handlePeerStatus(peer: VoiceMeshPeer): void {if (this.peers.get(peer.connectionId) !== peer) return;const {status} = peer;+ if (status === 'failed' && this.ownP2p() === false) return;VoiceMeshPeers.setStatus(peer.connectionId, status);if (status === 'connecting') return;this.reportPeer(peer, status);@@ -395,28 +396,30 @@ export class VoiceMeshEngine extends RTCEngine {const serverMuted =src === 'microphone' &&voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(peer.connectionId)?.mute === true;- if (receiver && receiver.track.readyState === 'live' && publication.isDesired && !serverMuted) {- if (publication.track?.mediaStreamTrack !== receiver.track) {- participant.addSubscribedMediaTrack(receiver.track, info.sid, new MediaStream([receiver.track]), receiver);- }- return;+ const live = receiver !== null && receiver.track.readyState === 'live' && publication.isDesired && !serverMuted;+ if (live && publication.track?.mediaStreamTrack === receiver.track) return;+ if (publication.track) {+ publication.track.stop();+ publication.setTrack(undefined);+ }+ if (live) {+ participant.addSubscribedMediaTrack(receiver.track, info.sid, new MediaStream([receiver.track]), receiver);}- if (!publication.track) return;- publication.track.stop();- publication.setTrack(undefined);}private readonly syncRoster = (): void => {if (!this.room) return;const {connectionId, channelId, guildId} = this.target;- const own = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId);- const ownP2p = own?.channel_id === channelId ? own.p2p : undefined;+ const ownP2p = this.ownP2p();this.syncConversionWatchdog(ownP2p === false && !this.conversionHeld);const roster = new Map<string, NormalizedVoiceState>();- if (ownP2p === true) {+ if (ownP2p !== undefined) {const states = voiceEngineV2AppVoiceStateAdapter.getAllVoiceStatesInChannel(guildId ?? ME, channelId);for (const state of Object.values(states)) {- if (state.p2p === true && state.connection_id !== connectionId) roster.set(state.connection_id, state);+ if (state.connection_id === connectionId) continue;+ if (this.peers.has(state.connection_id) || (ownP2p && state.p2p === true)) {+ roster.set(state.connection_id, state);+ }}}const before = this.peers.size;@@ -431,6 +434,12 @@ export class VoiceMeshEngine extends RTCEngine {for (const peer of this.peers.values()) this.syncPeerReceiver(peer, 'microphone');};+ private ownP2p(): boolean | undefined {+ const {connectionId, channelId} = this.target;+ const own = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId);+ return own?.channel_id === channelId ? own.p2p : undefined;+ }+private addPeer(state: NormalizedVoiceState): void {const peer = new VoiceMeshPeer(this, state.connection_id, state.user_id, this.target.connectionId);this.peers.set(peer.connectionId, peer);diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.tsindex 7d86b80a2..fcefd44db 100644--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts@@ -3,7 +3,11 @@import assert from 'node:assert/strict';import Authentication from '@app/features/auth/state/Authentication';import {Logger} from '@app/features/platform/utils/AppLogger';-import {createVoiceMeshRoom, holdVoiceMeshConversion} from '@app/features/voice/engine/mesh/VoiceMeshRoom';+import {+ createVoiceMeshRoom,+ holdVoiceMeshConversion,+ isVoiceMeshRoom,+} from '@app/features/voice/engine/mesh/VoiceMeshRoom';import {Store} from '@app/features/voice/engine/Store';import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector';import {@@ -785,8 +789,13 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {logger.warn('Region hot-swap: missing participant for track republishing');return;}+ const isMeshConversion = isVoiceMeshRoom(oldRoom);const publications = selectLocalMediaPublicationsForConnectionRepublish(Array.from(oldParticipant.trackPublications.values()),+ ).filter(+ (publication) =>+ !isMeshConversion ||+ (publication.source !== Track.Source.ScreenShare && publication.source !== Track.Source.ScreenShareAudio),);const errors: Array<{source: string; error: unknown}> = [];let codecPolicyFailure: Error | null = null;