Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(voice): one-way audio and stream loss in p2p calls (#3405)

Hampusbcb30eea8842
diff --git a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts
index 73783591e..9a8931712 100644
--- a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts
+++ b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts
@@ -468,9 +468,11 @@ export function useVoiceCallTracksAndLayout({channel, expandedUserIds}: UseVoice
useEffect(() => {
if (!pinnedParticipantIdentity) return;
if (cameraTracksAll.length === 0) return;
- const stillExists = cameraTracksAll.some((tr) => tr.participant.identity === pinnedParticipantIdentity);
- if (!stillExists) VoiceCallLayoutCommands.setPinnedParticipant(null);
- }, [cameraTracksAll, pinnedParticipantIdentity]);
+ if (cameraTracksAll.some((tr) => tr.participant.identity === pinnedParticipantIdentity)) return;
+ const {connectionId} = parseVoiceParticipantIdentity(pinnedParticipantIdentity);
+ if (connectionVoiceStates[connectionId]?.channel_id === channel.id) return;
+ VoiceCallLayoutCommands.setPinnedParticipant(null);
+ }, [cameraTracksAll, channel.id, connectionVoiceStates, pinnedParticipantIdentity]);
const {
mainTrack: focusMainTrack,
carouselTracks,
diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
index 6b94b3678..ded3d9171 100644
--- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
+++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
@@ -801,6 +801,7 @@ class MediaEngineFacade extends Store {
restoreStream: boolean;
screenShareSnapshot: ScreenShareReconnectSnapshot | null;
toastWhenStreamNotRestored: boolean;
+ sendUpdateWhenStreamNotRestored: boolean;
}): Promise<void> {
assert.equal(typeof args.restoreCamera, 'boolean', 'restoreLocalMedia requires a camera flag');
assert.equal(typeof args.restoreStream, 'boolean', 'restoreLocalMedia requires a stream flag');
@@ -828,7 +829,9 @@ class MediaEngineFacade extends Store {
}
if (restored) return;
if (streamRestoreFailed || args.screenShareSnapshot !== null) {
- voiceEngineV2AppMediaStateAdapter.applyScreenShareState(false, {sendUpdate: false});
+ voiceEngineV2AppMediaStateAdapter.applyScreenShareState(false, {
+ sendUpdate: args.sendUpdateWhenStreamNotRestored,
+ });
}
if (args.toastWhenStreamNotRestored && this.i18n) {
ToastCommands.createToast({
@@ -1690,7 +1693,7 @@ class MediaEngineFacade extends Store {
}
voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch();
this.reconcileLocalAudioStateInBackground('region hot-swap complete');
- if (isConversion) void this.reapplyVideoQualityLimits(newRoom);
+ if (isConversion) void this.finishMeshConversion(currentRoom, newRoom);
},
(_guildId, _channelId, _connectionId, _attemptId, error) => {
logger.warn('LiveKit connect failed, clearing gateway voice state', {error});
@@ -2715,6 +2718,19 @@ class MediaEngineFacade extends Store {
}
}
+ private async finishMeshConversion(meshRoom: Room, room: Room): Promise<void> {
+ await this.restoreLocalMedia({
+ reason: 'mesh conversion',
+ room,
+ restoreCamera: false,
+ restoreStream: false,
+ screenShareSnapshot: voiceEngineV2AppScreenShareExecutionAdapter.prepareScreenShareReconnect(meshRoom),
+ toastWhenStreamNotRestored: true,
+ sendUpdateWhenStreamNotRestored: true,
+ });
+ await this.reapplyVideoQualityLimits(room);
+ }
+
private async restoreLocalMediaState(roomOverride?: Room | null): Promise<void> {
const room = roomOverride ?? voiceEngineV2AppConnectionHostAdapter.room;
const participant = room?.localParticipant ?? null;
@@ -2728,6 +2744,7 @@ class MediaEngineFacade extends Store {
restoreStream: LocalVoiceState.getSelfStream() && !participant.isScreenShareEnabled,
screenShareSnapshot: pendingScreenShareReconnect,
toastWhenStreamNotRestored: false,
+ sendUpdateWhenStreamNotRestored: false,
});
}
@@ -2744,6 +2761,7 @@ class MediaEngineFacade extends Store {
restoreStream: pendingRestore.restoreStream,
screenShareSnapshot: null,
toastWhenStreamNotRestored: true,
+ sendUpdateWhenStreamNotRestored: false,
});
}
diff --git a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts
index cf38b49cf..719bd9d5a 100644
--- a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts
+++ b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts
@@ -284,6 +284,20 @@ export function bindRoomEvents(
participantSpeakingDisposers.get(identity)?.dispose();
participantSpeakingDisposers.delete(identity);
};
+ const observeRemoteScreenSharePublication = (participant: Participant, pub: RemoteTrackPublication): void => {
+ reportScreenSharePublicationObserved(participant.identity, pub.trackSid ?? null);
+ const streamKey = getStreamKeyForParticipantIdentity(guildId, channelId, participant.identity);
+ if (streamKey) {
+ cancelDeferredStopWatchingStreamKey(streamKey);
+ }
+ };
+ const observeExistingRemoteScreenSharePublications = (): void => {
+ room.remoteParticipants.forEach((participant) => {
+ for (const pub of participant.trackPublications.values()) {
+ if (pub.source === Track.Source.ScreenShare) observeRemoteScreenSharePublication(participant, pub);
+ }
+ });
+ };
const endScreenShareIfPublicationDidNotReturn = (): void => {
const publication = screenSharePublicationAwaitingReconnect;
screenSharePublicationAwaitingReconnect = null;
@@ -305,6 +319,7 @@ export function bindRoomEvents(
bindParticipantSpeakingEvents(room.localParticipant);
room.remoteParticipants.forEach((participant) => bindParticipantSpeakingEvents(participant));
dependencies.remoteSpeaking.hydrateFromRoom(room);
+ observeExistingRemoteScreenSharePublications();
bindScreenShareNegotiation();
dependencies.permissions.applyDeafen(room, getEffectiveAudioState().effectiveDeaf);
dependencies.connection.markConnected();
@@ -596,11 +611,7 @@ export function bindRoomEvents(
pub.setSubscribed(true);
}
if (pub.source === Track.Source.ScreenShare && !participant.isLocal) {
- reportScreenSharePublicationObserved(participant.identity, pub.trackSid ?? null);
- const streamKey = getStreamKeyForParticipantIdentity(guildId, channelId, participant.identity);
- if (streamKey) {
- cancelDeferredStopWatchingStreamKey(streamKey);
- }
+ observeRemoteScreenSharePublication(participant, pub);
dependencies.subscriptions.reattachScreenShareAfterPublish(participant.identity, pub);
dependencies.subscriptions.reconcileSubscriptions();
}
@@ -615,6 +626,7 @@ export function bindRoomEvents(
}),
);
if (room.state === ConnectionState.Connected) {
+ guard(attemptId, observeExistingRemoteScreenSharePublications)();
guard(attemptId, bindScreenShareNegotiation)();
}
}
diff --git a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts
index 66f3054d5..eb59103ca 100644
--- a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts
+++ b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts
@@ -320,6 +320,7 @@ export class VoiceMeshEngine extends RTCEngine {
handlePeerStatus(peer: VoiceMeshPeer): void {
if (this.peers.get(peer.connectionId) !== peer) return;
const {status} = peer;
+ if (status === 'failed' && this.ownP2p() === false) return;
VoiceMeshPeers.setStatus(peer.connectionId, status);
if (status === 'connecting') return;
this.reportPeer(peer, status);
@@ -395,28 +396,30 @@ export class VoiceMeshEngine extends RTCEngine {
const serverMuted =
src === 'microphone' &&
voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(peer.connectionId)?.mute === true;
- if (receiver && receiver.track.readyState === 'live' && publication.isDesired && !serverMuted) {
- if (publication.track?.mediaStreamTrack !== receiver.track) {
- participant.addSubscribedMediaTrack(receiver.track, info.sid, new MediaStream([receiver.track]), receiver);
- }
- return;
+ const live = receiver !== null && receiver.track.readyState === 'live' && publication.isDesired && !serverMuted;
+ if (live && publication.track?.mediaStreamTrack === receiver.track) return;
+ if (publication.track) {
+ publication.track.stop();
+ publication.setTrack(undefined);
+ }
+ if (live) {
+ participant.addSubscribedMediaTrack(receiver.track, info.sid, new MediaStream([receiver.track]), receiver);
}
- if (!publication.track) return;
- publication.track.stop();
- publication.setTrack(undefined);
}
private readonly syncRoster = (): void => {
if (!this.room) return;
const {connectionId, channelId, guildId} = this.target;
- const own = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId);
- const ownP2p = own?.channel_id === channelId ? own.p2p : undefined;
+ const ownP2p = this.ownP2p();
this.syncConversionWatchdog(ownP2p === false && !this.conversionHeld);
const roster = new Map<string, NormalizedVoiceState>();
- if (ownP2p === true) {
+ if (ownP2p !== undefined) {
const states = voiceEngineV2AppVoiceStateAdapter.getAllVoiceStatesInChannel(guildId ?? ME, channelId);
for (const state of Object.values(states)) {
- if (state.p2p === true && state.connection_id !== connectionId) roster.set(state.connection_id, state);
+ if (state.connection_id === connectionId) continue;
+ if (this.peers.has(state.connection_id) || (ownP2p && state.p2p === true)) {
+ roster.set(state.connection_id, state);
+ }
}
}
const before = this.peers.size;
@@ -431,6 +434,12 @@ export class VoiceMeshEngine extends RTCEngine {
for (const peer of this.peers.values()) this.syncPeerReceiver(peer, 'microphone');
};
+ private ownP2p(): boolean | undefined {
+ const {connectionId, channelId} = this.target;
+ const own = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId);
+ return own?.channel_id === channelId ? own.p2p : undefined;
+ }
+
private addPeer(state: NormalizedVoiceState): void {
const peer = new VoiceMeshPeer(this, state.connection_id, state.user_id, this.target.connectionId);
this.peers.set(peer.connectionId, peer);
diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts
index 7d86b80a2..fcefd44db 100644
--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts
+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts
@@ -3,7 +3,11 @@
import assert from 'node:assert/strict';
import Authentication from '@app/features/auth/state/Authentication';
import {Logger} from '@app/features/platform/utils/AppLogger';
-import {createVoiceMeshRoom, holdVoiceMeshConversion} from '@app/features/voice/engine/mesh/VoiceMeshRoom';
+import {
+ createVoiceMeshRoom,
+ holdVoiceMeshConversion,
+ isVoiceMeshRoom,
+} from '@app/features/voice/engine/mesh/VoiceMeshRoom';
import {Store} from '@app/features/voice/engine/Store';
import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector';
import {
@@ -785,8 +789,13 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
logger.warn('Region hot-swap: missing participant for track republishing');
return;
}
+ const isMeshConversion = isVoiceMeshRoom(oldRoom);
const publications = selectLocalMediaPublicationsForConnectionRepublish(
Array.from(oldParticipant.trackPublications.values()),
+ ).filter(
+ (publication) =>
+ !isMeshConversion ||
+ (publication.source !== Track.Source.ScreenShare && publication.source !== Track.Source.ScreenShareAudio),
);
const errors: Array<{source: string; error: unknown}> = [];
let codecPolicyFailure: Error | null = null;