Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(voice): republish screen share when its codec goes stale (#2426)

Hampusf090395c215b
diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
index 14ddbe8b1..56d03bfd1 100644
--- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
+++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
@@ -58,6 +58,7 @@ import {
saveCurrentVoiceSessionRestoreSnapshot,
type VoiceSessionRestoreSyncHandle,
} from '@app/features/voice/engine/media_engine_facade/VoiceSessionRestoreSync';
+import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
import {Store, useStoreVersion} from '@app/features/voice/engine/Store';
import {shouldMoveToAfkOnTick} from '@app/features/voice/engine/VoiceAfkTracking';
@@ -388,6 +389,17 @@ class MediaEngineFacade extends Store {
this.voiceEngineV2SourceLifecycleBridge = this.createSourceLifecycleBridge();
voiceEngineV2AppScreenShareExecutionAdapter.setControllerGateway(this.createScreenShareControllerGateway());
voiceEngineV2AppScreenShareExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);
+ ScreenShareCodecNegotiation.setSelectionChangeListener((activeRoom, codec, reason) => {
+ void voiceEngineV2AppScreenShareExecutionAdapter
+ .republishActiveScreenShareForNegotiatedCodecInternal(activeRoom, codec)
+ .catch((error) => {
+ logger.warn('Failed to republish active screen share after a codec negotiation change', {
+ error,
+ codec,
+ reason,
+ });
+ });
+ });
voiceEngineV2AppMediaExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);
setNativeAudioCaptureBridgeLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);
this.syncVoiceEngineV2AudioControlsFromAppState();
diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts
index bcc58dd53..d8fb37c40 100644
--- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts
+++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts
@@ -74,10 +74,45 @@ Object.defineProperty(globalThis, 'RTCRtpReceiver', {
value: {getCapabilities: () => VIDEO_CAPABILITIES},
});
-const {buildLocalCodecAdvertisements, computeNegotiatedVideoCodec, getScreenShareCodecPreferenceOrder} = await import(
- './ScreenShareCodecNegotiation'
-);
+const {
+ default: ScreenShareCodecNegotiation,
+ buildLocalCodecAdvertisements,
+ computeNegotiatedVideoCodec,
+ getScreenShareCodecPreferenceOrder,
+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+} = await import('./ScreenShareCodecNegotiation');
const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');
+const {RoomEvent} = await import('livekit-client');
+
+class FakeRoom {
+ private handlers = new Map<string, Set<(...args: Array<unknown>) => void>>();
+ localParticipant = {identity: 'local', publishData: vi.fn(async () => undefined)};
+ remoteParticipants = new Map<string, {identity: string}>();
+
+ on(event: string, handler: (...args: Array<unknown>) => void): this {
+ if (!this.handlers.has(event)) this.handlers.set(event, new Set());
+ this.handlers.get(event)?.add(handler);
+ return this;
+ }
+
+ off(event: string, handler: (...args: Array<unknown>) => void): this {
+ this.handlers.get(event)?.delete(handler);
+ return this;
+ }
+
+ emit(event: string, ...args: Array<unknown>): void {
+ for (const handler of this.handlers.get(event) ?? []) handler(...args);
+ }
+}
+
+function sessionUpdatePayload(codecs: Array<FluxerCodecAdvertisement>): Uint8Array {
+ return new TextEncoder().encode(
+ JSON.stringify({
+ op: 14,
+ d: {video_codec: 'H264', media_session_id: 'peer-session', reason: 'connected', codecs},
+ }),
+ );
+}
function videoAdvertisement(name: 'AV1' | 'VP9' | 'H265', encode: boolean, decode: boolean): FluxerCodecAdvertisement {
const payloadType = name === 'AV1' ? 101 : name === 'H265' ? 105 : 109;
@@ -180,3 +215,89 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => {
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('h265');
});
});
+
+describe('reacting to a mid-session codec selection change', () => {
+ let room: FakeRoom;
+
+ beforeEach(() => {
+ av1OptIn = false;
+ hevcOptIn = true;
+ preferredScreenShareCodec = 'auto';
+ gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};
+ resetCachedCodecCapabilities();
+ room = new FakeRoom();
+ ScreenShareCodecNegotiation.setSelectionChangeListener(null);
+ });
+
+ function flush(): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, 0));
+ }
+
+ async function establishBaseline(): Promise<void> {
+ ScreenShareCodecNegotiation.bind(room as never);
+ room.emit(RoomEvent.ParticipantDisconnected, {identity: 'nobody-was-connected'});
+ await flush();
+ }
+
+ it('picks the locally preferred codec while no remote participant is known', async () => {
+ await establishBaseline();
+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');
+ ScreenShareCodecNegotiation.dispose();
+ });
+
+ it('notifies the selection-change listener once a peer without HEVC decode is discovered', async () => {
+ await establishBaseline();
+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');
+ const listener = vi.fn();
+ ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
+ room.remoteParticipants.set('peer-without-hevc', {identity: 'peer-without-hevc'});
+ const payload = sessionUpdatePayload([videoAdvertisement('VP9', true, true)]);
+ room.emit(
+ RoomEvent.DataReceived,
+ payload,
+ {identity: 'peer-without-hevc'},
+ undefined,
+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+ );
+ await flush();
+ const negotiated = ScreenShareCodecNegotiation.getSelectedCodec();
+ expect(negotiated).not.toBe('h265');
+ expect(listener).toHaveBeenCalledTimes(1);
+ expect(listener).toHaveBeenCalledWith(room, negotiated, 'data');
+ ScreenShareCodecNegotiation.dispose();
+ });
+
+ it('does not notify the listener when the recomputed selection is unchanged', async () => {
+ await establishBaseline();
+ room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'});
+ const firstPayload = sessionUpdatePayload([
+ videoAdvertisement('H265', true, true),
+ videoAdvertisement('VP9', true, true),
+ ]);
+ room.emit(
+ RoomEvent.DataReceived,
+ firstPayload,
+ {identity: 'peer-with-hevc'},
+ undefined,
+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+ );
+ await flush();
+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');
+ const listener = vi.fn();
+ ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
+ const secondPayload = sessionUpdatePayload([
+ videoAdvertisement('H265', true, true),
+ videoAdvertisement('VP9', true, true),
+ ]);
+ room.emit(
+ RoomEvent.DataReceived,
+ secondPayload,
+ {identity: 'peer-with-hevc'},
+ undefined,
+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+ );
+ await flush();
+ expect(listener).not.toHaveBeenCalled();
+ ScreenShareCodecNegotiation.dispose();
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts
index 463cdebb6..3ed6fb826 100644
--- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts
+++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts
@@ -73,7 +73,7 @@ const VIDEO_CODEC_PROTOCOL_TABLE: Record<
type FluxerVideoCodecName = 'AV1' | 'H265' | 'H264' | 'VP9' | 'VP8';
type FluxerCodecName = 'opus' | FluxerVideoCodecName;
type FluxerCodecType = 'audio' | 'video';
-type NegotiationReason =
+export type NegotiationReason =
| 'connected'
| 'data'
| 'participant-connected'
@@ -499,11 +499,18 @@ class ScreenShareCodecNegotiation {
private mediaSessionId = createId('media');
private negotiationSnapshot = createScreenShareCodecNegotiationSnapshot();
private bindingRevision = 0;
+ private onSelectionChanged: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null = null;
getSelectedCodec(): VideoCodec | null {
return this.selectedCodec;
}
+ setSelectionChangeListener(
+ listener: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null,
+ ): void {
+ this.onSelectionChanged = listener;
+ }
+
selectScreenShareCodec(preference: CodecPreference = 'auto'): VideoCodec {
const selected = this.selectedCodec;
const selector = (codecPreference: CodecPreference): VideoCodec =>
@@ -716,6 +723,7 @@ class ScreenShareCodecNegotiation {
logger.info('Selected screen share codec from XState capability intersection', selection);
await this.publishSessionUpdate(room, selection);
if (!this.isBindingCurrent(room, bindingRevision)) return null;
+ this.onSelectionChanged?.(room, selection.codec, selection.reason);
return selection;
}
diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx
index 4efba2c53..79354de8a 100644
--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx
+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx
@@ -433,6 +433,23 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
this.stopScreenShareAfterEncoderFailure(room, participant, track, failedCodec, 'stalled');
}
+ async republishActiveScreenShareForNegotiatedCodecInternal(room: Room | null, codec: VideoCodec): Promise<void> {
+ const participant = room?.localParticipant;
+ if (!room || !participant) return;
+ const publication = participant.getTrackPublication(Track.Source.ScreenShare);
+ const track = publication?.videoTrack as LocalVideoTrack | undefined;
+ if (!track || !this.isScreenShareTrackPublishedInternal(participant, track)) return;
+ const publishedOptions = ((publication as {options?: TrackPublishOptions}).options ?? {}) as TrackPublishOptions;
+ if (publishedOptions.videoCodec === codec) return;
+ const republished = await this.liveKitFlows.republishActiveShareWithCodec(room, track, codec);
+ if (!republished) {
+ logger.warn('Failed to republish active screen share after a codec negotiation change', {
+ codec,
+ previousCodec: publishedOptions.videoCodec,
+ });
+ }
+ }
+
private async correctVerifiedScreenShareCodec(
room: Room | null,
participant: LocalParticipant,