A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(voice): republish screen share when its codec goes stale (#2426)
Hampusf090395c215b
diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.tsindex 14ddbe8b1..56d03bfd1 100644--- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts+++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts@@ -58,6 +58,7 @@ import {saveCurrentVoiceSessionRestoreSnapshot,type VoiceSessionRestoreSyncHandle,} from '@app/features/voice/engine/media_engine_facade/VoiceSessionRestoreSync';+import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';import {Store, useStoreVersion} from '@app/features/voice/engine/Store';import {shouldMoveToAfkOnTick} from '@app/features/voice/engine/VoiceAfkTracking';@@ -388,6 +389,17 @@ class MediaEngineFacade extends Store {this.voiceEngineV2SourceLifecycleBridge = this.createSourceLifecycleBridge();voiceEngineV2AppScreenShareExecutionAdapter.setControllerGateway(this.createScreenShareControllerGateway());voiceEngineV2AppScreenShareExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);+ ScreenShareCodecNegotiation.setSelectionChangeListener((activeRoom, codec, reason) => {+ void voiceEngineV2AppScreenShareExecutionAdapter+ .republishActiveScreenShareForNegotiatedCodecInternal(activeRoom, codec)+ .catch((error) => {+ logger.warn('Failed to republish active screen share after a codec negotiation change', {+ error,+ codec,+ reason,+ });+ });+ });voiceEngineV2AppMediaExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);setNativeAudioCaptureBridgeLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);this.syncVoiceEngineV2AudioControlsFromAppState();diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.tsindex bcc58dd53..d8fb37c40 100644--- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts+++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts@@ -74,10 +74,45 @@ Object.defineProperty(globalThis, 'RTCRtpReceiver', {value: {getCapabilities: () => VIDEO_CAPABILITIES},});-const {buildLocalCodecAdvertisements, computeNegotiatedVideoCodec, getScreenShareCodecPreferenceOrder} = await import(- './ScreenShareCodecNegotiation'-);+const {+ default: ScreenShareCodecNegotiation,+ buildLocalCodecAdvertisements,+ computeNegotiatedVideoCodec,+ getScreenShareCodecPreferenceOrder,+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,+} = await import('./ScreenShareCodecNegotiation');const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');+const {RoomEvent} = await import('livekit-client');++class FakeRoom {+ private handlers = new Map<string, Set<(...args: Array<unknown>) => void>>();+ localParticipant = {identity: 'local', publishData: vi.fn(async () => undefined)};+ remoteParticipants = new Map<string, {identity: string}>();++ on(event: string, handler: (...args: Array<unknown>) => void): this {+ if (!this.handlers.has(event)) this.handlers.set(event, new Set());+ this.handlers.get(event)?.add(handler);+ return this;+ }++ off(event: string, handler: (...args: Array<unknown>) => void): this {+ this.handlers.get(event)?.delete(handler);+ return this;+ }++ emit(event: string, ...args: Array<unknown>): void {+ for (const handler of this.handlers.get(event) ?? []) handler(...args);+ }+}++function sessionUpdatePayload(codecs: Array<FluxerCodecAdvertisement>): Uint8Array {+ return new TextEncoder().encode(+ JSON.stringify({+ op: 14,+ d: {video_codec: 'H264', media_session_id: 'peer-session', reason: 'connected', codecs},+ }),+ );+}function videoAdvertisement(name: 'AV1' | 'VP9' | 'H265', encode: boolean, decode: boolean): FluxerCodecAdvertisement {const payloadType = name === 'AV1' ? 101 : name === 'H265' ? 105 : 109;@@ -180,3 +215,89 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => {expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('h265');});});++describe('reacting to a mid-session codec selection change', () => {+ let room: FakeRoom;++ beforeEach(() => {+ av1OptIn = false;+ hevcOptIn = true;+ preferredScreenShareCodec = 'auto';+ gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};+ resetCachedCodecCapabilities();+ room = new FakeRoom();+ ScreenShareCodecNegotiation.setSelectionChangeListener(null);+ });++ function flush(): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, 0));+ }++ async function establishBaseline(): Promise<void> {+ ScreenShareCodecNegotiation.bind(room as never);+ room.emit(RoomEvent.ParticipantDisconnected, {identity: 'nobody-was-connected'});+ await flush();+ }++ it('picks the locally preferred codec while no remote participant is known', async () => {+ await establishBaseline();+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');+ ScreenShareCodecNegotiation.dispose();+ });++ it('notifies the selection-change listener once a peer without HEVC decode is discovered', async () => {+ await establishBaseline();+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');+ const listener = vi.fn();+ ScreenShareCodecNegotiation.setSelectionChangeListener(listener);+ room.remoteParticipants.set('peer-without-hevc', {identity: 'peer-without-hevc'});+ const payload = sessionUpdatePayload([videoAdvertisement('VP9', true, true)]);+ room.emit(+ RoomEvent.DataReceived,+ payload,+ {identity: 'peer-without-hevc'},+ undefined,+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,+ );+ await flush();+ const negotiated = ScreenShareCodecNegotiation.getSelectedCodec();+ expect(negotiated).not.toBe('h265');+ expect(listener).toHaveBeenCalledTimes(1);+ expect(listener).toHaveBeenCalledWith(room, negotiated, 'data');+ ScreenShareCodecNegotiation.dispose();+ });++ it('does not notify the listener when the recomputed selection is unchanged', async () => {+ await establishBaseline();+ room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'});+ const firstPayload = sessionUpdatePayload([+ videoAdvertisement('H265', true, true),+ videoAdvertisement('VP9', true, true),+ ]);+ room.emit(+ RoomEvent.DataReceived,+ firstPayload,+ {identity: 'peer-with-hevc'},+ undefined,+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,+ );+ await flush();+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');+ const listener = vi.fn();+ ScreenShareCodecNegotiation.setSelectionChangeListener(listener);+ const secondPayload = sessionUpdatePayload([+ videoAdvertisement('H265', true, true),+ videoAdvertisement('VP9', true, true),+ ]);+ room.emit(+ RoomEvent.DataReceived,+ secondPayload,+ {identity: 'peer-with-hevc'},+ undefined,+ SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,+ );+ await flush();+ expect(listener).not.toHaveBeenCalled();+ ScreenShareCodecNegotiation.dispose();+ });+});diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.tsindex 463cdebb6..3ed6fb826 100644--- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts+++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts@@ -73,7 +73,7 @@ const VIDEO_CODEC_PROTOCOL_TABLE: Record<type FluxerVideoCodecName = 'AV1' | 'H265' | 'H264' | 'VP9' | 'VP8';type FluxerCodecName = 'opus' | FluxerVideoCodecName;type FluxerCodecType = 'audio' | 'video';-type NegotiationReason =+export type NegotiationReason =| 'connected'| 'data'| 'participant-connected'@@ -499,11 +499,18 @@ class ScreenShareCodecNegotiation {private mediaSessionId = createId('media');private negotiationSnapshot = createScreenShareCodecNegotiationSnapshot();private bindingRevision = 0;+ private onSelectionChanged: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null = null;getSelectedCodec(): VideoCodec | null {return this.selectedCodec;}+ setSelectionChangeListener(+ listener: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null,+ ): void {+ this.onSelectionChanged = listener;+ }+selectScreenShareCodec(preference: CodecPreference = 'auto'): VideoCodec {const selected = this.selectedCodec;const selector = (codecPreference: CodecPreference): VideoCodec =>@@ -716,6 +723,7 @@ class ScreenShareCodecNegotiation {logger.info('Selected screen share codec from XState capability intersection', selection);await this.publishSessionUpdate(room, selection);if (!this.isBindingCurrent(room, bindingRevision)) return null;+ this.onSelectionChanged?.(room, selection.codec, selection.reason);return selection;}diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsxindex 4efba2c53..79354de8a 100644--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx@@ -433,6 +433,23 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {this.stopScreenShareAfterEncoderFailure(room, participant, track, failedCodec, 'stalled');}+ async republishActiveScreenShareForNegotiatedCodecInternal(room: Room | null, codec: VideoCodec): Promise<void> {+ const participant = room?.localParticipant;+ if (!room || !participant) return;+ const publication = participant.getTrackPublication(Track.Source.ScreenShare);+ const track = publication?.videoTrack as LocalVideoTrack | undefined;+ if (!track || !this.isScreenShareTrackPublishedInternal(participant, track)) return;+ const publishedOptions = ((publication as {options?: TrackPublishOptions}).options ?? {}) as TrackPublishOptions;+ if (publishedOptions.videoCodec === codec) return;+ const republished = await this.liveKitFlows.republishActiveShareWithCodec(room, track, codec);+ if (!republished) {+ logger.warn('Failed to republish active screen share after a codec negotiation change', {+ codec,+ previousCodec: publishedOptions.videoCodec,+ });+ }+ }+private async correctVerifiedScreenShareCodec(room: Room | null,participant: LocalParticipant,