A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(voice): let screen shares use the hardware H.264 encoder (#2845)
Hampusd93604afa2f4
diff --git a/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.test.ts b/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.test.tsindex 851d5cd9b..5b08c8151 100644--- a/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.test.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.test.ts@@ -19,19 +19,41 @@ function codec(}describe('selectPublisherCodecPreferences', () => {- it('prefers H.264 profiles that use Chromium external encoders before OpenH264', () => {- const openH264 = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f');- const externalBaseline = codec(+ it('offers the profiles a hardware encoder accepts before Constrained Baseline, which is always software on Windows', () => {+ const constrainedBaseline = codec('video/H264',- 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f',+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',);+ const baseline = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f');const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f');const rtx = codec('video/rtx');- const preferences = selectPublisherCodecPreferences('h264', [openH264, rtx, externalBaseline, highProfile]);- expect(preferences).toEqual([externalBaseline, openH264, highProfile, rtx]);+ const preferences = selectPublisherCodecPreferences('h264', [constrainedBaseline, rtx, baseline, highProfile]);+ expect(preferences).toEqual([highProfile, baseline, constrainedBaseline, rtx]);+ });++ it('offers High first out of the capabilities Chromium reports, so the only hardware profile this server registers wins', () => {+ const capabilities = [+ codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f'),+ codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f'),+ codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'),+ codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f'),+ codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f'),+ codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f'),+ codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034'),+ ];+ const preferences = selectPublisherCodecPreferences('h264', capabilities);+ expect(preferences.map((entry) => entry.sdpFmtpLine)).toEqual([+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034',+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f',+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f',+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',+ 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f',+ 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f',+ 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f',+ ]);});- it('ranks Constrained Baseline above Main, High and Constrained High', () => {+ it('ranks High, Constrained High, Main and Baseline above Constrained Baseline, whatever level each one carries', () => {const constrainedBaseline = codec('video/H264','level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',@@ -57,15 +79,15 @@ describe('selectPublisherCodecPreferences', () => {constrainedBaseline,]);expect(preferences).toEqual([- constrainedBaseline,- mainProfile,highProfileLevel31,highProfileLevel51,constrainedHigh,+ mainProfile,+ constrainedBaseline,]);});- it('keeps Constrained Baseline packetization-mode=1 ahead of Constrained Baseline packetization-mode=0 and High', () => {+ it('ranks packetization-mode=1 above packetization-mode=0, which no hardware encoder takes', () => {const constrainedBaselineMode0 = codec('video/H264','level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f',@@ -74,13 +96,26 @@ describe('selectPublisherCodecPreferences', () => {'video/H264','level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',);- const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033');+ const highProfileMode0 = codec(+ 'video/H264',+ 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=640033',+ );+ const highProfileMode1 = codec(+ 'video/H264',+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033',+ );const preferences = selectPublisherCodecPreferences('h264', [- highProfile,+ highProfileMode0,constrainedBaselineMode0,constrainedBaselineMode1,+ highProfileMode1,+ ]);+ expect(preferences).toEqual([+ highProfileMode1,+ constrainedBaselineMode1,+ highProfileMode0,+ constrainedBaselineMode0,]);- expect(preferences).toEqual([constrainedBaselineMode1, constrainedBaselineMode0, highProfile]);});it('puts the chosen codec first and keeps every other codec in browser capability order', () => {@@ -106,7 +141,7 @@ describe('selectPublisherCodecPreferences', () => {'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',);const preferences = selectPublisherCodecPreferences('vp8', [vp8, highProfile, constrainedBaseline]);- expect(preferences).toEqual([vp8, constrainedBaseline, highProfile]);+ expect(preferences).toEqual([vp8, highProfile, constrainedBaseline]);});it('returns nothing when the sender cannot encode the chosen codec', () => {diff --git a/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.ts b/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.tsindex 8b06c15b8..31eb3be99 100644--- a/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/RTCEngine.ts@@ -122,8 +122,16 @@ const videoCodecMimeTypes: Record<VideoCodec, Array<string>> = {vp9: ['video/vp9'],vp8: ['video/vp8'],};-const h264OpenH264ProfileLevelId = '42e01f';-const h264PreferredHardwareProfileLevelIds = new Set(['42001f']);+const h264ProfileRanks = new Map([+ ['6400', 0],+ ['640c', 1],+ ['4d00', 2],+ ['4200', 3],+ ['42e0', 4],+]);+const h264UnrankedProfileScore = 5;+const h264MissingProfileScore = 6;+const h264PacketizationMode0Score = 10;type RtpCodecCapability = RTCRtpCapabilities['codecs'][number] & {sdpFmtpLine?: string};enum PCState {@@ -1812,11 +1820,9 @@ function getFmtpParameter(sdpFmtpLine: string | undefined, key: string): stringfunction getH264PublisherCodecScore(codec: RtpCodecCapability): number {const profileLevelId = getFmtpParameter(codec.sdpFmtpLine, 'profile-level-id');const packetizationMode = getFmtpParameter(codec.sdpFmtpLine, 'packetization-mode');- const packetizationScore = packetizationMode === '1' ? 0 : 1;- if (profileLevelId && h264PreferredHardwareProfileLevelIds.has(profileLevelId)) return packetizationScore;- if (profileLevelId === h264OpenH264ProfileLevelId) return 10 + packetizationScore;- if (profileLevelId) return 20 + packetizationScore;- return 30 + packetizationScore;+ const packetizationScore = packetizationMode === '1' ? 0 : h264PacketizationMode0Score;+ if (!profileLevelId) return packetizationScore + h264MissingProfileScore;+ return packetizationScore + (h264ProfileRanks.get(profileLevelId.slice(0, 4)) ?? h264UnrankedProfileScore);}function preferHardwareH264Codecs(codecs: ReadonlyArray<RtpCodecCapability>): Array<RtpCodecCapability> {