A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(voice): start bitrate for non-SVC screen share codecs (#2434)
Hampusadd0a3dfc6eb
diff --git a/fluxer_app/pkgs/livekit-client/VENDORING.md b/fluxer_app/pkgs/livekit-client/VENDORING.mdindex 40193749e..7e803c5f6 100644--- a/fluxer_app/pkgs/livekit-client/VENDORING.md+++ b/fluxer_app/pkgs/livekit-client/VENDORING.md@@ -101,6 +101,21 @@ source edits in this package.sender track back, including `LocalVideoTrack`'s secondary simulcast senders,and aggregate every cleanup failure instead of discarding it.+13. **Start bitrate for every video codec** (`src/room/PCTransport.ts`,+ `src/room/participant/LocalParticipant.ts`, `src/room/participant/publishUtils.ts`)+ `x-google-start-bitrate` was reachable only by AV1 and VP9, gated twice: the+ publish path registered a track bitrate only for SVC codecs, and the offer+ munging returned early for everything else. H264, H265 and VP8 therefore+ opened at the Chromium default and had to ramp, which showed up as a 3000 kbps+ screen share encoding at 346 kbps twenty seconds in. The bitrate is now+ registered for every video codec from the highest encoding+ (`maxEncodingBitrate()`, so a simulcast ladder contributes its top layer), and+ the offer munging applies the start bitrate whenever a max bitrate is known.+ The dependency descriptor extension stays SVC-only.+ `appendStartBitrateToFmtp()` holds the fmtp edit so it can be tested, and+ `setTrackCodecBitrate()` now replaces an entry for the same cid or transceiver+ instead of appending, since `trackBitrates` is never cleared.+## Updating from upstream1. Check the upstream changelog for the target version.diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.tsnew file mode 100644index 000000000..bcfe6defc--- /dev/null+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts@@ -0,0 +1,46 @@+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.+//+// SPDX-License-Identifier: Apache-2.0+import type {MediaDescription} from 'sdp-transform';+import {describe, expect, it} from 'vitest';+import {appendStartBitrateToFmtp} from './PCTransport.ts';++function mediaWithFmtp(entries: Array<{payload: number; config: string}>): MediaDescription {+ return {fmtp: entries} as unknown as MediaDescription;+}++describe('appendStartBitrateToFmtp', () => {+ it('appends the start bitrate to a non-SVC codec fmtp line', () => {+ const media = mediaWithFmtp([+ {payload: 96, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'},+ ]);+ appendStartBitrateToFmtp(media, 96, 2100);+ expect(media.fmtp[0]?.config).toBe(+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f;x-google-start-bitrate=2100',+ );+ });++ it('only touches the fmtp line for the matching payload', () => {+ const media = mediaWithFmtp([+ {payload: 96, config: 'profile-level-id=42e01f'},+ {payload: 98, config: 'profile-id=0'},+ ]);+ appendStartBitrateToFmtp(media, 98, 1400);+ expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');+ expect(media.fmtp[1]?.config).toBe('profile-id=0;x-google-start-bitrate=1400');+ });++ it('never appends a second start bitrate', () => {+ const media = mediaWithFmtp([{payload: 96, config: 'profile-level-id=42e01f;x-google-start-bitrate=900'}]);+ appendStartBitrateToFmtp(media, 96, 2100);+ expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=900');+ });++ it('does nothing without a usable bitrate or a matching payload', () => {+ const media = mediaWithFmtp([{payload: 96, config: 'profile-level-id=42e01f'}]);+ appendStartBitrateToFmtp(media, 96, 0);+ appendStartBitrateToFmtp(media, 96, -1);+ appendStartBitrateToFmtp(media, 111, 2100);+ expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');+ });+});diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.tsindex f7688a9f1..55ae7a3bb 100644--- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts@@ -18,7 +18,7 @@ interface TrackBitrateInfo {maxbr: number;}-const startBitrateForSVC = 0.7;+const startBitrateFraction = 0.7;const opusMaxAverageBitrateBps = 510000;const opusPacketTimeMs = 10;const requiredOpusFmtpParameters = {@@ -35,6 +35,21 @@ export const PCEvents = {RTPVideoPayloadTypes: 'rtpVideoPayloadTypes',} as const;+export function appendStartBitrateToFmtp(media: MediaDescription, codecPayload: number, startBitrate: number): void {+ if (startBitrate <= 0) {+ return;+ }+ for (const fmtp of media.fmtp) {+ if (fmtp.payload !== codecPayload) {+ continue;+ }+ if (!fmtp.config.includes('x-google-start-bitrate')) {+ fmtp.config += `;x-google-start-bitrate=${startBitrate}`;+ }+ return;+ }+}+export default class PCTransport extends EventEmitter {private _pc: RTCPeerConnection | null;@@ -248,18 +263,10 @@ export default class PCTransport extends EventEmitter {if (isSVCCodec(trackbr.codec) && !isSafari()) {this.ensureVideoDDExtensionForSVC(media, sdpParsed);}- if (!isSVCCodec(trackbr.codec)) {+ if (trackbr.maxbr <= 0) {return true;}- const startBitrate = Math.round(trackbr.maxbr * startBitrateForSVC);- for (const fmtp of media.fmtp) {- if (fmtp.payload === codecPayload) {- if (!fmtp.config.includes('x-google-start-bitrate')) {- fmtp.config += `;x-google-start-bitrate=${startBitrate}`;- }- break;- }- }+ appendStartBitrateToFmtp(media, codecPayload, Math.round(trackbr.maxbr * startBitrateFraction));return true;});}@@ -336,7 +343,16 @@ export default class PCTransport extends EventEmitter {}setTrackCodecBitrate(info: TrackBitrateInfo) {- this.trackBitrates.push(info);+ const existing = this.trackBitrates.findIndex(+ (trackbr) =>+ (info.cid !== undefined && trackbr.cid === info.cid) ||+ (info.transceiver !== undefined && trackbr.transceiver === info.transceiver),+ );+ if (existing === -1) {+ this.trackBitrates.push(info);+ return;+ }+ this.trackBitrates[existing] = info;}setConfiguration(rtcConfig: RTCConfiguration) {diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.tsindex 08467433e..e751c89d3 100644--- a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts@@ -97,7 +97,12 @@ import {import Participant from './Participant.ts';import type {ParticipantTrackPermission} from './ParticipantTrackPermission.ts';import {trackPermissionToProto} from './ParticipantTrackPermission.ts';-import {computeTrackBackupEncodings, computeVideoEncodings, getDefaultDegradationPreference} from './publishUtils.ts';+import {+ computeTrackBackupEncodings,+ computeVideoEncodings,+ getDefaultDegradationPreference,+ maxEncodingBitrate,+} from './publishUtils.ts';import type RemoteParticipant from './RemoteParticipant.ts';type PendingSignalRequestValues = {@@ -1037,12 +1042,15 @@ export default class LocalParticipant extends Participant {maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,});}- } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {- this.engine.pcManager.publisher.setTrackCodecBitrate({- cid: req.cid,- codec: track.codec,- maxbr: encodings[0].maxBitrate / 1000,- });+ } else if (track.codec) {+ const maxBitrate = maxEncodingBitrate(encodings);+ if (maxBitrate > 0) {+ this.engine.pcManager.publisher.setTrackCodecBitrate({+ cid: req.cid,+ codec: track.codec,+ maxbr: maxBitrate / 1000,+ });+ }}}diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.maxEncodingBitrate.test.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.maxEncodingBitrate.test.tsnew file mode 100644index 000000000..bac9a1078--- /dev/null+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.maxEncodingBitrate.test.ts@@ -0,0 +1,21 @@+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.+//+// SPDX-License-Identifier: Apache-2.0+import {describe, expect, it} from 'vitest';+import {maxEncodingBitrate} from './publishUtils.ts';++describe('maxEncodingBitrate', () => {+ it('returns the highest layer bitrate for a simulcast ladder', () => {+ expect(maxEncodingBitrate([{maxBitrate: 150_000}, {maxBitrate: 500_000}, {maxBitrate: 1_700_000}])).toBe(1_700_000);+ });++ it('returns the single encoding bitrate for a non-simulcast publish', () => {+ expect(maxEncodingBitrate([{maxBitrate: 3_000_000}])).toBe(3_000_000);+ });++ it('returns zero when no encoding declares a bitrate', () => {+ expect(maxEncodingBitrate([{}])).toBe(0);+ expect(maxEncodingBitrate([])).toBe(0);+ expect(maxEncodingBitrate(undefined)).toBe(0);+ });+});diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.tsindex e5844c0a7..41f7fd8a0 100644--- a/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts@@ -370,6 +370,19 @@ export class ScalabilityMode {}}+export function maxEncodingBitrate(encodings: Array<RTCRtpEncodingParameters> | undefined): number {+ if (!encodings) {+ return 0;+ }+ let maxBitrate = 0;+ for (const encoding of encodings) {+ if (typeof encoding.maxBitrate === 'number' && encoding.maxBitrate > maxBitrate) {+ maxBitrate = encoding.maxBitrate;+ }+ }+ return maxBitrate;+}+export function getDefaultDegradationPreference(track: LocalVideoTrack): RTCDegradationPreference {if (track.source === Track.Source.ScreenShare) return 'maintain-resolution';if (track.constraints.height && unwrapConstraint(track.constraints.height) >= 1080) return 'maintain-resolution';