Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(voice): start bitrate for non-SVC screen share codecs (#2434)

Hampusadd0a3dfc6eb
diff --git a/fluxer_app/pkgs/livekit-client/VENDORING.md b/fluxer_app/pkgs/livekit-client/VENDORING.md
index 40193749e..7e803c5f6 100644
--- a/fluxer_app/pkgs/livekit-client/VENDORING.md
+++ b/fluxer_app/pkgs/livekit-client/VENDORING.md
@@ -101,6 +101,21 @@ source edits in this package.
sender track back, including `LocalVideoTrack`'s secondary simulcast senders,
and aggregate every cleanup failure instead of discarding it.
+13. **Start bitrate for every video codec** (`src/room/PCTransport.ts`,
+ `src/room/participant/LocalParticipant.ts`, `src/room/participant/publishUtils.ts`)
+ `x-google-start-bitrate` was reachable only by AV1 and VP9, gated twice: the
+ publish path registered a track bitrate only for SVC codecs, and the offer
+ munging returned early for everything else. H264, H265 and VP8 therefore
+ opened at the Chromium default and had to ramp, which showed up as a 3000 kbps
+ screen share encoding at 346 kbps twenty seconds in. The bitrate is now
+ registered for every video codec from the highest encoding
+ (`maxEncodingBitrate()`, so a simulcast ladder contributes its top layer), and
+ the offer munging applies the start bitrate whenever a max bitrate is known.
+ The dependency descriptor extension stays SVC-only.
+ `appendStartBitrateToFmtp()` holds the fmtp edit so it can be tested, and
+ `setTrackCodecBitrate()` now replaces an entry for the same cid or transceiver
+ instead of appending, since `trackBitrates` is never cleared.
+
## Updating from upstream
1. Check the upstream changelog for the target version.
diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts
new file mode 100644
index 000000000..bcfe6defc
--- /dev/null
+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts
@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type {MediaDescription} from 'sdp-transform';
+import {describe, expect, it} from 'vitest';
+import {appendStartBitrateToFmtp} from './PCTransport.ts';
+
+function mediaWithFmtp(entries: Array<{payload: number; config: string}>): MediaDescription {
+ return {fmtp: entries} as unknown as MediaDescription;
+}
+
+describe('appendStartBitrateToFmtp', () => {
+ it('appends the start bitrate to a non-SVC codec fmtp line', () => {
+ const media = mediaWithFmtp([
+ {payload: 96, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'},
+ ]);
+ appendStartBitrateToFmtp(media, 96, 2100);
+ expect(media.fmtp[0]?.config).toBe(
+ 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f;x-google-start-bitrate=2100',
+ );
+ });
+
+ it('only touches the fmtp line for the matching payload', () => {
+ const media = mediaWithFmtp([
+ {payload: 96, config: 'profile-level-id=42e01f'},
+ {payload: 98, config: 'profile-id=0'},
+ ]);
+ appendStartBitrateToFmtp(media, 98, 1400);
+ expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
+ expect(media.fmtp[1]?.config).toBe('profile-id=0;x-google-start-bitrate=1400');
+ });
+
+ it('never appends a second start bitrate', () => {
+ const media = mediaWithFmtp([{payload: 96, config: 'profile-level-id=42e01f;x-google-start-bitrate=900'}]);
+ appendStartBitrateToFmtp(media, 96, 2100);
+ expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=900');
+ });
+
+ it('does nothing without a usable bitrate or a matching payload', () => {
+ const media = mediaWithFmtp([{payload: 96, config: 'profile-level-id=42e01f'}]);
+ appendStartBitrateToFmtp(media, 96, 0);
+ appendStartBitrateToFmtp(media, 96, -1);
+ appendStartBitrateToFmtp(media, 111, 2100);
+ expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
+ });
+});
diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts
index f7688a9f1..55ae7a3bb 100644
--- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts
+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts
@@ -18,7 +18,7 @@ interface TrackBitrateInfo {
maxbr: number;
}
-const startBitrateForSVC = 0.7;
+const startBitrateFraction = 0.7;
const opusMaxAverageBitrateBps = 510000;
const opusPacketTimeMs = 10;
const requiredOpusFmtpParameters = {
@@ -35,6 +35,21 @@ export const PCEvents = {
RTPVideoPayloadTypes: 'rtpVideoPayloadTypes',
} as const;
+export function appendStartBitrateToFmtp(media: MediaDescription, codecPayload: number, startBitrate: number): void {
+ if (startBitrate <= 0) {
+ return;
+ }
+ for (const fmtp of media.fmtp) {
+ if (fmtp.payload !== codecPayload) {
+ continue;
+ }
+ if (!fmtp.config.includes('x-google-start-bitrate')) {
+ fmtp.config += `;x-google-start-bitrate=${startBitrate}`;
+ }
+ return;
+ }
+}
+
export default class PCTransport extends EventEmitter {
private _pc: RTCPeerConnection | null;
@@ -248,18 +263,10 @@ export default class PCTransport extends EventEmitter {
if (isSVCCodec(trackbr.codec) && !isSafari()) {
this.ensureVideoDDExtensionForSVC(media, sdpParsed);
}
- if (!isSVCCodec(trackbr.codec)) {
+ if (trackbr.maxbr <= 0) {
return true;
}
- const startBitrate = Math.round(trackbr.maxbr * startBitrateForSVC);
- for (const fmtp of media.fmtp) {
- if (fmtp.payload === codecPayload) {
- if (!fmtp.config.includes('x-google-start-bitrate')) {
- fmtp.config += `;x-google-start-bitrate=${startBitrate}`;
- }
- break;
- }
- }
+ appendStartBitrateToFmtp(media, codecPayload, Math.round(trackbr.maxbr * startBitrateFraction));
return true;
});
}
@@ -336,7 +343,16 @@ export default class PCTransport extends EventEmitter {
}
setTrackCodecBitrate(info: TrackBitrateInfo) {
- this.trackBitrates.push(info);
+ const existing = this.trackBitrates.findIndex(
+ (trackbr) =>
+ (info.cid !== undefined && trackbr.cid === info.cid) ||
+ (info.transceiver !== undefined && trackbr.transceiver === info.transceiver),
+ );
+ if (existing === -1) {
+ this.trackBitrates.push(info);
+ return;
+ }
+ this.trackBitrates[existing] = info;
}
setConfiguration(rtcConfig: RTCConfiguration) {
diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts
index 08467433e..e751c89d3 100644
--- a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts
+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts
@@ -97,7 +97,12 @@ import {
import Participant from './Participant.ts';
import type {ParticipantTrackPermission} from './ParticipantTrackPermission.ts';
import {trackPermissionToProto} from './ParticipantTrackPermission.ts';
-import {computeTrackBackupEncodings, computeVideoEncodings, getDefaultDegradationPreference} from './publishUtils.ts';
+import {
+ computeTrackBackupEncodings,
+ computeVideoEncodings,
+ getDefaultDegradationPreference,
+ maxEncodingBitrate,
+} from './publishUtils.ts';
import type RemoteParticipant from './RemoteParticipant.ts';
type PendingSignalRequestValues = {
@@ -1037,12 +1042,15 @@ export default class LocalParticipant extends Participant {
maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
});
}
- } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
- this.engine.pcManager.publisher.setTrackCodecBitrate({
- cid: req.cid,
- codec: track.codec,
- maxbr: encodings[0].maxBitrate / 1000,
- });
+ } else if (track.codec) {
+ const maxBitrate = maxEncodingBitrate(encodings);
+ if (maxBitrate > 0) {
+ this.engine.pcManager.publisher.setTrackCodecBitrate({
+ cid: req.cid,
+ codec: track.codec,
+ maxbr: maxBitrate / 1000,
+ });
+ }
}
}
diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.maxEncodingBitrate.test.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.maxEncodingBitrate.test.ts
new file mode 100644
index 000000000..bac9a1078
--- /dev/null
+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.maxEncodingBitrate.test.ts
@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {describe, expect, it} from 'vitest';
+import {maxEncodingBitrate} from './publishUtils.ts';
+
+describe('maxEncodingBitrate', () => {
+ it('returns the highest layer bitrate for a simulcast ladder', () => {
+ expect(maxEncodingBitrate([{maxBitrate: 150_000}, {maxBitrate: 500_000}, {maxBitrate: 1_700_000}])).toBe(1_700_000);
+ });
+
+ it('returns the single encoding bitrate for a non-simulcast publish', () => {
+ expect(maxEncodingBitrate([{maxBitrate: 3_000_000}])).toBe(3_000_000);
+ });
+
+ it('returns zero when no encoding declares a bitrate', () => {
+ expect(maxEncodingBitrate([{}])).toBe(0);
+ expect(maxEncodingBitrate([])).toBe(0);
+ expect(maxEncodingBitrate(undefined)).toBe(0);
+ });
+});
diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts
index e5844c0a7..41f7fd8a0 100644
--- a/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts
+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts
@@ -370,6 +370,19 @@ export class ScalabilityMode {
}
}
+export function maxEncodingBitrate(encodings: Array<RTCRtpEncodingParameters> | undefined): number {
+ if (!encodings) {
+ return 0;
+ }
+ let maxBitrate = 0;
+ for (const encoding of encodings) {
+ if (typeof encoding.maxBitrate === 'number' && encoding.maxBitrate > maxBitrate) {
+ maxBitrate = encoding.maxBitrate;
+ }
+ }
+ return maxBitrate;
+}
+
export function getDefaultDegradationPreference(track: LocalVideoTrack): RTCDegradationPreference {
if (track.source === Track.Source.ScreenShare) return 'maintain-resolution';
if (track.constraints.height && unwrapConstraint(track.constraints.height) >= 1080) return 'maintain-resolution';