A platform for community discussion. Free, open, simple.
FIX: Sharper, smoother camera video in LiveKit voice rooms (#44650)
Rafael dos Santos Silva58548dba48bd
diff --git a/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.js b/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.jsindex 0824a6be078..9353fd088ea 100644--- a/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.js+++ b/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.js@@ -9,6 +9,7 @@ import voiceLog from "discourse/plugins/voice/discourse/lib/voice/logger";// (or the module-level test override), which is what makes it unit-testable.import {cameraEncodingFor,+ cameraSimulcastLayersFor,screenEncodingFor,voiceBitrateFor,} from "./video-quality.js";@@ -63,9 +64,12 @@ function wait(ms) {// Content audio: higher Opus ceiling than the speech default so screen audio// doesn't sound underwater.const SCREEN_AUDIO_BITRATE = 128_000;-// Camera subscriptions drop to the LOW simulcast layer above this many-// active video publishers — the same threshold the mesh bitrate ladder uses.+// Camera subscriptions step down simulcast layers at the same publisher+// counts where the mesh ladder steps down its rungs.+const MEDIUM_LAYER_PUBLISHER_THRESHOLD = 3;const LOW_LAYER_PUBLISHER_THRESHOLD = 6;+// Fallback when the capture track doesn't report its dimensions.+const DEFAULT_CAMERA_DIMENSIONS = { width: 1280, height: 720 };export default class LivekitRoomSession {// Cheap pre-load check so obviously unsupported browsers fail with a@@ -103,6 +107,7 @@ export default class LivekitRoomSession {#closed = false;#reconnecting = false;#agentIdentities = new Map();+ #videoSyncQueue = Promise.resolve();constructor({roomId,@@ -227,15 +232,25 @@ export default class LivekitRoomSession {// Brings the video and screen-audio publications in line with the local// capture state. The mesh equivalent gates per watching peer; on the SFU// media is published once and receive gating moves to the subscriber side.- async syncLocalVideo(videoTrack, screenAudioTrack, kind) {- if (!this.#room) {- return;- }+ //+ // Serialized: roster updates re-sync while a publish is still in flight,+ // and an overlapping call would see no publication yet and publish the+ // same capture track a second time.+ syncLocalVideo(videoTrack, screenAudioTrack, kind) {+ const run = async () => {+ if (!this.#room) {+ return;+ }- await this.#syncVideoPublication(videoTrack ?? null, kind ?? null);- await this.#syncScreenAudioPublication(- videoTrack ? (screenAudioTrack ?? null) : null- );+ await this.#syncVideoPublication(videoTrack ?? null, kind ?? null);+ await this.#syncScreenAudioPublication(+ videoTrack ? (screenAudioTrack ?? null) : null+ );+ };++ const result = this.#videoSyncQueue.then(run, run);+ this.#videoSyncQueue = result.catch(() => {});+ return result;}// Receive gating for the watching state: non-watchers unsubscribe from@@ -456,6 +471,10 @@ export default class LivekitRoomSession {source: Track.Source.Camera,simulcast: true,videoEncoding: livekitEncoding,+ videoSimulcastLayers: this.#cameraSimulcastLayers(+ track,+ tiers.camera+ ),};try {@@ -472,6 +491,25 @@ export default class LivekitRoomSession {}}+ // Without explicit layers the SDK falls back to its 180p/360p presets with+ // low bitrate caps, which is what most subscribers would end up watching.+ // Lower layers mirror the mesh ladder's lower rungs instead.+ #cameraSimulcastLayers(track, tier) {+ const settings = track.getSettings?.() ?? {};+ const width = settings.width || DEFAULT_CAMERA_DIMENSIONS.width;+ const height = settings.height || DEFAULT_CAMERA_DIMENSIONS.height;++ return cameraSimulcastLayersFor(tier).map(+ (layer) =>+ new this.#sdk.VideoPreset(+ Math.round(width / layer.scaleResolutionDownBy),+ Math.round(height / layer.scaleResolutionDownBy),+ layer.maxBitrate,+ layer.maxFramerate+ )+ );+ }+async #syncScreenAudioPublication(track) {const current = this.#screenAudioPublication;if (current && current.track?.mediaStreamTrack !== track) {@@ -542,7 +580,8 @@ export default class LivekitRoomSession {// With adaptiveStream off the SDK would leave every subscription on the// HIGH simulcast layer, regressing downlink vs the mesh ladder in big- // rooms; pick layers explicitly instead.+ // rooms; pick layers explicitly instead, stepping down where the mesh+ // ladder does.#applySubscriberQuality(publication) {const { Track, VideoQuality } = this.#sdk;@@ -551,10 +590,13 @@ export default class LivekitRoomSession {quality = VideoQuality.HIGH;} else if (publication?.source === Track.Source.Camera) {const publisherCount = this.#getVideoPublisherCount?.() ?? 0;- quality =- publisherCount > LOW_LAYER_PUBLISHER_THRESHOLD- ? VideoQuality.LOW- : VideoQuality.MEDIUM;+ if (publisherCount > LOW_LAYER_PUBLISHER_THRESHOLD) {+ quality = VideoQuality.LOW;+ } else if (publisherCount > MEDIUM_LAYER_PUBLISHER_THRESHOLD) {+ quality = VideoQuality.MEDIUM;+ } else {+ quality = VideoQuality.HIGH;+ }} else {return;}diff --git a/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.js b/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.jsindex fcef3535cf9..0b748cca5a1 100644--- a/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.js+++ b/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.js@@ -87,6 +87,15 @@ export function cameraEncodingFor(tier) {return encodingFor("camera", 1, tier);}+// Lower simulcast layers for an SFU camera publish, lowest first. Maximum has+// no ladder of its own, so it borrows the high tier's lower rungs.+export function cameraSimulcastLayersFor(tier = QUALITY_STANDARD) {+ const ladder =+ CAMERA_LADDERS[tier === QUALITY_MAXIMUM ? QUALITY_HIGH : tier] ??+ CAMERA_LADDERS[QUALITY_STANDARD];+ return [ladder[2], ladder[1]];+}+export function screenEncodingFor(tier) {return encodingFor("screen", 1, tier);}diff --git a/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.js b/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.jsindex 7d2b39b2a8c..873303b0282 100644--- a/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.js+++ b/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.js@@ -192,9 +192,18 @@ function buildFakeSdk() {},};+ class VideoPreset {+ constructor(width, height, maxBitrate, maxFramerate) {+ this.width = width;+ this.height = height;+ this.encoding = { maxBitrate, maxFramerate };+ }+ }+class FakeLivekitRoom {static instances = [];static connectErrors = [];+ static publishDelayMs = 0;options;connectCalls = [];@@ -211,6 +220,9 @@ function buildFakeSdk() {published: [],unpublishCalls: [],async publishTrack(track, publishOptions) {+ if (FakeLivekitRoom.publishDelayMs) {+ await wait(FakeLivekitRoom.publishDelayMs);+ }const publication = {source: publishOptions?.source,options: publishOptions,@@ -272,6 +284,7 @@ function buildFakeSdk() {DisconnectReason,Track,VideoQuality,+ VideoPreset,isBrowserSupported: () => true,},FakeLivekitRoom,@@ -1274,14 +1287,26 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {this.cameraTrack,"publishes the camera track");- assert.deepEqual(+ assert.propEqual(publication.options,{source: "camera",simulcast: true,videoEncoding: { maxBitrate: 1_200_000, maxFramerate: 24 },+ videoSimulcastLayers: [+ {+ width: 640,+ height: 360,+ encoding: { maxBitrate: 400_000, maxFramerate: 15 },+ },+ {+ width: 853,+ height: 480,+ encoding: { maxBitrate: 700_000, maxFramerate: 24 },+ },+ ],},- "publishes with simulcast and the camera encoding preset"+ "publishes simulcast layers from the camera encoding ladder");assert.deepEqual(this.stateRequests.at(-1),@@ -1317,6 +1342,32 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {);});+ test("a roster update during a camera publish does not publish it twice", async function (assert) {+ await this.subject.join(this.room);+ await wait(50);++ this.FakeLivekitRoom.publishDelayMs = 30;+ const toggled = this.subject.toggleCamera();+ await wait(5);+ this.rooms.emit(1, {+ type: "participants",+ participants: [+ { id: this.currentUser.id, role: "participant", is_video_on: true },+ ],+ });+ await toggled;+ await wait(80);++ const lkRoom = this.FakeLivekitRoom.instances[0];+ assert.strictEqual(+ lkRoom.localParticipant.published.filter(+ ({ source }) => source === "camera"+ ).length,+ 1,+ "publishes the camera exactly once"+ );+ });+test("toggleScreenShare publishes the screen track and content audio", async function (assert) {await this.subject.join(this.room);await wait(50);@@ -1490,8 +1541,8 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {assert.deepEqual(cameraPublication.qualityCalls,- [1],- "cameras subscribe at the MEDIUM layer in small rooms"+ [2],+ "cameras subscribe at the HIGH layer in small rooms");assert.deepEqual(screenPublication.qualityCalls,@@ -1499,11 +1550,30 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {"screenshares subscribe at the HIGH layer");- const participants = [{ id: this.currentUser.id, role: "participant" }];- for (let id = 2; id <= 8; id++) {- participants.push({ id, role: "participant", is_video_on: true });- }- this.rooms.emit(1, { type: "participants", participants });+ const participantsWithCameras = (count) => {+ const participants = [{ id: this.currentUser.id, role: "participant" }];+ for (let id = 2; id < count + 2; id++) {+ participants.push({ id, role: "participant", is_video_on: true });+ }+ return participants;+ };++ this.rooms.emit(1, {+ type: "participants",+ participants: participantsWithCameras(4),+ });+ await wait(20);++ assert.strictEqual(+ cameraPublication.qualityCalls.at(-1),+ 1,+ "more than three publishers drops cameras to the MEDIUM layer"+ );++ this.rooms.emit(1, {+ type: "participants",+ participants: participantsWithCameras(7),+ });await wait(20);assert.strictEqual(