Skip to content

A platform for community discussion. Free, open, simple.

FIX: Sharper, smoother camera video in LiveKit voice rooms (#44650)

Rafael dos Santos Silva58548dba48bd
diff --git a/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.js b/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.js
index 0824a6be078..9353fd088ea 100644
--- a/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.js
+++ b/plugins/voice/assets/javascripts/discourse/lib/voice/livekit-session.js
@@ -9,6 +9,7 @@ import voiceLog from "discourse/plugins/voice/discourse/lib/voice/logger";
// (or the module-level test override), which is what makes it unit-testable.
import {
cameraEncodingFor,
+ cameraSimulcastLayersFor,
screenEncodingFor,
voiceBitrateFor,
} from "./video-quality.js";
@@ -63,9 +64,12 @@ function wait(ms) {
// Content audio: higher Opus ceiling than the speech default so screen audio
// doesn't sound underwater.
const SCREEN_AUDIO_BITRATE = 128_000;
-// Camera subscriptions drop to the LOW simulcast layer above this many
-// active video publishers — the same threshold the mesh bitrate ladder uses.
+// Camera subscriptions step down simulcast layers at the same publisher
+// counts where the mesh ladder steps down its rungs.
+const MEDIUM_LAYER_PUBLISHER_THRESHOLD = 3;
const LOW_LAYER_PUBLISHER_THRESHOLD = 6;
+// Fallback when the capture track doesn't report its dimensions.
+const DEFAULT_CAMERA_DIMENSIONS = { width: 1280, height: 720 };
export default class LivekitRoomSession {
// Cheap pre-load check so obviously unsupported browsers fail with a
@@ -103,6 +107,7 @@ export default class LivekitRoomSession {
#closed = false;
#reconnecting = false;
#agentIdentities = new Map();
+ #videoSyncQueue = Promise.resolve();
constructor({
roomId,
@@ -227,15 +232,25 @@ export default class LivekitRoomSession {
// Brings the video and screen-audio publications in line with the local
// capture state. The mesh equivalent gates per watching peer; on the SFU
// media is published once and receive gating moves to the subscriber side.
- async syncLocalVideo(videoTrack, screenAudioTrack, kind) {
- if (!this.#room) {
- return;
- }
+ //
+ // Serialized: roster updates re-sync while a publish is still in flight,
+ // and an overlapping call would see no publication yet and publish the
+ // same capture track a second time.
+ syncLocalVideo(videoTrack, screenAudioTrack, kind) {
+ const run = async () => {
+ if (!this.#room) {
+ return;
+ }
- await this.#syncVideoPublication(videoTrack ?? null, kind ?? null);
- await this.#syncScreenAudioPublication(
- videoTrack ? (screenAudioTrack ?? null) : null
- );
+ await this.#syncVideoPublication(videoTrack ?? null, kind ?? null);
+ await this.#syncScreenAudioPublication(
+ videoTrack ? (screenAudioTrack ?? null) : null
+ );
+ };
+
+ const result = this.#videoSyncQueue.then(run, run);
+ this.#videoSyncQueue = result.catch(() => {});
+ return result;
}
// Receive gating for the watching state: non-watchers unsubscribe from
@@ -456,6 +471,10 @@ export default class LivekitRoomSession {
source: Track.Source.Camera,
simulcast: true,
videoEncoding: livekitEncoding,
+ videoSimulcastLayers: this.#cameraSimulcastLayers(
+ track,
+ tiers.camera
+ ),
};
try {
@@ -472,6 +491,25 @@ export default class LivekitRoomSession {
}
}
+ // Without explicit layers the SDK falls back to its 180p/360p presets with
+ // low bitrate caps, which is what most subscribers would end up watching.
+ // Lower layers mirror the mesh ladder's lower rungs instead.
+ #cameraSimulcastLayers(track, tier) {
+ const settings = track.getSettings?.() ?? {};
+ const width = settings.width || DEFAULT_CAMERA_DIMENSIONS.width;
+ const height = settings.height || DEFAULT_CAMERA_DIMENSIONS.height;
+
+ return cameraSimulcastLayersFor(tier).map(
+ (layer) =>
+ new this.#sdk.VideoPreset(
+ Math.round(width / layer.scaleResolutionDownBy),
+ Math.round(height / layer.scaleResolutionDownBy),
+ layer.maxBitrate,
+ layer.maxFramerate
+ )
+ );
+ }
+
async #syncScreenAudioPublication(track) {
const current = this.#screenAudioPublication;
if (current && current.track?.mediaStreamTrack !== track) {
@@ -542,7 +580,8 @@ export default class LivekitRoomSession {
// With adaptiveStream off the SDK would leave every subscription on the
// HIGH simulcast layer, regressing downlink vs the mesh ladder in big
- // rooms; pick layers explicitly instead.
+ // rooms; pick layers explicitly instead, stepping down where the mesh
+ // ladder does.
#applySubscriberQuality(publication) {
const { Track, VideoQuality } = this.#sdk;
@@ -551,10 +590,13 @@ export default class LivekitRoomSession {
quality = VideoQuality.HIGH;
} else if (publication?.source === Track.Source.Camera) {
const publisherCount = this.#getVideoPublisherCount?.() ?? 0;
- quality =
- publisherCount > LOW_LAYER_PUBLISHER_THRESHOLD
- ? VideoQuality.LOW
- : VideoQuality.MEDIUM;
+ if (publisherCount > LOW_LAYER_PUBLISHER_THRESHOLD) {
+ quality = VideoQuality.LOW;
+ } else if (publisherCount > MEDIUM_LAYER_PUBLISHER_THRESHOLD) {
+ quality = VideoQuality.MEDIUM;
+ } else {
+ quality = VideoQuality.HIGH;
+ }
} else {
return;
}
diff --git a/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.js b/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.js
index fcef3535cf9..0b748cca5a1 100644
--- a/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.js
+++ b/plugins/voice/assets/javascripts/discourse/lib/voice/video-quality.js
@@ -87,6 +87,15 @@ export function cameraEncodingFor(tier) {
return encodingFor("camera", 1, tier);
}
+// Lower simulcast layers for an SFU camera publish, lowest first. Maximum has
+// no ladder of its own, so it borrows the high tier's lower rungs.
+export function cameraSimulcastLayersFor(tier = QUALITY_STANDARD) {
+ const ladder =
+ CAMERA_LADDERS[tier === QUALITY_MAXIMUM ? QUALITY_HIGH : tier] ??
+ CAMERA_LADDERS[QUALITY_STANDARD];
+ return [ladder[2], ladder[1]];
+}
+
export function screenEncodingFor(tier) {
return encodingFor("screen", 1, tier);
}
diff --git a/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.js b/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.js
index 7d2b39b2a8c..873303b0282 100644
--- a/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.js
+++ b/plugins/voice/test/javascripts/unit/services/voice-webrtc-livekit-test.js
@@ -192,9 +192,18 @@ function buildFakeSdk() {
},
};
+ class VideoPreset {
+ constructor(width, height, maxBitrate, maxFramerate) {
+ this.width = width;
+ this.height = height;
+ this.encoding = { maxBitrate, maxFramerate };
+ }
+ }
+
class FakeLivekitRoom {
static instances = [];
static connectErrors = [];
+ static publishDelayMs = 0;
options;
connectCalls = [];
@@ -211,6 +220,9 @@ function buildFakeSdk() {
published: [],
unpublishCalls: [],
async publishTrack(track, publishOptions) {
+ if (FakeLivekitRoom.publishDelayMs) {
+ await wait(FakeLivekitRoom.publishDelayMs);
+ }
const publication = {
source: publishOptions?.source,
options: publishOptions,
@@ -272,6 +284,7 @@ function buildFakeSdk() {
DisconnectReason,
Track,
VideoQuality,
+ VideoPreset,
isBrowserSupported: () => true,
},
FakeLivekitRoom,
@@ -1274,14 +1287,26 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {
this.cameraTrack,
"publishes the camera track"
);
- assert.deepEqual(
+ assert.propEqual(
publication.options,
{
source: "camera",
simulcast: true,
videoEncoding: { maxBitrate: 1_200_000, maxFramerate: 24 },
+ videoSimulcastLayers: [
+ {
+ width: 640,
+ height: 360,
+ encoding: { maxBitrate: 400_000, maxFramerate: 15 },
+ },
+ {
+ width: 853,
+ height: 480,
+ encoding: { maxBitrate: 700_000, maxFramerate: 24 },
+ },
+ ],
},
- "publishes with simulcast and the camera encoding preset"
+ "publishes simulcast layers from the camera encoding ladder"
);
assert.deepEqual(
this.stateRequests.at(-1),
@@ -1317,6 +1342,32 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {
);
});
+ test("a roster update during a camera publish does not publish it twice", async function (assert) {
+ await this.subject.join(this.room);
+ await wait(50);
+
+ this.FakeLivekitRoom.publishDelayMs = 30;
+ const toggled = this.subject.toggleCamera();
+ await wait(5);
+ this.rooms.emit(1, {
+ type: "participants",
+ participants: [
+ { id: this.currentUser.id, role: "participant", is_video_on: true },
+ ],
+ });
+ await toggled;
+ await wait(80);
+
+ const lkRoom = this.FakeLivekitRoom.instances[0];
+ assert.strictEqual(
+ lkRoom.localParticipant.published.filter(
+ ({ source }) => source === "camera"
+ ).length,
+ 1,
+ "publishes the camera exactly once"
+ );
+ });
+
test("toggleScreenShare publishes the screen track and content audio", async function (assert) {
await this.subject.join(this.room);
await wait(50);
@@ -1490,8 +1541,8 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {
assert.deepEqual(
cameraPublication.qualityCalls,
- [1],
- "cameras subscribe at the MEDIUM layer in small rooms"
+ [2],
+ "cameras subscribe at the HIGH layer in small rooms"
);
assert.deepEqual(
screenPublication.qualityCalls,
@@ -1499,11 +1550,30 @@ module("Voice | Unit | Service | voice-webrtc-livekit", function (hooks) {
"screenshares subscribe at the HIGH layer"
);
- const participants = [{ id: this.currentUser.id, role: "participant" }];
- for (let id = 2; id <= 8; id++) {
- participants.push({ id, role: "participant", is_video_on: true });
- }
- this.rooms.emit(1, { type: "participants", participants });
+ const participantsWithCameras = (count) => {
+ const participants = [{ id: this.currentUser.id, role: "participant" }];
+ for (let id = 2; id < count + 2; id++) {
+ participants.push({ id, role: "participant", is_video_on: true });
+ }
+ return participants;
+ };
+
+ this.rooms.emit(1, {
+ type: "participants",
+ participants: participantsWithCameras(4),
+ });
+ await wait(20);
+
+ assert.strictEqual(
+ cameraPublication.qualityCalls.at(-1),
+ 1,
+ "more than three publishers drops cameras to the MEDIUM layer"
+ );
+
+ this.rooms.emit(1, {
+ type: "participants",
+ participants: participantsWithCameras(7),
+ });
await wait(20);
assert.strictEqual(