Skip to content

A platform for community discussion. Free, open, simple.

FIX: Keep mesh Live calls alive through a server outage (#44846)

Rafael dos Santos Silva038312772449
diff --git a/plugins/live/app/controllers/live/rooms_controller.rb b/plugins/live/app/controllers/live/rooms_controller.rb
index b7895522b33..e4f9c33c294 100644
--- a/plugins/live/app/controllers/live/rooms_controller.rb
+++ b/plugins/live/app/controllers/live/rooms_controller.rb
@@ -375,9 +375,7 @@ module Live
session = close_session_for(@room.id, current_user.id)
Live::ParticipantTracker.mark_left(@room.id, current_user.id)
Live::ParticipantTracker.remove(@room.id, current_user.id)
- if Live::ParticipantTracker.human_user_ids(@room.id).empty?
- Live::AgentManager.evict_agents_in_room!(@room)
- end
+ Live::AgentManager.evict_agents_in_room!(@room) if Live::ParticipantTracker.vacant?(@room.id)
Live::UserStatusManager.clear_live_status(current_user)
Live::RoomBroadcaster.publish_participants(@room)
Live::BadgeGranterHooks.on_leave(current_user, session)
@@ -393,7 +391,18 @@ module Live
ensure_participant_session!
- Live::ParticipantTracker.add(@room.id, current_user.id)
+ # A session can outlive presence (see participant_session_ttl), so a
+ # beat resuming after an outage re-claims a slot like any admission.
+ admission =
+ Live::ParticipantTracker.add_within_capacity(
+ @room.id,
+ current_user.id,
+ @room.effective_max_participants,
+ )
+ if admission == :full
+ Live::ParticipantTracker.revoke_participant_session(@room.id, current_user.id)
+ return render_json_error(I18n.t("live.errors.room_full"), status: 410)
+ end
Live::ParticipantTracker.refresh_participant_session(@room.id, current_user.id)
metadata = Live::ParticipantTracker.get_metadata(@room.id, current_user.id)
diff --git a/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rb b/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rb
index 9bb3ebd9b25..743481ba75d 100644
--- a/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rb
+++ b/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rb
@@ -9,6 +9,7 @@ module Jobs
return unless SiteSetting.live_enabled && SiteSetting.live_analytics_enabled
participant_cache = {}
+ transport_cache = {}
::Live::Session.orphaned.find_each do |session|
participant_ids =
@@ -16,6 +17,19 @@ module Jobs
session.room_id,
)
next if participant_ids.include?(session.user_id)
+ # Lapsed presence with a resumable participant session is most likely
+ # a mesh client cut off by an outage, not one that left. LiveKit
+ # reports departures itself and resumes through its own token ladder.
+ transport =
+ transport_cache.fetch(session.room_id) do
+ transport_cache[session.room_id] = ::Live::ParticipantTracker.pinned_transport(
+ session.room_id,
+ )
+ end
+ if transport != "livekit" &&
+ ::Live::ParticipantTracker.participant_session?(session.room_id, session.user_id)
+ next
+ end
left_at =
::Live::ParticipantTracker.last_heartbeat_at(session.room_id, session.user_id) ||
@@ -25,7 +39,7 @@ module Jobs
::Live::ParticipantTracker.remove(session.room_id, session.user_id)
room = ::Live::Room.find_by(id: session.room_id)
- if room && ::Live::ParticipantTracker.human_user_ids(room.id).empty?
+ if room && ::Live::ParticipantTracker.vacant?(room.id)
::Live::AgentManager.evict_agents_in_room!(room)
end
diff --git a/plugins/live/app/jobs/scheduled/publish_room_participants.rb b/plugins/live/app/jobs/scheduled/publish_room_participants.rb
index af690edcbf6..01fce59f734 100644
--- a/plugins/live/app/jobs/scheduled/publish_room_participants.rb
+++ b/plugins/live/app/jobs/scheduled/publish_room_participants.rb
@@ -35,7 +35,7 @@ module Jobs
# Backstop for the pin-clear on last leave: a room that emptied
# without one (crashed clients, missed leave) must not hold its
# transport for the next call.
- if ::Live::ParticipantTracker.human_user_ids(room.id).empty?
+ if ::Live::ParticipantTracker.vacant?(room.id)
# Agent cleanup survives expired pins and retries provider failures.
::Live::AgentManager.evict_agents_in_room!(room)
else
diff --git a/plugins/live/app/services/live/ephemeral_room_manager.rb b/plugins/live/app/services/live/ephemeral_room_manager.rb
index b26c80d7d1f..38fc41da5f1 100644
--- a/plugins/live/app/services/live/ephemeral_room_manager.rb
+++ b/plugins/live/app/services/live/ephemeral_room_manager.rb
@@ -63,7 +63,7 @@ module Live
ttl = SiteSetting.live_ephemeral_room_ttl_minutes.minutes
Live::Room.ephemeral.find_each do |room|
- if Live::ParticipantTracker.human_user_ids(room.id).any?
+ if !Live::ParticipantTracker.vacant?(room.id)
room.update_column(:last_occupied_at, Time.current)
elsif (room.last_occupied_at || room.created_at) < ttl.ago
destroy!(room)
diff --git a/plugins/live/app/services/live/participant_tracker.rb b/plugins/live/app/services/live/participant_tracker.rb
index b2eedabb3ac..41c7cf10bc4 100644
--- a/plugins/live/app/services/live/participant_tracker.rb
+++ b/plugins/live/app/services/live/participant_tracker.rb
@@ -12,6 +12,10 @@ module Live
# Must outlive one client heartbeat interval (10s) plus request latency,
# so a beat already in flight when the user leaves can't resurrect them.
LEFT_TOMBSTONE_TTL = 15
+ # How long a client cut off from the server (a deploy, a network blip,
+ # sleep) can stay silent and still resume its call. Mesh media keeps
+ # flowing peer-to-peer meanwhile; leave/kick/takeover revoke explicitly.
+ PARTICIPANT_SESSION_RESUME_WINDOW = 10.minutes.to_i
# Purge-expired, existence check, capacity check, and add in one script:
# a separate ZCARD followed by ZADD would let concurrent joiners race the
@@ -175,10 +179,35 @@ module Live
redis.del(participant_session_key(room_id, user_id))
end
- # 2x the presence TTL (like the transport pin) so a briefly-lapsed
- # heartbeat can still self-heal presence before the session dies.
+ # Outlives presence so a heartbeat resuming after an outage can
+ # self-heal presence instead of being expelled from a call whose media
+ # never stopped.
def participant_session_ttl
- SiteSetting.live_participant_ttl_seconds.to_i * 2
+ [PARTICIPANT_SESSION_RESUME_WINDOW, SiteSetting.live_participant_ttl_seconds.to_i * 2].max
+ end
+
+ # No humans present, and none who could still resume a mesh call. LiveKit
+ # reports departures itself, so its rooms ignore resuming members.
+ def vacant?(room_id)
+ return false if human_user_ids(room_id).any?
+
+ pinned_transport(room_id) == "livekit" || resuming_user_ids(room_id).empty?
+ end
+
+ # Members whose presence lapsed while their participant session is still
+ # resumable: likely cut off from the server rather than gone, so clients
+ # keep their existing media connections instead of tearing them down.
+ def resuming_user_ids(room_id)
+ present = user_ids(room_id).to_set
+ candidates = redis.hkeys(metadata_key(room_id)).map(&:to_i)
+ candidates = candidates.select { |id| id.positive? && present.exclude?(id) }
+ return [] if candidates.empty?
+
+ exists =
+ redis.pipelined do |pipeline|
+ candidates.each { |id| pipeline.exists(participant_session_key(room_id, id)) }
+ end
+ candidates.zip(exists).filter_map { |id, count| id if count.to_i > 0 }
end
# Short-lived tombstone for a deliberate departure (leave/kick), letting
@@ -361,7 +390,7 @@ module Live
user_ids(room_id).sort.map do |id|
[id, (metadata[id] || {}).except(:last_heartbeat_at).sort.to_h]
end
- Digest::MD5.hexdigest(payload.to_json)
+ Digest::MD5.hexdigest([payload, resuming_user_ids(room_id).sort].to_json)
end
# Atomically store the new fingerprint and return the previous one, so a
@@ -381,9 +410,10 @@ module Live
# The transport pin ("mesh" | "livekit") holds a room instance to one
# transport for its whole life: joiners after the first ignore current
# settings, so a call is never split across transports. `SET NX` makes
- # the first join win under a race; the short TTL (refreshed by every
- # join/heartbeat) lets a crashed room self-heal instead of holding a
- # stale transport.
+ # the first join win under a race. The TTL (refreshed by every
+ # join/heartbeat) matches the participant session so members resuming
+ # after an outage find their transport still pinned; vacant rooms are
+ # unpinned explicitly by the minute sweep.
def pin_transport!(room_id, transport)
redis.set(transport_key(room_id), transport, nx: true, ex: transport_pin_ttl)
refresh_transport_pin(room_id)
@@ -421,7 +451,7 @@ module Live
end
def transport_pin_ttl
- SiteSetting.live_participant_ttl_seconds.to_i * 2
+ participant_session_ttl
end
def touch_recently_active(room_id)
diff --git a/plugins/live/app/services/live/room_broadcaster.rb b/plugins/live/app/services/live/room_broadcaster.rb
index 3fd9640cddf..bc08b85c9ca 100644
--- a/plugins/live/app/services/live/room_broadcaster.rb
+++ b/plugins/live/app/services/live/room_broadcaster.rb
@@ -63,6 +63,7 @@ module Live
guardian: Guardian.new(nil),
metadata: Live::ParticipantTracker.get_all_metadata(room.id),
),
+ resuming_ids: Live::ParticipantTracker.resuming_user_ids(room.id),
}
MessageBus.publish(Live.room_channel(room.id), payload, **room_message_bus_targets)
diff --git a/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.js b/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.js
index 27d7cade7c8..452f9a9e5dc 100644
--- a/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.js
+++ b/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.js
@@ -103,6 +103,9 @@ export default class RosterHandler {
const participantIds = new Set(
participants.map((participant) => Number(participant.id))
);
+ // Members the server lost contact with but who can still resume: their
+ // mesh media doesn't depend on the server, so it is left running.
+ const resumingIds = new Set((payload.resuming_ids || []).map(Number));
const currentUserId = this.#getCurrentUserId();
liveLog.info(
@@ -128,7 +131,10 @@ export default class RosterHandler {
peers?.forEach((pc, remoteUserId) => {
if (!participantIds.has(remoteUserId)) {
- if (this.#presencePending.has(roomId, remoteUserId)) {
+ if (
+ this.#presencePending.has(roomId, remoteUserId) ||
+ resumingIds.has(remoteUserId)
+ ) {
return;
}
hasPeerLeft = true;
@@ -189,7 +195,10 @@ export default class RosterHandler {
}
this.#syncRemoteMediaTracks(roomId, participants);
- this.#dropDisallowedStreams(roomId, participants, { isStage });
+ this.#dropDisallowedStreams(roomId, participants, {
+ isStage,
+ resumingIds,
+ });
this.#dropUnentitledMedia(roomId, participants);
if (!this.#isMeshRoom(roomId)) {
@@ -279,7 +288,7 @@ export default class RosterHandler {
// even when the demotion arrives as a roster refresh rather than a
// role_change message. The receive-side track policy stops new tracks;
// this drops the ones already registered.
- #dropDisallowedStreams(roomId, participants, { isStage }) {
+ #dropDisallowedStreams(roomId, participants, { isStage, resumingIds }) {
if (!isStage || !this.#isMeshRoom(roomId)) {
return;
}
@@ -294,7 +303,7 @@ export default class RosterHandler {
);
for (const userId of this.#getRemoteUserIds(roomId)) {
- if (!allowedToPublish.has(userId)) {
+ if (!allowedToPublish.has(userId) && !resumingIds.has(userId)) {
this.#removeRemoteStream(roomId, userId);
}
}
diff --git a/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rb b/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rb
index 9a1b964ec44..f05e5735daf 100644
--- a/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rb
+++ b/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rb
@@ -61,6 +61,18 @@ RSpec.describe Jobs::Live::CloseOrphanedSessions do
expect(session.left_at).to be_nil
end
+ it "keeps sessions open while a lapsed participant can still resume" do
+ session =
+ Fabricate(:live_session, user: user1, room: room, joined_at: 50.minutes.ago, left_at: nil)
+
+ set_stale_participant(room, user1, last_heartbeat: 2.minutes.ago)
+ Live::ParticipantTracker.create_participant_session!(room.id, user1.id)
+
+ job.execute({})
+
+ expect(session.reload.left_at).to be_nil
+ end
+
it "cleans up stale Redis entries after closing" do
Fabricate(:live_session, user: user1, room: room, joined_at: 50.minutes.ago, left_at: nil)
diff --git a/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rb b/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rb
index c0db74bbd8e..5482fb46b67 100644
--- a/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rb
+++ b/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rb
@@ -27,6 +27,24 @@ RSpec.describe Jobs::PublishRoomParticipants do
)
end
+ it "keeps a mesh room pinned while a member can still resume" do
+ Live::ParticipantTracker.pin_transport!(room.id, "mesh")
+ Live::ParticipantTracker.add(room.id, user1.id)
+ Live::ParticipantTracker.update_metadata(room.id, user1.id, { role: "participant" })
+ Live::ParticipantTracker.create_participant_session!(room.id, user1.id)
+ key = "#{Live::ParticipantTracker::KEY_NAMESPACE}:#{room.id}:participants"
+ Discourse.redis.zadd(key, 1.hour.ago.to_f, user1.id)
+
+ job.execute({})
+
+ expect(Live::ParticipantTracker.pinned_transport(room.id)).to eq("mesh")
+
+ Live::ParticipantTracker.revoke_participant_session(room.id, user1.id)
+ job.execute({})
+
+ expect(Live::ParticipantTracker.pinned_transport(room.id)).to be_nil
+ end
+
it "reflects TTL-expired participants in broadcast" do
Live::ParticipantTracker.add(room.id, user1.id)
Live::ParticipantTracker.add(room.id, user2.id)
diff --git a/plugins/live/spec/requests/live/rooms_controller_spec.rb b/plugins/live/spec/requests/live/rooms_controller_spec.rb
index 577a6fc45a6..e3bb7d05648 100644
--- a/plugins/live/spec/requests/live/rooms_controller_spec.rb
+++ b/plugins/live/spec/requests/live/rooms_controller_spec.rb
@@ -684,6 +684,45 @@ RSpec.describe Live::RoomsController do
expect(Live::ParticipantTracker.user_ids(room.id)).to include(user.id)
end
+ it "returns 410 and revokes the session when the room filled up while presence was lapsed" do
+ room.update!(max_participants: 2)
+ sign_in(user)
+ session_id = establish_presence!(room, user)
+ key = "#{Live::ParticipantTracker::KEY_NAMESPACE}:#{room.id}:participants"
+ Discourse.redis.zadd(key, 1.hour.ago.to_f, user.id)
+ establish_presence!(room, staff)
+ establish_presence!(room, other_participant)
+
+ post "/live/rooms/#{room.id}/heartbeat.json", params: { participant_session_id: session_id }
+
+ expect(response.status).to eq(410)
+ expect(Live::ParticipantTracker.user_ids(room.id)).not_to include(user.id)
+ expect(Live::ParticipantTracker.participant_session?(room.id, user.id)).to eq(false)
+ end
+
+ it "flags lapsed members with a resumable session so peers keep their media" do
+ sign_in(user)
+ session_id = establish_presence!(room, user)
+ establish_presence!(room, other_participant)
+ Live::ParticipantTracker.update_metadata(
+ room.id,
+ other_participant.id,
+ { role: "participant" },
+ )
+ key = "#{Live::ParticipantTracker::KEY_NAMESPACE}:#{room.id}:participants"
+ Discourse.redis.zadd(key, 1.hour.ago.to_f, other_participant.id)
+
+ published = []
+ allow(MessageBus).to receive(:publish) { |channel, data, _opts| published << data }
+
+ post "/live/rooms/#{room.id}/heartbeat.json", params: { participant_session_id: session_id }
+
+ expect(response.status).to eq(204)
+ participants_message = published.find { |data| data[:type] == "participants" }
+ expect(participants_message[:participants].map { |p| p[:id] }).to contain_exactly(user.id)
+ expect(participants_message[:resuming_ids]).to contain_exactly(other_participant.id)
+ end
+
it "rejects a heartbeat without a participant session and creates no presence" do
sign_in(user)
diff --git a/plugins/live/spec/services/live/participant_tracker_spec.rb b/plugins/live/spec/services/live/participant_tracker_spec.rb
index ab2b49142f9..5e45e2a35a1 100644
--- a/plugins/live/spec/services/live/participant_tracker_spec.rb
+++ b/plugins/live/spec/services/live/participant_tracker_spec.rb
@@ -144,6 +144,69 @@ RSpec.describe Live::ParticipantTracker do
expect(described_class.valid_participant_session?(room.id, user1.id, session_id)).to eq(false)
end
+
+ it "outlives presence long enough to resume after a server outage" do
+ expect(described_class.participant_session_ttl).to be >= 10.minutes.to_i
+ expect(described_class.participant_session_ttl).to be >
+ SiteSetting.live_participant_ttl_seconds
+ end
+ end
+
+ describe ".vacant?" do
+ def resuming!(user)
+ described_class.update_metadata(room.id, user.id, { role: "participant" })
+ described_class.create_participant_session!(room.id, user.id)
+ end
+
+ it "is false while a mesh member can still resume" do
+ resuming!(user1)
+
+ expect(described_class.vacant?(room.id)).to eq(false)
+ end
+
+ it "ignores resuming members in a LiveKit room" do
+ described_class.pin_transport!(room.id, "livekit")
+ resuming!(user1)
+
+ expect(described_class.vacant?(room.id)).to eq(true)
+ end
+
+ it "is false while a human is present" do
+ described_class.add(room.id, user1.id)
+
+ expect(described_class.vacant?(room.id)).to eq(false)
+ end
+ end
+
+ describe ".resuming_user_ids" do
+ def lapse_presence(user)
+ key = "#{described_class::KEY_NAMESPACE}:#{room.id}:participants"
+ Discourse.redis.zadd(key, 1.hour.ago.to_f, user.id)
+ end
+
+ it "lists members whose presence lapsed while their session is resumable" do
+ described_class.add(room.id, user1.id)
+ described_class.update_metadata(room.id, user1.id, { role: "participant" })
+ described_class.create_participant_session!(room.id, user1.id)
+ lapse_presence(user1)
+
+ expect(described_class.resuming_user_ids(room.id)).to contain_exactly(user1.id)
+ end
+
+ it "excludes present members, members without a session, and members who left" do
+ [user1, user2].each do |user|
+ described_class.add(room.id, user.id)
+ described_class.update_metadata(room.id, user.id, { role: "participant" })
+ described_class.create_participant_session!(room.id, user.id)
+ end
+ user3 = Fabricate(:user)
+ described_class.update_metadata(room.id, user3.id, { role: "participant" })
+ lapse_presence(user3)
+
+ described_class.remove(room.id, user2.id)
+
+ expect(described_class.resuming_user_ids(room.id)).to be_empty
+ end
end
describe ".raise_hand" do
@@ -189,11 +252,12 @@ RSpec.describe Live::ParticipantTracker do
expect(described_class.pinned_transport(room.id)).to eq("livekit")
end
- it "expires on its own so a crashed room self-heals" do
+ it "expires with the participant session so resuming members keep their transport" do
described_class.pin_transport!(room.id, "livekit")
ttl = Discourse.redis.ttl("#{described_class::KEY_NAMESPACE}:#{room.id}:transport")
- expect(ttl).to be_between(1, SiteSetting.live_participant_ttl_seconds * 2)
+ expect(ttl).to be > SiteSetting.live_participant_ttl_seconds * 2
+ expect(ttl).to be <= described_class.participant_session_ttl
end
end
diff --git a/plugins/live/test/javascripts/unit/lib/roster-handler-test.js b/plugins/live/test/javascripts/unit/lib/roster-handler-test.js
new file mode 100644
index 00000000000..6c83af4d45b
--- /dev/null
+++ b/plugins/live/test/javascripts/unit/lib/roster-handler-test.js
@@ -0,0 +1,55 @@
+import { module, test } from "qunit";
+import RosterHandler from "discourse/plugins/live/discourse/lib/live/roster-handler";
+
+const ROOM_ID = 1;
+const CURRENT_USER_ID = 10;
+
+function buildHandler({ peerIds }) {
+ const peers = new Map(peerIds.map((id) => [id, {}]));
+ const destroyed = [];
+
+ const handler = new RosterHandler({
+ peerManager: {
+ getRoomPeers: () => peers,
+ has: (_roomId, userId) => peers.has(userId),
+ destroy: (_roomId, userId) => {
+ destroyed.push(userId);
+ peers.delete(userId);
+ },
+ remoteVideoTrack: () => null,
+ },
+ presencePending: { has: () => false, clear() {} },
+ getCurrentUserId: () => CURRENT_USER_ID,
+ getChatSound: () => null,
+ getRoom: () => ({ id: ROOM_ID, room_type: "standard" }),
+ isActiveRoom: () => false,
+ isConnectingRoom: () => false,
+ isMeshRoom: () => true,
+ getLocalVideoKind: () => null,
+ });
+
+ return { handler, destroyed };
+}
+
+module("Live | Unit | Lib | roster-handler", function () {
+ test("keeps mesh peers the server marks as resuming", async function (assert) {
+ const { handler, destroyed } = buildHandler({ peerIds: [20, 30] });
+
+ await handler.handleParticipants(ROOM_ID, {
+ participants: [{ id: CURRENT_USER_ID }],
+ resuming_ids: [20],
+ });
+
+ assert.deepEqual(destroyed, [30], "only the departed peer is destroyed");
+ });
+
+ test("destroys peers absent from the roster when nothing is resuming", async function (assert) {
+ const { handler, destroyed } = buildHandler({ peerIds: [20] });
+
+ await handler.handleParticipants(ROOM_ID, {
+ participants: [{ id: CURRENT_USER_ID }],
+ });
+
+ assert.deepEqual(destroyed, [20]);
+ });
+});