A platform for community discussion. Free, open, simple.
FIX: Keep mesh Live calls alive through a server outage (#44846)
Rafael dos Santos Silva038312772449
diff --git a/plugins/live/app/controllers/live/rooms_controller.rb b/plugins/live/app/controllers/live/rooms_controller.rbindex b7895522b33..e4f9c33c294 100644--- a/plugins/live/app/controllers/live/rooms_controller.rb+++ b/plugins/live/app/controllers/live/rooms_controller.rb@@ -375,9 +375,7 @@ module Livesession = close_session_for(@room.id, current_user.id)Live::ParticipantTracker.mark_left(@room.id, current_user.id)Live::ParticipantTracker.remove(@room.id, current_user.id)- if Live::ParticipantTracker.human_user_ids(@room.id).empty?- Live::AgentManager.evict_agents_in_room!(@room)- end+ Live::AgentManager.evict_agents_in_room!(@room) if Live::ParticipantTracker.vacant?(@room.id)Live::UserStatusManager.clear_live_status(current_user)Live::RoomBroadcaster.publish_participants(@room)Live::BadgeGranterHooks.on_leave(current_user, session)@@ -393,7 +391,18 @@ module Liveensure_participant_session!- Live::ParticipantTracker.add(@room.id, current_user.id)+ # A session can outlive presence (see participant_session_ttl), so a+ # beat resuming after an outage re-claims a slot like any admission.+ admission =+ Live::ParticipantTracker.add_within_capacity(+ @room.id,+ current_user.id,+ @room.effective_max_participants,+ )+ if admission == :full+ Live::ParticipantTracker.revoke_participant_session(@room.id, current_user.id)+ return render_json_error(I18n.t("live.errors.room_full"), status: 410)+ endLive::ParticipantTracker.refresh_participant_session(@room.id, current_user.id)metadata = Live::ParticipantTracker.get_metadata(@room.id, current_user.id)diff --git a/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rb b/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rbindex 9bb3ebd9b25..743481ba75d 100644--- a/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rb+++ b/plugins/live/app/jobs/scheduled/live/close_orphaned_sessions.rb@@ -9,6 +9,7 @@ module Jobsreturn unless SiteSetting.live_enabled && SiteSetting.live_analytics_enabledparticipant_cache = {}+ transport_cache = {}::Live::Session.orphaned.find_each do |session|participant_ids =@@ -16,6 +17,19 @@ module Jobssession.room_id,)next if participant_ids.include?(session.user_id)+ # Lapsed presence with a resumable participant session is most likely+ # a mesh client cut off by an outage, not one that left. LiveKit+ # reports departures itself and resumes through its own token ladder.+ transport =+ transport_cache.fetch(session.room_id) do+ transport_cache[session.room_id] = ::Live::ParticipantTracker.pinned_transport(+ session.room_id,+ )+ end+ if transport != "livekit" &&+ ::Live::ParticipantTracker.participant_session?(session.room_id, session.user_id)+ next+ endleft_at =::Live::ParticipantTracker.last_heartbeat_at(session.room_id, session.user_id) ||@@ -25,7 +39,7 @@ module Jobs::Live::ParticipantTracker.remove(session.room_id, session.user_id)room = ::Live::Room.find_by(id: session.room_id)- if room && ::Live::ParticipantTracker.human_user_ids(room.id).empty?+ if room && ::Live::ParticipantTracker.vacant?(room.id)::Live::AgentManager.evict_agents_in_room!(room)enddiff --git a/plugins/live/app/jobs/scheduled/publish_room_participants.rb b/plugins/live/app/jobs/scheduled/publish_room_participants.rbindex af690edcbf6..01fce59f734 100644--- a/plugins/live/app/jobs/scheduled/publish_room_participants.rb+++ b/plugins/live/app/jobs/scheduled/publish_room_participants.rb@@ -35,7 +35,7 @@ module Jobs# Backstop for the pin-clear on last leave: a room that emptied# without one (crashed clients, missed leave) must not hold its# transport for the next call.- if ::Live::ParticipantTracker.human_user_ids(room.id).empty?+ if ::Live::ParticipantTracker.vacant?(room.id)# Agent cleanup survives expired pins and retries provider failures.::Live::AgentManager.evict_agents_in_room!(room)elsediff --git a/plugins/live/app/services/live/ephemeral_room_manager.rb b/plugins/live/app/services/live/ephemeral_room_manager.rbindex b26c80d7d1f..38fc41da5f1 100644--- a/plugins/live/app/services/live/ephemeral_room_manager.rb+++ b/plugins/live/app/services/live/ephemeral_room_manager.rb@@ -63,7 +63,7 @@ module Livettl = SiteSetting.live_ephemeral_room_ttl_minutes.minutesLive::Room.ephemeral.find_each do |room|- if Live::ParticipantTracker.human_user_ids(room.id).any?+ if !Live::ParticipantTracker.vacant?(room.id)room.update_column(:last_occupied_at, Time.current)elsif (room.last_occupied_at || room.created_at) < ttl.agodestroy!(room)diff --git a/plugins/live/app/services/live/participant_tracker.rb b/plugins/live/app/services/live/participant_tracker.rbindex b2eedabb3ac..41c7cf10bc4 100644--- a/plugins/live/app/services/live/participant_tracker.rb+++ b/plugins/live/app/services/live/participant_tracker.rb@@ -12,6 +12,10 @@ module Live# Must outlive one client heartbeat interval (10s) plus request latency,# so a beat already in flight when the user leaves can't resurrect them.LEFT_TOMBSTONE_TTL = 15+ # How long a client cut off from the server (a deploy, a network blip,+ # sleep) can stay silent and still resume its call. Mesh media keeps+ # flowing peer-to-peer meanwhile; leave/kick/takeover revoke explicitly.+ PARTICIPANT_SESSION_RESUME_WINDOW = 10.minutes.to_i# Purge-expired, existence check, capacity check, and add in one script:# a separate ZCARD followed by ZADD would let concurrent joiners race the@@ -175,10 +179,35 @@ module Liveredis.del(participant_session_key(room_id, user_id))end- # 2x the presence TTL (like the transport pin) so a briefly-lapsed- # heartbeat can still self-heal presence before the session dies.+ # Outlives presence so a heartbeat resuming after an outage can+ # self-heal presence instead of being expelled from a call whose media+ # never stopped.def participant_session_ttl- SiteSetting.live_participant_ttl_seconds.to_i * 2+ [PARTICIPANT_SESSION_RESUME_WINDOW, SiteSetting.live_participant_ttl_seconds.to_i * 2].max+ end++ # No humans present, and none who could still resume a mesh call. LiveKit+ # reports departures itself, so its rooms ignore resuming members.+ def vacant?(room_id)+ return false if human_user_ids(room_id).any?++ pinned_transport(room_id) == "livekit" || resuming_user_ids(room_id).empty?+ end++ # Members whose presence lapsed while their participant session is still+ # resumable: likely cut off from the server rather than gone, so clients+ # keep their existing media connections instead of tearing them down.+ def resuming_user_ids(room_id)+ present = user_ids(room_id).to_set+ candidates = redis.hkeys(metadata_key(room_id)).map(&:to_i)+ candidates = candidates.select { |id| id.positive? && present.exclude?(id) }+ return [] if candidates.empty?++ exists =+ redis.pipelined do |pipeline|+ candidates.each { |id| pipeline.exists(participant_session_key(room_id, id)) }+ end+ candidates.zip(exists).filter_map { |id, count| id if count.to_i > 0 }end# Short-lived tombstone for a deliberate departure (leave/kick), letting@@ -361,7 +390,7 @@ module Liveuser_ids(room_id).sort.map do |id|[id, (metadata[id] || {}).except(:last_heartbeat_at).sort.to_h]end- Digest::MD5.hexdigest(payload.to_json)+ Digest::MD5.hexdigest([payload, resuming_user_ids(room_id).sort].to_json)end# Atomically store the new fingerprint and return the previous one, so a@@ -381,9 +410,10 @@ module Live# The transport pin ("mesh" | "livekit") holds a room instance to one# transport for its whole life: joiners after the first ignore current# settings, so a call is never split across transports. `SET NX` makes- # the first join win under a race; the short TTL (refreshed by every- # join/heartbeat) lets a crashed room self-heal instead of holding a- # stale transport.+ # the first join win under a race. The TTL (refreshed by every+ # join/heartbeat) matches the participant session so members resuming+ # after an outage find their transport still pinned; vacant rooms are+ # unpinned explicitly by the minute sweep.def pin_transport!(room_id, transport)redis.set(transport_key(room_id), transport, nx: true, ex: transport_pin_ttl)refresh_transport_pin(room_id)@@ -421,7 +451,7 @@ module Liveenddef transport_pin_ttl- SiteSetting.live_participant_ttl_seconds.to_i * 2+ participant_session_ttlenddef touch_recently_active(room_id)diff --git a/plugins/live/app/services/live/room_broadcaster.rb b/plugins/live/app/services/live/room_broadcaster.rbindex 3fd9640cddf..bc08b85c9ca 100644--- a/plugins/live/app/services/live/room_broadcaster.rb+++ b/plugins/live/app/services/live/room_broadcaster.rb@@ -63,6 +63,7 @@ module Liveguardian: Guardian.new(nil),metadata: Live::ParticipantTracker.get_all_metadata(room.id),),+ resuming_ids: Live::ParticipantTracker.resuming_user_ids(room.id),}MessageBus.publish(Live.room_channel(room.id), payload, **room_message_bus_targets)diff --git a/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.js b/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.jsindex 27d7cade7c8..452f9a9e5dc 100644--- a/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.js+++ b/plugins/live/assets/javascripts/discourse/lib/live/roster-handler.js@@ -103,6 +103,9 @@ export default class RosterHandler {const participantIds = new Set(participants.map((participant) => Number(participant.id)));+ // Members the server lost contact with but who can still resume: their+ // mesh media doesn't depend on the server, so it is left running.+ const resumingIds = new Set((payload.resuming_ids || []).map(Number));const currentUserId = this.#getCurrentUserId();liveLog.info(@@ -128,7 +131,10 @@ export default class RosterHandler {peers?.forEach((pc, remoteUserId) => {if (!participantIds.has(remoteUserId)) {- if (this.#presencePending.has(roomId, remoteUserId)) {+ if (+ this.#presencePending.has(roomId, remoteUserId) ||+ resumingIds.has(remoteUserId)+ ) {return;}hasPeerLeft = true;@@ -189,7 +195,10 @@ export default class RosterHandler {}this.#syncRemoteMediaTracks(roomId, participants);- this.#dropDisallowedStreams(roomId, participants, { isStage });+ this.#dropDisallowedStreams(roomId, participants, {+ isStage,+ resumingIds,+ });this.#dropUnentitledMedia(roomId, participants);if (!this.#isMeshRoom(roomId)) {@@ -279,7 +288,7 @@ export default class RosterHandler {// even when the demotion arrives as a roster refresh rather than a// role_change message. The receive-side track policy stops new tracks;// this drops the ones already registered.- #dropDisallowedStreams(roomId, participants, { isStage }) {+ #dropDisallowedStreams(roomId, participants, { isStage, resumingIds }) {if (!isStage || !this.#isMeshRoom(roomId)) {return;}@@ -294,7 +303,7 @@ export default class RosterHandler {);for (const userId of this.#getRemoteUserIds(roomId)) {- if (!allowedToPublish.has(userId)) {+ if (!allowedToPublish.has(userId) && !resumingIds.has(userId)) {this.#removeRemoteStream(roomId, userId);}}diff --git a/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rb b/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rbindex 9a1b964ec44..f05e5735daf 100644--- a/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rb+++ b/plugins/live/spec/jobs/scheduled/live/close_orphaned_sessions_spec.rb@@ -61,6 +61,18 @@ RSpec.describe Jobs::Live::CloseOrphanedSessions doexpect(session.left_at).to be_nilend+ it "keeps sessions open while a lapsed participant can still resume" do+ session =+ Fabricate(:live_session, user: user1, room: room, joined_at: 50.minutes.ago, left_at: nil)++ set_stale_participant(room, user1, last_heartbeat: 2.minutes.ago)+ Live::ParticipantTracker.create_participant_session!(room.id, user1.id)++ job.execute({})++ expect(session.reload.left_at).to be_nil+ end+it "cleans up stale Redis entries after closing" doFabricate(:live_session, user: user1, room: room, joined_at: 50.minutes.ago, left_at: nil)diff --git a/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rb b/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rbindex c0db74bbd8e..5482fb46b67 100644--- a/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rb+++ b/plugins/live/spec/jobs/scheduled/publish_room_participants_spec.rb@@ -27,6 +27,24 @@ RSpec.describe Jobs::PublishRoomParticipants do)end+ it "keeps a mesh room pinned while a member can still resume" do+ Live::ParticipantTracker.pin_transport!(room.id, "mesh")+ Live::ParticipantTracker.add(room.id, user1.id)+ Live::ParticipantTracker.update_metadata(room.id, user1.id, { role: "participant" })+ Live::ParticipantTracker.create_participant_session!(room.id, user1.id)+ key = "#{Live::ParticipantTracker::KEY_NAMESPACE}:#{room.id}:participants"+ Discourse.redis.zadd(key, 1.hour.ago.to_f, user1.id)++ job.execute({})++ expect(Live::ParticipantTracker.pinned_transport(room.id)).to eq("mesh")++ Live::ParticipantTracker.revoke_participant_session(room.id, user1.id)+ job.execute({})++ expect(Live::ParticipantTracker.pinned_transport(room.id)).to be_nil+ end+it "reflects TTL-expired participants in broadcast" doLive::ParticipantTracker.add(room.id, user1.id)Live::ParticipantTracker.add(room.id, user2.id)diff --git a/plugins/live/spec/requests/live/rooms_controller_spec.rb b/plugins/live/spec/requests/live/rooms_controller_spec.rbindex 577a6fc45a6..e3bb7d05648 100644--- a/plugins/live/spec/requests/live/rooms_controller_spec.rb+++ b/plugins/live/spec/requests/live/rooms_controller_spec.rb@@ -684,6 +684,45 @@ RSpec.describe Live::RoomsController doexpect(Live::ParticipantTracker.user_ids(room.id)).to include(user.id)end+ it "returns 410 and revokes the session when the room filled up while presence was lapsed" do+ room.update!(max_participants: 2)+ sign_in(user)+ session_id = establish_presence!(room, user)+ key = "#{Live::ParticipantTracker::KEY_NAMESPACE}:#{room.id}:participants"+ Discourse.redis.zadd(key, 1.hour.ago.to_f, user.id)+ establish_presence!(room, staff)+ establish_presence!(room, other_participant)++ post "/live/rooms/#{room.id}/heartbeat.json", params: { participant_session_id: session_id }++ expect(response.status).to eq(410)+ expect(Live::ParticipantTracker.user_ids(room.id)).not_to include(user.id)+ expect(Live::ParticipantTracker.participant_session?(room.id, user.id)).to eq(false)+ end++ it "flags lapsed members with a resumable session so peers keep their media" do+ sign_in(user)+ session_id = establish_presence!(room, user)+ establish_presence!(room, other_participant)+ Live::ParticipantTracker.update_metadata(+ room.id,+ other_participant.id,+ { role: "participant" },+ )+ key = "#{Live::ParticipantTracker::KEY_NAMESPACE}:#{room.id}:participants"+ Discourse.redis.zadd(key, 1.hour.ago.to_f, other_participant.id)++ published = []+ allow(MessageBus).to receive(:publish) { |channel, data, _opts| published << data }++ post "/live/rooms/#{room.id}/heartbeat.json", params: { participant_session_id: session_id }++ expect(response.status).to eq(204)+ participants_message = published.find { |data| data[:type] == "participants" }+ expect(participants_message[:participants].map { |p| p[:id] }).to contain_exactly(user.id)+ expect(participants_message[:resuming_ids]).to contain_exactly(other_participant.id)+ end+it "rejects a heartbeat without a participant session and creates no presence" dosign_in(user)diff --git a/plugins/live/spec/services/live/participant_tracker_spec.rb b/plugins/live/spec/services/live/participant_tracker_spec.rbindex ab2b49142f9..5e45e2a35a1 100644--- a/plugins/live/spec/services/live/participant_tracker_spec.rb+++ b/plugins/live/spec/services/live/participant_tracker_spec.rb@@ -144,6 +144,69 @@ RSpec.describe Live::ParticipantTracker doexpect(described_class.valid_participant_session?(room.id, user1.id, session_id)).to eq(false)end++ it "outlives presence long enough to resume after a server outage" do+ expect(described_class.participant_session_ttl).to be >= 10.minutes.to_i+ expect(described_class.participant_session_ttl).to be >+ SiteSetting.live_participant_ttl_seconds+ end+ end++ describe ".vacant?" do+ def resuming!(user)+ described_class.update_metadata(room.id, user.id, { role: "participant" })+ described_class.create_participant_session!(room.id, user.id)+ end++ it "is false while a mesh member can still resume" do+ resuming!(user1)++ expect(described_class.vacant?(room.id)).to eq(false)+ end++ it "ignores resuming members in a LiveKit room" do+ described_class.pin_transport!(room.id, "livekit")+ resuming!(user1)++ expect(described_class.vacant?(room.id)).to eq(true)+ end++ it "is false while a human is present" do+ described_class.add(room.id, user1.id)++ expect(described_class.vacant?(room.id)).to eq(false)+ end+ end++ describe ".resuming_user_ids" do+ def lapse_presence(user)+ key = "#{described_class::KEY_NAMESPACE}:#{room.id}:participants"+ Discourse.redis.zadd(key, 1.hour.ago.to_f, user.id)+ end++ it "lists members whose presence lapsed while their session is resumable" do+ described_class.add(room.id, user1.id)+ described_class.update_metadata(room.id, user1.id, { role: "participant" })+ described_class.create_participant_session!(room.id, user1.id)+ lapse_presence(user1)++ expect(described_class.resuming_user_ids(room.id)).to contain_exactly(user1.id)+ end++ it "excludes present members, members without a session, and members who left" do+ [user1, user2].each do |user|+ described_class.add(room.id, user.id)+ described_class.update_metadata(room.id, user.id, { role: "participant" })+ described_class.create_participant_session!(room.id, user.id)+ end+ user3 = Fabricate(:user)+ described_class.update_metadata(room.id, user3.id, { role: "participant" })+ lapse_presence(user3)++ described_class.remove(room.id, user2.id)++ expect(described_class.resuming_user_ids(room.id)).to be_empty+ endenddescribe ".raise_hand" do@@ -189,11 +252,12 @@ RSpec.describe Live::ParticipantTracker doexpect(described_class.pinned_transport(room.id)).to eq("livekit")end- it "expires on its own so a crashed room self-heals" do+ it "expires with the participant session so resuming members keep their transport" dodescribed_class.pin_transport!(room.id, "livekit")ttl = Discourse.redis.ttl("#{described_class::KEY_NAMESPACE}:#{room.id}:transport")- expect(ttl).to be_between(1, SiteSetting.live_participant_ttl_seconds * 2)+ expect(ttl).to be > SiteSetting.live_participant_ttl_seconds * 2+ expect(ttl).to be <= described_class.participant_session_ttlendenddiff --git a/plugins/live/test/javascripts/unit/lib/roster-handler-test.js b/plugins/live/test/javascripts/unit/lib/roster-handler-test.jsnew file mode 100644index 00000000000..6c83af4d45b--- /dev/null+++ b/plugins/live/test/javascripts/unit/lib/roster-handler-test.js@@ -0,0 +1,55 @@+import { module, test } from "qunit";+import RosterHandler from "discourse/plugins/live/discourse/lib/live/roster-handler";++const ROOM_ID = 1;+const CURRENT_USER_ID = 10;++function buildHandler({ peerIds }) {+ const peers = new Map(peerIds.map((id) => [id, {}]));+ const destroyed = [];++ const handler = new RosterHandler({+ peerManager: {+ getRoomPeers: () => peers,+ has: (_roomId, userId) => peers.has(userId),+ destroy: (_roomId, userId) => {+ destroyed.push(userId);+ peers.delete(userId);+ },+ remoteVideoTrack: () => null,+ },+ presencePending: { has: () => false, clear() {} },+ getCurrentUserId: () => CURRENT_USER_ID,+ getChatSound: () => null,+ getRoom: () => ({ id: ROOM_ID, room_type: "standard" }),+ isActiveRoom: () => false,+ isConnectingRoom: () => false,+ isMeshRoom: () => true,+ getLocalVideoKind: () => null,+ });++ return { handler, destroyed };+}++module("Live | Unit | Lib | roster-handler", function () {+ test("keeps mesh peers the server marks as resuming", async function (assert) {+ const { handler, destroyed } = buildHandler({ peerIds: [20, 30] });++ await handler.handleParticipants(ROOM_ID, {+ participants: [{ id: CURRENT_USER_ID }],+ resuming_ids: [20],+ });++ assert.deepEqual(destroyed, [30], "only the departed peer is destroyed");+ });++ test("destroys peers absent from the roster when nothing is resuming", async function (assert) {+ const { handler, destroyed } = buildHandler({ peerIds: [20] });++ await handler.handleParticipants(ROOM_ID, {+ participants: [{ id: CURRENT_USER_ID }],+ });++ assert.deepEqual(destroyed, [20]);+ });+});