Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

livekit: Preserve tokens on channel rejoin (#59388)

Piotr Osiewicz68d2325b7f16
diff --git a/crates/call/src/call_impl/room.rs b/crates/call/src/call_impl/room.rs
index 9a5ee0d76c0..e72f375847e 100644
--- a/crates/call/src/call_impl/room.rs
+++ b/crates/call/src/call_impl/room.rs
@@ -1773,14 +1773,26 @@ impl Room {
}
}
+/// The number of times we attempt to establish the LiveKit connection before
+/// giving up. Retries cover transient failures (e.g. a network blip); a
+/// genuinely bad token won't be fixed by retrying, so the bound keeps us from
+/// hammering LiveKit indefinitely.
+const LIVEKIT_CONNECT_ATTEMPTS: usize = 5;
+
fn spawn_room_connection(
livekit_connection_info: Option<proto::LiveKitConnectionInfo>,
cx: &mut Context<Room>,
) -> Option<Task<()>> {
- let mut connection_info = livekit_connection_info?;
+ let connection_info = livekit_connection_info?;
Some(cx.spawn(async move |this, cx| {
+ // Retry the LiveKit connection with backoff, but deliberately do NOT
+ // ask collab for a fresh token here: a refreshed token is only needed
+ // when the collab connection itself is re-established, which is handled
+ // separately in `maintain_connection` via `ensure_livekit_connection`.
+ // Coupling this loop to `rejoin` previously turned an isolated LiveKit
+ // failure into a channel-wide `RejoinRoom` storm.
let mut backoff = Duration::from_secs(1);
- loop {
+ for attempt in 1..=LIVEKIT_CONNECT_ATTEMPTS {
match livekit::Room::connect(
connection_info.server_url.clone(),
connection_info.token.clone(),
@@ -1839,40 +1851,21 @@ fn spawn_room_connection(
return;
}
Err(error) => {
- log::error!("failed to connect to LiveKit room: {error:#}");
+ log::error!(
+ "failed to connect to LiveKit room (attempt {attempt}/{LIVEKIT_CONNECT_ATTEMPTS}): {error:#}"
+ );
}
}
- cx.background_executor().timer(backoff).await;
- backoff = (backoff * 2).min(Duration::from_secs(30));
-
- // The token we were given may no longer be valid: LiveKit revokes
- // it when this user's stale connection is cleaned up around the
- // time the token is issued (e.g. when rejoining a channel right
- // after a crash). Rejoin the room to obtain fresh connection info
- // before trying again.
- let Ok(rejoin) = this.update(cx, |this, cx| this.rejoin(cx)) else {
- return;
- };
- match rejoin.await {
- Ok(Some(new_connection_info)) => {
- log::info!(
- "refreshed LiveKit connection info (token changed: {})",
- new_connection_info.token != connection_info.token
- );
- connection_info = new_connection_info;
- }
- Ok(None) => {
- log::warn!(
- "server returned no fresh LiveKit connection info; \
- retrying with the existing token"
- );
- }
- Err(error) => {
- log::error!("failed to refresh LiveKit connection info: {error:#}");
- }
+ if attempt < LIVEKIT_CONNECT_ATTEMPTS {
+ cx.background_executor().timer(backoff).await;
+ backoff = (backoff * 2).min(Duration::from_secs(10));
}
}
+
+ log::error!(
+ "giving up on LiveKit room connection after {LIVEKIT_CONNECT_ATTEMPTS} attempts"
+ );
}))
}
diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs
index e89257b70d3..176d504ae1a 100644
--- a/crates/collab/src/rpc.rs
+++ b/crates/collab/src/rpc.rs
@@ -1339,7 +1339,7 @@ async fn connection_lost(
_ = executor.sleep(RECONNECT_TIMEOUT).fuse() => {
log::info!("connection lost, removing all resources for user:{}, connection:{:?}", session.user_id(), session.connection_id);
- leave_room_for_session(&session, session.connection_id).await.trace_err();
+ leave_room_for_session(&session, session.connection_id, None).await.trace_err();
leave_channel_buffers_for_session(&session)
.await
.trace_err();
@@ -1702,7 +1702,7 @@ async fn leave_room(
response: Response<proto::LeaveRoom>,
session: MessageContext,
) -> Result<()> {
- leave_room_for_session(&session, session.connection_id).await?;
+ leave_room_for_session(&session, session.connection_id, None).await?;
response.send(proto::Ack {})?;
Ok(())
}
@@ -3437,7 +3437,7 @@ async fn join_channel_internal(
"cleaning up stale connection",
);
drop(db);
- leave_room_for_session(&session, connection).await?;
+ leave_room_for_session(&session, connection, Some(channel_id)).await?;
db = session.db().await;
}
@@ -4067,7 +4067,11 @@ async fn update_user_contacts(user_id: UserId, session: &Session) -> Result<()>
Ok(())
}
-async fn leave_room_for_session(session: &Session, connection_id: ConnectionId) -> Result<()> {
+async fn leave_room_for_session(
+ session: &Session,
+ connection_id: ConnectionId,
+ rejoining_channel_id: Option<ChannelId>,
+) -> Result<()> {
let mut contacts_to_update = HashSet::default();
let room_id;
@@ -4076,6 +4080,7 @@ async fn leave_room_for_session(session: &Session, connection_id: ConnectionId)
let delete_livekit_room;
let room;
let channel;
+ let left_channel_id;
if let Some(mut left_room) = session.db().await.leave_room(connection_id).await? {
contacts_to_update.insert(session.user_id());
@@ -4090,12 +4095,23 @@ async fn leave_room_for_session(session: &Session, connection_id: ConnectionId)
delete_livekit_room = left_room.deleted;
room = mem::take(&mut left_room.room);
channel = mem::take(&mut left_room.channel);
+ left_channel_id = channel.as_ref().map(|channel| channel.id);
room_updated(&room, &session.peer);
} else {
return Ok(());
}
+ // When this cleanup is part of rejoining the same channel, the user is
+ // immediately re-entering this LiveKit room with the same identity, which
+ // LiveKit handles by evicting the prior session. Calling
+ // `remove_participant` here is therefore redundant, and worse: on LiveKit
+ // Cloud it revokes the identity's tokens (including the freshly issued one
+ // for the rejoin), leaving the user connected to collab but unable to join
+ // audio. Skip the removal in that case.
+ let skip_livekit_removal =
+ rejoining_channel_id.is_some() && left_channel_id == rejoining_channel_id;
+
if let Some(channel) = channel {
channel_updated(
&channel,
@@ -4128,10 +4144,12 @@ async fn leave_room_for_session(session: &Session, connection_id: ConnectionId)
}
if let Some(live_kit) = session.app_state.livekit_client.as_ref() {
- live_kit
- .remove_participant(livekit_room.clone(), session.user_id().to_string())
- .await
- .trace_err();
+ if !skip_livekit_removal {
+ live_kit
+ .remove_participant(livekit_room.clone(), session.user_id().to_string())
+ .await
+ .trace_err();
+ }
if delete_livekit_room {
live_kit.delete_room(livekit_room).await.trace_err();
diff --git a/crates/collab/tests/integration/channel_tests.rs b/crates/collab/tests/integration/channel_tests.rs
index eac9f579520..ab4754082ed 100644
--- a/crates/collab/tests/integration/channel_tests.rs
+++ b/crates/collab/tests/integration/channel_tests.rs
@@ -367,69 +367,73 @@ async fn test_joining_channel_ancestor_member(
}
#[gpui::test]
-async fn test_channel_call_recovers_from_livekit_connect_failure(
+async fn test_rejoining_channel_does_not_remove_livekit_participant(
executor: BackgroundExecutor,
- cx_a: &mut TestAppContext,
+ cx_a1: &mut TestAppContext,
+ cx_a2: &mut TestAppContext,
) {
let mut server = TestServer::start(executor.clone()).await;
- let client_a = server.create_client(cx_a, "user_a").await;
- client_a.initialize_channel_store(cx_a);
+ // Two connections for the same user. The second one joining the same
+ // channel reproduces the production scenario where the server runs
+ // stale-connection cleanup for the first connection (as it does after an
+ // abrupt disconnect/crash followed by a rejoin within RECONNECT_TIMEOUT).
+ // Mirror LiveKit Cloud: removing a participant revokes their tokens. This
+ // is what turns the redundant `remove_participant` during stale cleanup
+ // into a user-visible failure.
+ server
+ .test_livekit_server
+ .set_revoke_tokens_on_removal(true);
+
+ let client_a1 = server.create_client(cx_a1, "user_a").await;
+ let client_a2 = server.create_client(cx_a2, "user_a").await;
+ client_a1.initialize_channel_store(cx_a1);
let channel_id = server
- .make_channel("zed", None, (&client_a, cx_a), &mut [])
+ .make_channel("zed", None, (&client_a1, cx_a1), &mut [])
.await;
- // Simulate LiveKit rejecting the token issued for this join. In
- // production this happens when the collab server's stale connection
- // cleanup (`leave_room_for_session`) calls `remove_participant` for this
- // user's identity just before issuing a new token for the same identity
- // and room (e.g. rejoining a channel right after an abrupt disconnect or
- // restart): LiveKit Cloud revokes the freshly issued token and the
- // client's initial connection fails with "401 invalid token: revoked".
- let identity = client_a.user_id().unwrap().to_string();
- server
- .test_livekit_server
- .set_token_revoked(&identity, true);
-
- let active_call_a = cx_a.read(ActiveCall::global);
- active_call_a
- .update(cx_a, |active_call, cx| {
+ let active_call_a1 = cx_a1.read(ActiveCall::global);
+ active_call_a1
+ .update(cx_a1, |active_call, cx| {
active_call.join_channel(channel_id, cx)
})
.await
.unwrap();
executor.run_until_parked();
- // The collab server considers user A a participant in the call...
- let room_a =
- cx_a.read(|cx| active_call_a.read_with(cx, |call, _| call.room().unwrap().clone()));
- cx_a.read(|cx| {
- client_a.channel_store().read_with(cx, |channels, _| {
- assert_participants_eq(
- channels.channel_participants(channel_id),
- &[client_a.user_id().unwrap()],
- );
+ // The second connection joins the same channel, triggering stale-connection
+ // cleanup of the first connection on the server.
+ let active_call_a2 = cx_a2.read(ActiveCall::global);
+ active_call_a2
+ .update(cx_a2, |active_call, cx| {
+ active_call.join_channel(channel_id, cx)
})
- });
- // ...but the LiveKit connection failed, so they have no audio.
- cx_a.read(|cx| room_a.read_with(cx, |room, cx| assert!(!room.is_connected(cx))));
-
- // Once LiveKit accepts the user's tokens again, the client should
- // re-establish its LiveKit connection instead of silently staying in the
- // call without audio.
- server
- .test_livekit_server
- .set_token_revoked(&identity, false);
- executor.advance_clock(RECONNECT_TIMEOUT);
+ .await
+ .unwrap();
executor.run_until_parked();
- cx_a.read(|cx| {
- room_a.read_with(cx, |room, cx| {
+
+ // The user-visible symptom: with the bug, stale cleanup revokes the
+ // rejoining user's token and they end up in the call with no audio.
+ let room_a2 =
+ cx_a2.read(|cx| active_call_a2.read_with(cx, |call, _| call.room().unwrap().clone()));
+ cx_a2.read(|cx| {
+ room_a2.read_with(cx, |room, cx| {
assert!(
room.is_connected(cx),
- "client should have re-established its LiveKit connection"
+ "rejoining the same channel should not break audio"
)
})
});
+
+ // The mechanism: that cleanup must NOT have removed the user from the
+ // LiveKit room, since they are immediately rejoining it.
+ let identity = client_a2.user_id().unwrap().to_string();
+ assert!(
+ !server
+ .test_livekit_server
+ .participant_was_removed(&identity),
+ "rejoining the same channel should not remove the LiveKit participant"
+ );
}
#[gpui::test]
diff --git a/crates/livekit_client/src/test.rs b/crates/livekit_client/src/test.rs
index eff26af4258..b07087bbc2a 100644
--- a/crates/livekit_client/src/test.rs
+++ b/crates/livekit_client/src/test.rs
@@ -57,6 +57,13 @@ pub struct TestServer {
pub secret_key: String,
rooms: Mutex<HashMap<String, TestServerRoom>>,
revoked_identities: Mutex<HashSet<String>>,
+ /// Log of `(room, identity)` pairs passed to `remove_participant`, for
+ /// tests that need to assert whether a participant was forcibly removed.
+ removed_participants: Mutex<Vec<(String, String)>>,
+ /// When set, `remove_participant` revokes the identity's tokens, mirroring
+ /// LiveKit Cloud (whose docs state removal "immediately revokes their
+ /// access token"). Off by default so unrelated tests are unaffected.
+ revoke_tokens_on_removal: AtomicBool,
executor: BackgroundExecutor,
}
@@ -75,6 +82,8 @@ impl TestServer {
secret_key,
rooms: Default::default(),
revoked_identities: Default::default(),
+ removed_participants: Default::default(),
+ revoke_tokens_on_removal: AtomicBool::new(false),
executor,
});
e.insert(server.clone());
@@ -106,16 +115,18 @@ impl TestServer {
}
}
- /// Simulates LiveKit Cloud revoking an identity's tokens, as happens when
- /// `remove_participant` is called for that identity (e.g. by the collab
- /// server's stale connection cleanup) around the time a token is issued.
- pub fn set_token_revoked(&self, identity: &str, revoked: bool) {
- let mut revoked_identities = self.revoked_identities.lock();
- if revoked {
- revoked_identities.insert(identity.to_string());
- } else {
- revoked_identities.remove(identity);
- }
+ /// Makes `remove_participant` revoke the removed identity's tokens, as
+ /// LiveKit Cloud does. Used to reproduce the revoked-token failure.
+ pub fn set_revoke_tokens_on_removal(&self, revoke: bool) {
+ self.revoke_tokens_on_removal.store(revoke, SeqCst);
+ }
+
+ /// Whether `remove_participant` was ever called for the given identity.
+ pub fn participant_was_removed(&self, identity: &str) -> bool {
+ self.removed_participants
+ .lock()
+ .iter()
+ .any(|(_, removed_identity)| removed_identity == identity)
}
pub async fn create_room(&self, room: String) -> Result<()> {
@@ -266,6 +277,14 @@ impl TestServer {
) -> Result<()> {
self.simulate_random_delay().await;
+ self.removed_participants
+ .lock()
+ .push((room_name.clone(), identity.0.clone()));
+
+ if self.revoke_tokens_on_removal.load(SeqCst) {
+ self.revoked_identities.lock().insert(identity.0.clone());
+ }
+
let mut server_rooms = self.rooms.lock();
let room = server_rooms
.get_mut(&room_name)