Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(frontend) add an exponential backoff reconnection policy
briquet22a93947d5db
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c74c0ee9..68e2075b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,10 @@ and this project adheres to## [Unreleased]+### Added++- ✨(frontend) add jittered exponential backoff to LiveKit reconnections+### Changed- ⬆️(frontend) upgrade livekit-client to 2.23.0diff --git a/src/frontend/src/api/useConfig.ts b/src/frontend/src/api/useConfig.tsindex 50ab65ba..9a9d35b0 100644--- a/src/frontend/src/api/useConfig.ts+++ b/src/frontend/src/api/useConfig.ts@@ -75,6 +75,13 @@ export interface ApiConfig {enable_firefox_proxy_workaround: booleandefault_sources: Source[]default_video_codec: VideoCodec+ reconnect?: {+ backoff_enabled: boolean+ base_delay_ms: number+ max_delay_ms: number+ give_up_after_ms: number+ backoff_reset_ms: number+ }}transcription_destination?: stringmax_participants_for_sound: numberdiff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsxindex b67bee75..a66387d7 100644--- a/src/frontend/src/features/rooms/components/Conference.tsx+++ b/src/frontend/src/features/rooms/components/Conference.tsx@@ -10,7 +10,6 @@ import {ConnectionErrorReason,DisconnectReason,MediaDeviceFailure,- Room,type RoomOptions,VideoPresets,} from 'livekit-client'@@ -47,6 +46,25 @@ import { MeetDevtools } from '@/features/devtools'import { VOICE_AUDIO_CONSTRAINTS } from '@/features/rooms/livekit/utils/constants'import { resolveAudioOutputDeviceId } from '@/features/rooms/utils/resolveAudioOutputDeviceId'import { ConnectionEndReason } from '@/features/rooms/utils/connectionEndReason'+import { createLiveKitRoom } from '@/features/rooms/livekit/utils/reconnectPolicy'++const handleError = (e: Error) => {+ const failure = getMediaDeviceFailure(e)+ if (failure && failure !== MediaDeviceFailure.Other) return++ // connect() was aborted by a disconnect() before the join completed+ if (+ e instanceof ConnectionError &&+ e.reason === ConnectionErrorReason.Cancelled+ ) {+ void captureEvent('connection-cancelled')+ return+ }++ reportError('livekit_room_error', e, {+ path: 'connect_publish',+ })+}export const Conference = ({roomId,@@ -137,7 +155,11 @@ export const Conference = ({apiConfig?.livekit.default_video_codec,])- const room = useMemo(() => new Room(roomOptions), [roomOptions])+ const reconnect = apiConfig?.livekit.reconnect+ const room = useMemo(+ () => createLiveKitRoom(roomOptions, reconnect),+ [roomOptions, reconnect]+ )useEffect(() => {/**@@ -250,23 +272,7 @@ export const Conference = ({className={css({backgroundColor: 'primaryDark.50 !important',})}- onError={(e) => {- const failure = getMediaDeviceFailure(e)- if (failure && failure !== MediaDeviceFailure.Other) return-- // connect() was aborted by a disconnect() before the join completed- if (- e instanceof ConnectionError &&- e.reason === ConnectionErrorReason.Cancelled- ) {- void captureEvent('connection-cancelled')- return- }-- reportError('livekit_room_error', e, {- path: 'connect_publish',- })- }}+ onError={handleError}onConnected={async () => {hasConnectedRef.current = trueif (!apiConfig) returndiff --git a/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsx b/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsxindex d45f631b..2410d89e 100644--- a/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsx+++ b/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsx@@ -7,12 +7,23 @@ import {} from '@livekit/components-react'import { useEffect, useRef } from 'react'import { useSnapshot } from 'valtio'-import { ConnectionState, DisconnectReason, RoomEvent } from 'livekit-client'+import {+ ConnectionState,+ DisconnectReason,+ type Room,+ RoomEvent,+} from 'livekit-client'import { userPreferencesStore } from '@/stores/userPreferences'import { captureEvent, captureMediaEvent } from '@/features/analytics/telemetry'import { connectionObserverStore } from '@/stores/connectionObserver'+import { BackoffReconnectPolicy } from '@/features/rooms/livekit/utils/reconnectPolicy'++const getBackoffPolicy = (room: Room) => {+ const policy = room.options.reconnectPolicy+ return policy instanceof BackoffReconnectPolicy ? policy : undefined+}export const ConnectionObserver = () => {const room = useRoomContext()@@ -96,11 +107,13 @@ export const ConnectionObserver = () => {}const handleReconnected = async () => {+ const durationMs = getBackoffPolicy(room)?.lastReconnectionDurationMsconst participantSid = room.localParticipant.sidconst roomSid = await room.getSid().catch(() => undefined)captureEvent('reconnected-event', {livekit_room_sid: roomSid,livekit_participant_sid: participantSid,+ durationMs,})}@@ -123,9 +136,12 @@ export const ConnectionObserver = () => {sessionDuration: connectionStartTimeRef.current? connectionEndTime - connectionStartTimeRef.current: -1,- reason: disconnectReason- ? DisconnectReason[disconnectReason]- : 'UNKNOWN',+ // LiveKit reports no reason when the reconnect policy gives up+ reason: getBackoffPolicy(room)?.gaveUp+ ? 'RECONNECT_GAVE_UP'+ : disconnectReason+ ? DisconnectReason[disconnectReason]+ : 'UNKNOWN',})}diff --git a/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.test.ts b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.test.tsnew file mode 100644index 00000000..2e598262--- /dev/null+++ b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.test.ts@@ -0,0 +1,273 @@+import { ConnectionState, RoomEvent, type Room } from 'livekit-client'+import { afterEach, describe, expect, it, vi } from 'vitest'+import {+ BackoffReconnectPolicy,+ createLiveKitRoom,+ MAX_FIRST_DELAY_MS,+} from './reconnectPolicy'++const CONFIG = {+ backoff_enabled: true,+ base_delay_ms: 1_000,+ max_delay_ms: 15_000,+ give_up_after_ms: 300_000,+ backoff_reset_ms: 120_000,+}++const createRoom = () => {+ const listeners = new Map<RoomEvent, Array<(...args: unknown[]) => void>>()+ const room = {+ on(event: RoomEvent, listener: (...args: unknown[]) => void) {+ listeners.set(event, [...(listeners.get(event) ?? []), listener])+ return room+ },+ emit(event: RoomEvent, ...args: unknown[]) {+ listeners.get(event)?.forEach((listener) => listener(...args))+ },+ setState(state: ConnectionState, event?: RoomEvent) {+ room.emit(RoomEvent.ConnectionStateChanged, state)+ if (event) room.emit(event)+ },+ }+ return room+}++const setup = ({ config = CONFIG, random = 0.5 } = {}) => {+ const clock = { now: 0 }++ vi.spyOn(Math, 'random').mockReturnValue(random)+ vi.spyOn(performance, 'now').mockImplementation(() => clock.now)++ const room = createRoom()+ const policy = new BackoffReconnectPolicy(config)+ policy.attach(room as unknown as Room)++ const delay = (retryCount: number, elapsedMs = 0) =>+ policy.nextRetryDelayInMs({ retryCount, elapsedMs })++ return { clock, room, policy, delay }+}++afterEach(() => {+ vi.restoreAllMocks()+})++describe('BackoffReconnectPolicy', () => {+ it('doubles the delay ceiling after each failed attempt, up to the cap', () => {+ const { delay } = setup()++ expect([0, 1, 2, 3, 4, 10].map((retryCount) => delay(retryCount))).toEqual([+ 500, 1_500, 3_000, 6_000, 11_250, 11_250,+ ])+ })++ it('draws the first delay anywhere below its ceiling (check two extremes)', () => {+ expect(setup({ random: 0 }).delay(0)).toBe(0)+ expect(setup({ random: 0.9999 }).delay(0)).toBe(999)+ })++ it('draws the next delays between half their ceiling and their ceiling (check two extremes)', () => {+ expect(setup({ random: 0 }).delay(3)).toBe(4_000)+ expect(setup({ random: 0.9999 }).delay(3)).toBe(7_999)+ })++ it('keeps the first delay of a reconnection under 5 s', () => {+ const config = { ...CONFIG, base_delay_ms: 8_000 }+ const { delay } = setup({ config, random: 0.9999 })++ expect(delay(0)).toBeLessThan(MAX_FIRST_DELAY_MS)+ expect(delay(1)).toBe(14_999)+ })++ it('starts the first attempt within 5 s of the drop', () => {+ const config = { ...CONFIG, base_delay_ms: 8_000 }+ const { clock, delay } = setup({ config })+ expect(delay(0)).toBe(2_500)++ clock.now = 4_000+ expect(delay(0)).toBe(500)++ clock.now = 6_000+ expect(delay(0)).toBe(0)+ })++ it('never steps the backoff down within a reconnection', () => {+ const { delay } = setup()++ expect(delay(3)).toBe(6_000)+ expect(delay(0)).toBe(6_000)+ })++ it('ignores the elapsed time reported by LiveKit', () => {+ const { delay } = setup()++ expect(delay(1, Date.now())).toBe(1_500)+ })++ it('picks up the backoff of a reconnection that ended shortly before', () => {+ const { clock, room, delay } = setup()+ delay(0)+ delay(1)+ delay(2)+ room.emit(RoomEvent.Reconnected)++ clock.now += 60_000+ // One step above where the last reconnection ended (capped)+ expect(delay(0)).toBe(2_500)+ expect(delay(1)).toBe(11_250)+ })++ it('starts the backoff over once the connection held for the reset delay', () => {+ const { clock, room, delay } = setup()+ delay(0)+ delay(1)+ delay(2)+ room.emit(RoomEvent.Reconnected)++ clock.now += 120_000++ expect(delay(0)).toBe(500)+ })++ it('gives up once the reconnection lasts longer than the give-up delay', () => {+ const { clock, room, policy, delay } = setup()+ delay(0)+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)++ clock.now = 299_999++ expect(delay(20)).toBe(11_250)+ expect(policy.gaveUp).toBe(false)++ clock.now = 300_000++ expect(delay(21)).toBeNull()+ expect(policy.gaveUp).toBe(true)+ })++ it('keeps trying while only signaling is down', () => {+ const { clock, room, policy, delay } = setup()+ delay(0)+ room.setState(+ ConnectionState.SignalReconnecting,+ RoomEvent.SignalReconnecting+ )++ clock.now = 600_000++ expect(delay(40)).toBe(11_250)+ expect(policy.gaveUp).toBe(false)++ // The media connection failed as well+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)++ expect(delay(41)).toBeNull()+ expect(policy.gaveUp).toBe(true)+ })++ it('gives up once the media connection failed, even back to resuming', () => {+ const { clock, room, policy, delay } = setup()+ delay(0)+ room.setState(+ ConnectionState.SignalReconnecting,+ RoomEvent.SignalReconnecting+ )+ delay(1)++ // The peer connections fail, then LiveKit tries to resume again+ clock.now = 30_000+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)+ room.setState(+ ConnectionState.SignalReconnecting,+ RoomEvent.SignalReconnecting+ )++ clock.now = 299_999++ expect(delay(20)).toBe(11_250)+ expect(policy.gaveUp).toBe(false)++ clock.now = 300_000++ expect(delay(21)).toBeNull()+ expect(policy.gaveUp).toBe(true)+ })++ it('starts a new reconnection once the room connects again', () => {+ const { clock, room, policy, delay } = setup()+ delay(0)+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)+ clock.now = 300_000++ expect(delay(21)).toBeNull()++ room.setState(ConnectionState.Disconnected, RoomEvent.Disconnected)+ expect(policy.gaveUp).toBe(true)++ room.setState(ConnectionState.Connecting)+ room.setState(ConnectionState.Connected, RoomEvent.Connected)+ expect(policy.gaveUp).toBe(false)++ clock.now += 600_000++ expect(delay(0)).toBe(500)++ // The media connection of the previous reconnection is not lost anymore+ room.setState(+ ConnectionState.SignalReconnecting,+ RoomEvent.SignalReconnecting+ )+ clock.now += 300_000++ expect(delay(20)).toBe(11_250)+ })++ it('starts over when a new join replaces the reconnection', () => {+ const { clock, room, policy, delay } = setup()+ delay(0)+ delay(3)+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)++ // LiveKitRoom calls room.connect() again in the middle of the reconnection+ clock.now = 400_000+ room.setState(ConnectionState.Connecting)++ expect(delay(0)).toBe(500)+ expect(policy.gaveUp).toBe(false)+ })++ it('reports how long a reconnection lasted, once', () => {+ const { clock, room, policy, delay } = setup()+ clock.now = 1_000+ delay(0)++ clock.now = 4_000+ room.emit(RoomEvent.Reconnected)+ expect(policy.lastReconnectionDurationMs).toBe(3_000)++ // LiveKit can emit Reconnected several times for one recovery+ room.emit(RoomEvent.Reconnected)+ expect(policy.lastReconnectionDurationMs).toBeUndefined()+ })+})++describe('createLiveKitRoom', () => {+ it('builds the policy from the config endpoint values', () => {+ vi.spyOn(Math, 'random').mockReturnValue(0.5)++ const { reconnectPolicy } = createLiveKitRoom({}, CONFIG).options++ expect(reconnectPolicy).toBeInstanceOf(BackoffReconnectPolicy)+ expect(+ reconnectPolicy.nextRetryDelayInMs({ retryCount: 2, elapsedMs: 0 })+ ).toBe(3_000)+ })++ it('keeps the default policy when the backoff is missing or disabled', () => {+ for (const config of [undefined, { ...CONFIG, backoff_enabled: false }]) {+ expect(+ createLiveKitRoom({}, config).options.reconnectPolicy+ ).not.toBeInstanceOf(BackoffReconnectPolicy)+ }+ })+})diff --git a/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.ts b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.tsnew file mode 100644index 00000000..b03353c4--- /dev/null+++ b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.ts@@ -0,0 +1,146 @@+import {+ ConnectionState,+ Room,+ RoomEvent,+ type ReconnectContext,+ type ReconnectPolicy,+ type RoomOptions,+} from 'livekit-client'+import type { ApiConfig } from '@/api/useConfig'++/**+ * Until the first attempt of a reconnection starts, the room still reports+ * Connected and LiveKit keeps checking its transport for a while.+ * Around 10s after the initial drop, it will try a full websocket reconnection.+ * Hence, the first attempt should start well before that.+ */+export const MAX_FIRST_DELAY_MS = 5_000++type ReconnectConfig = NonNullable<ApiConfig['livekit']['reconnect']>++type Reconnection = {+ startedAt: number+ // Higher than 0 when the previous reconnection ended shortly before+ firstStep: number+ failedAttempts: number+ isMediaLost: boolean+ gaveUp: boolean+}++/**+ * When a LiveKit node crashes or Redis stalls, every client loses its signal+ * connection at the same time. This policy implements an exponential backoff+ * with some randomness to spread the reconnections out.+ *+ * The policy keeps its own clock since LiveKit calls the policy again on every+ * new disconnection trigger+ */+export class BackoffReconnectPolicy implements ReconnectPolicy {+ lastReconnectionDurationMs: number | undefined+ private readonly config: ReconnectConfig+ private reconnection: Reconnection | undefined+ private lastRecovery: { at: number; step: number } | undefined++ constructor(config: ReconnectConfig) {+ this.config = config+ }++ attach(room: Room) {+ room.on(RoomEvent.Connected, this.handleRecovery)+ room.on(RoomEvent.Reconnected, this.handleRecovery)+ room.on(RoomEvent.Reconnecting, this.handleMediaLost)+ room.on(RoomEvent.ConnectionStateChanged, this.handleStateChange)+ }++ get gaveUp() {+ return this.reconnection?.gaveUp ?? false+ }++ nextRetryDelayInMs({ retryCount }: ReconnectContext): number | null {+ const now = performance.now()+ this.reconnection ??= this.startReconnection(now)+ const reconnection = this.reconnection++ reconnection.failedAttempts = Math.max(+ reconnection.failedAttempts,+ retryCount+ )++ if (+ now - reconnection.startedAt >= this.config.give_up_after_ms &&+ reconnection.isMediaLost+ ) {+ reconnection.gaveUp = true+ return null+ }++ const ceiling = Math.min(+ this.config.max_delay_ms,+ this.config.base_delay_ms *+ 2 ** (reconnection.firstStep + reconnection.failedAttempts)+ )++ if (reconnection.failedAttempts === 0) {+ // LiveKit re-arms its timer on each trigger: count from the first drop+ const timeLeft = reconnection.startedAt + MAX_FIRST_DELAY_MS - now+ return Math.floor(+ Math.random() * Math.min(ceiling, Math.max(timeLeft, 0))+ )+ }+ return Math.floor((ceiling / 2) * (1 + Math.random()))+ }++ private startReconnection(now: number): Reconnection {+ const recovery = this.lastRecovery++ const isLastDisconnectionRecent =+ recovery !== undefined && now - recovery.at < this.config.backoff_reset_ms++ return {+ startedAt: now,+ firstStep: isLastDisconnectionRecent ? recovery.step + 1 : 0,+ failedAttempts: 0,+ isMediaLost: false,+ gaveUp: false,+ }+ }++ private handleMediaLost = () => {+ this.reconnection ??= this.startReconnection(performance.now())+ this.reconnection.isMediaLost = true+ }++ private handleStateChange = (state: ConnectionState) => {+ if (state === ConnectionState.Connecting) this.reconnection = undefined+ }++ private handleRecovery = () => {+ const now = performance.now()+ if (this.reconnection) {+ const { startedAt, firstStep, failedAttempts } = this.reconnection+ this.lastRecovery = { at: now, step: firstStep + failedAttempts }+ this.lastReconnectionDurationMs = now - startedAt+ this.reconnection = undefined+ } else {+ // LiveKit can emit Reconnected several times for a full reconnection+ // We want to avoid duplicating the reconnection duration in PostHog+ this.lastReconnectionDurationMs = undefined+ }+ }+}++/**+ * Build a room with the reconnection policy from the config endpoint.+ * Without it, the room keeps the LiveKit default policy+ */+export const createLiveKitRoom = (+ options: RoomOptions,+ config?: ReconnectConfig+): Room => {+ if (!config?.backoff_enabled) return new Room(options)++ const reconnectPolicy = new BackoffReconnectPolicy(config)+ const room = new Room({ ...options, reconnectPolicy })+ reconnectPolicy.attach(room)+ return room+}