Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(frontend) add an exponential backoff reconnection policy

briquet22a93947d5db
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c74c0ee9..68e2075b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,10 @@ and this project adheres to
## [Unreleased]
+### Added
+
+- ✨(frontend) add jittered exponential backoff to LiveKit reconnections
+
### Changed
- ⬆️(frontend) upgrade livekit-client to 2.23.0
diff --git a/src/frontend/src/api/useConfig.ts b/src/frontend/src/api/useConfig.ts
index 50ab65ba..9a9d35b0 100644
--- a/src/frontend/src/api/useConfig.ts
+++ b/src/frontend/src/api/useConfig.ts
@@ -75,6 +75,13 @@ export interface ApiConfig {
enable_firefox_proxy_workaround: boolean
default_sources: Source[]
default_video_codec: VideoCodec
+ reconnect?: {
+ backoff_enabled: boolean
+ base_delay_ms: number
+ max_delay_ms: number
+ give_up_after_ms: number
+ backoff_reset_ms: number
+ }
}
transcription_destination?: string
max_participants_for_sound: number
diff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsx
index b67bee75..a66387d7 100644
--- a/src/frontend/src/features/rooms/components/Conference.tsx
+++ b/src/frontend/src/features/rooms/components/Conference.tsx
@@ -10,7 +10,6 @@ import {
ConnectionErrorReason,
DisconnectReason,
MediaDeviceFailure,
- Room,
type RoomOptions,
VideoPresets,
} from 'livekit-client'
@@ -47,6 +46,25 @@ import { MeetDevtools } from '@/features/devtools'
import { VOICE_AUDIO_CONSTRAINTS } from '@/features/rooms/livekit/utils/constants'
import { resolveAudioOutputDeviceId } from '@/features/rooms/utils/resolveAudioOutputDeviceId'
import { ConnectionEndReason } from '@/features/rooms/utils/connectionEndReason'
+import { createLiveKitRoom } from '@/features/rooms/livekit/utils/reconnectPolicy'
+
+const handleError = (e: Error) => {
+ const failure = getMediaDeviceFailure(e)
+ if (failure && failure !== MediaDeviceFailure.Other) return
+
+ // connect() was aborted by a disconnect() before the join completed
+ if (
+ e instanceof ConnectionError &&
+ e.reason === ConnectionErrorReason.Cancelled
+ ) {
+ void captureEvent('connection-cancelled')
+ return
+ }
+
+ reportError('livekit_room_error', e, {
+ path: 'connect_publish',
+ })
+}
export const Conference = ({
roomId,
@@ -137,7 +155,11 @@ export const Conference = ({
apiConfig?.livekit.default_video_codec,
])
- const room = useMemo(() => new Room(roomOptions), [roomOptions])
+ const reconnect = apiConfig?.livekit.reconnect
+ const room = useMemo(
+ () => createLiveKitRoom(roomOptions, reconnect),
+ [roomOptions, reconnect]
+ )
useEffect(() => {
/**
@@ -250,23 +272,7 @@ export const Conference = ({
className={css({
backgroundColor: 'primaryDark.50 !important',
})}
- onError={(e) => {
- const failure = getMediaDeviceFailure(e)
- if (failure && failure !== MediaDeviceFailure.Other) return
-
- // connect() was aborted by a disconnect() before the join completed
- if (
- e instanceof ConnectionError &&
- e.reason === ConnectionErrorReason.Cancelled
- ) {
- void captureEvent('connection-cancelled')
- return
- }
-
- reportError('livekit_room_error', e, {
- path: 'connect_publish',
- })
- }}
+ onError={handleError}
onConnected={async () => {
hasConnectedRef.current = true
if (!apiConfig) return
diff --git a/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsx b/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsx
index d45f631b..2410d89e 100644
--- a/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsx
+++ b/src/frontend/src/features/rooms/livekit/components/ConnectionObserver.tsx
@@ -7,12 +7,23 @@ import {
} from '@livekit/components-react'
import { useEffect, useRef } from 'react'
import { useSnapshot } from 'valtio'
-import { ConnectionState, DisconnectReason, RoomEvent } from 'livekit-client'
+import {
+ ConnectionState,
+ DisconnectReason,
+ type Room,
+ RoomEvent,
+} from 'livekit-client'
import { userPreferencesStore } from '@/stores/userPreferences'
import { captureEvent, captureMediaEvent } from '@/features/analytics/telemetry'
import { connectionObserverStore } from '@/stores/connectionObserver'
+import { BackoffReconnectPolicy } from '@/features/rooms/livekit/utils/reconnectPolicy'
+
+const getBackoffPolicy = (room: Room) => {
+ const policy = room.options.reconnectPolicy
+ return policy instanceof BackoffReconnectPolicy ? policy : undefined
+}
export const ConnectionObserver = () => {
const room = useRoomContext()
@@ -96,11 +107,13 @@ export const ConnectionObserver = () => {
}
const handleReconnected = async () => {
+ const durationMs = getBackoffPolicy(room)?.lastReconnectionDurationMs
const participantSid = room.localParticipant.sid
const roomSid = await room.getSid().catch(() => undefined)
captureEvent('reconnected-event', {
livekit_room_sid: roomSid,
livekit_participant_sid: participantSid,
+ durationMs,
})
}
@@ -123,9 +136,12 @@ export const ConnectionObserver = () => {
sessionDuration: connectionStartTimeRef.current
? connectionEndTime - connectionStartTimeRef.current
: -1,
- reason: disconnectReason
- ? DisconnectReason[disconnectReason]
- : 'UNKNOWN',
+ // LiveKit reports no reason when the reconnect policy gives up
+ reason: getBackoffPolicy(room)?.gaveUp
+ ? 'RECONNECT_GAVE_UP'
+ : disconnectReason
+ ? DisconnectReason[disconnectReason]
+ : 'UNKNOWN',
})
}
diff --git a/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.test.ts b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.test.ts
new file mode 100644
index 00000000..2e598262
--- /dev/null
+++ b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.test.ts
@@ -0,0 +1,273 @@
+import { ConnectionState, RoomEvent, type Room } from 'livekit-client'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+ BackoffReconnectPolicy,
+ createLiveKitRoom,
+ MAX_FIRST_DELAY_MS,
+} from './reconnectPolicy'
+
+const CONFIG = {
+ backoff_enabled: true,
+ base_delay_ms: 1_000,
+ max_delay_ms: 15_000,
+ give_up_after_ms: 300_000,
+ backoff_reset_ms: 120_000,
+}
+
+const createRoom = () => {
+ const listeners = new Map<RoomEvent, Array<(...args: unknown[]) => void>>()
+ const room = {
+ on(event: RoomEvent, listener: (...args: unknown[]) => void) {
+ listeners.set(event, [...(listeners.get(event) ?? []), listener])
+ return room
+ },
+ emit(event: RoomEvent, ...args: unknown[]) {
+ listeners.get(event)?.forEach((listener) => listener(...args))
+ },
+ setState(state: ConnectionState, event?: RoomEvent) {
+ room.emit(RoomEvent.ConnectionStateChanged, state)
+ if (event) room.emit(event)
+ },
+ }
+ return room
+}
+
+const setup = ({ config = CONFIG, random = 0.5 } = {}) => {
+ const clock = { now: 0 }
+
+ vi.spyOn(Math, 'random').mockReturnValue(random)
+ vi.spyOn(performance, 'now').mockImplementation(() => clock.now)
+
+ const room = createRoom()
+ const policy = new BackoffReconnectPolicy(config)
+ policy.attach(room as unknown as Room)
+
+ const delay = (retryCount: number, elapsedMs = 0) =>
+ policy.nextRetryDelayInMs({ retryCount, elapsedMs })
+
+ return { clock, room, policy, delay }
+}
+
+afterEach(() => {
+ vi.restoreAllMocks()
+})
+
+describe('BackoffReconnectPolicy', () => {
+ it('doubles the delay ceiling after each failed attempt, up to the cap', () => {
+ const { delay } = setup()
+
+ expect([0, 1, 2, 3, 4, 10].map((retryCount) => delay(retryCount))).toEqual([
+ 500, 1_500, 3_000, 6_000, 11_250, 11_250,
+ ])
+ })
+
+ it('draws the first delay anywhere below its ceiling (check two extremes)', () => {
+ expect(setup({ random: 0 }).delay(0)).toBe(0)
+ expect(setup({ random: 0.9999 }).delay(0)).toBe(999)
+ })
+
+ it('draws the next delays between half their ceiling and their ceiling (check two extremes)', () => {
+ expect(setup({ random: 0 }).delay(3)).toBe(4_000)
+ expect(setup({ random: 0.9999 }).delay(3)).toBe(7_999)
+ })
+
+ it('keeps the first delay of a reconnection under 5 s', () => {
+ const config = { ...CONFIG, base_delay_ms: 8_000 }
+ const { delay } = setup({ config, random: 0.9999 })
+
+ expect(delay(0)).toBeLessThan(MAX_FIRST_DELAY_MS)
+ expect(delay(1)).toBe(14_999)
+ })
+
+ it('starts the first attempt within 5 s of the drop', () => {
+ const config = { ...CONFIG, base_delay_ms: 8_000 }
+ const { clock, delay } = setup({ config })
+ expect(delay(0)).toBe(2_500)
+
+ clock.now = 4_000
+ expect(delay(0)).toBe(500)
+
+ clock.now = 6_000
+ expect(delay(0)).toBe(0)
+ })
+
+ it('never steps the backoff down within a reconnection', () => {
+ const { delay } = setup()
+
+ expect(delay(3)).toBe(6_000)
+ expect(delay(0)).toBe(6_000)
+ })
+
+ it('ignores the elapsed time reported by LiveKit', () => {
+ const { delay } = setup()
+
+ expect(delay(1, Date.now())).toBe(1_500)
+ })
+
+ it('picks up the backoff of a reconnection that ended shortly before', () => {
+ const { clock, room, delay } = setup()
+ delay(0)
+ delay(1)
+ delay(2)
+ room.emit(RoomEvent.Reconnected)
+
+ clock.now += 60_000
+ // One step above where the last reconnection ended (capped)
+ expect(delay(0)).toBe(2_500)
+ expect(delay(1)).toBe(11_250)
+ })
+
+ it('starts the backoff over once the connection held for the reset delay', () => {
+ const { clock, room, delay } = setup()
+ delay(0)
+ delay(1)
+ delay(2)
+ room.emit(RoomEvent.Reconnected)
+
+ clock.now += 120_000
+
+ expect(delay(0)).toBe(500)
+ })
+
+ it('gives up once the reconnection lasts longer than the give-up delay', () => {
+ const { clock, room, policy, delay } = setup()
+ delay(0)
+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)
+
+ clock.now = 299_999
+
+ expect(delay(20)).toBe(11_250)
+ expect(policy.gaveUp).toBe(false)
+
+ clock.now = 300_000
+
+ expect(delay(21)).toBeNull()
+ expect(policy.gaveUp).toBe(true)
+ })
+
+ it('keeps trying while only signaling is down', () => {
+ const { clock, room, policy, delay } = setup()
+ delay(0)
+ room.setState(
+ ConnectionState.SignalReconnecting,
+ RoomEvent.SignalReconnecting
+ )
+
+ clock.now = 600_000
+
+ expect(delay(40)).toBe(11_250)
+ expect(policy.gaveUp).toBe(false)
+
+ // The media connection failed as well
+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)
+
+ expect(delay(41)).toBeNull()
+ expect(policy.gaveUp).toBe(true)
+ })
+
+ it('gives up once the media connection failed, even back to resuming', () => {
+ const { clock, room, policy, delay } = setup()
+ delay(0)
+ room.setState(
+ ConnectionState.SignalReconnecting,
+ RoomEvent.SignalReconnecting
+ )
+ delay(1)
+
+ // The peer connections fail, then LiveKit tries to resume again
+ clock.now = 30_000
+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)
+ room.setState(
+ ConnectionState.SignalReconnecting,
+ RoomEvent.SignalReconnecting
+ )
+
+ clock.now = 299_999
+
+ expect(delay(20)).toBe(11_250)
+ expect(policy.gaveUp).toBe(false)
+
+ clock.now = 300_000
+
+ expect(delay(21)).toBeNull()
+ expect(policy.gaveUp).toBe(true)
+ })
+
+ it('starts a new reconnection once the room connects again', () => {
+ const { clock, room, policy, delay } = setup()
+ delay(0)
+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)
+ clock.now = 300_000
+
+ expect(delay(21)).toBeNull()
+
+ room.setState(ConnectionState.Disconnected, RoomEvent.Disconnected)
+ expect(policy.gaveUp).toBe(true)
+
+ room.setState(ConnectionState.Connecting)
+ room.setState(ConnectionState.Connected, RoomEvent.Connected)
+ expect(policy.gaveUp).toBe(false)
+
+ clock.now += 600_000
+
+ expect(delay(0)).toBe(500)
+
+ // The media connection of the previous reconnection is not lost anymore
+ room.setState(
+ ConnectionState.SignalReconnecting,
+ RoomEvent.SignalReconnecting
+ )
+ clock.now += 300_000
+
+ expect(delay(20)).toBe(11_250)
+ })
+
+ it('starts over when a new join replaces the reconnection', () => {
+ const { clock, room, policy, delay } = setup()
+ delay(0)
+ delay(3)
+ room.setState(ConnectionState.Reconnecting, RoomEvent.Reconnecting)
+
+ // LiveKitRoom calls room.connect() again in the middle of the reconnection
+ clock.now = 400_000
+ room.setState(ConnectionState.Connecting)
+
+ expect(delay(0)).toBe(500)
+ expect(policy.gaveUp).toBe(false)
+ })
+
+ it('reports how long a reconnection lasted, once', () => {
+ const { clock, room, policy, delay } = setup()
+ clock.now = 1_000
+ delay(0)
+
+ clock.now = 4_000
+ room.emit(RoomEvent.Reconnected)
+ expect(policy.lastReconnectionDurationMs).toBe(3_000)
+
+ // LiveKit can emit Reconnected several times for one recovery
+ room.emit(RoomEvent.Reconnected)
+ expect(policy.lastReconnectionDurationMs).toBeUndefined()
+ })
+})
+
+describe('createLiveKitRoom', () => {
+ it('builds the policy from the config endpoint values', () => {
+ vi.spyOn(Math, 'random').mockReturnValue(0.5)
+
+ const { reconnectPolicy } = createLiveKitRoom({}, CONFIG).options
+
+ expect(reconnectPolicy).toBeInstanceOf(BackoffReconnectPolicy)
+ expect(
+ reconnectPolicy.nextRetryDelayInMs({ retryCount: 2, elapsedMs: 0 })
+ ).toBe(3_000)
+ })
+
+ it('keeps the default policy when the backoff is missing or disabled', () => {
+ for (const config of [undefined, { ...CONFIG, backoff_enabled: false }]) {
+ expect(
+ createLiveKitRoom({}, config).options.reconnectPolicy
+ ).not.toBeInstanceOf(BackoffReconnectPolicy)
+ }
+ })
+})
diff --git a/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.ts b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.ts
new file mode 100644
index 00000000..b03353c4
--- /dev/null
+++ b/src/frontend/src/features/rooms/livekit/utils/reconnectPolicy.ts
@@ -0,0 +1,146 @@
+import {
+ ConnectionState,
+ Room,
+ RoomEvent,
+ type ReconnectContext,
+ type ReconnectPolicy,
+ type RoomOptions,
+} from 'livekit-client'
+import type { ApiConfig } from '@/api/useConfig'
+
+/**
+ * Until the first attempt of a reconnection starts, the room still reports
+ * Connected and LiveKit keeps checking its transport for a while.
+ * Around 10s after the initial drop, it will try a full websocket reconnection.
+ * Hence, the first attempt should start well before that.
+ */
+export const MAX_FIRST_DELAY_MS = 5_000
+
+type ReconnectConfig = NonNullable<ApiConfig['livekit']['reconnect']>
+
+type Reconnection = {
+ startedAt: number
+ // Higher than 0 when the previous reconnection ended shortly before
+ firstStep: number
+ failedAttempts: number
+ isMediaLost: boolean
+ gaveUp: boolean
+}
+
+/**
+ * When a LiveKit node crashes or Redis stalls, every client loses its signal
+ * connection at the same time. This policy implements an exponential backoff
+ * with some randomness to spread the reconnections out.
+ *
+ * The policy keeps its own clock since LiveKit calls the policy again on every
+ * new disconnection trigger
+ */
+export class BackoffReconnectPolicy implements ReconnectPolicy {
+ lastReconnectionDurationMs: number | undefined
+ private readonly config: ReconnectConfig
+ private reconnection: Reconnection | undefined
+ private lastRecovery: { at: number; step: number } | undefined
+
+ constructor(config: ReconnectConfig) {
+ this.config = config
+ }
+
+ attach(room: Room) {
+ room.on(RoomEvent.Connected, this.handleRecovery)
+ room.on(RoomEvent.Reconnected, this.handleRecovery)
+ room.on(RoomEvent.Reconnecting, this.handleMediaLost)
+ room.on(RoomEvent.ConnectionStateChanged, this.handleStateChange)
+ }
+
+ get gaveUp() {
+ return this.reconnection?.gaveUp ?? false
+ }
+
+ nextRetryDelayInMs({ retryCount }: ReconnectContext): number | null {
+ const now = performance.now()
+ this.reconnection ??= this.startReconnection(now)
+ const reconnection = this.reconnection
+
+ reconnection.failedAttempts = Math.max(
+ reconnection.failedAttempts,
+ retryCount
+ )
+
+ if (
+ now - reconnection.startedAt >= this.config.give_up_after_ms &&
+ reconnection.isMediaLost
+ ) {
+ reconnection.gaveUp = true
+ return null
+ }
+
+ const ceiling = Math.min(
+ this.config.max_delay_ms,
+ this.config.base_delay_ms *
+ 2 ** (reconnection.firstStep + reconnection.failedAttempts)
+ )
+
+ if (reconnection.failedAttempts === 0) {
+ // LiveKit re-arms its timer on each trigger: count from the first drop
+ const timeLeft = reconnection.startedAt + MAX_FIRST_DELAY_MS - now
+ return Math.floor(
+ Math.random() * Math.min(ceiling, Math.max(timeLeft, 0))
+ )
+ }
+ return Math.floor((ceiling / 2) * (1 + Math.random()))
+ }
+
+ private startReconnection(now: number): Reconnection {
+ const recovery = this.lastRecovery
+
+ const isLastDisconnectionRecent =
+ recovery !== undefined && now - recovery.at < this.config.backoff_reset_ms
+
+ return {
+ startedAt: now,
+ firstStep: isLastDisconnectionRecent ? recovery.step + 1 : 0,
+ failedAttempts: 0,
+ isMediaLost: false,
+ gaveUp: false,
+ }
+ }
+
+ private handleMediaLost = () => {
+ this.reconnection ??= this.startReconnection(performance.now())
+ this.reconnection.isMediaLost = true
+ }
+
+ private handleStateChange = (state: ConnectionState) => {
+ if (state === ConnectionState.Connecting) this.reconnection = undefined
+ }
+
+ private handleRecovery = () => {
+ const now = performance.now()
+ if (this.reconnection) {
+ const { startedAt, firstStep, failedAttempts } = this.reconnection
+ this.lastRecovery = { at: now, step: firstStep + failedAttempts }
+ this.lastReconnectionDurationMs = now - startedAt
+ this.reconnection = undefined
+ } else {
+ // LiveKit can emit Reconnected several times for a full reconnection
+ // We want to avoid duplicating the reconnection duration in PostHog
+ this.lastReconnectionDurationMs = undefined
+ }
+ }
+}
+
+/**
+ * Build a room with the reconnection policy from the config endpoint.
+ * Without it, the room keeps the LiveKit default policy
+ */
+export const createLiveKitRoom = (
+ options: RoomOptions,
+ config?: ReconnectConfig
+): Room => {
+ if (!config?.backoff_enabled) return new Room(options)
+
+ const reconnectPolicy = new BackoffReconnectPolicy(config)
+ const room = new Room({ ...options, reconnectPolicy })
+ reconnectPolicy.attach(room)
+ return room
+}