A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import {advanceSpeakingHysteresis, type SpeakingHysteresis} from '@app/features/voice/engine/VoiceSpeakingThreshold';
import type {VoiceConnectionQuality} from '@app/features/voice/engine/VoiceTrackSource';
import {assign, initialTransition, type SnapshotFrom, setup, transition} from 'xstate';
export type LivekitParticipantSnapshot = Readonly<{
identity: string;
userId: string | null;
connectionId: string | null;
sid: string;
isLocal: boolean;
isSpeaking: boolean;
isAudioLevelSpeaking: boolean;
connectionQuality: VoiceConnectionQuality;
metadata?: string;
attributes: Readonly<Record<string, string>>;
audioTrackSids: ReadonlyArray<string>;
videoTrackSids: ReadonlyArray<string>;
isMicrophoneEnabled: boolean;
isCameraEnabled: boolean;
isScreenShareEnabled: boolean;
isScreenShareAudioEnabled: boolean;
joinedAt: number | null;
lastSpokeAt: number | null;
}>;
interface RemoteSpeakingAnalyserState extends SpeakingHysteresis {
identity: string;
track: unknown;
playbackBoost: number;
appliedBoost: number;
}
export type VoiceRemoteSpeakingCommand =
| {type: 'setAudioLevelSpeaking'; identity: string; speaking: boolean}
| {type: 'setPlaybackBoost'; identity: string; boost: number}
| {type: 'clearPlaybackBoost'; identity: string};
interface VoiceRemoteSpeakingMachineContext {
analysers: ReadonlyMap<string, RemoteSpeakingAnalyserState>;
commands: ReadonlyArray<VoiceRemoteSpeakingCommand>;
}
export type VoiceRemoteSpeakingEvent =
| {type: 'remote.attach'; identity: string; track: unknown}
| {type: 'remote.detach'; identity: string}
| {type: 'remote.tick'; identity: string; rms: number; threshold: number; nowMs: number; trackEnded?: boolean}
| {type: 'remote.clear'}
| {type: 'remote.clearCommands'};
const EMPTY_REMOTE_COMMANDS: ReadonlyArray<VoiceRemoteSpeakingCommand> = [];
const REMOTE_PLAYBACK_TARGET_RMS = 0.09;
const REMOTE_PLAYBACK_MIN_RMS = 0.0025;
const REMOTE_PLAYBACK_MAX_BOOST = 3;
const REMOTE_PLAYBACK_MIN_CHANGE_RATIO = 1.03;
const REMOTE_PLAYBACK_BOOST_UP_COEFF = 0.08;
const REMOTE_PLAYBACK_BOOST_DOWN_COEFF = 0.35;
const REMOTE_PLAYBACK_SILENCE_HOLD_MS = 3000;
const REMOTE_PLAYBACK_SILENCE_RELEASE = 0.99;
const REMOTE_PLAYBACK_UNITY_EPSILON = 0.01;
function remoteContext(
analysers: ReadonlyMap<string, RemoteSpeakingAnalyserState> = new Map(),
commands: ReadonlyArray<VoiceRemoteSpeakingCommand> = EMPTY_REMOTE_COMMANDS,
): VoiceRemoteSpeakingMachineContext {
return {analysers, commands};
}
function appendRemoteCommands(
context: VoiceRemoteSpeakingMachineContext,
commands: ReadonlyArray<VoiceRemoteSpeakingCommand>,
): VoiceRemoteSpeakingMachineContext {
if (commands.length === 0) return context;
return {...context, commands: [...context.commands, ...commands]};
}
function detachRemoteAnalyser(
context: VoiceRemoteSpeakingMachineContext,
identity: string,
): VoiceRemoteSpeakingMachineContext {
if (!context.analysers.has(identity)) return context;
const analysers = new Map(context.analysers);
analysers.delete(identity);
return appendRemoteCommands({...context, analysers}, [
{type: 'setAudioLevelSpeaking', identity, speaking: false},
{type: 'clearPlaybackBoost', identity},
]);
}
function attachRemoteAnalyser(
context: VoiceRemoteSpeakingMachineContext,
identity: string,
track: unknown,
): VoiceRemoteSpeakingMachineContext {
const existing = context.analysers.get(identity);
if (existing?.track === track) return context;
const analysers = new Map(context.analysers);
analysers.set(identity, {
identity,
track,
speaking: false,
belowSinceMs: null,
aboveSinceMs: null,
playbackBoost: 1,
appliedBoost: 1,
});
const commands: Array<VoiceRemoteSpeakingCommand> = [];
if (existing?.speaking) commands.push({type: 'setAudioLevelSpeaking', identity, speaking: false});
if (existing && existing.appliedBoost !== 1) commands.push({type: 'clearPlaybackBoost', identity});
commands.push({type: 'setPlaybackBoost', identity, boost: 1});
return appendRemoteCommands({...context, analysers}, commands);
}
function settleRemotePlaybackBoost(
entry: RemoteSpeakingAnalyserState,
nextBoost: number,
): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} {
const clampedBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, nextBoost));
const ratio =
clampedBoost >= entry.appliedBoost ? clampedBoost / entry.appliedBoost : entry.appliedBoost / clampedBoost;
if (ratio < REMOTE_PLAYBACK_MIN_CHANGE_RATIO) {
if (clampedBoost === entry.playbackBoost) return {entry, command: null};
return {entry: {...entry, playbackBoost: clampedBoost}, command: null};
}
return {
entry: {...entry, playbackBoost: clampedBoost, appliedBoost: clampedBoost},
command: {type: 'setPlaybackBoost', identity: entry.identity, boost: clampedBoost},
};
}
function snapRemotePlaybackBoost(entry: RemoteSpeakingAnalyserState): {
entry: RemoteSpeakingAnalyserState;
command: VoiceRemoteSpeakingCommand;
} {
return {
entry: {...entry, playbackBoost: 1, appliedBoost: 1},
command: {type: 'clearPlaybackBoost', identity: entry.identity},
};
}
function updateRemotePlaybackBoost(
entry: RemoteSpeakingAnalyserState,
rms: number,
threshold: number,
nowMs: number,
): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} {
const isActiveSpeech = Number.isFinite(rms) && rms >= REMOTE_PLAYBACK_MIN_RMS && rms >= threshold && entry.speaking;
if (isActiveSpeech) {
const desiredBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, REMOTE_PLAYBACK_TARGET_RMS / rms));
const coefficient =
desiredBoost > entry.playbackBoost ? REMOTE_PLAYBACK_BOOST_UP_COEFF : REMOTE_PLAYBACK_BOOST_DOWN_COEFF;
return settleRemotePlaybackBoost(entry, entry.playbackBoost + (desiredBoost - entry.playbackBoost) * coefficient);
}
if (entry.playbackBoost <= 1) return {entry, command: null};
if (entry.belowSinceMs === null || nowMs - entry.belowSinceMs < REMOTE_PLAYBACK_SILENCE_HOLD_MS) {
return {entry, command: null};
}
const releasedBoost = 1 + (entry.playbackBoost - 1) * REMOTE_PLAYBACK_SILENCE_RELEASE;
if (releasedBoost - 1 <= REMOTE_PLAYBACK_UNITY_EPSILON) return snapRemotePlaybackBoost(entry);
return settleRemotePlaybackBoost(entry, releasedBoost);
}
function tickRemoteAnalyser(
context: VoiceRemoteSpeakingMachineContext,
event: Extract<VoiceRemoteSpeakingEvent, {type: 'remote.tick'}>,
): VoiceRemoteSpeakingMachineContext {
const existing = context.analysers.get(event.identity);
if (!existing) return context;
if (event.trackEnded) return detachRemoteAnalyser(context, event.identity);
const commands: Array<VoiceRemoteSpeakingCommand> = [];
let entry: RemoteSpeakingAnalyserState = {
...existing,
...advanceSpeakingHysteresis(existing, event.rms, event.threshold, event.nowMs),
};
if (entry.speaking !== existing.speaking) {
commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: entry.speaking});
}
const boostUpdate = updateRemotePlaybackBoost(entry, event.rms, event.threshold, event.nowMs);
entry = boostUpdate.entry;
if (boostUpdate.command) commands.push(boostUpdate.command);
if (entry === existing && commands.length === 0) return context;
const analysers = new Map(context.analysers);
analysers.set(event.identity, entry);
return appendRemoteCommands({...context, analysers}, commands);
}
function detachAllRemoteAnalysers(context: VoiceRemoteSpeakingMachineContext): VoiceRemoteSpeakingMachineContext {
if (context.analysers.size === 0) return context;
const commands: Array<VoiceRemoteSpeakingCommand> = [];
for (const identity of context.analysers.keys()) {
commands.push({type: 'setAudioLevelSpeaking', identity, speaking: false});
commands.push({type: 'clearPlaybackBoost', identity});
}
return appendRemoteCommands(remoteContext(new Map(), context.commands), commands);
}
const voiceRemoteSpeakingStateMachine = setup({
types: {} as {
context: VoiceRemoteSpeakingMachineContext;
events: VoiceRemoteSpeakingEvent;
},
actions: {
attach: assign(({context, event}) =>
event.type === 'remote.attach' ? attachRemoteAnalyser(context, event.identity, event.track) : context,
),
detach: assign(({context, event}) =>
event.type === 'remote.detach' ? detachRemoteAnalyser(context, event.identity) : context,
),
tick: assign(({context, event}) => (event.type === 'remote.tick' ? tickRemoteAnalyser(context, event) : context)),
clear: assign(({context}) => detachAllRemoteAnalysers(context)),
clearCommands: assign(({context}) => ({...context, commands: EMPTY_REMOTE_COMMANDS})),
},
}).createMachine({
id: 'voiceRemoteSpeaking',
context: () => remoteContext(),
on: {
'remote.attach': {actions: 'attach'},
'remote.detach': {actions: 'detach'},
'remote.tick': {actions: 'tick'},
'remote.clear': {actions: 'clear'},
'remote.clearCommands': {actions: 'clearCommands'},
},
});
export type VoiceRemoteSpeakingSnapshot = SnapshotFrom<typeof voiceRemoteSpeakingStateMachine>;
export function createVoiceRemoteSpeakingSnapshot(): VoiceRemoteSpeakingSnapshot {
return initialTransition(voiceRemoteSpeakingStateMachine)[0];
}
export function transitionVoiceRemoteSpeakingSnapshot(
snapshot: VoiceRemoteSpeakingSnapshot,
event: VoiceRemoteSpeakingEvent,
): VoiceRemoteSpeakingSnapshot {
return transition(voiceRemoteSpeakingStateMachine, snapshot, event)[0] as VoiceRemoteSpeakingSnapshot;
}
export function clearVoiceRemoteSpeakingCommands(snapshot: VoiceRemoteSpeakingSnapshot): VoiceRemoteSpeakingSnapshot {
return transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});
}