A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(voice): retune the remote playback leveller and measure it in float (#1784)
Hampus17eab4324562
diff --git a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.tsindex dc1e696a3..1b87131b5 100644--- a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts+++ b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts@@ -8,6 +8,7 @@ import {import type {Participant, Room} from 'livekit-client';import {describe, expect, it} from 'vitest';import {+ __TEST__,createLivekitParticipantSnapshot,createLivekitParticipantSnapshotsFromRoom,createVoiceParticipantSnapshot,@@ -59,6 +60,48 @@ function commandsOf(snapshot: VoiceRemoteSpeakingSnapshot): ReadonlyArray<VoiceRreturn snapshot.context.commands;}+const LEVELLER_IDENTITY = 'user_1_a';+const LEVELLER_TICK_MS = 50;++function attachedRemoteSnapshot(): VoiceRemoteSpeakingSnapshot {+ const attached = transitionVoiceRemoteSpeakingSnapshot(createVoiceRemoteSpeakingSnapshot(), {+ type: 'remote.attach',+ identity: LEVELLER_IDENTITY,+ track: {},+ });+ return transitionVoiceRemoteSpeakingSnapshot(attached, {type: 'remote.clearCommands'});+}++function runRemoteTicks(+ snapshot: VoiceRemoteSpeakingSnapshot,+ options: {rms: number; threshold: number; ticks: number; startMs: number},+): {snapshot: VoiceRemoteSpeakingSnapshot; commands: Array<VoiceRemoteSpeakingCommand>; endMs: number} {+ let next = snapshot;+ let nowMs = options.startMs;+ const commands: Array<VoiceRemoteSpeakingCommand> = [];+ for (let i = 0; i < options.ticks; i++) {+ next = transitionVoiceRemoteSpeakingSnapshot(next, {+ type: 'remote.tick',+ identity: LEVELLER_IDENTITY,+ rms: options.rms,+ threshold: options.threshold,+ nowMs,+ });+ commands.push(...commandsOf(next));+ next = transitionVoiceRemoteSpeakingSnapshot(next, {type: 'remote.clearCommands'});+ nowMs += LEVELLER_TICK_MS;+ }+ return {snapshot: next, commands, endMs: nowMs};+}++function playbackBoostOf(snapshot: VoiceRemoteSpeakingSnapshot): number {+ return snapshot.context.analysers.get(LEVELLER_IDENTITY)?.playbackBoost ?? Number.NaN;+}++function appliedBoostOf(snapshot: VoiceRemoteSpeakingSnapshot): number {+ return snapshot.context.analysers.get(LEVELLER_IDENTITY)?.appliedBoost ?? Number.NaN;+}+describe('VoiceParticipantStateMachine participants', () => {it('keeps participant snapshot references stable when an upsert is equal', () => {const p = participant('user_42_conn-a', {@@ -304,43 +347,86 @@ describe('VoiceParticipantStateMachine remote speaking', () => {expect(commandsOf(snapshot)).toEqual([{type: 'rehydrateRemoteAnalysers'}]);});- it('resets playback boost after quiet audio and on detach', () => {- let snapshot = createVoiceRemoteSpeakingSnapshot();- snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {+ it('holds the playback boost through silence and releases it back to unity', () => {+ let snapshot = transitionVoiceRemoteSpeakingSnapshot(createVoiceRemoteSpeakingSnapshot(), {type: 'remote.attach',- identity: 'user_1_a',+ identity: LEVELLER_IDENTITY,track: {},});- expect(commandsOf(snapshot)).toEqual([{type: 'setPlaybackBoost', identity: 'user_1_a', boost: 1}]);+ expect(commandsOf(snapshot)).toEqual([{type: 'setPlaybackBoost', identity: LEVELLER_IDENTITY, boost: 1}]);+ expect(playbackBoostOf(snapshot)).toBe(1);+ expect(appliedBoostOf(snapshot)).toBe(1);snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});- snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {- type: 'remote.tick',- identity: 'user_1_a',- rms: 0.006,- threshold: 1,- nowMs: 10,- });- expect(commandsOf(snapshot)[0]).toMatchObject({type: 'setPlaybackBoost', identity: 'user_1_a'});- expect(snapshot.context.analysers.get('user_1_a')?.playbackBoost).toBeGreaterThan(1);- snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});+ const settled = runRemoteTicks(snapshot, {rms: 0.03, threshold: 0.006, ticks: 60, startMs: 0});+ expect(playbackBoostOf(settled.snapshot)).toBeGreaterThan(2.5);+ expect(settled.commands.some((command) => command.type === 'setPlaybackBoost')).toBe(true);- snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {- type: 'remote.tick',- identity: 'user_1_a',- rms: 0,- threshold: 1,- nowMs: 20,- });- expect(commandsOf(snapshot)).toEqual([{type: 'clearPlaybackBoost', identity: 'user_1_a'}]);- expect(snapshot.context.analysers.get('user_1_a')?.playbackBoost).toBe(1);+ const settledBoost = playbackBoostOf(settled.snapshot);+ const held = runRemoteTicks(settled.snapshot, {rms: 0, threshold: 0.006, ticks: 60, startMs: settled.endMs});+ expect(held.endMs - settled.endMs).toBe(__TEST__.REMOTE_PLAYBACK_SILENCE_HOLD_MS);+ expect(playbackBoostOf(held.snapshot)).toBe(settledBoost);+ expect(held.commands.some((command) => command.type === 'clearPlaybackBoost')).toBe(false);- snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});- snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.detach', identity: 'user_1_a'});+ const released = runRemoteTicks(held.snapshot, {rms: 0, threshold: 0.006, ticks: 700, startMs: held.endMs});+ expect(released.commands.filter((command) => command.type === 'clearPlaybackBoost')).toHaveLength(1);+ expect(playbackBoostOf(released.snapshot)).toBe(1);+ expect(appliedBoostOf(released.snapshot)).toBe(1);++ snapshot = transitionVoiceRemoteSpeakingSnapshot(released.snapshot, {+ type: 'remote.detach',+ identity: LEVELLER_IDENTITY,+ });expect(commandsOf(snapshot)).toEqual([- {type: 'setAudioLevelSpeaking', identity: 'user_1_a', speaking: false},- {type: 'clearPlaybackBoost', identity: 'user_1_a'},+ {type: 'setAudioLevelSpeaking', identity: LEVELLER_IDENTITY, speaking: false},+ {type: 'clearPlaybackBoost', identity: LEVELLER_IDENTITY},+ ]);+ });++ it('holds the boost through an inter-syllabic pause instead of pumping', () => {+ const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.02, threshold: 0.006, ticks: 120, startMs: 0});+ expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(__TEST__.REMOTE_PLAYBACK_MAX_BOOST, 2);++ const settledBoost = playbackBoostOf(settled.snapshot);+ const paused = runRemoteTicks(settled.snapshot, {rms: 0.002, threshold: 0.006, ticks: 4, startMs: settled.endMs});+ expect(Math.abs(playbackBoostOf(paused.snapshot) - settledBoost)).toBeLessThan(settledBoost * 0.01);+ expect(paused.commands).toHaveLength(0);+ });++ it('integrates steps the emit guard suppresses', () => {+ const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.086, threshold: 0.006, ticks: 200, startMs: 0});+ expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(1.0465, 4);+ expect(appliedBoostOf(settled.snapshot)).toBeLessThan(playbackBoostOf(settled.snapshot));+ });++ it('targets roughly -21 dBFS of playback level', () => {+ const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.045, threshold: 0.006, ticks: 200, startMs: 0});+ expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(2, 4);+ expect(playbackBoostOf(settled.snapshot) * 0.045).toBeCloseTo(__TEST__.REMOTE_PLAYBACK_TARGET_RMS, 4);+ });++ it('never integrates on a zero threshold', () => {+ const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0, threshold: 0, ticks: 10, startMs: 0});+ expect(playbackBoostOf(settled.snapshot)).toBe(1);+ expect(settled.commands.some((command) => command.type === 'setPlaybackBoost')).toBe(false);+ });++ it('clears a stale boost when reattaching a different track', () => {+ const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.02, threshold: 0.006, ticks: 60, startMs: 0});+ expect(appliedBoostOf(settled.snapshot)).toBeGreaterThan(1);++ const reattached = transitionVoiceRemoteSpeakingSnapshot(settled.snapshot, {+ type: 'remote.attach',+ identity: LEVELLER_IDENTITY,+ track: {},+ });+ expect(commandsOf(reattached)).toEqual([+ {type: 'setAudioLevelSpeaking', identity: LEVELLER_IDENTITY, speaking: false},+ {type: 'clearPlaybackBoost', identity: LEVELLER_IDENTITY},+ {type: 'setPlaybackBoost', identity: LEVELLER_IDENTITY, boost: 1},]);+ expect(playbackBoostOf(reattached)).toBe(1);+ expect(appliedBoostOf(reattached)).toBe(1);});it('clears stale speaking flags after detach and clear', () => {diff --git a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.tsindex 09af54d7d..4164d921b 100644--- a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts+++ b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts@@ -51,6 +51,7 @@ export interface RemoteSpeakingAnalyserState {belowSinceMs: number | null;aboveSinceMs: number | null;playbackBoost: number;+ appliedBoost: number;}export type VoiceRemoteSpeakingCommand =@@ -76,10 +77,15 @@ export type VoiceRemoteSpeakingEvent =const EMPTY_PARTICIPANTS: Readonly<Record<string, LivekitParticipantSnapshot>> = {};const EMPTY_REMOTE_COMMANDS: ReadonlyArray<VoiceRemoteSpeakingCommand> = [];-const REMOTE_PLAYBACK_TARGET_RMS = 0.025;-const REMOTE_PLAYBACK_MIN_RMS = 0.004;+const REMOTE_PLAYBACK_TARGET_RMS = 0.09;+const REMOTE_PLAYBACK_MIN_RMS = 0.0025;const REMOTE_PLAYBACK_MAX_BOOST = 3;-const REMOTE_PLAYBACK_MIN_CHANGE = 0.05;+const REMOTE_PLAYBACK_MIN_CHANGE_RATIO = 1.03;+const REMOTE_PLAYBACK_BOOST_UP_COEFF = 0.08;+const REMOTE_PLAYBACK_BOOST_DOWN_COEFF = 0.35;+const REMOTE_PLAYBACK_SILENCE_HOLD_MS = 3000;+const REMOTE_PLAYBACK_SILENCE_RELEASE = 0.99;+const REMOTE_PLAYBACK_UNITY_EPSILON = 0.01;export const extractParticipantUserId = (identity: string): string | null => {const match = identity.match(/^user_(\d+)(?:_(.+))?$/);@@ -308,40 +314,64 @@ function attachRemoteAnalyser(belowSinceMs: null,aboveSinceMs: null,playbackBoost: 1,+ appliedBoost: 1,});const commands: Array<VoiceRemoteSpeakingCommand> = [];if (existing?.speaking) commands.push({type: 'setAudioLevelSpeaking', identity, speaking: false});- if (existing && existing.playbackBoost !== 1) commands.push({type: 'clearPlaybackBoost', identity});+ if (existing && existing.appliedBoost !== 1) commands.push({type: 'clearPlaybackBoost', identity});commands.push({type: 'setPlaybackBoost', identity, boost: 1});return appendRemoteCommands({...context, analysers}, commands);}-function updateRemotePlaybackBoost(+function settleRemotePlaybackBoost(entry: RemoteSpeakingAnalyserState,- rms: number,+ nextBoost: number,): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} {- if (!Number.isFinite(rms) || rms <= REMOTE_PLAYBACK_MIN_RMS) {- if (entry.playbackBoost === 1) return {entry, command: null};- return {- entry: {...entry, playbackBoost: 1},- command: {type: 'clearPlaybackBoost', identity: entry.identity},- };- }- const desiredBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, REMOTE_PLAYBACK_TARGET_RMS / rms));- const nextBoost =- desiredBoost > entry.playbackBoost- ? entry.playbackBoost + (desiredBoost - entry.playbackBoost) * 0.4- : entry.playbackBoost + (desiredBoost - entry.playbackBoost) * 0.15;const clampedBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, nextBoost));- if (Math.abs(clampedBoost - entry.playbackBoost) < REMOTE_PLAYBACK_MIN_CHANGE) {- return {entry, command: null};+ const ratio =+ clampedBoost >= entry.appliedBoost ? clampedBoost / entry.appliedBoost : entry.appliedBoost / clampedBoost;+ if (ratio < REMOTE_PLAYBACK_MIN_CHANGE_RATIO) {+ if (clampedBoost === entry.playbackBoost) return {entry, command: null};+ return {entry: {...entry, playbackBoost: clampedBoost}, command: null};}return {- entry: {...entry, playbackBoost: clampedBoost},+ entry: {...entry, playbackBoost: clampedBoost, appliedBoost: clampedBoost},command: {type: 'setPlaybackBoost', identity: entry.identity, boost: clampedBoost},};}+function snapRemotePlaybackBoost(entry: RemoteSpeakingAnalyserState): {+ entry: RemoteSpeakingAnalyserState;+ command: VoiceRemoteSpeakingCommand;+} {+ return {+ entry: {...entry, playbackBoost: 1, appliedBoost: 1},+ command: {type: 'clearPlaybackBoost', identity: entry.identity},+ };+}++function updateRemotePlaybackBoost(+ entry: RemoteSpeakingAnalyserState,+ rms: number,+ threshold: number,+ nowMs: number,+): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} {+ const isActiveSpeech = Number.isFinite(rms) && rms >= REMOTE_PLAYBACK_MIN_RMS && rms >= threshold && entry.speaking;+ if (isActiveSpeech) {+ const desiredBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, REMOTE_PLAYBACK_TARGET_RMS / rms));+ const coefficient =+ desiredBoost > entry.playbackBoost ? REMOTE_PLAYBACK_BOOST_UP_COEFF : REMOTE_PLAYBACK_BOOST_DOWN_COEFF;+ return settleRemotePlaybackBoost(entry, entry.playbackBoost + (desiredBoost - entry.playbackBoost) * coefficient);+ }+ if (entry.playbackBoost <= 1) return {entry, command: null};+ if (entry.belowSinceMs === null || nowMs - entry.belowSinceMs < REMOTE_PLAYBACK_SILENCE_HOLD_MS) {+ return {entry, command: null};+ }+ const releasedBoost = 1 + (entry.playbackBoost - 1) * REMOTE_PLAYBACK_SILENCE_RELEASE;+ if (releasedBoost - 1 <= REMOTE_PLAYBACK_UNITY_EPSILON) return snapRemotePlaybackBoost(entry);+ return settleRemotePlaybackBoost(entry, releasedBoost);+}+function tickRemoteAnalyser(context: VoiceRemoteSpeakingMachineContext,event: Extract<VoiceRemoteSpeakingEvent, {type: 'remote.tick'}>,@@ -351,8 +381,7 @@ function tickRemoteAnalyser(if (event.trackEnded) return detachRemoteAnalyser(context, event.identity);const commands: Array<VoiceRemoteSpeakingCommand> = [];- let {entry, command} = updateRemotePlaybackBoost(existing, event.rms);- if (command) commands.push(command);+ let entry = existing;if (event.rms >= event.threshold) {const aboveSinceMs = entry.aboveSinceMs ?? event.nowMs;entry = {@@ -376,6 +405,9 @@ function tickRemoteAnalyser(commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: false});}}+ const boostUpdate = updateRemotePlaybackBoost(entry, event.rms, event.threshold, event.nowMs);+ entry = boostUpdate.entry;+ if (boostUpdate.command) commands.push(boostUpdate.command);if (entry === existing && commands.length === 0) return context;const analysers = new Map(context.analysers);analysers.set(event.identity, entry);@@ -509,6 +541,13 @@ export function clearVoiceRemoteSpeakingCommands(snapshot: VoiceRemoteSpeakingSnreturn transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});}+export const __TEST__ = {+ REMOTE_PLAYBACK_TARGET_RMS,+ REMOTE_PLAYBACK_MIN_RMS,+ REMOTE_PLAYBACK_MAX_BOOST,+ REMOTE_PLAYBACK_SILENCE_HOLD_MS,+};+export function findParticipantSnapshotByUserIdAndConnectionId(participants: Readonly<Record<string, LivekitParticipantSnapshot>>,userId: string,diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.test.tsnew file mode 100644index 000000000..383f2c420--- /dev/null+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.test.ts@@ -0,0 +1,35 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {describe, expect, it, vi} from 'vitest';+import {computeTimeDomainRms} from './VoiceEngineV2AppRemoteSpeakingAdapter';++vi.mock('@app/features/voice/state/ParticipantVolume', () => ({default: {}}));+vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: {getVadThreshold: () => 50}}));++describe('computeTimeDomainRms', () => {+ it('returns the amplitude of an alternating full-scale-relative signal', () => {+ const samples = new Float32Array(512);+ for (let i = 0; i < samples.length; i++) {+ samples[i] = i % 2 === 0 ? 0.09 : -0.09;+ }+ expect(computeTimeDomainRms(samples)).toBe(Math.fround(0.09));+ });++ it('returns zero for digital silence', () => {+ expect(computeTimeDomainRms(new Float32Array(512))).toBe(0);+ });++ it('returns zero for an empty buffer', () => {+ expect(computeTimeDomainRms(new Float32Array(0))).toBe(0);+ });++ it('spreads a single full-scale sample across the window', () => {+ const samples = new Float32Array(512);+ samples[17] = 1;+ expect(computeTimeDomainRms(samples)).toBeCloseTo(1 / Math.sqrt(512), 12);+ });++ it('resolves levels far below one 8-bit quantisation step', () => {+ expect(computeTimeDomainRms(Float32Array.of(0.0005, -0.0005))).toBe(Math.fround(0.0005));+ });+});diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.tsindex 05f7ac70a..08e671d8e 100644--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts@@ -30,12 +30,22 @@ const logger = new Logger('VoiceEngineV2AppRemoteSpeakingAdapter');export const REMOTE_SPEAKING_ANALYSER_INTERVAL_MS = 50;export const REMOTE_SPEAKING_ANALYSER_HANDLES_CAP = 256;+export function computeTimeDomainRms(samples: Float32Array): number {+ if (samples.length === 0) return 0;+ let sumSquares = 0;+ for (let i = 0; i < samples.length; i++) {+ const sample = samples[i]!;+ sumSquares += sample * sample;+ }+ return Math.sqrt(sumSquares / samples.length);+}+interface AnalyserHandle {identity: string;track: MediaStreamTrack;source: MediaStreamAudioSourceNode;analyser: AnalyserNode;- samples: Uint8Array<ArrayBuffer>;+ samples: Float32Array<ArrayBuffer>;nextTickAtMs: number;}@@ -73,9 +83,7 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter {break;case 'setPlaybackBoost':setRemoteVoicePlaybackBoost(command.identity, command.boost);- if (command.boost !== 1) {- this.reapplyParticipantVolume(command.identity);- }+ this.reapplyParticipantVolume(command.identity);break;case 'clearPlaybackBoost':clearRemoteVoicePlaybackBoost(command.identity);@@ -193,7 +201,7 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter {track: mediaStreamTrack,source,analyser,- samples: new Uint8Array(new ArrayBuffer(analyser.fftSize)),+ samples: new Float32Array(new ArrayBuffer(analyser.fftSize * Float32Array.BYTES_PER_ELEMENT)),nextTickAtMs: this.nowMs() + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS,};this.analysers.set(identity, handle);@@ -315,13 +323,8 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter {}return;}- handle.analyser.getByteTimeDomainData(handle.samples);- let sumSquares = 0;- for (let i = 0; i < handle.samples.length; i++) {- const normalized = (handle.samples[i]! - 128) / 128;- sumSquares += normalized * normalized;- }- const rms = Math.sqrt(sumSquares / handle.samples.length);+ handle.analyser.getFloatTimeDomainData(handle.samples);+ const rms = computeTimeDomainRms(handle.samples);const threshold = getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold());this.applyCommands(this.transition({type: 'remote.tick', identity: handle.identity, rms, threshold, nowMs}));handle.nextTickAtMs = nowMs + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS;