Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(calls): call noise cancellation (#4557)
Will Hutchinson0a4b867e853a
diff --git a/js/app/packages/app/lib/analytics/app-events.ts b/js/app/packages/app/lib/analytics/app-events.tsindex 90f168f62d..f366312b52 100644--- a/js/app/packages/app/lib/analytics/app-events.ts+++ b/js/app/packages/app/lib/analytics/app-events.ts@@ -140,6 +140,27 @@ export type AppEvents = {channelId: string;callId?: string;} & Record<string, unknown>;+ // Mic audio-processing state transitions (noise suppression attach /+ // detach / fallback). `constraintMismatch` flags engines that report+ // different track settings than were requested (mode changes that silently+ // no-oped); `sampleRate` <= 16000 indicates a Bluetooth HFP capture path.+ call_audio_processing: {+ event: string;+ channelId: string;+ preferredMode: string;+ activeMode: string;+ constraintMismatch: boolean;+ } & Record<string, unknown>;+ // Per-call summary of remote-audio decode health, flushed at room teardown.+ // High concealment alongside clean capture-side state points a "muddled+ // voices" report at the network/playback leg, not noise suppression.+ call_audio_receiver_stats: {+ channelId: string;+ callId?: string;+ sampledIntervals: number;+ badIntervals: number;+ maxConcealmentRate: number;+ } & Record<string, unknown>;block_pdf_definition_open: Record<string, unknown>;block_pdf_section_open: Record<string, unknown>;diff --git a/js/app/packages/channel/Call/CallContext.tsx b/js/app/packages/channel/Call/CallContext.tsxindex 784ec34933..22d6e2bc80 100644--- a/js/app/packages/channel/Call/CallContext.tsx+++ b/js/app/packages/channel/Call/CallContext.tsx@@ -123,20 +123,27 @@ function microphoneCaptureOptions(deviceId?: string | null): AudioCaptureOptions {const useBrowserProcessing = mode === 'browser';+ const noiseSuppressionSupported =+ supportsNativeAudioProcessingConstraint('noiseSuppression');return {- // AGC can raise residual background artifacts during quiet speech/silence.- // Keep it disabled for every mode; Krisp/browser NS should not be stacked- // with a gain stage that makes suppressed noise audible again.+ // Browser AGC runs inside the capture pipeline (AEC → AGC → NS), i.e.+ // upstream of any track processor — it feeds Krisp a healthy input level+ // and cannot re-amplify the filter's residue (that would need a gain+ // stage after the processor, which doesn't exist here). AGC-off left+ // quiet mics at low SNR, and Krisp/browser NS then gated their speech....(supportsNativeAudioProcessingConstraint('autoGainControl')- ? { autoGainControl: false }+ ? { autoGainControl: true }: {}),echoCancellation: true,- ...(supportsNativeAudioProcessingConstraint('noiseSuppression')+ ...(noiseSuppressionSupported? { noiseSuppression: useBrowserProcessing }: {}),+ // Voice isolation is a second, platform-level suppression stage; running+ // it on top of noiseSuppression cascades filters and attenuates voice.+ // Use it only as the fallback when noiseSuppression is unsupported....(supportsNativeAudioProcessingConstraint('voiceIsolation')- ? { voiceIsolation: useBrowserProcessing }+ ? { voiceIsolation: useBrowserProcessing && !noiseSuppressionSupported }: {}),...(deviceId ? { deviceId: { exact: deviceId } } : {}),};@@ -146,16 +153,25 @@ function nativeAudioProcessingConstraints(mode: MicNoiseSuppressionMode): NativeAudioProcessingConstraints {const useBrowserProcessing = mode === 'browser';+ const noiseSuppressionSupported =+ supportsNativeAudioProcessingConstraint('noiseSuppression');return {+ // applyConstraints() replaces the track's entire constraint set, so echo+ // cancellation must be restated here or every mode change silently drops+ // it back to unconstrained.+ echoCancellation: true,+ // AGC stays on in every mode — see microphoneCaptureOptions....(supportsNativeAudioProcessingConstraint('autoGainControl')- ? { autoGainControl: false }+ ? { autoGainControl: true }: {}),- ...(supportsNativeAudioProcessingConstraint('noiseSuppression')+ ...(noiseSuppressionSupported? { noiseSuppression: useBrowserProcessing }: {}),+ // See microphoneCaptureOptions: voiceIsolation is only the fallback when+ // noiseSuppression is unsupported, never a second stacked layer....(supportsNativeAudioProcessingConstraint('voiceIsolation')- ? { voiceIsolation: useBrowserProcessing }+ ? { voiceIsolation: useBrowserProcessing && !noiseSuppressionSupported }: {}),};}@@ -468,6 +484,21 @@ function createCallState() {| NativeAudioProcessingConstraints| undefined;+ // A requested constraint the engine reports back differently in settings+ // means the processing state was never actually applied (live-track+ // applyConstraints is not honored everywhere) — the key signal behind+ // "toggled noise suppression and nothing changed" reports.+ const constraintMismatch = (+ ['autoGainControl', 'echoCancellation', 'noiseSuppression'] as const+ ).some((name) => {+ const requested = constraints?.[name];+ return (+ typeof requested === 'boolean' &&+ settings?.[name] !== undefined &&+ settings[name] !== requested+ );+ });+console.debug('[call] mic audio processing', {event,preferredMode: persistedNoiseSuppressionMode(),@@ -476,6 +507,7 @@ function createCallState() {hasMicTrack: !!micTrack,micReadyState: mediaStreamTrack?.readyState,hasProcessor: !!micTrack?.getProcessor(),+ constraintMismatch,settings: {autoGainControl: settings?.autoGainControl,echoCancellation: settings?.echoCancellation,@@ -492,12 +524,51 @@ function createCallState() {},...extra,});++ analytics.track('call_audio_processing', {+ event,+ channelId: store.activeChannelId ?? '',+ callId: store.activeCallId ?? undefined,+ preferredMode: persistedNoiseSuppressionMode(),+ activeMode: store.noiseSuppressionMode,+ krispSupported: isKrispNoiseFilterSupported(),+ hasProcessor: !!micTrack?.getProcessor(),+ constraintMismatch,+ autoGainControl: settings?.autoGainControl,+ echoCancellation: settings?.echoCancellation,+ noiseSuppression: settings?.noiseSuppression,+ voiceIsolation: settings?.voiceIsolation,+ channelCount: settings?.channelCount,+ sampleRate: settings?.sampleRate,+ ...extra,+ });}// --- internal helpers ---+ // Noise-suppression processor work can be reached concurrently (background+ // media setup, mute toggle, device switch, NS toggle). Overlapping runs can+ // attach duplicate Krisp instances or land an attach after the user toggled+ // off, so it is serialized; each queued run re-reads the persisted mode,+ // which means the latest user intent wins regardless of arrival order.+ let micProcessingQueue: Promise<void> = Promise.resolve();++ function enqueueMicProcessing<T>(task: () => Promise<T>): Promise<T> {+ const run = micProcessingQueue.then(task);+ micProcessingQueue = run.then(+ () => {},+ () => {}+ );+ return run;+ }+/** Apply the preferred mic noise suppression mode to the current mic track. */- async function ensureNoiseSuppressionOnMicTrack(r: Room) {+ function ensureNoiseSuppressionOnMicTrack(r: Room): Promise<void> {+ return enqueueMicProcessing(() => applyNoiseSuppressionToMicTrack(r));+ }++ /** Serialized body of ensureNoiseSuppressionOnMicTrack — do not call directly. */+ async function applyNoiseSuppressionToMicTrack(r: Room) {if (room() !== r) return;const preferredMode = persistedNoiseSuppressionMode();@@ -553,7 +624,15 @@ function createCallState() {setKrispFilter(krisp);await micTrack.setProcessor(krisp);if (room() !== r || !isLiveLocalTrack(micTrack)) {- await krisp.destroy();+ // The room/track went away mid-attach. Destroying a processor that+ // is still wired into the track's audio graph leaves a dead node in+ // the pipeline — detach it via stopProcessor() instead (which+ // destroys the processor internally).+ if (micTrack.getProcessor() === krisp) {+ await micTrack.stopProcessor();+ } else {+ await krisp.destroy();+ }if (krispFilter() === krisp) setKrispFilter(null);return;}@@ -815,8 +894,12 @@ function createCallState() {await r.switchActiveDevice('audioinput', deviceId);setStore('activeAudioInputDeviceId', deviceId);- // If mic is currently live, republish with the new device to ensure it- // actually takes effect (switchActiveDevice alone can be unreliable).+ // If the mic is live, cycle mute/unmute as a nudge for devices where+ // switchActiveDevice's internal track restart doesn't take. Note this+ // is NOT a republish: setMicrophoneEnabled(true, opts) ignores the+ // capture options when the muted publication survives+ // (stopMicTrackOnMute is false), so the device change itself always+ // comes from switchActiveDevice above.if (!store.isAudioMuted) {await r.localParticipant.setMicrophoneEnabled(false);await r.localParticipant.setMicrophoneEnabled(@@ -1013,7 +1096,10 @@ function createCallState() {if (newMuted) {await r.localParticipant.setMicrophoneEnabled(false);} else {- // Re-enable with the user's selected device+ // Re-enable the mic. When the muted publication survived+ // (stopMicTrackOnMute is false) this only unmutes and the capture+ // options are ignored; they matter when the track was stopped and+ // must be recreated (e.g. after a device unplug).const deviceId = store.activeAudioInputDeviceId;await r.localParticipant.setMicrophoneEnabled(true,@@ -1107,13 +1193,11 @@ function createCallState() {if (!r) return;try {- if (newMode === 'off') {- await detachKrispFromMicTrack(r);- await applyNativeAudioProcessingToMicTrack(r, 'off');- logMicAudioProcessing('noise-suppression-toggled-off', r);- } else {- await ensureNoiseSuppressionOnMicTrack(r);- }+ // Both directions run through the serialized ensure path: its 'off'+ // branch detaches Krisp and clears native processing, and queueing+ // means a toggle issued while an attach is in flight still converges+ // on the mode the user picked last.+ await ensureNoiseSuppressionOnMicTrack(r);} catch (e) {console.error('failed to toggle noise suppression', e);}diff --git a/js/app/packages/channel/Call/LivekitJsCallController.ts b/js/app/packages/channel/Call/LivekitJsCallController.tsindex d5e0d8efad..2ddccb1df3 100644--- a/js/app/packages/channel/Call/LivekitJsCallController.ts+++ b/js/app/packages/channel/Call/LivekitJsCallController.ts@@ -8,6 +8,7 @@ import {RoomEvent,Track,} from 'livekit-client';+import { startReceiverStatsSampling } from './call-audio-receiver-stats';type LivekitJsCallControllerState = {activeChannelId: string | null;@@ -41,6 +42,13 @@ type LivekitJsCallControllerOptions = {export function createLivekitJsCallController(options: LivekitJsCallControllerOptions) {+ let stopReceiverStatsSampling: (() => void) | null = null;++ function stopReceiverStats() {+ stopReceiverStatsSampling?.();+ stopReceiverStatsSampling = null;+ }+function syncParticipantMap(room: Room) {options.setRemoteParticipants(new Map(room.remoteParticipants));options.bumpTrackVersion();@@ -90,6 +98,7 @@ export function createLivekitJsCallController(function destroyRoom() {options.cancelPendingMediaSetup();+ stopReceiverStats();options.destroyProcessors();const room = options.room();@@ -133,6 +142,12 @@ export function createLivekitJsCallController(const targetRoom = new Room({audioCaptureDefaults: options.currentMicrophoneCaptureOptions(),+ publishDefaults: {+ // Noise-suppressed audio has a near-silent noise floor, which makes+ // Opus DTX (on by default) misread quiet speech onsets as silence and+ // clip them. Always-on frames cost little at speech bitrates.+ dtx: false,+ },});attachRoomListeners(targetRoom);options.setRoom(targetRoom);@@ -151,6 +166,14 @@ export function createLivekitJsCallController(// Sync participants that were already in the room when we connected.syncParticipantMap(targetRoom);+ // Sample receive-side decode stats for the life of the room; the sampler+ // flushes one summary analytics event when stopped at teardown.+ stopReceiverStats();+ stopReceiverStatsSampling = startReceiverStatsSampling(targetRoom, {+ channelId: tokenResponse.channelId,+ callId: tokenResponse.callId,+ });+// Default to microphone on, video off as soon as the room is connected.options.setInitialMediaState();@@ -183,11 +206,13 @@ export function createLivekitJsCallController(if (!room) return;options.cancelPendingMediaSetup();+ stopReceiverStats();room.disconnect();}function dispose() {options.cancelPendingMediaSetup();+ stopReceiverStats();const room = options.room();if (!room) return;diff --git a/js/app/packages/channel/Call/call-audio-receiver-stats.ts b/js/app/packages/channel/Call/call-audio-receiver-stats.tsnew file mode 100644index 0000000000..b06e54e364--- /dev/null+++ b/js/app/packages/channel/Call/call-audio-receiver-stats.ts@@ -0,0 +1,118 @@+import { analytics } from '@app/lib/analytics';+import {+ type AudioReceiverStats,+ type RemoteAudioTrack,+ type Room,+ Track,+} from 'livekit-client';++const SAMPLE_INTERVAL_MS = 30_000;+// Concealment is the perceptual metric for "muddled/underwater" playback:+// the share of decoded samples the decoder synthesized (packet-loss+// concealment, jitter-buffer stretch) instead of received. Above ~5% of an+// interval, degradation is clearly audible.+const BAD_INTERVAL_CONCEALMENT_RATE = 0.05;+// Opus decodes at 48 kHz; totalSamplesDuration is reported in seconds.+const DECODE_SAMPLE_RATE = 48_000;++type Cursor = {+ concealedSamples: number;+ totalSamplesDuration: number;+ packetsLost: number;+ packetsReceived: number;+};++/**+ * Periodically samples remote microphone receiver stats and reports one+ * summary analytics event when stopped (room teardown). Receive-side decode+ * artifacts never show up in capture-side logs, so without this signal a+ * "muddled voices" report can't be split between network concealment and+ * over-eager noise suppression.+ */+export function startReceiverStatsSampling(+ room: Room,+ context: { channelId: string; callId?: string }+): () => void {+ const cursors = new Map<string, Cursor>();+ let sampledIntervals = 0;+ let badIntervals = 0;+ let maxConcealmentRate = 0;+ let maxPacketLossRate = 0;+ let maxJitter = 0;++ async function sampleTrack(identity: string, track: RemoteAudioTrack) {+ let stats: AudioReceiverStats | undefined;+ try {+ stats = await track.getReceiverStats();+ } catch {+ return;+ }+ if (!stats) return;++ const prev = cursors.get(identity);+ const cursor: Cursor = {+ concealedSamples: stats.concealedSamples ?? 0,+ totalSamplesDuration: stats.totalSamplesDuration ?? 0,+ packetsLost: stats.packetsLost ?? 0,+ packetsReceived: stats.packetsReceived ?? 0,+ };+ cursors.set(identity, cursor);+ if (stats.jitter !== undefined) {+ maxJitter = Math.max(maxJitter, stats.jitter);+ }+ if (!prev) return;++ const durationDelta =+ cursor.totalSamplesDuration - prev.totalSamplesDuration;+ const concealedDelta = cursor.concealedSamples - prev.concealedSamples;+ // Counters run backwards when the receiver was replaced mid-call+ // (reconnect / resubscribe); skip that interval.+ if (durationDelta <= 0 || concealedDelta < 0) return;++ sampledIntervals += 1;+ const concealmentRate =+ concealedDelta / (durationDelta * DECODE_SAMPLE_RATE);+ maxConcealmentRate = Math.max(maxConcealmentRate, concealmentRate);+ if (concealmentRate >= BAD_INTERVAL_CONCEALMENT_RATE) badIntervals += 1;++ const packetsDelta = cursor.packetsReceived - prev.packetsReceived;+ const lostDelta = cursor.packetsLost - prev.packetsLost;+ if (lostDelta >= 0 && packetsDelta + lostDelta > 0) {+ maxPacketLossRate = Math.max(+ maxPacketLossRate,+ lostDelta / (packetsDelta + lostDelta)+ );+ }+ }++ function sampleAll() {+ for (const participant of room.remoteParticipants.values()) {+ if (participant.isAgent) continue;+ const track = participant.getTrackPublication(+ Track.Source.Microphone+ )?.track;+ if (!track || track.kind !== Track.Kind.Audio) continue;+ void sampleTrack(participant.identity, track as RemoteAudioTrack);+ }+ }++ const timer = setInterval(sampleAll, SAMPLE_INTERVAL_MS);++ let stopped = false;+ return () => {+ if (stopped) return;+ stopped = true;+ clearInterval(timer);+ // Nothing sampled means a short or empty call — no event to send.+ if (sampledIntervals === 0) return;+ analytics.track('call_audio_receiver_stats', {+ channelId: context.channelId,+ callId: context.callId,+ sampledIntervals,+ badIntervals,+ maxConcealmentRate,+ maxPacketLossRate,+ maxJitter,+ });+ };+}