Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(calls): call noise cancellation (#4557)

Will Hutchinson0a4b867e853a
diff --git a/js/app/packages/app/lib/analytics/app-events.ts b/js/app/packages/app/lib/analytics/app-events.ts
index 90f168f62d..f366312b52 100644
--- a/js/app/packages/app/lib/analytics/app-events.ts
+++ b/js/app/packages/app/lib/analytics/app-events.ts
@@ -140,6 +140,27 @@ export type AppEvents = {
channelId: string;
callId?: string;
} & Record<string, unknown>;
+ // Mic audio-processing state transitions (noise suppression attach /
+ // detach / fallback). `constraintMismatch` flags engines that report
+ // different track settings than were requested (mode changes that silently
+ // no-oped); `sampleRate` <= 16000 indicates a Bluetooth HFP capture path.
+ call_audio_processing: {
+ event: string;
+ channelId: string;
+ preferredMode: string;
+ activeMode: string;
+ constraintMismatch: boolean;
+ } & Record<string, unknown>;
+ // Per-call summary of remote-audio decode health, flushed at room teardown.
+ // High concealment alongside clean capture-side state points a "muddled
+ // voices" report at the network/playback leg, not noise suppression.
+ call_audio_receiver_stats: {
+ channelId: string;
+ callId?: string;
+ sampledIntervals: number;
+ badIntervals: number;
+ maxConcealmentRate: number;
+ } & Record<string, unknown>;
block_pdf_definition_open: Record<string, unknown>;
block_pdf_section_open: Record<string, unknown>;
diff --git a/js/app/packages/channel/Call/CallContext.tsx b/js/app/packages/channel/Call/CallContext.tsx
index 784ec34933..22d6e2bc80 100644
--- a/js/app/packages/channel/Call/CallContext.tsx
+++ b/js/app/packages/channel/Call/CallContext.tsx
@@ -123,20 +123,27 @@ function microphoneCaptureOptions(
deviceId?: string | null
): AudioCaptureOptions {
const useBrowserProcessing = mode === 'browser';
+ const noiseSuppressionSupported =
+ supportsNativeAudioProcessingConstraint('noiseSuppression');
return {
- // AGC can raise residual background artifacts during quiet speech/silence.
- // Keep it disabled for every mode; Krisp/browser NS should not be stacked
- // with a gain stage that makes suppressed noise audible again.
+ // Browser AGC runs inside the capture pipeline (AEC → AGC → NS), i.e.
+ // upstream of any track processor — it feeds Krisp a healthy input level
+ // and cannot re-amplify the filter's residue (that would need a gain
+ // stage after the processor, which doesn't exist here). AGC-off left
+ // quiet mics at low SNR, and Krisp/browser NS then gated their speech.
...(supportsNativeAudioProcessingConstraint('autoGainControl')
- ? { autoGainControl: false }
+ ? { autoGainControl: true }
: {}),
echoCancellation: true,
- ...(supportsNativeAudioProcessingConstraint('noiseSuppression')
+ ...(noiseSuppressionSupported
? { noiseSuppression: useBrowserProcessing }
: {}),
+ // Voice isolation is a second, platform-level suppression stage; running
+ // it on top of noiseSuppression cascades filters and attenuates voice.
+ // Use it only as the fallback when noiseSuppression is unsupported.
...(supportsNativeAudioProcessingConstraint('voiceIsolation')
- ? { voiceIsolation: useBrowserProcessing }
+ ? { voiceIsolation: useBrowserProcessing && !noiseSuppressionSupported }
: {}),
...(deviceId ? { deviceId: { exact: deviceId } } : {}),
};
@@ -146,16 +153,25 @@ function nativeAudioProcessingConstraints(
mode: MicNoiseSuppressionMode
): NativeAudioProcessingConstraints {
const useBrowserProcessing = mode === 'browser';
+ const noiseSuppressionSupported =
+ supportsNativeAudioProcessingConstraint('noiseSuppression');
return {
+ // applyConstraints() replaces the track's entire constraint set, so echo
+ // cancellation must be restated here or every mode change silently drops
+ // it back to unconstrained.
+ echoCancellation: true,
+ // AGC stays on in every mode — see microphoneCaptureOptions.
...(supportsNativeAudioProcessingConstraint('autoGainControl')
- ? { autoGainControl: false }
+ ? { autoGainControl: true }
: {}),
- ...(supportsNativeAudioProcessingConstraint('noiseSuppression')
+ ...(noiseSuppressionSupported
? { noiseSuppression: useBrowserProcessing }
: {}),
+ // See microphoneCaptureOptions: voiceIsolation is only the fallback when
+ // noiseSuppression is unsupported, never a second stacked layer.
...(supportsNativeAudioProcessingConstraint('voiceIsolation')
- ? { voiceIsolation: useBrowserProcessing }
+ ? { voiceIsolation: useBrowserProcessing && !noiseSuppressionSupported }
: {}),
};
}
@@ -468,6 +484,21 @@ function createCallState() {
| NativeAudioProcessingConstraints
| undefined;
+ // A requested constraint the engine reports back differently in settings
+ // means the processing state was never actually applied (live-track
+ // applyConstraints is not honored everywhere) — the key signal behind
+ // "toggled noise suppression and nothing changed" reports.
+ const constraintMismatch = (
+ ['autoGainControl', 'echoCancellation', 'noiseSuppression'] as const
+ ).some((name) => {
+ const requested = constraints?.[name];
+ return (
+ typeof requested === 'boolean' &&
+ settings?.[name] !== undefined &&
+ settings[name] !== requested
+ );
+ });
+
console.debug('[call] mic audio processing', {
event,
preferredMode: persistedNoiseSuppressionMode(),
@@ -476,6 +507,7 @@ function createCallState() {
hasMicTrack: !!micTrack,
micReadyState: mediaStreamTrack?.readyState,
hasProcessor: !!micTrack?.getProcessor(),
+ constraintMismatch,
settings: {
autoGainControl: settings?.autoGainControl,
echoCancellation: settings?.echoCancellation,
@@ -492,12 +524,51 @@ function createCallState() {
},
...extra,
});
+
+ analytics.track('call_audio_processing', {
+ event,
+ channelId: store.activeChannelId ?? '',
+ callId: store.activeCallId ?? undefined,
+ preferredMode: persistedNoiseSuppressionMode(),
+ activeMode: store.noiseSuppressionMode,
+ krispSupported: isKrispNoiseFilterSupported(),
+ hasProcessor: !!micTrack?.getProcessor(),
+ constraintMismatch,
+ autoGainControl: settings?.autoGainControl,
+ echoCancellation: settings?.echoCancellation,
+ noiseSuppression: settings?.noiseSuppression,
+ voiceIsolation: settings?.voiceIsolation,
+ channelCount: settings?.channelCount,
+ sampleRate: settings?.sampleRate,
+ ...extra,
+ });
}
// --- internal helpers ---
+ // Noise-suppression processor work can be reached concurrently (background
+ // media setup, mute toggle, device switch, NS toggle). Overlapping runs can
+ // attach duplicate Krisp instances or land an attach after the user toggled
+ // off, so it is serialized; each queued run re-reads the persisted mode,
+ // which means the latest user intent wins regardless of arrival order.
+ let micProcessingQueue: Promise<void> = Promise.resolve();
+
+ function enqueueMicProcessing<T>(task: () => Promise<T>): Promise<T> {
+ const run = micProcessingQueue.then(task);
+ micProcessingQueue = run.then(
+ () => {},
+ () => {}
+ );
+ return run;
+ }
+
/** Apply the preferred mic noise suppression mode to the current mic track. */
- async function ensureNoiseSuppressionOnMicTrack(r: Room) {
+ function ensureNoiseSuppressionOnMicTrack(r: Room): Promise<void> {
+ return enqueueMicProcessing(() => applyNoiseSuppressionToMicTrack(r));
+ }
+
+ /** Serialized body of ensureNoiseSuppressionOnMicTrack — do not call directly. */
+ async function applyNoiseSuppressionToMicTrack(r: Room) {
if (room() !== r) return;
const preferredMode = persistedNoiseSuppressionMode();
@@ -553,7 +624,15 @@ function createCallState() {
setKrispFilter(krisp);
await micTrack.setProcessor(krisp);
if (room() !== r || !isLiveLocalTrack(micTrack)) {
- await krisp.destroy();
+ // The room/track went away mid-attach. Destroying a processor that
+ // is still wired into the track's audio graph leaves a dead node in
+ // the pipeline — detach it via stopProcessor() instead (which
+ // destroys the processor internally).
+ if (micTrack.getProcessor() === krisp) {
+ await micTrack.stopProcessor();
+ } else {
+ await krisp.destroy();
+ }
if (krispFilter() === krisp) setKrispFilter(null);
return;
}
@@ -815,8 +894,12 @@ function createCallState() {
await r.switchActiveDevice('audioinput', deviceId);
setStore('activeAudioInputDeviceId', deviceId);
- // If mic is currently live, republish with the new device to ensure it
- // actually takes effect (switchActiveDevice alone can be unreliable).
+ // If the mic is live, cycle mute/unmute as a nudge for devices where
+ // switchActiveDevice's internal track restart doesn't take. Note this
+ // is NOT a republish: setMicrophoneEnabled(true, opts) ignores the
+ // capture options when the muted publication survives
+ // (stopMicTrackOnMute is false), so the device change itself always
+ // comes from switchActiveDevice above.
if (!store.isAudioMuted) {
await r.localParticipant.setMicrophoneEnabled(false);
await r.localParticipant.setMicrophoneEnabled(
@@ -1013,7 +1096,10 @@ function createCallState() {
if (newMuted) {
await r.localParticipant.setMicrophoneEnabled(false);
} else {
- // Re-enable with the user's selected device
+ // Re-enable the mic. When the muted publication survived
+ // (stopMicTrackOnMute is false) this only unmutes and the capture
+ // options are ignored; they matter when the track was stopped and
+ // must be recreated (e.g. after a device unplug).
const deviceId = store.activeAudioInputDeviceId;
await r.localParticipant.setMicrophoneEnabled(
true,
@@ -1107,13 +1193,11 @@ function createCallState() {
if (!r) return;
try {
- if (newMode === 'off') {
- await detachKrispFromMicTrack(r);
- await applyNativeAudioProcessingToMicTrack(r, 'off');
- logMicAudioProcessing('noise-suppression-toggled-off', r);
- } else {
- await ensureNoiseSuppressionOnMicTrack(r);
- }
+ // Both directions run through the serialized ensure path: its 'off'
+ // branch detaches Krisp and clears native processing, and queueing
+ // means a toggle issued while an attach is in flight still converges
+ // on the mode the user picked last.
+ await ensureNoiseSuppressionOnMicTrack(r);
} catch (e) {
console.error('failed to toggle noise suppression', e);
}
diff --git a/js/app/packages/channel/Call/LivekitJsCallController.ts b/js/app/packages/channel/Call/LivekitJsCallController.ts
index d5e0d8efad..2ddccb1df3 100644
--- a/js/app/packages/channel/Call/LivekitJsCallController.ts
+++ b/js/app/packages/channel/Call/LivekitJsCallController.ts
@@ -8,6 +8,7 @@ import {
RoomEvent,
Track,
} from 'livekit-client';
+import { startReceiverStatsSampling } from './call-audio-receiver-stats';
type LivekitJsCallControllerState = {
activeChannelId: string | null;
@@ -41,6 +42,13 @@ type LivekitJsCallControllerOptions = {
export function createLivekitJsCallController(
options: LivekitJsCallControllerOptions
) {
+ let stopReceiverStatsSampling: (() => void) | null = null;
+
+ function stopReceiverStats() {
+ stopReceiverStatsSampling?.();
+ stopReceiverStatsSampling = null;
+ }
+
function syncParticipantMap(room: Room) {
options.setRemoteParticipants(new Map(room.remoteParticipants));
options.bumpTrackVersion();
@@ -90,6 +98,7 @@ export function createLivekitJsCallController(
function destroyRoom() {
options.cancelPendingMediaSetup();
+ stopReceiverStats();
options.destroyProcessors();
const room = options.room();
@@ -133,6 +142,12 @@ export function createLivekitJsCallController(
const targetRoom = new Room({
audioCaptureDefaults: options.currentMicrophoneCaptureOptions(),
+ publishDefaults: {
+ // Noise-suppressed audio has a near-silent noise floor, which makes
+ // Opus DTX (on by default) misread quiet speech onsets as silence and
+ // clip them. Always-on frames cost little at speech bitrates.
+ dtx: false,
+ },
});
attachRoomListeners(targetRoom);
options.setRoom(targetRoom);
@@ -151,6 +166,14 @@ export function createLivekitJsCallController(
// Sync participants that were already in the room when we connected.
syncParticipantMap(targetRoom);
+ // Sample receive-side decode stats for the life of the room; the sampler
+ // flushes one summary analytics event when stopped at teardown.
+ stopReceiverStats();
+ stopReceiverStatsSampling = startReceiverStatsSampling(targetRoom, {
+ channelId: tokenResponse.channelId,
+ callId: tokenResponse.callId,
+ });
+
// Default to microphone on, video off as soon as the room is connected.
options.setInitialMediaState();
@@ -183,11 +206,13 @@ export function createLivekitJsCallController(
if (!room) return;
options.cancelPendingMediaSetup();
+ stopReceiverStats();
room.disconnect();
}
function dispose() {
options.cancelPendingMediaSetup();
+ stopReceiverStats();
const room = options.room();
if (!room) return;
diff --git a/js/app/packages/channel/Call/call-audio-receiver-stats.ts b/js/app/packages/channel/Call/call-audio-receiver-stats.ts
new file mode 100644
index 0000000000..b06e54e364
--- /dev/null
+++ b/js/app/packages/channel/Call/call-audio-receiver-stats.ts
@@ -0,0 +1,118 @@
+import { analytics } from '@app/lib/analytics';
+import {
+ type AudioReceiverStats,
+ type RemoteAudioTrack,
+ type Room,
+ Track,
+} from 'livekit-client';
+
+const SAMPLE_INTERVAL_MS = 30_000;
+// Concealment is the perceptual metric for "muddled/underwater" playback:
+// the share of decoded samples the decoder synthesized (packet-loss
+// concealment, jitter-buffer stretch) instead of received. Above ~5% of an
+// interval, degradation is clearly audible.
+const BAD_INTERVAL_CONCEALMENT_RATE = 0.05;
+// Opus decodes at 48 kHz; totalSamplesDuration is reported in seconds.
+const DECODE_SAMPLE_RATE = 48_000;
+
+type Cursor = {
+ concealedSamples: number;
+ totalSamplesDuration: number;
+ packetsLost: number;
+ packetsReceived: number;
+};
+
+/**
+ * Periodically samples remote microphone receiver stats and reports one
+ * summary analytics event when stopped (room teardown). Receive-side decode
+ * artifacts never show up in capture-side logs, so without this signal a
+ * "muddled voices" report can't be split between network concealment and
+ * over-eager noise suppression.
+ */
+export function startReceiverStatsSampling(
+ room: Room,
+ context: { channelId: string; callId?: string }
+): () => void {
+ const cursors = new Map<string, Cursor>();
+ let sampledIntervals = 0;
+ let badIntervals = 0;
+ let maxConcealmentRate = 0;
+ let maxPacketLossRate = 0;
+ let maxJitter = 0;
+
+ async function sampleTrack(identity: string, track: RemoteAudioTrack) {
+ let stats: AudioReceiverStats | undefined;
+ try {
+ stats = await track.getReceiverStats();
+ } catch {
+ return;
+ }
+ if (!stats) return;
+
+ const prev = cursors.get(identity);
+ const cursor: Cursor = {
+ concealedSamples: stats.concealedSamples ?? 0,
+ totalSamplesDuration: stats.totalSamplesDuration ?? 0,
+ packetsLost: stats.packetsLost ?? 0,
+ packetsReceived: stats.packetsReceived ?? 0,
+ };
+ cursors.set(identity, cursor);
+ if (stats.jitter !== undefined) {
+ maxJitter = Math.max(maxJitter, stats.jitter);
+ }
+ if (!prev) return;
+
+ const durationDelta =
+ cursor.totalSamplesDuration - prev.totalSamplesDuration;
+ const concealedDelta = cursor.concealedSamples - prev.concealedSamples;
+ // Counters run backwards when the receiver was replaced mid-call
+ // (reconnect / resubscribe); skip that interval.
+ if (durationDelta <= 0 || concealedDelta < 0) return;
+
+ sampledIntervals += 1;
+ const concealmentRate =
+ concealedDelta / (durationDelta * DECODE_SAMPLE_RATE);
+ maxConcealmentRate = Math.max(maxConcealmentRate, concealmentRate);
+ if (concealmentRate >= BAD_INTERVAL_CONCEALMENT_RATE) badIntervals += 1;
+
+ const packetsDelta = cursor.packetsReceived - prev.packetsReceived;
+ const lostDelta = cursor.packetsLost - prev.packetsLost;
+ if (lostDelta >= 0 && packetsDelta + lostDelta > 0) {
+ maxPacketLossRate = Math.max(
+ maxPacketLossRate,
+ lostDelta / (packetsDelta + lostDelta)
+ );
+ }
+ }
+
+ function sampleAll() {
+ for (const participant of room.remoteParticipants.values()) {
+ if (participant.isAgent) continue;
+ const track = participant.getTrackPublication(
+ Track.Source.Microphone
+ )?.track;
+ if (!track || track.kind !== Track.Kind.Audio) continue;
+ void sampleTrack(participant.identity, track as RemoteAudioTrack);
+ }
+ }
+
+ const timer = setInterval(sampleAll, SAMPLE_INTERVAL_MS);
+
+ let stopped = false;
+ return () => {
+ if (stopped) return;
+ stopped = true;
+ clearInterval(timer);
+ // Nothing sampled means a short or empty call — no event to send.
+ if (sampledIntervals === 0) return;
+ analytics.track('call_audio_receiver_stats', {
+ channelId: context.channelId,
+ callId: context.callId,
+ sampledIntervals,
+ badIntervals,
+ maxConcealmentRate,
+ maxPacketLossRate,
+ maxJitter,
+ });
+ };
+}