Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(voice): preserve microphone channels during screen sharing (#3070)

Hampus5e0806f47963
diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts
index 0a7169e66..de4de0f6f 100644
--- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts
+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts
@@ -8,6 +8,7 @@ import type {TrackBitrateInfo} from './PCTransport.ts';
import {
applyVideoStartBitrate,
collectStereoMids,
+ conformBundledCodecFmtp,
ensureAudioNackAndStereo,
ensureOpusFmtp,
ensureVideoDDExtension,
@@ -282,15 +283,63 @@ describe('placeholderMidsFromTransceivers', () => {
return {mid, currentDirection, sender: {track}} as unknown as RTCRtpTransceiver;
}
- it('keeps the trackless recvonly sections that still hold an m-line', () => {
+ it('keeps unused trackless sections that still hold an m-line', () => {
const mids = placeholderMidsFromTransceivers([
- transceiver('3', null, 'recvonly'),
+ transceiver('3', null, 'inactive'),
+ transceiver('4', null, null),
transceiver('7', {} as MediaStreamTrack, 'sendonly'),
]);
- expect(mids).toEqual(new Set(['3']));
+ expect(mids).toEqual(new Set(['3', '4']));
});
it('drops a transceiver that unpublish stopped so its recycled m-section is not fmtp-conformed', () => {
expect(placeholderMidsFromTransceivers([transceiver('7', null, 'stopped')])).toEqual(new Set());
});
+
+ it.each(['recvonly', 'sendrecv'] as const)(
+ 'preserves a negotiated %s receiver without a local sender',
+ (direction) => {
+ expect(placeholderMidsFromTransceivers([transceiver('0', null, direction)])).toEqual(new Set());
+ },
+ );
+
+ it.each(['sendonly', 'sendrecv', undefined] as const)(
+ 'preserves a receiver activated by a remote %s answer before currentDirection updates',
+ (direction) => {
+ const incoming = opusMedia('useinbandfec=1');
+ incoming.direction = direction;
+ expect(placeholderMidsFromTransceivers([transceiver('0', null, null)], [incoming])).toEqual(new Set());
+ },
+ );
+
+ it('keeps inactive and rejected remote sections eligible for placeholder conformance', () => {
+ const inactive = opusMedia('useinbandfec=1', '0');
+ inactive.direction = 'inactive';
+ const rejected = opusMedia('useinbandfec=1', '1');
+ rejected.direction = 'sendonly';
+ rejected.port = 0;
+ expect(
+ placeholderMidsFromTransceivers(
+ [transceiver('0', null, 'inactive'), transceiver('1', null, null)],
+ [inactive, rejected],
+ ),
+ ).toEqual(new Set(['0', '1']));
+ });
+
+ it('starting a stereo share preserves incoming microphone fmtp while conforming unused placeholders', () => {
+ const microphone = opusMedia('useinbandfec=1;maxaveragebitrate=64000', '0');
+ const placeholder = opusMedia('useinbandfec=1;usedtx=1', '1');
+ const screenShare = opusMedia('useinbandfec=1', '2');
+ ensureOpusFmtp(screenShare, 128000, true);
+ const microphoneConfig = opusConfig(microphone);
+ const mids = placeholderMidsFromTransceivers([
+ transceiver('0', null, 'recvonly'),
+ transceiver('1', null, 'inactive'),
+ transceiver('2', {} as MediaStreamTrack, 'sendonly'),
+ ]);
+ conformBundledCodecFmtp([microphone, placeholder, screenShare], (media) => mids.has(String(media.mid)));
+ expect(opusConfig(microphone)).toBe(microphoneConfig);
+ expect(opusConfig(screenShare)).toContain('stereo=1');
+ expect(opusConfig(placeholder)).toBe(opusConfig(screenShare));
+ });
});
diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts
index 099635e6a..024877909 100644
--- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts
+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts
@@ -257,7 +257,7 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
}
}
});
- const placeholderMids = this.getPlaceholderMids();
+ const placeholderMids = this.getPlaceholderMids(sdpParsed.media);
if (placeholderMids.size > 0) {
conformBundledCodecFmtp(sdpParsed.media, (media) => placeholderMids.has(getMidString(media.mid!)));
}
@@ -461,8 +461,11 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
}
}
- private getPlaceholderMids(): Set<string> {
- return placeholderMidsFromTransceivers(this._pc?.getTransceivers() ?? []);
+ private getPlaceholderMids(remoteMedia?: ReadonlyArray<MediaDescription>): Set<string> {
+ return placeholderMidsFromTransceivers(
+ this._pc?.getTransceivers() ?? [],
+ remoteMedia ?? parse(this._pc?.remoteDescription?.sdp ?? '').media,
+ );
}
createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) {
@@ -853,10 +856,28 @@ export function collectStereoMids(
return stereoMids;
}
-export function placeholderMidsFromTransceivers(transceivers: ReadonlyArray<RTCRtpTransceiver>): Set<string> {
+export function placeholderMidsFromTransceivers(
+ transceivers: ReadonlyArray<RTCRtpTransceiver>,
+ remoteMedia: ReadonlyArray<MediaDescription> = [],
+): Set<string> {
+ const receivingMids = new Set(
+ remoteMedia
+ .filter(
+ (media) =>
+ media.mid !== undefined &&
+ media.port !== 0 &&
+ (media.direction === 'sendonly' || media.direction === 'sendrecv' || media.direction === undefined),
+ )
+ .map((media) => getMidString(media.mid!)),
+ );
const mids = new Set<string>();
for (const transceiver of transceivers) {
- if (transceiver.currentDirection === 'stopped') {
+ if (
+ transceiver.currentDirection === 'stopped' ||
+ transceiver.currentDirection === 'recvonly' ||
+ transceiver.currentDirection === 'sendrecv' ||
+ (transceiver.mid !== null && receivingMids.has(transceiver.mid))
+ ) {
continue;
}
if (transceiver.mid && !transceiver.sender.track) {
diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts
new file mode 100644
index 000000000..d13592f6a
--- /dev/null
+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts
@@ -0,0 +1,127 @@
+import {type AddTrackRequest, AudioTrackFeature, ParticipantPermission, TrackInfo} from '@livekit/protocol';
+import {EventEmitter} from 'events';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+import {SignalConnectionState} from '../../api/SignalClient.ts';
+import {publishDefaults, roomOptionDefaults} from '../defaults.ts';
+import {TrackEvent} from '../events.ts';
+import type RTCEngine from '../RTCEngine.ts';
+import LocalAudioTrack from '../track/LocalAudioTrack.ts';
+import {Track} from '../track/Track.ts';
+import LocalParticipant from './LocalParticipant.ts';
+
+class TestMediaStreamTrack extends EventTarget {
+ readonly id = 'microphone-input';
+ readonly kind = 'audio';
+ enabled = true;
+ readyState: MediaStreamTrackState = 'live';
+
+ constructor(private readonly channelCount: number) {
+ super();
+ }
+
+ getSettings(): MediaTrackSettings {
+ return {channelCount: this.channelCount, echoCancellation: true};
+ }
+
+ getConstraints(): MediaTrackConstraints {
+ return {};
+ }
+
+ stop() {
+ this.readyState = 'ended';
+ }
+}
+
+class TestMediaStream {
+ constructor(private readonly tracks: Array<MediaStreamTrack>) {}
+
+ getTracks() {
+ return this.tracks;
+ }
+}
+
+function createPublisher() {
+ const setTrackCodecBitrate = vi.fn();
+ const sendUpdateLocalAudioTrack = vi.fn();
+ const sender = {replaceTrack: vi.fn(async () => undefined)} as unknown as RTCRtpSender;
+ const transceiver = {sender} as RTCRtpTransceiver;
+ const addTrack = vi.fn(
+ async (request: AddTrackRequest) =>
+ new TrackInfo({
+ sid: 'TR_microphone',
+ name: request.name,
+ type: request.type,
+ source: request.source,
+ audioFeatures: request.audioFeatures,
+ }),
+ );
+ const engine = Object.assign(new EventEmitter(), {
+ isClosed: false,
+ logContext: {},
+ client: {currentState: SignalConnectionState.CONNECTED, sendUpdateLocalAudioTrack},
+ pcManager: {publisher: {getTransceivers: () => [transceiver], setTrackCodecBitrate}},
+ createSender: vi.fn(async () => sender),
+ negotiate: vi.fn(async () => undefined),
+ addTrack,
+ });
+ type ParticipantArgs = ConstructorParameters<typeof LocalParticipant>;
+ const participant = new LocalParticipant(
+ 'PA_publisher',
+ 'publisher',
+ engine as unknown as RTCEngine,
+ {...roomOptionDefaults, publishDefaults},
+ {} as ParticipantArgs[4],
+ {} as ParticipantArgs[5],
+ {} as ParticipantArgs[6],
+ {} as ParticipantArgs[7],
+ );
+ participant.permissions = new ParticipantPermission({canPublish: true});
+ return {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack};
+}
+
+describe('microphone publication stereo metadata', () => {
+ beforeEach(() => {
+ vi.stubGlobal('MediaStreamTrack', TestMediaStreamTrack);
+ vi.stubGlobal('MediaStream', TestMediaStream);
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it.each([
+ {name: 'mono policy on a two-channel device', channelCount: 2, forceStereo: false, stereo: false},
+ {name: 'automatic stereo on a two-channel device', channelCount: 2, forceStereo: undefined, stereo: true},
+ {name: 'forced stereo on a one-channel device', channelCount: 1, forceStereo: true, stereo: true},
+ {name: 'automatic mono on a one-channel device', channelCount: 1, forceStereo: undefined, stereo: false},
+ ])(
+ 'keeps the request, SDP policy, and feature updates consistent for $name',
+ async ({channelCount, forceStereo, stereo}) => {
+ const {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack} = createPublisher();
+ const source = new TestMediaStreamTrack(channelCount);
+ const track = new LocalAudioTrack(source as unknown as MediaStreamTrack);
+ track.source = Track.Source.Microphone;
+ await track.runWithTrackChangeLock(async () => undefined);
+ const publication = await participant.publishTrack(track, {
+ audioPreset: {maxBitrate: 64000},
+ forceStereo,
+ });
+
+ expect(addTrack).toHaveBeenCalledTimes(1);
+ const request = addTrack.mock.calls[0]![0];
+ expect(request.stereo).toBe(stereo);
+ expect(request.audioFeatures.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);
+ expect(request.audioFeatures).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION);
+ expect(setTrackCodecBitrate).toHaveBeenCalledWith(expect.objectContaining({codec: 'opus', maxbr: 64, stereo}));
+
+ expect(publication.getTrackFeatures().includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);
+ track.emit(TrackEvent.AudioTrackFeatureUpdate, track, AudioTrackFeature.TF_ECHO_CANCELLATION, true);
+ expect(sendUpdateLocalAudioTrack).toHaveBeenCalledTimes(1);
+ const [sid, features] = sendUpdateLocalAudioTrack.mock.calls[0]!;
+ expect(sid).toBe('TR_microphone');
+ expect(features.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);
+ expect(features).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION);
+ track.stop();
+ },
+ );
+});
diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts
index d25f73b35..8ae43e7cb 100644
--- a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts
+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts
@@ -966,7 +966,7 @@ export default class LocalParticipant extends Participant {
if (settings.noiseSuppression) {
audioFeatures.push(AudioTrackFeature.TF_NOISE_SUPPRESSION);
}
- if (settings.channelCount && settings.channelCount > 1) {
+ if (isStereo) {
audioFeatures.push(AudioTrackFeature.TF_STEREO);
}
if (disableDtx) {
diff --git a/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts b/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts
index de5b88d82..01095c574 100644
--- a/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts
+++ b/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts
@@ -90,7 +90,7 @@ export default class LocalTrackPublication extends TrackPublication {
if (settings.noiseSuppression) {
features.add(AudioTrackFeature.TF_NOISE_SUPPRESSION);
}
- if (settings.channelCount && settings.channelCount > 1) {
+ if (this.options?.forceStereo ?? (settings.channelCount !== undefined && settings.channelCount > 1)) {
features.add(AudioTrackFeature.TF_STEREO);
}
if (!this.options?.dtx) {
diff --git a/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx b/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx
index e33315459..6329b1ee5 100644
--- a/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx
+++ b/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx
@@ -294,7 +294,7 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
value: backend,
label: getNoiseSuppressionChoiceLabel(i18n, backend),
}));
- const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable();
+ const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable(activeInputLabel);
const setPushToTalkEnabled = (enabled: boolean) => {
const mode = enabled ? 'voice_push_to_talk' : 'voice_activity';
if (enabled && !isNativeDesktop) {
diff --git a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts
new file mode 100644
index 000000000..cd81f7570
--- /dev/null
+++ b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts
@@ -0,0 +1,63 @@
+import type {VoiceProcessingMode} from '@app/features/voice/utils/VoiceProcessingProfile';
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+const {deviceModes, voiceSettings} = vi.hoisted(() => ({
+ deviceModes: new Map<string, VoiceProcessingMode>(),
+ voiceSettings: {
+ voiceProcessingMode: 'voice' as VoiceProcessingMode,
+ echoCancellation: true,
+ autoGainControl: false,
+ getNoiseSuppressionBackend: () => 'none',
+ getStereoMicrophone: () => false,
+ getVoiceProcessingModeForDeviceLabel: vi.fn<(label: string | null | undefined) => VoiceProcessingMode>(),
+ },
+}));
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: voiceSettings}));
+vi.mock('@app/features/voice/commands/VoiceSettingsCommands', () => ({update: vi.fn()}));
+vi.mock('@app/features/voice/engine/VoiceDevicePermissionState', () => ({
+ default: {getState: () => ({inputDevices: []})},
+}));
+vi.mock('@app/features/voice/utils/VoiceDeviceManager', () => ({resolveEffectiveDeviceId: () => null}));
+
+import {isStereoMicrophoneChoiceAvailable} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionChoices';
+
+describe('stereo microphone choice for the selected device', () => {
+ beforeEach(() => {
+ deviceModes.clear();
+ voiceSettings.voiceProcessingMode = 'voice';
+ voiceSettings.echoCancellation = true;
+ voiceSettings.autoGainControl = false;
+ voiceSettings.getVoiceProcessingModeForDeviceLabel.mockReset();
+ voiceSettings.getVoiceProcessingModeForDeviceLabel.mockImplementation(
+ (label) => (label ? deviceModes.get(label) : undefined) ?? voiceSettings.voiceProcessingMode,
+ );
+ });
+
+ it('shows the switch for a device using direct input when the global mode is focused voice', () => {
+ deviceModes.set('USB audio interface', 'studio');
+ expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true);
+ expect(voiceSettings.getVoiceProcessingModeForDeviceLabel).toHaveBeenCalledWith('USB audio interface');
+ });
+
+ it('hides the switch for a device with custom processing when the global mode is direct input', () => {
+ voiceSettings.voiceProcessingMode = 'studio';
+ deviceModes.set('Headset microphone', 'custom');
+ expect(isStereoMicrophoneChoiceAvailable('Headset microphone')).toBe(false);
+ });
+
+ it('shows the switch for a custom device profile with processing disabled', () => {
+ deviceModes.set('USB audio interface', 'custom');
+ voiceSettings.echoCancellation = false;
+ expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true);
+ });
+
+ it.each([
+ {mode: 'voice' as const, available: false},
+ {mode: 'studio' as const, available: true},
+ ])('uses the global $mode profile when the device has no override', ({mode, available}) => {
+ voiceSettings.voiceProcessingMode = mode;
+ expect(isStereoMicrophoneChoiceAvailable('Unconfigured microphone')).toBe(available);
+ expect(isStereoMicrophoneChoiceAvailable(null)).toBe(available);
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts
index 302cdacb1..a0b3f1a8f 100644
--- a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts
+++ b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts
@@ -12,7 +12,7 @@ import {
readNoiseSuppressionRuntimeCapabilities,
supportsStereoCapture,
} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection';
-import {resolveVoiceProcessingFromState} from '@app/features/voice/utils/VoiceProcessingProfile';
+import {resolveVoiceProcessingFromStateForDeviceLabel} from '@app/features/voice/utils/VoiceProcessingProfile';
export function getNoiseSuppressionChoiceValues(): ReadonlyArray<VoiceNoiseSuppressionBackend> {
const capabilities = readNoiseSuppressionRuntimeCapabilities();
@@ -29,8 +29,8 @@ export function setNoiseSuppressionChoice(backend: VoiceNoiseSuppressionBackend)
VoiceSettingsCommands.update({noiseSuppressionBackend: backend});
}
-export function isStereoMicrophoneChoiceAvailable(): boolean {
- return supportsStereoCapture(resolveVoiceProcessingFromState(VoiceSettings));
+export function isStereoMicrophoneChoiceAvailable(deviceLabel: string | null): boolean {
+ return supportsStereoCapture(resolveVoiceProcessingFromStateForDeviceLabel(VoiceSettings, deviceLabel));
}
export function isStereoMicrophoneEnabled(): boolean {