A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(voice): preserve microphone channels during screen sharing (#3070)
Hampus5e0806f47963
diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.tsindex 0a7169e66..de4de0f6f 100644--- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts@@ -8,6 +8,7 @@ import type {TrackBitrateInfo} from './PCTransport.ts';import {applyVideoStartBitrate,collectStereoMids,+ conformBundledCodecFmtp,ensureAudioNackAndStereo,ensureOpusFmtp,ensureVideoDDExtension,@@ -282,15 +283,63 @@ describe('placeholderMidsFromTransceivers', () => {return {mid, currentDirection, sender: {track}} as unknown as RTCRtpTransceiver;}- it('keeps the trackless recvonly sections that still hold an m-line', () => {+ it('keeps unused trackless sections that still hold an m-line', () => {const mids = placeholderMidsFromTransceivers([- transceiver('3', null, 'recvonly'),+ transceiver('3', null, 'inactive'),+ transceiver('4', null, null),transceiver('7', {} as MediaStreamTrack, 'sendonly'),]);- expect(mids).toEqual(new Set(['3']));+ expect(mids).toEqual(new Set(['3', '4']));});it('drops a transceiver that unpublish stopped so its recycled m-section is not fmtp-conformed', () => {expect(placeholderMidsFromTransceivers([transceiver('7', null, 'stopped')])).toEqual(new Set());});++ it.each(['recvonly', 'sendrecv'] as const)(+ 'preserves a negotiated %s receiver without a local sender',+ (direction) => {+ expect(placeholderMidsFromTransceivers([transceiver('0', null, direction)])).toEqual(new Set());+ },+ );++ it.each(['sendonly', 'sendrecv', undefined] as const)(+ 'preserves a receiver activated by a remote %s answer before currentDirection updates',+ (direction) => {+ const incoming = opusMedia('useinbandfec=1');+ incoming.direction = direction;+ expect(placeholderMidsFromTransceivers([transceiver('0', null, null)], [incoming])).toEqual(new Set());+ },+ );++ it('keeps inactive and rejected remote sections eligible for placeholder conformance', () => {+ const inactive = opusMedia('useinbandfec=1', '0');+ inactive.direction = 'inactive';+ const rejected = opusMedia('useinbandfec=1', '1');+ rejected.direction = 'sendonly';+ rejected.port = 0;+ expect(+ placeholderMidsFromTransceivers(+ [transceiver('0', null, 'inactive'), transceiver('1', null, null)],+ [inactive, rejected],+ ),+ ).toEqual(new Set(['0', '1']));+ });++ it('starting a stereo share preserves incoming microphone fmtp while conforming unused placeholders', () => {+ const microphone = opusMedia('useinbandfec=1;maxaveragebitrate=64000', '0');+ const placeholder = opusMedia('useinbandfec=1;usedtx=1', '1');+ const screenShare = opusMedia('useinbandfec=1', '2');+ ensureOpusFmtp(screenShare, 128000, true);+ const microphoneConfig = opusConfig(microphone);+ const mids = placeholderMidsFromTransceivers([+ transceiver('0', null, 'recvonly'),+ transceiver('1', null, 'inactive'),+ transceiver('2', {} as MediaStreamTrack, 'sendonly'),+ ]);+ conformBundledCodecFmtp([microphone, placeholder, screenShare], (media) => mids.has(String(media.mid)));+ expect(opusConfig(microphone)).toBe(microphoneConfig);+ expect(opusConfig(screenShare)).toContain('stereo=1');+ expect(opusConfig(placeholder)).toBe(opusConfig(screenShare));+ });});diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.tsindex 099635e6a..024877909 100644--- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts@@ -257,7 +257,7 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter}}});- const placeholderMids = this.getPlaceholderMids();+ const placeholderMids = this.getPlaceholderMids(sdpParsed.media);if (placeholderMids.size > 0) {conformBundledCodecFmtp(sdpParsed.media, (media) => placeholderMids.has(getMidString(media.mid!)));}@@ -461,8 +461,11 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter}}- private getPlaceholderMids(): Set<string> {- return placeholderMidsFromTransceivers(this._pc?.getTransceivers() ?? []);+ private getPlaceholderMids(remoteMedia?: ReadonlyArray<MediaDescription>): Set<string> {+ return placeholderMidsFromTransceivers(+ this._pc?.getTransceivers() ?? [],+ remoteMedia ?? parse(this._pc?.remoteDescription?.sdp ?? '').media,+ );}createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) {@@ -853,10 +856,28 @@ export function collectStereoMids(return stereoMids;}-export function placeholderMidsFromTransceivers(transceivers: ReadonlyArray<RTCRtpTransceiver>): Set<string> {+export function placeholderMidsFromTransceivers(+ transceivers: ReadonlyArray<RTCRtpTransceiver>,+ remoteMedia: ReadonlyArray<MediaDescription> = [],+): Set<string> {+ const receivingMids = new Set(+ remoteMedia+ .filter(+ (media) =>+ media.mid !== undefined &&+ media.port !== 0 &&+ (media.direction === 'sendonly' || media.direction === 'sendrecv' || media.direction === undefined),+ )+ .map((media) => getMidString(media.mid!)),+ );const mids = new Set<string>();for (const transceiver of transceivers) {- if (transceiver.currentDirection === 'stopped') {+ if (+ transceiver.currentDirection === 'stopped' ||+ transceiver.currentDirection === 'recvonly' ||+ transceiver.currentDirection === 'sendrecv' ||+ (transceiver.mid !== null && receivingMids.has(transceiver.mid))+ ) {continue;}if (transceiver.mid && !transceiver.sender.track) {diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.tsnew file mode 100644index 000000000..d13592f6a--- /dev/null+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts@@ -0,0 +1,127 @@+import {type AddTrackRequest, AudioTrackFeature, ParticipantPermission, TrackInfo} from '@livekit/protocol';+import {EventEmitter} from 'events';+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';+import {SignalConnectionState} from '../../api/SignalClient.ts';+import {publishDefaults, roomOptionDefaults} from '../defaults.ts';+import {TrackEvent} from '../events.ts';+import type RTCEngine from '../RTCEngine.ts';+import LocalAudioTrack from '../track/LocalAudioTrack.ts';+import {Track} from '../track/Track.ts';+import LocalParticipant from './LocalParticipant.ts';++class TestMediaStreamTrack extends EventTarget {+ readonly id = 'microphone-input';+ readonly kind = 'audio';+ enabled = true;+ readyState: MediaStreamTrackState = 'live';++ constructor(private readonly channelCount: number) {+ super();+ }++ getSettings(): MediaTrackSettings {+ return {channelCount: this.channelCount, echoCancellation: true};+ }++ getConstraints(): MediaTrackConstraints {+ return {};+ }++ stop() {+ this.readyState = 'ended';+ }+}++class TestMediaStream {+ constructor(private readonly tracks: Array<MediaStreamTrack>) {}++ getTracks() {+ return this.tracks;+ }+}++function createPublisher() {+ const setTrackCodecBitrate = vi.fn();+ const sendUpdateLocalAudioTrack = vi.fn();+ const sender = {replaceTrack: vi.fn(async () => undefined)} as unknown as RTCRtpSender;+ const transceiver = {sender} as RTCRtpTransceiver;+ const addTrack = vi.fn(+ async (request: AddTrackRequest) =>+ new TrackInfo({+ sid: 'TR_microphone',+ name: request.name,+ type: request.type,+ source: request.source,+ audioFeatures: request.audioFeatures,+ }),+ );+ const engine = Object.assign(new EventEmitter(), {+ isClosed: false,+ logContext: {},+ client: {currentState: SignalConnectionState.CONNECTED, sendUpdateLocalAudioTrack},+ pcManager: {publisher: {getTransceivers: () => [transceiver], setTrackCodecBitrate}},+ createSender: vi.fn(async () => sender),+ negotiate: vi.fn(async () => undefined),+ addTrack,+ });+ type ParticipantArgs = ConstructorParameters<typeof LocalParticipant>;+ const participant = new LocalParticipant(+ 'PA_publisher',+ 'publisher',+ engine as unknown as RTCEngine,+ {...roomOptionDefaults, publishDefaults},+ {} as ParticipantArgs[4],+ {} as ParticipantArgs[5],+ {} as ParticipantArgs[6],+ {} as ParticipantArgs[7],+ );+ participant.permissions = new ParticipantPermission({canPublish: true});+ return {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack};+}++describe('microphone publication stereo metadata', () => {+ beforeEach(() => {+ vi.stubGlobal('MediaStreamTrack', TestMediaStreamTrack);+ vi.stubGlobal('MediaStream', TestMediaStream);+ });++ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it.each([+ {name: 'mono policy on a two-channel device', channelCount: 2, forceStereo: false, stereo: false},+ {name: 'automatic stereo on a two-channel device', channelCount: 2, forceStereo: undefined, stereo: true},+ {name: 'forced stereo on a one-channel device', channelCount: 1, forceStereo: true, stereo: true},+ {name: 'automatic mono on a one-channel device', channelCount: 1, forceStereo: undefined, stereo: false},+ ])(+ 'keeps the request, SDP policy, and feature updates consistent for $name',+ async ({channelCount, forceStereo, stereo}) => {+ const {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack} = createPublisher();+ const source = new TestMediaStreamTrack(channelCount);+ const track = new LocalAudioTrack(source as unknown as MediaStreamTrack);+ track.source = Track.Source.Microphone;+ await track.runWithTrackChangeLock(async () => undefined);+ const publication = await participant.publishTrack(track, {+ audioPreset: {maxBitrate: 64000},+ forceStereo,+ });++ expect(addTrack).toHaveBeenCalledTimes(1);+ const request = addTrack.mock.calls[0]![0];+ expect(request.stereo).toBe(stereo);+ expect(request.audioFeatures.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);+ expect(request.audioFeatures).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION);+ expect(setTrackCodecBitrate).toHaveBeenCalledWith(expect.objectContaining({codec: 'opus', maxbr: 64, stereo}));++ expect(publication.getTrackFeatures().includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);+ track.emit(TrackEvent.AudioTrackFeatureUpdate, track, AudioTrackFeature.TF_ECHO_CANCELLATION, true);+ expect(sendUpdateLocalAudioTrack).toHaveBeenCalledTimes(1);+ const [sid, features] = sendUpdateLocalAudioTrack.mock.calls[0]!;+ expect(sid).toBe('TR_microphone');+ expect(features.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);+ expect(features).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION);+ track.stop();+ },+ );+});diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.tsindex d25f73b35..8ae43e7cb 100644--- a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts@@ -966,7 +966,7 @@ export default class LocalParticipant extends Participant {if (settings.noiseSuppression) {audioFeatures.push(AudioTrackFeature.TF_NOISE_SUPPRESSION);}- if (settings.channelCount && settings.channelCount > 1) {+ if (isStereo) {audioFeatures.push(AudioTrackFeature.TF_STEREO);}if (disableDtx) {diff --git a/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts b/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.tsindex de5b88d82..01095c574 100644--- a/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts+++ b/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts@@ -90,7 +90,7 @@ export default class LocalTrackPublication extends TrackPublication {if (settings.noiseSuppression) {features.add(AudioTrackFeature.TF_NOISE_SUPPRESSION);}- if (settings.channelCount && settings.channelCount > 1) {+ if (this.options?.forceStereo ?? (settings.channelCount !== undefined && settings.channelCount > 1)) {features.add(AudioTrackFeature.TF_STEREO);}if (!this.options?.dtx) {diff --git a/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx b/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsxindex e33315459..6329b1ee5 100644--- a/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx+++ b/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx@@ -294,7 +294,7 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoRvalue: backend,label: getNoiseSuppressionChoiceLabel(i18n, backend),}));- const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable();+ const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable(activeInputLabel);const setPushToTalkEnabled = (enabled: boolean) => {const mode = enabled ? 'voice_push_to_talk' : 'voice_activity';if (enabled && !isNativeDesktop) {diff --git a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.tsnew file mode 100644index 000000000..cd81f7570--- /dev/null+++ b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts@@ -0,0 +1,63 @@+import type {VoiceProcessingMode} from '@app/features/voice/utils/VoiceProcessingProfile';+import {beforeEach, describe, expect, it, vi} from 'vitest';++const {deviceModes, voiceSettings} = vi.hoisted(() => ({+ deviceModes: new Map<string, VoiceProcessingMode>(),+ voiceSettings: {+ voiceProcessingMode: 'voice' as VoiceProcessingMode,+ echoCancellation: true,+ autoGainControl: false,+ getNoiseSuppressionBackend: () => 'none',+ getStereoMicrophone: () => false,+ getVoiceProcessingModeForDeviceLabel: vi.fn<(label: string | null | undefined) => VoiceProcessingMode>(),+ },+}));++vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: voiceSettings}));+vi.mock('@app/features/voice/commands/VoiceSettingsCommands', () => ({update: vi.fn()}));+vi.mock('@app/features/voice/engine/VoiceDevicePermissionState', () => ({+ default: {getState: () => ({inputDevices: []})},+}));+vi.mock('@app/features/voice/utils/VoiceDeviceManager', () => ({resolveEffectiveDeviceId: () => null}));++import {isStereoMicrophoneChoiceAvailable} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionChoices';++describe('stereo microphone choice for the selected device', () => {+ beforeEach(() => {+ deviceModes.clear();+ voiceSettings.voiceProcessingMode = 'voice';+ voiceSettings.echoCancellation = true;+ voiceSettings.autoGainControl = false;+ voiceSettings.getVoiceProcessingModeForDeviceLabel.mockReset();+ voiceSettings.getVoiceProcessingModeForDeviceLabel.mockImplementation(+ (label) => (label ? deviceModes.get(label) : undefined) ?? voiceSettings.voiceProcessingMode,+ );+ });++ it('shows the switch for a device using direct input when the global mode is focused voice', () => {+ deviceModes.set('USB audio interface', 'studio');+ expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true);+ expect(voiceSettings.getVoiceProcessingModeForDeviceLabel).toHaveBeenCalledWith('USB audio interface');+ });++ it('hides the switch for a device with custom processing when the global mode is direct input', () => {+ voiceSettings.voiceProcessingMode = 'studio';+ deviceModes.set('Headset microphone', 'custom');+ expect(isStereoMicrophoneChoiceAvailable('Headset microphone')).toBe(false);+ });++ it('shows the switch for a custom device profile with processing disabled', () => {+ deviceModes.set('USB audio interface', 'custom');+ voiceSettings.echoCancellation = false;+ expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true);+ });++ it.each([+ {mode: 'voice' as const, available: false},+ {mode: 'studio' as const, available: true},+ ])('uses the global $mode profile when the device has no override', ({mode, available}) => {+ voiceSettings.voiceProcessingMode = mode;+ expect(isStereoMicrophoneChoiceAvailable('Unconfigured microphone')).toBe(available);+ expect(isStereoMicrophoneChoiceAvailable(null)).toBe(available);+ });+});diff --git a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.tsindex 302cdacb1..a0b3f1a8f 100644--- a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts+++ b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts@@ -12,7 +12,7 @@ import {readNoiseSuppressionRuntimeCapabilities,supportsStereoCapture,} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection';-import {resolveVoiceProcessingFromState} from '@app/features/voice/utils/VoiceProcessingProfile';+import {resolveVoiceProcessingFromStateForDeviceLabel} from '@app/features/voice/utils/VoiceProcessingProfile';export function getNoiseSuppressionChoiceValues(): ReadonlyArray<VoiceNoiseSuppressionBackend> {const capabilities = readNoiseSuppressionRuntimeCapabilities();@@ -29,8 +29,8 @@ export function setNoiseSuppressionChoice(backend: VoiceNoiseSuppressionBackend)VoiceSettingsCommands.update({noiseSuppressionBackend: backend});}-export function isStereoMicrophoneChoiceAvailable(): boolean {- return supportsStereoCapture(resolveVoiceProcessingFromState(VoiceSettings));+export function isStereoMicrophoneChoiceAvailable(deviceLabel: string | null): boolean {+ return supportsStereoCapture(resolveVoiceProcessingFromStateForDeviceLabel(VoiceSettings, deviceLabel));}export function isStereoMicrophoneEnabled(): boolean {