Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(voice): report real state in voice diagnostics (#2600)

Hampusef067f36c669
diff --git a/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.test.ts b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.test.ts
new file mode 100644
index 000000000..ef75a9ea0
--- /dev/null
+++ b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.test.ts
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+const refreshMicrophoneFromSettings = vi.fn();
+const refreshCameraBackgroundFromSettings = vi.fn();
+const refreshCameraCaptureFromSettings = vi.fn();
+const refreshScreenShareCodecNegotiationFromSettings = vi.fn();
+
+const settings = {
+ cameraResolution: 720,
+ videoDeviceId: 'camera',
+ screenshareResolution: 1080,
+ videoFrameRate: 60,
+ streamingMode: 'smooth',
+ screenShareContentHint: 'auto',
+ preferredScreenShareCodec: 'auto',
+ screenShareAv1OptIn: true,
+ screenShareHevcOptIn: false,
+ screenShareEncoderMode: 'auto',
+ screenShareSoftwareQuality: 'auto',
+ screenShareScalabilityMode: 'auto',
+ screenShareBackupCodecMode: 'auto',
+ openH264Enabled: false,
+};
+
+vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({
+ default: {
+ room: null,
+ refreshMicrophoneFromSettings,
+ refreshCameraBackgroundFromSettings,
+ refreshCameraCaptureFromSettings,
+ refreshScreenShareCodecNegotiationFromSettings,
+ setScreenShareAudioMuted: vi.fn(),
+ applyAllLocalAudioPreferences: vi.fn(),
+ applyLocalInputVolume: vi.fn(),
+ },
+}));
+
+vi.mock('@app/features/voice/engine/ScreenShareUnderperformance', () => ({
+ noteDeliberateScreenShareQualityChange: vi.fn(),
+}));
+
+vi.mock('@app/features/voice/utils/VoiceProcessingProfile', () => ({
+ getActiveInputDeviceLabel: () => null,
+}));
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({
+ default: {
+ updateSettings: (patch: Record<string, unknown>) => Object.assign(settings, patch),
+ get cameraResolution() {
+ return settings.cameraResolution;
+ },
+ get videoDeviceId() {
+ return settings.videoDeviceId;
+ },
+ getScreenshareResolution: () => settings.screenshareResolution,
+ getVideoFrameRate: () => settings.videoFrameRate,
+ getStreamingMode: () => settings.streamingMode,
+ getScreenShareContentHint: () => settings.screenShareContentHint,
+ getPreferredScreenShareCodec: () => settings.preferredScreenShareCodec,
+ getScreenShareAv1OptIn: () => settings.screenShareAv1OptIn,
+ getScreenShareHevcOptIn: () => settings.screenShareHevcOptIn,
+ getScreenShareEncoderMode: () => settings.screenShareEncoderMode,
+ getScreenShareSoftwareQuality: () => settings.screenShareSoftwareQuality,
+ getScreenShareScalabilityMode: () => settings.screenShareScalabilityMode,
+ getScreenShareBackupCodecMode: () => settings.screenShareBackupCodecMode,
+ getOpenH264Enabled: () => settings.openH264Enabled,
+ },
+}));
+
+const VoiceSettingsCommands = await import('./VoiceSettingsCommands');
+
+describe('screen-share codec settings reactions', () => {
+ beforeEach(() => {
+ settings.screenShareAv1OptIn = true;
+ settings.screenShareHevcOptIn = false;
+ settings.preferredScreenShareCodec = 'auto';
+ refreshScreenShareCodecNegotiationFromSettings.mockClear();
+ });
+
+ it('renegotiates the live screen share when the AV1 opt-in is turned off', () => {
+ VoiceSettingsCommands.update({screenShareAv1OptIn: false});
+ expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
+ });
+
+ it('renegotiates the live screen share when the HEVC opt-in changes', () => {
+ VoiceSettingsCommands.update({screenShareHevcOptIn: true});
+ expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
+ });
+
+ it('renegotiates the live screen share when the preferred codec changes', () => {
+ VoiceSettingsCommands.update({preferredScreenShareCodec: 'vp9'});
+ expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
+ });
+
+ it('leaves the negotiation alone for unrelated screen share settings', () => {
+ VoiceSettingsCommands.update({videoFrameRate: 30});
+ expect(refreshScreenShareCodecNegotiationFromSettings).not.toHaveBeenCalled();
+ });
+});
diff --git a/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts
index a9de6afa4..bc69d199b 100644
--- a/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts
+++ b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts
@@ -103,6 +103,11 @@ const CAMERA_BACKGROUND_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
'mirrorCamera',
];
const CAMERA_CAPTURE_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = ['cameraResolution', 'videoDeviceId'];
+const SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
+ 'preferredScreenShareCodec',
+ 'screenShareAv1OptIn',
+ 'screenShareHevcOptIn',
+];
function refreshMicrophone(): void {
MediaEngine.refreshMicrophoneFromSettings();
}
@@ -115,6 +120,10 @@ function refreshCameraCapture(): void {
MediaEngine.refreshCameraCaptureFromSettings();
}
+function refreshScreenShareCodecNegotiation(): void {
+ MediaEngine.refreshScreenShareCodecNegotiationFromSettings();
+}
+
function shouldRefreshMicrophone(settings: VoiceSettingsPatch): boolean {
return MICROPHONE_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
@@ -123,6 +132,10 @@ function shouldRefreshCameraBackground(settings: VoiceSettingsPatch): boolean {
return CAMERA_BACKGROUND_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
+function shouldRefreshScreenShareCodecNegotiation(settings: VoiceSettingsPatch): boolean {
+ return SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS.some((key) => settings[key] !== undefined);
+}
+
function readCameraCaptureRefreshValues(): VoiceSettingsPatch {
return {
cameraResolution: VoiceSettings.cameraResolution,
@@ -196,6 +209,9 @@ function applyUpdatedVoiceSettings(
if (shouldRefreshCameraCapture(settings, cameraCaptureBefore, readCameraCaptureRefreshValues())) {
refreshCameraCapture();
}
+ if (shouldRefreshScreenShareCodecNegotiation(settings)) {
+ refreshScreenShareCodecNegotiation();
+ }
}
export function setVoiceProcessingModeForDeviceLabel(label: string, mode: VoiceProcessingMode): void {
diff --git a/fluxer_app/src/features/voice/components/useStatsForNerds.ts b/fluxer_app/src/features/voice/components/useStatsForNerds.ts
index 01a28f6f8..128c51804 100644
--- a/fluxer_app/src/features/voice/components/useStatsForNerds.ts
+++ b/fluxer_app/src/features/voice/components/useStatsForNerds.ts
@@ -9,7 +9,6 @@ import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/Med
import ScreenShareCodecNegotiation, {
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
-import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
@@ -17,6 +16,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import {
buildVoiceStatsForNerdsPresentation,
+ collectScreenShareAudioPublicationDiagnostics,
type StatsForNerdsData,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {VoiceEngineV2PerTrackStats, VoiceEngineV2TransportInfo} from '@fluxer/voice_engine_v2';
@@ -216,8 +216,8 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {})
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
},
screenShareAudioCapture: {
- pump: getScreenShareAudioPumpDiagnostics(),
nativeCapture: getNativeAudioCaptureDiagnosticState(),
+ publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
},
appInfo: {
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx
index ee123b115..a28a4d7b1 100644
--- a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx
+++ b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx
@@ -27,6 +27,7 @@ function createStatsData(): StatsForNerdsData {
subscriberTransport: null,
},
localVideo: null,
+ localVideoLayers: [],
localAudio: null,
localScreenShare: null,
localScreenShareAudio: null,
@@ -68,20 +69,8 @@ function createStatsData(): StatsForNerdsData {
openH264Enabled: true,
},
screenShareAudioCapture: {
- pump: {
- active: false,
- captureId: null,
- sampleRate: null,
- channels: null,
- usesNativeSink: false,
- publishStrategy: 'none',
- publishedFormatKey: null,
- eagerPublish: null,
- eagerPublishError: null,
- droppedPushFrames: 0,
- pendingPushFrames: 0,
- },
nativeCapture: {},
+ publications: [],
},
appInfo: {
appVersion: 'dev',
@@ -111,6 +100,50 @@ function createStatsData(): StatsForNerdsData {
};
}
+function createNativeCaptureRecord(): Record<string, unknown> {
+ return {
+ armedCapture: null,
+ activeBridge: {captureId: 'native-audio:live'},
+ supersededBridge: null,
+ lastStartedCapture: null,
+ lastArmFailure: null,
+ bridgeStats: {
+ active: true,
+ bridgeMode: 'generator',
+ captureId: 'native-audio:live',
+ startedAt: 1772000000000,
+ lastFrameAt: 1772000011240,
+ lastFrameTimestampUs: 11240000,
+ framesReceived: 1124,
+ framesDropped: 0,
+ lateFrameCount: 0,
+ rebufferCount: 0,
+ maxFrameArrivalGapMs: 32,
+ maxFrameTimestampGapMs: 10,
+ maxPendingFrames: 3,
+ maxBufferedDurationMs: 30,
+ lastFramePeak: 0.42,
+ lastFrameRms: 0.11,
+ maxFramePeak: 0.71,
+ maxFrameRms: 0.19,
+ nonSilentFrameCount: 1118,
+ prebufferTargetMs: 60,
+ frameDurationMs: 10,
+ endReason: null,
+ endDetail: null,
+ endedAt: null,
+ },
+ endedBridgeCaptures: [],
+ lifecycleFaults: [],
+ };
+}
+
+function screenShareAudioCaptureSection(html: string): string {
+ const match = /<section[^>]*><h3[^>]*>screenShareAudioCapture \(native\)<\/h3>[\s\S]*?<\/section>/.exec(html);
+ expect(match).not.toBeNull();
+ return match?.[0] ?? '';
+}
+
describe('renderVoiceDebugStatsHtml', () => {
it('renders pure SVG sparklines for key popout metrics', () => {
const html = renderVoiceDebugStatsHtml(createStatsData(), '2026-06-10T18:00:00.000Z');
@@ -124,4 +157,57 @@ describe('renderVoiceDebugStatsHtml', () => {
expect(html).toContain('heapUsedMB sparkline');
expect(html).toContain('mainProcessCpuPercent sparkline');
});
+
+ it('does not render a screen-share audio pump section while the native capture is live', () => {
+ const data = createStatsData();
+ data.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
+
+ const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
+
+ expect(html).toContain('screenShareAudioCapture (native)');
+ expect(html).toContain('framesReceived');
+ expect(html).not.toContain('screenShareAudioCapture (pump)');
+ expect(html).not.toContain('publishStrategy');
+ expect(html).not.toContain('droppedPushFrames');
+ expect(html).not.toContain('usesNativeSink');
+ });
+
+ it('renders the screen-share audio capture section identically for Chromium and Firefox dumps', () => {
+ const chromium = createStatsData();
+ chromium.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
+ chromium.system.platform = 'Win32';
+ chromium.system.userAgent =
+ '(Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
+ const firefox = createStatsData();
+ firefox.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
+ firefox.system.platform = 'Win32';
+ firefox.system.userAgent = '(Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0';
+ firefox.appInfo.electronVersion = null;
+ firefox.appInfo.chromiumVersion = null;
+ firefox.appInfo.hardwareAccelerationEnabled = null;
+
+ const chromiumHtml = renderVoiceDebugStatsHtml(chromium, '2026-06-10T18:00:00.000Z');
+ const firefoxHtml = renderVoiceDebugStatsHtml(firefox, '2026-06-10T18:00:00.000Z');
+
+ expect(screenShareAudioCaptureSection(chromiumHtml)).toBe(screenShareAudioCaptureSection(firefoxHtml));
+ expect(chromiumHtml).not.toContain('screenShareAudioCapture (pump)');
+ expect(firefoxHtml).not.toContain('screenShareAudioCapture (pump)');
+ });
+
+ it('renders every camera simulcast layer instead of only the projected localVideo row', () => {
+ const data = createStatsData();
+ data.localVideoLayers = [
+ {direction: 'send', kind: 'video', ssrc: 1001, rid: 'q', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
+ {direction: 'send', kind: 'video', ssrc: 1002, rid: 'h', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
+ {direction: 'send', kind: 'video', ssrc: 1003, rid: 'f', active: true, bitrateKbps: 1133, bitrateWindowMs: 2001},
+ ];
+ data.localVideo = data.localVideoLayers[0];
+
+ const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
+
+ expect(html).toContain('track:localVideoLayers[0]');
+ expect(html).toContain('track:localVideoLayers[1]');
+ expect(html).toContain('track:localVideoLayers[2]');
+ expect(html).toContain('bitrateWindowMs');
+ });
});
diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx
index 02f214bcd..4724d48e4 100644
--- a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx
+++ b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx
@@ -233,6 +233,10 @@ function TrackTables({data}: TrackTablesProps) {
const tracks: Array<[string, VoiceEngineV2PerTrackStats | null]> = [
['track:localAudio', data.localAudio],
['track:localVideo', data.localVideo],
+ ...data.localVideoLayers.map((layer, index): [string, VoiceEngineV2PerTrackStats | null] => [
+ `track:localVideoLayers[${index}]`,
+ layer,
+ ]),
['track:localScreenShare', data.localScreenShare],
['track:localScreenShareAudio', data.localScreenShareAudio],
['track:remoteAudio', data.remoteAudio],
@@ -372,11 +376,6 @@ function VoiceDebugStats({data, generatedAtIso}: VoiceDebugStatsProps) {
record={asStatsRecord(data.screenShareSettings)}
data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--7"
/>
- <StatsTable
- title="screenShareAudioCapture (pump)"
- record={asStatsRecord(data.screenShareAudioCapture.pump)}
- data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--8"
- />
<StatsTable
title="screenShareAudioCapture (native)"
record={asStatsRecord(data.screenShareAudioCapture.nativeCapture)}
diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
index 8475533be..26cbd4bb9 100644
--- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
+++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts
@@ -749,6 +749,16 @@ class MediaEngineFacade extends Store {
await voiceEngineV2AppMediaExecutionAdapter.refreshCameraCapture();
}
+ refreshScreenShareCodecNegotiationFromSettings(): void {
+ void this.refreshScreenShareCodecNegotiationFromCurrentEngine().catch((error) => {
+ logger.warn('Failed to refresh screen share codec negotiation from settings', {error});
+ });
+ }
+
+ private async refreshScreenShareCodecNegotiationFromCurrentEngine(): Promise<void> {
+ await ScreenShareCodecNegotiation.publishLocalCapabilities(this.room, 'manual');
+ }
+
private reconcileLocalAudioStateInBackground(reason: string): void {
const previous = this.localAudioReconcileCoalescer;
const next = transitionVoiceLocalAudioReconcileCoalescerSnapshot(previous, {type: 'run.requested', reason});
diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts
index d8fb37c40..5f76ca5c2 100644
--- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts
+++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts
@@ -1,13 +1,17 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
+import type {
+ FluxerCodecAdvertisement,
+ FluxerSelectProtocolMessage,
+} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
-import {beforeEach, describe, expect, it, vi} from 'vitest';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
let av1OptIn = false;
let hevcOptIn = false;
let preferredScreenShareCodec = 'auto';
let gpuReport: HardwareEncodeReport | null = null;
+let firefoxBrowser = false;
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
@@ -26,7 +30,7 @@ vi.mock('@app/features/ui/utils/NativeUtils', () => ({
guessPlatform: () => 'windows',
isChromiumBrowser: () => true,
isDesktop: () => true,
- isFirefoxBrowser: () => false,
+ isFirefoxBrowser: () => firefoxBrowser,
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
@@ -48,6 +52,8 @@ vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
}));
vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({
+ clearScreenShareDecodeFailures: () => undefined,
+ getScreenShareDecodeFailures: () => new Set(),
getVideoDecoderExclusionsSync: () => [],
loadVideoDecoderExclusions: async () => [],
}));
@@ -216,6 +222,92 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => {
});
});
+describe('screen-share codec negotiation on Gecko', () => {
+ beforeEach(() => {
+ av1OptIn = false;
+ hevcOptIn = false;
+ preferredScreenShareCodec = 'auto';
+ gpuReport = null;
+ firefoxBrowser = true;
+ resetCachedCodecCapabilities();
+ });
+
+ afterEach(() => {
+ firefoxBrowser = false;
+ resetCachedCodecCapabilities();
+ });
+
+ it('ranks VP8 ahead of H.264 because Gecko always software-encodes H.264', () => {
+ expect(getScreenShareCodecPreferenceOrder()).toEqual(['vp8', 'h264']);
+ });
+
+ it('negotiates VP8 when both ends can encode and decode VP8 and H.264', () => {
+ const local = buildLocalCodecAdvertisements();
+ const remote = [buildLocalCodecAdvertisements()];
+ expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('vp8');
+ });
+
+ it('still lets an explicit H.264 pin win over the Gecko order', () => {
+ expect(getScreenShareCodecPreferenceOrder('h264')[0]).toBe('h264');
+ });
+});
+
+describe('advertising local capabilities over the negotiation topic', () => {
+ let room: FakeRoom;
+
+ beforeEach(() => {
+ av1OptIn = false;
+ hevcOptIn = false;
+ preferredScreenShareCodec = 'auto';
+ gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};
+ resetCachedCodecCapabilities();
+ room = new FakeRoom();
+ ScreenShareCodecNegotiation.setSelectionChangeListener(null);
+ });
+
+ async function publishAndCaptureSelectProtocol(): Promise<{
+ payload: Uint8Array;
+ message: FluxerSelectProtocolMessage;
+ }> {
+ Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
+ try {
+ ScreenShareCodecNegotiation.bind(room as never);
+ await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
+ } finally {
+ Reflect.deleteProperty(globalThis, 'window');
+ }
+ const payloads = (room.localParticipant.publishData.mock.calls as Array<Array<unknown>>).map(
+ (call) => call[0] as Uint8Array,
+ );
+ const payload = payloads.findLast(
+ (candidate) => (JSON.parse(new TextDecoder().decode(candidate)) as {op: number}).op === 1,
+ );
+ if (!payload) throw new Error('no select-protocol message was published');
+ return {payload, message: JSON.parse(new TextDecoder().decode(payload)) as FluxerSelectProtocolMessage};
+ }
+
+ it('advertises no experiments because nothing on either side reads them', async () => {
+ const {message} = await publishAndCaptureSelectProtocol();
+ expect(message.op).toBe(1);
+ expect(message.d.experiments).toEqual([]);
+ ScreenShareCodecNegotiation.dispose();
+ });
+
+ it('keeps the empty experiments list acceptable to the receive-side validator', async () => {
+ const {payload} = await publishAndCaptureSelectProtocol();
+ ScreenShareCodecNegotiation.dispose();
+ const peerRoom = new FakeRoom();
+ peerRoom.remoteParticipants.set('peer', {identity: 'peer'});
+ ScreenShareCodecNegotiation.bind(peerRoom as never);
+ const listener = vi.fn();
+ ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
+ peerRoom.emit(RoomEvent.DataReceived, payload, {identity: 'peer'}, undefined, SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC);
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(listener).toHaveBeenCalled();
+ ScreenShareCodecNegotiation.dispose();
+ });
+});
+
describe('reacting to a mid-session codec selection change', () => {
let room: FakeRoom;
@@ -267,6 +359,47 @@ describe('reacting to a mid-session codec selection change', () => {
ScreenShareCodecNegotiation.dispose();
});
+ async function withWindowDefined(run: () => Promise<void>): Promise<void> {
+ Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
+ try {
+ await run();
+ } finally {
+ Reflect.deleteProperty(globalThis, 'window');
+ }
+ }
+
+ it('stops reusing a live AV1 selection once the AV1 opt-in is turned off mid-share', async () => {
+ av1OptIn = true;
+ resetCachedCodecCapabilities();
+ await establishBaseline();
+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
+ av1OptIn = false;
+ resetCachedCodecCapabilities();
+ expect(ScreenShareCodecNegotiation.selectScreenShareCodec('auto')).not.toBe('av1');
+ expect(ScreenShareCodecNegotiation.selectNativeScreenShareCodec('auto')).not.toBe('av1');
+ expect(getScreenShareCodecPreferenceOrder()).toContain(ScreenShareCodecNegotiation.selectScreenShareCodec('auto'));
+ ScreenShareCodecNegotiation.dispose();
+ });
+
+ it('renegotiates the live share off AV1 when the opt-in toggle refreshes the local capabilities', async () => {
+ av1OptIn = true;
+ resetCachedCodecCapabilities();
+ await establishBaseline();
+ expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
+ const listener = vi.fn();
+ ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
+ av1OptIn = false;
+ resetCachedCodecCapabilities();
+ await withWindowDefined(async () => {
+ await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
+ });
+ const negotiated = ScreenShareCodecNegotiation.getSelectedCodec();
+ expect(negotiated).not.toBe('av1');
+ expect(listener).toHaveBeenCalledTimes(1);
+ expect(listener).toHaveBeenCalledWith(room, negotiated, 'manual');
+ ScreenShareCodecNegotiation.dispose();
+ });
+
it('does not notify the listener when the recomputed selection is unchanged', async () => {
await establishBaseline();
room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'});
diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts
index 3ed6fb826..173acbb45 100644
--- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts
+++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
+import {isFirefoxBrowser} from '@app/features/ui/utils/NativeUtils';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
type CodecCapabilityReport,
@@ -17,6 +18,8 @@ import {loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabili
import {loadNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities';
import {loadOpenH264Status} from '@app/features/voice/utils/OpenH264Status';
import {
+ clearScreenShareDecodeFailures,
+ getScreenShareDecodeFailures,
getVideoDecoderExclusionsSync,
loadVideoDecoderExclusions,
} from '@app/features/voice/utils/VideoDecoderCapabilities';
@@ -39,6 +42,7 @@ const RTP_PAYLOAD_TYPE_MAX = 255;
const CODEC_PRIORITY_MAX = 65_535;
const CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
const SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'vp9', 'h264', 'vp8', 'h265'];
+const GECKO_SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['vp8', 'h264'];
const COMPATIBILITY_FALLBACK_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['h264', 'vp9', 'vp8'];
const BASELINE_VIDEO_CODEC: VideoCodec = 'vp8';
const VIDEO_CODEC_NAMES: Record<VideoCodec, FluxerVideoCodecName> = {
@@ -163,6 +167,9 @@ function hasReceiverCapability(codec: VideoCodec): boolean | null {
function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
const exclusions = new Set(getVideoDecoderExclusionsSync() ?? []);
+ for (const codec of getScreenShareDecodeFailures()) {
+ exclusions.add(codec);
+ }
const result: Record<VideoCodec, boolean> = {
av1: false,
h265: false,
@@ -172,7 +179,7 @@ function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
};
for (const codec of CODEC_PREFERENCE) {
const advertised = hasReceiverCapability(codec);
- result[codec] = advertised === null ? result[codec] : advertised && !exclusions.has(codec);
+ result[codec] = (advertised === null ? result[codec] : advertised) && !exclusions.has(codec);
}
return result;
}
@@ -181,8 +188,11 @@ export function getScreenShareCodecPreferenceOrder(
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
): ReadonlyArray<VideoCodec> {
const encoderMode = resolveEffectiveScreenShareEncoderMode(VoiceSettings.getScreenShareEncoderMode());
- const automaticOrder =
- encoderMode === 'software' ? SOFTWARE_CODEC_PREFERENCE : getHardwareFirstScreenShareCodecPreferenceOrder();
+ const automaticOrder = isFirefoxBrowser()
+ ? GECKO_SOFTWARE_CODEC_PREFERENCE
+ : encoderMode === 'software'
+ ? SOFTWARE_CODEC_PREFERENCE
+ : getHardwareFirstScreenShareCodecPreferenceOrder();
const order =
preference !== 'auto' ? [preference, ...automaticOrder.filter((codec) => codec !== preference)] : automaticOrder;
return order.filter((codec) => isVideoCodecAllowedForPublish(codec));
@@ -537,6 +547,7 @@ class ScreenShareCodecNegotiation {
}
private canUseSelectedCodecForCurrentParticipants(codec: VideoCodec): boolean {
+ if (!isVideoCodecAllowedForPublish(codec)) return false;
if (!this.canLocalEncode(codec)) return false;
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs();
if (unknownParticipants > 0) return false;
@@ -631,6 +642,7 @@ class ScreenShareCodecNegotiation {
this.remoteCodecsByIdentity.clear();
this.mediaSessionId = createId('media');
this.negotiationSnapshot = createScreenShareCodecNegotiationSnapshot();
+ clearScreenShareDecodeFailures();
}
async publishLocalCapabilities(
@@ -665,7 +677,7 @@ class ScreenShareCodecNegotiation {
},
codecs: this.localCodecs,
rtc_connection_id: this.rtcConnectionId,
- experiments: ['fixed_keyframe_interval', 'maintain_framerate', 'opus_red', 'transport_cc', 'loss_based_bwe_v2'],
+ experiments: [],
},
};
await this.publishMessage(room, message);
diff --git a/fluxer_app/src/features/voice/engine/ScreenShareDecodeStallRenegotiation.test.ts b/fluxer_app/src/features/voice/engine/ScreenShareDecodeStallRenegotiation.test.ts
new file mode 100644
index 000000000..774d9326b
--- /dev/null
+++ b/fluxer_app/src/features/voice/engine/ScreenShareDecodeStallRenegotiation.test.ts
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
+import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+const gpuReport: HardwareEncodeReport = {
+ av1: 'hardware',
+ h265: 'hardware',
+ h264: 'hardware',
+ vp9: 'software',
+ vp8: 'software',
+};
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({
+ default: {
+ getScreenShareAv1OptIn: () => false,
+ getScreenShareHevcOptIn: () => false,
+ getPreferredScreenShareCodec: () => 'auto',
+ getScreenShareEncoderMode: () => 'auto',
+ },
+}));
+
+vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({
+ getCachedDesktopTroubleshootingSettings: () => null,
+}));
+
+vi.mock('@app/features/ui/utils/NativeUtils', () => ({
+ guessPlatform: () => 'windows',
+ isChromiumBrowser: () => true,
+ isDesktop: () => true,
+ isFirefoxBrowser: () => false,
+}));
+
+vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
+ getGpuEncoderReportSync: () => gpuReport,
+ loadGpuEncoderReport: async () => gpuReport,
+}));
+
+vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({
+ getNativeHardwareEncoderCapabilitiesSync: () => null,
+ hasNativeHardwareEncoder: () => false,
+ resetNativeHardwareEncoderCapabilities: () => undefined,
+ loadNativeHardwareEncoderCapabilities: async () => null,
+}));
+
+vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
+ getOpenH264StatusSync: () => null,
+ resetOpenH264Status: () => undefined,
+ loadOpenH264Status: async () => null,
+}));
+
+const VIDEO_CAPABILITIES = {
+ codecs: [
+ {mimeType: 'video/VP8'},
+ {mimeType: 'video/VP9'},
+ {mimeType: 'video/H264'},
+ {mimeType: 'video/H265'},
+ {mimeType: 'video/AV1'},
+ ],
+};
+
+Object.defineProperty(globalThis, 'RTCRtpSender', {
+ configurable: true,
+ writable: true,
+ value: {getCapabilities: () => VIDEO_CAPABILITIES},
+});
+
+Object.defineProperty(globalThis, 'RTCRtpReceiver', {
+ configurable: true,
+ writable: true,
+ value: {getCapabilities: () => VIDEO_CAPABILITIES},
+});
+
+const {
+ default: ScreenShareCodecNegotiation,
+ buildLocalCodecAdvertisements,
+ computeNegotiatedVideoCodec,
+} = await import('./ScreenShareCodecNegotiation');
+const {findStalledVideoDecoder} = await import('@app/features/voice/utils/ScreenShareCodecDiagnostics');
+const {getVideoDecoderExclusionsSync, markScreenShareDecodeFailure, resetVideoDecoderExclusions} = await import(
+ '@app/features/voice/utils/VideoDecoderCapabilities'
+);
+const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');
+
+function stalledScreenShareStats(mimeType: string): RTCStatsReport {
+ const entries: Array<Record<string, unknown>> = [
+ {id: 'codec-1', type: 'codec', mimeType},
+ {
+ id: 'inbound-1',
+ type: 'inbound-rtp',
+ kind: 'video',
+ codecId: 'codec-1',
+ packetsReceived: 4200,
+ bytesReceived: 3_500_000,
+ framesReceived: 180,
+ framesDecoded: 0,
+ },
+ ];
+ return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
+}
+
+function decodeAdvertisedFor(name: 'H264' | 'VP8' | 'VP9'): boolean | undefined {
+ return buildLocalCodecAdvertisements().find((codec) => codec.name === name)?.decode;
+}
+
+function publisherThatEncodes(): Array<FluxerCodecAdvertisement> {
+ return [
+ {name: 'H264', type: 'video', payload_type: 102, priority: 1, encode: true, decode: true},
+ {name: 'VP9', type: 'video', payload_type: 109, priority: 2, encode: true, decode: true},
+ ];
+}
+
+describe('a stalled H.264 screen share decode', () => {
+ beforeEach(() => {
+ resetVideoDecoderExclusions();
+ resetCachedCodecCapabilities();
+ ScreenShareCodecNegotiation.dispose();
+ });
+
+ it('is reported as an h264 decode stall by the stats diagnostic', () => {
+ expect(findStalledVideoDecoder(stalledScreenShareStats('video/H264'))?.codec).toBe('h264');
+ });
+
+ it('withdraws the local H.264 decode advertisement so the publisher renegotiates away from it', () => {
+ const stall = findStalledVideoDecoder(stalledScreenShareStats('video/H264'));
+ if (!stall) throw new Error('expected the stats diagnostic to report a stalled decoder');
+ expect(stall.codec).toBe('h264');
+ expect(decodeAdvertisedFor('H264')).toBe(true);
+ expect(
+ computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
+ ).toBe('h264');
+
+ expect(markScreenShareDecodeFailure(stall.codec, 'screen-share-decode-stalled')).toBe(true);
+
+ expect(decodeAdvertisedFor('H264')).toBe(false);
+ expect(
+ computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
+ ).toBe('vp9');
+ });
+
+ it('never removes h264 or vp8 from the SDP-level subscriber exclusions', () => {
+ markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled');
+ expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('h264');
+ expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('vp8');
+ });
+
+ it('keeps a non-baseline runtime stall out of the SDP-level subscriber exclusions too', () => {
+ expect(markScreenShareDecodeFailure('av1', 'screen-share-decode-stalled')).toBe(true);
+ expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('av1');
+ });
+
+ it('restores the H.264 decode advertisement when the room is torn down', () => {
+ expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
+ expect(decodeAdvertisedFor('H264')).toBe(false);
+ ScreenShareCodecNegotiation.dispose();
+ expect(decodeAdvertisedFor('H264')).toBe(true);
+ });
+
+ it('stops after one codec change instead of cycling through the remaining codecs', () => {
+ expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
+ expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(false);
+ expect(markScreenShareDecodeFailure('vp8', 'screen-share-decode-stalled')).toBe(false);
+ expect(decodeAdvertisedFor('VP8')).toBe(true);
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts
index 1a6698d20..75f9404c9 100644
--- a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts
+++ b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts
@@ -18,7 +18,7 @@ import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import {scheduleScreenShareDecoderVerification} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
-import {markVideoDecoderRuntimeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
+import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {parseVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity';
import type {
LocalParticipant,
@@ -266,7 +266,7 @@ export function bindRoomEvents(
screenShareDecoderVerificationTimers.delete(trackSid);
},
(failure) => {
- if (!markVideoDecoderRuntimeFailure(failure.codec, 'screen-share-decode-stalled')) return;
+ if (!markScreenShareDecodeFailure(failure.codec, 'screen-share-decode-stalled')) return;
void ScreenShareCodecNegotiation.publishLocalCapabilities(room, 'manual').catch((error) => {
logger.warn('Failed to publish updated codec capabilities after decode stall', {
error,
diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts
deleted file mode 100644
index 87d854bee..000000000
--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts
+++ /dev/null
@@ -1,31 +0,0 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-export interface ScreenShareAudioPumpDiagnostics {
- active: boolean;
- captureId: string | null;
- sampleRate: number | null;
- channels: number | null;
- usesNativeSink: boolean;
- publishStrategy: 'eager' | 'lazy' | 'none';
- publishedFormatKey: string | null;
- eagerPublish: 'succeeded' | 'failed' | 'skipped' | null;
- eagerPublishError: string | null;
- droppedPushFrames: number;
- pendingPushFrames: number;
-}
-
-export function getScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics {
- return {
- active: false,
- captureId: null,
- sampleRate: null,
- channels: null,
- usesNativeSink: false,
- publishStrategy: 'none',
- publishedFormatKey: null,
- eagerPublish: null,
- eagerPublishError: null,
- droppedPushFrames: 0,
- pendingPushFrames: 0,
- };
-}
diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts
new file mode 100644
index 000000000..d8d7beb19
--- /dev/null
+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts
@@ -0,0 +1,337 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {
+ type PerTrackStats,
+ VoiceEngineV2AppStatsHostAdapter,
+ type VoiceEngineV2AppStatsHostAdapterScheduler,
+} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter';
+import {classifyVoiceEngineV2TrackStats} from '@fluxer/voice_engine_v2';
+import type {Room} from 'livekit-client';
+import {describe, expect, it} from 'vitest';
+
+interface FakeScheduler extends VoiceEngineV2AppStatsHostAdapterScheduler {
+ runStatsTick(): void;
+}
+
+function createScheduler(): FakeScheduler {
+ const handlers = new Map<number, () => void>();
+ return {
+ setInterval(handler, intervalMs) {
+ handlers.set(intervalMs, handler);
+ return intervalMs;
+ },
+ clearInterval(handle) {
+ handlers.delete(handle as number);
+ },
+ runStatsTick() {
+ handlers.get(2000)?.();
+ },
+ };
+}
+
+function chromiumCameraSimulcastReports(bytesSentByRid: Record<string, number>): Map<string, unknown> {
+ return new Map<string, unknown>([
+ [
+ 'SV1',
+ {
+ type: 'media-source',
+ id: 'SV1',
+ kind: 'video',
+ trackIdentifier: 'cam-track',
+ width: 1280,
+ height: 720,
+ framesPerSecond: 30,
+ frames: 1801,
+ },
+ ],
+ ['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
+ [
+ 'OT01V1',
+ {
+ type: 'outbound-rtp',
+ id: 'OT01V1',
+ kind: 'video',
+ ssrc: 1001,
+ rid: 'q',
+ mid: '2',
+ active: false,
+ mediaSourceId: 'SV1',
+ codecId: 'C1',
+ bytesSent: bytesSentByRid.q,
+ framesEncoded: 1,
+ targetBitrate: 0,
+ qualityLimitationReason: 'none',
+ },
+ ],
+ [
+ 'OT01V2',
+ {
+ type: 'outbound-rtp',
+ id: 'OT01V2',
+ kind: 'video',
+ ssrc: 1002,
+ rid: 'h',
+ mid: '2',
+ active: false,
+ mediaSourceId: 'SV1',
+ codecId: 'C1',
+ bytesSent: bytesSentByRid.h,
+ framesEncoded: 1,
+ targetBitrate: 0,
+ qualityLimitationReason: 'none',
+ },
+ ],
+ [
+ 'OT01V3',
+ {
+ type: 'outbound-rtp',
+ id: 'OT01V3',
+ kind: 'video',
+ ssrc: 1003,
+ rid: 'f',
+ mid: '2',
+ active: true,
+ mediaSourceId: 'SV1',
+ codecId: 'C1',
+ bytesSent: bytesSentByRid.f,
+ framesEncoded: 1,
+ frameWidth: 1280,
+ frameHeight: 720,
+ targetBitrate: 1133000,
+ qualityLimitationReason: 'none',
+ },
+ ],
+ ]);
+}
+
+async function collectChromiumCameraRows(
+ samples: ReadonlyArray<{now: number; bytesSentByRid: Record<string, number>}>,
+): Promise<Array<PerTrackStats>> {
+ const scheduler = createScheduler();
+ let sampleIndex = 0;
+ let now = samples[0].now;
+ const publisher = {
+ getStats() {
+ return Promise.resolve(chromiumCameraSimulcastReports(samples[sampleIndex].bytesSentByRid));
+ },
+ };
+ const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => now, scheduler});
+ adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
+ adapter.startStatsTracking();
+ for (sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
+ now = samples[sampleIndex].now;
+ scheduler.runStatsTick();
+ await Promise.resolve();
+ await Promise.resolve();
+ await Promise.resolve();
+ }
+ const rows = adapter.perTrackStats;
+ adapter.cleanup();
+ return rows;
+}
+
+describe('VoiceEngineV2AppStatsHostAdapter chromium outbound rows', () => {
+ it('collects the outbound-rtp active flag for every simulcast layer', async () => {
+ const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
+
+ expect(rows.map((row) => row.rid)).toEqual(['q', 'h', 'f']);
+ expect(rows.map((row) => row.active)).toEqual([false, false, true]);
+ });
+
+ it('omits the bitrate window on a first observation so a zero bitrate is not read as measured', async () => {
+ const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
+
+ expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
+ expect(rows.map((row) => row.bitrateWindowMs)).toEqual([undefined, undefined, undefined]);
+ });
+
+ it('reports the bitrate window once a delta was measured so a zero bitrate is falsifiable', async () => {
+ const rows = await collectChromiumCameraRows([
+ {now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
+ {now: 3001, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
+ ]);
+
+ expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
+ expect(rows.map((row) => row.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
+ });
+
+ it('keeps every pre-existing chromium row field byte-identical', async () => {
+ const rows = await collectChromiumCameraRows([
+ {now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
+ {now: 3000, bytesSentByRid: {q: 0, h: 0, f: 5_500_000}},
+ ]);
+ const {active, bitrateWindowMs, ...legacy} = rows[2];
+
+ expect(legacy).toEqual({
+ direction: 'send',
+ kind: 'video',
+ ssrc: 1003,
+ rid: 'f',
+ mid: '2',
+ trackIdentifier: 'cam-track',
+ mediaSourceId: 'SV1',
+ codec: 'video/H264',
+ payloadType: 108,
+ bitrateKbps: 2000,
+ framesEncoded: 1,
+ frameWidth: 1280,
+ frameHeight: 720,
+ sourceFrameWidth: 1280,
+ sourceFrameHeight: 720,
+ sourceFramesPerSecond: 30,
+ sourceFrames: 1801,
+ targetBitrateKbps: 1133,
+ qualityLimitationReason: 'none',
+ encoderAcceleration: 'unknown',
+ });
+ });
+});
+
+function firefoxCameraAndScreenShareReports(): Map<string, unknown> {
+ return new Map<string, unknown>([
+ ['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
+ ['C2', {type: 'codec', id: 'C2', mimeType: 'audio/opus', payloadType: 109}],
+ [
+ 'outbound_rtp_audio_1',
+ {
+ type: 'outbound-rtp',
+ id: 'outbound_rtp_audio_1',
+ kind: 'audio',
+ ssrc: 724915693,
+ mid: '1',
+ codecId: 'C2',
+ bytesSent: 100_000,
+ },
+ ],
+ [
+ 'outbound_rtp_video_3',
+ {
+ type: 'outbound-rtp',
+ id: 'outbound_rtp_video_3',
+ kind: 'video',
+ ssrc: 4089824766,
+ mid: '3',
+ codecId: 'C1',
+ bytesSent: 5_000_000,
+ framesEncoded: 900,
+ },
+ ],
+ [
+ 'outbound_rtp_video_4',
+ {
+ type: 'outbound-rtp',
+ id: 'outbound_rtp_video_4',
+ kind: 'video',
+ ssrc: 1477030229,
+ mid: '4',
+ codecId: 'C1',
+ bytesSent: 2_000_000,
+ framesEncoded: 600,
+ },
+ ],
+ [
+ 'inbound_rtp_audio_2',
+ {
+ type: 'inbound-rtp',
+ id: 'inbound_rtp_audio_2',
+ kind: 'audio',
+ ssrc: 1661789229,
+ mid: '2',
+ codecId: 'C2',
+ trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
+ bytesReceived: 50_000,
+ },
+ ],
+ ]);
+}
+
+async function collectFirefoxRows(
+ transceivers: ReadonlyArray<{mid: string | null; sender?: {track?: {id: string} | null} | null}> | null,
+): Promise<Array<PerTrackStats>> {
+ const scheduler = createScheduler();
+ const publisher = {
+ getStats() {
+ return Promise.resolve(firefoxCameraAndScreenShareReports());
+ },
+ ...(transceivers === null ? {} : {getTransceivers: () => transceivers}),
+ };
+ const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler});
+ adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
+ adapter.startStatsTracking();
+ scheduler.runStatsTick();
+ await Promise.resolve();
+ await Promise.resolve();
+ await Promise.resolve();
+ const rows = adapter.perTrackStats;
+ adapter.cleanup();
+ return rows;
+}
+
+describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => {
+ it('backfills the outbound trackIdentifier from the transceiver mid when the report carries neither identity member', async () => {
+ const rows = await collectFirefoxRows([
+ {mid: '1', sender: {track: {id: '{mic-track}'}}},
+ {mid: '2', sender: {track: null}},
+ {mid: '3', sender: {track: {id: '{camera-track}'}}},
+ {mid: '4', sender: {track: {id: '{screen-track}'}}},
+ ]);
+
+ expect(rows.map((row) => [row.direction, row.kind, row.mid, row.trackIdentifier])).toEqual([
+ ['send', 'audio', '1', '{mic-track}'],
+ ['send', 'video', '3', '{camera-track}'],
+ ['send', 'video', '4', '{screen-track}'],
+ ['recv', 'audio', '2', '{8a106956-d202-4916-8e19-0a1672c64b73}'],
+ ]);
+ });
+
+ it('leaves an outbound row unidentified when no transceiver carries its mid', async () => {
+ const rows = await collectFirefoxRows([{mid: '3', sender: {track: {id: '{camera-track}'}}}]);
+
+ expect(rows.map((row) => row.trackIdentifier)).toEqual([
+ undefined,
+ '{camera-track}',
+ undefined,
+ '{8a106956-d202-4916-8e19-0a1672c64b73}',
+ ]);
+ });
+
+ it('keeps the inbound trackIdentifier when the source exposes no transceivers at all', async () => {
+ const rows = await collectFirefoxRows(null);
+
+ expect(rows.map((row) => row.trackIdentifier)).toEqual([
+ undefined,
+ undefined,
+ undefined,
+ '{8a106956-d202-4916-8e19-0a1672c64b73}',
+ ]);
+ });
+
+ it('lets the classifier tell a firefox camera apart from a simultaneous screen share', async () => {
+ const rows = await collectFirefoxRows([
+ {mid: '1', sender: {track: {id: '{mic-track}'}}},
+ {mid: '3', sender: {track: {id: '{camera-track}'}}},
+ {mid: '4', sender: {track: {id: '{screen-track}'}}},
+ ]);
+ const classification = classifyVoiceEngineV2TrackStats({
+ tracks: rows.map((row) => ({
+ direction: row.direction,
+ kind: row.kind,
+ rid: row.rid,
+ trackIdentifier: row.trackIdentifier,
+ bitrateKbps: row.bitrateKbps,
+ })),
+ publications: {
+ localCameraTrackId: '{camera-track}',
+ localMicrophoneTrackId: '{mic-track}',
+ localScreenShareTrackId: '{screen-track}',
+ localScreenShareAudioTrackId: null,
+ remoteMicrophoneTrackIds: [],
+ remoteScreenShareTrackIds: [],
+ remoteScreenShareAudioTrackIds: [],
+ },
+ });
+
+ expect(rows[classification.localVideoTrackIndex!].mid).toBe('3');
+ expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4');
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts
index 71875c3f2..6d27f3adc 100644
--- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts
+++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts
@@ -54,12 +54,14 @@ export interface PerTrackStats {
kind: 'audio' | 'video' | 'unknown';
ssrc?: number;
rid?: string;
+ active?: boolean;
mid?: string;
trackIdentifier?: string;
mediaSourceId?: string;
codec?: string;
payloadType?: number;
bitrateKbps: number;
+ bitrateWindowMs?: number;
packetsLost?: number;
packetsLossPercent?: number;
jitterMs?: number;
@@ -160,6 +162,12 @@ type RoomWithEngine = Room & {
interface StatsSource {
getStats(): Promise<StatsReportMap>;
+ getTransceivers?(): ReadonlyArray<StatsTransceiver>;
+}
+
+interface StatsTransceiver {
+ mid: string | null;
+ sender?: {track?: {id: string} | null} | null;
}
interface StatsReportMap {
@@ -186,6 +194,7 @@ interface RTCStatsEntry {
payloadType?: number;
ssrc?: number;
rid?: string;
+ active?: boolean;
mid?: string;
trackId?: string;
trackIdentifier?: string;
@@ -248,6 +257,11 @@ const LATENCY_UPDATE_INTERVAL_MS = 2000;
const STATS_CLOCK_INTERVAL_MS = 1000;
const STATS_UPDATE_INTERVAL_MS = 2000;
+interface BitrateSample {
+ bitrateKbps: number;
+ windowMs?: number;
+}
+
function normalizeTrackKind(kind: string | undefined): PerTrackStats['kind'] {
if (kind === 'audio' || kind === 'video') return kind;
return 'unknown';
@@ -264,28 +278,32 @@ function getReportKind(report: RTCStatsEntry, reportsById: Map<string, RTCStatsE
return normalizeTrackKind(track?.kind ?? track?.mediaType ?? mediaSource?.kind ?? mediaSource?.mediaType);
}
-function getBitrateKbps(
+function getBitrateSample(
reportId: string,
currentBytes: number | undefined,
now: number,
rtpCounters: Map<string, VoiceStatsRtpCounter>,
-): number {
+): BitrateSample {
if (typeof currentBytes !== 'number' || !Number.isFinite(currentBytes)) {
- return 0;
+ return {bitrateKbps: 0};
}
const previous = rtpCounters.get(reportId);
let bitrateKbps = 0;
+ let windowMs: number | undefined;
if (previous?.bytes !== undefined) {
const dt = (now - previous.timestamp) / 1000;
const db = currentBytes - previous.bytes;
- if (dt > 0 && db >= 0) bitrateKbps = (db * 8) / 1000 / dt;
+ if (dt > 0 && db >= 0) {
+ bitrateKbps = (db * 8) / 1000 / dt;
+ windowMs = Math.round(now - previous.timestamp);
+ }
}
rtpCounters.set(reportId, {
...previous,
bytes: currentBytes,
timestamp: now,
});
- return bitrateKbps;
+ return {bitrateKbps, windowMs};
}
function hasUsableLossDeltas(lostDelta: number, receivedDelta: number): boolean {
@@ -344,6 +362,7 @@ function hasUsableJitterBuffer(report: RTCStatsEntry): boolean {
function buildOutboundTrackExtras(report: RTCStatsEntry): Partial<PerTrackStats> {
return {
+ active: report.active,
retransmittedPacketsSent: report.retransmittedPacketsSent,
retransmittedBytesSent: report.retransmittedBytesSent,
keyFramesEncoded: report.keyFramesEncoded,
@@ -379,11 +398,12 @@ function buildPerTrackStat(args: {
now: number;
rtpCounters: Map<string, VoiceStatsRtpCounter>;
reportsById: Map<string, RTCStatsEntry>;
+ midToSenderTrackId: Map<string, string>;
}): PerTrackStats {
- const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById} = args;
+ const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId} = args;
const id = `${sourceId}:${report.type}:${report.id}`;
const currentBytes = isOutbound ? report.bytesSent : report.bytesReceived;
- const bitrateKbps = getBitrateKbps(id, currentBytes, now, rtpCounters);
+ const bitrate = getBitrateSample(id, currentBytes, now, rtpCounters);
const codec = report.codecId ? reportsById.get(report.codecId) : undefined;
const mediaSource = report.mediaSourceId ? reportsById.get(report.mediaSourceId) : undefined;
const packetsLossPercent = isInbound ? getPacketLossPercent(id, report, now, rtpCounters) : undefined;
@@ -402,11 +422,15 @@ function buildPerTrackStat(args: {
ssrc: report.ssrc,
rid: report.rid,
mid: report.mid,
- trackIdentifier: report.trackIdentifier ?? mediaSource?.trackIdentifier,
+ trackIdentifier:
+ report.trackIdentifier ??
+ mediaSource?.trackIdentifier ??
+ (isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined),
mediaSourceId: report.mediaSourceId,
codec: codec?.mimeType,
payloadType: codec?.payloadType,
- bitrateKbps: Math.round(bitrateKbps),
+ bitrateKbps: Math.round(bitrate.bitrateKbps),
+ bitrateWindowMs: bitrate.windowMs,
packetsLost: report.packetsLost,
packetsLossPercent: packetsLossPercent !== undefined ? Math.round(packetsLossPercent * 10) / 10 : undefined,
jitterMs: typeof report.jitter === 'number' ? Math.round(report.jitter * 1000 * 10) / 10 : undefined,
@@ -485,6 +509,11 @@ async function collectFromStatsSource(
transport: TransportInfo | null;
}> {
const reports = await source.getStats();
+ const midToSenderTrackId = new Map<string, string>();
+ for (const transceiver of source.getTransceivers?.() ?? []) {
+ const senderTrackId = transceiver.sender?.track?.id;
+ if (transceiver.mid && senderTrackId) midToSenderTrackId.set(transceiver.mid, senderTrackId);
+ }
const reportsById = new Map<string, RTCStatsEntry>();
for (const report of reports.values()) {
const statsEntry = report as RTCStatsEntry;
@@ -511,7 +540,9 @@ async function collectFromStatsSource(
if (!isOutbound && !isInbound) continue;
const id = `${sourceId}:${report.type}:${report.id}`;
activeCounterIds.add(id);
- tracks.push(buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById}));
+ tracks.push(
+ buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId}),
+ );
}
if (!activePair && transportReport?.selectedCandidatePairId) {
const selectedPair = reportsById.get(transportReport.selectedCandidatePairId);
diff --git a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts
new file mode 100644
index 000000000..ab6bed850
--- /dev/null
+++ b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts
@@ -0,0 +1,61 @@
+// @vitest-environment happy-dom
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {findStalledVideoEncoder} from '@app/features/voice/engine/voice_screen_share_manager/shared';
+import {describe, expect, test, vi} from 'vitest';
+
+vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false}));
+
+function makeStats(entries: ReadonlyArray<Record<string, unknown>>): RTCStatsReport {
+ return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
+}
+
+const CODEC_REPORT = {type: 'codec', id: 'codec-h264', mimeType: 'video/H264'};
+const SOURCE_REPORT = {type: 'media-source', id: 'source-1', kind: 'video', frames: 240, framesPerSecond: 30};
+
+function outboundReport(overrides: Record<string, unknown>): Record<string, unknown> {
+ return {
+ type: 'outbound-rtp',
+ id: 'outbound-1',
+ kind: 'video',
+ codecId: 'codec-h264',
+ mediaSourceId: 'source-1',
+ framesEncoded: 0,
+ framesSent: 0,
+ ...overrides,
+ };
+}
+
+describe('findStalledVideoEncoder', () => {
+ test('reports a stall when the encoding is active and the source is producing frames', () => {
+ const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: true})]);
+ expect(findStalledVideoEncoder(stats, 'h264')).toEqual({
+ codec: 'h264',
+ framesEncoded: 0,
+ framesSent: 0,
+ sourceFrames: 240,
+ sourceFramesPerSecond: 30,
+ });
+ });
+
+ test('ignores an encoding the SFU deactivated even though the source is producing frames', () => {
+ const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: false})]);
+ expect(findStalledVideoEncoder(stats, 'h264')).toBeNull();
+ });
+
+ test('still reports a stall when active is absent, as on Firefox', () => {
+ const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({})]);
+ expect(findStalledVideoEncoder(stats, 'h264')?.codec).toBe('h264');
+ });
+
+ test('skips a deactivated layer and reports the still-active sibling instead', () => {
+ const stats = makeStats([
+ CODEC_REPORT,
+ SOURCE_REPORT,
+ {type: 'media-source', id: 'source-low', kind: 'video', frames: 90, framesPerSecond: 15},
+ outboundReport({id: 'outbound-low', mediaSourceId: 'source-low', active: false}),
+ outboundReport({id: 'outbound-high', active: true}),
+ ]);
+ expect(findStalledVideoEncoder(stats, 'h264')?.sourceFrames).toBe(240);
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts
index be791405f..92ced32df 100644
--- a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts
+++ b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts
@@ -469,6 +469,7 @@ interface OutboundVideoStatsEntry {
mediaType?: string;
codecId?: string;
mediaSourceId?: string;
+ active?: boolean;
framesEncoded?: number;
framesSent?: number;
encoderImplementation?: string;
@@ -592,6 +593,7 @@ export function findStalledVideoEncoder(stats: RTCStatsReport, codec?: VideoCode
}
for (const report of reports) {
if (getStatsKind(report, reportsById) !== 'video') continue;
+ if (report.active === false) continue;
const mimeType = report.codecId ? reportsById.get(report.codecId)?.mimeType : undefined;
if (report.codecId && !codecMatchesTarget(mimeType, codec)) continue;
const resolvedCodec = codec ?? getVideoCodecFromMimeType(mimeType);
diff --git a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts
new file mode 100644
index 000000000..1e0db0bf5
--- /dev/null
+++ b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
+import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
+import type {
+ NativeAudioBridgeEndedCapture,
+ NativeAudioBridgeStats,
+} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
+import {describe, expect, it} from 'vitest';
+
+describe('getNativeAudioCaptureDiagnosticState', () => {
+ it('keeps the six fields real dumps already carry and appends the retained histories', () => {
+ expect(Object.keys(getNativeAudioCaptureDiagnosticState())).toEqual([
+ 'armedCapture',
+ 'activeBridge',
+ 'supersededBridge',
+ 'lastStartedCapture',
+ 'lastArmFailure',
+ 'bridgeStats',
+ 'endedBridgeCaptures',
+ 'lifecycleFaults',
+ ]);
+ });
+
+ it('emits the superseded capture next to the live one instead of only the replacement', () => {
+ startBridgeStats('generator', 'native-audio:diag-failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
+ recordBridgeFrame('native-audio:diag-failed', {timestampUs: 0, durationUs: 10_000, peak: 0.31, rms: 0.08});
+ startBridgeStats('generator', 'native-audio:diag-live', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
+
+ const state = getNativeAudioCaptureDiagnosticState();
+
+ expect((state.bridgeStats as NativeAudioBridgeStats).captureId).toBe('native-audio:diag-live');
+ const ended = state.endedBridgeCaptures as Array<NativeAudioBridgeEndedCapture>;
+ const retained = ended.find((capture) => capture.captureId === 'native-audio:diag-failed');
+ expect(retained?.framesReceived).toBe(1);
+ expect(retained?.nonSilentFrameCount).toBe(1);
+ expect(retained?.endReason).toBe('superseded');
+ });
+
+ it('exposes a native-audio fault history that outlives the lifecycle unbind', () => {
+ expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true);
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts
index 304fb6272..836807bb5 100644
--- a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts
+++ b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts
@@ -4,7 +4,10 @@ import {Logger} from '@app/features/platform/utils/AppLogger';
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge';
import {getNativeAudioErrorDetail} from '@app/features/voice/utils/NativeAudioFailureUtils';
-import {getBridgeStats as getNativeAudioBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
+import {
+ getEndedBridgeCaptures,
+ getBridgeStats as getNativeAudioBridgeStats,
+} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import {createGeneratorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge';
import {createScriptProcessorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge';
import {
@@ -67,8 +70,18 @@ interface NativeAudioStartedCaptureDiagnostic {
includeSelfWindowAudio?: boolean;
}
+interface NativeAudioLifecycleFaultDiagnostic {
+ captureId: string;
+ sourceId: string;
+ message: string;
+ atMs: number;
+}
+
+const MAX_RETAINED_LIFECYCLE_FAULTS = 8;
+
let lastStartedCapture: NativeAudioStartedCaptureDiagnostic | null = null;
let sourceLifecycleBridge: VoiceEngineV2AppSourceLifecycleBridge | null = null;
+let lifecycleFaults: Array<NativeAudioLifecycleFaultDiagnostic> = [];
const lifecycleBoundCaptureIds = new Set<string>();
export {getNativeAudioBridgeStats};
@@ -93,9 +106,19 @@ function bindNativeAudioCaptureLifecycle(captureId: string): void {
}
}
+function recordLifecycleFault(captureId: string, message: string): void {
+ lifecycleFaults.push({captureId, sourceId: `native-audio-tap:${captureId}`, message, atMs: Date.now()});
+ if (lifecycleFaults.length > MAX_RETAINED_LIFECYCLE_FAULTS) {
+ lifecycleFaults = lifecycleFaults.slice(-MAX_RETAINED_LIFECYCLE_FAULTS);
+ }
+}
+
function unbindNativeAudioCaptureLifecycle(captureId: string, faulted: boolean): void {
if (!lifecycleBoundCaptureIds.has(captureId)) return;
const bridge = sourceLifecycleBridge;
+ if (faulted) {
+ recordLifecycleFault(captureId, 'native-audio-tap-track-ended');
+ }
if (bridge) {
if (faulted) {
bridge.reportLifecycle({captureId, kind: 'error', message: 'native-audio-tap-track-ended'});
@@ -167,6 +190,8 @@ export function getNativeAudioCaptureDiagnosticState(): Record<string, unknown>
lastStartedCapture: started,
lastArmFailure: getLastNativeAudioArmFailure(),
bridgeStats: getNativeAudioBridgeStats(),
+ endedBridgeCaptures: getEndedBridgeCaptures(),
+ lifecycleFaults: lifecycleFaults.map((fault) => ({...fault})),
};
}
@@ -1499,5 +1524,6 @@ export function resetNativeAudioCaptureBridgeForTests(): void {
activeBridge = null;
supersededBridge = null;
sourceLifecycleBridge = null;
+ lifecycleFaults = [];
lifecycleBoundCaptureIds.clear();
}
diff --git a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts
index 58347fdba..943dc7aa8 100644
--- a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts
+++ b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts
@@ -15,7 +15,6 @@ import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import ScreenShareCodecNegotiation, {
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
-import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump';
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
@@ -36,6 +35,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import {
buildVoiceStatsForNerdsPresentation,
+ collectScreenShareAudioPublicationDiagnostics,
type StatsForNerdsData,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {NativeAudioApplication, VirtmicNode} from '@app/types/electron.d';
@@ -519,8 +519,8 @@ export function collectStatsForNerdsSnapshot(): StatsForNerdsData {
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
},
screenShareAudioCapture: {
- pump: getScreenShareAudioPumpDiagnostics(),
nativeCapture: getNativeAudioCaptureDiagnosticState(),
+ publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
},
appInfo: {
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
diff --git a/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts b/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts
index 486068b51..56dbd9f7e 100644
--- a/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts
+++ b/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts
@@ -45,9 +45,11 @@ const CODEC_PROBES: ReadonlyArray<CodecProbe> = [
},
];
+const SCREEN_SHARE_DECODE_FAILURES_MAX = 1;
+
let cachedExclusions: Array<VideoCodec> | null = null;
let pendingProbe: Promise<Array<VideoCodec>> | null = null;
-const runtimeExcludedCodecs = new Set<VideoCodec>();
+const screenShareDecodeFailures = new Set<VideoCodec>();
interface VideoDecoderLike {
isConfigSupported(config: WebCodecsConfig): Promise<{supported: boolean}>;
@@ -80,15 +82,6 @@ function isBaselineWebRtcDecodeCodec(codec: VideoCodec): boolean {
return codec === 'h264' || codec === 'vp8';
}
-function mergeRuntimeExclusions(exclusions: Array<VideoCodec> | null): Array<VideoCodec> | null {
- if (runtimeExcludedCodecs.size === 0) return exclusions;
- const merged = new Set(exclusions ?? []);
- for (const codec of runtimeExcludedCodecs) {
- merged.add(codec);
- }
- return [...merged];
-}
-
function getPlatformPolicyExclusions(): Array<VideoCodec> | null {
if ((globalThis as Record<string, unknown>).RTCRtpReceiver === undefined) return null;
const excluded: Array<VideoCodec> = [];
@@ -145,17 +138,17 @@ async function probeAllCodecs(): Promise<Array<VideoCodec>> {
}
export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
- if (cachedExclusions) return Promise.resolve(mergeRuntimeExclusions(cachedExclusions) ?? []);
+ if (cachedExclusions) return Promise.resolve(cachedExclusions);
if (pendingProbe) return pendingProbe;
pendingProbe = probeAllCodecs()
.then((result) => {
cachedExclusions = result;
- return mergeRuntimeExclusions(result) ?? [];
+ return result;
})
.catch((error) => {
logger.warn('Video decoder probe failed entirely, not excluding any codecs', {error});
cachedExclusions = [];
- return mergeRuntimeExclusions([]) ?? [];
+ return [];
})
.finally(() => {
pendingProbe = null;
@@ -164,21 +157,29 @@ export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
}
export function getVideoDecoderExclusionsSync(): Array<VideoCodec> | null {
- return mergeRuntimeExclusions(cachedExclusions ?? getPlatformPolicyExclusions());
+ return cachedExclusions ?? getPlatformPolicyExclusions();
}
-export function markVideoDecoderRuntimeFailure(codec: VideoCodec, reason: string): boolean {
- if (isBaselineWebRtcDecodeCodec(codec)) return false;
- if (runtimeExcludedCodecs.has(codec)) return false;
- runtimeExcludedCodecs.add(codec);
- logger.warn('Excluding codec from subscriber decode after runtime failure', {codec, reason});
+export function markScreenShareDecodeFailure(codec: VideoCodec, reason: string): boolean {
+ if (screenShareDecodeFailures.has(codec)) return false;
+ if (screenShareDecodeFailures.size >= SCREEN_SHARE_DECODE_FAILURES_MAX) return false;
+ screenShareDecodeFailures.add(codec);
+ logger.warn('Withdrawing decode advertisement for a codec after a screen share decode stall', {codec, reason});
return true;
}
+export function getScreenShareDecodeFailures(): ReadonlySet<VideoCodec> {
+ return screenShareDecodeFailures;
+}
+
+export function clearScreenShareDecodeFailures(): void {
+ screenShareDecodeFailures.clear();
+}
+
export function resetVideoDecoderExclusions(): void {
cachedExclusions = null;
pendingProbe = null;
- runtimeExcludedCodecs.clear();
+ screenShareDecodeFailures.clear();
}
if (typeof window !== 'undefined') {
diff --git a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts
index de5ffa238..da97e430e 100644
--- a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts
+++ b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts
@@ -2,6 +2,7 @@
import {
buildVoiceStatsForNerdsPresentation,
+ collectScreenShareAudioPublicationDiagnostics,
type ParticipantPublicationLookup,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {
@@ -86,6 +87,64 @@ function nativeStats(overrides: Partial<VoiceEngineV2Stats>): VoiceEngineV2Stats
return {rttMs: null, outbound: [], inbound: [], ...overrides};
}
+function chromiumCameraSimulcastTracks(): Array<VoiceEngineV2PerTrackStats> {
+ return [
+ {
+ direction: 'send',
+ kind: 'video',
+ ssrc: 1001,
+ rid: 'q',
+ active: false,
+ mid: '2',
+ trackIdentifier: 'cam-track',
+ mediaSourceId: 'SV1',
+ bitrateKbps: 0,
+ bitrateWindowMs: 2001,
+ framesEncoded: 1,
+ targetBitrateKbps: 0,
+ },
+ {
+ direction: 'send',
+ kind: 'video',
+ ssrc: 1002,
+ rid: 'h',
+ active: false,
+ mid: '2',
+ trackIdentifier: 'cam-track',
+ mediaSourceId: 'SV1',
+ bitrateKbps: 0,
+ bitrateWindowMs: 2001,
+ framesEncoded: 1,
+ targetBitrateKbps: 0,
+ },
+ {
+ direction: 'send',
+ kind: 'video',
+ ssrc: 1003,
+ rid: 'f',
+ active: true,
+ mid: '2',
+ trackIdentifier: 'cam-track',
+ mediaSourceId: 'SV1',
+ bitrateKbps: 0,
+ bitrateWindowMs: 2001,
+ framesEncoded: 1,
+ frameWidth: 1280,
+ frameHeight: 720,
+ targetBitrateKbps: 1133,
+ },
+ {
+ direction: 'send',
+ kind: 'video',
+ ssrc: 2001,
+ mid: '3',
+ trackIdentifier: 'screen-track',
+ mediaSourceId: 'SV2',
+ bitrateKbps: 3500,
+ },
+ ];
+}
+
describe('buildVoiceStatsForNerdsPresentation', () => {
it('classifies browser per-track stats once for overlay and copy consumers', () => {
const localParticipant = participant({
@@ -180,6 +239,203 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
expect(presentation.remoteAudio).toBeNull();
});
+ it('classifies a firefox screen share whose outbound stats omit the track identifier', () => {
+ const localParticipant = participant({
+ microphone: 'b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36',
+ screen_share: '2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f',
+ });
+ const remoteParticipant = participant({microphone: '8a106956-d202-4916-8e19-0a1672c64b73'});
+ const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
+ {
+ direction: 'recv',
+ kind: 'audio',
+ mid: '2',
+ trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
+ bitrateKbps: 46,
+ },
+ {direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
+ {direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
+ ];
+
+ const presentation = buildVoiceStatsForNerdsPresentation({
+ connectionId: 'connection-a',
+ connectionQuality: 'excellent',
+ currentLatency: 37,
+ averageLatency: 41,
+ stats: voiceStats,
+ perTrackStats,
+ statsTimeSeries: timeSeries,
+ nativeStats: null,
+ publisherTransport: null,
+ subscriberTransport: null,
+ localParticipant,
+ remoteParticipants: [remoteParticipant],
+ });
+
+ expect(presentation.localScreenShare?.mid).toBe('3');
+ expect(presentation.localVideo).toBeNull();
+ });
+
+ it('classifies a firefox inbound screen share reported with a braced track identifier', () => {
+ const remoteParticipant = participant({screen_share: '34171e0d-174d-4d56-bb71-76b4fba330c3'});
+ const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
+ {
+ direction: 'recv',
+ kind: 'video',
+ mid: '3',
+ trackIdentifier: '{34171e0d-174d-4d56-bb71-76b4fba330c3}',
+ bitrateKbps: 3585,
+ },
+ ];
+
+ const presentation = buildVoiceStatsForNerdsPresentation({
+ connectionId: 'connection-a',
+ connectionQuality: 'excellent',
+ currentLatency: 37,
+ averageLatency: 41,
+ stats: voiceStats,
+ perTrackStats,
+ statsTimeSeries: timeSeries,
+ nativeStats: null,
+ publisherTransport: null,
+ subscriberTransport: null,
+ localParticipant: null,
+ remoteParticipants: [remoteParticipant],
+ });
+
+ expect(presentation.remoteScreenShare?.mid).toBe('3');
+ expect(presentation.remoteVideo).toBeNull();
+ });
+
+ it('exposes every camera simulcast layer next to the projected localVideo row', () => {
+ const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
+ const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
+
+ const presentation = buildVoiceStatsForNerdsPresentation({
+ connectionId: 'connection-a',
+ connectionQuality: 'excellent',
+ currentLatency: 37,
+ averageLatency: 41,
+ stats: voiceStats,
+ perTrackStats,
+ statsTimeSeries: timeSeries,
+ nativeStats: null,
+ publisherTransport: null,
+ subscriberTransport: null,
+ localParticipant,
+ remoteParticipants: null,
+ });
+
+ expect(presentation.localVideo?.rid).toBe('q');
+ expect(presentation.localVideoLayers).toHaveLength(3);
+ expect(presentation.localVideoLayers.map((layer) => layer.rid)).toEqual(['q', 'h', 'f']);
+ expect(presentation.localVideoLayers.map((layer) => layer.active)).toEqual([false, false, true]);
+ expect(presentation.localVideoLayers.map((layer) => layer.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
+ });
+
+ it('keeps the screen-share publication out of the camera layer view', () => {
+ const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
+ const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
+
+ const presentation = buildVoiceStatsForNerdsPresentation({
+ connectionId: 'connection-a',
+ connectionQuality: 'excellent',
+ currentLatency: 37,
+ averageLatency: 41,
+ stats: voiceStats,
+ perTrackStats,
+ statsTimeSeries: timeSeries,
+ nativeStats: null,
+ publisherTransport: null,
+ subscriberTransport: null,
+ localParticipant,
+ remoteParticipants: null,
+ });
+
+ expect(presentation.localScreenShare?.trackIdentifier).toBe('screen-track');
+ expect(presentation.localVideoLayers).not.toContain(perTrackStats[3]);
+ });
+
+ it('reports a single camera layer when firefox omits rid and media source ids', () => {
+ const localParticipant = participant({camera: 'cam-track'});
+ const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
+ {direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
+ {direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
+ ];
+
+ const presentation = buildVoiceStatsForNerdsPresentation({
+ connectionId: 'connection-a',
+ connectionQuality: 'excellent',
+ currentLatency: 37,
+ averageLatency: 41,
+ stats: voiceStats,
+ perTrackStats,
+ statsTimeSeries: timeSeries,
+ nativeStats: null,
+ publisherTransport: null,
+ subscriberTransport: null,
+ localParticipant,
+ remoteParticipants: null,
+ });
+
+ expect(presentation.localVideo?.mid).toBe('3');
+ expect(presentation.localVideoLayers).toEqual([perTrackStats[1]]);
+ });
+
+ it('reports no camera layers rather than a false single layer when firefox simulcast is unidentifiable', () => {
+ const localParticipant = participant({camera: 'cam-track'});
+ const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
+ {direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
+ {direction: 'send', kind: 'video', mid: '3', rid: 'q', bitrateKbps: 0},
+ {direction: 'send', kind: 'video', mid: '3', rid: 'h', bitrateKbps: 0},
+ {direction: 'send', kind: 'video', mid: '3', rid: 'f', bitrateKbps: 1133},
+ ];
+
+ const presentation = buildVoiceStatsForNerdsPresentation({
+ connectionId: 'connection-a',
+ connectionQuality: 'excellent',
+ currentLatency: 37,
+ averageLatency: 41,
+ stats: voiceStats,
+ perTrackStats,
+ statsTimeSeries: timeSeries,
+ nativeStats: null,
+ publisherTransport: null,
+ subscriberTransport: null,
+ localParticipant,
+ remoteParticipants: null,
+ });
+
+ expect(presentation.localVideo).toBeNull();
+ expect(presentation.localVideoLayers).toEqual([]);
+ });
+
+ it('keeps the chromium simulcast projection byte-identical', () => {
+ const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
+ const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
+
+ const presentation = buildVoiceStatsForNerdsPresentation({
+ connectionId: 'connection-a',
+ connectionQuality: 'excellent',
+ currentLatency: 37,
+ averageLatency: 41,
+ stats: voiceStats,
+ perTrackStats,
+ statsTimeSeries: timeSeries,
+ nativeStats: null,
+ publisherTransport: null,
+ subscriberTransport: null,
+ localParticipant,
+ remoteParticipants: null,
+ });
+
+ expect(presentation.localVideo).toBe(perTrackStats[0]);
+ expect(presentation.localScreenShare).toBe(perTrackStats[3]);
+ expect(presentation.localAudio).toBeNull();
+ expect(presentation.remoteVideo).toBeNull();
+ expect(presentation.network.videoSendBitrateKbps).toBe(1200);
+ });
+
it('uses the canonical v2 native stats projection when native stats are available', () => {
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{direction: 'send', kind: 'video', trackIdentifier: 'browser-camera', bitrateKbps: 500},
@@ -309,3 +565,87 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
expect(presentation.network.rttMs).toBeNull();
});
});
+
+describe('collectScreenShareAudioPublicationDiagnostics', () => {
+ it('reports the mute and upstream state of every local screen-share audio publication', () => {
+ const displayPublication = {
+ trackSid: 'TR_display',
+ source: 'screen_share_audio',
+ isMuted: false,
+ audioTrack: {
+ isUpstreamPaused: true,
+ mediaStreamTrack: {
+ id: 'display-audio',
+ readyState: 'live',
+ muted: true,
+ enabled: true,
+ } as MediaStreamTrack,
+ },
+ };
+ const devicePublication = {
+ trackSid: 'TR_device',
+ source: 'screen_share_audio',
+ isMuted: false,
+ audioTrack: {
+ isUpstreamPaused: false,
+ mediaStreamTrack: {
+ id: 'device-audio',
+ readyState: 'live',
+ muted: false,
+ enabled: true,
+ } as MediaStreamTrack,
+ },
+ };
+ const micPublication = {
+ trackSid: 'TR_mic',
+ source: 'microphone',
+ isMuted: false,
+ audioTrack: {
+ isUpstreamPaused: false,
+ mediaStreamTrack: {
+ id: 'mic-audio',
+ readyState: 'live',
+ muted: false,
+ enabled: true,
+ } as MediaStreamTrack,
+ },
+ };
+ const localParticipant: ParticipantPublicationLookup = {
+ getTrackPublication: (source) => (String(source) === 'screen_share_audio' ? displayPublication : undefined),
+ audioTrackPublications: new Map([
+ ['TR_display', displayPublication],
+ ['TR_device', devicePublication],
+ ['TR_mic', micPublication],
+ ]),
+ };
+
+ const diagnostics = collectScreenShareAudioPublicationDiagnostics(localParticipant);
+
+ expect(diagnostics).toEqual([
+ {
+ trackSid: 'TR_display',
+ source: 'screen_share_audio',
+ isMuted: false,
+ isUpstreamPaused: true,
+ mediaStreamTrackId: 'display-audio',
+ mediaStreamTrackReadyState: 'live',
+ mediaStreamTrackMuted: true,
+ mediaStreamTrackEnabled: true,
+ },
+ {
+ trackSid: 'TR_device',
+ source: 'screen_share_audio',
+ isMuted: false,
+ isUpstreamPaused: false,
+ mediaStreamTrackId: 'device-audio',
+ mediaStreamTrackReadyState: 'live',
+ mediaStreamTrackMuted: false,
+ mediaStreamTrackEnabled: true,
+ },
+ ]);
+ });
+
+ it('reports nothing when there is no local participant', () => {
+ expect(collectScreenShareAudioPublicationDiagnostics(null)).toEqual([]);
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts
index 8441f154b..42b289497 100644
--- a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts
+++ b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts
@@ -6,7 +6,6 @@ import {
type VoiceTrackPublicationSourceLike,
VoiceTrackSource,
} from '@app/features/voice/engine/VoiceTrackSource';
-import type {ScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {
classifyVoiceEngineV2TrackStats,
selectVoiceEngineV2StatsPresentationProjection,
@@ -21,6 +20,17 @@ import {
} from '@fluxer/voice_engine_v2';
import type {Track} from 'livekit-client';
+export interface ScreenShareAudioPublicationDiagnostic {
+ trackSid: string | null;
+ source: string | null;
+ isMuted: boolean | null;
+ isUpstreamPaused: boolean | null;
+ mediaStreamTrackId: string | null;
+ mediaStreamTrackReadyState: string | null;
+ mediaStreamTrackMuted: boolean | null;
+ mediaStreamTrackEnabled: boolean | null;
+}
+
export interface StatsForNerdsData {
session: {
connectionId: string;
@@ -44,6 +54,7 @@ export interface StatsForNerdsData {
subscriberTransport: VoiceEngineV2TransportInfo | null;
};
localVideo: VoiceEngineV2PerTrackStats | null;
+ localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
localAudio: VoiceEngineV2PerTrackStats | null;
localScreenShare: VoiceEngineV2PerTrackStats | null;
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
@@ -85,8 +96,8 @@ export interface StatsForNerdsData {
openH264Enabled: boolean;
};
screenShareAudioCapture: {
- pump: ScreenShareAudioPumpDiagnostics;
nativeCapture: Record<string, unknown>;
+ publications: Array<ScreenShareAudioPublicationDiagnostic>;
};
appInfo: {
appVersion: string;
@@ -119,6 +130,7 @@ export interface VoiceStatsForNerdsPresentation {
session: StatsForNerdsData['session'];
network: StatsForNerdsData['network'];
localVideo: VoiceEngineV2PerTrackStats | null;
+ localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
localAudio: VoiceEngineV2PerTrackStats | null;
localScreenShare: VoiceEngineV2PerTrackStats | null;
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
@@ -131,9 +143,12 @@ export interface VoiceStatsForNerdsPresentation {
interface PublicationTrackLike {
mediaStreamTrack?: MediaStreamTrack;
+ isUpstreamPaused?: boolean;
}
interface TrackPublicationLike extends VoiceTrackPublicationSourceLike {
+ trackSid?: string;
+ isMuted?: boolean;
audioTrack?: PublicationTrackLike | null;
videoTrack?: PublicationTrackLike | null;
track?: PublicationTrackLike | null;
@@ -234,6 +249,38 @@ function collectParticipantScreenShareAudioTrackIdentifiers(
return Array.from(identifiers);
}
+function summarizeScreenShareAudioPublication(
+ publication: TrackPublicationLike,
+): ScreenShareAudioPublicationDiagnostic {
+ const track = publication.audioTrack ?? publication.track ?? null;
+ const mediaStreamTrack = track?.mediaStreamTrack ?? null;
+ return {
+ trackSid: publication.trackSid ?? null,
+ source: typeof publication.source === 'string' ? publication.source : null,
+ isMuted: publication.isMuted ?? null,
+ isUpstreamPaused: track?.isUpstreamPaused ?? null,
+ mediaStreamTrackId: mediaStreamTrack?.id ?? null,
+ mediaStreamTrackReadyState: mediaStreamTrack?.readyState ?? null,
+ mediaStreamTrackMuted: mediaStreamTrack?.muted ?? null,
+ mediaStreamTrackEnabled: mediaStreamTrack?.enabled ?? null,
+ };
+}
+
+export function collectScreenShareAudioPublicationDiagnostics(
+ participant: ParticipantPublicationLookup | null | undefined,
+): Array<ScreenShareAudioPublicationDiagnostic> {
+ const publications = new Set<TrackPublicationLike>();
+ const directPublication = participant?.getTrackPublication(SCREEN_SHARE_AUDIO_SOURCE);
+ if (directPublication) {
+ publications.add(directPublication);
+ }
+ for (const publication of participant?.audioTrackPublications?.values() ?? []) {
+ if (!isScreenShareAudioPublicationLike(publication)) continue;
+ publications.add(publication);
+ }
+ return Array.from(publications, summarizeScreenShareAudioPublication);
+}
+
function collectRemoteScreenShareAudioTrackIdentifiers(
remoteParticipants: Iterable<ParticipantPublicationLookup> | null | undefined,
): Array<string> {
@@ -284,6 +331,25 @@ function getClassifiedTrack(
return perTrackStats[index] ?? null;
}
+function getPublicationGroupKey(track: VoiceEngineV2PerTrackStats): string | null {
+ if (track.mediaSourceId !== undefined) return `source:${track.mediaSourceId}`;
+ if (track.trackIdentifier !== undefined) return `track:${track.trackIdentifier}`;
+ if (track.mid !== undefined) return `mid:${track.mid}`;
+ return null;
+}
+
+function selectLocalVideoLayers(
+ perTrackStats: ReadonlyArray<VoiceEngineV2PerTrackStats>,
+ localVideo: VoiceEngineV2PerTrackStats | null,
+): Array<VoiceEngineV2PerTrackStats> {
+ if (!localVideo) return [];
+ const groupKey = getPublicationGroupKey(localVideo);
+ if (groupKey === null) return [localVideo];
+ return perTrackStats.filter(
+ (track) => track.direction === 'send' && track.kind === 'video' && getPublicationGroupKey(track) === groupKey,
+ );
+}
+
function selectNativeStatsProjection(
nativeStats: VoiceEngineV2Stats | null,
): VoiceEngineV2StatsPresentationProjection | null {
@@ -366,6 +432,7 @@ export function buildVoiceStatsForNerdsPresentation(
subscriberTransport,
},
localVideo: nativeProjection ? nativeProjection.localVideo : localVideo,
+ localVideoLayers: selectLocalVideoLayers(perTrackStats, localVideo),
localAudio: nativeProjection ? nativeProjection.localAudio : localAudio,
localScreenShare: nativeProjection ? nativeProjection.localScreenShare : localScreenShare,
localScreenShareAudio: nativeProjection ? nativeProjection.localScreenShareAudio : localScreenShareAudio,
diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.test.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.test.ts
new file mode 100644
index 000000000..29defd223
--- /dev/null
+++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.test.ts
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {describe, expect, it, vi} from 'vitest';
+import {
+ endBridgeStats,
+ getBridgeStats,
+ getEndedBridgeCaptures,
+ recordBridgeFrame,
+ startBridgeStats,
+} from './bridgeStats';
+
+function recordFrames(captureId: string, count: number, peak: number, rms: number): void {
+ for (let index = 0; index < count; index++) {
+ recordBridgeFrame(captureId, {timestampUs: index * 10_000, durationUs: 10_000, peak, rms});
+ }
+}
+
+function findEndedCapture(captureId: string) {
+ return getEndedBridgeCaptures().find((capture) => capture.captureId === captureId);
+}
+
+describe('bridgeStats', () => {
+ it('retains the counters of a capture that a mid-share restart supersedes', () => {
+ startBridgeStats('generator', 'native-audio:failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
+ recordFrames('native-audio:failed', 1124, 0.42, 0.11);
+
+ startBridgeStats('generator', 'native-audio:replacement', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
+
+ expect(getBridgeStats().captureId).toBe('native-audio:replacement');
+ expect(getBridgeStats().framesReceived).toBe(0);
+ const retained = findEndedCapture('native-audio:failed');
+ expect(retained?.bridgeMode).toBe('generator');
+ expect(retained?.framesReceived).toBe(1124);
+ expect(retained?.nonSilentFrameCount).toBe(1124);
+ expect(retained?.lastFramePeak).toBe(0.42);
+ expect(retained?.endReason).toBe('superseded');
+ });
+
+ it('records the end reason of a capture the newer capture already superseded', () => {
+ startBridgeStats('generator', 'native-audio:ends-late');
+ recordFrames('native-audio:ends-late', 3, 0.2, 0.05);
+ startBridgeStats('generator', 'native-audio:ends-late-replacement');
+
+ endBridgeStats('native-audio:ends-late', 'ended', 'source-disappeared');
+
+ const retained = findEndedCapture('native-audio:ends-late');
+ expect(retained?.endReason).toBe('ended');
+ expect(retained?.endDetail).toBe('source-disappeared');
+ expect(retained?.endedAt).not.toBeNull();
+ expect(getBridgeStats().captureId).toBe('native-audio:ends-late-replacement');
+ expect(getBridgeStats().active).toBe(true);
+ });
+
+ it('still ends the active capture in place', () => {
+ startBridgeStats('script-processor', 'native-audio:active-end');
+
+ endBridgeStats('native-audio:active-end', 'cleanup', 'caller-stopped');
+
+ const stats = getBridgeStats();
+ expect(stats.active).toBe(false);
+ expect(stats.endReason).toBe('cleanup');
+ expect(stats.endDetail).toBe('caller-stopped');
+ expect(stats.endedAt).not.toBeNull();
+ });
+
+ it('keeps the first end reason of the active capture', () => {
+ startBridgeStats('generator', 'native-audio:double-end');
+ endBridgeStats('native-audio:double-end', 'ended', 'source-disappeared');
+
+ endBridgeStats('native-audio:double-end', 'cleanup', 'caller-stopped');
+
+ expect(getBridgeStats().endReason).toBe('ended');
+ expect(getBridgeStats().endDetail).toBe('source-disappeared');
+ });
+
+ it('flags a capture whose frames keep arriving while every one of them is silence', () => {
+ const startMs = Date.UTC(2026, 8, 8, 12, 0, 0);
+ vi.useFakeTimers();
+ try {
+ vi.setSystemTime(startMs);
+ startBridgeStats('generator', 'native-audio:all-silent', {frameDurationUs: 10_000});
+ for (let index = 0; index < 400; index++) {
+ vi.setSystemTime(startMs + index * 100);
+ recordBridgeFrame('native-audio:all-silent', {
+ timestampUs: index * 10_000,
+ durationUs: 10_000,
+ peak: 0,
+ rms: 0,
+ });
+ }
+
+ const stats = getBridgeStats();
+ expect(stats.framesReceived).toBe(400);
+ expect(stats.nonSilentFrameCount).toBe(0);
+ expect(stats.lastNonSilentFrameAt).toBeNull();
+ expect(stats.silentFrameStreak).toBe(400);
+ expect(stats.maxSilentRunMs).toBe(39_900);
+ expect(stats.sustainedSilenceWarned).toBe(true);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('does not flag a capture that keeps producing audible frames', () => {
+ const startMs = Date.UTC(2026, 8, 8, 13, 0, 0);
+ vi.useFakeTimers();
+ try {
+ vi.setSystemTime(startMs);
+ startBridgeStats('generator', 'native-audio:audible', {frameDurationUs: 10_000});
+ for (let index = 0; index < 400; index++) {
+ vi.setSystemTime(startMs + index * 100);
+ const audible = index % 4 === 0;
+ recordBridgeFrame('native-audio:audible', {
+ timestampUs: index * 10_000,
+ durationUs: 10_000,
+ peak: audible ? 0.3 : 0,
+ rms: audible ? 0.1 : 0,
+ });
+ }
+
+ const stats = getBridgeStats();
+ expect(stats.nonSilentFrameCount).toBe(100);
+ expect(stats.lastNonSilentFrameAt).toBe(startMs + 396 * 100);
+ expect(stats.silentFrameStreak).toBe(3);
+ expect(stats.maxSilentRunMs).toBe(300);
+ expect(stats.sustainedSilenceWarned).toBe(false);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('bounds the retained capture history', () => {
+ for (let index = 0; index < 12; index++) {
+ startBridgeStats('generator', `native-audio:bounded-${index}`);
+ }
+
+ expect(getEndedBridgeCaptures().length).toBeLessThanOrEqual(8);
+ expect(findEndedCapture('native-audio:bounded-11')).toBeUndefined();
+ expect(findEndedCapture('native-audio:bounded-10')).toBeDefined();
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts
index 9501d31a2..9ee952ef0 100644
--- a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts
+++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts
@@ -1,22 +1,54 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
+import {Logger} from '@app/features/platform/utils/AppLogger';
import {
initialBridgeStats,
+ type NativeAudioBridgeEndedCapture,
type NativeAudioBridgeFrameMetrics,
type NativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
+const logger = new Logger('NativeAudioBridgeStats');
+
+const MAX_ENDED_BRIDGE_CAPTURES = 8;
+const SUPERSEDED_END_REASON = 'superseded';
+const SUSTAINED_SILENCE_WARN_MS = 30_000;
+
let bridgeStats: NativeAudioBridgeStats = {...initialBridgeStats};
+let endedBridgeCaptures: Array<NativeAudioBridgeEndedCapture> = [];
export function getBridgeStats(): NativeAudioBridgeStats {
return {...bridgeStats};
}
+export function getEndedBridgeCaptures(): Array<NativeAudioBridgeEndedCapture> {
+ return endedBridgeCaptures.map((capture) => ({...capture}));
+}
+
+function retireBridgeStats(): void {
+ if (bridgeStats.captureId === null) return;
+ endedBridgeCaptures.push({
+ captureId: bridgeStats.captureId,
+ bridgeMode: bridgeStats.bridgeMode,
+ startedAt: bridgeStats.startedAt,
+ endedAt: bridgeStats.endedAt ?? Date.now(),
+ endReason: bridgeStats.endReason ?? SUPERSEDED_END_REASON,
+ endDetail: bridgeStats.endDetail,
+ framesReceived: bridgeStats.framesReceived,
+ nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
+ lastFramePeak: bridgeStats.lastFramePeak,
+ });
+ if (endedBridgeCaptures.length > MAX_ENDED_BRIDGE_CAPTURES) {
+ endedBridgeCaptures = endedBridgeCaptures.slice(-MAX_ENDED_BRIDGE_CAPTURES);
+ }
+}
+
export function startBridgeStats(
mode: 'generator' | 'script-processor',
captureId: string,
options: {prebufferTargetUs?: number; frameDurationUs?: number} = {},
): void {
+ retireBridgeStats();
bridgeStats = {
...initialBridgeStats,
active: true,
@@ -57,6 +89,24 @@ export function recordBridgeFrame(captureId: string, metrics: NativeAudioBridgeF
bridgeStats.maxFrameRms = Math.max(bridgeStats.maxFrameRms, bridgeStats.lastFrameRms);
if (peak >= 0.0005 || rms >= 0.0001) {
bridgeStats.nonSilentFrameCount += 1;
+ bridgeStats.lastNonSilentFrameAt = now;
+ bridgeStats.silentFrameStreak = 0;
+ } else {
+ bridgeStats.silentFrameStreak += 1;
+ const silentSinceMs = bridgeStats.lastNonSilentFrameAt ?? bridgeStats.startedAt ?? now;
+ const silentRunMs = Math.max(0, now - silentSinceMs);
+ bridgeStats.maxSilentRunMs = Math.max(bridgeStats.maxSilentRunMs, silentRunMs);
+ if (!bridgeStats.sustainedSilenceWarned && silentRunMs >= SUSTAINED_SILENCE_WARN_MS) {
+ bridgeStats.sustainedSilenceWarned = true;
+ logger.warn('Native audio capture is delivering frames that contain only silence', {
+ captureId: bridgeStats.captureId,
+ bridgeMode: bridgeStats.bridgeMode,
+ framesReceived: bridgeStats.framesReceived,
+ nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
+ silentFrameStreak: bridgeStats.silentFrameStreak,
+ silentRunMs,
+ });
+ }
}
}
}
@@ -95,9 +145,16 @@ export function recordBridgeQueue(captureId: string, pendingFrames: number, buff
}
export function endBridgeStats(captureId: string, reason: string | null, detail: string | null): void {
- if (!bridgeStats.active || bridgeStats.captureId !== captureId) return;
- bridgeStats.active = false;
- bridgeStats.endReason = reason;
- bridgeStats.endDetail = detail;
- bridgeStats.endedAt = Date.now();
+ if (bridgeStats.active && bridgeStats.captureId === captureId) {
+ bridgeStats.active = false;
+ bridgeStats.endReason = reason;
+ bridgeStats.endDetail = detail;
+ bridgeStats.endedAt = Date.now();
+ return;
+ }
+ const ended = endedBridgeCaptures.find((capture) => capture.captureId === captureId);
+ if (!ended || ended.endReason !== SUPERSEDED_END_REASON) return;
+ ended.endReason = reason;
+ ended.endDetail = detail;
+ ended.endedAt = Date.now();
}
diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts
index 0d8b2608e..9bed519a9 100644
--- a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts
+++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts
@@ -122,6 +122,10 @@ export interface NativeAudioBridgeStats {
maxFramePeak: number;
maxFrameRms: number;
nonSilentFrameCount: number;
+ lastNonSilentFrameAt: number | null;
+ silentFrameStreak: number;
+ maxSilentRunMs: number;
+ sustainedSilenceWarned: boolean;
prebufferTargetMs: number | null;
frameDurationMs: number | null;
endReason: string | null;
@@ -129,6 +133,18 @@ export interface NativeAudioBridgeStats {
endedAt: number | null;
}
+export interface NativeAudioBridgeEndedCapture {
+ captureId: string;
+ bridgeMode: 'generator' | 'script-processor' | null;
+ startedAt: number | null;
+ endedAt: number | null;
+ endReason: string | null;
+ endDetail: string | null;
+ framesReceived: number;
+ nonSilentFrameCount: number;
+ lastFramePeak: number | null;
+}
+
export interface NativeAudioBridgeFrameMetrics {
timestampUs?: number;
durationUs?: number;
@@ -156,6 +172,10 @@ export const initialBridgeStats: NativeAudioBridgeStats = {
maxFramePeak: 0,
maxFrameRms: 0,
nonSilentFrameCount: 0,
+ lastNonSilentFrameAt: null,
+ silentFrameStreak: 0,
+ maxSilentRunMs: 0,
+ sustainedSilenceWarned: false,
prebufferTargetMs: null,
frameDurationMs: null,
endReason: null,
diff --git a/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json b/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json
index ad5c2f1ee..341c7c313 100644
--- a/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json
+++ b/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json
@@ -119,5 +119,177 @@
"remoteScreenShareTrackIndex": 3,
"remoteScreenShareAudioTrackIndex": null
}
+ },
+ {
+ "name": "classifies a firefox screen share whose outbound stats omit the track identifier",
+ "input": {
+ "tracks": [
+ {
+ "direction": "recv",
+ "kind": "audio",
+ "trackIdentifier": "{8a106956-d202-4916-8e19-0a1672c64b73}",
+ "bitrateKbps": 46
+ },
+ {"direction": "send", "kind": "audio", "bitrateKbps": 30},
+ {"direction": "send", "kind": "video", "bitrateKbps": 47}
+ ],
+ "publications": {
+ "localCameraTrackId": null,
+ "localMicrophoneTrackId": "b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36",
+ "localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f",
+ "localScreenShareAudioTrackId": null,
+ "remoteMicrophoneTrackIds": ["8a106956-d202-4916-8e19-0a1672c64b73"],
+ "remoteScreenShareTrackIds": [],
+ "remoteScreenShareAudioTrackIds": []
+ }
+ },
+ "expected": {
+ "localVideoTrackIndex": null,
+ "localAudioTrackIndex": 1,
+ "localScreenShareTrackIndex": 2,
+ "localScreenShareAudioTrackIndex": null,
+ "remoteVideoTrackIndex": null,
+ "remoteAudioTrackIndex": 0,
+ "remoteScreenShareTrackIndex": null,
+ "remoteScreenShareAudioTrackIndex": null
+ }
+ },
+ {
+ "name": "classifies a firefox inbound screen share reported with a braced track identifier",
+ "input": {
+ "tracks": [
+ {
+ "direction": "recv",
+ "kind": "audio",
+ "trackIdentifier": "{9010d313-264b-4c51-9f13-9a6aefafce83}",
+ "bitrateKbps": 29
+ },
+ {
+ "direction": "recv",
+ "kind": "video",
+ "trackIdentifier": "{34171e0d-174d-4d56-bb71-76b4fba330c3}",
+ "bitrateKbps": 3585
+ },
+ {"direction": "send", "kind": "audio", "bitrateKbps": 44}
+ ],
+ "publications": {
+ "localCameraTrackId": null,
+ "localMicrophoneTrackId": "7c5a1b98-3d24-4e6f-8a10-9b2c3d4e5f60",
+ "localScreenShareTrackId": null,
+ "localScreenShareAudioTrackId": null,
+ "remoteMicrophoneTrackIds": ["9010d313-264b-4c51-9f13-9a6aefafce83"],
+ "remoteScreenShareTrackIds": ["34171e0d-174d-4d56-bb71-76b4fba330c3"],
+ "remoteScreenShareAudioTrackIds": []
+ }
+ },
+ "expected": {
+ "localVideoTrackIndex": null,
+ "localAudioTrackIndex": 2,
+ "localScreenShareTrackIndex": null,
+ "localScreenShareAudioTrackIndex": null,
+ "remoteVideoTrackIndex": null,
+ "remoteAudioTrackIndex": 0,
+ "remoteScreenShareTrackIndex": 1,
+ "remoteScreenShareAudioTrackIndex": null
+ }
+ },
+ {
+ "name": "leaves both local video roles unresolved when unidentified sends could be camera or screen share",
+ "input": {
+ "tracks": [
+ {"direction": "send", "kind": "video", "bitrateKbps": 1600},
+ {"direction": "send", "kind": "video", "bitrateKbps": 500}
+ ],
+ "publications": {
+ "localCameraTrackId": "c1e7d3b0-88f4-45a9-9c62-0ab5d1e2f374",
+ "localMicrophoneTrackId": null,
+ "localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f",
+ "localScreenShareAudioTrackId": null,
+ "remoteMicrophoneTrackIds": [],
+ "remoteScreenShareTrackIds": [],
+ "remoteScreenShareAudioTrackIds": []
+ }
+ },
+ "expected": {
+ "localVideoTrackIndex": null,
+ "localAudioTrackIndex": null,
+ "localScreenShareTrackIndex": null,
+ "localScreenShareAudioTrackIndex": null,
+ "remoteVideoTrackIndex": null,
+ "remoteAudioTrackIndex": null,
+ "remoteScreenShareTrackIndex": null,
+ "remoteScreenShareAudioTrackIndex": null
+ }
+ },
+ {
+ "name": "keeps chromium outbound identifier matching for a screen share published without a camera",
+ "input": {
+ "tracks": [
+ {
+ "direction": "send",
+ "kind": "video",
+ "trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
+ "bitrateKbps": 1600
+ },
+ {"direction": "send", "kind": "audio", "bitrateKbps": 30}
+ ],
+ "publications": {
+ "localCameraTrackId": null,
+ "localMicrophoneTrackId": null,
+ "localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
+ "localScreenShareAudioTrackId": null,
+ "remoteMicrophoneTrackIds": [],
+ "remoteScreenShareTrackIds": [],
+ "remoteScreenShareAudioTrackIds": []
+ }
+ },
+ "expected": {
+ "localVideoTrackIndex": null,
+ "localAudioTrackIndex": 1,
+ "localScreenShareTrackIndex": 0,
+ "localScreenShareAudioTrackIndex": null,
+ "remoteVideoTrackIndex": null,
+ "remoteAudioTrackIndex": null,
+ "remoteScreenShareTrackIndex": null,
+ "remoteScreenShareAudioTrackIndex": null
+ }
+ },
+ {
+ "name": "keeps chromium identified sends eligible for the camera role while a screen share is published",
+ "input": {
+ "tracks": [
+ {
+ "direction": "send",
+ "kind": "video",
+ "trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
+ "bitrateKbps": 1600
+ },
+ {
+ "direction": "send",
+ "kind": "video",
+ "trackIdentifier": "1d0a4c62-5b3e-4f71-88d9-6c2a7e50b431",
+ "bitrateKbps": 400
+ }
+ ],
+ "publications": {
+ "localCameraTrackId": null,
+ "localMicrophoneTrackId": null,
+ "localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
+ "localScreenShareAudioTrackId": null,
+ "remoteMicrophoneTrackIds": [],
+ "remoteScreenShareTrackIds": [],
+ "remoteScreenShareAudioTrackIds": []
+ }
+ },
+ "expected": {
+ "localVideoTrackIndex": 1,
+ "localAudioTrackIndex": null,
+ "localScreenShareTrackIndex": 0,
+ "localScreenShareAudioTrackIndex": null,
+ "remoteVideoTrackIndex": null,
+ "remoteAudioTrackIndex": null,
+ "remoteScreenShareTrackIndex": null,
+ "remoteScreenShareAudioTrackIndex": null
+ }
}
]
diff --git a/packages/voice_engine_v2/src/policies/voiceStats.ts b/packages/voice_engine_v2/src/policies/voiceStats.ts
index 7b3d61336..97d7a6bde 100644
--- a/packages/voice_engine_v2/src/policies/voiceStats.ts
+++ b/packages/voice_engine_v2/src/policies/voiceStats.ts
@@ -513,24 +513,55 @@ function normalizedId(id: string | null | undefined): string | null {
return trimmed ? trimmed : null;
}
+function canonicalId(id: string | null | undefined): string | null {
+ const normalized = normalizedId(id);
+ if (normalized === null) return null;
+ const unwrapped =
+ normalized.startsWith('{') && normalized.endsWith('}') ? normalized.slice(1, -1).trim() : normalized;
+ return unwrapped ? unwrapped.toLowerCase() : null;
+}
+
function idList(ids: ReadonlyArray<string>): Array<string> {
- return ids.map((id) => normalizedId(id)).filter((id): id is string => id !== null);
+ return ids.map((id) => canonicalId(id)).filter((id): id is string => id !== null);
}
function matchesId(candidate: IndexedVoiceEngineV2StatsTrack, id: string | null): boolean {
- const trackId = normalizedId(candidate.track.trackIdentifier);
- return trackId !== null && id !== null && trackId === id;
+ const trackId = canonicalId(candidate.track.trackIdentifier);
+ const publicationId = canonicalId(id);
+ return trackId !== null && publicationId !== null && trackId === publicationId;
}
function idInList(candidate: IndexedVoiceEngineV2StatsTrack, ids: ReadonlyArray<string>): boolean {
- const trackId = normalizedId(candidate.track.trackIdentifier);
+ const trackId = canonicalId(candidate.track.trackIdentifier);
return trackId !== null && ids.includes(trackId);
}
+function hasId(candidate: IndexedVoiceEngineV2StatsTrack): boolean {
+ return canonicalId(candidate.track.trackIdentifier) !== null;
+}
+
function hasRid(candidate: IndexedVoiceEngineV2StatsTrack): boolean {
return normalizedId(candidate.track.rid) !== null;
}
+function couldBeLocalScreenShare(
+ candidate: IndexedVoiceEngineV2StatsTrack,
+ localScreenShareTrackId: string | null,
+): boolean {
+ if (localScreenShareTrackId === null) return false;
+ return matchesId(candidate, localScreenShareTrackId) || !hasId(candidate);
+}
+
+function unidentifiedLocalScreenShareIndex(
+ sentVideoTracks: ReadonlyArray<IndexedVoiceEngineV2StatsTrack>,
+ localCameraTrackId: string | null,
+ localScreenShareTrackId: string | null,
+): number | null {
+ if (localScreenShareTrackId === null || localCameraTrackId !== null || sentVideoTracks.length !== 1) return null;
+ const candidate = sentVideoTracks[0];
+ return candidate && !hasId(candidate) ? candidate.index : null;
+}
+
function filterById(
candidates: ReadonlyArray<IndexedVoiceEngineV2StatsTrack>,
id: string | null,
@@ -565,19 +596,25 @@ export function classifyVoiceEngineV2TrackStats(
const remoteScreenShareTracks = receivedVideoTracks.filter((track) => idInList(track, remoteScreenShareTrackIds));
const remoteScreenShareTrackIndex = pickActiveTrackIndex(remoteScreenShareTracks);
const fallbackRemoteVideoTracks = receivedVideoTracks.filter((track) => !idInList(track, remoteScreenShareTrackIds));
+ const localScreenShareFallbackIndex = unidentifiedLocalScreenShareIndex(
+ sentVideoTracks,
+ localCameraTrackId,
+ localScreenShareTrackId,
+ );
return {
localVideoTrackIndex:
pickActiveTrackIndex(filterById(sentVideoTracks, localCameraTrackId)) ??
pickActiveTrackIndex(
- sentVideoTracks.filter((track) => !hasRid(track) && !matchesId(track, localScreenShareTrackId)),
+ sentVideoTracks.filter((track) => !hasRid(track) && !couldBeLocalScreenShare(track, localScreenShareTrackId)),
),
localAudioTrackIndex:
pickActiveTrackIndex(filterById(sentAudioTracks, localMicrophoneTrackId)) ??
pickActiveTrackIndex(sentAudioTracks.filter((track) => !matchesId(track, localScreenShareAudioTrackId))),
localScreenShareTrackIndex:
pickActiveTrackIndex(filterById(sentVideoTracks, localScreenShareTrackId)) ??
- pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))),
+ pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))) ??
+ localScreenShareFallbackIndex,
localScreenShareAudioTrackIndex: pickActiveTrackIndex(filterById(sentAudioTracks, localScreenShareAudioTrackId)),
remoteVideoTrackIndex:
pickActiveTrackIndex(fallbackRemoteVideoTracks) ??
diff --git a/packages/voice_engine_v2/src/protocol/types.ts b/packages/voice_engine_v2/src/protocol/types.ts
index d2ca7969f..32dd415c8 100644
--- a/packages/voice_engine_v2/src/protocol/types.ts
+++ b/packages/voice_engine_v2/src/protocol/types.ts
@@ -499,12 +499,14 @@ export interface VoiceEngineV2PerTrackStats {
kind: 'audio' | 'video' | 'unknown';
ssrc?: number;
rid?: string;
+ active?: boolean;
mid?: string;
trackIdentifier?: string;
mediaSourceId?: string;
codec?: string;
payloadType?: number;
bitrateKbps: number;
+ bitrateWindowMs?: number;
packetsLost?: number;
packetsLossPercent?: number;
jitterMs?: number;