Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(voice): judge H.264 hardware support by negotiated format (#2680)

Hampus67e01be34afd
diff --git a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts
index ab6bed850..e174dc9f7 100644
--- a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts
+++ b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts
@@ -1,11 +1,38 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
-import {findStalledVideoEncoder} from '@app/features/voice/engine/voice_screen_share_manager/shared';
-import {describe, expect, test, vi} from 'vitest';
+import {
+ findStalledVideoEncoder,
+ scheduleScreenShareEncoderVerification,
+ shouldTriggerSoftwareEncoderWarning,
+} from '@app/features/voice/engine/voice_screen_share_manager/shared';
+import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
+import VoiceSettings from '@app/features/voice/state/VoiceSettings';
+import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';
vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false}));
+let h264Acceleration = 'hardware';
+const markSoftwareEncodeObserved = vi.fn();
+
+vi.mock('@app/features/voice/utils/CodecCapabilityDetector', () => ({
+ adjustScreenShareEncodingForCodec: (encoding: unknown) => encoding,
+ getCodecCapabilityReport: () => ({
+ vp8: {hardwareAccelerated: 'software'},
+ vp9: {hardwareAccelerated: 'software'},
+ h264: {hardwareAccelerated: h264Acceleration},
+ h265: {hardwareAccelerated: 'software'},
+ av1: {hardwareAccelerated: 'software'},
+ }),
+ markScreenShareCodecSoftwareEncodeObserved: (codec: string) => markSoftwareEncodeObserved(codec),
+ resolveVideoPublishCodecPolicy: (requested: string) => ({
+ allowed: [requested],
+ requested,
+ primary: requested,
+ backupCodec: false,
+ }),
+}));
+
function makeStats(entries: ReadonlyArray<Record<string, unknown>>): RTCStatsReport {
return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
}
@@ -59,3 +86,72 @@ describe('findStalledVideoEncoder', () => {
expect(findStalledVideoEncoder(stats, 'h264')?.sourceFrames).toBe(240);
});
});
+
+describe('the screen share encoder verification', () => {
+ function openH264Stats(): RTCStatsReport {
+ return makeStats([
+ CODEC_REPORT,
+ SOURCE_REPORT,
+ outboundReport({
+ active: true,
+ framesEncoded: 120,
+ framesSent: 120,
+ encoderImplementation: 'OpenH264',
+ powerEfficientEncoder: false,
+ }),
+ ]);
+ }
+
+ async function verifyOpenH264Share(): Promise<void> {
+ scheduleScreenShareEncoderVerification(async () => openH264Stats(), 'h264');
+ await vi.advanceTimersByTimeAsync(2500);
+ }
+
+ beforeEach(() => {
+ vi.useFakeTimers();
+ h264Acceleration = 'hardware';
+ markSoftwareEncodeObserved.mockClear();
+ SoftwareEncoderWarning.reset();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ SoftwareEncoderWarning.reset();
+ });
+
+ test('records a software session verdict when the encoder turns out to be OpenH264', async () => {
+ await verifyOpenH264Share();
+ expect(markSoftwareEncodeObserved).toHaveBeenCalledWith('h264');
+ });
+
+ test('warns about the software encoder while the capability layer still claims hardware', async () => {
+ await verifyOpenH264Share();
+ expect(SoftwareEncoderWarning.showWarning).toBe(true);
+ });
+
+ test('does not warn once the session verdict says the codec encodes in software', async () => {
+ h264Acceleration = 'software';
+ await verifyOpenH264Share();
+ expect(SoftwareEncoderWarning.showWarning).toBe(false);
+ expect(markSoftwareEncodeObserved).toHaveBeenCalledWith('h264');
+ });
+
+ test('records no verdict when the user asked for the software encoder', async () => {
+ vi.spyOn(VoiceSettings, 'getScreenShareEncoderMode').mockReturnValue('software');
+ await verifyOpenH264Share();
+ expect(markSoftwareEncodeObserved).not.toHaveBeenCalled();
+ });
+});
+
+describe('the software encoder warning gate', () => {
+ test('stays quiet once the session verdict says the codec encodes in software', () => {
+ h264Acceleration = 'software';
+ expect(shouldTriggerSoftwareEncoderWarning('h264')).toBe(false);
+ });
+
+ test('still speaks up while the capability layer claims hardware', () => {
+ h264Acceleration = 'hardware';
+ expect(shouldTriggerSoftwareEncoderWarning('h264')).toBe(true);
+ });
+});
diff --git a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts
index 92ced32df..d562bcde3 100644
--- a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts
+++ b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts
@@ -9,6 +9,7 @@ import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
adjustScreenShareEncodingForCodec,
getCodecCapabilityReport,
+ markScreenShareCodecSoftwareEncodeObserved,
resolveVideoPublishCodecPolicy,
type VideoPublishCodecPolicy,
} from '@app/features/voice/utils/CodecCapabilityDetector';
@@ -681,7 +682,11 @@ export function scheduleScreenShareEncoderVerification(
powerEfficientEncoder: encoder.powerEfficientEncoder,
expectedHardware,
});
- if (shouldTriggerSoftwareEncoderWarning(codec)) {
+ const warnAboutSoftwareEncoder = shouldTriggerSoftwareEncoderWarning(codec);
+ if (VoiceSettings.getScreenShareEncoderMode() !== 'software') {
+ markScreenShareCodecSoftwareEncodeObserved(codec);
+ }
+ if (warnAboutSoftwareEncoder) {
SoftwareEncoderWarning.triggerWarning(codec, encoder.implementation);
}
} else {
diff --git a/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.test.ts b/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.test.ts
index acb9aeee5..5f0c9fa44 100644
--- a/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.test.ts
+++ b/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.test.ts
@@ -7,7 +7,9 @@ import {afterEach, beforeEach, describe, expect, it, type MockInstance, vi} from
let av1OptIn = false;
let hevcOptIn = false;
let desktop = true;
+let platform = 'windows';
let gpuReport: HardwareEncodeReport | null = null;
+let nativeHardwareEncoder: {backend: string; codecs: ReadonlyArray<string>} | null = null;
let cameraPreference = 'auto';
let openH264Status: {enabled: boolean; downloaded: boolean} | null = null;
let codecCapabilityInfo: MockInstance<(...args: Array<unknown>) => void>;
@@ -25,7 +27,7 @@ vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({
}));
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
- guessPlatform: () => 'windows',
+ guessPlatform: () => platform,
isChromiumBrowser: () => true,
isDesktop: () => desktop,
isFirefoxBrowser: () => false,
@@ -36,8 +38,8 @@ vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
}));
vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({
- getNativeHardwareEncoderCapabilitiesSync: () => null,
- hasNativeHardwareEncoder: () => false,
+ getNativeHardwareEncoderCapabilitiesSync: () => nativeHardwareEncoder,
+ hasNativeHardwareEncoder: (codec: string) => nativeHardwareEncoder?.codecs.includes(codec) === true,
resetNativeHardwareEncoderCapabilities: () => undefined,
}));
@@ -65,6 +67,7 @@ const {
getRoomVideoPublishDefaults,
isVideoCodecAllowedForPublish,
markScreenShareCodecEncodeRuntimeFailure,
+ markScreenShareCodecSoftwareEncodeObserved,
resetCachedCodecCapabilities,
resolveScreenShareEncoderVerificationAction,
resolveVideoPublishCodecPolicy,
@@ -288,3 +291,66 @@ describe('video publish codec policy', () => {
}
});
});
+
+describe('H.264 hardware verdict', () => {
+ beforeEach(() => {
+ av1OptIn = false;
+ hevcOptIn = false;
+ desktop = true;
+ platform = 'windows';
+ gpuReport = null;
+ nativeHardwareEncoder = null;
+ cameraPreference = 'auto';
+ openH264Status = null;
+ senderCodecs = ALL_SENDER_CODECS;
+ resetCachedCodecCapabilities();
+ });
+
+ afterEach(() => {
+ platform = 'windows';
+ nativeHardwareEncoder = null;
+ resetCachedCodecCapabilities();
+ });
+
+ it('treats H.264 as software until the GPU probe has resolved', () => {
+ expect(getCodecCapabilityReport().h264.hardwareAccelerated).toBe('software');
+ expect(selectAutomaticScreenShareCodec('auto')).toEqual({codec: 'vp9', reason: 'software-vp9'});
+ });
+
+ it('does not pick hardware H.264 on a Windows AMD machine whose probe came back software', () => {
+ gpuReport = {...ALL_SOFTWARE, gpuFamily: 'amd-rdna3-plus'};
+ expect(getCodecCapabilityReport().h264.hardwareAccelerated).toBe('software');
+ expect(selectAutomaticScreenShareCodec('auto')).toEqual({codec: 'vp9', reason: 'software-vp9'});
+ });
+
+ it('keeps hardware H.264 on Linux when the probe reported a power efficient encoder', () => {
+ platform = 'linux';
+ gpuReport = {...ALL_SOFTWARE, h264: 'hardware'};
+ expect(getCodecCapabilityReport().h264.hardwareAccelerated).toBe('hardware');
+ expect(selectAutomaticScreenShareCodec('auto')).toEqual({codec: 'h264', reason: 'hardware-h264'});
+ });
+
+ it('does not let a macOS native VideoToolbox encoder stand in for hardware H.264', () => {
+ platform = 'macos';
+ nativeHardwareEncoder = {backend: 'videotoolbox', codecs: ['h264']};
+ const report = getCodecCapabilityReport();
+ expect(report.h264.supported).toBe(true);
+ expect(report.h264.hardwareAccelerated).toBe('software');
+ expect(selectAutomaticScreenShareCodec('auto').reason).not.toBe('hardware-h264');
+ });
+
+ it('still counts a native NVENC encoder as hardware H.264', () => {
+ platform = 'linux';
+ nativeHardwareEncoder = {backend: 'nvenc', codecs: ['h264']};
+ expect(getCodecCapabilityReport().h264.hardwareAccelerated).toBe('hardware');
+ expect(selectAutomaticScreenShareCodec('auto')).toEqual({codec: 'h264', reason: 'hardware-h264'});
+ });
+
+ it('stops selecting hardware H.264 once the session verdict says it encoded in software', () => {
+ gpuReport = {...ALL_SOFTWARE, h264: 'hardware'};
+ expect(selectAutomaticScreenShareCodec('auto')).toEqual({codec: 'h264', reason: 'hardware-h264'});
+ expect(markScreenShareCodecSoftwareEncodeObserved('h264')).toBe(true);
+ expect(getCodecCapabilityReport().h264.hardwareAccelerated).toBe('software');
+ expect(selectAutomaticScreenShareCodec('auto')).toEqual({codec: 'vp9', reason: 'software-vp9'});
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.ts b/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.ts
index 245416f14..63821c6d2 100644
--- a/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.ts
+++ b/fluxer_app/src/features/voice/utils/CodecCapabilityDetector.ts
@@ -92,6 +92,7 @@ let cachedReportHardwareAccelerationDisabled: boolean | undefined;
let cachedReportAv1OptIn: boolean | undefined;
let cachedReportHevcOptIn: boolean | undefined;
const runtimeEncodeFailureCodecs = new Set<VideoCodec>();
+const observedSoftwareEncodeCodecs = new Set<VideoCodec>();
interface RawProbeResult {
caps: CodecCapabilities;
@@ -223,6 +224,11 @@ function getEffectiveScreenShareCapabilities(caps: CodecCapabilities): CodecCapa
};
}
+function hasPublishPathNativeHardwareEncoder(codec: VideoCodec): boolean {
+ if (!hasNativeHardwareEncoder(codec)) return false;
+ return getNativeHardwareEncoderCapabilitiesSync()?.backend !== 'videotoolbox';
+}
+
function buildReport(): CodecCapabilityReport {
const {caps, probedSuccessfully} = probeRawCapabilities();
const context = buildScreenShareCodecPolicyContext();
@@ -231,18 +237,19 @@ function buildReport(): CodecCapabilityReport {
const linuxNvidiaWebRtcEncodeLimited =
context.platform === 'linux' && gpuReport?.gpuFamily?.startsWith('nvidia-') === true;
const hardwareAccelerationDisabled = isDesktopHardwareAccelerationDisabled();
- const openH264Status = getOpenH264StatusSync();
- const openH264Active = openH264Status?.enabled === true && openH264Status.downloaded === true;
function hwAccel(codec: keyof CodecCapabilities): HardwareEncodeAnswer {
if (hardwareAccelerationDisabled) {
return 'software';
}
- if (hasNativeHardwareEncoder(codec)) {
+ if (observedSoftwareEncodeCodecs.has(codec)) {
+ return 'software';
+ }
+ if (hasPublishPathNativeHardwareEncoder(codec)) {
return 'hardware';
}
const gpu = gpuReport ? gpuReport[codec] : 'unknown';
- if (codec === 'h264' && openH264Active && gpu === 'unknown') {
- return 'software';
+ if (codec === 'h264') {
+ return gpu === 'hardware' ? 'hardware' : 'software';
}
return gpu;
}
@@ -481,6 +488,14 @@ export function markScreenShareCodecEncodeRuntimeFailure(codec: VideoCodec, reas
return true;
}
+export function markScreenShareCodecSoftwareEncodeObserved(codec: VideoCodec): boolean {
+ if (observedSoftwareEncodeCodecs.has(codec)) return false;
+ observedSoftwareEncodeCodecs.add(codec);
+ cachedReport = null;
+ logger.warn('Treating this codec as software-encoded for the rest of the session', {codec});
+ return true;
+}
+
export type VideoPublishCodecDenial = 'sender-cannot-encode' | 'policy' | 'runtime-failed';
export interface VideoPublishCodecPolicy {
@@ -654,6 +669,7 @@ export function resetCachedCodecCapabilities(): void {
cachedReportAv1OptIn = undefined;
cachedReportHevcOptIn = undefined;
runtimeEncodeFailureCodecs.clear();
+ observedSoftwareEncodeCodecs.clear();
resetNativeHardwareEncoderCapabilities();
resetOpenH264Status();
}
diff --git a/fluxer_app/src/features/voice/utils/GpuEncoderCapabilities.test.ts b/fluxer_app/src/features/voice/utils/GpuEncoderCapabilities.test.ts
new file mode 100644
index 000000000..dc54078be
--- /dev/null
+++ b/fluxer_app/src/features/voice/utils/GpuEncoderCapabilities.test.ts
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+let streamingMode = 'custom';
+let screenshareResolution = 'high';
+let videoFrameRate = 60;
+let higherVideoQuality = true;
+let shareSettingsUnavailable = false;
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({
+ default: {
+ getStreamingMode: () => {
+ if (shareSettingsUnavailable) throw new ReferenceError('Cannot access VoiceSettings before initialization');
+ return streamingMode;
+ },
+ getScreenshareResolution: () => screenshareResolution,
+ getVideoFrameRate: () => videoFrameRate,
+ },
+}));
+
+vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({
+ hasHigherVideoQuality: () => higherVideoQuality,
+}));
+
+vi.mock('@app/features/ui/utils/NativeUtils', () => ({
+ getElectronAPI: () => null,
+ getNativePlatformSync: () => 'windows',
+ isDesktop: () => false,
+}));
+
+interface EncodingInfoConfig {
+ type: string;
+ video: {contentType: string; width: number; height: number; bitrate: number; framerate: number};
+}
+
+const encodingInfoConfigs: Array<EncodingInfoConfig> = [];
+let encodingInfoAnswer: () => {supported?: boolean; powerEfficient?: boolean} = () => ({
+ supported: true,
+ powerEfficient: false,
+});
+
+Object.defineProperty(globalThis, 'navigator', {
+ configurable: true,
+ writable: true,
+ value: {
+ mediaCapabilities: {
+ encodingInfo: async (config: EncodingInfoConfig) => {
+ encodingInfoConfigs.push(config);
+ return encodingInfoAnswer();
+ },
+ },
+ },
+});
+
+const {
+ AMD_RDNA3_PLUS,
+ NEGOTIABLE_H264_PROBE_CONTENT_TYPE,
+ NVIDIA_PRE_ADA,
+ PCI_VENDOR_AMD,
+ PCI_VENDOR_APPLE,
+ PCI_VENDOR_NVIDIA,
+ probeWebRtcEncodeEfficiency,
+ reconcileHardwareEncodeReport,
+} = await import('./GpuEncoderCapabilities');
+
+const AMD_RDNA3_REPORT: HardwareEncodeReport = {...AMD_RDNA3_PLUS.caps, gpuFamily: AMD_RDNA3_PLUS.family};
+const NVIDIA_REPORT: HardwareEncodeReport = {...NVIDIA_PRE_ADA.caps, gpuFamily: NVIDIA_PRE_ADA.family};
+const PROBE_SAYS_SOFTWARE = {
+ av1: 'software',
+ h265: 'software',
+ h264: 'software',
+ vp9: 'software',
+ vp8: 'software',
+} as const;
+
+function h264Probes(): Array<EncodingInfoConfig> {
+ return encodingInfoConfigs.filter((config) => config.video.contentType.startsWith('video/H264'));
+}
+
+describe('the WebRTC encode probe', () => {
+ beforeEach(() => {
+ encodingInfoConfigs.length = 0;
+ streamingMode = 'custom';
+ screenshareResolution = 'high';
+ videoFrameRate = 60;
+ higherVideoQuality = true;
+ shareSettingsUnavailable = false;
+ encodingInfoAnswer = () => ({supported: true, powerEfficient: false});
+ });
+
+ it('asks about the single H.264 format the SFU can negotiate', async () => {
+ await probeWebRtcEncodeEfficiency();
+ expect(h264Probes().map((config) => config.video.contentType)).toEqual([
+ 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
+ ]);
+ expect(NEGOTIABLE_H264_PROBE_CONTENT_TYPE).toBe(
+ 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
+ );
+ });
+
+ it('probes at the configured share size and frame rate', async () => {
+ await probeWebRtcEncodeEfficiency();
+ expect(h264Probes()[0]?.video).toMatchObject({width: 1920, height: 1080, framerate: 60, bitrate: 6_000_000});
+ });
+
+ it('follows the share settings down when the user picks a smaller share', async () => {
+ screenshareResolution = 'medium';
+ videoFrameRate = 30;
+ await probeWebRtcEncodeEfficiency();
+ expect(h264Probes()[0]?.video).toMatchObject({width: 1280, height: 720, framerate: 30, bitrate: 3_000_000});
+ });
+
+ it('answers software for H.264 when the negotiated format is not power efficient', async () => {
+ await expect(probeWebRtcEncodeEfficiency()).resolves.toMatchObject({h264: 'software'});
+ });
+
+ it('answers hardware for H.264 when the negotiated format is power efficient', async () => {
+ encodingInfoAnswer = () => ({supported: true, powerEfficient: true});
+ await expect(probeWebRtcEncodeEfficiency()).resolves.toMatchObject({h264: 'hardware'});
+ });
+
+ it('still answers at the largest share size when the share settings cannot be read', async () => {
+ shareSettingsUnavailable = true;
+ await expect(probeWebRtcEncodeEfficiency()).resolves.toMatchObject({h264: 'software'});
+ expect(h264Probes()[0]?.video).toMatchObject({width: 1920, height: 1080, framerate: 60, bitrate: 6_000_000});
+ });
+});
+
+describe('reconciling the GPU table against the probe', () => {
+ it('demotes H.264 to software on Windows when the negotiated format is not power efficient', () => {
+ const report = reconcileHardwareEncodeReport(AMD_RDNA3_REPORT, PROBE_SAYS_SOFTWARE, 'windows', PCI_VENDOR_AMD);
+ expect(report.h264).toBe('software');
+ });
+
+ it('demotes H.264 to software on macOS when the negotiated format is not power efficient', () => {
+ const report = reconcileHardwareEncodeReport(AMD_RDNA3_REPORT, PROBE_SAYS_SOFTWARE, 'macos', PCI_VENDOR_APPLE);
+ expect(report.h264).toBe('software');
+ });
+
+ it('demotes H.264 to software when the probe could not answer at all', () => {
+ expect(reconcileHardwareEncodeReport(AMD_RDNA3_REPORT, null, 'windows', PCI_VENDOR_AMD).h264).toBe('software');
+ });
+
+ it('keeps hardware H.264 on Windows when the probe reports a power efficient encoder', () => {
+ const report = reconcileHardwareEncodeReport(
+ AMD_RDNA3_REPORT,
+ {...PROBE_SAYS_SOFTWARE, h264: 'hardware'},
+ 'windows',
+ PCI_VENDOR_AMD,
+ );
+ expect(report.h264).toBe('hardware');
+ });
+
+ it('keeps hardware H.264 on Linux when the probe reports a power efficient encoder', () => {
+ const report = reconcileHardwareEncodeReport(
+ NVIDIA_REPORT,
+ {...PROBE_SAYS_SOFTWARE, h264: 'hardware'},
+ 'linux',
+ PCI_VENDOR_NVIDIA,
+ );
+ expect(report.h264).toBe('hardware');
+ });
+
+ it('leaves the other codecs on the table verdict outside Linux NVIDIA', () => {
+ const report = reconcileHardwareEncodeReport(AMD_RDNA3_REPORT, PROBE_SAYS_SOFTWARE, 'windows', PCI_VENDOR_AMD);
+ expect(report.av1).toBe('hardware');
+ expect(report.h265).toBe('hardware');
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/GpuEncoderCapabilities.ts b/fluxer_app/src/features/voice/utils/GpuEncoderCapabilities.ts
index 5045577bb..53b561cb3 100644
--- a/fluxer_app/src/features/voice/utils/GpuEncoderCapabilities.ts
+++ b/fluxer_app/src/features/voice/utils/GpuEncoderCapabilities.ts
@@ -7,6 +7,14 @@ import {
isDesktop,
type NativePlatform,
} from '@app/features/ui/utils/NativeUtils';
+import VoiceSettings from '@app/features/voice/state/VoiceSettings';
+import {
+ getScreenShareBitrateBps,
+ resolveEffectiveScreenShareDimensions,
+ resolveStreamingModeSettings,
+ SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
+} from '@app/features/voice/utils/ScreenShareOptions';
+import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import type {GpuDeviceInfo, GpuInfo} from '@app/types/electron.d';
import type {VideoCodec} from 'livekit-client';
@@ -236,19 +244,52 @@ export function reportFromGpuInfo(info: GpuInfo): HardwareEncodeReport {
};
}
+export const NEGOTIABLE_H264_PROBE_CONTENT_TYPE =
+ 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f';
+
const WEBRTC_ENCODE_PROBE_CONTENT_TYPES: Record<VideoCodec, ReadonlyArray<string>> = {
av1: ['video/AV1'],
h265: ['video/H265'],
- h264: [
- 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f',
- 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f',
- 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
- 'video/H264',
- ],
+ h264: [NEGOTIABLE_H264_PROBE_CONTENT_TYPE],
vp9: ['video/VP9'],
vp8: ['video/VP8'],
};
+interface EncodeProbeVideoConfig {
+ width: number;
+ height: number;
+ bitrate: number;
+ framerate: number;
+}
+
+const DEFAULT_ENCODE_PROBE_VIDEO_CONFIG: EncodeProbeVideoConfig = {
+ width: 1920,
+ height: 1080,
+ bitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
+ framerate: 60,
+};
+
+function resolveEncodeProbeVideoConfig(): EncodeProbeVideoConfig {
+ try {
+ const settings = resolveStreamingModeSettings(
+ VoiceSettings.getStreamingMode(),
+ VoiceSettings.getScreenshareResolution(),
+ VoiceSettings.getVideoFrameRate(),
+ hasHigherVideoQuality(),
+ );
+ const {width, height} = resolveEffectiveScreenShareDimensions(settings.resolution);
+ return {
+ width,
+ height,
+ bitrate: getScreenShareBitrateBps(settings.resolution, settings.frameRate),
+ framerate: settings.frameRate,
+ };
+ } catch (error) {
+ logger.debug('Share settings were unavailable, probing at the largest share size instead', {error});
+ return DEFAULT_ENCODE_PROBE_VIDEO_CONFIG;
+ }
+}
+
interface WebRtcEncodingInfoResult {
supported?: boolean;
powerEfficient?: boolean;
@@ -261,13 +302,14 @@ interface MediaCapabilitiesLike {
async function probeCodecEncodeEfficiency(
mediaCapabilities: MediaCapabilitiesLike,
contentTypes: ReadonlyArray<string>,
+ video: EncodeProbeVideoConfig,
): Promise<HardwareEncodeAnswer> {
let sawSupported = false;
for (const contentType of contentTypes) {
try {
const info = await mediaCapabilities.encodingInfo?.({
type: 'webrtc',
- video: {contentType, width: 1920, height: 1080, bitrate: 2_500_000, framerate: 30},
+ video: {contentType, ...video},
});
if (!info?.supported) continue;
sawSupported = true;
@@ -282,8 +324,11 @@ export async function probeWebRtcEncodeEfficiency(): Promise<Record<VideoCodec,
const mediaCapabilities = (navigator as Navigator & {mediaCapabilities?: MediaCapabilitiesLike}).mediaCapabilities;
if (!mediaCapabilities?.encodingInfo) return null;
const codecs: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
+ const video = resolveEncodeProbeVideoConfig();
const answers = await Promise.all(
- codecs.map((codec) => probeCodecEncodeEfficiency(mediaCapabilities, WEBRTC_ENCODE_PROBE_CONTENT_TYPES[codec])),
+ codecs.map((codec) =>
+ probeCodecEncodeEfficiency(mediaCapabilities, WEBRTC_ENCODE_PROBE_CONTENT_TYPES[codec], video),
+ ),
);
const result = {} as Record<VideoCodec, HardwareEncodeAnswer>;
codecs.forEach((codec, index) => {
@@ -299,19 +344,21 @@ export function reconcileHardwareEncodeReport(
vendorId: number,
): HardwareEncodeReport {
const isNvidiaReport = vendorId === PCI_VENDOR_NVIDIA || report.gpuFamily?.startsWith('nvidia-') === true;
- if (platform !== 'linux' || !isNvidiaReport) return report;
const adjust = (codec: VideoCodec): HardwareEncodeAnswer => {
if (report[codec] !== 'hardware') return report[codec];
return efficiency?.[codec] === 'hardware' ? 'hardware' : 'software';
};
- return {
- ...report,
- av1: adjust('av1'),
- h265: adjust('h265'),
- h264: adjust('h264'),
- vp9: adjust('vp9'),
- vp8: adjust('vp8'),
- };
+ if (platform === 'linux' && isNvidiaReport) {
+ return {
+ ...report,
+ av1: adjust('av1'),
+ h265: adjust('h265'),
+ h264: adjust('h264'),
+ vp9: adjust('vp9'),
+ vp8: adjust('vp8'),
+ };
+ }
+ return {...report, h264: adjust('h264')};
}
let cachedReport: HardwareEncodeReport | null = null;