Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
11.4 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {Logger} from '@app/features/platform/utils/AppLogger';
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
import {classifyVideoDecoderAcceleration} from '@app/features/voice/utils/VideoAccelerationClassification';
import type {VideoCodec} from 'livekit-client';

const logger = new Logger('ScreenShareCodecDiagnostics');
const DECODE_SAMPLE_INTERVAL_MS = 5000;
const UNKNOWN_DECODER_IMPLEMENTATION = 'software decoder';

interface CodecStatsEntry {
	type: string;
	id: string;
	mimeType?: string;
}

interface InboundVideoStatsEntry {
	type: string;
	kind?: string;
	mediaType?: string;
	codecId?: string;
	timestamp?: number;
	packetsReceived?: number;
	bytesReceived?: number;
	framesDecoded?: number;
	keyFramesDecoded?: number;
	framesReceived?: number;
	framesDropped?: number;
	freezeCount?: number;
	totalInterFrameDelay?: number;
	totalSquaredInterFrameDelay?: number;
	decoderImplementation?: string;
	powerEfficientDecoder?: boolean;
}

interface SoftwareVideoDecoderInfo {
	codec: string | null;
	implementation: string;
	powerEfficientDecoder: boolean | null;
}

export interface StalledVideoDecoderInfo {
	codec: VideoCodec;
	mimeType?: string;
	packetsReceived: number;
	bytesReceived: number;
	framesDecoded: number;
	framesReceived: number;
	framesDropped: number | null;
}

export interface InboundVideoDecodeSample {
	codec: VideoCodec | null;
	mimeType?: string;
	atMs: number;
	packetsReceived: number;
	bytesReceived: number;
	framesDecoded: number | null;
	keyFramesDecoded: number | null;
	framesReceived: number | null;
	framesDropped: number | null;
	freezeCount: number | null;
	totalInterFrameDelay: number | null;
	totalSquaredInterFrameDelay: number | null;
	decoderImplementation: string | null;
	powerEfficientDecoder: boolean | null;
}

interface InboundVideoDecodeHealth {
	codec: VideoCodec | null;
	intervalMs: number;
	decodedFps: number | null;
	framesDecodedDelta: number | null;
	meanInterFrameDelayMs: number | null;
	interFrameDelayStdDevMs: number | null;
	freezes: number | null;
	freezeCount: number | null;
	framesDroppedDelta: number | null;
	framesDecoded: number | null;
	bytesReceivedDelta: number;
}

function isSoftwareVideoStats(implementation: string | null, powerEfficient: boolean | null): boolean {
	return classifyVideoDecoderAcceleration(implementation, powerEfficient) === 'software';
}

function getCodecLabel(mimeType: string | undefined): string | null {
	if (!mimeType) return null;
	return mimeType.replace(/^video\//i, '').toUpperCase();
}

export function getVideoCodecFromMimeType(mimeType: string | undefined): VideoCodec | null {
	const lower = mimeType?.toLowerCase();
	if (!lower?.startsWith('video/')) return null;
	const codec = lower.slice('video/'.length);
	if (codec === 'av1' || codec === 'av1x') return 'av1';
	if (codec === 'h265' || codec === 'hevc') return 'h265';
	if (codec === 'h264') return 'h264';
	if (codec === 'vp9') return 'vp9';
	if (codec === 'vp8') return 'vp8';
	return null;
}

function finiteNumber(value: unknown): number | null {
	return typeof value === 'number' && Number.isFinite(value) ? value : null;
}

function counterDelta(previous: number | null, current: number | null): number | null {
	if (previous === null || current === null) return null;
	if (current < previous) return null;
	return current - previous;
}

function roundTo(value: number, decimals: number): number {
	const factor = 10 ** decimals;
	return Math.round(value * factor) / factor;
}

function getStatsKind(report: InboundVideoStatsEntry, codecs: Map<string, CodecStatsEntry>): string | undefined {
	if (report.kind || report.mediaType) return report.kind ?? report.mediaType;
	if (!report.codecId) return undefined;
	const codec = codecs.get(report.codecId);
	return codec?.mimeType?.startsWith('video/') ? 'video' : undefined;
}

function collectInboundVideoStats(stats: RTCStatsReport): {
	codecs: Map<string, CodecStatsEntry>;
	reports: Array<InboundVideoStatsEntry>;
} {
	const codecs = new Map<string, CodecStatsEntry>();
	const reports: Array<InboundVideoStatsEntry> = [];
	for (const raw of stats.values()) {
		const report = raw as CodecStatsEntry & InboundVideoStatsEntry;
		if (report.type === 'codec') {
			codecs.set(report.id, report);
		}
		if (report.type === 'inbound-rtp') {
			reports.push(report);
		}
	}
	return {codecs, reports};
}

export function findInboundVideoDecodeSample(stats: RTCStatsReport): InboundVideoDecodeSample | null {
	const {codecs, reports} = collectInboundVideoStats(stats);
	for (const report of reports) {
		if (getStatsKind(report, codecs) !== 'video') continue;
		const mimeType = report.codecId ? codecs.get(report.codecId)?.mimeType : undefined;
		return {
			codec: getVideoCodecFromMimeType(mimeType),
			mimeType,
			atMs: finiteNumber(report.timestamp) ?? Date.now(),
			packetsReceived: finiteNumber(report.packetsReceived) ?? 0,
			bytesReceived: finiteNumber(report.bytesReceived) ?? 0,
			framesDecoded: finiteNumber(report.framesDecoded),
			keyFramesDecoded: finiteNumber(report.keyFramesDecoded),
			framesReceived: finiteNumber(report.framesReceived),
			framesDropped: finiteNumber(report.framesDropped),
			freezeCount: finiteNumber(report.freezeCount),
			totalInterFrameDelay: finiteNumber(report.totalInterFrameDelay),
			totalSquaredInterFrameDelay: finiteNumber(report.totalSquaredInterFrameDelay),
			decoderImplementation:
				typeof report.decoderImplementation === 'string' && report.decoderImplementation.length > 0
					? report.decoderImplementation
					: null,
			powerEfficientDecoder: typeof report.powerEfficientDecoder === 'boolean' ? report.powerEfficientDecoder : null,
		};
	}
	return null;
}

function findSoftwareVideoDecoder(sample: InboundVideoDecodeSample): SoftwareVideoDecoderInfo | null {
	if (!isSoftwareVideoStats(sample.decoderImplementation, sample.powerEfficientDecoder)) return null;
	return {
		codec: getCodecLabel(sample.mimeType),
		implementation: sample.decoderImplementation ?? UNKNOWN_DECODER_IMPLEMENTATION,
		powerEfficientDecoder: sample.powerEfficientDecoder,
	};
}

export function findStalledVideoDecoder(sample: InboundVideoDecodeSample): StalledVideoDecoderInfo | null {
	if (sample.codec === null) return null;
	if (sample.framesDecoded !== 0) return null;
	if (sample.keyFramesDecoded !== 0) return null;
	if (sample.framesReceived === null || sample.framesReceived < 1) return null;
	return {
		codec: sample.codec,
		mimeType: sample.mimeType,
		packetsReceived: sample.packetsReceived,
		bytesReceived: sample.bytesReceived,
		framesDecoded: sample.framesDecoded,
		framesReceived: sample.framesReceived,
		framesDropped: sample.framesDropped,
	};
}

function confirmDecodeStall(
	first: StalledVideoDecoderInfo | null,
	second: StalledVideoDecoderInfo | null,
): StalledVideoDecoderInfo | null {
	if (!first || !second) return null;
	if (first.codec !== second.codec || first.mimeType !== second.mimeType) return null;
	if (first.framesDecoded !== 0 || second.framesDecoded !== 0) return null;
	if (second.framesReceived <= first.framesReceived) return null;
	return second;
}

function computeInboundVideoDecodeHealth(
	previous: InboundVideoDecodeSample,
	current: InboundVideoDecodeSample,
): InboundVideoDecodeHealth | null {
	const intervalMs = current.atMs - previous.atMs;
	if (intervalMs <= 0) return null;
	const framesDecodedDelta = counterDelta(previous.framesDecoded, current.framesDecoded);
	const interFrameDelayDelta = counterDelta(previous.totalInterFrameDelay, current.totalInterFrameDelay);
	const squaredInterFrameDelayDelta = counterDelta(
		previous.totalSquaredInterFrameDelay,
		current.totalSquaredInterFrameDelay,
	);
	const meanInterFrameDelay =
		framesDecodedDelta === null || framesDecodedDelta === 0 || interFrameDelayDelta === null
			? null
			: interFrameDelayDelta / framesDecodedDelta;
	const variance =
		meanInterFrameDelay === null || framesDecodedDelta === null || squaredInterFrameDelayDelta === null
			? null
			: squaredInterFrameDelayDelta / framesDecodedDelta - meanInterFrameDelay ** 2;
	return {
		codec: current.codec,
		intervalMs: Math.round(intervalMs),
		decodedFps: framesDecodedDelta === null ? null : roundTo((framesDecodedDelta * 1000) / intervalMs, 1),
		framesDecodedDelta,
		meanInterFrameDelayMs: meanInterFrameDelay === null ? null : roundTo(meanInterFrameDelay * 1000, 1),
		interFrameDelayStdDevMs: variance === null || variance < 0 ? null : roundTo(Math.sqrt(variance) * 1000, 1),
		freezes: counterDelta(previous.freezeCount, current.freezeCount),
		freezeCount: current.freezeCount,
		framesDroppedDelta: counterDelta(previous.framesDropped, current.framesDropped),
		framesDecoded: current.framesDecoded,
		bytesReceivedDelta: Math.max(0, current.bytesReceived - previous.bytesReceived),
	};
}

export function monitorScreenShareDecodeHealth(
	getStats: () => Promise<RTCStatsReport | undefined>,
	onDecodeStall?: (failure: StalledVideoDecoderInfo) => void,
): () => void {
	let cancelled = false;
	let timer: ReturnType<typeof setTimeout> | null = null;
	let previousSample: InboundVideoDecodeSample | null = null;
	let pendingStall: StalledVideoDecoderInfo | null = null;
	let reportedStall = false;
	let isSoftwareDecoder = false;
	let warnedSoftwareDecoder = false;
	let isFrozen = false;

	const noteSoftwareDecoder = (sample: InboundVideoDecodeSample): void => {
		const decoder = findSoftwareVideoDecoder(sample);
		if (!decoder) {
			isSoftwareDecoder = false;
			return;
		}
		if (isSoftwareDecoder) return;
		isSoftwareDecoder = true;
		logger.warn('Screen share is using a software decoder', decoder);
		if (warnedSoftwareDecoder) return;
		warnedSoftwareDecoder = true;
		SoftwareEncoderWarning.triggerDecoderWarning(decoder.codec, decoder.implementation);
	};

	const noteDecodeStall = (sample: InboundVideoDecodeSample): void => {
		const stall = findStalledVideoDecoder(sample);
		const confirmed = confirmDecodeStall(pendingStall, stall);
		pendingStall = stall;
		if (!confirmed || reportedStall) return;
		reportedStall = true;
		logger.warn('Screen share video decode is stalled', confirmed);
		onDecodeStall?.(confirmed);
	};

	const noteDecodeHealth = (sample: InboundVideoDecodeSample): void => {
		const health = previousSample === null ? null : computeInboundVideoDecodeHealth(previousSample, sample);
		if (!health) return;
		if (health.framesDecodedDelta === 0 && health.bytesReceivedDelta > 0) {
			if (isFrozen) return;
			isFrozen = true;
			logger.warn('Screen share video is receiving bytes but decoding no frames', health);
			return;
		}
		if (isFrozen && health.framesDecodedDelta !== null && health.framesDecodedDelta > 0) {
			isFrozen = false;
			logger.info('Screen share video decoding resumed', health);
			return;
		}
		logger.debug('Screen share video decode health', health);
	};

	const sampleOnce = async (): Promise<void> => {
		try {
			const stats = await getStats();
			if (cancelled || !stats) return;
			const sample = findInboundVideoDecodeSample(stats);
			if (!sample) return;
			noteSoftwareDecoder(sample);
			noteDecodeStall(sample);
			noteDecodeHealth(sample);
			previousSample = sample;
		} catch (error) {
			logger.debug('Failed to sample the screen share decoder', {error});
		}
	};

	const scheduleNextSample = (): void => {
		timer = setTimeout(async () => {
			timer = null;
			await sampleOnce();
			if (cancelled) return;
			scheduleNextSample();
		}, DECODE_SAMPLE_INTERVAL_MS);
	};

	scheduleNextSample();
	return () => {
		cancelled = true;
		if (timer === null) return;
		clearTimeout(timer);
		timer = null;
	};
}