Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
13.2 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {waitForAbortablePromise} from '@app/features/platform/utils/AbortablePromise';
import {throwCollectedFailures} from '@app/features/voice/utils/camera-effects/AggregateOperations';
import {runWithResponseDeadline} from '@app/features/voice/utils/camera-effects/BoundedResponse';
import {
	bytesEqualAt,
	normalizedMediaType,
	readWebCameraEffectCustomMediaBlob,
	requireWebCameraEffectCustomMediaURL,
	throwIfAborted,
	WEB_CAMERA_EFFECT_CUSTOM_MEDIA_OPERATION_TIMEOUT_MS,
} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomImage';
import {validateCameraEffectFrameDimensions} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol';

const MAX_CUSTOM_VIDEO_DURATION_SECONDS = 60;
const SUPPORTED_VIDEO_MEDIA_TYPES = new Set(['video/mp4', 'video/webm']);
const VIDEO_HEADER_SNIFF_BYTES = 128;
const MP4_FILE_TYPE_BOX_OFFSET = 4;
const MP4_FILE_TYPE_BOX: ReadonlyArray<number> = [0x66, 0x74, 0x79, 0x70];
const EBML_MAGIC: ReadonlyArray<number> = [0x1a, 0x45, 0xdf, 0xa3];
const EBML_DOC_TYPE_ELEMENT_ID: ReadonlyArray<number> = [0x42, 0x82];
const EBML_VINT_SINGLE_BYTE_MARKER = 0x80;
const EBML_VINT_SINGLE_BYTE_VALUE_MASK = 0x7f;
const MAX_EBML_DOC_TYPE_LENGTH = 16;
const WEBM_DOC_TYPE = 'webm';

export interface WebCameraEffectVideoFrameProducer {
	readonly readable: ReadableStream<VideoFrame>;
	readonly width: number;
	readonly height: number;
	stop(): void;
}

class WebCameraEffectVideoFrameProducerOwner implements WebCameraEffectVideoFrameProducer {
	readonly readable: ReadableStream<VideoFrame>;
	private readonly video: HTMLVideoElement;
	private readonly objectURL: string;
	private controller: ReadableStreamDefaultController<VideoFrame> | null = null;
	private pendingFrame: VideoFrame | null = null;
	private pullResolve: (() => void) | null = null;
	private callbackHandle: number | null = null;
	private started = false;
	private stopped = false;
	private videoWidth = 0;
	private videoHeight = 0;

	constructor(blob: Blob) {
		this.video = document.createElement('video');
		this.video.autoplay = true;
		this.video.crossOrigin = 'anonymous';
		this.video.loop = true;
		this.video.muted = true;
		this.video.playsInline = true;
		this.video.preload = 'auto';
		this.readable = new ReadableStream<VideoFrame>(
			{
				start: (controller): void => {
					this.controller = controller;
				},
				pull: (controller) => this.pullFrame(controller),
				cancel: () => this.cancelFromStream(),
			},
			{highWaterMark: 0},
		);
		this.objectURL = URL.createObjectURL(blob);
	}

	get width(): number {
		return this.videoWidth;
	}

	get height(): number {
		return this.videoHeight;
	}

	async start(signal: AbortSignal): Promise<void> {
		if (this.started) {
			throw new Error('Custom camera background video producer was started more than once');
		}
		this.started = true;
		if (typeof this.video.requestVideoFrameCallback !== 'function') {
			throw new Error('Custom camera background video requires requestVideoFrameCallback');
		}
		if (!('VideoFrame' in globalThis)) {
			throw new Error('Custom camera background video requires VideoFrame');
		}
		this.video.src = this.objectURL;
		this.video.load();
		await waitForVideoMetadata(this.video, signal);
		this.validateMetadata();
		this.video.addEventListener('error', this.handlePlaybackError);
		this.video.addEventListener('ended', this.handleUnexpectedEnd);
		await waitForAbortablePromise(this.video.play(), signal);
		throwIfAborted(signal);
		this.scheduleNextFrame();
	}

	stop(): void {
		const failures = this.releaseOwnedResources(true);
		throwCollectedFailures({failures: failures, message: 'Custom camera background video producer teardown failed'});
	}

	private pullFrame(controller: ReadableStreamDefaultController<VideoFrame>): void | Promise<void> {
		if (this.stopped) return;
		if (this.pendingFrame != null) {
			const frame = this.pendingFrame;
			this.pendingFrame = null;
			controller.enqueue(frame);
			return;
		}
		if (this.pullResolve != null) {
			throw new Error('Custom camera background video stream has more than one pending pull');
		}
		return new Promise<void>((resolve) => {
			this.pullResolve = resolve;
		});
	}

	private cancelFromStream(): void {
		const failures = this.releaseOwnedResources(false);
		throwCollectedFailures({failures: failures, message: 'Custom camera background video stream cancellation failed'});
	}

	private validateMetadata(): void {
		validateCameraEffectFrameDimensions(this.video.videoWidth, this.video.videoHeight);
		if (!Number.isFinite(this.video.duration) || this.video.duration <= 0) {
			throw new Error('Custom camera background video has an invalid duration');
		}
		if (this.video.duration > MAX_CUSTOM_VIDEO_DURATION_SECONDS) {
			throw new Error('Custom camera background video exceeds the supported duration');
		}
		this.videoWidth = this.video.videoWidth;
		this.videoHeight = this.video.videoHeight;
	}

	private scheduleNextFrame(): void {
		if (this.stopped) return;
		try {
			this.callbackHandle = this.video.requestVideoFrameCallback(this.handleVideoFrame);
		} catch (error) {
			this.fail(error);
		}
	}

	private readonly handleVideoFrame: VideoFrameRequestCallback = (_now, metadata): void => {
		this.callbackHandle = null;
		if (this.stopped) return;
		let frame: VideoFrame | null = null;
		try {
			if (!Number.isFinite(metadata.mediaTime) || metadata.mediaTime < 0) {
				throw new Error('Custom camera background video produced an invalid frame timestamp');
			}
			const timestamp = Math.round(metadata.mediaTime * 1_000_000);
			if (!Number.isSafeInteger(timestamp)) {
				throw new Error('Custom camera background video frame timestamp exceeds the safe range');
			}
			frame = new VideoFrame(this.video, {timestamp});
			validateCameraEffectFrameDimensions(frame.displayWidth, frame.displayHeight);
			this.publishFrame(frame);
			frame = null;
		} catch (error) {
			frame?.close();
			this.fail(error);
			return;
		}
		this.scheduleNextFrame();
	};

	private publishFrame(frame: VideoFrame): void {
		const controller = this.controller;
		if (controller == null) {
			throw new Error('Custom camera background video stream has no controller');
		}
		if (this.pullResolve != null) {
			const resolve = this.pullResolve;
			this.pullResolve = null;
			controller.enqueue(frame);
			resolve();
			return;
		}
		const replaced = this.pendingFrame;
		this.pendingFrame = frame;
		replaced?.close();
	}

	private readonly handlePlaybackError = (): void => {
		this.fail(videoElementError(this.video));
	};

	private readonly handleUnexpectedEnd = (): void => {
		this.fail(new Error('Custom camera background video playback ended unexpectedly'));
	};

	private fail(error: unknown): void {
		if (this.stopped) return;
		const failures = this.releaseOwnedResources(false);
		const reportedError =
			failures.length === 0
				? error
				: new AggregateError([error, ...failures], 'Custom camera background video failed during cleanup');
		this.controller?.error(reportedError);
	}

	private releaseOwnedResources(closeStream: boolean): Array<unknown> {
		if (this.stopped) return [];
		this.stopped = true;
		const failures: Array<unknown> = [];
		this.cancelFrameCallback(failures);
		this.closePendingFrame(failures);
		this.resolvePendingPull();
		this.releaseVideoElement(failures);
		if (closeStream) {
			try {
				this.controller?.close();
			} catch (error) {
				failures.push(error);
			}
		}
		return failures;
	}

	private cancelFrameCallback(failures: Array<unknown>): void {
		if (this.callbackHandle == null) return;
		try {
			this.video.cancelVideoFrameCallback(this.callbackHandle);
		} catch (error) {
			failures.push(error);
		}
		this.callbackHandle = null;
	}

	private closePendingFrame(failures: Array<unknown>): void {
		const pendingFrame = this.pendingFrame;
		this.pendingFrame = null;
		if (pendingFrame == null) return;
		try {
			pendingFrame.close();
		} catch (error) {
			failures.push(error);
		}
	}

	private resolvePendingPull(): void {
		const resolve = this.pullResolve;
		this.pullResolve = null;
		resolve?.();
	}

	private releaseVideoElement(failures: Array<unknown>): void {
		this.video.removeEventListener('error', this.handlePlaybackError);
		this.video.removeEventListener('ended', this.handleUnexpectedEnd);
		try {
			this.video.pause();
			this.video.removeAttribute('src');
			this.video.load();
		} catch (error) {
			failures.push(error);
		}
		try {
			URL.revokeObjectURL(this.objectURL);
		} catch (error) {
			failures.push(error);
		}
	}
}

function isEBMLDocTypeByte(byte: number): boolean {
	if (byte >= 0x30 && byte <= 0x39) return true;
	if (byte >= 0x41 && byte <= 0x5a) return true;
	if (byte >= 0x61 && byte <= 0x7a) return true;
	return byte === 0x2d || byte === 0x5f;
}

function readEBMLDocTypeValue(bytes: Uint8Array, start: number, length: number): string | null {
	let docType = '';
	for (let index = 0; index < length; index += 1) {
		const byte = bytes[start + index] ?? 0;
		if (byte === 0) break;
		if (!isEBMLDocTypeByte(byte)) return null;
		docType += String.fromCharCode(byte);
	}
	if (docType.length === 0) return null;
	return docType;
}

function readEBMLDocType(bytes: Uint8Array): string | null {
	const searchLimit = bytes.byteLength - EBML_DOC_TYPE_ELEMENT_ID.length;
	for (let offset = EBML_MAGIC.length; offset < searchLimit; offset += 1) {
		if (!bytesEqualAt(bytes, offset, EBML_DOC_TYPE_ELEMENT_ID)) continue;
		const sizeOffset = offset + EBML_DOC_TYPE_ELEMENT_ID.length;
		const sizeByte = bytes[sizeOffset] ?? 0;
		if ((sizeByte & EBML_VINT_SINGLE_BYTE_MARKER) === 0) continue;
		const length = sizeByte & EBML_VINT_SINGLE_BYTE_VALUE_MASK;
		if (length === 0 || length > MAX_EBML_DOC_TYPE_LENGTH) continue;
		const valueOffset = sizeOffset + 1;
		if (valueOffset + length > bytes.byteLength) return null;
		return readEBMLDocTypeValue(bytes, valueOffset, length);
	}
	return null;
}

function sniffVideoMediaType(bytes: Uint8Array): string | null {
	if (bytesEqualAt(bytes, MP4_FILE_TYPE_BOX_OFFSET, MP4_FILE_TYPE_BOX)) return 'video/mp4';
	if (!bytesEqualAt(bytes, 0, EBML_MAGIC)) return null;
	const docType = readEBMLDocType(bytes);
	if (docType == null) {
		throw new Error('Custom camera background video has an unreadable EBML document type');
	}
	if (docType !== WEBM_DOC_TYPE) {
		throw new Error(`Custom camera background video document type is unsupported: ${docType}`);
	}
	return 'video/webm';
}

async function normalizeCustomVideoBlob(blob: Blob, signal: AbortSignal): Promise<Blob> {
	const header = new Uint8Array(await blob.slice(0, VIDEO_HEADER_SNIFF_BYTES).arrayBuffer());
	throwIfAborted(signal);
	const detectedMediaType = sniffVideoMediaType(header);
	if (detectedMediaType == null) {
		throw new Error('Custom camera background is not a supported video format');
	}
	const declaredMediaType = normalizedMediaType(blob.type);
	if (SUPPORTED_VIDEO_MEDIA_TYPES.has(declaredMediaType) && declaredMediaType !== detectedMediaType) {
		throw new Error('Custom camera background video type does not match its encoded data');
	}
	if (declaredMediaType === detectedMediaType) return blob;
	return new Blob([blob], {type: detectedMediaType});
}

function videoElementError(video: HTMLVideoElement): Error {
	const code = video.error?.code ?? 0;
	const message = video.error?.message.trim() ?? '';
	if (message.length > 0) return new Error(`Custom camera background video failed (${code}): ${message}`);
	return new Error(`Custom camera background video failed with media error ${code}`);
}

function waitForVideoMetadata(video: HTMLVideoElement, signal: AbortSignal): Promise<void> {
	if (signal.aborted) return Promise.reject(signal.reason);
	if (video.readyState >= 1 && video.videoWidth > 0 && video.videoHeight > 0) return Promise.resolve();
	return new Promise<void>((resolve, reject) => {
		const cleanup = (): void => {
			video.removeEventListener('loadedmetadata', handleMetadata);
			video.removeEventListener('error', handleError);
			signal.removeEventListener('abort', handleAbort);
		};
		const handleMetadata = (): void => {
			cleanup();
			resolve();
		};
		const handleError = (): void => {
			cleanup();
			reject(videoElementError(video));
		};
		const handleAbort = (): void => {
			cleanup();
			reject(signal.reason);
		};
		video.addEventListener('loadedmetadata', handleMetadata, {once: true});
		video.addEventListener('error', handleError, {once: true});
		signal.addEventListener('abort', handleAbort, {once: true});
	});
}

export async function createWebCameraEffectVideoFrameProducer(
	mediaURL: string,
): Promise<WebCameraEffectVideoFrameProducer> {
	requireWebCameraEffectCustomMediaURL(mediaURL);
	return runWithResponseDeadline({
		timeoutMilliseconds: WEB_CAMERA_EFFECT_CUSTOM_MEDIA_OPERATION_TIMEOUT_MS,
		description: 'Custom camera background video initialization',
		signal: null,
		operation: async (signal) => {
			const blob = await readWebCameraEffectCustomMediaBlob(mediaURL, signal);
			const normalizedBlob = await normalizeCustomVideoBlob(blob, signal);
			const producer = new WebCameraEffectVideoFrameProducerOwner(normalizedBlob);
			try {
				await producer.start(signal);
				return producer;
			} catch (error) {
				try {
					producer.stop();
				} catch (cleanupError) {
					throw new AggregateError(
						[error, cleanupError],
						'Custom camera background video initialization failed during cleanup',
					);
				}
				throw error;
			}
		},
	});
}