A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import {getBackgroundMediaObjectURL} from '@app/features/theme/utils/BackgroundImageDB';
import VoiceSettings, {BLUR_BACKGROUND_ID, NONE_BACKGROUND_ID} from '@app/features/voice/state/VoiceSettings';
import {
type CameraVideoProcessorHandle,
type CameraVideoProcessorOptions,
createCameraVideoProcessor,
isCameraVideoProcessor,
} from '@app/features/voice/utils/CameraVideoProcessor';
import {CameraBackgroundMode} from '@app/features/voice/utils/camera-effects/CameraCaptureContract';
import type {WebCameraEffectConfig} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol';
import {areVoiceBackgroundsAvailable} from '@app/features/voice/utils/VoiceBackgroundAvailability';
import type {LocalVideoTrack} from 'livekit-client';
const logger = new Logger('VideoBackgroundProcessor');
export interface BackgroundProcessorOptions {
backgroundImageId?: string;
}
export interface AppliedBackgroundProcessor {
destroy: () => Promise<void>;
}
export async function clearCameraVideoProcessor(track: LocalVideoTrack): Promise<void> {
if (!track.getProcessor()) {
return;
}
await track.stopProcessor(false);
if (track.getProcessor()) {
throw new Error('Camera video processor remained attached after it was cleared');
}
logger.info('Cleared background processor');
}
async function recoverRawCameraAfterProcessorFailure(
track: LocalVideoTrack,
processor: CameraVideoProcessorHandle,
error: Error,
): Promise<void> {
try {
const recovered = await track.stopProcessorIfCurrent(processor, false);
if (recovered) {
logger.warn('Camera video processor failed; restored the raw camera track', {error});
}
} catch (recoveryError) {
if (track.mediaStreamTrack.readyState === 'ended') {
logger.warn('Camera video processor failed and the camera track was already gone during recovery', {
error,
recoveryError,
});
return;
}
logger.error('Camera video processor failed and raw camera recovery was incomplete', {
error,
recoveryError,
});
}
}
async function applyCameraVideoProcessor(
track: LocalVideoTrack,
options: CameraVideoProcessorOptions,
logLabel: string,
): Promise<AppliedBackgroundProcessor> {
const updatedProcessor = await track.runWithTrackChangeLock(async () => {
const activeProcessor = track.getProcessor();
if (!isCameraVideoProcessor(activeProcessor) || !activeProcessor.canUpdate(options)) {
return null;
}
await activeProcessor.update(options);
if (track.getProcessor() !== activeProcessor) {
throw new Error('Updated camera video processor is no longer attached to the active track');
}
if (activeProcessor.processedTrack?.readyState !== 'live') {
throw new Error('Updated camera video processor has no live output track');
}
return activeProcessor;
});
if (updatedProcessor != null) {
logger.info(logLabel);
return updatedProcessor;
}
const processor = createCameraVideoProcessor(options);
processor.setOperationalFailureHandler((error) => {
void recoverRawCameraAfterProcessorFailure(track, processor, error);
});
try {
await track.setProcessor(processor);
if (track.getProcessor() !== processor) {
throw new Error('Camera video processor was not retained by the active track');
}
if (processor.processedTrack?.readyState !== 'live') {
throw new Error('Camera video processor produced no live output track');
}
logger.info(logLabel);
return processor;
} catch (error) {
try {
if (track.getProcessor() === processor) {
await track.stopProcessor(false);
}
} catch (cleanupError) {
throw new AggregateError([error, cleanupError], 'Camera video processor apply and cleanup both failed');
}
throw error;
}
}
async function resolveBackgroundEffectConfig(
options?: BackgroundProcessorOptions,
): Promise<WebCameraEffectConfig | null> {
const backgroundImageId = options?.backgroundImageId ?? VoiceSettings.getBackgroundImageId();
if (backgroundImageId === NONE_BACKGROUND_ID) {
return null;
}
if (!areVoiceBackgroundsAvailable()) {
throw new Error('Camera background effects are unavailable');
}
const blurStrength = VoiceSettings.getBackgroundBlurStrength();
if (backgroundImageId === BLUR_BACKGROUND_ID) {
return {mode: CameraBackgroundMode.BLUR, blurStrength};
}
const backgroundImages = VoiceSettings.getBackgroundImages();
const hasImage = backgroundImages.some((image) => image.id === backgroundImageId);
if (!hasImage) {
throw new Error(`Custom camera background is not present in the saved media list: ${backgroundImageId}`);
}
const customMedia = await getBackgroundMediaObjectURL(backgroundImageId);
if (!customMedia) {
throw new Error(`Custom camera background media could not be resolved: ${backgroundImageId}`);
}
return {
mode: CameraBackgroundMode.CUSTOM,
blurStrength,
customMediaURL: customMedia.url,
customMediaKind: customMedia.mediaKind,
};
}
export async function applyBackgroundProcessor(
track: LocalVideoTrack,
options?: BackgroundProcessorOptions,
): Promise<AppliedBackgroundProcessor | null> {
try {
const background = await resolveBackgroundEffectConfig(options);
if (background == null) {
await clearCameraVideoProcessor(track);
logger.debug('No camera video processor applied');
return null;
}
return applyCameraVideoProcessor(track, {background}, `Applied camera background (${background.mode})`);
} catch (error) {
logger.warn('Failed to apply camera video processor', error);
throw error;
}
}