Skip to content

Jitsi Meet - Secure, Simple and Scalable Video Conferences that you use as a standalone app or embed in your web application.

5.6 KB
Raw Blame
/* eslint-disable lines-around-comment */
import {
    createVirtualBackgroundV2FailedEvent,
    createVirtualBackgroundV2StartedEvent
} from '../../analytics/AnalyticsEvents';
import { sendAnalytics } from '../../analytics/functions';
import { showWarningNotification } from '../../notifications/actions';
import { NOTIFICATION_TIMEOUT_TYPE } from '../../notifications/constants';
import { timeout } from '../../virtual-background/functions';
import logger from '../../virtual-background/logger';
import { IVirtualBackground } from '../../virtual-background/reducer';

import { detectDeviceTier } from './DeviceTierDetector';
import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
// @ts-ignore
import createTFLiteModule from './vendor/tflite/tflite';
// @ts-ignore
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
/* eslint-enable lines-around-comment */

const models = {
    modelLandscape: 'libs/selfie_segmentation_landscape.tflite'
};

let modelBuffer: ArrayBuffer;
let tflite: any;
let wasmCheck: any;
let isWasmDisabled = false;
const reportedV2Events = new Set<string>();

/**
 * Sends a V2 startup analytics event once per distinct payload, since an effect is created for every
 * preview and track (re)creation.
 *
 * @param {Object} event - The analytics event.
 * @returns {void}
 */
function sendV2StartupEvent(event: Object) {
    const key = JSON.stringify(event);

    if (!reportedV2Events.has(key)) {
        reportedV2Events.add(key);
        sendAnalytics(event);
    }
}

/**
 * Creates a new instance of the virtual background stream effect.
 *
 * @param {Object} virtualBackground - The virtual object that contains the background image source and
 * the isVirtualBackground flag that indicates if virtual image is activated.
 * @param {Function} dispatch - The Redux dispatch function.
 * @returns {Promise<JitsiStreamBackgroundEffect | undefined>}
 */
export async function createVirtualBackgroundEffect(virtualBackground: IVirtualBackground,
        dispatch?: Function
): Promise<JitsiStreamBackgroundEffect | undefined> {
    if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) {
        throw new Error('JitsiStreamBackgroundEffect not supported!');
    }
    const fullVbConfig = APP.store.getState()['features/base/config'].virtualBackground;

    // V2 is the default engine. V1 is only used when enableV2 is explicitly set to false.
    // (The type of enableV2 is `boolean | undefined` per IConfig, so `undefined` picks V2.)
    const useV2Engine = fullVbConfig?.enableV2 !== false;

    if (useV2Engine) {
        const vbConfig = fullVbConfig?.advanced;
        const capabilities = await detectDeviceTier(vbConfig);

        logger.info(
            '[VirtualBackground] Using V2 engine —'
            + ` tier: ${capabilities.tier}, backend: ${capabilities.backend}`
        );

        const effect = new JitsiStreamBackgroundEffect(undefined, virtualBackground, {
            capabilities,
            enableV2: true,
            vbConfig
        });

        try {
            await effect.init();
        } catch (error: any) {
            sendV2StartupEvent(createVirtualBackgroundV2FailedEvent({
                ...effect.v2StartupConfig,
                error: String(error?.message ?? error)
            }));

            throw error;
        }

        const startup = effect.v2StartupConfig;

        if (startup) {
            sendV2StartupEvent(createVirtualBackgroundV2StartedEvent({
                ...startup,
                workerFallback: startup.backend !== capabilities.backend
            }));
        }

        return effect;
    }

    if (isWasmDisabled) {
        dispatch?.(showWarningNotification({
            titleKey: 'virtualBackground.backgroundEffectError'
        }, NOTIFICATION_TIMEOUT_TYPE.LONG));

        return;
    }

    // Checks if WebAssembly feature is supported or enabled by/in the browser.
    // Conditional import of wasm-check package is done to prevent
    // the browser from crashing when the user opens the app.

    if (!tflite) {
        try {
            wasmCheck = require('wasm-check');
            const tfliteTimeout = 10000;

            if (wasmCheck?.feature?.simd) {
                tflite = await timeout(tfliteTimeout, createTFLiteSIMDModule());
            } else {
                tflite = await timeout(tfliteTimeout, createTFLiteModule());
            }
        } catch (err: any) {
            if (err?.message === '408') {
                logger.error('Failed to download tflite model!');
                dispatch?.(showWarningNotification({
                    titleKey: 'virtualBackground.backgroundEffectError'
                }, NOTIFICATION_TIMEOUT_TYPE.LONG));
            } else {
                isWasmDisabled = true;
                logger.error('Looks like WebAssembly is disabled or not supported on this browser', err);
                dispatch?.(showWarningNotification({
                    titleKey: 'virtualBackground.webAssemblyWarning',
                    descriptionKey: 'virtualBackground.webAssemblyWarningDescription'
                }, NOTIFICATION_TIMEOUT_TYPE.LONG));
            }

            return;
        }
    }

    if (!modelBuffer) {
        const modelResponse = await fetch(models.modelLandscape);

        if (!modelResponse.ok) {
            throw new Error('Failed to download tflite model!');
        }

        modelBuffer = await modelResponse.arrayBuffer();

        tflite.HEAPU8.set(new Uint8Array(modelBuffer), tflite._getModelBufferMemoryOffset());

        tflite._loadModel(modelBuffer.byteLength);
    }

    return new JitsiStreamBackgroundEffect(tflite, virtualBackground);
}