Skip to content

Jitsi Meet - Secure, Simple and Scalable Video Conferences that you use as a standalone app or embed in your web application.

19.6 KB
Raw Blame
import { AnyAction } from 'redux';

import { IStore } from '../app/types';
import { ENDPOINT_MESSAGE_RECEIVED, NON_PARTICIPANT_MESSAGE_RECEIVED } from '../base/conference/actionTypes';
import { MEET_FEATURES } from '../base/jwt/constants';
import { isJwtFeatureEnabled } from '../base/jwt/functions';
import JitsiMeetJS from '../base/lib-jitsi-meet';
import { TRANSCRIBER_ID } from '../base/participants/constants';
import MiddlewareRegistry from '../base/redux/MiddlewareRegistry';
import StateListenerRegistry from '../base/redux/StateListenerRegistry';
import { isE2EEOnInMeeting } from '../e2ee/functions';
import { showErrorNotification, showNotification } from '../notifications/actions';
import { NOTIFICATION_TIMEOUT_TYPE } from '../notifications/constants';
import { RECORDING_METADATA_ID } from '../recording/constants';
import { maybeNotifyRecordingStart } from '../recording/middleware';
import { TRANSCRIBER_JOINED } from '../transcribing/actionTypes';

import {
    SET_REQUESTING_SUBTITLES,
    TOGGLE_REQUESTING_SUBTITLES
} from './actionTypes';
import {
    removeCachedTranscriptMessage,
    removeTranscriptMessage,
    setRequestingSubtitles,
    setSubtitlesError,
    storeSubtitle,
    updateTranscriptMessage
} from './actions.any';
import { notifyTranscriptionChunkReceived } from './functions';
import { areClosedCaptionsEnabled, isCCTabEnabled } from './functions.any';
import logger from './logger';
import { ISubtitle, ITranscriptMessage } from './types';

/**
 * The type of json-message which indicates that json carries a
 * transcription result.
 */
const JSON_TYPE_TRANSCRIPTION_RESULT = 'transcription-result';

/**
 * The type of json-message which indicates that json carries a
 * translation result.
 */
const JSON_TYPE_TRANSLATION_RESULT = 'translation-result';

/**
 * The local participant property which is used to set whether the local
 * participant wants to have a transcriber in the room.
 */
const P_NAME_REQUESTING_TRANSCRIPTION = 'requestingTranscription';

/**
 * The local participant property which is used to store the language
 * preference for translation for a participant.
 */
const P_NAME_TRANSLATION_LANGUAGE = 'translation_language';

/**
 * The dial command to use for starting a transcriber.
 */
const TRANSCRIBER_DIAL_NUMBER = 'jitsi_meet_transcribe';

/**
* Time after which the rendered subtitles will be removed.
*/
const REMOVE_AFTER_MS = 3000;

/**
 * Stability factor for a transcription. We'll treat a transcript as stable
 * beyond this value.
 */
const STABLE_TRANSCRIPTION_FACTOR = 0.85;

/**
 * Middleware that catches actions related to transcript messages to be rendered
 * in {@link Captions}.
 *
 * @param {Store} store - The redux store.
 * @returns {Function}
 */
MiddlewareRegistry.register(store => next => action => {
    switch (action.type) {
    case ENDPOINT_MESSAGE_RECEIVED:
    case NON_PARTICIPANT_MESSAGE_RECEIVED:
        return _endpointMessageReceived(store, next, action);

    case TOGGLE_REQUESTING_SUBTITLES: {
        const state = store.getState()['features/subtitles'];
        const toggledValue = !state._requestingSubtitles;

        if (toggledValue && _refusedUnderE2EE(store)) {
            return;
        }
        _requestingSubtitlesChange(store, toggledValue, state._language);
        break;
    }
    case TRANSCRIBER_JOINED: {
        const { transcription } = store.getState()['features/base/config'];

        if (transcription?.autoCaptionOnTranscribe) {
            // The transcriber was started by someone else — skip metadata update
            // to avoid overwriting the initiator's isRecordingRequested flag.
            store.dispatch(setRequestingSubtitles(
                true, undefined, undefined, undefined, undefined, true));
        }

        break;
    }
    case SET_REQUESTING_SUBTITLES:
        if (action.enabled && _refusedUnderE2EE(store)) {
            return;
        }
        _requestingSubtitlesChange(
            store, action.enabled, action.language,
            action.forceBackendRecordingOn, action.isRecordingRequested,
            action.skipMetadataUpdate);
        break;
    }

    return next(action);
});

/**
 * Notifies the feature transcription that the action
 * {@code ENDPOINT_MESSAGE_RECEIVED} is being dispatched within a specific redux
 * store.
 *
 * @param {Store} store - The redux store in which the specified {@code action}
 * is being dispatched.
 * @param {Dispatch} next - The redux {@code dispatch} function to
 * dispatch the specified {@code action} to the specified {@code store}.
 * @param {Action} action - The redux action {@code ENDPOINT_MESSAGE_RECEIVED}
 * which is being dispatched in the specified {@code store}.
 * @private
 * @returns {Object} The value returned by {@code next(action)}.
 */
function _endpointMessageReceived(store: IStore, next: Function, action: AnyAction) {
    let json: any = {};

    if (action.type === ENDPOINT_MESSAGE_RECEIVED) {
        if (!action.participant.isHidden()) {
            return next(action);
        }
        json = action.data;
    } else if (action.type === NON_PARTICIPANT_MESSAGE_RECEIVED && action.id === TRANSCRIBER_ID) {
        json = action.json;
    } else {
        return next(action);
    }

    if (![ JSON_TYPE_TRANSCRIPTION_RESULT, JSON_TYPE_TRANSLATION_RESULT ].includes(json?.type)) {
        return next(action);
    }

    const { dispatch, getState } = store;
    const state = getState();
    const _areClosedCaptionsEnabled = areClosedCaptionsEnabled(store.getState());
    const transcriptMessageID = json.message_id;
    const { name, id, avatar_url: avatarUrl } = json.participant ?? {};
    const participant = {
        avatarUrl,
        id,
        name
    };
    const { timestamp } = json;
    const participantId = participant.id;
    const speaker = json.speaker;
    const renderTranscriptDetails = state['features/base/config'].transcription?.renderTranscriptDetails;
    let detailsPrefix = '';

    if (renderTranscriptDetails) {
        const parts = [];

        if (speaker != null) {
            parts.push(`Speaker ${speaker}`);
        }
        if (json.language) {
            parts.push(json.language);
        }
        if (parts.length > 0) {
            detailsPrefix = `[${parts.join(', ')}] `;
        }
    }

    // Handle transcript messages
    const language = state['features/base/conference'].conference
        ?.getLocalParticipantProperty(P_NAME_TRANSLATION_LANGUAGE);
    const { dumpTranscript, skipInterimTranscriptions } = state['features/base/config'].testing ?? {};

    let newTranscriptMessage: ITranscriptMessage | undefined;

    if (json.type === JSON_TYPE_TRANSLATION_RESULT) {
        if (!_areClosedCaptionsEnabled) {
            // If closed captions are not enabled, bail out.
            return next(action);
        }

        const translationText = json.text?.trim();
        const translation = translationText ? `${detailsPrefix}${translationText}` : translationText;

        if (isCCTabEnabled(state)) {
            dispatch(storeSubtitle({
                participantId,
                text: translation,
                language: json.language,
                interim: false,
                isTranscription: false,
                timestamp,
                id: transcriptMessageID
            }));

            return next(action);
        }

        if (json.language === language) {
            // Displays final results in the target language if translation is
            // enabled.
            newTranscriptMessage = {
                clearTimeOut: undefined,
                final: translation,
                participant
            };
        }
    } else if (json.type === JSON_TYPE_TRANSCRIPTION_RESULT) {
        const isInterim = json.is_interim;

        // Displays interim and final results without any translation if
        // translations are disabled.

        if (!Array.isArray(json.transcript) || !json.transcript[0]) {
            return next(action);
        }
        const { text } = json.transcript[0];
        const displayText = `${detailsPrefix}${text}`;

        // First, notify the external API.
        if (!(isInterim && skipInterimTranscriptions)) {
            const txt: any = {};

            if (!json.is_interim) {
                txt.final = text;
            } else if (json.stability > STABLE_TRANSCRIPTION_FACTOR) {
                txt.stable = text;
            } else {
                txt.unstable = text;
            }

            notifyTranscriptionChunkReceived(
                transcriptMessageID,
                json.language,
                participant,
                txt,
                store
            );

            if (navigator.product !== 'ReactNative') {

                // Dump transcript in a <transcript> element for debugging purposes.
                if (!json.is_interim && dumpTranscript) {
                    try {
                        let elem = document.body.getElementsByTagName('transcript')[0];

                        // eslint-disable-next-line max-depth
                        if (!elem) {
                            elem = document.createElement('transcript');
                            document.body.appendChild(elem);
                        }

                        elem.append(`${new Date(json.timestamp).toISOString()} ${participant.name}: ${text}`);
                    } catch (_) {
                        // Ignored.
                    }
                }
            }
        }

        if (!_areClosedCaptionsEnabled) {
            // If closed captions are not enabled, bail out.
            return next(action);
        }

        const subtitle: ISubtitle = {
            id: transcriptMessageID,
            participantId,
            language: json.language,
            text: displayText,
            interim: isInterim,
            timestamp,
            isTranscription: true
        };

        if (isCCTabEnabled(state)) {
            dispatch(storeSubtitle(subtitle));

            return next(action);
        }

        // Only display transcription results when the local user is showing subtitles and either
        // wants the source language (no translation language selected) or the transcription's
        // language matches the selected one. We rely on the redux _language (which is cleared when
        // switching back to the original language) rather than the translation_language participant
        // property, which is never cleared and can become stale.
        // The primary language code comparison catches all country-code notations like 'en-GB',
        // 'en_GB' and 'enGB' independent of the country code length.
        const { _language, _requestingSubtitles } = state['features/subtitles'];
        const selectedLanguage = _language?.replace('translation-languages:', '');

        if (!_requestingSubtitles
                || (selectedLanguage
                    && _getPrimaryLanguageCode(json.language) !== _getPrimaryLanguageCode(selectedLanguage))) {
            return next(action);
        }

        if (json.is_interim && skipInterimTranscriptions) {
            return next(action);
        }

        // We update the previous transcript message with the same
        // message ID or adds a new transcript message if it does not
        // exist in the map.
        const existingMessage = state['features/subtitles']._transcriptMessages.get(transcriptMessageID);

        newTranscriptMessage = {
            clearTimeOut: existingMessage?.clearTimeOut,
            participant
        };

        // If this is final result, update the state as a final result
        // and start a count down to remove the subtitle from the state
        if (!json.is_interim) {
            newTranscriptMessage.final = displayText;
        } else if (json.stability > STABLE_TRANSCRIPTION_FACTOR) {
            // If the message has a high stability, we can update the
            // stable field of the state and remove the previously
            // unstable results
            newTranscriptMessage.stable = displayText;
        } else {
            // Otherwise, this result has an unstable result, which we
            // add to the state. The unstable result will be appended
            // after the stable part.
            newTranscriptMessage.unstable = displayText;
        }
    }

    if (newTranscriptMessage) {
        if (newTranscriptMessage.final) {
            const cachedTranscriptMessage
                = state['features/subtitles']._cachedTranscriptMessages?.get(transcriptMessageID);

            if (cachedTranscriptMessage) {
                const cachedText = (cachedTranscriptMessage.stable || cachedTranscriptMessage.unstable)?.trim();
                const newText = newTranscriptMessage.final;

                if (cachedText && cachedText.length > 0 && newText && newText.length > 0
                    && newText.toLowerCase().startsWith(cachedText.toLowerCase())) {
                    newTranscriptMessage.final = newText.slice(cachedText.length)?.trim();
                }
                dispatch(removeCachedTranscriptMessage(transcriptMessageID));

                if (!newTranscriptMessage.final || newTranscriptMessage.final.length === 0) {
                    return next(action);
                }
            }
        }


        _setClearerOnTranscriptMessage(dispatch, transcriptMessageID, newTranscriptMessage);
        dispatch(updateTranscriptMessage(transcriptMessageID, newTranscriptMessage));
    }

    return next(action);
}

/**
 * Utility function to extract the primary language code like 'en-GB' 'en_GB'
 * 'enGB' 'zh-CN' and 'zh-TW'.
 *
 * @param {string} language - The language to use for translation or user requested.
 * @returns {string}
 */
function _getPrimaryLanguageCode(language: string) {
    return language.replace(/[-_A-Z].*/, '');
}

/**
 * Toggle the local property 'requestingTranscription'. This will cause Jicofo
 * and Jigasi to decide whether the transcriber needs to be in the room.
 *
 * @param {Store} store - The redux store.
 * @param {boolean} enabled - Whether subtitles should be enabled or not.
 * @param {string} language - The language to use for translation.
 * @param {boolean} forceBackendRecordingOn - Whether to force backend recording is on or not. This is used only when
 * we start recording, stopping is based on whether isTranscribingEnabled is already set.
 * @param {boolean} isRecordingRequested - Whether recording was also requested alongside transcription.
 * Passed through to metadata so remote clients receive both intent fields in a single atomic update.
 * @param {boolean} skipMetadataUpdate - When true, skips setting room metadata. Used when reacting
 * to a transcriber started by someone else (e.g. autoCaptionOnTranscribe) to avoid overwriting
 * the initiator's metadata.
 * @private
 * @returns {void}
 */
function _requestingSubtitlesChange(
        { dispatch, getState }: IStore,
        enabled: boolean,
        language?: string | null,
        forceBackendRecordingOn: boolean = false,
        isRecordingRequested: boolean = false,
        skipMetadataUpdate: boolean = false) {
    const state = getState();
    const { conference } = state['features/base/conference'];
    const backendRecordingOn = conference?.getMetadataHandler()?.getMetadata()?.asyncTranscription;

    conference?.setLocalParticipantProperty(
        P_NAME_REQUESTING_TRANSCRIPTION,
        enabled);

    if (enabled && conference?.getTranscriptionStatus() === JitsiMeetJS.constants.transcriptionStatus.OFF
        && isJwtFeatureEnabled(getState(), MEET_FEATURES.TRANSCRIPTION, false)) {

        if (!backendRecordingOn) {
            conference?.dial(TRANSCRIBER_DIAL_NUMBER)
                .catch((e: any) => {
                    logger.error('Error dialing', e);

                    // let's back to the correct state
                    dispatch(setRequestingSubtitles(false, false, null));

                    dispatch(showErrorNotification({
                        titleKey: 'transcribing.failed'
                    }));
                    dispatch(setSubtitlesError(true));

                    // Transcription failed — re-evaluate. _hasError is now true,
                    // so maybeNotifyRecordingStart will see transcription as resolved-failed
                    // and play recording-only sound if recording succeeded.
                    const intent = getState()['features/recording'].startRecordingIntent;

                    if (intent) {
                        maybeNotifyRecordingStart(dispatch, getState);
                    }
                });
        }

        if (!skipMetadataUpdate && (backendRecordingOn || forceBackendRecordingOn)) {
            conference?.getMetadataHandler()?.setMetadata(RECORDING_METADATA_ID, {
                isTranscribingEnabled: true,
                ...(isRecordingRequested && { isRecordingRequested: true })
            });
        }
    }

    // Advertise the requested translation language, and clear it when there is none. Clearing matters: the property
    // is what tells the backend which languages to translate into, so leaving a stale value behind would keep a
    // language being translated after the user switched back to the original one or turned subtitles off.
    conference?.setLocalParticipantProperty(
        P_NAME_TRANSLATION_LANGUAGE,
        enabled && language ? language.replace('translation-languages:', '') : '');

    if (!enabled && !skipMetadataUpdate && (backendRecordingOn || forceBackendRecordingOn)
        && conference?.getMetadataHandler()?.getMetadata()[RECORDING_METADATA_ID]?.isTranscribingEnabled) {
        conference?.getMetadataHandler()?.setMetadata(RECORDING_METADATA_ID, {
            isRecordingRequested: false,
            isTranscribingEnabled: false
        });
    }
}

/**
 * Set a timeout on a TranscriptMessage object so it clears itself when it's not
 * updated.
 *
 * @param {Function} dispatch - Dispatch remove action to store.
 * @param {string} transcriptMessageID - The id of the message to remove.
 * @param {Object} transcriptMessage - The message to remove.
 * @returns {void}
 */
function _setClearerOnTranscriptMessage(
        dispatch: IStore['dispatch'],
        transcriptMessageID: string,
        transcriptMessage: { clearTimeOut?: number; }) {
    if (transcriptMessage.clearTimeOut) {
        clearTimeout(transcriptMessage.clearTimeOut);
    }

    transcriptMessage.clearTimeOut
        = window.setTimeout(
            () => dispatch(removeTranscriptMessage(transcriptMessageID)),
            REMOVE_AFTER_MS);
}

/**
 * Refuses a subtitles request while anyone has E2EE on: nothing server-side can transcribe encrypted media. The
 * request never reaches the reducer.
 *
 * @param {IStore} store - The redux store.
 * @returns {boolean} Whether the request was refused.
 */
function _refusedUnderE2EE({ dispatch, getState }: IStore) {
    if (!isE2EEOnInMeeting(getState())) {
        return false;
    }
    logger.warn('Not requesting subtitles: end-to-end encryption is on');
    dispatch(showErrorNotification({ titleKey: 'transcribing.unavailableWithE2EE' }));
    dispatch(setSubtitlesError(true));

    // A recording started alongside still gets its start notification.
    if (getState()['features/recording'].startRecordingIntent) {
        maybeNotifyRecordingStart(dispatch, getState);
    }

    return true;
}

/**
 * E2EE came on with subtitles requested here (a race, or a client that doesn't block it): drop the local request.
 * A meeting-wide transcription is left to whoever runs it.
 */
StateListenerRegistry.register(
    state => isE2EEOnInMeeting(state),
    (e2eeOn, { dispatch, getState }) => {
        if (e2eeOn && getState()['features/subtitles']._requestingSubtitles) {
            dispatch(setRequestingSubtitles(false, false, null, false, false, true));
            dispatch(showNotification({ titleKey: 'transcribing.stoppedForE2EE' }, NOTIFICATION_TIMEOUT_TYPE.MEDIUM));
        }
    });