Skip to content

🎥 Make videos programmatically with React

12.3 KB
Raw Blame
import {useLayoutEffect} from 'react';
/* eslint-disable @typescript-eslint/no-use-before-define */
import {useContext, useEffect, useRef} from 'react';
import {Internals} from 'remotion';
import type {RemotionAudioContextState} from 'remotion';
import type {BrowserMediaControlsBehavior} from './browser-mediasession.js';
import {useBrowserMediaSession} from './browser-mediasession.js';
import {calculateNextFrame} from './calculate-next-frame.js';
import {useIsBackgrounded} from './is-backgrounded.js';
import {setGlobalTimeAnchor} from './set-global-time-anchor.js';
import {type UsePlayerMethods, usePlayerMethods} from './use-player-methods.js';

const shouldForceAnchorChange = (newState: RemotionAudioContextState) => {
	if (newState === 'suspended' || newState === 'running-to-suspended') {
		return true;
	}

	if (
		newState === 'closed' ||
		newState === 'interrupted' ||
		newState === 'running' ||
		newState === 'suspended-to-running'
	) {
		return false;
	}

	throw new Error(
		`Unexpected audio context state: ${newState satisfies never}`,
	);
};

// The start timestamp and the frames already advanced within the current
// frame are kept across re-runs of the playback effect, so that a re-run
// (e.g. because durationInFrames changed, see issue #12279) does not
// restart the clock from the current frame and lose the already elapsed
// portion of the frame.
// A fresh clock is created only where a restart is correct: when starting
// playback, or when the composition, fps or playback rate changed mid-playback.
type PlaybackClock = {
	compositionId: string;
	startedTime: number;
	framesAdvanced: number;
	fps: number;
	playbackRate: number;
};

export const usePlayback = ({
	loop,
	playbackRate,
	moveToBeginningWhenEnded,
	inFrame,
	outFrame,
	browserMediaControlsBehavior,
	getCurrentFrame,
	muted,
}: {
	loop: boolean;
	playbackRate: number;
	moveToBeginningWhenEnded: boolean;
	inFrame: number | null;
	outFrame: number | null;
	browserMediaControlsBehavior: BrowserMediaControlsBehavior;
	getCurrentFrame: UsePlayerMethods['getCurrentFrame'];
	muted: boolean;
}) => {
	const config = Internals.useUnsafeVideoConfig();
	const fps = config?.fps ?? null;
	const durationInFrames = config?.durationInFrames ?? null;
	const compositionId = config?.id ?? null;
	const frame = Internals.Timeline.useTimelinePosition();
	const playing = Internals.usePlaying();
	const {pause, emitter, isPlaying} = usePlayerMethods();
	const setFrameFromPlayback =
		Internals.Timeline.useTimelineSetFrameWithoutSeek();
	const sharedAudioContext = useContext(Internals.SharedAudioContext);
	const {setPlayerMuted} = useContext(Internals.SetMediaVolumeContext);
	const {isBuffering, subscribeBuffering, seek} = useContext(
		Internals.SetTimelineContext,
	);
	const logLevel = Internals.useLogLevel();

	// requestAnimationFrame() does not work if the tab is not active.
	// This means that audio will keep playing even if it has ended.
	// In that case, we use setTimeout() instead.
	const isBackgroundedRef = useIsBackgrounded();

	const lastTimeUpdateTimestamp = useRef<number>(0);
	const playbackClockRef = useRef<PlaybackClock | null>(null);
	const previousMutedRef = useRef(muted);
	const anchoredAudioContextRef = useRef<AudioContext | null>(null);

	useBrowserMediaSession({
		browserMediaControlsBehavior,
		playbackRate,
		videoConfig: config,
	});

	// Update time anchor when seeking:
	// If the user clicked on a different time in the timeline, we need to re-sync the anchor
	useLayoutEffect(() => {
		if (!sharedAudioContext) {
			return;
		}

		if (!sharedAudioContext.audioContext) {
			return;
		}

		if (fps === null) {
			return;
		}

		if (muted) {
			return;
		}

		const changed = setGlobalTimeAnchor({
			audioContext: sharedAudioContext.audioContext,
			audioSyncAnchor: sharedAudioContext.audioSyncAnchor,
			absoluteTimeInSeconds: frame / fps,
			globalPlaybackRate: playbackRate,
			logLevel,
			force: false,
		});
		if (changed) {
			sharedAudioContext.audioSyncAnchorEmitter.dispatch('changed');
		}
	}, [fps, frame, logLevel, playbackRate, sharedAudioContext, muted]);

	// When the audio context is suspended, we use the opportunity to
	// re-anchor the time to be exact.
	useLayoutEffect(() => {
		const audioContext = sharedAudioContext?.audioContext;
		if (!audioContext) {
			return;
		}

		if (fps === null) {
			return;
		}

		if (muted) {
			return;
		}

		const callback = () => {
			const newState = sharedAudioContext?.getAudioContextState();
			if (newState && shouldForceAnchorChange(newState)) {
				const changed = setGlobalTimeAnchor({
					audioContext,
					audioSyncAnchor: sharedAudioContext.audioSyncAnchor,
					absoluteTimeInSeconds: getCurrentFrame() / fps,
					globalPlaybackRate: playbackRate,
					logLevel,
					force: true,
				});
				if (changed) {
					sharedAudioContext.audioSyncAnchorEmitter.dispatch('changed');
				}
			}
		};

		audioContext?.addEventListener('statechange', callback);
		return () => {
			audioContext?.removeEventListener('statechange', callback);
		};
	}, [fps, getCurrentFrame, logLevel, muted, playbackRate, sharedAudioContext]);

	useEffect(() => {
		const wasMuted = previousMutedRef.current;
		previousMutedRef.current = muted;

		if (fps === null || durationInFrames === null || compositionId === null) {
			playbackClockRef.current = null;
			return;
		}

		if (!playing) {
			playbackClockRef.current = null;
			sharedAudioContext?.suspend?.();
			return;
		}

		// Duration and range changes update the endpoint without losing elapsed
		// sub-frame time. A different composition or timebase needs a fresh clock.
		const previousClock = playbackClockRef.current;
		const shouldRestartClock =
			previousClock === null ||
			previousClock.compositionId !== compositionId ||
			previousClock.fps !== fps ||
			previousClock.playbackRate !== playbackRate;
		const clock: PlaybackClock = shouldRestartClock
			? {
					compositionId,
					startedTime: performance.now(),
					framesAdvanced: 0,
					fps,
					playbackRate,
				}
			: previousClock;
		playbackClockRef.current = clock;

		if (
			sharedAudioContext?._experimentalKeepAudioContextAlive &&
			sharedAudioContext.audioContext &&
			!muted
		) {
			const didUnmute = wasMuted && !muted;
			const didReplaceAudioContext =
				anchoredAudioContextRef.current !== sharedAudioContext.audioContext;
			const shouldRescheduleAudio =
				shouldRestartClock || didUnmute || didReplaceAudioContext;

			// The native clock keeps running through pauses and muted playback.
			// These transitions need an exact anchor, as does a replacement context
			// with its own time origin. Other effect re-runs use the drift tolerance
			// so duration changes do not introduce frame-quantized audio shifts.
			const changed = setGlobalTimeAnchor({
				audioContext: sharedAudioContext.audioContext,
				audioSyncAnchor: sharedAudioContext.audioSyncAnchor,
				absoluteTimeInSeconds: getCurrentFrame() / fps,
				globalPlaybackRate: playbackRate,
				logLevel,
				force: shouldRescheduleAudio,
			});
			anchoredAudioContextRef.current = sharedAudioContext.audioContext;
			// A changed timebase can invalidate queued audio even if the anchor
			// happens to remain numerically equal (for example at frame zero).
			if (changed || shouldRescheduleAudio) {
				sharedAudioContext.audioSyncAnchorEmitter.dispatch('changed');
			}
		}

		let hasBeenStopped = false;
		let audioContextFailed = false;
		let reqAnimFrameCall:
			| {
					type: 'raf';
					id: number;
			  }
			| {
					type: 'timeout';
					id: Timer;
			  }
			| null = null;
		let playbackFrame = getCurrentFrame();
		let lastSeekRevision = seek?.revision.current ?? null;

		const cancelQueuedFrame = () => {
			if (reqAnimFrameCall !== null) {
				if (reqAnimFrameCall.type === 'raf') {
					cancelAnimationFrame(reqAnimFrameCall.id);
				} else {
					clearTimeout(reqAnimFrameCall.id);
				}
			}
		};

		const stop = () => {
			hasBeenStopped = true;
			cancelQueuedFrame();
		};

		const callback = () => {
			if (hasBeenStopped) {
				return;
			}

			if (!isPlaying()) {
				sharedAudioContext?.suspend?.();
				return;
			}

			if (!muted && !audioContextFailed && !isBuffering()) {
				sharedAudioContext?.resume?.();
			}

			const time = performance.now() - clock.startedTime;
			const actualLastFrame = outFrame ?? durationInFrames - 1;
			const actualFirstFrame = inFrame ?? 0;

			const seekRevision = seek?.revision.current ?? null;
			if (seekRevision !== lastSeekRevision) {
				playbackFrame = getCurrentFrame();
				lastSeekRevision = seekRevision;
			}

			// React may not have committed the previous playback update yet.
			const currentFrame = playbackFrame;
			const {nextFrame, framesToAdvance, hasEnded} = calculateNextFrame({
				time,
				currentFrame,
				playbackSpeed: playbackRate,
				fps,
				actualFirstFrame,
				actualLastFrame,
				framesAdvanced: clock.framesAdvanced,
				shouldLoop: loop,
			});

			clock.framesAdvanced += framesToAdvance;

			// A late callback can skip the endpoint. Hold that exact frame on end.
			const frameToDisplay =
				hasEnded && !moveToBeginningWhenEnded
					? playbackRate > 0
						? actualLastFrame
						: actualFirstFrame
					: nextFrame;

			if (frameToDisplay !== currentFrame && !isBuffering()) {
				playbackFrame = frameToDisplay;
				setFrameFromPlayback((c) => ({...c, [compositionId]: frameToDisplay}));
			}

			if (hasEnded) {
				stop();
				pause();
				emitter.dispatchEnded();
				return;
			}

			queueNextFrame();
		};

		const queueNextFrame = () => {
			if (hasBeenStopped) {
				return;
			}

			const getIsResumingAudioContext = audioContextFailed
				? null
				: (sharedAudioContext?.getIsResumingAudioContext?.() ?? null);
			if (getIsResumingAudioContext !== null && !muted) {
				getIsResumingAudioContext.then((result) => {
					if (hasBeenStopped) {
						return;
					}

					if (result === 'failed') {
						audioContextFailed = true;
						sharedAudioContext?.suspend();
						setPlayerMuted(true);
					}

					// The audio context is being re-established; start a fresh clock.
					clock.startedTime = performance.now();
					clock.framesAdvanced = 0;
					queueNextFrame();
				});

				return;
			}

			if (isBuffering()) {
				if (!muted && !audioContextFailed) {
					sharedAudioContext?.suspend?.();
				}

				const unsubscribe = subscribeBuffering((state) => {
					if (state.buffering) {
						return;
					}

					unsubscribe();
					if (
						!muted &&
						!audioContextFailed &&
						sharedAudioContext?._experimentalKeepAudioContextAlive
					) {
						sharedAudioContext.resume();
					}

					// Buffered media became reachable; start a fresh clock.
					clock.startedTime = performance.now();
					clock.framesAdvanced = 0;
					queueNextFrame();
				});

				return;
			}

			if (isBackgroundedRef.current) {
				reqAnimFrameCall = {
					type: 'timeout',
					// Note: Most likely, this will not be 1000 / fps, but the browser will throttle it to ~1/sec.
					id: setTimeout(callback, 1000 / fps),
				};
				return;
			}

			reqAnimFrameCall = {type: 'raf', id: requestAnimationFrame(callback)};
		};

		queueNextFrame();

		const onVisibilityChange = () => {
			if (document.visibilityState === 'visible') {
				return;
			}

			// If tab goes into the background, cancel requestAnimationFrame() and update immediately.
			// , so the transition to setTimeout() can be fulfilled.
			cancelQueuedFrame();
			callback();
		};

		window.addEventListener('visibilitychange', onVisibilityChange);

		return () => {
			window.removeEventListener('visibilitychange', onVisibilityChange);
			stop();
		};
	}, [
		compositionId,
		durationInFrames,
		fps,
		loop,
		pause,
		playing,
		setFrameFromPlayback,
		emitter,
		playbackRate,
		inFrame,
		outFrame,
		moveToBeginningWhenEnded,
		isBackgroundedRef,
		getCurrentFrame,
		isBuffering,
		isPlaying,
		sharedAudioContext,
		setPlayerMuted,
		subscribeBuffering,
		seek,
		logLevel,
		muted,
	]);

	useEffect(() => {
		const now = performance.now();
		const timeSinceLastUpdate = now - lastTimeUpdateTimestamp.current;

		if (timeSinceLastUpdate >= 250) {
			emitter.dispatchTimeUpdate({frame});
			lastTimeUpdateTimestamp.current = now;
			return;
		}

		const timeoutId = setTimeout(() => {
			emitter.dispatchTimeUpdate({frame});
			lastTimeUpdateTimestamp.current = performance.now();
		}, 250 - timeSinceLastUpdate);

		return () => clearTimeout(timeoutId);
	}, [emitter, frame]);

	useEffect(() => {
		emitter.dispatchFrameUpdate({frame});
	}, [emitter, frame]);
};