Skip to content

🎥 Make videos programmatically with React

18.3 KB
Raw Blame
import {
	AudioBufferSink,
	InputDisposedError,
	type InputAudioTrack,
} from 'mediabunny';
import type {LogLevel} from 'remotion';
import {Internals, type ScheduleAudioNodeResult} from 'remotion';
import {
	isAlreadyQueued,
	makeAudioIterator,
	type AudioIterator,
	type QueuedPeriod,
} from './audio/audio-preview-iterator';
import {getScheduledTime} from './audio/get-scheduled-time';
import {pitchShiftAudioIterator} from './audio/pitch-shift';
import {
	processNext,
	StaleWaiterError,
	waitForTurn,
} from './audio/sort-by-priority';
import type {DelayPlaybackIfNotPremounting} from './delay-playback-if-not-premounting';
import {
	makeIteratorWithPriming,
	makeLoopingIterator,
	type AudioBufferSlice,
} from './make-iterator-with-priming';
import type {Nonce} from './nonce-manager';
import type {SharedAudioContextForMediaPlayer} from './shared-audio-context-for-media-player';

type ScheduleAudioNode = (
	node: AudioBufferSourceNode,
	mediaTimestamp: number,
	originalUnloopedMediaTimestamp: number,
	sourceOffsetInSeconds: number,
	sourceDurationInSeconds: number,
) => ScheduleAudioNodeResult;

export const MINIMUM_AUDIO_BUFFERING_TIME_SECONDS = 0.1;

export const hasEnoughAudioToStartPlayback = (bufferedDuration: number) => {
	return bufferedDuration >= MINIMUM_AUDIO_BUFFERING_TIME_SECONDS;
};

export type AudioIteratorAnchor = {
	// The unlooped time in seconds at which the current iterator was started
	unloopedStartInSeconds: number;
	// The media timestamp emitted by the current iterator at that unlooped time
	mediaStartInSeconds: number;
};

// Convert an unlooped composition time into the iterator's continuous media
// timeline using the anchor established when the iterator was started. The
// looping iterator emits timestamps that continue monotonically across loop
// iterations, so both the scheduler (getTargetTime) and the seek dedup must
// map times into this same frame via the identical formula.
export const anchorToContinuousTime = ({
	anchor,
	unloopedTimeInSeconds,
	playbackRate,
}: {
	anchor: AudioIteratorAnchor;
	unloopedTimeInSeconds: number;
	playbackRate: number;
}): number => {
	return (
		anchor.mediaStartInSeconds +
		(unloopedTimeInSeconds - anchor.unloopedStartInSeconds) * playbackRate
	);
};

export const audioIteratorManager = ({
	audioTrack,
	delayPlaybackHandleIfNotPremounting,
	sharedAudioContext,
	getSequenceEndTimestamp,
	getSequenceDurationInSeconds,
	getMediaEndTimestamp,
	getStartTime,
	initialMuted,
	initialVolume,
	toneFrequency,
	drawDebugOverlay,
	onError,
}: {
	audioTrack: InputAudioTrack;
	delayPlaybackHandleIfNotPremounting: () => DelayPlaybackIfNotPremounting;
	sharedAudioContext: SharedAudioContextForMediaPlayer;
	getSequenceEndTimestamp: () => number;
	getSequenceDurationInSeconds: () => number;
	getMediaEndTimestamp: () => number;
	getStartTime: () => number;
	initialMuted: boolean;
	initialVolume: number;
	toneFrequency: number;
	drawDebugOverlay: () => void;
	onError: (error: Error) => void;
}) => {
	let muted = initialMuted;
	let currentVolume = Math.max(0, initialVolume);
	let currentToneFrequency = toneFrequency;
	let currentSeek: {
		time: number;
		playbackRate: number;
		trimBefore: number | undefined;
		trimAfter: number | undefined;
		sequenceOffset: number;
		sequenceDurationInFrames: number;
		loop: boolean;
		fps: number;
	} | null = null;

	const gainNode = sharedAudioContext.audioContext.createGain();
	gainNode.gain.value = muted ? 0 : currentVolume;
	gainNode.connect(sharedAudioContext.gainNode);

	const audioSink = new AudioBufferSink(audioTrack);
	let audioBufferIterator: AudioIterator | null = null;
	// When looping, the iterator emits timestamps that continue monotonically
	// across loop iterations instead of wrapping. This anchor maps the unlooped
	// time to that continuous timeline so the scheduler can align chunks.
	let currentAnchor: AudioIteratorAnchor | null = null;
	let audioIteratorsCreated = 0;
	let totalAudioScheduledInSeconds = 0;
	let currentDelayHandle: {unblock: () => void} | null = null;

	const unblockCurrentDelayHandle = () => {
		if (currentDelayHandle) {
			currentDelayHandle.unblock();
			currentDelayHandle = null;
		}
	};

	const pendingScheduleWaiters: {
		remaining: number;
		resolve: () => void;
	}[] = [];

	const notifyNodeScheduled = () => {
		for (let i = pendingScheduleWaiters.length - 1; i >= 0; i--) {
			const waiter = pendingScheduleWaiters[i];
			waiter.remaining--;
			if (waiter.remaining <= 0) {
				waiter.resolve();
				pendingScheduleWaiters.splice(i, 1);
			}
		}
	};

	const waitForNScheduledNodes = (n: number) => {
		if (n <= 0) {
			return Promise.resolve();
		}

		return new Promise<void>((resolve) => {
			pendingScheduleWaiters.push({remaining: n, resolve});
		});
	};

	const scheduleAudioChunk = ({
		buffer,
		mediaTimestamp,
		originalUnloopedMediaTimestamp,
		sourceOffsetInSeconds,
		sourceDurationInSeconds,
		playbackRate,
		scheduleAudioNode,
		logLevel,
	}: {
		buffer: AudioBuffer;
		mediaTimestamp: number;
		playbackRate: number;
		scheduleAudioNode: ScheduleAudioNode;
		logLevel: LogLevel;
		originalUnloopedMediaTimestamp: number;
		sourceOffsetInSeconds: number;
		sourceDurationInSeconds: number;
	}) => {
		if (!audioBufferIterator) {
			throw new Error('Audio buffer iterator not found');
		}

		if (muted) {
			return;
		}

		const node = sharedAudioContext.audioContext.createBufferSource();
		node.buffer = buffer;
		node.playbackRate.value = playbackRate;
		node.connect(gainNode);

		const started = scheduleAudioNode(
			node,
			mediaTimestamp,
			originalUnloopedMediaTimestamp,
			sourceOffsetInSeconds,
			sourceDurationInSeconds,
		);

		if (started.type === 'not-started') {
			Internals.Log.verbose(
				{logLevel, tag: 'audio-scheduling'},
				'not started, disconnected: %s %s',
				mediaTimestamp.toFixed(3),
				buffer.duration.toFixed(3),
			);

			node.disconnect();
			return;
		}

		audioBufferIterator.addQueuedAudioNode({
			node,
			timestamp: mediaTimestamp,
			sourceDurationInSeconds,
		});
	};

	const onAudioChunk = ({
		buffer,
		playbackRate,
		scheduleAudioNode,
		logLevel,
	}: {
		buffer: AudioBufferSlice;
		playbackRate: number;
		scheduleAudioNode: ScheduleAudioNode;
		logLevel: LogLevel;
	}) => {
		if (muted) {
			return;
		}

		const startTime = getStartTime();
		const sequenceEndTime = getSequenceEndTimestamp();

		// Skip chunks entirely outside the range
		if (
			buffer.timelineTimestamp + buffer.sourceDurationInSeconds <=
			startTime
		) {
			return;
		}

		if (buffer.timelineTimestamp >= sequenceEndTime) {
			return;
		}

		const scheduledStart = Math.max(buffer.timelineTimestamp, startTime);
		const scheduledEnd = Math.min(
			buffer.timelineTimestamp + buffer.sourceDurationInSeconds,
			sequenceEndTime,
		);
		totalAudioScheduledInSeconds += Math.max(0, scheduledEnd - scheduledStart);

		scheduleAudioChunk({
			buffer: buffer.buffer.buffer,
			mediaTimestamp: buffer.timelineTimestamp,
			playbackRate,
			scheduleAudioNode,
			logLevel,
			originalUnloopedMediaTimestamp: buffer.buffer.timestamp,
			sourceOffsetInSeconds: buffer.sourceOffsetInSeconds,
			sourceDurationInSeconds: Math.min(
				buffer.sourceDurationInSeconds,
				sequenceEndTime - buffer.timelineTimestamp,
			),
		});

		drawDebugOverlay();
	};

	const proceedScheduling = ({
		iterator,
		nonce,
		getTargetTime,
		playbackRate,
		scheduleAudioNode,
		onScheduled,
		onDestroyed,
		onDone,
		logLevel,
		currentTime,
		getAudioContextCurrentTimeMockedInTest,
	}: {
		iterator: AudioIterator;
		nonce: Nonce;
		getTargetTime: (
			mediaTimestamp: number,
			currentTime: number,
		) => number | null;
		playbackRate: number;
		scheduleAudioNode: ScheduleAudioNode;
		onScheduled: (
			sourceDurationInSeconds: number,
			timelineTimestamp: number,
		) => void;
		onDone: () => void;
		onDestroyed: () => void;
		logLevel: LogLevel;
		currentTime: number;
		getAudioContextCurrentTimeMockedInTest: () => number;
	}) => {
		waitForTurn({
			getPriority: () => {
				if (iterator.isDestroyed()) {
					onDestroyed();
					return null;
				}

				const guessedNextTimestamp = iterator.guessNextTimestamp();
				const targetTime = getTargetTime(guessedNextTimestamp, currentTime);
				if (targetTime === null) {
					// Time will not be mounted
					return null;
				}

				const scheduledTime = getScheduledTime({
					mediaTimestamp: guessedNextTimestamp,
					targetTime,
					currentTime,
					sequenceStartTime: getStartTime(),
				});

				return scheduledTime - getAudioContextCurrentTimeMockedInTest();
			},
			fn: () => iterator.getNextFn(),
			onDone: (result, next) => {
				if (iterator.isDestroyed()) {
					next();
					onDestroyed();
					return;
				}

				// We schedule even if nonce.isStale(), because the iterator is still alive and the seek did not destroy the
				// iterator. So the seek was non-destructive, and the schedule valid. The iterator already progressed, we cannot get it again.

				if (!result.value) {
					// media ended
					next();
					onDone();
					return;
				}

				onScheduled(
					result.value.sourceDurationInSeconds,
					result.value.timelineTimestamp,
				);
				notifyNodeScheduled();

				onAudioChunk({
					buffer: result.value,
					playbackRate,
					scheduleAudioNode,
					logLevel,
				});
				proceedScheduling({
					iterator,
					nonce,
					getTargetTime,
					playbackRate,
					scheduleAudioNode,
					onScheduled,
					onDestroyed,
					onDone,
					logLevel,
					currentTime,
					getAudioContextCurrentTimeMockedInTest,
				});
				next();
			},
			onError: (e) => {
				if (e instanceof InputDisposedError) {
					// iterator was disposed by a newer startAudioIterator call
					// this is expected during rapid seeking
					onDestroyed();
					return;
				}

				if (e instanceof StaleWaiterError) {
					onDestroyed();
					// iterator was stale before it got its turn
					return;
				}

				// A non-destructive seek can stale the nonce while retaining this
				// iterator. Only ignore errors when the iterator was replaced.
				if (iterator.isDestroyed()) {
					onDestroyed();
					return;
				}

				iterator.destroy();
				onDestroyed();
				onError(e as Error);
			},
		});
	};

	const startAudioIterator = ({
		nonce,
		playbackRate,
		startFromSecond,
		unloopedStartFromSecond,
		scheduleAudioNode,
		getTargetTime,
		logLevel,
		loop,
		unscheduleAudioNode,
		getAudioContextCurrentTimeMockedInTest,
	}: {
		startFromSecond: number;
		unloopedStartFromSecond: number;
		nonce: Nonce;
		playbackRate: number;
		scheduleAudioNode: ScheduleAudioNode;
		getTargetTime: (
			mediaTimestamp: number,
			currentTime: number,
		) => number | null;
		logLevel: LogLevel;
		loop: boolean;
		unscheduleAudioNode: (node: AudioBufferSourceNode) => boolean | void;
		getAudioContextCurrentTimeMockedInTest: () => number;
	}) => {
		// Audio iterators only schedule forward. Studio's reverse playback must
		// stay silent instead of producing negative AudioBufferSourceNode offsets.
		if (muted || playbackRate <= 0) {
			return;
		}

		const maximumTimestamp = getMediaEndTimestamp();
		if (startFromSecond >= maximumTimestamp) {
			return;
		}

		audioBufferIterator?.destroy();
		unblockCurrentDelayHandle();

		const delayHandle = delayPlaybackHandleIfNotPremounting();
		currentDelayHandle = delayHandle;

		currentAnchor = {
			unloopedStartInSeconds: unloopedStartFromSecond,
			mediaStartInSeconds: startFromSecond,
		};

		const maximumContinuousTimestamp =
			startFromSecond + getSequenceDurationInSeconds() * playbackRate;
		const unshiftedSource = loop
			? makeLoopingIterator({
					audioSink,
					seekTimeInSeconds: startFromSecond,
					loopStartInSeconds: getStartTime(),
					segmentEndInSeconds: maximumTimestamp,
					maximumContinuousTimestamp,
				})
			: makeIteratorWithPriming({
					audioSink,
					timeToSeek: startFromSecond,
					maximumTimestamp,
				});
		const source = pitchShiftAudioIterator({
			iterator: unshiftedSource,
			toneFrequency: currentToneFrequency,
		});
		const iterator = makeAudioIterator({
			startFromSecond,
			iterator: source,
			unscheduleAudioNode,
		});
		audioIteratorsCreated++;
		audioBufferIterator = iterator;

		let bufferedUntil = startFromSecond;
		let hasUnblockedPlayback = false;
		const unblockPlayback = () => {
			if (hasUnblockedPlayback) {
				return;
			}

			hasUnblockedPlayback = true;
			delayHandle.unblock();
		};

		proceedScheduling({
			iterator,
			nonce,
			getTargetTime,
			playbackRate,
			scheduleAudioNode,
			onScheduled: (sourceDurationInSeconds, timelineTimestamp) => {
				bufferedUntil = Math.max(
					bufferedUntil,
					timelineTimestamp + sourceDurationInSeconds,
				);
				const bufferedDuration = bufferedUntil - startFromSecond;
				// Need to schedule a bit into the future to unblock the buffer state,
				// otherwise we might be scheduling too late. This must be based on
				// timeline coverage, not audible duration or chunk count: silence is
				// already safe to play, and large PCM chunks can exceed the scheduling
				// horizon before enough chunks are queued:
				// https://github.com/remotion-dev/remotion/issues/9394
				if (hasEnoughAudioToStartPlayback(bufferedDuration)) {
					unblockPlayback();
				}
			},
			onDestroyed: () => {
				unblockPlayback();
			},
			onDone: () => {
				unblockPlayback();
			},
			logLevel,
			currentTime: sharedAudioContext.audioContext.currentTime,
			getAudioContextCurrentTimeMockedInTest,
		});
	};

	const seek = ({
		newTime,
		unloopedNewTime,
		nonce,
		playbackRate,
		localPlaybackRate,
		scheduleAudioNode,
		getTargetTime,
		logLevel,
		loop,
		trimBefore,
		trimAfter,
		sequenceOffset,
		sequenceDurationInFrames,
		fps,
		getAudioContextCurrentTimeMockedInTest,
	}: {
		newTime: number;
		unloopedNewTime: number;
		nonce: Nonce;
		playbackRate: number;
		localPlaybackRate: number;
		scheduleAudioNode: ScheduleAudioNode;
		getTargetTime: (
			mediaTimestamp: number,
			currentTime: number,
		) => number | null;
		logLevel: LogLevel;
		loop: boolean;
		trimBefore: number | undefined;
		trimAfter: number | undefined;
		sequenceOffset: number;
		sequenceDurationInFrames: number;
		fps: number;
		getAudioContextCurrentTimeMockedInTest: () => number;
	}) => {
		if (nonce.isStale()) {
			return;
		}

		if (
			currentSeek !== null &&
			currentSeek.time === newTime &&
			currentSeek.playbackRate === playbackRate &&
			currentSeek.trimBefore === trimBefore &&
			currentSeek.trimAfter === trimAfter &&
			currentSeek.sequenceOffset === sequenceOffset &&
			currentSeek.sequenceDurationInFrames === sequenceDurationInFrames &&
			currentSeek.loop === loop &&
			currentSeek.fps === fps &&
			(muted || (audioBufferIterator && !audioBufferIterator.isDestroyed()))
		) {
			return;
		}

		currentSeek = {
			time: newTime,
			playbackRate,
			trimBefore,
			trimAfter,
			sequenceOffset,
			sequenceDurationInFrames,
			loop,
			fps,
		};

		if (muted) {
			return;
		}

		if (audioBufferIterator && !audioBufferIterator.isDestroyed()) {
			// When looping, queued nodes carry continuous timestamps that don't
			// wrap at loop boundaries. Map the new time into that continuous
			// frame so it can be compared against the queued period.
			const timeToCheck =
				loop && currentAnchor
					? anchorToContinuousTime({
							anchor: currentAnchor,
							unloopedTimeInSeconds: unloopedNewTime,
							playbackRate: localPlaybackRate,
						})
					: newTime;
			const queuedPeriod = audioBufferIterator.getQueuedPeriod();
			// Queued timestamps are in media seconds, so convert the Player's
			// AudioContext tolerance using the combined media and Player rate.
			const anchorTolerance =
				Internals.getAudioSyncAnchorTolerance(sharedAudioContext.audioContext) *
				Math.abs(playbackRate);
			// If there is a missing period, but we'd have no chance to schedule nodes,
			// then let's not bother. Let's just leave the gap.
			const queuedPeriodMinusLatency: QueuedPeriod | null = queuedPeriod
				? {
						from: queuedPeriod.from - anchorTolerance,
						until: queuedPeriod.until,
					}
				: null;
			const currentTimeIsAlreadyQueued = isAlreadyQueued(
				timeToCheck,
				queuedPeriodMinusLatency,
			);
			if (currentTimeIsAlreadyQueued) {
				processNext();
				// current time is scheduled, will keep scheduling
				return;
			}

			const currentIteratorTimestamp = audioBufferIterator.guessNextTimestamp();
			const iteratorHasAdvancedThroughSilence =
				loop &&
				currentAnchor !== null &&
				unloopedNewTime >= currentAnchor.unloopedStartInSeconds &&
				currentIteratorTimestamp >= timeToCheck;
			if (
				iteratorHasAdvancedThroughSilence ||
				(currentIteratorTimestamp < timeToCheck &&
					Math.abs(currentIteratorTimestamp - timeToCheck) < 1)
			) {
				processNext();
				// The iterator has either advanced beyond the current time, meaning
				// the gap is known silence, or is less than 1 second behind. Let it run.
				return;
			}
		}

		startAudioIterator({
			nonce,
			playbackRate,
			startFromSecond: newTime,
			unloopedStartFromSecond: unloopedNewTime,
			scheduleAudioNode,
			getTargetTime,
			logLevel,
			loop,
			unscheduleAudioNode: sharedAudioContext.unscheduleAudioNode,
			getAudioContextCurrentTimeMockedInTest,
		});

		// Not further scheduling, initial iterator is already running
	};

	return {
		startAudioIterator,
		getAudioBufferIterator: () => audioBufferIterator,
		getCurrentAnchor: () => currentAnchor,
		destroyIterator: () => {
			audioBufferIterator?.destroy();
			audioBufferIterator = null;
			// Drop the anchor together with the iterator it described, so
			// getCurrentAnchor() cannot hand out a stale mapping (from a previous
			// rate/trim) during the window before a new iterator is started.
			currentAnchor = null;
			currentSeek = null;
			unblockCurrentDelayHandle();
		},
		seek,
		getAudioIteratorsCreated: () => audioIteratorsCreated,
		getTotalAudioScheduledInSeconds: () => totalAudioScheduledInSeconds,
		setMuted: (newMuted: boolean) => {
			if (muted === newMuted) {
				return;
			}

			muted = newMuted;
			gainNode.gain.value = muted ? 0 : currentVolume;
			if (muted) {
				audioBufferIterator?.destroy();
				audioBufferIterator = null;
				currentAnchor = null;
				unblockCurrentDelayHandle();
			}

			// An unmute at the same frame still needs a fresh iterator.
			currentSeek = null;
		},
		setVolume: (volume: number) => {
			currentVolume = Math.max(0, volume);
			gainNode.gain.value = muted ? 0 : currentVolume;
		},
		setToneFrequency: (newToneFrequency: number) => {
			currentToneFrequency = newToneFrequency;
		},
		scheduleAudioChunk,
		waitForNScheduledNodes,
	};
};

export type AudioIteratorManager = ReturnType<typeof audioIteratorManager>;