Skip to content

🎥 Make videos programmatically with React

11.2 KB
Raw Blame
import {ALL_FORMATS, Input, UrlSource} from 'mediabunny';
import type {ScheduleAudioNodeResult} from 'remotion';
import {expect, test} from 'vitest';
import {
	anchorToContinuousTime,
	audioIteratorManager,
	hasEnoughAudioToStartPlayback,
} from '../audio-iterator-manager';
import {makeNonceManager} from '../nonce-manager';

const prepare = async (options?: {
	src?: string;
	fps?: number;
	playbackRate?: number;
	localPlaybackRate?: number;
	mediaEndTimestamp?: number;
	sequenceEndTimestamp?: number;
	sequenceDurationInSeconds?: number;
	startTime?: number;
	loop?: boolean;
}) => {
	const {
		src = 'https://remotion.media/video.mp4',
		fps = 30,
		playbackRate = 1,
		localPlaybackRate = playbackRate,
		mediaEndTimestamp = Infinity,
		sequenceEndTimestamp = Infinity,
		sequenceDurationInSeconds = 10,
		startTime = 0,
		loop = false,
	} = options ?? {};
	const input = new Input({
		source: new UrlSource(src),
		formats: ALL_FORMATS,
	});
	const audioTrack = await input.getPrimaryAudioTrack();
	if (!audioTrack) {
		throw new Error('No audio track found');
	}

	const audioContext = new AudioContext();

	const audioContextCurrentTime = {
		current: 0,
	};

	const scheduleSharedAudioNode = (): ScheduleAudioNodeResult => {
		return {
			type: 'started',
			scheduledTime: 0,
		};
	};

	const unscheduleAudioNode = () => {};
	const audioSyncAnchor = {value: 0};
	let playbackUnblocks = 0;

	const manager = audioIteratorManager({
		audioTrack,
		delayPlaybackHandleIfNotPremounting: () => ({
			unblock: () => {
				playbackUnblocks++;
			},
			[Symbol.dispose]: () => {},
		}),
		sharedAudioContext: {
			audioContext,
			gainNode: audioContext.createGain(),
			audioSyncAnchor,
			scheduleAudioNode: scheduleSharedAudioNode,
			unscheduleAudioNode,
		},
		getMediaEndTimestamp: () => mediaEndTimestamp,
		getSequenceEndTimestamp: () => sequenceEndTimestamp,
		getSequenceDurationInSeconds: () => sequenceDurationInSeconds,
		getStartTime: () => startTime,
		initialMuted: false,
		initialVolume: 1,
		toneFrequency: 1,
		drawDebugOverlay: () => {},
		onError: (error) => {
			throw error;
		},
	});

	const scheduledChunks: number[] = [];
	const scheduledStartOffsets: number[] = [];
	const waiters: {count: number; resolve: () => void}[] = [];

	const scheduleAudioNode = (
		_node: AudioBufferSourceNode,
		mediaTimestamp: number,
		_originalUnloopedMediaTimestamp: number,
		sourceOffsetInSeconds: number,
		_sourceDurationInSeconds: number,
	): ScheduleAudioNodeResult => {
		scheduledChunks.push(mediaTimestamp);
		scheduledStartOffsets.push(sourceOffsetInSeconds);
		for (let i = waiters.length - 1; i >= 0; i--) {
			if (scheduledChunks.length >= waiters[i].count) {
				waiters[i].resolve();
				waiters.splice(i, 1);
			}
		}

		return {
			type: 'started',
			scheduledTime: mediaTimestamp,
		};
	};

	const seek = ({
		time,
		unloopedTime = time,
	}: {
		time: number;
		unloopedTime?: number;
	}) => {
		manager.seek({
			newTime: time,
			unloopedNewTime: unloopedTime,
			scheduleAudioNode,
			nonce: makeNonceManager().createAsyncOperation(),
			playbackRate,
			localPlaybackRate,
			getTargetTime: (mediaTimestamp: number) => mediaTimestamp,
			logLevel: 'info',
			loop,
			trimBefore: undefined,
			trimAfter: undefined,
			sequenceOffset: 0,
			sequenceDurationInFrames: 10,
			fps,
			getAudioContextCurrentTimeMockedInTest: () =>
				audioContextCurrentTime.current,
		});
	};

	return {
		manager,
		fps,
		scheduledChunks,
		scheduledStartOffsets,
		seek,
		audioContextCurrentTime,
		getPlaybackUnblocks: () => playbackUnblocks,
	};
};

test('anchor maps unlooped time using the local playback rate', () => {
	const anchor = {
		unloopedStartInSeconds: 10,
		mediaStartInSeconds: 4,
	};

	expect(
		anchorToContinuousTime({
			anchor,
			unloopedTimeInSeconds: 12,
			playbackRate: 0.5,
		}),
	).toBe(5);
	expect(
		anchorToContinuousTime({
			anchor,
			unloopedTimeInSeconds: 12,
			playbackRate: 2,
		}),
	).toBe(8);
});

test('audio startup buffering is based on duration instead of chunk count', () => {
	expect(hasEnoughAudioToStartPlayback(0.099)).toBe(false);
	expect(hasEnoughAudioToStartPlayback(0.1)).toBe(true);
	expect(hasEnoughAudioToStartPlayback(0.5)).toBe(true);
});

test('PCM chunks unblock playback based on duration before six chunks or EOF', async () => {
	const {manager, seek, scheduledChunks, getPlaybackUnblocks} = await prepare({
		src: '/junk.wav',
	});

	seek({time: 0});
	await manager.waitForNScheduledNodes(3);

	expect(scheduledChunks).toEqual([
		0, 0.046439909297052155, 0.09287981859410431,
	]);
	expect(getPlaybackUnblocks()).toBe(1);
	manager.destroyIterator();
});

// https://github.com/remotion-dev/remotion/issues/10658
test('silence before the next loop pass counts toward startup buffering', async () => {
	const {
		manager,
		seek,
		scheduledChunks,
		audioContextCurrentTime,
		getPlaybackUnblocks,
	} = await prepare({
		src: 'https://remotion.media/audio-shorter-than-video.mp4',
		mediaEndTimestamp: 10,
		sequenceDurationInSeconds: 12,
		loop: true,
	});

	audioContextCurrentTime.current = 8;
	seek({time: 8});
	await manager.waitForNScheduledNodes(1);

	expect(scheduledChunks[0]).toBeCloseTo(10);
	expect(getPlaybackUnblocks()).toBe(1);

	seek({time: 8 + 1 / 30});
	expect(manager.getAudioIteratorsCreated()).toBe(1);
	manager.destroyIterator();
});

test('media player should work', async () => {
	const {manager, scheduledChunks, seek, audioContextCurrentTime} =
		await prepare();

	audioContextCurrentTime.current = 9.96;
	seek({
		time: 9.96,
	});

	await manager.waitForNScheduledNodes(3);
	expect(scheduledChunks).toEqual([9.96, 9.962666666666667, 9.984]);

	scheduledChunks.length = 0;
	seek({
		time: 0,
	});

	await manager.waitForNScheduledNodes(3);

	expect(scheduledChunks).toEqual([
		0, 0.021333333333333333, 0.042666666666666665,
	]);
	scheduledChunks.length = 0;

	seek({
		time: 0.04,
	});

	await manager.waitForNScheduledNodes(2);

	manager.destroyIterator();

	expect(scheduledChunks).toEqual([0.064, 0.08533333333333333]);

	const created = manager.getAudioIteratorsCreated();
	expect(created).toBe(2);
	manager.destroyIterator();
});

test('should not create too many iterators when the audio ends', async () => {
	const {manager, seek, scheduledChunks, audioContextCurrentTime} =
		await prepare();

	audioContextCurrentTime.current = 9.97;
	seek({
		time: 9.97,
	});
	await manager.waitForNScheduledNodes(0);

	seek({
		time: 9.98,
	});
	await manager.waitForNScheduledNodes(2);

	const created = manager.getAudioIteratorsCreated();
	expect(created).toBe(1);

	expect(scheduledChunks).toEqual([9.97, 9.984]);
});

test('should not create too many iterators when the audio ends (variant)', async () => {
	const {manager, seek, scheduledChunks, audioContextCurrentTime} =
		await prepare();

	audioContextCurrentTime.current = 9.97;
	seek({
		time: 9.97,
	});
	await manager.waitForNScheduledNodes(2);

	seek({
		time: 9.98,
	});
	await manager.waitForNScheduledNodes(0);

	const created = manager.getAudioIteratorsCreated();
	expect(created).toBe(1);

	expect(scheduledChunks).toEqual([9.97, 9.984]);
});

test('should create more iterators when seeking ', async () => {
	const {manager, seek, scheduledChunks, audioContextCurrentTime} =
		await prepare();

	seek({
		time: 0,
	});
	await manager.waitForNScheduledNodes(6);
	expect(scheduledChunks).toEqual([
		0, 0.021333333333333333, 0.042666666666666665, 0.064, 0.08533333333333333,
		0.10666666666666667,
	]);
	scheduledChunks.length = 0;
	audioContextCurrentTime.current = 2;
	seek({
		time: 2,
	});
	await manager.waitForNScheduledNodes(6);

	const created = manager.getAudioIteratorsCreated();
	expect(created).toBe(2);

	expect(scheduledChunks).toEqual([
		2, 2.005333333333333, 2.026666666666667, 2.048, 2.0693333333333332,
		2.0906666666666665,
	]);
});

// https://github.com/remotion-dev/remotion/issues/5872#issuecomment-3541004403
test('should not schedule duplicate chunks with playbackRate=0.5', async () => {
	const input = new Input({
		source: new UrlSource('https://remotion.media/video.mp4'),
		formats: ALL_FORMATS,
	});
	const audioTrack = await input.getPrimaryAudioTrack();
	if (!audioTrack) {
		throw new Error('No audio track found');
	}

	const {seek, manager, scheduledChunks} = await prepare({
		fps: 25,
		playbackRate: 0.5,
	});

	const fps = 25;
	const playbackRate = 0.5;

	// Simulate sequential seeks like real playback over many frames
	// At 25fps with playbackRate=0.5, media time advances 20ms per frame
	// But AAC audio chunks are 21.33ms, so we keep re-encountering them

	// Simulate 30 frames of playback
	// (about 1.2 seconds real time, 0.6s media time)
	for (let frame = 0; frame < 30; frame++) {
		const mediaTime = frame * (1 / fps) * playbackRate;

		seek({
			time: mediaTime,
		});
		await manager.waitForNScheduledNodes(1);
	}

	const uniqueChunks = [...new Set(scheduledChunks)];
	expect(uniqueChunks.length).toEqual(30);
	expect(scheduledChunks.length).toBe(uniqueChunks.length);
});

test('looping keeps continuous timestamps and reuses the iterator after a wrapped seek', async () => {
	const {manager, seek, scheduledChunks, scheduledStartOffsets} = await prepare(
		{
			playbackRate: 1,
			localPlaybackRate: 0.5,
			mediaEndTimestamp: 0.1,
			sequenceDurationInSeconds: 1,
			loop: true,
		},
	);

	seek({time: 0.08, unloopedTime: 0.08});
	await manager.waitForNScheduledNodes(8);

	expect(scheduledChunks.length).toBeGreaterThanOrEqual(8);
	for (let i = 1; i < scheduledChunks.length; i++) {
		expect(scheduledChunks[i]).toBeGreaterThan(scheduledChunks[i - 1]);
	}

	expect(scheduledChunks.some((timestamp) => timestamp >= 0.1)).toBe(true);
	expect(scheduledStartOffsets.some((offset) => offset > 0)).toBe(true);

	// The wrapped media time differs, but the unlooped time maps into the
	// already queued continuous period using localPlaybackRate.
	seek({time: 0, unloopedTime: 0.12});
	expect(manager.getAudioIteratorsCreated()).toBe(1);

	// Seeking by exactly one loop duration produces the same wrapped media time
	// and must not recreate the iterator either.
	seek({time: 0, unloopedTime: 0.22});
	expect(manager.getAudioIteratorsCreated()).toBe(1);

	manager.destroyIterator();
	expect(manager.getCurrentAnchor()).toBe(null);
});

test('should not decode + schedule audio chunks beyond the end time', async () => {
	const endTime = 0.5;
	const fps = 30;

	const {seek, manager, scheduledChunks, audioContextCurrentTime} =
		await prepare({
			fps,
			mediaEndTimestamp: endTime,
			sequenceEndTimestamp: endTime,
		});

	const prom = manager.waitForNScheduledNodes(23);

	// Simulate playback frame by frame, seeking past the end time
	for (let frame = 0; frame < 30; frame++) {
		const mediaTime = frame / fps;
		audioContextCurrentTime.current = mediaTime;
		seek({time: mediaTime});
	}

	await prom;

	for (const timestamp of scheduledChunks) {
		expect(timestamp).toBeLessThanOrEqual(endTime);
	}

	expect(scheduledChunks.length).toBe(23);
});

test('seek to the same time after destroyIterator() starts a new iterator', async () => {
	const {seek, manager} = await prepare();

	seek({time: 2});
	expect(manager.getAudioIteratorsCreated()).toBe(1);
	expect(manager.getAudioBufferIterator()).not.toBe(null);

	manager.destroyIterator();
	expect(manager.getAudioBufferIterator()).toBe(null);

	seek({time: 2});
	expect(manager.getAudioIteratorsCreated()).toBe(2);
	expect(manager.getAudioBufferIterator()).not.toBe(null);
});