Skip to content

🎥 Make videos programmatically with React

7.1 KB
Raw Blame
import fs from 'fs';
import os from 'os';
import path from 'path';
import {RenderInternals} from '@remotion/renderer';
import execa from 'execa';
import {random} from 'remotion';
import sharp from 'sharp';

interface RgbColor {
	red: number;
	blue: number;
	green: number;
}
export const getMissedFramesForCodecs = async () => {
	const outputPath = await saveSequenceInTempDir(
		'video-testing-frame-accuracy',
	);
	const cases = [
		{codec: 'webm', type: 'normal', left: 0, top: 0},
		{codec: 'webm', type: 'offthread', left: 540, top: 0},
		{codec: 'mp4', type: 'normal', left: 0, top: 540},
		{codec: 'mp4', type: 'offthread', left: 540, top: 540},
	] as const;
	const missedFrames = {
		webm: {normal: 0, offthread: 0},
		mp4: {normal: 0, offthread: 0},
	};

	for (let frame = 0; frame < 100; frame++) {
		// each frame of the embedded video contains a (deterministically) random color which should appear correctly
		// in the rendered output
		const expectedColor = {
			red: selectColor('red', frame),
			green: selectColor('green', frame),
			blue: selectColor('blue', frame),
		};

		// extract the actual RGB color value of the top left pixel in the frame image that was generated by remotion
		const paddedIndex = String(frame).padStart(2, '0');
		const filename = path.join(outputPath, `element-${paddedIndex}.png`);
		const {data, info} = await sharp(filename)
			.raw()
			.toBuffer({resolveWithObject: true});

		for (const {codec, type, left, top} of cases) {
			const offset = (top * info.width + left) * info.channels;
			const actualColor = {
				red: data.readUInt8(offset),
				green: data.readUInt8(offset + 1),
				blue: data.readUInt8(offset + 2),
			};

			missedFrames[codec][type] = missedFrameChecker(
				expectedColor,
				actualColor,
				missedFrames[codec][type],
				frame,
				`${filename} (${codec} ${type})`,
			);
		}
	}
	RenderInternals.deleteDirectory(outputPath);

	return missedFrames;
};

export const getMissedFramesForMediaCodecs = async () => {
	const outputPath = await saveSequenceInTempDir(
		'video-testing-media-frame-accuracy',
	);
	const cases = [
		{codec: 'webm', left: 0},
		{codec: 'mp4', left: 540},
	] as const;
	const missedFrames = {webm: 0, mp4: 0};

	for (let frame = 0; frame < 100; frame++) {
		const expectedColor = {
			red: selectColor('red', frame),
			green: selectColor('green', frame),
			blue: selectColor('blue', frame),
		};
		const paddedIndex = String(frame).padStart(2, '0');
		const filename = path.join(outputPath, `element-${paddedIndex}.png`);
		const {data, info} = await sharp(filename)
			.raw()
			.toBuffer({resolveWithObject: true});

		for (const {codec, left} of cases) {
			const offset = left * info.channels;
			const actualColor = {
				red: data.readUInt8(offset),
				green: data.readUInt8(offset + 1),
				blue: data.readUInt8(offset + 2),
			};

			missedFrames[codec] = missedFrameChecker(
				expectedColor,
				actualColor,
				missedFrames[codec],
				frame,
				`${filename} (${codec})`,
			);
		}
	}
	RenderInternals.deleteDirectory(outputPath);

	return missedFrames;
};

export const getMissedFramesWithPlaybackrate = async (
	type: 'normal' | 'offthread' | 'codec',
) => {
	const outputPath = await saveSequenceInTempDir(
		`video-testing-playback${type === 'normal' ? '' : '-' + type}`,
	);

	let missedFrames = 0;

	//4x playbackrate
	for (let frame = 0; frame < 6; frame++) {
		//every 4th frame
		const expectedColor = {
			red: selectColor('red', frame * 4),
			green: selectColor('green', frame * 4),
			blue: selectColor('blue', frame * 4),
		};

		const paddedIndex = String(frame).padStart(2, '0');
		const filename = path.join(outputPath, `element-${paddedIndex}.png`);
		const img = await sharp(filename).raw().toBuffer();

		const actualColor = {
			red: img.readUInt8(0),
			green: img.readUInt8(1),
			blue: img.readUInt8(2),
		};

		missedFrames = missedFrameChecker(
			expectedColor,
			actualColor,
			missedFrames,
			frame,
			filename,
		);
	}
	RenderInternals.deleteDirectory(outputPath);

	return missedFrames;
};

export const getMissedFramesWithTrimApplied = async (
	type: 'normal' | 'offthread' | 'codec',
) => {
	const outputPath = await saveSequenceInTempDir(
		`video-testing-trim${type === 'normal' ? '' : '-' + type}`,
	);
	let missedFrames = 0;
	for (let frame = 20; frame < 22; frame++) {
		const expectedColor = {
			red: selectColor('red', frame),
			green: selectColor('green', frame),
			blue: selectColor('blue', frame),
		};

		const paddedIndex = String(frame - 20).padStart(2, '0');
		const filename = path.join(outputPath, `element-${paddedIndex}.png`);
		const img = await sharp(filename).raw().toBuffer();

		const actualColor = {
			red: img.readUInt8(0),
			green: img.readUInt8(1),
			blue: img.readUInt8(2),
		};
		missedFrames = missedFrameChecker(
			expectedColor,
			actualColor,
			missedFrames,
			frame,
			filename,
		);
	}
	RenderInternals.deleteDirectory(outputPath);

	return missedFrames;
};

export const getMissedFramesWithFractionalTrimApplied = async () => {
	const outputPath = await fs.promises.mkdtemp(
		path.join(os.tmpdir(), 'remotion-'),
	);
	const filename = path.join(outputPath, 'fractional-trim.png');

	await execa(
		'node_modules/.bin/remotion',
		[
			'still',
			'./build',
			'video-testing-mp4-offthread',
			filename,
			'--frame',
			'0',
			'--props',
			JSON.stringify({trimBefore: 20.75}),
		],
		{
			cwd: path.join(process.cwd(), '..', 'example'),
		},
	);

	const img = await sharp(filename).raw().toBuffer();
	const actualColor = {
		red: img.readUInt8(0),
		green: img.readUInt8(1),
		blue: img.readUInt8(2),
	};
	const expectedColor = {
		red: selectColor('red', 20),
		green: selectColor('green', 20),
		blue: selectColor('blue', 20),
	};

	const missedFrames = missedFrameChecker(
		expectedColor,
		actualColor,
		0,
		20,
		filename,
	);
	RenderInternals.deleteDirectory(outputPath);

	return missedFrames;
};

function selectColor(color: string, frame: number) {
	return Math.floor((random(`${color}-${frame}`) * 255) % 255);
}

async function saveSequenceInTempDir(id: string) {
	const outputPath = await fs.promises.mkdtemp(
		path.join(os.tmpdir(), 'remotion-'),
	);

	await execa(
		'bun',
		[
			'x',
			'remotion',
			'render',
			'./build',
			id,
			outputPath,
			'--image-format',
			'png',
			'--sequence',
			'--timeout',
			'60000',
			'--concurrency',
			'1',
		],
		{
			cwd: path.join(process.cwd(), '..', 'example'),
		},
	);
	return outputPath;
}

function missedFrameChecker(
	expectedColor: RgbColor,
	actualColor: RgbColor,
	missedFrames: number,
	frame: number,
	filename: string,
) {
	const colorDistance = {
		red: Math.abs(expectedColor.red - actualColor.red),
		green: Math.abs(expectedColor.green - actualColor.green),
		blue: Math.abs(expectedColor.blue - actualColor.blue),
	};

	// encoding sometimes shifts the color slightly - so measure the distance between the expected and actual
	// colors and consider any frame not within an acceptable range to be wrong
	const highestDistance = Math.max(
		colorDistance.red,
		colorDistance.blue,
		colorDistance.green,
	);
	const threshold = 35;
	if (highestDistance > threshold) {
		console.log(colorDistance, {threshold, frame, filename});
		return missedFrames + 1;
	}
	return missedFrames;
}