🎥 Make videos programmatically with React
import fs from 'fs';
import os from 'os';
import path from 'path';
import {RenderInternals} from '@remotion/renderer';
import execa from 'execa';
import {random} from 'remotion';
import sharp from 'sharp';
interface RgbColor {
red: number;
blue: number;
green: number;
}
export const getMissedFramesForCodecs = async () => {
const outputPath = await saveSequenceInTempDir(
'video-testing-frame-accuracy',
);
const cases = [
{codec: 'webm', type: 'normal', left: 0, top: 0},
{codec: 'webm', type: 'offthread', left: 540, top: 0},
{codec: 'mp4', type: 'normal', left: 0, top: 540},
{codec: 'mp4', type: 'offthread', left: 540, top: 540},
] as const;
const missedFrames = {
webm: {normal: 0, offthread: 0},
mp4: {normal: 0, offthread: 0},
};
for (let frame = 0; frame < 100; frame++) {
// each frame of the embedded video contains a (deterministically) random color which should appear correctly
// in the rendered output
const expectedColor = {
red: selectColor('red', frame),
green: selectColor('green', frame),
blue: selectColor('blue', frame),
};
// extract the actual RGB color value of the top left pixel in the frame image that was generated by remotion
const paddedIndex = String(frame).padStart(2, '0');
const filename = path.join(outputPath, `element-${paddedIndex}.png`);
const {data, info} = await sharp(filename)
.raw()
.toBuffer({resolveWithObject: true});
for (const {codec, type, left, top} of cases) {
const offset = (top * info.width + left) * info.channels;
const actualColor = {
red: data.readUInt8(offset),
green: data.readUInt8(offset + 1),
blue: data.readUInt8(offset + 2),
};
missedFrames[codec][type] = missedFrameChecker(
expectedColor,
actualColor,
missedFrames[codec][type],
frame,
`${filename} (${codec} ${type})`,
);
}
}
RenderInternals.deleteDirectory(outputPath);
return missedFrames;
};
export const getMissedFramesForMediaCodecs = async () => {
const outputPath = await saveSequenceInTempDir(
'video-testing-media-frame-accuracy',
);
const cases = [
{codec: 'webm', left: 0},
{codec: 'mp4', left: 540},
] as const;
const missedFrames = {webm: 0, mp4: 0};
for (let frame = 0; frame < 100; frame++) {
const expectedColor = {
red: selectColor('red', frame),
green: selectColor('green', frame),
blue: selectColor('blue', frame),
};
const paddedIndex = String(frame).padStart(2, '0');
const filename = path.join(outputPath, `element-${paddedIndex}.png`);
const {data, info} = await sharp(filename)
.raw()
.toBuffer({resolveWithObject: true});
for (const {codec, left} of cases) {
const offset = left * info.channels;
const actualColor = {
red: data.readUInt8(offset),
green: data.readUInt8(offset + 1),
blue: data.readUInt8(offset + 2),
};
missedFrames[codec] = missedFrameChecker(
expectedColor,
actualColor,
missedFrames[codec],
frame,
`${filename} (${codec})`,
);
}
}
RenderInternals.deleteDirectory(outputPath);
return missedFrames;
};
export const getMissedFramesWithPlaybackrate = async (
type: 'normal' | 'offthread' | 'codec',
) => {
const outputPath = await saveSequenceInTempDir(
`video-testing-playback${type === 'normal' ? '' : '-' + type}`,
);
let missedFrames = 0;
//4x playbackrate
for (let frame = 0; frame < 6; frame++) {
//every 4th frame
const expectedColor = {
red: selectColor('red', frame * 4),
green: selectColor('green', frame * 4),
blue: selectColor('blue', frame * 4),
};
const paddedIndex = String(frame).padStart(2, '0');
const filename = path.join(outputPath, `element-${paddedIndex}.png`);
const img = await sharp(filename).raw().toBuffer();
const actualColor = {
red: img.readUInt8(0),
green: img.readUInt8(1),
blue: img.readUInt8(2),
};
missedFrames = missedFrameChecker(
expectedColor,
actualColor,
missedFrames,
frame,
filename,
);
}
RenderInternals.deleteDirectory(outputPath);
return missedFrames;
};
export const getMissedFramesWithTrimApplied = async (
type: 'normal' | 'offthread' | 'codec',
) => {
const outputPath = await saveSequenceInTempDir(
`video-testing-trim${type === 'normal' ? '' : '-' + type}`,
);
let missedFrames = 0;
for (let frame = 20; frame < 22; frame++) {
const expectedColor = {
red: selectColor('red', frame),
green: selectColor('green', frame),
blue: selectColor('blue', frame),
};
const paddedIndex = String(frame - 20).padStart(2, '0');
const filename = path.join(outputPath, `element-${paddedIndex}.png`);
const img = await sharp(filename).raw().toBuffer();
const actualColor = {
red: img.readUInt8(0),
green: img.readUInt8(1),
blue: img.readUInt8(2),
};
missedFrames = missedFrameChecker(
expectedColor,
actualColor,
missedFrames,
frame,
filename,
);
}
RenderInternals.deleteDirectory(outputPath);
return missedFrames;
};
export const getMissedFramesWithFractionalTrimApplied = async () => {
const outputPath = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'remotion-'),
);
const filename = path.join(outputPath, 'fractional-trim.png');
await execa(
'node_modules/.bin/remotion',
[
'still',
'./build',
'video-testing-mp4-offthread',
filename,
'--frame',
'0',
'--props',
JSON.stringify({trimBefore: 20.75}),
],
{
cwd: path.join(process.cwd(), '..', 'example'),
},
);
const img = await sharp(filename).raw().toBuffer();
const actualColor = {
red: img.readUInt8(0),
green: img.readUInt8(1),
blue: img.readUInt8(2),
};
const expectedColor = {
red: selectColor('red', 20),
green: selectColor('green', 20),
blue: selectColor('blue', 20),
};
const missedFrames = missedFrameChecker(
expectedColor,
actualColor,
0,
20,
filename,
);
RenderInternals.deleteDirectory(outputPath);
return missedFrames;
};
function selectColor(color: string, frame: number) {
return Math.floor((random(`${color}-${frame}`) * 255) % 255);
}
async function saveSequenceInTempDir(id: string) {
const outputPath = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'remotion-'),
);
await execa(
'bun',
[
'x',
'remotion',
'render',
'./build',
id,
outputPath,
'--image-format',
'png',
'--sequence',
'--timeout',
'60000',
'--concurrency',
'1',
],
{
cwd: path.join(process.cwd(), '..', 'example'),
},
);
return outputPath;
}
function missedFrameChecker(
expectedColor: RgbColor,
actualColor: RgbColor,
missedFrames: number,
frame: number,
filename: string,
) {
const colorDistance = {
red: Math.abs(expectedColor.red - actualColor.red),
green: Math.abs(expectedColor.green - actualColor.green),
blue: Math.abs(expectedColor.blue - actualColor.blue),
};
// encoding sometimes shifts the color slightly - so measure the distance between the expected and actual
// colors and consider any frame not within an acceptable range to be wrong
const highestDistance = Math.max(
colorDistance.red,
colorDistance.blue,
colorDistance.green,
);
const threshold = 35;
if (highestDistance > threshold) {
console.log(colorDistance, {threshold, frame, filename});
return missedFrames + 1;
}
return missedFrames;
}