🎥 Make videos programmatically with React
import {afterEach, expect, mock, test} from 'bun:test';
import {act, cleanup, render, waitFor} from '@testing-library/react';
import {type ContextType, useContext} from 'react';
import {makeBrowserStudioOperations} from './make-browser-studio-operations';
let modelLoadCall:
| {
model: string;
}
| undefined;
let transcriptionCall:
| {
channelWaveform: Float32Array;
options: Record<string, unknown>;
}
| undefined;
let cacheCheck:
| {
model: string;
}
| undefined;
const getCacheCheck = () => cacheCheck;
const getModelLoadCall = () => modelLoadCall;
let modelIsCached = false;
let modelLoadProgressCallback:
| ((progress: {
file: string | null;
progress: number;
loadedBytes: number;
totalBytes: number;
}) => void)
| undefined;
let resolveModelLoading: (() => void) | null = null;
let disposalCalls = 0;
const resamplingCalls: Array<{
src: string;
audioStreamIndex: number | null;
requestInit: Omit<RequestInit, 'signal'> | null;
}> = [];
const resampledWaveform = new Float32Array(32_000).fill(0.25);
mock.module('../components/Transcription/resample-media-to-16-khz', () => ({
resampleMediaTo16Khz: ({
src,
audioStreamIndex,
requestInit,
onProgress,
}: {
src: string;
audioStreamIndex: number | null;
requestInit: Omit<RequestInit, 'signal'> | null;
onProgress: (progress: number) => void;
}) => {
resamplingCalls.push({src, audioStreamIndex, requestInit});
onProgress(0.5);
return Promise.resolve(resampledWaveform);
},
}));
mock.module('@remotion/whisper-webgpu', () => ({
canUseWhisperWebGpu: () => Promise.resolve({supported: true}),
clearStaleModels: () => Promise.resolve(),
disposeWhisperModel: () => {
disposalCalls++;
return Promise.resolve();
},
getAvailableModels: () => [
{
name: 'tiny',
modelId: 'onnx-community/whisper-tiny_timestamped',
parameters: 39_000_000,
multilingual: true,
supportsTranslation: true,
webGpuDownloadSize: 119_699_015,
},
{
name: 'tiny.en',
modelId: 'onnx-community/whisper-tiny.en_timestamped',
parameters: 39_000_000,
multilingual: false,
supportsTranslation: false,
webGpuDownloadSize: 119_697_479,
},
{
name: 'small.en',
modelId: 'onnx-community/whisper-small.en_timestamped',
parameters: 244_000_000,
multilingual: false,
supportsTranslation: false,
webGpuDownloadSize: 614_690_756,
},
],
isWhisperModelCached: ({model}: {model: string}) => {
cacheCheck = {model};
return Promise.resolve(modelIsCached);
},
downloadWhisperModel: async ({
model,
onProgress,
}: {
model: string;
onProgress?: typeof modelLoadProgressCallback;
}) => {
modelLoadCall = {model};
modelLoadProgressCallback = onProgress;
await new Promise<void>((resolve) => {
resolveModelLoading = resolve;
});
modelLoadProgressCallback?.({
file: null,
progress: 1,
loadedBytes: 119_699_015,
totalBytes: 119_699_015,
});
return {alreadyDownloaded: false};
},
loadWhisperModel: () => Promise.resolve({alreadyLoaded: false}),
removeWhisperModel: () => Promise.resolve(),
toCaptions: () => ({
captions: [
{
text: 'Hello',
startMs: 125,
endMs: 625,
timestampMs: 375,
confidence: null,
},
{
text: ' world.',
startMs: 750,
endMs: 1500,
timestampMs: 1125,
confidence: null,
},
],
}),
transcribe: ({
channelWaveform,
...options
}: {
channelWaveform: Float32Array;
[key: string]: unknown;
}) => {
transcriptionCall = {channelWaveform, options};
return Promise.resolve({
text: ' Hello world.',
chunks: [
{text: ' Hello', timestamp: [0.125, 0.625]},
{text: ' world.', timestamp: [0.75, 1.5]},
],
});
},
WHISPER_WEBGPU_SAMPLE_RATE: 16_000,
}));
afterEach(async () => {
cleanup();
resolveModelLoading?.();
const {disposeWhisperModel} = await import('@remotion/whisper-webgpu');
await disposeWhisperModel({model: 'tiny'});
resolveModelLoading = null;
});
test('downloads a model in the queued job before transcribing', async () => {
disposalCalls = 0;
resamplingCalls.length = 0;
modelLoadCall = undefined;
modelLoadProgressCallback = undefined;
transcriptionCall = undefined;
cacheCheck = undefined;
modelIsCached = false;
resolveModelLoading = null;
const {CaptionQueueProcessor} =
await import('../components/RenderQueue/CaptionQueueProcessor');
const {RenderQueueContext, RenderQueueContextProvider} =
await import('../components/RenderQueue/context');
const originalGpuDescriptor = Object.getOwnPropertyDescriptor(
navigator,
'gpu',
);
const originalSecureContextDescriptor = Object.getOwnPropertyDescriptor(
window,
'isSecureContext',
);
const originalBrowserStudioDescriptor = Object.getOwnPropertyDescriptor(
window,
'remotion_browserStudio',
);
const originalInstalledPackages = window.remotion_installedPackages;
const originalFetch = globalThis.fetch;
const writtenFiles: Array<{
contents: string | ArrayBuffer;
filePath: string;
}> = [];
Object.defineProperty(navigator, 'gpu', {
configurable: true,
value: {requestAdapter: () => Promise.resolve({})},
});
Object.defineProperty(window, 'isSecureContext', {
configurable: true,
value: true,
});
Object.defineProperty(window, 'remotion_browserStudio', {
configurable: true,
value: makeBrowserStudioOperations({
writeStaticFile: ({contents, filePath}) => {
writtenFiles.push({contents, filePath});
return Promise.resolve();
},
}),
});
let currentContext: ContextType<typeof RenderQueueContext> | null = null;
const ReadContext = () => {
currentContext = useContext(RenderQueueContext);
return null;
};
try {
render(
<RenderQueueContextProvider>
<CaptionQueueProcessor />
<ReadContext />
</RenderQueueContextProvider>,
);
const getContext = () => {
if (currentContext === null) {
throw new Error('Render queue context is not mounted');
}
return currentContext;
};
let jobId = '';
act(() => {
jobId = getContext().addCaptionJob({
captionStyle: null,
displayName: 'interview.wav',
audioStreamIndex: 2,
requestInit: {
credentials: 'include',
headers: {'x-remotion-test': 'transcription'},
},
language: 'de',
model: 'tiny',
chunkLengthInSeconds: 20,
strideLengthInSeconds: 3,
task: 'translate',
forceFullSequences: true,
doSample: true,
temperature: 0.7,
topK: 25,
repetitionPenalty: 1.2,
noRepeatNgramSize: 3,
outName: 'captions/interview.json',
target: null,
src: '/static/interview.wav',
});
});
await waitFor(() => expect(modelLoadCall).toBeDefined());
expect(getCacheCheck()).toEqual({model: 'tiny'});
expect(resamplingCalls).toEqual([]);
expect(transcriptionCall).toBeUndefined();
expect(writtenFiles).toEqual([]);
act(() => {
modelLoadProgressCallback?.({
file: null,
progress: 0.5,
loadedBytes: 119_699_015 / 2,
totalBytes: 119_699_015,
});
});
await waitFor(() => {
const job = getContext().captionJobs.find(({id}) => id === jobId);
if (job?.status !== 'running') {
throw new Error('Caption job is not running');
}
expect(job.progress.message).toBe('Downloading tiny 50%');
expect(job.progress.value).toBeCloseTo(0.165);
});
expect(resamplingCalls).toEqual([]);
expect(transcriptionCall).toBeUndefined();
expect(writtenFiles).toEqual([]);
act(() => resolveModelLoading?.());
await waitFor(() => {
expect(
getContext().captionJobs.find((job) => job.id === jobId),
).toMatchObject({status: 'done', captionCount: 2});
});
expect(resamplingCalls).toEqual([
{
src: '/static/interview.wav',
audioStreamIndex: 2,
requestInit: {
credentials: 'include',
headers: {'x-remotion-test': 'transcription'},
},
},
]);
expect(getModelLoadCall()).toEqual({model: 'tiny'});
await waitFor(() => {
expect(transcriptionCall).toEqual({
channelWaveform: resampledWaveform,
options: {
signal: expect.any(AbortSignal),
model: 'tiny',
language: 'de',
task: 'translate',
chunkLengthInSeconds: 20,
strideLengthInSeconds: 3,
forceFullSequences: true,
doSample: true,
temperature: 0.7,
topK: 25,
repetitionPenalty: 1.2,
noRepeatNgramSize: 3,
},
});
});
expect(writtenFiles).toEqual([
{
filePath: 'captions/interview.json',
contents: JSON.stringify(
[
{
text: 'Hello',
startMs: 125,
endMs: 625,
timestampMs: 375,
confidence: null,
},
{
text: ' world.',
startMs: 750,
endMs: 1500,
timestampMs: 1125,
confidence: null,
},
],
null,
2,
),
},
]);
expect(disposalCalls).toBe(1);
const insertedRequests: Array<{endpoint: string; body: unknown}> = [];
window.remotion_installedPackages = ['@remotion/captions'];
Object.defineProperty(window, 'remotion_browserStudio', {
configurable: true,
value: null,
});
globalThis.fetch = ((endpoint: string, init: RequestInit) => {
insertedRequests.push({
endpoint,
body: JSON.parse(String(init.body)),
});
return Promise.resolve({
json: () =>
Promise.resolve({
success: true,
data: {success: true, nodePathMutation: null},
}),
}) as Promise<Response>;
}) as typeof fetch;
modelLoadCall = undefined;
let inlineJobId = '';
act(() => {
inlineJobId = getContext().addCaptionJob({
captionStyle: null,
displayName: 'clip.mp4',
audioStreamIndex: null,
requestInit: null,
language: null,
model: 'tiny',
chunkLengthInSeconds: 30,
strideLengthInSeconds: 5,
task: 'transcribe',
forceFullSequences: false,
doSample: false,
temperature: 1,
topK: 50,
repetitionPenalty: 1,
noRepeatNgramSize: 0,
outName: 'Basic captions',
target: {
fileName: '/project/Root.tsx',
nodePath: {
absolutePath: '/project/Root.tsx',
nodePath: [0],
sequenceKeys: [],
effectKeys: [],
videoConfigValues: null,
},
durationInFrames: 90,
},
src: '/static/clip.mp4',
});
});
await waitFor(() => expect(modelLoadCall).toBeDefined());
act(() => resolveModelLoading?.());
await waitFor(() => {
expect(
getContext().captionJobs.find((job) => job.id === inlineJobId),
).toMatchObject({status: 'done', captionCount: 2});
});
expect(insertedRequests).toEqual([
{
endpoint: '/api/insert-basic-captions',
body: {
fileName: '/project/Root.tsx',
nodePath: [0],
durationInFrames: 90,
captions: [
{
text: 'Hello',
startMs: 125,
endMs: 625,
timestampMs: 375,
confidence: null,
},
{
text: ' world.',
startMs: 750,
endMs: 1500,
timestampMs: 1125,
confidence: null,
},
],
},
},
]);
expect(writtenFiles).toHaveLength(1);
const browserRequests: unknown[] = [];
Object.defineProperty(window, 'remotion_browserStudio', {
configurable: true,
value: makeBrowserStudioOperations({
insertBasicCaptions: (request) => {
browserRequests.push(request);
return Promise.resolve({
success: true,
nodePathMutation: {
mutationId: 'captions-browser',
timelineSelection: null,
files: [],
},
});
},
}),
});
const previousJob = getContext().captionJobs.find(
(job) => job.id === inlineJobId,
);
if (!previousJob || previousJob.target === null) {
throw new Error('Inline captions job is missing');
}
const previousTarget = previousJob.target;
modelLoadCall = undefined;
let browserJobId = '';
act(() => {
browserJobId = getContext().addCaptionJob({
...previousJob,
src: '/static/browser-clip.mp4',
target: {
...previousTarget,
fileName: '/project/Browser.tsx',
durationInFrames: 45,
},
});
});
await waitFor(() => expect(modelLoadCall).toBeDefined());
act(() => resolveModelLoading?.());
await waitFor(() => {
expect(
getContext().captionJobs.find((job) => job.id === browserJobId),
).toMatchObject({status: 'done', captionCount: 2});
});
expect(browserRequests).toEqual([
{
fileName: '/project/Browser.tsx',
nodePath: [0],
durationInFrames: 45,
captions: (insertedRequests[0].body as {captions: unknown}).captions,
},
]);
expect(insertedRequests).toHaveLength(1);
} finally {
globalThis.fetch = originalFetch;
window.remotion_installedPackages = originalInstalledPackages;
if (originalGpuDescriptor) {
Object.defineProperty(navigator, 'gpu', originalGpuDescriptor);
} else {
Reflect.deleteProperty(navigator, 'gpu');
}
if (originalSecureContextDescriptor) {
Object.defineProperty(
window,
'isSecureContext',
originalSecureContextDescriptor,
);
} else {
Reflect.deleteProperty(window, 'isSecureContext');
}
if (originalBrowserStudioDescriptor) {
Object.defineProperty(
window,
'remotion_browserStudio',
originalBrowserStudioDescriptor,
);
} else {
Reflect.deleteProperty(window, 'remotion_browserStudio');
}
}
});