🎥 Make videos programmatically with React
import {
canUseWhisperWebGpu,
clearStaleModels,
disposeWhisperModel,
downloadWhisperModel,
isWhisperModelCached,
loadWhisperModel,
toCaptions,
transcribe,
} from '@remotion/whisper-webgpu';
import {useCallback, useContext, useEffect} from 'react';
import {writeStaticFile} from '../../api/write-static-file';
import {getBrowserStudioOperations} from '../../helpers/browser-studio-operations';
import {installRequiredPackages} from '../../helpers/install-required-package';
import {callApi} from '../call-api';
import {installElement, prepareElementInstall} from '../element-install-api';
import {resampleMediaTo16Khz} from '../Transcription/resample-media-to-16-khz';
import type {CaptionJob} from './caption-job-types';
import {RenderQueueContext} from './context';
import {loadModelForJob} from './load-model-for-job';
export const CaptionQueueProcessor: React.FC = () => {
const {
updateCaptionJobProgress,
markCaptionJobDone,
markCaptionJobFailed,
markCaptionJobCancelled,
markCaptionJobSaving,
getAbortController,
setProcessCaptionJobCallback,
} = useContext(RenderQueueContext);
const processJob = useCallback(
async (job: CaptionJob) => {
const {signal} = getAbortController(job.id);
let captionCount: number | null = null;
let processingError: Error | null = null;
try {
signal.throwIfAborted();
updateCaptionJobProgress(job.id, {
message: 'Checking WebGPU support...',
value: 0,
});
const support = await canUseWhisperWebGpu();
signal.throwIfAborted();
if (!support.supported) {
throw new Error(support.detailedReason);
}
updateCaptionJobProgress(job.id, {
message: `Checking ${job.model}...`,
value: 0.02,
});
await clearStaleModels();
signal.throwIfAborted();
await loadModelForJob({
signal,
model: job.model,
progressStart: 0.03,
progressSpan: 0.27,
isModelCached: (model) => isWhisperModelCached({model}),
loadModel: async (model, onProgress) => {
await downloadWhisperModel({
signal,
model,
onProgress: (progress) => onProgress(progress.progress),
});
await loadWhisperModel({model, signal});
},
updateProgress: (progress) =>
updateCaptionJobProgress(job.id, progress),
});
updateCaptionJobProgress(job.id, {
message: `Preparing ${job.displayName}...`,
value: 0.3,
});
const channelWaveform = await resampleMediaTo16Khz({
signal,
src: job.src,
audioStreamIndex: job.audioStreamIndex,
requestInit: job.requestInit,
onProgress: (progress) => {
updateCaptionJobProgress(job.id, {
message: `Preparing ${job.displayName}...`,
value: 0.3 + progress * 0.2,
});
},
});
updateCaptionJobProgress(job.id, {
message: `Transcribing ${job.displayName}...`,
value: 0.5,
});
const transcription = await transcribe({
signal,
channelWaveform,
model: job.model,
task: job.task,
chunkLengthInSeconds: job.chunkLengthInSeconds,
strideLengthInSeconds: job.strideLengthInSeconds,
forceFullSequences: job.forceFullSequences,
doSample: job.doSample,
temperature: job.temperature,
topK: job.topK,
repetitionPenalty: job.repetitionPenalty,
noRepeatNgramSize: job.noRepeatNgramSize,
...(job.language === null ? {} : {language: job.language}),
});
const {captions} = toCaptions({
whisperWebGpuOutput: transcription,
});
signal.throwIfAborted();
markCaptionJobSaving(job.id);
if (job.target === null) {
updateCaptionJobProgress(job.id, {
message: `Saving ${job.outName}...`,
value: 0.95,
});
await writeStaticFile({
contents: JSON.stringify(captions, null, 2),
filePath: job.outName,
});
} else if (
job.captionStyle !== null &&
// The built-in path owns basic-captions.element.tsx, so the
// library's Basic Captions must not be installed under that name.
job.captionStyle.element.slug !== 'captions/basic-captions'
) {
const {element} = job.captionStyle;
updateCaptionJobProgress(job.id, {
message: `Adding ${element.displayName}...`,
value: 0.95,
});
const hash = await crypto.subtle.digest(
'SHA-256',
new TextEncoder().encode(element.sourceCode),
);
const sourceHash = Array.from(new Uint8Array(hash), (byte) =>
byte.toString(16).padStart(2, '0'),
).join('');
const baseName = element.slug.split('/').at(-1);
let installed = false;
for (let index = 0; index < 1000; index++) {
signal.throwIfAborted();
const installationName = `${baseName}${index === 0 ? '' : `-copy${index === 1 ? '' : `-${index}`}`}`;
const preflight = await prepareElementInstall({
installationName,
destination: {
type: 'selected-media',
compositionFile: job.target.fileName,
},
element,
});
if (!preflight.success) {
throw new Error(preflight.reason);
}
const {expectedFileState} = preflight.plan;
if (
expectedFileState.exists &&
expectedFileState.sourceHash !== sourceHash
) {
continue;
}
if (getBrowserStudioOperations() === null) {
await installRequiredPackages(element.dependencies);
}
signal.throwIfAborted();
const response = await installElement({
installationName,
compositionFile: null,
compositionId: null,
captionTarget: {
fileName: job.target.fileName,
nodePath: job.target.nodePath.nodePath,
durationInFrames: job.target.durationInFrames,
captions,
},
element,
expectedFileState,
from: null,
position: null,
overwriteExisting: expectedFileState.exists,
undoRedoNavigation: null,
newComposition: null,
});
if (!response.success) {
throw new Error(
response.type === 'error'
? response.reason
: `Element file changed: ${response.conflict.filePath}`,
);
}
installed = true;
break;
}
if (!installed) {
throw new Error(
'Could not find an available filename for the caption style',
);
}
} else {
updateCaptionJobProgress(job.id, {
message: 'Adding Basic captions...',
value: 0.95,
});
await installRequiredPackages([
{name: '@remotion/captions', version: null},
]);
const request = {
fileName: job.target.fileName,
nodePath: job.target.nodePath.nodePath,
durationInFrames: job.target.durationInFrames,
captions,
};
const browserStudioOperations = getBrowserStudioOperations();
const response = browserStudioOperations
? await browserStudioOperations.insertBasicCaptions(request)
: await callApi('/api/insert-basic-captions', request);
if (!response.success) {
throw new Error(response.reason);
}
}
captionCount = captions.length;
} catch (error) {
processingError =
error instanceof Error ? error : new Error(String(error));
}
try {
await disposeWhisperModel({model: job.model});
} catch {
// The pipeline is removed from the in-memory registry before disposal.
// A cleanup failure should not hide a successfully written caption file.
}
if (signal.aborted) {
markCaptionJobCancelled(job.id);
} else if (processingError) {
markCaptionJobFailed(job.id, processingError);
} else {
markCaptionJobDone(job.id, captionCount ?? 0);
}
},
[
getAbortController,
markCaptionJobCancelled,
markCaptionJobDone,
markCaptionJobFailed,
markCaptionJobSaving,
updateCaptionJobProgress,
],
);
useEffect(() => {
setProcessCaptionJobCallback(processJob);
return () => setProcessCaptionJobCallback(null);
}, [processJob, setProcessCaptionJobCallback]);
return null;
};