🎥 Make videos programmatically with React
import {createWriteStream, writeFileSync} from 'fs';
import {pipeline} from 'node:stream/promises';
import {join} from 'path';
import {
LambdaClientInternals,
ServerlessRoutines,
type GetBinaryPayloadSink,
type OnMessage,
type ServerlessPayload,
type RequestHandler,
} from '@remotion/lambda-client';
import type {EmittedArtifact, LogLevel} from '@remotion/renderer';
import {RenderInternals} from '@remotion/renderer';
import type {OnArtifactFromRenderer} from './artifact-registry';
import {getCurrentRegionInFunction} from './helpers/get-current-region';
import type {OverallProgressHelper} from './overall-render-progress';
import {artifactFromS3} from './s3-renderer-output';
type StreamRendererResponse =
| {
type: 'success';
}
| {
type: 'error';
error: string;
shouldRetry: boolean;
};
const streamRenderer = ({
payload,
functionName,
outdir,
overallProgress,
files,
logLevel,
onArtifact,
requestHandler,
}: {
payload: ServerlessPayload;
functionName: string;
outdir: string;
overallProgress: OverallProgressHelper;
files: string[];
logLevel: LogLevel;
onArtifact: OnArtifactFromRenderer;
requestHandler: RequestHandler | null;
expectedBucketOwner: string | null;
}) => {
if (payload.type !== ServerlessRoutines.renderer) {
throw new Error('Expected renderer type');
}
return new Promise<StreamRendererResponse>((resolve) => {
let chunkCompleted = false;
let invocation = 0;
const artifactInvocations = new Map<string, number>();
const chunkFilename = (type: 'video' | 'audio') =>
join(outdir, `chunk:${String(payload.chunk).padStart(8, '0')}:${type}`);
// Stream the chunks to disk as they arrive instead of buffering them
// in memory. With many chunks arriving concurrently, buffering can
// exhaust the memory of the main function.
const getBinaryPayloadSink: GetBinaryPayloadSink = ({messageType}) => {
const filename = chunkFilename(
messageType === 'video-chunk-rendered' ? 'video' : 'audio',
);
const writeStream = createWriteStream(filename);
return {
write: (data) =>
new Promise<void>((resolve_, reject) => {
writeStream.write(data, (err) => {
if (err) {
reject(err);
} else {
resolve_();
}
});
}),
end: () =>
new Promise<void>((resolve_) => {
writeStream.end(resolve_);
}),
};
};
const receivedStreamingPayload: OnMessage = ({message}) => {
if (message.type === 'lambda-invoked') {
invocation++;
overallProgress.setLambdaInvoked(payload.chunk);
return;
}
if (message.type === 'frames-uploaded') {
overallProgress.setUploadedFrames({
index: payload.chunk,
...message.payload,
});
return;
}
if (message.type === 'frames-rendered') {
overallProgress.setFrames({
index: payload.chunk,
encoded: message.payload.encoded,
rendered: message.payload.rendered,
});
return;
}
if (message.type === 'video-chunk-rendered') {
const filename = chunkFilename('video');
// An empty payload means the chunk was already streamed to disk
if (message.payload.length > 0) {
writeFileSync(filename, message.payload);
}
if (!files.includes(filename)) {
files.push(filename);
}
RenderInternals.Log.verbose(
{indent: false, logLevel},
`Received video chunk for chunk ${payload.chunk}`,
);
return;
}
if (message.type === 'audio-chunk-rendered') {
const filename = chunkFilename('audio');
// An empty payload means the chunk was already streamed to disk
if (message.payload.length > 0) {
writeFileSync(filename, message.payload);
}
RenderInternals.Log.verbose(
{indent: false, logLevel},
`Received audio chunk for chunk ${payload.chunk}`,
);
if (!files.includes(filename)) {
files.push(filename);
}
return;
}
if (message.type === 'chunk-complete') {
chunkCompleted = true;
RenderInternals.Log.verbose(
{indent: false, logLevel},
`Finished chunk ${payload.chunk}`,
);
overallProgress.addChunkCompleted(
payload.chunk,
message.payload.start,
message.payload.rendered,
);
return;
}
if (message.type === 'artifact-emitted') {
const artifact = LambdaClientInternals.deserializeArtifact(
message.payload.artifact,
);
RenderInternals.Log.info(
{indent: false, logLevel},
`Received artifact on frame ${message.payload.artifact.frame}:`,
artifact.filename,
artifact.content.length + 'bytes.',
);
const artifactKey = JSON.stringify([artifact.filename, artifact.frame]);
const previousInvocation = artifactInvocations.get(artifactKey);
if (
previousInvocation !== undefined &&
previousInvocation < invocation
) {
RenderInternals.Log.info(
{indent: false, logLevel},
`Ignoring artifact ${artifact.filename} replayed by a retried stream of chunk ${payload.chunk}`,
);
return;
}
artifactInvocations.set(artifactKey, invocation);
const artifactRegistration = onArtifact({
artifact,
chunk: payload.chunk,
attempt: payload.attempt,
});
if (artifactRegistration.type === 'conflict') {
return resolve({
type: 'error',
error: `Chunk ${payload.chunk} emitted an asset filename ${message.payload.artifact.filename} at frame ${message.payload.artifact.frame} but there is already another artifact with the same name. https://remotion.dev/docs/artifacts`,
shouldRetry: false,
});
}
if (artifactRegistration.type === 'retry-replay') {
RenderInternals.Log.info(
{indent: false, logLevel},
`Ignoring artifact ${artifact.filename} replayed by attempt ${payload.attempt} of chunk ${payload.chunk}`,
);
}
return;
}
if (message.type === 'error-occurred') {
overallProgress.addErrorWithoutUpload(message.payload.errorInfo);
overallProgress.setFrames({
encoded: 0,
index: payload.chunk,
rendered: 0,
});
RenderInternals.Log.error(
{
indent: false,
logLevel,
},
`Renderer function of chunk ${payload.chunk} failed with error: ${message.payload.error}`,
);
RenderInternals.Log.error(
{
indent: false,
logLevel,
},
`Will retry chunk = ${message.payload.shouldRetry}`,
);
resolve({
type: 'error',
error: message.payload.error,
shouldRetry: message.payload.shouldRetry,
});
return;
}
throw new Error(`Unknown message type ${message.type}`);
};
LambdaClientInternals.callFunctionWithStreamingImplementation({
functionName,
payload,
retriesRemaining: 1,
region: getCurrentRegionInFunction(),
timeoutInTest: 12000,
type: ServerlessRoutines.renderer,
receivedStreamingPayload,
requestHandler,
getBinaryPayloadSink,
})
.then(() => {
if (payload.output.type === 'sequence' && !chunkCompleted) {
resolve({
type: 'error',
error: `Image sequence chunk ${payload.chunk} ended without completing its uploads`,
shouldRetry: true,
});
return;
}
resolve({
type: 'success',
});
})
.catch((err) => {
const shouldRetry =
(err as Error).stack?.includes('Error: aborted') ||
(err as Error).stack?.includes('ETIMEDOUT') ||
(err as Error).stack?.includes('socket hang up') ||
(err as Error).stack?.includes('ECONNRESET') ||
false;
resolve({
type: 'error',
error: (err as Error).stack as string,
shouldRetry,
});
});
});
};
const wait = (duration: number) =>
new Promise<void>((resolve) => setTimeout(resolve, duration));
const isMissingObjectError = (err: unknown) => {
const error = err as {name?: string; Code?: string; code?: string};
return (
error.name === 'NoSuchKey' ||
error.name === 'NotFound' ||
error.Code === 'NoSuchKey' ||
error.code === 'NoSuchKey'
);
};
const readStreamAsBytes = async (
stream: NodeJS.ReadableStream,
): Promise<Uint8Array> => {
const chunks: Uint8Array[] = [];
for await (const chunk of stream) {
chunks.push(
typeof chunk === 'string'
? new TextEncoder().encode(chunk)
: new Uint8Array(chunk),
);
}
const size = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
const result = new Uint8Array(size);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.byteLength;
}
return result;
};
const s3Renderer = async ({
payload,
functionName,
outdir,
overallProgress,
files,
logLevel,
onArtifact,
requestHandler,
expectedBucketOwner,
}: {
payload: ServerlessPayload;
functionName: string;
outdir: string;
overallProgress: OverallProgressHelper;
files: string[];
logLevel: LogLevel;
onArtifact: OnArtifactFromRenderer;
requestHandler: RequestHandler | null;
expectedBucketOwner: string | null;
}): Promise<StreamRendererResponse> => {
if (payload.type !== ServerlessRoutines.renderer) {
throw new Error('Expected renderer type');
}
const region = getCurrentRegionInFunction();
try {
await LambdaClientInternals.callFunctionAsyncImplementation({
functionName,
payload,
region,
timeoutInTest: 12000,
type: ServerlessRoutines.renderer,
requestHandler,
});
} catch (err) {
return {
type: 'error',
error: (err as Error).stack ?? (err as Error).message,
shouldRetry: LambdaClientInternals.isFlakyError(err as Error),
};
}
const statusKey = LambdaClientInternals.rendererTransportStatusKey({
renderId: payload.renderId,
chunk: payload.chunk,
attempt: payload.attempt,
});
let lastRendered = 0;
let lastEncoded = 0;
let lambdaInvoked = false;
let pollingInterval = 250;
const timeoutTimestamp = overallProgress.get().timeoutTimestamp - 1000;
while (Date.now() < timeoutTimestamp) {
try {
const statusStream =
await LambdaClientInternals.lambdaReadFileImplementation({
bucketName: payload.bucketName,
key: statusKey,
region,
expectedBucketOwner,
forcePathStyle: payload.forcePathStyle,
requestHandler,
});
const status = LambdaClientInternals.parseS3RendererStatus({
value: JSON.parse(
await LambdaClientInternals.streamToString(statusStream),
),
expectedChunk: payload.chunk,
expectedAttempt: payload.attempt,
});
if (status.lambdaInvoked && !lambdaInvoked) {
lambdaInvoked = true;
overallProgress.setLambdaInvoked(payload.chunk);
}
if (
status.renderedFrames > lastRendered ||
status.encodedFrames > lastEncoded
) {
lastRendered = Math.max(lastRendered, status.renderedFrames);
lastEncoded = Math.max(lastEncoded, status.encodedFrames);
overallProgress.setFrames({
index: payload.chunk,
rendered: lastRendered,
encoded: lastEncoded,
});
}
if (payload.output.type === 'sequence') {
overallProgress.setUploadedFrames({
index: payload.chunk,
uploaded: status.uploadedFrames,
sizeInBytes: status.uploadedSizeInBytes,
});
}
if (status.state === 'failed') {
overallProgress.addErrorWithoutUpload(status.errorInfo);
overallProgress.setFrames({
encoded: 0,
index: payload.chunk,
rendered: 0,
});
await LambdaClientInternals.lambdaDeleteFileImplementation({
bucketName: payload.bucketName,
key: statusKey,
region,
customCredentials: null,
forcePathStyle: payload.forcePathStyle,
requestHandler,
}).catch(() => undefined);
return {
type: 'error',
error: status.errorInfo.stack,
shouldRetry: status.shouldRetry,
};
}
if (status.state === 'completed') {
const downloadedKeys: string[] = [];
const emittedArtifacts: EmittedArtifact[] = [];
for (const artifact of status.artifacts) {
let artifactStream;
try {
artifactStream =
await LambdaClientInternals.lambdaReadFileImplementation({
bucketName: payload.bucketName,
key: artifact.key,
region,
expectedBucketOwner,
forcePathStyle: payload.forcePathStyle,
requestHandler,
});
} catch (err) {
if (isMissingObjectError(err)) {
throw new Error(
`Renderer completed manifest references missing artifact object ${artifact.key}`,
);
}
throw err;
}
emittedArtifacts.push(
artifactFromS3({
metadata: artifact.metadata,
content: await readStreamAsBytes(artifactStream),
}),
);
downloadedKeys.push(artifact.key);
}
const downloadedFiles: string[] = [];
for (const [type, key] of [
['video', status.videoKey],
['audio', status.audioKey],
] as const) {
if (!key) {
continue;
}
const filename = join(
outdir,
`chunk-${String(payload.chunk).padStart(8, '0')}-${type}`,
);
let media;
try {
media = await LambdaClientInternals.lambdaReadFileImplementation({
bucketName: payload.bucketName,
key,
region,
expectedBucketOwner,
forcePathStyle: payload.forcePathStyle,
requestHandler,
});
} catch (err) {
if (isMissingObjectError(err)) {
throw new Error(
`Renderer completed manifest references missing media object ${key}`,
);
}
throw err;
}
await pipeline(media, createWriteStream(filename));
downloadedFiles.push(filename);
downloadedKeys.push(key);
}
for (const emittedArtifact of emittedArtifacts) {
const artifactRegistration = onArtifact({
artifact: emittedArtifact,
chunk: payload.chunk,
attempt: payload.attempt,
});
if (artifactRegistration.type === 'conflict') {
return {
type: 'error',
error: `Chunk ${payload.chunk} emitted an asset filename ${emittedArtifact.filename} at frame ${emittedArtifact.frame} but there is already another artifact with the same name. https://remotion.dev/docs/artifacts`,
shouldRetry: false,
};
}
if (artifactRegistration.type === 'retry-replay') {
RenderInternals.Log.info(
{indent: false, logLevel},
`Ignoring artifact ${emittedArtifact.filename} replayed by attempt ${payload.attempt} of chunk ${payload.chunk}`,
);
}
}
files.push(...downloadedFiles);
overallProgress.addChunkCompleted(
payload.chunk,
status.startedAt,
status.completedAt,
);
for (const key of [...downloadedKeys, statusKey]) {
await LambdaClientInternals.lambdaDeleteFileImplementation({
bucketName: payload.bucketName,
key,
region,
customCredentials: null,
forcePathStyle: payload.forcePathStyle,
requestHandler,
}).catch((err) => {
RenderInternals.Log.warn(
{indent: false, logLevel},
`Could not clean up renderer transport object ${key}`,
err,
);
});
}
return {type: 'success'};
}
} catch (err) {
if (
!isMissingObjectError(err) &&
!LambdaClientInternals.isFlakyError(err as Error)
) {
return {
type: 'error',
error: (err as Error).stack ?? (err as Error).message,
shouldRetry: false,
};
}
}
await wait(pollingInterval);
pollingInterval = Math.min(2000, Math.round(pollingInterval * 1.5));
}
return {
type: 'error',
error: `Renderer chunk ${payload.chunk}, attempt ${payload.attempt} did not publish a status before the launcher deadline. Logs: ${LambdaClientInternals.getCloudwatchRendererUrl(
{
region,
functionName,
rendererFunctionName: functionName,
renderId: payload.renderId,
chunk: payload.chunk,
},
)}`,
shouldRetry: false,
};
};
type RenderRendererFunctionWithRetryInput = {
payload: ServerlessPayload;
functionName: string;
outdir: string;
overallProgress: OverallProgressHelper;
files: string[];
logLevel: LogLevel;
onArtifact: OnArtifactFromRenderer;
requestHandler: RequestHandler | null;
expectedBucketOwner: string | null;
};
export const renderRendererFunctionWithRetry = async ({
payload,
files,
functionName,
outdir,
overallProgress,
logLevel,
onArtifact,
requestHandler,
expectedBucketOwner,
}: RenderRendererFunctionWithRetryInput): Promise<unknown> => {
if (payload.type !== ServerlessRoutines.renderer) {
throw new Error('Expected renderer type');
}
const transport = LambdaClientInternals.getRendererFunctionTransport(
getCurrentRegionInFunction(),
);
const result = await (transport === 's3' ? s3Renderer : streamRenderer)({
files,
functionName,
outdir,
overallProgress,
payload,
logLevel,
onArtifact,
requestHandler,
expectedBucketOwner,
});
if (result.type === 'error') {
if (!result.shouldRetry || payload.retriesLeft <= 0) {
throw new Error(result.error);
}
overallProgress.addRetry({
attempt: payload.attempt + 1,
time: Date.now(),
chunk: payload.chunk,
});
return renderRendererFunctionWithRetry({
files,
functionName,
outdir,
overallProgress,
payload: {
...payload,
attempt: payload.attempt + 1,
retriesLeft: payload.retriesLeft - 1,
},
logLevel,
onArtifact,
requestHandler,
expectedBucketOwner,
});
}
};