Skip to content

🎥 Make videos programmatically with React

16.8 KB
Raw Blame
import {createWriteStream, writeFileSync} from 'fs';
import {pipeline} from 'node:stream/promises';
import {join} from 'path';
import {
	LambdaClientInternals,
	ServerlessRoutines,
	type GetBinaryPayloadSink,
	type OnMessage,
	type ServerlessPayload,
	type RequestHandler,
} from '@remotion/lambda-client';
import type {EmittedArtifact, LogLevel} from '@remotion/renderer';
import {RenderInternals} from '@remotion/renderer';
import type {OnArtifactFromRenderer} from './artifact-registry';
import {getCurrentRegionInFunction} from './helpers/get-current-region';
import type {OverallProgressHelper} from './overall-render-progress';
import {artifactFromS3} from './s3-renderer-output';

type StreamRendererResponse =
	| {
			type: 'success';
	  }
	| {
			type: 'error';
			error: string;
			shouldRetry: boolean;
	  };

const streamRenderer = ({
	payload,
	functionName,
	outdir,
	overallProgress,
	files,
	logLevel,
	onArtifact,
	requestHandler,
}: {
	payload: ServerlessPayload;
	functionName: string;
	outdir: string;
	overallProgress: OverallProgressHelper;
	files: string[];
	logLevel: LogLevel;
	onArtifact: OnArtifactFromRenderer;
	requestHandler: RequestHandler | null;
	expectedBucketOwner: string | null;
}) => {
	if (payload.type !== ServerlessRoutines.renderer) {
		throw new Error('Expected renderer type');
	}

	return new Promise<StreamRendererResponse>((resolve) => {
		let chunkCompleted = false;
		let invocation = 0;
		const artifactInvocations = new Map<string, number>();
		const chunkFilename = (type: 'video' | 'audio') =>
			join(outdir, `chunk:${String(payload.chunk).padStart(8, '0')}:${type}`);

		// Stream the chunks to disk as they arrive instead of buffering them
		// in memory. With many chunks arriving concurrently, buffering can
		// exhaust the memory of the main function.
		const getBinaryPayloadSink: GetBinaryPayloadSink = ({messageType}) => {
			const filename = chunkFilename(
				messageType === 'video-chunk-rendered' ? 'video' : 'audio',
			);
			const writeStream = createWriteStream(filename);
			return {
				write: (data) =>
					new Promise<void>((resolve_, reject) => {
						writeStream.write(data, (err) => {
							if (err) {
								reject(err);
							} else {
								resolve_();
							}
						});
					}),
				end: () =>
					new Promise<void>((resolve_) => {
						writeStream.end(resolve_);
					}),
			};
		};

		const receivedStreamingPayload: OnMessage = ({message}) => {
			if (message.type === 'lambda-invoked') {
				invocation++;
				overallProgress.setLambdaInvoked(payload.chunk);
				return;
			}

			if (message.type === 'frames-uploaded') {
				overallProgress.setUploadedFrames({
					index: payload.chunk,
					...message.payload,
				});
				return;
			}

			if (message.type === 'frames-rendered') {
				overallProgress.setFrames({
					index: payload.chunk,
					encoded: message.payload.encoded,
					rendered: message.payload.rendered,
				});
				return;
			}

			if (message.type === 'video-chunk-rendered') {
				const filename = chunkFilename('video');
				// An empty payload means the chunk was already streamed to disk
				if (message.payload.length > 0) {
					writeFileSync(filename, message.payload);
				}

				if (!files.includes(filename)) {
					files.push(filename);
				}

				RenderInternals.Log.verbose(
					{indent: false, logLevel},
					`Received video chunk for chunk ${payload.chunk}`,
				);

				return;
			}

			if (message.type === 'audio-chunk-rendered') {
				const filename = chunkFilename('audio');
				// An empty payload means the chunk was already streamed to disk
				if (message.payload.length > 0) {
					writeFileSync(filename, message.payload);
				}

				RenderInternals.Log.verbose(
					{indent: false, logLevel},
					`Received audio chunk for chunk ${payload.chunk}`,
				);
				if (!files.includes(filename)) {
					files.push(filename);
				}

				return;
			}

			if (message.type === 'chunk-complete') {
				chunkCompleted = true;
				RenderInternals.Log.verbose(
					{indent: false, logLevel},
					`Finished chunk ${payload.chunk}`,
				);
				overallProgress.addChunkCompleted(
					payload.chunk,
					message.payload.start,
					message.payload.rendered,
				);
				return;
			}

			if (message.type === 'artifact-emitted') {
				const artifact = LambdaClientInternals.deserializeArtifact(
					message.payload.artifact,
				);
				RenderInternals.Log.info(
					{indent: false, logLevel},
					`Received artifact on frame ${message.payload.artifact.frame}:`,
					artifact.filename,
					artifact.content.length + 'bytes.',
				);
				const artifactKey = JSON.stringify([artifact.filename, artifact.frame]);
				const previousInvocation = artifactInvocations.get(artifactKey);
				if (
					previousInvocation !== undefined &&
					previousInvocation < invocation
				) {
					RenderInternals.Log.info(
						{indent: false, logLevel},
						`Ignoring artifact ${artifact.filename} replayed by a retried stream of chunk ${payload.chunk}`,
					);
					return;
				}

				artifactInvocations.set(artifactKey, invocation);
				const artifactRegistration = onArtifact({
					artifact,
					chunk: payload.chunk,
					attempt: payload.attempt,
				});
				if (artifactRegistration.type === 'conflict') {
					return resolve({
						type: 'error',
						error: `Chunk ${payload.chunk} emitted an asset filename ${message.payload.artifact.filename} at frame ${message.payload.artifact.frame} but there is already another artifact with the same name. https://remotion.dev/docs/artifacts`,
						shouldRetry: false,
					});
				}

				if (artifactRegistration.type === 'retry-replay') {
					RenderInternals.Log.info(
						{indent: false, logLevel},
						`Ignoring artifact ${artifact.filename} replayed by attempt ${payload.attempt} of chunk ${payload.chunk}`,
					);
				}

				return;
			}

			if (message.type === 'error-occurred') {
				overallProgress.addErrorWithoutUpload(message.payload.errorInfo);
				overallProgress.setFrames({
					encoded: 0,
					index: payload.chunk,
					rendered: 0,
				});

				RenderInternals.Log.error(
					{
						indent: false,
						logLevel,
					},
					`Renderer function of chunk ${payload.chunk} failed with error: ${message.payload.error}`,
				);
				RenderInternals.Log.error(
					{
						indent: false,
						logLevel,
					},
					`Will retry chunk = ${message.payload.shouldRetry}`,
				);

				resolve({
					type: 'error',
					error: message.payload.error,
					shouldRetry: message.payload.shouldRetry,
				});
				return;
			}

			throw new Error(`Unknown message type ${message.type}`);
		};

		LambdaClientInternals.callFunctionWithStreamingImplementation({
			functionName,
			payload,
			retriesRemaining: 1,
			region: getCurrentRegionInFunction(),
			timeoutInTest: 12000,
			type: ServerlessRoutines.renderer,
			receivedStreamingPayload,
			requestHandler,
			getBinaryPayloadSink,
		})
			.then(() => {
				if (payload.output.type === 'sequence' && !chunkCompleted) {
					resolve({
						type: 'error',
						error: `Image sequence chunk ${payload.chunk} ended without completing its uploads`,
						shouldRetry: true,
					});
					return;
				}

				resolve({
					type: 'success',
				});
			})
			.catch((err) => {
				const shouldRetry =
					(err as Error).stack?.includes('Error: aborted') ||
					(err as Error).stack?.includes('ETIMEDOUT') ||
					(err as Error).stack?.includes('socket hang up') ||
					(err as Error).stack?.includes('ECONNRESET') ||
					false;

				resolve({
					type: 'error',
					error: (err as Error).stack as string,
					shouldRetry,
				});
			});
	});
};

const wait = (duration: number) =>
	new Promise<void>((resolve) => setTimeout(resolve, duration));

const isMissingObjectError = (err: unknown) => {
	const error = err as {name?: string; Code?: string; code?: string};
	return (
		error.name === 'NoSuchKey' ||
		error.name === 'NotFound' ||
		error.Code === 'NoSuchKey' ||
		error.code === 'NoSuchKey'
	);
};

const readStreamAsBytes = async (
	stream: NodeJS.ReadableStream,
): Promise<Uint8Array> => {
	const chunks: Uint8Array[] = [];
	for await (const chunk of stream) {
		chunks.push(
			typeof chunk === 'string'
				? new TextEncoder().encode(chunk)
				: new Uint8Array(chunk),
		);
	}

	const size = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
	const result = new Uint8Array(size);
	let offset = 0;
	for (const chunk of chunks) {
		result.set(chunk, offset);
		offset += chunk.byteLength;
	}

	return result;
};

const s3Renderer = async ({
	payload,
	functionName,
	outdir,
	overallProgress,
	files,
	logLevel,
	onArtifact,
	requestHandler,
	expectedBucketOwner,
}: {
	payload: ServerlessPayload;
	functionName: string;
	outdir: string;
	overallProgress: OverallProgressHelper;
	files: string[];
	logLevel: LogLevel;
	onArtifact: OnArtifactFromRenderer;
	requestHandler: RequestHandler | null;
	expectedBucketOwner: string | null;
}): Promise<StreamRendererResponse> => {
	if (payload.type !== ServerlessRoutines.renderer) {
		throw new Error('Expected renderer type');
	}

	const region = getCurrentRegionInFunction();
	try {
		await LambdaClientInternals.callFunctionAsyncImplementation({
			functionName,
			payload,
			region,
			timeoutInTest: 12000,
			type: ServerlessRoutines.renderer,
			requestHandler,
		});
	} catch (err) {
		return {
			type: 'error',
			error: (err as Error).stack ?? (err as Error).message,
			shouldRetry: LambdaClientInternals.isFlakyError(err as Error),
		};
	}

	const statusKey = LambdaClientInternals.rendererTransportStatusKey({
		renderId: payload.renderId,
		chunk: payload.chunk,
		attempt: payload.attempt,
	});
	let lastRendered = 0;
	let lastEncoded = 0;
	let lambdaInvoked = false;
	let pollingInterval = 250;
	const timeoutTimestamp = overallProgress.get().timeoutTimestamp - 1000;

	while (Date.now() < timeoutTimestamp) {
		try {
			const statusStream =
				await LambdaClientInternals.lambdaReadFileImplementation({
					bucketName: payload.bucketName,
					key: statusKey,
					region,
					expectedBucketOwner,
					forcePathStyle: payload.forcePathStyle,
					requestHandler,
				});
			const status = LambdaClientInternals.parseS3RendererStatus({
				value: JSON.parse(
					await LambdaClientInternals.streamToString(statusStream),
				),
				expectedChunk: payload.chunk,
				expectedAttempt: payload.attempt,
			});

			if (status.lambdaInvoked && !lambdaInvoked) {
				lambdaInvoked = true;
				overallProgress.setLambdaInvoked(payload.chunk);
			}

			if (
				status.renderedFrames > lastRendered ||
				status.encodedFrames > lastEncoded
			) {
				lastRendered = Math.max(lastRendered, status.renderedFrames);
				lastEncoded = Math.max(lastEncoded, status.encodedFrames);
				overallProgress.setFrames({
					index: payload.chunk,
					rendered: lastRendered,
					encoded: lastEncoded,
				});
			}

			if (payload.output.type === 'sequence') {
				overallProgress.setUploadedFrames({
					index: payload.chunk,
					uploaded: status.uploadedFrames,
					sizeInBytes: status.uploadedSizeInBytes,
				});
			}

			if (status.state === 'failed') {
				overallProgress.addErrorWithoutUpload(status.errorInfo);
				overallProgress.setFrames({
					encoded: 0,
					index: payload.chunk,
					rendered: 0,
				});
				await LambdaClientInternals.lambdaDeleteFileImplementation({
					bucketName: payload.bucketName,
					key: statusKey,
					region,
					customCredentials: null,
					forcePathStyle: payload.forcePathStyle,
					requestHandler,
				}).catch(() => undefined);
				return {
					type: 'error',
					error: status.errorInfo.stack,
					shouldRetry: status.shouldRetry,
				};
			}

			if (status.state === 'completed') {
				const downloadedKeys: string[] = [];
				const emittedArtifacts: EmittedArtifact[] = [];
				for (const artifact of status.artifacts) {
					let artifactStream;
					try {
						artifactStream =
							await LambdaClientInternals.lambdaReadFileImplementation({
								bucketName: payload.bucketName,
								key: artifact.key,
								region,
								expectedBucketOwner,
								forcePathStyle: payload.forcePathStyle,
								requestHandler,
							});
					} catch (err) {
						if (isMissingObjectError(err)) {
							throw new Error(
								`Renderer completed manifest references missing artifact object ${artifact.key}`,
							);
						}

						throw err;
					}

					emittedArtifacts.push(
						artifactFromS3({
							metadata: artifact.metadata,
							content: await readStreamAsBytes(artifactStream),
						}),
					);
					downloadedKeys.push(artifact.key);
				}

				const downloadedFiles: string[] = [];
				for (const [type, key] of [
					['video', status.videoKey],
					['audio', status.audioKey],
				] as const) {
					if (!key) {
						continue;
					}

					const filename = join(
						outdir,
						`chunk-${String(payload.chunk).padStart(8, '0')}-${type}`,
					);
					let media;
					try {
						media = await LambdaClientInternals.lambdaReadFileImplementation({
							bucketName: payload.bucketName,
							key,
							region,
							expectedBucketOwner,
							forcePathStyle: payload.forcePathStyle,
							requestHandler,
						});
					} catch (err) {
						if (isMissingObjectError(err)) {
							throw new Error(
								`Renderer completed manifest references missing media object ${key}`,
							);
						}

						throw err;
					}

					await pipeline(media, createWriteStream(filename));
					downloadedFiles.push(filename);
					downloadedKeys.push(key);
				}

				for (const emittedArtifact of emittedArtifacts) {
					const artifactRegistration = onArtifact({
						artifact: emittedArtifact,
						chunk: payload.chunk,
						attempt: payload.attempt,
					});
					if (artifactRegistration.type === 'conflict') {
						return {
							type: 'error',
							error: `Chunk ${payload.chunk} emitted an asset filename ${emittedArtifact.filename} at frame ${emittedArtifact.frame} but there is already another artifact with the same name. https://remotion.dev/docs/artifacts`,
							shouldRetry: false,
						};
					}

					if (artifactRegistration.type === 'retry-replay') {
						RenderInternals.Log.info(
							{indent: false, logLevel},
							`Ignoring artifact ${emittedArtifact.filename} replayed by attempt ${payload.attempt} of chunk ${payload.chunk}`,
						);
					}
				}

				files.push(...downloadedFiles);
				overallProgress.addChunkCompleted(
					payload.chunk,
					status.startedAt,
					status.completedAt,
				);
				for (const key of [...downloadedKeys, statusKey]) {
					await LambdaClientInternals.lambdaDeleteFileImplementation({
						bucketName: payload.bucketName,
						key,
						region,
						customCredentials: null,
						forcePathStyle: payload.forcePathStyle,
						requestHandler,
					}).catch((err) => {
						RenderInternals.Log.warn(
							{indent: false, logLevel},
							`Could not clean up renderer transport object ${key}`,
							err,
						);
					});
				}

				return {type: 'success'};
			}
		} catch (err) {
			if (
				!isMissingObjectError(err) &&
				!LambdaClientInternals.isFlakyError(err as Error)
			) {
				return {
					type: 'error',
					error: (err as Error).stack ?? (err as Error).message,
					shouldRetry: false,
				};
			}
		}

		await wait(pollingInterval);
		pollingInterval = Math.min(2000, Math.round(pollingInterval * 1.5));
	}

	return {
		type: 'error',
		error: `Renderer chunk ${payload.chunk}, attempt ${payload.attempt} did not publish a status before the launcher deadline. Logs: ${LambdaClientInternals.getCloudwatchRendererUrl(
			{
				region,
				functionName,
				rendererFunctionName: functionName,
				renderId: payload.renderId,
				chunk: payload.chunk,
			},
		)}`,
		shouldRetry: false,
	};
};

type RenderRendererFunctionWithRetryInput = {
	payload: ServerlessPayload;
	functionName: string;
	outdir: string;
	overallProgress: OverallProgressHelper;
	files: string[];
	logLevel: LogLevel;
	onArtifact: OnArtifactFromRenderer;
	requestHandler: RequestHandler | null;
	expectedBucketOwner: string | null;
};

export const renderRendererFunctionWithRetry = async ({
	payload,
	files,
	functionName,
	outdir,
	overallProgress,
	logLevel,
	onArtifact,
	requestHandler,
	expectedBucketOwner,
}: RenderRendererFunctionWithRetryInput): Promise<unknown> => {
	if (payload.type !== ServerlessRoutines.renderer) {
		throw new Error('Expected renderer type');
	}

	const transport = LambdaClientInternals.getRendererFunctionTransport(
		getCurrentRegionInFunction(),
	);
	const result = await (transport === 's3' ? s3Renderer : streamRenderer)({
		files,
		functionName,
		outdir,
		overallProgress,
		payload,
		logLevel,
		onArtifact,
		requestHandler,
		expectedBucketOwner,
	});

	if (result.type === 'error') {
		if (!result.shouldRetry || payload.retriesLeft <= 0) {
			throw new Error(result.error);
		}

		overallProgress.addRetry({
			attempt: payload.attempt + 1,
			time: Date.now(),
			chunk: payload.chunk,
		});

		return renderRendererFunctionWithRetry({
			files,
			functionName,
			outdir,
			overallProgress,
			payload: {
				...payload,
				attempt: payload.attempt + 1,
				retriesLeft: payload.retriesLeft - 1,
			},
			logLevel,
			onArtifact,
			requestHandler,
			expectedBucketOwner,
		});
	}
};