🎥 Make videos programmatically with React
`@remotion/serverless`: Render single-concurrency Lambda directly
Jonny Burger7d9e601df2d3
diff --git a/packages/lambda/src/test/integration/renders/muted-render.test.ts b/packages/lambda/src/test/integration/renders/muted-render.test.tsindex e2c7c3ac16..f4b08de97b 100644--- a/packages/lambda/src/test/integration/renders/muted-render.test.ts+++ b/packages/lambda/src/test/integration/renders/muted-render.test.ts@@ -4,11 +4,12 @@ import {tmpdir} from 'node:os';import path from 'path';import {LambdaClientInternals} from '@remotion/lambda-client';import {RenderInternals, getVideoMetadata} from '@remotion/renderer';-import {rendersPrefix} from '@remotion/serverless';+import {rendersPrefix, ServerlessRoutines} from '@remotion/serverless';import {mockImplementation} from '../../mocks/mock-implementation';+import {getMockInvocationTypesForRender} from '../../mocks/mock-invocations';import {simulateLambdaRender} from '../simulate-lambda-render';-test('Should make muted render audio', async () => {+test('Should directly make a muted render with one Lambda', async () => {const {close, file, progress, renderId} = await simulateLambdaRender({codec: 'h264',composition: 'framer',@@ -18,6 +19,7 @@ test('Should make muted render audio', async () => {region: 'eu-central-1',inputProps: {},muted: true,+ concurrency: 1,});const tmpfile = path.join(tmpdir(), 'out.mp4');@@ -40,6 +42,11 @@ test('Should make muted render audio', async () => {});expect(out.stdout).not.toContain('Audio');+ expect(getMockInvocationTypesForRender(renderId)).toEqual([+ ServerlessRoutines.launch,+ ]);+ expect(progress.renderMetadata?.estimatedTotalLambdaInvokations).toBe(1);+ expect(progress.renderMetadata?.estimatedRenderLambdaInvokations).toBe(0);unlinkSync(tmpfile);diff --git a/packages/lambda/src/test/mocks/aws-clients.ts b/packages/lambda/src/test/mocks/aws-clients.tsindex d71c1e1984..5585abca44 100644--- a/packages/lambda/src/test/mocks/aws-clients.ts+++ b/packages/lambda/src/test/mocks/aws-clients.ts@@ -24,6 +24,7 @@ import {makeStreamer} from '@remotion/streaming';import {Log} from '../../cli/log';import {mockServerImplementation} from '../mock-implementation';import {mockImplementation} from './mock-implementation';+import {trackMockInvocation} from './mock-invocations';export const getMockCallFunctionStreaming: CallFunctionStreaming<AwsProvider@@ -34,6 +35,7 @@ export const getMockCallFunctionStreaming: CallFunctionStreaming<getBinaryPayloadSink: GetBinaryPayloadSink | null;},) => {+ trackMockInvocation(params.payload);const responseStream = new ResponseStream();const {onData, clear} = makeStreamer(@@ -95,6 +97,7 @@ export const getMockCallFunctionAsync: CallFunctionAsync<AwsProvider> = async <>(params: CallFunctionOptions<T, AwsProvider>,) => {+ trackMockInvocation(params.payload);const responseStream = new ResponseStream();await innerHandler<AwsProvider>({context: {diff --git a/packages/lambda/src/test/mocks/mock-invocations.ts b/packages/lambda/src/test/mocks/mock-invocations.tsnew file mode 100644index 0000000000..e73d5feefe--- /dev/null+++ b/packages/lambda/src/test/mocks/mock-invocations.ts@@ -0,0 +1,20 @@+import type {ServerlessRoutines} from '@remotion/serverless';++const invocationTypesByRender = new Map<string, ServerlessRoutines[]>();++export const trackMockInvocation = (payload: {+ type: ServerlessRoutines;+ renderId?: string;+}) => {+ if (!payload.renderId) {+ return;+ }++ const invocations = invocationTypesByRender.get(payload.renderId) ?? [];+ invocations.push(payload.type);+ invocationTypesByRender.set(payload.renderId, invocations);+};++export const getMockInvocationTypesForRender = (renderId: string) => {+ return invocationTypesByRender.get(renderId) ?? [];+};diff --git a/packages/serverless/src/handlers/launch.ts b/packages/serverless/src/handlers/launch.tsindex 73a1142cbe..a5cf0c6828 100644--- a/packages/serverless/src/handlers/launch.ts+++ b/packages/serverless/src/handlers/launch.ts@@ -1,5 +1,6 @@import {existsSync, mkdirSync, rmSync} from 'fs';import {type EventEmitter} from 'node:events';+import {rename} from 'node:fs/promises';import {join} from 'path';/* eslint-disable @typescript-eslint/no-use-before-define */import type {LogOptions} from '@remotion/renderer';@@ -17,6 +18,7 @@ import {compressInputProps,CONCAT_FOLDER_TOKEN,decompressInputProps,+ deserializeArtifact,DOCS_URL,getExpectedOutName,getNeedsToUpload,@@ -45,11 +47,169 @@ import type {InsideFunctionSpecifics} from '../provider-implementation';import {removeOutnameCredentials} from '../remove-outname-credentials';import {renderRendererFunctionWithRetry} from '../stream-renderer';import {validateComposition} from '../validate-composition';+import type {RequestContext} from './renderer';+import {rendererHandler} from './renderer';import {sendTelemetryEvent} from './send-telemetry-event';type Options = {expectedBucketOwner: string;getRemainingTimeInMillis: () => number;+ requestContext: RequestContext | null;+};++const renderRendererFunctionDirectly = async <Provider extends CloudProvider>({+ payload,+ files,+ outdir,+ overallProgress,+ onArtifact,+ providerSpecifics,+ insideFunctionSpecifics,+ expectedBucketOwner,+ requestContext,+}: {+ payload: ServerlessPayload<Provider>;+ files: string[];+ outdir: string;+ overallProgress: OverallProgressHelper<Provider>;+ onArtifact: OnArtifactFromRenderer;+ providerSpecifics: ProviderSpecifics<Provider>;+ insideFunctionSpecifics: InsideFunctionSpecifics<Provider>;+ expectedBucketOwner: string;+ requestContext: RequestContext;+}): Promise<void> => {+ if (payload.type !== ServerlessRoutines.renderer) {+ throw new Error('Expected renderer type');+ }++ const rendererState: {+ error: {+ error: string;+ shouldRetry: boolean;+ } | null;+ } = {error: null};+ const renderedFiles: string[] = [];++ await rendererHandler({+ params: payload,+ options: {+ expectedBucketOwner,+ isWarm: true,+ },+ onStream: (message) => {+ if (message.type === 'lambda-invoked') {+ overallProgress.setLambdaInvoked(payload.chunk);+ return Promise.resolve();+ }++ if (message.type === 'frames-rendered') {+ overallProgress.setFrames({+ index: payload.chunk,+ encoded: message.payload.encoded,+ rendered: message.payload.rendered,+ });+ return Promise.resolve();+ }++ if (message.type === 'chunk-complete') {+ overallProgress.addChunkCompleted(+ payload.chunk,+ message.payload.start,+ message.payload.rendered,+ );+ return Promise.resolve();+ }++ if (message.type === 'artifact-emitted') {+ const artifact = deserializeArtifact(message.payload.artifact);+ const registration = onArtifact({+ artifact,+ chunk: payload.chunk,+ attempt: payload.attempt,+ });+ if (registration.type === 'conflict') {+ rendererState.error = {+ error: `Chunk ${payload.chunk} emitted an asset filename ${artifact.filename} at frame ${artifact.frame} but there is already another artifact with the same name. https://remotion.dev/docs/artifacts`,+ shouldRetry: false,+ };+ }++ return Promise.resolve();+ }++ if (message.type === 'error-occurred') {+ overallProgress.addErrorWithoutUpload(message.payload.errorInfo);+ overallProgress.setFrames({+ encoded: 0,+ index: payload.chunk,+ rendered: 0,+ });+ rendererState.error = {+ error: message.payload.error,+ shouldRetry: message.payload.shouldRetry,+ };+ return Promise.resolve();+ }++ throw new Error(`Unexpected ${message.type} event from direct renderer`);+ },+ requestContext,+ providerSpecifics,+ insideFunctionSpecifics,+ executionMode: 'direct',+ onMediaFiles: async ({+ videoOutputLocation,+ audioOutputLocation,+ isAudioOnly,+ }) => {+ const chunkName = `chunk:${String(payload.chunk).padStart(8, '0')}`;+ const videoDestination = join(outdir, `${chunkName}:video`);+ const audioDestination = join(outdir, `${chunkName}:audio`);++ if (isAudioOnly) {+ await rename(videoOutputLocation, audioDestination);+ renderedFiles.push(audioDestination);+ return;+ }++ await rename(videoOutputLocation, videoDestination);+ renderedFiles.push(videoDestination);+ if (audioOutputLocation) {+ await rename(audioOutputLocation, audioDestination);+ renderedFiles.push(audioDestination);+ }+ },+ });++ if (rendererState.error) {+ if (!rendererState.error.shouldRetry || payload.retriesLeft <= 0) {+ throw new Error(rendererState.error.error);+ }++ overallProgress.addRetry({+ attempt: payload.attempt + 1,+ time: Date.now(),+ chunk: payload.chunk,+ });+ await renderRendererFunctionDirectly({+ payload: {+ ...payload,+ attempt: payload.attempt + 1,+ retriesLeft: payload.retriesLeft - 1,+ },+ files,+ outdir,+ overallProgress,+ onArtifact,+ providerSpecifics,+ insideFunctionSpecifics,+ expectedBucketOwner,+ requestContext,+ });+ return;+ }++ files.push(...renderedFiles);};const innerLaunchHandler = async <Provider extends CloudProvider>({@@ -334,17 +494,21 @@ const innerLaunchHandler = async <Provider extends CloudProvider>({const rendererFunctionName =params.rendererFunctionName ??insideFunctionSpecifics.getCurrentFunctionName();+ const directRequestContext =+ params.concurrency === 1 && params.rendererFunctionName === null+ ? options.requestContext+ : null;+ const shouldRenderDirectly = directRequestContext !== null;const renderMetadata: RenderMetadata<Provider> = {startedDate,totalChunks: chunks.length,estimatedTotalLambdaInvokations: [- // Direct invokations- chunks.length,+ shouldRenderDirectly ? 0 : chunks.length,// This function1,].reduce((a, b) => a + b, 0),- estimatedRenderLambdaInvokations: chunks.length,+ estimatedRenderLambdaInvokations: shouldRenderDirectly ? 0 : chunks.length,compositionId: comp.id,siteId: params.serveUrl,codec: params.codec,@@ -499,23 +663,37 @@ const innerLaunchHandler = async <Provider extends CloudProvider>({return artifactRegistration;};- await Promise.all(- lambdaPayloads.map(async (payload) => {- await renderRendererFunctionWithRetry({- files,- functionName: rendererFunctionName,- outdir,- overallProgress,- payload,- logLevel: params.logLevel,- onArtifact,- providerSpecifics,- insideFunctionSpecifics,- requestHandler: null,- expectedBucketOwner: options.expectedBucketOwner,- });- }),- );+ if (directRequestContext) {+ await renderRendererFunctionDirectly({+ payload: lambdaPayloads[0],+ files,+ outdir,+ overallProgress,+ onArtifact,+ providerSpecifics,+ insideFunctionSpecifics,+ expectedBucketOwner: options.expectedBucketOwner,+ requestContext: directRequestContext,+ });+ } else {+ await Promise.all(+ lambdaPayloads.map(async (payload) => {+ await renderRendererFunctionWithRetry({+ files,+ functionName: rendererFunctionName,+ outdir,+ overallProgress,+ payload,+ logLevel: params.logLevel,+ onArtifact,+ providerSpecifics,+ insideFunctionSpecifics,+ requestHandler: null,+ expectedBucketOwner: options.expectedBucketOwner,+ });+ }),+ );+ }const postRenderData = await mergeChunksAndFinishRender({bucketName: params.bucketName,diff --git a/packages/serverless/src/handlers/renderer.ts b/packages/serverless/src/handlers/renderer.tsindex 00e801e83c..a07fb75f23 100644--- a/packages/serverless/src/handlers/renderer.ts+++ b/packages/serverless/src/handlers/renderer.ts@@ -445,9 +445,9 @@ const renderHandler = async <Provider extends CloudProvider>({await Promise.all([- fs.promises.rm(videoOutputLocation, {recursive: true}),+ fs.promises.rm(videoOutputLocation, {recursive: true, force: true}),audioOutputLocation- ? fs.promises.rm(audioOutputLocation, {recursive: true})+ ? fs.promises.rm(audioOutputLocation, {recursive: true, force: true}): null,fs.promises.rm(outputPath, {recursive: true}),].filter(truthy),@@ -470,6 +470,7 @@ export const rendererHandler = async <Provider extends CloudProvider>({requestContext,insideFunctionSpecifics,onMediaFiles,+ executionMode,}: {params: ServerlessPayload<Provider>;options: Options;@@ -485,6 +486,7 @@ export const rendererHandler = async <Provider extends CloudProvider>({completedAt: number;}) => Promise<void>)| null;+ executionMode: 'invoked' | 'direct';}): Promise<void> => {if (params.type !== ServerlessRoutines.renderer) {throw new Error('Params must be renderer');@@ -581,7 +583,11 @@ export const rendererHandler = async <Provider extends CloudProvider>({});} finally {stopCancellationPolling();- if (shouldKeepBrowserOpen && instance) {+ if (executionMode === 'direct') {+ if (!shouldKeepBrowserOpen && instance) {+ await instance.instance.close({silent: true});+ }+ } else if (shouldKeepBrowserOpen && instance) {insideFunctionSpecifics.forgetBrowserEventLoop({logLevel: params.logLevel,launchedBrowser: instance,diff --git a/packages/serverless/src/inner-routine.ts b/packages/serverless/src/inner-routine.tsindex 0ad6b945a5..3b5b241290 100644--- a/packages/serverless/src/inner-routine.ts+++ b/packages/serverless/src/inner-routine.ts@@ -222,6 +222,7 @@ export const innerHandler = async <Provider extends CloudProvider>({options: {expectedBucketOwner: currentUserId,getRemainingTimeInMillis: context.getRemainingTimeInMillis,+ requestContext: context,},providerSpecifics,insideFunctionSpecifics,@@ -320,6 +321,7 @@ export const innerHandler = async <Provider extends CloudProvider>({});},onMediaFiles: s3Output?.uploadMediaAndComplete ?? null,+ executionMode: 'invoked',requestContext: context,providerSpecifics,insideFunctionSpecifics,