Skip to content

🎥 Make videos programmatically with React

@remotion/serverless: Fix Lambda browser cleanup lifecycle

anthod0f4e477bfb758
diff --git a/packages/lambda/src/functions/aws-server-implementation.ts b/packages/lambda/src/functions/aws-server-implementation.ts
index 62b6b5c304..f6ed501fd4 100644
--- a/packages/lambda/src/functions/aws-server-implementation.ts
+++ b/packages/lambda/src/functions/aws-server-implementation.ts
@@ -1,6 +1,7 @@
import {LambdaClientInternals, type AwsProvider} from '@remotion/lambda-client';
import type {InsideFunctionSpecifics} from '@remotion/serverless';
import {
+ closeBrowserInstanceImplementation,
forgetBrowserEventLoopImplementation,
getBrowserInstanceImplementation,
invokeWebhook,
@@ -17,6 +18,7 @@ export const serverAwsImplementation: InsideFunctionSpecifics<AwsProvider> = {
? 'veryfast'
: null,
forgetBrowserEventLoop: forgetBrowserEventLoopImplementation,
+ closeBrowserInstance: closeBrowserInstanceImplementation,
getBrowserInstance: getBrowserInstanceImplementation,
timer,
getCurrentRegionInFunction: getCurrentRegionInFunctionImplementation,
diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts
index 7a498c5341..efbbcf5528 100644
--- a/packages/lambda/src/test/mock-implementation.ts
+++ b/packages/lambda/src/test/mock-implementation.ts
@@ -44,6 +44,10 @@ export const mockServerImplementation: InsideFunctionSpecifics<AwsProvider> = {
);
launchedBrowser.instance.close({silent: false});
},
+ closeBrowserInstance: async ({launchedBrowser}) => {
+ browsersOpen.delete(launchedBrowser.instance.id);
+ await launchedBrowser.instance.close({silent: true});
+ },
getCurrentRegionInFunction: () => 'eu-central-1',
getBrowserInstance,
timer: () => ({
diff --git a/packages/serverless/src/get-browser-instance.ts b/packages/serverless/src/get-browser-instance.ts
index b0835b5124..79ed853dac 100644
--- a/packages/serverless/src/get-browser-instance.ts
+++ b/packages/serverless/src/get-browser-instance.ts
@@ -7,6 +7,7 @@ import type {
} from '@remotion/serverless-client';
import {VERSION} from '@remotion/serverless-client';
import type {
+ CloseBrowserInstance,
ForgetBrowserEventLoop,
GetBrowserInstance,
InsideFunctionSpecifics,
@@ -64,6 +65,16 @@ export const forgetBrowserEventLoopImplementation: ForgetBrowserEventLoop = ({
launchedBrowser.instance.runner.deleteBrowserCaches();
};
+export const closeBrowserInstanceImplementation: CloseBrowserInstance = async ({
+ launchedBrowser,
+}) => {
+ if (_browserInstance === launchedBrowser) {
+ _browserInstance = null;
+ }
+
+ await launchedBrowser.instance.close({silent: true});
+};
+
export const getBrowserInstanceImplementation: GetBrowserInstance = async <
Provider extends CloudProvider,
>({
@@ -132,31 +143,36 @@ export const getBrowserInstanceImplementation: GetBrowserInstance = async <
},
chromeMode: 'headless-shell',
});
+ const launchedBrowser = {
+ instance,
+ configurationString,
+ };
+ _browserInstance = launchedBrowser;
instance.on('disconnected', () => {
+ if (_browserInstance !== launchedBrowser) {
+ return;
+ }
+
+ _browserInstance = null;
RenderInternals.Log.info(
{indent: false, logLevel},
'Browser disconnected or crashed.',
);
insideFunctionSpecifics.forgetBrowserEventLoop({
logLevel,
- launchedBrowser: _browserInstance as LaunchedBrowser,
+ launchedBrowser,
});
- _browserInstance?.instance?.close({silent: true}).catch((err) => {
+ launchedBrowser.instance.close({silent: true}).catch((err) => {
RenderInternals.Log.info(
{indent: false, logLevel},
'Could not close browser instance',
err,
);
});
- _browserInstance = null;
});
- _browserInstance = {
- instance,
- configurationString,
- };
launching = false;
- return _browserInstance;
+ return launchedBrowser;
}
if (_browserInstance.configurationString !== configurationString) {
diff --git a/packages/serverless/src/handlers/renderer.ts b/packages/serverless/src/handlers/renderer.ts
index a07fb75f23..ce45c9a56a 100644
--- a/packages/serverless/src/handlers/renderer.ts
+++ b/packages/serverless/src/handlers/renderer.ts
@@ -583,32 +583,34 @@ export const rendererHandler = async <Provider extends CloudProvider>({
});
} finally {
stopCancellationPolling();
- if (executionMode === 'direct') {
- if (!shouldKeepBrowserOpen && instance) {
- await instance.instance.close({silent: true});
+ if (!shouldKeepBrowserOpen && instance) {
+ try {
+ await insideFunctionSpecifics.closeBrowserInstance({
+ launchedBrowser: instance,
+ });
+ } catch (err) {
+ RenderInternals.Log.info(
+ {indent: false, logLevel: params.logLevel},
+ 'Could not close browser instance after flaky error',
+ err,
+ );
}
- } else if (shouldKeepBrowserOpen && instance) {
+ } else if (
+ executionMode === 'invoked' &&
+ shouldKeepBrowserOpen &&
+ instance
+ ) {
insideFunctionSpecifics.forgetBrowserEventLoop({
logLevel: params.logLevel,
launchedBrowser: instance,
});
- } else {
+ }
+
+ if (!shouldKeepBrowserOpen) {
RenderInternals.Log.info(
{indent: false, logLevel: params.logLevel},
'Function did not succeed with flaky error, not keeping browser open.',
);
- RenderInternals.Log.info(
- {indent: false, logLevel: params.logLevel},
- 'Waiting 2 seconds to allow for response to be sent',
- );
-
- setTimeout(() => {
- RenderInternals.Log.info(
- {indent: false, logLevel: params.logLevel},
- 'Quitting Function forcefully now to force not keeping the Function warm.',
- );
- process.exit(0);
- }, 2000);
}
if (ENABLE_SLOW_LEAK_DETECTION) {
diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts
index f43d4bc32b..b60f7b9f95 100644
--- a/packages/serverless/src/index.ts
+++ b/packages/serverless/src/index.ts
@@ -1,5 +1,6 @@
export * from '@remotion/serverless-client';
export {
+ closeBrowserInstanceImplementation,
forgetBrowserEventLoopImplementation,
getBrowserInstanceImplementation,
} from './get-browser-instance';
diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts
index cc4651a451..9b0fac866f 100644
--- a/packages/serverless/src/provider-implementation.ts
+++ b/packages/serverless/src/provider-implementation.ts
@@ -49,6 +49,10 @@ export type ForgetBrowserEventLoop = (options: {
launchedBrowser: LaunchedBrowser;
}) => void;
+export type CloseBrowserInstance = (options: {
+ launchedBrowser: LaunchedBrowser;
+}) => Promise<void>;
+
export type GenerateRenderId = (options: {
deleteAfter: DeleteAfter | null;
randomHashFn: () => string;
@@ -123,6 +127,7 @@ export type InsideFunctionSpecifics<Provider extends CloudProvider> = {
defaultX264Preset: X264Preset | null;
getBrowserInstance: GetBrowserInstance;
forgetBrowserEventLoop: ForgetBrowserEventLoop;
+ closeBrowserInstance: CloseBrowserInstance;
timer: DebuggingTimer;
generateRandomId: GenerateRenderId;
deleteTmpDir: () => Promise<void>;
diff --git a/packages/serverless/src/test/renderer-lifecycle.test.ts b/packages/serverless/src/test/renderer-lifecycle.test.ts
new file mode 100644
index 0000000000..712303fe0e
--- /dev/null
+++ b/packages/serverless/src/test/renderer-lifecycle.test.ts
@@ -0,0 +1,256 @@
+import {expect, mock, spyOn, test} from 'bun:test';
+import {RenderInternals} from '@remotion/renderer';
+import type {
+ CloudProvider,
+ ProviderSpecifics,
+ StreamingPayload,
+} from '@remotion/serverless-client';
+import {VERSION} from '@remotion/serverless-client';
+import {
+ closeBrowserInstanceImplementation,
+ getBrowserInstanceImplementation,
+} from '../get-browser-instance';
+import type {LaunchedBrowser} from '../get-browser-instance';
+import {rendererHandler} from '../handlers/renderer';
+import type {InsideFunctionSpecifics} from '../provider-implementation';
+
+type MockProvider = CloudProvider<
+ 'mock-region',
+ Record<string, never>,
+ Record<string, never>,
+ 'normal',
+ Record<string, never>
+>;
+
+test('a flaky renderer invocation cannot terminate the next invocation', async () => {
+ const previousNodeEnv = process.env.NODE_ENV;
+ const scheduledCallbacks: Array<() => void> = [];
+ let currentInvocation = 'A';
+ const exits: string[] = [];
+ const timeoutSpy = spyOn(globalThis, 'setTimeout').mockImplementation(((
+ callback: (...args: never[]) => void,
+ ) => {
+ scheduledCallbacks.push(callback);
+
+ return 0 as unknown as ReturnType<typeof setTimeout>;
+ }) as unknown as typeof setTimeout);
+ const exitSpy = spyOn(process, 'exit').mockImplementation((() => {
+ exits.push(currentInvocation);
+ }) as typeof process.exit);
+ const streamedMessages: StreamingPayload<MockProvider>[] = [];
+
+ try {
+ process.env.NODE_ENV = 'production';
+ await rendererHandler<MockProvider>({
+ params: {
+ type: 'renderer',
+ chromiumOptions: {gl: null},
+ launchFunctionConfig: {version: `${VERSION}-mismatch`},
+ logLevel: 'error',
+ retriesLeft: 1,
+ attempt: 1,
+ chunk: 0,
+ enableCancellation: false,
+ } as never,
+ options: {expectedBucketOwner: 'owner', isWarm: true},
+ onStream: (message) => {
+ streamedMessages.push(message);
+ return Promise.resolve();
+ },
+ providerSpecifics: {
+ isFlakyError: () => true,
+ } as unknown as ProviderSpecifics<MockProvider>,
+ requestContext: {
+ awsRequestId: 'invocation-a',
+ invokedFunctionArn: 'arn',
+ getRemainingTimeInMillis: () => 120_000,
+ },
+ insideFunctionSpecifics:
+ {} as unknown as InsideFunctionSpecifics<MockProvider>,
+ onMediaFiles: null,
+ executionMode: 'invoked',
+ });
+
+ expect(streamedMessages).toHaveLength(1);
+ expect(streamedMessages[0]).toMatchObject({
+ type: 'error-occurred',
+ payload: {shouldRetry: true},
+ });
+
+ currentInvocation = 'B';
+ for (const callback of scheduledCallbacks) {
+ callback();
+ }
+
+ expect(exits).toEqual([]);
+ } finally {
+ timeoutSpy.mockRestore();
+ exitSpy.mockRestore();
+ if (previousNodeEnv === undefined) {
+ delete process.env.NODE_ENV;
+ } else {
+ process.env.NODE_ENV = previousNodeEnv;
+ }
+ }
+});
+
+test('a flaky renderer invocation closes its browser before returning', async () => {
+ const previousNodeEnv = process.env.NODE_ENV;
+ const tmpDirSpy = spyOn(RenderInternals, 'tmpDir').mockImplementation(
+ () => '/tmp/remotion-renderer-lifecycle-test',
+ );
+ const browser = {
+ instance: {
+ close: mock(() => Promise.resolve()),
+ },
+ configurationString: 'configuration',
+ };
+ let notifyCloseStarted: () => void = () => undefined;
+ const closeStarted = new Promise<void>((resolve) => {
+ notifyCloseStarted = resolve;
+ });
+ let failClosing: () => void = () => undefined;
+ const closing = new Promise<void>((_resolve, reject) => {
+ failClosing = () => reject(new Error('Could not close browser'));
+ });
+ const closeBrowserInstance = mock(() => {
+ notifyCloseStarted();
+ return closing;
+ });
+
+ try {
+ process.env.NODE_ENV = 'production';
+ const handlerPromise = rendererHandler<MockProvider>({
+ params: {
+ type: 'renderer',
+ chromiumOptions: {gl: null},
+ launchFunctionConfig: {version: VERSION},
+ inputProps: {type: 'payload', payload: '{}'},
+ resolvedProps: {type: 'payload', payload: '{}'},
+ bucketName: 'bucket',
+ forcePathStyle: false,
+ logLevel: 'error',
+ retriesLeft: 1,
+ attempt: 1,
+ enableCancellation: false,
+ } as never,
+ options: {expectedBucketOwner: 'owner', isWarm: true},
+ onStream: () => Promise.resolve(),
+ providerSpecifics: {
+ isFlakyError: () => true,
+ } as unknown as ProviderSpecifics<MockProvider>,
+ requestContext: {
+ awsRequestId: 'invocation-a',
+ invokedFunctionArn: 'arn',
+ getRemainingTimeInMillis: () => 120_000,
+ },
+ insideFunctionSpecifics: {
+ getCurrentRegionInFunction: () => 'mock-region',
+ getBrowserInstance: () => Promise.resolve(browser as never),
+ closeBrowserInstance,
+ } as unknown as InsideFunctionSpecifics<MockProvider>,
+ onMediaFiles: null,
+ executionMode: 'invoked',
+ });
+
+ expect(
+ await Promise.race([
+ closeStarted.then(() => 'close-started'),
+ handlerPromise.then(() => 'handler-returned'),
+ ]),
+ ).toBe('close-started');
+ expect(closeBrowserInstance).toHaveBeenCalledTimes(1);
+ expect(closeBrowserInstance).toHaveBeenCalledWith({
+ launchedBrowser: browser,
+ });
+
+ let handlerReturned = false;
+ handlerPromise.then(() => {
+ handlerReturned = true;
+ });
+ await Promise.resolve();
+ expect(handlerReturned).toBe(false);
+
+ failClosing();
+ await handlerPromise;
+ expect(handlerReturned).toBe(true);
+ } finally {
+ failClosing();
+ await Promise.resolve();
+ tmpDirSpy.mockRestore();
+ if (previousNodeEnv === undefined) {
+ delete process.env.NODE_ENV;
+ } else {
+ process.env.NODE_ENV = previousNodeEnv;
+ }
+ }
+});
+
+test('closing a cached browser makes the next invocation launch a new one', async () => {
+ const makeBrowser = (id: string) => {
+ let onDisconnected: (() => void) | null = null;
+
+ return {
+ id,
+ on: mock((event: string, callback: () => void) => {
+ if (event === 'disconnected') {
+ onDisconnected = callback;
+ }
+ }),
+ close: mock(() => {
+ onDisconnected?.();
+ return Promise.resolve();
+ }),
+ };
+ };
+
+ const firstBrowser = makeBrowser('first');
+ const secondBrowser = makeBrowser('second');
+ const openBrowserSpy = spyOn(
+ RenderInternals,
+ 'internalOpenBrowser',
+ ).mockImplementationOnce(() => Promise.resolve(firstBrowser as never));
+ openBrowserSpy.mockImplementationOnce(() =>
+ Promise.resolve(secondBrowser as never),
+ );
+ const providerSpecifics = {
+ getChromiumPath: () => '/mock-chrome',
+ } as unknown as ProviderSpecifics<MockProvider>;
+ const insideFunctionSpecifics = {
+ getBrowserInstance: getBrowserInstanceImplementation,
+ } as unknown as InsideFunctionSpecifics<MockProvider>;
+ let first: LaunchedBrowser | null = null;
+ let second: LaunchedBrowser | null = null;
+
+ try {
+ first = await getBrowserInstanceImplementation({
+ logLevel: 'error',
+ indent: false,
+ chromiumOptions: {},
+ providerSpecifics,
+ insideFunctionSpecifics,
+ });
+ await closeBrowserInstanceImplementation({launchedBrowser: first});
+ second = await getBrowserInstanceImplementation({
+ logLevel: 'error',
+ indent: false,
+ chromiumOptions: {},
+ providerSpecifics,
+ insideFunctionSpecifics,
+ });
+
+ expect(openBrowserSpy).toHaveBeenCalledTimes(2);
+ expect(first.instance as unknown).toBe(firstBrowser);
+ expect(second.instance as unknown).toBe(secondBrowser);
+ } finally {
+ if (second) {
+ await closeBrowserInstanceImplementation({launchedBrowser: second});
+ }
+
+ if (first) {
+ await closeBrowserInstanceImplementation({launchedBrowser: first});
+ }
+
+ openBrowserSpy.mockRestore();
+ }
+});