Skip to content

🎥 Make videos programmatically with React

@remotion/serverless: Recover from failed browser launches

anthod0cc020ec58fd2
diff --git a/packages/serverless/src/get-browser-instance.ts b/packages/serverless/src/get-browser-instance.ts
index 79ed853dac..3a63a1bbd7 100644
--- a/packages/serverless/src/get-browser-instance.ts
+++ b/packages/serverless/src/get-browser-instance.ts
@@ -128,51 +128,54 @@ export const getBrowserInstanceImplementation: GetBrowserInstance = async <
);
launching = true;
- const execPath = providerSpecifics.getChromiumPath();
-
- const instance = await RenderInternals.internalOpenBrowser({
- browser: 'chrome',
- browserExecutable: execPath,
- chromiumOptions: actualChromiumOptions,
- forceDeviceScaleFactor: undefined,
- indent: false,
- viewport: null,
- logLevel,
- onBrowserDownload: () => {
- throw new Error('Should not download a browser in serverless');
- },
- chromeMode: 'headless-shell',
- });
- const launchedBrowser = {
- instance,
- configurationString,
- };
- _browserInstance = launchedBrowser;
- instance.on('disconnected', () => {
- if (_browserInstance !== launchedBrowser) {
- return;
- }
-
- _browserInstance = null;
- RenderInternals.Log.info(
- {indent: false, logLevel},
- 'Browser disconnected or crashed.',
- );
- insideFunctionSpecifics.forgetBrowserEventLoop({
+ try {
+ const execPath = providerSpecifics.getChromiumPath();
+
+ const instance = await RenderInternals.internalOpenBrowser({
+ browser: 'chrome',
+ browserExecutable: execPath,
+ chromiumOptions: actualChromiumOptions,
+ forceDeviceScaleFactor: undefined,
+ indent: false,
+ viewport: null,
logLevel,
- launchedBrowser,
+ onBrowserDownload: () => {
+ throw new Error('Should not download a browser in serverless');
+ },
+ chromeMode: 'headless-shell',
});
- launchedBrowser.instance.close({silent: true}).catch((err) => {
+ const launchedBrowser = {
+ instance,
+ configurationString,
+ };
+ _browserInstance = launchedBrowser;
+ instance.on('disconnected', () => {
+ if (_browserInstance !== launchedBrowser) {
+ return;
+ }
+
+ _browserInstance = null;
RenderInternals.Log.info(
{indent: false, logLevel},
- 'Could not close browser instance',
- err,
+ 'Browser disconnected or crashed.',
);
+ insideFunctionSpecifics.forgetBrowserEventLoop({
+ logLevel,
+ launchedBrowser,
+ });
+ launchedBrowser.instance.close({silent: true}).catch((err) => {
+ RenderInternals.Log.info(
+ {indent: false, logLevel},
+ 'Could not close browser instance',
+ err,
+ );
+ });
});
- });
- launching = false;
- return launchedBrowser;
+ return launchedBrowser;
+ } finally {
+ launching = false;
+ }
}
if (_browserInstance.configurationString !== configurationString) {
diff --git a/packages/serverless/src/test/renderer-lifecycle.test.ts b/packages/serverless/src/test/renderer-lifecycle.test.ts
index 712303fe0e..15bd1dd8b1 100644
--- a/packages/serverless/src/test/renderer-lifecycle.test.ts
+++ b/packages/serverless/src/test/renderer-lifecycle.test.ts
@@ -1,4 +1,7 @@
import {expect, mock, spyOn, test} from 'bun:test';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
import {RenderInternals} from '@remotion/renderer';
import type {
CloudProvider,
@@ -22,77 +25,157 @@ type MockProvider = CloudProvider<
Record<string, never>
>;
-test('a flaky renderer invocation cannot terminate the next invocation', async () => {
- const previousNodeEnv = process.env.NODE_ENV;
- const scheduledCallbacks: Array<() => void> = [];
- let currentInvocation = 'A';
- const exits: string[] = [];
- const timeoutSpy = spyOn(globalThis, 'setTimeout').mockImplementation(((
- callback: (...args: never[]) => void,
- ) => {
- scheduledCallbacks.push(callback);
+test.each(['browser launch', 'Chromium path lookup'] as const)(
+ 'a failed %s allows the next renderer invocation to complete without exiting',
+ async (failure) => {
+ const previousNodeEnv = process.env.NODE_ENV;
+ const testRoot = fs.mkdtempSync(
+ path.join(os.tmpdir(), 'renderer-lifecycle-'),
+ );
+ const tmpDirSpy = spyOn(RenderInternals, 'tmpDir').mockImplementation(() =>
+ fs.mkdtempSync(path.join(testRoot, 'render-')),
+ );
+ const launchError = new Error(`Failed ${failure}`);
+ const browser = {
+ on: mock(() => undefined),
+ close: mock(() => Promise.resolve()),
+ };
+ // Fake Chromium startup and rendering: a real connection timeout is
+ // nondeterministic. Keep the handler and browser cache implementation real.
+ const openBrowserSpy = spyOn(RenderInternals, 'internalOpenBrowser');
+ if (failure === 'browser launch') {
+ openBrowserSpy.mockRejectedValueOnce(launchError);
+ }
- return 0 as unknown as ReturnType<typeof setTimeout>;
- }) as unknown as typeof setTimeout);
- const exitSpy = spyOn(process, 'exit').mockImplementation((() => {
- exits.push(currentInvocation);
- }) as typeof process.exit);
- const streamedMessages: StreamingPayload<MockProvider>[] = [];
+ openBrowserSpy.mockResolvedValue(browser as never);
+ const renderMediaSpy = spyOn(
+ RenderInternals,
+ 'internalRenderMedia',
+ ).mockResolvedValue({
+ slowestFrames: [],
+ } as never);
+ const onMediaFiles = mock(() => Promise.resolve());
+ const forgetBrowserEventLoop = mock(() => undefined);
+ let launchedBrowser: LaunchedBrowser | null = null;
+ const scheduledCallbacks: Array<() => void> = [];
+ let currentInvocation = 'A';
+ const exits: string[] = [];
+ const timeoutSpy = spyOn(globalThis, 'setTimeout').mockImplementation(((
+ callback: (...args: never[]) => void,
+ ) => {
+ scheduledCallbacks.push(callback);
- try {
- process.env.NODE_ENV = 'production';
- await rendererHandler<MockProvider>({
- params: {
- type: 'renderer',
- chromiumOptions: {gl: null},
- launchFunctionConfig: {version: `${VERSION}-mismatch`},
- logLevel: 'error',
- retriesLeft: 1,
- attempt: 1,
- chunk: 0,
- enableCancellation: false,
- } as never,
- options: {expectedBucketOwner: 'owner', isWarm: true},
- onStream: (message) => {
- streamedMessages.push(message);
- return Promise.resolve();
- },
- providerSpecifics: {
- isFlakyError: () => true,
- } as unknown as ProviderSpecifics<MockProvider>,
- requestContext: {
- awsRequestId: 'invocation-a',
- invokedFunctionArn: 'arn',
- getRemainingTimeInMillis: () => 120_000,
- },
- insideFunctionSpecifics:
- {} as unknown as InsideFunctionSpecifics<MockProvider>,
- onMediaFiles: null,
- executionMode: 'invoked',
- });
+ return 0 as unknown as ReturnType<typeof setTimeout>;
+ }) as unknown as typeof setTimeout);
+ const exitSpy = spyOn(process, 'exit').mockImplementation((() => {
+ exits.push(currentInvocation);
+ }) as typeof process.exit);
+ const streamedMessages: StreamingPayload<MockProvider>[] = [];
- expect(streamedMessages).toHaveLength(1);
- expect(streamedMessages[0]).toMatchObject({
- type: 'error-occurred',
- payload: {shouldRetry: true},
- });
+ try {
+ process.env.NODE_ENV = 'production';
+ const invoke = () =>
+ rendererHandler<MockProvider>({
+ params: {
+ type: 'renderer',
+ chromiumOptions: {gl: null},
+ launchFunctionConfig: {version: VERSION},
+ inputProps: {type: 'payload', payload: '{}'},
+ resolvedProps: {type: 'payload', payload: '{}'},
+ bucketName: 'bucket',
+ forcePathStyle: false,
+ frameRange: [0, 0],
+ everyNthFrame: 1,
+ codec: 'h264',
+ muted: true,
+ framesPerLambda: 1,
+ logLevel: 'error',
+ retriesLeft: 1,
+ attempt: 1,
+ chunk: 0,
+ enableCancellation: false,
+ } as never,
+ options: {expectedBucketOwner: 'owner', isWarm: true},
+ onStream: (message) => {
+ streamedMessages.push(message);
+ return Promise.resolve();
+ },
+ providerSpecifics: {
+ isFlakyError: (err: Error) => err === launchError,
+ getChromiumPath: () => {
+ if (
+ failure === 'Chromium path lookup' &&
+ currentInvocation === 'A'
+ ) {
+ throw launchError;
+ }
- currentInvocation = 'B';
- for (const callback of scheduledCallbacks) {
- callback();
- }
+ return '/mock-chrome';
+ },
+ } as unknown as ProviderSpecifics<MockProvider>,
+ requestContext: {
+ awsRequestId: currentInvocation,
+ invokedFunctionArn: 'arn',
+ getRemainingTimeInMillis: () => 120_000,
+ },
+ insideFunctionSpecifics: {
+ getCurrentRegionInFunction: () => 'mock-region',
+ getBrowserInstance: async (options) => {
+ launchedBrowser = await getBrowserInstanceImplementation(options);
+ return launchedBrowser;
+ },
+ closeBrowserInstance: closeBrowserInstanceImplementation,
+ forgetBrowserEventLoop,
+ timer: () => ({end: () => undefined}),
+ } satisfies Partial<
+ InsideFunctionSpecifics<MockProvider>
+ > as unknown as InsideFunctionSpecifics<MockProvider>,
+ onMediaFiles,
+ executionMode: 'invoked',
+ });
- expect(exits).toEqual([]);
- } finally {
- timeoutSpy.mockRestore();
- exitSpy.mockRestore();
- if (previousNodeEnv === undefined) {
- delete process.env.NODE_ENV;
- } else {
- process.env.NODE_ENV = previousNodeEnv;
+ await invoke();
+ expect(streamedMessages).toHaveLength(1);
+ expect(streamedMessages[0]).toMatchObject({
+ type: 'error-occurred',
+ payload: {shouldRetry: true, errorInfo: {message: launchError.message}},
+ });
+
+ currentInvocation = 'B';
+ const nextInvocation = invoke();
+ for (const callback of [...scheduledCallbacks]) {
+ callback();
+ }
+
+ await nextInvocation;
+ expect(streamedMessages).toHaveLength(2);
+ expect(streamedMessages[1]).toMatchObject({type: 'chunk-complete'});
+ expect(openBrowserSpy).toHaveBeenCalledTimes(
+ failure === 'browser launch' ? 2 : 1,
+ );
+ expect(renderMediaSpy).toHaveBeenCalledTimes(1);
+ expect(onMediaFiles).toHaveBeenCalledTimes(1);
+ expect(forgetBrowserEventLoop).toHaveBeenCalledTimes(1);
+ expect(exits).toEqual([]);
+ } finally {
+ if (launchedBrowser) {
+ await closeBrowserInstanceImplementation({launchedBrowser});
+ }
+
+ tmpDirSpy.mockRestore();
+ openBrowserSpy.mockRestore();
+ renderMediaSpy.mockRestore();
+ fs.rmSync(testRoot, {recursive: true, force: true});
+ timeoutSpy.mockRestore();
+ exitSpy.mockRestore();
+ if (previousNodeEnv === undefined) {
+ delete process.env.NODE_ENV;
+ } else {
+ process.env.NODE_ENV = previousNodeEnv;
+ }
}
- }
-});
+ },
+);
test('a flaky renderer invocation closes its browser before returning', async () => {
const previousNodeEnv = process.env.NODE_ENV;
@@ -113,6 +196,7 @@ test('a flaky renderer invocation closes its browser before returning', async ()
const closing = new Promise<void>((_resolve, reject) => {
failClosing = () => reject(new Error('Could not close browser'));
});
+ const streamedMessages: StreamingPayload<MockProvider>[] = [];
const closeBrowserInstance = mock(() => {
notifyCloseStarted();
return closing;
@@ -135,7 +219,10 @@ test('a flaky renderer invocation closes its browser before returning', async ()
enableCancellation: false,
} as never,
options: {expectedBucketOwner: 'owner', isWarm: true},
- onStream: () => Promise.resolve(),
+ onStream: (message) => {
+ streamedMessages.push(message);
+ return Promise.resolve();
+ },
providerSpecifics: {
isFlakyError: () => true,
} as unknown as ProviderSpecifics<MockProvider>,
@@ -174,6 +261,18 @@ test('a flaky renderer invocation closes its browser before returning', async ()
failClosing();
await handlerPromise;
expect(handlerReturned).toBe(true);
+ expect(streamedMessages).toHaveLength(1);
+ expect(streamedMessages[0]).toMatchObject({
+ type: 'error-occurred',
+ payload: {
+ shouldRetry: true,
+ errorInfo: {
+ message: 'must pass chunk',
+ willRetry: true,
+ isFatal: false,
+ },
+ },
+ });
} finally {
failClosing();
await Promise.resolve();