🎥 Make videos programmatically with React
@remotion/serverless: Recover from failed browser launches
anthod0cc020ec58fd2
diff --git a/packages/serverless/src/get-browser-instance.ts b/packages/serverless/src/get-browser-instance.tsindex 79ed853dac..3a63a1bbd7 100644--- a/packages/serverless/src/get-browser-instance.ts+++ b/packages/serverless/src/get-browser-instance.ts@@ -128,51 +128,54 @@ export const getBrowserInstanceImplementation: GetBrowserInstance = async <);launching = true;- const execPath = providerSpecifics.getChromiumPath();-- const instance = await RenderInternals.internalOpenBrowser({- browser: 'chrome',- browserExecutable: execPath,- chromiumOptions: actualChromiumOptions,- forceDeviceScaleFactor: undefined,- indent: false,- viewport: null,- logLevel,- onBrowserDownload: () => {- throw new Error('Should not download a browser in serverless');- },- chromeMode: 'headless-shell',- });- const launchedBrowser = {- instance,- configurationString,- };- _browserInstance = launchedBrowser;- instance.on('disconnected', () => {- if (_browserInstance !== launchedBrowser) {- return;- }-- _browserInstance = null;- RenderInternals.Log.info(- {indent: false, logLevel},- 'Browser disconnected or crashed.',- );- insideFunctionSpecifics.forgetBrowserEventLoop({+ try {+ const execPath = providerSpecifics.getChromiumPath();++ const instance = await RenderInternals.internalOpenBrowser({+ browser: 'chrome',+ browserExecutable: execPath,+ chromiumOptions: actualChromiumOptions,+ forceDeviceScaleFactor: undefined,+ indent: false,+ viewport: null,logLevel,- launchedBrowser,+ onBrowserDownload: () => {+ throw new Error('Should not download a browser in serverless');+ },+ chromeMode: 'headless-shell',});- launchedBrowser.instance.close({silent: true}).catch((err) => {+ const launchedBrowser = {+ instance,+ configurationString,+ };+ _browserInstance = launchedBrowser;+ instance.on('disconnected', () => {+ if (_browserInstance !== launchedBrowser) {+ return;+ }++ _browserInstance = null;RenderInternals.Log.info({indent: false, logLevel},- 'Could not close browser instance',- err,+ 'Browser disconnected or crashed.',);+ insideFunctionSpecifics.forgetBrowserEventLoop({+ logLevel,+ launchedBrowser,+ });+ launchedBrowser.instance.close({silent: true}).catch((err) => {+ RenderInternals.Log.info(+ {indent: false, logLevel},+ 'Could not close browser instance',+ err,+ );+ });});- });- launching = false;- return launchedBrowser;+ return launchedBrowser;+ } finally {+ launching = false;+ }}if (_browserInstance.configurationString !== configurationString) {diff --git a/packages/serverless/src/test/renderer-lifecycle.test.ts b/packages/serverless/src/test/renderer-lifecycle.test.tsindex 712303fe0e..15bd1dd8b1 100644--- a/packages/serverless/src/test/renderer-lifecycle.test.ts+++ b/packages/serverless/src/test/renderer-lifecycle.test.ts@@ -1,4 +1,7 @@import {expect, mock, spyOn, test} from 'bun:test';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';import {RenderInternals} from '@remotion/renderer';import type {CloudProvider,@@ -22,77 +25,157 @@ type MockProvider = CloudProvider<Record<string, never>>;-test('a flaky renderer invocation cannot terminate the next invocation', async () => {- const previousNodeEnv = process.env.NODE_ENV;- const scheduledCallbacks: Array<() => void> = [];- let currentInvocation = 'A';- const exits: string[] = [];- const timeoutSpy = spyOn(globalThis, 'setTimeout').mockImplementation(((- callback: (...args: never[]) => void,- ) => {- scheduledCallbacks.push(callback);+test.each(['browser launch', 'Chromium path lookup'] as const)(+ 'a failed %s allows the next renderer invocation to complete without exiting',+ async (failure) => {+ const previousNodeEnv = process.env.NODE_ENV;+ const testRoot = fs.mkdtempSync(+ path.join(os.tmpdir(), 'renderer-lifecycle-'),+ );+ const tmpDirSpy = spyOn(RenderInternals, 'tmpDir').mockImplementation(() =>+ fs.mkdtempSync(path.join(testRoot, 'render-')),+ );+ const launchError = new Error(`Failed ${failure}`);+ const browser = {+ on: mock(() => undefined),+ close: mock(() => Promise.resolve()),+ };+ // Fake Chromium startup and rendering: a real connection timeout is+ // nondeterministic. Keep the handler and browser cache implementation real.+ const openBrowserSpy = spyOn(RenderInternals, 'internalOpenBrowser');+ if (failure === 'browser launch') {+ openBrowserSpy.mockRejectedValueOnce(launchError);+ }- return 0 as unknown as ReturnType<typeof setTimeout>;- }) as unknown as typeof setTimeout);- const exitSpy = spyOn(process, 'exit').mockImplementation((() => {- exits.push(currentInvocation);- }) as typeof process.exit);- const streamedMessages: StreamingPayload<MockProvider>[] = [];+ openBrowserSpy.mockResolvedValue(browser as never);+ const renderMediaSpy = spyOn(+ RenderInternals,+ 'internalRenderMedia',+ ).mockResolvedValue({+ slowestFrames: [],+ } as never);+ const onMediaFiles = mock(() => Promise.resolve());+ const forgetBrowserEventLoop = mock(() => undefined);+ let launchedBrowser: LaunchedBrowser | null = null;+ const scheduledCallbacks: Array<() => void> = [];+ let currentInvocation = 'A';+ const exits: string[] = [];+ const timeoutSpy = spyOn(globalThis, 'setTimeout').mockImplementation(((+ callback: (...args: never[]) => void,+ ) => {+ scheduledCallbacks.push(callback);- try {- process.env.NODE_ENV = 'production';- await rendererHandler<MockProvider>({- params: {- type: 'renderer',- chromiumOptions: {gl: null},- launchFunctionConfig: {version: `${VERSION}-mismatch`},- logLevel: 'error',- retriesLeft: 1,- attempt: 1,- chunk: 0,- enableCancellation: false,- } as never,- options: {expectedBucketOwner: 'owner', isWarm: true},- onStream: (message) => {- streamedMessages.push(message);- return Promise.resolve();- },- providerSpecifics: {- isFlakyError: () => true,- } as unknown as ProviderSpecifics<MockProvider>,- requestContext: {- awsRequestId: 'invocation-a',- invokedFunctionArn: 'arn',- getRemainingTimeInMillis: () => 120_000,- },- insideFunctionSpecifics:- {} as unknown as InsideFunctionSpecifics<MockProvider>,- onMediaFiles: null,- executionMode: 'invoked',- });+ return 0 as unknown as ReturnType<typeof setTimeout>;+ }) as unknown as typeof setTimeout);+ const exitSpy = spyOn(process, 'exit').mockImplementation((() => {+ exits.push(currentInvocation);+ }) as typeof process.exit);+ const streamedMessages: StreamingPayload<MockProvider>[] = [];- expect(streamedMessages).toHaveLength(1);- expect(streamedMessages[0]).toMatchObject({- type: 'error-occurred',- payload: {shouldRetry: true},- });+ try {+ process.env.NODE_ENV = 'production';+ const invoke = () =>+ rendererHandler<MockProvider>({+ params: {+ type: 'renderer',+ chromiumOptions: {gl: null},+ launchFunctionConfig: {version: VERSION},+ inputProps: {type: 'payload', payload: '{}'},+ resolvedProps: {type: 'payload', payload: '{}'},+ bucketName: 'bucket',+ forcePathStyle: false,+ frameRange: [0, 0],+ everyNthFrame: 1,+ codec: 'h264',+ muted: true,+ framesPerLambda: 1,+ logLevel: 'error',+ retriesLeft: 1,+ attempt: 1,+ chunk: 0,+ enableCancellation: false,+ } as never,+ options: {expectedBucketOwner: 'owner', isWarm: true},+ onStream: (message) => {+ streamedMessages.push(message);+ return Promise.resolve();+ },+ providerSpecifics: {+ isFlakyError: (err: Error) => err === launchError,+ getChromiumPath: () => {+ if (+ failure === 'Chromium path lookup' &&+ currentInvocation === 'A'+ ) {+ throw launchError;+ }- currentInvocation = 'B';- for (const callback of scheduledCallbacks) {- callback();- }+ return '/mock-chrome';+ },+ } as unknown as ProviderSpecifics<MockProvider>,+ requestContext: {+ awsRequestId: currentInvocation,+ invokedFunctionArn: 'arn',+ getRemainingTimeInMillis: () => 120_000,+ },+ insideFunctionSpecifics: {+ getCurrentRegionInFunction: () => 'mock-region',+ getBrowserInstance: async (options) => {+ launchedBrowser = await getBrowserInstanceImplementation(options);+ return launchedBrowser;+ },+ closeBrowserInstance: closeBrowserInstanceImplementation,+ forgetBrowserEventLoop,+ timer: () => ({end: () => undefined}),+ } satisfies Partial<+ InsideFunctionSpecifics<MockProvider>+ > as unknown as InsideFunctionSpecifics<MockProvider>,+ onMediaFiles,+ executionMode: 'invoked',+ });- expect(exits).toEqual([]);- } finally {- timeoutSpy.mockRestore();- exitSpy.mockRestore();- if (previousNodeEnv === undefined) {- delete process.env.NODE_ENV;- } else {- process.env.NODE_ENV = previousNodeEnv;+ await invoke();+ expect(streamedMessages).toHaveLength(1);+ expect(streamedMessages[0]).toMatchObject({+ type: 'error-occurred',+ payload: {shouldRetry: true, errorInfo: {message: launchError.message}},+ });++ currentInvocation = 'B';+ const nextInvocation = invoke();+ for (const callback of [...scheduledCallbacks]) {+ callback();+ }++ await nextInvocation;+ expect(streamedMessages).toHaveLength(2);+ expect(streamedMessages[1]).toMatchObject({type: 'chunk-complete'});+ expect(openBrowserSpy).toHaveBeenCalledTimes(+ failure === 'browser launch' ? 2 : 1,+ );+ expect(renderMediaSpy).toHaveBeenCalledTimes(1);+ expect(onMediaFiles).toHaveBeenCalledTimes(1);+ expect(forgetBrowserEventLoop).toHaveBeenCalledTimes(1);+ expect(exits).toEqual([]);+ } finally {+ if (launchedBrowser) {+ await closeBrowserInstanceImplementation({launchedBrowser});+ }++ tmpDirSpy.mockRestore();+ openBrowserSpy.mockRestore();+ renderMediaSpy.mockRestore();+ fs.rmSync(testRoot, {recursive: true, force: true});+ timeoutSpy.mockRestore();+ exitSpy.mockRestore();+ if (previousNodeEnv === undefined) {+ delete process.env.NODE_ENV;+ } else {+ process.env.NODE_ENV = previousNodeEnv;+ }}- }-});+ },+);test('a flaky renderer invocation closes its browser before returning', async () => {const previousNodeEnv = process.env.NODE_ENV;@@ -113,6 +196,7 @@ test('a flaky renderer invocation closes its browser before returning', async ()const closing = new Promise<void>((_resolve, reject) => {failClosing = () => reject(new Error('Could not close browser'));});+ const streamedMessages: StreamingPayload<MockProvider>[] = [];const closeBrowserInstance = mock(() => {notifyCloseStarted();return closing;@@ -135,7 +219,10 @@ test('a flaky renderer invocation closes its browser before returning', async ()enableCancellation: false,} as never,options: {expectedBucketOwner: 'owner', isWarm: true},- onStream: () => Promise.resolve(),+ onStream: (message) => {+ streamedMessages.push(message);+ return Promise.resolve();+ },providerSpecifics: {isFlakyError: () => true,} as unknown as ProviderSpecifics<MockProvider>,@@ -174,6 +261,18 @@ test('a flaky renderer invocation closes its browser before returning', async ()failClosing();await handlerPromise;expect(handlerReturned).toBe(true);+ expect(streamedMessages).toHaveLength(1);+ expect(streamedMessages[0]).toMatchObject({+ type: 'error-occurred',+ payload: {+ shouldRetry: true,+ errorInfo: {+ message: 'must pass chunk',+ willRetry: true,+ isFatal: false,+ },+ },+ });} finally {failClosing();await Promise.resolve();