π₯ Make videos programmatically with React
`@remotion/lambda`: Fix rendering to write-only output buckets
Jonny Burger7d190b17f27a
diff --git a/packages/docs/docs/lambda/without-iam/index.mdx b/packages/docs/docs/lambda/without-iam/index.mdxindex e72ba52928..7860b244e5 100644--- a/packages/docs/docs/lambda/without-iam/index.mdx+++ b/packages/docs/docs/lambda/without-iam/index.mdx@@ -34,7 +34,7 @@ The steps below provide authorization for the Lambda function to execute `render- Click on JSON- Copy the JSON policy template below:<details>- <summary>Show full role permissions JSON file for latest Remotion Lambda version</summary>+ <summary>Show full role permissions JSON file for latest Remotion Lambda version</summary><UserPolicy /></details>- Click next. On the tags page, you don't need to fill in anything. Click next again.@@ -104,23 +104,20 @@ In the example above, `renderMediaOnLambda()` is configured to output the render- Click "Create inline policy"- Click the "JSON" tab-Add the policy below to grant `s3:PutObject` and `s3:GetObject` on the destination bucket's objects, and `s3:ListBucket` on the bucket itself.+Add the policy below to grant `s3:PutObject` on the destination bucket's objects.-Remotion checks whether the output file exists using an S3 `HeadObject` request. This requires `s3:GetObject`. The `s3:ListBucket` permission lets S3 return `404 Not Found` while the output file does not exist, instead of `403 Forbidden`. See the [S3 HeadObject permissions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).+From <AvailableFrom v="4.0.523" inline />, AWS S3 destinations can use a write-only policy. When [`overwrite`](/docs/lambda/rendermediaonlambda#overwrite) is `false`, a conditional upload prevents replacing an existing file. If the role cannot check whether the file exists before rendering, a conflict is reported when uploading the finished video.++In versions 4.0.472β4.0.522, custom destination buckets also require `s3:GetObject` on `arn:aws:s3:::{bucketname}/*` and `s3:ListBucket` on `arn:aws:s3:::{bucketname}`. Upgrade to restore write-only access. These additional permissions remain useful for detecting existing files before rendering and recovering completion if the final progress update is missing. See the [S3 HeadObject permissions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).```json title="Destination bucket policy"{"Version": "2012-10-17","Statement": [{- "Action": ["s3:PutObject", "s3:GetObject"],+ "Action": ["s3:PutObject"],"Resource": ["arn:aws:s3:::{bucketname}/*"],"Effect": "Allow"- },- {- "Action": ["s3:ListBucket"],- "Resource": ["arn:aws:s3:::{bucketname}"],- "Effect": "Allow"}]}diff --git a/packages/lambda-client/src/aws-provider.ts b/packages/lambda-client/src/aws-provider.tsindex 34caed2f5e..8e5cd993e5 100644--- a/packages/lambda-client/src/aws-provider.ts+++ b/packages/lambda-client/src/aws-provider.ts@@ -53,7 +53,10 @@ import {randomHashImplementation} from './random-hash';import {lambdaReadFileImplementation} from './read-file';import type {RuntimePreference} from './runtime-preference';import type {RequestHandler} from './types';-import {lambdaWriteFileImplementation} from './write-file';+import {+ lambdaWriteFileImplementation,+ lambdaWriteFileIfNotExistsImplementation,+} from './write-file';if (/^AWS_Lambda_nodejs(?:18|20|24)[.]x$/.test(@@ -112,6 +115,7 @@ export const awsImplementation: ProviderSpecifics<AwsProvider> = {randomHash: randomHashImplementation,readFile: lambdaReadFileImplementation,writeFile: lambdaWriteFileImplementation,+ writeFileIfNotExists: lambdaWriteFileIfNotExistsImplementation,headFile: lambdaHeadFileImplementation,convertToServeUrl: convertToServeUrlImplementation,printLoggingHelper: true,diff --git a/packages/lambda-client/src/head-file.ts b/packages/lambda-client/src/head-file.tsindex ba88342925..28ecfbd09d 100644--- a/packages/lambda-client/src/head-file.ts+++ b/packages/lambda-client/src/head-file.ts@@ -12,6 +12,7 @@ export const lambdaHeadFileImplementation: ProviderSpecifics<AwsProvider>['headFforcePathStyle,requestHandler,}): Promise<{+ renderId: string | null;LastModified?: Date | undefined;ContentLength?: number | undefined;}> => {@@ -26,5 +27,5 @@ export const lambdaHeadFileImplementation: ProviderSpecifics<AwsProvider>['headFKey: key,}),);- return head;+ return {...head, renderId: head.Metadata?.['remotion-render-id'] ?? null};};diff --git a/packages/lambda-client/src/test/aws-china-regions.test.ts b/packages/lambda-client/src/test/aws-china-regions.test.tsindex 3144e13853..e2f58c5abe 100644--- a/packages/lambda-client/src/test/aws-china-regions.test.ts+++ b/packages/lambda-client/src/test/aws-china-regions.test.ts@@ -115,6 +115,7 @@ test('AWS service and Console URLs use the region partition metadata', () => {customCredentials: null,currentRegion: 'cn-north-1',renderMetadata: {+ outputFileIsConditional: null,outName: 'out.mp4',privacy: 'public',renderId: 'render-id',diff --git a/packages/lambda-client/src/test/price-calculation.test.ts b/packages/lambda-client/src/test/price-calculation.test.tsindex 0a315e4fec..a7a947dea5 100644--- a/packages/lambda-client/src/test/price-calculation.test.ts+++ b/packages/lambda-client/src/test/price-calculation.test.ts@@ -10,6 +10,7 @@ test('Should not throw while calculating prices when time shifts occur', () => {const price = estimatePriceFromMetadata({memorySizeInMb: 1024,renderMetadata: {+ outputFileIsConditional: null,audioBitrate: null,codec: 'h264',compositionId: 'react-svg',diff --git a/packages/lambda-client/src/test/write-file.test.ts b/packages/lambda-client/src/test/write-file.test.tsindex 82d1e022f2..8e498cbeae 100644--- a/packages/lambda-client/src/test/write-file.test.ts+++ b/packages/lambda-client/src/test/write-file.test.ts@@ -1,52 +1,211 @@-import {beforeAll, expect, mock, test} from 'bun:test';+import {afterAll, beforeAll, expect, test} from 'bun:test';import {Readable} from 'node:stream';-import type {lambdaWriteFileImplementation as LambdaWriteFileImplementation} from '../write-file';+import type {S3Client} from '@aws-sdk/client-s3';+import type {WriteFileInput} from '@remotion/serverless-client';+import {awsImplementation, type AwsProvider} from '../aws-provider';+import type {RequestHandler} from '../types';-const uploadDone = mock();--mock.module('@aws-sdk/lib-storage', () => ({- Upload: class {- done = uploadDone;- },-}));--mock.module('../get-s3-client', () => ({- getS3Client: () => ({- send: mock(),- }),-}));--let lambdaWriteFileImplementation: typeof LambdaWriteFileImplementation;--beforeAll(async () => {- const mod = await import('../write-file');- lambdaWriteFileImplementation = mod.lambdaWriteFileImplementation;+const previousAccessKey = process.env.REMOTION_AWS_ACCESS_KEY_ID;+const previousSecretKey = process.env.REMOTION_AWS_SECRET_ACCESS_KEY;+beforeAll(() => {+ process.env.REMOTION_AWS_ACCESS_KEY_ID = 'test-access-key';+ process.env.REMOTION_AWS_SECRET_ACCESS_KEY = 'test-secret-key';+});+afterAll(() => {+ if (previousAccessKey === undefined)+ delete process.env.REMOTION_AWS_ACCESS_KEY_ID;+ else process.env.REMOTION_AWS_ACCESS_KEY_ID = previousAccessKey;+ if (previousSecretKey === undefined)+ delete process.env.REMOTION_AWS_SECRET_ACCESS_KEY;+ else process.env.REMOTION_AWS_SECRET_ACCESS_KEY = previousSecretKey;});-test('does not retry a failed upload with a consumed Readable', async () => {- const originalError = new Error('The upload failed');- uploadDone.mockRejectedValueOnce(originalError);+test.each(['buffer', 'stream', 'multipart'] as const)(+ 'write-only S3 %s uploads preserve existing outputs and allow explicit overwrites',+ async (kind) => {+ const objects = new Map<string, {body: Buffer; renderId: string | null}>();+ const uploads = new Map<+ string,+ {parts: Map<number, Buffer>; renderId: string | null}+ >();+ let nextUpload = 0;+ let conditionalRequests = 0;+ let completedMultipart = 0;+ let aborts = 0;+ let denyWrites = false;+ let allowReads = false;+ let denyAbort = false;+ let writeRequests = 0;+ // Only the AWS HTTP boundary is faked. Exercise the real SDK and lib-storage.+ const requestHandler: RequestHandler = {+ handle: (+ request: Parameters<S3Client['config']['requestHandler']['handle']>[0],+ ) => {+ let statusCode = 200;+ let responseBody = '';+ const headers: Record<string, string> = {};+ const uploadId = String(request.query.uploadId ?? '');+ if (request.method === 'HEAD') {+ const object = objects.get(request.path);+ if (!allowReads) {+ statusCode = 403;+ } else if (!object) {+ statusCode = 404;+ } else {+ headers['content-length'] = String(object.body.length);+ if (object.renderId) {+ headers['x-amz-meta-remotion-render-id'] = object.renderId;+ }+ }+ } else if (request.method === 'DELETE') {+ aborts++;+ if (denyAbort) {+ statusCode = 403;+ responseBody =+ '<Error><Code>AccessDenied</Code><Message>Cleanup denied</Message></Error>';+ } else {+ uploads.delete(uploadId);+ statusCode = 204;+ }+ } else if (denyWrites) {+ writeRequests++;+ statusCode = 403;+ responseBody =+ '<Error><Code>AccessDenied</Code><Message>Write denied</Message></Error>';+ } else if (request.method === 'POST' && 'uploads' in request.query) {+ const id = `upload-${++nextUpload}`;+ uploads.set(id, {+ parts: new Map(),+ renderId: request.headers['x-amz-meta-remotion-render-id'] ?? null,+ });+ responseBody = `<InitiateMultipartUploadResult><UploadId>${id}</UploadId></InitiateMultipartUploadResult>`;+ } else if (request.method === 'PUT' && uploadId) {+ uploads+ .get(uploadId)!+ .parts.set(+ Number(request.query.partNumber),+ Buffer.from(request.body),+ );+ headers.etag = `"part-${request.query.partNumber}"`;+ } else if (+ request.method === 'PUT' ||+ (request.method === 'POST' && uploadId)+ ) {+ const conditional = request.headers['if-none-match'] === '*';+ conditionalRequests += Number(conditional);+ if (conditional && objects.has(request.path)) {+ statusCode = 412;+ responseBody =+ '<Error><Code>PreconditionFailed</Code><Message>Already exists</Message></Error>';+ } else {+ const upload = uploads.get(uploadId);+ const body = upload+ ? Buffer.concat(+ [...upload.parts.entries()]+ .sort((a, b) => a[0] - b[0])+ .map((part) => Uint8Array.from(part[1])),+ )+ : Buffer.from(request.body);+ objects.set(request.path, {+ body,+ renderId:+ upload?.renderId ??+ request.headers['x-amz-meta-remotion-render-id'] ??+ null,+ });+ headers.etag = '"output"';+ if (upload) {+ completedMultipart++;+ uploads.delete(uploadId);+ responseBody =+ '<CompleteMultipartUploadResult><Location>https://bucket.s3.amazonaws.com/video.mp4</Location><Bucket>bucket</Bucket><Key>video.mp4</Key><ETag>"output"</ETag></CompleteMultipartUploadResult>';+ }+ }+ } else {+ throw new Error(+ `Unexpected AWS request: ${request.method} ${request.path}`,+ );+ }- let thrownError: unknown;- try {- await lambdaWriteFileImplementation({- body: Readable.from(['video']),+ return Promise.resolve({+ response: {statusCode, headers, body: Readable.from([responseBody])},+ });+ },+ };+ const content = Buffer.alloc(+ kind === 'multipart' ? 6 * 1024 * 1024 : 16,+ 42,+ );+ const options = (): WriteFileInput<AwsProvider> => ({+ body:+ kind === 'buffer' ? Uint8Array.from(content) : Readable.from([content]),bucketName: 'bucket',- customCredentials: null,- downloadBehavior: null,- expectedBucketOwner: null,- forcePathStyle: false,key: 'video.mp4',- privacy: 'private',region: 'us-east-1',- requestHandler: null,- retries: 1,+ privacy: 'no-acl',+ customCredentials: null,+ expectedBucketOwner: null,+ downloadBehavior: null,+ forcePathStyle: true,storageClass: null,+ requestHandler,+ });+ const write = (renderId: string) =>+ awsImplementation.writeFileIfNotExists!({...options(), renderId});+ await expect(awsImplementation.headFile(options())).rejects.toMatchObject({+ $metadata: {httpStatusCode: 403},});- } catch (err) {- thrownError = err;- }+ await write('first');+ allowReads = true;+ await expect(awsImplementation.headFile(options())).resolves.toMatchObject({+ renderId: 'first',+ ContentLength: content.length,+ });+ allowReads = false;+ expect([...objects.keys()]).toEqual(['/video.mp4']);+ expect(+ objects.get('/video.mp4')!.body.equals(Uint8Array.from(content)),+ ).toBe(true);+ expect(objects.get('/video.mp4')!.renderId).toBe('first');+ await expect(write('second')).rejects.toThrow('already exists');+ expect(objects.get('/video.mp4')!.renderId).toBe('first');+ expect(conditionalRequests).toBe(2);+ if (kind === 'multipart') {+ expect(completedMultipart).toBe(1);+ expect(aborts).toBe(1);+ expect(uploads.size).toBe(0);+ }- expect(thrownError).toBe(originalError);- expect(uploadDone).toHaveBeenCalledTimes(1);-});+ objects.clear();+ const concurrent = await Promise.allSettled([write('a'), write('b')]);+ expect(+ concurrent.filter((result) => result.status === 'fulfilled'),+ ).toHaveLength(1);+ expect(+ concurrent.filter((result) => result.status === 'rejected'),+ ).toHaveLength(1);+ expect(+ objects.get('/video.mp4')!.body.equals(Uint8Array.from(content)),+ ).toBe(true);++ await awsImplementation.writeFile({...options(), body: 'replacement'});+ expect(objects.get('/video.mp4')).toEqual({+ body: Buffer.from('replacement'),+ renderId: null,+ });+ if (kind === 'multipart') {+ denyAbort = true;+ await expect(write('cleanup-denied')).rejects.toThrow('already exists');+ expect(objects.get('/video.mp4')!.body.toString()).toBe('replacement');+ }++ if (kind === 'stream') {+ // Upload has already consumed the stream: never retry it with an empty body.+ denyWrites = true;+ await expect(awsImplementation.writeFile(options())).rejects.toThrow(+ 'Write denied',+ );+ expect(writeRequests).toBe(1);+ }+ },+);diff --git a/packages/lambda-client/src/write-file.ts b/packages/lambda-client/src/write-file.tsindex a65027c166..ab2f9d33ee 100644--- a/packages/lambda-client/src/write-file.ts+++ b/packages/lambda-client/src/write-file.ts@@ -1,6 +1,9 @@/* eslint-disable no-console */import type {ObjectCannedACL, PutObjectCommandInput} from '@aws-sdk/client-s3';-import {PutObjectCommand} from '@aws-sdk/client-s3';+import {+ AbortMultipartUploadCommand,+ PutObjectCommand,+} from '@aws-sdk/client-s3';import {Upload} from '@aws-sdk/lib-storage';import type {CustomCredentials,@@ -26,7 +29,8 @@ const tryLambdaWriteFile = async ({forcePathStyle,storageClass,requestHandler,-}: WriteFileInput<AwsProvider>): Promise<void> => {+ renderId,+}: WriteFileInput<AwsProvider> & {renderId: string | null}): Promise<void> => {const client = getS3Client({region,customCredentials: customCredentials as CustomCredentials<AwsProvider>,@@ -50,6 +54,8 @@ const tryLambdaWriteFile = async ({ContentType: mimeTypes.lookup(key) || 'application/octet-stream',ContentDisposition: getContentDispositionHeader(downloadBehavior),StorageClass: storageClass ?? undefined,+ IfNoneMatch: renderId === null ? undefined : '*',+ Metadata: renderId === null ? undefined : {'remotion-render-id': renderId},};// Determine file size@@ -71,22 +77,60 @@ const tryLambdaWriteFile = async ({partSize: 5 * 1024 * 1024, // chunk size of 5MB});- await upload.done();+ try {+ await upload.done();+ } catch (err) {+ const status = (err as {$metadata: {httpStatusCode: number} | undefined})+ .$metadata?.httpStatusCode;+ // lib-storage does not abort when CompleteMultipartUpload rejects a condition.+ if (+ renderId !== null &&+ upload.uploadId &&+ (status === 412 || status === 409)+ ) {+ await client+ .send(+ new AbortMultipartUploadCommand({+ Bucket: bucketName,+ Key: key,+ UploadId: upload.uploadId,+ ExpectedBucketOwner: params.ExpectedBucketOwner,+ }),+ )+ .catch(() => {+ // A write-only policy may not allow cleanup. Preserve the upload error.+ });+ }++ throw err;+ }} else {// Use regular PutObject for small filesawait client.send(new PutObjectCommand(params));}};-export const lambdaWriteFileImplementation = async (+const writeFileWithRetries = async (params: WriteFileInput<AwsProvider> & {retries?: number;+ renderId: string | null;},): Promise<void> => {const remainingRetries = params.retries ?? 2;try {await tryLambdaWriteFile(params);} catch (err) {+ if (+ params.renderId !== null &&+ (err as {$metadata: {httpStatusCode: number} | undefined}).$metadata+ ?.httpStatusCode === 412+ ) {+ throw new Error(+ `Output file "${params.key}" in bucket "${params.bucketName}" already exists. Delete it before re-rendering, or set the 'overwrite' option in renderMediaOnLambda() to overwrite it.`,+ {cause: err},+ );+ }+// A failed upload may have consumed a Readable, and we cannot recreate it here.// Retrying it could upload an empty body and mask the original error.const bodyCanBeRetried =@@ -104,9 +148,23 @@ export const lambdaWriteFileImplementation = async (console.warn(err);console.warn(`Retrying (${remainingRetries} retries remaining)...`);- return lambdaWriteFileImplementation({+ return writeFileWithRetries({...params,retries: remainingRetries - 1,});}};++export const lambdaWriteFileImplementation = (+ params: WriteFileInput<AwsProvider> & {retries?: number},+): Promise<void> => writeFileWithRetries({...params, renderId: null});++export const lambdaWriteFileIfNotExistsImplementation = (+ params: WriteFileInput<AwsProvider> & {renderId: string},+): Promise<void> => {+ if (params.customCredentials !== null) {+ throw new Error('Conditional output uploads are only supported for AWS S3');+ }++ return writeFileWithRetries(params);+};diff --git a/packages/lambda/src/test/integration/renders/write-only-bucket.test.ts b/packages/lambda/src/test/integration/renders/write-only-bucket.test.tsnew file mode 100644index 0000000000..da6a8c837c--- /dev/null+++ b/packages/lambda/src/test/integration/renders/write-only-bucket.test.ts@@ -0,0 +1,127 @@+import {expect, spyOn, test} from 'bun:test';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {bundle} from '@remotion/bundler';+import {LambdaClientInternals} from '@remotion/lambda-client';+import {RenderInternals} from '@remotion/renderer';+import {ServerlessRoutines} from '@remotion/serverless';+import {mockImplementation} from '../../mocks/mock-implementation';+import {readMockS3File} from '../../mocks/mock-store';+import {waitUntilDone} from '../wait-until-done';++test.each([1, 2])(+ 'renders to a write-only destination with concurrency %s without overwriting existing files',+ async (concurrency) => {+ const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'write-only-render-'));+ const destination = `write-only-output-${concurrency}`;+ const region = 'eu-central-1';+ const key = 'output.mp4';+ const originalHead = mockImplementation.headFile;+ const originalConditionalWrite = mockImplementation.writeFileIfNotExists;+ let destinationChecks = 0;+ const headSpy = spyOn(mockImplementation, 'headFile').mockImplementation(+ (input) => {+ if (input.bucketName === destination) {+ destinationChecks++;+ throw Object.assign(new Error('Forbidden'), {+ $metadata: {httpStatusCode: 403},+ });+ }++ return originalHead(input);+ },+ );+ let close: (() => Promise<void>) | null = null;+ try {+ const serveUrl = await bundle({+ entryPoint: path.join(__dirname, 'write-only-bucket.tsx'),+ outDir: temp,+ publicDir: null,+ });+ const server = await RenderInternals.serveStatic(serveUrl, {+ binariesDirectory: null,+ offthreadVideoThreads: 1,+ downloadMap: RenderInternals.makeDownloadMap(48000),+ indent: false,+ logLevel: 'error',+ offthreadVideoCacheSizeInBytes: null,+ port: null,+ remotionRoot: process.cwd(),+ forceIPv4: false,+ });+ close = server.close;+ const render = async (overwrite: boolean | null) => {+ const payload =+ await LambdaClientInternals.makeLambdaRenderMediaPayload(+ LambdaClientInternals.renderMediaOnLambdaOptionalToRequired({+ codec: 'h264',+ composition: 'write-only',+ concurrency,+ region,+ functionName: 'remotion-dev-render',+ serveUrl: `http://localhost:${server.port}`,+ outName: {bucketName: destination, key},+ privacy: 'no-acl',+ inputProps: {color: overwrite ? 'blue' : 'red'},+ logLevel: 'error',+ ...(overwrite === null ? {} : {overwrite}),+ }),+ );+ const result =+ await mockImplementation.callFunctionSync<ServerlessRoutines.start>({+ type: ServerlessRoutines.start,+ payload,+ functionName: 'remotion-dev-lambda',+ region,+ timeoutInTest: 120000,+ requestHandler: null,+ });+ return waitUntilDone(result.bucketName, result.renderId);+ };++ const first = await render(null);+ expect(first.done).toBe(true);+ expect(first.outBucket).toBe(destination);+ const original = Buffer.from(+ readMockS3File({region, bucketName: destination, key})!.content,+ );+ expect(original.includes('ftyp')).toBe(true);+ await expect(render(false)).rejects.toThrow('already exists');+ expect(+ Buffer.from(+ readMockS3File({region, bucketName: destination, key})!.content,+ ).equals(Uint8Array.from(original)),+ ).toBe(true);+ const replaced = await render(true);+ expect(replaced.done).toBe(true);+ expect(+ Buffer.from(+ readMockS3File({region, bucketName: destination, key})!.content,+ ).equals(Uint8Array.from(original)),+ ).toBe(false);+ expect(destinationChecks).toBeGreaterThanOrEqual(2);++ // Without conditional uploads, denied preflight reads must remain fatal.+ mockImplementation.writeFileIfNotExists = null;+ await expect(render(false)).rejects.toThrow('s3:GetObject');+ mockImplementation.writeFileIfNotExists = originalConditionalWrite;++ // With read access, reject an existing file before uploading.+ headSpy.mockImplementation(originalHead);+ const uploadSpy = spyOn(mockImplementation, 'writeFileIfNotExists');+ try {+ await expect(render(false)).rejects.toThrow('already exists');+ expect(uploadSpy).not.toHaveBeenCalled();+ } finally {+ uploadSpy.mockRestore();+ }+ } finally {+ mockImplementation.writeFileIfNotExists = originalConditionalWrite;+ headSpy.mockRestore();+ await close?.();+ fs.rmSync(temp, {recursive: true, force: true});+ }+ },+ 120000,+);diff --git a/packages/lambda/src/test/integration/renders/write-only-bucket.tsx b/packages/lambda/src/test/integration/renders/write-only-bucket.tsxnew file mode 100644index 0000000000..be8821bf98--- /dev/null+++ b/packages/lambda/src/test/integration/renders/write-only-bucket.tsx@@ -0,0 +1,18 @@+/* eslint-disable no-restricted-imports -- Browser entry bundled for the Lambda integration test. */+import React from 'react';+import {AbsoluteFill, Composition, getInputProps, registerRoot} from 'remotion';++const Video: React.FC = () => (+ <AbsoluteFill style={{backgroundColor: getInputProps().color as string}} />+);++registerRoot(() => (+ <Composition+ id="write-only"+ component={Video}+ width={32}+ height={32}+ fps={30}+ durationInFrames={4}+ />+));diff --git a/packages/lambda/src/test/integration/wait-until-done.ts b/packages/lambda/src/test/integration/wait-until-done.tsindex c08466fcf2..b98ea91993 100644--- a/packages/lambda/src/test/integration/wait-until-done.ts+++ b/packages/lambda/src/test/integration/wait-until-done.ts@@ -25,7 +25,7 @@ export const waitUntilDone = async (bucketName: string, renderId: string) => {}if (progress.fatalErrorEncountered) {- throw new Error(progress.errors.join('\n'));+ throw new Error(progress.errors.map((error) => error.message).join('\n'));}await new Promise((resolve) => {diff --git a/packages/lambda/src/test/mocks/mock-implementation.ts b/packages/lambda/src/test/mocks/mock-implementation.tsindex 16c7333240..f563acac38 100644--- a/packages/lambda/src/test/mocks/mock-implementation.ts+++ b/packages/lambda/src/test/mocks/mock-implementation.ts@@ -18,9 +18,12 @@ import {mockBucketExists,mockDeleteS3File,readMockS3File,+ streamToUint8Array,writeMockS3File,} from './mock-store';+const conditionalRenderIds = new Map<string, string>();+export const mockImplementation: ProviderSpecifics<AwsProvider> = {getAccountId: () =>Promise.resolve('aws:iam::123456789'.match(/aws:iam::([0-9]+)/)?.[1]!),@@ -105,6 +108,7 @@ export const mockImplementation: ProviderSpecifics<AwsProvider> = {return Promise.resolve(createReadStream(tmp));},writeFile: async ({body, bucketName, key, privacy, region}) => {+ conditionalRenderIds.delete(`${region}/${bucketName}/${key}`);await writeMockS3File({body,bucketName,@@ -114,6 +118,24 @@ export const mockImplementation: ProviderSpecifics<AwsProvider> = {});return Promise.resolve(undefined);},+ writeFileIfNotExists: async ({+ body,+ bucketName,+ key,+ privacy,+ region,+ renderId,+ }) => {+ const content = await streamToUint8Array(body);+ if (readMockS3File({region, bucketName, key})) {+ throw new Error(+ `Output file "${key}" in bucket "${bucketName}" already exists.`,+ );+ }++ await writeMockS3File({body: content, bucketName, key, privacy, region});+ conditionalRenderIds.set(`${region}/${bucketName}/${key}`, renderId);+ },headFile: ({bucketName, key, region}) => {const read = readMockS3File({bucketName,@@ -127,6 +149,8 @@ export const mockImplementation: ProviderSpecifics<AwsProvider> = {}return Promise.resolve({+ renderId:+ conditionalRenderIds.get(`${region}/${bucketName}/${key}`) ?? null,ContentLength: read.content.toString().length,LastModified: new Date(),});diff --git a/packages/serverless-client/src/find-output-file-in-bucket.ts b/packages/serverless-client/src/find-output-file-in-bucket.tsindex 6185e8106a..fdd4ec89cb 100644--- a/packages/serverless-client/src/find-output-file-in-bucket.ts+++ b/packages/serverless-client/src/find-output-file-in-bucket.ts@@ -7,8 +7,11 @@ import type {CloudProvider} from './types';export type OutputFileMetadata = {url: string;sizeInBytes: number | null;+ renderId: string | null;};+export class OutputFileAccessDeniedError extends Error {}+export const findOutputFileInBucket = async <Provider extends CloudProvider>({region,renderMetadata,@@ -46,6 +49,7 @@ export const findOutputFileInBucket = async <Provider extends CloudProvider>({});return {+ renderId: metadata.renderId ?? null,url: providerSpecifics.getOutputUrl({renderMetadata,bucketName,@@ -64,7 +68,12 @@ export const findOutputFileInBucket = async <Provider extends CloudProvider>({(err as {$metadata: {httpStatusCode: number}}).$metadata?.httpStatusCode === 403) {- throw new Error(+ const ErrorClass =+ (err as {$metadata: {httpStatusCode: number} | undefined}).$metadata+ ?.httpStatusCode === 403+ ? OutputFileAccessDeniedError+ : Error;+ throw new ErrorClass(`Unable to access item "${key}" from bucket "${renderBucketName}" ${customCredentials?.endpoint? `(S3 Endpoint = ${customCredentials?.endpoint})`diff --git a/packages/serverless-client/src/index.ts b/packages/serverless-client/src/index.tsindex f66046a2cf..b1a7e9dfd5 100644--- a/packages/serverless-client/src/index.ts+++ b/packages/serverless-client/src/index.ts@@ -79,6 +79,7 @@ export {} from './expected-out-name';export {findOutputFileInBucket,+ OutputFileAccessDeniedError,type OutputFileMetadata,} from './find-output-file-in-bucket';export {formatCostsInfo} from './format-costs-info';diff --git a/packages/serverless-client/src/progress.ts b/packages/serverless-client/src/progress.tsindex 23891a7963..e16fb8db08 100644--- a/packages/serverless-client/src/progress.ts+++ b/packages/serverless-client/src/progress.ts@@ -3,7 +3,10 @@ import {calculateChunkTimes} from './calculate-chunk-times';import type {CustomCredentials} from './constants';import {estimatePriceFromMetadata} from './estimate-price-from-bucket';import {getExpectedOutName} from './expected-out-name';-import {findOutputFileInBucket} from './find-output-file-in-bucket';+import {+ findOutputFileInBucket,+ OutputFileAccessDeniedError,+} from './find-output-file-in-bucket';import {formatCostsInfo} from './format-costs-info';import {getOverallProgress} from './get-overall-progress';import {getOverallProgressFromStorage} from './get-overall-progress-from-storage';@@ -254,6 +257,7 @@ export const getProgress = async <Provider extends CloudProvider>({const shouldCheckForCompletedOutput =allChunks &&+ !errorExplanations.some((error) => error.isFatal && !error.willRetry) &&(isBeyondTimeoutAndHasStitchTimeout ||(overallProgress.combinedFrames >= frameCount &&overallProgress.timeToCombine !== null));@@ -268,9 +272,20 @@ export const getProgress = async <Provider extends CloudProvider>({providerSpecifics,forcePathStyle,requestHandler,+ }).catch((err) => {+ if (err instanceof OutputFileAccessDeniedError) {+ // Recovery is optional; normal progress remains authoritative.+ return null;+ }++ throw err;});- if (outputFile) {+ if (+ outputFile &&+ (!renderMetadata.outputFileIsConditional ||+ outputFile.renderId === renderId)+ ) {const outData = getExpectedOutName({renderMetadata,bucketName,diff --git a/packages/serverless-client/src/provider-implementation.ts b/packages/serverless-client/src/provider-implementation.tsindex 29808029db..e030481e96 100644--- a/packages/serverless-client/src/provider-implementation.ts+++ b/packages/serverless-client/src/provider-implementation.ts@@ -103,6 +103,7 @@ type ConvertToServeUrl<Provider extends CloudProvider> = (params: {}) => string;type HeadFileOutput = {+ renderId: string | null;LastModified?: Date | undefined;ContentLength?: number | undefined;};@@ -248,6 +249,9 @@ export type ProviderSpecifics<Provider extends CloudProvider> = {randomHash: RandomHash;readFile: ReadFile<Provider>;writeFile: WriteFile<Provider>;+ writeFileIfNotExists:+ | ((params: WriteFileInput<Provider> & {renderId: string}) => Promise<void>)+ | null;headFile: HeadFile<Provider>;convertToServeUrl: ConvertToServeUrl<Provider>;printLoggingHelper: boolean;diff --git a/packages/serverless-client/src/render-metadata.ts b/packages/serverless-client/src/render-metadata.tsindex b51abf50d9..9db007724d 100644--- a/packages/serverless-client/src/render-metadata.ts+++ b/packages/serverless-client/src/render-metadata.ts@@ -50,6 +50,8 @@ export type RenderMetadata<Provider extends CloudProvider> = Discriminated & {region: Provider['region'];renderId: string;outName: OutNameInputWithoutCredentials | undefined;+ // Conditional outputs carry the render ID so recovery cannot use another render's file.+ outputFileIsConditional: boolean | null;privacy: Privacy;deleteAfter: DeleteAfter | null;numberOfGifLoops: number | null;diff --git a/packages/serverless-client/src/test/expected-out-name.test.ts b/packages/serverless-client/src/test/expected-out-name.test.tsindex 5cecfb25ed..e0769dc2fa 100644--- a/packages/serverless-client/src/test/expected-out-name.test.ts+++ b/packages/serverless-client/src/test/expected-out-name.test.ts@@ -17,6 +17,7 @@ type MockProvider = {};const testRenderMetadata: RenderMetadata<MockProvider> = {+ outputFileIsConditional: null,compositionId: 'react-svg',estimatedRenderLambdaInvokations: 100,estimatedTotalLambdaInvokations: 100,diff --git a/packages/serverless-client/src/test/progress.test.ts b/packages/serverless-client/src/test/progress.test.tsindex b7e9611c8c..4261848fc0 100644--- a/packages/serverless-client/src/test/progress.test.ts+++ b/packages/serverless-client/src/test/progress.test.ts@@ -24,6 +24,7 @@ const bucketName = 'source-bucket';const startedDate = Date.now() - 30000;const renderMetadata: RenderMetadata<MockProvider> = {+ outputFileIsConditional: null,audioBitrate: null,audioCodec: null,codec: 'h264',@@ -164,6 +165,7 @@ const makeProviderSpecifics = ({serverStorageProductName: () => 'S3',validateDeleteAfter: () => undefined,writeFile: () => Promise.resolve(),+ writeFileIfNotExists: null,headFile: onHeadFile,};};@@ -174,7 +176,7 @@ test('getProgress treats an existing output file as finished if postRenderData wbillingCurrency: 'CNY',onHeadFile: ({bucketName: headedBucketName, key}) => {headedFiles.push({bucketName: headedBucketName, key});- return Promise.resolve({ContentLength: 1234});+ return Promise.resolve({ContentLength: 1234, renderId: null});},renderProgress: null,});@@ -222,7 +224,7 @@ test('getProgress estimates costs from invoked lambdas only', async () => {priceInputs.push(input);return input.durationInMilliseconds;},- onHeadFile: () => Promise.resolve({ContentLength: 0}),+ onHeadFile: () => Promise.resolve({ContentLength: 0, renderId: null}),renderProgress: null,});@@ -295,3 +297,121 @@ test('getProgress keeps direct render progress finite', async () => {expect(renderProgress.overallProgress).toBeLessThanOrEqual(1);}});++test('getProgress falls back to persisted progress when destination reads are denied', async () => {+ let heads = 0;+ const providerSpecifics = makeProviderSpecifics({+ renderProgress: null,+ onHeadFile: () => {+ heads++;+ return Promise.reject(+ Object.assign(new Error('Forbidden'), {+ $metadata: {httpStatusCode: 403},+ }),+ );+ },+ });+ for (const timeoutInMilliseconds of [120000, 1000]) {+ const result = await getProgress({+ bucketName,+ renderId,+ customCredentials: null,+ expectedBucketOwner: null,+ forcePathStyle: false,+ functionName: renderMetadata.functionName,+ memorySizeInMb: 2048,+ providerSpecifics,+ region: 'eu-central-1',+ requestHandler: null,+ timeoutInMilliseconds,+ });+ expect(result.done).toBe(false);+ expect(result.framesRendered).toBe(20);+ expect(result.outputFile).toBeNull();+ expect(result.fatalErrorEncountered).toBe(timeoutInMilliseconds === 1000);+ }++ expect(heads).toBe(2);+});++test('getProgress only recovers a conditional output belonging to this render', async () => {+ for (const outputRenderId of [null, 'another-render', renderId]) {+ const providerSpecifics = makeProviderSpecifics({+ renderProgress: {+ ...progress,+ renderMetadata: {...renderMetadata, outputFileIsConditional: true},+ },+ onHeadFile: () =>+ Promise.resolve({renderId: outputRenderId, ContentLength: 1234}),+ });+ const result = await getProgress({+ bucketName,+ renderId,+ customCredentials: null,+ expectedBucketOwner: null,+ forcePathStyle: false,+ functionName: renderMetadata.functionName,+ memorySizeInMb: 2048,+ providerSpecifics,+ region: 'eu-central-1',+ requestHandler: null,+ timeoutInMilliseconds: 120000,+ });+ expect(result.done).toBe(outputRenderId === renderId);+ expect(result.outputSizeInBytes).toBe(+ outputRenderId === renderId ? 1234 : null,+ );+ }+});++test('getProgress preserves upload failures and unexpected storage errors', async () => {+ const uploadFailure = {+ type: 'stitcher' as const,+ message: 'Output already exists',+ name: 'Error',+ stack: '',+ frame: null,+ chunk: null,+ isFatal: true,+ attempt: 1,+ willRetry: false,+ totalAttempts: 1,+ tmpDir: null,+ };+ let heads = 0;+ const storageError = Object.assign(new Error('Storage unavailable'), {+ $metadata: {httpStatusCode: 500},+ });+ for (const errors of [[uploadFailure], []]) {+ const providerSpecifics = makeProviderSpecifics({+ renderProgress: {...progress, errors},+ onHeadFile: () => {+ heads++;+ return Promise.reject(storageError);+ },+ });+ const pending = getProgress({+ bucketName,+ renderId,+ customCredentials: null,+ expectedBucketOwner: null,+ forcePathStyle: false,+ functionName: renderMetadata.functionName,+ memorySizeInMb: 2048,+ providerSpecifics,+ region: 'eu-central-1',+ requestHandler: null,+ timeoutInMilliseconds: 120000,+ });+ if (errors.length) {+ const result = await pending;+ expect(result.done).toBe(false);+ expect(result.fatalErrorEncountered).toBe(true);+ expect(result.errors[0].message).toBe('Output already exists');+ } else {+ await expect(pending).rejects.toBe(storageError);+ }+ }++ expect(heads).toBe(1);+});diff --git a/packages/serverless/src/finish-render.ts b/packages/serverless/src/finish-render.tsindex 350e08de29..86ba242248 100644--- a/packages/serverless/src/finish-render.ts+++ b/packages/serverless/src/finish-render.ts@@ -8,6 +8,7 @@ import type {Privacy,ProviderSpecifics,RenderMetadata,+ WriteFileInput,SerializedInputProps,} from '@remotion/serverless-client';import {inspectErrors} from '@remotion/serverless-client';@@ -64,7 +65,7 @@ export const finishRender = async <Provider extends CloudProvider>({logLevel,);- await providerSpecifics.writeFile({+ const writeOptions: WriteFileInput<Provider> = {bucketName: renderBucketName,key,body: fs.createReadStream(outputFile),@@ -76,7 +77,22 @@ export const finishRender = async <Provider extends CloudProvider>({forcePathStyle,storageClass,requestHandler,- });+ };++ if (renderMetadata.outputFileIsConditional) {+ if (!providerSpecifics.writeFileIfNotExists) {+ throw new Error(+ 'The provider does not support conditional output uploads',+ );+ }++ await providerSpecifics.writeFileIfNotExists({+ ...writeOptions,+ renderId: renderMetadata.renderId,+ });+ } else {+ await providerSpecifics.writeFile(writeOptions);+ }writeToBucket.end();@@ -106,6 +122,7 @@ export const finishRender = async <Provider extends CloudProvider>({errorExplanations,timeToDelete: (await cleanupProm).reduce((a, b) => Math.max(a, b), 0),outputFile: {+ renderId: renderMetadata.renderId,sizeInBytes: outputSize,url: outputUrl,},diff --git a/packages/serverless/src/handlers/launch.ts b/packages/serverless/src/handlers/launch.tsindex a294a2f1fa..2eda4d89b5 100644--- a/packages/serverless/src/handlers/launch.ts+++ b/packages/serverless/src/handlers/launch.ts@@ -18,9 +18,11 @@ import {CONCAT_FOLDER_TOKEN,decompressInputProps,DOCS_URL,+ getCredentialsFromOutName,getExpectedOutName,getNeedsToUpload,MAX_FUNCTIONS_PER_RENDER,+ OutputFileAccessDeniedError,rendererTransportPrefix,serializeOrThrow,ServerlessRoutines,@@ -344,6 +346,10 @@ const innerLaunchHandler = async <Provider extends CloudProvider>({options.requestContext !== null;const renderMetadata: RenderMetadata<Provider> = {+ outputFileIsConditional:+ !params.overwrite &&+ getCredentialsFromOutName(params.outName ?? null) === null &&+ providerSpecifics.writeFileIfNotExists !== null,startedDate,totalChunks: chunks.length,estimatedTotalLambdaInvokations: [@@ -410,6 +416,16 @@ const innerLaunchHandler = async <Provider extends CloudProvider>({providerSpecifics,forcePathStyle: params.forcePathStyle,requestHandler: null,+ }).catch((err) => {+ if (+ err instanceof OutputFileAccessDeniedError &&+ renderMetadata.outputFileIsConditional+ ) {+ // The final conditional upload still enforces overwrite: false.+ return null;+ }++ throw err;});if (output) {throw new TypeError(diff --git a/packages/serverless/src/handlers/still.ts b/packages/serverless/src/handlers/still.tsindex 1467dac26b..e6295f8019 100644--- a/packages/serverless/src/handlers/still.ts+++ b/packages/serverless/src/handlers/still.ts@@ -186,6 +186,7 @@ const innerStillHandler = async <Provider extends CloudProvider>(});const renderMetadata: RenderMetadata<Provider> = {+ outputFileIsConditional: null,startedDate: Date.now(),codec: null,compositionId: params.composition,diff --git a/packages/serverless/src/test/overall-render-progress.test.ts b/packages/serverless/src/test/overall-render-progress.test.tsindex 5db6424958..1ff50fbc24 100644--- a/packages/serverless/src/test/overall-render-progress.test.ts+++ b/packages/serverless/src/test/overall-render-progress.test.ts@@ -48,7 +48,7 @@ const makeProviderSpecifics = ({getMaxNonInlinePayloadSizePerFunction: () => 0,getMaxStillInlinePayloadSize: () => 0,getOutputUrl: () => ({key: 'out.mp4', url: 'https://example.com/out.mp4'}),- headFile: () => Promise.resolve({}),+ headFile: () => Promise.resolve({renderId: null}),isFlakyError: () => false,listObjects: () => Promise.resolve([]),parseFunctionName: () => null,@@ -58,6 +58,7 @@ const makeProviderSpecifics = ({serverStorageProductName: () => 'S3',validateDeleteAfter: () => undefined,writeFile,+ writeFileIfNotExists: null,};};