Skip to content

πŸŽ₯ Make videos programmatically with React

`@remotion/lambda`: Fix rendering to write-only output buckets

Jonny Burger7d190b17f27a
diff --git a/packages/docs/docs/lambda/without-iam/index.mdx b/packages/docs/docs/lambda/without-iam/index.mdx
index e72ba52928..7860b244e5 100644
--- a/packages/docs/docs/lambda/without-iam/index.mdx
+++ b/packages/docs/docs/lambda/without-iam/index.mdx
@@ -34,7 +34,7 @@ The steps below provide authorization for the Lambda function to execute `render
- Click on JSON
- Copy the JSON policy template below:
<details>
- <summary>Show full role permissions JSON file for latest Remotion Lambda version</summary>
+ <summary>Show full role permissions JSON file for latest Remotion Lambda version</summary>
<UserPolicy />
</details>
- Click next. On the tags page, you don't need to fill in anything. Click next again.
@@ -104,23 +104,20 @@ In the example above, `renderMediaOnLambda()` is configured to output the render
- Click "Create inline policy"
- Click the "JSON" tab
-Add the policy below to grant `s3:PutObject` and `s3:GetObject` on the destination bucket's objects, and `s3:ListBucket` on the bucket itself.
+Add the policy below to grant `s3:PutObject` on the destination bucket's objects.
-Remotion checks whether the output file exists using an S3 `HeadObject` request. This requires `s3:GetObject`. The `s3:ListBucket` permission lets S3 return `404 Not Found` while the output file does not exist, instead of `403 Forbidden`. See the [S3 HeadObject permissions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
+From <AvailableFrom v="4.0.523" inline />, AWS S3 destinations can use a write-only policy. When [`overwrite`](/docs/lambda/rendermediaonlambda#overwrite) is `false`, a conditional upload prevents replacing an existing file. If the role cannot check whether the file exists before rendering, a conflict is reported when uploading the finished video.
+
+In versions 4.0.472–4.0.522, custom destination buckets also require `s3:GetObject` on `arn:aws:s3:::{bucketname}/*` and `s3:ListBucket` on `arn:aws:s3:::{bucketname}`. Upgrade to restore write-only access. These additional permissions remain useful for detecting existing files before rendering and recovering completion if the final progress update is missing. See the [S3 HeadObject permissions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
```json title="Destination bucket policy"
{
"Version": "2012-10-17",
"Statement": [
{
- "Action": ["s3:PutObject", "s3:GetObject"],
+ "Action": ["s3:PutObject"],
"Resource": ["arn:aws:s3:::{bucketname}/*"],
"Effect": "Allow"
- },
- {
- "Action": ["s3:ListBucket"],
- "Resource": ["arn:aws:s3:::{bucketname}"],
- "Effect": "Allow"
}
]
}
diff --git a/packages/lambda-client/src/aws-provider.ts b/packages/lambda-client/src/aws-provider.ts
index 34caed2f5e..8e5cd993e5 100644
--- a/packages/lambda-client/src/aws-provider.ts
+++ b/packages/lambda-client/src/aws-provider.ts
@@ -53,7 +53,10 @@ import {randomHashImplementation} from './random-hash';
import {lambdaReadFileImplementation} from './read-file';
import type {RuntimePreference} from './runtime-preference';
import type {RequestHandler} from './types';
-import {lambdaWriteFileImplementation} from './write-file';
+import {
+ lambdaWriteFileImplementation,
+ lambdaWriteFileIfNotExistsImplementation,
+} from './write-file';
if (
/^AWS_Lambda_nodejs(?:18|20|24)[.]x$/.test(
@@ -112,6 +115,7 @@ export const awsImplementation: ProviderSpecifics<AwsProvider> = {
randomHash: randomHashImplementation,
readFile: lambdaReadFileImplementation,
writeFile: lambdaWriteFileImplementation,
+ writeFileIfNotExists: lambdaWriteFileIfNotExistsImplementation,
headFile: lambdaHeadFileImplementation,
convertToServeUrl: convertToServeUrlImplementation,
printLoggingHelper: true,
diff --git a/packages/lambda-client/src/head-file.ts b/packages/lambda-client/src/head-file.ts
index ba88342925..28ecfbd09d 100644
--- a/packages/lambda-client/src/head-file.ts
+++ b/packages/lambda-client/src/head-file.ts
@@ -12,6 +12,7 @@ export const lambdaHeadFileImplementation: ProviderSpecifics<AwsProvider>['headF
forcePathStyle,
requestHandler,
}): Promise<{
+ renderId: string | null;
LastModified?: Date | undefined;
ContentLength?: number | undefined;
}> => {
@@ -26,5 +27,5 @@ export const lambdaHeadFileImplementation: ProviderSpecifics<AwsProvider>['headF
Key: key,
}),
);
- return head;
+ return {...head, renderId: head.Metadata?.['remotion-render-id'] ?? null};
};
diff --git a/packages/lambda-client/src/test/aws-china-regions.test.ts b/packages/lambda-client/src/test/aws-china-regions.test.ts
index 3144e13853..e2f58c5abe 100644
--- a/packages/lambda-client/src/test/aws-china-regions.test.ts
+++ b/packages/lambda-client/src/test/aws-china-regions.test.ts
@@ -115,6 +115,7 @@ test('AWS service and Console URLs use the region partition metadata', () => {
customCredentials: null,
currentRegion: 'cn-north-1',
renderMetadata: {
+ outputFileIsConditional: null,
outName: 'out.mp4',
privacy: 'public',
renderId: 'render-id',
diff --git a/packages/lambda-client/src/test/price-calculation.test.ts b/packages/lambda-client/src/test/price-calculation.test.ts
index 0a315e4fec..a7a947dea5 100644
--- a/packages/lambda-client/src/test/price-calculation.test.ts
+++ b/packages/lambda-client/src/test/price-calculation.test.ts
@@ -10,6 +10,7 @@ test('Should not throw while calculating prices when time shifts occur', () => {
const price = estimatePriceFromMetadata({
memorySizeInMb: 1024,
renderMetadata: {
+ outputFileIsConditional: null,
audioBitrate: null,
codec: 'h264',
compositionId: 'react-svg',
diff --git a/packages/lambda-client/src/test/write-file.test.ts b/packages/lambda-client/src/test/write-file.test.ts
index 82d1e022f2..8e498cbeae 100644
--- a/packages/lambda-client/src/test/write-file.test.ts
+++ b/packages/lambda-client/src/test/write-file.test.ts
@@ -1,52 +1,211 @@
-import {beforeAll, expect, mock, test} from 'bun:test';
+import {afterAll, beforeAll, expect, test} from 'bun:test';
import {Readable} from 'node:stream';
-import type {lambdaWriteFileImplementation as LambdaWriteFileImplementation} from '../write-file';
+import type {S3Client} from '@aws-sdk/client-s3';
+import type {WriteFileInput} from '@remotion/serverless-client';
+import {awsImplementation, type AwsProvider} from '../aws-provider';
+import type {RequestHandler} from '../types';
-const uploadDone = mock();
-
-mock.module('@aws-sdk/lib-storage', () => ({
- Upload: class {
- done = uploadDone;
- },
-}));
-
-mock.module('../get-s3-client', () => ({
- getS3Client: () => ({
- send: mock(),
- }),
-}));
-
-let lambdaWriteFileImplementation: typeof LambdaWriteFileImplementation;
-
-beforeAll(async () => {
- const mod = await import('../write-file');
- lambdaWriteFileImplementation = mod.lambdaWriteFileImplementation;
+const previousAccessKey = process.env.REMOTION_AWS_ACCESS_KEY_ID;
+const previousSecretKey = process.env.REMOTION_AWS_SECRET_ACCESS_KEY;
+beforeAll(() => {
+ process.env.REMOTION_AWS_ACCESS_KEY_ID = 'test-access-key';
+ process.env.REMOTION_AWS_SECRET_ACCESS_KEY = 'test-secret-key';
+});
+afterAll(() => {
+ if (previousAccessKey === undefined)
+ delete process.env.REMOTION_AWS_ACCESS_KEY_ID;
+ else process.env.REMOTION_AWS_ACCESS_KEY_ID = previousAccessKey;
+ if (previousSecretKey === undefined)
+ delete process.env.REMOTION_AWS_SECRET_ACCESS_KEY;
+ else process.env.REMOTION_AWS_SECRET_ACCESS_KEY = previousSecretKey;
});
-test('does not retry a failed upload with a consumed Readable', async () => {
- const originalError = new Error('The upload failed');
- uploadDone.mockRejectedValueOnce(originalError);
+test.each(['buffer', 'stream', 'multipart'] as const)(
+ 'write-only S3 %s uploads preserve existing outputs and allow explicit overwrites',
+ async (kind) => {
+ const objects = new Map<string, {body: Buffer; renderId: string | null}>();
+ const uploads = new Map<
+ string,
+ {parts: Map<number, Buffer>; renderId: string | null}
+ >();
+ let nextUpload = 0;
+ let conditionalRequests = 0;
+ let completedMultipart = 0;
+ let aborts = 0;
+ let denyWrites = false;
+ let allowReads = false;
+ let denyAbort = false;
+ let writeRequests = 0;
+ // Only the AWS HTTP boundary is faked. Exercise the real SDK and lib-storage.
+ const requestHandler: RequestHandler = {
+ handle: (
+ request: Parameters<S3Client['config']['requestHandler']['handle']>[0],
+ ) => {
+ let statusCode = 200;
+ let responseBody = '';
+ const headers: Record<string, string> = {};
+ const uploadId = String(request.query.uploadId ?? '');
+ if (request.method === 'HEAD') {
+ const object = objects.get(request.path);
+ if (!allowReads) {
+ statusCode = 403;
+ } else if (!object) {
+ statusCode = 404;
+ } else {
+ headers['content-length'] = String(object.body.length);
+ if (object.renderId) {
+ headers['x-amz-meta-remotion-render-id'] = object.renderId;
+ }
+ }
+ } else if (request.method === 'DELETE') {
+ aborts++;
+ if (denyAbort) {
+ statusCode = 403;
+ responseBody =
+ '<Error><Code>AccessDenied</Code><Message>Cleanup denied</Message></Error>';
+ } else {
+ uploads.delete(uploadId);
+ statusCode = 204;
+ }
+ } else if (denyWrites) {
+ writeRequests++;
+ statusCode = 403;
+ responseBody =
+ '<Error><Code>AccessDenied</Code><Message>Write denied</Message></Error>';
+ } else if (request.method === 'POST' && 'uploads' in request.query) {
+ const id = `upload-${++nextUpload}`;
+ uploads.set(id, {
+ parts: new Map(),
+ renderId: request.headers['x-amz-meta-remotion-render-id'] ?? null,
+ });
+ responseBody = `<InitiateMultipartUploadResult><UploadId>${id}</UploadId></InitiateMultipartUploadResult>`;
+ } else if (request.method === 'PUT' && uploadId) {
+ uploads
+ .get(uploadId)!
+ .parts.set(
+ Number(request.query.partNumber),
+ Buffer.from(request.body),
+ );
+ headers.etag = `"part-${request.query.partNumber}"`;
+ } else if (
+ request.method === 'PUT' ||
+ (request.method === 'POST' && uploadId)
+ ) {
+ const conditional = request.headers['if-none-match'] === '*';
+ conditionalRequests += Number(conditional);
+ if (conditional && objects.has(request.path)) {
+ statusCode = 412;
+ responseBody =
+ '<Error><Code>PreconditionFailed</Code><Message>Already exists</Message></Error>';
+ } else {
+ const upload = uploads.get(uploadId);
+ const body = upload
+ ? Buffer.concat(
+ [...upload.parts.entries()]
+ .sort((a, b) => a[0] - b[0])
+ .map((part) => Uint8Array.from(part[1])),
+ )
+ : Buffer.from(request.body);
+ objects.set(request.path, {
+ body,
+ renderId:
+ upload?.renderId ??
+ request.headers['x-amz-meta-remotion-render-id'] ??
+ null,
+ });
+ headers.etag = '"output"';
+ if (upload) {
+ completedMultipart++;
+ uploads.delete(uploadId);
+ responseBody =
+ '<CompleteMultipartUploadResult><Location>https://bucket.s3.amazonaws.com/video.mp4</Location><Bucket>bucket</Bucket><Key>video.mp4</Key><ETag>"output"</ETag></CompleteMultipartUploadResult>';
+ }
+ }
+ } else {
+ throw new Error(
+ `Unexpected AWS request: ${request.method} ${request.path}`,
+ );
+ }
- let thrownError: unknown;
- try {
- await lambdaWriteFileImplementation({
- body: Readable.from(['video']),
+ return Promise.resolve({
+ response: {statusCode, headers, body: Readable.from([responseBody])},
+ });
+ },
+ };
+ const content = Buffer.alloc(
+ kind === 'multipart' ? 6 * 1024 * 1024 : 16,
+ 42,
+ );
+ const options = (): WriteFileInput<AwsProvider> => ({
+ body:
+ kind === 'buffer' ? Uint8Array.from(content) : Readable.from([content]),
bucketName: 'bucket',
- customCredentials: null,
- downloadBehavior: null,
- expectedBucketOwner: null,
- forcePathStyle: false,
key: 'video.mp4',
- privacy: 'private',
region: 'us-east-1',
- requestHandler: null,
- retries: 1,
+ privacy: 'no-acl',
+ customCredentials: null,
+ expectedBucketOwner: null,
+ downloadBehavior: null,
+ forcePathStyle: true,
storageClass: null,
+ requestHandler,
+ });
+ const write = (renderId: string) =>
+ awsImplementation.writeFileIfNotExists!({...options(), renderId});
+ await expect(awsImplementation.headFile(options())).rejects.toMatchObject({
+ $metadata: {httpStatusCode: 403},
});
- } catch (err) {
- thrownError = err;
- }
+ await write('first');
+ allowReads = true;
+ await expect(awsImplementation.headFile(options())).resolves.toMatchObject({
+ renderId: 'first',
+ ContentLength: content.length,
+ });
+ allowReads = false;
+ expect([...objects.keys()]).toEqual(['/video.mp4']);
+ expect(
+ objects.get('/video.mp4')!.body.equals(Uint8Array.from(content)),
+ ).toBe(true);
+ expect(objects.get('/video.mp4')!.renderId).toBe('first');
+ await expect(write('second')).rejects.toThrow('already exists');
+ expect(objects.get('/video.mp4')!.renderId).toBe('first');
+ expect(conditionalRequests).toBe(2);
+ if (kind === 'multipart') {
+ expect(completedMultipart).toBe(1);
+ expect(aborts).toBe(1);
+ expect(uploads.size).toBe(0);
+ }
- expect(thrownError).toBe(originalError);
- expect(uploadDone).toHaveBeenCalledTimes(1);
-});
+ objects.clear();
+ const concurrent = await Promise.allSettled([write('a'), write('b')]);
+ expect(
+ concurrent.filter((result) => result.status === 'fulfilled'),
+ ).toHaveLength(1);
+ expect(
+ concurrent.filter((result) => result.status === 'rejected'),
+ ).toHaveLength(1);
+ expect(
+ objects.get('/video.mp4')!.body.equals(Uint8Array.from(content)),
+ ).toBe(true);
+
+ await awsImplementation.writeFile({...options(), body: 'replacement'});
+ expect(objects.get('/video.mp4')).toEqual({
+ body: Buffer.from('replacement'),
+ renderId: null,
+ });
+ if (kind === 'multipart') {
+ denyAbort = true;
+ await expect(write('cleanup-denied')).rejects.toThrow('already exists');
+ expect(objects.get('/video.mp4')!.body.toString()).toBe('replacement');
+ }
+
+ if (kind === 'stream') {
+ // Upload has already consumed the stream: never retry it with an empty body.
+ denyWrites = true;
+ await expect(awsImplementation.writeFile(options())).rejects.toThrow(
+ 'Write denied',
+ );
+ expect(writeRequests).toBe(1);
+ }
+ },
+);
diff --git a/packages/lambda-client/src/write-file.ts b/packages/lambda-client/src/write-file.ts
index a65027c166..ab2f9d33ee 100644
--- a/packages/lambda-client/src/write-file.ts
+++ b/packages/lambda-client/src/write-file.ts
@@ -1,6 +1,9 @@
/* eslint-disable no-console */
import type {ObjectCannedACL, PutObjectCommandInput} from '@aws-sdk/client-s3';
-import {PutObjectCommand} from '@aws-sdk/client-s3';
+import {
+ AbortMultipartUploadCommand,
+ PutObjectCommand,
+} from '@aws-sdk/client-s3';
import {Upload} from '@aws-sdk/lib-storage';
import type {
CustomCredentials,
@@ -26,7 +29,8 @@ const tryLambdaWriteFile = async ({
forcePathStyle,
storageClass,
requestHandler,
-}: WriteFileInput<AwsProvider>): Promise<void> => {
+ renderId,
+}: WriteFileInput<AwsProvider> & {renderId: string | null}): Promise<void> => {
const client = getS3Client({
region,
customCredentials: customCredentials as CustomCredentials<AwsProvider>,
@@ -50,6 +54,8 @@ const tryLambdaWriteFile = async ({
ContentType: mimeTypes.lookup(key) || 'application/octet-stream',
ContentDisposition: getContentDispositionHeader(downloadBehavior),
StorageClass: storageClass ?? undefined,
+ IfNoneMatch: renderId === null ? undefined : '*',
+ Metadata: renderId === null ? undefined : {'remotion-render-id': renderId},
};
// Determine file size
@@ -71,22 +77,60 @@ const tryLambdaWriteFile = async ({
partSize: 5 * 1024 * 1024, // chunk size of 5MB
});
- await upload.done();
+ try {
+ await upload.done();
+ } catch (err) {
+ const status = (err as {$metadata: {httpStatusCode: number} | undefined})
+ .$metadata?.httpStatusCode;
+ // lib-storage does not abort when CompleteMultipartUpload rejects a condition.
+ if (
+ renderId !== null &&
+ upload.uploadId &&
+ (status === 412 || status === 409)
+ ) {
+ await client
+ .send(
+ new AbortMultipartUploadCommand({
+ Bucket: bucketName,
+ Key: key,
+ UploadId: upload.uploadId,
+ ExpectedBucketOwner: params.ExpectedBucketOwner,
+ }),
+ )
+ .catch(() => {
+ // A write-only policy may not allow cleanup. Preserve the upload error.
+ });
+ }
+
+ throw err;
+ }
} else {
// Use regular PutObject for small files
await client.send(new PutObjectCommand(params));
}
};
-export const lambdaWriteFileImplementation = async (
+const writeFileWithRetries = async (
params: WriteFileInput<AwsProvider> & {
retries?: number;
+ renderId: string | null;
},
): Promise<void> => {
const remainingRetries = params.retries ?? 2;
try {
await tryLambdaWriteFile(params);
} catch (err) {
+ if (
+ params.renderId !== null &&
+ (err as {$metadata: {httpStatusCode: number} | undefined}).$metadata
+ ?.httpStatusCode === 412
+ ) {
+ throw new Error(
+ `Output file "${params.key}" in bucket "${params.bucketName}" already exists. Delete it before re-rendering, or set the 'overwrite' option in renderMediaOnLambda() to overwrite it.`,
+ {cause: err},
+ );
+ }
+
// A failed upload may have consumed a Readable, and we cannot recreate it here.
// Retrying it could upload an empty body and mask the original error.
const bodyCanBeRetried =
@@ -104,9 +148,23 @@ export const lambdaWriteFileImplementation = async (
console.warn(err);
console.warn(`Retrying (${remainingRetries} retries remaining)...`);
- return lambdaWriteFileImplementation({
+ return writeFileWithRetries({
...params,
retries: remainingRetries - 1,
});
}
};
+
+export const lambdaWriteFileImplementation = (
+ params: WriteFileInput<AwsProvider> & {retries?: number},
+): Promise<void> => writeFileWithRetries({...params, renderId: null});
+
+export const lambdaWriteFileIfNotExistsImplementation = (
+ params: WriteFileInput<AwsProvider> & {renderId: string},
+): Promise<void> => {
+ if (params.customCredentials !== null) {
+ throw new Error('Conditional output uploads are only supported for AWS S3');
+ }
+
+ return writeFileWithRetries(params);
+};
diff --git a/packages/lambda/src/test/integration/renders/write-only-bucket.test.ts b/packages/lambda/src/test/integration/renders/write-only-bucket.test.ts
new file mode 100644
index 0000000000..da6a8c837c
--- /dev/null
+++ b/packages/lambda/src/test/integration/renders/write-only-bucket.test.ts
@@ -0,0 +1,127 @@
+import {expect, spyOn, test} from 'bun:test';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {bundle} from '@remotion/bundler';
+import {LambdaClientInternals} from '@remotion/lambda-client';
+import {RenderInternals} from '@remotion/renderer';
+import {ServerlessRoutines} from '@remotion/serverless';
+import {mockImplementation} from '../../mocks/mock-implementation';
+import {readMockS3File} from '../../mocks/mock-store';
+import {waitUntilDone} from '../wait-until-done';
+
+test.each([1, 2])(
+ 'renders to a write-only destination with concurrency %s without overwriting existing files',
+ async (concurrency) => {
+ const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'write-only-render-'));
+ const destination = `write-only-output-${concurrency}`;
+ const region = 'eu-central-1';
+ const key = 'output.mp4';
+ const originalHead = mockImplementation.headFile;
+ const originalConditionalWrite = mockImplementation.writeFileIfNotExists;
+ let destinationChecks = 0;
+ const headSpy = spyOn(mockImplementation, 'headFile').mockImplementation(
+ (input) => {
+ if (input.bucketName === destination) {
+ destinationChecks++;
+ throw Object.assign(new Error('Forbidden'), {
+ $metadata: {httpStatusCode: 403},
+ });
+ }
+
+ return originalHead(input);
+ },
+ );
+ let close: (() => Promise<void>) | null = null;
+ try {
+ const serveUrl = await bundle({
+ entryPoint: path.join(__dirname, 'write-only-bucket.tsx'),
+ outDir: temp,
+ publicDir: null,
+ });
+ const server = await RenderInternals.serveStatic(serveUrl, {
+ binariesDirectory: null,
+ offthreadVideoThreads: 1,
+ downloadMap: RenderInternals.makeDownloadMap(48000),
+ indent: false,
+ logLevel: 'error',
+ offthreadVideoCacheSizeInBytes: null,
+ port: null,
+ remotionRoot: process.cwd(),
+ forceIPv4: false,
+ });
+ close = server.close;
+ const render = async (overwrite: boolean | null) => {
+ const payload =
+ await LambdaClientInternals.makeLambdaRenderMediaPayload(
+ LambdaClientInternals.renderMediaOnLambdaOptionalToRequired({
+ codec: 'h264',
+ composition: 'write-only',
+ concurrency,
+ region,
+ functionName: 'remotion-dev-render',
+ serveUrl: `http://localhost:${server.port}`,
+ outName: {bucketName: destination, key},
+ privacy: 'no-acl',
+ inputProps: {color: overwrite ? 'blue' : 'red'},
+ logLevel: 'error',
+ ...(overwrite === null ? {} : {overwrite}),
+ }),
+ );
+ const result =
+ await mockImplementation.callFunctionSync<ServerlessRoutines.start>({
+ type: ServerlessRoutines.start,
+ payload,
+ functionName: 'remotion-dev-lambda',
+ region,
+ timeoutInTest: 120000,
+ requestHandler: null,
+ });
+ return waitUntilDone(result.bucketName, result.renderId);
+ };
+
+ const first = await render(null);
+ expect(first.done).toBe(true);
+ expect(first.outBucket).toBe(destination);
+ const original = Buffer.from(
+ readMockS3File({region, bucketName: destination, key})!.content,
+ );
+ expect(original.includes('ftyp')).toBe(true);
+ await expect(render(false)).rejects.toThrow('already exists');
+ expect(
+ Buffer.from(
+ readMockS3File({region, bucketName: destination, key})!.content,
+ ).equals(Uint8Array.from(original)),
+ ).toBe(true);
+ const replaced = await render(true);
+ expect(replaced.done).toBe(true);
+ expect(
+ Buffer.from(
+ readMockS3File({region, bucketName: destination, key})!.content,
+ ).equals(Uint8Array.from(original)),
+ ).toBe(false);
+ expect(destinationChecks).toBeGreaterThanOrEqual(2);
+
+ // Without conditional uploads, denied preflight reads must remain fatal.
+ mockImplementation.writeFileIfNotExists = null;
+ await expect(render(false)).rejects.toThrow('s3:GetObject');
+ mockImplementation.writeFileIfNotExists = originalConditionalWrite;
+
+ // With read access, reject an existing file before uploading.
+ headSpy.mockImplementation(originalHead);
+ const uploadSpy = spyOn(mockImplementation, 'writeFileIfNotExists');
+ try {
+ await expect(render(false)).rejects.toThrow('already exists');
+ expect(uploadSpy).not.toHaveBeenCalled();
+ } finally {
+ uploadSpy.mockRestore();
+ }
+ } finally {
+ mockImplementation.writeFileIfNotExists = originalConditionalWrite;
+ headSpy.mockRestore();
+ await close?.();
+ fs.rmSync(temp, {recursive: true, force: true});
+ }
+ },
+ 120000,
+);
diff --git a/packages/lambda/src/test/integration/renders/write-only-bucket.tsx b/packages/lambda/src/test/integration/renders/write-only-bucket.tsx
new file mode 100644
index 0000000000..be8821bf98
--- /dev/null
+++ b/packages/lambda/src/test/integration/renders/write-only-bucket.tsx
@@ -0,0 +1,18 @@
+/* eslint-disable no-restricted-imports -- Browser entry bundled for the Lambda integration test. */
+import React from 'react';
+import {AbsoluteFill, Composition, getInputProps, registerRoot} from 'remotion';
+
+const Video: React.FC = () => (
+ <AbsoluteFill style={{backgroundColor: getInputProps().color as string}} />
+);
+
+registerRoot(() => (
+ <Composition
+ id="write-only"
+ component={Video}
+ width={32}
+ height={32}
+ fps={30}
+ durationInFrames={4}
+ />
+));
diff --git a/packages/lambda/src/test/integration/wait-until-done.ts b/packages/lambda/src/test/integration/wait-until-done.ts
index c08466fcf2..b98ea91993 100644
--- a/packages/lambda/src/test/integration/wait-until-done.ts
+++ b/packages/lambda/src/test/integration/wait-until-done.ts
@@ -25,7 +25,7 @@ export const waitUntilDone = async (bucketName: string, renderId: string) => {
}
if (progress.fatalErrorEncountered) {
- throw new Error(progress.errors.join('\n'));
+ throw new Error(progress.errors.map((error) => error.message).join('\n'));
}
await new Promise((resolve) => {
diff --git a/packages/lambda/src/test/mocks/mock-implementation.ts b/packages/lambda/src/test/mocks/mock-implementation.ts
index 16c7333240..f563acac38 100644
--- a/packages/lambda/src/test/mocks/mock-implementation.ts
+++ b/packages/lambda/src/test/mocks/mock-implementation.ts
@@ -18,9 +18,12 @@ import {
mockBucketExists,
mockDeleteS3File,
readMockS3File,
+ streamToUint8Array,
writeMockS3File,
} from './mock-store';
+const conditionalRenderIds = new Map<string, string>();
+
export const mockImplementation: ProviderSpecifics<AwsProvider> = {
getAccountId: () =>
Promise.resolve('aws:iam::123456789'.match(/aws:iam::([0-9]+)/)?.[1]!),
@@ -105,6 +108,7 @@ export const mockImplementation: ProviderSpecifics<AwsProvider> = {
return Promise.resolve(createReadStream(tmp));
},
writeFile: async ({body, bucketName, key, privacy, region}) => {
+ conditionalRenderIds.delete(`${region}/${bucketName}/${key}`);
await writeMockS3File({
body,
bucketName,
@@ -114,6 +118,24 @@ export const mockImplementation: ProviderSpecifics<AwsProvider> = {
});
return Promise.resolve(undefined);
},
+ writeFileIfNotExists: async ({
+ body,
+ bucketName,
+ key,
+ privacy,
+ region,
+ renderId,
+ }) => {
+ const content = await streamToUint8Array(body);
+ if (readMockS3File({region, bucketName, key})) {
+ throw new Error(
+ `Output file "${key}" in bucket "${bucketName}" already exists.`,
+ );
+ }
+
+ await writeMockS3File({body: content, bucketName, key, privacy, region});
+ conditionalRenderIds.set(`${region}/${bucketName}/${key}`, renderId);
+ },
headFile: ({bucketName, key, region}) => {
const read = readMockS3File({
bucketName,
@@ -127,6 +149,8 @@ export const mockImplementation: ProviderSpecifics<AwsProvider> = {
}
return Promise.resolve({
+ renderId:
+ conditionalRenderIds.get(`${region}/${bucketName}/${key}`) ?? null,
ContentLength: read.content.toString().length,
LastModified: new Date(),
});
diff --git a/packages/serverless-client/src/find-output-file-in-bucket.ts b/packages/serverless-client/src/find-output-file-in-bucket.ts
index 6185e8106a..fdd4ec89cb 100644
--- a/packages/serverless-client/src/find-output-file-in-bucket.ts
+++ b/packages/serverless-client/src/find-output-file-in-bucket.ts
@@ -7,8 +7,11 @@ import type {CloudProvider} from './types';
export type OutputFileMetadata = {
url: string;
sizeInBytes: number | null;
+ renderId: string | null;
};
+export class OutputFileAccessDeniedError extends Error {}
+
export const findOutputFileInBucket = async <Provider extends CloudProvider>({
region,
renderMetadata,
@@ -46,6 +49,7 @@ export const findOutputFileInBucket = async <Provider extends CloudProvider>({
});
return {
+ renderId: metadata.renderId ?? null,
url: providerSpecifics.getOutputUrl({
renderMetadata,
bucketName,
@@ -64,7 +68,12 @@ export const findOutputFileInBucket = async <Provider extends CloudProvider>({
(err as {$metadata: {httpStatusCode: number}}).$metadata
?.httpStatusCode === 403
) {
- throw new Error(
+ const ErrorClass =
+ (err as {$metadata: {httpStatusCode: number} | undefined}).$metadata
+ ?.httpStatusCode === 403
+ ? OutputFileAccessDeniedError
+ : Error;
+ throw new ErrorClass(
`Unable to access item "${key}" from bucket "${renderBucketName}" ${
customCredentials?.endpoint
? `(S3 Endpoint = ${customCredentials?.endpoint})`
diff --git a/packages/serverless-client/src/index.ts b/packages/serverless-client/src/index.ts
index f66046a2cf..b1a7e9dfd5 100644
--- a/packages/serverless-client/src/index.ts
+++ b/packages/serverless-client/src/index.ts
@@ -79,6 +79,7 @@ export {
} from './expected-out-name';
export {
findOutputFileInBucket,
+ OutputFileAccessDeniedError,
type OutputFileMetadata,
} from './find-output-file-in-bucket';
export {formatCostsInfo} from './format-costs-info';
diff --git a/packages/serverless-client/src/progress.ts b/packages/serverless-client/src/progress.ts
index 23891a7963..e16fb8db08 100644
--- a/packages/serverless-client/src/progress.ts
+++ b/packages/serverless-client/src/progress.ts
@@ -3,7 +3,10 @@ import {calculateChunkTimes} from './calculate-chunk-times';
import type {CustomCredentials} from './constants';
import {estimatePriceFromMetadata} from './estimate-price-from-bucket';
import {getExpectedOutName} from './expected-out-name';
-import {findOutputFileInBucket} from './find-output-file-in-bucket';
+import {
+ findOutputFileInBucket,
+ OutputFileAccessDeniedError,
+} from './find-output-file-in-bucket';
import {formatCostsInfo} from './format-costs-info';
import {getOverallProgress} from './get-overall-progress';
import {getOverallProgressFromStorage} from './get-overall-progress-from-storage';
@@ -254,6 +257,7 @@ export const getProgress = async <Provider extends CloudProvider>({
const shouldCheckForCompletedOutput =
allChunks &&
+ !errorExplanations.some((error) => error.isFatal && !error.willRetry) &&
(isBeyondTimeoutAndHasStitchTimeout ||
(overallProgress.combinedFrames >= frameCount &&
overallProgress.timeToCombine !== null));
@@ -268,9 +272,20 @@ export const getProgress = async <Provider extends CloudProvider>({
providerSpecifics,
forcePathStyle,
requestHandler,
+ }).catch((err) => {
+ if (err instanceof OutputFileAccessDeniedError) {
+ // Recovery is optional; normal progress remains authoritative.
+ return null;
+ }
+
+ throw err;
});
- if (outputFile) {
+ if (
+ outputFile &&
+ (!renderMetadata.outputFileIsConditional ||
+ outputFile.renderId === renderId)
+ ) {
const outData = getExpectedOutName({
renderMetadata,
bucketName,
diff --git a/packages/serverless-client/src/provider-implementation.ts b/packages/serverless-client/src/provider-implementation.ts
index 29808029db..e030481e96 100644
--- a/packages/serverless-client/src/provider-implementation.ts
+++ b/packages/serverless-client/src/provider-implementation.ts
@@ -103,6 +103,7 @@ type ConvertToServeUrl<Provider extends CloudProvider> = (params: {
}) => string;
type HeadFileOutput = {
+ renderId: string | null;
LastModified?: Date | undefined;
ContentLength?: number | undefined;
};
@@ -248,6 +249,9 @@ export type ProviderSpecifics<Provider extends CloudProvider> = {
randomHash: RandomHash;
readFile: ReadFile<Provider>;
writeFile: WriteFile<Provider>;
+ writeFileIfNotExists:
+ | ((params: WriteFileInput<Provider> & {renderId: string}) => Promise<void>)
+ | null;
headFile: HeadFile<Provider>;
convertToServeUrl: ConvertToServeUrl<Provider>;
printLoggingHelper: boolean;
diff --git a/packages/serverless-client/src/render-metadata.ts b/packages/serverless-client/src/render-metadata.ts
index b51abf50d9..9db007724d 100644
--- a/packages/serverless-client/src/render-metadata.ts
+++ b/packages/serverless-client/src/render-metadata.ts
@@ -50,6 +50,8 @@ export type RenderMetadata<Provider extends CloudProvider> = Discriminated & {
region: Provider['region'];
renderId: string;
outName: OutNameInputWithoutCredentials | undefined;
+ // Conditional outputs carry the render ID so recovery cannot use another render's file.
+ outputFileIsConditional: boolean | null;
privacy: Privacy;
deleteAfter: DeleteAfter | null;
numberOfGifLoops: number | null;
diff --git a/packages/serverless-client/src/test/expected-out-name.test.ts b/packages/serverless-client/src/test/expected-out-name.test.ts
index 5cecfb25ed..e0769dc2fa 100644
--- a/packages/serverless-client/src/test/expected-out-name.test.ts
+++ b/packages/serverless-client/src/test/expected-out-name.test.ts
@@ -17,6 +17,7 @@ type MockProvider = {
};
const testRenderMetadata: RenderMetadata<MockProvider> = {
+ outputFileIsConditional: null,
compositionId: 'react-svg',
estimatedRenderLambdaInvokations: 100,
estimatedTotalLambdaInvokations: 100,
diff --git a/packages/serverless-client/src/test/progress.test.ts b/packages/serverless-client/src/test/progress.test.ts
index b7e9611c8c..4261848fc0 100644
--- a/packages/serverless-client/src/test/progress.test.ts
+++ b/packages/serverless-client/src/test/progress.test.ts
@@ -24,6 +24,7 @@ const bucketName = 'source-bucket';
const startedDate = Date.now() - 30000;
const renderMetadata: RenderMetadata<MockProvider> = {
+ outputFileIsConditional: null,
audioBitrate: null,
audioCodec: null,
codec: 'h264',
@@ -164,6 +165,7 @@ const makeProviderSpecifics = ({
serverStorageProductName: () => 'S3',
validateDeleteAfter: () => undefined,
writeFile: () => Promise.resolve(),
+ writeFileIfNotExists: null,
headFile: onHeadFile,
};
};
@@ -174,7 +176,7 @@ test('getProgress treats an existing output file as finished if postRenderData w
billingCurrency: 'CNY',
onHeadFile: ({bucketName: headedBucketName, key}) => {
headedFiles.push({bucketName: headedBucketName, key});
- return Promise.resolve({ContentLength: 1234});
+ return Promise.resolve({ContentLength: 1234, renderId: null});
},
renderProgress: null,
});
@@ -222,7 +224,7 @@ test('getProgress estimates costs from invoked lambdas only', async () => {
priceInputs.push(input);
return input.durationInMilliseconds;
},
- onHeadFile: () => Promise.resolve({ContentLength: 0}),
+ onHeadFile: () => Promise.resolve({ContentLength: 0, renderId: null}),
renderProgress: null,
});
@@ -295,3 +297,121 @@ test('getProgress keeps direct render progress finite', async () => {
expect(renderProgress.overallProgress).toBeLessThanOrEqual(1);
}
});
+
+test('getProgress falls back to persisted progress when destination reads are denied', async () => {
+ let heads = 0;
+ const providerSpecifics = makeProviderSpecifics({
+ renderProgress: null,
+ onHeadFile: () => {
+ heads++;
+ return Promise.reject(
+ Object.assign(new Error('Forbidden'), {
+ $metadata: {httpStatusCode: 403},
+ }),
+ );
+ },
+ });
+ for (const timeoutInMilliseconds of [120000, 1000]) {
+ const result = await getProgress({
+ bucketName,
+ renderId,
+ customCredentials: null,
+ expectedBucketOwner: null,
+ forcePathStyle: false,
+ functionName: renderMetadata.functionName,
+ memorySizeInMb: 2048,
+ providerSpecifics,
+ region: 'eu-central-1',
+ requestHandler: null,
+ timeoutInMilliseconds,
+ });
+ expect(result.done).toBe(false);
+ expect(result.framesRendered).toBe(20);
+ expect(result.outputFile).toBeNull();
+ expect(result.fatalErrorEncountered).toBe(timeoutInMilliseconds === 1000);
+ }
+
+ expect(heads).toBe(2);
+});
+
+test('getProgress only recovers a conditional output belonging to this render', async () => {
+ for (const outputRenderId of [null, 'another-render', renderId]) {
+ const providerSpecifics = makeProviderSpecifics({
+ renderProgress: {
+ ...progress,
+ renderMetadata: {...renderMetadata, outputFileIsConditional: true},
+ },
+ onHeadFile: () =>
+ Promise.resolve({renderId: outputRenderId, ContentLength: 1234}),
+ });
+ const result = await getProgress({
+ bucketName,
+ renderId,
+ customCredentials: null,
+ expectedBucketOwner: null,
+ forcePathStyle: false,
+ functionName: renderMetadata.functionName,
+ memorySizeInMb: 2048,
+ providerSpecifics,
+ region: 'eu-central-1',
+ requestHandler: null,
+ timeoutInMilliseconds: 120000,
+ });
+ expect(result.done).toBe(outputRenderId === renderId);
+ expect(result.outputSizeInBytes).toBe(
+ outputRenderId === renderId ? 1234 : null,
+ );
+ }
+});
+
+test('getProgress preserves upload failures and unexpected storage errors', async () => {
+ const uploadFailure = {
+ type: 'stitcher' as const,
+ message: 'Output already exists',
+ name: 'Error',
+ stack: '',
+ frame: null,
+ chunk: null,
+ isFatal: true,
+ attempt: 1,
+ willRetry: false,
+ totalAttempts: 1,
+ tmpDir: null,
+ };
+ let heads = 0;
+ const storageError = Object.assign(new Error('Storage unavailable'), {
+ $metadata: {httpStatusCode: 500},
+ });
+ for (const errors of [[uploadFailure], []]) {
+ const providerSpecifics = makeProviderSpecifics({
+ renderProgress: {...progress, errors},
+ onHeadFile: () => {
+ heads++;
+ return Promise.reject(storageError);
+ },
+ });
+ const pending = getProgress({
+ bucketName,
+ renderId,
+ customCredentials: null,
+ expectedBucketOwner: null,
+ forcePathStyle: false,
+ functionName: renderMetadata.functionName,
+ memorySizeInMb: 2048,
+ providerSpecifics,
+ region: 'eu-central-1',
+ requestHandler: null,
+ timeoutInMilliseconds: 120000,
+ });
+ if (errors.length) {
+ const result = await pending;
+ expect(result.done).toBe(false);
+ expect(result.fatalErrorEncountered).toBe(true);
+ expect(result.errors[0].message).toBe('Output already exists');
+ } else {
+ await expect(pending).rejects.toBe(storageError);
+ }
+ }
+
+ expect(heads).toBe(1);
+});
diff --git a/packages/serverless/src/finish-render.ts b/packages/serverless/src/finish-render.ts
index 350e08de29..86ba242248 100644
--- a/packages/serverless/src/finish-render.ts
+++ b/packages/serverless/src/finish-render.ts
@@ -8,6 +8,7 @@ import type {
Privacy,
ProviderSpecifics,
RenderMetadata,
+ WriteFileInput,
SerializedInputProps,
} from '@remotion/serverless-client';
import {inspectErrors} from '@remotion/serverless-client';
@@ -64,7 +65,7 @@ export const finishRender = async <Provider extends CloudProvider>({
logLevel,
);
- await providerSpecifics.writeFile({
+ const writeOptions: WriteFileInput<Provider> = {
bucketName: renderBucketName,
key,
body: fs.createReadStream(outputFile),
@@ -76,7 +77,22 @@ export const finishRender = async <Provider extends CloudProvider>({
forcePathStyle,
storageClass,
requestHandler,
- });
+ };
+
+ if (renderMetadata.outputFileIsConditional) {
+ if (!providerSpecifics.writeFileIfNotExists) {
+ throw new Error(
+ 'The provider does not support conditional output uploads',
+ );
+ }
+
+ await providerSpecifics.writeFileIfNotExists({
+ ...writeOptions,
+ renderId: renderMetadata.renderId,
+ });
+ } else {
+ await providerSpecifics.writeFile(writeOptions);
+ }
writeToBucket.end();
@@ -106,6 +122,7 @@ export const finishRender = async <Provider extends CloudProvider>({
errorExplanations,
timeToDelete: (await cleanupProm).reduce((a, b) => Math.max(a, b), 0),
outputFile: {
+ renderId: renderMetadata.renderId,
sizeInBytes: outputSize,
url: outputUrl,
},
diff --git a/packages/serverless/src/handlers/launch.ts b/packages/serverless/src/handlers/launch.ts
index a294a2f1fa..2eda4d89b5 100644
--- a/packages/serverless/src/handlers/launch.ts
+++ b/packages/serverless/src/handlers/launch.ts
@@ -18,9 +18,11 @@ import {
CONCAT_FOLDER_TOKEN,
decompressInputProps,
DOCS_URL,
+ getCredentialsFromOutName,
getExpectedOutName,
getNeedsToUpload,
MAX_FUNCTIONS_PER_RENDER,
+ OutputFileAccessDeniedError,
rendererTransportPrefix,
serializeOrThrow,
ServerlessRoutines,
@@ -344,6 +346,10 @@ const innerLaunchHandler = async <Provider extends CloudProvider>({
options.requestContext !== null;
const renderMetadata: RenderMetadata<Provider> = {
+ outputFileIsConditional:
+ !params.overwrite &&
+ getCredentialsFromOutName(params.outName ?? null) === null &&
+ providerSpecifics.writeFileIfNotExists !== null,
startedDate,
totalChunks: chunks.length,
estimatedTotalLambdaInvokations: [
@@ -410,6 +416,16 @@ const innerLaunchHandler = async <Provider extends CloudProvider>({
providerSpecifics,
forcePathStyle: params.forcePathStyle,
requestHandler: null,
+ }).catch((err) => {
+ if (
+ err instanceof OutputFileAccessDeniedError &&
+ renderMetadata.outputFileIsConditional
+ ) {
+ // The final conditional upload still enforces overwrite: false.
+ return null;
+ }
+
+ throw err;
});
if (output) {
throw new TypeError(
diff --git a/packages/serverless/src/handlers/still.ts b/packages/serverless/src/handlers/still.ts
index 1467dac26b..e6295f8019 100644
--- a/packages/serverless/src/handlers/still.ts
+++ b/packages/serverless/src/handlers/still.ts
@@ -186,6 +186,7 @@ const innerStillHandler = async <Provider extends CloudProvider>(
});
const renderMetadata: RenderMetadata<Provider> = {
+ outputFileIsConditional: null,
startedDate: Date.now(),
codec: null,
compositionId: params.composition,
diff --git a/packages/serverless/src/test/overall-render-progress.test.ts b/packages/serverless/src/test/overall-render-progress.test.ts
index 5db6424958..1ff50fbc24 100644
--- a/packages/serverless/src/test/overall-render-progress.test.ts
+++ b/packages/serverless/src/test/overall-render-progress.test.ts
@@ -48,7 +48,7 @@ const makeProviderSpecifics = ({
getMaxNonInlinePayloadSizePerFunction: () => 0,
getMaxStillInlinePayloadSize: () => 0,
getOutputUrl: () => ({key: 'out.mp4', url: 'https://example.com/out.mp4'}),
- headFile: () => Promise.resolve({}),
+ headFile: () => Promise.resolve({renderId: null}),
isFlakyError: () => false,
listObjects: () => Promise.resolve([]),
parseFunctionName: () => null,
@@ -58,6 +58,7 @@ const makeProviderSpecifics = ({
serverStorageProductName: () => 'S3',
validateDeleteAfter: () => undefined,
writeFile,
+ writeFileIfNotExists: null,
};
};