🎥 Make videos programmatically with React
import type {Dirent} from 'node:fs';
import {createReadStream, promises as fs} from 'node:fs';
import path from 'node:path';
import {Upload} from '@aws-sdk/lib-storage';
import {
LambdaClientInternals,
type AwsRegion,
type Privacy,
type RequestHandler,
} from '@remotion/lambda-client';
import mimeTypes from 'mime-types';
import type {UploadDirProgress} from '../functions/types';
import {makeS3Key} from '../shared/make-s3-key';
import {multipartUploadPartSize} from '../shared/multipart-upload-part-size';
import {waitForPromisesToFinish} from '../shared/wait-for-promises-to-finish';
type FileInfo = {
name: string;
size: number;
};
export type MockFile = {
name: string;
content: string;
};
async function getFiles(
directory: string,
originalDirectory: string,
toUpload: string[],
): Promise<FileInfo[]> {
const dirents = await fs.readdir(directory, {withFileTypes: true});
const _files = await Promise.all(
dirents
.map((dirent): [Dirent, string] => {
const res = path.resolve(directory, dirent.name);
return [dirent, res];
})
.filter(([dirent, res]) => {
const relative = path
.relative(originalDirectory, res)
.split(path.sep)
.join('/');
if (dirent.isDirectory()) {
return true;
}
if (!toUpload.includes(relative)) {
return false;
}
return true;
})
.map(async ([dirent, res]) => {
const {size} = await fs.stat(res);
return dirent.isDirectory()
? getFiles(res, originalDirectory, toUpload)
: [
{
name: res,
size,
},
];
}),
);
return _files.flat(1);
}
const limit = LambdaClientInternals.pLimit(5);
type UploadDirInput = {
bucket: string;
region: AwsRegion;
localDir: string;
keyPrefix: string;
onProgress: (progress: UploadDirProgress) => void;
privacy: Privacy;
toUpload: string[];
forcePathStyle: boolean;
requestHandler: RequestHandler | null;
};
export const uploadDir = async ({
bucket,
region,
localDir,
onProgress,
keyPrefix,
privacy,
toUpload,
forcePathStyle,
requestHandler,
}: UploadDirInput): Promise<void> => {
const files = await getFiles(localDir, localDir, toUpload);
const progresses: {[key: string]: number} = {};
for (const file of files) {
progresses[file.name] = 0;
}
const client = LambdaClientInternals.getS3Client({
region,
customCredentials: null,
forcePathStyle,
requestHandler,
});
const uploadWithoutRetry = async (filePath: FileInfo) => {
const Key = makeS3Key(keyPrefix, localDir, filePath.name);
const Body = createReadStream(filePath.name);
const ContentType = mimeTypes.lookup(Key) || 'application/octet-stream';
const ACL =
privacy === 'no-acl'
? undefined
: privacy === 'private'
? 'private'
: 'public-read';
const parallelUploadsS3 = new Upload({
client,
queueSize: 2,
partSize: multipartUploadPartSize,
params: {
Key,
Bucket: bucket,
Body,
ACL,
ContentType,
},
});
parallelUploadsS3.on('httpUploadProgress', (progress) => {
progresses[filePath.name] = progress.loaded ?? 0;
});
const prom = await parallelUploadsS3.done();
return prom;
};
const uploadWithRetry = async (filePath: FileInfo) => {
let error: Error | null = null;
for (let i = 0; i < 3; i++) {
try {
return await uploadWithoutRetry(filePath);
} catch (err) {
error = err as Error;
}
}
if (error) {
throw error;
}
};
const uploads = files.map((filePath) =>
limit(async () => {
await uploadWithRetry(filePath);
}),
);
const interval = setInterval(() => {
onProgress({
totalSize: files.map((f) => f.size).reduce((a, b) => a + b, 0),
sizeUploaded: Object.values(progresses).reduce((a, b) => a + b, 0),
totalFiles: files.length,
filesUploaded: files.filter((f) => progresses[f.name] === f.size).length,
});
}, 1000);
try {
await waitForPromisesToFinish(uploads);
} finally {
clearInterval(interval);
}
};