Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(media-server): upload multipart output parts concurrently
Richie McIlroy03a6111266c5
diff --git a/apps/media-server/src/__tests__/lib/storage-upload.test.ts b/apps/media-server/src/__tests__/lib/storage-upload.test.tsindex f260ea8ae..4b57d3ed5 100644--- a/apps/media-server/src/__tests__/lib/storage-upload.test.ts+++ b/apps/media-server/src/__tests__/lib/storage-upload.test.ts@@ -3,7 +3,10 @@ import { createHash } from "node:crypto";import { mkdtemp, rm, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";-import { uploadFileToStorage } from "../../lib/media-video";+import {+ multipartUploadConcurrencyFromEnv,+ uploadFileToStorage,+} from "../../lib/media-video";import { verifyRemoteRecordingBytes } from "../../lib/recording-verification";const originalFetch = globalThis.fetch;@@ -314,6 +317,74 @@ describe("uploadFileToStorage", () => {});});+ test("uploads multipart parts concurrently and completes them in order", async () => {+ const uploadFile = await createTempUploadFile(partSize * 4 + 1);+ let inFlight = 0;+ let maxInFlight = 0;+ let completed: unknown;+ globalThis.fetch = (async (input, init) => {+ const url = String(input);+ if (init?.method === "POST") {+ const json = JSON.parse(String(init.body));+ if (url.endsWith("/sign"))+ return Response.json({+ url: `https://storage.example.com/part-${json.partNumber}`,+ });+ completed = json;+ return Response.json({ objectIdentity: '"completed"' });+ }+ const partNumber = Number(url.split("part-")[1]);+ inFlight++;+ maxInFlight = Math.max(maxInFlight, inFlight);+ await Bun.sleep((6 - partNumber) * 5);+ inFlight--;+ return new Response(null, {+ status: 200,+ headers: { etag: `"etag-${partNumber}"` },+ });+ }) as typeof fetch;+ process.env.MEDIA_SERVER_UPLOAD_CONCURRENCY = "3";+ try {+ await uploadFileToStorage(+ uploadFile.path,+ {+ type: "multipart",+ videoId: "video-id",+ key: "user-id/video-id/result.mp4",+ uploadId: "upload-id",+ partSize,+ signPartUrl: "https://cap.example.com/sign",+ completeUrl: "https://cap.example.com/complete",+ abortUrl: "https://cap.example.com/abort",+ },+ "video/mp4",+ );+ } finally {+ delete process.env.MEDIA_SERVER_UPLOAD_CONCURRENCY;+ await uploadFile.cleanup();+ }+ expect(maxInFlight).toBe(3);+ expect(+ (completed as { parts: Array<{ partNumber: number }> }).parts.map(+ (part) => part.partNumber,+ ),+ ).toEqual([1, 2, 3, 4, 5]);+ });++ test("reads a bounded multipart upload concurrency", () => {+ expect(multipartUploadConcurrencyFromEnv({})).toBe(1);+ expect(+ multipartUploadConcurrencyFromEnv({+ MEDIA_SERVER_UPLOAD_CONCURRENCY: "8",+ }),+ ).toBe(8);+ expect(+ multipartUploadConcurrencyFromEnv({+ MEDIA_SERVER_UPLOAD_CONCURRENCY: "99",+ }),+ ).toBe(16);+ });+test("aborts multipart uploads when a part fails", async () => {const uploadFile = await createTempUploadFile(partSize + 1);const requests: Array<{ url: string; method: string }> = [];diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.tsindex a58d944d9..f61cd5b2e 100644--- a/apps/media-server/src/lib/media-video.ts+++ b/apps/media-server/src/lib/media-video.ts@@ -2730,6 +2730,13 @@ async function abortMultipartUpload(await response.body?.cancel().catch(() => {});}+export function multipartUploadConcurrencyFromEnv(+ env: Record<string, string | undefined> = process.env,+): number {+ const value = Number.parseInt(env.MEDIA_SERVER_UPLOAD_CONCURRENCY ?? "", 10);+ return Number.isFinite(value) && value > 0 ? Math.min(value, 16) : 1;+}+async function uploadFileMultipart(filePath: string,target: Extract<StorageUploadTarget, { type: "multipart" }>,@@ -2757,25 +2764,45 @@ async function uploadFileMultipart();}- for (let partNumber = 1; partNumber <= partCount; partNumber++) {- const start = (partNumber - 1) * partSize;- const end = Math.min(start + partSize, contentLength);- const partLength = end - start;- const url = await getMultipartPartUrl(- target,- partNumber,- partLength,- abortSignal,- );- const etag = await uploadMultipartPart(- url,- fileHandle.slice(start, end),- partLength,- partNumber,- abortSignal,- );- parts.push({ partNumber, etag, size: partLength });+ const failed = new AbortController();+ const signal = abortSignal+ ? AbortSignal.any([abortSignal, failed.signal])+ : failed.signal;+ let nextPart = 1;+ const worker = async () => {+ while (nextPart <= partCount) {+ const partNumber = nextPart++;+ const start = (partNumber - 1) * partSize;+ const end = Math.min(start + partSize, contentLength);+ const partLength = end - start;+ const url = await getMultipartPartUrl(+ target,+ partNumber,+ partLength,+ signal,+ );+ const etag = await uploadMultipartPart(+ url,+ fileHandle.slice(start, end),+ partLength,+ partNumber,+ signal,+ );+ parts.push({ partNumber, etag, size: partLength });+ }+ };+ const workers = Array.from(+ { length: Math.min(multipartUploadConcurrencyFromEnv(), partCount) },+ () => worker(),+ );+ try {+ await Promise.all(workers);+ } catch (error) {+ failed.abort();+ await Promise.allSettled(workers);+ throw error;}+ parts.sort((a, b) => a.partNumber - b.partNumber);return await completeMultipartUpload(target, parts, abortSignal);} catch (error) {