Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(media-server): upload multipart recording outputs while they are verified
Richie McIlroyc45b2482f2dc
diff --git a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.tsindex f59eb5b58..49a3d070c 100644--- a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts+++ b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts@@ -858,7 +858,8 @@ describe("media routes real-world integration tests", () => {}}, 30_000);- test("refuses a local output changed between full decode and byte binding", async () => {+ // Hashing now overlaps the decode check, so whichever check reads the changed bytes first refuses it.+ test("refuses a local output changed while it is decoded and hashed", async () => {const originalHash = recordingVerification.hashRecordingFile;const hashing = spyOn(recordingVerification,@@ -882,8 +883,8 @@ describe("media routes real-world integration tests", () => {jobId = ((await response.json()) as { jobId: string }).jobId;const job = await waitForTerminalJob(jobId);expect(job.phase).toBe("error");- expect(job.error).toContain(- "Local recording changed during verification",+ expect(job.error).toMatch(+ /Local recording changed during verification|does not preserve the source/,);expect(job.errorCode).toBe("output-invalid");expect(job.recordingVerification).toBeUndefined();@@ -930,6 +931,31 @@ describe("media routes real-world integration tests", () => {30_000,);+ test.each(["corrupt", "corrupt-audio"] as const)(+ "never completes a multipart output whose source fails verification (%s)",+ async (fault) => {+ sourceFault = fault;+ const body = multipartMuxRequest(`multipart-unverified-${fault}`);+ const response = await app.fetch(+ mediaPostRequest("/video/mux-segments", body),+ );+ expect(response.status).toBe(200);+ const { jobId } = (await response.json()) as { jobId: string };+ try {+ const job = await waitForTerminalJob(jobId);+ expect(job.phase).toBe("error");+ expect(job.recordingVerification).toBeUndefined();+ const actions = multipartCallbacks.map(({ action }) => action);+ // Parts may already be uploaded while the check runs, but the object is never created.+ expect(actions).not.toContain("complete");+ expect(actions.at(-1)).toBe("abort");+ } finally {+ deleteJob(jobId);+ }+ },+ 30_000,+ );+test.each(["changed", "missing", "corrupt", "corrupt-audio"] as const)("withholds upload and proof after a pinned source is %s",async (fault) => {diff --git a/apps/media-server/src/__tests__/lib/storage-upload.test.ts b/apps/media-server/src/__tests__/lib/storage-upload.test.tsindex 4df363215..bbc3cf7ed 100644--- a/apps/media-server/src/__tests__/lib/storage-upload.test.ts+++ b/apps/media-server/src/__tests__/lib/storage-upload.test.ts@@ -4,6 +4,7 @@ import { mkdtemp, open, rm, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";import {+ beginStorageUpload,multipartUploadConcurrencyFromEnv,uploadFileToStorage,} from "../../lib/media-video";@@ -496,3 +497,132 @@ describe("uploadFileToStorage", () => {);});});++describe("beginStorageUpload", () => {+ afterEach(() => {+ globalThis.fetch = originalFetch;+ });++ const multipartTarget = {+ type: "multipart" as const,+ videoId: "video",+ key: "owner/video/result.mp4",+ uploadId: "upload",+ partSize,+ signPartUrl: "https://cap.test/sign-part",+ completeUrl: "https://cap.test/complete",+ abortUrl: "https://cap.test/abort",+ };++ async function waitUntil(condition: () => boolean) {+ for (let tries = 0; tries < 100 && !condition(); tries++)+ await Bun.sleep(5);+ }++ function recordMultipart(holdParts?: Promise<void>) {+ const calls: string[] = [];+ globalThis.fetch = (async (input, init) => {+ const url = String(input);+ if (url.endsWith("/sign-part")) {+ const { partNumber } = JSON.parse(String(init?.body));+ calls.push(`sign-${partNumber}`);+ return Response.json({+ url: `https://storage.test/part/${partNumber}`,+ });+ }+ if (init?.method === "PUT") {+ calls.push(`put-${url.split("/").at(-1)}`);+ if (holdParts)+ await Promise.race([+ holdParts,+ new Promise((_, reject) =>+ init.signal?.addEventListener("abort", () =>+ reject(new DOMException("Aborted", "AbortError")),+ ),+ ),+ ]);+ return new Response(null, { headers: { ETag: `"etag-${url}"` } });+ }+ if (url.endsWith("/complete")) {+ calls.push("complete");+ return Response.json({ objectIdentity: '"done"' });+ }+ if (url.endsWith("/abort")) {+ calls.push("abort");+ return Response.json({ success: true });+ }+ throw new Error(`Unexpected request ${url}`);+ }) as typeof fetch;+ return calls;+ }++ const puts = (calls: string[]) =>+ calls.filter((call) => call.startsWith("put"));++ test("uploads multipart parts before finish and completes only when finished", async () => {+ const uploadFile = await createTempUploadFile(partSize * 2 + 1);+ const calls = recordMultipart();+ try {+ const pending = beginStorageUpload(+ uploadFile.path,+ multipartTarget,+ "video/mp4",+ );+ await waitUntil(() => puts(calls).length === 3);+ expect(puts(calls)).toHaveLength(3);+ expect(calls).not.toContain("complete");+ expect(await pending.finish()).toEqual({ objectIdentity: '"done"' });+ expect(calls.at(-1)).toBe("complete");+ expect(calls).not.toContain("abort");+ } finally {+ await uploadFile.cleanup();+ }+ });++ test("cancel stops in-flight parts without completing and leaves the abort to the caller", async () => {+ const uploadFile = await createTempUploadFile(partSize * 2 + 1);+ let release = () => {};+ const calls = recordMultipart(+ new Promise<void>((resolve) => {+ release = resolve;+ }),+ );+ try {+ const pending = beginStorageUpload(+ uploadFile.path,+ multipartTarget,+ "video/mp4",+ );+ await waitUntil(() => puts(calls).length > 0);+ expect(puts(calls).length).toBeGreaterThan(0);+ await pending.cancel();+ expect(calls).not.toContain("complete");+ expect(calls).not.toContain("abort");+ } finally {+ release();+ await uploadFile.cleanup();+ }+ });++ test("sends nothing for a single PUT target until finish", async () => {+ const uploadFile = await createTempUploadFile(16);+ const calls: string[] = [];+ globalThis.fetch = (async (_input, init) => {+ calls.push(String(init?.method));+ return new Response(null, { headers: { ETag: '"put"' } });+ }) as typeof fetch;+ try {+ const pending = beginStorageUpload(+ uploadFile.path,+ { type: "put", url: "https://storage.test/result.mp4" },+ "video/mp4",+ );+ await Bun.sleep(20);+ expect(calls).toHaveLength(0);+ await pending.finish();+ expect(calls).toEqual(["PUT"]);+ } finally {+ await uploadFile.cleanup();+ }+ });+});diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.tsindex da16eceb2..e1030ae5f 100644--- a/apps/media-server/src/lib/media-video.ts+++ b/apps/media-server/src/lib/media-video.ts@@ -2796,85 +2796,141 @@ export function multipartUploadConcurrencyFromEnv(return Number.isFinite(value) && value > 0 ? Math.min(value, 16) : 1;}-async function uploadFileMultipart(+async function uploadMultipartParts(filePath: string,target: Extract<StorageUploadTarget, { type: "multipart" }>,abortSignal?: AbortSignal,-): Promise<StorageUploadReceipt> {+): Promise<UploadedPart[]> {const fileHandle = file(filePath);const contentLength = fileHandle.size;const partSize = Math.floor(target.partSize);const parts: UploadedPart[] = [];- try {- abortSignal?.throwIfAborted();- if (contentLength <= 0) {- throw new Error("Multipart upload requires a non-empty file");- }+ abortSignal?.throwIfAborted();+ if (contentLength <= 0) {+ throw new Error("Multipart upload requires a non-empty file");+ }- if (partSize < MULTIPART_MIN_PART_SIZE_BYTES) {- throw new Error("Multipart upload part size is too small");- }+ if (partSize < MULTIPART_MIN_PART_SIZE_BYTES) {+ throw new Error("Multipart upload part size is too small");+ }- const partCount = Math.ceil(contentLength / partSize);- if (partCount > MULTIPART_MAX_PARTS) {- throw new Error(- `Multipart upload would require too many parts: ${partCount}/${MULTIPART_MAX_PARTS}`,+ const partCount = Math.ceil(contentLength / partSize);+ if (partCount > MULTIPART_MAX_PARTS) {+ throw new Error(+ `Multipart upload would require too many parts: ${partCount}/${MULTIPART_MAX_PARTS}`,+ );+ }++ const failed = new AbortController();+ const signal = abortSignal+ ? AbortSignal.any([abortSignal, failed.signal])+ : failed.signal;+ let nextPart = 1;+ const worker = async () => {+ while (nextPart <= partCount) {+ const partNumber = nextPart++;+ const start = (partNumber - 1) * partSize;+ const end = Math.min(start + partSize, contentLength);+ const partLength = end - start;+ const url = await getMultipartPartUrl(+ target,+ partNumber,+ partLength,+ signal,+ );+ const etag = await uploadMultipartPart(+ url,+ fileHandle.slice(start, end),+ partLength,+ partNumber,+ signal,);+ parts.push({ partNumber, etag, size: partLength });}+ };+ const workers = Array.from(+ { length: Math.min(multipartUploadConcurrencyFromEnv(), partCount) },+ () => worker(),+ );+ try {+ await Promise.all(workers);+ } catch (error) {+ failed.abort();+ await Promise.allSettled(workers);+ throw error;+ }+ return parts.sort((a, b) => a.partNumber - b.partNumber);+}- const failed = new AbortController();- const signal = abortSignal- ? AbortSignal.any([abortSignal, failed.signal])- : failed.signal;- let nextPart = 1;- const worker = async () => {- while (nextPart <= partCount) {- const partNumber = nextPart++;- const start = (partNumber - 1) * partSize;- const end = Math.min(start + partSize, contentLength);- const partLength = end - start;- const url = await getMultipartPartUrl(- target,- partNumber,- partLength,- signal,- );- const etag = await uploadMultipartPart(- url,- fileHandle.slice(start, end),- partLength,- partNumber,- signal,- );- parts.push({ partNumber, etag, size: partLength });- }- };- const workers = Array.from(- { length: Math.min(multipartUploadConcurrencyFromEnv(), partCount) },- () => worker(),+async function abortMultipartUploadQuietly(+ target: Extract<StorageUploadTarget, { type: "multipart" }>,+) {+ await abortMultipartUpload(target).catch((abortError) => {+ console.warn(+ "Multipart upload abort failed:",+ abortError instanceof Error ? abortError.message : abortError,);- try {- await Promise.all(workers);- } catch (error) {- failed.abort();- await Promise.allSettled(workers);- throw error;- }- parts.sort((a, b) => a.partNumber - b.partNumber);+ });+}+async function uploadFileMultipart(+ filePath: string,+ target: Extract<StorageUploadTarget, { type: "multipart" }>,+ abortSignal?: AbortSignal,+): Promise<StorageUploadReceipt> {+ try {+ const parts = await uploadMultipartParts(filePath, target, abortSignal);return await completeMultipartUpload(target, parts, abortSignal);} catch (error) {- await abortMultipartUpload(target).catch((abortError) => {- console.warn(- "Multipart upload abort failed:",- abortError instanceof Error ? abortError.message : abortError,- );- });+ await abortMultipartUploadQuietly(target);throw error;}}+export interface PendingStorageUpload {+ // Publishes the upload; for multipart targets this is the only step that creates the object.+ finish(): Promise<StorageUploadReceipt>;+ // Stops early work and waits for it to settle, leaving the target to the caller's abort.+ cancel(): Promise<void>;+}++// Multipart parts are invisible until the upload is completed, so they can be sent while the caller+// is still checking the file; single PUT targets only upload once finish() is called.+export function beginStorageUpload(+ filePath: string,+ target: StorageUploadTarget,+ contentType: string,+ abortSignal?: AbortSignal,+): PendingStorageUpload {+ if (target.type !== "multipart")+ return {+ finish: () =>+ uploadFileToStorage(filePath, target, contentType, abortSignal),+ cancel: async () => {},+ };+ const early = new AbortController();+ const signal = abortSignal+ ? AbortSignal.any([abortSignal, early.signal])+ : early.signal;+ const parts = uploadMultipartParts(filePath, target, signal);+ parts.catch(() => {});+ return {+ finish: async () => {+ try {+ return await completeMultipartUpload(target, await parts, abortSignal);+ } catch (error) {+ await abortMultipartUploadQuietly(target);+ throw error;+ }+ },+ cancel: async () => {+ early.abort();+ await parts.catch(() => {});+ },+ };+}+export async function abortStorageUploadTarget(target: StorageUploadTarget,): Promise<void> {diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.tsindex 1bedd8777..bbcd6af24 100644--- a/apps/media-server/src/routes/video.ts+++ b/apps/media-server/src/routes/video.ts@@ -62,12 +62,14 @@ import {withMediaTransfers,} from "../lib/media-transfer";import type {+ PendingStorageUpload,ResilientInputFlags,StorageUploadReceipt,StorageUploadTarget,} from "../lib/media-video";import {abortStorageUploadTarget,+ beginStorageUpload,downloadVideoToTemp,generatePreviewGif,generateThumbnail,@@ -2964,6 +2966,7 @@ async function muxSegmentsAsync(const abortController = new AbortController();updateJob(jobId, { abortController });let outputUploadStarted = false;+ let pendingOutputUpload: PendingStorageUpload | undefined;let processingTimeout: ReturnType<typeof setTimeout> | undefined;let errorCode: RecordingErrorCode = "processing-unavailable";const sources = context.sourceObjects@@ -3116,6 +3119,16 @@ async function muxSegmentsAsync(const beforeDecode = await lstat(resultPath, { bigint: true });if (!beforeDecode.isFile())throw new Error("Recording verification requires a local regular file");+ // Hashing and multipart parts only read the finished file, so they run during the decode check; the+ // upload is only completed (the object only exists) after everything below has passed.+ pendingOutputUpload = beginStorageUpload(+ resultPath,+ outputUpload,+ "video/mp4",+ abortController.signal,+ );+ const outputHash = hashRecordingFile(resultPath, abortController.signal);+ outputHash.catch(() => {});updateJob(jobId, {progress: 70,message: "Checking the processed recording...",@@ -3161,9 +3174,7 @@ async function muxSegmentsAsync(message: "Verifying the recording file...",});sendCurrentJobWebhook(jobId);- const outputSha256 = await withJobHeartbeat(jobId, () =>- hashRecordingFile(resultPath, abortController.signal),- );+ const outputSha256 = await withJobHeartbeat(jobId, () => outputHash);const afterHash = await lstat(resultPath, { bigint: true });if (!afterHash.isFile() ||@@ -3226,13 +3237,11 @@ async function muxSegmentsAsync(outputUploadStarted = true;errorCode = "processing-unavailable";+ const startedOutputUpload = pendingOutputUpload;+ if (!startedOutputUpload)+ throw new Error("Recording output upload was not started");const uploadReceipt = await withJobHeartbeat(jobId, () =>- uploadFileToStorage(- resultPath,- outputUpload,- "video/mp4",- abortController.signal,- ),+ startedOutputUpload.finish(),);if (outputVerificationUrl) {errorCode = "output-invalid";@@ -3401,6 +3410,8 @@ async function muxSegmentsAsync(resources: getSystemResources(),});if (!outputUploadStarted) {+ // Parts still in flight could land after the abort, so let them settle first.+ await pendingOutputUpload?.cancel();await abortStorageUploadTarget(outputUpload).catch((abortError) => {console.warn(`[mux-segments] Failed to abort output upload for ${videoId}:`,