Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(media-server): upload multipart recording outputs while they are verified

Richie McIlroyc45b2482f2dc
diff --git a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts
index f59eb5b58..49a3d070c 100644
--- a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts
+++ b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts
@@ -858,7 +858,8 @@ describe("media routes real-world integration tests", () => {
}
}, 30_000);
- test("refuses a local output changed between full decode and byte binding", async () => {
+ // Hashing now overlaps the decode check, so whichever check reads the changed bytes first refuses it.
+ test("refuses a local output changed while it is decoded and hashed", async () => {
const originalHash = recordingVerification.hashRecordingFile;
const hashing = spyOn(
recordingVerification,
@@ -882,8 +883,8 @@ describe("media routes real-world integration tests", () => {
jobId = ((await response.json()) as { jobId: string }).jobId;
const job = await waitForTerminalJob(jobId);
expect(job.phase).toBe("error");
- expect(job.error).toContain(
- "Local recording changed during verification",
+ expect(job.error).toMatch(
+ /Local recording changed during verification|does not preserve the source/,
);
expect(job.errorCode).toBe("output-invalid");
expect(job.recordingVerification).toBeUndefined();
@@ -930,6 +931,31 @@ describe("media routes real-world integration tests", () => {
30_000,
);
+ test.each(["corrupt", "corrupt-audio"] as const)(
+ "never completes a multipart output whose source fails verification (%s)",
+ async (fault) => {
+ sourceFault = fault;
+ const body = multipartMuxRequest(`multipart-unverified-${fault}`);
+ const response = await app.fetch(
+ mediaPostRequest("/video/mux-segments", body),
+ );
+ expect(response.status).toBe(200);
+ const { jobId } = (await response.json()) as { jobId: string };
+ try {
+ const job = await waitForTerminalJob(jobId);
+ expect(job.phase).toBe("error");
+ expect(job.recordingVerification).toBeUndefined();
+ const actions = multipartCallbacks.map(({ action }) => action);
+ // Parts may already be uploaded while the check runs, but the object is never created.
+ expect(actions).not.toContain("complete");
+ expect(actions.at(-1)).toBe("abort");
+ } finally {
+ deleteJob(jobId);
+ }
+ },
+ 30_000,
+ );
+
test.each(["changed", "missing", "corrupt", "corrupt-audio"] as const)(
"withholds upload and proof after a pinned source is %s",
async (fault) => {
diff --git a/apps/media-server/src/__tests__/lib/storage-upload.test.ts b/apps/media-server/src/__tests__/lib/storage-upload.test.ts
index 4df363215..bbc3cf7ed 100644
--- a/apps/media-server/src/__tests__/lib/storage-upload.test.ts
+++ b/apps/media-server/src/__tests__/lib/storage-upload.test.ts
@@ -4,6 +4,7 @@ import { mkdtemp, open, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
+ beginStorageUpload,
multipartUploadConcurrencyFromEnv,
uploadFileToStorage,
} from "../../lib/media-video";
@@ -496,3 +497,132 @@ describe("uploadFileToStorage", () => {
);
});
});
+
+describe("beginStorageUpload", () => {
+ afterEach(() => {
+ globalThis.fetch = originalFetch;
+ });
+
+ const multipartTarget = {
+ type: "multipart" as const,
+ videoId: "video",
+ key: "owner/video/result.mp4",
+ uploadId: "upload",
+ partSize,
+ signPartUrl: "https://cap.test/sign-part",
+ completeUrl: "https://cap.test/complete",
+ abortUrl: "https://cap.test/abort",
+ };
+
+ async function waitUntil(condition: () => boolean) {
+ for (let tries = 0; tries < 100 && !condition(); tries++)
+ await Bun.sleep(5);
+ }
+
+ function recordMultipart(holdParts?: Promise<void>) {
+ const calls: string[] = [];
+ globalThis.fetch = (async (input, init) => {
+ const url = String(input);
+ if (url.endsWith("/sign-part")) {
+ const { partNumber } = JSON.parse(String(init?.body));
+ calls.push(`sign-${partNumber}`);
+ return Response.json({
+ url: `https://storage.test/part/${partNumber}`,
+ });
+ }
+ if (init?.method === "PUT") {
+ calls.push(`put-${url.split("/").at(-1)}`);
+ if (holdParts)
+ await Promise.race([
+ holdParts,
+ new Promise((_, reject) =>
+ init.signal?.addEventListener("abort", () =>
+ reject(new DOMException("Aborted", "AbortError")),
+ ),
+ ),
+ ]);
+ return new Response(null, { headers: { ETag: `"etag-${url}"` } });
+ }
+ if (url.endsWith("/complete")) {
+ calls.push("complete");
+ return Response.json({ objectIdentity: '"done"' });
+ }
+ if (url.endsWith("/abort")) {
+ calls.push("abort");
+ return Response.json({ success: true });
+ }
+ throw new Error(`Unexpected request ${url}`);
+ }) as typeof fetch;
+ return calls;
+ }
+
+ const puts = (calls: string[]) =>
+ calls.filter((call) => call.startsWith("put"));
+
+ test("uploads multipart parts before finish and completes only when finished", async () => {
+ const uploadFile = await createTempUploadFile(partSize * 2 + 1);
+ const calls = recordMultipart();
+ try {
+ const pending = beginStorageUpload(
+ uploadFile.path,
+ multipartTarget,
+ "video/mp4",
+ );
+ await waitUntil(() => puts(calls).length === 3);
+ expect(puts(calls)).toHaveLength(3);
+ expect(calls).not.toContain("complete");
+ expect(await pending.finish()).toEqual({ objectIdentity: '"done"' });
+ expect(calls.at(-1)).toBe("complete");
+ expect(calls).not.toContain("abort");
+ } finally {
+ await uploadFile.cleanup();
+ }
+ });
+
+ test("cancel stops in-flight parts without completing and leaves the abort to the caller", async () => {
+ const uploadFile = await createTempUploadFile(partSize * 2 + 1);
+ let release = () => {};
+ const calls = recordMultipart(
+ new Promise<void>((resolve) => {
+ release = resolve;
+ }),
+ );
+ try {
+ const pending = beginStorageUpload(
+ uploadFile.path,
+ multipartTarget,
+ "video/mp4",
+ );
+ await waitUntil(() => puts(calls).length > 0);
+ expect(puts(calls).length).toBeGreaterThan(0);
+ await pending.cancel();
+ expect(calls).not.toContain("complete");
+ expect(calls).not.toContain("abort");
+ } finally {
+ release();
+ await uploadFile.cleanup();
+ }
+ });
+
+ test("sends nothing for a single PUT target until finish", async () => {
+ const uploadFile = await createTempUploadFile(16);
+ const calls: string[] = [];
+ globalThis.fetch = (async (_input, init) => {
+ calls.push(String(init?.method));
+ return new Response(null, { headers: { ETag: '"put"' } });
+ }) as typeof fetch;
+ try {
+ const pending = beginStorageUpload(
+ uploadFile.path,
+ { type: "put", url: "https://storage.test/result.mp4" },
+ "video/mp4",
+ );
+ await Bun.sleep(20);
+ expect(calls).toHaveLength(0);
+ await pending.finish();
+ expect(calls).toEqual(["PUT"]);
+ } finally {
+ await uploadFile.cleanup();
+ }
+ });
+});
diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts
index da16eceb2..e1030ae5f 100644
--- a/apps/media-server/src/lib/media-video.ts
+++ b/apps/media-server/src/lib/media-video.ts
@@ -2796,85 +2796,141 @@ export function multipartUploadConcurrencyFromEnv(
return Number.isFinite(value) && value > 0 ? Math.min(value, 16) : 1;
}
-async function uploadFileMultipart(
+async function uploadMultipartParts(
filePath: string,
target: Extract<StorageUploadTarget, { type: "multipart" }>,
abortSignal?: AbortSignal,
-): Promise<StorageUploadReceipt> {
+): Promise<UploadedPart[]> {
const fileHandle = file(filePath);
const contentLength = fileHandle.size;
const partSize = Math.floor(target.partSize);
const parts: UploadedPart[] = [];
- try {
- abortSignal?.throwIfAborted();
- if (contentLength <= 0) {
- throw new Error("Multipart upload requires a non-empty file");
- }
+ abortSignal?.throwIfAborted();
+ if (contentLength <= 0) {
+ throw new Error("Multipart upload requires a non-empty file");
+ }
- if (partSize < MULTIPART_MIN_PART_SIZE_BYTES) {
- throw new Error("Multipart upload part size is too small");
- }
+ if (partSize < MULTIPART_MIN_PART_SIZE_BYTES) {
+ throw new Error("Multipart upload part size is too small");
+ }
- const partCount = Math.ceil(contentLength / partSize);
- if (partCount > MULTIPART_MAX_PARTS) {
- throw new Error(
- `Multipart upload would require too many parts: ${partCount}/${MULTIPART_MAX_PARTS}`,
+ const partCount = Math.ceil(contentLength / partSize);
+ if (partCount > MULTIPART_MAX_PARTS) {
+ throw new Error(
+ `Multipart upload would require too many parts: ${partCount}/${MULTIPART_MAX_PARTS}`,
+ );
+ }
+
+ const failed = new AbortController();
+ const signal = abortSignal
+ ? AbortSignal.any([abortSignal, failed.signal])
+ : failed.signal;
+ let nextPart = 1;
+ const worker = async () => {
+ while (nextPart <= partCount) {
+ const partNumber = nextPart++;
+ const start = (partNumber - 1) * partSize;
+ const end = Math.min(start + partSize, contentLength);
+ const partLength = end - start;
+ const url = await getMultipartPartUrl(
+ target,
+ partNumber,
+ partLength,
+ signal,
+ );
+ const etag = await uploadMultipartPart(
+ url,
+ fileHandle.slice(start, end),
+ partLength,
+ partNumber,
+ signal,
);
+ parts.push({ partNumber, etag, size: partLength });
}
+ };
+ const workers = Array.from(
+ { length: Math.min(multipartUploadConcurrencyFromEnv(), partCount) },
+ () => worker(),
+ );
+ try {
+ await Promise.all(workers);
+ } catch (error) {
+ failed.abort();
+ await Promise.allSettled(workers);
+ throw error;
+ }
+ return parts.sort((a, b) => a.partNumber - b.partNumber);
+}
- const failed = new AbortController();
- const signal = abortSignal
- ? AbortSignal.any([abortSignal, failed.signal])
- : failed.signal;
- let nextPart = 1;
- const worker = async () => {
- while (nextPart <= partCount) {
- const partNumber = nextPart++;
- const start = (partNumber - 1) * partSize;
- const end = Math.min(start + partSize, contentLength);
- const partLength = end - start;
- const url = await getMultipartPartUrl(
- target,
- partNumber,
- partLength,
- signal,
- );
- const etag = await uploadMultipartPart(
- url,
- fileHandle.slice(start, end),
- partLength,
- partNumber,
- signal,
- );
- parts.push({ partNumber, etag, size: partLength });
- }
- };
- const workers = Array.from(
- { length: Math.min(multipartUploadConcurrencyFromEnv(), partCount) },
- () => worker(),
+async function abortMultipartUploadQuietly(
+ target: Extract<StorageUploadTarget, { type: "multipart" }>,
+) {
+ await abortMultipartUpload(target).catch((abortError) => {
+ console.warn(
+ "Multipart upload abort failed:",
+ abortError instanceof Error ? abortError.message : abortError,
);
- try {
- await Promise.all(workers);
- } catch (error) {
- failed.abort();
- await Promise.allSettled(workers);
- throw error;
- }
- parts.sort((a, b) => a.partNumber - b.partNumber);
+ });
+}
+async function uploadFileMultipart(
+ filePath: string,
+ target: Extract<StorageUploadTarget, { type: "multipart" }>,
+ abortSignal?: AbortSignal,
+): Promise<StorageUploadReceipt> {
+ try {
+ const parts = await uploadMultipartParts(filePath, target, abortSignal);
return await completeMultipartUpload(target, parts, abortSignal);
} catch (error) {
- await abortMultipartUpload(target).catch((abortError) => {
- console.warn(
- "Multipart upload abort failed:",
- abortError instanceof Error ? abortError.message : abortError,
- );
- });
+ await abortMultipartUploadQuietly(target);
throw error;
}
}
+export interface PendingStorageUpload {
+ // Publishes the upload; for multipart targets this is the only step that creates the object.
+ finish(): Promise<StorageUploadReceipt>;
+ // Stops early work and waits for it to settle, leaving the target to the caller's abort.
+ cancel(): Promise<void>;
+}
+
+// Multipart parts are invisible until the upload is completed, so they can be sent while the caller
+// is still checking the file; single PUT targets only upload once finish() is called.
+export function beginStorageUpload(
+ filePath: string,
+ target: StorageUploadTarget,
+ contentType: string,
+ abortSignal?: AbortSignal,
+): PendingStorageUpload {
+ if (target.type !== "multipart")
+ return {
+ finish: () =>
+ uploadFileToStorage(filePath, target, contentType, abortSignal),
+ cancel: async () => {},
+ };
+ const early = new AbortController();
+ const signal = abortSignal
+ ? AbortSignal.any([abortSignal, early.signal])
+ : early.signal;
+ const parts = uploadMultipartParts(filePath, target, signal);
+ parts.catch(() => {});
+ return {
+ finish: async () => {
+ try {
+ return await completeMultipartUpload(target, await parts, abortSignal);
+ } catch (error) {
+ await abortMultipartUploadQuietly(target);
+ throw error;
+ }
+ },
+ cancel: async () => {
+ early.abort();
+ await parts.catch(() => {});
+ },
+ };
+}
+
export async function abortStorageUploadTarget(
target: StorageUploadTarget,
): Promise<void> {
diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts
index 1bedd8777..bbcd6af24 100644
--- a/apps/media-server/src/routes/video.ts
+++ b/apps/media-server/src/routes/video.ts
@@ -62,12 +62,14 @@ import {
withMediaTransfers,
} from "../lib/media-transfer";
import type {
+ PendingStorageUpload,
ResilientInputFlags,
StorageUploadReceipt,
StorageUploadTarget,
} from "../lib/media-video";
import {
abortStorageUploadTarget,
+ beginStorageUpload,
downloadVideoToTemp,
generatePreviewGif,
generateThumbnail,
@@ -2964,6 +2966,7 @@ async function muxSegmentsAsync(
const abortController = new AbortController();
updateJob(jobId, { abortController });
let outputUploadStarted = false;
+ let pendingOutputUpload: PendingStorageUpload | undefined;
let processingTimeout: ReturnType<typeof setTimeout> | undefined;
let errorCode: RecordingErrorCode = "processing-unavailable";
const sources = context.sourceObjects
@@ -3116,6 +3119,16 @@ async function muxSegmentsAsync(
const beforeDecode = await lstat(resultPath, { bigint: true });
if (!beforeDecode.isFile())
throw new Error("Recording verification requires a local regular file");
+ // Hashing and multipart parts only read the finished file, so they run during the decode check; the
+ // upload is only completed (the object only exists) after everything below has passed.
+ pendingOutputUpload = beginStorageUpload(
+ resultPath,
+ outputUpload,
+ "video/mp4",
+ abortController.signal,
+ );
+ const outputHash = hashRecordingFile(resultPath, abortController.signal);
+ outputHash.catch(() => {});
updateJob(jobId, {
progress: 70,
message: "Checking the processed recording...",
@@ -3161,9 +3174,7 @@ async function muxSegmentsAsync(
message: "Verifying the recording file...",
});
sendCurrentJobWebhook(jobId);
- const outputSha256 = await withJobHeartbeat(jobId, () =>
- hashRecordingFile(resultPath, abortController.signal),
- );
+ const outputSha256 = await withJobHeartbeat(jobId, () => outputHash);
const afterHash = await lstat(resultPath, { bigint: true });
if (
!afterHash.isFile() ||
@@ -3226,13 +3237,11 @@ async function muxSegmentsAsync(
outputUploadStarted = true;
errorCode = "processing-unavailable";
+ const startedOutputUpload = pendingOutputUpload;
+ if (!startedOutputUpload)
+ throw new Error("Recording output upload was not started");
const uploadReceipt = await withJobHeartbeat(jobId, () =>
- uploadFileToStorage(
- resultPath,
- outputUpload,
- "video/mp4",
- abortController.signal,
- ),
+ startedOutputUpload.finish(),
);
if (outputVerificationUrl) {
errorCode = "output-invalid";
@@ -3401,6 +3410,8 @@ async function muxSegmentsAsync(
resources: getSystemResources(),
});
if (!outputUploadStarted) {
+ // Parts still in flight could land after the abort, so let them settle first.
+ await pendingOutputUpload?.cancel();
await abortStorageUploadTarget(outputUpload).catch((abortError) => {
console.warn(
`[mux-segments] Failed to abort output upload for ${videoId}:`,