Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(media-server): bound preverified run reads and stop a failed run's downloads

Richie McIlroy92aabf44c46c
diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile
index 697c354cb..83b944ecf 100644
--- a/apps/media-server/Dockerfile
+++ b/apps/media-server/Dockerfile
@@ -19,10 +19,11 @@ RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/au
&& bun test src/__tests__/lib/loom-import-limit.test.ts \
&& bun test src/__tests__/lib/media-size.test.ts src/__tests__/lib/media-probe.integration.test.ts \
&& bun test src/__tests__/lib/media-audio.integration.test.ts \
- && bun test src/__tests__/lib/recording-verification.integration.test.ts src/__tests__/lib/job-manager.test.ts \
+ && bun test src/__tests__/lib/recording-verification.integration.test.ts src/__tests__/lib/job-manager.test.ts src/__tests__/lib/recording-preverify.test.ts \
&& bun test src/__tests__/lib/media-transfer.test.ts src/__tests__/lib/capcodec-encoder.test.ts src/__tests__/lib/nvenc-encoder.test.ts \
&& bun test src/__tests__/routes/recording-verification.test.ts \
&& bun test src/__tests__/routes/video.test.ts \
+ && bun test src/__tests__/routes/preverify.test.ts \
&& bun test src/__tests__/lib/lifecycle.test.ts src/__tests__/lib/temp-files.test.ts \
&& bun test src/__tests__/routes/lifecycle.test.ts \
&& bun test src/__tests__/routes/peer-jobs.test.ts \
diff --git a/apps/media-server/src/__tests__/lib/recording-preverify.test.ts b/apps/media-server/src/__tests__/lib/recording-preverify.test.ts
index 2ccd47bbc..69f1131cf 100644
--- a/apps/media-server/src/__tests__/lib/recording-preverify.test.ts
+++ b/apps/media-server/src/__tests__/lib/recording-preverify.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import {
type PreverifiedRunPayload,
+ readBoundedText,
readPreverifiedRun,
resolvePreverifiedRuns,
signPreverifiedRun,
@@ -109,3 +110,26 @@ describe("chaining preverified runs", () => {
).toEqual([]);
});
});
+
+describe("bounded run reads", () => {
+ const stream = (sizes: number[]) =>
+ new Response(
+ new ReadableStream({
+ start(controller) {
+ for (const size of sizes)
+ controller.enqueue(new Uint8Array(size).fill(97));
+ controller.close();
+ },
+ }),
+ );
+
+ test("reads a body within the limit", async () => {
+ expect(await readBoundedText(stream([3, 4]), 7)).toBe("aaaaaaa");
+ });
+
+ test("stops at the limit even without a Content-Length", async () => {
+ const response = stream([4, 4, 4]);
+ expect(response.headers.get("content-length")).toBeNull();
+ expect(await readBoundedText(response, 7)).toBeNull();
+ });
+});
diff --git a/apps/media-server/src/__tests__/routes/preverify.test.ts b/apps/media-server/src/__tests__/routes/preverify.test.ts
new file mode 100644
index 000000000..9a554a601
--- /dev/null
+++ b/apps/media-server/src/__tests__/routes/preverify.test.ts
@@ -0,0 +1,234 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtemp, readdir, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import app from "../../app";
+import { readPreverifiedRun } from "../../lib/recording-preverify";
+
+const SECRET = "test-secret";
+process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET;
+
+type Served = {
+ body: Uint8Array<ArrayBuffer>;
+ etag: string;
+ status?: number;
+ hang?: boolean;
+ delayMs?: number;
+};
+const objects = new Map<string, Served>();
+const uploads = new Map<string, string>();
+let openHangs = 0;
+let server: ReturnType<typeof Bun.serve>;
+let base: string;
+
+async function run(command: string[]) {
+ const child = Bun.spawn(command, { stdout: "pipe", stderr: "pipe" });
+ if ((await child.exited) !== 0)
+ throw new Error(await new Response(child.stderr).text());
+}
+
+beforeAll(async () => {
+ server = Bun.serve({
+ port: 0,
+ async fetch(request) {
+ const key = new URL(request.url).pathname;
+ if (request.method === "PUT") {
+ uploads.set(key, await request.text());
+ return new Response(null, { status: 200, headers: { ETag: '"put"' } });
+ }
+ const served = objects.get(key);
+ if (!served) return new Response(null, { status: 404 });
+ if (served.hang) {
+ openHangs++;
+ await new Promise<void>((resolve) =>
+ request.signal.addEventListener("abort", () => resolve(), {
+ once: true,
+ }),
+ );
+ openHangs--;
+ return new Response(null, { status: 499 });
+ }
+ if (served.delayMs) await Bun.sleep(served.delayMs);
+ if (served.status) return new Response(null, { status: served.status });
+ return new Response(served.body, {
+ headers: {
+ ETag: served.etag,
+ "Content-Length": String(served.body.length),
+ },
+ });
+ },
+ });
+ base = `http://127.0.0.1:${server.port}`;
+});
+
+afterAll(() => server.stop(true));
+
+function source(key: string) {
+ const served = objects.get(key);
+ if (!served) throw new Error(`Missing ${key}`);
+ return {
+ url: `${base}${key}`,
+ objectIdentity: served.etag,
+ size: served.body.length || 1,
+ };
+}
+
+function request(body: unknown, secret = SECRET) {
+ return app.fetch(
+ new Request("http://localhost/video/preverify-segments", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "x-media-server-secret": secret,
+ },
+ body: JSON.stringify(body),
+ }),
+ );
+}
+
+async function until(check: () => boolean, timeoutMs = 20_000) {
+ const deadline = Date.now() + timeoutMs;
+ while (!check()) {
+ if (Date.now() > deadline) throw new Error("Timed out waiting");
+ await Bun.sleep(50);
+ }
+}
+
+async function seedFragments(name: string) {
+ const directory = await mkdtemp(join(tmpdir(), `preverify-${name}-`));
+ await run([
+ "ffmpeg",
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=320x180:rate=30",
+ "-frames:v",
+ "60",
+ "-c:v",
+ "libx264",
+ "-preset",
+ "ultrafast",
+ "-g",
+ "15",
+ "-keyint_min",
+ "15",
+ "-sc_threshold",
+ "0",
+ "-pix_fmt",
+ "yuv420p",
+ "-f",
+ "dash",
+ "-seg_duration",
+ "0.5",
+ "-init_seg_name",
+ "init.mp4",
+ "-media_seg_name",
+ "fragment-$Number%05d$.m4s",
+ join(directory, "manifest.mpd"),
+ ]);
+ const files = (await readdir(directory))
+ .filter((file) => /^fragment-\d+\.m4s$/.test(file))
+ .sort();
+ const keys: string[] = [];
+ for (const [index, file] of ["init.mp4", ...files].entries()) {
+ const key = `/${name}/${file}`;
+ objects.set(key, {
+ body: new Uint8Array(await readFile(join(directory, file))),
+ etag: `"${name}-${index}"`,
+ });
+ keys.push(key);
+ }
+ return keys;
+}
+
+describe("POST /video/preverify-segments", () => {
+ test("requires the media server secret and a complete body", async () => {
+ expect((await request({}, "wrong")).status).toBe(401);
+ expect((await request({ videoId: "video" })).status).toBe(400);
+ });
+
+ test("stores a signed run that names the exact fragments it decoded", async () => {
+ const [init, ...fragments] = await seedFragments("signed");
+ if (!init) throw new Error("Missing init");
+ const response = await request({
+ videoId: "video",
+ init: source(init),
+ segments: fragments.map(source),
+ resultUploadUrl: `${base}/results/signed.json`,
+ });
+ expect(response.status).toBe(202);
+ await until(() => uploads.has("/results/signed.json"));
+ const run = readPreverifiedRun(
+ uploads.get("/results/signed.json") ?? "",
+ SECRET,
+ );
+ expect(run?.init).toEqual({
+ objectIdentity: source(init).objectIdentity,
+ size: source(init).size,
+ });
+ expect(run?.segments.map((segment) => segment.objectIdentity)).toEqual(
+ fragments.map((key) => source(key).objectIdentity),
+ );
+ expect(run?.lines.filter((line) => !line.startsWith("#"))).toHaveLength(60);
+ });
+
+ test("stops every download and frees its slot when one fragment fails", async () => {
+ const [init, first, ...rest] = await seedFragments("failing");
+ if (!init || !first) throw new Error("Missing fragments");
+ objects.set(first, {
+ ...(objects.get(first) as Served),
+ status: 403,
+ delayMs: 300,
+ });
+ for (const key of rest)
+ objects.set(key, { ...(objects.get(key) as Served), hang: true });
+ const response = await request({
+ videoId: "video",
+ init: source(init),
+ segments: [first, ...rest].map(source),
+ resultUploadUrl: `${base}/results/failing.json`,
+ });
+ expect(response.status).toBe(202);
+ await until(() => openHangs > 0);
+ await until(() => openHangs === 0);
+ const [, ...healthy] = await seedFragments("after-failure");
+ const accepted = await Promise.all(
+ [0, 1].map(
+ async () =>
+ (
+ await request({
+ videoId: "video",
+ init: source(`/after-failure/init.mp4`),
+ segments: healthy.slice(0, 2).map(source),
+ resultUploadUrl: `${base}/results/after-${Math.random()}.json`,
+ })
+ ).status,
+ ),
+ );
+ expect(accepted).toEqual([202, 202]);
+ expect(uploads.has("/results/failing.json")).toBe(false);
+ await until(
+ () =>
+ [...uploads.keys()].filter((key) => key.startsWith("/results/after-"))
+ .length === 2,
+ );
+ });
+
+ test("declines a third concurrent run", async () => {
+ const [init, ...fragments] = await seedFragments("busy");
+ if (!init) throw new Error("Missing init");
+ for (const key of fragments)
+ objects.set(key, { ...(objects.get(key) as Served), hang: true });
+ const body = (name: string) => ({
+ videoId: "video",
+ init: source(init),
+ segments: fragments.slice(0, 1).map(source),
+ resultUploadUrl: `${base}/results/${name}.json`,
+ });
+ expect((await request(body("busy-1"))).status).toBe(202);
+ expect((await request(body("busy-2"))).status).toBe(202);
+ expect((await request(body("busy-3"))).status).toBe(503);
+ });
+});
diff --git a/apps/media-server/src/lib/recording-preverify.ts b/apps/media-server/src/lib/recording-preverify.ts
index 25e447b2f..e609fa197 100644
--- a/apps/media-server/src/lib/recording-preverify.ts
+++ b/apps/media-server/src/lib/recording-preverify.ts
@@ -93,3 +93,29 @@ export function resolvePreverifiedRuns(
}
return resolved;
}
+
+// Reads at most `limit` bytes, whatever the response claims about its length.
+export async function readBoundedText(
+ response: Response,
+ limit: number,
+): Promise<string | null> {
+ const reader = response.body?.getReader();
+ if (!reader) return null;
+ const chunks: Uint8Array[] = [];
+ let bytes = 0;
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ bytes += value.byteLength;
+ if (bytes > limit) {
+ await reader.cancel().catch(() => {});
+ return null;
+ }
+ chunks.push(value);
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ return new TextDecoder().decode(Buffer.concat(chunks));
+}
diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts
index 87782dd75..2f4cf6ea2 100644
--- a/apps/media-server/src/routes/video.ts
+++ b/apps/media-server/src/routes/video.ts
@@ -87,6 +87,7 @@ import { findJobOnPeer, isPeerLookup } from "../lib/peer-jobs";
import {
MAX_PREVERIFIED_RUN_BYTES,
type PreverifiedRunPayload,
+ readBoundedText,
readPreverifiedRun,
resolvePreverifiedRuns,
signPreverifiedRun,
@@ -2735,21 +2736,31 @@ async function preverifySegments(
path: segmentPaths[index] as string,
})),
];
+ // One failed download stops the others, and the run only ends once none is still writing.
+ const downloadAbort = new AbortController();
+ const downloadSignal = AbortSignal.any([signal, downloadAbort.signal]);
let next = 0;
- await Promise.all(
+ const workers = await Promise.allSettled(
Array.from({ length: Math.min(8, downloads.length) }, async () => {
- while (next < downloads.length) {
+ while (next < downloads.length && !downloadSignal.aborted) {
const download = downloads[next++];
if (!download) return;
- await downloadUrlToFile(
- download.source.url,
- download.path,
- signal,
- download.source,
- );
+ try {
+ await downloadUrlToFile(
+ download.source.url,
+ download.path,
+ downloadSignal,
+ download.source,
+ );
+ } catch (error) {
+ downloadAbort.abort();
+ throw error;
+ }
}
}),
);
+ const failed = workers.find((worker) => worker.status === "rejected");
+ if (failed?.status === "rejected") throw failed.reason;
const lines = await decodePreverifiedRun(
initPath,
segmentPaths,
@@ -2816,16 +2827,14 @@ async function loadPreverifiedRuns(
if (!url) return;
try {
const response = await fetch(url, { signal: budget });
- const length = Number(response.headers.get("content-length"));
- if (
- !response.ok ||
- !Number.isSafeInteger(length) ||
- length > MAX_PREVERIFIED_RUN_BYTES
- ) {
+ const text = response.ok
+ ? await readBoundedText(response, MAX_PREVERIFIED_RUN_BYTES)
+ : null;
+ if (text === null) {
await response.body?.cancel().catch(() => {});
continue;
}
- const run = readPreverifiedRun(await response.text(), secret);
+ const run = readPreverifiedRun(text, secret);
if (run) runs.push(run);
} catch {
signal.throwIfAborted();