Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(media-server): verify recording fragments during the recording and reuse them at Stop

Richie McIlroy07d80bb6fe50
diff --git a/apps/media-server/src/__tests__/lib/recording-preverify.test.ts b/apps/media-server/src/__tests__/lib/recording-preverify.test.ts
new file mode 100644
index 000000000..2ccd47bbc
--- /dev/null
+++ b/apps/media-server/src/__tests__/lib/recording-preverify.test.ts
@@ -0,0 +1,111 @@
+import { describe, expect, test } from "bun:test";
+import {
+ type PreverifiedRunPayload,
+ readPreverifiedRun,
+ resolvePreverifiedRuns,
+ signPreverifiedRun,
+} from "../../lib/recording-preverify";
+
+const init = { objectIdentity: '"init"', size: 800 };
+const segment = (index: number) => ({
+ objectIdentity: `"segment-${index}"`,
+ size: 1000 + index,
+});
+const segments = Array.from({ length: 8 }, (_, index) => segment(index + 1));
+const run = (from: number, to: number, label = ""): PreverifiedRunPayload => ({
+ version: 1,
+ init,
+ segments: segments.slice(from, to),
+ lines: [`#tb 0: 1/30${label}`, `0, 0, 0, 1, 100, 0x0000000${from}`],
+});
+
+describe("signed preverified runs", () => {
+ test("round-trips a run signed with the same secret", () => {
+ const payload = run(0, 3);
+ expect(
+ readPreverifiedRun(signPreverifiedRun(payload, "secret"), "secret"),
+ ).toEqual(payload);
+ });
+
+ test.each([
+ ["another secret", (text: string) => text, "other"],
+ [
+ "a changed line",
+ (text: string) => text.replace("0x00000000", "0x00000001"),
+ "secret",
+ ],
+ [
+ "a changed fragment identity",
+ (text: string) => text.replace('\\"segment-1\\"', '\\"segment-9\\"'),
+ "secret",
+ ],
+ [
+ "a removed signature",
+ (text: string) =>
+ JSON.stringify({ ...JSON.parse(text), signature: undefined }),
+ "secret",
+ ],
+ ["invalid JSON", () => "{", "secret"],
+ ])("rejects %s", (_, tamper, secret) => {
+ expect(
+ readPreverifiedRun(
+ tamper(signPreverifiedRun(run(0, 3), "secret")),
+ secret,
+ ),
+ ).toBeNull();
+ });
+});
+
+describe("chaining preverified runs", () => {
+ test("chains runs that cover fragments from the start in order", () => {
+ const resolved = resolvePreverifiedRuns(
+ [run(3, 6), run(0, 3)],
+ init,
+ segments,
+ );
+ expect(resolved.map((entry) => entry.segments)).toEqual([3, 3]);
+ });
+
+ test("stops at the first fragment no run starts at", () => {
+ expect(
+ resolvePreverifiedRuns([run(0, 2), run(3, 5)], init, segments).map(
+ (entry) => entry.segments,
+ ),
+ ).toEqual([2]);
+ });
+
+ test("prefers the longest run starting at a fragment", () => {
+ expect(
+ resolvePreverifiedRuns(
+ [run(0, 2), run(0, 4, "-long")],
+ init,
+ segments,
+ ).map((entry) => entry.segments),
+ ).toEqual([4]);
+ });
+
+ test("ignores runs over another recording's init or different fragment bytes", () => {
+ expect(
+ resolvePreverifiedRuns(
+ [{ ...run(0, 3), init: { ...init, objectIdentity: '"other"' } }],
+ init,
+ segments,
+ ),
+ ).toEqual([]);
+ expect(
+ resolvePreverifiedRuns(
+ [run(0, 3)],
+ init,
+ segments.map((entry, index) =>
+ index === 1 ? { ...entry, size: entry.size + 1 } : entry,
+ ),
+ ),
+ ).toEqual([]);
+ });
+
+ test("ignores a run longer than the recording", () => {
+ expect(
+ resolvePreverifiedRuns([run(0, 8)], init, segments.slice(0, 5)),
+ ).toEqual([]);
+ });
+});
diff --git a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts
index d30f14b0d..3da0952f5 100644
--- a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts
+++ b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts
@@ -25,8 +25,10 @@ import {
timingMetadataBudget,
} from "../../lib/recording-timing";
import {
+ decodePreverifiedRun,
inspectRecordingSources,
isRetryableRecordingVerificationError,
+ type PreverifiedVideoRun,
verifyRecording,
verifyRemoteRecording,
verifyRemoteRecordingBytes,
@@ -2971,6 +2973,184 @@ describe("chunked packet-proven verification", () => {
delete process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS;
}
});
+
+ async function preverify(
+ recording: { init: string; segments: string[] },
+ from: number,
+ to: number,
+ ): Promise<PreverifiedVideoRun> {
+ const work = await mkdtemp(join(directory, "preverify-"));
+ const lines = await decodePreverifiedRun(
+ recording.init,
+ recording.segments.slice(from, to),
+ work,
+ );
+ if (!lines) throw new Error("Expected a decodable preverified run");
+ return { segments: to - from, lines };
+ }
+
+ async function withRuns(
+ recording: Awaited<ReturnType<typeof segmentedRecording>>,
+ runs: PreverifiedVideoRun[],
+ chunks?: string,
+ ) {
+ if (chunks === undefined)
+ delete process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS;
+ else process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS = chunks;
+ const warn = spyOn(console, "warn");
+ try {
+ const result = await verifyRemuxedRecording(
+ recording.combined,
+ silent,
+ recording.output,
+ { requireAudio: false },
+ { init: recording.init, segments: recording.segments },
+ runs,
+ );
+ const fallbacks = warn.mock.calls.filter((call) =>
+ String(call[0]).includes("recording_chunked_decode_fallback"),
+ );
+ return { result, fallbacks };
+ } finally {
+ warn.mockRestore();
+ delete process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS;
+ }
+ }
+
+ test.each([undefined, "3"])(
+ "returns the same evidence from preverified runs as one full decode (chunks %s)",
+ async (chunks) => {
+ const recording = await segmentedRecording("preverified-parity", 15);
+ expect(recording.segments.length).toBeGreaterThanOrEqual(6);
+ const { single } = await both(recording);
+ const runs = [
+ await preverify(recording, 0, 2),
+ await preverify(recording, 2, 4),
+ ];
+ const { result, fallbacks } = await withRuns(recording, runs, chunks);
+ expect(fallbacks).toHaveLength(0);
+ expect(result).toEqual(single);
+ expect(result.video.frameCount).toBe(120);
+ },
+ );
+
+ test("accepts runs that cover every fragment", async () => {
+ const recording = await segmentedRecording("preverified-complete", 15);
+ const { single } = await both(recording);
+ const { result, fallbacks } = await withRuns(recording, [
+ await preverify(recording, 0, recording.segments.length),
+ ]);
+ expect(fallbacks).toHaveLength(0);
+ expect(result).toEqual(single);
+ });
+
+ test.each([
+ "dropped frame",
+ "extra frame",
+ "shifted timestamps",
+ "changed header",
+ ])(
+ "falls back to one full decode when a cached run has a %s",
+ async (defect) => {
+ const recording = await segmentedRecording(
+ `preverified-${defect.replaceAll(" ", "-")}`,
+ 15,
+ );
+ const { single } = await both(recording);
+ const run = await preverify(recording, 0, 3);
+ const frames = run.lines.filter((line) => !line.startsWith("#"));
+ const header = run.lines.filter((line) => line.startsWith("#"));
+ const changed =
+ defect === "dropped frame"
+ ? [...header, ...frames.slice(0, -1)]
+ : defect === "extra frame"
+ ? [...header, ...frames, frames.at(-1) as string]
+ : defect === "shifted timestamps"
+ ? [
+ ...header,
+ ...frames.map((line) =>
+ line.replace(
+ /^0,\s*(\d+),\s*(\d+)/,
+ (_, dts, pts) =>
+ `0, ${Number(dts) + 1}, ${Number(pts) + 1}`,
+ ),
+ ),
+ ]
+ : [
+ ...header.map((line) =>
+ line.replace(/^#dimensions 0: .*/, "#dimensions 0: 1x1"),
+ ),
+ ...frames,
+ ];
+ const { result, fallbacks } = await withRuns(recording, [
+ { segments: run.segments, lines: changed },
+ ]);
+ expect(fallbacks).toHaveLength(1);
+ expect(result).toEqual(single);
+ },
+ );
+
+ test("drops a cached run whose next fragment cannot be decoded on its own", async () => {
+ const recording = await misalignedRecording("preverified-misaligned");
+ const { single } = await both(recording);
+ const work = await mkdtemp(join(directory, "preverify-misaligned-"));
+ const lines = await decodePreverifiedRun(
+ recording.init,
+ recording.segments.slice(0, 1),
+ work,
+ );
+ expect(lines).not.toBeNull();
+ const { result } = await withRuns(recording, [
+ { segments: 1, lines: lines ?? [] },
+ ]);
+ expect(result).toEqual(single);
+ });
+
+ test("does not preverify a run that does not start on a keyframe", async () => {
+ const recording = await misalignedRecording("preverified-unaligned-start");
+ const work = await mkdtemp(join(directory, "preverify-unaligned-"));
+ expect(
+ await decodePreverifiedRun(
+ recording.init,
+ recording.segments.slice(1, 3),
+ work,
+ ),
+ ).toBeNull();
+ });
+
+ test("does not preverify fragments that fail to decode", async () => {
+ const recording = await segmentedRecording("preverified-corrupt", 15);
+ const [first, second] = recording.segments;
+ if (!first || !second) throw new Error("Expected two fragments");
+ const packets = JSON.parse(
+ await run([
+ "ffprobe",
+ "-v",
+ "error",
+ "-protocol_whitelist",
+ "file,concat",
+ "-select_streams",
+ "v:0",
+ "-show_packets",
+ "-show_entries",
+ "packet=pos,size",
+ "-of",
+ "json",
+ `concat:${recording.init}|${second}`,
+ ]),
+ ) as { packets: { pos: string; size: string }[] };
+ const last = packets.packets.at(-1);
+ if (!last) throw new Error("Fragment has no video packet");
+ const offset = Number(last.pos) - (await readFile(recording.init)).length;
+ const bytes = Buffer.from(await readFile(second));
+ bytes.fill(0, offset, offset + Number(last.size));
+ const corrupt = join(directory, "preverified-corrupt-fragment.m4s");
+ await writeFile(corrupt, bytes);
+ const work = await mkdtemp(join(directory, "preverify-corrupt-"));
+ expect(
+ await decodePreverifiedRun(recording.init, [first, corrupt], work),
+ ).toBeNull();
+ });
});
describe("ranged remote byte verification", () => {
diff --git a/apps/media-server/src/lib/recording-preverify.ts b/apps/media-server/src/lib/recording-preverify.ts
new file mode 100644
index 000000000..25e447b2f
--- /dev/null
+++ b/apps/media-server/src/lib/recording-preverify.ts
@@ -0,0 +1,95 @@
+import { createHmac, timingSafeEqual } from "node:crypto";
+import { z } from "zod";
+import type { PreverifiedVideoRun } from "./recording-verification";
+
+const sourceSchema = z.object({
+ objectIdentity: z.string().min(1).max(200),
+ size: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
+});
+const payloadSchema = z.object({
+ version: z.literal(1),
+ init: sourceSchema,
+ segments: z.array(sourceSchema).min(1).max(4_096),
+ lines: z.array(z.string().max(16_384)).min(1).max(1_000_000),
+});
+const signedSchema = payloadSchema.extend({
+ signature: z.string().regex(/^[a-f0-9]{64}$/),
+});
+
+export type PreverifiedSource = z.infer<typeof sourceSchema>;
+export type PreverifiedRunPayload = z.infer<typeof payloadSchema>;
+
+export const MAX_PREVERIFIED_RUN_BYTES = 64 * 1024 * 1024;
+
+function sign(payload: PreverifiedRunPayload, secret: string) {
+ return createHmac("sha256", secret)
+ .update(
+ JSON.stringify([
+ payload.version,
+ payload.init,
+ payload.segments,
+ payload.lines,
+ ]),
+ )
+ .digest("hex");
+}
+
+export function signPreverifiedRun(
+ payload: PreverifiedRunPayload,
+ secret: string,
+): string {
+ return JSON.stringify({ ...payload, signature: sign(payload, secret) });
+}
+
+// Only this fleet holds the secret, so a run that verifies was decoded by a media server from the exact
+// fragments it names; anything else is ignored and those fragments are decoded again.
+export function readPreverifiedRun(
+ text: string,
+ secret: string,
+): PreverifiedRunPayload | null {
+ let value: unknown;
+ try {
+ value = JSON.parse(text);
+ } catch {
+ return null;
+ }
+ const parsed = signedSchema.safeParse(value);
+ if (!parsed.success) return null;
+ const { signature, ...payload } = parsed.data;
+ const expected = Buffer.from(sign(payload, secret), "hex");
+ const actual = Buffer.from(signature, "hex");
+ return expected.length === actual.length && timingSafeEqual(expected, actual)
+ ? payload
+ : null;
+}
+
+const sameSource = (left: PreverifiedSource, right: PreverifiedSource) =>
+ left.objectIdentity === right.objectIdentity && left.size === right.size;
+
+// Chains runs that cover the recording's fragments from the first one, in order and by content identity;
+// the chain stops at the first fragment no run starts at.
+export function resolvePreverifiedRuns(
+ runs: PreverifiedRunPayload[],
+ init: PreverifiedSource,
+ segments: PreverifiedSource[],
+): PreverifiedVideoRun[] {
+ const resolved: PreverifiedVideoRun[] = [];
+ const candidates = runs.filter((run) => sameSource(run.init, init));
+ let position = 0;
+ while (position < segments.length) {
+ const next = candidates
+ .filter(
+ (run) =>
+ position + run.segments.length <= segments.length &&
+ run.segments.every((source, offset) => {
+ const expected = segments[position + offset];
+ return expected !== undefined && sameSource(source, expected);
+ }),
+ )
+ .sort((left, right) => right.segments.length - left.segments.length)[0];
+ if (!next) break;
+ resolved.push({ segments: next.segments.length, lines: next.lines });
+ position += next.segments.length;
+ }
+ return resolved;
+}
diff --git a/apps/media-server/src/lib/recording-verification.ts b/apps/media-server/src/lib/recording-verification.ts
index 6bbe6442c..540d5a08b 100644
--- a/apps/media-server/src/lib/recording-verification.ts
+++ b/apps/media-server/src/lib/recording-verification.ts
@@ -919,6 +919,95 @@ async function keyframeAlignedGroups(
);
}
+const CHUNK_DECODE_BASE_ARGS = [
+ "-hide_banner",
+ "-nostdin",
+ "-copyts",
+ "-v",
+ "error",
+ "-xerror",
+ "-err_detect",
+ "explode+crccheck",
+ "-filter_threads",
+ "1",
+ "-filter_complex_threads",
+ "1",
+];
+
+function videoChunkDecodeArgs(path: string, threads: string) {
+ return [
+ ...CHUNK_DECODE_BASE_ARGS,
+ "-threads",
+ threads,
+ "-protocol_whitelist",
+ "file",
+ "-f",
+ "mov",
+ "-i",
+ path,
+ "-map",
+ "0:v:0",
+ "-fps_mode",
+ "passthrough",
+ "-enc_time_base:v",
+ "-1",
+ "-c:v",
+ "wrapped_avframe",
+ "-bsf:v",
+ "setts=pts=N:dts=N:duration=1",
+ "-max_interleave_delta",
+ "1",
+ "-f",
+ "framecrc",
+ "pipe:1",
+ ];
+}
+
+// Frame lines of whole fragments that were decoded during the recording, as one chunk of this decoder
+// would print them; the caller has matched each fragment's content identity to this recording's source.
+export interface PreverifiedVideoRun {
+ segments: number;
+ lines: string[];
+}
+
+// Decodes one run of fragments exactly as a chunk of the final verification would. Returns null when the
+// run cannot stand alone or the decoder complains, so the final verification decodes those fragments itself.
+export async function decodePreverifiedRun(
+ init: string,
+ segments: string[],
+ directory: string,
+ signal?: AbortSignal,
+): Promise<string[] | null> {
+ const first = segments[0];
+ if (!first || !(await decodesCleanlyFromStart(init, first, signal)))
+ return null;
+ const chunkPath = join(directory, "preverify-run.mp4");
+ await concatFiles([init, ...segments], chunkPath, signal);
+ const decoder = registerSubprocess(
+ await spawnRecordingDecoder(videoChunkDecodeArgs(chunkPath, "2"), 0),
+ );
+ const kill = () => {
+ if (decoder.exitCode === null) decoder.kill("SIGKILL");
+ };
+ signal?.addEventListener("abort", kill, { once: true });
+ const lines: string[] = [];
+ try {
+ const [exit, , stderr] = await Promise.all([
+ decoder.exited,
+ collectDecoderLines(decoder.stdout, lines, () => {}),
+ readDecoderErrors(decoder.stderr, chunkPath),
+ ]);
+ signal?.throwIfAborted();
+ if (exit !== 0 || stderr.error || decoder.signalCode) return null;
+ return lines.some((line) => !line.startsWith("#")) ? lines : null;
+ } finally {
+ signal?.removeEventListener("abort", kill);
+ kill();
+ unregisterSubprocess(decoder);
+ await rm(chunkPath, { force: true });
+ }
+}
+
// Decodes the packet-proven video as independent runs of whole fragments in
// parallel, then replays their frame lines through the same parser and evidence
// checks as one decode. Any decoder complaint falls back to the single decode,
@@ -929,13 +1018,33 @@ async function decodePacketProvenChunks(
chunkCount: number,
options: RecordingVerificationOptions,
packetProof: Awaited<ReturnType<typeof proveRecordingPackets>>,
+ preverified: PreverifiedVideoRun[] = [],
): Promise<RecordingVerificationResult> {
- const groups = await keyframeAlignedGroups(
- source,
- chunkCount,
- options.abortSignal,
- );
- if (groups.length < 2)
+ const cached = [...preverified];
+ let covered = cached.reduce((total, run) => total + run.segments, 0);
+ if (covered > source.segments.length)
+ throw new ChunkedDecodeUnavailable("preverified runs exceed the source");
+ // The fragments after the cached runs are decoded from their first one, so it has to decode on its own.
+ while (
+ cached.length > 0 &&
+ covered < source.segments.length &&
+ !(await decodesCleanlyFromStart(
+ source.init,
+ source.segments[covered],
+ options.abortSignal,
+ ))
+ )
+ covered -= cached.pop()?.segments ?? 0;
+ const remaining = source.segments.slice(covered);
+ const groups =
+ remaining.length === 0
+ ? []
+ : await keyframeAlignedGroups(
+ { init: source.init, segments: remaining },
+ chunkCount,
+ options.abortSignal,
+ );
+ if (cached.length === 0 && groups.length < 2)
throw new ChunkedDecodeUnavailable("no cleanly decodable fragment runs");
const directory = dirname(outputPath);
const chunkPaths = groups.map((_, index) =>
@@ -949,24 +1058,11 @@ async function decodePacketProvenChunks(
1,
Math.floor(
(configuredThreads > 0 ? configuredThreads : availableParallelism()) /
- groups.length,
+ Math.max(1, groups.length),
),
),
);
- const base = [
- "-hide_banner",
- "-nostdin",
- "-copyts",
- "-v",
- "error",
- "-xerror",
- "-err_detect",
- "explode+crccheck",
- "-filter_threads",
- "1",
- "-filter_complex_threads",
- "1",
- ];
+ const base = CHUNK_DECODE_BASE_ARGS;
const decoders: Awaited<ReturnType<typeof spawnRecordingDecoder>>[] = [];
const outputs: string[][] = [];
const readers: Promise<unknown>[] = [];
@@ -1028,32 +1124,7 @@ async function decodePacketProvenChunks(
),
);
const jobs = [
- ...chunkPaths.map((path) => [
- ...base,
- "-threads",
- threads,
- "-protocol_whitelist",
- "file",
- "-f",
- "mov",
- "-i",
- path,
- "-map",
- "0:v:0",
- "-fps_mode",
- "passthrough",
- "-enc_time_base:v",
- "-1",
- "-c:v",
- "wrapped_avframe",
- "-bsf:v",
- "setts=pts=N:dts=N:duration=1",
- "-max_interleave_delta",
- "1",
- "-f",
- "framecrc",
- "pipe:1",
- ]),
+ ...chunkPaths.map((path) => videoChunkDecodeArgs(path, threads)),
...(packetProof.hasAudio
? [
[
@@ -1132,8 +1203,11 @@ async function decodePacketProvenChunks(
);
}
const streams = new Map<number, DecodedStream>();
- const videoOutputs = outputs.slice(0, chunkPaths.length);
- const videoMetadata = metadataOf(videoOutputs[0]);
+ const videoOutputs = [
+ ...cached.map((run) => run.lines),
+ ...outputs.slice(0, chunkPaths.length),
+ ];
+ const videoMetadata = metadataOf(videoOutputs[0] ?? []);
let offset = 0;
for (const [index, lines] of videoOutputs.entries()) {
const metadata = metadataOf(lines);
@@ -1207,6 +1281,7 @@ export async function verifyRemuxedRecording(
"requireAudio" | "abortSignal" | "timeoutMs" | "onProgress"
>,
videoChunks?: RecordingVideoChunkSource,
+ preverified: PreverifiedVideoRun[] = [],
): Promise<RecordingVerificationResult & { sourcePreserved: true }> {
const signal = AbortSignal.any([
...(options.abortSignal ? [options.abortSignal] : []),
@@ -1247,8 +1322,8 @@ export async function verifyRemuxedRecording(
let result: RecordingVerificationResult | undefined;
if (
videoChunks &&
- chunkCount > 1 &&
- videoChunks.segments.length >= chunkCount * 2
+ (preverified.length > 0 ||
+ (chunkCount > 1 && videoChunks.segments.length >= chunkCount * 2))
) {
try {
result = await decodePacketProvenChunks(
@@ -1257,6 +1332,7 @@ export async function verifyRemuxedRecording(
chunkCount,
decodeOptions,
proof,
+ preverified,
);
} catch (error) {
signal.throwIfAborted();
diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts
index baf27cac4..87782dd75 100644
--- a/apps/media-server/src/routes/video.ts
+++ b/apps/media-server/src/routes/video.ts
@@ -84,10 +84,19 @@ import {
} from "../lib/media-video";
import { encodeAttemptsBudgetMs } from "../lib/nvenc-encoder";
import { findJobOnPeer, isPeerLookup } from "../lib/peer-jobs";
+import {
+ MAX_PREVERIFIED_RUN_BYTES,
+ type PreverifiedRunPayload,
+ readPreverifiedRun,
+ resolvePreverifiedRuns,
+ signPreverifiedRun,
+} from "../lib/recording-preverify";
import { RecordingTimingError } from "../lib/recording-timing";
import {
+ decodePreverifiedRun,
hashRecordingFile,
isRetryableRecordingVerificationError,
+ type PreverifiedVideoRun,
verifyDecodeChunks,
verifyRecording,
verifyRemoteRecording,
@@ -2399,6 +2408,7 @@ const muxSegmentsSchema = z
videoSegmentUrls: z.array(z.string().url()),
audioInitUrl: z.string().url().optional(),
audioSegmentUrls: z.array(z.string().url()).optional(),
+ preverifiedRunUrls: z.array(z.string().url()).max(4_096).optional(),
})
.refine((body) => body.outputPresignedUrl || body.outputUpload, {
message: "outputPresignedUrl or outputUpload is required",
@@ -2491,6 +2501,7 @@ type MuxContext = Pick<
| "sourceObjects"
| "requiredAudio"
| "audioLevels"
+ | "preverifiedRunUrls"
>;
function getMuxSegmentsOutputUpload(
@@ -2643,6 +2654,188 @@ video.post("/mux-segments", async (c) => {
});
});
+const preverifySourceSchema = z.object({
+ url: z.string().url(),
+ objectIdentity: strongObjectIdentitySchema,
+ size: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
+});
+const preverifySchema = z.object({
+ videoId: z.string().min(1).max(200),
+ init: preverifySourceSchema,
+ segments: z.array(preverifySourceSchema).min(1).max(256),
+ resultUploadUrl: z.string().url(),
+});
+const MAX_ACTIVE_PREVERIFY_RUNS = 2;
+const PREVERIFIED_RUN_LOAD_BUDGET_MS = 10_000;
+const PREVERIFY_RUN_TIMEOUT_MS = 5 * 60 * 1000;
+let activePreverifyRuns = 0;
+
+// Decodes fragments while the recording is still going, so the final verification only decodes the tail.
+// The work is optional: a busy or draining server declines it and the final verification does it instead.
+video.post("/preverify-segments", async (c) => {
+ if (!validateMediaServerSecret(c)) {
+ return c.json({ error: "Unauthorized" }, 401);
+ }
+ const body = preverifySchema.safeParse(await c.req.json());
+ if (!body.success) {
+ return c.json(
+ { error: "Invalid request", details: body.error.issues },
+ 400,
+ );
+ }
+ const secret = process.env.MEDIA_SERVER_WEBHOOK_SECRET;
+ if (
+ !secret ||
+ isDraining() ||
+ activePreverifyRuns >= MAX_ACTIVE_PREVERIFY_RUNS
+ ) {
+ return c.json({ error: "Server busy", code: "SERVER_BUSY" }, 503);
+ }
+ activePreverifyRuns++;
+ void trackBackgroundTask(
+ preverifySegments(body.data, secret)
+ .catch((error: unknown) => {
+ logVideoEvent("recording_preverify_failed", {
+ videoId: body.data.videoId,
+ segments: body.data.segments.length,
+ error: logReason(error, MUX_FAILURE_LOG_LENGTH),
+ });
+ })
+ .finally(() => {
+ activePreverifyRuns--;
+ }),
+ );
+ return c.json({ status: "queued" }, 202);
+});
+
+async function preverifySegments(
+ body: z.infer<typeof preverifySchema>,
+ secret: string,
+): Promise<void> {
+ const { ensureTempDir, getTempDir, retainTempPath } = await import(
+ "../lib/temp-files"
+ );
+ const { mkdir, rm } = await import("node:fs/promises");
+ const { join } = await import("node:path");
+ const startedAt = Date.now();
+ const signal = AbortSignal.timeout(PREVERIFY_RUN_TIMEOUT_MS);
+ await ensureTempDir();
+ const workDir = join(getTempDir(), `preverify-${randomUUID()}`);
+ const releaseWorkDir = retainTempPath(workDir);
+ try {
+ await mkdir(workDir, { recursive: true });
+ const initPath = join(workDir, "init.mp4");
+ const segmentPaths = body.segments.map((_, index) =>
+ join(workDir, `segment-${index}.m4s`),
+ );
+ const downloads = [
+ { source: body.init, path: initPath },
+ ...body.segments.map((source, index) => ({
+ source,
+ path: segmentPaths[index] as string,
+ })),
+ ];
+ let next = 0;
+ await Promise.all(
+ Array.from({ length: Math.min(8, downloads.length) }, async () => {
+ while (next < downloads.length) {
+ const download = downloads[next++];
+ if (!download) return;
+ await downloadUrlToFile(
+ download.source.url,
+ download.path,
+ signal,
+ download.source,
+ );
+ }
+ }),
+ );
+ const lines = await decodePreverifiedRun(
+ initPath,
+ segmentPaths,
+ workDir,
+ signal,
+ );
+ if (!lines) {
+ logVideoEvent("recording_preverify_skipped", {
+ videoId: body.videoId,
+ segments: body.segments.length,
+ durationMs: Date.now() - startedAt,
+ });
+ return;
+ }
+ const run = signPreverifiedRun(
+ {
+ version: 1,
+ init: {
+ objectIdentity: body.init.objectIdentity,
+ size: body.init.size,
+ },
+ segments: body.segments.map(({ objectIdentity, size }) => ({
+ objectIdentity,
+ size,
+ })),
+ lines,
+ },
+ secret,
+ );
+ await uploadToS3(
+ new TextEncoder().encode(run),
+ body.resultUploadUrl,
+ "application/json",
+ signal,
+ );
+ logVideoEvent("recording_preverify_complete", {
+ videoId: body.videoId,
+ segments: body.segments.length,
+ frames: lines.filter((line) => !line.startsWith("#")).length,
+ durationMs: Date.now() - startedAt,
+ });
+ } finally {
+ await rm(workDir, { recursive: true, force: true }).catch(() => {});
+ releaseWorkDir();
+ }
+}
+
+async function loadPreverifiedRuns(
+ urls: string[],
+ secret: string,
+ signal: AbortSignal,
+): Promise<PreverifiedRunPayload[]> {
+ const runs: PreverifiedRunPayload[] = [];
+ // Cached runs only save decoding, so loading them never holds a job back for long.
+ const budget = AbortSignal.any([
+ signal,
+ AbortSignal.timeout(PREVERIFIED_RUN_LOAD_BUDGET_MS),
+ ]);
+ let next = 0;
+ await Promise.all(
+ Array.from({ length: Math.min(16, urls.length) }, async () => {
+ while (next < urls.length && !budget.aborted) {
+ const url = urls[next++];
+ if (!url) return;
+ try {
+ const response = await fetch(url, { signal: budget });
+ const length = Number(response.headers.get("content-length"));
+ if (
+ !response.ok ||
+ !Number.isSafeInteger(length) ||
+ length > MAX_PREVERIFIED_RUN_BYTES
+ ) {
+ await response.body?.cancel().catch(() => {});
+ continue;
+ }
+ const run = readPreverifiedRun(await response.text(), secret);
+ if (run) runs.push(run);
+ } catch {
+ signal.throwIfAborted();
+ }
+ }
+ }),
+ );
+ return runs;
+}
+
async function streamConcatFiles(
inputPaths: string[],
outputPath: string,
@@ -3047,6 +3240,15 @@ async function muxSegmentsAsync(
await ensureTempDir();
updateJob(jobId, { phase: "downloading", progress: 0 });
sendCurrentJobWebhook(jobId);
+ const webhookSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET;
+ const preverifiedRuns =
+ sources && webhookSecret && context.preverifiedRunUrls?.length
+ ? loadPreverifiedRuns(
+ context.preverifiedRunUrls,
+ webhookSecret,
+ abortController.signal,
+ ).catch((): PreverifiedRunPayload[] => [])
+ : Promise.resolve([]);
await mkdir(workDir, { recursive: true });
const videoDir = join(workDir, "video");
@@ -3128,9 +3330,19 @@ async function muxSegmentsAsync(
abortController.signal,
),
);
+ const initSource = sources?.get(videoInitUrl);
+ const segmentSources = videoSegmentUrls.map((url) => sources?.get(url));
+ const preverified: PreverifiedVideoRun[] =
+ initSource && segmentSources.every((source) => source !== undefined)
+ ? resolvePreverifiedRuns(
+ await preverifiedRuns,
+ initSource,
+ segmentSources as MuxSourceObject[],
+ )
+ : [];
// Parallel verification decodes runs of these fragments; otherwise free them now.
const videoChunks =
- verifyDecodeChunks() > 1
+ verifyDecodeChunks() > 1 || preverified.length > 0
? { init: videoInitPath, segments: videoSegmentFiles }
: undefined;
if (!videoChunks) await rm(videoDir, { recursive: true, force: true });
@@ -3225,6 +3437,7 @@ async function muxSegmentsAsync(
},
},
videoChunks,
+ preverified,
),
),
);
@@ -3233,6 +3446,10 @@ async function muxSegmentsAsync(
jobId,
videoId,
durationMs: Date.now() - verificationStartedAt,
+ preverifiedSegments: preverified.reduce(
+ (total, run) => total + run.segments,
+ 0,
+ ),
method: localVerified.integrity ? "decoded-source" : "encoded-packets",
frames: localVerified.video.frameCount,
recordingDuration: localVerified.video.duration,