Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(render-farm): clean up failed video outputs

Aditya Garuda42705b1571d
diff --git a/apps/render-farm/src/worker.test.ts b/apps/render-farm/src/worker.test.ts
index a9e9e99c0..39d144ece 100644
--- a/apps/render-farm/src/worker.test.ts
+++ b/apps/render-farm/src/worker.test.ts
@@ -2,7 +2,12 @@ import { describe, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import { readFileSync } from "node:fs";
import { join } from "node:path";
-import { MIN_PART, type TranscodeTask, type WorkItem } from "./protocol";
+import {
+ MIN_PART,
+ type TranscodeTask,
+ type VideoTask,
+ type WorkItem,
+} from "./protocol";
import * as stitch from "./stitch";
import * as transcode from "./transcode";
@@ -26,6 +31,8 @@ function harness(
randomUUID,
join,
...transcode,
+ processCpuSeconds: async () => 0,
+ sampleThreads: async () => ({}),
availableParallelism: () => 1,
hostname: () => "worker-test",
mkdirSync: () => {},
@@ -34,6 +41,13 @@ function harness(
s3ConfigFromEnv: () => ({}),
...stitch,
MIN_PART,
+ ProjectCache: class {
+ root = "/work/job";
+ async materialize() {
+ return { bytes: 0, ms: 0, requests: 0 };
+ }
+ close() {}
+ },
S3: class {
async presignFresh() {
await options.presignGate;
@@ -97,8 +111,18 @@ function harness(
const compiled = new Bun.Transpiler({ loader: "ts" }).transformSync(source);
const worker = new Function(
...Object.keys(deps),
- `${compiled}\nreturn {runTranscode, cancel, busy, progress, transcoders, engineEnv, uploadLayout};`,
+ `${compiled}\nreturn {runVideo, runTranscode, cancel, busy, progress, transcoders, engineEnv, uploadLayout};`,
)(...Object.values(deps)) as {
+ runVideo: (
+ task: VideoTask,
+ engine: {
+ pid: number;
+ onEvent: ((event: Record<string, unknown>) => void) | null;
+ onProgress: ((frames: number) => void) | null;
+ request: () => Promise<never>;
+ },
+ queuedMs: number,
+ ) => Promise<unknown>;
runTranscode: (task: TranscodeTask, slot: number) => Promise<number>;
cancel: (taskIds: string[]) => void;
busy: Map<number, WorkItem>;
@@ -227,3 +251,41 @@ describe("chunk upload", () => {
});
});
});
+
+test("a failed video render removes its temporary output", async () => {
+ const worker = harness();
+ const task: VideoTask = {
+ kind: "video",
+ taskId: "job:v0",
+ jobId: "job",
+ chunk: 0,
+ fps: 30,
+ resolution: [1920, 1080],
+ bpp: 0.1,
+ frames: [0, 30],
+ threads: 1,
+ files: [],
+ upload: {
+ key: "out/job.mp4",
+ uploadId: "upload",
+ firstPart: 2,
+ partLimit: 3,
+ partTarget: MIN_PART,
+ stashKey: "stash/job/0",
+ },
+ audio: null,
+ hls: null,
+ };
+ const engine = {
+ pid: 1,
+ onEvent: null,
+ onProgress: null,
+ request: () => Promise.reject(new Error("render failed")),
+ };
+ await expect(worker.runVideo(task, engine, 0)).rejects.toThrow(
+ "render failed",
+ );
+ expect(worker.removed).toHaveLength(1);
+ expect(worker.removed[0]).toStartWith("/work/job/v0-");
+ expect(worker.removed[0]).toEndWith(".h264");
+});
diff --git a/apps/render-farm/src/worker.ts b/apps/render-farm/src/worker.ts
index 4de9dfdbf..e6e129f21 100644
--- a/apps/render-farm/src/worker.ts
+++ b/apps/render-farm/src/worker.ts
@@ -505,110 +505,116 @@ async function runVideo(
cache.root,
`v${task.chunk}-${randomUUID().slice(0, 6)}.h264`,
);
- const engineStarted = performance.now();
- const slot = engines.indexOf(engine);
- const entry = {
- taskId: task.taskId,
- kind: "video" as const,
- frames: 0,
- total: task.frames[1] - task.frames[0],
- startedAt: Date.now(),
- lastProgressAt: Date.now(),
- };
- progress.set(slot, entry);
- const stream = task.hls
- ? new SegmentStream({ ...task, hls: task.hls }, out, engineStarted)
- : null;
- engine.onEvent = stream ? (event) => stream.onEvent(event) : null;
- // Ask for the next task once this one is ~PREFETCH_LEAD_MS from done, so
- // its sources download while this one finishes instead of after.
- engine.onProgress = (frames) => {
- entry.frames = frames;
- entry.lastProgressAt = Date.now();
- const elapsed = performance.now() - engineStarted;
- const remainingMs =
- ((entry.total - frames) * elapsed) / Math.max(1, frames);
- if (frames > 0 && remainingMs <= PREFETCH_LEAD_MS) nearEnd();
- };
- const request = engine.request<{
- width: number;
- height: number;
- sizes: number[];
- keyframes: number[];
- extradata: string;
- bytes: number;
- timings: Record<string, number>;
- }>("video", {
- project: cache.root,
- fps: task.fps,
- resolution: task.resolution,
- bpp: task.bpp,
- frames: task.frames,
- threads: task.threads,
- out,
- });
- const threadsPromise: Promise<Record<string, number>> =
- process.env.RF_PROFILE_THREADS === "1"
- ? sampleThreads(engine.pid, request)
- : Promise.resolve({});
- const result = await request.finally(() => {
- engine.onProgress = null;
- engine.onEvent = null;
- progress.delete(slot);
- });
- nearEnd();
- const segmentsDone = stream?.finish(result.sizes, result.extradata);
- segmentsDone?.catch(() => {});
- const engineMs = performance.now() - engineStarted;
- const threads = await threadsPromise;
- const waitStarted = performance.now();
- const audio = await audioPromise;
- const audioWaitMs = performance.now() - waitStarted;
- const uploadStarted = performance.now();
- const plan = layout(task, out, result.sizes, audio);
- const [{ parts, stash }, segments] = await Promise.all([
- uploadLayout(task, plan.segments, plan.bytes),
- segmentsDone ?? Promise.resolve(0),
- ]);
- const uploadMs = performance.now() - uploadStarted;
- rmSync(out, { force: true });
- const timings: TaskTimings = {
- queuedMs,
- fetch: fetchStats,
- engine: {
- ...result.timings,
- ...Object.fromEntries(
- Object.entries(threads).map(([name, value]) => [
- `thread:${name}`,
- value,
- ]),
- ),
- ...(stream
- ? { segments, firstSegmentMs: Math.round(stream.firstUploadedMs ?? -1) }
- : {}),
- },
- engineMs,
- audioWaitMs,
- uploadMs,
- totalMs: performance.now() - started,
- cpuSeconds: (await processCpuSeconds(engine.pid)) - cpuBefore,
- };
- return {
- taskId: task.taskId,
- worker: WORKER_ID,
- sizes: result.sizes,
- keyframes: result.keyframes,
- extradata: result.extradata,
- width: result.width,
- height: result.height,
- videoRuns: plan.videoRuns,
- audioRuns: plan.audioRuns,
- firstPart: task.upload.firstPart,
- stash,
- parts,
- bytes: plan.bytes,
- timings,
- };
+ try {
+ const engineStarted = performance.now();
+ const slot = engines.indexOf(engine);
+ const entry = {
+ taskId: task.taskId,
+ kind: "video" as const,
+ frames: 0,
+ total: task.frames[1] - task.frames[0],
+ startedAt: Date.now(),
+ lastProgressAt: Date.now(),
+ };
+ progress.set(slot, entry);
+ const stream = task.hls
+ ? new SegmentStream({ ...task, hls: task.hls }, out, engineStarted)
+ : null;
+ engine.onEvent = stream ? (event) => stream.onEvent(event) : null;
+ // Ask for the next task once this one is ~PREFETCH_LEAD_MS from done, so
+ // its sources download while this one finishes instead of after.
+ engine.onProgress = (frames) => {
+ entry.frames = frames;
+ entry.lastProgressAt = Date.now();
+ const elapsed = performance.now() - engineStarted;
+ const remainingMs =
+ ((entry.total - frames) * elapsed) / Math.max(1, frames);
+ if (frames > 0 && remainingMs <= PREFETCH_LEAD_MS) nearEnd();
+ };
+ const request = engine.request<{
+ width: number;
+ height: number;
+ sizes: number[];
+ keyframes: number[];
+ extradata: string;
+ bytes: number;
+ timings: Record<string, number>;
+ }>("video", {
+ project: cache.root,
+ fps: task.fps,
+ resolution: task.resolution,
+ bpp: task.bpp,
+ frames: task.frames,
+ threads: task.threads,
+ out,
+ });
+ const threadsPromise: Promise<Record<string, number>> =
+ process.env.RF_PROFILE_THREADS === "1"
+ ? sampleThreads(engine.pid, request)
+ : Promise.resolve({});
+ const result = await request.finally(() => {
+ engine.onProgress = null;
+ engine.onEvent = null;
+ progress.delete(slot);
+ });
+ nearEnd();
+ const segmentsDone = stream?.finish(result.sizes, result.extradata);
+ segmentsDone?.catch(() => {});
+ const engineMs = performance.now() - engineStarted;
+ const threads = await threadsPromise;
+ const waitStarted = performance.now();
+ const audio = await audioPromise;
+ const audioWaitMs = performance.now() - waitStarted;
+ const uploadStarted = performance.now();
+ const plan = layout(task, out, result.sizes, audio);
+ const [{ parts, stash }, segments] = await Promise.all([
+ uploadLayout(task, plan.segments, plan.bytes),
+ segmentsDone ?? Promise.resolve(0),
+ ]);
+ const uploadMs = performance.now() - uploadStarted;
+ const timings: TaskTimings = {
+ queuedMs,
+ fetch: fetchStats,
+ engine: {
+ ...result.timings,
+ ...Object.fromEntries(
+ Object.entries(threads).map(([name, value]) => [
+ `thread:${name}`,
+ value,
+ ]),
+ ),
+ ...(stream
+ ? {
+ segments,
+ firstSegmentMs: Math.round(stream.firstUploadedMs ?? -1),
+ }
+ : {}),
+ },
+ engineMs,
+ audioWaitMs,
+ uploadMs,
+ totalMs: performance.now() - started,
+ cpuSeconds: (await processCpuSeconds(engine.pid)) - cpuBefore,
+ };
+ return {
+ taskId: task.taskId,
+ worker: WORKER_ID,
+ sizes: result.sizes,
+ keyframes: result.keyframes,
+ extradata: result.extradata,
+ width: result.width,
+ height: result.height,
+ videoRuns: plan.videoRuns,
+ audioRuns: plan.audioRuns,
+ firstPart: task.upload.firstPart,
+ stash,
+ parts,
+ bytes: plan.bytes,
+ timings,
+ };
+ } finally {
+ rmSync(out, { force: true });
+ }
}
async function runAudio(task: AudioTask, engine: Engine, queuedMs: number) {