Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
import { randomUUID } from "node:crypto";
import { mkdirSync, rmSync } from "node:fs";
import { availableParallelism, hostname } from "node:os";
import { join } from "node:path";
import { Engine, processCpuSeconds, sampleThreads } from "./engine";
import { segmentHeader } from "./fmp4";
import { closesSegment, segmentKey } from "./hls";
import { ProjectCache } from "./materialize";
import type { Run } from "./mp4";
import {
type AudioResultMeta,
type AudioTask,
MIN_PART,
type SegmentReport,
type TaskTimings,
type TranscodeTask,
type VideoResult,
type VideoTask,
type WorkItem,
} from "./protocol";
import { mediaS3ConfigFromEnv, S3, s3ConfigFromEnv } from "./s3";
import { stashBytes } from "./stitch";
import {
canRemux,
encodedSeconds,
probeArgs,
remuxArgs,
transcodeArgs,
} from "./transcode";
const s3 = new S3(mediaS3ConfigFromEnv());
/** The farm's own bucket, for chunk stashes (see stitch.ts). */
const stashS3 = new S3(s3ConfigFromEnv());
const COORDINATOR = (
process.env.RF_COORDINATOR_URL ?? "http://127.0.0.1:8080"
).replace(/\/$/, "");
const TOKEN = process.env.RF_TOKEN ?? "";
const ENGINE_BIN = process.env.RF_ENGINE_BIN ?? "cap-render-farm";
const WORK_DIR = process.env.RF_WORK_DIR ?? "/tmp/rf-worker";
const CPUS = Number(process.env.RF_CPUS ?? availableParallelism());
// Measured best on one L4 with 8 vCPUs (g6.2xlarge): 5 render slots overlap
// decode latency; 4 Studio Sound lanes use the otherwise idle cores.
const SLOTS = Number(process.env.RF_SLOTS ?? 5);
const THREADS = Math.max(1, Math.floor(CPUS / SLOTS));
// CPU-only audio lanes next to the GPU slots: with frames kept on the GPU the
// host cores sit idle, and Studio Sound sections are pure CPU work. When set,
// the GPU slots only take video.
const AUDIO_SLOTS = Number(process.env.RF_AUDIO_SLOTS ?? 4);
const SLOT_KINDS = AUDIO_SLOTS > 0 ? ["video"] : undefined;
const PREFETCH_LEAD_MS = Number(process.env.RF_PREFETCH_LEAD_MS ?? 2500);
const AUDIO_NICE = Number(process.env.RF_AUDIO_NICE ?? 10);
// A video engine that reports no frames for this long is hung (driver or
// decoder stall): kill it so the chunk fails over instead of waiting forever.
const STALL_MS = Number(process.env.RF_STALL_MS ?? 30_000);
// Recycle an engine whose device has less free VRAM than this after a task,
// so a slow leak can never grow into CUDA_ERROR_OUT_OF_MEMORY mid-export.
const MIN_FREE_VRAM_MB = Number(process.env.RF_MIN_FREE_VRAM_MB ?? 3000);
let degradedFailures: number[] = [];
const WORKER_ID = `${process.env.RF_WORKER_NAME ?? hostname()}-${randomUUID().slice(0, 6)}`;
const SERVICE = process.env.RF_SERVICE ?? "local";
const headers = {
"content-type": "application/json",
...(TOKEN ? { authorization: `Bearer ${TOKEN}` } : {}),
};
/**
* Engine environment. Decoder readahead stays off by default: with it on,
* the first chunk of a clip can hold the recording's opening frame for
* seconds, because the readahead window overfills the decoder's frame cache
* before the first request and every later request falls behind it.
*/
function engineEnv(env: Record<string, string | undefined>) {
return {
LP_NUM_THREADS: String(THREADS),
RAYON_NUM_THREADS: String(THREADS),
CAP_EXPORT_DISABLE_ZERO_COPY: "1",
CAP_RENDER_LOOP_STATS: "1",
CAP_DECODER_READAHEAD: env.CAP_DECODER_READAHEAD ?? "0",
};
}
const engines = Array.from(
{ length: SLOTS + AUDIO_SLOTS },
(_, slot) =>
new Engine(
ENGINE_BIN,
engineEnv(process.env),
`slot${slot}`,
// Studio Sound lanes are CPU-heavy; on a 4 vCPU host they would
// otherwise starve the render slots' decode/encode threads.
slot >= SLOTS ? AUDIO_NICE : 0,
),
);
// GPU slots create their device and a spare layer set at boot, not on the
// first task.
function warm(engine: Engine, slot: number) {
if (slot < SLOTS) engine.request("warm", {}).catch(() => {});
return engine;
}
engines.forEach(warm);
const caches = new Map<string, ProjectCache>();
function cacheFor(jobId: string) {
let cache = caches.get(jobId);
if (!cache) {
const root = join(WORK_DIR, jobId);
mkdirSync(root, { recursive: true });
cache = new ProjectCache(s3, root);
caches.set(jobId, cache);
}
return cache;
}
function dropJobs(finished: string[]) {
for (const jobId of finished) {
const cache = caches.get(jobId);
if (!cache) continue;
if (
[...busy.values()].some(
(task) => task.kind !== "transcode" && task.jobId === jobId,
)
)
continue;
cache.close();
caches.delete(jobId);
rmSync(join(WORK_DIR, jobId), { recursive: true, force: true });
}
}
const busy = new Map<number, WorkItem>();
type TranscodeRun = {
controller: AbortController;
process?: Bun.Subprocess;
};
const transcoders = new Map<number, TranscodeRun>();
function stopTranscode(slot: number) {
const run = transcoders.get(slot);
if (!run) return;
run.controller.abort(new Error("transcode cancelled"));
run.process?.kill("SIGKILL");
}
const progress = new Map<
number,
{
taskId: string;
kind: "video" | "audio" | "transcode";
frames: number;
total: number;
startedAt: number;
lastProgressAt: number;
}
>();
async function post(path: string, body: unknown, signal?: AbortSignal) {
for (let attempt = 0; ; attempt++) {
try {
const response = await fetch(`${COORDINATOR}${path}`, {
method: "POST",
headers,
body: JSON.stringify(body),
signal,
});
if (!response.ok) throw new Error(`${path} -> ${response.status}`);
return response;
} catch (error) {
if (attempt >= 5 || signal?.aborted) throw error;
await Bun.sleep(250 * 2 ** attempt);
}
}
}
// Open /work long-polls, aborted when the worker starts draining so the
// coordinator stops handing it tasks it would never run.
const openPolls = new Set<AbortController>();
async function fetchAudio(task: VideoTask) {
if (!task.audio) return { sizes: [] as number[], data: new Uint8Array() };
return fetchAudioRange(task.jobId, task.audio.first, task.audio.end);
}
/** AAC packets [from, to) on the global grid; waits until they are rendered. */
async function fetchAudioRange(jobId: string, from: number, to: number) {
if (to <= from) return { sizes: [] as number[], data: new Uint8Array() };
const url = `${COORDINATOR}/jobs/${jobId}/audio?from=${from}&to=${to}`;
for (let attempt = 0; ; attempt++) {
try {
const response = await fetch(url, { headers });
if (!response.ok) throw new Error(`audio -> ${response.status}`);
const bytes = new Uint8Array(await response.arrayBuffer());
const view = new DataView(bytes.buffer, bytes.byteOffset);
const count = view.getUint32(0);
const sizes = Array.from({ length: count }, (_, index) =>
view.getUint32(4 + index * 4),
);
return { sizes, data: bytes.subarray(4 + count * 4) };
} catch (error) {
if (attempt >= 5) throw error;
await Bun.sleep(500);
}
}
}
type Segment = {
file?: { path: string; start: number; end: number };
bytes?: Uint8Array;
};
/**
* Publishes a chunk as HLS media segments while the engine is still encoding
* it: every finished GOP the engine announces is on disk, so once a segment's
* worth has accumulated it is wrapped in a moof, muxed with its audio and
* uploaded. Players can start on chunk 0 seconds after the request.
*/
class SegmentStream {
private sizes: number[] = [];
private offsets = [0];
private start = 0;
private index = 0;
private extradata = "";
private uploads: Promise<void>[] = [];
private failed: unknown = null;
firstUploadedMs: number | null = null;
constructor(
private task: VideoTask & { hls: NonNullable<VideoTask["hls"]> },
private path: string,
private started: number,
) {}
onEvent(event: Record<string, unknown>) {
if (typeof event.extradata === "string") this.extradata = event.extradata;
const gop = event.gop as { first: number; sizes: number[] } | undefined;
if (!gop || gop.first !== this.sizes.length) return;
this.append(gop.sizes);
if (
closesSegment(this.start, this.sizes.length, this.task.hls.segmentFrames)
) {
this.cut(this.sizes.length, false);
}
}
/** After the engine returns: publish the tail and wait for every upload. */
async finish(allSizes: number[], extradata: string) {
this.extradata ||= extradata;
this.append(allSizes.slice(this.sizes.length));
if (this.start < this.sizes.length) this.cut(this.sizes.length, true);
await Promise.all(this.uploads);
if (this.failed) throw this.failed;
return this.index;
}
private append(sizes: number[]) {
for (const size of sizes) {
this.sizes.push(size);
this.offsets.push((this.offsets[this.offsets.length - 1] ?? 0) + size);
}
}
private cut(end: number, last: boolean) {
const range: [number, number] = [this.start, end];
this.start = end;
const index = this.index++;
this.uploads.push(
this.publish(range, index, last).catch((error) => {
this.failed ??= error;
}),
);
}
/** First AAC packet at or after output frame `frame` (the layout() rule). */
private packetAt(frame: number) {
const audio = this.task.audio;
if (!audio) return 0;
const sample = frame * (48_000 / this.task.fps);
const packet = Math.ceil(sample / 1024) + 1;
return Math.min(audio.end, Math.max(audio.first, packet));
}
private async publish(
[a, b]: [number, number],
index: number,
last: boolean,
) {
const task = this.task;
const first = task.frames[0];
const video = new Uint8Array(
await Bun.file(this.path)
.slice(this.offsets[a] ?? 0, this.offsets[b] ?? 0)
.arrayBuffer(),
);
const packets: [number, number] = task.audio
? [
a === 0 ? task.audio.first : this.packetAt(first + a),
last ? task.audio.end : this.packetAt(first + b),
]
: [0, 0];
const audio = await fetchAudioRange(task.jobId, packets[0], packets[1]);
const header = segmentHeader({
sequence: first + a + 1,
firstFrame: first + a,
videoSizes: this.sizes.slice(a, b),
firstPacket: packets[0],
audioSizes: audio.sizes,
});
const body = new Uint8Array(
header.byteLength + video.byteLength + audio.data.byteLength,
);
body.set(header, 0);
body.set(video, header.byteLength);
body.set(audio.data, header.byteLength + video.byteLength);
const key = segmentKey(
task.hls.prefix,
task.chunk,
task.upload.firstPart,
index,
);
await s3.put(key, body, "video/iso.segment");
this.firstUploadedMs ??= performance.now() - this.started;
const report: SegmentReport = {
chunk: task.chunk,
index,
frames: [first + a, first + b],
key,
last,
extradata: this.extradata,
};
await post(`/tasks/${encodeURIComponent(task.taskId)}/segment`, report);
}
}
/**
* Video samples interleaved with the chunk's audio packets once per second,
* the way ffmpeg's muxer lays out a progressive MP4.
*/
function layout(
task: VideoTask,
videoPath: string,
videoSizes: number[],
audio: { sizes: number[]; data: Uint8Array },
) {
const segments: Segment[] = [];
const videoRuns: Run[] = [];
const audioRuns: Run[] = [];
const samplesPerFrame = 48_000 / task.fps;
let offset = 0;
let videoFile = 0;
let frame = 0;
let packet = 0;
let audioByte = 0;
const flushAudio = (untilSample: number) => {
const firstPacket = packet;
const startByte = audioByte;
while (packet < audio.sizes.length) {
const global = (task.audio?.first ?? 0) + packet;
if ((global - 1) * 1024 >= untilSample) break;
audioByte += audio.sizes[packet] ?? 0;
packet++;
}
if (packet > firstPacket) {
audioRuns.push({
first: (task.audio?.first ?? 0) + firstPacket,
count: packet - firstPacket,
offset,
});
segments.push({ bytes: audio.data.subarray(startByte, audioByte) });
offset += audioByte - startByte;
}
};
const total = videoSizes.length;
while (frame < total) {
const blockEnd = Math.min(total, frame + task.fps);
let blockBytes = 0;
for (let index = frame; index < blockEnd; index++)
blockBytes += videoSizes[index] ?? 0;
videoRuns.push({
first: task.frames[0] + frame,
count: blockEnd - frame,
offset,
});
segments.push({
file: { path: videoPath, start: videoFile, end: videoFile + blockBytes },
});
offset += blockBytes;
videoFile += blockBytes;
frame = blockEnd;
flushAudio((task.frames[0] + frame) * samplesPerFrame);
}
flushAudio(Number.POSITIVE_INFINITY);
return { segments, videoRuns, audioRuns, bytes: offset };
}
async function uploadLayout(
task: VideoTask,
segments: Segment[],
bytes: number,
) {
const parts: { partNumber: number; etag: string; size: number }[] = [];
const { upload } = task;
let pending: Uint8Array[] = [];
let pendingBytes = 0;
let nextPart = upload.firstPart;
const inflight = new Set<Promise<void>>();
let failure: unknown = null;
const track = async (promise: Promise<void>) => {
inflight.add(promise);
// Handle both outcomes here: a bare .finally() re-rejects unhandled and
// kills the whole worker (e.g. a hedge loser hitting NoSuchUpload after
// the job completed). An upload can settle and leave the set before
// anything awaits it, so its failure is kept to be thrown below.
promise.then(
() => inflight.delete(promise),
(error) => {
failure ??= error;
inflight.delete(promise);
},
);
if (inflight.size >= 4) await Promise.race(inflight);
};
const send = (body: Uint8Array) => {
if (nextPart >= upload.firstPart + upload.partLimit) {
throw new Error("chunk needs more parts than reserved");
}
const partNumber = nextPart++;
return track(
s3
.uploadPart(upload.key, upload.uploadId, partNumber, body)
.then((etag) => {
parts.push({ partNumber, etag, size: body.byteLength });
}),
);
};
const take = (count: number) => {
const out = new Uint8Array(count);
let filled = 0;
while (filled < count) {
const head = pending[0];
if (!head) break;
const use = Math.min(head.byteLength, count - filled);
out.set(head.subarray(0, use), filled);
filled += use;
if (use === head.byteLength) pending.shift();
else pending[0] = head.subarray(use);
}
pendingBytes -= count;
return out;
};
// The opening bytes go to the stash, which the coordinator joins to its
// neighbours; everything after them is a part of at least 5 MiB.
const stashSize = stashBytes(bytes);
let stashed = false;
const files = new Map<string, ReturnType<typeof Bun.file>>();
for (const segment of segments) {
let data: Uint8Array;
if (segment.bytes) data = segment.bytes;
else if (segment.file) {
let file = files.get(segment.file.path);
if (!file) {
file = Bun.file(segment.file.path);
files.set(segment.file.path, file);
}
data = new Uint8Array(
await file.slice(segment.file.start, segment.file.end).arrayBuffer(),
);
} else continue;
pending.push(data);
pendingBytes += data.byteLength;
if (!stashed && pendingBytes >= stashSize) {
stashed = true;
await track(stashS3.put(upload.stashKey, take(stashSize)));
}
// Always keep >= 5 MiB back so the chunk's last part is never too small.
while (stashed && pendingBytes >= upload.partTarget + MIN_PART)
await send(take(upload.partTarget));
}
if (!stashed) throw new Error("chunk ended before its stash filled");
if (pendingBytes > 0) await send(take(pendingBytes));
await Promise.all(inflight);
if (failure) throw failure;
pending = [];
if (parts.reduce((sum, part) => sum + part.size, 0) + stashSize !== bytes) {
throw new Error("uploaded byte count mismatch");
}
return {
parts: parts.sort((a, b) => a.partNumber - b.partNumber),
stash: { key: upload.stashKey, bytes: stashSize },
};
}
async function runVideo(
task: VideoTask,
engine: Engine,
queuedMs: number,
nearEnd: () => void = () => {},
): Promise<VideoResult> {
const started = performance.now();
const cpuBefore = await processCpuSeconds(engine.pid);
const cache = cacheFor(task.jobId);
const fetchStats = await cache.materialize(task.files);
const audioPromise = fetchAudio(task);
// Awaited only after the render; without a handler until then, a failure
// (job cancelled, coordinator unreachable) would kill the whole worker.
audioPromise.catch(() => {});
const out = join(
cache.root,
`v${task.chunk}-${randomUUID().slice(0, 6)}.h264`,
);
try {
const engineStarted = performance.now();
const slot = engines.indexOf(engine);
const entry = {
taskId: task.taskId,
kind: "video" as const,
frames: 0,
total: task.frames[1] - task.frames[0],
startedAt: Date.now(),
lastProgressAt: Date.now(),
};
progress.set(slot, entry);
const stream = task.hls
? new SegmentStream({ ...task, hls: task.hls }, out, engineStarted)
: null;
engine.onEvent = stream ? (event) => stream.onEvent(event) : null;
// Ask for the next task once this one is ~PREFETCH_LEAD_MS from done, so
// its sources download while this one finishes instead of after.
engine.onProgress = (frames) => {
entry.frames = frames;
entry.lastProgressAt = Date.now();
const elapsed = performance.now() - engineStarted;
const remainingMs =
((entry.total - frames) * elapsed) / Math.max(1, frames);
if (frames > 0 && remainingMs <= PREFETCH_LEAD_MS) nearEnd();
};
const request = engine.request<{
width: number;
height: number;
sizes: number[];
keyframes: number[];
extradata: string;
bytes: number;
timings: Record<string, number>;
}>("video", {
project: cache.root,
fps: task.fps,
resolution: task.resolution,
bpp: task.bpp,
frames: task.frames,
threads: task.threads,
out,
});
const threadsPromise: Promise<Record<string, number>> =
process.env.RF_PROFILE_THREADS === "1"
? sampleThreads(engine.pid, request)
: Promise.resolve({});
const result = await request.finally(() => {
engine.onProgress = null;
engine.onEvent = null;
progress.delete(slot);
});
nearEnd();
const segmentsDone = stream?.finish(result.sizes, result.extradata);
segmentsDone?.catch(() => {});
const engineMs = performance.now() - engineStarted;
const threads = await threadsPromise;
const waitStarted = performance.now();
const audio = await audioPromise;
const audioWaitMs = performance.now() - waitStarted;
const uploadStarted = performance.now();
const plan = layout(task, out, result.sizes, audio);
const [{ parts, stash }, segments] = await Promise.all([
uploadLayout(task, plan.segments, plan.bytes),
segmentsDone ?? Promise.resolve(0),
]);
const uploadMs = performance.now() - uploadStarted;
const timings: TaskTimings = {
queuedMs,
fetch: fetchStats,
engine: {
...result.timings,
...Object.fromEntries(
Object.entries(threads).map(([name, value]) => [
`thread:${name}`,
value,
]),
),
...(stream
? {
segments,
firstSegmentMs: Math.round(stream.firstUploadedMs ?? -1),
}
: {}),
},
engineMs,
audioWaitMs,
uploadMs,
totalMs: performance.now() - started,
cpuSeconds: (await processCpuSeconds(engine.pid)) - cpuBefore,
};
return {
taskId: task.taskId,
worker: WORKER_ID,
sizes: result.sizes,
keyframes: result.keyframes,
extradata: result.extradata,
width: result.width,
height: result.height,
videoRuns: plan.videoRuns,
audioRuns: plan.audioRuns,
firstPart: task.upload.firstPart,
stash,
parts,
bytes: plan.bytes,
timings,
};
} finally {
rmSync(out, { force: true });
}
}
async function runAudio(task: AudioTask, engine: Engine, queuedMs: number) {
const started = performance.now();
const cpuBefore = await processCpuSeconds(engine.pid);
const cache = cacheFor(task.jobId);
const fetchStats = await cache.materialize(task.files);
const out = join(
cache.root,
`a${task.section}-${randomUUID().slice(0, 6)}.aac`,
);
const engineStarted = performance.now();
// Reported in heartbeats so a restarted coordinator can re-attach it.
const slot = engines.indexOf(engine);
progress.set(slot, {
taskId: task.taskId,
kind: "audio",
frames: 0,
total: 0,
startedAt: Date.now(),
lastProgressAt: Date.now(),
});
// Studio Sound runs ~15x realtime; a section taking longer than a third
// of its own length (or a minute) is a hung engine.
const seconds = (task.range[1] - task.range[0] + task.preroll) / 48_000;
const watchdog = setTimeout(
() => {
console.error(`${task.taskId}: audio section timed out, killing engine`);
engine.kill();
},
Math.max(60_000, (seconds * 1000) / 3),
);
const result = await engine
.request<{
first_packet: number;
sizes: number[];
extradata: string;
timings: Record<string, number>;
}>("audio", {
project: cache.root,
fps: task.fps,
section: task.range,
preroll: task.preroll,
out,
})
.finally(() => {
clearTimeout(watchdog);
progress.delete(slot);
});
const engineMs = performance.now() - engineStarted;
const data = new Uint8Array(await Bun.file(out).arrayBuffer());
rmSync(out, { force: true });
const meta: AudioResultMeta = {
taskId: task.taskId,
worker: WORKER_ID,
firstPacket: result.first_packet,
sizes: result.sizes,
extradata: result.extradata,
timings: {
queuedMs,
fetch: fetchStats,
engine: result.timings,
engineMs,
audioWaitMs: 0,
uploadMs: 0,
totalMs: performance.now() - started,
cpuSeconds: (await processCpuSeconds(engine.pid)) - cpuBefore,
},
};
const json = new TextEncoder().encode(JSON.stringify(meta));
const body = new Uint8Array(4 + json.byteLength + data.byteLength);
new DataView(body.buffer).setUint32(0, json.byteLength);
body.set(json, 4);
body.set(data, 4 + json.byteLength);
for (let attempt = 0; ; attempt++) {
try {
const response = await fetch(
`${COORDINATOR}/tasks/${encodeURIComponent(task.taskId)}/audio`,
{
method: "POST",
headers: TOKEN ? { authorization: `Bearer ${TOKEN}` } : {},
body,
},
);
if (!response.ok) throw new Error(`audio result -> ${response.status}`);
break;
} catch (error) {
if (attempt >= 5) throw error;
await Bun.sleep(500 * 2 ** attempt);
}
}
}
const TRANSCODE_ENCODER = process.env.RF_TRANSCODE_ENCODER ?? "h264_nvenc";
async function runTranscode(task: TranscodeTask, slot: number) {
const dir = join(WORK_DIR, `transcode-${randomUUID()}`);
mkdirSync(dir, { recursive: true });
const entry = {
taskId: task.taskId,
kind: "transcode" as const,
frames: 0,
total: 0,
startedAt: Date.now(),
lastProgressAt: Date.now(),
};
progress.set(slot, entry);
const run: TranscodeRun = { controller: new AbortController() };
transcoders.set(slot, run);
try {
const output = join(dir, "output.mp4");
const input = await s3.presignFresh("GET", task.source, 6 * 3600);
run.controller.signal.throwIfAborted();
const probe = Bun.spawn(["ffprobe", ...probeArgs(input)], {
stdout: "pipe",
stderr: "ignore",
});
run.process = probe;
const probeDecoder = new TextDecoder();
let probed = "";
for await (const bytes of probe.stdout) {
probed += probeDecoder.decode(bytes, { stream: true });
entry.lastProgressAt = Date.now();
}
probed += probeDecoder.decode();
const remux = (await probe.exited) === 0 && canRemux(probed);
run.controller.signal.throwIfAborted();
entry.lastProgressAt = Date.now();
console.log(`${task.taskId}: ${remux ? "remuxing" : "transcoding"}`);
const ffmpeg = Bun.spawn(
[
"ffmpeg",
...(remux
? remuxArgs(input, output)
: transcodeArgs(
input,
output,
task.keyframeSeconds,
TRANSCODE_ENCODER,
)),
],
{ stdout: "pipe", stderr: "pipe" },
);
run.process = ffmpeg;
const stderr = new Response(ffmpeg.stderr).text();
const decoder = new TextDecoder();
let buffered = "";
for await (const bytes of ffmpeg.stdout) {
buffered += decoder.decode(bytes, { stream: true });
const lines = buffered.split("\n");
buffered = lines.pop() ?? "";
for (const line of lines) {
const seconds = encodedSeconds(line.trim());
if (seconds !== null && seconds > entry.frames) {
entry.frames = Math.floor(seconds);
entry.lastProgressAt = Date.now();
}
}
}
const code = await ffmpeg.exited;
run.controller.signal.throwIfAborted();
if (code !== 0) {
throw new Error(
`ffmpeg exited ${code}: ${(await stderr).trim().slice(-500)}`,
);
}
return await s3.uploadFile(task.output, output, "video/mp4", {
ifNoneMatch: true,
});
} finally {
if (run.process?.exitCode === null) {
run.process.kill("SIGKILL");
await run.process.exited;
}
transcoders.delete(slot);
progress.delete(slot);
rmSync(dir, { recursive: true, force: true });
}
}
async function poll(kinds: string[] | undefined, prefetch = false) {
const controller = new AbortController();
openPolls.add(controller);
const response = await post(
"/work",
{
worker: WORKER_ID,
slots: SLOTS,
cpus: CPUS,
service: SERVICE,
kinds,
audioSlots: AUDIO_SLOTS,
prefetch,
draining,
},
controller.signal,
).finally(() => openPolls.delete(controller));
const body = (await response.json()) as {
task: WorkItem | null;
finished: string[];
};
dropJobs(body.finished ?? []);
return body.task;
}
// A stray rejection must not take down every slot's task with the process.
process.on("unhandledRejection", (error) => {
console.error(`unhandled rejection: ${error}`);
});
let draining = false;
// Slots holding a prefetched (already reserved) next task, once it is known.
const reserved = new Map<number, WorkItem | null>();
const busySince = new Map<number, number>();
function exitWhenIdle() {
if (draining && busy.size === 0 && reserved.size === 0) {
console.log("drained; exiting");
process.exit(0);
}
}
for (const signal of ["SIGTERM", "SIGINT"] as const) {
process.on(signal, () => {
if (draining) return;
draining = true;
for (const controller of openPolls) controller.abort();
console.log(
`${signal}: finishing ${busy.size} running task(s), taking no new work`,
);
exitWhenIdle();
});
}
async function slotLoop(slot: number) {
let engine = engines[slot] as Engine;
const kinds = slot >= SLOTS ? ["audio"] : SLOT_KINDS;
let next: Promise<WorkItem | null> | null = null;
for (;;) {
if (draining && !next) {
exitWhenIdle();
await Bun.sleep(1000);
continue;
}
let task: WorkItem | null = null;
try {
task = next ? await next : await poll(kinds);
} catch (error) {
console.error(`poll failed: ${error}`);
await Bun.sleep(1000);
continue;
} finally {
next = null;
reserved.delete(slot);
}
if (!task) continue;
// Reserve the following task near the end of this one and start its
// source download (queued work only; never a hedge).
const prefetchNext = () => {
if (next || draining || PREFETCH_LEAD_MS <= 0) return;
reserved.set(slot, null);
next = poll(kinds, true)
.then((upcoming) => {
if (upcoming) {
reserved.set(slot, upcoming);
if (upcoming.kind !== "transcode") {
cacheFor(upcoming.jobId)
.materialize(upcoming.files)
.catch(() => {});
}
}
return upcoming;
})
.catch(() => null);
};
const received = performance.now();
busy.set(slot, task);
busySince.set(slot, Date.now());
try {
if (!engine.alive) {
engine = warm(
new Engine(ENGINE_BIN, engine.env, `slot${slot}`, engine.nice),
slot,
);
engines[slot] = engine;
}
if (task.kind === "transcode") {
const size = await runTranscode(task, slot);
await post(`/transcodes/${task.taskId.slice(3)}/done`, {
worker: WORKER_ID,
attempt: task.attempt,
size,
});
} else if (task.kind === "video") {
const result = await runVideo(task, engine, 0, prefetchNext);
await post(`/tasks/${encodeURIComponent(task.taskId)}/done`, result);
const vram = (
result.timings.engine as unknown as { gpu_mem?: number[][] }
).gpu_mem;
const freeMb = vram?.[vram.length - 1]?.[2];
if (freeMb !== undefined && freeMb > 0 && freeMb < MIN_FREE_VRAM_MB) {
console.warn(
`slot${slot}: ${freeMb} MiB VRAM free, recycling engine`,
);
engine.kill();
engine = warm(
new Engine(ENGINE_BIN, engine.env, `slot${slot}`),
slot,
);
engines[slot] = engine;
}
} else {
await runAudio(task, engine, 0);
}
console.log(
`${task.taskId} done in ${Math.round(performance.now() - received)}ms`,
);
} catch (error) {
console.error(`${task.taskId} failed: ${error}`);
if (String(error).includes("gpu path degraded")) {
// A fresh process usually recovers the CUDA/Vulkan state; if
// it keeps happening, restart the whole app (boot.ts relaunches).
engine.kill();
const now = Date.now();
degradedFailures = [
...degradedFailures.filter((at) => now - at < 300_000),
now,
];
if (degradedFailures.length >= 3) {
console.error(
"GPU path degraded 3 times in 5 min; restarting worker",
);
setTimeout(() => process.exit(75), 500);
}
}
const failure = {
worker: WORKER_ID,
attempt: task.attempt,
error: String(error instanceof Error ? error.message : error),
};
await post(
task.kind === "transcode"
? `/transcodes/${task.taskId.slice(3)}/fail`
: `/tasks/${encodeURIComponent(task.taskId)}/fail`,
failure,
).catch(() => {});
} finally {
busy.delete(slot);
busySince.delete(slot);
exitWhenIdle();
}
}
}
/** Container CPU seconds and memory (cgroup v2), for per-job cost accounting. */
async function usage() {
try {
const stat = await Bun.file("/sys/fs/cgroup/cpu.stat").text();
const usec = Number(stat.match(/usage_usec (\d+)/)?.[1] ?? 0);
const memory = Number(
(await Bun.file("/sys/fs/cgroup/memory.current").text()).trim(),
);
const throttledUsec = Number(stat.match(/throttled_usec (\d+)/)?.[1] ?? 0);
const throttledPeriods = Number(stat.match(/nr_throttled (\d+)/)?.[1] ?? 0);
return {
cpuSeconds: usec / 1e6,
memoryBytes: memory,
throttledSeconds: throttledUsec / 1e6,
throttledPeriods,
cpuMax: (
await Bun.file("/sys/fs/cgroup/cpu.max")
.text()
.catch(() => "")
).trim(),
load: (
await Bun.file("/proc/loadavg")
.text()
.catch(() => "")
).trim(),
model: (
await Bun.file("/proc/cpuinfo")
.text()
.catch(() => "")
).match(/model name\s*:\s*(.*)/)?.[1],
hostCpus: (
(
await Bun.file("/proc/cpuinfo")
.text()
.catch(() => "")
).match(/^processor/gm) ?? []
).length,
// Host-wide jiffies: user nice system idle iowait irq softirq steal.
hostStat: (
await Bun.file("/proc/stat")
.text()
.catch(() => "")
).split("\n")[0],
at: Date.now(),
};
} catch {
return null;
}
}
function cancel(taskIds: string[]) {
for (const [slot, task] of busy) {
if (!taskIds.includes(task.taskId)) continue;
console.log(`cancelling ${task.taskId}`);
if (task.kind === "transcode") {
stopTranscode(slot);
continue;
}
// SIGKILL: a stopped or wedged engine never acts on SIGTERM and would
// hold its slot until the watchdog gave up on it too.
engines[slot]?.kill("SIGKILL");
}
}
setInterval(() => {
const now = Date.now();
for (const [slot, entry] of progress) {
if (entry.kind === "transcode") {
// Decoding a long source can start slowly; a minute without any
// output means a wedged ffmpeg.
if (now - entry.lastProgressAt >= Math.max(STALL_MS, 60_000)) {
console.error(`${entry.taskId}: no progress, stopping transcode`);
entry.lastProgressAt = now;
stopTranscode(slot);
}
continue;
}
if (entry.kind !== "video" || now - entry.lastProgressAt < STALL_MS)
continue;
console.error(
`${entry.taskId}: no progress for ${now - entry.lastProgressAt} ms, killing slot${slot}`,
);
entry.lastProgressAt = now;
engines[slot]?.kill("SIGKILL");
}
}, 5_000);
setInterval(async () => {
// Every task this worker holds, in every phase (fetch, render, upload) and
// reserved ahead: the coordinator requeues anything a live worker stops
// listing, and a restarted coordinator re-attaches what is listed.
const running = [];
for (const [slot, task] of busy) {
const entry = progress.get(slot);
running.push({
taskId: task.taskId,
attempt: task.attempt,
phase: "running",
frames: entry?.taskId === task.taskId ? entry.frames : 0,
total: entry?.taskId === task.taskId ? entry.total : 0,
elapsedMs: Date.now() - (busySince.get(slot) ?? Date.now()),
});
}
for (const task of reserved.values()) {
if (!task) continue;
running.push({
taskId: task.taskId,
attempt: task.attempt,
phase: "reserved",
frames: 0,
total: 0,
elapsedMs: 0,
});
}
post("/heartbeat", {
worker: WORKER_ID,
slots: SLOTS,
cpus: CPUS,
usage: await usage(),
running,
})
.then(async (response) => {
const body = (await response.json()) as {
finished: string[];
cancel?: string[];
};
cancel(body.cancel ?? []);
dropJobs(body.finished ?? []);
})
.catch(() => {});
}, 3_000);
console.log(
`render-farm worker ${WORKER_ID}: ${SLOTS} slots x ${THREADS} threads, coordinator ${COORDINATOR}`,
);
for (let slot = 0; slot < SLOTS + AUDIO_SLOTS; slot++) slotLoop(slot);