Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
import {
createHash,
createHmac,
randomUUID,
timingSafeEqual,
} from "node:crypto";
import { mkdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import { Engine } from "./engine";
import { initSegment, playlist } from "./fmp4";
import { checkSegmentReport, segmentCuts, segmentKey } from "./hls";
import { type FileSpec, ProjectCache } from "./materialize";
import {
avcC,
buildHeader,
byteRangeFor,
indexVideoTrack,
locateMoov,
type Run,
type TrackIndex,
} from "./mp4";
import {
chunkPartLayout,
MIN_RANGE_PARTS,
planChunkBoundaries,
} from "./planning";
import { ProbeEngine } from "./probe-engine";
import {
type AudioResultMeta,
type AudioTask,
COMPRESSION_BPP,
type JobRequest,
MIN_PART,
type SegmentReport,
type Task,
type TranscodeTask,
type VideoResult,
type VideoTask,
type WorkItem,
} from "./protocol";
import { acceptOnce, completeUpload, reservePartRange } from "./recovery";
import { mediaS3ConfigFromEnv, S3, s3ConfigFromEnv } from "./s3";
import { pickQueued as pickQueuedTask } from "./scheduler";
import {
planStitch,
StitchLimiter,
type StitchPart,
uploadProblem,
} from "./stitch";
import {
AUDIO_FILE,
checkManifestBounds,
isKey,
sourceLimitsFromEnv,
validateJobRequest,
} from "./validate";
// Recordings and exports live in the media bucket (possibly the product's
// own); the journal stays in the render farm's bucket.
const s3 = new S3(mediaS3ConfigFromEnv());
const journalS3 = new S3(s3ConfigFromEnv());
const ENGINE_BIN = process.env.RF_ENGINE_BIN ?? "cap-render-farm";
const WORK_DIR = process.env.RF_WORK_DIR ?? "/tmp/rf-coordinator";
const PORT = Number(process.env.PORT ?? 8080);
const PUBLIC_URL = process.env.RF_COORDINATOR_URL ?? `http://127.0.0.1:${PORT}`;
const TOKEN = process.env.RF_TOKEN ?? "";
if (!TOKEN && process.env.RF_ALLOW_NO_TOKEN !== "1") {
throw new Error(
"RF_TOKEN is required (set RF_ALLOW_NO_TOKEN=1 for local use)",
);
}
const MAX_ACTIVE_JOBS = Number(process.env.RF_MAX_ACTIVE_JOBS ?? 32);
// A job whose tasks stop completing (every attempt failing slowly, a lost
// dependency) would otherwise hold its upload and waiters forever.
const JOB_STALL_MS = Number(process.env.RF_JOB_STALL_MS ?? 10 * 60_000);
// Finished jobs stay queryable this long, as a summary without media data.
const JOB_RETENTION_MS = Number(process.env.RF_JOB_RETENTION_MS ?? 60 * 60_000);
/** Presigned segment and playlist URLs last 6 h; older jobs are abandoned. */
const JOB_LIFETIME_MS = 5 * 3600_000;
const SAMPLE_RATE = 48_000;
const PACKET = 1024;
const probeEngine = new ProbeEngine(
() => new Engine(ENGINE_BIN, {}, "coordinator-engine"),
);
// Audio sections (Studio Sound) are CPU work. These lanes render them on the
// coordinator's cores, pulling from the same queue as workers' audio lanes.
const LOCAL_AUDIO_SLOTS = Number(process.env.RF_LOCAL_AUDIO_SLOTS ?? 2);
// Progressive playback: chunks stream fMP4 segments while rendering and the
// coordinator keeps an HLS EVENT playlist of the contiguous prefix, so an
// export is watchable seconds after the request whatever its length. The
// flat MP4 is still assembled for download.
const HLS = process.env.RF_HLS !== "0";
const HLS_SEGMENT_SECONDS = Number(process.env.RF_HLS_SEGMENT_SECONDS ?? 2);
/** Unfinished chunks per job (from the front) that outrank other work. */
const HEAD_CHUNKS = Number(process.env.RF_HEAD_CHUNKS ?? 2);
const LEAD_IN_SECONDS = Number(process.env.RF_LEAD_IN_SECONDS ?? 4);
const stitchLimiter = new StitchLimiter({
total: 16,
ahead: 8,
aheadPerJob: 4,
});
/**
* Shortest audio section. Each one after the first also renders a 10 s
* preroll, but a short export's audio is what its video chunks wait on, so
* spreading it over every lane finishes sooner than fewer, longer sections.
*/
const MIN_AUDIO_SECTION_SECONDS = Number(
process.env.RF_MIN_AUDIO_SECTION_SECONDS ?? 10,
);
const SLOT_MEGAPIXELS_PER_SEC = Number(
process.env.RF_SLOT_MEGAPIXELS_PER_SEC ?? 450,
);
const CHUNK_WORK_SECONDS = Number(process.env.RF_CHUNK_WORK_SECONDS ?? 6);
const audioEngines = Array.from(
{ length: LOCAL_AUDIO_SLOTS },
(_, index) => new Engine(ENGINE_BIN, {}, `audio${index}`),
);
const idleAudioEngines = [...audioEngines];
const localAudioQueue: (() => void)[] = [];
async function withAudioEngine<T>(
fn: (engine: Engine) => Promise<T>,
timeoutMs: number,
) {
while (idleAudioEngines.length === 0) {
await new Promise<void>((resolve) => localAudioQueue.push(resolve));
}
let engine = idleAudioEngines.pop() as Engine;
if (!engine.alive) engine = new Engine(ENGINE_BIN, {}, engine.label);
const watchdog = setTimeout(() => {
console.error(`${engine.label}: audio section timed out, killing engine`);
engine.kill("SIGKILL");
}, timeoutMs);
try {
return await fn(engine);
} finally {
clearTimeout(watchdog);
idleAudioEngines.push(engine);
localAudioQueue.shift()?.();
}
}
/**
* Local audio lanes pull from the shared queue like any fleet audio lane, so
* sections spread over the coordinator's cores and the workers' idle CPUs.
*/
let localAudioRunning = 0;
function pumpLocalAudio() {
while (localAudioRunning < LOCAL_AUDIO_SLOTS) {
const index = pickQueued((kind) => kind === "audio");
if (index < 0) return;
const state = queue.splice(index, 1)[0] as TaskState;
const job = jobs.get(state.task.jobId);
if (!job || job.status !== "rendering") continue;
state.attempts++;
job.t.firstTaskStarted ??= now();
localAudioRunning++;
runAudioLocally(job, state).finally(() => {
localAudioRunning--;
pumpLocalAudio();
});
}
}
async function runAudioLocally(job: Job, state: TaskState) {
const task = state.task;
if (task.kind !== "audio") return;
state.state = "running";
state.startedAt = now();
state.worker = "coordinator";
try {
const started = performance.now();
if (!job.audioCache) {
job.audioCache = new ProjectCache(s3, join(WORK_DIR, job.id, "audio"));
}
const cache = job.audioCache;
mkdirSync(cache.root, { recursive: true });
const fetch = await cache.materialize(task.files);
const out = join(cache.root, `a${task.section}.aac`);
const engineStarted = performance.now();
const seconds =
(task.range[1] - task.range[0] + task.preroll) / SAMPLE_RATE;
const result = await withAudioEngine(
(engine) =>
engine.request<{
first_packet: number;
sizes: number[];
extradata: string;
timings: Record<string, number>;
}>("audio", {
project: cache.root,
fps: task.fps,
section: task.range,
preroll: task.preroll,
out,
}),
Math.max(60_000, (seconds * 1000) / 3),
);
const data = new Uint8Array(await Bun.file(out).arrayBuffer());
rmSync(out, { force: true });
const meta: AudioResultMeta = {
taskId: task.taskId,
worker: "coordinator",
firstPacket: result.first_packet,
sizes: result.sizes,
extradata: result.extradata,
timings: {
queuedMs: 0,
fetch,
engine: result.timings,
engineMs: performance.now() - engineStarted,
audioWaitMs: 0,
uploadMs: 0,
totalMs: performance.now() - started,
cpuSeconds: 0,
},
};
if (job.status === "rendering") await onAudioDone(job, state, meta, data);
} catch (error) {
if (state.attempts >= 3) failJob(job, error);
else
setTimeout(() => {
if (job.status !== "rendering" || job.audioSections.has(task.section))
return;
requeue(state, "local audio lane failed");
pumpLocalAudio();
dispatch();
}, 1000 * state.attempts);
}
}
type Usage = { cpuSeconds: number; memoryBytes: number; at: number };
type Worker = {
id: string;
slots: number;
cpus: number;
lastSeen: number;
service: string;
audioSlots?: number;
usage?: Usage | null;
};
type Section = {
index: number;
range: [number, number];
preroll: number;
exact: boolean;
packets: [number, number];
};
type Chunk = {
index: number;
frames: [number, number];
packets: [number, number];
files: FileSpec[];
firstPart: number;
partLimit: number;
/** Copies dispatched so far; picks each dispatch's part range. */
dispatches: number;
};
type TaskState = {
task: Task;
state: "queued" | "running" | "done";
worker?: string;
startedAt?: number;
attempts: number;
dispatching?: boolean;
reattach?: boolean;
firstPart?: number;
duplicateOf?: string;
duplicated?: boolean;
/** Reserved by a busy slot ahead of time; not started until it reports progress. */
prefetched?: boolean;
/**
* Resumed after a coordinator restart: a worker may still be running it.
* Held back until then so its heartbeat can re-attach it instead of the
* chunk being rendered twice.
*/
heldUntil?: number;
/** Last heartbeat in which the owning worker listed this task. */
lastReportedAt?: number;
progress?: {
frames: number;
total: number;
elapsedMs: number;
at: number;
advancedAt: number;
};
};
type HlsState = {
prefix: string;
url: string;
initUrl?: string;
extradata?: string;
segments: Map<number, Map<number, SegmentReport>>;
listed: { url: string; duration: number }[];
cursor: { chunk: number; index: number };
writing: boolean;
dirty: boolean;
ended: boolean;
audioExtradata?: string;
retry?: ReturnType<typeof setTimeout>;
};
type Job = {
verified?: boolean;
acceptances: Map<string, Promise<void>>;
stitchParts?: Map<string, Promise<{ partNumber: number; etag: string }>>;
headerStashes?: Map<string, Promise<Uint8Array>>;
/** Summary frozen when the job ends; the job's media data is released then. */
final?: ReturnType<typeof summary>;
hls?: HlsState;
id: string;
request: JobRequest;
status: "planning" | "rendering" | "assembling" | "ready" | "error";
error?: string;
key: string;
uploadId?: string;
t: Record<string, number>;
fps: number;
bpp: number;
resolution: [number, number];
totalFrames: number;
totalSamples: number;
width: number;
height: number;
chunks: Chunk[];
sections: Section[];
tasks: Map<string, TaskState>;
videoResults: Map<number, VideoResult>;
audioSections: Map<number, { meta: AudioResultMeta; data: Uint8Array }>;
audioWaiters: (() => void)[];
cpuSeconds: number;
fetchedBytes: number;
workersUsed: Set<string>;
url?: string;
outputBytes?: number;
probe?: Record<string, unknown>;
plan?: Record<string, unknown>;
waiters: (() => void)[];
taskStats: Record<string, unknown>[];
audioCache?: ProjectCache;
};
const workers = new Map<string, Worker>();
const jobs = new Map<string, Job>();
const queue: TaskState[] = [];
type Poller = {
worker: string;
kinds?: string[];
/** A busy slot reserving its next task: queued work only, never a hedge. */
prefetch?: boolean;
resolve: (task: WorkItem | null) => void;
};
const pollers: Poller[] = [];
const finishedJobs: string[] = [];
function now() {
return performance.timeOrigin + performance.now();
}
function liveSlots() {
const cutoff = Date.now() - 30_000;
let slots = 0;
let audioSlots = 0;
let count = 0;
for (const worker of workers.values()) {
if (worker.lastSeen >= cutoff) {
slots += worker.slots;
audioSlots += worker.audioSlots ?? 0;
count++;
}
}
return { slots, audioSlots, workers: count };
}
async function newJob(
id: string,
request: JobRequest,
requestedAt = now(),
): Promise<Job> {
const compression = request.compression ?? "Maximum";
const job: Job = {
id,
acceptances: new Map(),
request,
status: "planning",
key: request.output?.key ?? `out/${id}.mp4`,
t: { requested: requestedAt },
fps: request.fps ?? 30,
bpp: COMPRESSION_BPP[compression],
resolution: request.resolution ?? [1920, 1080],
totalFrames: 0,
totalSamples: 0,
width: 0,
height: 0,
chunks: [],
sections: [],
tasks: new Map(),
videoResults: new Map(),
audioSections: new Map(),
audioWaiters: [],
cpuSeconds: 0,
fetchedBytes: 0,
workersUsed: new Set(),
waiters: [],
taskStats: [],
};
if (HLS) {
job.hls = await newHlsState(`${request.output?.hlsPrefix ?? "hls"}/${id}`);
}
return job;
}
// ---------------------------------------------------------------- planning ---
type Manifest = {
files: {
path: string;
size: number;
key?: string;
/** Transcode this source to `key` first (see TranscodeTask). */
transcodeFrom?: string;
}[];
};
type RecordingMeta = {
segments?: {
display: { path: string; start_time?: number };
camera?: { path: string; start_time?: number };
mic?: { path: string };
system_audio?: { path: string };
}[];
display?: { path: string };
camera?: { path: string };
audio?: { path: string };
};
type Mp4Meta = {
head: [number, number];
moov: [number, number];
index: ReturnType<typeof indexVideoTrack>;
};
// Every web Save renders from a new project folder, so the per-recording
// source index never hits; the parsed moov of an unchanged source file can.
const mp4Metas = new Map<string, Mp4Meta>();
const MP4_META_CACHE_ENTRIES = 16;
// A manifest can list thousands of videos; their index reads share this many
// slots across every job so one can't flood storage or the coordinator.
const INDEX_READ_CONCURRENCY = Math.max(
1,
Math.floor(Number(process.env.RF_INDEX_READ_CONCURRENCY)) || 16,
);
let indexReadsActive = 0;
const indexReadQueue: (() => void)[] = [];
async function withIndexRead<T>(run: () => Promise<T>): Promise<T> {
if (indexReadsActive < INDEX_READ_CONCURRENCY) indexReadsActive++;
else await new Promise<void>((resolve) => indexReadQueue.push(resolve));
try {
return await run();
} finally {
const next = indexReadQueue.shift();
if (next) next();
else indexReadsActive--;
}
}
async function mp4MetaRanges(key: string, size: number): Promise<Mp4Meta> {
const headEnd = Math.min(size, 128 * 1024);
const { bytes: head, etag } = await s3.getRangeTagged(key, 0, headEnd - 1);
const cacheKey =
etag && process.env.RF_INDEX_CACHE !== "0"
? `${key}\n${size}\n${etag}`
: null;
const cached = cacheKey ? mp4Metas.get(cacheKey) : undefined;
if (cacheKey && cached) {
mp4Metas.delete(cacheKey);
mp4Metas.set(cacheKey, cached);
return cached;
}
const meta = await readMp4Meta(key, size, head);
if (cacheKey) {
mp4Metas.set(cacheKey, meta);
if (mp4Metas.size > MP4_META_CACHE_ENTRIES)
mp4Metas.delete(mp4Metas.keys().next().value as string);
}
return meta;
}
async function readMp4Meta(
key: string,
size: number,
head: Uint8Array,
): Promise<Mp4Meta> {
const location = locateMoov(head, size);
let moovStart: number;
let moovBytes: Uint8Array;
if (location && "start" in location && location.start !== undefined) {
if (location.size > SOURCE_LIMITS.moovBytes) {
throw new Error(`${key} has a ${location.size} byte moov`);
}
moovStart = location.start;
moovBytes =
location.start + location.size <= head.byteLength
? head.subarray(location.start, location.start + location.size)
: await s3.getRange(
key,
location.start,
location.start + location.size - 1,
);
} else if (location && "next" in location && location.next !== undefined) {
// Moov after mdat (Cap's recorder): fetch the tail in one request.
if (size - location.next > SOURCE_LIMITS.moovBytes) {
throw new Error(`${key} has ${size - location.next} bytes after mdat`);
}
const tail = await s3.getRange(key, location.next, size - 1);
const found = locateMoov(tail, tail.byteLength);
if (!found || !("start" in found) || found.start === undefined) {
throw new Error(`no moov in ${key}`);
}
moovStart = location.next + found.start;
moovBytes = tail.subarray(found.start, found.start + found.size);
} else {
throw new Error(`no moov in ${key}`);
}
return {
head: [0, Math.min(size, 128 * 1024)],
moov: [moovStart, moovStart + moovBytes.byteLength],
index: indexVideoTrack(moovBytes),
};
}
function mergeRanges(ranges: [number, number][]) {
const sorted = ranges
.filter(([start, end]) => end > start)
.sort((a, b) => a[0] - b[0]);
const merged: [number, number][] = [];
for (const range of sorted) {
const last = merged[merged.length - 1];
if (last && range[0] <= last[1] + 256 * 1024)
last[1] = Math.max(last[1], range[1]);
else merged.push([range[0], range[1]]);
}
return merged;
}
type SourceIndex = {
manifest: Manifest;
recordingMeta: RecordingMeta;
mediaMeta: Map<
string,
{
head: [number, number];
moov: [number, number];
index: TrackIndex;
size: number;
key: string;
}
>;
};
// Stand-in for an index built once at upload time: recordings are immutable
// once uploaded, so their moov indexes never need re-reading per export.
const sourceIndexes = new Map<string, SourceIndex>();
// Shared prefixes a manifest may point sources at besides its own recording
// prefix (e.g. assets reused across recordings), comma-separated.
const SOURCE_KEY_PREFIXES = (process.env.RF_SOURCE_KEY_PREFIXES ?? "")
.split(",")
.filter(Boolean);
const SOURCE_LIMITS = sourceLimitsFromEnv(process.env);
async function getBounded(key: string, limit: number) {
const bytes = await s3.getRange(key, 0, limit);
if (bytes.byteLength > limit) {
throw new Error(`${key} is larger than ${limit} bytes`);
}
return bytes;
}
/** Manifests name local paths and bucket keys; neither may escape its scope. */
function checkManifest(
manifest: Manifest,
prefix: string,
sourceRoot?: string,
) {
const bounds = checkManifestBounds(manifest, SOURCE_LIMITS);
if (bounds) throw new Error(bounds);
for (const file of manifest.files) {
const parts = file.path.split("/");
if (
file.path.startsWith("/") ||
parts.includes("..") ||
parts.includes("")
) {
throw new Error(`manifest path ${file.path} is not a relative path`);
}
const inScope = (key: string) =>
key.startsWith(`${prefix}/`) ||
(sourceRoot !== undefined && key.startsWith(sourceRoot)) ||
SOURCE_KEY_PREFIXES.some((allowed) => key.startsWith(allowed));
for (const key of [file.key, file.transcodeFrom]) {
if (key !== undefined && (!isKey(key) || !inScope(key))) {
throw new Error(
`manifest key for ${file.path} is outside the recording`,
);
}
}
if (file.transcodeFrom !== undefined && file.key === undefined) {
throw new Error(`manifest transcode for ${file.path} names no key`);
}
}
}
async function sourceIndex(
prefix: string,
sourceRoot?: string,
onManifest?: (manifest: Manifest) => void,
): Promise<SourceIndex> {
const cached =
process.env.RF_INDEX_CACHE !== "0" ? sourceIndexes.get(prefix) : undefined;
if (cached) {
checkManifest(cached.manifest, prefix, sourceRoot);
onManifest?.(cached.manifest);
return cached;
}
const manifest = JSON.parse(
new TextDecoder().decode(
await getBounded(`${prefix}/manifest.json`, SOURCE_LIMITS.metadataBytes),
),
) as Manifest;
checkManifest(manifest, prefix, sourceRoot);
onManifest?.(manifest);
const keyOf = (file: { path: string; key?: string }) =>
file.key ?? `${prefix}/${file.path}`;
const sourceFiles = await Promise.all(
manifest.files.map(async (file) => ({
...file,
size:
file.transcodeFrom === undefined
? file.size
: ((await storedTranscodeSize(keyOf(file))) ??
(await transcodeSourceSize(file.transcodeFrom))),
})),
);
const sourceBounds = checkManifestBounds(
{ files: sourceFiles },
SOURCE_LIMITS,
);
if (sourceBounds) throw new Error(sourceBounds);
await Promise.all(
manifest.files.map(async (file) => {
if (file.transcodeFrom === undefined) return;
file.size = await awaitTranscode(
await ensureTranscode(file.transcodeFrom, keyOf(file)),
);
}),
);
const bounds = checkManifestBounds(manifest, SOURCE_LIMITS);
if (bounds) throw new Error(bounds);
const metaFile = manifest.files.find(
(file) => file.path === "recording-meta.json",
);
if (!metaFile) throw new Error("recording has no recording-meta.json");
const mediaMeta: SourceIndex["mediaMeta"] = new Map();
const [recordingMeta] = await Promise.all([
getBounded(keyOf(metaFile), SOURCE_LIMITS.metadataBytes).then(
(bytes) => JSON.parse(new TextDecoder().decode(bytes)) as RecordingMeta,
),
...manifest.files
.filter((file) => file.path.endsWith(".mp4"))
.map(async (file) => {
const meta = await withIndexRead(() =>
mp4MetaRanges(keyOf(file), file.size),
);
mediaMeta.set(file.path, {
...meta,
size: file.size,
key: keyOf(file),
});
}),
]);
const index = { manifest, recordingMeta, mediaMeta };
sourceIndexes.set(prefix, index);
if (sourceIndexes.size > 32)
sourceIndexes.delete(sourceIndexes.keys().next().value as string);
return index;
}
function manifestFiles(manifest: Manifest) {
const mediaFiles = manifest.files.filter((file) =>
file.path.endsWith(".mp4"),
);
const audioFiles = manifest.files.filter((file) =>
AUDIO_FILE.test(file.path),
);
const smallFiles = manifest.files.filter(
(file) => !mediaFiles.includes(file) && !audioFiles.includes(file),
);
return { mediaFiles, audioFiles, smallFiles };
}
/** Every file but the videos: whole small files, and audio whole or just its ends. */
function baseFileSpecs(
manifest: Manifest,
prefix: string,
audioMode: "probe" | "all",
): FileSpec[] {
const { audioFiles, smallFiles } = manifestFiles(manifest);
const keyOf = (file: { path: string; key?: string }) =>
file.key ?? `${prefix}/${file.path}`;
return [
...smallFiles.map((file) => ({
path: file.path,
key: keyOf(file),
size: file.size,
ranges: "all" as const,
})),
...audioFiles.map((file) => ({
path: file.path,
key: keyOf(file),
size: file.size,
ranges:
audioMode === "all"
? ("all" as const)
: mergeRanges([
[0, Math.min(file.size, 256 * 1024)],
[Math.max(0, file.size - 256 * 1024), file.size],
]),
})),
];
}
async function planJob(job: Job) {
const request = job.request;
const prefix = request.recording.replace(/\/$/, "");
const probeDir = join(WORK_DIR, job.id);
mkdirSync(probeDir, { recursive: true });
const cache = new ProjectCache(s3, probeDir);
// The probe's small files and audio ends need only the manifest, so they
// download while the videos are indexed.
let baseFetch = null as Promise<unknown> | null;
let index: SourceIndex;
try {
index = await sourceIndex(prefix, request.sourceRoot, (manifest) => {
baseFetch = cache.materialize(baseFileSpecs(manifest, prefix, "probe"));
baseFetch.catch(() => {});
});
} catch (error) {
await baseFetch?.catch(() => {});
cache.close();
throw error;
}
const { manifest, recordingMeta, mediaMeta } = index;
const { mediaFiles, audioFiles } = manifestFiles(manifest);
job.t.indexed = now();
const baseSpecs = (audioMode: "probe" | "all") =>
baseFileSpecs(manifest, prefix, audioMode);
const mediaSpec = (path: string, extra: [number, number][]): FileSpec => {
const meta = mediaMeta.get(path);
if (!meta) throw new Error(`no index for ${path}`);
return {
path,
key: meta.key,
size: meta.size,
ranges: mergeRanges([meta.head, meta.moov, ...extra]),
};
};
// Both downloads write through the cache, so neither may outlive it.
const fetched = await Promise.allSettled([
baseFetch ?? cache.materialize(baseSpecs("probe")),
cache.materialize(mediaFiles.map((file) => mediaSpec(file.path, []))),
]);
const failed = fetched.find((result) => result.status === "rejected");
if (failed) {
cache.close();
throw failed.reason;
}
job.t.materialized = now();
const gop = job.fps * 2;
type Probe = {
total_frames: number;
total_samples: number;
width: number;
height: number;
span_step: number;
spans: { clip: number; start: number; end: number }[][];
audio_cuts: number[];
clips: { index: number; camera: number; mic: number }[];
};
const probe = await probeEngine
.request<Probe>({
project: probeDir,
fps: job.fps,
resolution: job.resolution,
span_step: gop,
})
.finally(() => cache.close());
job.t.probed = now();
if (request.frameLimit && request.frameLimit < probe.total_frames) {
probe.total_frames = request.frameLimit;
probe.total_samples = Math.round(
(request.frameLimit * SAMPLE_RATE) / job.fps,
);
}
if (probe.total_frames > job.fps * SOURCE_LIMITS.exportSeconds) {
throw new Error(
`export is ${Math.round(probe.total_frames / job.fps)} s (limit ${SOURCE_LIMITS.exportSeconds} s)`,
);
}
job.totalFrames = probe.total_frames;
job.totalSamples = probe.total_samples;
job.width = probe.width;
job.height = probe.height;
job.probe = {
totalFrames: probe.total_frames,
width: probe.width,
height: probe.height,
audioCuts: probe.audio_cuts.length,
};
// Which media files belong to which recording clip.
const clipMedia: { display?: string; camera?: string; offset: number }[] = [];
const segments: NonNullable<RecordingMeta["segments"]> =
recordingMeta.segments ?? [
{
display: recordingMeta.display ?? { path: "" },
camera: recordingMeta.camera,
},
];
segments.forEach((segment, index) => {
const cameraOffset =
probe.clips.find((clip) => clip.index === index)?.camera ?? 0;
const starts = [
segment.display.start_time ?? 0,
segment.camera?.start_time ?? 0,
];
clipMedia[index] = {
display: segment.display.path,
camera: segment.camera?.path,
offset:
Math.abs(cameraOffset) + Math.max(...starts) - Math.min(...starts),
};
});
// ---- video chunks
const { slots } = liveSlots();
const minChunkFrames = Math.max(
1,
Math.round((request.minChunkSeconds ?? 1) * job.fps),
);
// Chunks are sized by render work, not video length, so 4K gets shorter
// chunks than 1080p. ~6 s of work per chunk measured the same single-job
// speed as 12 s while freeing slots twice as often for exports that arrive
// during a long one.
const megapixels = (probe.width * probe.height) / 1e6;
const slotFps = SLOT_MEGAPIXELS_PER_SEC / megapixels;
const targetFrames = Math.max(
minChunkFrames,
Math.round(slotFps * (request.chunkWorkSeconds ?? CHUNK_WORK_SECONDS)),
);
const boundaries = planChunkBoundaries({
totalFrames: probe.total_frames,
fps: job.fps,
slots,
targetFrames,
minChunkFrames,
chunks: request.chunks,
chunksPerSlot: request.chunksPerSlot,
maxChunks: request.maxChunks,
leadInFrames: job.hls ? LEAD_IN_SECONDS * job.fps : 0,
});
const chunkCount = boundaries.length - 1;
const { partsPerChunk, partLimit } = chunkPartLayout(chunkCount);
if (partLimit < MIN_RANGE_PARTS) {
throw new Error("too many chunks for one multipart upload");
}
const samplesPerFrame = SAMPLE_RATE / job.fps;
const totalPackets = Math.ceil(probe.total_samples / PACKET) + 1;
const packetAt = (frame: number, isStart: boolean, isEnd: boolean) => {
if (isStart) return 0;
if (isEnd) return totalPackets;
return Math.ceil((frame * samplesPerFrame) / PACKET) + 1;
};
const margin = 2;
job.chunks = [];
for (let index = 0; index < chunkCount; index++) {
const f0 = boundaries[index] ?? 0;
const f1 = boundaries[index + 1] ?? probe.total_frames;
const perFile = new Map<string, [number, number][]>();
for (let step = Math.floor(f0 / gop); step < Math.ceil(f1 / gop); step++) {
for (const span of probe.spans[step] ?? []) {
const media = clipMedia[span.clip];
if (!media) continue;
const pad = margin + media.offset;
for (const path of [media.display, media.camera]) {
if (!path) continue;
const meta = mediaMeta.get(path);
if (!meta) continue;
const range = byteRangeFor(
meta.index,
span.start - pad,
span.end + pad,
);
if (!range) continue;
const list = perFile.get(path) ?? [];
list.push([range.start, range.end]);
perFile.set(path, list);
}
}
}
job.chunks.push({
index,
frames: [f0, f1],
packets: [
packetAt(f0, index === 0, false),
packetAt(f1, false, index === chunkCount - 1),
],
files: [
...baseSpecs("probe"),
...mediaFiles.map((file) =>
mediaSpec(file.path, perFile.get(file.path) ?? []),
),
],
firstPart: 2 + index * partsPerChunk + 1,
partLimit,
dispatches: 0,
});
}
// ---- audio sections, split on clip boundaries where Studio Sound resets
const hasAudio = audioFiles.length > 0 && !request.frameLimit;
job.sections = [];
if (hasAudio) {
const total = probe.total_samples;
// One wave of sections across whoever renders audio (the coordinator's
// local slots, or the fleet).
const fleetAudio = liveSlots().audioSlots;
const audioLanes =
LOCAL_AUDIO_SLOTS + fleetAudio > 0
? LOCAL_AUDIO_SLOTS + fleetAudio
: Math.max(8, slots);
const target = Math.max(
MIN_AUDIO_SECTION_SECONDS * SAMPLE_RATE,
Math.ceil(total / Math.min(64, audioLanes)),
);
const cuts = probe.audio_cuts;
let start = 0;
let preroll = 0;
let exact = true;
// With HLS the first segment waits on section 0's audio: keep it short
// so playback can start well before the long sections finish.
const firstTarget = job.hls ? Math.min(target, 6 * SAMPLE_RATE) : target;
while (start < total) {
const size = job.sections.length === 0 ? firstTarget : target;
const ideal = start + size;
if (ideal >= total - size / 2) {
job.sections.push({
index: job.sections.length,
range: [start, total],
preroll,
exact,
packets: [start === 0 ? 0 : (start + PACKET) / PACKET, totalPackets],
});
break;
}
const cut = cuts.find(
(value) => value >= ideal - size / 2 && value <= ideal + size / 2,
);
let end: number;
let nextPreroll: number;
let nextExact: boolean;
if (cut !== undefined) {
end = Math.ceil((cut + PACKET) / PACKET) * PACKET;
const inputStart = Math.max(
0,
Math.floor((cut - 2 * SAMPLE_RATE) / PACKET) * PACKET,
);
nextPreroll = end - inputStart;
nextExact = true;
} else {
end = Math.floor(ideal / PACKET) * PACKET;
nextPreroll = Math.min(
end,
Math.floor((10 * SAMPLE_RATE) / PACKET) * PACKET,
);
nextExact = false;
}
job.sections.push({
index: job.sections.length,
range: [start, end],
preroll,
exact,
packets: [
start === 0 ? 0 : (start + PACKET) / PACKET,
(end + PACKET) / PACKET,
],
});
start = end;
preroll = nextPreroll;
exact = nextExact;
}
}
job.uploadId = await s3.createMultipart(job.key, "video/mp4");
job.t.planned = now();
const bitrate =
job.width * job.height * (30 + Math.max(0, job.fps - 30) * 0.6) * job.bpp;
// Sized for the largest chunk, with 2.5x headroom for bitrate peaks.
const largestChunkFrames = Math.max(
...job.chunks.map((chunk) => chunk.frames[1] - chunk.frames[0]),
);
const largestChunkBytes = (bitrate / 8) * (largestChunkFrames / job.fps);
const partTarget = Math.max(
16 << 20,
Math.ceil((largestChunkBytes * 2.5) / Math.max(1, partLimit - 2)),
);
job.plan = {
chunks: chunkCount,
sections: job.sections.length,
exactSeams: job.sections.filter((section) => section.exact).length,
slots,
partsPerChunk,
partTarget,
sourceBytesPerChunk: Math.round(
job.chunks.reduce(
(sum, chunk) =>
sum +
chunk.files.reduce(
(fileSum, file) =>
fileSum +
(file.ranges === "all"
? file.size
: file.ranges.reduce((r, [a, b]) => r + b - a, 0)),
0,
),
0,
) / chunkCount,
),
};
for (const section of job.sections) {
const task: AudioTask = {
kind: "audio",
taskId: `${job.id}:a${section.index}`,
jobId: job.id,
section: section.index,
fps: job.fps,
range: section.range,
preroll: section.preroll,
files: [
...baseSpecs("all"),
...mediaFiles.map((file) => mediaSpec(file.path, [])),
],
};
enqueue(job, task);
}
for (const chunk of job.chunks) {
const task: VideoTask = {
kind: "video",
taskId: `${job.id}:v${chunk.index}`,
jobId: job.id,
chunk: chunk.index,
fps: job.fps,
resolution: job.resolution,
bpp: job.bpp,
frames: chunk.frames,
threads: 8,
files: chunk.files,
upload: {
key: job.key,
uploadId: job.uploadId,
firstPart: chunk.firstPart,
partLimit: chunk.partLimit,
partTarget,
stashKey: stashKey(job.id, chunk.index, chunk.firstPart),
},
audio: hasAudio
? {
first: chunk.packets[0],
end: chunk.packets[1],
coordinator: PUBLIC_URL,
}
: null,
hls: job.hls
? {
prefix: job.hls.prefix,
segmentFrames: Math.round(job.fps * HLS_SEGMENT_SECONDS),
}
: null,
};
enqueue(job, task);
}
await journalJob(job);
job.status = "rendering";
dispatch();
}
// ----------------------------------------------------------------- journal ---
// With RF_JOURNAL=1 a coordinator restart (crash, deploy) resumes in-flight
// exports instead of losing them: the plan is written before dispatch, each
// accepted chunk/audio result as it lands, and a marker when the job ends.
// On boot, unfinished jobs are rebuilt and only their missing work re-queued;
// the multipart upload and every uploaded part are reused.
const JOURNAL = process.env.RF_JOURNAL !== "0";
// Workers heartbeat every 3 s; this is enough for all of them to report in.
const RESUME_HOLD_MS = 10_000;
const journalKey = (id: string, name: string) => `jobs/${id}/${name}`;
/**
* Where a dispatch stores its chunk's opening bytes (see stitch.ts). Outside
* `jobs/` so a sweep can list every stash without reading journals.
*/
const stashKey = (id: string, chunk: number, firstPart: number) =>
`stash/${id}/c${chunk}-p${firstPart}`;
async function journalPut(key: string, body: Uint8Array | string) {
if (!JOURNAL) return;
await journalS3.put(key, body);
}
type JournaledJob = Pick<
Job,
| "id"
| "request"
| "key"
| "uploadId"
| "t"
| "fps"
| "bpp"
| "resolution"
| "totalFrames"
| "totalSamples"
| "width"
| "height"
| "chunks"
| "sections"
| "plan"
| "probe"
> & { version: 3; tasks: Task[]; hls: { prefix: string } | null };
function journalJob(job: Job) {
const record: JournaledJob = {
version: 3,
id: job.id,
request: job.request,
key: job.key,
uploadId: job.uploadId,
t: job.t,
fps: job.fps,
bpp: job.bpp,
resolution: job.resolution,
totalFrames: job.totalFrames,
totalSamples: job.totalSamples,
width: job.width,
height: job.height,
chunks: job.chunks,
sections: job.sections,
plan: job.plan,
probe: job.probe,
tasks: [...job.tasks.values()]
.filter((state) => !state.duplicateOf)
.map((state) => state.task),
hls: job.hls ? { prefix: job.hls.prefix } : null,
};
return journalPut(journalKey(job.id, "job.json"), JSON.stringify(record));
}
function audioFrame(meta: AudioResultMeta, data: Uint8Array) {
const json = new TextEncoder().encode(JSON.stringify(meta));
const body = new Uint8Array(4 + json.byteLength + data.byteLength);
new DataView(body.buffer).setUint32(0, json.byteLength);
body.set(json, 4);
body.set(data, 4 + json.byteLength);
return body;
}
async function newHlsState(prefix: string): Promise<HlsState> {
return {
prefix,
url: await s3.presignFresh("GET", `${prefix}/index.m3u8`, 6 * 3600),
segments: new Map(),
listed: [],
cursor: { chunk: 0, index: 0 },
writing: false,
dirty: false,
ended: false,
};
}
/**
* The segments a worker published for a finished chunk, re-derived from its
* keyframes with the worker's own cut rule (a segment closes at the first
* GOP boundary at or past `segmentFrames`), so the playlist can be rebuilt.
*/
function segmentsFromResult(
job: Job,
chunk: Chunk,
result: VideoResult,
): SegmentReport[] {
if (!job.hls) return [];
const firstPart = result.firstPart;
const cuts = segmentCuts(
result.keyframes,
result.sizes.length,
Math.round(job.fps * HLS_SEGMENT_SECONDS),
);
return cuts.map(([a, b], index) => ({
chunk: chunk.index,
index,
frames: [chunk.frames[0] + a, chunk.frames[0] + b],
key: segmentKey(job.hls?.prefix ?? "", chunk.index, firstPart, index),
last: index === cuts.length - 1,
extradata: result.extradata,
}));
}
async function resumeJobs() {
// The stash sweep stays off until every journaled job is back in memory:
// until then it would take a resumable job for a finished one.
for (let attempt = 0; ; attempt++) {
try {
await resumeJournal();
break;
} catch (error) {
console.error(`resume failed: ${error}`);
await new Promise((resolve) =>
setTimeout(resolve, Math.min(60_000, 1000 * 2 ** attempt)),
);
}
}
journalResumed = true;
await sweepStashes().catch((error) =>
console.error(`stash sweep failed: ${error}`),
);
}
async function resumeJournal() {
if (!JOURNAL) return;
const keys = await journalS3.list("jobs/");
const byJob = new Map<string, Set<string>>();
for (const { key } of keys) {
const [, id, ...rest] = key.split("/");
if (!id) continue;
const names = byJob.get(id) ?? new Set<string>();
names.add(rest.join("/"));
byJob.set(id, names);
}
for (const [id, names] of byJob) {
if (names.has("done") || jobs.has(id)) continue;
try {
if (!names.has("job.json")) {
if (!names.has("request.json")) continue;
const receipt = JSON.parse(
new TextDecoder().decode(
await journalS3.get(journalKey(id, "request.json")),
),
) as { request: JobRequest; requestedAt: number };
if (Date.now() - receipt.requestedAt > JOB_LIFETIME_MS) {
await journalPut(journalKey(id, "done"), "expired");
continue;
}
const job = await newJob(id, receipt.request, receipt.requestedAt);
jobs.set(id, job);
planJob(job).catch((error) => failJob(job, error));
continue;
}
const record = JSON.parse(
new TextDecoder().decode(
await journalS3.get(journalKey(id, "job.json")),
),
) as JournaledJob;
if (Date.now() - (record.t.requested ?? 0) > JOB_LIFETIME_MS) {
await journalPut(journalKey(id, "done"), "expired");
continue;
}
// Version 3 numbers parts around the coordinator's stash parts;
// earlier journals' uploads cannot be completed with it.
if (record.version !== 3) {
if (record.uploadId)
await s3.abortMultipart(record.key, record.uploadId);
await journalPut(
journalKey(id, "done"),
"incompatible journal version",
);
console.error(
`job ${id}: legacy journal has no durable upload reservations`,
);
continue;
}
const reservations: {
task: Task;
worker: string;
duplicateOf?: string;
}[] = [];
const job: Job = {
...record,
acceptances: new Map(),
status: "rendering",
tasks: new Map(),
videoResults: new Map(),
audioSections: new Map(),
audioWaiters: [],
cpuSeconds: 0,
fetchedBytes: 0,
workersUsed: new Set(),
waiters: [],
taskStats: [],
hls: record.hls ? await newHlsState(record.hls.prefix) : undefined,
};
job.t.resumed = now();
job.t.lastProgress = now();
for (const name of names) {
const dispatch = name.match(/^dispatches\/(\d+)\/(\d+)\.json$/);
if (dispatch) {
const chunk = job.chunks[Number(dispatch[1])];
if (!chunk) throw new Error("invalid reserved chunk");
chunk.dispatches = Math.max(
chunk.dispatches,
Number(dispatch[2]) + 1,
);
reservations.push(
JSON.parse(
new TextDecoder().decode(
await journalS3.get(journalKey(id, name)),
),
),
);
continue;
}
const video = name.match(/^v\/(\d+)\.json$/);
if (video) {
const result = JSON.parse(
new TextDecoder().decode(await journalS3.get(journalKey(id, name))),
) as VideoResult;
job.videoResults.set(Number(video[1]), result);
continue;
}
const audio = name.match(/^a\/(\d+)\.bin$/);
if (audio) {
const bytes = await journalS3.get(journalKey(id, name));
const length = new DataView(bytes.buffer, bytes.byteOffset).getUint32(
0,
);
const meta = JSON.parse(
new TextDecoder().decode(bytes.subarray(4, 4 + length)),
) as AudioResultMeta;
job.audioSections.set(Number(audio[1]), {
meta,
data: bytes.slice(4 + length),
});
}
}
jobs.set(id, job);
let requeued = 0;
for (const task of record.tasks) {
const done =
task.kind === "video"
? job.videoResults.has(task.chunk)
: job.audioSections.has(task.section);
const state: TaskState = {
task,
state: done ? "done" : "queued",
attempts: 0,
heldUntil: done ? undefined : Date.now() + RESUME_HOLD_MS,
};
job.tasks.set(task.taskId, state);
if (!done) {
queue.push(state);
requeued++;
}
}
for (const reservation of reservations) {
const { task, worker, duplicateOf } = reservation;
let state = job.tasks.get(task.taskId);
if (!state) {
const done =
task.kind === "video" && job.videoResults.has(task.chunk);
state = {
task,
state: done ? "done" : "queued",
attempts: 0,
duplicateOf,
heldUntil: done ? undefined : Date.now() + RESUME_HOLD_MS,
};
job.tasks.set(task.taskId, state);
if (!done) queue.push(state);
}
if ((task.attempt ?? 0) >= state.attempts) {
state.attempts = task.attempt ?? 0;
state.worker = worker;
state.reattach = true;
if (task.kind === "video") state.firstPart = task.upload.firstPart;
}
if (duplicateOf) {
const original = job.tasks.get(duplicateOf);
if (original) original.duplicated = true;
}
}
for (const chunk of job.chunks) {
chunk.dispatches = Math.max(chunk.dispatches, 1);
}
// A first dispatch has no reservation of its own: any worker still
// rendering it may re-attach it as attempt 1.
for (const state of job.tasks.values()) {
if (
state.task.kind === "video" &&
state.state === "queued" &&
!state.duplicateOf &&
!state.reattach
) {
state.attempts = 1;
state.reattach = true;
state.firstPart = job.chunks[state.task.chunk]?.firstPart;
}
}
if (job.hls) {
job.hls.audioExtradata = job.audioSections
.values()
.next().value?.meta.extradata;
for (const chunk of job.chunks) {
const result = job.videoResults.get(chunk.index);
if (!result) continue;
job.hls.extradata ??= result.extradata;
job.hls.segments.set(
chunk.index,
new Map(
segmentsFromResult(job, chunk, result).map((segment) => [
segment.index,
segment,
]),
),
);
}
publishPlaylist(job);
}
console.log(
`resumed job ${id}: ${job.videoResults.size}/${job.chunks.length} chunks, ${job.audioSections.size}/${job.sections.length} audio done, ${requeued} tasks held for re-attach`,
);
setTimeout(dispatch, RESUME_HOLD_MS + 50);
if (audioReady(job)) job.t.audioDone ??= now();
if (job.videoResults.size === job.chunks.length)
job.t.videoDone ??= now();
maybeAssemble(job);
} catch (error) {
console.error(`resume ${id} failed: ${error}`);
}
}
dispatch();
}
// ---------------------------------------------------------------- dispatch ---
function enqueue(job: Job, task: Task, duplicateOf?: string) {
const state: TaskState = { task, state: "queued", attempts: 0, duplicateOf };
job.tasks.set(task.taskId, state);
queue.push(state);
}
function pickQueued(accepts: (kind: string) => boolean) {
for (const state of [...queue]) {
const job = jobs.get(state.task.jobId);
if (job && taskAccepted(job, state.task)) retireTask(state);
}
const schedulable = [...jobs.values()].map((job) => {
let runningTasks = 0;
for (const state of job.tasks.values()) {
if (state.state === "running") runningTasks++;
}
return {
id: job.id,
status: job.status,
requestedAt: job.t.requested ?? 0,
chunks: job.chunks.map((chunk) => chunk.index),
finishedChunks: job.videoResults,
runningTasks,
};
});
return pickQueuedTask(queue, schedulable, accepts, {
headChunks: HEAD_CHUNKS,
fifo: process.env.RF_SCHEDULER === "fifo",
now: Date.now(),
});
}
/** The task as sent for this attempt: its attempt number and part range. */
async function dispatchedTask(
job: Job | undefined,
state: TaskState,
): Promise<Task> {
const task = state.task;
if (task.kind !== "video" || !job)
return { ...task, attempt: state.attempts };
const chunk = job.chunks[task.chunk];
if (!chunk) throw new Error("unknown chunk");
const { range, firstPart } = reservePartRange(chunk);
const dispatched = {
...task,
attempt: state.attempts,
upload: {
...task.upload,
firstPart,
partLimit: chunk.partLimit,
stashKey: stashKey(job.id, chunk.index, firstPart),
},
};
// The journaled plan already reserves every chunk's first range (a resumed
// coordinator never reuses it), so first dispatches skip this write; it
// cost ~1 s of time to first segment on long exports.
state.firstPart = firstPart;
if (range === 0) return dispatched;
await journalPut(
journalKey(job.id, `dispatches/${chunk.index}/${range}.json`),
JSON.stringify({
task: dispatched,
worker: state.worker,
duplicateOf: state.duplicateOf,
}),
);
return dispatched;
}
function taskAccepted(job: Job, task: Task) {
return task.kind === "video"
? job.videoResults.has(task.chunk)
: job.audioSections.has(task.section);
}
function retireTask(state: TaskState) {
state.state = "done";
const index = queue.indexOf(state);
if (index >= 0) queue.splice(index, 1);
}
/** Put a task back in the queue, at most once. */
function requeue(state: TaskState, reason: string) {
const job = jobs.get(state.task.jobId);
if (!job || job.status !== "rendering" || taskAccepted(job, state.task)) {
retireTask(state);
return;
}
console.warn(`requeue ${state.task.taskId}: ${reason}`);
state.state = "queued";
state.worker = undefined;
state.progress = undefined;
state.reattach = false;
if (!queue.includes(state)) queue.unshift(state);
}
// -------------------------------------------------------------- transcodes ---
const TRANSCODE_KEYFRAME_SECONDS = 1;
const TRANSCODE_ATTEMPTS = 3;
type Transcode = {
id: string;
task: TranscodeTask;
state: "checking" | "queued" | "running" | "ready" | "error";
worker?: string;
attempts: number;
lastReportedAt?: number;
notBefore?: number;
size?: number;
error?: string;
/** Seconds of source encoded so far, from the worker's heartbeat. */
encodedSeconds?: number;
settledAt?: number;
settled: Promise<void>;
settle: () => void;
};
/** By output key: one transcode per target, shared by every job needing it. */
const transcodes = new Map<string, Transcode>();
async function storedTranscodeSize(output: string) {
const head = await s3.head(output);
if (!head) return null;
if (
!Number.isSafeInteger(head.size) ||
head.size <= 0 ||
head.size > SOURCE_LIMITS.sourceBytes
) {
throw new Error("stored transcode output has an invalid size");
}
return head.size;
}
async function transcodeSourceSize(source: string) {
const head = await s3.head(source);
if (!head || !Number.isSafeInteger(head.size) || head.size <= 0) {
throw new Error(
`transcode source ${source} is missing or has an invalid size`,
);
}
if (head.size > SOURCE_LIMITS.sourceBytes) {
throw new Error(
`transcode source ${source} is ${head.size} bytes (limit ${SOURCE_LIMITS.sourceBytes})`,
);
}
return head.size;
}
async function ensureTranscode(source: string, output: string) {
const existing = transcodes.get(output);
if (existing && existing.state !== "error") return existing;
const { promise, resolve } = Promise.withResolvers<void>();
const id = createHash("sha256").update(output).digest("hex").slice(0, 16);
const transcode: Transcode = {
id,
task: {
kind: "transcode",
taskId: `tc:${id}`,
source,
output,
keyframeSeconds: TRANSCODE_KEYFRAME_SECONDS,
},
state: "checking",
attempts: 0,
settled: promise,
settle: resolve,
};
transcodes.set(output, transcode);
try {
const size = await storedTranscodeSize(output);
if (size !== null) {
settleTranscode(transcode, "ready", size);
} else {
await transcodeSourceSize(source);
transcode.state = "queued";
dispatch();
}
} catch (error) {
settleTranscode(transcode, "error", String(error));
}
return transcode;
}
async function awaitTranscode(transcode: Transcode) {
await transcode.settled;
if (transcode.state !== "ready" || transcode.size === undefined) {
throw new Error(
`transcode of ${transcode.task.source} failed: ${transcode.error}`,
);
}
return transcode.size;
}
function nextTranscode() {
const at = Date.now();
for (const transcode of transcodes.values()) {
if (transcode.state === "queued" && (transcode.notBefore ?? 0) <= at) {
return transcode;
}
}
}
function transcodeById(id: string) {
for (const transcode of transcodes.values()) {
if (transcode.id === id) return transcode;
}
}
function settleTranscode(
transcode: Transcode,
state: "ready" | "error",
detail: number | string,
) {
transcode.state = state;
transcode.worker = undefined;
if (state === "ready") transcode.size = Number(detail);
else transcode.error = String(detail);
transcode.settledAt = Date.now();
transcode.settle();
}
function requeueTranscode(transcode: Transcode, reason: string) {
console.warn(`requeue ${transcode.task.taskId}: ${reason}`);
if (transcode.attempts >= TRANSCODE_ATTEMPTS) {
settleTranscode(transcode, "error", reason);
return;
}
const delay = 1000 * 2 ** transcode.attempts;
transcode.state = "queued";
transcode.worker = undefined;
transcode.notBefore = Date.now() + delay;
setTimeout(dispatch, delay + 50).unref();
}
function transcodeSummary(transcode: Transcode) {
return {
id: transcode.id,
status: transcode.state === "checking" ? "queued" : transcode.state,
source: transcode.task.source,
output: transcode.task.output,
size: transcode.size,
error: transcode.error,
encodedSeconds: transcode.encodedSeconds,
};
}
setInterval(() => {
// Workers heartbeat every 3 s and report a running transcode each time.
const cutoff = Date.now() - 30_000;
for (const [output, transcode] of transcodes) {
if (transcode.state === "running") {
const worker = transcode.worker ? workers.get(transcode.worker) : null;
if (
!worker ||
worker.lastSeen < cutoff ||
(transcode.lastReportedAt ?? 0) < cutoff
) {
requeueTranscode(transcode, "worker stopped reporting it");
}
} else if (
transcode.settledAt !== undefined &&
Date.now() - transcode.settledAt > JOB_RETENTION_MS
) {
transcodes.delete(output);
}
}
}, 5_000);
function dispatch() {
pumpLocalAudio();
for (let p = 0; p < pollers.length; ) {
const poller = pollers[p] as Poller;
const accepts = (kind: string) =>
!poller.kinds || poller.kinds.includes(kind);
// Transcodes gate an export's planning, so they go before any chunk.
const transcode =
accepts("video") && !poller.prefetch ? nextTranscode() : undefined;
if (transcode) {
pollers.splice(p, 1);
transcode.state = "running";
transcode.worker = poller.worker;
transcode.attempts++;
transcode.lastReportedAt = Date.now();
transcode.encodedSeconds = 0;
poller.resolve({ ...transcode.task, attempt: transcode.attempts });
continue;
}
const next = pickQueued(accepts);
let state: TaskState | undefined;
if (next >= 0) {
state = queue.splice(next, 1)[0];
} else if (accepts("video") && !poller.prefetch) {
state = straggler();
}
if (!state) {
p++;
continue;
}
pollers.splice(p, 1);
state.prefetched = poller.prefetch;
state.progress = undefined;
state.state = "running";
state.worker = poller.worker;
state.startedAt = now();
state.attempts++;
state.lastReportedAt = undefined;
const job = jobs.get(state.task.jobId);
if (job) {
job.t.firstTaskStarted ??= now();
if (state.task.kind === "video") job.t.firstVideoStarted ??= now();
job.workersUsed.add(poller.worker);
}
state.dispatching = true;
state.reattach = false;
const dispatchedState = state;
dispatchedTask(job, state).then(
(task) => {
dispatchedState.dispatching = false;
dispatchedState.startedAt = now();
poller.resolve(job?.status === "rendering" ? task : null);
},
(error) => {
dispatchedState.dispatching = false;
if (job) failJob(job, error);
poller.resolve(null);
},
);
}
}
/**
* Speculative execution: once the queue has drained, hedge the running chunk
* whose projected finish is furthest behind what a fresh slot could do. Slow
* replicas (noisy neighbours) otherwise set the whole export's tail.
*/
const FROZEN_MS = 5_000;
function straggler(): TaskState | undefined {
let best: { state: TaskState; gain: number } | undefined;
for (const job of jobs.values()) {
if (job.status !== "rendering" || job.request.duplicateStragglers === false)
continue;
const rates = job.taskStats
.filter(
(stat) => stat.kind === "video" && Number(stat.engineRenderMs) > 0,
)
.map((stat) => Number(stat.frames) / Number(stat.engineRenderMs));
for (const state of job.tasks.values()) {
if (state.state !== "running" || state.task.kind !== "video") continue;
const frames = state.progress?.frames ?? 0;
if (frames > 0)
rates.push(frames / Math.max(1, state.progress?.elapsedMs ?? 1));
}
if (rates.length < 3) continue;
rates.sort((a, b) => a - b);
const typicalRate = rates[Math.floor(rates.length / 2)] ?? 0;
if (typicalRate <= 0) continue;
for (const state of job.tasks.values()) {
if (
state.task.kind !== "video" ||
state.dispatching ||
state.state !== "running" ||
state.duplicated ||
state.duplicateOf ||
(state.prefetched && !state.progress) ||
job.videoResults.has(state.task.chunk)
) {
continue;
}
const total = state.task.frames[1] - state.task.frames[0];
const elapsed =
state.progress?.elapsedMs ?? now() - (state.startedAt ?? now());
const frames = state.progress?.frames ?? 0;
const expected = total / typicalRate + 2500;
// Engines report frames several times a second. A copy part-way
// through whose count stopped is hung: projecting from its average
// rate hedged one frozen near its end only when the engine watchdog
// fired, 30 s later.
const frozen =
frames > 0 &&
frames < total &&
now() - (state.progress?.advancedAt ?? now()) > FROZEN_MS;
// A copy with no frames long past its expected time is stuck (not
// merely slow): its remaining time is unknown, so always hedge it.
const remaining = frozen
? Number.POSITIVE_INFINITY
: frames > 0
? ((total - frames) * (state.progress?.elapsedMs ?? elapsed)) / frames
: elapsed > expected * 2
? Number.POSITIVE_INFINITY
: Math.max(0, expected - elapsed) * 2;
const fresh = 2500 + total / typicalRate;
const gain = remaining - fresh;
if (gain > 3000 && (!best || gain > best.gain)) best = { state, gain };
}
}
if (!best || best.state.task.kind !== "video") return undefined;
best.state.duplicated = true;
const job = jobs.get(best.state.task.jobId);
if (!job) return undefined;
const original = best.state.task;
const task: VideoTask = { ...original, taskId: `${original.taskId}:dup` };
const state: TaskState = {
task,
state: "queued",
attempts: 0,
duplicateOf: best.state.task.taskId,
};
job.tasks.set(task.taskId, state);
return state;
}
/** Running tasks whose result is already in (hedge losers) or whose job ended. */
function cancellations(worker: string) {
const out: string[] = [];
for (const job of jobs.values()) {
const ended = job.status !== "rendering";
if (ended && Date.now() - (job.t.ready ?? job.t.failed ?? now()) > 120_000)
continue;
for (const state of job.tasks.values()) {
if (state.state !== "running" || state.worker !== worker) continue;
if (
ended ||
(state.task.kind === "video" && job.videoResults.has(state.task.chunk))
) {
out.push(state.task.taskId);
}
}
}
return out;
}
/**
* A worker process restarted on the same host (crash, app hot-swap): its old
* id is dead, so hand its tasks back now instead of waiting for the timeout.
*/
function supersede(id: string) {
const host = id.slice(0, id.lastIndexOf("-"));
// Only predecessors that have gone quiet: two live workers sharing a
// hostname (several GPUs on one host) must not evict each other.
const quietSince = Date.now() - 4_500;
for (const [old, worker] of [...workers.entries()]) {
if (old === id || old.slice(0, old.lastIndexOf("-")) !== host) continue;
if (worker.lastSeen > quietSince) continue;
workers.delete(old);
for (const transcode of transcodes.values()) {
if (transcode.state === "running" && transcode.worker === old) {
requeueTranscode(transcode, `worker ${old} restarted`);
}
}
for (const job of jobs.values()) {
if (job.status !== "rendering") continue;
for (const state of job.tasks.values()) {
if (
state.state === "running" &&
!state.dispatching &&
state.worker === old
) {
requeue(state, `worker ${old} restarted`);
}
}
}
}
}
setInterval(() => {
// Requeue work held by workers that stopped heartbeating (they beat every
// 3 s). Local audio lanes run in this process and never heartbeat.
const cutoff = Date.now() - 12_000;
for (const job of jobs.values()) {
if (job.status !== "rendering") continue;
const lastProgress = job.t.lastProgress ?? job.t.planned ?? now();
if (now() - lastProgress > JOB_STALL_MS) {
failJob(
job,
new Error(
`no progress for ${Math.round((now() - lastProgress) / 1000)} s`,
),
);
continue;
}
for (const state of job.tasks.values()) {
if (
state.state !== "running" ||
state.dispatching ||
!state.worker ||
state.worker === "coordinator"
)
continue;
const worker = workers.get(state.worker);
if (!worker || worker.lastSeen < cutoff) {
requeue(state, `worker ${state.worker} went away`);
continue;
}
// A live worker that stops listing a task (lost /work response,
// crashed slot) no longer holds it.
if ((state.lastReportedAt ?? state.startedAt ?? 0) < cutoff) {
requeue(state, `not reported by ${state.worker}`);
}
}
}
dispatch();
}, 5_000);
// ---------------------------------------------------------------- results ---
function failJob(job: Job, error: unknown) {
if (job.status === "error" || job.status === "ready") return;
job.status = "error";
job.error = error instanceof Error ? error.message : String(error);
job.t.failed = now();
console.error(`job ${job.id} failed: ${job.error}`);
for (const state of job.tasks.values()) {
const index = queue.indexOf(state);
if (index >= 0) queue.splice(index, 1);
}
if (job.uploadId) s3.abortMultipart(job.key, job.uploadId).catch(() => {});
finish(job);
}
async function writeInit(job: Job, hls: HlsState) {
if (!hls.extradata) return false;
let asc: Uint8Array | null = null;
if (job.sections.length > 0) {
if (!hls.audioExtradata) return false;
asc = Buffer.from(hls.audioExtradata, "base64");
}
const key = `${hls.prefix}/init.mp4`;
await s3.put(
key,
initSegment({
width: job.width,
height: job.height,
fps: job.fps,
avcC: avcC(Buffer.from(hls.extradata, "base64")),
asc,
}),
"video/mp4",
);
hls.initUrl = await s3.presignFresh("GET", key, 6 * 3600);
return true;
}
/** Rewrites the playlist whenever the contiguous run of segments grows. */
async function publishPlaylist(job: Job) {
const hls = job.hls;
if (!hls || hls.ended || job.status === "error") return;
clearTimeout(hls.retry);
if (hls.writing) {
hls.dirty = true;
return;
}
hls.writing = true;
try {
do {
hls.dirty = false;
if (!hls.initUrl && !(await writeInit(job, hls))) return;
let grew = false;
let cursor = { ...hls.cursor };
const listed = [...hls.listed];
for (;;) {
const segment = hls.segments.get(cursor.chunk)?.get(cursor.index);
if (!segment) break;
listed.push({
url: await s3.presignFresh("GET", segment.key, 6 * 3600),
duration: (segment.frames[1] - segment.frames[0]) / job.fps,
});
grew = true;
cursor = segment.last
? { chunk: cursor.chunk + 1, index: 0 }
: { chunk: cursor.chunk, index: cursor.index + 1 };
}
const ended = cursor.chunk >= job.chunks.length;
if (!grew && !ended) continue;
await s3.put(
`${hls.prefix}/index.m3u8`,
playlist(listed, {
initUrl: hls.initUrl as string,
targetDuration: HLS_SEGMENT_SECONDS + 2,
ended,
}),
"application/vnd.apple.mpegurl",
);
hls.cursor = cursor;
hls.listed = listed;
if (hls.listed.length > 0) job.t.firstSegment ??= now();
if (ended) {
hls.ended = true;
job.t.hlsEnded = now();
if (job.status === "ready") persistFinished(job);
}
} while (hls.dirty && !hls.ended);
} catch (error) {
console.error(`job ${job.id} playlist: ${error}`);
if (now() - (job.t.requested ?? 0) < JOB_LIFETIME_MS) {
hls.retry = setTimeout(() => publishPlaylist(job), 1000);
hls.retry.unref();
}
} finally {
hls.writing = false;
}
}
function persistFinished(job: Job) {
journalPut(journalKey(job.id, "done"), job.status).catch((error) => {
console.error(`job ${job.id} terminal journal: ${error}`);
if (jobs.has(job.id)) setTimeout(() => persistFinished(job), 1000).unref();
});
}
// ---------------------------------------------------------------- callbacks ---
// Host suffixes a job may name as its callback (e.g. "vercel.app,cap.so").
const CALLBACK_HOSTS = (process.env.RF_CALLBACK_HOSTS ?? "")
.split(",")
.map((host) => host.trim().toLowerCase())
.filter(Boolean);
const CALLBACK_SECRET = process.env.RF_CALLBACK_SECRET || TOKEN;
function callbackAllowed(callbackUrl: string) {
const host = new URL(callbackUrl).hostname.toLowerCase();
return CALLBACK_HOSTS.some(
(allowed) => host === allowed || host.endsWith(`.${allowed}`),
);
}
function callbackPayload(job: Job) {
return {
id: job.id,
reference: job.request.reference,
status: job.status,
error: job.error,
key: job.key,
bytes: job.outputBytes,
width: job.width,
height: job.height,
frames: job.totalFrames,
fps: job.fps,
durationSeconds: job.totalFrames / job.fps,
};
}
/** Posts the outcome, signed `sha256=<hex hmac of the body>`, with retries. */
function notify(
job: Job,
body = JSON.stringify(callbackPayload(job)),
attempt = 0,
) {
const url = job.request.callbackUrl;
if (!url) return;
const signature = createHmac("sha256", CALLBACK_SECRET)
.update(body)
.digest("hex");
fetch(url, {
method: "POST",
redirect: "error",
headers: {
"content-type": "application/json",
"x-render-farm-signature": `sha256=${signature}`,
},
body,
signal: AbortSignal.timeout(15_000),
})
.then((response) => {
if (!response.ok) throw new Error(`callback -> ${response.status}`);
})
.catch((error) => {
if (attempt >= 8) {
console.error(`job ${job.id} callback gave up: ${error}`);
return;
}
setTimeout(
() => notify(job, body, attempt + 1),
Math.min(60_000, 1000 * 2 ** attempt),
).unref();
});
}
function finish(job: Job) {
if (job.status === "error" || !job.hls || job.hls.ended) persistFinished(job);
notify(job);
job.audioCache?.close();
// Freeze the summary, then drop the media: rendered audio alone is ~290 MB
// for a 2 h export, and finished jobs were never evicted.
job.final = summary(job);
job.audioSections.clear();
job.videoResults.clear();
job.taskStats = [];
for (const [taskId, state] of job.tasks) {
const index = queue.indexOf(state);
if (index >= 0) queue.splice(index, 1);
// Running copies stay so their workers are told to cancel them.
if (state.state !== "running") job.tasks.delete(taskId);
}
setTimeout(() => jobs.delete(job.id), JOB_RETENTION_MS).unref();
for (const waiter of job.waiters.splice(0)) waiter();
for (const waiter of job.audioWaiters.splice(0)) waiter();
finishedJobs.push(job.id);
if (finishedJobs.length > 50) finishedJobs.shift();
rmSync(join(WORK_DIR, job.id), { recursive: true, force: true });
stitchLimiter.cancel(job.id);
job.headerStashes = undefined;
job.stitchParts = undefined;
dropStashes(job.id).catch((error) =>
console.error(`job ${job.id}: stash cleanup failed: ${error}`),
);
}
function recordStat(
job: Job,
kind: string,
index: number,
worker: string,
timings: object,
) {
job.taskStats.push({ kind, index, worker, ...timings });
}
async function onVideoDone(job: Job, state: TaskState, result: VideoResult) {
if (state.task.kind !== "video") return Promise.resolve();
await acceptOnce(job.acceptances, `v/${state.task.chunk}`, () =>
acceptVideo(job, state, result),
);
retireTask(state);
}
async function acceptVideo(job: Job, state: TaskState, result: VideoResult) {
if (state.task.kind !== "video") return;
const chunk = state.task.chunk;
if (job.videoResults.has(chunk) || job.status !== "rendering") return;
const plan = job.chunks[chunk];
const firstPart = state.firstPart ?? plan?.firstPart;
const problem =
plan && firstPart !== undefined && result.firstPart === firstPart
? uploadProblem(
{
stashKey: stashKey(job.id, chunk, firstPart),
firstPart,
partLimit: plan.partLimit,
},
result,
)
: "result is not from this chunk's dispatch";
if (problem) throw new Error(`chunk ${chunk}: ${problem}`);
await journalPut(
journalKey(job.id, `v/${chunk}.json`),
JSON.stringify(result),
);
if (job.status !== "rendering") return;
state.state = "done";
const queued = queue.indexOf(state);
if (queued >= 0) queue.splice(queued, 1);
recordStat(job, "video", chunk, result.worker, {
...result.timings,
fetchBytes: result.timings.fetch.bytes,
fetchMs: result.timings.fetch.ms,
engineRenderMs: result.timings.engine.render_ms,
frames: result.sizes.length,
bytes: result.bytes,
duplicate: Boolean(state.duplicateOf),
});
job.cpuSeconds += result.timings.cpuSeconds;
job.fetchedBytes += result.timings.fetch.bytes;
for (const other of job.tasks.values()) {
// The sibling copy (original or hedge) is now redundant.
if (
other !== state &&
other.task.kind === "video" &&
other.task.chunk === chunk &&
other.state === "queued"
) {
other.state = "done";
const index = queue.indexOf(other);
if (index >= 0) queue.splice(index, 1);
}
}
job.videoResults.set(chunk, result);
job.t.lastProgress = now();
stitchAhead(job);
// Segments reported to a previous coordinator process (before a restart)
// are not reported again; derive any missing ones from the result.
const chunkPlan = job.chunks[chunk];
if (job.hls && chunkPlan) {
job.hls.extradata ??= result.extradata;
const known =
job.hls.segments.get(chunk) ?? new Map<number, SegmentReport>();
for (const segment of segmentsFromResult(job, chunkPlan, result)) {
if (!known.has(segment.index)) known.set(segment.index, segment);
}
job.hls.segments.set(chunk, known);
publishPlaylist(job);
}
if (job.videoResults.size === job.chunks.length) {
job.t.videoDone = now();
maybeAssemble(job);
}
}
async function onAudioDone(
job: Job,
state: TaskState,
meta: AudioResultMeta,
data: Uint8Array,
) {
if (state.task.kind !== "audio") return Promise.resolve();
await acceptOnce(job.acceptances, `a/${state.task.section}`, () =>
acceptAudio(job, state, meta, data),
);
retireTask(state);
}
async function acceptAudio(
job: Job,
state: TaskState,
meta: AudioResultMeta,
data: Uint8Array,
) {
if (
state.task.kind !== "audio" ||
job.status !== "rendering" ||
job.audioSections.has(state.task.section)
)
return;
await journalPut(
journalKey(job.id, `a/${state.task.section}.bin`),
audioFrame(meta, data),
);
if (job.status !== "rendering") return;
state.state = "done";
const queued = queue.indexOf(state);
if (queued >= 0) queue.splice(queued, 1);
if (job.hls) job.hls.audioExtradata ??= meta.extradata;
job.audioSections.set(state.task.section, { meta, data });
job.t.lastProgress = now();
if (job.hls && !job.hls.initUrl) publishPlaylist(job);
recordStat(job, "audio", state.task.section, meta.worker, {
...meta.timings,
fetchMs: meta.timings.fetch.ms,
fetchBytes: meta.timings.fetch.bytes,
});
job.cpuSeconds += meta.timings.cpuSeconds;
job.fetchedBytes += meta.timings.fetch.bytes;
if (audioReady(job)) job.t.audioDone = now();
for (const waiter of job.audioWaiters.splice(0)) waiter();
if (audioReady(job)) maybeAssemble(job);
}
function audioReady(job: Job) {
return job.audioSections.size === job.sections.length;
}
function maybeAssemble(job: Job) {
if (job.status !== "rendering") return;
if (job.videoResults.size !== job.chunks.length || !audioReady(job)) return;
job.status = "assembling";
assemble(job).catch((error) => failJob(job, error));
}
async function assemble(job: Job) {
const results = job.chunks.map((chunk) => {
const result = job.videoResults.get(chunk.index);
if (!result) throw new Error(`missing chunk ${chunk.index}`);
return result;
});
const extradata = results[0]?.extradata ?? "";
for (const result of results) {
if (result.extradata !== extradata) {
throw new Error("chunks disagree on SPS/PPS; cannot join them");
}
}
const sizes = new Uint32Array(job.totalFrames);
const keyframes: number[] = [];
const videoRuns: Run[] = [];
const audioRuns: Run[] = [];
const parts: { partNumber: number; etag: string }[] = [];
const stashed: Parameters<typeof planStitch>[1] = [];
let base = 0;
job.chunks.forEach((chunk, index) => {
const result = results[index] as VideoResult;
sizes.set(result.sizes, chunk.frames[0]);
for (const key of result.keyframes) keyframes.push(chunk.frames[0] + key);
for (const run of result.videoRuns)
videoRuns.push({ ...run, offset: run.offset + base });
for (const run of result.audioRuns)
audioRuns.push({ ...run, offset: run.offset + base });
for (const part of result.parts) parts.push(part);
stashed.push({
slot: chunk.firstPart - 1,
stash: result.stash,
parts: result.parts,
});
base += result.bytes;
});
const payloadSize = base;
let audio: Parameters<typeof buildHeader>[0]["audio"] = null;
if (job.sections.length > 0) {
const totalPackets = Math.ceil(job.totalSamples / PACKET) + 1;
const audioSizes = new Uint32Array(totalPackets);
let asc = "";
for (const { meta } of job.audioSections.values()) {
audioSizes.set(meta.sizes, meta.firstPacket);
asc = meta.extradata;
}
audio = {
sizes: audioSizes,
runs: audioRuns,
asc: Buffer.from(asc, "base64"),
totalSamples: job.totalSamples,
priming: PACKET,
};
const covered = audioRuns.reduce((sum, run) => sum + run.count, 0);
if (covered !== totalPackets) {
throw new Error(`audio runs cover ${covered}/${totalPackets} packets`);
}
}
const videoCovered = videoRuns.reduce((sum, run) => sum + run.count, 0);
if (videoCovered !== job.totalFrames) {
throw new Error(
`video runs cover ${videoCovered}/${job.totalFrames} frames`,
);
}
const header = buildHeader({
width: results[0]?.width ?? job.width,
height: results[0]?.height ?? job.height,
fps: job.fps,
video: {
sizes,
runs: videoRuns,
keyframes: Uint32Array.from(keyframes),
avcC: avcC(Buffer.from(extradata, "base64")),
},
audio,
payloadSize,
minimumSize: 0,
});
job.t.headerBuilt = now();
if (!job.uploadId) throw new Error("no upload");
const uploadId = job.uploadId;
const stitch = planStitch(header.byteLength, stashed);
job.outputBytes = await completeUpload(
s3,
{
key: job.key,
uploadId,
header,
payloadSize,
parts,
prepare: () => writeStitchParts(job, uploadId, header, stitch),
},
(intent) => journalPut(journalKey(job.id, "assembly.json"), intent),
);
job.t.completed = now();
await s3.ready();
job.url = await s3.presignFresh("GET", job.key, 24 * 3600);
if (process.env.RF_VERIFY_ASYNC !== "0") {
// Watchable as soon as the object exists; the ffprobe check still
// runs and is reported on the job, it just isn't on the user's path.
verifyPlayable(job).then(
() => {
job.verified = true;
},
(error) => {
job.verified = false;
console.error(`job ${job.id} FAILED verification: ${error}`);
},
);
} else {
await verifyPlayable(job);
job.verified = true;
}
job.t.ready = now();
job.status = "ready";
console.log(
`job ${job.id} ready in ${Math.round(job.t.ready - (job.t.requested ?? 0))}ms (${job.chunks.length} chunks)`,
);
finish(job);
}
const stitchPartKey = (part: StitchPart) =>
`${part.partNumber}:${part.sources
.map((source) => (source.kind === "header" ? "header" : source.key))
.join(",")}`;
/**
* Uploads a part the coordinator owns: a stash that stands alone is copied
* server side; the header and runs of small stashes are joined and uploaded.
*/
async function writeStitchPart(
key: string,
uploadId: string,
header: Uint8Array | null,
part: StitchPart,
readStash: (key: string) => Promise<Uint8Array> = (stash) =>
journalS3.get(stash),
) {
const [only] = part.sources;
if (
part.sources.length === 1 &&
only?.kind === "stash" &&
s3.sharesStoreWith(journalS3)
) {
return {
partNumber: part.partNumber,
etag: await s3.uploadPartCopy(key, uploadId, part.partNumber, {
bucket: journalS3.config.bucket,
key: only.key,
}),
};
}
const pieces = await Promise.all(
part.sources.map(async (source) => {
if (source.kind === "stash") return readStash(source.key);
if (!header) throw new Error("the header part waits for assembly");
return header;
}),
);
const body = new Uint8Array(part.bytes);
let offset = 0;
for (const [index, piece] of pieces.entries()) {
if (piece.byteLength !== part.sources[index]?.bytes) {
throw new Error(`stash for part ${part.partNumber} is incomplete`);
}
body.set(piece, offset);
offset += piece.byteLength;
}
return {
partNumber: part.partNumber,
etag: await s3.uploadPart(key, uploadId, part.partNumber, body),
};
}
/**
* While chunks render, writes every coordinator part whose chunks are all in:
* the plan for an accepted run of chunks never changes as later chunks
* arrive, except for the part holding the header and the part still open at
* the run's end. Assembly then only writes those.
*/
function stitchAhead(job: Job) {
if (!job.uploadId) return;
const accepted: Parameters<typeof planStitch>[1] = [];
for (const chunk of job.chunks) {
const result = job.videoResults.get(chunk.index);
if (!result) break;
accepted.push({
slot: chunk.firstPart - 1,
stash: result.stash,
parts: result.parts,
});
}
if (accepted.length === 0) return;
// The header's part is the one assembly must write; read its stashes now
// so that write is only an upload.
job.headerStashes ??= new Map();
let carried = 0;
for (const chunk of accepted) {
if (!job.headerStashes.has(chunk.stash.key)) {
const read = stitchLimiter.run(job.id, true, () =>
journalS3.get(chunk.stash.key),
);
read.catch(() => job.headerStashes?.delete(chunk.stash.key));
job.headerStashes.set(chunk.stash.key, read);
}
carried += chunk.stash.bytes;
if (chunk.parts.length > 0 || carried >= MIN_PART) break;
}
const complete = accepted.length === job.chunks.length;
const plan = planStitch(0, accepted, { partial: !complete });
job.stitchParts ??= new Map();
for (const part of plan) {
if (part.sources.some((source) => source.kind === "header")) continue;
const key = stitchPartKey(part);
if (job.stitchParts.has(key)) continue;
const uploadId = job.uploadId;
const written = stitchLimiter.run(job.id, true, () =>
writeStitchPart(job.key, uploadId, null, part),
);
written.catch(() => job.stitchParts?.delete(key));
job.stitchParts.set(key, written);
}
}
async function writeStitchParts(
job: Job,
uploadId: string,
header: Uint8Array,
plan: StitchPart[],
) {
const ahead = job.stitchParts ?? new Map();
stitchLimiter.promote(job.id);
// A part written ahead under a grouping the final plan changed must land
// before the plan rewrites its number. Header reads are awaited here too,
// not inside a stitch slot, so assembly never holds slots while waiting.
await Promise.allSettled([
...ahead.values(),
...(job.headerStashes?.values() ?? []),
]);
const readStash = (key: string) =>
(job.headerStashes?.get(key) ?? Promise.reject()).catch(() =>
journalS3.get(key),
);
return Promise.all(
plan.map((part) => {
const write = () =>
stitchLimiter.run(job.id, false, () =>
writeStitchPart(job.key, uploadId, header, part, readStash),
);
const early = ahead.get(stitchPartKey(part));
return early ? early.catch(write) : write();
}),
);
}
async function dropStashes(id: string) {
const keys = await journalS3.list(`stash/${id}/`);
await Promise.all(keys.map(({ key }) => journalS3.delete(key)));
}
const STASH_SWEEP_MS = 10 * 60_000;
let journalResumed = false;
/**
* Deletes stashes a job's own cleanup missed: those a losing copy writes after
* it, and any left when the coordinator stops mid-cleanup (restart never
* reopens a finished job). A stash of a job this process doesn't know is
* kept until the job could no longer be resumed.
*/
async function sweepStashes() {
if (!journalResumed) return;
const stale = (await journalS3.list("stash/")).filter(
({ key, modifiedAt }) => {
const job = jobs.get(key.split("/")[1] ?? "");
if (!job) return Date.now() - modifiedAt > JOB_LIFETIME_MS;
return job.status === "ready" || job.status === "error";
},
);
await Promise.all(stale.map(({ key }) => journalS3.delete(key)));
}
setInterval(() => {
sweepStashes().catch((error) =>
console.error(`stash sweep failed: ${error}`),
);
}, STASH_SWEEP_MS);
async function verifyPlayable(job: Job) {
if (!job.url) return;
const probe = Bun.spawn(
[
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration:stream=codec_name,nb_frames",
"-of",
"json",
job.url,
],
{ stdout: "pipe", stderr: "pipe" },
);
const [out, err, code] = await Promise.all([
new Response(probe.stdout).text(),
new Response(probe.stderr).text(),
probe.exited,
]);
if (code !== 0) throw new Error(`ffprobe failed: ${err.slice(0, 300)}`);
const parsed = JSON.parse(out) as {
format?: { duration?: string };
streams?: { codec_name?: string; nb_frames?: string }[];
};
const duration = Number(parsed.format?.duration ?? 0);
const expected = job.totalFrames / job.fps;
if (Math.abs(duration - expected) > 0.5) {
throw new Error(`output duration ${duration}s, expected ${expected}s`);
}
const frames = Number(
parsed.streams?.find((stream) => stream.codec_name === "h264")?.nb_frames ??
0,
);
if (frames !== job.totalFrames) {
throw new Error(`output has ${frames} frames, expected ${job.totalFrames}`);
}
}
// ---------------------------------------------------------------- summary ---
function percentile(values: number[], p: number) {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
return (
sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0
);
}
/** Share of the export rendered, 0-1; 1 only once the MP4 is complete. */
function jobProgress(job: Job) {
if (job.status === "ready") return 1;
if (job.totalFrames <= 0) return 0;
let frames = 0;
for (const chunk of job.chunks) {
if (job.videoResults.has(chunk.index)) {
frames += chunk.frames[1] - chunk.frames[0];
}
}
for (const state of job.tasks.values()) {
if (
state.task.kind === "video" &&
state.state === "running" &&
!state.duplicateOf &&
!job.videoResults.has(state.task.chunk)
) {
frames += state.progress?.frames ?? 0;
}
}
return Math.min(0.99, frames / job.totalFrames);
}
function summary(job: Job) {
const start = job.t.requested ?? 0;
const rel = Object.fromEntries(
Object.entries(job.t).map(([name, value]) => [
name,
Math.round(value - start),
]),
);
const video = job.taskStats.filter(
(stat) => stat.kind === "video" && !stat.duplicate,
);
const audio = job.taskStats.filter((stat) => stat.kind === "audio");
const pick = (list: Record<string, unknown>[], field: string) =>
list.map((stat) => Number(stat[field] ?? 0));
const describe = (list: Record<string, unknown>[], field: string) => ({
p50: Math.round(percentile(pick(list, field), 0.5)),
p95: Math.round(percentile(pick(list, field), 0.95)),
max: Math.round(Math.max(0, ...pick(list, field))),
});
return {
id: job.id,
label: job.request.label,
status: job.status,
error: job.error,
recording: job.request.recording,
fps: job.fps,
resolution: job.resolution,
output: {
width: job.width,
height: job.height,
frames: job.totalFrames,
bytes: job.outputBytes,
},
url: job.status === "ready" ? job.url : undefined,
reference: job.request.reference,
progress: jobProgress(job),
timeline: rel,
plan: job.plan,
probe: job.probe,
workersUsed: job.workersUsed.size,
verified: job.verified,
hlsUrl: job.hls?.url,
hlsSegments: job.hls?.listed.length,
cpuSeconds: Math.round(job.cpuSeconds * 10) / 10,
fetchedBytes: job.fetchedBytes,
duplicates: job.taskStats.filter((stat) => stat.duplicate).length,
video: {
total: describe(video, "totalMs"),
fetch: describe(video, "fetchMs"),
render: describe(video, "engineRenderMs"),
engine: describe(video, "engineMs"),
audioWait: describe(video, "audioWaitMs"),
upload: describe(video, "uploadMs"),
queued: describe(video, "queuedMs"),
},
audio: {
total: describe(audio, "totalMs"),
engine: describe(audio, "engineMs"),
fetch: describe(audio, "fetchMs"),
},
tasks: job.taskStats,
};
}
// ------------------------------------------------------------------ server ---
function authorized(request: Request) {
if (!TOKEN) return true;
const given = Buffer.from(request.headers.get("authorization") ?? "");
const expected = Buffer.from(`Bearer ${TOKEN}`);
return given.length === expected.length && timingSafeEqual(given, expected);
}
Bun.serve({
port: PORT,
hostname: "::",
idleTimeout: 120,
async fetch(request) {
const url = new URL(request.url);
if (url.pathname === "/health")
return Response.json({ ok: true, ...liveSlots() });
if (!authorized(request))
return new Response("unauthorized", { status: 401 });
if (url.pathname === "/work" && request.method === "POST") {
const body = (await request.json()) as {
worker: string;
slots: number;
cpus: number;
service?: string;
kinds?: string[];
audioSlots?: number;
prefetch?: boolean;
draining?: boolean;
};
const worker = workers.get(body.worker) ?? {
id: body.worker,
slots: body.slots,
cpus: body.cpus,
lastSeen: Date.now(),
service: body.service ?? "",
};
worker.lastSeen = Date.now();
worker.slots = body.slots;
worker.audioSlots = body.audioSlots ?? 0;
supersede(body.worker);
workers.set(body.worker, worker);
if (body.draining)
return Response.json({ task: null, finished: finishedJobs });
const task = await new Promise<WorkItem | null>((resolve) => {
const poller: Poller = {
worker: body.worker,
kinds: body.kinds,
prefetch: body.prefetch,
resolve,
};
pollers.push(poller);
dispatch();
if (poller.prefetch) {
const index = pollers.indexOf(poller);
if (index >= 0) {
pollers.splice(index, 1);
resolve(null);
}
}
setTimeout(() => {
const index = pollers.indexOf(poller);
if (index >= 0) {
pollers.splice(index, 1);
resolve(null);
}
}, 20_000);
});
return Response.json({ task, finished: finishedJobs });
}
if (url.pathname === "/heartbeat" && request.method === "POST") {
const body = (await request.json()) as {
worker: string;
slots: number;
cpus: number;
usage?: Usage | null;
running?: {
taskId: string;
attempt?: number;
phase?: "reserved" | "running";
frames: number;
total: number;
elapsedMs: number;
}[];
};
const staleTranscodes: string[] = [];
for (const entry of body.running ?? []) {
if (entry.taskId.startsWith("tc:")) {
const transcode = transcodeById(entry.taskId.slice(3));
if (
transcode?.state === "running" &&
transcode.worker === body.worker &&
transcode.attempts === entry.attempt
) {
transcode.lastReportedAt = Date.now();
transcode.encodedSeconds = entry.frames;
} else {
staleTranscodes.push(entry.taskId);
}
continue;
}
const job = jobs.get(entry.taskId.split(":")[0] ?? "");
const state = job?.tasks.get(entry.taskId);
if (!state) continue;
if (
state.state === "queued" &&
state.heldUntil &&
(state.task.kind === "audio" ||
(state.reattach &&
(state.worker === undefined || state.worker === body.worker) &&
state.attempts === entry.attempt))
) {
// Still running from before a coordinator restart: adopt it.
state.state = "running";
state.worker = body.worker;
state.startedAt = now() - entry.elapsedMs;
state.heldUntil = undefined;
state.reattach = false;
state.attempts = entry.attempt ?? 1;
state.prefetched = entry.phase === "reserved";
const index = queue.indexOf(state);
if (index >= 0) queue.splice(index, 1);
console.log(`re-attached ${entry.taskId} to ${body.worker}`);
}
if (
state.state !== "running" ||
state.worker !== body.worker ||
(entry.attempt !== undefined && state.attempts !== entry.attempt)
)
continue;
state.lastReportedAt = Date.now();
if (entry.phase === "reserved") continue;
if (state.task.kind === "video") {
const advanced = entry.frames > (state.progress?.frames ?? 0);
if (advanced && job) job.t.lastProgress = now();
state.progress = {
...entry,
at: now(),
advancedAt:
advanced || !state.progress ? now() : state.progress.advancedAt,
};
}
}
const worker = workers.get(body.worker);
supersede(body.worker);
if (worker) {
worker.lastSeen = Date.now();
worker.usage = body.usage;
} else {
workers.set(body.worker, {
id: body.worker,
slots: body.slots,
cpus: body.cpus,
lastSeen: Date.now(),
service: "",
usage: body.usage,
});
}
return Response.json({
finished: finishedJobs,
cancel: [...cancellations(body.worker), ...staleTranscodes],
});
}
const segmentMatch = url.pathname.match(/^\/tasks\/([^/]+)\/segment$/);
if (segmentMatch && request.method === "POST") {
const taskId = decodeURIComponent(segmentMatch[1] ?? "");
const job = jobs.get(taskId.split(":")[0] ?? "");
if (!job?.hls) return new Response("unknown job", { status: 404 });
const state = job.tasks.get(taskId);
const plan =
state?.task.kind === "video" ? job.chunks[state.task.chunk] : undefined;
const report =
plan && state?.firstPart !== undefined
? checkSegmentReport(
await request.json(),
job.hls.prefix,
plan,
state.firstPart,
)
: null;
if (!report) return new Response("invalid segment", { status: 400 });
job.hls.extradata ??= report.extradata || undefined;
let chunk = job.hls.segments.get(report.chunk);
if (!chunk) {
chunk = new Map();
job.hls.segments.set(report.chunk, chunk);
}
// First copy wins; other copies wrote their own objects.
if (!chunk.has(report.index)) chunk.set(report.index, report);
publishPlaylist(job);
return Response.json({ ok: true });
}
if (url.pathname === "/transcodes" && request.method === "POST") {
const { source, output, sourceRoot } = (await request.json()) as Record<
string,
unknown
>;
if (
!isKey(sourceRoot, { trailingSlash: true }) ||
!sourceRoot.endsWith("/") ||
!isKey(source) ||
!isKey(output) ||
!source.startsWith(sourceRoot) ||
!output.startsWith(sourceRoot) ||
!output.endsWith(".mp4")
) {
return new Response(
"source and output must be keys inside sourceRoot, output an .mp4",
{ status: 400 },
);
}
return Response.json(
transcodeSummary(await ensureTranscode(source, output)),
);
}
const transcodeMatch = url.pathname.match(
/^\/transcodes\/([0-9a-f]{16})(?:\/(done|fail))?$/,
);
if (transcodeMatch) {
const transcode = transcodeById(transcodeMatch[1] ?? "");
if (!transcode) return new Response("unknown transcode", { status: 404 });
if (!transcodeMatch[2] && request.method === "GET") {
return Response.json(transcodeSummary(transcode));
}
if (request.method !== "POST") {
return new Response("method not allowed", { status: 405 });
}
const report = (await request.json()) as {
worker: string;
attempt?: number;
size?: number;
error?: string;
};
if (transcodeMatch[2] === "done") {
if (transcode.state === "ready") return Response.json({ ok: true });
const current = () =>
transcodes.get(transcode.task.output) === transcode &&
transcode.state === "running" &&
transcode.worker === report.worker &&
transcode.attempts === report.attempt;
if (!current()) {
return new Response("stale transcode report", { status: 409 });
}
if (!Number.isSafeInteger(report.size) || !(Number(report.size) > 0)) {
return new Response("invalid transcode size", { status: 400 });
}
const object = await s3.head(transcode.task.output);
if (!current()) {
return new Response("stale transcode report", { status: 409 });
}
if (!object || object.size !== report.size) {
return new Response("transcode output does not match report", {
status: 409,
});
}
settleTranscode(transcode, "ready", object.size);
dispatch();
} else if (
transcode.state === "running" &&
transcode.worker === report.worker &&
transcode.attempts === report.attempt
) {
requeueTranscode(transcode, report.error ?? "failed");
}
return Response.json({ ok: true });
}
const taskMatch = url.pathname.match(
/^\/tasks\/([^/]+)\/(done|fail|audio)$/,
);
if (taskMatch && request.method === "POST") {
const taskId = decodeURIComponent(taskMatch[1] ?? "");
const jobId = taskId.split(":")[0] ?? "";
const job = jobs.get(jobId);
const state = job?.tasks.get(taskId);
if (!job || !state) return new Response("unknown task", { status: 404 });
if (taskMatch[2] === "fail") {
const body = (await request.json()) as {
error: string;
worker: string;
attempt?: number;
};
console.warn(`task ${taskId} failed on ${body.worker}: ${body.error}`);
if (job.status !== "rendering") return Response.json({ ok: true });
// A stale copy (already requeued, re-dispatched or resumed) failing
// says nothing about the attempt now in charge of the task.
if (
state.state !== "running" ||
state.worker !== body.worker ||
(body.attempt !== undefined && body.attempt !== state.attempts)
) {
return Response.json({ ok: true });
}
if (
state.task.kind === "video" &&
job.videoResults.has(state.task.chunk)
) {
state.state = "done";
return Response.json({ ok: true });
}
if (state.duplicateOf) {
state.state = "done";
return Response.json({ ok: true });
}
if (state.attempts >= 5)
failJob(job, new Error(`${taskId}: ${body.error}`));
else {
// Back off before retrying so a transient storage hiccup
// can't burn every attempt within a second.
state.state = "queued";
state.worker = undefined;
state.heldUntil = Date.now() + 1000 * state.attempts;
if (!queue.includes(state)) queue.unshift(state);
setTimeout(dispatch, 1000 * state.attempts + 50);
}
return Response.json({ ok: true });
}
if (taskMatch[2] === "done") {
const result = (await request.json()) as VideoResult;
if (job.status === "rendering") await onVideoDone(job, state, result);
return Response.json({ ok: true });
}
// audio: [u32 json length][json][packet bytes]
const bytes = new Uint8Array(await request.arrayBuffer());
const length = new DataView(bytes.buffer, bytes.byteOffset).getUint32(0);
const meta = JSON.parse(
new TextDecoder().decode(bytes.subarray(4, 4 + length)),
) as AudioResultMeta;
const data = bytes.slice(4 + length);
if (state.task.kind === "audio" && job.status === "rendering") {
await onAudioDone(job, state, meta, data);
}
return Response.json({ ok: true });
}
const audioMatch = url.pathname.match(/^\/jobs\/([^/]+)\/audio$/);
if (audioMatch && request.method === "GET") {
const job = jobs.get(audioMatch[1] ?? "");
if (!job) return new Response("unknown job", { status: 404 });
const from = Number(url.searchParams.get("from"));
const to = Number(url.searchParams.get("to"));
const collect = () => {
const sizes: number[] = [];
const pieces: Uint8Array[] = [];
let packet = from;
while (packet < to) {
let found = false;
for (const { meta, data } of job.audioSections.values()) {
const end = meta.firstPacket + meta.sizes.length;
if (packet < meta.firstPacket || packet >= end) continue;
let offset = 0;
for (let index = 0; index < packet - meta.firstPacket; index++) {
offset += meta.sizes[index] ?? 0;
}
const last = Math.min(to, end);
for (
let index = packet - meta.firstPacket;
index < last - meta.firstPacket;
index++
) {
const size = meta.sizes[index] ?? 0;
sizes.push(size);
pieces.push(data.subarray(offset, offset + size));
offset += size;
}
packet = last;
found = true;
break;
}
if (!found) return null;
}
return { sizes, pieces };
};
let collected = collect();
while (!collected) {
if (job.status !== "rendering" && job.status !== "planning")
return new Response("job finished", { status: 410 });
await new Promise<void>((resolve) => {
job.audioWaiters.push(resolve);
setTimeout(resolve, 10_000);
});
collected = collect();
}
const header = new Uint8Array(4 + collected.sizes.length * 4);
const view = new DataView(header.buffer);
view.setUint32(0, collected.sizes.length);
for (const [index, size] of collected.sizes.entries()) {
view.setUint32(4 + index * 4, size);
}
return new Response(
new Blob([header, ...collected.pieces] as BlobPart[]),
);
}
if (url.pathname === "/index" && request.method === "POST") {
// Upload-time hook: index a recording's sources before any export.
const parsed = validateJobRequest(await request.json().catch(() => null));
if (typeof parsed === "string")
return new Response(parsed, { status: 400 });
const body = parsed;
const started = performance.now();
await sourceIndex(body.recording.replace(/\/$/, ""), body.sourceRoot);
return Response.json({ ms: Math.round(performance.now() - started) });
}
if (url.pathname === "/jobs" && request.method === "POST") {
const parsed = validateJobRequest(await request.json().catch(() => null));
if (typeof parsed === "string")
return new Response(parsed, { status: 400 });
const body = parsed;
if (body.callbackUrl && !callbackAllowed(body.callbackUrl)) {
return new Response("callbackUrl host is not in RF_CALLBACK_HOSTS", {
status: 400,
});
}
const active = [...jobs.values()].filter(
(job) => job.status !== "ready" && job.status !== "error",
).length;
if (active >= MAX_ACTIVE_JOBS) {
return new Response("too many active exports", {
status: 429,
headers: { "retry-after": "10" },
});
}
const id = randomUUID().slice(0, 12);
if (body.output && (await s3.head(body.output.key))) {
return new Response("output key already exists", { status: 409 });
}
const job = await newJob(id, body);
jobs.set(id, job);
try {
await journalPut(
journalKey(id, "request.json"),
JSON.stringify({
id,
request: body,
requestedAt: job.t.requested,
}),
);
} catch (error) {
jobs.delete(id);
throw error;
}
planJob(job).catch((error) => failJob(job, error));
return Response.json({ id, hlsUrl: job.hls?.url });
}
const jobMatch = url.pathname.match(/^\/jobs\/([^/]+)(\/wait)?$/);
if (jobMatch && request.method === "GET") {
const job = jobs.get(jobMatch[1] ?? "");
if (!job) return new Response("unknown job", { status: 404 });
if (jobMatch[2] && job.status !== "ready" && job.status !== "error") {
await new Promise<void>((resolve) => {
job.waiters.push(resolve);
setTimeout(resolve, 100_000);
});
}
const body = job.final
? { ...job.final, verified: job.verified }
: summary(job);
// Re-signed per read: a URL issued at submission lapses with its
// signature or the instance credentials behind it.
if (job.hls) {
body.hlsUrl = await s3.presignFresh(
"GET",
`${job.hls.prefix}/index.m3u8`,
6 * 3600,
);
}
return Response.json(body);
}
if (url.pathname === "/usage") {
// Fleet totals for cost accounting: diff two snapshots around a job.
const live = [...workers.values()].filter(
(worker) => Date.now() - worker.lastSeen < 30_000,
);
return Response.json({
at: Date.now(),
workers: live.length,
cpuSeconds: live.reduce(
(sum, worker) => sum + (worker.usage?.cpuSeconds ?? 0),
0,
),
memoryBytes: live.reduce(
(sum, worker) => sum + (worker.usage?.memoryBytes ?? 0),
0,
),
perWorker: live.map((worker) => ({ id: worker.id, ...worker.usage })),
});
}
if (url.pathname === "/workers") {
return Response.json({
...liveSlots(),
workers: [...workers.values()].map((worker) => ({
...worker,
ageMs: Date.now() - worker.lastSeen,
})),
queue: queue.length,
});
}
return new Response("not found", { status: 404 });
},
});
console.log(`render-farm coordinator on :${PORT} (${PUBLIC_URL})`);
// Pick up exports a previous coordinator process left unfinished.
resumeJobs().catch((error) => console.error(`resume failed: ${error}`));
export type { Job, TaskState };