Skip to content

Code-first video & image, rendered natively on WebGPU.

8.7 KB
Raw Blame
import {
	accessSync,
	constants,
	mkdirSync,
	readdirSync,
	realpathSync,
	rmSync,
} from "node:fs";
import path from "node:path";
import {
	describeConfig,
	loadConfig,
	type RendererConfig,
	scrubSecretEnv,
} from "../config/index.js";
import { RenderError } from "../errors.js";
import { checkout } from "../git/checkout.js";
import { exec } from "../git/exec.js";
import { repoHash, workerId } from "../git/mirror.js";
import { install } from "../install/installer.js";
import { CrashJournal } from "../jobs/journal.js";
import type { Pipeline } from "../jobs/pipeline.js";
import { JobRunner } from "../jobs/runner.js";
import { rendererLogger } from "../logger.js";
import { deliver, findExisting } from "../output/deliver.js";
import { ReadinessTracker } from "../readiness.js";
import { redact } from "../redact.js";
import { boundEngine } from "../render/engine.js";
import { installEngineAlias } from "../render/engine-alias.js";
import { render } from "../render/render-step.js";
import { installProcessGuards, strayErrorCount } from "../render/sandbox.js";
import { applyGpuPlan, gpuStatus, planGpu, probeGpu } from "./gpu.js";
import { workerHealth } from "./health.js";
import { CacheJanitor, type DiskSummary } from "./janitor.js";
import { currentStats, recycleReason } from "./recycle.js";
import { startWatchdog } from "./watchdog.js";

/** The job sandbox swaps process.exit while repo code runs; the worker exits through the real one. */
export const exit = process.exit.bind(process);

export interface Worker {
	config: RendererConfig;
	runner: JobRunner;
	readiness: ReadinessTracker;
	/** True once the worker stopped taking jobs (recycling or shutting down). */
	isDraining(): boolean;
	/** Stops taking jobs and waits for the current one. */
	drain(reason: RenderError, abortActive: boolean): Promise<void>;
	/** Disk space of the work and cache dirs (cached; see CacheJanitor.disks). */
	disks(): Record<string, DiskSummary> | undefined;
	/** Checks git, git-lfs, ffprobe and the GPU; rejects with TOOL_MISSING or GPU_UNAVAILABLE. */
	checkTools(): Promise<void>;
	stop(): void;
}

/**
 * Everything the HTTP server and the CLI share (PLAN.md §5): config, engine
 * alias, process guards, crash journal, watchdog, cache janitor, recycling.
 */
export function bootWorker(
	options: { onRecycle?: (reason: string) => void } = {},
): Worker {
	const config = loadConfig();
	scrubSecretEnv();
	rendererLogger.info(
		{ config: describeConfig(config) },
		"Renderer config loaded",
	);

	// Before any engine loads: Vulkan reads its driver list once (§8.7).
	const gpu = planGpu(config.gpu, probeGpu());
	applyGpuPlan(gpu);
	rendererLogger[gpu.problem ? "error" : "info"](
		{ gpu: { ...gpuStatus(), icds: gpu.icds }, problem: gpu.problem },
		gpu.expect === "software"
			? "No GPU in use: rendering on the CPU"
			: "Rendering on the GPU",
	);

	// Temp files (the engine renders video there before copying it out,
	// installs put their HOME there) live under the work dir, where the
	// janitor sees their disk use and a crash's leftovers get cleared.
	// os.tmpdir() reads TMPDIR on every call.
	const tmpDir = path.join(config.paths.workDir, "tmp");
	writableDir(config.paths.workDir, "FF_WORK_DIR");
	writableDir(config.paths.cacheDir, "FF_CACHE_DIR");
	rmSync(tmpDir, { recursive: true, force: true });
	mkdirSync(tmpDir, { recursive: true });
	process.env.TMPDIR = tmpDir;
	// Job dirs left by a process that died mid-job; the crash journal reports
	// the job, so only kept failures (FF_KEEP_FAILED_JOBS) are worth keeping.
	if (!config.keepFailedJobs) {
		rmSync(path.join(config.paths.workDir, "jobs"), {
			recursive: true,
			force: true,
		});
	}

	// Workspaces live here; repo code under it gets the engine alias (§7).
	const workspaces = path.join(config.paths.workDir, "ws");
	mkdirSync(workspaces, { recursive: true });
	mkdirSync(config.paths.cacheDir, { recursive: true });
	installEngineAlias([realpathSync(workspaces)]);

	installProcessGuards((msg, err) =>
		rendererLogger.error(
			{
				err: redact(
					err instanceof Error ? (err.stack ?? err.message) : String(err),
				),
			},
			msg,
		),
	);

	const readiness = new ReadinessTracker();
	// The bound engine is loaded from a workspace: never evict that one.
	const engineRepo = () => {
		const engine = boundEngine();
		return engine ? [path.basename(path.dirname(engine.workspace))] : [];
	};
	const janitor = new CacheJanitor(
		config,
		workerId(),
		undefined,
		undefined,
		engineRepo,
	);
	const journal = new CrashJournal(
		path.join(config.paths.workDir, "inflight.json"),
	);
	const pipeline: Pipeline = {
		checkout,
		install,
		render,
		findExisting,
		deliver,
	};
	let baselineHandles = 0;
	let boundVersion: string | undefined;
	let draining = false;

	const runner = new JobRunner(config, pipeline, {
		journal,
		beforeJob: (job) => janitor.maybeRun([repoHash(job.repo)]),
		afterJob: (completed) => {
			emptyTmp(tmpDir);
			const engine = boundEngine();
			if (engine && boundVersion !== engine.version) {
				// The engine's own timers and GPU handles aren't leaks.
				boundVersion = engine.version;
				readiness.setEngine(engine.version);
				baselineHandles = process.getActiveResourcesInfo().length;
			}
			const reason = recycleReason(
				currentStats(
					completed,
					baselineHandles,
					strayErrorCount(),
					workerHealth.recycleReason,
				),
				config,
			);
			if (reason && !draining) {
				draining = true;
				rendererLogger.warn({ reason }, "Recycling worker");
				runner.queue.close(
					new RenderError("SHUTDOWN", `Worker is recycling (${reason}).`),
					false,
				);
				options.onRecycle?.(reason);
			}
		},
	});

	const crashed = journal.recover();
	if (crashed) {
		runner.store.restore(crashed);
		rendererLogger.error(
			{ jobId: crashed.id, error: crashed.error },
			"Previous process died during a job",
		);
	}
	const stopWatchdog = startWatchdog({
		blockSec: config.recycle.eventLoopBlockSec,
		journalPath: journal.path,
	});

	return {
		config,
		runner,
		readiness,
		isDraining: () => draining,
		disks: () => janitor.disks(),
		async drain(reason, abortActive) {
			draining = true;
			runner.queue.close(reason, abortActive);
			await runner.queue.idle();
		},
		async checkTools() {
			const missing: string[] = [];
			for (const [tool, args] of [
				["git", ["--version"]],
				["git", ["lfs", "version"]],
				["ffprobe", ["-version"]],
			] as const) {
				await exec(tool, [...args], { timeoutMs: 10_000 }).catch(() =>
					missing.push(
						`${tool} ${args[0]}`
							.replace(" --version", "")
							.replace(" -version", ""),
					),
				);
			}
			// ffprobe only sharpens output checks; git and git-lfs are required.
			const required = missing.filter((m) => m.startsWith("git"));
			if (missing.length) rendererLogger.warn({ missing }, "Tools missing");
			if (required.length) {
				readiness.setFailure(`TOOL_MISSING: ${required.join(", ")}`);
				throw new RenderError(
					"TOOL_MISSING",
					`Missing: ${required.join(", ")}`,
				);
			}
			if (gpu.problem) {
				readiness.setFailure(`GPU_UNAVAILABLE: ${gpu.problem}`);
				throw new RenderError("GPU_UNAVAILABLE", gpu.problem, {
					hint: gpu.hint,
				});
			}
			readiness.setToolsReady(true);
			baselineHandles = process.getActiveResourcesInfo().length;
		},
		stop: stopWatchdog,
	};
}

/**
 * Kept between jobs: compile caches (tsx's `tsx-<uid>`, Node's
 * `node-compile-cache`), keyed by content, so the next job doesn't recompile
 * the project; and `gatewai-fonts`, where engine versions up to
 * 2.2.0-beta.3 cache downloaded fonts by URL for the life of the process.
 */
function keepInTmp(name: string): boolean {
	return (
		name === "gatewai-fonts" ||
		name === "node-compile-cache" ||
		name.startsWith("tsx-")
	);
}

/**
 * Creates `dir` and checks this process can write to it, so a volume mounted
 * read-only or owned by another user fails with the setting to fix instead of
 * a bare EACCES stack trace.
 */
function writableDir(dir: string, setting: string): void {
	try {
		mkdirSync(dir, { recursive: true });
		accessSync(dir, constants.W_OK);
	} catch (err) {
		const uid = process.getuid?.();
		console.error(
			`❌ CONFIG_INVALID\n  - ${setting}: ${dir} is not writable${uid === undefined ? "" : ` by uid ${uid}`} (${(err as NodeJS.ErrnoException).code}). Point it at a writable path, or start the image as root so its entrypoint can hand the dir to the framefields user.`,
		);
		exit(1);
	}
}

/** Removes what the job left in the temp dir (jobs run one at a time). */
function emptyTmp(dir: string): void {
	try {
		for (const name of readdirSync(dir)) {
			if (keepInTmp(name)) continue;
			rmSync(path.join(dir, name), { recursive: true, force: true });
		}
	} catch (err) {
		rendererLogger.warn({ dir, err: String(err) }, "Temp dir cleanup failed");
	}
}