Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(renderer-service): CLI mode and cache eviction (phase 7)

okanaslc6341a0137bf
diff --git a/apps/renderer-service/PLAN.md b/apps/renderer-service/PLAN.md
index 7e951aab..c0469773 100644
--- a/apps/renderer-service/PLAN.md
+++ b/apps/renderer-service/PLAN.md
@@ -277,10 +277,12 @@ Why the files are split this way:
### 6.4 Cache eviction
-`FF_CACHE_MAX_GB` and `FF_WORK_MAX_GB` set the size limits. Eviction runs before each job, and again whenever a preflight needs space:
+`FF_CACHE_MAX_GB` and `FF_WORK_MAX_GB` set the size limits (0 = no limit). Eviction runs between jobs, at most every 5 minutes, and again whenever the LFS preflight needs space. Mirrors and workspaces record their last use in a `.ff-last-used` marker.
-- Least-recently-leased workspaces are evicted first, then LFS objects by atime, then whole mirrors.
-- A workspace that is currently leased is never evicted.
+- In `FF_WORK_DIR`: the least recently used workspaces are evicted. Failed-job dirs kept by `FF_KEEP_FAILED_JOBS` are deleted after a day.
+- In `FF_CACHE_DIR`, in this order: LFS objects by last access, then git mirrors by last use, then whole package caches (largest first). Pinned vision models are never evicted.
+- The next job's own workspace and mirror are never evicted.
+- Another worker's mirror on a shared volume is only evicted once it has been idle for over an hour.
### 6.5 Git hosts
@@ -530,10 +532,12 @@ Each phase is its own PR-sized step with tests.
- `console.*` from repo code is not yet captured into a per-job log.
6. **Outputs.** ✅ Done: verification (size, SHA-256, ffprobe frame count ±1), the QA `failOn` gate (codes or severities), and the s3 (multipart via `@aws-sdk/lib-storage`), presigned, local (atomic) and inline sinks. Transient errors are retried; auth and expiry errors fail fast (`UPLOAD_AUTH_FAILED`, `UPLOAD_URL_EXPIRED`). The idempotency manifest is in place.
- Verified end to end over HTTP on this Mac: a local git repo was checked out, the install skipped, rendered on the GPU in-process, verified and delivered in 1.2 s cold. A warm rerun after a push took 0.7 s (243 ms render). Also verified: SSE progress, cancel mid-render, a bad ref, and an inline QA report.
-7. **Transports.**
+7. **Transports.** ✅ Done:
- HTTP v2 with SSE progress and cancel.
- - New `runpod_worker.py` (localhost client, `progress_update`, error passthrough with `retryable`, `refresh_worker` on recycle).
- - CLI `--job-file` and `--result-file`.
+ - `runpod_worker.py`: a localhost client with `progress_update`, error passthrough including `retryable`, and `refresh_worker` on recycle.
+ - CLI `--job-file` / `--job -` with `--result-file`. Exit codes: 0 completed, 1 failed, 2 bad invocation or job.
+ - Server and CLI share `worker/boot.ts`.
+ - Also done here: cache eviction (§6.4, `worker/janitor.ts`). It runs between jobs (at most every 5 min), and on demand when the LFS disk preflight finds too little space.
8. **Image + template + compose + docs.**
- Release workflow.
- GPU smoke test: render the fixture repo on a RunPod pod. Run it by hand first, then as a nightly job.
diff --git a/apps/renderer-service/package.json b/apps/renderer-service/package.json
index 96b73554..294848e1 100644
--- a/apps/renderer-service/package.json
+++ b/apps/renderer-service/package.json
@@ -14,6 +14,8 @@
"dev": "tsx watch --tsconfig tsconfig.dev.json --conditions=development src/index.ts",
"typecheck": "tsc -b tsconfig.json",
"build": "tsc -b tsconfig.json",
+ "start": "node dist/src/index.js",
+ "render": "node dist/src/cli.js",
"test": "vitest run"
},
"dependencies": {
diff --git a/apps/renderer-service/src/cli.ts b/apps/renderer-service/src/cli.ts
new file mode 100644
index 00000000..3d4238b6
--- /dev/null
+++ b/apps/renderer-service/src/cli.ts
@@ -0,0 +1,90 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+import { RenderError } from "./errors.js";
+import { isTerminal, type JobRecord } from "./jobs/store.js";
+import { bootWorker, exit } from "./worker/boot.js";
+
+/**
+ * One-shot render (FF_MODE=cli, PLAN.md §5):
+ * node dist/src/cli.js --job-file job.json [--result-file result.json]
+ * cat job.json | node dist/src/cli.js --job -
+ * Logs go to stdout as JSON lines; the job record is written to
+ * --result-file, or printed last on stdout as `{"ffResult": …}`.
+ * Exit codes: 0 completed, 1 failed/cancelled, 2 bad invocation or job.
+ */
+const USAGE =
+ "usage: cli.js (--job-file <path> | --job -) [--result-file <path>]";
+
+function readJob(): unknown {
+ const { values } = parseArgs({
+ options: {
+ "job-file": { type: "string" },
+ job: { type: "string" },
+ "result-file": { type: "string" },
+ },
+ });
+ const source =
+ values["job-file"] ?? (values.job === "-" ? "/dev/stdin" : undefined);
+ if (!source) {
+ console.error(USAGE);
+ exit(2);
+ }
+ try {
+ return JSON.parse(readFileSync(source as string, "utf8"));
+ } catch (err) {
+ console.error(
+ `Could not read the job from ${source}: ${err instanceof Error ? err.message : err}`,
+ );
+ return exit(2);
+ }
+}
+
+function report(record: JobRecord | { status: "FAILED"; error: unknown }) {
+ const { values } = parseArgs({
+ options: { "result-file": { type: "string" } },
+ strict: false,
+ });
+ const file = values["result-file"];
+ if (typeof file === "string")
+ writeFileSync(file, JSON.stringify(record, null, 2));
+ else process.stdout.write(`${JSON.stringify({ ffResult: record })}\n`);
+}
+
+process.env.FF_MODE ??= "cli";
+const raw = readJob();
+const worker = bootWorker();
+
+let interrupted = false;
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+ process.on(signal, () => {
+ if (interrupted) exit(130);
+ interrupted = true;
+ void worker.drain(
+ new RenderError("CANCELLED", `Received ${signal}.`),
+ true,
+ );
+ });
+}
+
+try {
+ await worker.initGpu();
+ const record = worker.runner.submit(raw);
+ const done = await new Promise<JobRecord>((resolve) => {
+ const check = (r: JobRecord) => {
+ if (r.id === record.id && isTerminal(r.status)) resolve(r);
+ };
+ worker.runner.store.on("update", check);
+ check(worker.runner.store.get(record.id) ?? record);
+ });
+ report(done);
+ worker.stop();
+ exit(done.status === "COMPLETED" ? 0 : 1);
+} catch (err) {
+ const error =
+ err instanceof RenderError
+ ? err.toJSON()
+ : { code: "INTERNAL", message: String(err) };
+ report({ status: "FAILED", error });
+ worker.stop();
+ exit(err instanceof RenderError && err.code === "JOB_INVALID" ? 2 : 1);
+}
diff --git a/apps/renderer-service/src/git/checkout.ts b/apps/renderer-service/src/git/checkout.ts
index 4b2bafd4..40c719b1 100644
--- a/apps/renderer-service/src/git/checkout.ts
+++ b/apps/renderer-service/src/git/checkout.ts
@@ -3,6 +3,7 @@ import path from "node:path";
import { RenderError } from "../errors.js";
import type { CheckoutResult, Pipeline } from "../jobs/pipeline.js";
import { resolveInside } from "../paths.js";
+import { activeJanitor, markUsed } from "../worker/janitor.js";
import { openGitSession } from "./auth.js";
import { hostProfile } from "./hosts.js";
import { findPointers, pullLfs } from "./lfs.js";
@@ -10,6 +11,7 @@ import {
type GitContext,
git,
listRemote,
+ repoHash,
resolveRef,
syncMirror,
} from "./mirror.js";
@@ -36,6 +38,7 @@ export const checkout: Pipeline["checkout"] = async (job, ctx) => {
ctx.progress({ phase: "fetching" });
const mirror = await syncMirror(gctx, repo, spec);
+ await markUsed(mirror.path);
if (source.commit && !mirror.sha.startsWith(source.commit.toLowerCase())) {
throw new RenderError(
"COMMIT_MISMATCH",
@@ -50,6 +53,7 @@ export const checkout: Pipeline["checkout"] = async (job, ctx) => {
const ws = await checkoutWorkspace(gctx, repo, mirror.path, mirror.sha, {
sparse: sparsePaths(source.sparse, source.workdir),
});
+ await markUsed(path.dirname(ws.root));
if (source.submodules) await initSubmodules(gctx, ws.root, repo);
const workdir = await resolveInside(ws.root, source.workdir).catch(
@@ -76,6 +80,9 @@ export const checkout: Pipeline["checkout"] = async (job, ctx) => {
include,
exclude: source.lfs.exclude,
scope: include.length ? [] : scope,
+ makeRoom: async () => {
+ await activeJanitor()?.enforce([repoHash(repo)]);
+ },
});
const leftover = await findPointers(ws.root, source.workdir);
if (leftover.length > 0) {
diff --git a/apps/renderer-service/src/git/lfs.ts b/apps/renderer-service/src/git/lfs.ts
index 4bf0f56f..3c8d7399 100644
--- a/apps/renderer-service/src/git/lfs.ts
+++ b/apps/renderer-service/src/git/lfs.ts
@@ -13,6 +13,8 @@ export interface LfsOptions {
exclude: string[];
/** Directories that must be fully usable (workdir, sparse paths). */
scope: string[];
+ /** Frees disk (cache eviction) when the preflight finds too little. */
+ makeRoom?: () => Promise<void>;
}
export interface LfsResult {
@@ -115,8 +117,21 @@ export async function pullLfs(
const needed = missing.reduce((sum, f) => sum + (f.size ?? 0), 0);
const total = files.reduce((sum, f) => sum + (f.size ?? 0), 0);
// Objects land in the store, then get copied into the checkout.
- await assertFreeSpace(storage, needed);
- await assertFreeSpace(root, total);
+ const fits = async () => {
+ await assertFreeSpace(storage, needed);
+ await assertFreeSpace(root, total);
+ };
+ try {
+ await fits();
+ } catch (err) {
+ if (
+ !(err instanceof RenderError && err.code === "DISK_FULL") ||
+ !options.makeRoom
+ )
+ throw err;
+ await options.makeRoom();
+ await fits();
+ }
const attempts = 3;
for (let attempt = 1; ; attempt++) {
diff --git a/apps/renderer-service/src/git/mirror.ts b/apps/renderer-service/src/git/mirror.ts
index 92f53731..ea466104 100644
--- a/apps/renderer-service/src/git/mirror.ts
+++ b/apps/renderer-service/src/git/mirror.ts
@@ -43,11 +43,12 @@ export function repoHash(repo: RepoRef): string {
}
/** Mirrors are per worker: a shared volume isn't safe for concurrent git writers (§6.1). */
+export function workerId(): string {
+ return (process.env.RUNPOD_POD_ID || os.hostname()).replace(/[^\w.-]/g, "_");
+}
+
export function mirrorPath(config: RendererConfig, repo: RepoRef): string {
- const worker = (process.env.RUNPOD_POD_ID || os.hostname()).replace(
- /[^\w.-]/g,
- "_",
- );
+ const worker = workerId();
return path.join(
config.paths.cacheDir,
"git",
diff --git a/apps/renderer-service/src/index.ts b/apps/renderer-service/src/index.ts
index 4133085b..8d74d4e5 100644
--- a/apps/renderer-service/src/index.ts
+++ b/apps/renderer-service/src/index.ts
@@ -1,104 +1,29 @@
-import { mkdirSync, realpathSync } from "node:fs";
import { Server as HttpServer } from "node:http";
-import path from "node:path";
-import { HeadlessMediaRenderer } from "@framefields/renderer";
-import "@framefields/renderer/offscreen-gl-polyfill";
import { rendererLogger } from "@framefields/server-utils";
import { serve } from "@hono/node-server";
-import { describeConfig, loadConfig, scrubSecretEnv } from "./config/index.js";
import { RenderError } from "./errors.js";
-import { checkout } from "./git/checkout.js";
import { createApp } from "./http/app.js";
-import { install } from "./install/installer.js";
-import { CrashJournal } from "./jobs/journal.js";
-import type { Pipeline } from "./jobs/pipeline.js";
-import { JobRunner } from "./jobs/runner.js";
-import { deliver, findExisting } from "./output/deliver.js";
-import { ReadinessTracker } from "./readiness.js";
-import { redact } from "./redact.js";
-import { installEngineAlias } from "./render/engine-alias.js";
-import { render } from "./render/render-step.js";
-import { installProcessGuards, strayErrorCount } from "./render/sandbox.js";
+import { bootWorker, exit } from "./worker/boot.js";
import { workerHealth } from "./worker/health.js";
-import { currentStats, recycleReason } from "./worker/recycle.js";
-import { startWatchdog } from "./worker/watchdog.js";
-// The job sandbox swaps process.exit while repo code runs; the worker itself
-// always exits through the real one.
-const exit = process.exit.bind(process);
-
-const config = loadConfig();
-scrubSecretEnv();
-rendererLogger.info(
- { config: describeConfig(config) },
- "Renderer config loaded",
-);
-
-// Workspaces live here; repo code under it gets the engine alias (§7).
-const workspaces = path.join(config.paths.workDir, "ws");
-mkdirSync(workspaces, { recursive: true });
-mkdirSync(config.paths.cacheDir, { recursive: true });
-installEngineAlias([realpathSync(workspaces)]);
-
-installProcessGuards((msg, err) =>
- rendererLogger.error(
- {
- err: redact(
- err instanceof Error ? (err.stack ?? err.message) : String(err),
- ),
- },
- msg,
- ),
-);
-
-const readiness = new ReadinessTracker();
-let draining = false;
-
-const pipeline: Pipeline = {
- checkout,
- install,
- render,
- findExisting,
- deliver,
-};
-
-const journal = new CrashJournal(
- path.join(config.paths.workDir, "inflight.json"),
-);
-let baselineHandles = 0;
-
-const runner = new JobRunner(config, pipeline, {
- journal,
- afterJob: (completed) => {
- const reason = recycleReason(
- currentStats(
- completed,
- baselineHandles,
- strayErrorCount(),
- workerHealth.recycleReason,
- ),
- config,
- );
- if (reason) void recycle(reason);
+/**
+ * HTTP worker (FF_MODE=http or runpod-serverless). On RunPod the Python shim
+ * in front of it asks for a fresh worker (refresh_worker) when /health says
+ * "draining", so a recycling serverless worker just stays drained.
+ */
+const worker = bootWorker({
+ onRecycle: () => {
+ void worker.runner.queue.idle().then(() => {
+ if (config.mode !== "runpod-serverless") shutdown(0);
+ });
},
});
-
-const crashed = journal.recover();
-if (crashed) {
- runner.store.restore(crashed);
- rendererLogger.error(
- { jobId: crashed.id, error: crashed.error },
- "Previous process died during a job",
- );
-}
-const stopWatchdog = startWatchdog({
- blockSec: config.recycle.eventLoopBlockSec,
- journalPath: journal.path,
-});
+const { config, runner, readiness } = worker;
const app = createApp(config, runner, {
health: () => {
const status = readiness.status;
+ const draining = worker.isDraining();
return {
ok: readiness.isReady && !draining,
body: {
@@ -123,7 +48,7 @@ const app = createApp(config, runner, {
},
};
},
- draining: () => draining,
+ draining: () => worker.isDraining(),
});
const server = serve(
@@ -142,81 +67,42 @@ if (server instanceof HttpServer) {
server.keepAliveTimeout = 65_000;
}
-async function bootstrap() {
- try {
- rendererLogger.info("Initializing renderer (GPU, node discovery)...");
- await HeadlessMediaRenderer.initialize();
- readiness.setRendererReady(true);
- readiness.setGpuReady(true);
- baselineHandles = process.getActiveResourcesInfo().length;
- rendererLogger.info("Renderer initialized.");
- } catch (error) {
- const errMsg = error instanceof Error ? error.message : String(error);
- rendererLogger.fatal(
- { err: error },
- "❌ GPU / Renderer initialization failed",
- );
- readiness.setFailure(`GPU_UNAVAILABLE: ${errMsg}`);
- // RunPod should replace a worker without a usable GPU.
- if (config.mode === "runpod-serverless") exit(1);
- }
-}
-
-void bootstrap();
+worker.initGpu().catch(() => {
+ // RunPod should replace a worker without a usable GPU; /health reports it otherwise.
+ if (config.mode === "runpod-serverless") exit(1);
+});
-/**
- * Stops taking jobs and restarts the process once the queue is empty. On
- * RunPod serverless the shim asks for a fresh worker (refresh_worker) when
- * it sees /health report "draining", so the process just stays drained.
- */
-async function recycle(reason: string) {
- if (draining) return;
- draining = true;
- rendererLogger.warn({ reason }, "Recycling worker");
- runner.queue.close(
- new RenderError("SHUTDOWN", `Worker is recycling (${reason}).`),
- false,
- );
- await runner.queue.idle();
- if (config.mode !== "runpod-serverless") {
- stopWatchdog();
- server.close();
- exit(0);
- }
+function shutdown(code: number) {
+ worker.stop();
+ server.close();
+ exit(code);
}
-// ─── Graceful Shutdown ──────────────────────────────────────────────────────────
-
let shuttingDown = false;
async function gracefulShutdown(signal: string) {
if (shuttingDown) return;
shuttingDown = true;
- draining = true;
rendererLogger.info(
{ signal, active: runner.queue.activeId },
"Shutting down...",
);
-
+ const forceExit = setTimeout(
+ () => {
+ rendererLogger.error("Shutdown timed out, forcing exit.");
+ exit(1);
+ },
+ (config.limits.killGraceSec + 5) * 1000,
+ );
// The platform is stopping us: abort the running job so it's reported as
// SHUTDOWN (retryable) instead of vanishing mid-render.
- runner.queue.close(
+ await worker.drain(
new RenderError("SHUTDOWN", `Worker received ${signal}.`),
true,
);
-
- const graceMs = (config.limits.killGraceSec + 5) * 1000;
- const forceExit = setTimeout(() => {
- rendererLogger.error("Shutdown timed out, forcing exit.");
- exit(1);
- }, graceMs);
-
- await runner.queue.idle();
- stopWatchdog();
- server.close();
clearTimeout(forceExit);
rendererLogger.info("Shutdown complete.");
- exit(0);
+ shutdown(0);
}
process.on("SIGTERM", () => void gracefulShutdown("SIGTERM"));
diff --git a/apps/renderer-service/src/jobs/runner.ts b/apps/renderer-service/src/jobs/runner.ts
index ca6ee52e..be4040a2 100644
--- a/apps/renderer-service/src/jobs/runner.ts
+++ b/apps/renderer-service/src/jobs/runner.ts
@@ -25,6 +25,8 @@ export class JobRunner {
private readonly pipeline: Pipeline,
private readonly hooks: {
afterJob?: (completed: number) => void;
+ /** Awaited before checkout, between jobs (e.g. cache eviction). */
+ beforeJob?: (job: ValidatedJob) => Promise<void>;
journal?: CrashJournal;
} = {},
) {
@@ -137,6 +139,7 @@ export class JobRunner {
let succeeded = false;
let settle: (() => unknown) | undefined;
try {
+ await this.hooks.beforeJob?.(job);
await fs.mkdir(path.join(jobDir, "out"), { recursive: true });
const checkout = await timed("checkout", () =>
diff --git a/apps/renderer-service/src/worker/boot.ts b/apps/renderer-service/src/worker/boot.ts
new file mode 100644
index 00000000..75ad579d
--- /dev/null
+++ b/apps/renderer-service/src/worker/boot.ts
@@ -0,0 +1,160 @@
+import { mkdirSync, realpathSync } from "node:fs";
+import path from "node:path";
+import { HeadlessMediaRenderer } from "@framefields/renderer";
+import "@framefields/renderer/offscreen-gl-polyfill";
+import { rendererLogger } from "@framefields/server-utils";
+import {
+ describeConfig,
+ loadConfig,
+ type RendererConfig,
+ scrubSecretEnv,
+} from "../config/index.js";
+import { RenderError } from "../errors.js";
+import { checkout } from "../git/checkout.js";
+import { repoHash, workerId } from "../git/mirror.js";
+import { install } from "../install/installer.js";
+import { CrashJournal } from "../jobs/journal.js";
+import type { Pipeline } from "../jobs/pipeline.js";
+import { JobRunner } from "../jobs/runner.js";
+import { deliver, findExisting } from "../output/deliver.js";
+import { ReadinessTracker } from "../readiness.js";
+import { redact } from "../redact.js";
+import { installEngineAlias } from "../render/engine-alias.js";
+import { render } from "../render/render-step.js";
+import { installProcessGuards, strayErrorCount } from "../render/sandbox.js";
+import { workerHealth } from "./health.js";
+import { CacheJanitor } from "./janitor.js";
+import { currentStats, recycleReason } from "./recycle.js";
+import { startWatchdog } from "./watchdog.js";
+
+/** The job sandbox swaps process.exit while repo code runs; the worker exits through the real one. */
+export const exit = process.exit.bind(process);
+
+export interface Worker {
+ config: RendererConfig;
+ runner: JobRunner;
+ readiness: ReadinessTracker;
+ /** True once the worker stopped taking jobs (recycling or shutting down). */
+ isDraining(): boolean;
+ /** Stops taking jobs and waits for the current one. */
+ drain(reason: RenderError, abortActive: boolean): Promise<void>;
+ /** Initialises the GPU; rejects when no usable device is found. */
+ initGpu(): Promise<void>;
+ stop(): void;
+}
+
+/**
+ * Everything the HTTP server and the CLI share (PLAN.md §5): config, engine
+ * alias, process guards, crash journal, watchdog, cache janitor, recycling.
+ */
+export function bootWorker(
+ options: { onRecycle?: (reason: string) => void } = {},
+): Worker {
+ const config = loadConfig();
+ scrubSecretEnv();
+ rendererLogger.info(
+ { config: describeConfig(config) },
+ "Renderer config loaded",
+ );
+
+ // Workspaces live here; repo code under it gets the engine alias (§7).
+ const workspaces = path.join(config.paths.workDir, "ws");
+ mkdirSync(workspaces, { recursive: true });
+ mkdirSync(config.paths.cacheDir, { recursive: true });
+ installEngineAlias([realpathSync(workspaces)]);
+
+ installProcessGuards((msg, err) =>
+ rendererLogger.error(
+ {
+ err: redact(
+ err instanceof Error ? (err.stack ?? err.message) : String(err),
+ ),
+ },
+ msg,
+ ),
+ );
+
+ const readiness = new ReadinessTracker();
+ const janitor = new CacheJanitor(config, workerId());
+ const journal = new CrashJournal(
+ path.join(config.paths.workDir, "inflight.json"),
+ );
+ const pipeline: Pipeline = {
+ checkout,
+ install,
+ render,
+ findExisting,
+ deliver,
+ };
+ let baselineHandles = 0;
+ let draining = false;
+
+ const runner = new JobRunner(config, pipeline, {
+ journal,
+ beforeJob: (job) => janitor.maybeRun([repoHash(job.repo)]),
+ afterJob: (completed) => {
+ const reason = recycleReason(
+ currentStats(
+ completed,
+ baselineHandles,
+ strayErrorCount(),
+ workerHealth.recycleReason,
+ ),
+ config,
+ );
+ if (reason && !draining) {
+ draining = true;
+ rendererLogger.warn({ reason }, "Recycling worker");
+ runner.queue.close(
+ new RenderError("SHUTDOWN", `Worker is recycling (${reason}).`),
+ false,
+ );
+ options.onRecycle?.(reason);
+ }
+ },
+ });
+
+ const crashed = journal.recover();
+ if (crashed) {
+ runner.store.restore(crashed);
+ rendererLogger.error(
+ { jobId: crashed.id, error: crashed.error },
+ "Previous process died during a job",
+ );
+ }
+ const stopWatchdog = startWatchdog({
+ blockSec: config.recycle.eventLoopBlockSec,
+ journalPath: journal.path,
+ });
+
+ return {
+ config,
+ runner,
+ readiness,
+ isDraining: () => draining,
+ async drain(reason, abortActive) {
+ draining = true;
+ runner.queue.close(reason, abortActive);
+ await runner.queue.idle();
+ },
+ async initGpu() {
+ try {
+ rendererLogger.info("Initializing renderer (GPU, node discovery)...");
+ await HeadlessMediaRenderer.initialize();
+ readiness.setRendererReady(true);
+ readiness.setGpuReady(true);
+ baselineHandles = process.getActiveResourcesInfo().length;
+ rendererLogger.info("Renderer initialized.");
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ rendererLogger.fatal(
+ { err: error },
+ "❌ GPU / Renderer initialization failed",
+ );
+ readiness.setFailure(`GPU_UNAVAILABLE: ${message}`);
+ throw new RenderError("GPU_UNAVAILABLE", message, { cause: error });
+ }
+ },
+ stop: stopWatchdog,
+ };
+}
diff --git a/apps/renderer-service/src/worker/janitor.test.ts b/apps/renderer-service/src/worker/janitor.test.ts
new file mode 100644
index 00000000..6afe6713
--- /dev/null
+++ b/apps/renderer-service/src/worker/janitor.test.ts
@@ -0,0 +1,102 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { testConfig } from "../test-helpers.js";
+import { CacheJanitor, dirSize, LAST_USED } from "./janitor.js";
+
+const MB = 1024 * 1024;
+let tmp: string;
+
+beforeAll(async () => {
+ tmp = await fs.mkdtemp(path.join(os.tmpdir(), "ff-janitor-"));
+});
+afterAll(async () => {
+ await fs.rm(tmp, { recursive: true, force: true });
+});
+
+async function blob(p: string, mb: number, ageMin = 0) {
+ await fs.mkdir(path.dirname(p), { recursive: true });
+ await fs.writeFile(p, Buffer.alloc(mb * MB));
+ const t = new Date(Date.now() - ageMin * 60_000);
+ await fs.utimes(p, t, t);
+}
+
+/** Limits are whole GB, so tests scale sizes via a fractional limit hack: 1 GB = 1024 MB. */
+function janitor(root: string, env: Record<string, string>) {
+ const config = testConfig({
+ FF_WORK_DIR: path.join(root, "work"),
+ FF_CACHE_DIR: path.join(root, "cache"),
+ ...env,
+ });
+ return { j: new CacheJanitor(config, "me", 0), config };
+}
+
+describe("CacheJanitor", () => {
+ it("evicts least-recently-used workspaces, sparing the protected one", async () => {
+ const root = path.join(tmp, "ws-case");
+ const ws = path.join(root, "work/ws");
+ await blob(path.join(ws, "old/0/big.bin"), 600);
+ await blob(path.join(ws, "old", LAST_USED), 0, 120);
+ await blob(path.join(ws, "mid/0/big.bin"), 300);
+ await blob(path.join(ws, "mid", LAST_USED), 0, 60);
+ await blob(path.join(ws, "protected/0/big.bin"), 600);
+ await blob(path.join(ws, "protected", LAST_USED), 0, 500);
+ const { j } = janitor(root, { FF_WORK_MAX_GB: "1", FF_CACHE_MAX_GB: "0" });
+
+ const removed = await j.enforce(["protected"]);
+ expect(removed.map((p) => path.basename(p))).toEqual(["old"]);
+ expect(await dirSize(ws)).toBeLessThanOrEqual(1024 * MB);
+ });
+
+ it("evicts LFS objects before mirrors, and leaves other workers' busy mirrors", async () => {
+ const root = path.join(tmp, "cache-case");
+ const cache = path.join(root, "cache");
+ await blob(path.join(cache, "lfs/objects/aa/bb/old-oid"), 500, 300);
+ await blob(path.join(cache, "lfs/objects/cc/dd/new-oid"), 100, 1);
+ await blob(path.join(cache, "git/me/repo1.git/pack.bin"), 300);
+ await blob(path.join(cache, "git/me/repo1.git", LAST_USED), 0, 200);
+ await blob(path.join(cache, "git/other/repo2.git/pack.bin"), 300);
+ await blob(path.join(cache, "git/other/repo2.git", LAST_USED), 0, 5);
+ await blob(path.join(cache, "models/vision.onnx"), 100);
+ const { j } = janitor(root, { FF_CACHE_MAX_GB: "1", FF_WORK_MAX_GB: "0" });
+
+ // 1300 MB total, limit 1024: the old LFS object (500) alone is enough.
+ const removed = await j.enforce([]);
+ expect(removed.map((p) => path.basename(p))).toEqual(["old-oid"]);
+
+ // Over again: the oldest LFS object covers it on its own.
+ await blob(path.join(cache, "lfs/objects/ee/ff/more"), 700, 2);
+ expect((await j.enforce([])).map((p) => path.basename(p))).toEqual([
+ "more",
+ ]);
+
+ // Far over: LFS first, then our own idle mirror, then the package store.
+ // The other worker's recently used mirror and the models stay.
+ await blob(path.join(cache, "pnpm-store/v10/blob"), 1200);
+ const again = await j.enforce([]);
+ expect(again.map((p) => path.basename(p))).toEqual([
+ "new-oid",
+ "repo1.git",
+ "pnpm-store",
+ ]);
+ await expect(
+ fs.stat(path.join(cache, "git/other/repo2.git")),
+ ).resolves.toBeTruthy();
+ await expect(
+ fs.stat(path.join(cache, "models/vision.onnx")),
+ ).resolves.toBeTruthy();
+ });
+
+ it("drops failed-job dirs older than a day", async () => {
+ const root = path.join(tmp, "jobs-case");
+ const old = path.join(root, "work/jobs/old-job");
+ await blob(path.join(old, "out/x.mp4"), 1);
+ const t = new Date(Date.now() - 2 * 24 * 3600_000);
+ await fs.utimes(old, t, t);
+ await blob(path.join(root, "work/jobs/new-job/out/x.mp4"), 1);
+ const { j } = janitor(root, {});
+ const removed = await j.enforce([]);
+ expect(removed.map((p) => path.basename(p))).toEqual(["old-job"]);
+ });
+});
diff --git a/apps/renderer-service/src/worker/janitor.ts b/apps/renderer-service/src/worker/janitor.ts
new file mode 100644
index 00000000..c32facb9
--- /dev/null
+++ b/apps/renderer-service/src/worker/janitor.ts
@@ -0,0 +1,255 @@
+import { existsSync } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { rendererLogger } from "@framefields/server-utils";
+import type { RendererConfig } from "../config/index.js";
+
+/** Marker whose mtime records when a mirror or workspace was last used. */
+export const LAST_USED = ".ff-last-used";
+
+const GB = 1024 ** 3;
+const DAY_MS = 24 * 60 * 60 * 1000;
+/** Mirrors owned by other workers on a shared volume are only evicted when idle this long. */
+const FOREIGN_MIRROR_IDLE_MS = 60 * 60 * 1000;
+/** Package caches, evicted whole and only as a last resort. */
+const STORES = [
+ "pnpm-store",
+ "npm-cache",
+ "yarn-cache",
+ "yarn-berry",
+ "corepack",
+];
+
+let current: CacheJanitor | undefined;
+
+/** The worker's janitor, for emergency eviction (e.g. the LFS disk preflight). */
+export function activeJanitor(): CacheJanitor | undefined {
+ return current;
+}
+
+export async function markUsed(dir: string): Promise<void> {
+ const marker = path.join(dir, LAST_USED);
+ const now = new Date();
+ await fs.utimes(marker, now, now).catch(() => fs.writeFile(marker, ""));
+}
+
+interface Candidate {
+ path: string;
+ bytes: number;
+ lastUsed: number;
+ kind: string;
+}
+
+/**
+ * Keeps FF_WORK_DIR and FF_CACHE_DIR under their size limits (PLAN.md §6.4).
+ * Runs between jobs, so nothing it deletes is in use. Eviction order:
+ * workspaces (LRU), then LFS objects (LRU), then git mirrors (LRU), then
+ * whole package caches. Pinned vision models are never evicted.
+ */
+export class CacheJanitor {
+ private lastRun = 0;
+
+ constructor(
+ private readonly config: RendererConfig,
+ private readonly workerId: string,
+ private readonly intervalMs = 5 * 60 * 1000,
+ ) {
+ current = this;
+ }
+
+ /** Runs at most once per interval unless forced. Never throws. */
+ async maybeRun(protect: readonly string[], force = false): Promise<void> {
+ if (!force && Date.now() - this.lastRun < this.intervalMs) return;
+ this.lastRun = Date.now();
+ try {
+ const removed = await this.enforce(protect);
+ if (removed.length) {
+ rendererLogger.info(
+ { removed: removed.length, paths: removed.slice(0, 20) },
+ "Cache eviction",
+ );
+ }
+ } catch (err) {
+ rendererLogger.warn({ err: String(err) }, "Cache eviction failed");
+ }
+ }
+
+ /**
+ * Evicts until both limits hold. `protect` lists repo hashes the next job
+ * needs (its workspace and mirror stay).
+ */
+ async enforce(protect: readonly string[]): Promise<string[]> {
+ const removed: string[] = [];
+ const { workDir, cacheDir, workMaxGb, cacheMaxGb } = this.config.paths;
+
+ // Failed-job dirs kept for debugging expire after a day.
+ for (const dir of await children(path.join(workDir, "jobs"))) {
+ if (Date.now() - (await mtime(dir)) > DAY_MS) {
+ await remove(dir, removed);
+ }
+ }
+
+ if (workMaxGb > 0) {
+ const workspaces = (await children(path.join(workDir, "ws"))).filter(
+ (d) => !protect.includes(path.basename(d)),
+ );
+ const candidates = await Promise.all(
+ workspaces.map(async (d) => ({
+ path: d,
+ bytes: await dirSize(d),
+ lastUsed: await mtime(path.join(d, LAST_USED), d),
+ kind: "workspace",
+ })),
+ );
+ const total =
+ (await dirSize(path.join(workDir, "ws"))) +
+ (await dirSize(path.join(workDir, "jobs")));
+ await evict(candidates, total - workMaxGb * GB, removed);
+ }
+
+ if (cacheMaxGb > 0) {
+ let total = await dirSize(cacheDir);
+ const over = () => total - cacheMaxGb * GB;
+ if (over() > 0) {
+ const lfs = await lfsObjects(path.join(cacheDir, "lfs"));
+ total -= await evict(lfs, over(), removed);
+ }
+ if (over() > 0) {
+ total -= await evict(await this.mirrors(protect), over(), removed);
+ }
+ if (over() > 0) {
+ const stores = await Promise.all(
+ STORES.map(async (s) => {
+ const p = path.join(cacheDir, s);
+ return {
+ path: p,
+ bytes: await dirSize(p),
+ lastUsed: 0,
+ kind: "store",
+ };
+ }),
+ );
+ // Largest first: one eviction usually suffices.
+ stores.sort((a, b) => b.bytes - a.bytes);
+ total -= await evict(
+ stores.filter((s) => s.bytes > 0),
+ over(),
+ removed,
+ false,
+ );
+ }
+ }
+ return removed;
+ }
+
+ private async mirrors(protect: readonly string[]): Promise<Candidate[]> {
+ const out: Candidate[] = [];
+ for (const workerDir of await children(
+ path.join(this.config.paths.cacheDir, "git"),
+ )) {
+ const own = path.basename(workerDir) === this.workerId;
+ for (const mirror of await children(workerDir)) {
+ const hash = path.basename(mirror).replace(/\.git$/, "");
+ if (own && protect.includes(hash)) continue;
+ const lastUsed = await mtime(path.join(mirror, LAST_USED), mirror);
+ // Another worker may be using its mirror right now.
+ if (!own && Date.now() - lastUsed < FOREIGN_MIRROR_IDLE_MS) continue;
+ out.push({
+ path: mirror,
+ bytes: await dirSize(mirror),
+ lastUsed,
+ kind: "mirror",
+ });
+ }
+ }
+ return out;
+ }
+}
+
+/** Removes candidates (oldest first unless `sort` is false) until `bytes` are freed. */
+async function evict(
+ candidates: Candidate[],
+ bytes: number,
+ removed: string[],
+ sort = true,
+): Promise<number> {
+ if (bytes <= 0) return 0;
+ if (sort) candidates.sort((a, b) => a.lastUsed - b.lastUsed);
+ let freed = 0;
+ for (const c of candidates) {
+ if (freed >= bytes) break;
+ await remove(c.path, removed);
+ freed += c.bytes;
+ }
+ return freed;
+}
+
+async function remove(p: string, removed: string[]): Promise<void> {
+ await fs.rm(p, { recursive: true, force: true });
+ removed.push(p);
+}
+
+async function children(dir: string): Promise<string[]> {
+ const entries = await fs
+ .readdir(dir, { withFileTypes: true })
+ .catch(() => []);
+ return entries
+ .filter((e) => e.isDirectory())
+ .map((e) => path.join(dir, e.name));
+}
+
+async function mtime(p: string, fallback?: string): Promise<number> {
+ const stat = await fs
+ .stat(p)
+ .catch(() =>
+ fallback ? fs.stat(fallback).catch(() => undefined) : undefined,
+ );
+ return stat?.mtimeMs ?? 0;
+}
+
+/** git-lfs stores objects at lfs/objects/aa/bb/<oid>; LRU by access time. */
+async function lfsObjects(root: string): Promise<Candidate[]> {
+ const out: Candidate[] = [];
+ const objects = path.join(root, "objects");
+ if (!existsSync(objects)) return out;
+ for (const a of await children(objects)) {
+ for (const b of await children(a)) {
+ for (const name of await fs.readdir(b).catch(() => [])) {
+ const p = path.join(b, name);
+ const stat = await fs.stat(p).catch(() => undefined);
+ if (stat?.isFile()) {
+ out.push({
+ path: p,
+ bytes: stat.size,
+ lastUsed: Math.max(stat.atimeMs, stat.mtimeMs),
+ kind: "lfs",
+ });
+ }
+ }
+ }
+ }
+ return out;
+}
+
+/** Bytes on disk under `dir` (symlinks not followed). */
+export async function dirSize(dir: string): Promise<number> {
+ let total = 0;
+ const walk = async (d: string): Promise<void> => {
+ const entries = await fs
+ .readdir(d, { withFileTypes: true })
+ .catch(() => []);
+ await Promise.all(
+ entries.map(async (e) => {
+ const p = path.join(d, e.name);
+ if (e.isDirectory()) await walk(p);
+ else if (e.isFile()) {
+ // Read first: `total += await …` would add to a stale total.
+ const size = (await fs.lstat(p).catch(() => undefined))?.size ?? 0;
+ total += size;
+ }
+ }),
+ );
+ };
+ await walk(dir);
+ return total;
+}
diff --git a/apps/renderer-service/vitest.config.ts b/apps/renderer-service/vitest.config.ts
index d84aabcb..64e5fe46 100644
--- a/apps/renderer-service/vitest.config.ts
+++ b/apps/renderer-service/vitest.config.ts
@@ -13,7 +13,7 @@ export default defineConfig({
alias: [
{
find: "@",
- replacement: path.resolve(__dirname, "./src"),
+ replacement: path.resolve(import.meta.dirname, "./src"),
},
],
},