Code-first video & image, rendered natively on WebGPU.
feat(renderer-service): CLI mode and cache eviction (phase 7)
okanaslc6341a0137bf
diff --git a/apps/renderer-service/PLAN.md b/apps/renderer-service/PLAN.mdindex 7e951aab..c0469773 100644--- a/apps/renderer-service/PLAN.md+++ b/apps/renderer-service/PLAN.md@@ -277,10 +277,12 @@ Why the files are split this way:### 6.4 Cache eviction-`FF_CACHE_MAX_GB` and `FF_WORK_MAX_GB` set the size limits. Eviction runs before each job, and again whenever a preflight needs space:+`FF_CACHE_MAX_GB` and `FF_WORK_MAX_GB` set the size limits (0 = no limit). Eviction runs between jobs, at most every 5 minutes, and again whenever the LFS preflight needs space. Mirrors and workspaces record their last use in a `.ff-last-used` marker.-- Least-recently-leased workspaces are evicted first, then LFS objects by atime, then whole mirrors.-- A workspace that is currently leased is never evicted.+- In `FF_WORK_DIR`: the least recently used workspaces are evicted. Failed-job dirs kept by `FF_KEEP_FAILED_JOBS` are deleted after a day.+- In `FF_CACHE_DIR`, in this order: LFS objects by last access, then git mirrors by last use, then whole package caches (largest first). Pinned vision models are never evicted.+- The next job's own workspace and mirror are never evicted.+- Another worker's mirror on a shared volume is only evicted once it has been idle for over an hour.### 6.5 Git hosts@@ -530,10 +532,12 @@ Each phase is its own PR-sized step with tests.- `console.*` from repo code is not yet captured into a per-job log.6. **Outputs.** ✅ Done: verification (size, SHA-256, ffprobe frame count ±1), the QA `failOn` gate (codes or severities), and the s3 (multipart via `@aws-sdk/lib-storage`), presigned, local (atomic) and inline sinks. Transient errors are retried; auth and expiry errors fail fast (`UPLOAD_AUTH_FAILED`, `UPLOAD_URL_EXPIRED`). The idempotency manifest is in place.- Verified end to end over HTTP on this Mac: a local git repo was checked out, the install skipped, rendered on the GPU in-process, verified and delivered in 1.2 s cold. A warm rerun after a push took 0.7 s (243 ms render). Also verified: SSE progress, cancel mid-render, a bad ref, and an inline QA report.-7. **Transports.**+7. **Transports.** ✅ Done:- HTTP v2 with SSE progress and cancel.- - New `runpod_worker.py` (localhost client, `progress_update`, error passthrough with `retryable`, `refresh_worker` on recycle).- - CLI `--job-file` and `--result-file`.+ - `runpod_worker.py`: a localhost client with `progress_update`, error passthrough including `retryable`, and `refresh_worker` on recycle.+ - CLI `--job-file` / `--job -` with `--result-file`. Exit codes: 0 completed, 1 failed, 2 bad invocation or job.+ - Server and CLI share `worker/boot.ts`.+ - Also done here: cache eviction (§6.4, `worker/janitor.ts`). It runs between jobs (at most every 5 min), and on demand when the LFS disk preflight finds too little space.8. **Image + template + compose + docs.**- Release workflow.- GPU smoke test: render the fixture repo on a RunPod pod. Run it by hand first, then as a nightly job.diff --git a/apps/renderer-service/package.json b/apps/renderer-service/package.jsonindex 96b73554..294848e1 100644--- a/apps/renderer-service/package.json+++ b/apps/renderer-service/package.json@@ -14,6 +14,8 @@"dev": "tsx watch --tsconfig tsconfig.dev.json --conditions=development src/index.ts","typecheck": "tsc -b tsconfig.json","build": "tsc -b tsconfig.json",+ "start": "node dist/src/index.js",+ "render": "node dist/src/cli.js","test": "vitest run"},"dependencies": {diff --git a/apps/renderer-service/src/cli.ts b/apps/renderer-service/src/cli.tsnew file mode 100644index 00000000..3d4238b6--- /dev/null+++ b/apps/renderer-service/src/cli.ts@@ -0,0 +1,90 @@+import { readFileSync, writeFileSync } from "node:fs";+import { parseArgs } from "node:util";+import { RenderError } from "./errors.js";+import { isTerminal, type JobRecord } from "./jobs/store.js";+import { bootWorker, exit } from "./worker/boot.js";++/**+ * One-shot render (FF_MODE=cli, PLAN.md §5):+ * node dist/src/cli.js --job-file job.json [--result-file result.json]+ * cat job.json | node dist/src/cli.js --job -+ * Logs go to stdout as JSON lines; the job record is written to+ * --result-file, or printed last on stdout as `{"ffResult": …}`.+ * Exit codes: 0 completed, 1 failed/cancelled, 2 bad invocation or job.+ */+const USAGE =+ "usage: cli.js (--job-file <path> | --job -) [--result-file <path>]";++function readJob(): unknown {+ const { values } = parseArgs({+ options: {+ "job-file": { type: "string" },+ job: { type: "string" },+ "result-file": { type: "string" },+ },+ });+ const source =+ values["job-file"] ?? (values.job === "-" ? "/dev/stdin" : undefined);+ if (!source) {+ console.error(USAGE);+ exit(2);+ }+ try {+ return JSON.parse(readFileSync(source as string, "utf8"));+ } catch (err) {+ console.error(+ `Could not read the job from ${source}: ${err instanceof Error ? err.message : err}`,+ );+ return exit(2);+ }+}++function report(record: JobRecord | { status: "FAILED"; error: unknown }) {+ const { values } = parseArgs({+ options: { "result-file": { type: "string" } },+ strict: false,+ });+ const file = values["result-file"];+ if (typeof file === "string")+ writeFileSync(file, JSON.stringify(record, null, 2));+ else process.stdout.write(`${JSON.stringify({ ffResult: record })}\n`);+}++process.env.FF_MODE ??= "cli";+const raw = readJob();+const worker = bootWorker();++let interrupted = false;+for (const signal of ["SIGINT", "SIGTERM"] as const) {+ process.on(signal, () => {+ if (interrupted) exit(130);+ interrupted = true;+ void worker.drain(+ new RenderError("CANCELLED", `Received ${signal}.`),+ true,+ );+ });+}++try {+ await worker.initGpu();+ const record = worker.runner.submit(raw);+ const done = await new Promise<JobRecord>((resolve) => {+ const check = (r: JobRecord) => {+ if (r.id === record.id && isTerminal(r.status)) resolve(r);+ };+ worker.runner.store.on("update", check);+ check(worker.runner.store.get(record.id) ?? record);+ });+ report(done);+ worker.stop();+ exit(done.status === "COMPLETED" ? 0 : 1);+} catch (err) {+ const error =+ err instanceof RenderError+ ? err.toJSON()+ : { code: "INTERNAL", message: String(err) };+ report({ status: "FAILED", error });+ worker.stop();+ exit(err instanceof RenderError && err.code === "JOB_INVALID" ? 2 : 1);+}diff --git a/apps/renderer-service/src/git/checkout.ts b/apps/renderer-service/src/git/checkout.tsindex 4b2bafd4..40c719b1 100644--- a/apps/renderer-service/src/git/checkout.ts+++ b/apps/renderer-service/src/git/checkout.ts@@ -3,6 +3,7 @@ import path from "node:path";import { RenderError } from "../errors.js";import type { CheckoutResult, Pipeline } from "../jobs/pipeline.js";import { resolveInside } from "../paths.js";+import { activeJanitor, markUsed } from "../worker/janitor.js";import { openGitSession } from "./auth.js";import { hostProfile } from "./hosts.js";import { findPointers, pullLfs } from "./lfs.js";@@ -10,6 +11,7 @@ import {type GitContext,git,listRemote,+ repoHash,resolveRef,syncMirror,} from "./mirror.js";@@ -36,6 +38,7 @@ export const checkout: Pipeline["checkout"] = async (job, ctx) => {ctx.progress({ phase: "fetching" });const mirror = await syncMirror(gctx, repo, spec);+ await markUsed(mirror.path);if (source.commit && !mirror.sha.startsWith(source.commit.toLowerCase())) {throw new RenderError("COMMIT_MISMATCH",@@ -50,6 +53,7 @@ export const checkout: Pipeline["checkout"] = async (job, ctx) => {const ws = await checkoutWorkspace(gctx, repo, mirror.path, mirror.sha, {sparse: sparsePaths(source.sparse, source.workdir),});+ await markUsed(path.dirname(ws.root));if (source.submodules) await initSubmodules(gctx, ws.root, repo);const workdir = await resolveInside(ws.root, source.workdir).catch(@@ -76,6 +80,9 @@ export const checkout: Pipeline["checkout"] = async (job, ctx) => {include,exclude: source.lfs.exclude,scope: include.length ? [] : scope,+ makeRoom: async () => {+ await activeJanitor()?.enforce([repoHash(repo)]);+ },});const leftover = await findPointers(ws.root, source.workdir);if (leftover.length > 0) {diff --git a/apps/renderer-service/src/git/lfs.ts b/apps/renderer-service/src/git/lfs.tsindex 4bf0f56f..3c8d7399 100644--- a/apps/renderer-service/src/git/lfs.ts+++ b/apps/renderer-service/src/git/lfs.ts@@ -13,6 +13,8 @@ export interface LfsOptions {exclude: string[];/** Directories that must be fully usable (workdir, sparse paths). */scope: string[];+ /** Frees disk (cache eviction) when the preflight finds too little. */+ makeRoom?: () => Promise<void>;}export interface LfsResult {@@ -115,8 +117,21 @@ export async function pullLfs(const needed = missing.reduce((sum, f) => sum + (f.size ?? 0), 0);const total = files.reduce((sum, f) => sum + (f.size ?? 0), 0);// Objects land in the store, then get copied into the checkout.- await assertFreeSpace(storage, needed);- await assertFreeSpace(root, total);+ const fits = async () => {+ await assertFreeSpace(storage, needed);+ await assertFreeSpace(root, total);+ };+ try {+ await fits();+ } catch (err) {+ if (+ !(err instanceof RenderError && err.code === "DISK_FULL") ||+ !options.makeRoom+ )+ throw err;+ await options.makeRoom();+ await fits();+ }const attempts = 3;for (let attempt = 1; ; attempt++) {diff --git a/apps/renderer-service/src/git/mirror.ts b/apps/renderer-service/src/git/mirror.tsindex 92f53731..ea466104 100644--- a/apps/renderer-service/src/git/mirror.ts+++ b/apps/renderer-service/src/git/mirror.ts@@ -43,11 +43,12 @@ export function repoHash(repo: RepoRef): string {}/** Mirrors are per worker: a shared volume isn't safe for concurrent git writers (§6.1). */+export function workerId(): string {+ return (process.env.RUNPOD_POD_ID || os.hostname()).replace(/[^\w.-]/g, "_");+}+export function mirrorPath(config: RendererConfig, repo: RepoRef): string {- const worker = (process.env.RUNPOD_POD_ID || os.hostname()).replace(- /[^\w.-]/g,- "_",- );+ const worker = workerId();return path.join(config.paths.cacheDir,"git",diff --git a/apps/renderer-service/src/index.ts b/apps/renderer-service/src/index.tsindex 4133085b..8d74d4e5 100644--- a/apps/renderer-service/src/index.ts+++ b/apps/renderer-service/src/index.ts@@ -1,104 +1,29 @@-import { mkdirSync, realpathSync } from "node:fs";import { Server as HttpServer } from "node:http";-import path from "node:path";-import { HeadlessMediaRenderer } from "@framefields/renderer";-import "@framefields/renderer/offscreen-gl-polyfill";import { rendererLogger } from "@framefields/server-utils";import { serve } from "@hono/node-server";-import { describeConfig, loadConfig, scrubSecretEnv } from "./config/index.js";import { RenderError } from "./errors.js";-import { checkout } from "./git/checkout.js";import { createApp } from "./http/app.js";-import { install } from "./install/installer.js";-import { CrashJournal } from "./jobs/journal.js";-import type { Pipeline } from "./jobs/pipeline.js";-import { JobRunner } from "./jobs/runner.js";-import { deliver, findExisting } from "./output/deliver.js";-import { ReadinessTracker } from "./readiness.js";-import { redact } from "./redact.js";-import { installEngineAlias } from "./render/engine-alias.js";-import { render } from "./render/render-step.js";-import { installProcessGuards, strayErrorCount } from "./render/sandbox.js";+import { bootWorker, exit } from "./worker/boot.js";import { workerHealth } from "./worker/health.js";-import { currentStats, recycleReason } from "./worker/recycle.js";-import { startWatchdog } from "./worker/watchdog.js";-// The job sandbox swaps process.exit while repo code runs; the worker itself-// always exits through the real one.-const exit = process.exit.bind(process);--const config = loadConfig();-scrubSecretEnv();-rendererLogger.info(- { config: describeConfig(config) },- "Renderer config loaded",-);--// Workspaces live here; repo code under it gets the engine alias (§7).-const workspaces = path.join(config.paths.workDir, "ws");-mkdirSync(workspaces, { recursive: true });-mkdirSync(config.paths.cacheDir, { recursive: true });-installEngineAlias([realpathSync(workspaces)]);--installProcessGuards((msg, err) =>- rendererLogger.error(- {- err: redact(- err instanceof Error ? (err.stack ?? err.message) : String(err),- ),- },- msg,- ),-);--const readiness = new ReadinessTracker();-let draining = false;--const pipeline: Pipeline = {- checkout,- install,- render,- findExisting,- deliver,-};--const journal = new CrashJournal(- path.join(config.paths.workDir, "inflight.json"),-);-let baselineHandles = 0;--const runner = new JobRunner(config, pipeline, {- journal,- afterJob: (completed) => {- const reason = recycleReason(- currentStats(- completed,- baselineHandles,- strayErrorCount(),- workerHealth.recycleReason,- ),- config,- );- if (reason) void recycle(reason);+/**+ * HTTP worker (FF_MODE=http or runpod-serverless). On RunPod the Python shim+ * in front of it asks for a fresh worker (refresh_worker) when /health says+ * "draining", so a recycling serverless worker just stays drained.+ */+const worker = bootWorker({+ onRecycle: () => {+ void worker.runner.queue.idle().then(() => {+ if (config.mode !== "runpod-serverless") shutdown(0);+ });},});--const crashed = journal.recover();-if (crashed) {- runner.store.restore(crashed);- rendererLogger.error(- { jobId: crashed.id, error: crashed.error },- "Previous process died during a job",- );-}-const stopWatchdog = startWatchdog({- blockSec: config.recycle.eventLoopBlockSec,- journalPath: journal.path,-});+const { config, runner, readiness } = worker;const app = createApp(config, runner, {health: () => {const status = readiness.status;+ const draining = worker.isDraining();return {ok: readiness.isReady && !draining,body: {@@ -123,7 +48,7 @@ const app = createApp(config, runner, {},};},- draining: () => draining,+ draining: () => worker.isDraining(),});const server = serve(@@ -142,81 +67,42 @@ if (server instanceof HttpServer) {server.keepAliveTimeout = 65_000;}-async function bootstrap() {- try {- rendererLogger.info("Initializing renderer (GPU, node discovery)...");- await HeadlessMediaRenderer.initialize();- readiness.setRendererReady(true);- readiness.setGpuReady(true);- baselineHandles = process.getActiveResourcesInfo().length;- rendererLogger.info("Renderer initialized.");- } catch (error) {- const errMsg = error instanceof Error ? error.message : String(error);- rendererLogger.fatal(- { err: error },- "❌ GPU / Renderer initialization failed",- );- readiness.setFailure(`GPU_UNAVAILABLE: ${errMsg}`);- // RunPod should replace a worker without a usable GPU.- if (config.mode === "runpod-serverless") exit(1);- }-}--void bootstrap();+worker.initGpu().catch(() => {+ // RunPod should replace a worker without a usable GPU; /health reports it otherwise.+ if (config.mode === "runpod-serverless") exit(1);+});-/**- * Stops taking jobs and restarts the process once the queue is empty. On- * RunPod serverless the shim asks for a fresh worker (refresh_worker) when- * it sees /health report "draining", so the process just stays drained.- */-async function recycle(reason: string) {- if (draining) return;- draining = true;- rendererLogger.warn({ reason }, "Recycling worker");- runner.queue.close(- new RenderError("SHUTDOWN", `Worker is recycling (${reason}).`),- false,- );- await runner.queue.idle();- if (config.mode !== "runpod-serverless") {- stopWatchdog();- server.close();- exit(0);- }+function shutdown(code: number) {+ worker.stop();+ server.close();+ exit(code);}-// ─── Graceful Shutdown ──────────────────────────────────────────────────────────-let shuttingDown = false;async function gracefulShutdown(signal: string) {if (shuttingDown) return;shuttingDown = true;- draining = true;rendererLogger.info({ signal, active: runner.queue.activeId },"Shutting down...",);-+ const forceExit = setTimeout(+ () => {+ rendererLogger.error("Shutdown timed out, forcing exit.");+ exit(1);+ },+ (config.limits.killGraceSec + 5) * 1000,+ );// The platform is stopping us: abort the running job so it's reported as// SHUTDOWN (retryable) instead of vanishing mid-render.- runner.queue.close(+ await worker.drain(new RenderError("SHUTDOWN", `Worker received ${signal}.`),true,);-- const graceMs = (config.limits.killGraceSec + 5) * 1000;- const forceExit = setTimeout(() => {- rendererLogger.error("Shutdown timed out, forcing exit.");- exit(1);- }, graceMs);-- await runner.queue.idle();- stopWatchdog();- server.close();clearTimeout(forceExit);rendererLogger.info("Shutdown complete.");- exit(0);+ shutdown(0);}process.on("SIGTERM", () => void gracefulShutdown("SIGTERM"));diff --git a/apps/renderer-service/src/jobs/runner.ts b/apps/renderer-service/src/jobs/runner.tsindex ca6ee52e..be4040a2 100644--- a/apps/renderer-service/src/jobs/runner.ts+++ b/apps/renderer-service/src/jobs/runner.ts@@ -25,6 +25,8 @@ export class JobRunner {private readonly pipeline: Pipeline,private readonly hooks: {afterJob?: (completed: number) => void;+ /** Awaited before checkout, between jobs (e.g. cache eviction). */+ beforeJob?: (job: ValidatedJob) => Promise<void>;journal?: CrashJournal;} = {},) {@@ -137,6 +139,7 @@ export class JobRunner {let succeeded = false;let settle: (() => unknown) | undefined;try {+ await this.hooks.beforeJob?.(job);await fs.mkdir(path.join(jobDir, "out"), { recursive: true });const checkout = await timed("checkout", () =>diff --git a/apps/renderer-service/src/worker/boot.ts b/apps/renderer-service/src/worker/boot.tsnew file mode 100644index 00000000..75ad579d--- /dev/null+++ b/apps/renderer-service/src/worker/boot.ts@@ -0,0 +1,160 @@+import { mkdirSync, realpathSync } from "node:fs";+import path from "node:path";+import { HeadlessMediaRenderer } from "@framefields/renderer";+import "@framefields/renderer/offscreen-gl-polyfill";+import { rendererLogger } from "@framefields/server-utils";+import {+ describeConfig,+ loadConfig,+ type RendererConfig,+ scrubSecretEnv,+} from "../config/index.js";+import { RenderError } from "../errors.js";+import { checkout } from "../git/checkout.js";+import { repoHash, workerId } from "../git/mirror.js";+import { install } from "../install/installer.js";+import { CrashJournal } from "../jobs/journal.js";+import type { Pipeline } from "../jobs/pipeline.js";+import { JobRunner } from "../jobs/runner.js";+import { deliver, findExisting } from "../output/deliver.js";+import { ReadinessTracker } from "../readiness.js";+import { redact } from "../redact.js";+import { installEngineAlias } from "../render/engine-alias.js";+import { render } from "../render/render-step.js";+import { installProcessGuards, strayErrorCount } from "../render/sandbox.js";+import { workerHealth } from "./health.js";+import { CacheJanitor } from "./janitor.js";+import { currentStats, recycleReason } from "./recycle.js";+import { startWatchdog } from "./watchdog.js";++/** The job sandbox swaps process.exit while repo code runs; the worker exits through the real one. */+export const exit = process.exit.bind(process);++export interface Worker {+ config: RendererConfig;+ runner: JobRunner;+ readiness: ReadinessTracker;+ /** True once the worker stopped taking jobs (recycling or shutting down). */+ isDraining(): boolean;+ /** Stops taking jobs and waits for the current one. */+ drain(reason: RenderError, abortActive: boolean): Promise<void>;+ /** Initialises the GPU; rejects when no usable device is found. */+ initGpu(): Promise<void>;+ stop(): void;+}++/**+ * Everything the HTTP server and the CLI share (PLAN.md §5): config, engine+ * alias, process guards, crash journal, watchdog, cache janitor, recycling.+ */+export function bootWorker(+ options: { onRecycle?: (reason: string) => void } = {},+): Worker {+ const config = loadConfig();+ scrubSecretEnv();+ rendererLogger.info(+ { config: describeConfig(config) },+ "Renderer config loaded",+ );++ // Workspaces live here; repo code under it gets the engine alias (§7).+ const workspaces = path.join(config.paths.workDir, "ws");+ mkdirSync(workspaces, { recursive: true });+ mkdirSync(config.paths.cacheDir, { recursive: true });+ installEngineAlias([realpathSync(workspaces)]);++ installProcessGuards((msg, err) =>+ rendererLogger.error(+ {+ err: redact(+ err instanceof Error ? (err.stack ?? err.message) : String(err),+ ),+ },+ msg,+ ),+ );++ const readiness = new ReadinessTracker();+ const janitor = new CacheJanitor(config, workerId());+ const journal = new CrashJournal(+ path.join(config.paths.workDir, "inflight.json"),+ );+ const pipeline: Pipeline = {+ checkout,+ install,+ render,+ findExisting,+ deliver,+ };+ let baselineHandles = 0;+ let draining = false;++ const runner = new JobRunner(config, pipeline, {+ journal,+ beforeJob: (job) => janitor.maybeRun([repoHash(job.repo)]),+ afterJob: (completed) => {+ const reason = recycleReason(+ currentStats(+ completed,+ baselineHandles,+ strayErrorCount(),+ workerHealth.recycleReason,+ ),+ config,+ );+ if (reason && !draining) {+ draining = true;+ rendererLogger.warn({ reason }, "Recycling worker");+ runner.queue.close(+ new RenderError("SHUTDOWN", `Worker is recycling (${reason}).`),+ false,+ );+ options.onRecycle?.(reason);+ }+ },+ });++ const crashed = journal.recover();+ if (crashed) {+ runner.store.restore(crashed);+ rendererLogger.error(+ { jobId: crashed.id, error: crashed.error },+ "Previous process died during a job",+ );+ }+ const stopWatchdog = startWatchdog({+ blockSec: config.recycle.eventLoopBlockSec,+ journalPath: journal.path,+ });++ return {+ config,+ runner,+ readiness,+ isDraining: () => draining,+ async drain(reason, abortActive) {+ draining = true;+ runner.queue.close(reason, abortActive);+ await runner.queue.idle();+ },+ async initGpu() {+ try {+ rendererLogger.info("Initializing renderer (GPU, node discovery)...");+ await HeadlessMediaRenderer.initialize();+ readiness.setRendererReady(true);+ readiness.setGpuReady(true);+ baselineHandles = process.getActiveResourcesInfo().length;+ rendererLogger.info("Renderer initialized.");+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ rendererLogger.fatal(+ { err: error },+ "❌ GPU / Renderer initialization failed",+ );+ readiness.setFailure(`GPU_UNAVAILABLE: ${message}`);+ throw new RenderError("GPU_UNAVAILABLE", message, { cause: error });+ }+ },+ stop: stopWatchdog,+ };+}diff --git a/apps/renderer-service/src/worker/janitor.test.ts b/apps/renderer-service/src/worker/janitor.test.tsnew file mode 100644index 00000000..6afe6713--- /dev/null+++ b/apps/renderer-service/src/worker/janitor.test.ts@@ -0,0 +1,102 @@+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { testConfig } from "../test-helpers.js";+import { CacheJanitor, dirSize, LAST_USED } from "./janitor.js";++const MB = 1024 * 1024;+let tmp: string;++beforeAll(async () => {+ tmp = await fs.mkdtemp(path.join(os.tmpdir(), "ff-janitor-"));+});+afterAll(async () => {+ await fs.rm(tmp, { recursive: true, force: true });+});++async function blob(p: string, mb: number, ageMin = 0) {+ await fs.mkdir(path.dirname(p), { recursive: true });+ await fs.writeFile(p, Buffer.alloc(mb * MB));+ const t = new Date(Date.now() - ageMin * 60_000);+ await fs.utimes(p, t, t);+}++/** Limits are whole GB, so tests scale sizes via a fractional limit hack: 1 GB = 1024 MB. */+function janitor(root: string, env: Record<string, string>) {+ const config = testConfig({+ FF_WORK_DIR: path.join(root, "work"),+ FF_CACHE_DIR: path.join(root, "cache"),+ ...env,+ });+ return { j: new CacheJanitor(config, "me", 0), config };+}++describe("CacheJanitor", () => {+ it("evicts least-recently-used workspaces, sparing the protected one", async () => {+ const root = path.join(tmp, "ws-case");+ const ws = path.join(root, "work/ws");+ await blob(path.join(ws, "old/0/big.bin"), 600);+ await blob(path.join(ws, "old", LAST_USED), 0, 120);+ await blob(path.join(ws, "mid/0/big.bin"), 300);+ await blob(path.join(ws, "mid", LAST_USED), 0, 60);+ await blob(path.join(ws, "protected/0/big.bin"), 600);+ await blob(path.join(ws, "protected", LAST_USED), 0, 500);+ const { j } = janitor(root, { FF_WORK_MAX_GB: "1", FF_CACHE_MAX_GB: "0" });++ const removed = await j.enforce(["protected"]);+ expect(removed.map((p) => path.basename(p))).toEqual(["old"]);+ expect(await dirSize(ws)).toBeLessThanOrEqual(1024 * MB);+ });++ it("evicts LFS objects before mirrors, and leaves other workers' busy mirrors", async () => {+ const root = path.join(tmp, "cache-case");+ const cache = path.join(root, "cache");+ await blob(path.join(cache, "lfs/objects/aa/bb/old-oid"), 500, 300);+ await blob(path.join(cache, "lfs/objects/cc/dd/new-oid"), 100, 1);+ await blob(path.join(cache, "git/me/repo1.git/pack.bin"), 300);+ await blob(path.join(cache, "git/me/repo1.git", LAST_USED), 0, 200);+ await blob(path.join(cache, "git/other/repo2.git/pack.bin"), 300);+ await blob(path.join(cache, "git/other/repo2.git", LAST_USED), 0, 5);+ await blob(path.join(cache, "models/vision.onnx"), 100);+ const { j } = janitor(root, { FF_CACHE_MAX_GB: "1", FF_WORK_MAX_GB: "0" });++ // 1300 MB total, limit 1024: the old LFS object (500) alone is enough.+ const removed = await j.enforce([]);+ expect(removed.map((p) => path.basename(p))).toEqual(["old-oid"]);++ // Over again: the oldest LFS object covers it on its own.+ await blob(path.join(cache, "lfs/objects/ee/ff/more"), 700, 2);+ expect((await j.enforce([])).map((p) => path.basename(p))).toEqual([+ "more",+ ]);++ // Far over: LFS first, then our own idle mirror, then the package store.+ // The other worker's recently used mirror and the models stay.+ await blob(path.join(cache, "pnpm-store/v10/blob"), 1200);+ const again = await j.enforce([]);+ expect(again.map((p) => path.basename(p))).toEqual([+ "new-oid",+ "repo1.git",+ "pnpm-store",+ ]);+ await expect(+ fs.stat(path.join(cache, "git/other/repo2.git")),+ ).resolves.toBeTruthy();+ await expect(+ fs.stat(path.join(cache, "models/vision.onnx")),+ ).resolves.toBeTruthy();+ });++ it("drops failed-job dirs older than a day", async () => {+ const root = path.join(tmp, "jobs-case");+ const old = path.join(root, "work/jobs/old-job");+ await blob(path.join(old, "out/x.mp4"), 1);+ const t = new Date(Date.now() - 2 * 24 * 3600_000);+ await fs.utimes(old, t, t);+ await blob(path.join(root, "work/jobs/new-job/out/x.mp4"), 1);+ const { j } = janitor(root, {});+ const removed = await j.enforce([]);+ expect(removed.map((p) => path.basename(p))).toEqual(["old-job"]);+ });+});diff --git a/apps/renderer-service/src/worker/janitor.ts b/apps/renderer-service/src/worker/janitor.tsnew file mode 100644index 00000000..c32facb9--- /dev/null+++ b/apps/renderer-service/src/worker/janitor.ts@@ -0,0 +1,255 @@+import { existsSync } from "node:fs";+import fs from "node:fs/promises";+import path from "node:path";+import { rendererLogger } from "@framefields/server-utils";+import type { RendererConfig } from "../config/index.js";++/** Marker whose mtime records when a mirror or workspace was last used. */+export const LAST_USED = ".ff-last-used";++const GB = 1024 ** 3;+const DAY_MS = 24 * 60 * 60 * 1000;+/** Mirrors owned by other workers on a shared volume are only evicted when idle this long. */+const FOREIGN_MIRROR_IDLE_MS = 60 * 60 * 1000;+/** Package caches, evicted whole and only as a last resort. */+const STORES = [+ "pnpm-store",+ "npm-cache",+ "yarn-cache",+ "yarn-berry",+ "corepack",+];++let current: CacheJanitor | undefined;++/** The worker's janitor, for emergency eviction (e.g. the LFS disk preflight). */+export function activeJanitor(): CacheJanitor | undefined {+ return current;+}++export async function markUsed(dir: string): Promise<void> {+ const marker = path.join(dir, LAST_USED);+ const now = new Date();+ await fs.utimes(marker, now, now).catch(() => fs.writeFile(marker, ""));+}++interface Candidate {+ path: string;+ bytes: number;+ lastUsed: number;+ kind: string;+}++/**+ * Keeps FF_WORK_DIR and FF_CACHE_DIR under their size limits (PLAN.md §6.4).+ * Runs between jobs, so nothing it deletes is in use. Eviction order:+ * workspaces (LRU), then LFS objects (LRU), then git mirrors (LRU), then+ * whole package caches. Pinned vision models are never evicted.+ */+export class CacheJanitor {+ private lastRun = 0;++ constructor(+ private readonly config: RendererConfig,+ private readonly workerId: string,+ private readonly intervalMs = 5 * 60 * 1000,+ ) {+ current = this;+ }++ /** Runs at most once per interval unless forced. Never throws. */+ async maybeRun(protect: readonly string[], force = false): Promise<void> {+ if (!force && Date.now() - this.lastRun < this.intervalMs) return;+ this.lastRun = Date.now();+ try {+ const removed = await this.enforce(protect);+ if (removed.length) {+ rendererLogger.info(+ { removed: removed.length, paths: removed.slice(0, 20) },+ "Cache eviction",+ );+ }+ } catch (err) {+ rendererLogger.warn({ err: String(err) }, "Cache eviction failed");+ }+ }++ /**+ * Evicts until both limits hold. `protect` lists repo hashes the next job+ * needs (its workspace and mirror stay).+ */+ async enforce(protect: readonly string[]): Promise<string[]> {+ const removed: string[] = [];+ const { workDir, cacheDir, workMaxGb, cacheMaxGb } = this.config.paths;++ // Failed-job dirs kept for debugging expire after a day.+ for (const dir of await children(path.join(workDir, "jobs"))) {+ if (Date.now() - (await mtime(dir)) > DAY_MS) {+ await remove(dir, removed);+ }+ }++ if (workMaxGb > 0) {+ const workspaces = (await children(path.join(workDir, "ws"))).filter(+ (d) => !protect.includes(path.basename(d)),+ );+ const candidates = await Promise.all(+ workspaces.map(async (d) => ({+ path: d,+ bytes: await dirSize(d),+ lastUsed: await mtime(path.join(d, LAST_USED), d),+ kind: "workspace",+ })),+ );+ const total =+ (await dirSize(path.join(workDir, "ws"))) ++ (await dirSize(path.join(workDir, "jobs")));+ await evict(candidates, total - workMaxGb * GB, removed);+ }++ if (cacheMaxGb > 0) {+ let total = await dirSize(cacheDir);+ const over = () => total - cacheMaxGb * GB;+ if (over() > 0) {+ const lfs = await lfsObjects(path.join(cacheDir, "lfs"));+ total -= await evict(lfs, over(), removed);+ }+ if (over() > 0) {+ total -= await evict(await this.mirrors(protect), over(), removed);+ }+ if (over() > 0) {+ const stores = await Promise.all(+ STORES.map(async (s) => {+ const p = path.join(cacheDir, s);+ return {+ path: p,+ bytes: await dirSize(p),+ lastUsed: 0,+ kind: "store",+ };+ }),+ );+ // Largest first: one eviction usually suffices.+ stores.sort((a, b) => b.bytes - a.bytes);+ total -= await evict(+ stores.filter((s) => s.bytes > 0),+ over(),+ removed,+ false,+ );+ }+ }+ return removed;+ }++ private async mirrors(protect: readonly string[]): Promise<Candidate[]> {+ const out: Candidate[] = [];+ for (const workerDir of await children(+ path.join(this.config.paths.cacheDir, "git"),+ )) {+ const own = path.basename(workerDir) === this.workerId;+ for (const mirror of await children(workerDir)) {+ const hash = path.basename(mirror).replace(/\.git$/, "");+ if (own && protect.includes(hash)) continue;+ const lastUsed = await mtime(path.join(mirror, LAST_USED), mirror);+ // Another worker may be using its mirror right now.+ if (!own && Date.now() - lastUsed < FOREIGN_MIRROR_IDLE_MS) continue;+ out.push({+ path: mirror,+ bytes: await dirSize(mirror),+ lastUsed,+ kind: "mirror",+ });+ }+ }+ return out;+ }+}++/** Removes candidates (oldest first unless `sort` is false) until `bytes` are freed. */+async function evict(+ candidates: Candidate[],+ bytes: number,+ removed: string[],+ sort = true,+): Promise<number> {+ if (bytes <= 0) return 0;+ if (sort) candidates.sort((a, b) => a.lastUsed - b.lastUsed);+ let freed = 0;+ for (const c of candidates) {+ if (freed >= bytes) break;+ await remove(c.path, removed);+ freed += c.bytes;+ }+ return freed;+}++async function remove(p: string, removed: string[]): Promise<void> {+ await fs.rm(p, { recursive: true, force: true });+ removed.push(p);+}++async function children(dir: string): Promise<string[]> {+ const entries = await fs+ .readdir(dir, { withFileTypes: true })+ .catch(() => []);+ return entries+ .filter((e) => e.isDirectory())+ .map((e) => path.join(dir, e.name));+}++async function mtime(p: string, fallback?: string): Promise<number> {+ const stat = await fs+ .stat(p)+ .catch(() =>+ fallback ? fs.stat(fallback).catch(() => undefined) : undefined,+ );+ return stat?.mtimeMs ?? 0;+}++/** git-lfs stores objects at lfs/objects/aa/bb/<oid>; LRU by access time. */+async function lfsObjects(root: string): Promise<Candidate[]> {+ const out: Candidate[] = [];+ const objects = path.join(root, "objects");+ if (!existsSync(objects)) return out;+ for (const a of await children(objects)) {+ for (const b of await children(a)) {+ for (const name of await fs.readdir(b).catch(() => [])) {+ const p = path.join(b, name);+ const stat = await fs.stat(p).catch(() => undefined);+ if (stat?.isFile()) {+ out.push({+ path: p,+ bytes: stat.size,+ lastUsed: Math.max(stat.atimeMs, stat.mtimeMs),+ kind: "lfs",+ });+ }+ }+ }+ }+ return out;+}++/** Bytes on disk under `dir` (symlinks not followed). */+export async function dirSize(dir: string): Promise<number> {+ let total = 0;+ const walk = async (d: string): Promise<void> => {+ const entries = await fs+ .readdir(d, { withFileTypes: true })+ .catch(() => []);+ await Promise.all(+ entries.map(async (e) => {+ const p = path.join(d, e.name);+ if (e.isDirectory()) await walk(p);+ else if (e.isFile()) {+ // Read first: `total += await …` would add to a stale total.+ const size = (await fs.lstat(p).catch(() => undefined))?.size ?? 0;+ total += size;+ }+ }),+ );+ };+ await walk(dir);+ return total;+}diff --git a/apps/renderer-service/vitest.config.ts b/apps/renderer-service/vitest.config.tsindex d84aabcb..64e5fe46 100644--- a/apps/renderer-service/vitest.config.ts+++ b/apps/renderer-service/vitest.config.ts@@ -13,7 +13,7 @@ export default defineConfig({alias: [{find: "@",- replacement: path.resolve(__dirname, "./src"),+ replacement: path.resolve(import.meta.dirname, "./src"),},],},