Skip to content

Code-first video & image, rendered natively on WebGPU.

15.1 KB
Raw Blame
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import type { RendererConfig } from "../config/index.js";
import { ASSET_CACHE } from "../git/assets.js";
import {
	moduleStoreDir,
	readManifests,
	releaseKeys,
	sweepStore,
} from "../install/shared-modules.js";
import { rendererLogger } from "../logger.js";

/** Marker whose mtime records when a mirror or workspace was last used. */
export const LAST_USED = ".ff-last-used";

const GB = 1024 ** 3;
const DAY_MS = 24 * 60 * 60 * 1000;
/** Mirrors owned by other workers on a shared volume are only evicted when idle this long. */
const FOREIGN_MIRROR_IDLE_MS = 60 * 60 * 1000;
/** Package caches, evicted whole and only as a last resort. */
const STORES = [
	"pnpm-store",
	"npm-cache",
	"yarn-cache",
	"yarn-berry",
	"corepack",
];

let current: CacheJanitor | undefined;

/** The worker's janitor, for emergency eviction (e.g. the LFS disk preflight). */
export function activeJanitor(): CacheJanitor | undefined {
	return current;
}

export async function markUsed(dir: string): Promise<void> {
	const marker = path.join(dir, LAST_USED);
	const now = new Date();
	await fs.utimes(marker, now, now).catch(() => fs.writeFile(marker, ""));
}

/** A filesystem as statfs sees it. */
export interface DiskInfo {
	/** Identifies the filesystem: dirs with equal ids share free space. */
	id: string;
	totalBytes: number;
	freeBytes: number;
}

export type DiskProbe = (dir: string) => Promise<DiskInfo>;

const statDisk: DiskProbe = async (dir) => {
	const [fsStat, stat] = await Promise.all([fs.statfs(dir), fs.stat(dir)]);
	return {
		id: String(stat.dev),
		totalBytes: Number(fsStat.blocks) * Number(fsStat.bsize),
		freeBytes: Number(fsStat.bavail) * Number(fsStat.bsize),
	};
};

/** One disk as /health and the eviction log report it. */
export interface DiskSummary {
	path: string;
	freeGb: number;
	totalGb: number;
	/** Eviction starts below this much free space (null: not measurable). */
	evictBelowGb: number | null;
	lowOnSpace: boolean;
	/**
	 * False when the filesystem reports an implausible size (some network and
	 * host-shared mounts do): free-space eviction is off there; only the
	 * optional FF_WORK_MAX_GB / FF_CACHE_MAX_GB caps apply.
	 */
	measurable?: false;
	/** Set when the cache dir is on the work dir's disk. */
	sameDiskAs?: string;
}

/** Percentages never ask for more free space than this. */
const PCT_CAP = 200 * GB;
/** Larger reported sizes are taken as unknown (e.g. a shared cluster's total). */
const MAX_PLAUSIBLE_DISK = 64 * 1024 * GB;

function measurable(d: DiskInfo): boolean {
	return (
		d.totalBytes > 0 &&
		d.totalBytes <= MAX_PLAUSIBLE_DISK &&
		d.freeBytes <= d.totalBytes
	);
}

const warned = new Set<string>();
function warnUnmeasurable(dir: string, d: DiskInfo): void {
	if (warned.has(dir)) return;
	warned.add(dir);
	rendererLogger.warn(
		{
			dir,
			totalGb: Math.round(d.totalBytes / GB),
			freeGb: Math.round(d.freeBytes / GB),
		},
		"Disk reports an implausible size; free-space eviction is off for it (set FF_WORK_MAX_GB / FF_CACHE_MAX_GB to cap usage)",
	);
}

interface Candidate {
	path: string;
	bytes: number;
	lastUsed: number;
	kind: string;
}

/** One eviction stage on one disk: removes items until `enough()` says stop. */
type Stage = (
	enough: () => Promise<boolean>,
	removed: string[],
) => Promise<void>;

/**
 * Keeps the workspace and cache disks from filling up (PLAN.md §6.4). Runs
 * between jobs, so nothing it deletes is in use.
 *
 * Free space decides: when a disk has less free than
 * max(FF_DISK_MIN_FREE_PCT, FF_DISK_MIN_FREE_GB), the janitor evicts until
 * FF_DISK_TARGET_FREE_PCT is free. Order: workspaces (LRU) and the shared
 * module files only they used, then LFS objects (LRU), then git mirrors (LRU),
 * then whole package caches. When FF_WORK_DIR and FF_CACHE_DIR are on the
 * same disk the stages run in that order against one free-space figure.
 * FF_WORK_MAX_GB / FF_CACHE_MAX_GB optionally cap usage on top. Pinned vision
 * models are never evicted.
 */
export class CacheJanitor {
	private lastRun = 0;

	constructor(
		private readonly config: RendererConfig,
		private readonly workerId: string,
		private readonly intervalMs = 5 * 60 * 1000,
		private readonly disk: DiskProbe = statDisk,
		/** Repo hashes never evicted, whoever asks (e.g. the bound engine's). */
		private readonly alwaysProtect: () => readonly string[] = () => [],
	) {
		current = this;
	}

	/**
	 * Before each job: evicts right away when a disk is low on space (cheap
	 * statfs check), otherwise at most once per interval. Never throws.
	 */
	async maybeRun(protect: readonly string[], force = false): Promise<void> {
		const due = force || Date.now() - this.lastRun >= this.intervalMs;
		if (!due && !(await this.lowOnSpace().catch(() => false))) return;
		this.lastRun = Date.now();
		try {
			await this.enforce(protect);
		} catch (err) {
			rendererLogger.warn({ err: String(err) }, "Cache eviction failed");
		}
	}

	private snapshot:
		| { at: number; disks: Record<string, DiskSummary> }
		| undefined;
	private refreshing: Promise<void> | undefined;

	/**
	 * The last measured disks, for the synchronous /health handler. Starts a
	 * background refresh when older than `maxAgeMs`; undefined until the
	 * first measurement finishes.
	 */
	disks(maxAgeMs = 10_000): Record<string, DiskSummary> | undefined {
		const stale = !this.snapshot || Date.now() - this.snapshot.at > maxAgeMs;
		if (stale && !this.refreshing) {
			this.refreshing = this.describeDisks()
				.then(
					() => undefined,
					() => undefined,
				)
				.finally(() => {
					this.refreshing = undefined;
				});
		}
		return this.snapshot?.disks;
	}

	/** Free space per disk, for logs and /health. */
	async describeDisks(): Promise<Record<string, DiskSummary>> {
		const { workDir, cacheDir } = this.config.paths;
		const [work, cache] = await Promise.all([
			this.disk(workDir),
			this.disk(cacheDir),
		]);
		const summary = (dir: string, d: DiskInfo): DiskSummary => {
			const { low } = this.thresholds(d);
			const gb = (bytes: number) => Math.round((bytes / GB) * 10) / 10;
			return {
				path: dir,
				freeGb: gb(d.freeBytes),
				totalGb: gb(d.totalBytes),
				...(measurable(d)
					? { evictBelowGb: gb(low), lowOnSpace: d.freeBytes < low }
					: { evictBelowGb: null, lowOnSpace: false, measurable: false }),
			};
		};
		const disks = {
			work: summary(workDir, work),
			cache: {
				...summary(cacheDir, cache),
				...(cache.id === work.id && { sameDiskAs: "work" }),
			},
		};
		this.snapshot = { at: Date.now(), disks };
		return disks;
	}

	private thresholds(d: DiskInfo, extra = 0) {
		const { minFreePct, minFreeGb, targetFreePct } = this.config.paths;
		// A fixed GB floor can't demand most of a small disk, and a percentage
		// can't demand terabytes of a huge one.
		const floor = Math.min(minFreeGb * GB, d.totalBytes * 0.5);
		const pct = (p: number) => Math.min((d.totalBytes * p) / 100, PCT_CAP);
		const low = Math.max(pct(minFreePct), floor) + extra;
		const target = Math.max(
			Math.min(pct(targetFreePct), d.totalBytes * 0.6),
			low,
		);
		return { low, target };
	}

	private async lowOnSpace(): Promise<boolean> {
		const { workDir, cacheDir } = this.config.paths;
		for (const dir of [workDir, cacheDir]) {
			if (!existsSync(dir)) continue;
			const d = await this.disk(dir);
			if (measurable(d) && d.freeBytes < this.thresholds(d).low) return true;
		}
		return false;
	}

	/**
	 * Evicts until each disk has its target free space and the optional caps
	 * hold. `protect` lists repo hashes the next job needs (its workspace and
	 * mirror stay). `extraBytes` raises the bar, for a download that is about
	 * to need that much (LFS).
	 */
	async enforce(
		protectOnly: readonly string[],
		extraBytes = 0,
	): Promise<string[]> {
		const protect = [...protectOnly, ...this.alwaysProtect()];
		const removed: string[] = [];
		const { workDir, cacheDir, workMaxGb, cacheMaxGb } = this.config.paths;
		await fs.mkdir(workDir, { recursive: true });
		await fs.mkdir(cacheDir, { recursive: true });

		// Failed-job dirs kept for debugging expire after a day.
		for (const dir of await children(path.join(workDir, "jobs"))) {
			if (Date.now() - (await mtime(dir)) > DAY_MS) {
				await remove(dir, removed);
			}
		}

		const workStages: Stage[] = [
			(enough, out) => this.evictWorkspaces(protect, enough, out),
			async () => {
				await sweepStore(moduleStoreDir(workDir));
			},
		];
		const cacheStages: Stage[] = [
			(enough, out) =>
				evictEach(lfsObjects(path.join(cacheDir, "lfs")), enough, out),
			(enough, out) =>
				evictEach(assetObjects(path.join(cacheDir, ASSET_CACHE)), enough, out),
			async (enough, out) => evictEach(this.mirrors(protect), enough, out),
			(enough, out) => evictEach(this.stores(), enough, out, false),
		];

		// Free space, per disk.
		const [work, cache] = await Promise.all([
			this.disk(workDir),
			this.disk(cacheDir),
		]);
		const disks =
			work.id === cache.id
				? [{ dir: workDir, stages: [...workStages, ...cacheStages] }]
				: [
						{ dir: workDir, stages: workStages },
						{ dir: cacheDir, stages: cacheStages },
					];
		for (const { dir, stages } of disks) {
			const first = await this.disk(dir);
			if (!measurable(first)) {
				warnUnmeasurable(dir, first);
				continue;
			}
			const { low, target } = this.thresholds(first, extraBytes);
			if (first.freeBytes >= low) continue;
			const enough = async () => (await this.disk(dir)).freeBytes >= target;
			for (const stage of stages) {
				if (await enough()) break;
				await stage(enough, removed);
			}
		}

		// Optional caps on top, measured by walking the trees.
		if (workMaxGb > 0) {
			const over = async () =>
				(await usage([
					path.join(workDir, "ws"),
					path.join(workDir, "jobs"),
					path.join(workDir, "store"),
				])) >
				workMaxGb * GB;
			const enough = async () => !(await over());
			if (await over()) {
				for (const stage of workStages) {
					if (await enough()) break;
					await stage(enough, removed);
				}
			}
		}
		if (cacheMaxGb > 0) {
			const enough = async () => (await usage([cacheDir])) <= cacheMaxGb * GB;
			for (const stage of cacheStages) {
				if (await enough()) break;
				await stage(enough, removed);
			}
		}
		if (removed.length) {
			rendererLogger.info(
				{
					removed: removed.length,
					paths: removed.slice(0, 20),
					...(extraBytes > 0 && {
						neededMb: Math.round(extraBytes / 1024 ** 2),
					}),
					disks: await this.describeDisks().catch(() => undefined),
				},
				"Cache eviction",
			);
		}
		return removed;
	}

	/** Oldest workspace first; each takes its now-unused shared module files along. */
	private async evictWorkspaces(
		protect: readonly string[],
		enough: () => Promise<boolean>,
		removed: string[],
	): Promise<void> {
		const { workDir } = this.config.paths;
		const store = moduleStoreDir(workDir);
		const workspaces = await Promise.all(
			(await children(path.join(workDir, "ws")))
				.filter((d) => !protect.includes(path.basename(d)))
				.map(async (d) => ({
					path: d,
					lastUsed: await mtime(path.join(d, LAST_USED), d),
				})),
		);
		workspaces.sort((a, b) => a.lastUsed - b.lastUsed);
		for (const ws of workspaces) {
			if (await enough()) return;
			// Remove the workspace first, so its store files drop to one link.
			const keys = await readManifests(ws.path);
			await remove(ws.path, removed);
			await releaseKeys(store, keys);
		}
	}

	private async mirrors(protect: readonly string[]): Promise<Candidate[]> {
		const out: Candidate[] = [];
		for (const workerDir of await children(
			path.join(this.config.paths.cacheDir, "git"),
		)) {
			const own = path.basename(workerDir) === this.workerId;
			for (const mirror of await children(workerDir)) {
				const hash = path.basename(mirror).replace(/\.git$/, "");
				if (own && protect.includes(hash)) continue;
				const lastUsed = await mtime(path.join(mirror, LAST_USED), mirror);
				// Another worker may be using its mirror right now.
				if (!own && Date.now() - lastUsed < FOREIGN_MIRROR_IDLE_MS) continue;
				out.push({ path: mirror, bytes: 0, lastUsed, kind: "mirror" });
			}
		}
		return out;
	}

	/** Package caches, largest first: one eviction usually suffices. */
	private async stores(): Promise<Candidate[]> {
		const stores = await Promise.all(
			STORES.map(async (s) => {
				const p = path.join(this.config.paths.cacheDir, s);
				return { path: p, bytes: await usage([p]), lastUsed: 0, kind: "store" };
			}),
		);
		return stores.filter((s) => s.bytes > 0).sort((a, b) => b.bytes - a.bytes);
	}
}

/** Removes candidates (oldest first unless `sort` is false) until `enough()`. */
async function evictEach(
	candidates: Promise<Candidate[]>,
	enough: () => Promise<boolean>,
	removed: string[],
	sort = true,
): Promise<void> {
	const list = await candidates;
	if (sort) list.sort((a, b) => a.lastUsed - b.lastUsed);
	for (const c of list) {
		if (await enough()) return;
		await remove(c.path, removed);
	}
}

async function remove(p: string, removed: string[]): Promise<void> {
	await fs.rm(p, { recursive: true, force: true });
	removed.push(p);
}

async function children(dir: string): Promise<string[]> {
	const entries = await fs
		.readdir(dir, { withFileTypes: true })
		.catch(() => []);
	return entries
		.filter((e) => e.isDirectory())
		.map((e) => path.join(dir, e.name));
}

async function mtime(p: string, fallback?: string): Promise<number> {
	const stat = await fs
		.stat(p)
		.catch(() =>
			fallback ? fs.stat(fallback).catch(() => undefined) : undefined,
		);
	return stat?.mtimeMs ?? 0;
}

/** git-lfs stores objects at lfs/objects/aa/bb/<oid>; LRU by access time. */
async function lfsObjects(root: string): Promise<Candidate[]> {
	const out: Candidate[] = [];
	for (const a of await children(path.join(root, "objects")))
		for (const b of await children(a)) await cachedFiles(b, "lfs", out);
	return out;
}

/** framefields.json assets are cached at assets/aa/<sha256>. */
async function assetObjects(root: string): Promise<Candidate[]> {
	const out: Candidate[] = [];
	for (const a of await children(root)) await cachedFiles(a, "asset", out);
	return out;
}

async function cachedFiles(
	dir: string,
	kind: string,
	out: Candidate[],
): Promise<void> {
	for (const name of await fs.readdir(dir).catch(() => [])) {
		const p = path.join(dir, name);
		const stat = await fs.stat(p).catch(() => undefined);
		if (stat?.isFile())
			out.push({
				path: p,
				bytes: stat.size,
				lastUsed: Math.max(stat.atimeMs, stat.mtimeMs),
				kind,
			});
	}
}

/**
 * Bytes on disk under `dirs` (symlinks not followed). A file hard-linked in
 * several places (shared node_modules) counts once.
 */
export async function usage(dirs: string[]): Promise<number> {
	let total = 0;
	const seen = new Set<string>();
	const walk = async (d: string): Promise<void> => {
		const entries = await fs
			.readdir(d, { withFileTypes: true })
			.catch(() => []);
		await Promise.all(
			entries.map(async (e) => {
				const p = path.join(d, e.name);
				if (e.isDirectory()) await walk(p);
				else if (e.isFile()) {
					const stat = await fs.lstat(p).catch(() => undefined);
					if (!stat) return;
					const id = `${stat.dev}:${stat.ino}`;
					if (seen.has(id)) return;
					seen.add(id);
					total += stat.size;
				}
			}),
		);
	};
	for (const dir of dirs) await walk(dir);
	return total;
}