Skip to content

Code-first video & image, rendered natively on WebGPU.

7.2 KB
Raw Blame
import fs from "node:fs/promises";
import path from "node:path";
import { RenderError } from "../errors.js";
import { CommandError, classifyGitError } from "./exec.js";
import { type GitContext, git } from "./mirror.js";

const POINTER_HEADER = "version https://git-lfs.github.com/spec/v1";
const DISK_MARGIN_BYTES = 2 * 1024 * 1024 * 1024;

export interface LfsOptions {
	enabled?: boolean;
	include: string[];
	exclude: string[];
	/** Directories that must be fully usable (workdir, sparse paths). */
	scope: string[];
	/** Frees disk (cache eviction) when the preflight finds too little. */
	/** Evicts caches so `bytes` more fit (the janitor). */
	makeRoom?: (bytes: number) => Promise<void>;
}

export interface LfsResult {
	used: boolean;
	files: number;
	bytes: number;
	cache: "hit" | "partial" | "miss" | "off";
}

interface LfsFile {
	name: string;
	size: number;
	downloaded?: boolean;
}

/** True when any tracked .gitattributes routes files through LFS. */
export async function usesLfs(root: string, ctx: GitContext): Promise<boolean> {
	const { stdout } = await git(ctx, [
		"-C",
		root,
		"ls-files",
		"-z",
		"--",
		":(glob)**/.gitattributes",
		".gitattributes",
	]).catch(() => ({ stdout: "" }));
	for (const file of stdout.split("\0").filter(Boolean)) {
		const text = await fs
			.readFile(path.join(root, file), "utf8")
			.catch(() => "");
		if (/filter=lfs/.test(text)) return true;
	}
	return false;
}

/**
 * Pulls LFS objects for the include/exclude patterns into the shared object
 * store, with a disk-space preflight and a pointer-file postflight (§6.2.5).
 */
export async function pullLfs(
	root: string,
	ctx: GitContext,
	options: LfsOptions,
): Promise<LfsResult> {
	const enabled = options.enabled ?? (await usesLfs(root, ctx));
	if (!enabled) return { used: false, files: 0, bytes: 0, cache: "off" };

	const storage = path.join(ctx.config.paths.cacheDir, "lfs");
	await fs.mkdir(storage, { recursive: true });
	// The shared object store goes in the repo's own config, not `-c`: with
	// `-c`, git-lfs's file:// transfer agent (which reads the remote repo)
	// also used it, and looked for the remote's objects in our store.
	await git(ctx, ["-C", root, "config", "lfs.storage", storage]);
	const lfsArgs = [
		"-C",
		root,
		"-c",
		`lfs.concurrenttransfers=${Math.max(1, ctx.config.source.lfsConcurrency)}`,
		"-c",
		"lfs.fetchrecentalways=false",
		"-c",
		"lfs.skipdownloaderrors=false",
	];
	const filters = [
		...(options.include.length ? ["-I", options.include.join(",")] : []),
		...(options.exclude.length ? ["-X", options.exclude.join(",")] : []),
	];
	const lfsError = (err: unknown) =>
		err instanceof RenderError
			? err
			: classifyGitError(err, "lfs", {
					lfs: true,
					extraRules: ctx.profile.lfsRules,
					fallback: "GIT_NETWORK",
				});

	// Preflight: what would be downloaded, and does it fit?
	let files: LfsFile[];
	try {
		const { stdout } = await git(ctx, [
			...lfsArgs,
			"lfs",
			"ls-files",
			"--json",
			...filters,
		]);
		files = (JSON.parse(stdout || "{}").files ?? []) as LfsFile[];
	} catch (err) {
		if (
			err instanceof CommandError &&
			/unknown flag: --json/.test(err.result.stderr)
		) {
			throw new RenderError(
				"TOOL_MISSING",
				"git-lfs ≥ 3.3 is required (ls-files --json).",
				{ phase: "lfs" },
			);
		}
		throw lfsError(err);
	}
	const missing = files.filter((f) => !f.downloaded);
	const needed = missing.reduce((sum, f) => sum + (f.size ?? 0), 0);
	const total = files.reduce((sum, f) => sum + (f.size ?? 0), 0);
	// Objects land in the store, then get copied into the checkout.
	const fits = async () => {
		await assertFreeSpace(storage, needed);
		await assertFreeSpace(root, total);
	};
	try {
		await fits();
	} catch (err) {
		if (
			!(err instanceof RenderError && err.code === "DISK_FULL") ||
			!options.makeRoom
		)
			throw err;
		await options.makeRoom(Math.max(needed, total) + DISK_MARGIN_BYTES);
		await fits();
	}

	const attempts = 3;
	for (let attempt = 1; ; attempt++) {
		try {
			await git(ctx, [...lfsArgs, "lfs", "pull", ...filters], {
				timeoutSec: ctx.config.limits.timeouts.lfs,
			});
			break;
		} catch (err) {
			const classified = lfsError(err);
			if (!classified.retryable || attempt >= attempts || ctx.signal.aborted)
				throw classified;
			await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
		}
	}

	await assertNoPointers(root, files, options.scope);

	return {
		used: true,
		files: files.length,
		bytes: total,
		cache:
			missing.length === 0
				? "hit"
				: missing.length === files.length
					? "miss"
					: "partial",
	};
}

/**
 * After the pull, any file in scope that still holds a pointer is an asset
 * the render would choke on. Fail with the exact paths instead.
 */
async function assertNoPointers(
	root: string,
	files: LfsFile[],
	scope: string[],
): Promise<void> {
	const inScope = (name: string) =>
		scope.length === 0 ||
		scope.some(
			(s) =>
				s === "." || name === s || name.startsWith(`${s.replace(/\/$/, "")}/`),
		);
	const unresolved: string[] = [];
	for (const file of files) {
		if (!inScope(file.name)) continue;
		const handle = await fs
			.open(path.join(root, file.name), "r")
			.catch(() => undefined);
		if (!handle) continue;
		try {
			const buf = Buffer.alloc(POINTER_HEADER.length);
			await handle.read(buf, 0, buf.length, 0);
			if (buf.toString("utf8") === POINTER_HEADER) unresolved.push(file.name);
		} finally {
			await handle.close();
		}
	}
	if (unresolved.length > 0) {
		throw new RenderError(
			"LFS_POINTER_UNRESOLVED",
			`${unresolved.length} LFS file(s) were not downloaded: ${unresolved.slice(0, 10).join(", ")}${unresolved.length > 10 ? ", …" : ""}`,
			{ phase: "lfs", details: { files: unresolved.slice(0, 100) } },
		);
	}
}

/**
 * Pointer files that are outside the job's include patterns but inside its
 * workdir: likely assets the render needs. Reported as a warning up front.
 */
export async function findPointers(
	root: string,
	scope: string,
	limit = 20,
): Promise<string[]> {
	const found: string[] = [];
	const walk = async (dir: string): Promise<void> => {
		if (found.length >= limit) return;
		const entries = await fs
			.readdir(dir, { withFileTypes: true })
			.catch(() => []);
		for (const entry of entries) {
			if (found.length >= limit) return;
			if (entry.name === ".git" || entry.name === "node_modules") continue;
			const full = path.join(dir, entry.name);
			if (entry.isDirectory()) await walk(full);
			else if (entry.isFile()) {
				const stat = await fs.stat(full);
				if (stat.size > 200) continue;
				const text = await fs.readFile(full, "utf8").catch(() => "");
				if (text.startsWith(POINTER_HEADER))
					found.push(path.relative(root, full));
			}
		}
	};
	await walk(path.join(root, scope));
	return found;
}

export async function assertFreeSpace(
	dir: string,
	bytes: number,
): Promise<void> {
	if (bytes <= 0) return;
	const stat = await fs.statfs(dir);
	const free = Number(stat.bavail) * Number(stat.bsize);
	if (free < bytes + DISK_MARGIN_BYTES) {
		throw new RenderError(
			"DISK_FULL",
			`Need ${gb(bytes)} (+${gb(DISK_MARGIN_BYTES)} margin) in ${dir} but only ${gb(free)} is free.`,
			{
				phase: "lfs",
				hint: "Raise containerDiskInGb / the volume size, or narrow source.lfs.include.",
			},
		);
	}
}

const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GB`;