Skip to content

Code-first video & image, rendered natively on WebGPU.

12.7 KB
Raw Blame
import { createHash } from "node:crypto";
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import semver from "semver";
import type { RendererConfig } from "../config/index.js";
import { RenderError } from "../errors.js";
import { CommandError, exec } from "../git/exec.js";
import type { Pipeline, StepContext } from "../jobs/pipeline.js";
import { rendererLogger } from "../logger.js";
import { redact } from "../redact.js";
import { depsLoadedFrom } from "../render/engine.js";
import { workerHealth } from "../worker/health.js";
import { activeJanitor } from "../worker/janitor.js";
import {
	declaredEngineRange,
	inspectProject,
	type ProjectInfo,
	readPackageJson,
} from "./project.js";
import {
	expectedInstallSize,
	moduleStoreDir,
	recordInstallSize,
	removeModules,
	shareModules,
} from "./shared-modules.js";

export const STAMP_FILE = ".ff-install-stamp";

/**
 * Pipeline step: installs the project's dependencies, including the
 * framefields version it renders with (PLAN.md §7). Skipped when the
 * lockfile hasn't changed since the last install in this workspace slot.
 */
export const install: Pipeline["install"] = async (checkout, ctx) => {
	const project = await inspectProject(checkout.root, checkout.workdir);
	if (!project) {
		throw new RenderError(
			"ENGINE_NOT_FOUND",
			"The workdir has no package.json, so there's no framefields to render with.",
			{
				phase: "install",
				hint: 'Point source.workdir at the package that depends on "framefields".',
			},
		);
	}

	const rootPkg = project.inWorkspace
		? await readPackageJson(project.installRoot)
		: undefined;
	const declaredEngine =
		declaredEngineRange(project.pkg) ??
		(rootPkg && declaredEngineRange(rootPkg));
	if (!declaredEngine) {
		throw new RenderError(
			"ENGINE_NOT_FOUND",
			"The project doesn't depend on framefields.",
			{
				phase: "install",
				hint: 'Add "framefields" to the dependencies in package.json and commit the lockfile.',
			},
		);
	}
	checkNodeEngine(project, ctx);

	if (!project.lockfile && !ctx.config.install.allowUnlocked) {
		throw new RenderError(
			"LOCKFILE_MISSING",
			"The project has no lockfile, so the install wouldn't be reproducible.",
			{
				phase: "install",
				hint: "Commit pnpm-lock.yaml / package-lock.json / yarn.lock (or set FF_ALLOW_UNLOCKED_INSTALL=true).",
			},
		);
	}

	const stampPath = path.join(project.installRoot, STAMP_FILE);
	const stamp = await installStamp(project, ctx.config, checkout.root);
	const previous = await fs.readFile(stampPath, "utf8").catch(() => "");
	if (
		previous === stamp &&
		existsSync(path.join(project.installRoot, "node_modules"))
	) {
		return {
			skipped: false,
			packageManager: project.manager,
			declaredEngine,
			cache: { install: "hit" },
		};
	}

	// A half-finished install must never be trusted by the next job.
	await fs.rm(stampPath, { force: true });
	const shared = ctx.config.paths.sharedModules;
	// Shared files are read-only hard links: a package manager must never
	// write into them, so the install starts from an empty node_modules.
	if (shared) await removeModules(checkout.root);
	// Room for a full extraction: sharing happens only after the install.
	const expected = await expectedInstallSize(ctx.config.paths.workDir, stamp);
	if (expected > 0) {
		await activeJanitor()?.enforce(
			[path.basename(path.dirname(checkout.root))],
			expected,
		);
	}
	await runInstall(project, ctx, checkout.root);
	if (shared) {
		const result = await shareModules(
			checkout.root,
			moduleStoreDir(ctx.config.paths.workDir),
		);
		await recordInstallSize(
			ctx.config.paths.workDir,
			stamp,
			result.installBytes,
		);
		rendererLogger.info(
			{
				files: result.files,
				linked: result.linked,
				savedMb: Math.round(result.savedBytes / 1024 ** 2),
				skipped: result.skipped,
			},
			"Shared node_modules with other workspaces",
		);
	}
	await fs.writeFile(stampPath, stamp);

	// This process holds the old dependencies in its module cache; only a
	// fresh worker can load the new ones (engine-alias.ts).
	if (depsLoadedFrom(checkout.root)) {
		workerHealth.requestRecycle("project dependencies changed");
		throw new RenderError(
			"ENGINE_SWITCH",
			"The project's dependencies changed since this worker loaded them.",
			{
				phase: "install",
				hint: "Retry: the worker restarts and loads the new dependencies (already installed).",
			},
		);
	}
	return {
		skipped: false,
		packageManager: project.manager,
		declaredEngine,
		cache: { install: "miss" },
	};
};

function checkNodeEngine(project: ProjectInfo, ctx: StepContext): void {
	const range = project.pkg.engines?.node;
	if (
		range &&
		semver.validRange(range) &&
		!semver.satisfies(process.versions.node, range)
	) {
		ctx.warn(
			"NODE_ENGINE_MISMATCH",
			`The project wants Node ${range}; the worker runs ${process.versions.node}.`,
		);
	}
}

/** Everything that changes what an install produces. */
async function installStamp(
	project: ProjectInfo,
	config: RendererConfig,
	repoRoot: string,
): Promise<string> {
	const hash = createHash("sha256");
	hash.update(
		project.lockfile ? await fs.readFile(project.lockfile) : "no-lockfile",
	);
	hash.update(
		await fs.readFile(path.join(project.installRoot, "package.json")),
	);
	if (project.inWorkspace)
		hash.update(
			await fs.readFile(path.join(project.packageDir, "package.json")),
		);
	hash.update(
		JSON.stringify({
			manager: project.managerSpec ?? project.manager,
			node: process.versions.modules,
			platform: `${process.platform}-${process.arch}`,
			scripts: config.install.scripts,
			filter: path.relative(repoRoot, project.packageDir),
			npmrc: config.install.npmrc
				? createHash("sha256").update(config.install.npmrc).digest("hex")
				: "",
		}),
	);
	return hash.digest("hex");
}

function installCommand(
	project: ProjectInfo,
	config: RendererConfig,
): string[] {
	const cache = config.paths.cacheDir;
	const rel = path.relative(project.installRoot, project.packageDir) || ".";
	const noScripts = !config.install.scripts;
	switch (project.manager) {
		case "pnpm":
			return [
				"pnpm",
				"install",
				project.lockfile ? "--frozen-lockfile" : "--no-frozen-lockfile",
				"--prefer-offline",
				"--reporter=append-only",
				"--config.confirmModulesPurge=false",
				`--store-dir=${path.join(cache, "pnpm-store")}`,
				...(project.inWorkspace ? [`--filter=./${rel}...`] : []),
				...(noScripts ? ["--ignore-scripts"] : []),
			];
		case "npm":
			return [
				"npm",
				project.lockfile ? "ci" : "install",
				"--prefer-offline",
				"--no-audit",
				"--no-fund",
				`--cache=${path.join(cache, "npm-cache")}`,
				...(project.inWorkspace
					? [`--workspace=${rel}`, "--include-workspace-root=false"]
					: []),
				...(noScripts ? ["--ignore-scripts"] : []),
			];
		case "yarn":
			return project.yarnBerry
				? [
						"yarn",
						"install",
						project.lockfile ? "--immutable" : "--no-immutable",
						"--inline-builds",
					]
				: [
						"yarn",
						"install",
						project.lockfile ? "--frozen-lockfile" : "--no-lockfile",
						"--prefer-offline",
						"--non-interactive",
						`--cache-folder=${path.join(cache, "yarn-cache")}`,
						...(noScripts ? ["--ignore-scripts"] : []),
					];
	}
}

async function runInstall(
	project: ProjectInfo,
	ctx: StepContext,
	repoRoot: string,
): Promise<void> {
	const config = ctx.config;
	const home = await fs.mkdtemp(path.join(os.tmpdir(), "ff-install-"));
	try {
		const env: NodeJS.ProcessEnv = {
			PATH: process.env.PATH,
			HOME: home,
			TMPDIR: home,
			CI: "true",
			LANG: "C.UTF-8",
			TZ: "UTC",
			COREPACK_ENABLE_DOWNLOAD_PROMPT: "0",
			COREPACK_ENABLE_STRICT: "0",
			COREPACK_HOME: path.join(config.paths.cacheDir, "corepack"),
			npm_config_update_notifier: "false",
			YARN_ENABLE_TELEMETRY: "0",
			YARN_ENABLE_GLOBAL_CACHE: "true",
			YARN_GLOBAL_FOLDER: path.join(config.paths.cacheDir, "yarn-berry"),
			...(project.yarnBerry &&
				!config.install.scripts && { YARN_ENABLE_SCRIPTS: "false" }),
		};
		if (config.install.npmrc) {
			const npmrc = path.join(home, ".npmrc");
			await fs.writeFile(npmrc, config.install.npmrc, { mode: 0o600 });
			env.npm_config_userconfig = npmrc;
			env.NPM_CONFIG_USERCONFIG = npmrc;
		}

		const [pm, ...args] = installCommand(project, config);
		// npm ships with Node, so corepack only runs it when the project pins a
		// version; an unpinned npm via corepack can be a different major than
		// the one that wrote the lockfile. pnpm and yarn always go through it.
		const useCorepack =
			(pm !== "npm" || Boolean(project.managerSpec)) && (await hasCorepack());
		const file = useCorepack ? "corepack" : pm;
		const argv = useCorepack ? [pm, ...args] : args;

		ctx.progress({
			phase: "installing",
			message: `${project.managerSpec ?? pm} ${args[0]}`,
		});
		const attempt = () =>
			exec(file, argv, {
				cwd: project.installRoot,
				env,
				signal: ctx.signal,
				timeoutMs: config.limits.timeouts.install * 1000,
				maxStdoutBytes: 1024 * 1024,
			});
		try {
			await attempt();
		} catch (err) {
			const classified = classifyInstallError(err, repoRoot);
			// One retry for registry hiccups; everything else is deterministic.
			if (classified.details?.transient && !ctx.signal.aborted) {
				await attempt().catch((retryErr) => {
					throw classifyInstallError(retryErr, repoRoot);
				});
			} else {
				throw classified;
			}
		}
	} finally {
		await fs.rm(home, { recursive: true, force: true });
	}
}

let corepackAvailable: Promise<boolean> | undefined;
function hasCorepack(): Promise<boolean> {
	corepackAvailable ??= exec("corepack", ["--version"], {
		timeoutMs: 10_000,
	}).then(
		() => true,
		() => false,
	);
	return corepackAvailable;
}

export function classifyInstallError(
	err: unknown,
	repoRoot: string,
): RenderError {
	if (err instanceof RenderError) return err;
	if (!(err instanceof CommandError)) {
		const message = err instanceof Error ? err.message : String(err);
		if (/ENOENT/.test(message)) {
			return new RenderError(
				"TOOL_MISSING",
				`Package manager not found: ${message}`,
				{ phase: "install" },
			);
		}
		return new RenderError("INSTALL_FAILED", redact(message), {
			phase: "install",
			cause: err,
		});
	}
	const output = `${err.result.stderr}\n${err.result.stdout}`;
	// Package managers put the cause first and usage/help text last: keep both ends.
	const lines = output
		.trim()
		.split("\n")
		.filter((l) => !/^npm error\s*$/.test(l));
	const excerpt =
		lines.length > 45
			? [...lines.slice(0, 15), "…", ...lines.slice(-30)]
			: lines;
	const logTail = redact(excerpt.join("\n")).replaceAll(repoRoot, "<repo>");
	// The first line that names the problem, e.g. npm's "`npm ci` can only install…".
	const cause =
		lines.find(
			(l) =>
				/ERR_PNPM_|error (?!code\b)|YN\d{4}|ERR!/i.test(l) &&
				l.replace(/^npm error\s*/, "").length > 12,
		) ??
		lines.at(-1) ??
		"unknown error";
	const fail = (hint?: string, transient = false) =>
		new RenderError(
			"INSTALL_FAILED",
			`${err.command} failed: ${redact(cause).replaceAll(repoRoot, "<repo>")}`,
			{
				phase: "install",
				hint,
				details: { log: logTail, ...(transient && { transient }) },
				cause: err,
			},
		);

	if (err.result.timedOut) {
		return new RenderError(
			"INSTALL_TIMEOUT",
			"The dependency install timed out.",
			{
				phase: "install",
				hint: "Raise FF_TIMEOUT_INSTALL_SEC, or keep a network volume so the package store stays warm.",
				details: { log: logTail },
			},
		);
	}
	if (/ENOSPC|No space left on device/i.test(output)) {
		return new RenderError("DISK_FULL", "Ran out of disk during the install.", {
			phase: "install",
			hint: "The install extracts every package before sharing them; give FF_WORK_DIR more disk (containerDiskInGb on RunPod).",
		});
	}
	if (
		/ERR_PNPM_OUTDATED_LOCKFILE|ERR_PNPM_LOCKFILE|frozen-lockfile|can only install packages when your package\.json and package-lock\.json|lockfile would have been modified|YN0028/i.test(
			output,
		)
	) {
		return fail(
			"The lockfile is out of date with package.json; run the install locally and commit the lockfile.",
		);
	}
	if (
		/E401|E403|ERR_PNPM_FETCH_40[13]|Unauthorized|authentication token/i.test(
			output,
		)
	) {
		return fail(
			"A private registry rejected the request; set FF_NPMRC with an auth token.",
		);
	}
	if (
		/E404|ERR_PNPM_FETCH_404|ERR_PNPM_NO_MATCHING_VERSION|Not Found - GET/i.test(
			output,
		)
	) {
		return fail("A dependency (or version) doesn't exist on the registry.");
	}
	if (
		/node-gyp|gyp ERR|make: \*\*\*|g\+\+|cc1plus|Python is not set/i.test(
			output,
		)
	) {
		return fail(
			"A native dependency needed compiling; use the -full image, which has a toolchain.",
		);
	}
	if (
		/ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|socket hang up|ERR_SOCKET_TIMEOUT|\b50[234]\b/i.test(
			output,
		)
	) {
		return fail("The package registry was unreachable.", true);
	}
	return fail();
}