Skip to content

Code-first video & image, rendered natively on WebGPU.

5.3 KB
Raw Blame
import { constants, existsSync } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import {
	type AssetEntry,
	AssetError,
	downloadVerified,
	hashFile,
	ManifestError,
	readProjectManifest,
	readUrl,
	type StorageConfig,
} from "@framefields/storage";
import { RenderError } from "../errors.js";
import { resolveInside } from "../paths.js";

/** Under the cache dir: verified asset bytes at assets/aa/<sha256>. */
export const ASSET_CACHE = "assets";
const CONCURRENCY = 4;

export interface AssetOptions {
	cacheDir: string;
	/** Signed GET URLs for private assets, by sha256 (`source.assets`). */
	signed?: Readonly<Record<string, string>>;
	signal?: AbortSignal;
	/** Evicts caches so `bytes` more fit (the janitor). */
	makeRoom?: (bytes: number) => Promise<void>;
	fetch?: typeof fetch;
}

export interface AssetResult {
	files: number;
	bytes: number;
	cache: "hit" | "partial" | "miss" | "off";
}

/**
 * Pipeline step after LFS: puts every asset the project's framefields.json
 * pins in place, from the worker's cache or its URL (the remote's public
 * domain, the asset's own URL, or a signature the job carries), checked
 * against its sha256.
 */
export async function fetchAssets(
	workdir: string,
	options: AssetOptions,
): Promise<AssetResult> {
	const manifest = await readManifest(workdir);
	const entries = Object.entries(manifest?.assets ?? {});
	if (!manifest || entries.length === 0)
		return { files: 0, bytes: 0, cache: "off" };

	const plan = await Promise.all(
		entries.map(async ([rel, entry]) => ({
			rel,
			entry,
			dest: path.join(workdir, rel),
			cached: cachePath(options.cacheDir, entry.sha256),
		})),
	);
	const todo: typeof plan = [];
	for (const item of plan)
		if (!(await present(item.dest, item.entry))) todo.push(item);
	const misses = todo.filter((t) => !existsSync(t.cached));
	const missBytes = misses.reduce((n, t) => n + t.entry.size, 0);
	if (missBytes > 0) await options.makeRoom?.(missBytes);

	let next = 0;
	const worker = async () => {
		while (next < todo.length) {
			const item = todo[next++];
			if (!existsSync(item.cached))
				await download(
					item.rel,
					item.entry,
					item.cached,
					manifest.storage,
					options,
				);
			await place(workdir, item.rel, item.cached);
		}
	};
	await Promise.all(
		Array.from({ length: Math.min(CONCURRENCY, todo.length) }, worker),
	);
	return {
		files: entries.length,
		bytes: missBytes,
		cache:
			misses.length === 0
				? "hit"
				: misses.length === entries.length
					? "miss"
					: "partial",
	};
}

async function readManifest(workdir: string) {
	try {
		return await readProjectManifest(workdir);
	} catch (err) {
		if (err instanceof ManifestError)
			throw new RenderError("ASSET_MANIFEST_INVALID", err.message, {
				hint: "`npx framefields assets status` reads the same file locally.",
			});
		throw err;
	}
}

/**
 * Copies the cached bytes to `rel`, refusing folders that symlink out of
 * the checkout and replacing (not writing through) a symlink at `rel`.
 */
async function place(workdir: string, rel: string, cached: string) {
	let existing = path.posix.dirname(rel);
	while (existing !== "." && !existsSync(path.join(workdir, existing)))
		existing = path.posix.dirname(existing);
	await resolveInside(workdir, existing);
	const dest = path.join(workdir, rel);
	await fs.mkdir(path.dirname(dest), { recursive: true });
	await fs.rm(dest, { force: true });
	await fs.copyFile(cached, dest, constants.COPYFILE_FICLONE);
}

/** A file already in the checkout counts when its bytes match. */
async function present(dest: string, entry: AssetEntry): Promise<boolean> {
	const stat = await fs.lstat(dest).catch(() => undefined);
	if (!stat?.isFile() || stat.size !== entry.size) return false;
	return (await hashFile(dest)).sha256 === entry.sha256;
}

async function download(
	rel: string,
	entry: AssetEntry,
	cached: string,
	storage: StorageConfig | undefined,
	options: AssetOptions,
): Promise<void> {
	const url = readUrl(entry, storage, options.signed);
	if (!url)
		throw new RenderError(
			"ASSET_PRIVATE",
			`${rel} is on a private remote and the job carries no signed URL for it.`,
			{
				hint: "Render from the CLI or renderCloud (they sign reads with your keys), or give the remote a publicUrl.",
			},
		);
	try {
		await downloadVerified(url, cached, entry, {
			fetch: options.fetch,
			signal: options.signal,
		});
	} catch (err) {
		if (!(err instanceof AssetError)) throw err;
		throw new RenderError(ERROR_CODES[err.code], `${rel}: ${err.message}`, {
			hint: HINTS[err.code],
			cause: err,
		});
	}
}

const ERROR_CODES = {
	ASSET_NOT_FOUND: "ASSET_NOT_UPLOADED",
	ASSET_FORBIDDEN: "ASSET_FORBIDDEN",
	ASSET_DOWNLOAD_FAILED: "ASSET_DOWNLOAD_FAILED",
	ASSET_HASH_MISMATCH: "ASSET_HASH_MISMATCH",
} as const;

const HINTS: Record<AssetError["code"], string> = {
	ASSET_NOT_FOUND: "Run `npx framefields assets push` and render again.",
	ASSET_FORBIDDEN:
		"The publicUrl doesn't serve the bucket, or a signed URL expired: render again from the CLI.",
	ASSET_DOWNLOAD_FAILED: "The bucket or CDN didn't answer; retried.",
	ASSET_HASH_MISMATCH:
		"The bucket holds other bytes than framefields.json pins: `npx framefields assets push` from the machine with the right file.",
};

function cachePath(cacheDir: string, sha256: string): string {
	return path.join(cacheDir, ASSET_CACHE, sha256.slice(0, 2), sha256);
}