Skip to content

Code-first video & image, rendered natively on WebGPU.

4.5 KB
Raw Blame
/**
 * Asset pipeline for the Framefields film.
 *
 *   pnpm assets            → fonts, plates (gpt-image-2), shots (h3-max / h3-max-turbo), score (ElevenLabs)
 *   pnpm assets images     → only the stills
 *   pnpm assets videos     → only the image-to-video shots
 *   pnpm assets music      → only the score
 *
 * Every step is idempotent: existing files are kept, delete one to regenerate it.
 */
import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import dotenv from "dotenv";
import { ai } from "framefields";
import { SCORE, SHOTS } from "./shots.js";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(HERE, "../../..");
const ASSETS = path.resolve(HERE, "../assets");
dotenv.config({ path: path.join(ROOT, ".env"), quiet: true });

function checkApiKey() {
	const key = process.env.FRAMEFIELDS_API_KEY;
	if (!key) throw new Error("FRAMEFIELDS_API_KEY missing from .env");
}

const FONTS: Record<string, string> = {
	"InstrumentSerif-Regular.ttf":
		"https://github.com/google/fonts/raw/main/ofl/instrumentserif/InstrumentSerif-Regular.ttf",
	"InstrumentSerif-Italic.ttf":
		"https://github.com/google/fonts/raw/main/ofl/instrumentserif/InstrumentSerif-Italic.ttf",
	"JetBrainsMono.ttf":
		"https://github.com/google/fonts/raw/main/ofl/jetbrainsmono/JetBrainsMono%5Bwght%5D.ttf",
};

const exists = (p: string) =>
	fs.access(p).then(
		() => true,
		() => false,
	);

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

/** fetch that rides out transient network failures (long polls drop connections). */
async function request(url: string, init?: RequestInit): Promise<Response> {
	for (let attempt = 1; ; attempt++) {
		try {
			return await fetch(url, init);
		} catch (err) {
			if (attempt === 5) throw err;
			await sleep(2000 * attempt);
		}
	}
}

async function download(url: string, dest: string): Promise<void> {
	const res = await request(url);
	if (!res.ok) throw new Error(`GET ${url} → ${res.status}`);
	await fs.mkdir(path.dirname(dest), { recursive: true });
	await fs.writeFile(dest, Buffer.from(await res.arrayBuffer()));
}

async function fonts() {
	const dir = path.join(ROOT, "assets/fonts");
	for (const [file, url] of Object.entries(FONTS)) {
		const dest = path.join(dir, file);
		if (await exists(dest)) continue;
		await download(url, dest);
		console.log(`font   ${file}`);
	}
}

async function images() {
	checkApiKey();
	await Promise.all(
		SHOTS.map(async (shot) => {
			const dest = path.join(ASSETS, `${shot.id}.png`);
			if (await exists(dest)) return;
			const t0 = Date.now();
			const { data: out } = await ai.subscribe<{ images: { url: string }[] }>(
				"openai/gpt-image-2",
				{
					input: {
						prompt: shot.image,
						image_size: { width: 2048, height: 1152 },
						quality: "high",
						output_format: "png",
					},
				},
			);
			await download(out.images[0].url, dest);
			console.log(
				`image  ${shot.id} (${((Date.now() - t0) / 1000).toFixed(0)}s)`,
			);
		}),
	);
}

/** The gateway accepts data URIs for image inputs, so the local plate is sent as-is. */
async function videos() {
	checkApiKey();
	await Promise.all(
		SHOTS.map(async (shot) => {
			const dest = path.join(ASSETS, `${shot.id}.mp4`);
			if (await exists(dest)) return;
			const plate = await fs.readFile(path.join(ASSETS, `${shot.id}.png`));
			const t0 = Date.now();
			const { data: out } = await ai.subscribe<{ video: { url: string } }>(
				shot.model ?? "minimax/h3-max-turbo/image-to-video",
				{
					input: {
						prompt: shot.motion,
						image_url: `data:image/png;base64,${plate.toString("base64")}`,
						duration: shot.seconds,
						resolution: "1080P",
						prompt_expansion_mode: "disabled",
					},
				},
			);
			await download(out.video.url, dest);
			console.log(
				`video  ${shot.id} (${((Date.now() - t0) / 1000).toFixed(0)}s)`,
			);
		}),
	);
}

async function music() {
	checkApiKey();
	const dest = path.join(ASSETS, "score.mp3");
	if (await exists(dest)) return;
	const t0 = Date.now();
	const { data: out } = await ai.subscribe<{ audio: { url: string } }>(
		"fal-ai/elevenlabs/music",
		{
			input: {
				composition_plan: SCORE,
				respect_sections_durations: true,
			},
		},
	);
	await download(out.audio.url, dest);
	console.log(`music  score (${((Date.now() - t0) / 1000).toFixed(0)}s)`);
}

const step = process.argv[2];
await fs.mkdir(ASSETS, { recursive: true });
if (!step || step === "fonts") await fonts();
if (!step || step === "images") await images();
if (!step || step === "videos") await videos();
if (!step || step === "music") await music();