Skip to content

Code-first video & image, rendered natively on WebGPU.

11.2 KB
Raw Blame
/**
 * pnpm audio       → assets/voice/<line>.wav, assets/music.wav, assets/sfx/click.wav,
 *                    and what was measured in assets/audio.json
 * pnpm audio cost  → print the estimate, no calls
 *
 * Gemini 3.8 Flash TTS reads each line of the narration, Lyria 3.5 plays the
 * music bed and Seed Audio 1.0 makes the click. Scribe v2 checks that every
 * line was read as written and finds its first and last word, so each line is
 * trimmed to its speech and the film can start it exactly where it belongs;
 * it also checks the music has no voice in it. Idempotent: delete a file to
 * make it again. Raw takes are kept in assets/takes/.
 */
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { existsSync, mkdirSync } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { ai } from "framefields";
import {
	type AudioManifest,
	CLICK_MODEL,
	CLICK_PATH,
	CLICK_PROMPT,
	LINE_IDS,
	LINES,
	type LineId,
	MANIFEST,
	MUSIC_MODEL,
	MUSIC_PATH,
	MUSIC_PROMPT,
	MUSIC_SECONDS,
	TRANSCRIBER,
	VOICE_LEAD,
	VOICE_MODEL,
	voiceInput,
	voicePath,
} from "./audio.js";
import { ASSETS } from "./theme.js";

/**
 * The gateway's rates (November 2026: fal's price plus 25 %): per minute of
 * audio, per 1000 characters read, and per music track.
 */
const PER_MINUTE = { [CLICK_MODEL]: 0.234375, [TRANSCRIBER]: 0.0125 };
const PER_1000_CHARACTERS = 0.05625;
const PER_TRACK = 0.125;
const SAMPLE_RATE = 48000;
const MAX_TAKES = 4;
const TAKES = path.join(ASSETS, "takes");
/** Kept after a line's last word, faded over its second half. */
const VOICE_TAIL = 0.3;
const CLICK_SECONDS = 0.12;
/** Longest silence allowed between two words of a sentence; a full stop may hold twice as long. */
const MAX_PAUSE = 0.45;
/** A click puts at least this share of its energy above 1.5 kHz. */
const MIN_BRIGHTNESS = 0.25;

/** A take's file, named by what was asked for, so a changed prompt is made afresh. */
const takePath = (
	kind: string,
	input: Record<string, unknown>,
	take: number,
	ext = "wav",
) =>
	path.join(
		TAKES,
		`${kind}-${createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 8)}-${take}.${ext}`,
	);

let spent = 0;

/** Runs `model` and saves the audio it returns to `dest`. */
async function generate(
	model: string,
	input: Record<string, unknown>,
	dest: string,
): Promise<void> {
	const { data } = await ai.subscribe<{ audio: { url: string } }>(model, {
		input,
	});
	const res = await fetch(data.audio.url);
	if (!res.ok) throw new Error(`GET ${data.audio.url} → ${res.status}`);
	await fs.mkdir(path.dirname(dest), { recursive: true });
	await fs.writeFile(dest, Buffer.from(await res.arrayBuffer()));
}

interface Word {
	text: string;
	start: number;
	end: number;
}

async function transcribe(file: string): Promise<Word[]> {
	const audio = new Blob([await fs.readFile(file)], { type: "audio/wav" });
	const { data } = await ai.subscribe<{
		words: Array<Word & { type: string }>;
	}>(TRANSCRIBER, {
		input: {
			audio_url: audio,
			language_code: "eng",
			keyterms: ["Relay", "Maya"],
			tag_audio_events: false,
		},
	});
	spent += (PER_MINUTE[TRANSCRIBER] * duration(file)) / 60;
	return data.words
		.filter((w) => w.type === "word")
		.map(({ text, start, end }) => ({ text: text.trim(), start, end }));
}

function duration(file: string): number {
	const out = execFileSync("ffprobe", [
		"-v",
		"error",
		"-show_entries",
		"format=duration",
		"-of",
		"csv=p=0",
		file,
	]);
	return Number(out.toString().trim());
}

/** Mono float samples at the film's rate. */
function decode(file: string): Float32Array {
	const raw = execFileSync(
		"ffmpeg",
		[
			"-v",
			"error",
			"-i",
			file,
			"-ac",
			"1",
			"-ar",
			`${SAMPLE_RATE}`,
			"-f",
			"f32le",
			"pipe:1",
		],
		{ maxBuffer: 1 << 28 },
	);
	return new Float32Array(raw.buffer, raw.byteOffset, raw.byteLength / 4);
}

/** Cuts `from`–`to` seconds of `src` through `filters` into a 48 kHz mono WAV. */
function cut(
	src: string,
	dest: string,
	from: number,
	to: number,
	filters: string[],
) {
	mkdirSync(path.dirname(dest), { recursive: true });
	execFileSync("ffmpeg", [
		"-v",
		"error",
		"-y",
		"-ss",
		from.toFixed(3),
		"-to",
		to.toFixed(3),
		"-i",
		src,
		"-af",
		filters.join(","),
		"-ar",
		`${SAMPLE_RATE}`,
		"-ac",
		"1",
		dest,
	]);
}

/** The longest gap between words, counting a gap after a full stop at half its length. */
function longestPause(heard: Word[]): number {
	let longest = 0;
	for (let i = 1; i < heard.length; i++) {
		const before = heard[i - 1];
		const word = heard[i];
		if (!before || !word) continue;
		const gap = word.start - before.end;
		longest = Math.max(longest, before.text.endsWith(".") ? gap / 2 : gap);
	}
	return longest;
}

const words = (s: string) =>
	s
		.toLowerCase()
		.replace(/[^a-z' ]/g, " ")
		.split(/\s+/)
		.filter(Boolean)
		.join(" ");

async function voiceLine(id: LineId): Promise<AudioManifest["voice"][LineId]> {
	const text = LINES[id];
	for (let take = 1; take <= MAX_TAKES; take++) {
		const input = voiceInput(id);
		const raw = takePath(`voice-${id}`, input, take);
		if (!existsSync(raw)) {
			await generate(VOICE_MODEL, input, raw);
			spent += (PER_1000_CHARACTERS * text.length) / 1000;
		}
		const heard = await transcribe(raw);
		const said = heard.map((w) => w.text).join(" ");
		const first = heard[0];
		const last = heard.at(-1);
		// Same words, no question where the script states something, no long pauses.
		const asks = said.includes("?") && !text.includes("?");
		const pause = longestPause(heard);
		if (
			!first ||
			!last ||
			words(said) !== words(text) ||
			asks ||
			pause > MAX_PAUSE
		) {
			console.log(
				`voice  ${id} take ${take} heard "${said}" (longest pause ${pause.toFixed(2)} s), retrying`,
			);
			continue;
		}
		const from = Math.max(0, first.start - VOICE_LEAD);
		const to = Math.min(duration(raw), last.end + VOICE_TAIL);
		const length = to - from;
		cut(raw, voicePath(id), from, to, [
			"afade=t=in:d=0.01",
			`afade=t=out:st=${(length - VOICE_TAIL / 2).toFixed(3)}:d=${(VOICE_TAIL / 2).toFixed(3)}`,
			"loudnorm=I=-16:TP=-1.5:LRA=11",
		]);
		const seconds = duration(voicePath(id));
		console.log(`voice  ${id} ${seconds.toFixed(2)} s "${said}"`);
		return { text, input, heard: said, seconds };
	}
	throw new Error(
		`No take of "${text}" was read as written in ${MAX_TAKES} tries.`,
	);
}

async function music(): Promise<AudioManifest["music"]> {
	const input = { prompt: MUSIC_PROMPT };
	for (let take = 1; take <= MAX_TAKES; take++) {
		const raw = takePath("music", input, take, "mp3");
		if (!existsSync(raw)) {
			await generate(MUSIC_MODEL, input, raw);
			spent += PER_TRACK;
		}
		const length = duration(raw);
		const heard = await transcribe(raw);
		if (heard.length > 0 || length < MUSIC_SECONDS) {
			console.log(
				`music  take ${take}: ${length.toFixed(1)} s, heard "${heard.map((w) => w.text).join(" ")}", retrying`,
			);
			continue;
		}
		cut(raw, MUSIC_PATH, 0, MUSIC_SECONDS, [
			`afade=t=out:st=${(MUSIC_SECONDS - 1.5).toFixed(3)}:d=1.5`,
			"loudnorm=I=-18:TP=-2:LRA=11",
		]);
		const seconds = duration(MUSIC_PATH);
		console.log(`music  ${seconds.toFixed(1)} s of ${length.toFixed(1)} s`);
		return { prompt: MUSIC_PROMPT, seconds };
	}
	throw new Error(
		`No instrumental music take of ${MUSIC_SECONDS} s or more in ${MAX_TAKES} tries.`,
	);
}

/** Share of energy above ~1.5 kHz, through a one-pole high-pass. */
function brightness(x: Float32Array): number {
	const rc = 1 / (2 * Math.PI * 1500);
	const a = rc / (rc + 1 / SAMPLE_RATE);
	let y = 0;
	let high = 0;
	let all = 0;
	for (let i = 1; i < x.length; i++) {
		y = a * (y + (x[i] ?? 0) - (x[i - 1] ?? 0));
		high += y * y;
		all += (x[i] ?? 0) ** 2;
	}
	return all > 0 ? high / all : 0;
}

/** The loudest transient of a take: where it starts and how far it can be raised. */
function strike(x: Float32Array): { onset: number; gainDb: number } {
	let peak = 0;
	for (let i = 1; i < x.length; i++)
		if (Math.abs(x[i] ?? 0) > Math.abs(x[peak] ?? 0)) peak = i;
	const level = Math.abs(x[peak] ?? 0);
	let onset = peak;
	const earliest = Math.max(0, peak - Math.round(0.02 * SAMPLE_RATE));
	for (let i = peak; i >= earliest; i--)
		if (Math.abs(x[i] ?? 0) > level * 0.1) onset = i;
	return { onset: onset / SAMPLE_RATE, gainDb: -3 - 20 * Math.log10(level) };
}

async function click(): Promise<AudioManifest["click"]> {
	for (let take = 1; take <= MAX_TAKES; take++) {
		const input = { prompt: CLICK_PROMPT };
		const raw = takePath("click", input, take);
		if (!existsSync(raw)) {
			await generate(
				CLICK_MODEL,
				{ output_format: "wav", sample_rate: SAMPLE_RATE, ...input },
				raw,
			);
			spent += (PER_MINUTE[CLICK_MODEL] * duration(raw)) / 60;
		}
		const { onset, gainDb } = strike(decode(raw));
		const from = Math.max(0, onset - 0.002);
		cut(raw, CLICK_PATH, from, from + CLICK_SECONDS, [
			`afade=t=out:st=${(CLICK_SECONDS / 2).toFixed(3)}:d=${(CLICK_SECONDS / 2).toFixed(3)}`,
			`volume=${gainDb.toFixed(2)}dB`,
		]);
		const bright = brightness(decode(CLICK_PATH));
		if (bright < MIN_BRIGHTNESS) {
			console.log(
				`click  take ${take} is a thump (${(bright * 100).toFixed(0)} % above 1.5 kHz), retrying`,
			);
			continue;
		}
		console.log(`click  ${(bright * 100).toFixed(0)} % above 1.5 kHz`);
		return {
			seconds: duration(CLICK_PATH),
			brightness: Number(bright.toFixed(3)),
		};
	}
	throw new Error(`No click take was bright enough in ${MAX_TAKES} tries.`);
}

async function readPartial(): Promise<Partial<AudioManifest>> {
	if (!existsSync(MANIFEST)) return {};
	return JSON.parse(
		await fs.readFile(MANIFEST, "utf8"),
	) as Partial<AudioManifest>;
}

async function save(manifest: Partial<AudioManifest>): Promise<void> {
	await fs.writeFile(MANIFEST, `${JSON.stringify(manifest, null, "\t")}\n`);
}

async function main(): Promise<void> {
	process.loadEnvFile(new URL("../../../.env", import.meta.url).pathname);
	if (!process.env.FRAMEFIELDS_API_KEY)
		throw new Error("FRAMEFIELDS_API_KEY missing from the repo root .env");
	const manifest = await readPartial();
	const voice: Partial<AudioManifest["voice"]> = {};
	for (const id of LINE_IDS) {
		const kept = manifest.voice?.[id];
		const same = JSON.stringify(kept?.input) === JSON.stringify(voiceInput(id));
		if (kept && same && existsSync(voicePath(id))) {
			voice[id] = kept;
			continue;
		}
		voice[id] = await voiceLine(id);
		await save({ ...manifest, voice: voice as AudioManifest["voice"] });
	}
	manifest.voice = voice as AudioManifest["voice"];
	if (manifest.music?.prompt !== MUSIC_PROMPT || !existsSync(MUSIC_PATH)) {
		manifest.music = await music();
		await save(manifest);
	}
	if (!manifest.click || !existsSync(CLICK_PATH)) {
		manifest.click = await click();
		await save(manifest);
	}
	console.log(`spent  ~$${spent.toFixed(3)}`);
}

if (process.argv[2] === "cost") {
	const characters = Object.values(LINES).join("").length;
	const speech = Object.values(LINES).join(" ").split(" ").length / 2.9;
	const usd =
		(PER_1000_CHARACTERS * characters) / 1000 +
		PER_TRACK +
		(PER_MINUTE[CLICK_MODEL] * 5) / 60 +
		(PER_MINUTE[TRANSCRIBER] * (speech + 30)) / 60;
	console.log(
		`estimate $${usd.toFixed(2)} per pass (${characters} characters of narration, a ${MUSIC_SECONDS} s music track, a click), up to ${MAX_TAKES}× with retries`,
	);
} else {
	await main();
}
process.exit(0);