Skip to content

Code-first video & image, rendered natively on WebGPU.

9.1 KB
Raw Blame
/**
 * pnpm audio       → assets/voice/<line>.wav, assets/music.wav, and what was
 *                    measured in assets/audio.json
 * pnpm audio cost  → print the estimate, no calls
 *
 * Gemini 3.8 Flash TTS reads each line of the narration and Lyria 3.5 plays
 * the bed. Scribe v2 checks that every line was read as written and finds its
 * first and last word, so each line is trimmed to its speech and the film can
 * start it exactly where it belongs; it also checks the music has no voice in
 * it. Idempotent: delete a file to make it again. Raw takes are kept in
 * assets/takes/.
 */
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { existsSync, mkdirSync } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { ai } from "framefields";
import {
	type AudioManifest,
	LINE_IDS,
	LINES,
	type LineId,
	MANIFEST,
	MUSIC_MODEL,
	MUSIC_PATH,
	MUSIC_PROMPT,
	MUSIC_SECONDS,
	type SpokenWord,
	TRANSCRIBER,
	VOICE_LEAD,
	VOICE_MODEL,
	voiceInput,
	voicePath,
} from "./audio.js";
import { spokenWords } from "./spoken.js";
import { ASSETS } from "./theme.js";

/**
 * The gateway's rates (November 2026: fal's price plus 25 %): per minute
 * transcribed, per 1000 characters read, and per music track.
 */
const PER_MINUTE = 0.0125;
const PER_1000_CHARACTERS = 0.05625;
const PER_TRACK = 0.125;
const SAMPLE_RATE = 48000;
const MAX_TAKES = 4;
const TAKES = path.join(ASSETS, "takes");
/** Kept after a line's last word, faded over its second half. */
const VOICE_TAIL = 0.3;
/** Longest silence allowed between two words of a sentence; punctuation may hold longer (`HOLD`). */
const MAX_PAUSE = 0.5;
const KEYTERMS = [
	"Learjet",
	"Eleuthera",
	"Palm Beach",
	"FAA",
	"Military Trail",
];

/** A take's file, named by what was asked for, so a changed prompt is made afresh. */
const takePath = (
	kind: string,
	input: Record<string, unknown>,
	take: number,
	ext = "wav",
) =>
	path.join(
		TAKES,
		`${kind}-${createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 8)}-${take}.${ext}`,
	);

let spent = 0;

/** Runs `model` and saves the audio it returns to `dest`. */
async function generate(
	model: string,
	input: Record<string, unknown>,
	dest: string,
): Promise<void> {
	const { data } = await ai.subscribe<{ audio: { url: string } }>(model, {
		input,
	});
	const res = await fetch(data.audio.url);
	if (!res.ok) throw new Error(`GET ${data.audio.url} → ${res.status}`);
	await fs.mkdir(path.dirname(dest), { recursive: true });
	await fs.writeFile(dest, Buffer.from(await res.arrayBuffer()));
}

interface Word {
	text: string;
	start: number;
	end: number;
}

async function transcribe(file: string): Promise<Word[]> {
	const type = file.endsWith(".mp3") ? "audio/mpeg" : "audio/wav";
	const audio = new Blob([await fs.readFile(file)], { type });
	const { data } = await ai.subscribe<{
		words: Array<Word & { type: string }>;
	}>(TRANSCRIBER, {
		input: {
			audio_url: audio,
			language_code: "eng",
			keyterms: KEYTERMS,
			tag_audio_events: false,
		},
	});
	spent += (PER_MINUTE * duration(file)) / 60;
	return data.words
		.filter((w) => w.type === "word")
		.map(({ text, start, end }) => ({ text: text.trim(), start, end }));
}

function duration(file: string): number {
	const out = execFileSync("ffprobe", [
		"-v",
		"error",
		"-show_entries",
		"format=duration",
		"-of",
		"csv=p=0",
		file,
	]);
	return Number(out.toString().trim());
}

/** Cuts `from`–`to` seconds of `src` through `filters` into a 48 kHz WAV. */
function cut(
	src: string,
	dest: string,
	from: number,
	to: number,
	filters: string[],
	channels: 1 | 2,
) {
	mkdirSync(path.dirname(dest), { recursive: true });
	execFileSync("ffmpeg", [
		"-v",
		"error",
		"-y",
		"-ss",
		from.toFixed(3),
		"-to",
		to.toFixed(3),
		"-i",
		src,
		"-af",
		filters.join(","),
		"-ar",
		`${SAMPLE_RATE}`,
		"-ac",
		`${channels}`,
		dest,
	]);
}

/** Word times moved by `by` seconds, rounded to the millisecond. */
const shift = (heard: Word[], by: number): SpokenWord[] =>
	heard.map(({ text, start, end }) => ({
		text,
		start: Number((start + by).toFixed(3)),
		end: Number((end + by).toFixed(3)),
	}));

/** How much longer than a gap between words a punctuation mark may hold. */
const HOLD = (word: string) =>
	/[.:;]$/.test(word) ? 2 : /,$/.test(word) ? 1.4 : 1;

/** The longest gap between words, each measured against what its punctuation allows. */
function longestPause(heard: Word[]): number {
	let longest = 0;
	for (let i = 1; i < heard.length; i++) {
		const before = heard[i - 1];
		const word = heard[i];
		if (!before || !word) continue;
		longest = Math.max(longest, (word.start - before.end) / HOLD(before.text));
	}
	return longest;
}

async function voiceLine(id: LineId): Promise<AudioManifest["voice"][LineId]> {
	const text = LINES[id];
	for (let take = 1; take <= MAX_TAKES; take++) {
		const input = voiceInput(id);
		const raw = takePath(`voice-${id}`, input, take);
		if (!existsSync(raw)) {
			await generate(VOICE_MODEL, input, raw);
			spent += (PER_1000_CHARACTERS * text.length) / 1000;
		}
		const heard = await transcribe(raw);
		const said = heard.map((w) => w.text).join(" ");
		const first = heard[0];
		const last = heard.at(-1);
		// Same words, no question where the script states something, no long pauses.
		const asks = said.includes("?") && !text.includes("?");
		const pause = longestPause(heard);
		if (
			!first ||
			!last ||
			spokenWords(said) !== spokenWords(text) ||
			asks ||
			pause > MAX_PAUSE
		) {
			console.log(
				`voice  ${id} take ${take} heard "${said}" (longest pause ${pause.toFixed(2)} s), retrying`,
			);
			continue;
		}
		const from = Math.max(0, first.start - VOICE_LEAD);
		const to = Math.min(duration(raw), last.end + VOICE_TAIL);
		const length = to - from;
		cut(
			raw,
			voicePath(id),
			from,
			to,
			[
				"afade=t=in:d=0.01",
				`afade=t=out:st=${(length - VOICE_TAIL / 2).toFixed(3)}:d=${(VOICE_TAIL / 2).toFixed(3)}`,
				"loudnorm=I=-16:TP=-1.5:LRA=11",
			],
			1,
		);
		const seconds = duration(voicePath(id));
		console.log(`voice  ${id} ${seconds.toFixed(2)} s "${said}"`);
		return { text, input, heard: said, seconds, words: shift(heard, -from) };
	}
	throw new Error(
		`No take of "${text}" was read as written in ${MAX_TAKES} tries.`,
	);
}

async function music(): Promise<AudioManifest["music"]> {
	const input = { prompt: MUSIC_PROMPT };
	for (let take = 1; take <= MAX_TAKES; take++) {
		const raw = takePath("music", input, take, "mp3");
		if (!existsSync(raw)) {
			await generate(MUSIC_MODEL, input, raw);
			spent += PER_TRACK;
		}
		const length = duration(raw);
		const heard = await transcribe(raw);
		if (heard.length > 0 || length < MUSIC_SECONDS) {
			console.log(
				`music  take ${take}: ${length.toFixed(1)} s, heard "${heard.map((w) => w.text).join(" ")}", retrying`,
			);
			continue;
		}
		cut(
			raw,
			MUSIC_PATH,
			0,
			MUSIC_SECONDS,
			[
				`afade=t=out:st=${(MUSIC_SECONDS - 3).toFixed(3)}:d=3`,
				"loudnorm=I=-20:TP=-2:LRA=11",
			],
			2,
		);
		const seconds = duration(MUSIC_PATH);
		console.log(`music  ${seconds.toFixed(1)} s of ${length.toFixed(1)} s`);
		return { prompt: MUSIC_PROMPT, seconds };
	}
	throw new Error(
		`No instrumental music take of ${MUSIC_SECONDS} s or more in ${MAX_TAKES} tries.`,
	);
}

async function readPartial(): Promise<Partial<AudioManifest>> {
	if (!existsSync(MANIFEST)) return {};
	return JSON.parse(
		await fs.readFile(MANIFEST, "utf8"),
	) as Partial<AudioManifest>;
}

async function save(manifest: Partial<AudioManifest>): Promise<void> {
	await fs.writeFile(MANIFEST, `${JSON.stringify(manifest, null, "\t")}\n`);
}

async function main(): Promise<void> {
	process.loadEnvFile(new URL("../../../.env", import.meta.url).pathname);
	if (!process.env.FRAMEFIELDS_API_KEY)
		throw new Error("FRAMEFIELDS_API_KEY missing from the repo root .env");
	const manifest = await readPartial();
	const voice: Partial<AudioManifest["voice"]> = {};
	for (const id of LINE_IDS) {
		const kept = manifest.voice?.[id];
		const same = JSON.stringify(kept?.input) === JSON.stringify(voiceInput(id));
		if (kept && same && existsSync(voicePath(id))) {
			// Lines made before word timings were kept: read them off the trimmed file.
			voice[id] = kept.words
				? kept
				: { ...kept, words: shift(await transcribe(voicePath(id)), 0) };
			continue;
		}
		voice[id] = await voiceLine(id);
		await save({ ...manifest, voice: voice as AudioManifest["voice"] });
	}
	manifest.voice = voice as AudioManifest["voice"];
	if (manifest.music?.prompt !== MUSIC_PROMPT || !existsSync(MUSIC_PATH)) {
		manifest.music = await music();
		await save(manifest);
	}
	console.log(`spent  ~$${spent.toFixed(3)}`);
}

if (process.argv[2] === "cost") {
	const characters = Object.values(LINES).join("").length;
	const speech = Object.values(LINES).join(" ").split(" ").length / 2.6;
	const usd =
		(PER_1000_CHARACTERS * characters) / 1000 +
		PER_TRACK +
		(PER_MINUTE * (speech + MUSIC_SECONDS)) / 60;
	console.log(
		`estimate $${usd.toFixed(2)} per pass (${characters} characters of narration, a ${MUSIC_SECONDS} s music track), up to ${MAX_TAKES}× with retries`,
	);
} else {
	await main();
}
process.exit(0);