Skip to content

Code-first video & image, rendered natively on WebGPU.

11.0 KB
Raw Blame
/**
 * pnpm song        → the whole soundtrack pipeline (idempotent; delete a file to redo its step)
 * pnpm song cost   → print the estimate, no calls
 * pnpm song measure → re-run the free steps (grid, structure, lyrics, envelopes)
 *
 *   1. song       ElevenLabs Music v2.5, one chunk per section of song.ts      → assets/song.mp3
 *   2. stems      fal-ai/demucs: vocals, drums                                   → assets/stems/
 *   3. words      ElevenLabs Scribe v2 on the vocal stem, word timestamps        → assets/words.json
 *   4. grid       the drum stem's tempo (beat-grid.ts), checked against 180 BPM
 *   5. lyrics     written words aligned to heard words                           → assets/lyrics.json
 *   6. chapters   lyric line starts and the final hit (chapters.ts)              → assets/structure.json
 *   7. envelopes  kick / snare from the drum stem, vocal level                   → assets/envelopes.json
 *
 * The soundtrack is song.mp3 itself, untouched: the film's clock is the take's
 * clock, so nothing here stretches, shifts or re-masters the audio.
 */
import { execFileSync } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { ai } from "framefields";
import { download, exists, PRICING } from "./gateway.js";
import { BAR_SEC, BPM, PLAN_SEC, SAMPLE_RATE } from "./grid.js";
import { alignLyrics, type HeardWord } from "./music/align.js";
import { analyzeBeatGrid } from "./music/beat-grid.js";
import { chaptersOf } from "./music/chapters.js";
import { type Stereo, stereo } from "./music/dsp.js";
import { envelopes } from "./music/envelopes.js";
import { findFinalHit } from "./music/structure.js";
import { ASSETS, asset, ROOT } from "./paths.js";
import {
	GLOBAL_AVOID,
	GLOBAL_STYLES,
	SECTIONS,
	sectionDurationsMs,
} from "./song.js";

const SONG = asset("song.mp3");
const SONG_META = asset("song.json");
const WORDS = asset("words.json");
const STEMS = asset("stems.json");
const stem = (name: string) => asset(`stems/${name}.wav`);

const step = process.argv[2] ?? "all";

let spent = 0;

function checkApiKey(): void {
	process.loadEnvFile(path.join(ROOT, ".env"));
	const key = process.env.FRAMEFIELDS_API_KEY;
	if (!key)
		throw new Error("FRAMEFIELDS_API_KEY missing from the repo root .env");
}

/** Generous upper bound on the music model's compute time, for the estimate. */
const SONG_COMPUTE_SEC = 120;

async function song(seed: number): Promise<void> {
	if (await exists(SONG)) return;
	const durations = sectionDurationsMs();
	// v2.5 has no global styles: the first chunk's styles set the whole song's
	// tone, so it carries the global ones; every chunk avoids the global faults.
	const plan = {
		chunks: SECTIONS.map((s, i) => ({
			duration_ms: durations[i],
			text: [`[${s.name}]`, ...s.lines].join("\n"),
			positive_styles: i === 0 ? [...GLOBAL_STYLES, ...s.styles] : s.styles,
			negative_styles: [...GLOBAL_AVOID, ...s.avoid],
			context_adherence: "high",
		})),
	};
	const t0 = Date.now();
	const { data: out } = await ai.subscribe<{ audio: { url: string } }>(
		"fal-ai/elevenlabs/music/v2.5",
		{
			input: { composition_plan: plan, seed, output_format: "mp3_48000_192" },
		},
	);
	await download(out.audio.url, SONG);
	await fs.writeFile(
		SONG_META,
		JSON.stringify({ url: out.audio.url, seed, plan }, null, 2),
	);
	spent += PRICING["fal-ai/elevenlabs/music/v2.5"].usd * SONG_COMPUTE_SEC;
	console.log(
		`song   ${SONG} seed ${seed} (${((Date.now() - t0) / 1000).toFixed(0)} s)`,
	);
}

/** The vocal stem is transcribed; the drum stem gives the grid and the hits. */
type StemName = "vocals" | "drums";
const STEM_NAMES: StemName[] = ["vocals", "drums"];

async function stems(): Promise<void> {
	if (await exists(STEMS)) return;
	const { url } = JSON.parse(await fs.readFile(SONG_META, "utf8")) as {
		url: string;
	};
	const { data: out } = await ai.subscribe<
		Record<StemName, { url: string } | null>
	>("fal-ai/demucs", {
		input: {
			audio_url: url,
			model: "htdemucs_ft",
			stems: STEM_NAMES,
			output_format: "wav",
		},
	});
	const urls: Partial<Record<StemName, string>> = {};
	for (const name of STEM_NAMES) {
		const file = out[name];
		if (!file) throw new Error(`demucs returned no ${name} stem`);
		await download(file.url, stem(name));
		urls[name] = file.url;
	}
	await fs.writeFile(STEMS, JSON.stringify(urls, null, 2));
	spent += PRICING["fal-ai/demucs"].usd * PLAN_SEC;
	console.log(`stems  ${STEM_NAMES.join(", ")}`);
}

interface ScribeWord {
	text: string;
	start: number;
	end: number;
	type: "word" | "spacing" | "audio_event";
}

/** Names the transcriber would otherwise spell as ordinary words. */
const KEYTERMS = [
	"Framefields",
	"Chromium",
	"GPU",
	"glTF",
	"After Effects",
	"Photoshop",
	"Premiere",
	"Cinema 4D",
	"Blender",
	"Illustrator",
];

async function words(): Promise<void> {
	if (await exists(WORDS)) return;
	const { vocals: url } = JSON.parse(
		await fs.readFile(STEMS, "utf8"),
	) as Record<StemName, string>;
	const { data: out } = await ai.subscribe<{ words: ScribeWord[] }>(
		"fal-ai/elevenlabs/speech-to-text/scribe-v2",
		{
			input: {
				audio_url: url,
				language_code: "eng",
				keyterms: KEYTERMS,
				tag_audio_events: false,
			},
		},
	);
	const heard: HeardWord[] = out.words
		.filter((w) => w.type === "word")
		.map(({ text, start, end }) => ({ text: text.trim(), start, end }));
	await fs.writeFile(WORDS, JSON.stringify(heard, null, 1));
	spent +=
		PRICING["fal-ai/elevenlabs/speech-to-text/scribe-v2"].usd * (PLAN_SEC / 60);
	console.log(
		`words  ${heard.length} heard: ${heard.map((h) => h.text).join(" ")}`,
	);
}

/** Decodes through ffmpeg to interleaved float32 at the film's sample rate. */
function decode(
	file: string,
	filters: string[],
	channels: 1 | 2,
): Float32Array {
	const args = ["-v", "error", "-i", file];
	if (filters.length) args.push("-af", filters.join(","));
	args.push(
		"-ac",
		String(channels),
		"-ar",
		String(SAMPLE_RATE),
		"-f",
		"f32le",
		"pipe:1",
	);
	const raw = execFileSync("ffmpeg", args, { maxBuffer: 1 << 30 });
	return new Float32Array(raw.buffer, raw.byteOffset, raw.byteLength / 4);
}

/** How far the take may stray from 180 BPM before the grid drifts audibly off it (0.2 % ≈ 0.1 s a minute). */
const MAX_TEMPO_ERROR = 0.002;

interface Grid {
	measuredBpm: number;
	/** Where the take's downbeat phase sits relative to the film's bar lines (ms, for the record). */
	phaseMs: number;
	/** Film length in bars: from the start of the take to its last sounding bar. */
	bars: number;
}

/** Bars quieter than this are the ring-out past the end. */
const SILENCE_DB = -60;

function barLevels(mix: Stereo): number[] {
	const per = Math.round(BAR_SEC * SAMPLE_RATE);
	const out: number[] = [];
	for (let at = 0; at + per <= mix[0].length; at += per) {
		let e = 0;
		for (let i = at; i < at + per; i++)
			e += (mix[0][i] ** 2 + mix[1][i] ** 2) / 2;
		out.push(10 * Math.log10(e / per + 1e-12));
	}
	return out;
}

/** A file decoded on the film's clock (which is the take's clock), padded or cut to `bars`. */
function onClock(file: string, bars: number): Stereo {
	const seconds = bars * BAR_SEC;
	const pcm = decode(file, ["apad", `atrim=end=${seconds.toFixed(4)}`], 2);
	const out = stereo(Math.round(seconds * SAMPLE_RATE));
	for (let i = 0; i < out[0].length && 2 * i + 1 < pcm.length; i++) {
		out[0][i] = pcm[2 * i];
		out[1][i] = pcm[2 * i + 1];
	}
	return out;
}

/**
 * The take is used as it is, so its tempo must already be the film's: measure
 * it on the drum stem and refuse a take that would drift off the bar grid.
 */
function measureGrid(): Grid {
	const mono = decode(stem("drums"), [], 1);
	const grid = analyzeBeatGrid(mono, SAMPLE_RATE, {
		bpmHint: BPM,
		minBpm: 150,
		maxBpm: 210,
	});
	const error = Math.abs(grid.bpm - BPM) / BPM;
	if (error > MAX_TEMPO_ERROR)
		throw new Error(
			`take runs at ${grid.bpm.toFixed(2)} BPM, ${(error * 100).toFixed(2)} % off ${BPM}: regenerate it`,
		);
	const phase =
		Math.round(grid.downbeatSec / BAR_SEC) * BAR_SEC - grid.downbeatSec;
	const levels = barLevels(onClock(SONG, Math.ceil(PLAN_SEC / BAR_SEC) + 2));
	const bars =
		levels.length - [...levels].reverse().findIndex((db) => db > SILENCE_DB);
	console.log(
		`grid   ${grid.bpm.toFixed(2)} BPM, downbeats ${(phase * 1000).toFixed(0)} ms off the bar lines, ${bars} bars (${(bars * BAR_SEC).toFixed(2)} s)`,
	);
	return { measuredBpm: grid.bpm, phaseMs: Math.round(phase * 1000), bars };
}

async function measure(): Promise<void> {
	const grid = measureGrid();
	const song = onClock(SONG, grid.bars);

	const heard = JSON.parse(await fs.readFile(WORDS, "utf8")) as HeardWord[];
	const words = alignLyrics(SECTIONS, heard);
	console.log(
		`lyrics ${words.filter((w) => w.heard).length}/${words.length} words timed from the transcript`,
	);

	const chapters = chaptersOf(findFinalHit(song), words, grid.bars);
	console.log(
		`chapters ${Object.entries(chapters)
			.map(([k, v]) => `${k} ${v}`)
			.join(", ")}`,
	);
	await fs.writeFile(
		asset("structure.json"),
		JSON.stringify({ grid, chapters }, null, 1),
	);
	await fs.writeFile(asset("lyrics.json"), JSON.stringify(words, null, 1));
	await fs.writeFile(
		asset("envelopes.json"),
		JSON.stringify(
			envelopes(
				onClock(stem("drums"), grid.bars),
				onClock(stem("vocals"), grid.bars),
			),
		),
	);
}

/**
 * The film puts every written line on screen when it is sung, so a take that
 * skips a line (the model sometimes repeats one instead) cannot be cut to:
 * the lines most of whose words the transcript never heard.
 */
async function skippedLines(): Promise<string[]> {
	const heard = JSON.parse(await fs.readFile(WORDS, "utf8")) as HeardWord[];
	const words = alignLyrics(SECTIONS, heard);
	const lines = [...new Set(words.map((w) => w.line))];
	return lines
		.map((n) => words.filter((w) => w.line === n))
		.filter((ws) => ws.filter((w) => w.heard).length < ws.length / 2)
		.map((ws) => ws.map((w) => w.text).join(" "));
}

const MAX_TAKES = 4;

/** Generates takes until one sings every line, setting rejected ones aside. */
async function generate(): Promise<void> {
	for (let take = 1; ; take++) {
		await song(take);
		await stems();
		await words();
		const skipped = await skippedLines();
		if (!skipped.length) return;
		const rejected = path.resolve(
			ASSETS,
			"../scratch/rejected-takes",
			`take-${take}`,
		);
		await fs.mkdir(rejected, { recursive: true });
		for (const file of [SONG, SONG_META, STEMS, WORDS, asset("stems")])
			if (await exists(file))
				await fs.rename(file, path.join(rejected, path.basename(file)));
		console.log(`take ${take} skipped: ${skipped.join(" / ")}`);
		if (take === MAX_TAKES)
			throw new Error(`no take sang every line in ${MAX_TAKES} tries`);
	}
}

if (step === "cost") {
	const usd =
		PRICING["fal-ai/elevenlabs/music/v2.5"].usd * SONG_COMPUTE_SEC +
		PRICING["fal-ai/demucs"].usd * PLAN_SEC +
		PRICING["fal-ai/elevenlabs/speech-to-text/scribe-v2"].usd * (PLAN_SEC / 60);
	console.log(
		`estimate $${usd.toFixed(2)} (song ${PLAN_SEC.toFixed(0)} s, stems, transcription)`,
	);
} else {
	if (step === "all") {
		checkApiKey();
		await generate();
		if (spent > 0) console.log(`spent  ~$${spent.toFixed(3)}`);
	}
	await measure();
}
process.exit(0);