Skip to content

Code-first video & image, rendered natively on WebGPU.

8.5 KB
Raw Blame
import fs from "node:fs/promises";
import path from "node:path";
import {
	AudioSignal,
	AudioSignalExtractor,
	Blur,
	ColorBalance,
	Composition,
	computed,
	FontManager,
	Layer,
	LayerAnimation,
	Media,
	Signal,
	signal,
	Vignette,
} from "../packages/framefields/dist/index.mjs";
import { HeadlessMediaRenderer } from "../packages/framefields/dist/renderer/index.mjs";

async function main() {
	console.log("=== Framefields Signal-Driven Animation Pipeline ===");
	const inputVideo =
		process.env.INPUT_VIDEO || path.resolve(process.cwd(), "export.mp4");
	const outputPng = path.resolve(process.cwd(), "rendered_preview.png");
	const outputMp4 = path.resolve(process.cwd(), "rendered_composition.mp4");

	console.log("1. Registering TrueType fonts via FontManager singleton...");
	const font = await FontManager.register("assets/fonts/Inter.ttf");
	console.log(
		`   FontManager registered: "${font.family}" (${font.postscriptName}), unitsPerEm: ${font.unitsPerEm}`,
	);

	console.log(
		"2. Initializing AudioSignalExtractor node & extracting WebGPU audio features...",
	);
	const audioExtractor = new AudioSignalExtractor({
		nodeId: "speech_extractor",
		extractionMode: "rms_envelope",
		attackMs: 12,
		releaseMs: 140,
		sensitivity: 1.25,
		noiseFloorDb: -42,
		dynamicRangeDb: 40,
		smoothing: 0.18,
		curve: "smoothstep",
	});

	const audio = await audioExtractor.extract(inputVideo, { fps: 24 });
	console.log(
		`   Extracted WebGPU audio speech signals (${audio.numFrames} frames) via AudioSignalExtractor`,
	);

	console.log("3. Initializing composition and defining reactive signals...");
	const comp = new Composition({
		width: 1280,
		height: 720,
		fps: 24,
		backgroundColor: "#0d1117",
	});

	// Define reactive signals
	// 1. Rhythmic breathing pulse signal (1.4 Hz sinusoidal LFO)
	comp.addSignal("pulse_lfo", (frame, fps) => {
		const t = frame / fps;
		const wave = Math.sin(t * Math.PI * 2 * 1.4);
		return (wave + 1) * 0.5; // normalized 0..1
	});

	// 2. Real-time speech energy signal extracted via AudioSignalExtractor
	comp.addSignal("energy_signal", (frame) => {
		audio.seekFrame(frame);
		return audio.energy.value;
	});

	// Base video layer
	comp.add(
		Layer.video(inputVideo, {
			x: 0,
			y: 0,
			width: 1280,
			height: 720,
			fit: "cover",
		}),
	);

	// Audio track (mixed directly from video source)
	comp.addAudio(
		Layer.audio(inputVideo, {
			volume: 1.0,
		}),
	);

	// 1. Frosted dark card container behind typography
	comp.add(
		Layer.shape("rect", {
			id: "card-container",
			x: 70,
			y: 490,
			width: 1140,
			height: 165,
			borderRadius: 18,
			fillColor: "rgba(15, 23, 42, 0.88)",
		}).animate(
			new LayerAnimation()
				.slideInY(590, 490, 0, 22, "power3.out")
				.fadeIn(0, 16),
		),
	);

	// 2. Pulsing glowing indicator circle (scale & color modulated by pulse_lfo signal)
	comp.add(
		Layer.shape("circle", {
			id: "pulse-dot",
			x: 100,
			y: 520,
			width: 14,
			height: 14,
			borderRadius: 7,
			fillColor: "#38bdf8",
		}).animate(
			new LayerAnimation()
				.signal("scale", "pulse_lfo", { multiplier: 0.35, offset: 0.85 })
				.colorSignal("fillColor", "pulse_lfo", {
					colorA: "#38bdf8",
					colorB: "#ec4899",
				})
				.fadeIn(0, 18),
		),
	);

	// 3. Accent pill badge (scale modulated by signal, organic rotation wiggle)
	comp.add(
		Layer.shape("rect", {
			id: "vfx-badge",
			x: 126,
			y: 514,
			width: 185,
			height: 26,
			borderRadius: 6,
			fillColor: "#38bdf8",
		}).animate(
			new LayerAnimation()
				.signal("scale", "pulse_lfo", { multiplier: 0.08, offset: 1.0 })
				.wiggle("rotation", { frequency: 1.2, amplitude: 3.0, seed: 101 })
				.slideInY(536, 514, 6, 22)
				.fadeIn(6, 18),
		),
	);

	// 4. Badge label
	comp.add(
		Layer.text("AUDIO SIGNAL EXTRACTOR", {
			id: "badge-label",
			x: 136,
			y: 520,
			fontSize: 11,
			fill: "#0f172a",
			fontFamily: "Inter",
			fontWeight: "bold",
		}).animate(new LayerAnimation().slideInY(542, 520, 6, 22).fadeIn(6, 18)),
	);

	// 5. Signal reactive level track (meter background)
	comp.add(
		Layer.shape("rect", {
			id: "meter-bg",
			x: 330,
			y: 524,
			width: 220,
			height: 6,
			borderRadius: 3,
			fillColor: "rgba(255, 255, 255, 0.12)",
		}).animate(new LayerAnimation().fadeIn(10, 24)),
	);

	// 6. Active signal meter bar (width & color dynamically modulated by speech energy_signal)
	comp.add(
		Layer.shape("rect", {
			id: "meter-active",
			x: 330,
			y: 524,
			width: 60,
			height: 6,
			borderRadius: 3,
			fillColor: "#38bdf8",
		}).animate(
			new LayerAnimation()
				.signal("width", "energy_signal", { multiplier: 190, offset: 25 })
				.colorSignal("fillColor", "energy_signal", {
					colorA: "#38bdf8",
					colorB: "#f43f5e",
				})
				.fadeIn(10, 24),
		),
	);

	// 7. Signal meter label
	comp.add(
		Layer.text("SIGNAL MODULATION", {
			id: "meter-label",
			x: 566,
			y: 522,
			fontSize: 11,
			fill: "#94a3b8",
			fontFamily: "Inter",
			fontWeight: "bold",
		}).animate(new LayerAnimation().fadeIn(12, 26)),
	);

	// 8. Main title (entrance slide-up & fade-in)
	comp.add(
		Layer.text("Framefields Render Engine", {
			id: "main-title",
			x: 100,
			y: 554,
			fontSize: 32,
			fill: "#ffffff",
			fontFamily: "Inter",
		}).animate(new LayerAnimation().slideInY(580, 554, 8, 26).fadeIn(8, 22)),
	);

	// 9. Subtitle (entrance slide-up & fade-in)
	comp.add(
		Layer.text(
			"node-audio-signal-extractor • WebGPU Audio DSP • Reactive Speech Blur",
			{
				id: "subtitle",
				x: 100,
				y: 605,
				fontSize: 15,
				fill: "#94a3b8",
				fontFamily: "Inter",
			},
		).animate(new LayerAnimation().slideInY(625, 605, 14, 32).fadeIn(14, 28)),
	);

	console.log(
		"3. Dynamically computing composition duration & configuring reactive effects...",
	);
	const durationMs = await comp.computeDuration();
	console.log(
		`   Dynamically computed duration: ${durationMs}ms (${(durationMs / 1000).toFixed(2)}s)`,
	);

	console.log(
		"4. Instantiating reactive effect classes with game-engine per-frame lifecycle hooks...",
	);

	// 1. First-class ColorBalance effect
	const colorBalance = new ColorBalance({
		shadows: { cyanRed: 12, magentaGreen: -4, yellowBlue: -12 },
		midtones: { cyanRed: 4, magentaGreen: 6, yellowBlue: 2 },
		highlights: { cyanRed: -4, magentaGreen: 0, yellowBlue: 8 },
		preserveLuminosity: true,
	});

	// 2. Procedural breathing LFO for subtle vignette modulation
	const vignetteBreathing = Signal.builder({
		type: "sine",
		frequency: 0.8,
		amplitude: 8.0,
		offset: 50.0,
	});

	// 3. First-class Vignette effect bound to reactive signals with per-frame mutation
	const vignette = new Vignette({
		strength: vignetteBreathing,
		radius: 0.85,
		softness: 0.55,
	});

	// Per-frame game engine hook: dynamically adjust radius over frames
	vignette.onRequestFrame(({ frame }) => {
		if (frame < 50) {
			vignette.radius = 0.75 + frame * 0.002;
		}
	});

	// 4. Reactive Speech Blur: Screen blurs dynamically when the man is talking!
	// When quiet/silent: 0 blur (crystal clear).
	// When speech detected: blur smoothly scales with voice energy (up to 20px).
	const speechBlurStrength = computed(() => {
		const energy = audio.energy.value;
		if (energy < 0.15) return 0;
		return Math.min(22, Math.max(0, (energy - 0.15) * 32 + 2));
	});

	const blur = new Blur({
		blurType: "Gaussian",
		strength: speechBlurStrength,
	});

	blur.onRequestFrame(({ frame }) => {
		audio.seekFrame(frame);
	});

	const compVM = await comp.toVirtualMediaAsync();
	const styledMedia = new Media(compVM)
		.apply(colorBalance)
		.apply(vignette)
		.apply(blur);

	const finalVM = styledMedia.toVirtualMedia();
	const renderer = new HeadlessMediaRenderer();

	console.log("5. Rendering high-quality preview frame at frame 48 (2.0s)...");
	const t0 = Date.now();
	const pngBuffer = await renderer.renderImage(finalVM, 48, 24);
	await fs.writeFile(outputPng, pngBuffer);
	console.log(
		`   Saved preview PNG (${(pngBuffer.length / 1024).toFixed(1)} KB) in ${Date.now() - t0}ms`,
	);

	console.log(
		`6. Rendering and encoding full-duration MP4 video (${(durationMs / 1000).toFixed(2)}s, ${Math.round((durationMs / 1000) * 24)} frames @ 24fps)...`,
	);
	const t1 = Date.now();
	const { filePath: tempVideoPath, cleanup } = await renderer.renderVideo(
		finalVM,
		{
			codec: "h264",
			quality: "medium",
		},
	);

	await fs.copyFile(tempVideoPath, outputMp4);
	await cleanup();
	const stat = await fs.stat(outputMp4);
	console.log(
		`   Saved full-length MP4 video (${(stat.size / 1024 / 1024).toFixed(2)} MB) in ${Date.now() - t1}ms`,
	);

	console.log("=== Render Completed Successfully ===");
}

main().catch((err) => {
	console.error("Pipeline execution failed:", err);
	process.exit(1);
});