Skip to content

Code-first video & image, rendered natively on WebGPU.

3.7 KB
Raw Blame
/**
 * 07 — Vision, on the drop. The dancer clip, analysed on device as it renders: a giant word sits
 * behind her (a matte cutout of the same clip is layered above it), embers part around her
 * silhouette, and animated sparkle emoji ride her tracked wrists.
 */
import fs from "node:fs";
import {
	Color,
	type Composition,
	Layer,
	LayerAnimation,
	Signal,
} from "framefields";
import { full, hud, keys, type Node, scene } from "../kit.js";
import { DANCER, emoji } from "../paths.js";
import {
	COBALT,
	DISPLAY,
	EMBER,
	GOLD,
	H,
	MONO,
	NAVY,
	PAPER,
	W,
} from "../theme.js";
import { chapter } from "../timeline.js";

export async function visionScene(comp: Composition): Promise<Node> {
	const c = chapter("vision");
	if (!fs.existsSync(DANCER))
		throw new Error(
			`Missing ${DANCER}. Run \`pnpm assets\` in examples/19_framefields_film.`,
		);
	// The clip starts on this chapter's downbeat; vision results use the composition clock.
	const dancer = comp.vision(DANCER, { startFrame: c.start });
	const person = dancer.person.place({ width: W, height: H, fit: "cover" });

	const video = (id: string, muted: boolean) =>
		Layer.video(DANCER, {
			id,
			position: "absolute",
			x: 0,
			y: 0,
			width: W,
			height: H,
			fit: "cover",
			muted,
		});

	const embers = Layer.particles({
		id: "vision-embers",
		width: W,
		height: H,
		max: 14000,
		seed: 21,
		emitter: { shape: "rect", x: 0, y: 360, width: W, height: 640, rate: 2600 },
		lifetime: [3, 5],
		speed: [0, 30],
		drag: 2.2,
		gravity: -8,
		noise: { strength: 220, scale: 160, speed: 0.3 },
		push: { person, strength: 9000, sweep: 120000 },
		floor: 1010,
		bounce: 0.1,
		size: [10, 6],
		sizeJitter: 0.6,
		fade: [0.15, 0.3],
		color: [NAVY, EMBER, GOLD, "#DBEAFE"],
		colorBySpeed: 500,
		prewarm: 3,
	} as never);

	// Sparkles riding each wrist (pose keypoints, read as the frame renders).
	const sparkle = (side: "left" | "right") => {
		const size = 150;
		const at = (frame: number) => person.point(`${side}Wrist`, frame);
		return Layer.lottie(emoji(side === "left" ? "sparkles" : "glowingStar"), {
			id: `vision-${side}-sparkle`,
			position: "absolute",
			x: 0,
			y: 0,
			width: size,
			height: size,
		}).animate(
			LayerAnimation.create()
				.signal(
					"x",
					Signal.programmatic((ctx) => at(ctx.frame).x - size / 2),
				)
				.signal(
					"y",
					Signal.programmatic((ctx) => at(ctx.frame).y - size / 2),
				),
			keys("opacity", [
				[0, 0],
				[6, 0],
				[14, 1],
			]),
		);
	};

	return scene("vision", "#E9D3B4", [
		video("vision-plate", true),
		// The word behind her.
		Layer.text("VISION", {
			id: "vision-word",
			position: "absolute",
			x: 0,
			y: 210,
			width: W,
			align: "center",
			fontFamily: DISPLAY,
			fontWeight: 800,
			fontSize: 360,
			letterSpacing: -14,
			fill: Color.alpha(COBALT, 0.94),
		}).animate(
			keys("scale", [
				[0, 1.3],
				[20, 1, "expo.out"],
			]),
			keys("opacity", [
				[0, 0],
				[4, 1],
			]),
		),
		embers,
		video("vision-cutout", true).withVision({ mode: "matte" }),
		sparkle("left"),
		sparkle("right"),
		Layer.text(
			"comp.vision(clip) · person.place() · withVision({ mode: matte }) · particles push: { person }",
			{
				id: "vision-caption",
				position: "absolute",
				x: 80,
				y: 70,
				fontFamily: MONO,
				fontSize: 20,
				fill: Color.alpha(NAVY, 0.8),
			},
		).animate(
			keys("opacity", [
				[0, 0],
				[16, 0],
				[28, 1],
			]),
		),
		full({ id: "vision-flash", background: PAPER, opacity: 0 }).animate(
			keys("opacity", [
				[0, 0.9],
				[10, 0, "power2.out"],
			]),
		),
		hud(
			"vision",
			[
				"on-device pose + segmentation",
				"matte cutout",
				"pose-tracked Lotties",
				"particles pushed by silhouette",
			],
			{ dark: true },
		),
	]);
}