Skip to content

Code-first video & image, rendered natively on WebGPU.

9.6 KB
Raw Blame
/**
 * Character animation for the compositor: poses a model from its clip
 * layers, look-at and IK constraints (targets in scene coordinates) and
 * spring bones (replayed from frame 0 and cached, so frames render the
 * same in any order); and the per-instance records of skinned crowds.
 */
import {
	type AimConstraint,
	type CrowdConfig,
	type JointPose,
	type LookAtConstraint,
	type Model3DNode,
	numberAt,
	type TrackClock,
	type TwoBoneIkConstraint,
	type Vec3Track,
	vec3At,
} from "@framefields/compositions/program";
import {
	type AnimationClip3D,
	applyConstraints,
	bakeCrowdPalette,
	blendClipLayers,
	type CrowdBake,
	createRandom,
	type Mat4,
	Matrix4Math,
	type Model3DData,
	poseNodes,
	type Quat,
	type RigClipLayer,
	type RigConstraints,
	type RigJointPose,
	type RigPose,
	resolveClips,
	SpringBones,
	type Vec3,
} from "@framefields/webgpu-renderers";

const transformPoint = (m: Mat4, p: Vec3): Vec3 => [
	m[0] * p[0] + m[4] * p[1] + m[8] * p[2] + m[12],
	m[1] * p[0] + m[5] * p[1] + m[9] * p[2] + m[13],
	m[2] * p[0] + m[6] * p[1] + m[10] * p[2] + m[14],
];
const transformDir = (m: Mat4, v: Vec3): Vec3 => [
	m[0] * v[0] + m[4] * v[1] + m[8] * v[2],
	m[1] * v[0] + m[5] * v[1] + m[9] * v[2],
	m[2] * v[0] + m[6] * v[1] + m[10] * v[2],
];

type CharacterNode = Pick<
	Model3DNode,
	| "animationLayers"
	| "ik"
	| "lookAt"
	| "springBones"
	| "loop"
	| "pose"
	| "aim"
	| "drive"
>;

/**
 * A mesh's morph weights: the clip's (or the file's defaults), with any
 * target the layer names in `morphs` set from its track. Names match
 * case-insensitively against the mesh's `morphTargetNames`.
 */
export function namedMorphWeights(
	names: readonly string[] | undefined,
	base: number[],
	morphs: Model3DNode["morphs"],
	clock: TrackClock,
): number[] {
	if (!morphs || !names?.length) return base;
	const out = [...base];
	const index = new Map(names.map((n, i) => [n.toLowerCase(), i]));
	for (const [name, track] of Object.entries(morphs)) {
		const i = index.get(name.toLowerCase());
		if (i !== undefined) out[i] = numberAt(track, clock, out[i] ?? 0);
	}
	return out;
}

/** Whether a node needs the rig evaluator (anything beyond one plain clip). */
export function hasRig(n: CharacterNode): boolean {
	return Boolean(
		n.animationLayers?.length ||
			n.ik?.length ||
			n.lookAt?.length ||
			n.springBones?.length ||
			(n.pose && Object.keys(n.pose).length) ||
			n.aim?.length ||
			n.drive?.length,
	);
}

/** Local rotation for Euler degrees, applied Z, then Y, then X (R = Rz · Ry · Rx), like layers. */
function eulerQuat(rx: number, ry: number, rz: number): Quat {
	const h = (d: number) => (d * Math.PI) / 360;
	const [sx, cx] = [Math.sin(h(rx)), Math.cos(h(rx))];
	const [sy, cy] = [Math.sin(h(ry)), Math.cos(h(ry))];
	const [sz, cz] = [Math.sin(h(rz)), Math.cos(h(rz))];
	// qz · qy · qx
	return [
		cz * cy * sx - sz * sy * cx,
		cz * sy * cx + sz * cy * sx,
		sz * cy * cx - cz * sy * sx,
		cz * cy * cx + sz * sy * sx,
	];
}

function jointPoses(p: PoseInput, clock: TrackClock): RigJointPose[] {
	return Object.entries(p.node.pose ?? {}).map(
		([joint, j]: [string, JointPose]) => ({
			joint,
			rotation: eulerQuat(
				numberAt(j.rotateX, clock, 0),
				numberAt(j.rotateY, clock, 0),
				numberAt(j.rotateZ, clock, 0),
			),
			offset: [
				numberAt(j.x, clock, 0),
				numberAt(j.y, clock, 0),
				numberAt(j.z, clock, 0),
			] as Vec3,
			weight: numberAt(j.weight, clock, 1),
			mode: j.mode ?? "additive",
		}),
	);
}

export interface PoseInput {
	model: Model3DData;
	node: CharacterNode;
	/** The layer's own frame. */
	frame: number;
	fps: number;
	/** The single clip and its time when there are no `animationLayers`. */
	clip: AnimationClip3D | undefined;
	clipSec: number;
	/** Scene ← model (the model's world transform, root transform included). */
	sceneRoot: Mat4;
	/** Identifies the node across frames (spring state). */
	key: string;
}

function clipLayers(p: PoseInput, frame: number): RigClipLayer[] {
	const layers = p.node.animationLayers;
	if (!layers?.length) {
		return p.clip ? [{ clip: p.clip, time: p.clipSec, weight: 1 }] : [];
	}
	const out: RigClipLayer[] = [];
	for (const l of layers) {
		const [clip] = resolveClips(p.model, [l.clip]);
		if (!clip) continue;
		let time = (l.timeOffset ?? 0) + (frame / p.fps) * (l.speed ?? 1);
		if (clip.duration > 0) {
			time =
				(l.loop ?? true)
					? ((time % clip.duration) + clip.duration) % clip.duration
					: Math.min(Math.max(time, 0), clip.duration);
		}
		out.push({
			clip,
			time,
			weight: numberAt(l.weight, { frame, fps: p.fps }, 1),
			mask: l.mask,
		});
	}
	return out;
}

function constraintsAt(
	p: PoseInput,
	frame: number,
	toModel: Mat4,
): RigConstraints {
	const clock: TrackClock = { frame, fps: p.fps };
	const point = (t: Vec3Track) => transformPoint(toModel, vec3At(t, clock));
	const direction = (t: Vec3Track) => transformDir(toModel, vec3At(t, clock));
	return {
		pose: jointPoses(p, clock),
		aim: (p.node.aim ?? []).map((a: AimConstraint) => ({
			joint: a.joint,
			direction: direction(a.direction),
			weight: numberAt(a.weight, clock, 1),
			...(a.axis && { axis: a.axis }),
		})),
		ik: (p.node.ik ?? []).map((c: TwoBoneIkConstraint) => ({
			chain: c.chain,
			target: point(c.target),
			pole: c.poleTarget ? point(c.poleTarget) : undefined,
			weight: numberAt(c.weight, clock, 1),
		})),
		// The file's own drivers, then the layer's.
		drive: [...(p.model.drivers ?? []), ...(p.node.drive ?? [])],
		lookAt: (p.node.lookAt ?? []).map((c: LookAtConstraint) => ({
			joint: c.joint,
			target: point(c.target),
			weight: numberAt(c.weight, clock, 1),
			maxAngleDeg: c.maxAngleDeg ?? 70,
			axis: c.axis ?? [0, 0, 1],
		})),
	};
}

/** The pose before springs: blended clips, then constraints. */
function basePose(p: PoseInput, frame: number, toModel: Mat4): RigPose {
	const rig = blendClipLayers(p.model, clipLayers(p, frame));
	applyConstraints(rig, constraintsAt(p, frame, toModel));
	return rig;
}

interface SpringState {
	model: Model3DData;
	config: string;
	springs: SpringBones;
	/** Last frame stepped. */
	frame: number;
}

const springStates = new Map<string, SpringState>();

function springsAt(
	p: PoseInput,
	toModel: Mat4,
	rig: RigPose,
): SpringBones | undefined {
	const chains = p.node.springBones;
	if (!chains?.length) return undefined;
	const config = JSON.stringify([
		chains,
		p.node.animationLayers,
		p.node.pose,
		p.node.aim,
		p.fps,
	]);
	let s = springStates.get(p.key);
	const frame = Math.max(0, Math.floor(p.frame));
	if (!s || s.model !== p.model || s.config !== config || s.frame > frame) {
		const gravityModel = (g: Vec3 | undefined): Vec3 =>
			transformDir(toModel, g ?? [0, 0, 0]);
		s = {
			model: p.model,
			config,
			springs: new SpringBones(
				rig,
				chains.map((c) => ({ ...c, gravity: gravityModel(c.gravity) })),
			),
			frame: -1,
		};
		if (springStates.size > 256) springStates.clear();
		springStates.set(p.key, s);
	}
	for (let f = s.frame + 1; f <= frame; f++) {
		s.springs.step(basePose(p, f, toModel), 1 / p.fps);
		s.frame = f;
	}
	return s.springs;
}

/** Model-space node matrices for this frame. */
export function poseCharacter(p: PoseInput): Mat4[] {
	if (!hasRig(p.node) && !p.model.drivers?.length)
		return poseNodes(p.model, p.clip, p.clipSec);
	const toModel = Matrix4Math.invert(p.sceneRoot) ?? Matrix4Math.identity();
	const rig = basePose(p, p.frame, toModel);
	springsAt(p, toModel, rig)?.apply(rig);
	return rig.world();
}

// --- Crowds -------------------------------------------------------------------

const bakes = new WeakMap<Model3DData, Map<string, CrowdBake>>();
const palettes = new WeakMap<Float32Array, GPUBuffer>();

/** A baked palette on the GPU, uploaded once. */
export function paletteBuffer(
	device: GPUDevice,
	palette: Float32Array,
): GPUBuffer {
	let buffer = palettes.get(palette);
	if (!buffer) {
		buffer = device.createBuffer({
			label: "CrowdPalette",
			size: Math.max(64, palette.byteLength),
			usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
		});
		device.queue.writeBuffer(buffer, 0, palette);
		palettes.set(palette, buffer);
	}
	return buffer;
}

/** The model's crowd palette for these clips, baked once. */
export function crowdBake(
	model: Model3DData,
	crowd: CrowdConfig,
): CrowdBake | undefined {
	const key = JSON.stringify([crowd.clips, crowd.fps ?? 30]);
	let byKey = bakes.get(model);
	if (!byKey) {
		byKey = new Map();
		bakes.set(model, byKey);
	}
	let bake = byKey.get(key);
	if (!bake) {
		const clips = resolveClips(model, crowd.clips);
		if (!clips.length) return undefined;
		bake = bakeCrowdPalette(model, clips, crowd.fps ?? 30);
		byKey.set(key, bake);
	}
	return bake;
}

export interface CrowdInstanceAnim {
	clip?: number;
	animOffset?: number;
	animSpeed?: number;
}

/**
 * Instance `i`'s second record: (first baked frame, frames per loop, phase
 * in frames, frames per second of playback). Unset fields come from the seed: clip i % n, a random
 * phase, a speed within `speedRange`.
 */
export function crowdRecord(
	bake: CrowdBake,
	crowd: CrowdConfig,
	i: number,
	anim: CrowdInstanceAnim,
): [number, number, number, number] {
	const rand = createRandom((crowd.seed ?? 1) * 7919 + i * 104729);
	const pick = rand();
	const speedDraw = rand();
	const clipIndex = (anim.clip ?? i) % bake.clips.length;
	const range = bake.clips[Math.max(0, clipIndex)]!;
	const [s0, s1] = crowd.speedRange ?? [1, 1];
	const offset =
		anim.animOffset ??
		((crowd.phase ?? "random") === "random" ? pick * range.duration : 0);
	// Baked samples per second of this clip's time.
	const rate = range.duration > 0 ? range.count / range.duration : bake.fps;
	return [
		range.base,
		range.count,
		offset * rate,
		(anim.animSpeed ?? s0 + (s1 - s0) * speedDraw) * rate,
	];
}