Code-first video & image, rendered natively on WebGPU.
/**
* Character animation through the compositor: clip layers, IK, look-at,
* spring bones and skinned crowds, on a GLB built here: a 20-wide strip
* skinned to a three-joint chain (root at 0, knee 80 up, foot 160 up,
* the strip reaching 200), with a "fold" clip (knee +90° about z over 1 s)
* and a "curl" clip (knee -90°).
*/
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { GlbBuilder } from "@framefields/gltf-fixtures";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import {
Composition,
CompositorProgramSchema,
Layer,
Layer3D,
type LayoutNode,
type PoseSource,
retargetPose,
Signal,
} from "./index.js";
import { decodePngPixels } from "./test-utils/pixels.js";
const SIZE = 400;
const C = SIZE / 2;
let dir = "";
let file = "";
function quarter(sign: number): Float32Array {
const s = Math.SQRT1_2;
return new Float32Array([0, 0, 0, 1, 0, 0, sign * s, s]);
}
function buildRig(): Buffer {
const g = new GlbBuilder();
const ys = [0, 40, 80, 120, 160, 200];
const joint = [0, 0, 1, 1, 2, 2];
const positions: number[] = [];
const joints: number[] = [];
const weights: number[] = [];
ys.forEach((y, k) => {
for (const x of [-10, 10]) {
positions.push(x, y, 0);
joints.push(joint[k]!, 0, 0, 0);
weights.push(1, 0, 0, 0);
}
});
const indices: number[] = [];
for (let k = 0; k + 1 < ys.length; k++) {
const a = k * 2;
indices.push(a, a + 1, a + 3, a, a + 3, a + 2);
}
const ibm = [0, -80, -160].flatMap((ty) => [
1,
0,
0,
0,
0,
1,
0,
0,
0,
0,
1,
0,
0,
ty,
0,
1,
]);
const times = g.accessor(new Float32Array([0, 1]), "SCALAR");
const clip = (name: string, sign: number) => ({
name,
channels: [{ sampler: 0, target: { node: 2, path: "rotation" } }],
samplers: [{ input: times, output: g.accessor(quarter(sign), "VEC4") }],
});
return g.build({
scenes: [{ nodes: [0, 1] }],
nodes: [
{ mesh: 0, skin: 0 },
{ name: "root", children: [2] },
{ name: "knee", translation: [0, 80, 0], children: [3] },
{ name: "foot", translation: [0, 80, 0] },
],
skins: [
{
joints: [1, 2, 3],
inverseBindMatrices: g.accessor(new Float32Array(ibm), "MAT4"),
},
],
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(positions), "VEC3"),
JOINTS_0: g.accessor(new Uint16Array(joints), "VEC4"),
WEIGHTS_0: g.accessor(new Float32Array(weights), "VEC4"),
},
indices: g.accessor(new Uint16Array(indices), "SCALAR"),
material: 0,
},
],
},
],
materials: [{ pbrMetallicRoughness: { baseColorFactor: [1, 1, 1, 1] } }],
animations: [clip("fold", 1), clip("curl", -1)],
});
}
beforeAll(async () => {
dir = await fs.mkdtemp(path.join(os.tmpdir(), "framefields-character-"));
file = path.join(dir, "leg.glb");
await fs.writeFile(file, buildRig());
});
afterAll(async () => {
await fs.rm(dir, { recursive: true, force: true });
});
function scene(layers: LayoutNode[]): Composition {
const comp = new Composition({
width: SIZE,
height: SIZE,
fps: 30,
durationFrames: 120,
backgroundColor: "#000000",
});
comp.add([
Layer.camera({
x: C,
y: C,
// A 50° field of view at this distance maps one model unit to one pixel.
z: -C / Math.tan((25 * Math.PI) / 180),
targetX: C,
targetY: C,
targetZ: 0,
}),
...layers,
]);
return comp;
}
/** The leg, its root at the frame's center; model +y is up on screen. */
const leg = (extra: Record<string, unknown> = {}) =>
Layer.glb(file, {
id: "leg",
x: C,
y: C,
z: 0,
rotateX: 180,
material: "unlit",
...extra,
});
/** Whether model point (x, y) is drawn (white), for a model rooted at screen (ox, oy). */
async function probe(comp: Composition, frame = 0) {
const { pixels } = await decodePngPixels(await comp.renderFrame({ frame }));
return (x: number, y: number, ox = C, oy = C) => {
const i = (Math.round(oy - y) * SIZE + Math.round(ox + x)) * 4;
return (pixels[i] ?? 0) > 128;
};
}
describe("character animation", () => {
it("blends clip layers by weight, over the pose beneath", async () => {
const at = (weight: number) =>
probe(
scene([
leg({
animationLayers: [
{ clip: "fold", timeOffset: 1, loop: false },
{ clip: "curl", timeOffset: 1, loop: false, weight },
],
}),
]),
);
// Folded +90°: the shin points -x. Blended halfway back toward -90°… the
// two cancel at weight 0.5 by slerp: straight up again.
const folded = await at(0);
expect(folded(-60, 80)).toBe(true);
expect(folded(0, 140)).toBe(false);
const straight = await at(0.5);
expect(straight(0, 140)).toBe(true);
const curled = await at(1);
expect(curled(60, 80)).toBe(true);
});
it("reaches an IK target with the foot, the knee toward the pole", async () => {
// Target model (100, 60): scene (C + 100, C - 60).
const lit = await probe(
scene([
leg({
ik: [
{
chain: ["root", "knee", "foot"],
target: [C + 100, C - 60, 0],
poleTarget: [C + 200, C - 200, 0],
},
],
}),
]),
);
expect(lit(100, 60)).toBe(true);
expect(lit(0, 140)).toBe(false);
// The knee, 80 from both root and target, sits on the pole's side of
// the root→target line: (50, 30) + 54.8 · (-0.514, 0.857).
const knee: [number, number] = [22, 77];
expect(lit(...knee)).toBe(true);
});
it("turns a joint toward a look-at target within its limit", async () => {
const look = (max: number) =>
probe(
scene([
leg({
lookAt: [
{
joint: "root",
target: [C + 300, C, 0],
axis: [0, 1, 0],
maxAngleDeg: max,
},
],
}),
]),
);
const full = await look(180);
expect(full(150, 0)).toBe(true);
expect(full(0, 150)).toBe(false);
const limited = await look(45);
expect(limited(150 * Math.SQRT1_2, 150 * Math.SQRT1_2)).toBe(true);
});
it("simulates spring bones deterministically from frame 0", async () => {
const comp = scene([
leg({
// The foot's chain swings under sideways gravity; the knee clip is still.
springBones: [
{
joints: ["knee"],
stiffness: 0,
damping: 0.9,
gravity: [600, 0, 0],
},
],
}),
]);
const first = await probe(comp, 0);
expect(first(0, 140)).toBe(true);
const later = await probe(comp, 40);
expect(later(0, 150)).toBe(false);
// Rendering an earlier frame, then frame 40 again, gives the same picture.
await comp.renderFrame({ frame: 10 });
const again = await comp.renderFrame({ frame: 40 });
expect(again).toEqual(await comp.renderFrame({ frame: 40 }));
const { pixels: a } = await decodePngPixels(again);
const fresh = scene([
leg({
springBones: [
{
joints: ["knee"],
stiffness: 0,
damping: 0.9,
gravity: [600, 0, 0],
},
],
}),
]);
const { pixels: b } = await decodePngPixels(
await fresh.renderFrame({ frame: 40 }),
);
expect(a).toEqual(b);
});
it("draws a skinned crowd, each instance at its own point in its clip", async () => {
// The mesh node's own rotateX (the leg is y-up) applies under each instance's.
const crowd = Layer3D.instances({
mesh: leg(),
x: C,
y: C,
z: 0,
material: "unlit",
crowd: { clips: ["fold", "curl"], phase: "sync" },
transforms: [
{ x: -120, y: 0, z: 0, clip: 0, animOffset: 0 },
{ x: 0, y: 0, z: 0, clip: 0, animOffset: 0.5 },
{ x: 120, y: 0, z: 0, clip: 1, animOffset: 0.98 },
],
});
const lit = await probe(scene([crowd]));
// Straight, folded 45° left, curled ~90° right.
expect(lit(0, 140, C - 120)).toBe(true);
expect(lit(-40, 120, C)).toBe(true);
expect(lit(0, 150, C)).toBe(false);
expect(lit(60, 80, C + 120)).toBe(true);
expect(lit(0, 150, C + 120)).toBe(false);
});
it("keeps character options schema-valid", () => {
const comp = scene([
leg({
animationLayers: [
{
clip: "fold",
weight: [
[0, 0],
[30, 1],
],
mask: ["knee"],
},
],
ik: [{ chain: ["root", "knee", "foot"], target: [[0, 1, 2, 3]] }],
lookAt: [{ joint: "root", target: [0, 0, 0], maxAngleDeg: 30 }],
springBones: [{ joints: ["knee"], gravity: [0, 980, 0] }],
pose: {
knee: {
rotateZ: [
[0, 0],
[30, 45],
],
mode: "additive",
},
},
aim: [
{
joint: "root",
direction: [
1,
[
[0, 0],
[30, 1],
],
0,
],
},
],
}),
Layer3D.instances({
mesh: leg(),
count: 2,
crowd: { clips: ["fold"], fps: 24 },
}),
]);
expect(CompositorProgramSchema.safeParse(comp.toSpec()).success).toBe(true);
const bad = scene([leg({ springBones: [{ joints: [] }] })]);
expect(CompositorProgramSchema.safeParse(bad.toSpec()).success).toBe(false);
});
it("poses joints directly, from numbers, keys or signals", async () => {
// Additive: the knee turns +90° about z over the (still) pose.
const folded = await probe(
scene([leg({ pose: { knee: { rotateZ: 90 } } })]),
);
expect(folded(-60, 80)).toBe(true);
expect(folded(0, 140)).toBe(false);
// A signal drives it, read with the render's clock.
const swing = Signal.builder({
type: "custom",
fn: (ctx) => -3 * ctx.frame,
});
const comp = scene([leg({ pose: { knee: { rotateZ: swing } } })]);
const curled = await probe(comp, 30);
expect(curled(60, 80)).toBe(true);
// Keyed weight fades the override in.
const fading = scene([
leg({
pose: {
knee: {
rotateZ: 90,
weight: [
[0, 0],
[30, 1],
],
},
},
}),
]);
expect((await probe(fading, 0))(0, 140)).toBe(true);
// (Frame 30: the looping fold clip underneath is back at rest.)
expect((await probe(fading, 30))(-60, 80)).toBe(true);
});
it("poses from rest over a clip", async () => {
const lit = await probe(
scene([
leg({
animationLayers: [{ clip: "fold", timeOffset: 1, loop: false }],
pose: { knee: { mode: "rest" } },
}),
]),
);
// The clip folds the knee; the rest pose override straightens it.
expect(lit(0, 140)).toBe(true);
});
it("aims bones along scene directions", async () => {
const lit = await probe(
scene([leg({ aim: [{ joint: "root", direction: [1, 0, 0] }] })]),
);
expect(lit(150, 0)).toBe(true);
expect(lit(0, 150)).toBe(false);
});
it("retargets a performer's 2D pose onto the rig", async () => {
// Screen px: hip at the origin, knee to the right, ankle below the knee.
const pose = (visibility: number): PoseSource => ({
point: (name) =>
({
leftHip: { x: 0, y: 0, visibility },
leftKnee: { x: 100, y: 0, visibility },
leftAnkle: { x: 100, y: 100, visibility },
})[name as "leftHip"] ?? { x: 0, y: 0, visibility: 0 },
});
const rig = { leftUpperLeg: "root", leftLowerLeg: "knee" };
const lit = await probe(
scene([leg({ aim: retargetPose(pose(1), { rig }) })]),
);
// Thigh along +x to the knee at (80, 0), shin down the screen (model −y).
expect(lit(40, 0)).toBe(true);
expect(lit(80, -40)).toBe(true);
expect(lit(0, 140)).toBe(false);
// Unsure keypoints hand the bones back: the leg stands as animated.
const unsure = await probe(
scene([leg({ aim: retargetPose(pose(0.1), { rig }) })]),
);
expect(unsure(0, 140)).toBe(true);
// Mirrored: the performer's left drives the right side, unmapped here.
const mirrored = await probe(
scene([leg({ aim: retargetPose(pose(1), { rig, mirror: true }) })]),
);
expect(mirrored(0, 140)).toBe(true);
});
});