Skip to content

Code-first video & image, rendered natively on WebGPU.

6.2 KB
Raw Blame
/**
 * Pose retargeting: a performer's 2D pose (vision keypoints, or any source
 * of named points) drives a 3D character's bones. Each bone is aimed along
 * its segment in the picture (shoulder → elbow, hip → knee, …), parents
 * first, so the character copies the performer's silhouette at its own
 * proportions. Aims run after clips and `pose`, before `lookAt` and `ik`,
 * so clips can carry what the camera can't see and IK can plant the feet.
 *
 * ```ts
 * const dancer = comp.vision("dance.mp4").person.place({ width: W, height: H });
 * Layer.glb("robot.glb", { aim: retargetPose(dancer, { rig: "mixamo" }) });
 * ```
 *
 * The picture has no depth: limbs swing in the plane facing the camera.
 */
import {
	type AimConstraint,
	type NumberTrack,
	numberAt,
	type TrackClock,
} from "@framefields/compositions/program";
import type { PoseKeypointName } from "./clip-vision.js";

/** Anything that answers where a keypoint is at a frame (a vision `PersonTrack`, a mocap reader). */
export interface PoseSource {
	point(
		name: PoseKeypointName,
		frame: number,
	): { x: number; y: number; visibility: number };
}

/** The bones a performer's 2D pose can aim. */
export type RetargetBone =
	| "spine"
	| "neck"
	| "leftUpperArm"
	| "leftLowerArm"
	| "rightUpperArm"
	| "rightLowerArm"
	| "leftUpperLeg"
	| "leftLowerLeg"
	| "rightUpperLeg"
	| "rightLowerLeg";

/** Joint names per bone; names match without a namespace prefix and case (`"LeftArm"` finds `"mixamorig:LeftArm"`). */
export type RigMap = Partial<Record<RetargetBone, string>>;

/** Joint names of common rigs. */
export const RIG_PRESETS = {
	/** Mixamo (`mixamorig:` prefix, any number). */
	mixamo: {
		spine: "Spine",
		neck: "Neck",
		leftUpperArm: "LeftArm",
		leftLowerArm: "LeftForeArm",
		rightUpperArm: "RightArm",
		rightLowerArm: "RightForeArm",
		leftUpperLeg: "LeftUpLeg",
		leftLowerLeg: "LeftLeg",
		rightUpperLeg: "RightUpLeg",
		rightLowerLeg: "RightLeg",
	},
	/** Unity humanoid / VRM bone names. */
	humanoid: {
		spine: "Spine",
		neck: "Neck",
		leftUpperArm: "LeftUpperArm",
		leftLowerArm: "LeftLowerArm",
		rightUpperArm: "RightUpperArm",
		rightLowerArm: "RightLowerArm",
		leftUpperLeg: "LeftUpperLeg",
		leftLowerLeg: "LeftLowerLeg",
		rightUpperLeg: "RightUpperLeg",
		rightLowerLeg: "RightLowerLeg",
	},
	/** Blender Rigify deform bones. */
	rigify: {
		spine: "spine",
		neck: "neck",
		leftUpperArm: "upper_arm.L",
		leftLowerArm: "forearm.L",
		rightUpperArm: "upper_arm.R",
		rightLowerArm: "forearm.R",
		leftUpperLeg: "thigh.L",
		leftLowerLeg: "shin.L",
		rightUpperLeg: "thigh.R",
		rightLowerLeg: "shin.R",
	},
} as const satisfies Record<string, RigMap>;

export interface RetargetOptions {
	/** The character's joint names: a preset or a map. Default "mixamo". */
	rig?: keyof typeof RIG_PRESETS | RigMap;
	/** Only these bones (e.g. arms, over a walk clip). Default: every mapped bone. */
	bones?: RetargetBone[];
	/**
	 * Drive the character's right side with the performer's left: for a
	 * character seen from behind. Default false (both face the camera).
	 */
	mirror?: boolean;
	/**
	 * Keypoints less sure than this hand their bone back to the animated
	 * pose, fading over the next 0.2 of confidence. Default 0.3.
	 */
	minVisibility?: number;
	/** Overall blend over the animated pose, 0..1: a number, keys, function or signal. Default 1. */
	weight?: NumberTrack;
}

type Point = PoseKeypointName | readonly [PoseKeypointName, PoseKeypointName];

/** Each bone's segment in the picture, parents first. */
const SEGMENTS: ReadonlyArray<[RetargetBone, Point, Point]> = [
	["spine", ["leftHip", "rightHip"], ["leftShoulder", "rightShoulder"]],
	["neck", ["leftShoulder", "rightShoulder"], "nose"],
	["leftUpperArm", "leftShoulder", "leftElbow"],
	["leftLowerArm", "leftElbow", "leftWrist"],
	["rightUpperArm", "rightShoulder", "rightElbow"],
	["rightLowerArm", "rightElbow", "rightWrist"],
	["leftUpperLeg", "leftHip", "leftKnee"],
	["leftLowerLeg", "leftKnee", "leftAnkle"],
	["rightUpperLeg", "rightHip", "rightKnee"],
	["rightLowerLeg", "rightKnee", "rightAnkle"],
];

const mirrored = (name: PoseKeypointName): PoseKeypointName =>
	(name.startsWith("left")
		? `right${name.slice(4)}`
		: name.startsWith("right")
			? `left${name.slice(5)}`
			: name) as PoseKeypointName;

/** A keypoint, or the midpoint of two, with the lesser visibility. */
function read(source: PoseSource, p: Point, frame: number, mirror: boolean) {
	const name = (n: PoseKeypointName) => (mirror ? mirrored(n) : n);
	if (typeof p === "string") return source.point(name(p), frame);
	const a = source.point(name(p[0]), frame);
	const b = source.point(name(p[1]), frame);
	return {
		x: (a.x + b.x) / 2,
		y: (a.y + b.y) / 2,
		visibility: Math.min(a.visibility, b.visibility),
	};
}

const smoothstep = (lo: number, hi: number, v: number) => {
	const t = Math.min(1, Math.max(0, (v - lo) / (hi - lo)));
	return t * t * (3 - 2 * t);
};

/**
 * Aim constraints that make a character copy `source`'s pose: put them on
 * a glTF model's `aim`. Bones whose joints the rig map doesn't name, or
 * whose keypoints the vision isn't sure of, keep the animated pose.
 */
export function retargetPose(
	source: PoseSource,
	options: RetargetOptions = {},
): AimConstraint[] {
	const rig: RigMap =
		typeof options.rig === "object"
			? options.rig
			: RIG_PRESETS[options.rig ?? "mixamo"];
	const only = options.bones ? new Set(options.bones) : undefined;
	const mirror = options.mirror ?? false;
	const lo = options.minVisibility ?? 0.3;
	return SEGMENTS.filter(
		([bone]) => rig[bone] && (!only || only.has(bone)),
	).map(([bone, from, to]) => ({
		joint: rig[bone] as string,
		direction: (frame: number): [number, number, number] => {
			const a = read(source, from, frame, mirror);
			const b = read(source, to, frame, mirror);
			return [b.x - a.x, b.y - a.y, 0];
		},
		// A signal, so the user's weight reads with the render's own clock.
		weight: {
			get: (ctx?: unknown) => {
				const clock = ctx as TrackClock;
				const a = read(source, from, clock.frame, mirror);
				const b = read(source, to, clock.frame, mirror);
				const sure = smoothstep(
					lo,
					lo + 0.2,
					Math.min(a.visibility, b.visibility),
				);
				return sure * numberAt(options.weight, clock, 1);
			},
		},
	}));
}