Motion capture for any creature from one AI video: two cameras in one frame, a DINOv2 joint reader, the rig fitted in both views, a GLB out.
"""Put a solved (or made-up) motion on the rig: keyframes, true joint positions, renders and a GLB.
blender -b -P blender/apply_motion.py -- rigged.blend rig_info.json motion.json out_dir [options]
--views front,side,persp which cameras to render (default none)
--engine cycles|workbench (default cycles)
--paint keep the leg paint (the exam needs it; the showcase does not)
--blur motion blur from the keyframes, like a video model's fast limbs
--glb export out_dir/animation.glb
--frames 10,40,80 render only these frames (stills), for quick comparisons
--springs fur.json simulate the fur bones (blender/add_fur_bones.py) as damped springs
--scale 0.75 render at this fraction of the layout size
--ghost render the mesh as an x-ray ghost (workbench), for drawing a skeleton over
Always writes out_dir/joints.json, the world position of every joint per frame from Blender itself, and for every
rendered view out_dir/cam_<view>.json, its pixel projection (common.camera_json).
"""
import json, math, pathlib, sys
import bpy
import numpy as np
from mathutils import Matrix, Quaternion, Vector
sys.path.insert(0, str(pathlib.Path(__file__).parent))
import common # noqa: E402
args = sys.argv[sys.argv.index("--") + 1:]
blend, info_path, motion_path, out = args[:4]
opts = args[4:]
opt = lambda k, d=None: opts[opts.index(k) + 1] if k in opts else d
out = pathlib.Path(out)
out.mkdir(parents=True, exist_ok=True)
bpy.ops.wm.open_mainfile(filepath=blend)
scene = bpy.context.scene
info = json.load(open(info_path))
motion = json.load(open(motion_path))
rig, body = bpy.data.objects["rig"], bpy.data.objects["body"]
names = motion["bones"]
T = len(motion["euler"])
fps = motion.get("fps", 24)
scene.render.fps = round(fps)
scene.render.fps_base = round(fps) / fps
scene.frame_start, scene.frame_end = 0, T - 1
if "--paint" not in opts:
attr = body.data.color_attributes["legpaint"]
rgba = np.zeros(len(attr.data) * 4)
attr.data.foreach_set("color", rgba)
root = names[0]
B = rig.data.bones[root].matrix_local.to_3x3()
for pb in rig.pose.bones:
pb.rotation_mode = "XYZ"
rig.pose.bones[root].rotation_mode = "QUATERNION"
for t in range(T):
pr = rig.pose.bones[root]
pr.rotation_quaternion = Quaternion(motion["root_quat"][t])
pr.location = B.transposed() @ Vector(motion["trans"][t])
pr.keyframe_insert("rotation_quaternion", frame=t)
pr.keyframe_insert("location", frame=t)
for i, n in enumerate(names[1:], 1):
pb = rig.pose.bones[n]
pb.rotation_euler = motion["euler"][t][i]
pb.keyframe_insert("rotation_euler", frame=t)
for fc in rig.animation_data.action.fcurves if hasattr(rig.animation_data.action, "fcurves") else []:
for k in fc.keyframe_points:
k.interpolation = "LINEAR"
if opt("--springs"):
# Secondary motion for the fur bones: each tip is a damped spring chasing a target, so fur lags behind the
# body, overshoots and settles. The target is the rigid (sculpted) direction pulled part of the way back toward
# the bone's rest direction in the world ("gravity": fur keeps hanging down when the body rears or sits).
# Sub-stepped, the tip kept at bone length and within max_deg of the target.
springs = json.load(open(opt("--springs")))
sub = 6
dt = 1.0 / (fps * sub)
state, swing = {}, {n: [] for n in springs}
rest_world = {}
for name in springs:
b = rig.data.bones[name]
rest_world[name] = (rig.matrix_world.to_3x3() @ (b.tail_local - b.head_local)).normalized()
for t in range(T):
scene.frame_set(t)
for name, s in springs.items():
pb = rig.pose.bones[name]
par = pb.parent
rigid = rig.matrix_world @ par.matrix @ (par.bone.matrix_local.inverted() @ pb.bone.matrix_local)
L = pb.bone.length
head = rigid.translation.copy()
rdir = (rigid @ Vector((0, L, 0)) - head).normalized()
gdir = rdir.slerp(rest_world[name], s.get("gravity", 0.0))
target = head + gdir * L
if name not in state:
state[name] = {"x": target.copy(), "v": Vector((0, 0, 0)), "h": head, "g": target}
st = state[name]
w = 2 * math.pi * s["freq"]
k, c = w * w, 2 * s["damping"] * w
cos_max = math.cos(math.radians(s["max_deg"]))
for i in range(1, sub + 1):
a = i / sub
h, g = st["h"].lerp(head, a), st["g"].lerp(target, a)
st["v"] += (k * (g - st["x"]) - c * st["v"]) * dt
st["x"] += st["v"] * dt
d = (st["x"] - h).normalized()
gd = (g - h).normalized()
if d.dot(gd) < cos_max: # past the swing limit: put it on the cone's edge
axis = gd.cross(d).normalized()
d = gd.copy()
d.rotate(Quaternion(axis, math.radians(s["max_deg"])))
st["x"] = h + d * L
st["h"], st["g"] = head, target
final = (st["x"] - head).normalized()
swing[name].append(math.degrees(math.acos(max(-1.0, min(1.0, final.dot(rdir))))))
local = rigid.to_3x3().inverted() @ final
pb.rotation_mode = "QUATERNION"
pb.rotation_quaternion = Vector((0, 1, 0)).rotation_difference(local)
pb.keyframe_insert("rotation_quaternion", frame=t)
print("springs simulated (swing from the sculpted direction, mean / max deg):",
{n: (round(sum(v) / len(v), 1), round(max(v), 1)) for n, v in swing.items()})
joints = []
for t in range(T):
scene.frame_set(t)
mw = rig.matrix_world
row = []
for kp in info["keypoints"]:
pb = rig.pose.bones[info["bones"][kp["bone"]]["name"]]
row.append(list(mw @ (pb.head if kp["end"] == "head" else pb.tail)))
joints.append(row)
json.dump({"names": [k["name"] for k in info["keypoints"]], "joints": joints, "fps": fps}, open(out / "joints.json", "w"))
print("joints written", T, "frames")
views = [v for v in opt("--views", "").split(",") if v]
if views:
engine = "ghost" if "--ghost" in opts else opt("--engine", "cycles")
if engine == "ghost":
scene.render.engine = "BLENDER_WORKBENCH"
scene.display.shading.light = "FLAT"
scene.display.shading.color_type = "SINGLE"
scene.display.shading.single_color = (0.55, 0.56, 0.6)
scene.display.shading.show_xray = True
scene.display.shading.xray_alpha = 0.18
scene.render.film_transparent = True
elif engine == "cycles":
common.studio(scene, samples=int(opt("--samples", 32)))
scene.cycles.use_adaptive_sampling = True
scene.cycles.denoising_use_gpu = True
scene.render.threads_mode = "FIXED"
scene.render.threads = 4
else:
scene.render.engine = "BLENDER_WORKBENCH"
scene.display.shading.light = "STUDIO"
scene.display.shading.color_type = "TEXTURE"
scene.render.film_transparent = True
scene.render.use_motion_blur = "--blur" in opts
scene.render.motion_blur_shutter = 0.5
cams = common.cameras(scene)
if "persp" in views: # a three-quarter showcase camera (--cam x,y,z --target x,y,z --lens mm --res WxH)
cam = bpy.data.objects.new("cam_persp", bpy.data.cameras.new("cam_persp"))
scene.collection.objects.link(cam)
cam.data.lens = float(opt("--lens", 50))
vec = lambda k, d: Vector([float(x) for x in opt(k, d).split(",")])
target = vec("--target", "0,0,0.5")
cam.location = vec("--cam", "-3.2,-3.0,1.6")
cam.rotation_euler = (target - cam.location).to_track_quat("-Z", "Y").to_euler()
cam.data.clip_end = 200
pw, ph = (int(x) for x in opt("--res", "1280x720").split("x"))
cams["cam_persp"] = (cam, (pw, ph))
floor = None
if "persp" in views and engine == "cycles": # a shadow catcher, so the creature stands on the ground in 3/4 shots
bpy.ops.mesh.primitive_plane_add(size=80, location=(0, 0, 0))
floor = bpy.context.object
floor.is_shadow_catcher = True
for v in views:
cam, w = cams[f"cam_{v}"]
scene.camera = cam
if floor is not None:
floor.hide_render = v != "persp"
k = float(opt("--scale", 1.0))
w, hgt = w if isinstance(w, tuple) else (w, common.H)
scene.render.resolution_x = round(w * k)
scene.render.resolution_y = round(hgt * k)
json.dump(common.camera_json(cam, scene.render.resolution_x, scene.render.resolution_y), open(out / f"cam_{v}.json", "w"))
d = out / v
d.mkdir(exist_ok=True)
scene.render.image_settings.file_format = "PNG"
scene.render.image_settings.color_mode = "RGBA"
if opt("--frames"):
for f in [int(x) for x in opt("--frames").split(",")]:
scene.frame_set(f)
scene.render.filepath = str(d / f"f_{f:04d}.png")
bpy.ops.render.render(write_still=True)
else:
scene.render.filepath = str(d / "f_")
bpy.ops.render.render(animation=True)
print("rendered", v, flush=True)
if "--glb" in opts:
bpy.ops.object.select_all(action="DESELECT")
rig.select_set(True)
body.select_set(True)
for o in list(scene.objects):
if o.type in ("CAMERA", "LIGHT"):
o.select_set(False)
bpy.ops.export_scene.gltf(filepath=str(out / "animation.glb"), use_selection=True, export_animations=True,
export_force_sampling=True, export_frame_range=True)
print("glb written")