Motion capture for any creature from one AI video: two cameras in one frame, a DINOv2 joint reader, the rig fitted in both views, a GLB out.
"""Shared Blender helpers: the rig file, the two-camera layout, the studio, limb paint and debug renders.
The layout is the contract between every stage. One 1280 x 720 frame holds two orthographic views at the same scale:
the front camera (left) looks along +Y at the creature's face, the side camera (right) looks along +X at its right
flank, head to the right. Both share the vertical axis, so a joint's height is the same row in both halves; that is
what makes two views enough for 3D and what the sync check measures.
The creature is chosen with MOCAP_CREATURE=<assets dir>, whose creature.json can set its own framing (a dragon's
wingspan needs a wider front view) and limb paint groups. Without it, the wolf's settings below apply.
"""
import json, math, os, pathlib
import bpy
import numpy as np
from mathutils import Vector
H = 720
W_FRONT, W_SIDE = 420, 860
M_PER_PX = 1.9 / H # 1.9 m of height in frame
GROUND_V = 630 # pixel row of the ground (z = 0); the 90 rows below it are the floor band
LEG_COLORS = {"FL": "#F53CD1", "FR": "#2F6BFF", "HL": "#FFD21F", "HR": "#19D3E0"}
LEG_BONES = {"FL": ["upperarm_L", "forearm_L", "metacarpus_L", "fpaw_L"],
"FR": ["upperarm_R", "forearm_R", "metacarpus_R", "fpaw_R"],
"HL": ["thigh_L", "shin_L", "metatarsus_L", "hpaw_L"],
"HR": ["thigh_R", "shin_R", "metatarsus_R", "hpaw_R"]}
CREATURE = os.environ.get("MOCAP_CREATURE")
if CREATURE and (pathlib.Path(CREATURE) / "creature.json").exists():
_c = json.load(open(pathlib.Path(CREATURE) / "creature.json"))
_l = _c.get("layout", {})
W_FRONT, W_SIDE = _l.get("W_FRONT", W_FRONT), _l.get("W_SIDE", W_SIDE)
M_PER_PX = _l.get("M_PER_PX", M_PER_PX)
GROUND_V = _l.get("GROUND_V", GROUND_V)
if "paint" in _c:
LEG_COLORS = {k: v["color"] for k, v in _c["paint"].items()}
LEG_BONES = {k: v["bones"] for k, v in _c["paint"].items()}
Z_CENTER = (GROUND_V - H / 2) * M_PER_PX
LAYOUT = {"H": H, "W_FRONT": W_FRONT, "W_SIDE": W_SIDE, "M_PER_PX": M_PER_PX, "GROUND_V": GROUND_V,
"Z_CENTER": Z_CENTER, "Y_CENTER": 0.0, "X_CENTER": 0.0}
def mirror(name):
return name[:-2] + "_R" if name and name.endswith("_L") else name
def load_rig(path):
"""Joints and bones from rig.json with every _L mirrored to _R, and per-bone Euler limits in degrees."""
rig = json.load(open(path))
joints = dict(rig["joints"])
for k, v in rig["joints"].items():
if k.endswith("_L") and mirror(k) not in rig["joints"]:
joints[mirror(k)] = [-v[0], v[1], v[2]]
bones = []
for b in rig["bones"]:
bones.append(b)
if b[0].endswith("_L"):
bones.append([mirror(x) for x in b])
limits = {}
for name, *_ in bones:
key = name if name in rig["limits_deg"] else name[:-2]
limits[name] = rig["limits_deg"][key]
return joints, bones, limits
def srgb_to_linear(hexcol):
c = np.array([int(hexcol[i:i + 2], 16) / 255 for i in (1, 3, 5)])
return np.where(c <= 0.04045, c / 12.92, ((c + 0.055) / 1.055) ** 2.4)
def paint_legs(body, threshold=0.55):
"""Colour each leg from the shoulder or hip down to the paw, as a color attribute mixed over the texture."""
idx = {g.name: g.index for g in body.vertex_groups}
n = len(body.data.vertices)
w = {leg: np.zeros(n) for leg in LEG_BONES}
for v in body.data.vertices:
for g in v.groups:
for leg, names in LEG_BONES.items():
if body.vertex_groups[g.group].name in names:
w[leg][v.index] += g.weight
rgba = np.zeros((n, 4))
for leg, col in LEG_COLORS.items():
m = w[leg] > threshold
rgba[m, :3] = srgb_to_linear(col)
rgba[m, 3] = 1.0
attr = body.data.color_attributes.get("legpaint") or body.data.color_attributes.new("legpaint", "FLOAT_COLOR", "POINT")
attr.data.foreach_set("color", rgba.ravel())
for slot in body.material_slots:
nt = slot.material.node_tree
bsdf = next(nd for nd in nt.nodes if nd.type == "BSDF_PRINCIPLED")
src = bsdf.inputs["Base Color"].links[0].from_socket if bsdf.inputs["Base Color"].links else None
ca = nt.nodes.new("ShaderNodeVertexColor")
ca.layer_name = "legpaint"
mix = nt.nodes.new("ShaderNodeMix")
mix.data_type = "RGBA"
nt.links.new(ca.outputs["Alpha"], mix.inputs["Factor"])
if src is not None:
nt.links.new(src, mix.inputs["A"])
else:
mix.inputs["A"].default_value = bsdf.inputs["Base Color"].default_value
nt.links.new(ca.outputs["Color"], mix.inputs["B"])
nt.links.new(mix.outputs["Result"], bsdf.inputs["Base Color"])
print("painted vertices", {leg: int((w[leg] > threshold).sum()) for leg in w})
def write_rig_info(arm, joints, bones, limits, path):
"""Everything the solver needs to redo Blender's forward kinematics, plus which bone end each joint sits on."""
order = [b[0] for b in bones]
info = {"bones": [], "keypoints": [], "layout": LAYOUT}
for name, h, t, parent in bones:
bb = arm.data.bones[name]
info["bones"].append({"name": name, "parent": order.index(parent) if parent else -1,
"matrix_local": [list(r) for r in bb.matrix_local], "length": bb.length,
"head_joint": h, "tail_joint": t, "limits_deg": limits[name]})
for j in joints:
src = next(((i, "head") for i, b in enumerate(bones) if b[1] == j), None) or \
next(((i, "tail") for i, b in enumerate(bones) if b[2] == j), None)
info["keypoints"].append({"name": j, "bone": src[0], "end": src[1], "rest": joints[j]})
json.dump(info, open(path, "w"), indent=1)
def cameras(scene):
"""The front and side orthographic cameras of the layout (created once, reused)."""
cams = {}
for name, loc, rot, w in (("cam_front", (0, -20, Z_CENTER), (math.pi / 2, 0, 0), W_FRONT),
("cam_side", (-20, 0, Z_CENTER), (math.pi / 2, 0, -math.pi / 2), W_SIDE)):
cam = bpy.data.objects.get(name)
if cam is None:
cam = bpy.data.objects.new(name, bpy.data.cameras.new(name))
scene.collection.objects.link(cam)
cam.location, cam.rotation_euler = loc, rot
cam.data.type = "ORTHO"
cam.data.ortho_scale = M_PER_PX * max(w, H)
cam.data.clip_end = 100
cams[name] = (cam, w)
return cams
def camera_json(cam, w, h):
"""The camera's pixel projection, so 2D drawings can be laid exactly over its renders:
pc = R @ p + t; perspective u = cx + f * pc.x / -pc.z, v = cy - f * pc.y / -pc.z; ortho u = cx + f * pc.x."""
bpy.context.view_layer.update() # a camera placed from Python has a stale matrix_world until the next update
mw = cam.matrix_world.normalized()
R = mw.to_3x3().transposed()
t = -(R @ mw.translation)
if cam.data.type == "ORTHO":
f = max(w, h) / cam.data.ortho_scale
else:
f = cam.data.lens / cam.data.sensor_width * max(w, h)
return {"type": cam.data.type, "f": f, "cx": w / 2, "cy": h / 2, "res": [w, h],
"R": [list(r) for r in R], "t": list(t), "location": list(cam.location)}
def studio(scene, samples=32, seed=None):
"""Cycles on CUDA, transparent film, Standard view transform (keeps the leg paint saturated), three suns."""
scene.render.engine = "CYCLES"
prefs = bpy.context.preferences.addons["cycles"].preferences
prefs.compute_device_type = "CUDA"
prefs.get_devices()
for d in prefs.devices:
d.use = d.type == "CUDA"
scene.cycles.device = "GPU"
scene.cycles.samples = samples
scene.cycles.use_denoising = True
scene.cycles.max_bounces = 4
scene.render.film_transparent = True
scene.view_settings.view_transform = "Standard"
scene.render.image_settings.file_format = "PNG"
scene.render.image_settings.color_mode = "RGBA"
world = scene.world or bpy.data.worlds.new("world")
scene.world = world
world.use_nodes = True
bg = world.node_tree.nodes["Background"]
bg.inputs["Color"].default_value = (0.5, 0.5, 0.5, 1)
bg.inputs["Strength"].default_value = 0.55
for name, rot, energy in (("key", (math.radians(50), 0, math.radians(-35)), 3.2),
("fill", (math.radians(60), 0, math.radians(120)), 1.0),
("rim", (math.radians(35), 0, math.radians(200)), 1.6)):
ob = bpy.data.objects.get(name)
if ob is None:
ob = bpy.data.objects.new(name, bpy.data.lights.new(name, "SUN"))
scene.collection.objects.link(ob)
ob.rotation_euler = rot
ob.data.energy = energy
ob.data.angle = math.radians(8)
def render_views(out, prefix, scale=1.0, res=None):
"""Render the front and side views as RGBA PNGs at the layout's pixel scale times `scale`."""
scene = bpy.context.scene
paths = {}
for name, (cam, w) in cameras(scene).items():
scene.camera = cam
scene.render.resolution_x = round(w * scale)
scene.render.resolution_y = round(H * scale)
scene.render.resolution_percentage = 100
view = name.split("_")[1]
scene.render.filepath = str(out / f"{prefix}_{view}.png")
bpy.ops.render.render(write_still=True)
paths[view] = scene.render.filepath
return paths
def stick_figure(joints, bones, radius=0.012):
"""Spheres at joints and rods along bones (left pink, right blue, middle orange), drawn over the mesh."""
mats = {}
for side, col in (("L", (1, 0.1, 0.8, 1)), ("R", (0.1, 0.3, 1, 1)), ("M", (1, 0.5, 0, 1))):
m = bpy.data.materials.new(f"stick_{side}")
m.diffuse_color = col
mats[side] = m
side = lambda n: "L" if n.endswith("_L") else "R" if n.endswith("_R") else "M"
for name, p in joints.items():
bpy.ops.mesh.primitive_uv_sphere_add(radius=radius * 1.6, location=p, segments=12, ring_count=8)
bpy.context.object.data.materials.append(mats[side(name)])
for name, h, t, _ in bones:
a, b = Vector(joints[h]), Vector(joints[t])
bpy.ops.mesh.primitive_cylinder_add(radius=radius * 0.6, depth=(b - a).length, location=(a + b) / 2, vertices=8)
rod = bpy.context.object
rod.rotation_euler = (b - a).to_track_quat("Z", "Y").to_euler()
rod.data.materials.append(mats[side(name)])
def xray_views(out, prefix):
"""Workbench x-ray renders (side, front, top) so joints inside the mesh are visible."""
scene = bpy.context.scene
scene.render.engine = "BLENDER_WORKBENCH"
scene.display.shading.light = "STUDIO"
scene.display.shading.color_type = "MATERIAL"
scene.display.shading.show_xray = True
scene.display.shading.xray_alpha = 0.35
scene.render.film_transparent = False
cams = cameras(scene)
top = bpy.data.objects.new("cam_top", bpy.data.cameras.new("cam_top"))
scene.collection.objects.link(top)
top.location, top.rotation_euler = (0, 0, 20), (0, 0, 0)
top.data.type, top.data.ortho_scale = "ORTHO", M_PER_PX * W_SIDE
for name, cam, w, h in (("front", cams["cam_front"][0], W_FRONT, H), ("side", cams["cam_side"][0], W_SIDE, H),
("top", top, 600, W_SIDE)):
scene.camera = cam
scene.render.resolution_x, scene.render.resolution_y = w, h
scene.render.filepath = str(out / f"{prefix}_{name}.png")
bpy.ops.render.render(write_still=True)