Skip to content

Motion capture for any creature from one AI video: two cameras in one frame, a DINOv2 joint reader, the rig fitted in both views, a GLB out.

2.5 KB
Raw Blame
"""The two-camera frame: compose renders into it, split video frames out of it, map pixels to metres and back.

Mirrors blender/common.py. Front view (left, 420 px) looks along +Y: u grows with X. Side view (right, 860 px) looks
along +X: u grows with -Y, so the head (-Y) is on the right. In both, v grows downward with -Z and the ground (z = 0)
is row 630.
"""
import json, os, pathlib

import numpy as np

H, W_FRONT, W_SIDE = 720, 420, 860
M_PER_PX = 1.9 / H
GROUND_V = 630
# MOCAP_CREATURE=<assets dir> with a creature.json "layout" overrides the wolf's framing (see blender/common.py)
_cfg = pathlib.Path(os.environ.get("MOCAP_CREATURE", "")) / "creature.json"
if os.environ.get("MOCAP_CREATURE") and _cfg.exists():
    _l = json.load(open(_cfg)).get("layout", {})
    W_FRONT, W_SIDE = _l.get("W_FRONT", W_FRONT), _l.get("W_SIDE", W_SIDE)
    M_PER_PX, GROUND_V = _l.get("M_PER_PX", M_PER_PX), _l.get("GROUND_V", GROUND_V)
W = W_FRONT + W_SIDE
Z_CENTER = (GROUND_V - H / 2) * M_PER_PX
BG, FLOOR, DIVIDER = (128, 130, 134), (152, 154, 158), (34, 34, 38)


def compose(front_rgba, side_rgba, bg=BG, floor=FLOOR, divider=DIVIDER):
    """Alpha-composite the two RGBA renders over the studio backdrop (grey wall, lighter floor band, divider)."""
    out = np.zeros((H, W, 3), np.float32)
    out[:] = bg
    out[GROUND_V:] = floor
    for img, x0 in ((front_rgba, 0), (side_rgba, W_FRONT)):
        img = img.astype(np.float32)
        a = img[..., 3:4] / 255.0
        h, w = img.shape[:2]
        out[:h, x0:x0 + w] = img[..., :3] * a + out[:h, x0:x0 + w] * (1 - a)
    out[:, W_FRONT - 2:W_FRONT + 2] = divider
    return out.clip(0, 255).astype(np.uint8)


def split(frame):
    """A 1280 x 720 frame (resize first if needed) into its front and side views."""
    return frame[:, :W_FRONT], frame[:, W_FRONT:]


def project(p):
    """World points (..., 3) to pixel coordinates in each view: front (u, v) and side (u, v)."""
    p = np.asarray(p, np.float64)
    v = H / 2 - (p[..., 2] - Z_CENTER) / M_PER_PX
    front = np.stack([W_FRONT / 2 + p[..., 0] / M_PER_PX, v], -1)
    side = np.stack([W_SIDE / 2 - p[..., 1] / M_PER_PX, v], -1)
    return front, side


def unproject_front(uv):
    """Front-view pixels to (x, z) in metres."""
    uv = np.asarray(uv, np.float64)
    return np.stack([(uv[..., 0] - W_FRONT / 2) * M_PER_PX, Z_CENTER + (H / 2 - uv[..., 1]) * M_PER_PX], -1)


def unproject_side(uv):
    """Side-view pixels to (y, z) in metres."""
    uv = np.asarray(uv, np.float64)
    return np.stack([-(uv[..., 0] - W_SIDE / 2) * M_PER_PX, Z_CENTER + (H / 2 - uv[..., 1]) * M_PER_PX], -1)