Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
"""
Make-It-Animatable driver: mesh in, skeleton + skinning weights out.
Wraps jasongzy/Make-It-Animatable (MIT, CVPR 2025) as a pure predictor. We run
its prepare/preprocess/infer/vis stages headlessly and read the results off
its DB state object; we deliberately do NOT run its Blender/FBX export stage.
The visual mesh never round-trips through Blender, so the original GLB's
materials and PBR textures survive untouched; rig_glb.py grafts the predicted
skeleton straight into the original bytes.
MIA specifics this module encodes:
- init must happen on the MAIN thread: importing MIA's app module builds the
Mixamo kinematic tree through bpy, and bpy crashes when scene data is
touched from a child thread. Per-job prediction never touches bpy.
- MIA normalizes the input into its own working frame and composes the
transform onto db.global_transform; we invert it to bring joints and
working vertices back into the caller's GLB space.
- Output skeleton is MIA's 52-bone Mixamo subset (MIXAMO_JOINTS in
util/dataset_mixamo.py: 30 finger bones included, no HeadTop_End and no eye
bones), names prefixed "mixamorig:". The platform's glb-canonicalize maps
these onto the canonical clip skeleton at ingest, so every clip in the
library retargets; tests/glb-canonicalize.test.js pins the same 52 names.
Environment:
MIA_DIR path to the Make-It-Animatable checkout (default /app/mia); its
output/best/new/*.pth weights and data/Mixamo/bones.fbx template
must be in place before init() (see stage_assets.py).
"""
from __future__ import annotations
import logging
import os
import sys
import numpy as np
log = logging.getLogger("rig.engine_mia")
MIA_DIR = os.environ.get("MIA_DIR", "/app/mia")
_mia = None
_names: list[str] | None = None
_parents: list[int] | None = None
def init() -> None:
"""Import MIA and load its six checkpoints. Main thread only; ~seconds."""
global _mia, _names, _parents
if _mia is not None:
return
if not os.path.isdir(MIA_DIR):
raise RuntimeError(f"MIA_DIR not found: {MIA_DIR}")
sys.path.insert(0, MIA_DIR)
import app as mia_app # noqa: PLC0415 (heavy import by design, once)
# MIA replaces gradio.helpers.log_message with a version that reads
# LocalContext.blocks.get(); on current gradio that ContextVar has no
# default, so every Timing(print_fn=gr.Info) raises LookupError outside a
# live Gradio request. Re-replace it with a print-only logger: this
# service has no UI to toast to.
import warnings # noqa: PLC0415
import gradio.helpers # noqa: PLC0415
def _headless_log(message, level="info", *args, **kwargs):
if level == "warning":
warnings.warn(message)
else:
print(message)
gradio.helpers.log_message = _headless_log
mia_app.init_models()
# The stage functions return Gradio component-update dicts keyed by UI
# globals that only exist after init_blocks(); build the (unlaunched)
# Blocks so those names resolve. No server is started here.
mia_app.init_blocks()
from util.dataset_mixamo import BONES_IDX_DICT, KINEMATIC_TREE # noqa: PLC0415
ordered = sorted(BONES_IDX_DICT.items(), key=lambda kv: kv[1])
_names = [name for name, _ in ordered]
_parents = list(KINEMATIC_TREE.parent_indices)
if len(_parents) != len(_names):
raise RuntimeError(
f"kinematic tree size {len(_parents)} != bone count {len(_names)}"
)
_mia = mia_app
log.info("MIA ready: %d bones, root=%s", len(_names), _names[0])
def is_ready() -> bool:
return _mia is not None
def joint_names() -> list[str]:
if _names is None:
raise RuntimeError("engine not initialized")
return list(_names)
def predict(glb_path: str) -> dict:
"""Run MIA on a GLB file and return predictions in the GLB's own space.
Returns:
verts (V,3) MIA's working vertices, back in input space (weight rows
align to these; rig_glb KD-trees them onto the GLB primitives)
weights (V,J)
joints (J,3) bone heads, input space
joints_tail (J,3)
parents (J,) with -1 for the root
names [J] "mixamorig:*" bone names
"""
if _mia is None:
raise RuntimeError("engine not initialized")
import shutil # noqa: PLC0415
import torch # noqa: PLC0415
db = _mia.DB()
try:
_mia.prepare_input(glb_path, False, 0.0, db, export_temp=True)
_mia.preprocess(db)
_mia.infer(False, db)
# bw_fix applies MIA's weight post-process (head/finger cleanups);
# vis also converts db fields to numpy and fills joints_tail.
_mia.vis(True, "LeftArm", False, db)
finally:
# MIA caches GPU tensors on the DB and writes stage visualizations into
# a per-job temp dir; drop both even on failure so a poison mesh cannot
# leak VRAM or disk across jobs.
torch.cuda.empty_cache()
if db.output_dir:
shutil.rmtree(db.output_dir, ignore_errors=True)
if db.bw is None or db.joints is None or db.joints_tail is None:
raise RuntimeError("MIA produced no rig prediction")
# db.global_transform maps input space -> MIA's normalized frame (it was
# composed across prepare_input and infer); invert to go home.
inv = db.global_transform.inverse()
def _back(points: np.ndarray) -> np.ndarray:
t = torch.as_tensor(np.asarray(points, dtype=np.float32)).reshape(1, -1, 3)
return inv.transform_points(t).squeeze(0).cpu().numpy().astype(np.float32)
verts = _back(db.verts)
joints = _back(db.joints)
joints_tail = _back(db.joints_tail)
weights = np.asarray(db.bw, dtype=np.float32)
if weights.shape[0] != verts.shape[0]:
raise RuntimeError(
f"weights ({weights.shape[0]}) misaligned with working verts ({verts.shape[0]})"
)
if weights.shape[1] != len(_names):
raise RuntimeError(
f"weight columns ({weights.shape[1]}) != bone count ({len(_names)})"
)
return {
"verts": verts,
"weights": weights,
"joints": joints,
"joints_tail": joints_tail,
"parents": np.asarray(_parents, dtype=np.int64),
"names": list(_names),
}