Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
#!/usr/bin/env python3
"""Convert one NVIDIA SimReady OpenUSD asset into a web-ready GLB.
Usage: simready-to-glb.py <asset_root> <out.glb> [--max-tex 1024] [--meta out.json]
<asset_root> is the asset folder of the nvidia/simready-assets dataset (the one that
holds simready_usd/ and .metadata/). Needs `usd-core`, `numpy`, `Pillow`, `pygltflib`.
What it does: opens the root USD with instance proxies, bakes every mesh into world
space (Z-up to Y-up, meters), triangulates, splits face-varying normals/UVs into
glTF vertices, maps each GeomSubset's UsdPreviewSurface to a glTF PBR material
(albedo, ORM packed as occlusion + metallicRoughness, tangent-space normal), bakes the
UsdTransform2d tiling into the UVs, and downsizes textures to JPEG.
"""
import argparse
import io
import json
import sys
from pathlib import Path
import numpy as np
from PIL import Image
from pxr import Gf, Usd, UsdGeom, UsdShade
import pygltflib as gl
def attr_or_primvar(mesh, name):
"""Return (values, indices_or_None, interpolation) for a primvar, else None."""
pv = UsdGeom.PrimvarsAPI(mesh.GetPrim()).GetPrimvar(name)
if pv and pv.HasValue():
vals = pv.Get()
idx = pv.GetIndices() if pv.IsIndexed() else None
return np.asarray(vals, dtype=np.float32), (np.asarray(idx, dtype=np.int64) if idx is not None else None), pv.GetInterpolation()
return None
def shader_input_source(shader, name):
inp = shader.GetInput(name)
if not inp:
return None, None
if inp.HasConnectedSource():
src, out, _ = inp.GetConnectedSource()
return src, out
return inp.Get(), None
class TexCache:
def __init__(self, max_tex):
self.max_tex = max_tex
self.images = {}
def load(self, path):
key = str(path)
if key not in self.images:
im = Image.open(path)
im.load()
self.images[key] = im
return self.images[key]
def resolve_tex(shader_prim):
"""For a UsdUVTexture prim return (absolute path, tiling scale, offset)."""
sh = UsdShade.Shader(shader_prim)
f = sh.GetInput('file').Get()
path = f.resolvedPath if f and f.resolvedPath else None
scale, offset = (1.0, 1.0), (0.0, 0.0)
st = sh.GetInput('st')
if st and st.HasConnectedSource():
t = st.GetConnectedSource()[0].GetPrim()
if UsdShade.Shader(t).GetIdAttr().Get() == 'UsdTransform2d':
tsh = UsdShade.Shader(t)
s = tsh.GetInput('scale').Get() or Gf.Vec2f(1, 1)
o = tsh.GetInput('translation').Get() or Gf.Vec2f(0, 0)
scale, offset = (float(s[0]), float(s[1])), (float(o[0]), float(o[1]))
return path, scale, offset
def to_jpeg(im, max_tex, channels=None, quality=86):
im = im.copy()
if channels is not None:
im = im.convert('RGB') if len(channels) == 3 else im
else:
im = im.convert('RGB')
im.thumbnail((max_tex, max_tex), Image.LANCZOS)
buf = io.BytesIO()
im.save(buf, 'JPEG', quality=quality, optimize=True)
return buf.getvalue()
def extract_material(mat_prim, cache, max_tex):
"""Describe one USD material as plain data: factors plus texture paths."""
mat = UsdShade.Material(mat_prim)
surf = mat.ComputeSurfaceSource()[0]
out = {'name': mat_prim.GetName(), 'base': (0.8, 0.8, 0.8), 'rough': 0.6, 'metal': 0.0,
'albedo': None, 'orm': None, 'rough_tex': None, 'metal_tex': None, 'normal': None,
'scale': (1.0, 1.0), 'offset': (0.0, 0.0), 'opacity': 1.0}
if not surf:
return out
def tex_of(name):
v, o = shader_input_source(surf, name)
if o is None:
return v, None, None
prim = v.GetPrim()
if UsdShade.Shader(prim).GetIdAttr().Get() != 'UsdUVTexture':
return None, None, None
path, scale, offset = resolve_tex(prim)
if not path or not Path(path).exists():
print(f'warning: texture missing for {mat_prim.GetName()}.{name}: {path}', file=sys.stderr)
return None, None, None
return None, (path, str(o)), (scale, offset)
v, t, tr = tex_of('diffuseColor')
if t:
out['albedo'] = t[0]
out['scale'], out['offset'] = tr
elif v is not None:
out['base'] = tuple(float(x) for x in v)
v, t, tr = tex_of('roughness')
if t:
out['rough_tex'] = t
elif v is not None:
out['rough'] = float(v)
v, t, tr = tex_of('metallic')
if t:
out['metal_tex'] = t
elif v is not None:
out['metal'] = float(v)
v, t, tr = tex_of('normal')
if t:
out['normal'] = t[0]
v, _ = shader_input_source(surf, 'opacity')
if isinstance(v, (int, float)):
out['opacity'] = float(v)
return out
def triangulate(mesh):
counts = np.asarray(mesh.GetFaceVertexCountsAttr().Get(), dtype=np.int64)
idx = np.asarray(mesh.GetFaceVertexIndicesAttr().Get(), dtype=np.int64)
starts = np.concatenate([[0], np.cumsum(counts)[:-1]])
tri_face, tri_corner = [], []
# fan triangulation, vectorised per polygon size
for n in np.unique(counts):
if n < 3:
continue
fs = np.nonzero(counts == n)[0]
for k in range(1, n - 1):
tri_face.append(np.stack([fs, fs, fs], 1))
tri_corner.append(np.stack([starts[fs], starts[fs] + k, starts[fs] + k + 1], 1))
if not tri_face:
return None
face = np.concatenate(tri_face)[:, 0]
corners = np.concatenate(tri_corner)
order = np.argsort(face, kind='stable')
return face[order], corners[order], idx, counts, starts
def mesh_to_arrays(mesh, xf, normal_xf):
tri = triangulate(mesh)
if tri is None:
return None
face, corners, fv_idx, counts, starts = tri
pts = np.asarray(mesh.GetPointsAttr().Get(), dtype=np.float64)
if mesh.GetOrientationAttr().Get() == 'leftHanded':
corners = corners[:, ::-1]
corner_flat = corners.reshape(-1)
p_idx = fv_idx[corner_flat]
nrm = None
nattr = mesh.GetNormalsAttr().Get()
ninterp = mesh.GetNormalsInterpolation()
pvn = attr_or_primvar(mesh, 'normals')
if nattr is not None:
arr = np.asarray(nattr, dtype=np.float32)
if ninterp == 'faceVarying':
nrm = arr[corner_flat]
elif ninterp == 'vertex':
nrm = arr[p_idx]
elif ninterp == 'uniform':
nrm = arr[np.repeat(face, 3)]
elif ninterp == 'constant':
nrm = np.broadcast_to(arr[0], (len(corner_flat), 3))
elif pvn is not None:
arr, ix, interp = pvn
arr = arr[ix] if ix is not None else arr
nrm = arr[corner_flat] if interp == 'faceVarying' else arr[p_idx]
uv = None
pv = attr_or_primvar(mesh, 'st')
if pv is not None:
arr, ix, interp = pv
if interp == 'faceVarying':
a = arr[ix] if ix is not None else arr
uv = a[corner_flat]
elif interp == 'vertex':
a = arr[ix] if ix is not None else arr
uv = a[p_idx]
pts_w = (np.c_[pts, np.ones(len(pts))] @ np.array(xf, dtype=np.float64))[:, :3]
pos = pts_w[p_idx].astype(np.float32)
if nrm is not None:
nm = np.array(normal_xf, dtype=np.float64)
nn = (nrm.astype(np.float64) @ nm[:3, :3])
ln = np.linalg.norm(nn, axis=1, keepdims=True)
ln[ln == 0] = 1
nrm = (nn / ln).astype(np.float32)
return pos, nrm, uv, face
def weld(pos, nrm, uv):
"""Collapse identical corner vertices so the index buffer is shared."""
cols = [pos]
if nrm is not None:
cols.append(nrm)
if uv is not None:
cols.append(uv)
key = np.concatenate(cols, axis=1)
key_r = np.round(key * 1e5).astype(np.int64)
_, first, inv = np.unique(key_r, axis=0, return_index=True, return_inverse=True)
inv = inv.reshape(-1)
return first, inv.astype(np.uint32)
def build(asset_root, out_path, max_tex):
root = Path(asset_root)
usd_dir = root / 'simready_usd'
usds = [f for pat in ('sm_*.usd', 'sm_*.usda', 'sm_*.usdc') for f in sorted(usd_dir.glob(pat))] \
or [f for pat in ('*.usd', '*.usda', '*.usdc') for f in sorted(usd_dir.glob(pat))]
if not usds:
raise SystemExit(f'no root usd in {root}/simready_usd')
stage = Usd.Stage.Open(str(usds[0]))
up = UsdGeom.GetStageUpAxis(stage)
mpu = UsdGeom.GetStageMetersPerUnit(stage) or 1.0
cache = UsdGeom.XformCache()
zup = np.array([[1, 0, 0, 0], [0, 0, -1, 0], [0, 1, 0, 0], [0, 0, 0, 1]], dtype=np.float64) # Z-up to Y-up (row vectors)
fix = zup if up == 'Z' else np.eye(4)
scale = np.diag([mpu, mpu, mpu, 1.0])
gltf = gl.GLTF2(asset=gl.Asset(version='2.0', generator='three.ws simready-to-glb'))
gltf.scenes = [gl.Scene(nodes=[])]
gltf.scene = 0
blob = bytearray()
def add_view(data, target=None):
while len(blob) % 4:
blob.append(0)
off = len(blob)
blob.extend(data)
gltf.bufferViews.append(gl.BufferView(buffer=0, byteOffset=off, byteLength=len(data), target=target))
return len(gltf.bufferViews) - 1
def add_accessor(arr, comp, typ, target, minmax=False):
v = add_view(arr.tobytes(), target)
a = gl.Accessor(bufferView=v, componentType=comp, count=len(arr), type=typ)
if minmax:
a.min = arr.min(0).tolist()
a.max = arr.max(0).tolist()
gltf.accessors.append(a)
return len(gltf.accessors) - 1
cache_tex = TexCache(max_tex)
img_ids = {}
mat_ids = {}
def add_image(key, data):
if key not in img_ids:
v = add_view(data)
gltf.images.append(gl.Image(bufferView=v, mimeType='image/jpeg'))
gltf.textures.append(gl.Texture(source=len(gltf.images) - 1, sampler=0))
img_ids[key] = len(gltf.textures) - 1
return img_ids[key]
gltf.samplers = [gl.Sampler(magFilter=9729, minFilter=9987, wrapS=10497, wrapT=10497)]
def get_material(prim):
key = str(prim.GetPath().name)
if key in mat_ids:
return mat_ids[key][0], mat_ids[key][1]
d = extract_material(prim, cache_tex, max_tex)
pbr = gl.PbrMetallicRoughness(baseColorFactor=[*d['base'], d['opacity']],
roughnessFactor=d['rough'], metallicFactor=d['metal'])
m = gl.Material(name=d['name'], pbrMetallicRoughness=pbr, doubleSided=False)
if d['albedo'] and Path(d['albedo']).exists():
im = cache_tex.load(d['albedo'])
pbr.baseColorTexture = gl.TextureInfo(index=add_image(('a', d['albedo']), to_jpeg(im, max_tex)))
pbr.baseColorFactor = [1, 1, 1, d['opacity']]
rt, mt = d['rough_tex'], d['metal_tex']
if rt and mt and rt[0] == mt[0] and Path(rt[0]).exists():
im = cache_tex.load(rt[0]).convert('RGB')
pbr.metallicRoughnessTexture = gl.TextureInfo(index=add_image(('o', rt[0]), to_jpeg(im, max_tex)))
m.occlusionTexture = gl.OcclusionTextureInfo(index=pbr.metallicRoughnessTexture.index)
pbr.roughnessFactor, pbr.metallicFactor = 1.0, 1.0
elif rt and Path(rt[0]).exists():
im = cache_tex.load(rt[0]).convert('RGB')
ch = {'r': 0, 'g': 1, 'b': 2}.get(rt[1], 1)
r = np.asarray(im)[..., ch]
packed = np.zeros(r.shape + (3,), np.uint8)
packed[..., 1] = r
pbr.metallicRoughnessTexture = gl.TextureInfo(index=add_image(('r', rt[0], ch), to_jpeg(Image.fromarray(packed), max_tex)))
pbr.roughnessFactor = 1.0
if d['normal'] and Path(d['normal']).exists():
im = cache_tex.load(d['normal'])
m.normalTexture = gl.NormalMaterialTexture(index=add_image(('n', d['normal']), to_jpeg(im, max_tex, quality=90)))
if d['opacity'] < 1.0:
m.alphaMode = 'BLEND'
gltf.materials.append(m)
mat_ids[key] = (len(gltf.materials) - 1, d)
return mat_ids[key]
stats = {'meshes': 0, 'tris': 0, 'materials': 0}
bounds_lo = np.full(3, np.inf)
bounds_hi = np.full(3, -np.inf)
for prim in Usd.PrimRange.Stage(stage, Usd.TraverseInstanceProxies()):
if not prim.IsA(UsdGeom.Mesh):
continue
img = UsdGeom.Imageable(prim)
if img.ComputeVisibility() == UsdGeom.Tokens.invisible or img.ComputePurpose() in (UsdGeom.Tokens.guide, UsdGeom.Tokens.proxy):
continue
if prim.GetAttribute('physics:collisionEnabled').IsValid() and not prim.GetAttribute('primvars:st').IsValid() \
and not UsdGeom.PrimvarsAPI(prim).GetPrimvar('st'):
if 'collision' in prim.GetName().lower():
continue
mesh = UsdGeom.Mesh(prim)
world = cache.GetLocalToWorldTransform(prim)
xf = np.array(world, dtype=np.float64) @ scale @ fix
xf_n = np.linalg.inv(xf).T
arrs = mesh_to_arrays(mesh, xf, xf_n)
if arrs is None:
continue
pos, nrm, uv, face = arrs
if np.linalg.det(xf[:3, :3]) < 0:
pos = pos.reshape(-1, 3, 3)[:, ::-1].reshape(-1, 3)
nrm = nrm.reshape(-1, 3, 3)[:, ::-1].reshape(-1, 3) if nrm is not None else None
uv = uv.reshape(-1, 3, 2)[:, ::-1].reshape(-1, 2) if uv is not None else None
subsets = UsdGeom.Subset.GetAllGeomSubsets(mesh)
groups = []
if subsets:
covered = np.zeros(int(face.max()) + 1, bool)
for ss in subsets:
fi = np.asarray(ss.GetIndicesAttr().Get() or [], dtype=np.int64)
covered[fi[fi < len(covered)]] = True
mb = UsdShade.MaterialBindingAPI(ss.GetPrim()).ComputeBoundMaterial()[0]
sel = np.isin(face, fi)
if sel.any():
groups.append((mb.GetPrim() if mb else None, sel))
rest = ~covered[face]
if rest.any():
mb = UsdShade.MaterialBindingAPI(prim).ComputeBoundMaterial()[0]
groups.append((mb.GetPrim() if mb else None, rest))
else:
mb = UsdShade.MaterialBindingAPI(prim).ComputeBoundMaterial()[0]
groups.append((mb.GetPrim() if mb else None, np.ones(len(face), bool)))
prims_out = []
for mprim, sel in groups:
tri_sel = np.nonzero(sel)[0]
c = (tri_sel[:, None] * 3 + np.arange(3)).reshape(-1)
p_, n_, u_ = pos[c], (nrm[c] if nrm is not None else None), (uv[c] if uv is not None else None)
mid = None
if mprim is not None:
mid, d = get_material(mprim)
if u_ is not None:
u_ = u_ * np.array(d['scale'], np.float32) + np.array(d['offset'], np.float32)
if u_ is not None:
u_ = u_.copy()
u_[:, 1] = 1.0 - u_[:, 1] # USD v is up, glTF v is down
first, inv = weld(p_, n_, u_)
vp = np.ascontiguousarray(p_[first], np.float32)
attrs = gl.Attributes(POSITION=add_accessor(vp, gl.FLOAT, 'VEC3', gl.ARRAY_BUFFER, True))
if n_ is not None:
attrs.NORMAL = add_accessor(np.ascontiguousarray(n_[first], np.float32), gl.FLOAT, 'VEC3', gl.ARRAY_BUFFER)
if u_ is not None:
attrs.TEXCOORD_0 = add_accessor(np.ascontiguousarray(u_[first], np.float32), gl.FLOAT, 'VEC2', gl.ARRAY_BUFFER)
ia = add_accessor(inv.astype(np.uint32), gl.UNSIGNED_INT, 'SCALAR', gl.ELEMENT_ARRAY_BUFFER)
prims_out.append(gl.Primitive(attributes=attrs, indices=ia, material=mid, mode=4))
bounds_lo = np.minimum(bounds_lo, vp.min(0))
bounds_hi = np.maximum(bounds_hi, vp.max(0))
stats['tris'] += len(inv) // 3
if not prims_out:
continue
gltf.meshes.append(gl.Mesh(name=prim.GetName(), primitives=prims_out))
gltf.nodes.append(gl.Node(name=prim.GetName(), mesh=len(gltf.meshes) - 1))
gltf.scenes[0].nodes.append(len(gltf.nodes) - 1)
stats['meshes'] += 1
if not stats['meshes']:
raise SystemExit('no renderable meshes found')
stats['materials'] = len(gltf.materials)
stats['size_m'] = [round(float(x), 4) for x in (bounds_hi - bounds_lo)]
gltf.buffers = [gl.Buffer(byteLength=len(blob))]
gltf.set_binary_blob(bytes(blob))
gltf.save_binary(out_path)
stats['bytes'] = Path(out_path).stat().st_size
return stats
def main():
ap = argparse.ArgumentParser()
ap.add_argument('asset_root')
ap.add_argument('out')
ap.add_argument('--max-tex', type=int, default=1024)
ap.add_argument('--meta')
a = ap.parse_args()
st = build(a.asset_root, a.out, a.max_tex)
if a.meta:
Path(a.meta).write_text(json.dumps(st))
print(json.dumps(st))
if __name__ == '__main__':
main()