Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// GLB to USDZ, in the browser.
//
// This is what makes "Place in your space" real on an iPhone. Apple's AR Quick
// Look is the only way to get true ARKit placement on iOS, and it reads USDZ,
// not glTF. There is no server here doing the conversion: the model is loaded
// into a scene, prepared, and exported with three.js's own USDZExporter, on the
// device, in a couple of seconds.
//
// Three preparation steps matter, and skipping any of them produces a model
// that loads but looks broken:
//
// 1. Skinned meshes are baked to static geometry. USDZExporter carries no
// skeletons: it writes raw vertex attributes plus each node's world
// matrix. Export a rigged character without baking and Quick Look shows a
// collapsed pile with limbs balled at the hips.
// 2. Materials are coerced to MeshStandardMaterial. USDZ speaks
// UsdPreviewSurface, which maps cleanly from Standard only; unlit, Phong
// and Toon materials are dropped with a warning.
// 3. Missing normals are computed. Position-only meshes (decimated props,
// procedural geometry) otherwise export flat and unlit.
//
// Ported from the three.ws USDZ pipeline (Apache-2.0).
import {
Box3, BufferAttribute, Color, DoubleSide, Group, Mesh, MeshStandardMaterial, Vector3,
} from 'three';
import { clone as cloneSkinnedScene } from 'three/addons/utils/SkeletonUtils.js';
import { sharedGLTFLoaderReady } from './loaders.js';
import { fitTransform } from './scene-math.js';
/**
* CPU-skin one SkinnedMesh at its current pose, returning deformed positions in
* the mesh's own local space. The caller must have updated world matrices.
*
* @param {import('three').SkinnedMesh} mesh
* @returns {Float32Array} vertexCount * 3
*/
export function bakedLocalPositions(mesh) {
const posAttr = mesh.geometry.getAttribute('position');
const out = new Float32Array(posAttr.count * 3);
const v = new Vector3();
for (let i = 0; i < posAttr.count; i++) {
v.fromBufferAttribute(posAttr, i);
mesh.applyBoneTransform(i, v);
out[i * 3] = v.x;
out[i * 3 + 1] = v.y;
out[i * 3 + 2] = v.z;
}
return out;
}
/** Replace every SkinnedMesh with a static Mesh frozen at the current pose. */
export function bakeSkinnedMeshes(scene) {
scene.updateMatrixWorld(true);
const skinned = [];
scene.traverse((obj) => {
if (obj.isSkinnedMesh && obj.skeleton?.bones?.length) skinned.push(obj);
});
for (const mesh of skinned) {
if (!mesh.geometry.getAttribute('position')) continue;
const baked = mesh.geometry.clone();
baked.setAttribute('position', new BufferAttribute(bakedLocalPositions(mesh), 3));
// Skinning data is meaningless on a static mesh and confuses the exporter.
baked.deleteAttribute('skinIndex');
baked.deleteAttribute('skinWeight');
// Normals were authored for the bind pose; recompute so the shading
// matches the geometry Quick Look actually receives.
baked.computeVertexNormals();
const replacement = new Mesh(baked, mesh.material);
replacement.name = mesh.name;
replacement.visible = mesh.visible;
// applyBoneTransform returns local-space vertices, so the replacement
// keeps the original's local transform and the exporter applies it.
replacement.position.copy(mesh.position);
replacement.quaternion.copy(mesh.quaternion);
replacement.scale.copy(mesh.scale);
const parent = mesh.parent || scene;
parent.add(replacement);
parent.remove(mesh);
}
}
/** Coerce every material to MeshStandardMaterial, in place. */
export function coerceMaterialsToStandard(scene) {
scene.traverse((obj) => {
if (!obj.isMesh) return;
const mats = Array.isArray(obj.material) ? obj.material : [obj.material];
mats.forEach((m, i) => {
if (!m || m.isMeshStandardMaterial) return;
const replacement = new MeshStandardMaterial({
color: m.color ? m.color.clone() : new Color(0xffffff),
map: m.map || null,
normalMap: m.normalMap || null,
roughness: typeof m.roughness === 'number' ? m.roughness : 0.85,
metalness: typeof m.metalness === 'number' ? m.metalness : 0,
transparent: !!m.transparent,
opacity: typeof m.opacity === 'number' ? m.opacity : 1,
side: m.side ?? DoubleSide,
});
if (Array.isArray(obj.material)) obj.material[i] = replacement;
else obj.material = replacement;
});
});
}
/** Give every renderable mesh a normal attribute. */
export function ensureNormals(scene) {
scene.traverse((obj) => {
if (!obj.isMesh || !obj.geometry) return;
if (obj.geometry.getAttribute('normal')) return;
if (!obj.geometry.getAttribute('position')) return;
obj.geometry.computeVertexNormals();
});
}
/**
* Wrap content in a fresh, identity-transform root and a `stage` node that
* carries the export's own placement.
*
* THIS WRAPPER IS NOT TIDINESS, IT IS A CORRECTNESS FIX. `USDZExporter` walks
* `scene.children` and writes each node's LOCAL matrix, so the transform on the
* object you hand it is never written at all. Set a scale on the root and the
* export silently comes out at the original size; set a position and it is
* silently dropped. Everything this module needs to place a model (the pinch
* scale, the floor offset, the recentre) therefore lives on `stage`, which is a
* child and so does get written.
*
* @param {import('three').Object3D} object
* @returns {{root: import('three').Group, stage: import('three').Group}}
*/
export function usdzExportRoot(object) {
const root = new Group();
root.name = 'Root';
const stage = new Group();
stage.name = 'Model';
stage.add(object);
root.add(stage);
root.updateMatrixWorld(true);
return { root, stage };
}
/** World-space bounds of a prepared export root, or null when it holds nothing. */
function measure(root) {
root.updateMatrixWorld(true);
const box = new Box3().setFromObject(root);
if (box.isEmpty()) return null;
const min = box.min, max = box.max;
if (![min.x, min.y, min.z, max.x, max.y, max.z].every(Number.isFinite)) return null;
return box;
}
/**
* Stand the model on y=0 with its footprint centred on the origin.
*
* This is the single thing that decides whether AR Quick Look puts a model on
* someone's floor or leaves it hanging in mid-air. Quick Look anchors the
* scene's ORIGIN to the horizontal plane it detects and does not look at the
* geometry: a GLB authored with its origin at the bounding-box centre arrives
* half-buried, one authored around a distant scene origin arrives floating
* across the room at eye level, and in both cases the model reads as "not
* tracking" because it never touches the ground the person is pointing at.
* Neither is a Quick Look bug and no anchoring property fixes it; the content
* has to be moved.
*
* @param {import('three').Group} stage The node from `usdzExportRoot`.
* @param {import('three').Group} root
* @returns {boolean} false when there was nothing to ground.
*/
export function groundOnFloor(stage, root) {
const box = measure(root);
if (!box) return false;
const centre = box.getCenter(new Vector3());
stage.position.x -= centre.x;
stage.position.y -= box.min.y;
stage.position.z -= centre.z;
root.updateMatrixWorld(true);
return true;
}
/**
* Give the model a believable real-world size.
*
* USDZ is metres (`metersPerUnit = 1`), and a model authored in centimetres
* arrives a hundred times too big. A ten-metre chair cannot sit on a plane
* ARKit found in a living room, so Quick Look shows it swimming around the
* viewer rather than resting anywhere: the same symptom as a broken anchor,
* from a completely different cause.
*
* Uses the studio's own normalization rule, so a model placed through the
* studio and the same model converted straight from its URL land at the same
* size. Models already within 2x of a believable size are left exactly as
* authored: real furniture scans stay real.
*
* @param {import('three').Group} stage
* @param {import('three').Group} root
* @returns {number} the scale applied (1 when the model was already sane).
*/
export function fitToRoomScale(stage, root) {
const box = measure(root);
if (!box) return 1;
let skinned = false;
root.traverse((o) => { if (o.isSkinnedMesh) skinned = true; });
const { scale } = fitTransform(
{ min: { x: box.min.x, y: box.min.y, z: box.min.z }, max: { x: box.max.x, y: box.max.y, z: box.max.z } },
{ skinned },
);
if (!(scale > 0) || scale === 1) return 1;
stage.scale.multiplyScalar(scale);
root.updateMatrixWorld(true);
return scale;
}
/**
* Largest footprint (m) AR Quick Look can be relied on to find a plane for.
*
* Quick Look does not place a model until ARKit has found a horizontal plane
* BIG ENOUGH TO FIT IT. Until then the model hangs aligned to the camera and
* travels with the phone, which every person reads as broken tracking rather
* than as "still looking". Past roughly this size there is no such plane in an
* ordinary room, so the model never lands at all.
*/
export const MAX_AR_FOOTPRINT_M = 2.5;
/** Tallest (m) a model can be and still belong in a room with a ceiling. */
export const MAX_AR_HEIGHT_M = 2.5;
/**
* Keep the export inside a size ARKit can actually place.
*
* A ceiling, not a normalizer: real furniture, a person, a car-door-sized prop
* all pass through untouched. It exists because the size someone pinched a
* model to is now honoured (it used to be silently dropped), and the studio
* lets that reach 4x. Four times a standing avatar is a seven-metre figure that
* no living-room floor plane will ever fit.
*
* @param {import('three').Group} stage
* @param {import('three').Group} root
* @param {{maxFootprint?: number, maxHeight?: number}} [limits]
* @returns {number} the scale applied (1 when it already fit).
*/
export function clampToPlaceableSize(stage, root, {
maxFootprint = MAX_AR_FOOTPRINT_M, maxHeight = MAX_AR_HEIGHT_M,
} = {}) {
const box = measure(root);
if (!box) return 1;
const size = box.getSize(new Vector3());
const footprint = Math.max(size.x, size.z);
// The floor plane has to fit the footprint; the room has to fit the height.
// Whichever is tighter decides.
const scale = Math.min(
footprint > 0 ? maxFootprint / footprint : 1,
size.y > 0 ? maxHeight / size.y : 1,
);
if (!(scale < 1) || !Number.isFinite(scale)) return 1;
stage.scale.multiplyScalar(scale);
root.updateMatrixWorld(true);
return scale;
}
/**
* Convert a loaded scene to USDZ bytes. Mutates the scene, so pass a clone or a
* scene you are done with.
*
* The model is always grounded before it is written: see `groundOnFloor` for
* why that, and not any anchoring flag, is what makes AR Quick Look put it on
* the floor.
*
* @param {import('three').Object3D} scene
* @param {object} [options]
* @param {boolean} [options.fit] Normalize an absurdly authored size to a
* believable real-world one. On for a raw file straight off the network, off
* for a model the studio has already normalized and the person has resized.
* Everything else is passed through to three's USDZExporter.
* @returns {Promise<Blob>} model/vnd.usdz+zip
*/
export async function sceneToUsdzBlob(scene, { fit = false, ...options } = {}) {
const { root, stage } = usdzExportRoot(scene);
bakeSkinnedMeshes(root);
coerceMaterialsToStandard(root);
ensureNormals(root);
// Size first, then stand it on the floor: grounding measures the model at the
// size it will actually be exported at, and a model too big for any plane in
// the room never gets placed on one at all.
if (fit) fitToRoomScale(stage, root);
clampToPlaceableSize(stage, root);
groundOnFloor(stage, root);
// Loaded on demand: nobody who never taps "Place in your space" should pay
// for the exporter.
const { USDZExporter } = await import('three/addons/exporters/USDZExporter.js');
const bytes = await new USDZExporter().parseAsync(root, {
// Quick Look is the only thing that ever reads these bytes, and it applies
// texture repeat and offset in a different order to every other USD
// renderer (Apple FB10036297). This flag pre-compensates, so a model with
// tiled textures does not arrive in someone's room with the tiling
// visibly wrong. It is a no-op on the untiled default.
quickLookCompatible: true,
// Rest the model on the horizontal surface the person points at, which is
// what "place it on your floor" means. Stated rather than left to the
// exporter's default so a future default cannot quietly change it.
includeAnchoringProperties: true,
ar: { anchoring: { type: 'plane' }, planeAnchoring: { alignment: 'horizontal' } },
...options,
});
return new Blob([bytes], { type: 'model/vnd.usdz+zip' });
}
/**
* Convert an object that is ALREADY in a live scene to USDZ, without touching
* the network.
*
* This is the fast path, and on a phone the difference is the whole feature.
* Re-fetching and re-parsing the GLB costs seconds and a second CORS round trip
* on a file the page has already decoded; worse, it exports the model's rest
* pose at its authored size. Cloning the live object exports what the person is
* actually looking at: the current animation pose, and the size they pinched it
* to, which is the size Quick Look will stand in their room.
*
* The clone is what gets baked and mutated, so the live scene is untouched.
* Position and yaw are zeroed because the AR viewer re-anchors the model to the
* surface the user picks; carrying the studio's floor coordinates through would
* only offset it from their own reticle. World scale is preserved: it is the
* real-world size, and `sceneToUsdzBlob` is what makes it survive the export.
*
* No `fit`: this model went through the studio's own normalization when it was
* placed, and the size on top of that is the one the person pinched.
*
* @param {import('three').Object3D} object
* @returns {Promise<Blob>} model/vnd.usdz+zip
*/
export async function objectToUsdzBlob(object) {
object.updateMatrixWorld(true);
// SkeletonUtils.clone rebinds skinned meshes to the cloned bones and copies
// their current transforms, so the pose survives; Object3D.clone does not.
const copy = cloneSkinnedScene(object);
copy.position.set(0, 0, 0);
copy.rotation.set(0, 0, 0);
copy.scale.copy(object.getWorldScale(new Vector3()));
copy.updateMatrixWorld(true);
return sceneToUsdzBlob(copy);
}
/**
* Fetch a GLB and convert it to a USDZ blob.
*
* @param {string} glbUrl
* @param {{signal?: AbortSignal, onProgress?: (stage: string) => void}} [opts]
* @returns {Promise<Blob>}
*/
export async function glbUrlToUsdzBlob(glbUrl, { signal, onProgress } = {}) {
onProgress?.('download');
const res = await fetch(glbUrl, { signal });
if (!res.ok) throw new Error(`model fetch ${res.status}`);
const buffer = await res.arrayBuffer();
onProgress?.('parse');
// Awaited rather than taken synchronously: a meshopt-compressed GLB handed to
// a loader whose decoder is still downloading fails on a valid file.
const loader = await sharedGLTFLoaderReady();
const gltf = await new Promise((resolve, reject) => {
loader.parse(buffer, '', resolve, reject);
});
const scene = gltf.scene || gltf.scenes?.[0];
if (!scene) throw new Error('that model contains no scene');
onProgress?.('convert');
// Straight off the network and never seen by the studio's normalizer, so this
// is the one path that has to defend against a model authored in centimetres.
return sceneToUsdzBlob(scene, { fit: true });
}