Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* A3S packer - turns one GLB into a progressive A3S stream. Node only.
*
* The packer's whole job is to earn the format's central promise: that a prefix
* of the file is a complete asset. It does that in four moves.
*
* 1. Build a level-of-detail chain by simplifying *successively*, finest to
* coarsest. Each step collapses edges of the previous step's index buffer,
* so the vertices surviving at level k are a strict subset of those at
* level k+1. That nesting is not incidental, it is the property the whole
* format rests on, and `verifyNesting` asserts it rather than assuming it.
* 2. Rank every vertex by the coarsest level it survives into, then reorder
* the vertex buffer by that rank. Now level k's vertices are exactly the
* first V(k) entries, and a patch can append instead of rewrite.
* 3. Emit level 0 as a standalone, spec-valid GLB carrying the full node
* hierarchy and skin, so the coarse mesh is already skinned and posed.
* 4. Emit each finer level as a patch: the newly revealed vertex bytes plus a
* replacement index buffer.
*
* Skinning survives all of this for free: meshopt's simplifier only ever emits a
* new index buffer over the original vertex array, so JOINTS_0 and WEIGHTS_0
* ride along untouched. Morph targets are carried explicitly, reordered and
* sliced in lockstep with the base attributes they deform.
*/
import { createHash } from 'node:crypto';
import { NodeIO } from '@gltf-transform/core';
import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
import { cloneDocument, dequantize, prune, weldPrimitive } from '@gltf-transform/functions';
import { MeshoptSimplifier, MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
import draco3d from 'draco3dgltf';
import sharp from 'sharp';
import { VERSION_TAG, encodeContainer, decodeHeader, decodePreamble, align4 } from './format.js';
/** Default level ratios, coarsest first. The trailing 1.0 level is the original mesh. */
export const DEFAULT_LEVELS = [0.03, 0.1, 0.3, 1.0];
/** Longest edge, in pixels, of the textures baked into the base layer. */
export const DEFAULT_BASE_TEXTURE_SIZE = 64;
/** glTF primitive mode for triangles; the only mode a mesh simplifier can reason about. */
const MODE_TRIANGLES = 4;
/** Compression extensions the base layer deliberately does not depend on. */
const COMPRESSION_EXTENSIONS = new Set(['EXT_meshopt_compression', 'KHR_draco_mesh_compression']);
const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
let ioPromise = null;
/** Build (once) a NodeIO wired for every extension the platform's avatars use. */
export async function getIO() {
if (!ioPromise) {
ioPromise = (async () => {
await MeshoptDecoder.ready;
await MeshoptEncoder.ready;
return new NodeIO().registerExtensions(ALL_EXTENSIONS).registerDependencies({
'draco3d.decoder': await draco3d.createDecoderModule(),
'draco3d.encoder': await draco3d.createEncoderModule(),
'meshopt.decoder': MeshoptDecoder,
'meshopt.encoder': MeshoptEncoder,
});
})();
}
return ioPromise;
}
/**
* Assert that a chain of index buffers has strictly nested vertex sets, coarsest
* first. A packer that skipped this could still produce a plausible-looking file
* whose patches silently reference vertices the client was never sent.
*/
export function verifyNesting(chain) {
const sets = chain.map((indices) => new Set(indices));
for (let i = 0; i < sets.length - 1; i++) {
for (const v of sets[i]) {
if (!sets[i + 1].has(v)) {
throw new Error(`a3s: level ${i} is not nested inside level ${i + 1}; cannot pack progressively`);
}
}
}
return sets;
}
/**
* Build the coarse-to-fine index chain for one primitive.
* @returns {Uint32Array[]} index buffers, coarsest first, finest (original) last
*/
export function buildLodChain(indices, positions, levels) {
const original = indices instanceof Uint32Array ? indices : new Uint32Array(indices);
const triangles = original.length / 3;
// Walk fine -> coarse so each simplification consumes the previous result,
// which is what makes the surviving vertex sets nest.
const descending = levels.filter((r) => r < 1).sort((a, b) => b - a);
const coarseToFine = [];
let current = original;
for (const ratio of descending) {
const target = Math.max(1, Math.floor(triangles * ratio)) * 3;
if (target >= current.length) continue;
const [simplified] = MeshoptSimplifier.simplify(current, positions, 3, target, 1.0, ['LockBorder']);
const next = simplified instanceof Uint32Array ? simplified : new Uint32Array(simplified);
// A tiny primitive can collapse to nothing. An empty level would slice its
// accessors to count 0, which is invalid glTF, so keep the finer buffer
// rather than emitting a level that renders as a hole.
if (next.length < 3) continue;
current = next;
coarseToFine.unshift(current);
}
// Pad so every primitive reports the same number of levels even when a small
// mesh could not be decimated further; readers index layers positionally.
while (coarseToFine.length < levels.length - 1) coarseToFine.unshift(coarseToFine[0] || original);
coarseToFine.push(original);
return coarseToFine;
}
/**
* Rank vertices by the coarsest level they appear in, then produce the
* old->new remap that groups each level's vertices into a contiguous prefix.
*/
export function computeVertexOrder(chain, vertexCount) {
const RANK_UNUSED = chain.length; // vertices no level references sort last
const rank = new Uint32Array(vertexCount).fill(RANK_UNUSED);
for (let level = chain.length - 1; level >= 0; level--) {
for (const v of chain[level]) rank[v] = level;
}
const sorted = Array.from({ length: vertexCount }, (_, i) => i).sort((a, b) => rank[a] - rank[b] || a - b);
const remap = new Uint32Array(vertexCount);
sorted.forEach((oldIndex, newIndex) => {
remap[oldIndex] = newIndex;
});
const levelVertexCounts = [];
let seen = 0;
for (let level = 0; level < chain.length; level++) {
while (seen < vertexCount && rank[sorted[seen]] <= level) seen++;
levelVertexCounts.push(seen);
}
return { remap, order: Uint32Array.from(sorted), levelVertexCounts };
}
/** Reorder a flat typed array of `order.length` elements of `stride` components. */
function reorderArray(array, order, stride) {
const out = new array.constructor(order.length * stride);
for (let i = 0; i < order.length; i++) {
const src = order[i] * stride;
const dst = i * stride;
for (let c = 0; c < stride; c++) out[dst + c] = array[src + c];
}
return out;
}
function indexArrayFor(vertexCount, indices) {
return vertexCount <= 0xffff ? Uint16Array.from(indices) : Uint32Array.from(indices);
}
/**
* Every per-vertex accessor a primitive owns, including morph target deltas.
*
* Accessors are routinely shared between primitives in a real GLB, so each one
* is cloned into sole ownership first. Without that, reordering the vertices of
* one primitive silently scrambles every sibling that pointed at the same
* buffer, which surfaces much later as "all accessors of the same primitive must
* have the same count".
*/
function claimVertexAccessors(prim) {
const entries = [];
for (const semantic of prim.listSemantics()) {
const owned = prim.getAttribute(semantic).clone();
prim.setAttribute(semantic, owned);
entries.push({ key: semantic, accessor: owned });
}
prim.listTargets().forEach((target, t) => {
for (const semantic of target.listSemantics()) {
const owned = target.getAttribute(semantic).clone();
target.setAttribute(semantic, owned);
entries.push({ key: `targets/${t}/${semantic}`, accessor: owned });
}
});
return entries;
}
async function resizeTexture(bytes, mimeType, maxSize) {
const image = sharp(Buffer.from(bytes), { failOn: 'none' });
const meta = await image.metadata();
const longest = Math.max(meta.width || 0, meta.height || 0);
if (!longest) return null;
const pipeline = longest > maxSize ? image.resize({ width: maxSize, height: maxSize, fit: 'inside' }) : image;
// Re-encode into the source container so the base layer stays loadable in
// every glTF viewer, not just the ones that negotiated a newer image codec.
let out;
if (mimeType === 'image/jpeg') out = await pipeline.jpeg({ quality: 80 }).toBuffer();
else if (mimeType === 'image/webp') out = await pipeline.webp({ quality: 80 }).toBuffer();
else out = await pipeline.png({ compressionLevel: 9 }).toBuffer();
const outMeta = await sharp(out).metadata();
return { bytes: new Uint8Array(out), width: outMeta.width, height: outMeta.height };
}
/**
* Pack a GLB into an A3S container.
*
* @param {Uint8Array|Buffer} glbBytes
* @param {object} [options]
* @param {number[]} [options.levels] LOD ratios, coarsest first, ending at 1.0
* @param {number} [options.baseTextureSize] longest texture edge in the base layer
* @param {string} [options.name] source name recorded in the header
* @returns {Promise<{ container: Uint8Array, header: object, stats: object }>}
*/
export async function pack(glbBytes, options = {}) {
const levels = options.levels || DEFAULT_LEVELS;
const baseTextureSize = options.baseTextureSize || DEFAULT_BASE_TEXTURE_SIZE;
const source = glbBytes instanceof Uint8Array ? glbBytes : new Uint8Array(glbBytes);
const levelCount = levels.length;
await MeshoptSimplifier.ready;
const io = await getIO();
const doc = await io.readBinary(new Uint8Array(source));
const root = doc.getRoot();
// Positions arrive quantized on most platform avatars; the simplifier needs
// real floats, and the base layer is small enough that the trade is cheap.
await doc.transform(dequantize());
// Layer 0's whole promise is that it opens anywhere, so it must not require a
// decoder the reader may not have installed. Dropping the compression
// extensions the source declared costs a few kilobytes and buys a base layer
// that loads in a stock GLTFLoader, in Blender, and in the Khronos validator.
for (const extension of root.listExtensionsUsed()) {
if (COMPRESSION_EXTENSIONS.has(extension.extensionName)) extension.dispose();
}
const primPlans = [];
let skipped = 0;
let primOrdinal = 0;
for (const mesh of root.listMeshes()) {
for (const prim of mesh.listPrimitives()) {
const ordinal = primOrdinal++;
const position = prim.getAttribute('POSITION');
// Points, lines and strips have no triangles to collapse. They ride in
// the base layer at full detail rather than being dropped.
if (!position || prim.getMode() !== MODE_TRIANGLES) {
skipped++;
continue;
}
if (!prim.getIndices()) {
// A non-indexed triangle soup shares no vertices between triangles, so
// every edge reads as a border and the simplifier can collapse nothing.
// Welding restores the shared topology the format needs.
weldPrimitive(prim);
}
const vertexCount = prim.getAttribute('POSITION').getCount();
let sourceIndices = prim.getIndices()?.getArray();
if (!sourceIndices) {
sourceIndices = new Uint32Array(vertexCount);
for (let i = 0; i < vertexCount; i++) sourceIndices[i] = i;
}
if (sourceIndices.length < 3) {
skipped++;
continue;
}
const entries = claimVertexAccessors(prim);
const positions = prim.getAttribute('POSITION').getArray();
const chain = buildLodChain(new Uint32Array(sourceIndices), positions, levels);
verifyNesting(chain);
const { remap, order, levelVertexCounts } = computeVertexOrder(chain, vertexCount);
const snapshot = {};
for (const { key, accessor } of entries) {
const stride = accessor.getElementSize();
const reordered = reorderArray(accessor.getArray(), order, stride);
accessor.setArray(reordered);
snapshot[key] = {
array: reordered,
elementSize: stride,
componentType: accessor.getComponentType(),
type: accessor.getType(),
normalized: accessor.getNormalized(),
};
}
const remapped = chain.map((indices) => {
const out = new Uint32Array(indices.length);
for (let i = 0; i < indices.length; i++) out[i] = remap[indices[i]];
return out;
});
// An indexed primitive is required from here on, since every level ships
// its own index buffer.
if (!prim.getIndices()) {
prim.setIndices(doc.createAccessor().setType('SCALAR').setArray(new Uint32Array(sourceIndices)));
} else {
prim.setIndices(prim.getIndices().clone());
}
prim.setExtras({ ...(prim.getExtras() || {}), a3sPrim: ordinal });
const morphTargetNames = mesh.getExtras()?.targetNames || null;
primPlans.push({
ordinal,
prim,
mesh,
entries,
snapshot,
chain: remapped,
levelVertexCounts,
vertexCount,
morphTargetCount: prim.listTargets().length,
morphTargetNames: Array.isArray(morphTargetNames) ? morphTargetNames : null,
morphWeights: mesh.getWeights() || null,
});
}
}
if (!primPlans.length) {
throw new Error('a3s: no triangle primitives found; nothing to stream progressively');
}
// --- Texture pyramid ------------------------------------------------------
// The base layer carries thumbnails so it stays tiny; the final layer carries
// the originals. The format permits more steps; the packer emits two.
const texturePlans = [];
const textures = root.listTextures();
for (let i = 0; i < textures.length; i++) {
const texture = textures[i];
const image = texture.getImage();
const mimeType = texture.getMimeType();
if (!image || !/^image\/(png|jpeg|webp)$/.test(mimeType || '')) continue;
const small = await resizeTexture(image, mimeType, baseTextureSize);
if (!small) continue;
texturePlans.push({
index: i,
mimeType,
full: new Uint8Array(image),
small: small.bytes,
slots: describeTextureSlots(root, texture),
});
texture.setImage(small.bytes);
}
// --- Layer 0: a standalone, valid GLB ------------------------------------
for (const plan of primPlans) sliceToLevel(plan, 0);
// Animation tracks are the heaviest thing in a rigged avatar: on a 65-joint
// character they routinely outweigh the mesh several times over, and they are
// the one part nobody can see in the first frame. The base layer keeps the
// skeleton and sheds the clips, which ship as a geometry-free companion GLB
// in the first patch. Clip tracks bind to nodes by name, so they drop onto
// the base skeleton the moment they land.
let animationGlb = null;
const animationNames = root.listAnimations().map((a) => a.getName() || null);
if (animationNames.length) {
const animDoc = cloneDocument(doc);
const animRoot = animDoc.getRoot();
// A node may not keep a `skin` once its `mesh` is gone: that pairing is a
// hard glTF constraint, and the companion carries clips, not geometry.
for (const node of animRoot.listNodes()) {
node.setMesh(null);
node.setSkin(null);
}
for (const skin of animRoot.listSkins()) skin.dispose();
for (const mesh of animRoot.listMeshes()) mesh.dispose();
for (const material of animRoot.listMaterials()) material.dispose();
for (const texture of animRoot.listTextures()) texture.dispose();
disposeAccessorsExcept(animRoot, collectAnimationAccessors(animRoot));
await animDoc.transform(prune({ keepAttributes: false, keepLeaves: true }));
animationGlb = new Uint8Array(await io.writeBinary(animDoc));
for (const animation of root.listAnimations()) {
// Disposing an Animation orphans its sampler accessors rather than
// deleting them, and they are the heaviest thing in the file.
for (const sampler of animation.listSamplers()) {
sampler.getInput()?.dispose();
sampler.getOutput()?.dispose();
}
animation.dispose();
}
}
await doc.transform(prune({ keepAttributes: false, keepLeaves: true }));
const baseGlb = new Uint8Array(await io.writeBinary(doc));
// --- Layers 1..N: patches -------------------------------------------------
const patches = [];
const layerDescriptors = [];
for (let level = 1; level < levelCount; level++) {
const chunks = [];
let cursor = 0;
const prims = [];
for (const plan of primPlans) {
const from = plan.levelVertexCounts[level - 1];
const to = plan.levelVertexCounts[level];
const attributes = {};
for (const [key, meta] of Object.entries(plan.snapshot)) {
// Each attribute tracks its own high-water mark, because morph deltas
// start from zero here while base attributes resume from the base layer.
const start = plan.delivered[key] ?? 0;
if (to <= start) continue;
const stride = meta.elementSize;
const slice = meta.array.subarray(start * stride, to * stride);
const bytes = new Uint8Array(slice.buffer, slice.byteOffset, slice.byteLength);
attributes[key] = {
offset: cursor,
length: bytes.byteLength,
componentType: meta.componentType,
type: meta.type,
normalized: meta.normalized,
elementSize: stride,
start,
count: to - start,
};
chunks.push(bytes);
cursor = align4(cursor + bytes.byteLength);
plan.delivered[key] = to;
}
const indices = indexArrayFor(to, plan.chain[level]);
const indexBytes = new Uint8Array(indices.buffer, indices.byteOffset, indices.byteLength);
prims.push({
prim: plan.ordinal,
newVertexStart: from,
newVertexCount: to - from,
vertexCount: to,
morphTargetCount: plan.morphTargetCount,
morphTargetNames: plan.morphTargetNames,
morphWeights: plan.morphWeights,
attributes,
indices: {
offset: cursor,
length: indexBytes.byteLength,
componentType: indices.BYTES_PER_ELEMENT === 2 ? 5123 : 5125,
count: indices.length,
},
});
chunks.push(indexBytes);
cursor = align4(cursor + indexBytes.byteLength);
}
// The first patch carries the animation bundle: clips matter to a viewer
// long before the finest triangles do.
let animationChunk = null;
if (level === 1 && animationGlb) {
animationChunk = { offset: cursor, length: animationGlb.byteLength, clips: animationNames };
chunks.push(animationGlb);
cursor = align4(cursor + animationGlb.byteLength);
}
// The finest level also delivers the full-resolution textures.
const texturePatches = [];
if (level === levelCount - 1) {
for (const plan of texturePlans) {
texturePatches.push({
texture: plan.index,
offset: cursor,
length: plan.full.byteLength,
mimeType: plan.mimeType,
slots: plan.slots,
});
chunks.push(plan.full);
cursor = align4(cursor + plan.full.byteLength);
}
}
const payload = new Uint8Array(cursor);
let write = 0;
for (const chunk of chunks) {
payload.set(chunk, write);
write = align4(write + chunk.byteLength);
}
patches.push(payload);
layerDescriptors.push({
level,
kind: 'patch',
sha256: sha256(payload),
triangleCount: primPlans.reduce((n, p) => n + p.chain[level].length / 3, 0),
vertexCount: primPlans.reduce((n, p) => n + p.levelVertexCounts[level], 0),
prims,
textures: texturePatches,
animations: animationChunk,
});
}
const header = {
version: VERSION_TAG,
generator: '@three-ws/avatar-stream',
source: {
name: options.name || null,
sha256: sha256(source),
byteLength: source.byteLength,
},
geometry: {
vertexCount: primPlans.reduce((n, p) => n + p.vertexCount, 0),
triangleCount: primPlans.reduce((n, p) => n + p.chain[levelCount - 1].length / 3, 0),
primitiveCount: primPlans.length,
passthroughPrimitiveCount: skipped,
},
levels,
layers: [
{
level: 0,
kind: 'base',
sha256: sha256(baseGlb),
triangleCount: primPlans.reduce((n, p) => n + p.chain[0].length / 3, 0),
vertexCount: primPlans.reduce((n, p) => n + p.levelVertexCounts[0], 0),
},
...layerDescriptors,
],
};
const container = encodeContainer({ header, baseGlb, patches });
// Read the offsets straight back out of the encoded bytes, so what the caller
// inspects is exactly what a client will parse, never a hopeful copy.
const encodedHeader = decodeHeader(container, decodePreamble(container));
return {
container,
header: encodedHeader,
stats: {
sourceBytes: source.byteLength,
containerBytes: container.byteLength,
baseBytes: baseGlb.byteLength,
baseTriangles: header.layers[0].triangleCount,
fullTriangles: header.geometry.triangleCount,
layerCount: levelCount,
},
};
}
/** Every accessor an animation clip needs in order to play. */
function collectAnimationAccessors(root) {
const keep = new Set();
for (const animation of root.listAnimations()) {
for (const sampler of animation.listSamplers()) {
const input = sampler.getInput();
const output = sampler.getOutput();
if (input) keep.add(input);
if (output) keep.add(output);
}
}
return keep;
}
/** Drop every accessor the document owns that is not in `keep`. */
function disposeAccessorsExcept(root, keep) {
for (const accessor of root.listAccessors()) {
if (!keep.has(accessor)) accessor.dispose();
}
}
/** Map a texture to the material slots that reference it, for client-side swap-in. */
function describeTextureSlots(root, texture) {
const slots = [];
for (const material of root.listMaterials()) {
const pairs = [
['baseColorTexture', material.getBaseColorTexture()],
['metallicRoughnessTexture', material.getMetallicRoughnessTexture()],
['normalTexture', material.getNormalTexture()],
['occlusionTexture', material.getOcclusionTexture()],
['emissiveTexture', material.getEmissiveTexture()],
];
for (const [slot, tex] of pairs) {
if (tex === texture) slots.push({ material: material.getName() || null, slot });
}
}
return slots;
}
/** A morph target delta, as opposed to a base vertex attribute. */
const isMorphKey = (key) => key.startsWith('targets/');
/**
* Slice one primitive down to a level for the base layer.
*
* Morph target deltas are dropped entirely rather than sliced. A blendshape is
* invisible until something drives it, so on a face-rigged avatar its deltas are
* pure weight in the first frame: they were up to 72% of the base layer before
* this. They arrive with the first geometry patch instead, and `delivered`
* records that nothing has been sent for them yet.
*/
function sliceToLevel(plan, level) {
const keep = plan.levelVertexCounts[level];
plan.delivered = {};
for (const { key, accessor } of plan.entries) {
const meta = plan.snapshot[key];
if (isMorphKey(key)) {
plan.delivered[key] = 0;
continue;
}
accessor.setArray(meta.array.slice(0, keep * meta.elementSize));
plan.delivered[key] = keep;
}
for (const target of plan.prim.listTargets()) plan.prim.removeTarget(target);
// A mesh may not keep a weights array once its targets are gone: the two must
// agree in length. The weights ride along in the patch and are restored with
// the deltas they describe.
if (plan.mesh.listPrimitives().every((p) => p.listTargets().length === 0)) {
plan.mesh.setWeights([]);
}
plan.prim.getIndices().setArray(indexArrayFor(keep, plan.chain[level]));
}