Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(avatar-stream): reconstruct a full asset from the layers already received

nirholas51cfd178b0e2
diff --git a/packages/avatar-stream/src/pack.js b/packages/avatar-stream/src/pack.js
index b48d470e5..2834d9643 100644
--- a/packages/avatar-stream/src/pack.js
+++ b/packages/avatar-stream/src/pack.js
@@ -290,7 +290,20 @@ export async function pack(glbBytes, options = {}) {
}
prim.setExtras({ ...(prim.getExtras() || {}), a3sPrim: ordinal });
- primPlans.push({ ordinal, prim, entries, snapshot, chain: remapped, levelVertexCounts, vertexCount });
+ const morphTargetNames = mesh.getExtras()?.targetNames || null;
+ primPlans.push({
+ ordinal,
+ prim,
+ mesh,
+ entries,
+ snapshot,
+ chain: remapped,
+ levelVertexCounts,
+ vertexCount,
+ morphTargetCount: prim.listTargets().length,
+ morphTargetNames: Array.isArray(morphTargetNames) ? morphTargetNames : null,
+ morphWeights: mesh.getWeights() || null,
+ });
}
}
@@ -373,8 +386,12 @@ export async function pack(glbBytes, options = {}) {
const to = plan.levelVertexCounts[level];
const attributes = {};
for (const [key, meta] of Object.entries(plan.snapshot)) {
+ // Each attribute tracks its own high-water mark, because morph deltas
+ // start from zero here while base attributes resume from the base layer.
+ const start = plan.delivered[key] ?? 0;
+ if (to <= start) continue;
const stride = meta.elementSize;
- const slice = meta.array.subarray(from * stride, to * stride);
+ const slice = meta.array.subarray(start * stride, to * stride);
const bytes = new Uint8Array(slice.buffer, slice.byteOffset, slice.byteLength);
attributes[key] = {
offset: cursor,
@@ -383,9 +400,12 @@ export async function pack(glbBytes, options = {}) {
type: meta.type,
normalized: meta.normalized,
elementSize: stride,
+ start,
+ count: to - start,
};
chunks.push(bytes);
cursor = align4(cursor + bytes.byteLength);
+ plan.delivered[key] = to;
}
const indices = indexArrayFor(to, plan.chain[level]);
const indexBytes = new Uint8Array(indices.buffer, indices.byteOffset, indices.byteLength);
@@ -394,6 +414,9 @@ export async function pack(glbBytes, options = {}) {
newVertexStart: from,
newVertexCount: to - from,
vertexCount: to,
+ morphTargetCount: plan.morphTargetCount,
+ morphTargetNames: plan.morphTargetNames,
+ morphWeights: plan.morphWeights,
attributes,
indices: {
offset: cursor,
@@ -534,12 +557,36 @@ function describeTextureSlots(root, texture) {
return slots;
}
-/** Slice one primitive's accessors, morph targets included, down to a level. */
+/** A morph target delta, as opposed to a base vertex attribute. */
+const isMorphKey = (key) => key.startsWith('targets/');
+
+/**
+ * Slice one primitive down to a level for the base layer.
+ *
+ * Morph target deltas are dropped entirely rather than sliced. A blendshape is
+ * invisible until something drives it, so on a face-rigged avatar its deltas are
+ * pure weight in the first frame: they were up to 72% of the base layer before
+ * this. They arrive with the first geometry patch instead, and `delivered`
+ * records that nothing has been sent for them yet.
+ */
function sliceToLevel(plan, level) {
const keep = plan.levelVertexCounts[level];
+ plan.delivered = {};
for (const { key, accessor } of plan.entries) {
const meta = plan.snapshot[key];
+ if (isMorphKey(key)) {
+ plan.delivered[key] = 0;
+ continue;
+ }
accessor.setArray(meta.array.slice(0, keep * meta.elementSize));
+ plan.delivered[key] = keep;
+ }
+ for (const target of plan.prim.listTargets()) plan.prim.removeTarget(target);
+ // A mesh may not keep a weights array once its targets are gone: the two must
+ // agree in length. The weights ride along in the patch and are restored with
+ // the deltas they describe.
+ if (plan.mesh.listPrimitives().every((p) => p.listTargets().length === 0)) {
+ plan.mesh.setWeights([]);
}
plan.prim.getIndices().setArray(indexArrayFor(keep, plan.chain[level]));
}
diff --git a/packages/avatar-stream/src/reconstruct.js b/packages/avatar-stream/src/reconstruct.js
index 2d49b0d4a..e6fa951b7 100644
--- a/packages/avatar-stream/src/reconstruct.js
+++ b/packages/avatar-stream/src/reconstruct.js
@@ -82,13 +82,17 @@ export async function reconstruct(target, options = {}) {
const primitive = primitives.get(entry.prim);
if (!primitive) continue;
for (const [key, meta] of Object.entries(entry.attributes)) {
+ // An attribute carries its own window, since morph deltas begin at zero
+ // in the patch that introduces them while base attributes resume.
+ const start = meta.start ?? entry.newVertexStart;
+ const count = meta.count ?? entry.newVertexCount;
const incoming = typedArrayFrom(A3SStream.chunk(payload, meta), meta.componentType);
const stride = meta.elementSize;
const TypedArray = COMPONENT_ARRAYS[meta.componentType];
const existing = primitive.attributes[key]?.array;
- const merged = new TypedArray((entry.newVertexStart + entry.newVertexCount) * stride);
- if (existing) merged.set(existing.subarray(0, Math.min(existing.length, entry.newVertexStart * stride)), 0);
- merged.set(incoming.subarray(0, entry.newVertexCount * stride), entry.newVertexStart * stride);
+ const merged = new TypedArray((start + count) * stride);
+ if (existing) merged.set(existing.subarray(0, Math.min(existing.length, start * stride)), 0);
+ merged.set(incoming.subarray(0, count * stride), start * stride);
primitive.attributes[key] = { array: merged, elementSize: stride };
}
primitive.indices = new Uint32Array(typedArrayFrom(A3SStream.chunk(payload, entry.indices), entry.indices.componentType));
diff --git a/packages/avatar-stream/src/three.js b/packages/avatar-stream/src/three.js
index 9be32b9e6..ea4e946c3 100644
--- a/packages/avatar-stream/src/three.js
+++ b/packages/avatar-stream/src/three.js
@@ -111,22 +111,53 @@ function growMorphAttribute(THREE, geometry, targetIndex, semantic, meta, bytes,
/** Apply one primitive's patch to its geometry. */
export function applyPrimitivePatch(THREE, mesh, entry, payload) {
const geometry = mesh.geometry;
+ let morphChanged = false;
for (const [key, meta] of Object.entries(entry.attributes)) {
const bytes = A3SStream.chunk(payload, meta);
+ // Each attribute carries its own window: morph deltas begin at zero in the
+ // patch that introduces them, base attributes resume where layer 0 stopped.
+ const start = meta.start ?? entry.newVertexStart;
+ const count = meta.count ?? entry.newVertexCount;
const morph = key.match(/^targets\/(\d+)\/(.+)$/);
if (morph) {
- growMorphAttribute(THREE, geometry, Number(morph[1]), morph[2], meta, bytes, entry.newVertexStart, entry.newVertexCount);
+ growMorphAttribute(THREE, geometry, Number(morph[1]), morph[2], meta, bytes, start, count);
+ morphChanged = true;
continue;
}
const name = SEMANTIC_TO_ATTRIBUTE[key] || key.toLowerCase();
- growAttribute(THREE, geometry, name, meta, bytes, entry.newVertexStart, entry.newVertexCount);
+ growAttribute(THREE, geometry, name, meta, bytes, start, count);
}
+ if (morphChanged) restoreMorphTargets(mesh, entry);
const indexBytes = A3SStream.chunk(payload, entry.indices);
geometry.setIndex(new THREE.BufferAttribute(typedArrayFrom(indexBytes, entry.indices.componentType), 1));
geometry.computeBoundingSphere();
geometry.computeBoundingBox();
}
+/**
+ * Re-attach the morph plumbing three.js builds at parse time.
+ *
+ * The base layer ships no targets, so the loader created neither an influence
+ * array nor a name dictionary. Both are restored here, and the material is
+ * flagged so its shader is recompiled with morph support switched on.
+ */
+function restoreMorphTargets(mesh, entry) {
+ const count = entry.morphTargetCount || (mesh.geometry.morphAttributes.position || []).length;
+ if (!count) return;
+ if (!mesh.morphTargetInfluences || mesh.morphTargetInfluences.length !== count) {
+ const weights = entry.morphWeights;
+ mesh.morphTargetInfluences = Array.from({ length: count }, (_, i) => weights?.[i] ?? 0);
+ }
+ if (entry.morphTargetNames && !mesh.morphTargetDictionary) {
+ mesh.morphTargetDictionary = {};
+ entry.morphTargetNames.forEach((name, i) => {
+ mesh.morphTargetDictionary[name] = i;
+ });
+ }
+ const materials = Array.isArray(mesh.material) ? mesh.material : mesh.material ? [mesh.material] : [];
+ for (const material of materials) material.needsUpdate = true;
+}
+
/** Decode a texture patch and bind it to every material slot that referenced it. */
export async function applyTexturePatch(THREE, materialsByName, entry, payload) {
const bytes = A3SStream.chunk(payload, entry);