Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(avatar-studio): add free-sculpt brushing and freeze the scene while a brush paints
nirholasf813626f460d
diff --git a/api/_lib/bake-sculpt.js b/api/_lib/bake-sculpt.jsnew file mode 100644index 000000000..ef60820a5--- /dev/null+++ b/api/_lib/bake-sculpt.js@@ -0,0 +1,159 @@+// Server-side free-sculpt baker.+//+// The runtime counterpart is src/avatar-sculpt-brush.js: it records a brush+// stroke into an extra morph target named `customSculpt` on the live mesh and+// serializes it, sparse and quantised, into `appearance.sculpt`. This module+// writes that same target into a @gltf-transform Document inside+// bakeAppearance(), which is what carries a free-sculpt edit through the+// /avatars/:id/edit path, where the server renders the GLB from the pristine+// base and never sees the browser's scene.+//+// Recorded as a morph target rather than folded into POSITION for two reasons+// that both matter downstream: the edit stays additive with every library+// slider, and it stays reversible, so re-baking the same base with the sculpt+// removed returns the catalogue body exactly.+//+// A document whose vertex counts do not match the base it is baked onto is a+// sculpt from a different topology. Those meshes are skipped and reported, not+// force-fitted: a delta applied to the wrong vertices is a disfigured avatar,+// which is worse than an un-sculpted one.++// The pure document module, not the brush: the server has no three.js scene to+// paint into and should not drag the renderer into its import graph.+import {+ decodeSculptMesh,+ SCULPT_TARGET_NAME,+ SCULPT_VERSION,+} from '../../src/avatar-sculpt-doc.js';++/** True when this appearance carries a free-sculpt document worth baking. */+export function hasSculpt(appearance) {+ const doc = appearance?.sculpt;+ return !!(doc && doc.version === SCULPT_VERSION && doc.meshes && Object.keys(doc.meshes).length);+}++/**+ * Index of `name` in a mesh's target-name list, or -1.+ * glTF keeps morph names in `mesh.extras.targetNames` by convention (three.js,+ * Blender and gltf-transform all read it); the target's own name is a+ * gltf-transform nicety that does not survive every writer, so the extras list+ * is the authority.+ */+function targetNames(mesh) {+ const extras = mesh.getExtras() || {};+ const names = Array.isArray(extras.targetNames) ? [...extras.targetNames] : [];+ const declared = mesh.listPrimitives()[0]?.listTargets().length || 0;+ while (names.length < declared) names.push(`morph_${names.length}`);+ return names;+}++/**+ * Apply a serialized free-sculpt document to `doc` in place.+ *+ * @param {import('@gltf-transform/core').Document} doc+ * @param {{version:number, meshes:Record<string, object>}} sculpt+ * @returns {{ applied: string[], skipped: Array<{mesh:string, reason:string}> }}+ */+export function applySculpt(doc, sculpt) {+ const applied = [];+ const skipped = [];+ if (!sculpt || sculpt.version !== SCULPT_VERSION || !sculpt.meshes) return { applied, skipped };++ const buffer = doc.getRoot().listBuffers()[0] || doc.createBuffer('sculpt');+ const meshesByName = new Map();+ for (const mesh of doc.getRoot().listMeshes()) {+ if (!meshesByName.has(mesh.getName())) meshesByName.set(mesh.getName(), mesh);+ }++ for (const [name, entry] of Object.entries(sculpt.meshes)) {+ const mesh = meshesByName.get(name);+ if (!mesh) {+ skipped.push({ mesh: name, reason: 'no mesh with that name' });+ continue;+ }+ const decoded = decodeSculptMesh(entry);+ if (!decoded) {+ skipped.push({ mesh: name, reason: 'unreadable delta block' });+ continue;+ }++ const prims = mesh.listPrimitives();+ const target = prims.find(+ (p) => p.getAttribute('POSITION')?.getCount() === decoded.vertexCount,+ );+ if (!target) {+ skipped.push({+ mesh: name,+ reason: `vertex count ${decoded.vertexCount} matches no primitive`,+ });+ continue;+ }++ const names = targetNames(mesh);+ const existing = names.indexOf(SCULPT_TARGET_NAME);++ // Every primitive of a glTF mesh must declare the same number of morph+ // targets, so a mesh with several primitives gets a zero target on the+ // ones the sculpt does not touch.+ for (const prim of prims) {+ const count = prim.getAttribute('POSITION').getCount();+ const array = new Float32Array(count * 3);+ if (prim === target) {+ for (let k = 0; k < decoded.indices.length; k++) {+ const i = decoded.indices[k];+ if (i >= count) continue;+ array[i * 3] = decoded.deltas[k * 3];+ array[i * 3 + 1] = decoded.deltas[k * 3 + 1];+ array[i * 3 + 2] = decoded.deltas[k * 3 + 2];+ }+ }+ const accessor = doc+ .createAccessor(`${name}-${SCULPT_TARGET_NAME}`)+ .setType('VEC3')+ .setArray(array)+ .setBuffer(buffer)+ .setSparse(true);++ if (existing >= 0) {+ const prior = prim.listTargets()[existing];+ if (prior) {+ prior.setAttribute('POSITION', accessor);+ continue;+ }+ }+ const morphTarget = doc.createPrimitiveTarget(SCULPT_TARGET_NAME);+ morphTarget.setAttribute('POSITION', accessor);+ prim.addTarget(morphTarget);+ }++ const index = existing >= 0 ? existing : names.length;+ if (existing < 0) names.push(SCULPT_TARGET_NAME);+ mesh.setExtras({ ...(mesh.getExtras() || {}), targetNames: names });++ // Weight 1: the sculpt is not a slider, it IS the shape. Written on the+ // mesh so a viewer with no node override still sees it, and on every+ // node that already carries its own weight list so the mesh default+ // cannot be shadowed by a stale array.+ const meshWeights = padWeights(mesh.getWeights(), names.length);+ meshWeights[index] = 1;+ mesh.setWeights(meshWeights);+ for (const node of doc.getRoot().listNodes()) {+ if (node.getMesh() !== mesh) continue;+ const nodeWeights = node.getWeights();+ if (!nodeWeights?.length) continue;+ const padded = padWeights(nodeWeights, names.length);+ padded[index] = 1;+ node.setWeights(padded);+ }++ applied.push(name);+ }+ return { applied, skipped };+}++function padWeights(weights, length) {+ const out = new Array(length).fill(0);+ const src = weights || [];+ for (let i = 0; i < Math.min(src.length, length); i++) out[i] = src[i] ?? 0;+ return out;+}diff --git a/api/_lib/bake.js b/api/_lib/bake.jsindex af5a49461..30964fa4e 100644--- a/api/_lib/bake.js+++ b/api/_lib/bake.js@@ -66,6 +66,8 @@ export function isBakeable(appearance) {// nothing else, and treating that as "no customisation" is what left the// baked GLB at default height while the editor showed the edit.if (appearance.proportions && Object.keys(appearance.proportions).length > 0) return true;+ // Free-sculpt deltas (src/avatar-sculpt-brush.js, baked by bake-sculpt.js).+ if (appearance.sculpt?.meshes && Object.keys(appearance.sculpt.meshes).length > 0) return true;return false;}@@ -73,7 +75,7 @@ export function isBakeable(appearance) {* Bake `appearance` into a copy of the base GLB. Returns Uint8Array of the GLB.** @param {Uint8Array|Buffer} baseGlbBytes- * @param {object} appearance — { outfit?, accessories?, morphs?, proportions?, garments? }+ * @param {object} appearance — { outfit?, accessories?, morphs?, proportions?, sculpt?, garments? }* @returns {Promise<Uint8Array>}*/export async function bakeAppearance(baseGlbBytes, appearance) {@@ -100,6 +102,27 @@ export async function bakeAppearance(baseGlbBytes, appearance) {if (isValidPresetId(id)) presetIds.push(id);}+ // 0) Free-sculpt deltas, first: they add a morph target, and the weight+ // bookkeeping below pads every node's weight array to the target count.+ // Running this after that padding would leave the new target unweighted+ // on any node that already carried an explicit list.+ if (appearance?.sculpt) {+ try {+ const { applySculpt, hasSculpt } = await import('./bake-sculpt.js');+ if (hasSculpt(appearance)) {+ const { applied, skipped } = applySculpt(doc, appearance.sculpt);+ if (applied.length) console.log(`[bake] free sculpt baked: ${applied.join(', ')}`);+ if (skipped.length) {+ console.warn(+ `[bake] free sculpt skipped: ${skipped.map((s) => `${s.mesh} (${s.reason})`).join(', ')}`,+ );+ }+ }+ } catch (err) {+ console.warn(`[bake] free-sculpt pass failed, baking without it: ${err.message}`);+ }+ }+// 1) Aggregate morph weights: outfit.morphBinding + appearance.morphs.// appearance.morphs wins on key conflict (user override beats preset).const morphMap = {};diff --git a/api/_lib/render-glb.js b/api/_lib/render-glb.jsindex 98f6bba30..6955577f1 100644--- a/api/_lib/render-glb.js+++ b/api/_lib/render-glb.js@@ -13,9 +13,9 @@// so Vercel's NFT doesn't statically trace the chromium tree for every route// that transitively imports this module — that trace caused 45-min build hangs.import { env } from './env.js';-import { DEFAULT_THREE_BASE, resolveThreeCdn, THREE_VERSION, threeImportMap } from './three-cdn.js';import { fetchModel } from './fetch-model.js';import { scriptJson, safeCssColor } from './render-safe.js';+import { DEFAULT_THREE_BASE, resolveThreeCdn, THREE_VERSION, threeImportMap } from './three-cdn.js';// Cap on GLB bytes pulled into the renderer. Anything larger risks OOM /// blowing the render budget; callers may tighten this via `maxBytes`.@@ -32,7 +32,6 @@ const DEFAULT_CHROMIUM_PACK ='https://github.com/Sparticuz/chromium/releases/download/v148.0.0/chromium-v148.0.0-pack.x64.tar';const CHROMIUM_PACK = env.CHROMIUM_PACK_URL || DEFAULT_CHROMIUM_PACK;-// Poster composition. A 26° yaw reads as a natural 3/4 portrait without hiding// the front of the model; 1.22 leaves enough margin that a T-pose's fingertips// stay inside even after the yaw shrinks the projected width.@@ -140,8 +139,9 @@ function releaseRenderSlot() {}// Inline viewer HTML — bundled into the function so the renderer needs no-// extra static assets. three.js + GLTFLoader load from unpkg pinned to the-// installed version. window.__renderDone signals readiness to puppeteer.+// extra static assets. three.js + GLTFLoader load from whichever CDN+// resolveThreeCdn() found alive, pinned to THREE_VERSION.+// window.__renderDone signals readiness to puppeteer.function viewerHtml({ glbBase64, width, height, background, backdrop, threeBase = DEFAULT_THREE_BASE }) {// A gradient backdrop renders as page CSS behind a transparent canvas: the// screenshot composites the two, so the scene itself stays background-free.diff --git a/src/avatar-edit.js b/src/avatar-edit.jsindex 8dff0d794..33bd2cc8a 100644--- a/src/avatar-edit.js+++ b/src/avatar-edit.js@@ -25,6 +25,8 @@ import { IdleAnimation } from './idle-animation.js';import { renderSculptPanel } from './avatar-sculpt.js';import { applyProportionsToRoot, captureProportionRest, normalizeProportions, proportionsEqual } from './avatar-proportions.js';import { canonicalBoneNodesFromObject } from './animation-retarget.js';+import { sanitizeSculptDoc, sculptEqual } from './avatar-sculpt-doc.js';+import { applySculptToRoot } from './avatar-sculpt-brush.js';import { renderWardrobePanel } from './avatar-wardrobe.js';import { GarmentCloset, renderClosetSection } from './garment-closet.js';import { renderRigPanel } from './avatar-rig.js';@@ -983,6 +985,7 @@ function normalizeAppearance(a) {// Skeleton-space build (src/avatar-proportions.js). Normalized on the way// in so an out-of-range or hand-edited record can never drive the rig.proportions: normalizeProportions(a.proportions),+ sculpt: sanitizeSculptDoc(a.sculpt),};}@@ -998,6 +1001,8 @@ function collapseAppearance(a) {if (a.garments?.length) out.garments = a.garments.map((g) => ({ slot: g.slot, id: g.id }));const proportions = normalizeProportions(a.proportions);if (Object.keys(proportions).length) out.proportions = proportions;+ const sculpt = sanitizeSculptDoc(a.sculpt);+ if (sculpt) out.sculpt = sculpt;return Object.keys(out).length ? out : null;}@@ -1028,5 +1033,6 @@ function appearanceEquals(a, b) {if (ga.size !== gb.size) return false;for (const v of ga) if (!gb.has(v)) return false;if (!proportionsEqual(a?.proportions, b?.proportions)) return false;+ if (!sculptEqual(a?.sculpt, b?.sculpt)) return false;return true;}diff --git a/src/avatar-sculpt-brush.js b/src/avatar-sculpt-brush.jsnew file mode 100644index 000000000..96dcd345f--- /dev/null+++ b/src/avatar-sculpt-brush.js@@ -0,0 +1,598 @@+/**+ * Free sculpt: the "change literally anything" guarantee.+ *+ * The morph library and the proportion table cover the shapes MakeHuman+ * authored and the shapes a skeleton can express. Neither can give someone a+ * dent in one temple, a crooked nose bridge, or a brow ridge that exists in no+ * catalogue. This module does: a radius-and-falloff brush that pushes vertices+ * along their own surface normal, recorded as ONE extra morph target on each+ * mesh it touches.+ *+ * Recording it as a morph target rather than editing POSITION is the whole+ * design. It means a free-sculpt edit:+ * - composes additively with every library slider instead of fighting it,+ * - survives GLTFExporter untouched (Avatar Studio saves the live scene),+ * - can be re-applied to the pristine base from a serialized document, which+ * is what api/_lib/bake-sculpt.js does for the /avatars/:id/edit path,+ * - is reversible: clearing the target restores the catalogue body exactly.+ *+ * Space matters and is easy to get wrong. Morph deltas live in BIND space, but+ * the user is pointing at a skinned, morphed, proportion-edited body on screen.+ * Every stroke therefore builds the exact per-vertex bind-to-world map+ * (bindMatrixInverse x blended bone matrices x bindMatrix, then the mesh's+ * world matrix), moves the point in world space, and maps the displacement back+ * through the inverse of that map. That map is sampled once per drag, at+ * pointer-down: sculpting is correct for the pose you sculpted on, so the host+ * settles the rig before enabling the brush rather than painting onto a moving+ * target.+ *+ * Symmetry is a mirror of the BRUSH, not of the vertices: the stroke is applied+ * a second time at the point reflected across the avatar's own X = 0 plane. On+ * an asymmetric body that still does the right thing, and turning it off is+ * what makes asymmetry reachable at all.+ */++import { Matrix4, Raycaster, Vector2, Vector3, Float32BufferAttribute } from 'three';+import {+ SCULPT_TARGET_NAME,+ SCULPT_VERSION,+ SCULPT_MAX_VERTS,+ clampDisplacement,+ bytesToBase64,+ decodeSculptMesh,+} from './avatar-sculpt-doc.js';++// Re-exported so a caller that already holds the brush does not need to know+// the document module exists.+export {+ SCULPT_TARGET_NAME,+ SCULPT_VERSION,+ SCULPT_MAX_VERTS,+ SCULPT_MAX_DISPLACEMENT,+ decodeSculptMesh,+ sanitizeSculptDoc,+ sculptVertexCount,+ sculptEqual,+} from './avatar-sculpt-doc.js';++/** Brush defaults, in metres / metres-per-stroke-step. */++/* ────────────────────────────────────────────────────────────────────────── *+ * Morph-target plumbing+ * ────────────────────────────────────────────────────────────────────────── */++/**+ * The custom morph attribute for `mesh`, created (zeroed, influence 1) if the+ * mesh does not carry one yet.+ *+ * A mesh with no morphs at all also gets `morphTargetsRelative = true`, because+ * a delta target read as absolute positions collapses the mesh to a point.+ *+ * @param {import('three').Mesh} mesh+ * @returns {import('three').BufferAttribute}+ */+export function ensureSculptTarget(mesh) {+ const geo = mesh.geometry;+ if (!geo?.attributes?.position) return null;++ geo.morphAttributes = geo.morphAttributes || {};+ const list = (geo.morphAttributes.position = geo.morphAttributes.position || []);+ mesh.morphTargetDictionary = mesh.morphTargetDictionary || {};+ mesh.morphTargetInfluences = mesh.morphTargetInfluences || [];++ const existing = mesh.morphTargetDictionary[SCULPT_TARGET_NAME];+ if (existing !== undefined && list[existing]) {+ mesh.morphTargetInfluences[existing] = 1;+ return list[existing];+ }++ if (list.length === 0) geo.morphTargetsRelative = true;+ const attr = new Float32BufferAttribute(new Float32Array(geo.attributes.position.count * 3), 3);+ attr.name = SCULPT_TARGET_NAME;+ const index = list.length;+ list.push(attr);+ mesh.morphTargetDictionary[SCULPT_TARGET_NAME] = index;+ mesh.morphTargetInfluences[index] = 1;+ // The target count is part of the shader's cache key; three rebuilds the+ // morph texture on its own but the program has to be told.+ if (mesh.material) {+ for (const m of Array.isArray(mesh.material) ? mesh.material : [mesh.material]) {+ m.needsUpdate = true;+ }+ }+ return attr;+}++/** The custom morph attribute for `mesh`, or null when it has never been sculpted. */+export function getSculptTarget(mesh) {+ const index = mesh?.morphTargetDictionary?.[SCULPT_TARGET_NAME];+ if (index === undefined) return null;+ return mesh.geometry?.morphAttributes?.position?.[index] || null;+}++/** Every sculptable mesh under `root`, in traversal order. */+export function sculptableMeshes(root) {+ const out = [];+ root?.traverse?.((node) => {+ if (node.isMesh && node.geometry?.attributes?.position) out.push(node);+ });+ return out;+}++/** Zero every custom target under `root`. Returns the number of meshes reset. */+export function clearSculpt(root) {+ let n = 0;+ for (const mesh of sculptableMeshes(root)) {+ const attr = getSculptTarget(mesh);+ if (!attr) continue;+ attr.array.fill(0);+ attr.needsUpdate = true;+ n++;+ }+ return n;+}++/** True when no mesh under `root` carries a non-zero free-sculpt delta. */+export function sculptIsEmpty(root) {+ for (const mesh of sculptableMeshes(root)) {+ const attr = getSculptTarget(mesh);+ if (!attr) continue;+ for (let i = 0; i < attr.array.length; i++) if (attr.array[i] !== 0) return false;+ }+ return true;+}++/* ────────────────────────────────────────────────────────────────────────── *+ * Serialization: sparse, quantised, self-describing+ * ────────────────────────────────────────────────────────────────────────── */++/**+ * Serialize every non-zero free-sculpt delta under `root` into the document+ * that `appearance.sculpt` carries (specs/PARAMETRIC_AVATAR.md).+ *+ * Sparse because a stroke touches hundreds of vertices out of ~15k, and+ * quantised to int16 because a 0.12 m range at 16 bits resolves to under+ * 4 micrometres, which is four orders of magnitude finer than anything a+ * viewer can see. Together those turn a megabyte of floats into tens of KB of+ * JSON-safe text.+ *+ * @param {object} root three.js Object3D+ * @returns {{version:number, meshes:Record<string, object>}|null} null when nothing is sculpted+ */+export function serializeSculpt(root) {+ const meshes = {};+ for (const mesh of sculptableMeshes(root)) {+ const attr = getSculptTarget(mesh);+ if (!attr) continue;+ const arr = attr.array;++ const indices = [];+ let peak = 0;+ for (let i = 0, n = arr.length / 3; i < n; i++) {+ const x = arr[i * 3];+ const y = arr[i * 3 + 1];+ const z = arr[i * 3 + 2];+ if (x === 0 && y === 0 && z === 0) continue;+ indices.push(i);+ peak = Math.max(peak, Math.abs(x), Math.abs(y), Math.abs(z));+ if (indices.length >= SCULPT_MAX_VERTS) break;+ }+ if (!indices.length || peak === 0) continue;++ const scale = peak / 32767;+ const idx = new Uint32Array(indices);+ const deltas = new Int16Array(indices.length * 3);+ for (let k = 0; k < indices.length; k++) {+ const i = indices[k];+ deltas[k * 3] = Math.round(arr[i * 3] / scale);+ deltas[k * 3 + 1] = Math.round(arr[i * 3 + 1] / scale);+ deltas[k * 3 + 2] = Math.round(arr[i * 3 + 2] / scale);+ }+ meshes[mesh.name || 'Mesh'] = {+ count: indices.length,+ vertexCount: arr.length / 3,+ scale,+ indices: bytesToBase64(new Uint8Array(idx.buffer)),+ deltas: bytesToBase64(new Uint8Array(deltas.buffer)),+ };+ }+ return Object.keys(meshes).length ? { version: SCULPT_VERSION, meshes } : null;+}++/**+ * Re-apply a serialized document to a freshly loaded model. Meshes named in the+ * document that this model does not have (a different base) are skipped, and a+ * vertex index past the end of a mesh is dropped: a sculpt is tied to a+ * topology, and refusing to load the rest of a valid document over one stale+ * entry would be worse than ignoring it.+ *+ * @returns {{applied: string[], skipped: string[]}}+ */+export function applySculptToRoot(root, doc) {+ const applied = [];+ const skipped = [];+ if (!doc || doc.version !== SCULPT_VERSION || !doc.meshes) return { applied, skipped };++ const byName = new Map(sculptableMeshes(root).map((m) => [m.name, m]));+ for (const [name, entry] of Object.entries(doc.meshes)) {+ const mesh = byName.get(name);+ const decoded = decodeSculptMesh(entry);+ if (!mesh || !decoded) {+ skipped.push(name);+ continue;+ }+ const attr = ensureSculptTarget(mesh);+ if (!attr) {+ skipped.push(name);+ continue;+ }+ attr.array.fill(0);+ const limit = attr.array.length / 3;+ for (let k = 0; k < decoded.indices.length; k++) {+ const i = decoded.indices[k];+ if (i >= limit) continue;+ attr.array[i * 3] = decoded.deltas[k * 3];+ attr.array[i * 3 + 1] = decoded.deltas[k * 3 + 1];+ attr.array[i * 3 + 2] = decoded.deltas[k * 3 + 2];+ }+ attr.needsUpdate = true;+ applied.push(name);+ }+ return { applied, skipped };+}++/* ────────────────────────────────────────────────────────────────────────── *+ * The brush+ * ────────────────────────────────────────────────────────────────────────── */++const _ndc = new Vector2();+const _v = new Vector3();+const _n = new Vector3();+const _delta = new Vector3();+const _a = new Vector3();+const _b = new Vector3();+const _skin = new Matrix4();+const _bone = new Matrix4();+const _inv = new Matrix4();+const _rootInv = new Matrix4();++/** Blender-style smooth falloff: 1 at the centre, 0 at the rim, flat at both. */+function falloff(t) {+ const s = 1 - t * t;+ return s * s * s;+}++/**+ * The bind-space-to-world matrix for one vertex of `mesh`, i.e. everything+ * between the morph-target space the deltas live in and the pixels the user is+ * pointing at. For a plain Mesh that is just the world matrix.+ */+function vertexBindToWorld(mesh, index, target) {+ if (!mesh.isSkinnedMesh || !mesh.skeleton) return target.copy(mesh.matrixWorld);++ const skinIndex = mesh.geometry.attributes.skinIndex;+ const skinWeight = mesh.geometry.attributes.skinWeight;+ if (!skinIndex || !skinWeight) return target.copy(mesh.matrixWorld);++ _skin.set(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);+ let total = 0;+ for (let i = 0; i < 4; i++) {+ const w = skinWeight.getComponent(index, i);+ if (w === 0) continue;+ const b = skinIndex.getComponent(index, i);+ const bone = mesh.skeleton.bones[b];+ const inverse = mesh.skeleton.boneInverses[b];+ if (!bone || !inverse) continue;+ _bone.multiplyMatrices(bone.matrixWorld, inverse);+ for (let e = 0; e < 16; e++) _skin.elements[e] += _bone.elements[e] * w;+ total += w;+ }+ if (total === 0) return target.copy(mesh.matrixWorld);++ // bindMatrixInverse x blendedBones x bindMatrix, then out to world.+ target.copy(mesh.bindMatrixInverse).multiply(_skin).multiply(mesh.bindMatrix);+ return target.premultiply(mesh.matrixWorld);+}++/**+ * A free-sculpt brush bound to one canvas, one camera and one model root.+ *+ * Lifecycle: `new SculptBrush(opts)` then `enable()` / `disable()`. While+ * enabled, a pointer press that hits the model sculpts and suppresses the+ * orbit controls for that drag; a press that misses the model orbits as usual,+ * so the camera never becomes unreachable.+ */+export class SculptBrush {+ /**+ * @param {object} opts+ * @param {object} opts.root model root (three.js Object3D)+ * @param {object} opts.camera+ * @param {HTMLElement} opts.domElement the renderer canvas+ * @param {object} [opts.controls] OrbitControls, suppressed during a stroke+ * @param {(info: {vertices:number}) => void} [opts.onStroke] after each stroke step+ * @param {() => void} [opts.onStrokeEnd] after the pointer lifts+ */+ constructor({ root, camera, domElement, controls = null, onStroke, onStrokeEnd }) {+ this.root = root;+ this.camera = camera;+ this.domElement = domElement;+ this.controls = controls;+ this.onStroke = onStroke;+ this.onStrokeEnd = onStrokeEnd;++ this.params = { ...BRUSH_DEFAULTS };+ this.enabled = false;++ this._raycaster = new Raycaster();+ this._meshes = [];+ this._pointerId = null;+ this._controlsWere = null;+ this._cursor = null;+ this._worldCache = new WeakMap(); // mesh -> Float32Array of world positions+ this._touched = 0;++ this._onDown = this._onDown.bind(this);+ this._onMove = this._onMove.bind(this);+ this._onUp = this._onUp.bind(this);+ this._onLeave = this._onLeave.bind(this);+ }++ setParams(patch) {+ Object.assign(this.params, patch);+ this._syncCursorSize();+ }++ enable() {+ if (this.enabled || !this.domElement) return;+ this.enabled = true;+ this._meshes = sculptableMeshes(this.root);+ this.domElement.addEventListener('pointerdown', this._onDown);+ this.domElement.addEventListener('pointermove', this._onMove);+ this.domElement.addEventListener('pointerleave', this._onLeave);+ window.addEventListener('pointerup', this._onUp);+ this.domElement.style.cursor = 'crosshair';+ this._ensureCursor();+ }++ disable() {+ if (!this.enabled) return;+ this.enabled = false;+ this._endStroke();+ this.domElement.removeEventListener('pointerdown', this._onDown);+ this.domElement.removeEventListener('pointermove', this._onMove);+ this.domElement.removeEventListener('pointerleave', this._onLeave);+ window.removeEventListener('pointerup', this._onUp);+ this.domElement.style.cursor = '';+ this._cursor?.remove();+ this._cursor = null;+ }++ dispose() {+ this.disable();+ this.root = null;+ this._meshes = [];+ }++ /* ── pointer plumbing ──────────────────────────────────────────────── */++ _ndcFrom(event) {+ const rect = this.domElement.getBoundingClientRect();+ _ndc.set(+ ((event.clientX - rect.left) / rect.width) * 2 - 1,+ -((event.clientY - rect.top) / rect.height) * 2 + 1,+ );+ return _ndc;+ }++ _hit(event) {+ if (!this._meshes.length) this._meshes = sculptableMeshes(this.root);+ this._raycaster.setFromCamera(this._ndcFrom(event), this.camera);+ const hits = this._raycaster.intersectObjects(this._meshes, false);+ return hits.length ? hits[0] : null;+ }++ _onDown(event) {+ if (!this.enabled || event.button !== 0) return;+ const hit = this._hit(event);+ if (!hit) return; // background press: let the orbit controls have it+ event.preventDefault();+ this._pointerId = event.pointerId;+ this._touched = 0;+ if (this.controls) {+ this._controlsWere = this.controls.enabled;+ this.controls.enabled = false;+ }+ this._worldCache = new WeakMap();+ this.domElement.setPointerCapture?.(event.pointerId);+ this._stroke(hit);+ }++ _onMove(event) {+ if (!this.enabled) return;+ this._moveCursor(event);+ if (this._pointerId === null || event.pointerId !== this._pointerId) return;+ const hit = this._hit(event);+ if (hit) this._stroke(hit);+ }++ _onUp(event) {+ if (this._pointerId === null || event.pointerId !== this._pointerId) return;+ this._endStroke();+ }++ _onLeave() {+ if (this._cursor) this._cursor.style.opacity = '0';+ }++ _endStroke() {+ if (this._pointerId !== null) {+ this.domElement?.releasePointerCapture?.(this._pointerId);+ this._pointerId = null;+ if (this.controls && this._controlsWere !== null) {+ this.controls.enabled = this._controlsWere;+ this._controlsWere = null;+ }+ if (this._touched > 0) this.onStrokeEnd?.();+ }+ }++ /* ── the stroke itself ─────────────────────────────────────────────── */++ /**+ * Cached skinned+morphed world positions for one mesh. Recomputed at the+ * start of every drag: the whole point of caching is that the rig does not+ * move during a stroke, and a stale cache after an animation frame would+ * sculpt the wrong vertices.+ */+ _worldPositions(mesh) {+ let cached = this._worldCache.get(mesh);+ if (cached) return cached;+ const count = mesh.geometry.attributes.position.count;+ cached = new Float32Array(count * 3);+ for (let i = 0; i < count; i++) {+ mesh.getVertexPosition(i, _v);+ _v.applyMatrix4(mesh.matrixWorld);+ cached[i * 3] = _v.x;+ cached[i * 3 + 1] = _v.y;+ cached[i * 3 + 2] = _v.z;+ }+ this._worldCache.set(mesh, cached);+ return cached;+ }++ _stroke(hit) {+ const mesh = hit.object;+ if (!mesh?.geometry?.attributes?.position) return;++ let touched = this._applyAt(mesh, hit.point, hit.face);+ if (this.params.symmetry) {+ const mirrored = this._mirrorPoint(hit.point);+ if (mirrored) touched += this._applyAt(mesh, mirrored, hit.face, true);+ }+ if (touched) {+ this._touched += touched;+ this.onStroke?.({ vertices: touched });+ }+ }++ /** Reflect a world point across the model's own X = 0 plane. */+ _mirrorPoint(point) {+ if (!this.root) return null;+ this.root.updateMatrixWorld();+ const local = this.root.worldToLocal(_a.copy(point));+ local.x *= -1;+ return this.root.localToWorld(local).clone();+ }++ /**+ * Apply one stroke step centred on `center` (world space). Returns the+ * number of vertices moved.+ */+ _applyAt(mesh, center, face, mirrored = false) {+ const attr = ensureSculptTarget(mesh);+ if (!attr) return 0;++ const world = this._worldPositions(mesh);+ const positions = mesh.geometry.attributes.position;+ const normals = mesh.geometry.attributes.normal;+ const { radius, strength, direction } = this.params;+ const r2 = radius * radius;++ // Stroke direction: the surface normal at the hit, in world space. The+ // mirrored pass reflects it so both sides bulge outward rather than one+ // pushing in while the other pulls out.+ if (face && normals) {+ _n.fromBufferAttribute(normals, face.a);+ _n.transformDirection(mesh.matrixWorld);+ } else {+ _n.set(0, 0, 1);+ }+ if (mirrored && this.root) {+ // Reflect the direction in the model's own frame: into root space,+ // negate X, back out. transformDirection ignores translation and+ // renormalises, which is exactly what a unit normal wants.+ _rootInv.copy(this.root.matrixWorld).invert();+ _n.transformDirection(_rootInv);+ _n.x *= -1;+ _n.transformDirection(this.root.matrixWorld);+ }++ let moved = 0;+ const count = positions.count;+ for (let i = 0; i < count; i++) {+ const dx = world[i * 3] - center.x;+ const dy = world[i * 3 + 1] - center.y;+ const dz = world[i * 3 + 2] - center.z;+ const d2 = dx * dx + dy * dy + dz * dz;+ if (d2 > r2) continue;++ const w = falloff(Math.sqrt(d2) / radius) * strength * direction;+ if (w === 0) continue;++ // World displacement -> bind-space displacement through the exact+ // inverse of this vertex's own skinning map.+ vertexBindToWorld(mesh, i, _inv);+ _inv.invert();+ _a.set(world[i * 3], world[i * 3 + 1], world[i * 3 + 2]);+ _b.copy(_a).addScaledVector(_n, w);+ _a.applyMatrix4(_inv);+ _b.applyMatrix4(_inv);+ _delta.subVectors(_b, _a);++ const o = i * 3;+ attr.array[o] = clampDisplacement(attr.array[o] + _delta.x);+ attr.array[o + 1] = clampDisplacement(attr.array[o + 1] + _delta.y);+ attr.array[o + 2] = clampDisplacement(attr.array[o + 2] + _delta.z);+ moved++;+ }+ if (moved) attr.needsUpdate = true;+ return moved;+ }++ /* ── on-canvas brush ring ──────────────────────────────────────────── */++ _ensureCursor() {+ if (this._cursor || typeof document === 'undefined') return;+ const ring = document.createElement('div');+ ring.className = 'ae-brush-ring';+ ring.setAttribute('aria-hidden', 'true');+ ring.style.cssText = [+ 'position:fixed',+ 'pointer-events:none',+ 'z-index:40',+ 'border:1.5px solid rgba(255,255,255,0.9)',+ 'box-shadow:0 0 0 1px rgba(0,0,0,0.45)',+ 'border-radius:50%',+ 'transform:translate(-50%,-50%)',+ 'opacity:0',+ 'transition:opacity 0.12s',+ ].join(';');+ document.body.appendChild(ring);+ this._cursor = ring;+ }++ _moveCursor(event) {+ if (!this._cursor) return;+ this._cursor.style.left = `${event.clientX}px`;+ this._cursor.style.top = `${event.clientY}px`;+ this._cursor.style.opacity = '1';+ this._syncCursorSize();+ }++ /**+ * Size the ring to the brush's real footprint by projecting a point one+ * radius to the camera's right of the orbit target. A fixed-pixel ring+ * would lie about the brush every time the user zoomed.+ */+ _syncCursorSize() {+ if (!this._cursor || !this.camera || !this.domElement) return;+ const target = this.controls?.target || _a.set(0, 1.4, 0);+ const dist = this.camera.position.distanceTo(target) || 1;+ const fov = ((this.camera.fov || 35) * Math.PI) / 180;+ const worldHeight = 2 * Math.tan(fov / 2) * dist;+ const px = (this.params.radius * 2 * this.domElement.clientHeight) / worldHeight;+ const size = Math.max(8, Math.min(400, px));+ this._cursor.style.width = `${size}px`;+ this._cursor.style.height = `${size}px`;+ }+}diff --git a/src/avatar-sculpt-doc.js b/src/avatar-sculpt-doc.jsnew file mode 100644index 000000000..911c66d35--- /dev/null+++ b/src/avatar-sculpt-doc.js@@ -0,0 +1,138 @@+/**+ * The free-sculpt document: `appearance.sculpt`, on the wire.+ *+ * Dependency-free on purpose. The same encode/decode/validate runs in the+ * browser (src/avatar-sculpt-brush.js, while the user is painting), on the+ * server (api/_lib/bake-sculpt.js, writing the target into the served GLB) and+ * in the pure appearance helpers (src/avatar-studio-utils.js, which must not+ * pull three.js into its unit tests). Field-by-field contract:+ * specs/PARAMETRIC_AVATAR.md.+ *+ * Shape:+ * { version: 1, meshes: { "<three.js mesh name>": {+ * count, number of recorded vertices+ * vertexCount, the mesh's vertex count, so a stale topology is caught+ * scale, metres per quantisation step+ * indices, base64 of Uint32Array[count], little-endian+ * deltas base64 of Int16Array[count * 3], little-endian+ * } } }+ */++/** Name of the morph target every free-sculpt edit is recorded into. */+export const SCULPT_TARGET_NAME = 'customSculpt';++/** Document version. Bump on any wire-format change; readers reject mismatches. */+export const SCULPT_VERSION = 1;++/** Per-mesh ceiling on recorded vertices, so an appearance record stays small. */+export const SCULPT_MAX_VERTS = 20000;++/** Hard cap on how far one vertex can travel, in metres of bind space. */+export const SCULPT_MAX_DISPLACEMENT = 0.12;++export function clampDisplacement(v) {+ if (!Number.isFinite(v)) return 0;+ return Math.max(-SCULPT_MAX_DISPLACEMENT, Math.min(SCULPT_MAX_DISPLACEMENT, v));+}++/* ── base64 for typed arrays ─────────────────────────────────────────────── *+ * Twenty lines rather than a dependency, and chunked because btoa is fed+ * through String.fromCharCode.apply, which blows the argument limit on a+ * megabyte of deltas.+ * ────────────────────────────────────────────────────────────────────────── */++const B64_CHUNK = 0x8000;++export function bytesToBase64(bytes) {+ if (typeof btoa === 'function') {+ let bin = '';+ for (let i = 0; i < bytes.length; i += B64_CHUNK) {+ bin += String.fromCharCode.apply(null, bytes.subarray(i, i + B64_CHUNK));+ }+ return btoa(bin);+ }+ return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64');+}++export function base64ToBytes(b64) {+ if (typeof atob === 'function') {+ const bin = atob(b64);+ const out = new Uint8Array(bin.length);+ for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);+ return out;+ }+ const buf = Buffer.from(b64, 'base64');+ return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);+}++/**+ * Decode one mesh entry into plain typed arrays.+ *+ * @returns {{indices: Uint32Array, deltas: Float32Array, vertexCount: number}|null}+ */+export function decodeSculptMesh(entry) {+ if (!entry || typeof entry !== 'object') return null;+ const scale = Number(entry.scale);+ if (!Number.isFinite(scale) || scale <= 0) return null;+ let idxBytes;+ let deltaBytes;+ try {+ idxBytes = base64ToBytes(String(entry.indices || ''));+ deltaBytes = base64ToBytes(String(entry.deltas || ''));+ } catch {+ return null;+ }+ if (idxBytes.byteLength % 4 !== 0 || deltaBytes.byteLength % 2 !== 0) return null;+ const count = idxBytes.byteLength / 4;+ if (!count || count > SCULPT_MAX_VERTS || deltaBytes.byteLength / 2 !== count * 3) return null;++ // A base64 decoder can hand back a view whose byteOffset is not a multiple+ // of the element size, which the typed-array constructors reject outright.+ // DataView has no such alignment rule, so read through it.+ const indices = new Uint32Array(count);+ const dv = new DataView(idxBytes.buffer, idxBytes.byteOffset, idxBytes.byteLength);+ for (let i = 0; i < count; i++) indices[i] = dv.getUint32(i * 4, true);++ const deltas = new Float32Array(count * 3);+ const dd = new DataView(deltaBytes.buffer, deltaBytes.byteOffset, deltaBytes.byteLength);+ for (let i = 0; i < count * 3; i++) deltas[i] = clampDisplacement(dd.getInt16(i * 2, true) * scale);++ return { indices, deltas, vertexCount: Number(entry.vertexCount) || 0 };+}++/**+ * Validate an incoming document, dropping mesh entries that cannot be decoded.+ * Returns null when nothing usable survives, which is the canonical "no free+ * sculpt" value and lets `collapseAppearance` omit the field entirely.+ */+export function sanitizeSculptDoc(raw) {+ if (!raw || typeof raw !== 'object') return null;+ if (raw.version !== SCULPT_VERSION) return null;+ if (!raw.meshes || typeof raw.meshes !== 'object') return null;++ const meshes = {};+ for (const [name, entry] of Object.entries(raw.meshes)) {+ if (typeof name !== 'string' || !name) continue;+ const decoded = decodeSculptMesh(entry);+ if (!decoded) continue;+ meshes[name] = {+ count: decoded.indices.length,+ vertexCount: decoded.vertexCount,+ scale: Number(entry.scale),+ indices: String(entry.indices),+ deltas: String(entry.deltas),+ };+ }+ return Object.keys(meshes).length ? { version: SCULPT_VERSION, meshes } : null;+}++/** Total recorded vertices across every mesh in a document. */+export function sculptVertexCount(doc) {+ if (!doc?.meshes) return 0;+ return Object.values(doc.meshes).reduce((n, m) => n + (Number(m?.count) || 0), 0);+}++/** True when two sculpt documents mean the same shape. */+export function sculptEqual(a, b) {+ return JSON.stringify(sanitizeSculptDoc(a)) === JSON.stringify(sanitizeSculptDoc(b));+}diff --git a/src/avatar-sculpt.js b/src/avatar-sculpt.jsindex 06e79ecc4..cb0b7e5f5 100644--- a/src/avatar-sculpt.js+++ b/src/avatar-sculpt.js@@ -30,6 +30,16 @@ import {wireProportionsGroup,} from './avatar-proportions.js';import { canonicalBoneNodesFromObject } from './animation-retarget.js';+import {+ SculptBrush,+ BRUSH_DEFAULTS,+ BRUSH_LIMITS,+ SCULPT_TARGET_NAME,+ serializeSculpt,+ applySculptToRoot,+ clearSculpt,+} from './avatar-sculpt-brush.js';+import { sculptVertexCount } from './avatar-sculpt-doc.js';import { log } from './shared/log.js';/* ────────────────────────────────────────────────────────────────────────── *@@ -279,6 +289,11 @@ export function discoverMorphs(root) {if (obj.isMesh && obj.morphTargetDictionary) {for (const k of Object.keys(obj.morphTargetDictionary)) {if (k === 'tongueOut') continue;+ // The free-sculpt delta is a shape, not a slider: its weight is+ // pinned at 1 and a control that could dial it back to 0.4 would+ // mean "half of the edits I made", which is not a thing a user+ // asks for. It is cleared from the Free Sculpt group instead.+ if (k === SCULPT_TARGET_NAME) continue;found.add(k);}}@@ -315,11 +330,17 @@ export function applyMorphsToRoot(root, morphs) {* @param {() => void} [opts.onRigChanged]: called after a proportion edit settles,* so the host can re-measure root motion (see AnimationManager.remeasureRigProportions)*/-export function renderSculptPanel({ container, root, working, onDirty, onRigChanged }) {+export function renderSculptPanel({ container, root, working, onDirty, onRigChanged, viewport = null }) {const all = discoverMorphs(root);const boneMap = canonicalBoneNodesFromObject(root);const proportionIds = availableProportionParams(root, { boneMap });+ // A panel rebuild replaces the DOM the brush's controls live on, so the+ // previous instance has to give up its canvas listeners and its ring before+ // the new one claims them. Without this a few tab switches leave several+ // brushes painting on the same stroke.+ teardownBrush();+if (!all.length && !proportionIds.length) {container.innerHTML = `<div class="ae-empty">@@ -364,6 +385,8 @@ export function renderSculptPanel({ container, root, working, onDirty, onRigChanskeleton. Double-click any slider to reset it.`}</p>+ ${freeSculptGroupHtml(viewport, working)}+${proportionsGroupHtml(proportionIds, working.proportions)}${all.length ? blendWheelHtml() : ''}@@ -374,6 +397,7 @@ export function renderSculptPanel({ container, root, working, onDirty, onRigChanensureProportionCss();wireSliders(container, root, working, onDirty);wireFilter(container);+ wireFreeSculpt({ container, root, working, viewport, onDirty });if (all.length) wireBlendWheel(container, root, working, available, onDirty);wireProportionsGroup({ container, root, working, boneMap, onDirty, onRigChanged, rerender });@@ -385,8 +409,10 @@ export function renderSculptPanel({ container, root, working, onDirty, onRigChancontainer.querySelector('#ae-sculpt-reset')?.addEventListener('click', () => {working.morphs = {};working.proportions = {};+ working.sculpt = null;applyMorphsToRoot(root, clearAll(all));applyProportionsToRoot(root, {}, { boneMap });+ clearSculpt(root);rerender();onDirty?.();onRigChanged?.();diff --git a/src/avatar-studio-utils.js b/src/avatar-studio-utils.jsindex fcc92f6b7..450093851 100644--- a/src/avatar-studio-utils.js+++ b/src/avatar-studio-utils.js@@ -4,6 +4,7 @@*/import { normalizeProportions } from './avatar-proportions.js';+import { sanitizeSculptDoc } from './avatar-sculpt-doc.js';export const DRAFT_KEY = 'avatar-studio-draft';export const DRAFT_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days@@ -37,13 +38,21 @@ export function collapseAppearance(a) {// hash stays stable across a slider that was dragged and put back.const proportions = normalizeProportions(a.proportions);if (Object.keys(proportions).length) out.proportions = proportions;+ // Free-sculpt deltas (src/avatar-sculpt-brush.js). Sanitizing here is what+ // keeps the record canonical: an undecodable or stale-version document+ // collapses to nothing rather than riding along and failing at bake time.+ const sculpt = sanitizeSculptDoc(a.sculpt);+ if (sculpt) out.sculpt = sculpt;return Object.keys(out).length ? out : null;}/** Hydrate a saved appearance record into a mutable working object. */export function hydrateAppearance(raw) {if (!raw || typeof raw !== 'object') {- return { outfit: null, accessories: [], morphs: {}, colors: {}, hidden: [], garments: [], proportions: {} };+ return {+ outfit: null, accessories: [], morphs: {}, colors: {}, hidden: [],+ garments: [], proportions: {}, sculpt: null,+ };}return {outfit: typeof raw.outfit === 'string' && raw.outfit ? raw.outfit : null,@@ -53,6 +62,7 @@ export function hydrateAppearance(raw) {hidden: Array.isArray(raw.hidden) ? [...raw.hidden] : [],garments: copyGarments(raw.garments),proportions: normalizeProportions(raw.proportions),+ sculpt: sanitizeSculptDoc(raw.sculpt),};}@@ -66,6 +76,10 @@ export function cloneAppearance(a) {hidden: [...a.hidden],garments: copyGarments(a.garments),proportions: { ...(a.proportions || {}) },+ // The sculpt document is immutable once serialized: the brush replaces+ // it wholesale rather than mutating in place, so sharing the reference+ // is safe and a structural clone of tens of KB per history step is not.+ sculpt: a.sculpt || null,};}diff --git a/src/idle-animation.js b/src/idle-animation.jsindex 163d13185..bdfe50635 100644--- a/src/idle-animation.js+++ b/src/idle-animation.js@@ -149,6 +149,17 @@ export class IdleAnimation {Object.assign(this._channels, partial);}+ /**+ * The current channel flags, copied. Callers that need to silence the idle+ * for a while (the free-sculpt brush paints onto a frozen rig) snapshot this+ * first so they can put back exactly what they found instead of guessing at+ * a default that may not have been the state.+ * @returns {{breathing:boolean,saccade:boolean,blink:boolean,weightShift:boolean}}+ */+ getChannels() {+ return { ...this._channels };+ }+/*** Pause or resume saccade micro-movements externally (e.g. while user is speaking).* @param {boolean} active — true = pause indefinitely, false = release immediatelydiff --git a/src/voice/talk-scene.js b/src/voice/talk-scene.jsindex 1c3810723..c6a54b425 100644--- a/src/voice/talk-scene.js+++ b/src/voice/talk-scene.js@@ -42,6 +42,7 @@ export class TalkScene {this.gltf = null;this.root = null;this.mixer = null;+ this._rigPaused = false;this._clips = [];this._currentAction = null;this._clock = new Timer();@@ -388,6 +389,16 @@ export class TalkScene {this._lastFrameAt = 0;}+ /**+ * Freeze or resume clip playback. Paused means the mixer and the emote+ * controller stop advancing, so the skinned mesh holds the pose it is in;+ * rendering, orbiting and everything else carry on.+ * @param {boolean} paused+ */+ setRigPaused(paused) {+ this._rigPaused = !!paused;+ }+_start() {if (this._running) return;this._running = true;@@ -404,8 +415,14 @@ export class TalkScene {this._clock.update();const dt = this._clock.getDelta();this.controls?.update();- this.mixer?.update(dt);- this._emotes?.update(dt);+ // A paused rig still renders and still orbits; it just stops being+ // re-posed. The free-sculpt brush needs a mesh that holds still+ // between pointer-down and pointer-up, and freezing the clip stack+ // is cheaper and more honest than trying to sculpt a moving target.+ if (!this._rigPaused) {+ this.mixer?.update(dt);+ this._emotes?.update(dt);+ }for (const fn of this._onTickFns) fn(dt);this.renderer?.render(this.scene, this.camera);};