Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(avatar-studio): replay saved sculpt strokes onto the live mesh

nirholaseda4ed386f3c
diff --git a/src/avatar-edit.js b/src/avatar-edit.js
index 33bd2cc8a..1a7eedcd8 100644
--- a/src/avatar-edit.js
+++ b/src/avatar-edit.js
@@ -26,7 +26,7 @@ import { renderSculptPanel } from './avatar-sculpt.js';
import { applyProportionsToRoot, captureProportionRest, normalizeProportions, proportionsEqual } from './avatar-proportions.js';
import { canonicalBoneNodesFromObject } from './animation-retarget.js';
import { sanitizeSculptDoc, sculptEqual } from './avatar-sculpt-doc.js';
-import { applySculptToRoot } from './avatar-sculpt-brush.js';
+import { applySculptToRoot, clearSculpt } from './avatar-sculpt-brush.js';
import { renderWardrobePanel } from './avatar-wardrobe.js';
import { GarmentCloset, renderClosetSection } from './garment-closet.js';
import { renderRigPanel } from './avatar-rig.js';
@@ -250,6 +250,20 @@ function applyProportions() {
walkPreview?.remeasureProportions();
}
+/**
+ * Replay `workingAppearance.sculpt` onto the live mesh.
+ *
+ * Called wherever the whole appearance is restored (boot, draft restore, undo,
+ * revert) rather than folded into applyProportions(), which fires on every
+ * slider frame: rewriting a 15k-vertex delta array 60 times a second would be
+ * wasted work, and it would stamp on a free-sculpt stroke that is mid-drag.
+ */
+function applySculpt() {
+ if (!scene?.root) return;
+ if (workingAppearance.sculpt) applySculptToRoot(scene.root, workingAppearance.sculpt);
+ else clearSculpt(scene.root);
+}
+
async function bootScene() {
// IMPORTANT: load the BASE GLB, not the baked one. The customizer applies
// appearance on the client; loading the already-baked URL would stack
@@ -268,6 +282,7 @@ async function bootScene() {
boneNodes = canonicalBoneNodesFromObject(scene.root);
captureProportionRest(scene.root, { boneMap: boneNodes });
applyProportions();
+ applySculpt();
accessoryManager = new AccessoryManager({
content: scene.root,
@@ -391,6 +406,38 @@ function renderTabs() {
});
}
+/**
+ * The stage handed to the sculpt panel's free-sculpt brush. Returning null
+ * (no scene yet) simply means the Free Sculpt group is not rendered, which is
+ * the correct outcome: a brush with no canvas is a dead control.
+ *
+ * `setPaused` freezes clip playback AND the procedural idle. The brush caches
+ * the skinned world positions of every vertex at pointer-down, so a breathing
+ * chest under a live drag would make every stroke after the first land a few
+ * millimetres off the pixel the user aimed at.
+ */
+let idleChannelsBeforeSculpt = null;
+
+function sculptViewport() {
+ if (!scene?.camera || !scene?.renderer?.domElement) return null;
+ return {
+ camera: scene.camera,
+ domElement: scene.renderer.domElement,
+ controls: scene.controls || null,
+ setPaused: (paused) => {
+ scene.setRigPaused?.(paused);
+ if (!idle) return;
+ if (paused) {
+ idleChannelsBeforeSculpt = idleChannelsBeforeSculpt || idle.getChannels?.() || null;
+ idle.setChannels({ breathing: false, saccade: false, blink: false, weightShift: false });
+ } else if (idleChannelsBeforeSculpt) {
+ idle.setChannels(idleChannelsBeforeSculpt);
+ idleChannelsBeforeSculpt = null;
+ }
+ },
+ };
+}
+
function renderActivePanel() {
const tab = TABS.find((t) => t.id === activeTab);
const panel = $('ae-panel');
@@ -435,6 +482,8 @@ function renderActivePanel() {
// Sculpt tab gets its own UI — morph sliders, not a tile grid. The face-
// capture modal lives inside that module, so the avatar-edit shell doesn't
// need to know anything about MediaPipe loading or webcam plumbing.
+
+
if (tab.sculpt) {
if (!scene?.root) {
panel.innerHTML = `<div class="ae-empty">Waiting for avatar to load…</div>`;
@@ -444,6 +493,7 @@ function renderActivePanel() {
container: panel,
root: scene.root,
working: workingAppearance,
+ viewport: sculptViewport(),
onDirty: () => {
renderChips();
updateDirtyState();
@@ -930,6 +980,7 @@ function bindHeader() {
}
workingAppearance = clone(currentAppearance);
applyProportions();
+ applySculpt();
if (accessoryManager) await accessoryManager.hydrateFromAppearance(workingAppearance);
if (closet) {
closet.clear();
diff --git a/src/avatar-sculpt-brush.js b/src/avatar-sculpt-brush.js
index 96dcd345f..e1dd0ec2e 100644
--- a/src/avatar-sculpt-brush.js
+++ b/src/avatar-sculpt-brush.js
@@ -55,7 +55,19 @@ export {
sculptEqual,
} from './avatar-sculpt-doc.js';
-/** Brush defaults, in metres / metres-per-stroke-step. */
+/** Brush defaults, in metres and metres-per-stroke-step. */
+export const BRUSH_DEFAULTS = Object.freeze({
+ radius: 0.05,
+ strength: 0.006,
+ direction: 1, // +1 pulls the surface out, -1 pushes it in
+ symmetry: true,
+});
+
+/** Slider ranges for the brush controls, in the same units. */
+export const BRUSH_LIMITS = Object.freeze({
+ radius: { min: 0.01, max: 0.3, step: 0.005 },
+ strength: { min: 0.001, max: 0.03, step: 0.001 },
+});
/* ────────────────────────────────────────────────────────────────────────── *
* Morph-target plumbing
diff --git a/src/avatar-sculpt.js b/src/avatar-sculpt.js
index cb0b7e5f5..5ff2db061 100644
--- a/src/avatar-sculpt.js
+++ b/src/avatar-sculpt.js
@@ -502,6 +502,51 @@ function ensureProportionCss() {
padding: 2px 7px;
border-radius: 999px;
}
+
+ /* Free sculpt */
+ .ae-brush-toggle {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ font-size: 12px;
+ color: var(--text-2, var(--text));
+ cursor: pointer;
+ padding: 4px 0;
+ }
+ .ae-brush-toggle input { accent-color: var(--accent); cursor: pointer; }
+ .ae-brush-modes { display: flex; gap: 6px; margin: 2px 0 4px; }
+ .ae-brush-mode {
+ flex: 1 1 0;
+ padding: 7px 8px;
+ font: inherit;
+ font-size: 12px;
+ font-weight: 600;
+ color: var(--text-2, var(--text));
+ background: var(--panel);
+ border: 1px solid var(--border-2);
+ border-radius: 8px;
+ cursor: pointer;
+ transition: background 0.12s, border-color 0.12s, color 0.12s;
+ }
+ .ae-brush-mode:hover { border-color: var(--text-3); }
+ .ae-brush-mode:focus-visible {
+ outline: none;
+ box-shadow: 0 0 0 2px var(--accent-dim, rgba(255, 255, 255, 0.14));
+ }
+ .ae-brush-mode[aria-pressed='true'] {
+ background: var(--accent);
+ border-color: var(--accent);
+ color: var(--accent-ink, #000);
+ }
+ .ae-brush-foot {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 10px;
+ margin-top: 6px;
+ }
+ .ae-brush-status { font-size: 11px; color: var(--text-3); min-width: 0; }
+ .ae-brush-clear:disabled { opacity: 0.45; cursor: not-allowed; }
`;
document.head.appendChild(style);
}
@@ -763,6 +808,186 @@ function wireFilter(container) {
});
}
+
+/* ────────────────────────────────────────────────────────────────────────── *
+ * Free sculpt: the brush group.
+ *
+ * The library sliders and the proportion table are both catalogues. This is the
+ * escape hatch from them, so it sits at the top of the panel rather than buried
+ * under three hundred sliders. It only renders when the host handed us a
+ * viewport, because a brush with no canvas to paint on would be a dead control.
+ * ────────────────────────────────────────────────────────────────────────── */
+
+let _brush = null;
+let _brushParams = { ...BRUSH_DEFAULTS };
+
+/** Drop the live brush: canvas listeners, cursor ring, rig pause. */
+function teardownBrush() {
+ if (!_brush) return;
+ _brush.viewport?.setPaused?.(false);
+ _brush.instance.dispose();
+ _brush = null;
+}
+
+function freeSculptGroupHtml(viewport, working) {
+ if (!viewport?.camera || !viewport?.domElement) return '';
+ const p = _brushParams;
+ const r = BRUSH_LIMITS.radius;
+ const st = BRUSH_LIMITS.strength;
+ const recorded = sculptVertexCount(working.sculpt);
+ return `
+ <details class="ae-sculpt-group ae-brush-group" open>
+ <summary>
+ <span>Free Sculpt</span>
+ <span class="ae-sculpt-edited" data-edited="${recorded}"${recorded ? '' : ' hidden'}>${recorded} points</span>
+ <span class="ae-sculpt-count">brush</span>
+ </summary>
+ <div class="ae-sculpt-rows">
+ <p class="ae-prop-note">
+ Drag on the avatar to push or pull the surface. Dragging the
+ background still orbits the camera. Playback pauses while the
+ brush is on so the mesh holds still under your cursor.
+ </p>
+ <label class="ae-brush-toggle">
+ <input type="checkbox" id="ae-brush-enable">
+ <span>Brush on</span>
+ </label>
+ <div class="ae-brush-modes" role="group" aria-label="Brush direction">
+ <button type="button" class="ae-brush-mode" data-direction="1"
+ aria-pressed="${p.direction > 0}">Pull out</button>
+ <button type="button" class="ae-brush-mode" data-direction="-1"
+ aria-pressed="${p.direction < 0}">Push in</button>
+ </div>
+ <div class="ae-sculpt-row ae-prop-row">
+ <div class="ae-sculpt-label"><span class="ae-sculpt-name">Size</span></div>
+ <input type="range" id="ae-brush-radius"
+ min="${r.min}" max="${r.max}" step="${r.step}" value="${p.radius}"
+ aria-label="Brush size in centimetres">
+ <output class="ae-sculpt-value" id="ae-brush-radius-out">${(p.radius * 100).toFixed(1)} cm</output>
+ </div>
+ <div class="ae-sculpt-row ae-prop-row">
+ <div class="ae-sculpt-label"><span class="ae-sculpt-name">Strength</span></div>
+ <input type="range" id="ae-brush-strength"
+ min="${st.min}" max="${st.max}" step="${st.step}" value="${p.strength}"
+ aria-label="Brush strength in millimetres per stroke">
+ <output class="ae-sculpt-value" id="ae-brush-strength-out">${(p.strength * 1000).toFixed(1)} mm</output>
+ </div>
+ <label class="ae-brush-toggle">
+ <input type="checkbox" id="ae-brush-symmetry" ${p.symmetry ? 'checked' : ''}>
+ <span>Symmetry</span>
+ </label>
+ <div class="ae-brush-foot">
+ <span class="ae-brush-status" id="ae-brush-status" role="status" aria-live="polite">
+ ${recorded ? `${recorded} points sculpted` : 'Nothing sculpted yet'}
+ </span>
+ <button type="button" class="ae-btn ae-brush-clear" id="ae-brush-clear"
+ ${recorded ? '' : 'disabled'}>Clear sculpt</button>
+ </div>
+ </div>
+ </details>
+ `;
+}
+
+function wireFreeSculpt({ container, root, working, viewport, onDirty }) {
+ const enable = container.querySelector('#ae-brush-enable');
+ if (!enable || !viewport) return;
+
+ const status = container.querySelector('#ae-brush-status');
+ const clearBtn = container.querySelector('#ae-brush-clear');
+ const badge = container.querySelector('.ae-brush-group .ae-sculpt-edited');
+ const radius = container.querySelector('#ae-brush-radius');
+ const radiusOut = container.querySelector('#ae-brush-radius-out');
+ const strength = container.querySelector('#ae-brush-strength');
+ const strengthOut = container.querySelector('#ae-brush-strength-out');
+ const symmetry = container.querySelector('#ae-brush-symmetry');
+
+ // Rehydrate whatever the record already carries onto the live mesh. The
+ // panel is the only place that knows both the document and the model, and
+ // doing it here means a saved sculpt reappears on every tab visit rather
+ // than only on the load that happened to run the host's boot path.
+ if (working.sculpt) applySculptToRoot(root, working.sculpt);
+
+ const report = (message) => {
+ const count = sculptVertexCount(working.sculpt);
+ if (status) status.textContent = message ?? (count ? `${count} points sculpted` : 'Nothing sculpted yet');
+ if (clearBtn) clearBtn.disabled = count === 0;
+ if (badge) {
+ badge.dataset.edited = String(count);
+ badge.textContent = `${count} points`;
+ badge.hidden = count === 0;
+ }
+ };
+
+ const commit = () => {
+ working.sculpt = serializeSculpt(root);
+ report();
+ onDirty?.();
+ };
+
+ const ensureBrush = () => {
+ if (_brush) return _brush.instance;
+ const instance = new SculptBrush({
+ root,
+ camera: viewport.camera,
+ domElement: viewport.domElement,
+ controls: viewport.controls || null,
+ onStroke: ({ vertices }) => {
+ if (status) status.textContent = `Sculpting… ${vertices} vertices this step`;
+ },
+ onStrokeEnd: commit,
+ });
+ instance.setParams(_brushParams);
+ _brush = { instance, viewport };
+ return instance;
+ };
+
+ enable.addEventListener('change', () => {
+ if (enable.checked) {
+ const instance = ensureBrush();
+ viewport.setPaused?.(true);
+ instance.enable();
+ report('Brush on. Drag the avatar to sculpt.');
+ } else {
+ teardownBrush();
+ report();
+ }
+ });
+
+ const pushParams = (patch) => {
+ Object.assign(_brushParams, patch);
+ _brush?.instance.setParams(_brushParams);
+ };
+
+ radius?.addEventListener('input', () => {
+ const v = Number(radius.value);
+ pushParams({ radius: v });
+ if (radiusOut) radiusOut.textContent = `${(v * 100).toFixed(1)} cm`;
+ });
+ strength?.addEventListener('input', () => {
+ const v = Number(strength.value);
+ pushParams({ strength: v });
+ if (strengthOut) strengthOut.textContent = `${(v * 1000).toFixed(1)} mm`;
+ });
+ symmetry?.addEventListener('change', () => pushParams({ symmetry: symmetry.checked }));
+
+ for (const btn of container.querySelectorAll('.ae-brush-mode')) {
+ btn.addEventListener('click', () => {
+ const direction = Number(btn.dataset.direction);
+ pushParams({ direction });
+ for (const other of container.querySelectorAll('.ae-brush-mode')) {
+ other.setAttribute('aria-pressed', String(Number(other.dataset.direction) === direction));
+ }
+ });
+ }
+
+ clearBtn?.addEventListener('click', () => {
+ clearSculpt(root);
+ working.sculpt = null;
+ report('Free sculpt cleared.');
+ onDirty?.();
+ });
+}
+
/**
* "3 edited" pill on a collapsed group's summary. With hundreds of sliders the
* panel is mostly closed, and without this the user has no way to tell which
diff --git a/src/avatar-studio.js b/src/avatar-studio.js
index 2e878c4d3..fd3714b10 100644
--- a/src/avatar-studio.js
+++ b/src/avatar-studio.js
@@ -23,6 +23,7 @@ import { TalkScene } from './voice/talk-scene.js';
import { AccessoryManager } from './agent-accessories.js';
import { IdleAnimation } from './idle-animation.js';
import { renderSculptPanel, applyMorphsToRoot } from './avatar-sculpt.js';
+import { applySculptToRoot, clearSculpt } from './avatar-sculpt-brush.js';
import {
applyProportionsToRoot,
captureProportionRest,
@@ -504,6 +505,7 @@ function maybeSuggestDraft() {
applyAllLayers();
applyMorphsToRoot(scene.root, workingAppearance.morphs);
applyProportions();
+ applySculpt();
if (accessoryManager) accessoryManager.hydrateFromAppearance(workingAppearance);
}
pushHistory();
@@ -573,6 +575,7 @@ async function applyHistoryState(state) {
applyAllLayers();
applyMorphsToRoot(scene.root, workingAppearance.morphs);
applyProportions();
+ applySculpt();
}
renderChips();
renderActivePanel();
@@ -686,6 +689,20 @@ function applyProportions() {
walkPanel?.remeasureProportions();
}
+/**
+ * Replay `workingAppearance.sculpt` onto the live mesh.
+ *
+ * Called wherever the whole appearance is restored (boot, draft restore, undo,
+ * revert) rather than folded into applyProportions(), which fires on every
+ * slider frame: rewriting a 15k-vertex delta array 60 times a second would be
+ * wasted work, and it would stamp on a free-sculpt stroke that is mid-drag.
+ */
+function applySculpt() {
+ if (!scene?.root) return;
+ if (workingAppearance.sculpt) applySculptToRoot(scene.root, workingAppearance.sculpt);
+ else clearSculpt(scene.root);
+}
+
// ── Boot scene ────────────────────────────────────────────────────────
async function bootScene(glbUrl, editAvatar) {
@@ -985,6 +1002,38 @@ function renderTabs() {
});
}
+/**
+ * The stage handed to the sculpt panel's free-sculpt brush. Returning null
+ * (no scene yet) simply means the Free Sculpt group is not rendered, which is
+ * the correct outcome: a brush with no canvas is a dead control.
+ *
+ * `setPaused` freezes clip playback AND the procedural idle. The brush caches
+ * the skinned world positions of every vertex at pointer-down, so a breathing
+ * chest under a live drag would make every stroke after the first land a few
+ * millimetres off the pixel the user aimed at.
+ */
+let idleChannelsBeforeSculpt = null;
+
+function sculptViewport() {
+ if (!scene?.camera || !scene?.renderer?.domElement) return null;
+ return {
+ camera: scene.camera,
+ domElement: scene.renderer.domElement,
+ controls: scene.controls || null,
+ setPaused: (paused) => {
+ scene.setRigPaused?.(paused);
+ if (!idle) return;
+ if (paused) {
+ idleChannelsBeforeSculpt = idleChannelsBeforeSculpt || idle.getChannels?.() || null;
+ idle.setChannels({ breathing: false, saccade: false, blink: false, weightShift: false });
+ } else if (idleChannelsBeforeSculpt) {
+ idle.setChannels(idleChannelsBeforeSculpt);
+ idleChannelsBeforeSculpt = null;
+ }
+ },
+ };
+}
+
function renderActivePanel() {
// A re-render replaces the swatch DOM the popover is anchored to; close it
// first so it never points at a detached node.
@@ -1063,6 +1112,8 @@ function renderActivePanel() {
return;
}
+
+
if (tab.sculpt) {
if (!scene?.root) {
panel.innerHTML = `<div class="as-empty">Waiting for avatar to load…</div>`;
@@ -1072,6 +1123,7 @@ function renderActivePanel() {
container: panel,
root: scene.root,
working: workingAppearance,
+ viewport: sculptViewport(),
onDirty: () => {
pushHistory();
renderChips();