Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// /sign-language: the dedicated home for three.ws's signing avatars.
//
// A live avatar that signs on arrival, an input that spells or signs anything
// you type, a webcam demo that transcribes your own fingerspelling, and a plain
// explanation of what works today. This is the front door for the feature that
// otherwise lives inside the Animation Studio panel and the chat toggles.
//
// The 3D + animation stack is the same PoseStage the studio uses; SignSpeaker
// (src/sign-speech.js) drives it exactly like a text-to-speech engine.
import { PoseStage } from './avatar-pose.js';
import { SignSpeaker, scaledTiming } from './sign-speech.js';
import { normalizeWord } from './fingerspelling.js';
import { SIGNS, signGloss, signLookup } from './sign-dictionary.js';
import { initSignApiConsole } from './sign-api-console.js';
import { buildRigPicker, loadSignPrefs, resolveRig, saveSignPrefs } from './sign-avatars.js';
import { log } from './shared/log.js';
// Signing speed. Learners and many Deaf viewers want it slower, and signing is
// content, so it cannot simply be reduced away like decorative motion.
const SPEEDS = [
{ label: '0.5×', rate: 0.5 },
{ label: '0.75×', rate: 0.75 },
{ label: '1×', rate: 1 },
];
// Phrases the hero cycles through so a first-time visitor immediately sees the
// avatar signing real content, not a static pose. Each mixes lexical signs with
// fingerspelling, which is what the feature actually does.
const DEMO_PHRASES = ['HELLO', 'HAPPY TO MEET YOU', 'WELCOME TO THREE WS', 'THANK YOU YALL'];
const $ = (sel, root = document) => root.querySelector(sel);
async function boot() {
const stageHost = $('#sl-stage');
if (!stageHost) return;
// Speed, hand, and rig survive a reload and carry across every sign surface:
// a left-handed signer should not have to re-declare themselves per page.
const prefs = loadSignPrefs();
let avatar = resolveRig(prefs);
let stage = null;
let speaker = null;
let heroTimer = 0;
let heroIndex = 0;
let heroActive = true;
const status = $('#sl-status');
const setStatus = (msg) => {
if (status) status.textContent = msg || '';
};
// The stage is a several-MB GLB over the network, so it has the same states
// as any other fetch. Every control reads `signingState` before it tries to
// sign, which is what keeps a click during the download from being a dead
// one: it queues instead.
const stageWrap = $('#sl-stage-wrap');
const stageFallback = $('#sl-stage-fallback');
const stageMsg = $('#sl-stage-msg');
/** @type {'loading'|'ready'|'mute'|'error'} */
let signingState = 'loading';
const setStageState = (state, message = '') => {
signingState = state;
if (stageWrap) stageWrap.dataset.state = state;
if (stageFallback) stageFallback.hidden = state !== 'error';
if (stageMsg) stageMsg.textContent = message;
};
/** A phrase asked for before the avatar was ready, performed on arrival. */
let pending = null;
// Speed and dominant hand are baked into the compiled clips, so changing
// either rebuilds the speaker (cheap: the vocabulary is compiled lazily and
// cached per setting).
let rate = SPEEDS.some((s) => s.rate === prefs.rate) ? prefs.rate : 1;
let dominant = prefs.dominant === 'Left' ? 'Left' : 'Right';
/** The last thing signed, so a settings change can show itself immediately. */
let lastPhrase = null;
const rebuildSpeaker = () => {
if (!stage?.anim) return;
speaker?.cancel();
speaker = new SignSpeaker({
manager: stage.anim,
dominant,
signs: signLookup({ dominant, rate }),
timing: rate === 1 ? null : scaledTiming(rate),
});
};
const replay = async () => {
if (!speaker || !lastPhrase) return;
try {
const result = await speaker.speak(lastPhrase);
if (!result.superseded) setStatus(describeResult(result));
} catch {
/* a superseded replay is not an error worth surfacing */
}
};
const applySetting = (fn) => {
fn();
saveSignPrefs({ rate, dominant, avatar: avatar.id });
rebuildSpeaker();
replay();
};
const setRate = (value) => applySetting(() => { rate = value; });
const setDominant = (value) => applySetting(() => { dominant = value; });
// Tell the visitor which words were SIGNED and which were spelled: the
// distinction is the whole point of having a dictionary.
const describeResult = ({ signed, spelled }) => {
const parts = [];
if (signed?.length) parts.push(`signed ${signed.map((w) => w.toLowerCase()).join(', ')}`);
if (spelled?.length) parts.push(`spelled ${spelled.map((w) => w.toLowerCase()).join(', ')}`);
return parts.length ? parts.join(' · ') : 'Type anything and watch the avatar sign it.';
};
/** Mount (or remount) the hero on the current rig. Returns whether it signs. */
const mountStage = async () => {
clearTimeout(heroTimer);
speaker?.cancel();
speaker = null;
stage?.dispose();
setStageState('loading');
setStatus(`Loading ${avatar.label}…`);
// Full-body framing: the whole avatar stays in frame, and the orbit
// controls let anyone zoom into the signing space when they want detail.
stage = new PoseStage(stageHost, {
glbUrl: avatar.url,
label: 'A 3D avatar signing in American Sign Language',
});
try {
const { supported } = await stage.mount();
stage.start();
if (!supported) {
setStageState('mute');
setStatus(`${avatar.label} can’t sign: it has no usable skeleton. Pick another avatar, or compile the signing over HTTP below.`);
return false;
}
rebuildSpeaker();
setStageState('ready');
return true;
} catch (err) {
log.warn('[sign-language] stage mount failed', err?.message);
// A failed mount is a network failure like any other, so it gets a
// way back rather than a permanently dead box.
setStageState(
'error',
'The 3D preview did not load. That is usually the network, so a retry fixes it; the sign API below runs on the server and works either way.',
);
setStatus('');
return false;
}
};
// ── Hero auto-signing loop ────────────────────────────────────────────────
// Signing is content, so it is never disabled: but auto-PLAYING on arrival
// is motion the visitor did not ask for. Under prefers-reduced-motion the
// hero waits for an explicit action (typing, a chip, a vocab word) instead
// of looping on its own.
const reducedMotion = window.matchMedia?.('(prefers-reduced-motion: reduce)').matches ?? false;
async function heroTick() {
if (!heroActive || !speaker) return;
const phrase = DEMO_PHRASES[heroIndex % DEMO_PHRASES.length];
heroIndex++;
setStatus(`Signing: “${phrase.toLowerCase()}”`);
try {
const { clip } = await speaker.speak(phrase);
heroTimer = window.setTimeout(heroTick, clip.duration * 1000 + 1400);
} catch {
heroTimer = window.setTimeout(heroTick, 3000);
}
}
const stopHero = () => {
heroActive = false;
clearTimeout(heroTimer);
speaker?.cancel();
};
// ── Spell-anything input ──────────────────────────────────────────────────
const spellInput = $('#sl-spell-input');
const spellBtn = $('#sl-spell-btn');
// Why the avatar cannot sign right now, phrased as something to do about it.
const unavailableReason = () =>
signingState === 'mute'
? `${avatar.label} has no signing skeleton. Pick another avatar above, or compile this over HTTP in the sign API below.`
: 'The 3D preview did not load. Retry it above, or compile this over HTTP in the sign API below.';
/**
* Sign a phrase in whatever state the stage is in: queue it while the GLB
* is still downloading, and say why when it cannot be performed at all.
* @param {string} phrase
* @param {{ label?: string, onSettled?: () => void }} [opts] `label` replaces
* the default "Signing: …" line; `onSettled` runs once the clip is over.
* @returns {Promise<boolean>} whether it was performed now.
*/
const requestSign = async (phrase, opts = {}) => {
lastPhrase = phrase;
syncShare();
if (signingState === 'loading') {
pending = { phrase, opts };
setStatus(`Loading the avatar: it signs “${phrase.trim().toLowerCase()}” as soon as it arrives.`);
return false;
}
if (!speaker) {
setStatus(unavailableReason());
opts.onSettled?.();
return false;
}
setStatus(opts.label || `Signing: “${phrase.trim().toLowerCase()}”`);
try {
const result = await speaker.speak(phrase);
if (!result.superseded) setStatus(describeResult(result));
opts.onSettled?.();
return true;
} catch (e) {
setStatus(e?.message || 'Could not sign that.');
opts.onSettled?.();
return false;
}
};
/** Perform whatever was asked for while the avatar was still loading. */
const flushPending = async () => {
if (!pending) return false;
const { phrase, opts } = pending;
pending = null;
await requestSign(phrase, opts);
return true;
};
const spellIt = async () => {
const raw = spellInput.value.trim();
if (!normalizeWord(raw)) {
setStatus('Type letters or numbers to sign.');
return;
}
stopHero();
await requestSign(raw);
};
spellBtn?.addEventListener('click', spellIt);
spellInput?.addEventListener('keydown', (e) => {
if (e.key === 'Enter') {
e.preventDefault();
spellIt();
}
});
// "/" focuses the input from anywhere on the page, the way search boxes do.
document.addEventListener('keydown', (e) => {
const typing = /^(INPUT|TEXTAREA|SELECT)$/.test(document.activeElement?.tagName || '');
if (e.key === '/' && !typing && !e.metaKey && !e.ctrlKey && spellInput) {
e.preventDefault();
spellInput.focus();
}
});
// ── Share the signed phrase ───────────────────────────────────────────────
// Every phrase is a URL (?say=), so anything the avatar just signed can be
// handed to someone else as a link that signs on arrival.
const shareBtn = $('#sl-share-btn');
const shareUrl = () =>
`${location.origin}/sign-language?say=${encodeURIComponent(lastPhrase.trim())}`;
const syncShare = () => {
if (shareBtn) shareBtn.hidden = !lastPhrase || !normalizeWord(lastPhrase);
};
shareBtn?.addEventListener('click', async () => {
if (!lastPhrase) return;
const url = shareUrl();
try {
if (navigator.share) {
await navigator.share({ title: 'Watch this signed in ASL', url });
setStatus('Shared.');
} else {
await navigator.clipboard.writeText(url);
setStatus('Link copied. Anyone who opens it sees this signed.');
}
} catch (e) {
if (e?.name !== 'AbortError') setStatus('Could not share: copy the URL from the address bar.');
}
});
// Quick-phrase chips.
document.querySelectorAll('[data-sl-phrase]').forEach((chip) => {
chip.addEventListener('click', () => {
if (spellInput) spellInput.value = chip.dataset.slPhrase;
spellIt();
});
});
// ── Signing speed, dominant hand, and hero rig ────────────────────────────
/** A pill group: one button per option, exactly one pressed at a time. */
const buildOptions = (hostSel, options, pressed, onPick) => {
const host = $(hostSel);
if (!host) return;
options.forEach((option) => {
const btn = document.createElement('button');
btn.type = 'button';
btn.className = 'sl-opt';
btn.textContent = option.label;
btn.setAttribute('aria-pressed', String(pressed(option)));
btn.addEventListener('click', () => {
host.querySelectorAll('.sl-opt').forEach((b) => b.setAttribute('aria-pressed', 'false'));
btn.setAttribute('aria-pressed', 'true');
onPick(option);
});
host.appendChild(btn);
});
};
/** Move the pressed state in a pill group to whichever button `match` picks. */
const syncPills = (hostSel, match) => {
$(hostSel)
?.querySelectorAll('.sl-opt')
.forEach((btn) => btn.setAttribute('aria-pressed', String(match(btn))));
};
buildOptions('#sl-speed', SPEEDS, (s) => s.rate === rate, (s) => setRate(s.rate));
buildOptions(
'#sl-hand',
[
{ label: 'Right-handed', side: 'Right' },
{ label: 'Left-handed', side: 'Left' },
],
(o) => o.side === dominant,
(o) => setDominant(o.side),
);
// Switching rigs replaces the whole stage: the clips are rig-independent
// (they retarget on attach), so the speaker just rebuilds against the new
// skeleton and whatever was signing resumes on the new avatar. That is why
// any avatar on three.ws can take the hero's place, not only the two rigs
// that ship with it.
buildRigPicker({
host: '#sl-rig',
optionClass: 'sl-opt',
active: avatar,
onStatus: setStatus,
apply: async (picked) => {
const previous = avatar;
avatar = picked;
saveSignPrefs({ rate, dominant, avatar: avatar.id });
setStatus('Loading avatar…');
if (!(await mountStage())) {
// A rig with no usable skeleton cannot sign, and leaving a mute
// avatar on stage would read as the feature being broken. Put the
// working one back and say which one is signing.
avatar = previous;
saveSignPrefs({ avatar: avatar.id });
const restored = await mountStage();
setStatus(`${picked.label} can’t sign: it has no usable skeleton. ${previous.label} is back on stage.`);
if (restored && heroActive && !reducedMotion) heroTick();
return false;
}
if (picked.id === 'expressive') {
setStatus('This avatar has a face: questions raise the brows, negation furrows them.');
} else if (picked.custom) {
setStatus(`${picked.label} is signing. Speed and signing hand carry over.`);
}
if (heroActive && !reducedMotion) heroTick();
else replay();
return true;
},
});
// ── Vocabulary: every word with a real sign, playable ─────────────────────
const vocabHost = $('#sl-vocab-chips');
if (vocabHost) {
const words = Object.keys(SIGNS).sort();
let active = null;
for (const word of words) {
const chip = document.createElement('button');
chip.type = 'button';
chip.className = 'sl-vocab-chip';
// No role="listitem": overriding a button's role forbids aria-pressed
// (axe aria-allowed-attr, critical). The host is a role="group" with an
// aria-label instead of a faked list.
chip.setAttribute('aria-pressed', 'false');
chip.textContent = word.toLowerCase();
const gloss = signGloss(word);
if (gloss) chip.title = gloss;
chip.addEventListener('click', () => {
stopHero();
active?.setAttribute('aria-pressed', 'false');
active = chip;
chip.setAttribute('aria-pressed', 'true');
requestSign(word, {
label: gloss ? `${word.toLowerCase()}: ${gloss}` : `Signing “${word.toLowerCase()}”`,
onSettled: () => chip.setAttribute('aria-pressed', 'false'),
});
});
vocabHost.appendChild(chip);
}
}
// ── Webcam sign-in demo ───────────────────────────────────────────────────
wireWebcamDemo(setStatus);
// ── The sign API, live on the same page ───────────────────────────────────
// The console calls /api/sign for real and draws the timeline that comes
// back; "play it on the avatar" hands the same utterance to the hero, with
// the console's hand and speed applied, so the request and the performance
// on screen are provably the same thing.
initSignApiConsole({
defaults: { hand: dominant === 'Left' ? 'left' : 'right', speed: rate },
sign: async ({ text, hand, speed }) => {
if (signingState === 'loading') throw new Error('the avatar has not finished loading');
if (!speaker) throw new Error(unavailableReason());
stopHero();
const wanted = hand === 'left' ? 'Left' : 'Right';
if (wanted !== dominant || speed !== rate) {
dominant = wanted;
rate = speed;
saveSignPrefs({ rate, dominant, avatar: avatar.id });
syncPills('#sl-hand', (btn) => btn.textContent.toLowerCase().startsWith(hand));
syncPills('#sl-speed', (btn) => btn.textContent === `${speed}×`);
rebuildSpeaker();
}
// Move to the avatar before it starts, not after it finishes.
stageHost.scrollIntoView({ behavior: reducedMotion ? 'auto' : 'smooth', block: 'center' });
await requestSign(text);
},
});
// ── ?say= deep link ───────────────────────────────────────────────────────
// A shareable signing link, mirroring the studio's ?spell=: /sign-language
// ?say=hello+world signs the phrase on arrival (dictionary + spelling).
const say = new URLSearchParams(location.search).get('say');
if (say && normalizeWord(say)) {
stopHero();
if (spellInput) spellInput.value = say.slice(0, 48);
requestSign(say);
}
// ── Mount the stage last ──────────────────────────────────────────────────
// Everything above is wired before the several-MB GLB starts downloading, so
// a visitor who types during the load gets a queued phrase rather than a
// button with no listener on it.
$('#sl-stage-retry')?.addEventListener('click', () => startStage());
async function startStage() {
if (!(await mountStage())) return;
if (await flushPending()) return;
if (heroActive && !reducedMotion) heroTick();
else if (lastPhrase) replay();
else setStatus('Type anything and watch the avatar sign it.');
}
await startStage();
}
function wireWebcamDemo(setStatus) {
const btn = $('#sl-cam-btn');
const previewWrap = $('#sl-cam-preview');
const resultEl = $('#sl-cam-result');
if (!btn) return;
let input = null;
let active = false;
btn.addEventListener('click', async () => {
if (active) {
btn.disabled = true;
btn.textContent = 'Reading…';
try {
const { text, confidence } = await input.stop();
if (text) {
resultEl.textContent = text;
resultEl.dataset.state = confidence != null && confidence < 0.4 ? 'low' : 'ok';
} else {
resultEl.textContent = 'No fingerspelling read: try again, a little slower.';
resultEl.dataset.state = 'low';
}
} catch (e) {
resultEl.textContent = e?.message || 'Transcription failed.';
resultEl.dataset.state = 'low';
}
active = false;
btn.disabled = false;
btn.textContent = '🎥 Start signing';
btn.classList.remove('is-live');
previewWrap.hidden = true;
previewWrap.innerHTML = '';
return;
}
try {
if (!input) {
const { SignInput } = await import('./sign-input.js');
input = new SignInput({
onState: (s, d) => {
if (s === 'capturing') setStatus(`Reading your signing… ${d?.frames ?? 0} frames`);
else if (s === 'transcribing') setStatus('Transcribing…');
else setStatus('');
},
});
}
await input.start();
previewWrap.innerHTML = '';
const v = input.videoElement;
v.className = 'sl-cam-video';
previewWrap.appendChild(v);
previewWrap.hidden = false;
active = true;
btn.textContent = '⏹ Stop & read';
btn.classList.add('is-live');
resultEl.textContent = '';
resultEl.removeAttribute('data-state');
} catch (e) {
resultEl.textContent = e?.message || 'Camera unavailable: check browser permissions.';
resultEl.dataset.state = 'low';
input?.cancel();
}
});
}
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', boot);
} else {
boot();
}