Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

11.1 KB
Raw Blame
// Sign speech: the signed-language counterpart of text-to-speech.
//
// Give it text and it produces ONE canonical-skeleton AnimationClip document
// that performs the whole utterance: each word is looked up in a sign
// dictionary (lexical clips captured from real signers via the video2motion
// lane), and any word without a dictionary entry is fingerspelled through
// src/fingerspelling.js. Compiling the utterance to a single clip means every
// chat surface integrates the same way a TTS call does: compile → inject →
// play once: no per-word sequencing, no playback-completion plumbing.
//
// The module is deliberately platform-free (no three.js, no DOM, no fetch):
// the dictionary is injected, and the player is any object with the
// AnimationManager shape (injectClip + playOnce). That keeps it extractable as
// a standalone package.

import { DEFAULT_TIMING, buildFingerspellingClip, normalizeWord } from './fingerspelling.js';
import { lookupSign, signLookup } from './sign-dictionary.js';

// Chat replies favour pace over ceremony: quicker holds and transitions than
// the studio default, so a sentence stays watchable.
export const CHAT_TIMING = Object.freeze({
	...DEFAULT_TIMING,
	holdSeconds: 0.34,
	transitionSeconds: 0.16,
	leadSeconds: 0.25,
	tailSeconds: 0.3,
});

const WORD_GAP_SECONDS = 0.18;
const NORMAL_BLEND_MODE = 2500;

/**
 * Timing for a given playback rate: every authored duration divided by it, so
 * 0.5× is a signer taking twice as long over the same signs rather than a clip
 * played back slowly. Pair it with `signLookup({ rate })`, which scales the
 * lexical signs the same way.
 *
 * @param {number} rate  1 is citation speed; below 1 is slower
 * @param {object} [base]  the timing table to scale (chat pacing by default)
 */
export function scaledTiming(rate, base = CHAT_TIMING) {
	if (!(rate > 0) || rate === 1) return { ...base };
	const out = {};
	for (const [key, value] of Object.entries(base)) out[key] = value / rate;
	return out;
}

/**
 * Split text into the word sequence the signer will perform. Letters and
 * digits survive (A-Z fingerspelling plus the ASL number handshapes 0-9);
 * punctuation is dropped.
 */
export function utteranceWords(text) {
	return normalizeWord(text).split(' ').filter(Boolean);
}

// A lexical sign runs about this long in citation form: used only by the
// estimator, which decides how much text is worth signing before anything is
// compiled.
const SIGN_SECONDS = 1.1;

/**
 * Rough signing duration in seconds for `text` under `timing`, before
 * compiling anything: used to cap chat replies to a watchable length. Words
 * with a dictionary sign count as one sign; the rest are counted per letter.
 */
export function estimateDuration(text, timing = CHAT_TIMING, { signed = true } = {}) {
	const words = utteranceWords(text);
	let seconds = timing.leadSeconds + timing.tailSeconds;
	for (const word of words) {
		if (signed && lookupSign(word)) seconds += SIGN_SECONDS;
		else seconds += word.length * (timing.holdSeconds + timing.transitionSeconds);
	}
	return seconds + Math.max(0, words.length - 1) * WORD_GAP_SECONDS;
}

// Append `clip`'s tracks to the merged timeline starting at `offset`. Bones
// the clip does not drive hold their previous value across the segment so a
// lexical sign that only animates the arms never snaps the fingers to rest.
function appendSegment(merged, clip, offset) {
	const seen = new Set();
	for (const track of clip.tracks) {
		seen.add(track.name);
		let lane = merged.get(track.name);
		if (!lane) {
			lane = { type: track.type, times: [], values: [] };
			merged.set(track.name, lane);
		}
		const stride = track.type === 'quaternion' ? 4 : 3;
		// Bridge from wherever this bone was left to the segment's first key.
		if (lane.times.length && track.times.length) {
			const first = track.times[0] + offset;
			const lastT = lane.times[lane.times.length - 1];
			if (first - lastT > 1e-6) {
				lane.times.push(lastT + Math.min(WORD_GAP_SECONDS, first - lastT) * 0.5);
				lane.values.push(...lane.values.slice(-stride));
			}
		}
		for (let i = 0; i < track.times.length; i++) {
			lane.times.push(track.times[i] + offset);
			lane.values.push(...track.values.slice(i * stride, i * stride + stride));
		}
	}
	// Bones already in the timeline that this clip does not drive: hold their
	// last value through the segment end so nothing snaps.
	const end = offset + clip.duration;
	for (const [name, lane] of merged) {
		if (seen.has(name)) continue;
		const stride = lane.type === 'quaternion' ? 4 : 3;
		lane.times.push(end);
		lane.values.push(...lane.values.slice(-stride));
	}
	return end;
}

/**
 * Compile `text` into one signed-utterance AnimationClip document.
 *
 * @param {string} text
 * @param {{
 *   signs?: Map<string, object>|((word: string) => object|null),
 *   timing?: object,
 *   maxSeconds?: number,
 *   name?: string,
 * }} [opts]
 *   `signs` resolves a word to a lexical sign clip document (canonical-bone
 *   tracks, the animation library shape); words it misses are fingerspelled.
 * @returns {{
 *   clip: object,
 *   words: string[],
 *   signed: string[],
 *   spelled: string[],
 *   segments: { word: string, signed: boolean, gloss: string|null, start: number, end: number,
 *               letters: { letter: string, start: number, end: number }[]|null }[],
 *   truncated: boolean,
 * }}
 *   `segments` is the performed timeline: one entry per word in clip order, with
 *   the seconds it occupies and, for a spelled word, where each letter lands.
 * @throws when the text has no spellable content.
 */
export function compileUtterance(text, opts = {}) {
	const timing = { ...CHAT_TIMING, ...(opts.timing ?? {}) };
	const maxSeconds = opts.maxSeconds ?? 45;
	const lookup =
		typeof opts.signs === 'function' ? opts.signs : (w) => (opts.signs ? opts.signs.get(w) : null) ?? null;

	const words = utteranceWords(text);
	if (!words.length) throw new Error('text has no signable characters (A-Z, 0-9)');

	// A signer raises their hands once for a sentence and lowers them once at the
	// end. Only the first word leads in from rest and only the last settles back
	// to it; every word between keeps the hands up in signing space, which is what
	// makes a sentence read as one utterance instead of as a list of words.
	// Each word compiles to its own segment, and a spelled one also reports where
	// each letter falls inside it, so a caller can follow the utterance along
	// without re-deriving the cadence it was built from.
	const build = (word, position) => {
		const sign = lookup(word, position);
		if (sign) return { segment: sign, signed: true, letters: null };
		const marks = [];
		const segment = buildFingerspellingClip(word, {
			...timing,
			name: `fs-${word.toLowerCase()}`,
			lead: position.first,
			settle: position.last,
			dominant: opts.dominant,
			marks,
		});
		return { segment, signed: false, letters: marks };
	};

	const chosen = [];
	let planned = 0;
	let truncated = false;
	for (let i = 0; i < words.length; i++) {
		const position = { first: i === 0, last: false };
		const built = build(words[i], position);
		const gap = chosen.length ? WORD_GAP_SECONDS : 0;
		// Reserve room for the closing settle so the cap still holds once the last
		// word is rebuilt with it.
		if (chosen.length && planned + gap + built.segment.duration + timing.tailSeconds > maxSeconds) {
			truncated = true;
			break;
		}
		planned += gap + built.segment.duration;
		chosen.push({ word: words[i], position, ...built });
	}

	// Whichever word ended up last (the real last, or the one truncation stopped
	// at) is the one that lowers the hands.
	const closing = chosen[chosen.length - 1];
	closing.position = { ...closing.position, last: true };
	Object.assign(closing, build(closing.word, closing.position));

	const merged = new Map();
	let cursor = 0;
	const signed = [];
	const spelled = [];
	const segments = [];
	for (const entry of chosen) {
		if (cursor > 0) cursor += WORD_GAP_SECONDS;
		const start = cursor;
		cursor = appendSegment(merged, entry.segment, cursor);
		(entry.signed ? signed : spelled).push(entry.word);
		segments.push({
			word: entry.word,
			signed: entry.signed,
			gloss: entry.signed ? (lookupSign(entry.word)?.sign.gloss ?? null) : null,
			start,
			end: cursor,
			letters: entry.letters
				? entry.letters
						.filter((m) => m.letter !== ' ')
						.map((m) => ({ letter: m.letter, start: start + m.start, end: start + m.end }))
				: null,
		});
	}

	// Rotation lanes only. A signed utterance is upper-body: it must never write
	// the root's translation, which on a rig whose hips rest a metre off the
	// floor would drop the whole avatar through it.
	const tracks = [];
	for (const [name, lane] of merged) {
		if (name.endsWith('.position') || name.endsWith('.scale')) continue;
		tracks.push({ type: lane.type, name, times: lane.times, values: lane.values });
	}

	return {
		clip: {
			name: opts.name ?? `sign-${words.join('-').toLowerCase()}`.slice(0, 60),
			duration: cursor,
			tracks,
			uuid: undefined,
			blendMode: NORMAL_BLEND_MODE,
		},
		words,
		signed,
		spelled,
		segments,
		truncated,
	};
}

/**
 * SignSpeaker: drive an avatar's AnimationManager like a TTS engine.
 *
 *   const speaker = new SignSpeaker({ manager });
 *   await speaker.speak('hello world');   // resolves when the signing ends
 *
 * `manager` is anything with the AnimationManager shape:
 *   injectClip(name, clipJSON, opts) and playOnce(name, opts).
 * `signs` (optional) is the lexical dictionary passed through to the compiler.
 */
export class SignSpeaker {
	constructor({ manager, signs = undefined, timing = null, maxSeconds = 45, dominant = 'Right' } = {}) {
		if (!manager) throw new Error('SignSpeaker needs a manager');
		this.manager = manager;
		/** Which hand leads. About one signer in ten signs left-dominant. */
		this.dominant = dominant === 'Left' ? 'Left' : 'Right';
		// Default to the built-in vocabulary: a signer signs the words they have
		// signs for and only spells the rest. Pass `signs: null` for spelling-only,
		// or your own lookup to override.
		this.signs = signs === undefined ? signLookup({ dominant: this.dominant }) : signs;
		this.timing = timing;
		this.maxSeconds = maxSeconds;
		this._counter = 0;
		this._activeToken = 0;
		this.speaking = false;
	}

	/**
	 * Sign `text` on the avatar. A newer speak() supersedes an in-flight one.
	 * Resolves after the clip's duration with the compile summary.
	 */
	async speak(text, opts = {}) {
		const token = ++this._activeToken;
		const result = compileUtterance(text, {
			signs: this.signs,
			timing: this.timing ?? undefined,
			maxSeconds: this.maxSeconds,
			dominant: this.dominant,
			...opts,
		});
		const name = `sign-speech-${++this._counter}`;
		this.manager.injectClip(name, result.clip, { loop: false });
		if (token !== this._activeToken) return { ...result, superseded: true };
		this.speaking = true;
		this.manager.playOnce(name, opts.playOpts ?? {});
		await new Promise((r) => setTimeout(r, result.clip.duration * 1000));
		if (token === this._activeToken) this.speaking = false;
		return { ...result, superseded: token !== this._activeToken };
	}

	/** Abandon the current utterance (the next speak() takes over cleanly). */
	cancel() {
		this._activeToken++;
		this.speaking = false;
	}
}