Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(avatar): make emoji in agent speech drive facial expression, not just keywords
nirholascc63dba58dfc
diff --git a/src/agent-avatar.js b/src/agent-avatar.jsindex 8715118c0..81bb7f73a 100644--- a/src/agent-avatar.js+++ b/src/agent-avatar.js@@ -50,6 +50,12 @@ const MOOD_GESTURE_THRESHOLD = {calmArousal: 0.25, // below this = "subdued"};+// Active-listening backchannel: how often the avatar nods while the user is+// speaking (a real VAD/STT signal via setUserSpeaking(), never a fabricated+// loading timer). TalkingHead/airi-style companions use the same cadence of+// small acknowledgment gestures to read as "tracking you" rather than idle.+const LISTENING_NOD_INTERVAL_S = 2.8;+// Emotion decay rates (units per second — larger = fades faster)const DECAY = {concern: 0.08, // half-life ~12s — lingers, builds empathy@@ -155,6 +161,41 @@ const VOCAB = {],};+// Emoji carry explicit emotional signal independent of any surrounding+// English words, and are often glued directly to punctuation with no word+// boundary (e.g. "done!🎉"), so they are scored as direct valence/arousal+// deltas rather than through VOCAB's whitespace-token matching.+const EMOJI_SENTIMENT = {+ '🎉': { valence: 0.9, arousal: 0.6 },+ '🥳': { valence: 0.9, arousal: 0.7 },+ '😀': { valence: 0.7, arousal: 0.3 },+ '😃': { valence: 0.7, arousal: 0.4 },+ '😄': { valence: 0.8, arousal: 0.4 },+ '😁': { valence: 0.7, arousal: 0.4 },+ '😊': { valence: 0.6, arousal: 0.2 },+ '🙂': { valence: 0.4, arousal: 0.1 },+ '👍': { valence: 0.6, arousal: 0.2 },+ '❤️': { valence: 0.7, arousal: 0.3 },+ '✅': { valence: 0.6, arousal: 0.2 },+ '🔥': { valence: 0.6, arousal: 0.6 },+ '✨': { valence: 0.5, arousal: 0.3 },+ '🙏': { valence: 0.3, arousal: 0.1 },+ '😢': { valence: -0.6, arousal: 0.2 },+ '😭': { valence: -0.7, arousal: 0.4 },+ '😞': { valence: -0.5, arousal: 0.1 },+ '😕': { valence: -0.4, arousal: 0.1 },+ '😟': { valence: -0.5, arousal: 0.2 },+ '😠': { valence: -0.6, arousal: 0.6 },+ '😡': { valence: -0.8, arousal: 0.8 },+ '😬': { valence: -0.3, arousal: 0.4 },+ '⚠️': { valence: -0.5, arousal: 0.4 },+ '❌': { valence: -0.6, arousal: 0.3 },+ '🚨': { valence: -0.5, arousal: 0.7 },+ '🤔': { valence: 0, arousal: 0.6 },+ '😮': { valence: 0.1, arousal: 0.6 },+ '😲': { valence: 0.1, arousal: 0.7 },+};+export class AgentAvatar {/*** @param {import('./viewer.js').Viewer} viewer@@ -198,6 +239,7 @@ export class AgentAvatar {// Gaze bias statethis._userSpeaking = false;+ this._listeningNodTimer = 0;this._agentThinking = false;this._agentThinkTimer = 0;this._thinkGazeYaw = 0.1; // randomised on each THINK event@@ -299,6 +341,8 @@ export class AgentAvatar {this._sub(ACTION_TYPES.LOAD_END, this._onLoadEnd.bind(this));this._sub(ACTION_TYPES.VALIDATE, this._onValidate.bind(this));this._sub(ACTION_TYPES.INTERRUPTED, this._onInterrupted.bind(this));+ this._sub(ACTION_TYPES.LISTEN_START, this._onListenStart.bind(this));+ this._sub(ACTION_TYPES.LISTEN_END, this._onListenEnd.bind(this));// First-encounter curiosity burstif (this._firstEncounter) {@@ -929,6 +973,27 @@ export class AgentAvatar {this._setMorphTarget('mouthOpen', 0.3);}+ /**+ * The user started talking (real VAD from livekit-voice.js's agent-server+ * speech_started/ActiveSpeakersChanged, or the generic STT loop's mic-open+ * signal relayed through element.js). Switches the gaze bias in+ * _applyMoodLayer/_applyHeadTransform to attentive eye contact and arms the+ * backchannel nod cadence in _tickEmotion.+ */+ _onListenStart(_action) {+ this.setUserSpeaking(true);+ this.fireAnimationEvent('listen');+ this._listeningNodTimer = 0;+ this._injectStimulus('curiosity', 0.15);+ }++ /** The user stopped talking: release the attentive gaze bias and the nod cadence. */+ _onListenEnd(_action) {+ this.setUserSpeaking(false);+ this.fireAnimationEvent('listen-end');+ this._listeningNodTimer = 0;+ }+// ── Empathy Layer — The Novel Part ────────────────────────────────────────/**@@ -980,6 +1045,21 @@ export class AgentAvatar {// a step whose gesture just expired is replaced on the same frame.this._routinePlayer?.update(dt);+ // Stage 2c: Active-listening backchannel. While a real listen-start signal+ // is active (_onListenStart, never a fabricated loading timer) the avatar+ // nods periodically to show it is tracking the user, same as the nod+ // _onLoadEnd already plays on a finished load. A routine or another+ // one-shot in flight always wins; the timer just holds at 0 until clear.+ if (this._userSpeaking && !this._isPlayingOneShot && !this._routinePlayer) {+ this._listeningNodTimer += dt;+ if (this._listeningNodTimer >= LISTENING_NOD_INTERVAL_S) {+ this._listeningNodTimer = 0;+ this._playSlot('nod', 0.6);+ }+ } else if (!this._userSpeaking) {+ this._listeningNodTimer = 0;+ }+// Stage 3: Emotion-threshold gesture triggers (routed through slot map).// A routine outranks them: an owner who composed a performance did not ask// for a stray celebrate to cut into it at the seam between two steps.@@ -1494,6 +1574,15 @@ export class AgentAvatar {const questions = (text.match(/\?/g) || []).length;arousal += Math.min(exclamations * 0.2 + questions * 0.1, 0.5);+ // Emoji sentiment, scored as substrings so punctuation-adjacent emoji+ // (no word boundary) still register.+ for (const [emoji, delta] of Object.entries(EMOJI_SENTIMENT)) {+ if (text.includes(emoji)) {+ valence += delta.valence;+ arousal += delta.arousal;+ }+ }+return {valence: Math.max(-1, Math.min(1, valence)),arousal: Math.max(0, Math.min(1, arousal)),diff --git a/tests/agent-avatar-emoji-expression.test.js b/tests/agent-avatar-emoji-expression.test.jsnew file mode 100644index 000000000..61050ae9b--- /dev/null+++ b/tests/agent-avatar-emoji-expression.test.js@@ -0,0 +1,94 @@+/**+ * AgentAvatar, emoji in agent text maps to expression.+ *+ * `_onSpeak()` runs every outgoing utterance through `_analyzeSentiment()`+ * and injects the resulting valence/arousal into `_emotion` (celebration,+ * concern, curiosity, ...), which Stage 4 of `_tickEmotion()` renders onto+ * the avatar's face. Before this fix, `_analyzeSentiment()` only scored a+ * plain-English keyword VOCAB, so a reply carrying nothing but emoji (e.g.+ * "🎉" or "😢" with no English sentiment words) produced zero emotional+ * signal and the avatar stayed neutral regardless of what the emoji said.+ */++import { describe, it, expect, beforeEach } from 'vitest';++let AgentAvatar;++function makeRoot() {+ return {+ isObject3D: true,+ children: [],+ traverse(fn) {+ fn(this);+ },+ };+}++function makeFakeAnimationManager() {+ return {+ currentName: null,+ isLoaded: () => false,+ getAnimationDefs: () => [],+ play: () => Promise.resolve(true),+ crossfadeTo: () => {},+ };+}++function makeFakeViewer() {+ return {+ content: makeRoot(),+ animationManager: makeFakeAnimationManager(),+ state: {},+ };+}++function makeAvatar() {+ const viewer = makeFakeViewer();+ const protocol = {}; // _onSpeak is called directly, bus unused+ const identity = { id: 'test' };+ return new AgentAvatar(viewer, protocol, identity);+}++beforeEach(async () => {+ if (!AgentAvatar) {+ const mod = await import('../src/agent-avatar.js');+ AgentAvatar = mod.AgentAvatar;+ }+});++describe('AgentAvatar, emoji in agent text maps to expression', () => {+ it('a celebratory emoji with no English sentiment words still raises celebration', () => {+ const avatar = makeAvatar();+ expect(avatar._emotion.celebration).toBe(0);+ avatar._onSpeak({ payload: { text: 'here you go 🎉' } });+ expect(avatar._emotion.celebration).toBeGreaterThan(0.1);+ });++ it('a sad emoji with no English sentiment words raises concern', () => {+ const avatar = makeAvatar();+ expect(avatar._emotion.concern).toBe(0);+ avatar._onSpeak({ payload: { text: 'that did not work out 😢' } });+ expect(avatar._emotion.concern).toBeGreaterThan(0.1);+ });++ it('an emoji glued directly to punctuation with no space still registers', () => {+ // VOCAB's single-token matching splits on whitespace, so an emoji+ // butted up against punctuation (no space) must not be missed.+ const avatar = makeAvatar();+ avatar._onSpeak({ payload: { text: 'done!🎉' } });+ expect(avatar._emotion.celebration).toBeGreaterThan(0.1);+ });++ it('a thinking emoji raises curiosity/arousal, mirroring a question mark', () => {+ const avatar = makeAvatar();+ avatar._onSpeak({ payload: { text: 'let me look into that 🤔' } });+ expect(avatar._emotion.curiosity).toBeGreaterThan(0.1);+ });++ it('plain text with no emoji and no keywords stays neutral (no false positives)', () => {+ const avatar = makeAvatar();+ avatar._onSpeak({ payload: { text: 'the quick brown fox jumps over the lazy dog' } });+ expect(avatar._emotion.celebration).toBe(0);+ expect(avatar._emotion.concern).toBe(0);+ });+});