Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Built-in scene-tools — available to every agent without any skill installed.
// See specs/AGENT_MANIFEST.md § Built-in tools.
import { log } from '../shared/log.js';
import { SLOTS } from './animation-slots.js';
// Gestures are the subset of the slot vocabulary (animation-slots.js) an agent
// can request by name as a deliberate body-language choice. 'idle' is the rest
// state, not a performable gesture, so it is excluded from the tool's enum.
const GESTURE_NAMES = SLOTS.filter((s) => s !== 'idle');
export const BUILTIN_TOOLS = [
{
name: 'wave',
description: 'Wave at the user. Use for greetings, farewells, or acknowledgments.',
input_schema: {
type: 'object',
properties: {
duration_ms: { type: 'integer', minimum: 500, maximum: 5000, default: 1500 },
},
},
},
{
name: 'lookAt',
description: "Direct the agent's gaze toward a target.",
input_schema: {
type: 'object',
properties: {
target: {
type: 'string',
enum: ['user', 'camera', 'center'],
description: 'Where to look',
},
},
required: ['target'],
},
},
{
name: 'play_clip',
description: 'Play a named animation clip from the loaded body.',
input_schema: {
type: 'object',
properties: {
name: { type: 'string' },
loop: { type: 'boolean', default: false },
fade_ms: { type: 'integer', default: 200 },
},
required: ['name'],
},
},
{
name: 'gesture',
description:
'Perform a body-language gesture. Use this to react physically while speaking or ' +
'listening: nod to agree or acknowledge, shake the head to disagree, wave to greet or ' +
'say goodbye, shrug when unsure, bow to thank or apologize, point to direct attention, ' +
'celebrate good news, show concern for bad news, think while working something out, ' +
'inspect something closely, present a result, or signal patience, curiosity, or fidgety ' +
'restlessness. The gesture plays once and the body returns to its resting pose.',
input_schema: {
type: 'object',
properties: {
name: { type: 'string', enum: GESTURE_NAMES, description: 'Which gesture to perform.' },
},
required: ['name'],
},
},
{
name: 'setExpression',
description: 'Set facial expression via morph target preset.',
input_schema: {
type: 'object',
properties: {
preset: {
type: 'string',
enum: ['neutral', 'happy', 'sad', 'surprised', 'confused', 'focused'],
},
intensity: { type: 'number', minimum: 0, maximum: 1, default: 1 },
},
required: ['preset'],
},
},
{
name: 'speak',
description:
'Say something out loud via TTS. Use this instead of plain text when voice is on.',
input_schema: {
type: 'object',
properties: { text: { type: 'string' } },
required: ['text'],
},
},
{
name: 'remember',
description:
'Save a durable memory about the user, a feedback rule, a project fact, or a reference.',
input_schema: {
type: 'object',
properties: {
key: { type: 'string', description: 'short snake_case filename, e.g. "user_role"' },
name: { type: 'string' },
description: { type: 'string' },
type: { type: 'string', enum: ['user', 'feedback', 'project', 'reference'] },
body: { type: 'string' },
},
required: ['key', 'name', 'description', 'type', 'body'],
},
},
{
name: 'see_screen',
description:
'Capture what is currently visible on screen and describe it to inform your response.',
input_schema: {
type: 'object',
properties: {
mode: {
type: 'string',
enum: ['canvas', 'text', 'screen'],
description:
'canvas=3D viewport only (default), text=page text content, screen=full screen capture (requires user permission)',
},
},
required: [],
},
},
];
// Stage-scoped tools — appended to an agent's tool list only when Runtime is
// attached to an <agent-stage>. See specs/STAGE_SPEC.md.
export const STAGE_TOOLS = [
{
name: 'observe_agents',
description:
'List the other agents currently sharing this stage, with their names and positions.',
input_schema: { type: 'object', properties: {} },
},
{
name: 'say_to_agent',
description:
'Send a message to another agent on the same stage. The target agent will receive it as a user turn and reply in their own chat chrome.',
input_schema: {
type: 'object',
properties: {
agentId: {
type: 'string',
description: 'Target agent id as returned by observe_agents.',
},
text: { type: 'string' },
},
required: ['agentId', 'text'],
},
},
];
export const BUILTIN_HANDLERS = {
async wave(args, ctx) {
const dur = args.duration_ms || 1500;
await ctx.viewer.playAnimationByHint?.('wave', { duration: dur });
ctx.memory.note('waved', { duration_ms: dur });
return { ok: true };
},
async lookAt(args, ctx) {
const target = args.target || 'user';
ctx.viewer.lookAt?.(target);
return { ok: true, target };
},
async play_clip(args, ctx) {
const { name, loop = false, fade_ms = 200 } = args;
// Resolve slot name (e.g. 'celebrate') to actual clip via agent's mapping
const resolved = ctx.viewer.resolveAnimationSlot?.(name) ?? name;
const played = ctx.viewer.playClipByName?.(resolved, { loop, fade_ms });
if (!played) {
if (resolved !== name)
log.warn(`[play_clip] slot "${name}" → "${resolved}" not found in library`);
return { ok: false, error: `No clip named "${resolved}"` };
}
return { ok: true, name, resolved };
},
async gesture(args, ctx) {
const { name } = args;
const resolved = ctx.viewer.resolveAnimationSlot?.(name) ?? name;
const played = ctx.viewer.playClipByName?.(resolved, { loop: false, fade_ms: 200 });
if (!played) {
log.warn(`[gesture] slot "${name}" → "${resolved}" not found in library`);
return { ok: false, error: `No clip for gesture "${name}"` };
}
return { ok: true, name, resolved };
},
async setExpression(args, ctx) {
const { preset, intensity = 1 } = args;
ctx.viewer.setExpression?.(preset, intensity);
return { ok: true, preset };
},
async speak(args, ctx) {
await ctx.speak(args.text);
return { ok: true };
},
async remember(args, ctx) {
const { key, name, description, type, body } = args;
ctx.memory.write(key, { name, description, type, body });
return { ok: true, saved: key };
},
async see_screen(args, ctx) {
const { captureContext } = await import('./screen-context.js');
const mode = args.mode ?? 'canvas';
try {
const result = await captureContext(ctx.viewer.renderer, { mode });
if (result.type === 'image') {
return { ok: true, imageData: result.data, description: 'Screen captured.' };
}
return { ok: true, text: result.data };
} catch (err) {
return { ok: false, error: err.message || 'Screen capture failed.' };
}
},
async observe_agents(args, ctx) {
if (!ctx.stage) return { ok: false, error: 'not on a stage' };
const others = ctx.stage
.getAgents()
.filter((a) => a.agentId !== ctx.agentId)
.map(({ agentId, name, position }) => ({ agentId, name, position }));
return { ok: true, agents: others };
},
async say_to_agent(args, ctx) {
if (!ctx.stage) return { ok: false, error: 'not on a stage' };
const { agentId, text } = args;
if (!agentId || !text) return { ok: false, error: 'agentId and text required' };
const from = ctx.agentId || 'anon';
ctx.stage.broadcast(from, { kind: 'direct', to: agentId, text });
return ctx.stage.routeMessage(from, agentId, text);
},
};