Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

7.3 KB
Raw Blame
// Built-in scene-tools — available to every agent without any skill installed.
// See specs/AGENT_MANIFEST.md § Built-in tools.

import { log } from '../shared/log.js';
import { SLOTS } from './animation-slots.js';

// Gestures are the subset of the slot vocabulary (animation-slots.js) an agent
// can request by name as a deliberate body-language choice. 'idle' is the rest
// state, not a performable gesture, so it is excluded from the tool's enum.
const GESTURE_NAMES = SLOTS.filter((s) => s !== 'idle');

export const BUILTIN_TOOLS = [
	{
		name: 'wave',
		description: 'Wave at the user. Use for greetings, farewells, or acknowledgments.',
		input_schema: {
			type: 'object',
			properties: {
				duration_ms: { type: 'integer', minimum: 500, maximum: 5000, default: 1500 },
			},
		},
	},
	{
		name: 'lookAt',
		description: "Direct the agent's gaze toward a target.",
		input_schema: {
			type: 'object',
			properties: {
				target: {
					type: 'string',
					enum: ['user', 'camera', 'center'],
					description: 'Where to look',
				},
			},
			required: ['target'],
		},
	},
	{
		name: 'play_clip',
		description: 'Play a named animation clip from the loaded body.',
		input_schema: {
			type: 'object',
			properties: {
				name: { type: 'string' },
				loop: { type: 'boolean', default: false },
				fade_ms: { type: 'integer', default: 200 },
			},
			required: ['name'],
		},
	},
	{
		name: 'gesture',
		description:
			'Perform a body-language gesture. Use this to react physically while speaking or ' +
			'listening: nod to agree or acknowledge, shake the head to disagree, wave to greet or ' +
			'say goodbye, shrug when unsure, bow to thank or apologize, point to direct attention, ' +
			'celebrate good news, show concern for bad news, think while working something out, ' +
			'inspect something closely, present a result, or signal patience, curiosity, or fidgety ' +
			'restlessness. The gesture plays once and the body returns to its resting pose.',
		input_schema: {
			type: 'object',
			properties: {
				name: { type: 'string', enum: GESTURE_NAMES, description: 'Which gesture to perform.' },
			},
			required: ['name'],
		},
	},
	{
		name: 'setExpression',
		description: 'Set facial expression via morph target preset.',
		input_schema: {
			type: 'object',
			properties: {
				preset: {
					type: 'string',
					enum: ['neutral', 'happy', 'sad', 'surprised', 'confused', 'focused'],
				},
				intensity: { type: 'number', minimum: 0, maximum: 1, default: 1 },
			},
			required: ['preset'],
		},
	},
	{
		name: 'speak',
		description:
			'Say something out loud via TTS. Use this instead of plain text when voice is on.',
		input_schema: {
			type: 'object',
			properties: { text: { type: 'string' } },
			required: ['text'],
		},
	},
	{
		name: 'remember',
		description:
			'Save a durable memory about the user, a feedback rule, a project fact, or a reference.',
		input_schema: {
			type: 'object',
			properties: {
				key: { type: 'string', description: 'short snake_case filename, e.g. "user_role"' },
				name: { type: 'string' },
				description: { type: 'string' },
				type: { type: 'string', enum: ['user', 'feedback', 'project', 'reference'] },
				body: { type: 'string' },
			},
			required: ['key', 'name', 'description', 'type', 'body'],
		},
	},
	{
		name: 'see_screen',
		description:
			'Capture what is currently visible on screen and describe it to inform your response.',
		input_schema: {
			type: 'object',
			properties: {
				mode: {
					type: 'string',
					enum: ['canvas', 'text', 'screen'],
					description:
						'canvas=3D viewport only (default), text=page text content, screen=full screen capture (requires user permission)',
				},
			},
			required: [],
		},
	},
];

// Stage-scoped tools — appended to an agent's tool list only when Runtime is
// attached to an <agent-stage>. See specs/STAGE_SPEC.md.
export const STAGE_TOOLS = [
	{
		name: 'observe_agents',
		description:
			'List the other agents currently sharing this stage, with their names and positions.',
		input_schema: { type: 'object', properties: {} },
	},
	{
		name: 'say_to_agent',
		description:
			'Send a message to another agent on the same stage. The target agent will receive it as a user turn and reply in their own chat chrome.',
		input_schema: {
			type: 'object',
			properties: {
				agentId: {
					type: 'string',
					description: 'Target agent id as returned by observe_agents.',
				},
				text: { type: 'string' },
			},
			required: ['agentId', 'text'],
		},
	},
];

export const BUILTIN_HANDLERS = {
	async wave(args, ctx) {
		const dur = args.duration_ms || 1500;
		await ctx.viewer.playAnimationByHint?.('wave', { duration: dur });
		ctx.memory.note('waved', { duration_ms: dur });
		return { ok: true };
	},

	async lookAt(args, ctx) {
		const target = args.target || 'user';
		ctx.viewer.lookAt?.(target);
		return { ok: true, target };
	},

	async play_clip(args, ctx) {
		const { name, loop = false, fade_ms = 200 } = args;
		// Resolve slot name (e.g. 'celebrate') to actual clip via agent's mapping
		const resolved = ctx.viewer.resolveAnimationSlot?.(name) ?? name;
		const played = ctx.viewer.playClipByName?.(resolved, { loop, fade_ms });
		if (!played) {
			if (resolved !== name)
				log.warn(`[play_clip] slot "${name}" → "${resolved}" not found in library`);
			return { ok: false, error: `No clip named "${resolved}"` };
		}
		return { ok: true, name, resolved };
	},

	async gesture(args, ctx) {
		const { name } = args;
		const resolved = ctx.viewer.resolveAnimationSlot?.(name) ?? name;
		const played = ctx.viewer.playClipByName?.(resolved, { loop: false, fade_ms: 200 });
		if (!played) {
			log.warn(`[gesture] slot "${name}" → "${resolved}" not found in library`);
			return { ok: false, error: `No clip for gesture "${name}"` };
		}
		return { ok: true, name, resolved };
	},

	async setExpression(args, ctx) {
		const { preset, intensity = 1 } = args;
		ctx.viewer.setExpression?.(preset, intensity);
		return { ok: true, preset };
	},

	async speak(args, ctx) {
		await ctx.speak(args.text);
		return { ok: true };
	},

	async remember(args, ctx) {
		const { key, name, description, type, body } = args;
		ctx.memory.write(key, { name, description, type, body });
		return { ok: true, saved: key };
	},

	async see_screen(args, ctx) {
		const { captureContext } = await import('./screen-context.js');
		const mode = args.mode ?? 'canvas';
		try {
			const result = await captureContext(ctx.viewer.renderer, { mode });
			if (result.type === 'image') {
				return { ok: true, imageData: result.data, description: 'Screen captured.' };
			}
			return { ok: true, text: result.data };
		} catch (err) {
			return { ok: false, error: err.message || 'Screen capture failed.' };
		}
	},

	async observe_agents(args, ctx) {
		if (!ctx.stage) return { ok: false, error: 'not on a stage' };
		const others = ctx.stage
			.getAgents()
			.filter((a) => a.agentId !== ctx.agentId)
			.map(({ agentId, name, position }) => ({ agentId, name, position }));
		return { ok: true, agents: others };
	},

	async say_to_agent(args, ctx) {
		if (!ctx.stage) return { ok: false, error: 'not on a stage' };
		const { agentId, text } = args;
		if (!agentId || !text) return { ok: false, error: 'agentId and text required' };
		const from = ctx.agentId || 'anon';
		ctx.stage.broadcast(from, { kind: 'direct', to: agentId, text });
		return ctx.stage.routeMessage(from, agentId, text);
	},
};