Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Scene Studio: Agent tab.
//
// Sibling to the vendored r184 editor (never edits vendor/**): adds a chat
// panel to the sidebar's tab row (Sidebar.addTab, see vendor/js/Sidebar.js)
// that drives the open scene through tool calls instead of the mouse.
//
// The model only picks tools and arguments: every tool EXECUTES in this
// module, as a real editor.execute(Command) the same File/Add menus use, so
// Edit > Undo removes an agent action exactly like a manual one and autosave
// (main.js) persists it. Generation tools reuse the diorama pipeline
// (src/diorama/compose.js: composeWorld/forgeObject) that already backs the
// Forge flow, and glTF import reuses the shared Draco/KTX2/Meshopt loader
// (loader.js) the ?model= deep link and "Import from Forge" use.
//
// One reasoning round per turn is POSTed to /api/scene/agent (free, public,
// rate-limited, built from the same provider chain + tool-calling building
// blocks as the metered /api/v1/chat/completions client-tool lane, see that
// file's header) and the loop re-posts with the tool results until the model
// answers with no more tool calls or a per-turn round cap is hit.
//
// Every prompt that changes the scene pushes a version: a scene.toJSON()
// snapshot plus a viewport screenshot, both captured synchronously right
// after the mutating execute() calls (toDataURL must run in the same task as
// the render, since the editor's renderer has no preserveDrawingBuffer;
// Viewport.js renders synchronously inside each signal dispatch). The
// version list lets the user step back to any earlier state: a MultiCmdsCommand
// that removes the scene's current children and re-adds the snapshot's (plain
// SetSceneCommand only APPENDS a parsed scene's children, since the vendored
// Loader.js always runs it right after editor.clear(); that clear also wipes
// undo history, which stepping back through versions must never do), so
// restoring a version is itself one more undoable edit.
import * as THREE from 'three';
import { UIElement } from './vendor/js/libs/ui.js';
import { AddObjectCommand } from './vendor/js/commands/AddObjectCommand.js';
import { RemoveObjectCommand } from './vendor/js/commands/RemoveObjectCommand.js';
import { SetPositionCommand } from './vendor/js/commands/SetPositionCommand.js';
import { SetRotationCommand } from './vendor/js/commands/SetRotationCommand.js';
import { SetScaleCommand } from './vendor/js/commands/SetScaleCommand.js';
import { SetMaterialColorCommand } from './vendor/js/commands/SetMaterialColorCommand.js';
import { MultiCmdsCommand } from './vendor/js/commands/MultiCmdsCommand.js';
import { addGltfBufferToScene } from './loader.js';
import { composeWorld, forgeObject } from '../diorama/compose.js';
import { toastError } from '../shared/toast.js';
const STYLE_ID = 'tws-studio-agent-styles';
const ENDPOINT = '/api/scene/agent';
const MAX_ROUNDS = 6; // tool-call round-trips per user turn, bounds a runaway loop on a free/rate-limited endpoint
const MAX_VERSIONS = 30;
function ensureStyles() {
if (typeof document === 'undefined' || document.getElementById(STYLE_ID)) return;
const el = document.createElement('style');
el.id = STYLE_ID;
el.textContent = CSS;
document.head.appendChild(el);
}
// ── Tool catalog ─────────────────────────────────────────────────────────
// OpenAI-wire function tools (api/_lib/client-tool-round.js normalizes this
// exact shape). Kept intentionally small and scene-scoped: the agent reads
// and mutates the OPEN scene, it does not manage named server-side scenes
// (that is packages/scene-mcp's job against /api/diorama, for an MCP client).
const TOOLS = [
{
type: 'function',
function: {
name: 'list_objects',
description: "List every object currently in the scene: uuid, name, type, and position. Call this first to see what's there before moving, scaling, recoloring, or deleting anything.",
parameters: { type: 'object', properties: {} },
},
},
{
type: 'function',
function: {
name: 'add_object',
description: 'Generate one new 3D object from a short description and add it to the scene (e.g. "a red wooden chair"). Takes 10-60 seconds to forge a real mesh.',
parameters: {
type: 'object',
properties: { prompt: { type: 'string', description: 'A short description of the single object to generate.' } },
required: ['prompt'],
},
},
},
{
type: 'function',
function: {
name: 'build_world',
description: 'Generate a whole small scene (ground, several placed objects) from one sentence and add every object to the scene. Use for "build me a..." requests with multiple things; use add_object for a single item.',
parameters: {
type: 'object',
properties: { prompt: { type: 'string', description: 'A sentence describing the world to build.' } },
required: ['prompt'],
},
},
},
{
type: 'function',
function: {
name: 'move_object',
description: 'Move an object to an absolute position (scene units).',
parameters: {
type: 'object',
properties: {
uuid: { type: 'string' },
x: { type: 'number' }, y: { type: 'number' }, z: { type: 'number' },
},
required: ['uuid', 'x', 'y', 'z'],
},
},
},
{
type: 'function',
function: {
name: 'rotate_object',
description: 'Set an object\'s rotation, in degrees, around each axis.',
parameters: {
type: 'object',
properties: {
uuid: { type: 'string' },
x: { type: 'number' }, y: { type: 'number' }, z: { type: 'number' },
},
required: ['uuid', 'x', 'y', 'z'],
},
},
},
{
type: 'function',
function: {
name: 'scale_object',
description: 'Scale an object uniformly or per-axis (1 = original size).',
parameters: {
type: 'object',
properties: {
uuid: { type: 'string' },
x: { type: 'number' }, y: { type: 'number' }, z: { type: 'number' },
},
required: ['uuid', 'x', 'y', 'z'],
},
},
},
{
type: 'function',
function: {
name: 'set_material_color',
description: "Set an object's base material color.",
parameters: {
type: 'object',
properties: { uuid: { type: 'string' }, color: { type: 'string', description: 'CSS color, e.g. "#ff6a3d" or "tomato".' } },
required: ['uuid', 'color'],
},
},
},
{
type: 'function',
function: {
name: 'delete_object',
description: 'Remove an object from the scene.',
parameters: { type: 'object', properties: { uuid: { type: 'string' } }, required: ['uuid'] },
},
},
];
const SYSTEM_PROMPT = [
'You are the Scene Studio agent on three.ws, a 3D scene editor. You help the visitor build and edit the 3D scene they have open by calling tools: every tool call is a real, undoable edit to their live scene.',
"Call list_objects before you move, scale, recolor, or delete anything you haven't just created, so you act on the right uuid.",
'Use add_object for one new item, build_world for a whole small scene from a sentence. Both take real time to generate; say so briefly rather than going quiet.',
'After acting, reply with a short, concrete sentence about what changed. Never claim to have done something you did not call a tool for.',
'If a request is ambiguous (which object, what color), make the most reasonable choice yourself rather than asking a clarifying question. Only ask when truly nothing reasonable can be inferred.',
].join(' ');
function uid() {
return typeof crypto !== 'undefined' && crypto.randomUUID ? crypto.randomUUID() : `v-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
}
/** Synchronous screenshot right after a render, see module header on timing. */
function captureScreenshot() {
try {
const canvas = document.querySelector('#viewport canvas');
if (!canvas) return null;
return canvas.toDataURL('image/jpeg', 0.72);
} catch {
return null; // tainted canvas (cross-origin texture) or no canvas yet, never fatal
}
}
function forceRender(editor) {
editor.signals.sceneGraphChanged.dispatch();
}
export class SceneAgent {
constructor(editor) {
this.editor = editor;
this.messages = [{ role: 'system', content: SYSTEM_PROMPT }];
this.versions = [];
}
objectSummary() {
const out = [];
this.editor.scene.traverse((obj) => {
if (obj === this.editor.scene) return;
out.push({
uuid: obj.uuid,
name: obj.name || obj.type,
type: obj.type,
position: { x: round(obj.position.x), y: round(obj.position.y), z: round(obj.position.z) },
});
});
return out;
}
findObject(uuidOrName) {
const byUuid = this.editor.objectByUuid(uuidOrName);
if (byUuid) return byUuid;
let found = null;
this.editor.scene.traverse((obj) => {
if (!found && obj.name === uuidOrName) found = obj;
});
return found;
}
async addGlbAsObject(glbUrl, label) {
const res = await fetch(glbUrl);
if (!res.ok) throw new Error(`HTTP ${res.status} fetching the generated model`);
const contents = await res.arrayBuffer();
return addGltfBufferToScene(this.editor, contents, label);
}
/** Executes one named tool call; returns the text that goes back to the model as the tool result. */
async runTool(name, rawArgs, { onStatus } = {}) {
let args;
try {
args = JSON.parse(rawArgs || '{}');
} catch {
return `Error: arguments were not valid JSON.`;
}
switch (name) {
case 'list_objects': {
const objects = this.objectSummary();
return objects.length ? JSON.stringify(objects) : 'The scene is empty.';
}
case 'add_object': {
const prompt = String(args.prompt || '').slice(0, 300);
if (!prompt) return 'Error: prompt is required.';
onStatus?.(`Forging "${prompt}"…`);
const object = { id: uid(), prompt, status: 'pending', glbUrl: null };
const result = await forgeObject(object);
if (result.status !== 'ready' || !result.glbUrl) return `Could not generate "${prompt}". Try a simpler description.`;
const added = await this.addGlbAsObject(result.glbUrl, prompt.slice(0, 48));
return `Added "${added.name || prompt}" (uuid ${added.uuid}) at position (${round(added.position.x)}, ${round(added.position.y)}, ${round(added.position.z)}).`;
}
case 'build_world': {
const prompt = String(args.prompt || '').slice(0, 400);
if (!prompt) return 'Error: prompt is required.';
onStatus?.(`Composing a scene for "${prompt}"…`);
let diorama;
try {
diorama = await composeWorld(prompt, {
onObject: (id, patch) => {
if (patch.status === 'forging') onStatus?.(`Forging piece ${id.slice(0, 6)}…`);
},
});
} catch (err) {
return `Could not compose that world: ${err.message}`;
}
const ready = diorama.objects.filter((o) => o.status === 'ready' && o.glbUrl);
if (!ready.length) return 'The world composed but no pieces forged successfully. Try a more concrete sentence.';
const added = [];
for (const obj of ready) {
try {
added.push(await this.addGlbAsObject(obj.glbUrl, obj.label || obj.prompt || 'Object'));
} catch {
// one failed import never sinks the rest of the world
}
}
return `Built the scene and added ${added.length} of ${diorama.objects.length} object(s): ${added.map((o) => o.name).join(', ') || 'none imported cleanly'}.`;
}
case 'move_object': {
const object = this.findObject(args.uuid);
if (!object) return `Error: no object with uuid "${args.uuid}".`;
this.editor.execute(new SetPositionCommand(this.editor, object, new THREE.Vector3(num(args.x), num(args.y), num(args.z))));
return `Moved ${object.name || object.uuid} to (${num(args.x)}, ${num(args.y)}, ${num(args.z)}).`;
}
case 'rotate_object': {
const object = this.findObject(args.uuid);
if (!object) return `Error: no object with uuid "${args.uuid}".`;
const rad = (d) => (num(d) * Math.PI) / 180;
this.editor.execute(new SetRotationCommand(this.editor, object, new THREE.Euler(rad(args.x), rad(args.y), rad(args.z))));
return `Rotated ${object.name || object.uuid} to (${num(args.x)}°, ${num(args.y)}°, ${num(args.z)}°).`;
}
case 'scale_object': {
const object = this.findObject(args.uuid);
if (!object) return `Error: no object with uuid "${args.uuid}".`;
this.editor.execute(new SetScaleCommand(this.editor, object, new THREE.Vector3(num(args.x, 1), num(args.y, 1), num(args.z, 1))));
return `Scaled ${object.name || object.uuid} to (${num(args.x, 1)}, ${num(args.y, 1)}, ${num(args.z, 1)}).`;
}
case 'set_material_color': {
const object = this.findObject(args.uuid);
if (!object) return `Error: no object with uuid "${args.uuid}".`;
const color = new THREE.Color();
try {
color.set(String(args.color || '#ffffff'));
} catch {
return `Error: "${args.color}" is not a color this editor understands.`;
}
this.editor.execute(new SetMaterialColorCommand(this.editor, object, 'color', color.getHex()));
return `Set ${object.name || object.uuid}'s color to ${args.color}.`;
}
case 'delete_object': {
const object = this.findObject(args.uuid);
if (!object) return `Error: no object with uuid "${args.uuid}".`;
const label = object.name || object.uuid;
this.editor.execute(new RemoveObjectCommand(this.editor, object));
return `Deleted ${label}.`;
}
default:
return `Error: unknown tool "${name}".`;
}
}
/** One user turn: loops tool rounds against /api/scene/agent until the model stops calling tools. */
async send(userText, { onStatus, onAssistant } = {}) {
this.messages.push({ role: 'user', content: userText });
let mutated = false;
for (let round = 0; round < MAX_ROUNDS; round++) {
let data;
try {
const res = await fetch(ENDPOINT, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ messages: this.messages, tools: TOOLS }),
});
data = await res.json().catch(() => ({}));
if (!res.ok) throw new Error(data?.error_description || data?.message || `HTTP ${res.status}`);
} catch (err) {
onAssistant?.(`Could not reach the scene agent: ${err.message}. Try again in a moment.`, { error: true });
return;
}
const toolCalls = Array.isArray(data.tool_calls) ? data.tool_calls : [];
if (data.content) onAssistant?.(data.content, {});
if (!toolCalls.length) {
this.messages.push({ role: 'assistant', content: data.content || '' });
break;
}
this.messages.push({
role: 'assistant',
content: data.content || null,
tool_calls: toolCalls.map((tc) => ({ id: tc.id, type: 'function', function: { name: tc.name, arguments: tc.arguments } })),
});
for (const call of toolCalls) {
onStatus?.(`${call.name.replace(/_/g, ' ')}…`);
let result;
try {
result = await this.runTool(call.name, call.arguments, { onStatus });
} catch (err) {
result = `Error running ${call.name}: ${err.message}`;
}
if (!/^Error:/.test(result) && call.name !== 'list_objects') mutated = true;
this.messages.push({ role: 'tool', tool_call_id: call.id, content: result });
}
}
onStatus?.(null);
if (mutated) this.pushVersion(userText);
}
pushVersion(label) {
forceRender(this.editor);
const snapshot = this.editor.scene.toJSON();
const screenshot = captureScreenshot();
this.versions.push({ id: uid(), label, at: Date.now(), snapshot, screenshot });
if (this.versions.length > MAX_VERSIONS) this.versions.shift();
return this.versions[this.versions.length - 1];
}
/**
* Restores a prior version as one more undoable edit. SetSceneCommand only
* APPENDS a parsed scene's children (vendor/js/Loader.js always calls it
* right after editor.clear(), which also wipes undo history); doing the
* same here would litter the current scene with the version's objects
* instead of replacing them. Composing explicit Remove + Add commands
* under one MultiCmdsCommand gets a real replace that stays undoable.
*/
restoreVersion(versionId) {
const version = this.versions.find((v) => v.id === versionId);
if (!version) return false;
const loaded = new THREE.ObjectLoader().parse(version.snapshot);
const cmds = [];
for (const child of [...this.editor.scene.children]) {
cmds.push(new RemoveObjectCommand(this.editor, child));
}
while (loaded.children.length > 0) {
cmds.push(new AddObjectCommand(this.editor, loaded.children.pop()));
}
this.editor.execute(new MultiCmdsCommand(this.editor, cmds), `Restore "${version.label}"`);
return true;
}
}
function round(n) {
return Math.round(n * 100) / 100;
}
function num(v, fallback = 0) {
const n = Number(v);
return Number.isFinite(n) ? n : fallback;
}
/**
* Builds the Agent tab's panel and returns a UIElement ready for
* sidebar.addTab('agent', 'Agent', panel), see main.js wiring.
* @param {import('./vendor/js/Editor.js').Editor} editor
*/
export function createAgentPanel(editor) {
ensureStyles();
const agent = new SceneAgent(editor);
const root = document.createElement('div');
root.className = 'tws-agent-panel';
root.innerHTML = `
<p class="tws-agent-intro">Describe what you want in the scene. The agent reads it, makes real edits, and every change is undoable (Edit ▸ Undo).</p>
<div class="tws-agent-log" role="log" aria-live="polite"></div>
<p class="tws-agent-status" aria-live="polite" hidden></p>
<form class="tws-agent-form">
<textarea class="tws-agent-input" placeholder="e.g. add a red wooden chair by the window" rows="2" maxlength="500"></textarea>
<button type="submit" class="tws-agent-send">Send</button>
</form>
<div class="tws-agent-versions">
<h4>Versions</h4>
<p class="tws-agent-versions-empty">Changes the agent makes will show up here, step back to any of them.</p>
<ol class="tws-agent-versions-list"></ol>
</div>
`;
const log = root.querySelector('.tws-agent-log');
const statusEl = root.querySelector('.tws-agent-status');
const form = root.querySelector('.tws-agent-form');
const input = root.querySelector('.tws-agent-input');
const sendBtn = root.querySelector('.tws-agent-send');
const versionsList = root.querySelector('.tws-agent-versions-list');
const versionsEmpty = root.querySelector('.tws-agent-versions-empty');
function appendMessage(role, text) {
const row = document.createElement('div');
row.className = `tws-agent-msg tws-agent-msg-${role}`;
row.textContent = text;
log.appendChild(row);
log.scrollTop = log.scrollHeight;
}
function renderVersions() {
versionsList.innerHTML = '';
versionsEmpty.hidden = agent.versions.length > 0;
for (const version of agent.versions) {
const li = document.createElement('li');
li.className = 'tws-agent-version';
const time = new Date(version.at).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
li.innerHTML = `
${version.screenshot ? `<img loading="lazy" decoding="async" src="${version.screenshot}" alt="" width="48" height="36" />` : '<span class="tws-agent-version-noimg" aria-hidden="true"></span>'}
<span class="tws-agent-version-label">${escapeHtml(version.label).slice(0, 60)}</span>
<span class="tws-agent-version-time">${time}</span>
<button type="button" data-restore="${version.id}">Restore</button>
`;
versionsList.appendChild(li);
}
}
versionsList.addEventListener('click', (e) => {
const btn = e.target.closest('[data-restore]');
if (!btn) return;
agent.restoreVersion(btn.dataset.restore);
appendMessage('agent', 'Restored that version.');
});
form.addEventListener('submit', async (e) => {
e.preventDefault();
const text = input.value.trim();
if (!text || sendBtn.disabled) return;
appendMessage('user', text);
input.value = '';
sendBtn.disabled = true;
input.disabled = true;
try {
await agent.send(text, {
onStatus: (text) => {
statusEl.hidden = !text;
if (text) statusEl.textContent = text;
},
onAssistant: (text, { error } = {}) => {
if (text) appendMessage(error ? 'error' : 'agent', text);
},
});
} catch (err) {
toastError(`Scene agent failed: ${err.message}`, { duration: 7000 });
} finally {
statusEl.hidden = true;
sendBtn.disabled = false;
input.disabled = false;
input.focus();
renderVersions();
}
});
input.addEventListener('keydown', (e) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
form.requestSubmit();
}
});
return new UIElement(root);
}
function escapeHtml(s) {
return String(s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
}
const CSS = `
.tws-agent-panel {
display: flex; flex-direction: column; gap: 10px;
padding: 10px; font: 400 12.5px/1.45 system-ui, -apple-system, sans-serif;
color: var(--ink, #e8e8ec);
height: 100%; box-sizing: border-box;
}
.tws-agent-intro { margin: 0; color: #9a9aa8; font-size: 12px; }
.tws-agent-log {
display: flex; flex-direction: column; gap: 6px;
min-height: 120px; max-height: 260px; overflow-y: auto;
border: 1px solid rgba(255,255,255,0.12); border-radius: 8px;
padding: 8px; background: rgba(0,0,0,0.2);
}
.tws-agent-msg { padding: 6px 9px; border-radius: 8px; max-width: 92%; white-space: pre-wrap; word-break: break-word; }
.tws-agent-msg-user { align-self: flex-end; background: #2a6ef0; color: #fff; }
.tws-agent-msg-agent { align-self: flex-start; background: rgba(255,255,255,0.08); }
.tws-agent-msg-error { align-self: flex-start; background: rgba(255,90,90,0.18); color: #ffb3b3; }
.tws-agent-status { margin: 0; color: #6ee7b7; font-size: 11.5px; font-style: italic; }
.tws-agent-form { display: flex; gap: 6px; }
.tws-agent-input {
flex: 1; resize: vertical; min-height: 36px;
border: 1px solid rgba(255,255,255,0.18); border-radius: 8px;
padding: 8px 9px; background: rgba(10,10,14,0.8); color: var(--ink, #e8e8ec);
font: inherit;
}
.tws-agent-input:focus-visible { outline: 2px solid #6ee7b7; outline-offset: 1px; }
.tws-agent-send {
appearance: none; cursor: pointer; align-self: flex-end;
border: 0; border-radius: 8px; padding: 9px 14px;
background: #e8e8ec; color: #0a0a0f; font: 600 12.5px/1 system-ui, sans-serif;
}
.tws-agent-send:hover { background: #fff; }
.tws-agent-send:disabled { opacity: 0.55; cursor: default; }
.tws-agent-send:focus-visible { outline: 2px solid #6ee7b7; outline-offset: 2px; }
.tws-agent-versions { border-top: 1px solid rgba(255,255,255,0.12); padding-top: 8px; }
.tws-agent-versions h4 { margin: 0 0 6px; font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.04em; color: #9a9aa8; }
.tws-agent-versions-empty { margin: 0; color: #70707c; font-size: 11.5px; }
.tws-agent-versions-list { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 4px; max-height: 180px; overflow-y: auto; }
.tws-agent-version { display: flex; align-items: center; gap: 8px; padding: 4px; border-radius: 6px; }
.tws-agent-version:hover { background: rgba(255,255,255,0.06); }
.tws-agent-version img, .tws-agent-version-noimg { width: 48px; height: 36px; border-radius: 4px; object-fit: cover; background: #15151b; flex-shrink: 0; }
.tws-agent-version-label { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 11.5px; }
.tws-agent-version-time { color: #70707c; font-size: 10.5px; flex-shrink: 0; }
.tws-agent-version button {
appearance: none; cursor: pointer; flex-shrink: 0;
border: 1px solid rgba(255,255,255,0.18); border-radius: 6px;
padding: 4px 8px; background: transparent; color: var(--ink, #e8e8ec);
font: 600 10.5px/1 system-ui, sans-serif;
}
.tws-agent-version button:hover { background: rgba(255,255,255,0.1); }
.tws-agent-version button:focus-visible { outline: 2px solid #6ee7b7; outline-offset: 1px; }
`;