Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// The 3D viewer: a custom editor for .glb/.gltf files in the workspace, and a
// panel for models that live on a URL.
//
// The host owns the file: it reads the bytes, runs the inspector, and streams
// the report to the webview. The webview owns the picture, and sends back what
// only a running scene can produce: a retargeted clip to bake, a turntable, a
// render for the quality check.
import * as vscode from 'vscode';
import { reportFor } from './inspect.js';
import { notify, trackPanel } from './active-panel.js';
import { uniqueName } from './naming.js';
import { writeClipIntoGlb } from './bake-clip.js';
import { clipSlug } from './animations.js';
export const VIEW_TYPE = 'threews3d.modelViewer';
/** Models larger than this are rendered but not inspected; the report would stall. */
const MAX_INSPECT_BYTES = 96 * 1024 * 1024;
/** Custom editor for model files. One webview per open document. */
export class ModelViewerProvider {
/** @param {vscode.ExtensionContext} context @param {import('./links.js').ModelLinks} links */
constructor(context, links, output) {
this.context = context;
this.links = links;
this.output = output;
}
static register(context, links, output) {
return vscode.window.registerCustomEditorProvider(
VIEW_TYPE,
new ModelViewerProvider(context, links, output),
{
webviewOptions: { retainContextWhenHidden: true },
supportsMultipleEditorsPerDocument: false,
},
);
}
async openCustomDocument(uri) {
return { uri, dispose: () => {} };
}
async resolveCustomEditor(document, panel) {
const dir = vscode.Uri.joinPath(document.uri, '..');
panel.webview.options = {
enableScripts: true,
localResourceRoots: [vscode.Uri.joinPath(this.context.extensionUri, 'media'), dir],
};
panel.webview.html = viewerHtml(panel.webview, this.context.extensionUri);
const src = panel.webview.asWebviewUri(document.uri).toString();
const wire = attachViewer(this.context, panel, {
src,
resource: document.uri,
links: this.links,
output: this.output,
readBytes: () => vscode.workspace.fs.readFile(document.uri),
});
trackPanel(panel);
// Regenerate the model on disk and the open viewer catches up on its own.
const watcher = vscode.workspace.createFileSystemWatcher(
new vscode.RelativePattern(dir, path(document.uri)),
);
watcher.onDidChange(() => wire.reload());
panel.onDidDispose(() => {
watcher.dispose();
wire.dispose();
});
}
}
/** Open a model that lives on an http(s) URL in its own panel. */
export function openRemoteModel(context, links, output, url, title, column = vscode.ViewColumn.Active) {
const panel = vscode.window.createWebviewPanel(
VIEW_TYPE,
title || 'three.ws 3D',
column,
{
enableScripts: true,
retainContextWhenHidden: true,
localResourceRoots: [vscode.Uri.joinPath(context.extensionUri, 'media')],
},
);
panel.webview.html = viewerHtml(panel.webview, context.extensionUri);
const wire = attachViewer(context, panel, {
src: url,
remoteUrl: url,
links,
output,
readBytes: async () => {
const res = await fetch(url);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const declared = Number(res.headers.get('content-length') || 0);
if (declared > MAX_INSPECT_BYTES) {
res.body?.cancel();
throw new Error(`model is ${declared} bytes, too large to inspect`);
}
return new Uint8Array(await res.arrayBuffer());
},
});
trackPanel(panel);
panel.onDidDispose(() => wire.dispose());
return panel;
}
/**
* Open a model that lives outside the workspace (a committed version pulled
* from git) beside the current editor.
*/
export function openLocalModel(context, links, output, uri, title, column = vscode.ViewColumn.Beside) {
const dir = vscode.Uri.joinPath(uri, '..');
const panel = vscode.window.createWebviewPanel(VIEW_TYPE, title, column, {
enableScripts: true,
retainContextWhenHidden: true,
localResourceRoots: [vscode.Uri.joinPath(context.extensionUri, 'media'), dir],
});
panel.webview.html = viewerHtml(panel.webview, context.extensionUri);
const wire = attachViewer(context, panel, {
src: panel.webview.asWebviewUri(uri).toString(),
resource: uri,
links,
output,
readBytes: () => vscode.workspace.fs.readFile(uri),
});
trackPanel(panel);
panel.onDidDispose(() => wire.dispose());
return panel;
}
/**
* Wire one webview: hand it the model, run the report, and route its actions
* back into the extension's commands.
*/
function attachViewer(context, panel, { src, resource, remoteUrl, links, output, readBytes }) {
let disposed = false;
const post = (message) => {
if (!disposed) panel.webview.postMessage(message);
};
// What the commands need to know about this viewer.
panel.threews = { resource, remoteUrl, readBytes, stats: null, fileSize: 0, post, pending: new Map() };
const sendReport = async () => {
try {
const bytes = await readBytes();
panel.threews.fileSize = bytes.byteLength;
if (bytes.byteLength > MAX_INSPECT_BYTES) return;
const { rows, suggestions } = await reportFor(bytes);
post({ type: 'report', rows, suggestions });
} catch (err) {
output.appendLine(`report failed for ${resource?.fsPath || remoteUrl}: ${err?.message || err}`);
}
};
const sub = panel.webview.onDidReceiveMessage(async (msg) => {
if (msg?.type === 'ready') {
post({ type: 'load', src });
sendReport();
} else if (msg?.type === 'loaded') {
panel.threews.stats = msg.stats || null;
notify();
} else if (msg?.type === 'error') {
output.appendLine(`viewer: ${msg.message}`);
} else if (msg?.type === 'snapshot') {
await saveBeside(msg.dataUrl, { resource, output, ext: '.png', suffix: '', what: 'Snapshot' });
} else if (msg?.type === 'turntable') {
await saveBeside(msg.dataUrl, { resource, output, ext: '.gif', suffix: '-turntable', what: 'Turntable' });
} else if (msg?.type === 'bake-clip') {
await bakeClip(msg, { resource, readBytes, links, output });
} else if (msg?.type === 'clip-result') {
const waiter = panel.threews.pending.get(msg.requestId);
if (waiter) {
panel.threews.pending.delete(msg.requestId);
waiter(msg);
}
} else if (msg?.type === 'action') {
await runAction(msg.action, { resource, remoteUrl });
}
});
return {
reload: () => {
post({ type: 'load', src: `${src}${src.includes('?') ? '&' : '?'}t=${Date.now()}` });
sendReport();
},
dispose: () => {
disposed = true;
sub.dispose();
},
};
}
async function runAction(action, { resource, remoteUrl }) {
const target = resource || (remoteUrl ? vscode.Uri.parse(remoteUrl) : undefined);
const commands = {
openInBrowser: 'threews3d.openInBrowser',
embed: 'threews3d.insertEmbed',
rig: 'threews3d.rigModel',
animate: 'threews3d.animate',
refine: 'threews3d.refineModel',
quality: 'threews3d.checkQuality',
optimize: 'threews3d.optimizeModel',
compare: 'threews3d.compareWithHead',
};
if (commands[action]) await vscode.commands.executeCommand(commands[action], target);
}
/**
* Ask the webview for something only the running scene can produce and wait
* for its answer (a `clip-result` message carrying the same requestId).
*
* @param {vscode.WebviewPanel} panel
* @param {object} message
* @param {number} [timeoutMs]
* @returns {Promise<any>}
*/
export function request(panel, message, timeoutMs = 30_000) {
const requestId = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
return new Promise((resolve) => {
panel.threews.pending.set(requestId, resolve);
panel.threews.post({ ...message, requestId });
setTimeout(() => {
if (panel.threews.pending.delete(requestId)) {
resolve({ ok: false, coverage: 0, matched: 0, total: 0, message: 'the viewer did not answer' });
}
}, timeoutMs);
});
}
/**
* Send a clip to the webview and wait for it to say whether the rig took it.
* @param {vscode.WebviewPanel} panel
* @param {{ clip: object, label: string, loop: boolean }} payload
* @returns {Promise<{ ok: boolean, coverage: number, matched: number, total: number, message?: string }>}
*/
export function playClipIn(panel, payload) {
return request(panel, { type: 'play-clip', ...payload });
}
/** Write the retargeted clip into a copy of the model, next to it. */
async function bakeClip(msg, { resource, readBytes, links, output }) {
const label = String(msg.label || msg.clip?.name || 'clip');
try {
const bytes = await readBytes();
const result = await writeClipIntoGlb(bytes, msg.clip, { name: label });
const folder = resource ? vscode.Uri.joinPath(resource, '..') : await defaultFolder();
const stem = `${resource ? path(resource).replace(/\.(glb|gltf)$/i, '') : 'model'}-${clipSlug(label)}`;
const target = await freeName(folder, stem, '.glb');
await vscode.workspace.fs.writeFile(target, result.bytes);
if (resource) {
const meta = links.meta(resource);
if (meta) await links.set(target, meta.url, { prompt: meta.prompt });
}
output.appendLine(`baked "${label}" into ${target.fsPath} (${result.channels} channels${result.dropped.length ? `, ${result.dropped.length} tracks had no node` : ''})`);
const open = await vscode.window.showInformationMessage(
`Saved ${path(target)} with the "${label}" clip baked in.`,
'Open',
);
if (open === 'Open') await vscode.commands.executeCommand('vscode.openWith', target, VIEW_TYPE);
} catch (err) {
output.appendLine(`bake failed: ${err?.message || err}`);
vscode.window.showErrorMessage(`three.ws 3D: baking the clip failed. ${err?.message || err}`);
}
}
/** Write a data: URL image next to the model (or into the workspace root). */
async function saveBeside(dataUrl, { resource, output, ext, suffix, what }) {
const match = /^data:image\/(png|gif);base64,(.+)$/.exec(String(dataUrl || ''));
if (!match) {
vscode.window.showErrorMessage(`three.ws 3D: the ${what.toLowerCase()} came back empty.`);
return;
}
const bytes = Buffer.from(match[2], 'base64');
let folder;
try {
folder = resource ? vscode.Uri.joinPath(resource, '..') : await defaultFolder();
} catch (err) {
vscode.window.showErrorMessage(`three.ws 3D: ${err.message}`);
return;
}
const stem = `${resource ? path(resource).replace(/\.(glb|gltf)$/i, '') : 'model'}${suffix}`;
const target = await freeName(folder, stem, ext);
await vscode.workspace.fs.writeFile(target, bytes);
output.appendLine(`${what.toLowerCase()} written to ${target.fsPath}`);
const open = await vscode.window.showInformationMessage(`${what} saved as ${path(target)}`, 'Open');
if (open === 'Open') await vscode.commands.executeCommand('vscode.open', target);
}
async function defaultFolder() {
const folder = vscode.workspace.workspaceFolders?.[0]?.uri;
if (!folder) throw new Error('open a folder to save into.');
return folder;
}
async function freeName(folder, stem, ext) {
const taken = await vscode.workspace.fs.readDirectory(folder).then(
(entries) => new Set(entries.map(([name]) => name.toLowerCase())),
() => new Set(),
);
return vscode.Uri.joinPath(folder, uniqueName(stem, ext, (name) => taken.has(name.toLowerCase())));
}
function path(uri) {
return uri.path.split('/').pop() || '';
}
/** The webview document. Scripts are nonce-gated; nothing loads from a CDN. */
export function viewerHtml(webview, extensionUri) {
const nonce = randomNonce();
const asset = (...parts) =>
webview.asWebviewUri(vscode.Uri.joinPath(extensionUri, 'media', ...parts)).toString();
const config = {
dracoPath: `${asset('vendor', 'draco')}/`,
basisPath: `${asset('vendor', 'basis')}/`,
environment: setting('environment', 'studio'),
showGrid: setting('showGrid', true),
autoRotate: setting('autoRotate', false),
turntableFrames: setting('turntableFrames', 36),
turntableSize: setting('turntableSize', 480),
};
// wasm-unsafe-eval: the meshopt, Draco, and Basis decoders are WebAssembly.
// blob: workers: Draco and KTX2 build their worker from a blob at runtime.
const csp = [
"default-src 'none'",
`img-src ${webview.cspSource} data: blob:`,
`style-src ${webview.cspSource}`,
`script-src 'nonce-${nonce}' 'wasm-unsafe-eval' blob:`,
`connect-src ${webview.cspSource} https: data: blob:`,
'worker-src blob:',
'child-src blob:',
].join('; ');
return `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="Content-Security-Policy" content="${csp}">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="${asset('viewer.css')}">
<title>three.ws 3D</title>
</head>
<body>
<div id="stage"></div>
<div id="loading">
<div class="spinner" aria-hidden="true"></div>
<p id="loading-label">Loading model…</p>
</div>
<div id="error" hidden role="alert">
<h2>This model could not be rendered</h2>
<p id="error-text"></p>
<p>Check that the file is a valid glTF 2.0 asset. A .gltf that points at
external buffers needs those files next to it in the workspace.</p>
</div>
<div id="busy" hidden role="status" aria-live="polite">
<div class="spinner" aria-hidden="true"></div>
<span id="busy-label"></span>
</div>
<aside id="report" hidden aria-label="Model report">
<h2>Model report</h2>
<dl id="report-body"></dl>
<ul id="stats"></ul>
</aside>
<div id="toolbar" role="toolbar" aria-label="Viewer controls">
<span id="playback" hidden>
<button id="play" data-action="play" aria-pressed="true" title="Play or pause the animation">Pause</button>
<input id="scrub" type="range" min="0" max="1" step="0.001" value="0" aria-label="Animation time">
<select id="clips" hidden aria-label="Animation clip"></select>
<button id="bake" data-action="bake" hidden title="Write this library clip into a copy of the model so it plays in any engine">Bake clip</button>
</span>
<button data-toggle="grid" aria-pressed="${config.showGrid ? 'true' : 'false'}" title="Show the ground grid">Grid</button>
<button data-toggle="wireframe" aria-pressed="false" title="Draw the mesh as a wireframe">Wireframe</button>
<button data-toggle="skeleton" aria-pressed="false" title="Overlay the skeleton">Skeleton</button>
<button data-toggle="rotate" aria-pressed="${config.autoRotate ? 'true' : 'false'}" title="Orbit the camera automatically">Rotate</button>
<button data-action="reset" title="Frame the model again">Reset view</button>
<button data-toggle="report" aria-pressed="false" title="Triangles, materials, textures, rig, and optimization notes">Report</button>
<span class="sep" aria-hidden="true"></span>
<button data-action="animate" title="Try any of 2,800 library animations on this rig, or describe a motion in words">Animate</button>
<button data-action="refine" title="Describe a change and generate a new version anchored to this model">Refine</button>
<button data-action="quality" title="Ask a vision model to score this render for realism and completeness">Check quality</button>
<button data-action="optimize" title="Dedup, weld, resample and meshopt-compress a copy for the web">Optimize</button>
<span class="sep" aria-hidden="true"></span>
<button data-action="snapshot" title="Save the current view as a PNG next to the model">Snapshot</button>
<button data-action="turntable" title="Save a looping turntable GIF next to the model">Turntable</button>
<button data-action="embed" title="Insert an <agent-3d> embed snippet">Embed</button>
<button data-action="rig" title="Add a humanoid skeleton with the three.ws rigger">Rig</button>
<button data-action="openInBrowser" title="Open this model in the three.ws viewer">three.ws</button>
</div>
<script nonce="${nonce}">window.__THREEWS_CONFIG__ = ${JSON.stringify(config)};</script>
<script nonce="${nonce}" src="${asset('viewer.js')}"></script>
</body>
</html>`;
}
function setting(key, fallback) {
return vscode.workspace.getConfiguration('threews3d').get(key, fallback);
}
function randomNonce() {
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
let out = '';
for (let i = 0; i < 32; i++) out += chars[Math.floor(Math.random() * chars.length)];
return out;
}