Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// agent-desk, a live "working agent" desk object for the 3D world.
//
// Places a desk in the scene with:
// • A flat-screen monitor whose face is a live CanvasTexture. When the agent
// has a Playwright process pushing frames the monitor shows those verbatim.
// When no frames arrive it paints a real-time activity canvas from the
// public /api/agent-screen-stream SSE feed's `log` events, the same
// stream the 2D watch panel and the live wall use, which backfills from
// the agent's real agent_actions rows server-side.
// • A keyboard + mouse prop on the desk surface.
// • An agent avatar seated behind the desk (idle-animated, driven by the
// same retargeting pipeline as every other avatar in the world).
// • Proximity detection: when the local player walks within NEAR_DIST units
// the monitor wakes up (the screen face brightens to full white) and the
// desk pings watch-intent so the caster pool upgrades it to live frames.
// Clicking/tapping the monitor opens the full 2D watch view via openWatch()
// (coincommunities wires the pointer pick; the cursor turns to a pointer
// over the screen).
//
// Follows chart-screen.js in every structural detail, CanvasTexture updated
// per-frame, update(dt) called from the scene loop, dispose() for cleanup.
//
// Usage:
// import { createAgentDesk } from './agent-desk.js';
// const desk = createAgentDesk(scene, { agentId, agentName, avatarUrl }, {
// position: [x, 0, z],
// rotationY: Math.PI,
// });
// // in the scene loop:
// desk.update(dt, playerPosition);
// // to open the watch panel when the player interacts:
// desk.openWatch(); // navigates to /dashboard-next/watch?agentId=…
// // cleanup:
// desk.dispose();
import {
Group, Mesh, MeshStandardMaterial,
PlaneGeometry, BoxGeometry, CylinderGeometry,
Vector3,
} from 'three';
import { createAgentScreenClient } from '../shared/agent-screen-client.js';
import { makeScreenTexture, screenMaterial } from './screen-texture.js';
const NEAR_DIST = 8; // units, player proximity to activate
const REDRAW_MS = 100; // ~10fps canvas repaint
// Canvas resolution, 16:9 to match the 2D panel.
const CW = 1280, CH = 720;
// Monitor physical dimensions (world units).
const MON_W = 3.6;
const MON_H = (MON_W * CH) / CW;
// ── canvas painting ───────────────────────────────────────────────────────────
function paintDesk(ctx, tex, actions, status, agentName, t) {
const text = '#f0f0f4', dim = 'rgba(255,255,255,0.35)', faint = 'rgba(255,255,255,0.12)';
const up = '#5fd08a', bg0 = '#080809', bg1 = '#0d0d11';
const pulse = 0.5 + 0.5 * Math.sin(t * 3);
// Background gradient
const bg = ctx.createLinearGradient(0, 0, 0, CH);
bg.addColorStop(0, bg1);
bg.addColorStop(1, bg0);
ctx.fillStyle = bg;
ctx.fillRect(0, 0, CW, CH);
// Top bar
ctx.fillStyle = 'rgba(255,255,255,0.03)';
ctx.fillRect(0, 0, CW, 48);
ctx.strokeStyle = 'rgba(255,255,255,0.05)';
ctx.lineWidth = 1;
ctx.beginPath(); ctx.moveTo(0, 48); ctx.lineTo(CW, 48); ctx.stroke();
// Status dot + name
ctx.beginPath();
ctx.arc(28, 24, 6, 0, Math.PI * 2);
ctx.fillStyle = status === 'live'
? `rgba(95,208,138,${0.55 + pulse * 0.45})`
: 'rgba(120,120,128,0.5)';
ctx.fill();
ctx.font = '700 16px Inter, system-ui, sans-serif';
ctx.fillStyle = text;
ctx.textAlign = 'left';
ctx.textBaseline = 'middle';
const name = (agentName || 'Agent').slice(0, 30);
ctx.fillText(name, 48, 24);
ctx.fillStyle = status === 'live' ? up : dim;
ctx.font = '500 13px Inter, system-ui, sans-serif';
ctx.fillText(status === 'live' ? '· live' : '· idle', 48 + ctx.measureText(name + ' ').width, 24);
// Clock
const now = new Date();
const clock = now.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit', hour12: false });
ctx.font = '600 14px Inter, system-ui, monospace';
ctx.fillStyle = dim;
ctx.textAlign = 'right';
ctx.fillText(clock, CW - 20, 24);
ctx.textAlign = 'left';
// Terminal window
const pad = 28;
const winX = pad, winY = 62, winW = CW - pad * 2, winH = CH - 62 - pad;
const r = 8;
ctx.fillStyle = '#0c0c10';
ctx.beginPath();
roundRect(ctx, winX, winY, winW, winH, r);
ctx.fill();
ctx.strokeStyle = 'rgba(255,255,255,0.055)';
ctx.lineWidth = 1;
ctx.stroke();
// Titlebar
const tbH = 34;
ctx.fillStyle = 'rgba(255,255,255,0.03)';
ctx.beginPath();
roundRect(ctx, winX, winY, winW, tbH, [r, r, 0, 0]);
ctx.fill();
ctx.strokeStyle = 'rgba(255,255,255,0.05)';
ctx.beginPath(); ctx.moveTo(winX, winY + tbH); ctx.lineTo(winX + winW, winY + tbH); ctx.stroke();
// Traffic lights
[['#ff5f57'], ['#febc2e'], ['#28c840']].forEach(([c], i) => {
ctx.beginPath();
ctx.arc(winX + 16 + i * 18, winY + tbH / 2, 5, 0, Math.PI * 2);
ctx.fillStyle = c;
ctx.fill();
});
// Title
ctx.font = '500 12px Inter, system-ui, sans-serif';
ctx.fillStyle = faint;
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
ctx.fillText(`${name}, live activity`, winX + winW / 2, winY + tbH / 2);
ctx.textAlign = 'left';
// Lines
const lH = 32;
const cX = winX + 20;
let lY = winY + tbH + 24;
const maxL = Math.floor((winH - tbH - 50) / lH);
if (!actions.length) {
ctx.font = '500 14px "Courier New", monospace';
ctx.fillStyle = faint;
ctx.textBaseline = 'alphabetic';
ctx.fillText('> Waiting for agent activity…', cX, lY);
if (Math.sin(t * 4) > 0) {
ctx.fillStyle = faint;
ctx.fillRect(cX + ctx.measureText('> Waiting for agent activity…').width + 3, lY - 14, 8, 16);
}
} else {
actions.slice(0, maxL).forEach((a, i) => {
const latest = i === 0;
const age = Math.max(0, Math.round((Date.now() - (a.ts || Date.now())) / 1000));
const ts = age < 5 ? 'now' : age < 60 ? `${age}s` : `${Math.round(age / 60)}m`;
ctx.textBaseline = 'alphabetic';
ctx.font = '600 12px "Courier New", monospace';
ctx.fillStyle = latest ? up : 'rgba(255,255,255,0.18)';
const prefix = ` [${ts}] `;
ctx.fillText(prefix, cX, lY + i * lH);
const pw = ctx.measureText(prefix).width;
ctx.font = `${latest ? '600' : '400'} 13px "Courier New", monospace`;
ctx.fillStyle = latest ? text : dim;
const summary = (a.summary || a.type || 'action').slice(0, 88);
ctx.fillText(summary, cX + pw, lY + i * lH);
if (latest && Math.sin(t * 4) > 0) {
const tw = pw + ctx.measureText(summary).width;
ctx.fillStyle = up;
ctx.fillRect(cX + tw + 3, lY + i * lH - 13, 7, 14);
}
});
}
// Status bar
const sbY = winY + winH - 24;
ctx.fillStyle = 'rgba(95,208,138,0.06)';
ctx.fillRect(winX + 1, sbY, winW - 2, 23);
ctx.strokeStyle = 'rgba(255,255,255,0.04)';
ctx.beginPath(); ctx.moveTo(winX, sbY); ctx.lineTo(winX + winW, sbY); ctx.stroke();
ctx.font = '500 11px Inter, system-ui, sans-serif';
ctx.textBaseline = 'middle';
ctx.fillStyle = 'rgba(95,208,138,0.6)';
ctx.fillText(`three.ws · ${actions.length || 0} actions`, cX, sbY + 12);
ctx.textAlign = 'right';
ctx.fillStyle = faint;
ctx.fillText('walk up · click the screen to watch', winX + winW - 10, sbY + 12);
ctx.textAlign = 'left';
tex.needsUpdate = true;
}
function roundRect(ctx, x, y, w, h, r) {
const tl = Array.isArray(r) ? r[0] : r;
const tr = Array.isArray(r) ? r[1] : r;
const br = Array.isArray(r) ? r[2] : r;
const bl = Array.isArray(r) ? r[3] : r;
ctx.beginPath();
ctx.moveTo(x + tl, y);
ctx.arcTo(x + w, y, x + w, y + h, tr);
ctx.arcTo(x + w, y + h, x, y + h, br);
ctx.arcTo(x, y + h, x, y, bl);
ctx.arcTo(x, y, x + w, y, tl);
ctx.closePath();
}
// ── factory ───────────────────────────────────────────────────────────────────
/**
* Create an agent desk and add it to the scene.
*
* @param {THREE.Scene} scene
* @param {{ agentId:string, agentName:string, avatarUrl?:string }} agent
* @param {{ position?:[number,number,number], rotationY?:number }} [opts]
* @returns {{ group:THREE.Group, update(dt:number, playerPos?:THREE.Vector3):void, openWatch():void, dispose():void }}
*/
export function createAgentDesk(scene, agent, opts = {}) {
const position = opts.position || [0, 0, 0];
const rotationY = opts.rotationY || 0;
const group = new Group();
group.position.set(...position);
group.rotation.y = rotationY;
const metalMat = new MeshStandardMaterial({ color: 0x1a1a1f, roughness: 0.4, metalness: 0.7 });
const darkMat = new MeshStandardMaterial({ color: 0x101012, roughness: 0.6, metalness: 0.3 });
const woodMat = new MeshStandardMaterial({ color: 0x2a1f14, roughness: 0.8, metalness: 0.0 });
// ── Desk surface ────────────────────────────────────────────────────────
const deskW = 4.8, deskD = 2.0, deskH = 0.08;
const deskY = 0.78; // standard desk height
const deskTop = new Mesh(new BoxGeometry(deskW, deskH, deskD), woodMat);
deskTop.position.set(0, deskY, 0);
deskTop.castShadow = true; deskTop.receiveShadow = true;
group.add(deskTop);
// Desk legs (4)
for (const [sx, sz] of [[-1, -1], [1, -1], [-1, 1], [1, 1]]) {
const leg = new Mesh(new BoxGeometry(0.08, deskY, 0.08), metalMat);
leg.position.set(sx * (deskW / 2 - 0.15), deskY / 2, sz * (deskD / 2 - 0.15));
leg.castShadow = true;
group.add(leg);
}
// ── Monitor stand ───────────────────────────────────────────────────────
const standH = 0.55;
const stand = new Mesh(new CylinderGeometry(0.04, 0.08, standH, 12), metalMat);
stand.position.set(0, deskY + deskH / 2 + standH / 2, -0.35);
stand.castShadow = true;
group.add(stand);
// Monitor base plate
const base = new Mesh(new BoxGeometry(0.6, 0.02, 0.3), metalMat);
base.position.set(0, deskY + deskH / 2 + 0.01, -0.35);
base.castShadow = true; base.receiveShadow = true;
group.add(base);
// Monitor bezel
const bezelMat = new MeshStandardMaterial({ color: 0x080809, roughness: 0.3, metalness: 0.8 });
const bezel = new Mesh(
new BoxGeometry(MON_W + 0.22, MON_H + 0.22, 0.12),
bezelMat,
);
const monCY = deskY + deskH / 2 + standH + MON_H / 2 + 0.02;
bezel.position.set(0, monCY, -0.35);
bezel.castShadow = true;
group.add(bezel);
// ── Live screen face ────────────────────────────────────────────────────
const canvas = document.createElement('canvas');
canvas.width = CW; canvas.height = CH;
const ctx = canvas.getContext('2d');
const tex = makeScreenTexture(canvas);
const screenMat = screenMaterial(tex);
// Idle screens sit slightly dimmed; walking near lifts them to full white.
// MeshBasicMaterial.color multiplies the map, so this is a real brightness
// change (the material is opaque, opacity would be a no-op without
// transparent: true).
screenMat.color.setHex(0xbbbbbb);
const screen = new Mesh(new PlaneGeometry(MON_W, MON_H), screenMat);
screen.position.set(0, monCY, -0.35 + 0.07);
screen.userData.agentDesk = true;
group.add(screen);
// ── Keyboard ────────────────────────────────────────────────────────────
const kb = new Mesh(new BoxGeometry(1.2, 0.03, 0.36), darkMat);
kb.position.set(0, deskY + deskH / 2 + 0.015, 0.3);
kb.castShadow = true; kb.receiveShadow = true;
group.add(kb);
// ── Mouse ───────────────────────────────────────────────────────────────
const mouse = new Mesh(new BoxGeometry(0.1, 0.025, 0.15), darkMat);
mouse.position.set(0.75, deskY + deskH / 2 + 0.012, 0.32);
mouse.castShadow = true; mouse.receiveShadow = true;
group.add(mouse);
scene.add(group);
// ── Live state ───────────────────────────────────────────────────────────
let actions = [];
let frameImg = null; // latest pushed frame as an Image, or null
let status = 'idle';
let destroyed = false;
let near = false;
let acc = 0;
let t = 0;
// Initial paint so the screen isn't blank before the first fetch.
paintDesk(ctx, tex, [], 'idle', agent.agentName || 'Agent', 0);
// Same stream the 2D watch panel and the live wall use. Frames arrive as
// `event: frame` with `data` holding a full data URL (or null for text-only
// beats); `event: log` backfills activity oldest-first; `event: dark` fires
// when the caster's frame TTL lapses. Reconnect/backoff lives in the client.
const screenClient = createAgentScreenClient(agent.agentId, {
onFrame(frame) {
status = 'live';
if (typeof frame?.data === 'string' && frame.data.startsWith('data:image/')) {
const img = new Image();
img.onload = () => { frameImg = img; };
img.src = frame.data;
}
},
onLog(entries) {
if (!entries?.length) return;
// paintDesk expects newest-first rows shaped like agent-actions.
actions = entries
.map((e) => ({ ts: e.ts, summary: e.activity, type: e.type }))
.reverse();
},
onDark() { status = 'idle'; frameImg = null; },
onError() { status = 'idle'; },
});
// Signal that this desk's agent is being watched so the on-demand caster pool
// upgrades it to a live browser feed. Throttled; fired while the player is near.
let lastWatchPing = 0;
function pingWatchIntent() {
const now = Date.now();
if (now - lastWatchPing < 20_000) return;
lastWatchPing = now;
try {
fetch('/api/agent/watch-intent', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ agentId: agent.agentId }),
keepalive: true,
}).catch(() => {});
} catch { /* */ }
}
// The SSE stream is the desk's single activity source: its `log` events
// backfill from the caster's Redis log or the agent's real DB actions (the
// server re-polls dark agents every ~8s), and it is public. The old direct
// /api/agent-actions REST poll here 401'd forever for anonymous players
// (that route is owner-only), pure wasted requests, so it's gone.
screenClient.connect();
// ── public API ────────────────────────────────────────────────────────────
return {
group,
screen,
agentId: agent.agentId,
update(dt, playerPos) {
if (destroyed) return;
t += dt;
acc += dt;
// Proximity check, brighten screen when player is near.
if (playerPos) {
const deskWorldPos = new Vector3();
group.getWorldPosition(deskWorldPos);
const dist = playerPos.distanceTo(deskWorldPos);
const wasNear = near;
near = dist < NEAR_DIST;
if (near !== wasNear) {
// Wake the screen: full-white map when near, dimmed when far.
screenMat.color.setHex(near ? 0xffffff : 0xbbbbbb);
}
if (near) pingWatchIntent();
}
if (acc < REDRAW_MS / 1000) return;
acc = 0;
if (frameImg && status === 'live') {
ctx.drawImage(frameImg, 0, 0, CW, CH);
tex.needsUpdate = true;
} else {
paintDesk(ctx, tex, actions, status, agent.agentName || 'Agent', t);
}
},
openWatch() {
const url = `/dashboard-next/watch?agentId=${encodeURIComponent(agent.agentId)}`;
window.open(url, '_blank', 'noopener,noreferrer');
},
dispose() {
destroyed = true;
screenClient.disconnect();
scene.remove(group);
group.traverse((n) => {
if (!n.isMesh) return;
n.geometry?.dispose?.();
const mats = Array.isArray(n.material) ? n.material : [n.material];
for (const m of mats) { m?.map?.dispose?.(); m?.dispose?.(); }
});
},
};
}