3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
// The local page behind scene_ask_user (issue #922): it shows one rendered view, lets the person pin points on it with a short
// note each, and returns the marks when they press Send. It listens on loopback only, under a random path, with no remote assets.
import { createServer } from 'node:http';
import { randomBytes } from 'node:crypto';
import { isPreviewHost } from './preview-host.mjs';
import { ANNOTATE_LIMITS, validateMarks, isBoundedPng } from '../src/agent-annotate.js';
const MAXIMUM_BODY_BYTES = 2 * 1024 * 1024;
// Shared with the inline MCP Apps view (scripts/annotate-app-view.mjs) so both look the same.
export function annotateStyle(width) {
return `
*{box-sizing:border-box}
body{margin:0;min-height:100vh;background:#182126;color:#e8eef0;font:15px/1.4 system-ui,sans-serif;display:flex;flex-direction:column;gap:16px;padding:20px}
h1{margin:0;font-size:18px;font-weight:600}
main{display:flex;gap:20px;align-items:flex-start;flex-wrap:wrap}
.stage{position:relative;line-height:0;flex:0 1 ${width}px;max-width:100%;cursor:crosshair;user-select:none}
.stage img{width:100%;height:auto;border-radius:8px;display:block}
.stage.lasso{cursor:cell;touch-action:none}
.outline{position:absolute;inset:0;width:100%;height:100%;pointer-events:none;overflow:visible}
.outline polygon,.outline polyline{fill:rgba(255,90,79,.22);stroke:#ff5a4f;stroke-width:2;stroke-linejoin:round;vector-effect:non-scaling-stroke}
.outline polyline{fill:none;stroke:#fff}
.tools{display:flex;gap:4px}
button[aria-pressed=true]{background:#26343b;box-shadow:inset 0 0 0 2px #6fb7ff}
.pin{position:absolute;width:26px;height:26px;margin:-13px 0 0 -13px;border-radius:50%;background:#ff5a4f;color:#fff;font:600 13px/26px system-ui,sans-serif;text-align:center;box-shadow:0 0 0 2px #fff;pointer-events:none}
aside{flex:1 1 280px;max-width:380px;display:flex;flex-direction:column;gap:12px;user-select:none}
.row{display:flex;gap:10px;align-items:center}
.badge{flex:none;width:26px;height:26px;border-radius:50%;background:#ff5a4f;color:#fff;font:600 13px/26px system-ui,sans-serif;text-align:center}
input{flex:1;min-width:0;height:36px;border:0;border-radius:6px;background:#26343b;color:inherit;padding:0 10px;font:inherit;user-select:text}
input:focus{outline:2px solid #6fb7ff}
button{height:36px;border:0;border-radius:6px;padding:0 14px;font:inherit;color:inherit;background:transparent;cursor:pointer;display:inline-flex;gap:8px;align-items:center}
button:hover{background:#26343b}
button.primary{background:#2f7dd1;color:#fff}
button.primary:hover{background:#3a8ae0}
button.icon{width:36px;padding:0;justify-content:center;flex:none}
button:disabled{opacity:.5;cursor:default}
svg{width:18px;height:18px;flex:none}
.actions{display:flex;gap:8px;margin-top:4px}
.hint,.count{color:#9db0b8}
.done{display:flex;gap:10px;align-items:center;font-size:17px}
.done svg{width:26px;height:26px}
.ok{color:#58d68d}.stop{color:#f0b429}
`;
}
function pageHtml({ prompt, width, height, maxMarks, maxNoteLength }) {
const escaped = prompt.replace(/[&<>"]/g, character => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[character]);
return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Mark the scene</title><style>${annotateStyle(width)}</style></head><body>
<h1>${escaped || 'Click the scene to mark what you mean'}</h1>
<main>
<div class="stage" id="stage"><img id="frame" src="frame.png" width="${width}" height="${height}" alt="Rendered scene"></div>
<aside id="side">
<div class="tools" role="group" aria-label="Mark type">
<button class="icon" id="tool-pin" aria-pressed="true" aria-label="Pin" title="Pin: click a spot"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 21s-6-5.5-6-11a6 6 0 0 1 12 0c0 5.500-6 11-6 11z"/><circle cx="12" cy="10" r="2"/></svg></button>
<button class="icon" id="tool-lasso" aria-pressed="false" aria-label="Lasso" title="Lasso: draw around a region"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 4C7 4 3 6.500 3 10s4 6 9 6 9-2.500 9-6-4-6-9-6z"/><path d="M9 16c-1 2-1 3.500 1 4"/></svg></button>
</div>
<div class="hint" id="hint">Click the picture to drop a pin, or draw around a region with the lasso, then say what to change.</div>
<div id="rows" style="display:flex;flex-direction:column;gap:8px"></div>
<div class="count" id="count"></div>
<div class="actions"><button class="primary" id="send">Send</button><button id="cancel">Cancel</button></div>
</aside>
</main>
<script type="application/json" id="limits">${JSON.stringify({ width, height, maxMarks, maxNoteLength, minLassoPoints: ANNOTATE_LIMITS.minLassoPoints, maxLassoPoints: ANNOTATE_LIMITS.maxLassoPoints })}</script>
<script src="app.js"></script>
</body></html>`;
}
// Served as its own file because the page's content security policy allows scripts from this origin only.
const PAGE_SCRIPT = `(() => {
const limits = JSON.parse(document.getElementById('limits').textContent);
const marks = [];
const stage = document.getElementById('stage');
const frame = document.getElementById('frame');
const rows = document.getElementById('rows');
const count = document.getElementById('count');
const sendButton = document.getElementById('send');
const trash = '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 6h18M8 6V4h8v2M6 6l1 14h10l1-14"/></svg>';
const svgNamespace = 'http://www.w3.org/2000/svg';
const outline = document.createElementNS(svgNamespace, 'svg');
outline.setAttribute('class', 'outline');
outline.setAttribute('viewBox', '0 0 ' + limits.width + ' ' + limits.height);
outline.setAttribute('preserveAspectRatio', 'none');
stage.appendChild(outline);
let tool = 'pin';
let drawing = null;
const toolButtons = { pin: document.getElementById('tool-pin'), lasso: document.getElementById('tool-lasso') };
function chooseTool(name) {
tool = name;
stage.classList.toggle('lasso', name === 'lasso');
Object.entries(toolButtons).forEach(([key, button]) => button.setAttribute('aria-pressed', String(key === name)));
}
toolButtons.pin.addEventListener('click', () => chooseTool('pin'));
toolButtons.lasso.addEventListener('click', () => chooseTool('lasso'));
function pointsText(points) {
return points.map(point => point[0].toFixed(1) + ',' + point[1].toFixed(1)).join(' ');
}
function markCentre(mark) {
if (mark.points === undefined) return [mark.x, mark.y];
const total = mark.points.reduce((sum, point) => [sum[0] + point[0], sum[1] + point[1]], [0, 0]);
return [total[0] / mark.points.length, total[1] / mark.points.length];
}
// A pen stroke has hundreds of points; the answer allows maxLassoPoints, so keep evenly spaced ones.
function thinned(points) {
if (points.length <= limits.maxLassoPoints) return points;
return Array.from({ length: limits.maxLassoPoints }, (unused, index) => points[Math.floor(index * points.length / limits.maxLassoPoints)]);
}
function render() {
stage.querySelectorAll('.pin').forEach(pin => pin.remove());
outline.textContent = '';
rows.textContent = '';
marks.forEach((mark, index) => {
const centre = markCentre(mark);
if (mark.points !== undefined) {
const shape = document.createElementNS(svgNamespace, 'polygon');
shape.setAttribute('points', pointsText(mark.points));
outline.appendChild(shape);
}
const pin = document.createElement('div');
pin.className = 'pin';
pin.textContent = index + 1;
pin.style.left = (centre[0] / limits.width * 100) + '%';
pin.style.top = (centre[1] / limits.height * 100) + '%';
stage.appendChild(pin);
const row = document.createElement('div');
row.className = 'row';
const badge = document.createElement('span');
badge.className = 'badge';
badge.textContent = index + 1;
const input = document.createElement('input');
input.maxLength = limits.maxNoteLength;
input.value = mark.note;
input.placeholder = 'What should change here?';
input.addEventListener('input', () => { mark.note = input.value; });
const remove = document.createElement('button');
remove.className = 'icon';
remove.title = 'Remove mark';
remove.setAttribute('aria-label', 'Remove mark ' + (index + 1));
remove.innerHTML = trash;
remove.addEventListener('click', () => { marks.splice(index, 1); render(); });
row.append(badge, input, remove);
rows.appendChild(row);
mark.input = input;
});
count.textContent = marks.length + ' of ' + limits.maxMarks + ' marks';
sendButton.disabled = marks.length === 0;
}
function pixelOf(event) {
const box = frame.getBoundingClientRect();
const x = Math.min(limits.width, Math.max(0, (event.clientX - box.left) / box.width * limits.width));
const y = Math.min(limits.height, Math.max(0, (event.clientY - box.top) / box.height * limits.height));
return [x, y];
}
function polygonArea(points) {
let twice = 0;
points.forEach((point, index) => {
const next = points[(index + 1) % points.length];
twice += point[0] * next[1] - next[0] * point[1];
});
return Math.abs(twice) / 2;
}
stage.addEventListener('click', event => {
if (tool !== 'pin' || marks.length >= limits.maxMarks) return;
const [x, y] = pixelOf(event);
marks.push({ x, y, note: '' });
render();
marks[marks.length - 1].input.focus();
});
stage.addEventListener('pointerdown', event => {
if (tool !== 'lasso' || marks.length >= limits.maxMarks || event.button !== 0) return;
event.preventDefault();
stage.setPointerCapture(event.pointerId);
const trail = document.createElementNS(svgNamespace, 'polyline');
outline.appendChild(trail);
drawing = { points: [pixelOf(event)], trail };
});
stage.addEventListener('pointermove', event => {
if (!drawing) return;
const point = pixelOf(event);
const last = drawing.points[drawing.points.length - 1];
if (Math.hypot(point[0] - last[0], point[1] - last[1]) < 3) return;
drawing.points.push(point);
drawing.trail.setAttribute('points', pointsText(drawing.points));
});
function endLasso() {
if (!drawing) return;
const points = thinned(drawing.points);
drawing = null;
if (points.length >= limits.minLassoPoints && polygonArea(points) >= 1) marks.push({ kind: 'lasso', points, note: '' });
render();
if (marks.length && marks[marks.length - 1].points === points) marks[marks.length - 1].input.focus();
}
stage.addEventListener('pointerup', endLasso);
stage.addEventListener('pointercancel', endLasso);
function markedImage() {
const canvas = document.createElement('canvas');
canvas.width = limits.width;
canvas.height = limits.height;
const context = canvas.getContext('2d');
context.drawImage(frame, 0, 0, limits.width, limits.height);
context.font = '600 15px system-ui, sans-serif';
context.textAlign = 'center';
context.textBaseline = 'middle';
marks.forEach(mark => {
if (mark.points === undefined) return;
context.beginPath();
mark.points.forEach((point, index) => (index === 0 ? context.moveTo(point[0], point[1]) : context.lineTo(point[0], point[1])));
context.closePath();
context.fillStyle = 'rgba(255,90,79,0.22)';
context.fill();
context.lineWidth = 2;
context.strokeStyle = '#ff5a4f';
context.stroke();
});
marks.forEach((mark, index) => {
const [centreX, centreY] = markCentre(mark);
context.beginPath();
context.arc(centreX, centreY, 14, 0, Math.PI * 2);
context.fillStyle = '#ff5a4f';
context.fill();
context.lineWidth = 2;
context.strokeStyle = '#fff';
context.stroke();
context.fillStyle = '#fff';
context.fillText(String(index + 1), centreX, centreY + 1);
});
return canvas.toDataURL('image/png').split(',')[1];
}
function finish(kind, text) {
const icon = kind === 'ok'
? '<svg class="ok" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><path d="M7.5 12.5l3 3 6-6.5"/></svg>'
: '<svg class="stop" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><path d="M8 12h8"/></svg>';
document.getElementById('side').innerHTML = '<div class="done">' + icon + '<span>' + text + '</span></div>';
}
async function post(path, body) {
const response = await fetch(path, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
if (!response.ok) throw new Error((await response.text()) || 'The request failed.');
}
sendButton.addEventListener('click', async () => {
sendButton.disabled = true;
try {
await post('send', { marks: marks.map(mark => (mark.points === undefined ? { x: mark.x, y: mark.y, note: mark.note } : { kind: 'lasso', points: mark.points, note: mark.note })), image: markedImage() });
finish('ok', 'Sent to the agent. You can close this tab.');
} catch (error) {
sendButton.disabled = false;
document.getElementById('hint').textContent = error.message;
}
});
document.getElementById('cancel').addEventListener('click', async () => {
await post('cancel', {}).catch(() => {});
finish('stop', 'Cancelled. You can close this tab.');
});
render();
})();`;
function readBody(request) {
return new Promise((resolve, reject) => {
const chunks = [];
let total = 0;
request.on('data', chunk => {
total += chunk.length;
if (total > MAXIMUM_BODY_BYTES) {
reject(new RangeError('The request body is too large.'));
request.destroy();
return;
}
chunks.push(chunk);
});
request.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
request.on('error', reject);
});
}
/**
* Serves one annotate page and waits for the person. `open(url)` shows the page (the runtime opens the browser; a test fetches it).
* `signal` ends the wait early with outcome 'aborted'; the server and the timer are released at once.
* Resolves { outcome: 'sent' | 'cancelled' | 'timeout' | 'aborted', url, marks?, image? }; a rejected `open` rejects with the page address.
*/
export async function runAnnotateSession({ png, width, height, prompt = '', maxMarks = ANNOTATE_LIMITS.maxMarks, waitMs, open, signal }) {
const token = randomBytes(16).toString('hex');
const base = `/${token}/`;
const frameBytes = Buffer.from(png, 'base64');
const html = pageHtml({ prompt, width, height, maxMarks, maxNoteLength: ANNOTATE_LIMITS.maxNoteLength });
let finish;
const finished = new Promise(resolve => { finish = resolve; });
const server = createServer(async (request, response) => {
response.setHeader('Cache-Control', 'no-store');
response.setHeader('Content-Security-Policy', "default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self'");
if (!isPreviewHost(request.headers.host, server.address().port) || !request.url.startsWith(base)) {
response.writeHead(404);
response.end();
return;
}
const route = request.url.slice(base.length);
try {
if (request.method === 'GET' && route === '') {
response.setHeader('Content-Type', 'text/html; charset=utf-8');
response.end(html);
} else if (request.method === 'GET' && route === 'app.js') {
response.setHeader('Content-Type', 'application/javascript; charset=utf-8');
response.end(PAGE_SCRIPT);
} else if (request.method === 'GET' && route === 'frame.png') {
response.setHeader('Content-Type', 'image/png');
response.end(frameBytes);
} else if (request.method === 'POST' && route === 'cancel') {
response.end('ok');
finish({ outcome: 'cancelled' });
} else if (request.method === 'POST' && route === 'send') {
const body = JSON.parse(await readBody(request));
const marks = validateMarks(body.marks, { width, height, maxMarks });
if (!marks.length) throw new TypeError('Add at least one pin before sending.');
if (!isBoundedPng(body.image)) throw new TypeError('The marked picture is not a PNG within 1 MiB.');
response.end('ok');
finish({ outcome: 'sent', marks, image: body.image });
} else {
response.writeHead(404);
response.end();
}
} catch (error) {
response.writeHead(400, { 'Content-Type': 'text/plain; charset=utf-8' });
response.end(error.message);
}
});
await new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolve);
});
const url = `http://127.0.0.1:${server.address().port}${base}`;
let timer;
const onAbort = () => finish({ outcome: 'aborted' });
if (signal?.aborted) onAbort();
else signal?.addEventListener('abort', onAbort, { once: true });
try {
try {
await open(url);
} catch (error) {
throw new Error(`Could not open the browser (${error.message}). The annotate page was ${url}`);
}
const timeout = new Promise(resolve => { timer = setTimeout(() => resolve({ outcome: 'timeout' }), waitMs); });
return { url, ...await Promise.race([finished, timeout]) };
} finally {
clearTimeout(timer);
signal?.removeEventListener('abort', onAbort);
// Let the answer to the page finish: close idle connections now and any straggler after a second.
const straggler = setTimeout(() => server.closeAllConnections?.(), 1000);
await new Promise(resolve => {
server.close(resolve);
server.closeIdleConnections?.();
});
clearTimeout(straggler);
}
}