Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): editor MCP server over the automation surface (stdio)

esengine4a875fa0eeb9
diff --git a/desktop/electron/mcp/server.mjs b/desktop/electron/mcp/server.mjs
new file mode 100644
index 000000000..91918bdf5
--- /dev/null
+++ b/desktop/electron/mcp/server.mjs
@@ -0,0 +1,249 @@
+/**
+ * @file Editor MCP server (docs/REARCH_EDITOR_AUTOMATION.md P2).
+ *
+ * Exposes the editor's EditorControlSurface as MCP tools + resources so an agent
+ * can open a scene, mutate it, advance frames, and read back the scene tree and
+ * the rendered viewport. It is a TRANSPORT ADAPTER over the surface, not a
+ * parallel API: it hosts the proven headless render host (static-serves dist/,
+ * opens a show:false window on headless.html) and maps each MCP call to
+ * `window.__estellaHeadless.api.<method>` via executeJavaScript — the same path
+ * the verify runner uses.
+ *
+ * Transport: stdio (an agent spawns this as a subprocess). stdout carries ONLY
+ * JSON-RPC; every log goes to stderr so the protocol stream stays clean.
+ *
+ * Permission: reads + observation always allowed; mutations are gated behind
+ * ESTELLA_MCP_ALLOW_WRITES so an agent can't silently rewrite a project.
+ */
+import { app, BrowserWindow } from 'electron';
+import http from 'node:http';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import {
+ ListToolsRequestSchema,
+ CallToolRequestSchema,
+ ListResourcesRequestSchema,
+ ReadResourceRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+
+const DIST = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'dist');
+const ALLOW_WRITES = process.env.ESTELLA_MCP_ALLOW_WRITES === '1';
+const W = Number(process.env.ESTELLA_MCP_W) || 1280;
+const H = Number(process.env.ESTELLA_MCP_H) || 720;
+
+// stdout is the JSON-RPC channel — keep ALL diagnostics on stderr.
+const log = (...a) => process.stderr.write('[mcp] ' + a.join(' ') + '\n');
+app.commandLine.appendSwitch('enable-unsafe-swiftshader');
+process.env.ELECTRON_DISABLE_SECURITY_WARNINGS = 'true';
+
+const MIME = {
+ '.html': 'text/html', '.js': 'text/javascript', '.mjs': 'text/javascript',
+ '.css': 'text/css', '.json': 'application/json', '.esscene': 'application/json',
+ '.wasm': 'application/wasm', '.png': 'image/png', '.jpg': 'image/jpeg',
+ '.webp': 'image/webp', '.woff': 'font/woff', '.woff2': 'font/woff2', '.svg': 'image/svg+xml',
+};
+
+function serveDist() {
+ const server = http.createServer(async (req, res) => {
+ try {
+ const rel = decodeURIComponent(new URL(req.url, 'http://x').pathname).replace(/^\/+/, '') || 'index.html';
+ const abs = path.join(DIST, rel);
+ if (!abs.startsWith(DIST)) {
+ res.writeHead(403).end();
+ return;
+ }
+ const bytes = await readFile(abs);
+ res.writeHead(200, { 'content-type': MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' });
+ res.end(bytes);
+ } catch {
+ res.writeHead(404).end('not found');
+ }
+ });
+ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server)));
+}
+
+// Render the viewport to a top-down PNG (base64) for an MCP image content block.
+// Runs in the renderer (has document): captureViewport returns bottom-up GL rows,
+// so flip Y into a 2D canvas before toDataURL.
+const CAPTURE_PNG = `(() => {
+ const c = window.__estellaHeadless.api.captureViewport();
+ const { rgba, width, height } = c;
+ const cv = document.createElement('canvas'); cv.width = width; cv.height = height;
+ const ctx = cv.getContext('2d');
+ const img = ctx.createImageData(width, height);
+ for (let y = 0; y < height; y++) {
+ const src = (height - 1 - y) * width * 4;
+ img.data.set(rgba.subarray(src, src + width * 4), y * width * 4);
+ }
+ ctx.putImageData(img, 0, 0);
+ return cv.toDataURL('image/png').split(',')[1];
+})()`;
+
+const text = (t) => ({ content: [{ type: 'text', text: String(t) }] });
+const errorResult = (msg) => ({ content: [{ type: 'text', text: msg }], isError: true });
+
+// — Tool catalog. Each maps to a surface call run in the renderer. `write` tools
+// need ESTELLA_MCP_ALLOW_WRITES. Prescriptive descriptions state when to call. —
+const TOOLS = [
+ {
+ name: 'editor_load_scene',
+ description: 'Load a scene (.esscene) into the editor World, resolving asset refs via the texture manifest. Call this first to set up a known scene before stepping or capturing.',
+ inputSchema: {
+ type: 'object', additionalProperties: false, required: ['scene'],
+ properties: {
+ scene: { type: 'string', description: 'Scene URL served by the editor, e.g. /scenes/sprite-rendering.esscene' },
+ manifest: { type: 'string', description: 'Optional uuid→texture manifest URL' },
+ },
+ },
+ run: (exec, a) =>
+ exec(`window.__estellaHeadless.api.loadScene(${JSON.stringify(a.scene)}${a.manifest ? `, ${JSON.stringify(a.manifest)}` : ''})`)
+ .then((n) => text(`loaded ${n} entities`)),
+ },
+ {
+ name: 'editor_step',
+ description: 'Advance the engine by N fixed-delta frames (no wall-clock) so a following capture is reproducible. Call after loading a scene and before capturing.',
+ inputSchema: {
+ type: 'object', additionalProperties: false,
+ properties: {
+ frames: { type: 'integer', minimum: 1, description: 'Frames to advance (default 1)' },
+ dt: { type: 'number', description: 'Seconds per frame (default 1/60)' },
+ },
+ },
+ run: (exec, a) =>
+ exec(`window.__estellaHeadless.api.step(${Number(a.frames) || 1}, ${Number(a.dt) || 1 / 60})`).then(() => text('ok')),
+ },
+ {
+ name: 'editor_get_scene_tree',
+ description: 'Read the scene outliner tree (entity ids, names, kinds, hierarchy) as JSON.',
+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },
+ run: (exec) => exec(`JSON.stringify(window.__estellaHeadless.api.getSceneTree())`).then(text),
+ },
+ {
+ name: 'editor_get_inspector',
+ description: 'Read the full editable inspector model (components + fields) for one entity, as JSON.',
+ inputSchema: {
+ type: 'object', additionalProperties: false, required: ['entity'],
+ properties: { entity: { type: 'integer', description: 'Entity id from the scene tree' } },
+ },
+ run: (exec, a) => exec(`JSON.stringify(window.__estellaHeadless.api.getInspector(${Number(a.entity)}))`).then(text),
+ },
+ {
+ name: 'editor_get_stats',
+ description: 'Read live engine stats (entity count) as JSON.',
+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },
+ run: (exec) => exec(`JSON.stringify(window.__estellaHeadless.api.getStats())`).then(text),
+ },
+ {
+ name: 'editor_capture_viewport',
+ description: 'Render the current viewport and return it as a PNG image, so you can SEE what the editor drew. Step the scene first for a settled frame.',
+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },
+ run: (exec) => exec(CAPTURE_PNG).then((b64) => ({ content: [{ type: 'image', data: b64, mimeType: 'image/png' }] })),
+ },
+ {
+ name: 'editor_add_entity',
+ description: 'Spawn a new empty entity (with a Transform). Returns its id. Undoable. Requires write permission.',
+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },
+ write: true,
+ run: (exec) => exec(`window.__estellaHeadless.api.addEntity()`).then((id) => text(`entity ${id}`)),
+ },
+ {
+ name: 'editor_set_field',
+ description: 'Write one inspector field on a component (e.g. Transform.position). Undoable. Requires write permission.',
+ inputSchema: {
+ type: 'object', additionalProperties: false,
+ required: ['entity', 'component', 'key', 'type', 'value'],
+ properties: {
+ entity: { type: 'integer' },
+ component: { type: 'string', description: 'Component name, e.g. Transform' },
+ key: { type: 'string', description: 'Field key, e.g. position' },
+ type: { type: 'string', enum: ['number', 'bool', 'string', 'vec2', 'vec3', 'angle', 'color'] },
+ value: { description: 'Value matching the field type (vec2/vec3 = number array, color = hex string)' },
+ },
+ },
+ write: true,
+ run: (exec, a) =>
+ exec(`window.__estellaHeadless.api.setField(${Number(a.entity)}, ${JSON.stringify(a.component)}, ${JSON.stringify(a.key)}, ${JSON.stringify(a.type)}, ${JSON.stringify(a.value)})`).then(() => text('ok')),
+ },
+];
+
+const RESOURCES = [
+ { uri: 'editor://scene/tree', name: 'Scene tree', mimeType: 'application/json', code: `JSON.stringify(window.__estellaHeadless.api.getSceneTree())` },
+ { uri: 'editor://stats', name: 'Engine stats', mimeType: 'application/json', code: `JSON.stringify(window.__estellaHeadless.api.getStats())` },
+];
+
+function buildMcpServer(exec) {
+ const server = new Server(
+ { name: 'estella-editor', version: '0.1.0' },
+ { capabilities: { tools: {}, resources: {} } },
+ );
+
+ server.setRequestHandler(ListToolsRequestSchema, async () => ({
+ tools: TOOLS.filter((t) => !t.write || ALLOW_WRITES).map(({ name, description, inputSchema }) => ({ name, description, inputSchema })),
+ }));
+
+ server.setRequestHandler(CallToolRequestSchema, async (req) => {
+ const tool = TOOLS.find((t) => t.name === req.params.name);
+ if (!tool) return errorResult(`unknown tool: ${req.params.name}`);
+ if (tool.write && !ALLOW_WRITES) return errorResult(`tool ${tool.name} needs write permission (set ESTELLA_MCP_ALLOW_WRITES=1)`);
+ try {
+ return await tool.run(exec, req.params.arguments ?? {});
+ } catch (e) {
+ return errorResult(`${tool.name} failed: ${String((e && e.message) || e)}`);
+ }
+ });
+
+ server.setRequestHandler(ListResourcesRequestSchema, async () => ({
+ resources: RESOURCES.map(({ uri, name, mimeType }) => ({ uri, name, mimeType })),
+ }));
+
+ server.setRequestHandler(ReadResourceRequestSchema, async (req) => {
+ const r = RESOURCES.find((x) => x.uri === req.params.uri);
+ if (!r) throw new Error(`unknown resource: ${req.params.uri}`);
+ const t = await exec(r.code);
+ return { contents: [{ uri: r.uri, mimeType: r.mimeType, text: String(t) }] };
+ });
+
+ return server;
+}
+
+app.whenReady().then(async () => {
+ // Boot the headless host in the background; the MCP transport connects first so
+ // the handshake / tools.list answer instantly, and tool calls await the host.
+ let win = null;
+ const hostReady = (async () => {
+ const httpServer = await serveDist();
+ const url = `http://127.0.0.1:${httpServer.address().port}/headless.html?w=${W}&h=${H}`;
+ win = new BrowserWindow({ show: false, width: W, height: H, webPreferences: { offscreen: false } });
+ win.webContents.on('console-message', (...args) => {
+ const msg = args.map((x) => (x && typeof x === 'object' ? x.message ?? '' : String(x))).join(' ');
+ if (/error|fail|unwind|exception/i.test(msg)) log('[renderer]', msg.slice(0, 200));
+ });
+ await win.loadURL(url);
+ await win.webContents.executeJavaScript('window.__estellaHeadless.ready', true);
+ log(`headless host ready (writes ${ALLOW_WRITES ? 'ENABLED' : 'disabled'})`);
+ return win;
+ })();
+ hostReady.catch((e) => log('host boot failed:', String((e && e.stack) || e)));
+
+ const exec = async (code) => {
+ const w = await hostReady;
+ return w.webContents.executeJavaScript(code, true);
+ };
+
+ try {
+ await buildMcpServer(exec).connect(new StdioServerTransport());
+ log('mcp server connected on stdio');
+ } catch (e) {
+ log('fatal:', String((e && e.stack) || e));
+ app.quit();
+ }
+});
+
+// Keep the process alive for the stdio session; exit when stdin closes (client gone).
+process.stdin.on('close', () => app.quit());
+app.on('window-all-closed', () => {
+ /* keep running for stdio */
+});
diff --git a/desktop/electron/mcp/smoke.mjs b/desktop/electron/mcp/smoke.mjs
new file mode 100644
index 000000000..9246d4881
--- /dev/null
+++ b/desktop/electron/mcp/smoke.mjs
@@ -0,0 +1,82 @@
+/**
+ * @file MCP smoke test (docs/REARCH_EDITOR_AUTOMATION.md P2). Run by node: spawns
+ * the Electron-hosted editor MCP server over stdio (the way a real MCP
+ * client would), then lists tools/resources and drives a load → step →
+ * read tree → capture → mutate round-trip, asserting each. Proves the MCP
+ * transport correctly exposes the editor surface end-to-end.
+ */
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const electronBin = path.resolve(here, '..', '..', 'node_modules', '.bin', 'electron');
+const serverPath = path.resolve(here, 'server.mjs');
+
+const checks = [];
+const check = (name, cond, detail = '') => {
+ checks.push({ name, ok: !!cond });
+ console.log(`${cond ? '✓' : '✗'} ${name}${detail ? ' → ' + detail : ''}`);
+};
+
+const transport = new StdioClientTransport({
+ command: electronBin,
+ args: [serverPath],
+ env: { ...process.env, ESTELLA_MCP_ALLOW_WRITES: '1' },
+ stderr: 'inherit',
+});
+const client = new Client({ name: 'estella-smoke', version: '0.0.0' }, { capabilities: {} });
+
+try {
+ await client.connect(transport);
+
+ const { tools } = await client.listTools();
+ const names = tools.map((t) => t.name);
+ check(
+ 'listTools includes core tools',
+ ['editor_load_scene', 'editor_step', 'editor_capture_viewport', 'editor_get_scene_tree'].every((n) => names.includes(n)),
+ names.join(','),
+ );
+ check('write tool exposed under ALLOW_WRITES', names.includes('editor_add_entity'));
+
+ const load = await client.callTool({
+ name: 'editor_load_scene',
+ arguments: { scene: '/scenes/sprite-rendering.esscene', manifest: '/scenes/sprite-rendering.textures.json' },
+ });
+ check('load_scene returns an entity count', /loaded \d+ entities/.test(load.content?.[0]?.text ?? ''), load.content?.[0]?.text);
+
+ await client.callTool({ name: 'editor_step', arguments: { frames: 30 } });
+
+ const tree = await client.callTool({ name: 'editor_get_scene_tree' });
+ let treeLen = 0;
+ try {
+ treeLen = JSON.parse(tree.content[0].text).length;
+ } catch {
+ /* leave 0 */
+ }
+ check('get_scene_tree returns nodes', treeLen > 0, `${treeLen} roots`);
+
+ const cap = await client.callTool({ name: 'editor_capture_viewport' });
+ const img = cap.content?.[0];
+ check(
+ 'capture_viewport returns a PNG image block',
+ img?.type === 'image' && img?.mimeType === 'image/png' && (img?.data?.length ?? 0) > 100,
+ `${img?.data?.length ?? 0} base64 chars`,
+ );
+
+ const add = await client.callTool({ name: 'editor_add_entity' });
+ check('add_entity (write) spawns an entity', /entity \d+/.test(add.content?.[0]?.text ?? ''), add.content?.[0]?.text);
+
+ const { resources } = await client.listResources();
+ check('listResources includes scene tree', resources.some((r) => r.uri === 'editor://scene/tree'));
+ const res = await client.readResource({ uri: 'editor://scene/tree' });
+ check('readResource(scene/tree) returns json', (res.contents?.[0]?.text?.length ?? 0) > 2);
+} catch (e) {
+ check('connect + drive the server', false, String((e && e.stack) || e));
+} finally {
+ await client.close().catch(() => {});
+ const failed = checks.filter((c) => !c.ok);
+ console.log(`\n[mcp:smoke] ${failed.length === 0 ? 'PASS' : 'FAIL'} — ${checks.length - failed.length}/${checks.length}`);
+ process.exit(failed.length === 0 ? 0 : 1);
+}
diff --git a/desktop/package.json b/desktop/package.json
index 8d18d0f1a..9e9649bf4 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -12,12 +12,15 @@
"typecheck": "tsc --noEmit",
"test": "vitest run",
"test:watch": "vitest",
- "verify:render": "vite build && electron scripts/headless-verify.mjs"
+ "verify:render": "vite build && electron scripts/headless-verify.mjs",
+ "mcp": "electron electron/mcp/server.mjs",
+ "mcp:smoke": "vite build && node electron/mcp/smoke.mjs"
},
"dependencies": {
- "esengine": "workspace:*",
- "esbuild": "^0.27.0",
+ "@modelcontextprotocol/sdk": "^1.29.0",
"dockview": "^6.6.1",
+ "esbuild": "^0.27.0",
+ "esengine": "workspace:*",
"lucide-react": "^1.21.0",
"react": "^19.2.0",
"react-dom": "^19.2.0",