A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(editor): editor MCP server over the automation surface (stdio)
esengine4a875fa0eeb9
diff --git a/desktop/electron/mcp/server.mjs b/desktop/electron/mcp/server.mjsnew file mode 100644index 000000000..91918bdf5--- /dev/null+++ b/desktop/electron/mcp/server.mjs@@ -0,0 +1,249 @@+/**+ * @file Editor MCP server (docs/REARCH_EDITOR_AUTOMATION.md P2).+ *+ * Exposes the editor's EditorControlSurface as MCP tools + resources so an agent+ * can open a scene, mutate it, advance frames, and read back the scene tree and+ * the rendered viewport. It is a TRANSPORT ADAPTER over the surface, not a+ * parallel API: it hosts the proven headless render host (static-serves dist/,+ * opens a show:false window on headless.html) and maps each MCP call to+ * `window.__estellaHeadless.api.<method>` via executeJavaScript — the same path+ * the verify runner uses.+ *+ * Transport: stdio (an agent spawns this as a subprocess). stdout carries ONLY+ * JSON-RPC; every log goes to stderr so the protocol stream stays clean.+ *+ * Permission: reads + observation always allowed; mutations are gated behind+ * ESTELLA_MCP_ALLOW_WRITES so an agent can't silently rewrite a project.+ */+import { app, BrowserWindow } from 'electron';+import http from 'node:http';+import { readFile } from 'node:fs/promises';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { Server } from '@modelcontextprotocol/sdk/server/index.js';+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';+import {+ ListToolsRequestSchema,+ CallToolRequestSchema,+ ListResourcesRequestSchema,+ ReadResourceRequestSchema,+} from '@modelcontextprotocol/sdk/types.js';++const DIST = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'dist');+const ALLOW_WRITES = process.env.ESTELLA_MCP_ALLOW_WRITES === '1';+const W = Number(process.env.ESTELLA_MCP_W) || 1280;+const H = Number(process.env.ESTELLA_MCP_H) || 720;++// stdout is the JSON-RPC channel — keep ALL diagnostics on stderr.+const log = (...a) => process.stderr.write('[mcp] ' + a.join(' ') + '\n');+app.commandLine.appendSwitch('enable-unsafe-swiftshader');+process.env.ELECTRON_DISABLE_SECURITY_WARNINGS = 'true';++const MIME = {+ '.html': 'text/html', '.js': 'text/javascript', '.mjs': 'text/javascript',+ '.css': 'text/css', '.json': 'application/json', '.esscene': 'application/json',+ '.wasm': 'application/wasm', '.png': 'image/png', '.jpg': 'image/jpeg',+ '.webp': 'image/webp', '.woff': 'font/woff', '.woff2': 'font/woff2', '.svg': 'image/svg+xml',+};++function serveDist() {+ const server = http.createServer(async (req, res) => {+ try {+ const rel = decodeURIComponent(new URL(req.url, 'http://x').pathname).replace(/^\/+/, '') || 'index.html';+ const abs = path.join(DIST, rel);+ if (!abs.startsWith(DIST)) {+ res.writeHead(403).end();+ return;+ }+ const bytes = await readFile(abs);+ res.writeHead(200, { 'content-type': MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' });+ res.end(bytes);+ } catch {+ res.writeHead(404).end('not found');+ }+ });+ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server)));+}++// Render the viewport to a top-down PNG (base64) for an MCP image content block.+// Runs in the renderer (has document): captureViewport returns bottom-up GL rows,+// so flip Y into a 2D canvas before toDataURL.+const CAPTURE_PNG = `(() => {+ const c = window.__estellaHeadless.api.captureViewport();+ const { rgba, width, height } = c;+ const cv = document.createElement('canvas'); cv.width = width; cv.height = height;+ const ctx = cv.getContext('2d');+ const img = ctx.createImageData(width, height);+ for (let y = 0; y < height; y++) {+ const src = (height - 1 - y) * width * 4;+ img.data.set(rgba.subarray(src, src + width * 4), y * width * 4);+ }+ ctx.putImageData(img, 0, 0);+ return cv.toDataURL('image/png').split(',')[1];+})()`;++const text = (t) => ({ content: [{ type: 'text', text: String(t) }] });+const errorResult = (msg) => ({ content: [{ type: 'text', text: msg }], isError: true });++// — Tool catalog. Each maps to a surface call run in the renderer. `write` tools+// need ESTELLA_MCP_ALLOW_WRITES. Prescriptive descriptions state when to call. —+const TOOLS = [+ {+ name: 'editor_load_scene',+ description: 'Load a scene (.esscene) into the editor World, resolving asset refs via the texture manifest. Call this first to set up a known scene before stepping or capturing.',+ inputSchema: {+ type: 'object', additionalProperties: false, required: ['scene'],+ properties: {+ scene: { type: 'string', description: 'Scene URL served by the editor, e.g. /scenes/sprite-rendering.esscene' },+ manifest: { type: 'string', description: 'Optional uuid→texture manifest URL' },+ },+ },+ run: (exec, a) =>+ exec(`window.__estellaHeadless.api.loadScene(${JSON.stringify(a.scene)}${a.manifest ? `, ${JSON.stringify(a.manifest)}` : ''})`)+ .then((n) => text(`loaded ${n} entities`)),+ },+ {+ name: 'editor_step',+ description: 'Advance the engine by N fixed-delta frames (no wall-clock) so a following capture is reproducible. Call after loading a scene and before capturing.',+ inputSchema: {+ type: 'object', additionalProperties: false,+ properties: {+ frames: { type: 'integer', minimum: 1, description: 'Frames to advance (default 1)' },+ dt: { type: 'number', description: 'Seconds per frame (default 1/60)' },+ },+ },+ run: (exec, a) =>+ exec(`window.__estellaHeadless.api.step(${Number(a.frames) || 1}, ${Number(a.dt) || 1 / 60})`).then(() => text('ok')),+ },+ {+ name: 'editor_get_scene_tree',+ description: 'Read the scene outliner tree (entity ids, names, kinds, hierarchy) as JSON.',+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },+ run: (exec) => exec(`JSON.stringify(window.__estellaHeadless.api.getSceneTree())`).then(text),+ },+ {+ name: 'editor_get_inspector',+ description: 'Read the full editable inspector model (components + fields) for one entity, as JSON.',+ inputSchema: {+ type: 'object', additionalProperties: false, required: ['entity'],+ properties: { entity: { type: 'integer', description: 'Entity id from the scene tree' } },+ },+ run: (exec, a) => exec(`JSON.stringify(window.__estellaHeadless.api.getInspector(${Number(a.entity)}))`).then(text),+ },+ {+ name: 'editor_get_stats',+ description: 'Read live engine stats (entity count) as JSON.',+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },+ run: (exec) => exec(`JSON.stringify(window.__estellaHeadless.api.getStats())`).then(text),+ },+ {+ name: 'editor_capture_viewport',+ description: 'Render the current viewport and return it as a PNG image, so you can SEE what the editor drew. Step the scene first for a settled frame.',+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },+ run: (exec) => exec(CAPTURE_PNG).then((b64) => ({ content: [{ type: 'image', data: b64, mimeType: 'image/png' }] })),+ },+ {+ name: 'editor_add_entity',+ description: 'Spawn a new empty entity (with a Transform). Returns its id. Undoable. Requires write permission.',+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },+ write: true,+ run: (exec) => exec(`window.__estellaHeadless.api.addEntity()`).then((id) => text(`entity ${id}`)),+ },+ {+ name: 'editor_set_field',+ description: 'Write one inspector field on a component (e.g. Transform.position). Undoable. Requires write permission.',+ inputSchema: {+ type: 'object', additionalProperties: false,+ required: ['entity', 'component', 'key', 'type', 'value'],+ properties: {+ entity: { type: 'integer' },+ component: { type: 'string', description: 'Component name, e.g. Transform' },+ key: { type: 'string', description: 'Field key, e.g. position' },+ type: { type: 'string', enum: ['number', 'bool', 'string', 'vec2', 'vec3', 'angle', 'color'] },+ value: { description: 'Value matching the field type (vec2/vec3 = number array, color = hex string)' },+ },+ },+ write: true,+ run: (exec, a) =>+ exec(`window.__estellaHeadless.api.setField(${Number(a.entity)}, ${JSON.stringify(a.component)}, ${JSON.stringify(a.key)}, ${JSON.stringify(a.type)}, ${JSON.stringify(a.value)})`).then(() => text('ok')),+ },+];++const RESOURCES = [+ { uri: 'editor://scene/tree', name: 'Scene tree', mimeType: 'application/json', code: `JSON.stringify(window.__estellaHeadless.api.getSceneTree())` },+ { uri: 'editor://stats', name: 'Engine stats', mimeType: 'application/json', code: `JSON.stringify(window.__estellaHeadless.api.getStats())` },+];++function buildMcpServer(exec) {+ const server = new Server(+ { name: 'estella-editor', version: '0.1.0' },+ { capabilities: { tools: {}, resources: {} } },+ );++ server.setRequestHandler(ListToolsRequestSchema, async () => ({+ tools: TOOLS.filter((t) => !t.write || ALLOW_WRITES).map(({ name, description, inputSchema }) => ({ name, description, inputSchema })),+ }));++ server.setRequestHandler(CallToolRequestSchema, async (req) => {+ const tool = TOOLS.find((t) => t.name === req.params.name);+ if (!tool) return errorResult(`unknown tool: ${req.params.name}`);+ if (tool.write && !ALLOW_WRITES) return errorResult(`tool ${tool.name} needs write permission (set ESTELLA_MCP_ALLOW_WRITES=1)`);+ try {+ return await tool.run(exec, req.params.arguments ?? {});+ } catch (e) {+ return errorResult(`${tool.name} failed: ${String((e && e.message) || e)}`);+ }+ });++ server.setRequestHandler(ListResourcesRequestSchema, async () => ({+ resources: RESOURCES.map(({ uri, name, mimeType }) => ({ uri, name, mimeType })),+ }));++ server.setRequestHandler(ReadResourceRequestSchema, async (req) => {+ const r = RESOURCES.find((x) => x.uri === req.params.uri);+ if (!r) throw new Error(`unknown resource: ${req.params.uri}`);+ const t = await exec(r.code);+ return { contents: [{ uri: r.uri, mimeType: r.mimeType, text: String(t) }] };+ });++ return server;+}++app.whenReady().then(async () => {+ // Boot the headless host in the background; the MCP transport connects first so+ // the handshake / tools.list answer instantly, and tool calls await the host.+ let win = null;+ const hostReady = (async () => {+ const httpServer = await serveDist();+ const url = `http://127.0.0.1:${httpServer.address().port}/headless.html?w=${W}&h=${H}`;+ win = new BrowserWindow({ show: false, width: W, height: H, webPreferences: { offscreen: false } });+ win.webContents.on('console-message', (...args) => {+ const msg = args.map((x) => (x && typeof x === 'object' ? x.message ?? '' : String(x))).join(' ');+ if (/error|fail|unwind|exception/i.test(msg)) log('[renderer]', msg.slice(0, 200));+ });+ await win.loadURL(url);+ await win.webContents.executeJavaScript('window.__estellaHeadless.ready', true);+ log(`headless host ready (writes ${ALLOW_WRITES ? 'ENABLED' : 'disabled'})`);+ return win;+ })();+ hostReady.catch((e) => log('host boot failed:', String((e && e.stack) || e)));++ const exec = async (code) => {+ const w = await hostReady;+ return w.webContents.executeJavaScript(code, true);+ };++ try {+ await buildMcpServer(exec).connect(new StdioServerTransport());+ log('mcp server connected on stdio');+ } catch (e) {+ log('fatal:', String((e && e.stack) || e));+ app.quit();+ }+});++// Keep the process alive for the stdio session; exit when stdin closes (client gone).+process.stdin.on('close', () => app.quit());+app.on('window-all-closed', () => {+ /* keep running for stdio */+});diff --git a/desktop/electron/mcp/smoke.mjs b/desktop/electron/mcp/smoke.mjsnew file mode 100644index 000000000..9246d4881--- /dev/null+++ b/desktop/electron/mcp/smoke.mjs@@ -0,0 +1,82 @@+/**+ * @file MCP smoke test (docs/REARCH_EDITOR_AUTOMATION.md P2). Run by node: spawns+ * the Electron-hosted editor MCP server over stdio (the way a real MCP+ * client would), then lists tools/resources and drives a load → step →+ * read tree → capture → mutate round-trip, asserting each. Proves the MCP+ * transport correctly exposes the editor surface end-to-end.+ */+import { Client } from '@modelcontextprotocol/sdk/client/index.js';+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';+import path from 'node:path';+import { fileURLToPath } from 'node:url';++const here = path.dirname(fileURLToPath(import.meta.url));+const electronBin = path.resolve(here, '..', '..', 'node_modules', '.bin', 'electron');+const serverPath = path.resolve(here, 'server.mjs');++const checks = [];+const check = (name, cond, detail = '') => {+ checks.push({ name, ok: !!cond });+ console.log(`${cond ? '✓' : '✗'} ${name}${detail ? ' → ' + detail : ''}`);+};++const transport = new StdioClientTransport({+ command: electronBin,+ args: [serverPath],+ env: { ...process.env, ESTELLA_MCP_ALLOW_WRITES: '1' },+ stderr: 'inherit',+});+const client = new Client({ name: 'estella-smoke', version: '0.0.0' }, { capabilities: {} });++try {+ await client.connect(transport);++ const { tools } = await client.listTools();+ const names = tools.map((t) => t.name);+ check(+ 'listTools includes core tools',+ ['editor_load_scene', 'editor_step', 'editor_capture_viewport', 'editor_get_scene_tree'].every((n) => names.includes(n)),+ names.join(','),+ );+ check('write tool exposed under ALLOW_WRITES', names.includes('editor_add_entity'));++ const load = await client.callTool({+ name: 'editor_load_scene',+ arguments: { scene: '/scenes/sprite-rendering.esscene', manifest: '/scenes/sprite-rendering.textures.json' },+ });+ check('load_scene returns an entity count', /loaded \d+ entities/.test(load.content?.[0]?.text ?? ''), load.content?.[0]?.text);++ await client.callTool({ name: 'editor_step', arguments: { frames: 30 } });++ const tree = await client.callTool({ name: 'editor_get_scene_tree' });+ let treeLen = 0;+ try {+ treeLen = JSON.parse(tree.content[0].text).length;+ } catch {+ /* leave 0 */+ }+ check('get_scene_tree returns nodes', treeLen > 0, `${treeLen} roots`);++ const cap = await client.callTool({ name: 'editor_capture_viewport' });+ const img = cap.content?.[0];+ check(+ 'capture_viewport returns a PNG image block',+ img?.type === 'image' && img?.mimeType === 'image/png' && (img?.data?.length ?? 0) > 100,+ `${img?.data?.length ?? 0} base64 chars`,+ );++ const add = await client.callTool({ name: 'editor_add_entity' });+ check('add_entity (write) spawns an entity', /entity \d+/.test(add.content?.[0]?.text ?? ''), add.content?.[0]?.text);++ const { resources } = await client.listResources();+ check('listResources includes scene tree', resources.some((r) => r.uri === 'editor://scene/tree'));+ const res = await client.readResource({ uri: 'editor://scene/tree' });+ check('readResource(scene/tree) returns json', (res.contents?.[0]?.text?.length ?? 0) > 2);+} catch (e) {+ check('connect + drive the server', false, String((e && e.stack) || e));+} finally {+ await client.close().catch(() => {});+ const failed = checks.filter((c) => !c.ok);+ console.log(`\n[mcp:smoke] ${failed.length === 0 ? 'PASS' : 'FAIL'} — ${checks.length - failed.length}/${checks.length}`);+ process.exit(failed.length === 0 ? 0 : 1);+}diff --git a/desktop/package.json b/desktop/package.jsonindex 8d18d0f1a..9e9649bf4 100644--- a/desktop/package.json+++ b/desktop/package.json@@ -12,12 +12,15 @@"typecheck": "tsc --noEmit","test": "vitest run","test:watch": "vitest",- "verify:render": "vite build && electron scripts/headless-verify.mjs"+ "verify:render": "vite build && electron scripts/headless-verify.mjs",+ "mcp": "electron electron/mcp/server.mjs",+ "mcp:smoke": "vite build && node electron/mcp/smoke.mjs"},"dependencies": {- "esengine": "workspace:*",- "esbuild": "^0.27.0",+ "@modelcontextprotocol/sdk": "^1.29.0","dockview": "^6.6.1",+ "esbuild": "^0.27.0",+ "esengine": "workspace:*","lucide-react": "^1.21.0","react": "^19.2.0","react-dom": "^19.2.0",