Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): MCP server over EditorControlSurface (headless host)

esengineaf56c695dddd
diff --git a/desktop/package.json b/desktop/package.json
index 28216b463..0be242d3f 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -14,6 +14,9 @@
"test": "vitest run",
"test:watch": "vitest",
"verify:render": "vite build && electron scripts/headless-verify.mjs",
+ "editor:mcp": "electron scripts/editor-mcp.mjs",
+ "editor:mcp:build": "vite build && electron scripts/editor-mcp.mjs",
+ "editor:mcp:e2e": "vite build && node scripts/editor-mcp-e2e.mjs",
"verify:render:spine": "vite build && ESTELLA_VERIFY_SCENE=/scenes/spine-test.esscene ESTELLA_VERIFY_MANIFEST=/scenes/spine-test.textures.json electron scripts/headless-verify.mjs",
"verify:render:particles": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SCENE=/scenes/particle-demo.esscene ESTELLA_VERIFY_MANIFEST=/scenes/particle-demo.textures.json ESTELLA_VERIFY_STEPS=90 electron scripts/headless-verify.mjs",
"verify:render:particles:gradient": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SCENE=/scenes/particle-gradient.esscene ESTELLA_VERIFY_MANIFEST=/scenes/particle-gradient.textures.json ESTELLA_VERIFY_W=256 ESTELLA_VERIFY_H=256 ESTELLA_VERIFY_STEPS=20 ESTELLA_VERIFY_EXPECT=\"[{\\\"x\\\":0.5,\\\"y\\\":0.5,\\\"rgb\\\":[0,255,0],\\\"tol\\\":90}]\" electron scripts/headless-verify.mjs",
diff --git a/desktop/scripts/editor-mcp-e2e.mjs b/desktop/scripts/editor-mcp-e2e.mjs
new file mode 100644
index 000000000..82b7eefb6
--- /dev/null
+++ b/desktop/scripts/editor-mcp-e2e.mjs
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file editor-mcp-e2e.mjs — end-to-end smoke for the editor MCP server.
+ * Spawns editor-mcp.mjs and drives the MCP stdio handshake (initialize →
+ * tools/list → tools/call), exercising the full path: stdio transport → SDK
+ * Server → executeJavaScript driver → EditorControlSurface → engine. Run from
+ * desktop/ after a dist build: pnpm editor:mcp:e2e
+ */
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const DESKTOP = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const electron = path.join(DESKTOP, 'node_modules', '.bin', 'electron');
+
+const child = spawn(electron, ['scripts/editor-mcp.mjs'], { cwd: DESKTOP, stdio: ['pipe', 'pipe', 'pipe'] });
+
+let buf = '';
+const waiters = new Map();
+child.stdout.on('data', (d) => {
+ buf += d.toString();
+ let nl;
+ while ((nl = buf.indexOf('\n')) >= 0) {
+ const line = buf.slice(0, nl).trim();
+ buf = buf.slice(nl + 1);
+ if (!line) continue;
+ let msg;
+ try { msg = JSON.parse(line); } catch { continue; } // skip stray non-JSON stdout
+ if (msg.id != null && waiters.has(msg.id)) { waiters.get(msg.id)(msg); waiters.delete(msg.id); }
+ }
+});
+child.stderr.on('data', (d) => process.stderr.write(d));
+
+const send = (msg) => child.stdin.write(JSON.stringify(msg) + '\n');
+const rpc = (id, method, params) => new Promise((resolve, reject) => {
+ const t = setTimeout(() => reject(new Error(`timeout waiting for ${method}`)), 30000);
+ waiters.set(id, (m) => { clearTimeout(t); resolve(m); });
+ send({ jsonrpc: '2.0', id, method, params });
+});
+const fail = (m) => { console.log('E2E FAIL:', m); child.kill(); process.exit(1); };
+
+try {
+ const init = await rpc(1, 'initialize', {
+ protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'e2e', version: '0' },
+ });
+ if (!init.result?.serverInfo) fail('no serverInfo in initialize');
+ console.log('initialize OK — server:', init.result.serverInfo.name);
+ send({ jsonrpc: '2.0', method: 'notifications/initialized' });
+
+ const list = await rpc(2, 'tools/list', {});
+ const names = (list.result?.tools ?? []).map((t) => t.name);
+ if (!names.includes('load_scene') || !names.includes('get_scene_tree')) fail(`tools/list missing tools: ${names}`);
+ console.log(`tools/list OK — ${names.length} tools`);
+
+ const load = await rpc(3, 'tools/call', {
+ name: 'load_scene',
+ arguments: { sceneUrl: '/scenes/sprite-rendering.esscene', manifestUrl: '/scenes/sprite-rendering.textures.json' },
+ });
+ if (load.result?.isError) fail(`load_scene error: ${load.result?.content?.[0]?.text}`);
+ console.log('load_scene OK — entityCount:', load.result?.content?.[0]?.text);
+
+ const tree = await rpc(4, 'tools/call', { name: 'get_scene_tree', arguments: {} });
+ if (tree.result?.isError) fail(`get_scene_tree error: ${tree.result?.content?.[0]?.text}`);
+ const parsed = JSON.parse(tree.result?.content?.[0]?.text ?? 'null');
+ if (!Array.isArray(parsed) || parsed.length === 0) fail('get_scene_tree returned no nodes');
+ console.log(`get_scene_tree OK — ${parsed.length} root nodes`);
+
+ console.log('\nE2E PASS — MCP server drives EditorControlSurface end-to-end');
+ child.kill();
+ process.exit(0);
+} catch (e) {
+ fail(String(e?.message ?? e));
+}
diff --git a/desktop/scripts/editor-mcp-tools.mjs b/desktop/scripts/editor-mcp-tools.mjs
new file mode 100644
index 000000000..35224377c
--- /dev/null
+++ b/desktop/scripts/editor-mcp-tools.mjs
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file editor-mcp-tools.mjs
+ * The editor MCP tool registry. Each tool maps 1:1 to an EditorControlSurface
+ * method, so the MCP server adds NO new editor truth — it is a transport over
+ * the surface (exactly what EditorControlSurface.ts:7-9 anticipated). Kept
+ * dependency-free (JSON-Schema + light manual validation, no zod import) so the
+ * dispatch unit-tests without Electron and resolves under vite/vitest; the
+ * Electron entry (editor-mcp.mjs) supplies the executeJavaScript driver and the
+ * MCP SDK wiring. See docs/REARCH_EDITOR_ARCH.md §11.
+ */
+
+const obj = (properties, required = []) => ({ type: 'object', properties, required });
+
+/** name → { description, schema (JSON Schema), method (surface), args(input)→[] }. */
+export const TOOLS = [
+ { name: 'load_scene',
+ description: 'Load a scene (and optional asset manifest) into the headless World; returns the spawned entity count.',
+ schema: obj({ sceneUrl: { type: 'string' }, manifestUrl: { type: 'string' } }, ['sceneUrl']),
+ method: 'loadScene', args: (i) => [i.sceneUrl, i.manifestUrl] },
+ { name: 'get_scene_tree',
+ description: 'The scene entity tree (id, name, kind, children).',
+ schema: obj({}), method: 'getSceneTree', args: () => [] },
+ { name: 'get_entity',
+ description: 'One entity: name, kind, and its component list.',
+ schema: obj({ id: { type: 'number' } }, ['id']), method: 'getEntity', args: (i) => [i.id] },
+ { name: 'get_inspector',
+ description: "An entity's components and fields — the Details-panel data.",
+ schema: obj({ entity: { type: 'number' } }, ['entity']), method: 'getInspector', args: (i) => [i.entity] },
+ { name: 'serialize_scene',
+ description: 'The full lossless scene JSON (the model truth).',
+ schema: obj({}), method: 'serializeScene', args: () => [] },
+ { name: 'add_entity',
+ description: 'Create a new entity; returns its id.',
+ schema: obj({}), method: 'addEntity', args: () => [] },
+ { name: 'delete_entity',
+ description: 'Delete an entity.',
+ schema: obj({ id: { type: 'number' } }, ['id']), method: 'deleteEntity', args: (i) => [i.id] },
+ { name: 'rename_entity',
+ description: 'Rename an entity.',
+ schema: obj({ id: { type: 'number' }, name: { type: 'string' } }, ['id', 'name']),
+ method: 'renameEntity', args: (i) => [i.id, i.name] },
+ { name: 'set_field',
+ description: 'Set a component field (undoable). `type` is the inspector field type (e.g. float, int, bool, string, vec2, vec3, color, enum).',
+ schema: obj({
+ entity: { type: 'number' }, component: { type: 'string' }, key: { type: 'string' },
+ type: { type: 'string' }, value: {},
+ }, ['entity', 'component', 'key', 'type', 'value']),
+ method: 'setField', args: (i) => [i.entity, i.component, i.key, i.type, i.value] },
+ { name: 'set_parent',
+ description: 'Re-parent an entity in the transform hierarchy (parent=null → scene root).',
+ schema: obj({ id: { type: 'number' }, parent: { type: ['number', 'null'] } }, ['id', 'parent']),
+ method: 'setParent', args: (i) => [i.id, i.parent] },
+ { name: 'select',
+ description: 'Select an entity (id=null clears the selection).',
+ schema: obj({ id: { type: ['number', 'null'] } }, ['id']), method: 'select', args: (i) => [i.id] },
+ { name: 'get_selection',
+ description: 'The primary selected entity id, or null.',
+ schema: obj({}), method: 'getSelection', args: () => [] },
+ { name: 'set_run_mode',
+ description: 'Enter or leave play mode (playing=true runs gameplay; Stop rebuilds the edit World).',
+ schema: obj({ playing: { type: 'boolean' }, paused: { type: 'boolean' } }, ['playing']),
+ method: 'setRunMode', args: (i) => [i.playing, i.paused] },
+ { name: 'step',
+ description: 'Advance the engine by N fixed-dt frames deterministically (no rAF).',
+ schema: obj({ frames: { type: 'number' }, dt: { type: 'number' } }),
+ method: 'step', args: (i) => [i.frames, i.dt] },
+ { name: 'get_stats',
+ description: 'Live counts: entity count + last frame draw calls (cheap render evidence).',
+ schema: obj({}), method: 'getStats', args: () => [] },
+ { name: 'undo', description: 'Undo the last edit.', schema: obj({}), method: 'undo', args: () => [] },
+ { name: 'redo', description: 'Redo the last undone edit.', schema: obj({}), method: 'redo', args: () => [] },
+];
+
+/** The MCP `tools/list` payload — name, description, JSON-Schema inputSchema. */
+export function listTools() {
+ return TOOLS.map((t) => ({ name: t.name, description: t.description, inputSchema: t.schema }));
+}
+
+function typeMatches(spec, val) {
+ return (Array.isArray(spec) ? spec : [spec]).some((t) =>
+ t === 'null' ? val === null
+ : t === 'number' ? typeof val === 'number'
+ : t === 'string' ? typeof val === 'string'
+ : t === 'boolean' ? typeof val === 'boolean'
+ : t === 'object' ? (val !== null && typeof val === 'object')
+ : true);
+}
+
+/** Light validation: required args present + declared scalar types match. */
+function validate(schema, raw) {
+ const input = raw ?? {};
+ for (const req of schema.required ?? []) {
+ if (input[req] === undefined) throw new Error(`missing required argument: ${req}`);
+ }
+ for (const [key, spec] of Object.entries(schema.properties ?? {})) {
+ if (input[key] !== undefined && spec.type && !typeMatches(spec.type, input[key])) {
+ throw new Error(`argument "${key}" must be ${[].concat(spec.type).join(' | ')}`);
+ }
+ }
+ return input;
+}
+
+/**
+ * Validate `rawInput`, invoke the surface via `driver(method, args)`, and wrap the
+ * result as an MCP CallToolResult. A validation failure or a driver throw becomes an
+ * `isError` result rather than crashing the server.
+ */
+export async function runTool(tool, driver, rawInput) {
+ try {
+ const input = validate(tool.schema, rawInput);
+ const result = await driver(tool.method, tool.args(input));
+ return { content: [{ type: 'text', text: result === undefined ? 'ok' : JSON.stringify(result) }] };
+ } catch (err) {
+ return { content: [{ type: 'text', text: `error: ${err?.message ?? String(err)}` }], isError: true };
+ }
+}
diff --git a/desktop/scripts/editor-mcp.mjs b/desktop/scripts/editor-mcp.mjs
new file mode 100644
index 000000000..166869b73
--- /dev/null
+++ b/desktop/scripts/editor-mcp.mjs
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file editor-mcp.mjs — the editor MCP server (headless host).
+ *
+ * An Electron entry that boots the SAME headless render host the render-verify runner
+ * uses (serves dist/, opens headless.html, which publishes EditorControlSurface on
+ * `window.__estellaHeadless`), then runs an MCP server over stdio whose tools are the
+ * surface methods (see editor-mcp-tools.mjs). So an MCP client (an AI agent) can drive
+ * the real editor surface headlessly — load/inspect/edit scenes, step, read stats —
+ * with no live Electron UI. The surface stays the single source of truth; this is a
+ * transport over it (EditorControlSurface.ts:7-9). Requires a built dist/ (vite build).
+ *
+ * stdout is the MCP JSON-RPC channel — ALL logging goes to stderr.
+ */
+import { app, BrowserWindow } from 'electron';
+import http from 'node:http';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { TOOLS, listTools, runTool } from './editor-mcp-tools.mjs';
+
+const DIST = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'dist');
+const W = Number(process.env.ESTELLA_MCP_W) || 1280;
+const H = Number(process.env.ESTELLA_MCP_H) || 720;
+
+app.commandLine.appendSwitch('enable-unsafe-swiftshader'); // GPU-less WebGL2 fallback
+process.env.ELECTRON_DISABLE_SECURITY_WARNINGS = 'true';
+const log = (...a) => process.stderr.write(`[editor-mcp] ${a.join(' ')}\n`);
+
+const MIME = {
+ '.html': 'text/html', '.js': 'text/javascript', '.mjs': 'text/javascript',
+ '.css': 'text/css', '.json': 'application/json', '.esscene': 'application/json',
+ '.wasm': 'application/wasm', '.png': 'image/png', '.jpg': 'image/jpeg',
+ '.webp': 'image/webp', '.woff': 'font/woff', '.woff2': 'font/woff2', '.svg': 'image/svg+xml',
+};
+
+function serveDist() {
+ const server = http.createServer(async (req, res) => {
+ try {
+ const rel = decodeURIComponent(new URL(req.url, 'http://x').pathname).replace(/^\/+/, '') || 'index.html';
+ const abs = path.join(DIST, rel);
+ if (!abs.startsWith(DIST)) { res.writeHead(403).end(); return; }
+ const bytes = await readFile(abs);
+ res.writeHead(200, { 'content-type': MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' });
+ res.end(bytes);
+ } catch { res.writeHead(404).end('not found'); }
+ });
+ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server)));
+}
+
+// Keep the process alive on the hidden window (this is a long-running server).
+app.on('window-all-closed', () => {});
+
+app.whenReady().then(async () => {
+ try {
+ const server = await serveDist();
+ const url = `http://127.0.0.1:${server.address().port}/headless.html?w=${W}&h=${H}`;
+ const win = new BrowserWindow({ show: false, width: W, height: H, webPreferences: { offscreen: false } });
+ win.webContents.on('console-message', (...args) => {
+ const msg = args.map((a) => (a && typeof a === 'object' ? a.message ?? '' : String(a))).join(' ');
+ if (/error|fail|unwind|exception|webgl/i.test(msg)) log('[renderer]', msg.slice(0, 240));
+ });
+ await win.loadURL(url);
+ await win.webContents.executeJavaScript('window.__estellaHeadless.ready', true);
+ log('headless engine ready');
+
+ // Marshal a surface call into the headless renderer. `undefined` args become the JS
+ // `undefined` literal so the surface's default parameters apply.
+ const driver = (method, args) =>
+ win.webContents.executeJavaScript(
+ `window.__estellaHeadless.api.${method}(${(args ?? [])
+ .map((a) => (a === undefined ? 'undefined' : JSON.stringify(a)))
+ .join(',')})`,
+ true,
+ );
+
+ const mcp = new Server({ name: 'estella-editor', version: '0.1.0' }, { capabilities: { tools: {} } });
+ mcp.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: listTools() }));
+ mcp.setRequestHandler(CallToolRequestSchema, async (req) => {
+ const tool = TOOLS.find((t) => t.name === req.params.name);
+ if (!tool) return { content: [{ type: 'text', text: `unknown tool: ${req.params.name}` }], isError: true };
+ return runTool(tool, driver, req.params.arguments);
+ });
+ await mcp.connect(new StdioServerTransport());
+ log('MCP server connected over stdio');
+ } catch (e) {
+ log('FATAL', String((e && e.stack) || e));
+ app.exit(1);
+ }
+});
diff --git a/desktop/tests/editor-mcp-tools.test.ts b/desktop/tests/editor-mcp-tools.test.ts
new file mode 100644
index 000000000..34d967537
--- /dev/null
+++ b/desktop/tests/editor-mcp-tools.test.ts
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+import { describe, it, expect, vi } from 'vitest';
+// @ts-expect-error — plain-.mjs tool registry shared with the Electron MCP entry.
+import { TOOLS, runTool, listTools } from '../scripts/editor-mcp-tools.mjs';
+
+// The editor MCP server is a transport over EditorControlSurface: each tool maps to one
+// surface method. These cover the pure dispatch layer (no Electron) — the Electron entry
+// only supplies the executeJavaScript driver.
+describe('editor MCP tool registry', () => {
+ it('every tool has a unique name, a surface method, and an args builder', () => {
+ const names = new Set<string>();
+ for (const t of TOOLS as Array<{ name: string; method: string; args: unknown }>) {
+ expect(typeof t.method).toBe('string');
+ expect(typeof t.args).toBe('function');
+ expect(names.has(t.name)).toBe(false);
+ names.add(t.name);
+ }
+ expect(TOOLS.length).toBeGreaterThan(10);
+ });
+
+ it('runTool validates input and calls the driver with (method, args)', async () => {
+ const driver = vi.fn(async () => 42);
+ const setField = TOOLS.find((t: { name: string }) => t.name === 'set_field');
+ const res = await runTool(setField, driver, {
+ entity: 1, component: 'Transform', key: 'position.x', type: 'float', value: 5,
+ });
+ expect(driver).toHaveBeenCalledWith('setField', [1, 'Transform', 'position.x', 'float', 5]);
+ expect(res.content[0].text).toBe('42');
+ expect(res.isError).toBeFalsy();
+ });
+
+ it('runTool returns an error result on invalid input (driver not called)', async () => {
+ const driver = vi.fn();
+ const getEntity = TOOLS.find((t: { name: string }) => t.name === 'get_entity');
+ const res = await runTool(getEntity, driver, { id: 'not-a-number' });
+ expect(res.isError).toBe(true);
+ expect(driver).not.toHaveBeenCalled();
+ });
+
+ it('runTool wraps a driver throw as an error result', async () => {
+ const driver = vi.fn(async () => { throw new Error('boom'); });
+ const tree = TOOLS.find((t: { name: string }) => t.name === 'get_scene_tree');
+ const res = await runTool(tree, driver, {});
+ expect(res.isError).toBe(true);
+ expect(res.content[0].text).toContain('boom');
+ });
+
+ it('runTool reports "ok" for a void surface call', async () => {
+ const driver = vi.fn(async () => undefined);
+ const undo = TOOLS.find((t: { name: string }) => t.name === 'undo');
+ const res = await runTool(undo, driver, {});
+ expect(res.content[0].text).toBe('ok');
+ });
+
+ it('listTools advertises every tool with a description + JSON-Schema inputSchema', () => {
+ const listed = listTools();
+ expect(listed.map((t: { name: string }) => t.name)).toEqual((TOOLS as Array<{ name: string }>).map((t) => t.name));
+ for (const t of listed as Array<{ description: string; inputSchema: { type: string } }>) {
+ expect(typeof t.description).toBe('string');
+ expect(t.inputSchema.type).toBe('object');
+ }
+ });
+});