A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(editor): MCP server over EditorControlSurface (headless host)
esengineaf56c695dddd
diff --git a/desktop/package.json b/desktop/package.jsonindex 28216b463..0be242d3f 100644--- a/desktop/package.json+++ b/desktop/package.json@@ -14,6 +14,9 @@"test": "vitest run","test:watch": "vitest","verify:render": "vite build && electron scripts/headless-verify.mjs",+ "editor:mcp": "electron scripts/editor-mcp.mjs",+ "editor:mcp:build": "vite build && electron scripts/editor-mcp.mjs",+ "editor:mcp:e2e": "vite build && node scripts/editor-mcp-e2e.mjs","verify:render:spine": "vite build && ESTELLA_VERIFY_SCENE=/scenes/spine-test.esscene ESTELLA_VERIFY_MANIFEST=/scenes/spine-test.textures.json electron scripts/headless-verify.mjs","verify:render:particles": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SCENE=/scenes/particle-demo.esscene ESTELLA_VERIFY_MANIFEST=/scenes/particle-demo.textures.json ESTELLA_VERIFY_STEPS=90 electron scripts/headless-verify.mjs","verify:render:particles:gradient": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SCENE=/scenes/particle-gradient.esscene ESTELLA_VERIFY_MANIFEST=/scenes/particle-gradient.textures.json ESTELLA_VERIFY_W=256 ESTELLA_VERIFY_H=256 ESTELLA_VERIFY_STEPS=20 ESTELLA_VERIFY_EXPECT=\"[{\\\"x\\\":0.5,\\\"y\\\":0.5,\\\"rgb\\\":[0,255,0],\\\"tol\\\":90}]\" electron scripts/headless-verify.mjs",diff --git a/desktop/scripts/editor-mcp-e2e.mjs b/desktop/scripts/editor-mcp-e2e.mjsnew file mode 100644index 000000000..82b7eefb6--- /dev/null+++ b/desktop/scripts/editor-mcp-e2e.mjs@@ -0,0 +1,74 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file editor-mcp-e2e.mjs — end-to-end smoke for the editor MCP server.+ * Spawns editor-mcp.mjs and drives the MCP stdio handshake (initialize →+ * tools/list → tools/call), exercising the full path: stdio transport → SDK+ * Server → executeJavaScript driver → EditorControlSurface → engine. Run from+ * desktop/ after a dist build: pnpm editor:mcp:e2e+ */+import { spawn } from 'node:child_process';+import path from 'node:path';+import { fileURLToPath } from 'node:url';++const DESKTOP = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');+const electron = path.join(DESKTOP, 'node_modules', '.bin', 'electron');++const child = spawn(electron, ['scripts/editor-mcp.mjs'], { cwd: DESKTOP, stdio: ['pipe', 'pipe', 'pipe'] });++let buf = '';+const waiters = new Map();+child.stdout.on('data', (d) => {+ buf += d.toString();+ let nl;+ while ((nl = buf.indexOf('\n')) >= 0) {+ const line = buf.slice(0, nl).trim();+ buf = buf.slice(nl + 1);+ if (!line) continue;+ let msg;+ try { msg = JSON.parse(line); } catch { continue; } // skip stray non-JSON stdout+ if (msg.id != null && waiters.has(msg.id)) { waiters.get(msg.id)(msg); waiters.delete(msg.id); }+ }+});+child.stderr.on('data', (d) => process.stderr.write(d));++const send = (msg) => child.stdin.write(JSON.stringify(msg) + '\n');+const rpc = (id, method, params) => new Promise((resolve, reject) => {+ const t = setTimeout(() => reject(new Error(`timeout waiting for ${method}`)), 30000);+ waiters.set(id, (m) => { clearTimeout(t); resolve(m); });+ send({ jsonrpc: '2.0', id, method, params });+});+const fail = (m) => { console.log('E2E FAIL:', m); child.kill(); process.exit(1); };++try {+ const init = await rpc(1, 'initialize', {+ protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'e2e', version: '0' },+ });+ if (!init.result?.serverInfo) fail('no serverInfo in initialize');+ console.log('initialize OK — server:', init.result.serverInfo.name);+ send({ jsonrpc: '2.0', method: 'notifications/initialized' });++ const list = await rpc(2, 'tools/list', {});+ const names = (list.result?.tools ?? []).map((t) => t.name);+ if (!names.includes('load_scene') || !names.includes('get_scene_tree')) fail(`tools/list missing tools: ${names}`);+ console.log(`tools/list OK — ${names.length} tools`);++ const load = await rpc(3, 'tools/call', {+ name: 'load_scene',+ arguments: { sceneUrl: '/scenes/sprite-rendering.esscene', manifestUrl: '/scenes/sprite-rendering.textures.json' },+ });+ if (load.result?.isError) fail(`load_scene error: ${load.result?.content?.[0]?.text}`);+ console.log('load_scene OK — entityCount:', load.result?.content?.[0]?.text);++ const tree = await rpc(4, 'tools/call', { name: 'get_scene_tree', arguments: {} });+ if (tree.result?.isError) fail(`get_scene_tree error: ${tree.result?.content?.[0]?.text}`);+ const parsed = JSON.parse(tree.result?.content?.[0]?.text ?? 'null');+ if (!Array.isArray(parsed) || parsed.length === 0) fail('get_scene_tree returned no nodes');+ console.log(`get_scene_tree OK — ${parsed.length} root nodes`);++ console.log('\nE2E PASS — MCP server drives EditorControlSurface end-to-end');+ child.kill();+ process.exit(0);+} catch (e) {+ fail(String(e?.message ?? e));+}diff --git a/desktop/scripts/editor-mcp-tools.mjs b/desktop/scripts/editor-mcp-tools.mjsnew file mode 100644index 000000000..35224377c--- /dev/null+++ b/desktop/scripts/editor-mcp-tools.mjs@@ -0,0 +1,118 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file editor-mcp-tools.mjs+ * The editor MCP tool registry. Each tool maps 1:1 to an EditorControlSurface+ * method, so the MCP server adds NO new editor truth — it is a transport over+ * the surface (exactly what EditorControlSurface.ts:7-9 anticipated). Kept+ * dependency-free (JSON-Schema + light manual validation, no zod import) so the+ * dispatch unit-tests without Electron and resolves under vite/vitest; the+ * Electron entry (editor-mcp.mjs) supplies the executeJavaScript driver and the+ * MCP SDK wiring. See docs/REARCH_EDITOR_ARCH.md §11.+ */++const obj = (properties, required = []) => ({ type: 'object', properties, required });++/** name → { description, schema (JSON Schema), method (surface), args(input)→[] }. */+export const TOOLS = [+ { name: 'load_scene',+ description: 'Load a scene (and optional asset manifest) into the headless World; returns the spawned entity count.',+ schema: obj({ sceneUrl: { type: 'string' }, manifestUrl: { type: 'string' } }, ['sceneUrl']),+ method: 'loadScene', args: (i) => [i.sceneUrl, i.manifestUrl] },+ { name: 'get_scene_tree',+ description: 'The scene entity tree (id, name, kind, children).',+ schema: obj({}), method: 'getSceneTree', args: () => [] },+ { name: 'get_entity',+ description: 'One entity: name, kind, and its component list.',+ schema: obj({ id: { type: 'number' } }, ['id']), method: 'getEntity', args: (i) => [i.id] },+ { name: 'get_inspector',+ description: "An entity's components and fields — the Details-panel data.",+ schema: obj({ entity: { type: 'number' } }, ['entity']), method: 'getInspector', args: (i) => [i.entity] },+ { name: 'serialize_scene',+ description: 'The full lossless scene JSON (the model truth).',+ schema: obj({}), method: 'serializeScene', args: () => [] },+ { name: 'add_entity',+ description: 'Create a new entity; returns its id.',+ schema: obj({}), method: 'addEntity', args: () => [] },+ { name: 'delete_entity',+ description: 'Delete an entity.',+ schema: obj({ id: { type: 'number' } }, ['id']), method: 'deleteEntity', args: (i) => [i.id] },+ { name: 'rename_entity',+ description: 'Rename an entity.',+ schema: obj({ id: { type: 'number' }, name: { type: 'string' } }, ['id', 'name']),+ method: 'renameEntity', args: (i) => [i.id, i.name] },+ { name: 'set_field',+ description: 'Set a component field (undoable). `type` is the inspector field type (e.g. float, int, bool, string, vec2, vec3, color, enum).',+ schema: obj({+ entity: { type: 'number' }, component: { type: 'string' }, key: { type: 'string' },+ type: { type: 'string' }, value: {},+ }, ['entity', 'component', 'key', 'type', 'value']),+ method: 'setField', args: (i) => [i.entity, i.component, i.key, i.type, i.value] },+ { name: 'set_parent',+ description: 'Re-parent an entity in the transform hierarchy (parent=null → scene root).',+ schema: obj({ id: { type: 'number' }, parent: { type: ['number', 'null'] } }, ['id', 'parent']),+ method: 'setParent', args: (i) => [i.id, i.parent] },+ { name: 'select',+ description: 'Select an entity (id=null clears the selection).',+ schema: obj({ id: { type: ['number', 'null'] } }, ['id']), method: 'select', args: (i) => [i.id] },+ { name: 'get_selection',+ description: 'The primary selected entity id, or null.',+ schema: obj({}), method: 'getSelection', args: () => [] },+ { name: 'set_run_mode',+ description: 'Enter or leave play mode (playing=true runs gameplay; Stop rebuilds the edit World).',+ schema: obj({ playing: { type: 'boolean' }, paused: { type: 'boolean' } }, ['playing']),+ method: 'setRunMode', args: (i) => [i.playing, i.paused] },+ { name: 'step',+ description: 'Advance the engine by N fixed-dt frames deterministically (no rAF).',+ schema: obj({ frames: { type: 'number' }, dt: { type: 'number' } }),+ method: 'step', args: (i) => [i.frames, i.dt] },+ { name: 'get_stats',+ description: 'Live counts: entity count + last frame draw calls (cheap render evidence).',+ schema: obj({}), method: 'getStats', args: () => [] },+ { name: 'undo', description: 'Undo the last edit.', schema: obj({}), method: 'undo', args: () => [] },+ { name: 'redo', description: 'Redo the last undone edit.', schema: obj({}), method: 'redo', args: () => [] },+];++/** The MCP `tools/list` payload — name, description, JSON-Schema inputSchema. */+export function listTools() {+ return TOOLS.map((t) => ({ name: t.name, description: t.description, inputSchema: t.schema }));+}++function typeMatches(spec, val) {+ return (Array.isArray(spec) ? spec : [spec]).some((t) =>+ t === 'null' ? val === null+ : t === 'number' ? typeof val === 'number'+ : t === 'string' ? typeof val === 'string'+ : t === 'boolean' ? typeof val === 'boolean'+ : t === 'object' ? (val !== null && typeof val === 'object')+ : true);+}++/** Light validation: required args present + declared scalar types match. */+function validate(schema, raw) {+ const input = raw ?? {};+ for (const req of schema.required ?? []) {+ if (input[req] === undefined) throw new Error(`missing required argument: ${req}`);+ }+ for (const [key, spec] of Object.entries(schema.properties ?? {})) {+ if (input[key] !== undefined && spec.type && !typeMatches(spec.type, input[key])) {+ throw new Error(`argument "${key}" must be ${[].concat(spec.type).join(' | ')}`);+ }+ }+ return input;+}++/**+ * Validate `rawInput`, invoke the surface via `driver(method, args)`, and wrap the+ * result as an MCP CallToolResult. A validation failure or a driver throw becomes an+ * `isError` result rather than crashing the server.+ */+export async function runTool(tool, driver, rawInput) {+ try {+ const input = validate(tool.schema, rawInput);+ const result = await driver(tool.method, tool.args(input));+ return { content: [{ type: 'text', text: result === undefined ? 'ok' : JSON.stringify(result) }] };+ } catch (err) {+ return { content: [{ type: 'text', text: `error: ${err?.message ?? String(err)}` }], isError: true };+ }+}diff --git a/desktop/scripts/editor-mcp.mjs b/desktop/scripts/editor-mcp.mjsnew file mode 100644index 000000000..166869b73--- /dev/null+++ b/desktop/scripts/editor-mcp.mjs@@ -0,0 +1,94 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file editor-mcp.mjs — the editor MCP server (headless host).+ *+ * An Electron entry that boots the SAME headless render host the render-verify runner+ * uses (serves dist/, opens headless.html, which publishes EditorControlSurface on+ * `window.__estellaHeadless`), then runs an MCP server over stdio whose tools are the+ * surface methods (see editor-mcp-tools.mjs). So an MCP client (an AI agent) can drive+ * the real editor surface headlessly — load/inspect/edit scenes, step, read stats —+ * with no live Electron UI. The surface stays the single source of truth; this is a+ * transport over it (EditorControlSurface.ts:7-9). Requires a built dist/ (vite build).+ *+ * stdout is the MCP JSON-RPC channel — ALL logging goes to stderr.+ */+import { app, BrowserWindow } from 'electron';+import http from 'node:http';+import { readFile } from 'node:fs/promises';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { Server } from '@modelcontextprotocol/sdk/server/index.js';+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';+import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';+import { TOOLS, listTools, runTool } from './editor-mcp-tools.mjs';++const DIST = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'dist');+const W = Number(process.env.ESTELLA_MCP_W) || 1280;+const H = Number(process.env.ESTELLA_MCP_H) || 720;++app.commandLine.appendSwitch('enable-unsafe-swiftshader'); // GPU-less WebGL2 fallback+process.env.ELECTRON_DISABLE_SECURITY_WARNINGS = 'true';+const log = (...a) => process.stderr.write(`[editor-mcp] ${a.join(' ')}\n`);++const MIME = {+ '.html': 'text/html', '.js': 'text/javascript', '.mjs': 'text/javascript',+ '.css': 'text/css', '.json': 'application/json', '.esscene': 'application/json',+ '.wasm': 'application/wasm', '.png': 'image/png', '.jpg': 'image/jpeg',+ '.webp': 'image/webp', '.woff': 'font/woff', '.woff2': 'font/woff2', '.svg': 'image/svg+xml',+};++function serveDist() {+ const server = http.createServer(async (req, res) => {+ try {+ const rel = decodeURIComponent(new URL(req.url, 'http://x').pathname).replace(/^\/+/, '') || 'index.html';+ const abs = path.join(DIST, rel);+ if (!abs.startsWith(DIST)) { res.writeHead(403).end(); return; }+ const bytes = await readFile(abs);+ res.writeHead(200, { 'content-type': MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' });+ res.end(bytes);+ } catch { res.writeHead(404).end('not found'); }+ });+ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server)));+}++// Keep the process alive on the hidden window (this is a long-running server).+app.on('window-all-closed', () => {});++app.whenReady().then(async () => {+ try {+ const server = await serveDist();+ const url = `http://127.0.0.1:${server.address().port}/headless.html?w=${W}&h=${H}`;+ const win = new BrowserWindow({ show: false, width: W, height: H, webPreferences: { offscreen: false } });+ win.webContents.on('console-message', (...args) => {+ const msg = args.map((a) => (a && typeof a === 'object' ? a.message ?? '' : String(a))).join(' ');+ if (/error|fail|unwind|exception|webgl/i.test(msg)) log('[renderer]', msg.slice(0, 240));+ });+ await win.loadURL(url);+ await win.webContents.executeJavaScript('window.__estellaHeadless.ready', true);+ log('headless engine ready');++ // Marshal a surface call into the headless renderer. `undefined` args become the JS+ // `undefined` literal so the surface's default parameters apply.+ const driver = (method, args) =>+ win.webContents.executeJavaScript(+ `window.__estellaHeadless.api.${method}(${(args ?? [])+ .map((a) => (a === undefined ? 'undefined' : JSON.stringify(a)))+ .join(',')})`,+ true,+ );++ const mcp = new Server({ name: 'estella-editor', version: '0.1.0' }, { capabilities: { tools: {} } });+ mcp.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: listTools() }));+ mcp.setRequestHandler(CallToolRequestSchema, async (req) => {+ const tool = TOOLS.find((t) => t.name === req.params.name);+ if (!tool) return { content: [{ type: 'text', text: `unknown tool: ${req.params.name}` }], isError: true };+ return runTool(tool, driver, req.params.arguments);+ });+ await mcp.connect(new StdioServerTransport());+ log('MCP server connected over stdio');+ } catch (e) {+ log('FATAL', String((e && e.stack) || e));+ app.exit(1);+ }+});diff --git a/desktop/tests/editor-mcp-tools.test.ts b/desktop/tests/editor-mcp-tools.test.tsnew file mode 100644index 000000000..34d967537--- /dev/null+++ b/desktop/tests/editor-mcp-tools.test.ts@@ -0,0 +1,64 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+import { describe, it, expect, vi } from 'vitest';+// @ts-expect-error — plain-.mjs tool registry shared with the Electron MCP entry.+import { TOOLS, runTool, listTools } from '../scripts/editor-mcp-tools.mjs';++// The editor MCP server is a transport over EditorControlSurface: each tool maps to one+// surface method. These cover the pure dispatch layer (no Electron) — the Electron entry+// only supplies the executeJavaScript driver.+describe('editor MCP tool registry', () => {+ it('every tool has a unique name, a surface method, and an args builder', () => {+ const names = new Set<string>();+ for (const t of TOOLS as Array<{ name: string; method: string; args: unknown }>) {+ expect(typeof t.method).toBe('string');+ expect(typeof t.args).toBe('function');+ expect(names.has(t.name)).toBe(false);+ names.add(t.name);+ }+ expect(TOOLS.length).toBeGreaterThan(10);+ });++ it('runTool validates input and calls the driver with (method, args)', async () => {+ const driver = vi.fn(async () => 42);+ const setField = TOOLS.find((t: { name: string }) => t.name === 'set_field');+ const res = await runTool(setField, driver, {+ entity: 1, component: 'Transform', key: 'position.x', type: 'float', value: 5,+ });+ expect(driver).toHaveBeenCalledWith('setField', [1, 'Transform', 'position.x', 'float', 5]);+ expect(res.content[0].text).toBe('42');+ expect(res.isError).toBeFalsy();+ });++ it('runTool returns an error result on invalid input (driver not called)', async () => {+ const driver = vi.fn();+ const getEntity = TOOLS.find((t: { name: string }) => t.name === 'get_entity');+ const res = await runTool(getEntity, driver, { id: 'not-a-number' });+ expect(res.isError).toBe(true);+ expect(driver).not.toHaveBeenCalled();+ });++ it('runTool wraps a driver throw as an error result', async () => {+ const driver = vi.fn(async () => { throw new Error('boom'); });+ const tree = TOOLS.find((t: { name: string }) => t.name === 'get_scene_tree');+ const res = await runTool(tree, driver, {});+ expect(res.isError).toBe(true);+ expect(res.content[0].text).toContain('boom');+ });++ it('runTool reports "ok" for a void surface call', async () => {+ const driver = vi.fn(async () => undefined);+ const undo = TOOLS.find((t: { name: string }) => t.name === 'undo');+ const res = await runTool(undo, driver, {});+ expect(res.content[0].text).toBe('ok');+ });++ it('listTools advertises every tool with a description + JSON-Schema inputSchema', () => {+ const listed = listTools();+ expect(listed.map((t: { name: string }) => t.name)).toEqual((TOOLS as Array<{ name: string }>).map((t) => t.name));+ for (const t of listed as Array<{ description: string; inputSchema: { type: string } }>) {+ expect(typeof t.description).toBe('string');+ expect(t.inputSchema.type).toBe('object');+ }+ });+});