A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(editor): play_probe - MCP eyes and hands inside the running game
yhhf985e960f5fe
diff --git a/desktop/electron/mcpEndpoint.ts b/desktop/electron/mcpEndpoint.tsindex 77af6a6f9..81486819b 100644--- a/desktop/electron/mcpEndpoint.ts+++ b/desktop/electron/mcpEndpoint.ts@@ -53,8 +53,9 @@ export async function startMcpEndpoint(getWin: () => BrowserWindow | null): Promconst server = await createExecEndpoint({token,- run: async ({ method, args, root, js, op }: {+ run: async ({ method, args, root, js, op, code, frame }: {method?: string; args?: unknown[]; root?: string; js?: string; op?: string;+ code?: string; frame?: number;}) => {if (op === 'screenshot') {const win = getWin();@@ -62,6 +63,16 @@ export async function startMcpEndpoint(getWin: () => BrowserWindow | null): Promconst image = await win.webContents.capturePage();return image.toPNG().toString('base64');}+ if (op === 'play_probe') {+ // The play realm is an estella:// OOPIF — only a main-process frame eval+ // reaches it (the SHOT_PLAY_EVAL idiom; window.__estellaPlay is the probe).+ const win = getWin();+ if (!win) throw new Error('no editor window');+ const playFrames = win.webContents.mainFrame.frames.filter((f) => f.url.startsWith('estella://'));+ const playFrame = playFrames[frame ?? 0];+ if (!playFrame) throw new Error(`no play realm at index ${frame ?? 0} (${playFrames.length} running — enter play first)`);+ return playFrame.executeJavaScript(code ?? 'true');+ }if (js) return exec(js);const target = root === 'editor' ? 'window.__estellaEditor' : 'window.__estellaEditor.surface';return exec(`${target}.${method}(${(args ?? [])diff --git a/desktop/scripts/editor-mcp-editor-e2e.mjs b/desktop/scripts/editor-mcp-editor-e2e.mjsindex f02a65987..70dceb6a4 100644--- a/desktop/scripts/editor-mcp-editor-e2e.mjs+++ b/desktop/scripts/editor-mcp-editor-e2e.mjs@@ -134,8 +134,11 @@ try {if (!play?.ready) await fail(`play realm never became ready (${JSON.stringify(play)})`);const playShot = await call('screenshot');if (playShot.type !== 'image') await fail('screenshot during play failed');+ // The gameplay-state probe reaches the estella:// OOPIF and sees __estellaPlay.+ const probe = JSON.parse((await call('play_probe', { code: 'typeof window.__estellaPlay' })).text);+ if (probe !== 'object') await fail(`play_probe: __estellaPlay not present (${probe})`);await call('toggle_play', {}, 60_000);- console.log('play OK — realm ready, screenshot captured, stopped');+ console.log('play OK — realm ready, screenshot + probe captured, stopped');if (process.env.ESTELLA_E2E_EXPORT === '1') {const outDir = path.join(tmpRoot, 'export-web');diff --git a/desktop/scripts/editor-mcp-tools.mjs b/desktop/scripts/editor-mcp-tools.mjsindex f0e6571bd..dbae36161 100644--- a/desktop/scripts/editor-mcp-tools.mjs+++ b/desktop/scripts/editor-mcp-tools.mjs@@ -201,6 +201,10 @@ export const TOOLS = [{ name: 'screenshot',description: 'Capture the composited editor window as a PNG (includes the play realm iframe — use this to SEE gameplay; capture_viewport only sees the edit viewport).',schema: obj({}), op: 'screenshot', image: true },+ { name: 'play_probe', write: true,+ description: "Evaluate JS inside the RUNNING play realm and return the result — the gameplay probe. window.__estellaPlay = { app, getComponent } for state reads; to drive gameplay input, dispatch KeyboardEvents on DOCUMENT (the engine listens there, not on window): document.dispatchEvent(new KeyboardEvent('keydown', {code:'ArrowRight'})). frame picks the realm in multiplayer previews (0 = host).",+ schema: obj({ code: { type: 'string' }, frame: { type: 'number' } }, ['code']),+ op: 'play_probe' },{ name: 'world_component',description: "A LIVE World component's data for a source entity, resolved by component name — verifies an edit actually reached the engine.",schema: obj({ id: { type: 'number' }, component: { type: 'string' } }, ['id', 'component']),@@ -282,9 +286,8 @@ export async function runTool(tool, driver, rawInput, allowWrites = true) {const input = validate(tool.schema, rawInput);if (tool.op) {const data = await driver.op(tool.op, input);- return tool.image- ? { content: [{ type: 'image', data, mimeType: 'image/png' }] }- : { content: [{ type: 'text', text: String(data) }] };+ if (tool.image) return { content: [{ type: 'image', data, mimeType: 'image/png' }] };+ return { content: [{ type: 'text', text: data === undefined ? 'ok' : JSON.stringify(data) }] };}if (tool.js) {const data = await driver.js(tool.js(input));