Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): play_probe - MCP eyes and hands inside the running game

yhhf985e960f5fe
diff --git a/desktop/electron/mcpEndpoint.ts b/desktop/electron/mcpEndpoint.ts
index 77af6a6f9..81486819b 100644
--- a/desktop/electron/mcpEndpoint.ts
+++ b/desktop/electron/mcpEndpoint.ts
@@ -53,8 +53,9 @@ export async function startMcpEndpoint(getWin: () => BrowserWindow | null): Prom
const server = await createExecEndpoint({
token,
- run: async ({ method, args, root, js, op }: {
+ run: async ({ method, args, root, js, op, code, frame }: {
method?: string; args?: unknown[]; root?: string; js?: string; op?: string;
+ code?: string; frame?: number;
}) => {
if (op === 'screenshot') {
const win = getWin();
@@ -62,6 +63,16 @@ export async function startMcpEndpoint(getWin: () => BrowserWindow | null): Prom
const image = await win.webContents.capturePage();
return image.toPNG().toString('base64');
}
+ if (op === 'play_probe') {
+ // The play realm is an estella:// OOPIF — only a main-process frame eval
+ // reaches it (the SHOT_PLAY_EVAL idiom; window.__estellaPlay is the probe).
+ const win = getWin();
+ if (!win) throw new Error('no editor window');
+ const playFrames = win.webContents.mainFrame.frames.filter((f) => f.url.startsWith('estella://'));
+ const playFrame = playFrames[frame ?? 0];
+ if (!playFrame) throw new Error(`no play realm at index ${frame ?? 0} (${playFrames.length} running — enter play first)`);
+ return playFrame.executeJavaScript(code ?? 'true');
+ }
if (js) return exec(js);
const target = root === 'editor' ? 'window.__estellaEditor' : 'window.__estellaEditor.surface';
return exec(`${target}.${method}(${(args ?? [])
diff --git a/desktop/scripts/editor-mcp-editor-e2e.mjs b/desktop/scripts/editor-mcp-editor-e2e.mjs
index f02a65987..70dceb6a4 100644
--- a/desktop/scripts/editor-mcp-editor-e2e.mjs
+++ b/desktop/scripts/editor-mcp-editor-e2e.mjs
@@ -134,8 +134,11 @@ try {
if (!play?.ready) await fail(`play realm never became ready (${JSON.stringify(play)})`);
const playShot = await call('screenshot');
if (playShot.type !== 'image') await fail('screenshot during play failed');
+ // The gameplay-state probe reaches the estella:// OOPIF and sees __estellaPlay.
+ const probe = JSON.parse((await call('play_probe', { code: 'typeof window.__estellaPlay' })).text);
+ if (probe !== 'object') await fail(`play_probe: __estellaPlay not present (${probe})`);
await call('toggle_play', {}, 60_000);
- console.log('play OK — realm ready, screenshot captured, stopped');
+ console.log('play OK — realm ready, screenshot + probe captured, stopped');
if (process.env.ESTELLA_E2E_EXPORT === '1') {
const outDir = path.join(tmpRoot, 'export-web');
diff --git a/desktop/scripts/editor-mcp-tools.mjs b/desktop/scripts/editor-mcp-tools.mjs
index f0e6571bd..dbae36161 100644
--- a/desktop/scripts/editor-mcp-tools.mjs
+++ b/desktop/scripts/editor-mcp-tools.mjs
@@ -201,6 +201,10 @@ export const TOOLS = [
{ name: 'screenshot',
description: 'Capture the composited editor window as a PNG (includes the play realm iframe — use this to SEE gameplay; capture_viewport only sees the edit viewport).',
schema: obj({}), op: 'screenshot', image: true },
+ { name: 'play_probe', write: true,
+ description: "Evaluate JS inside the RUNNING play realm and return the result — the gameplay probe. window.__estellaPlay = { app, getComponent } for state reads; to drive gameplay input, dispatch KeyboardEvents on DOCUMENT (the engine listens there, not on window): document.dispatchEvent(new KeyboardEvent('keydown', {code:'ArrowRight'})). frame picks the realm in multiplayer previews (0 = host).",
+ schema: obj({ code: { type: 'string' }, frame: { type: 'number' } }, ['code']),
+ op: 'play_probe' },
{ name: 'world_component',
description: "A LIVE World component's data for a source entity, resolved by component name — verifies an edit actually reached the engine.",
schema: obj({ id: { type: 'number' }, component: { type: 'string' } }, ['id', 'component']),
@@ -282,9 +286,8 @@ export async function runTool(tool, driver, rawInput, allowWrites = true) {
const input = validate(tool.schema, rawInput);
if (tool.op) {
const data = await driver.op(tool.op, input);
- return tool.image
- ? { content: [{ type: 'image', data, mimeType: 'image/png' }] }
- : { content: [{ type: 'text', text: String(data) }] };
+ if (tool.image) return { content: [{ type: 'image', data, mimeType: 'image/png' }] };
+ return { content: [{ type: 'text', text: data === undefined ? 'ok' : JSON.stringify(data) }] };
}
if (tool.js) {
const data = await driver.js(tool.js(input));