A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(mcp): a screenshot a model without vision can read
yhhfb4e0de33cfe
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.tsindex 6c87a4619..763dc9f78 100644--- a/desktop/electron/agent/kernel.ts+++ b/desktop/electron/agent/kernel.ts@@ -185,9 +185,11 @@ export async function runTurn(if (context) session.pushContext(context);if (!acceptsImages) {session.pushContext(- 'This endpoint cannot carry images, so capture_viewport and the other screenshot '- + 'tools will not let you SEE anything — their frames cannot reach you. Verify with '- + 'get_diagnostics and by reading fields back instead of by looking.',+ 'This endpoint cannot carry images: a PNG from screenshot or capture_viewport cannot '+ + "reach you. `screenshot` with `format: 'grid'` can — it answers with the same picture "+ + 'as a coarse colour grid in text, cropped to the running game. That is your eyes; use '+ + 'it for the questions no field-read can answer (did anything draw, is it on camera, '+ + 'what colour did it come out, did the picture change after that input).',);}session.pushUser(text, images);@@ -258,13 +260,15 @@ export async function runTurn(+ 'reads, whether it is where you meant, are all things only the picture answers. '+ 'capture_viewport now; if it is something that has to be PLAYED, toggle_play, '+ 'drive it with play_input and screenshot that. Then fix what you see, or report.'- : 'You changed the scene this turn and never ran it. You cannot receive images, so '- + 'do not spend a call on capture_viewport — RUN it instead: toggle_play, then '+ : 'You changed the scene this turn and never ran it. RUN it: toggle_play, then '+ 'play_probe to read the state back (`find(NAME)` gives every entity carrying a '+ 'component, with its data), play_input to drive the controls a player would use, '- + 'and play_probe again to prove they did something. get_logs shows what the '- + 'running game complained about. Reading your own source again is not this: it '- + 'is the thing that already convinced you it works.',+ + 'step to advance the frames, and play_probe again to prove they did something. '+ + "You cannot receive images, but `screenshot` with `format: 'grid'` gives you the "+ + 'picture as text — take one, because whether anything DREW is not a question the '+ + 'component values answer. get_logs shows what the running game complained about. '+ + 'Reading your own source again is not this: it is the thing that already '+ + 'convinced you it works.',);continue;}diff --git a/desktop/electron/agent/prompt.ts b/desktop/electron/agent/prompt.tsindex 64e94ab65..ccc357cba 100644--- a/desktop/electron/agent/prompt.ts+++ b/desktop/electron/agent/prompt.ts@@ -30,7 +30,7 @@ How work lands here:Working style:- Look before you edit. get_scene_tree, get_inspector and get_document cost one round trip and are the difference between editing the right entity and editing a plausible one.-- You can see the result: capture_viewport renders what the user is looking at, and screenshot takes in the whole window including a running game. LOOK before you call a visual change done — a material, an effect, a layout, anything whose whole point is what it looks like. Reading back the data you just wrote proves you wrote it, not that it did anything; the difference between "the component says 1.0" and "the sprite is gone" is the entire job.+- You can see the result: capture_viewport renders what the user is looking at, and screenshot takes in the whole window including a running game — as a picture, or, with \`format: 'grid'\`, as a coarse colour grid in TEXT, which is the form to ask for when images cannot reach you. LOOK before you call a visual change done — a material, an effect, a layout, anything whose whole point is what it looks like. Reading back the data you just wrote proves you wrote it, not that it did anything; the difference between "the component says 1.0" and "the sprite is gone" is the entire job.- A game you have built is not tested until you have PLAYED it: toggle_play, then step to advance it (its loop is throttled to about a frame a second whenever the editor window is not the focused one, so waiting shows you a frozen picture of a healthy game), play_input to press the keys a player would, play_probe to read what the game now thinks, and screenshot to see what a player would see. Step-look-step is the loop; twenty probes without a screenshot is how a project ships game-over-on-arrival with every component reporting exactly the value it should.- Prefer one apply_scene_ops over many small calls when building something: it is one undo step for the user and one round trip for you, and it is the ONE write the user sees BEFORE it lands — they get a line-by-line preview and can strike parts of it out. A dozen set_field calls give them no such chance, so build with a batch and save the single writes for genuinely single changes.- Units are the editor's: 1 world unit = 1 design pixel. Y is up in the world, and UI is laid out top-down.diff --git a/desktop/electron/surfaceDriver.ts b/desktop/electron/surfaceDriver.tsindex 9655c76fa..fa568d3da 100644--- a/desktop/electron/surfaceDriver.ts+++ b/desktop/electron/surfaceDriver.ts@@ -54,6 +54,86 @@ const carryError = (code: string, preamble = ''): string => {return `(async () => { try { ${preamble} return { v: ${value} }; } catch (e) { return { __estellaExecError: (e && e.message) || String(e) }; } })()`;};+interface GameRect {+ x: number; y: number; width: number; height: number;+ windowWidth: number; windowHeight: number; playing: boolean;+}++/**+ * A capture, as sixteen colour letters — the screenshot for a caller with no vision.+ *+ * Half the models an editor gets pointed at cannot receive an image, and for those the+ * whole verifying-by-looking half of the job was closed: the agent kernel told them, in+ * so many words, not to spend a call on a picture they could not see. So they verified+ * by reading their own fields back, which is exactly the reading that cannot see a+ * sprite that never drew, content off camera, or a shader that came out black.+ *+ * Coarse on purpose. At 64 columns a cell is a dozen-odd pixels, which is enough for+ * "five rows of colour across the top, one white bar near the bottom, a dot between+ * them" — a Breakout, recognisably — and small enough to read in a tool result. The+ * palette is fixed rather than a per-capture legend, so the letters mean the same thing+ * in every reply and can be reasoned about directly.+ */+const PALETTE: ReadonlyArray<[string, number, number, number]> = [+ ['.', 0, 0, 0], ['r', 128, 0, 0], ['g', 0, 128, 0], ['y', 128, 128, 0],+ ['b', 0, 0, 128], ['m', 128, 0, 128], ['c', 0, 128, 128], ['w', 170, 170, 170],+ ['K', 85, 85, 85], ['R', 255, 0, 0], ['G', 0, 255, 0], ['Y', 255, 255, 0],+ ['B', 0, 0, 255], ['M', 255, 0, 255], ['C', 0, 255, 255], ['W', 255, 255, 255],+];++function nearestInk(r: number, g: number, b: number): string {+ let best = '.';+ let bestDist = Infinity;+ for (const [ink, pr, pg, pb] of PALETTE) {+ const d = (r - pr) ** 2 + (g - pg) ** 2 + (b - pb) ** 2;+ if (d < bestDist) { bestDist = d; best = ink; }+ }+ return best;+}++function colorGrid(image: Electron.NativeImage, rect: GameRect | null, cols: number, rows: number): string {+ const w = Math.max(8, Math.min(160, Math.round(cols)));+ const h = Math.max(4, Math.min(90, Math.round(rows)));+ const full = image.getSize();+ let shot = image;+ let what = 'the whole editor window';+ if (rect && rect.windowWidth > 0) {+ // capturePage is in device pixels and the rect is in CSS pixels.+ const scale = full.width / rect.windowWidth;+ const crop = {+ x: Math.max(0, Math.round(rect.x * scale)),+ y: Math.max(0, Math.round(rect.y * scale)),+ width: Math.min(full.width, Math.round(rect.width * scale)),+ height: Math.min(full.height, Math.round(rect.height * scale)),+ };+ if (crop.width > 8 && crop.height > 8) {+ shot = image.crop(crop);+ what = rect.playing ? 'the RUNNING GAME' : 'the edit viewport';+ }+ }+ const small = shot.resize({ width: w, height: h, quality: 'good' });+ const size = small.getSize();+ // BGRA, row-major. Electron 42's own typings declare this `void`; it returns the+ // buffer the docs promise, so the cast is over their declaration, not over reality.+ const bitmap = small.getBitmap() as unknown as Buffer;+ const lines: string[] = [];+ for (let y = 0; y < size.height; y++) {+ let line = '';+ for (let x = 0; x < size.width; x++) {+ const i = (y * size.width + x) * 4;+ line += nearestInk(bitmap[i + 2], bitmap[i + 1], bitmap[i]);+ }+ lines.push(`${String(y).padStart(2, ' ')} ${line}`);+ }+ const px = rect ? `${Math.round(rect.width / size.width)}x${Math.round(rect.height / size.height)} css px` : 'unknown';+ return [+ `${size.width}x${size.height} colour grid of ${what} (one letter per ${px} cell).`,+ 'Palette: . black r/R red g/G green y/Y yellow b/B blue m/M magenta c/C cyan w/W grey/white'+ + ' (capitals are the bright form). Column 0 is the LEFT edge, row 0 the TOP.',+ ...lines,+ ].join('\n');+}+/*** What a play probe finds already in scope.*@@ -128,7 +208,10 @@ export function createSurfaceDriver(// The composited window, not the viewport canvas: this is the only way to// see the play realm, which renders in its own frame.const image = await requireWin().webContents.capturePage();- return image.toPNG().toString('base64');+ if (input.format !== 'grid') return image.toPNG().toString('base64');+ // The same picture, in the only form a caller without vision can read.+ const rect = await exec('window.__estellaEditor.gameRect()') as GameRect | null;+ return colorGrid(image, rect, Number(input.cols ?? 64), Number(input.rows ?? 32));}case 'play_probe': {// The play realm is an estella:// OOPIF — only a main-process frame evaldiff --git a/desktop/scripts/editor-checks/play-stepping.mjs b/desktop/scripts/editor-checks/play-stepping.mjsindex c2f6da0ff..123a30af7 100644--- a/desktop/scripts/editor-checks/play-stepping.mjs+++ b/desktop/scripts/editor-checks/play-stepping.mjs@@ -71,5 +71,17 @@ export async function run(ed) {const shot = await ed.screenshot('play-stepping');check(shot.w > 0 && shot.h > 0, 'the window did not capture');+ // And the same picture in the form a model without vision can read. Half the+ // endpoints an editor gets pointed at cannot receive a PNG, and for those this is+ // the ONLY answer to "did anything draw at all".+ const grid = await ed.json('screenshot', { format: 'grid', cols: 48, rows: 24 }, 60000);+ const rows = String(grid).split('\n').filter((l) => /^\s*\d+ /.test(l));+ check(rows.length === 24, `the grid came back ${rows.length} rows deep, not 24`);+ check(/colour grid of the RUNNING GAME/.test(String(grid)), 'the grid did not crop to the running game');+ // A grid of one repeated letter is a capture of nothing — a black frame, or a crop+ // that landed off the canvas. The example draws sprites, so it must have colour.+ const inks = new Set(rows.flatMap((l) => [...l.replace(/^\s*\d+ /, '')]));+ check(inks.size >= 3, `the grid is all one colour (${[...inks].join('')}) — nothing drew, or the crop missed`);+return check.failures;}diff --git a/desktop/shared/toolCatalog.mjs b/desktop/shared/toolCatalog.mjsindex f1a40e744..2df694350 100644--- a/desktop/shared/toolCatalog.mjs+++ b/desktop/shared/toolCatalog.mjs@@ -487,8 +487,11 @@ export const TOOLS = [+ 'so a game that looks frozen (an animation that never advances, a tween stuck at its start) is usually a background window, not a bug. `fps` is null until the first heartbeat (~0.5s after Play).',schema: obj({}), method: 'playState', args: () => [], root: 'editor' },{ name: 'screenshot',- description: 'Capture the composited editor window as a PNG (includes the play realm iframe — use this to SEE gameplay; capture_viewport only sees the edit viewport).',- schema: obj({}), op: 'screenshot', image: true },+ description: 'Capture the composited editor window — the only capture that includes the play realm, so this is how you SEE gameplay (capture_viewport sees the edit viewport only). '+ + "`format: 'grid'` returns the same picture as TEXT instead of a PNG: a coarse colour grid, cropped to the running game (or the edit viewport when nothing is playing), one letter per cell from a fixed 16-colour palette. "+ + 'That is the form to ask for if you cannot receive images — it still answers the questions no amount of reading fields back can (did anything draw at all, is it on camera, what colour did it come out, did the picture change after that input). `cols`/`rows` set the resolution (default 64x32).',+ schema: obj({ format: { type: 'string' }, cols: { type: 'number' }, rows: { type: 'number' } }),+ op: 'screenshot', image: (i) => i.format !== 'grid' },// — The editor's OWN agent, for a driver that wants to run it: an eval harness,// a dogfood session, a regression over its behaviour. `driverOnly` keeps these// out of the built-in agent's own tool list — an agent messaging itself is a@@ -647,9 +650,12 @@ export async function runTool(tool, driver, rawInput, allowWrites = true) {throw new Error(`tool ${tool.name} mutates the scene — start the server with ESTELLA_MCP_ALLOW_WRITES=1`);}const input = validate(tool.schema, rawInput);+ // `image` may be a PREDICATE: one tool answers with a picture or with text+ // depending on what the caller asked for (screenshot's `format: 'grid'`).+ const wantsImage = typeof tool.image === 'function' ? tool.image(input) : tool.image;if (tool.op) {const data = await driver.op(tool.op, input);- if (tool.image) return { content: [{ type: 'image', data, mimeType: 'image/png' }] };+ if (wantsImage) return { content: [{ type: 'image', data, mimeType: 'image/png' }] };return { content: [{ type: 'text', text: data === undefined ? 'ok' : JSON.stringify(data) }] };}// A js template may DECLINE (return null) for inputs it has nothing special@@ -658,7 +664,7 @@ export async function runTool(tool, driver, rawInput, allowWrites = true) {const js = tool.js ? tool.js(input) : null;if (js) {const data = await driver.js(js);- if (tool.image) return { content: [{ type: 'image', data, mimeType: 'image/png' }] };+ if (wantsImage) return { content: [{ type: 'image', data, mimeType: 'image/png' }] };return { content: [{ type: 'text', text: data === undefined ? 'ok' : JSON.stringify(data) }] };}const result = await driver(tool.method, tool.args(input), tool.root);diff --git a/desktop/src/engine/PlayRealm.ts b/desktop/src/engine/PlayRealm.tsindex 47476f02f..5405d6bdb 100644--- a/desktop/src/engine/PlayRealm.ts+++ b/desktop/src/engine/PlayRealm.ts@@ -304,6 +304,16 @@ export class PlayRealmInstance {});}+ /** Where the running game is on screen, in CSS pixels — the crop a capture of the+ * window has to take to be a picture of the GAME rather than of the editor around+ * it. Null when nothing is playing. */+ viewRect(): { x: number; y: number; width: number; height: number } | null {+ const box = this.iframe?.getBoundingClientRect();+ return box && box.width > 0 && box.height > 0+ ? { x: box.left, y: box.top, width: box.width, height: box.height }+ : null;+ }+/** The running game's subsystem (module) health, for the editor's Modules indicator.* Null if not ready. */subsystems(): Promise<SubsystemStatus[] | null> {diff --git a/desktop/src/main.tsx b/desktop/src/main.tsxindex 3da6fea42..708fb5665 100644--- a/desktop/src/main.tsx+++ b/desktop/src/main.tsx@@ -271,6 +271,25 @@ function buildEditorAutomation(): unknown {}},playState: () => PlayRealm.getSnapshot(),+ /**+ * Where the picture worth taking is, in CSS pixels, with the window's own size so+ * a capture (which is in device pixels) can be scaled to it: the running game if+ * one is playing, else the edit viewport's canvas.+ *+ * A capture of the whole window is mostly panels. Cropping to this is what makes a+ * COARSE capture — the text form a model without vision can actually read — carry+ * anything: at 64 columns, the editor chrome would have eaten most of them.+ */+ gameRect: () => {+ const rect = PlayRealm.viewRect()+ ?? document.getElementById('estella-viewport-canvas')?.getBoundingClientRect();+ if (!rect) return null;+ return {+ x: rect.x, y: rect.y, width: rect.width, height: rect.height,+ windowWidth: window.innerWidth, windowHeight: window.innerHeight,+ playing: PlayRealm.getSnapshot().playing,+ };+ },/** Player count for the next Play (1 = single, 2-4 = listen server + clients). */setPlayPlayers: (n: number) => useEditorStore.getState().setPlayPlayers(n),/** Advance the EDIT-realm engine n frames (headless drivers: rAF may stall). */diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.tsindex d2bcfb054..4d0b08b12 100644--- a/desktop/tests/agent-kernel.test.ts+++ b/desktop/tests/agent-kernel.test.ts@@ -423,11 +423,16 @@ describe('the agent turn', () => {// A gateway that speaks the core format may still refuse image blocks. The// agent has no way to find that out except by spending a call on a screenshot// it will then be told it cannot see.- it('tells a blind endpoint\'s agent not to rely on looking', async () => {+ //+ // What it is told changed once there was something to tell it: the answer used to+ // be "verify by reading fields back instead of by looking", which closed off the+ // only question fields cannot answer — did anything DRAW. A screenshot in text+ // form reaches any endpoint, so the blind agent is pointed at that instead.+ it('points a blind endpoint\'s agent at the picture it CAN read', async () => {const s = fakeSession([ends()]);await runTurn({ ...deps(s), acceptsImages: false }, 'how does it look', null, new AbortController().signal);expect(s.context.some((c) => /cannot carry images/i.test(c))).toBe(true);- expect(s.context.some((c) => /get_diagnostics/.test(c))).toBe(true);+ expect(s.context.some((c) => /format: 'grid'/.test(c))).toBe(true);});it('says nothing about it where images do cross', async () => {diff --git a/desktop/tests/editor-mcp-tools.test.ts b/desktop/tests/editor-mcp-tools.test.tsindex d7d7e7fcf..ea2a6a9b9 100644--- a/desktop/tests/editor-mcp-tools.test.ts+++ b/desktop/tests/editor-mcp-tools.test.ts@@ -46,6 +46,19 @@ describe('editor MCP tool registry', () => {expect(res.content[0]).toEqual({ type: 'image', data: 'cGpn', mimeType: 'image/png' });});+ it("a screenshot asked for as a grid comes back as TEXT, because that is the point", async () => {+ // Half the endpoints an editor gets pointed at cannot receive an image. Wrapped+ // as one anyway, the grid would arrive as a base64 blob in an image block and+ // reach the model as nothing at all.+ const driver = vi.fn() as unknown as { op: ReturnType<typeof vi.fn> } & ((...a: unknown[]) => unknown);+ driver.op = vi.fn(async () => '8x2 colour grid\n 0 RRRR\n 1 ....');+ const shot = toolNamed('screenshot');+ const res = await runTool(shot, driver, { format: 'grid', cols: 8, rows: 2 });+ expect(driver.op).toHaveBeenCalledWith('screenshot', { format: 'grid', cols: 8, rows: 2 });+ expect(res.content[0].type).toBe('text');+ expect(res.content[0].text).toContain('colour grid');+ });+it('runTool validates input and calls the driver with (method, args)', async () => {const driver = vi.fn(async () => 42);const setField = TOOLS.find((t: { name: string }) => t.name === 'set_field');