A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(editor): the built-in agent can be driven, so it can be watched
yhhed3c8fc4f543
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.tsindex c92813585..7f718cd46 100644--- a/desktop/electron/agent/kernel.ts+++ b/desktop/electron/agent/kernel.ts@@ -19,14 +19,18 @@ import type {// @ts-expect-error untyped shared moduleimport { TOOLS, runTool, mutates, irreversible } from '../../shared/toolCatalog.mjs';-const catalog = TOOLS as CatalogTool[];+// `driverOnly` tools are doors for an EXTERNAL driver (a harness running the+// editor's agent as a subject). The agent itself must not see them: a tool that+// messages the agent, handed to the agent, is a loop with a bill attached.+const catalog = (TOOLS as CatalogTool[]).filter((t) => !(t as { driverOnly?: boolean }).driverOnly);const byName = new Map(catalog.map((t) => [t.name, t]));/*** The tools an in-editor agent may call: the built-in catalog, plus whatever* loaded plugins contributed.*- * Everything built-in is offered. The coarse ESTELLA_MCP_ALLOW_WRITES door+ * Everything built-in is offered except the driver-only doors (see `catalog`).+ * The coarse ESTELLA_MCP_ALLOW_WRITES door* exists because a REMOTE client has nobody to ask, and this one has the user* right there (see the confirm gate below).*diff --git a/desktop/shared/toolCatalog.mjs b/desktop/shared/toolCatalog.mjsindex d5bad1566..ee0662a6a 100644--- a/desktop/shared/toolCatalog.mjs+++ b/desktop/shared/toolCatalog.mjs@@ -423,6 +423,48 @@ export const TOOLS = [{ name: 'screenshot',description: 'Capture the composited editor window as a PNG (includes the play realm iframe — use this to SEE gameplay; capture_viewport only sees the edit viewport).',schema: obj({}), op: 'screenshot', image: true },+ // — The editor's OWN agent, for a driver that wants to run it: an eval harness,+ // a dogfood session, a regression over its behaviour. `driverOnly` keeps these+ // out of the built-in agent's own tool list — an agent messaging itself is a+ // loop with a bill attached, and nothing it could learn that way is true.+ { name: 'agent_send', effect: 'irreversible', driverOnly: true,+ description: "Send a message to the editor's BUILT-IN agent (Settings › AI Agents supplies its key and model) and return its status. It works asynchronously: poll agent_status until phase is 'idle', or 'awaiting_confirm' when it is waiting on a batch preview.",+ schema: obj({ text: { type: 'string' } }, ['text']),+ js: (i) => `window.estella.agent.send(${JSON.stringify(i.text)})` },+ { name: 'agent_status', driverOnly: true,+ description: "The built-in agent's phase (idle / thinking / awaiting_confirm / error), its model, and how full its context is.",+ schema: obj({}), js: () => `window.estella.agent.status().then(async (s) => {+ if (s.phase !== 'awaiting_confirm') return s;+ // What it is waiting ON, not just that it waits: a driver cannot answer a+ // question it has to go digging through the transcript to read.+ const t = await window.estella.agent.transcript();+ const last = [...t].reverse().find((e) => e.type === 'awaiting_confirm');+ return { ...s, pending: last?.request ?? null };+ })` },+ { name: 'agent_confirm', effect: 'irreversible', driverOnly: true,+ description: "Answer the built-in agent's pending confirmation (agent_status reports it as `pending` while phase is 'awaiting_confirm'). `answer`: 'once' this call, 'turn' every call of that tool for the rest of the run, 'no' declined. `declined` strikes out lines of a previewed batch by index; the rest still runs.",+ schema: obj({+ answer: { type: 'string' }, callId: { type: 'string' },+ declined: { type: 'array', items: { type: 'number' } },+ }, ['answer']),+ js: (i) => `(async () => {+ let callId = ${JSON.stringify(i.callId ?? null)};+ if (!callId) {+ const t = await window.estella.agent.transcript();+ callId = [...t].reverse().find((e) => e.type === 'awaiting_confirm')?.request?.callId ?? null;+ }+ if (!callId) return { ok: false, error: 'the agent is not waiting on a confirmation' };+ await window.estella.agent.confirm(callId, ${JSON.stringify(i.answer)}, ${JSON.stringify(i.declined ?? undefined)});+ return { ok: true, callId };+ })()` },+ { name: 'agent_transcript', driverOnly: true,+ description: "The built-in agent's conversation as events (text deltas, tool calls, results, errors) — what it did and what came back. Returns `{ total, from, events }`: `tail` (default 40) is how many of the most recent to send, and `total` is how many exist, so a POLLING driver can tell what it missed rather than counting a rolling window and silently going blind.",+ schema: obj({ tail: { type: 'number' } }),+ js: (i) => `window.estella.agent.transcript().then((t) => {+ const tail = ${Number(i.tail) > 0 ? Number(i.tail) : 40};+ const from = Math.max(0, t.length - tail);+ return { total: t.length, from, events: t.slice(from) };+ })` },{ name: 'play_probe', effect: 'irreversible',description: "Evaluate JS inside the RUNNING play realm and return the result — the gameplay probe. window.__estellaPlay = { app, getComponent } for state reads; to drive gameplay input, dispatch KeyboardEvents on DOCUMENT (the engine listens there, not on window): document.dispatchEvent(new KeyboardEvent('keydown', {code:'ArrowRight'})). frame picks the realm in multiplayer previews (0 = host).",schema: obj({ code: { type: 'string' }, frame: { type: 'number' } }, ['code']),diff --git a/desktop/tests/agent-plugin-tools.test.ts b/desktop/tests/agent-plugin-tools.test.tsindex 7e0bab6f8..c69d7182c 100644--- a/desktop/tests/agent-plugin-tools.test.ts+++ b/desktop/tests/agent-plugin-tools.test.ts@@ -28,6 +28,17 @@ describe('the catalog a session is given', () => {expect(agentTools().some((t) => t.name === 'get_scene_tree')).toBe(true);});+ // The catalog also carries doors for an external DRIVER — a harness running+ // the editor's agent as its subject. Handing those to the agent would let it+ // message itself: a loop with a bill attached, and nothing it could learn+ // that way is true.+ it('withholds the driver-only doors from the agent itself', () => {+ const names = agentTools().map((t) => t.name);+ for (const door of ['agent_send', 'agent_status', 'agent_confirm', 'agent_transcript']) {+ expect(names).not.toContain(door);+ }+ });+it('appends what plugins offered, after the built-ins', () => {const merged = agentTools([tool()]);expect(merged).toHaveLength(agentTools().length + 1);