Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): the built-in agent can be driven, so it can be watched

yhhed3c8fc4f543
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.ts
index c92813585..7f718cd46 100644
--- a/desktop/electron/agent/kernel.ts
+++ b/desktop/electron/agent/kernel.ts
@@ -19,14 +19,18 @@ import type {
// @ts-expect-error untyped shared module
import { TOOLS, runTool, mutates, irreversible } from '../../shared/toolCatalog.mjs';
-const catalog = TOOLS as CatalogTool[];
+// `driverOnly` tools are doors for an EXTERNAL driver (a harness running the
+// editor's agent as a subject). The agent itself must not see them: a tool that
+// messages the agent, handed to the agent, is a loop with a bill attached.
+const catalog = (TOOLS as CatalogTool[]).filter((t) => !(t as { driverOnly?: boolean }).driverOnly);
const byName = new Map(catalog.map((t) => [t.name, t]));
/**
* The tools an in-editor agent may call: the built-in catalog, plus whatever
* loaded plugins contributed.
*
- * Everything built-in is offered. The coarse ESTELLA_MCP_ALLOW_WRITES door
+ * Everything built-in is offered except the driver-only doors (see `catalog`).
+ * The coarse ESTELLA_MCP_ALLOW_WRITES door
* exists because a REMOTE client has nobody to ask, and this one has the user
* right there (see the confirm gate below).
*
diff --git a/desktop/shared/toolCatalog.mjs b/desktop/shared/toolCatalog.mjs
index d5bad1566..ee0662a6a 100644
--- a/desktop/shared/toolCatalog.mjs
+++ b/desktop/shared/toolCatalog.mjs
@@ -423,6 +423,48 @@ export const TOOLS = [
{ name: 'screenshot',
description: 'Capture the composited editor window as a PNG (includes the play realm iframe — use this to SEE gameplay; capture_viewport only sees the edit viewport).',
schema: obj({}), op: 'screenshot', image: true },
+ // — The editor's OWN agent, for a driver that wants to run it: an eval harness,
+ // a dogfood session, a regression over its behaviour. `driverOnly` keeps these
+ // out of the built-in agent's own tool list — an agent messaging itself is a
+ // loop with a bill attached, and nothing it could learn that way is true.
+ { name: 'agent_send', effect: 'irreversible', driverOnly: true,
+ description: "Send a message to the editor's BUILT-IN agent (Settings › AI Agents supplies its key and model) and return its status. It works asynchronously: poll agent_status until phase is 'idle', or 'awaiting_confirm' when it is waiting on a batch preview.",
+ schema: obj({ text: { type: 'string' } }, ['text']),
+ js: (i) => `window.estella.agent.send(${JSON.stringify(i.text)})` },
+ { name: 'agent_status', driverOnly: true,
+ description: "The built-in agent's phase (idle / thinking / awaiting_confirm / error), its model, and how full its context is.",
+ schema: obj({}), js: () => `window.estella.agent.status().then(async (s) => {
+ if (s.phase !== 'awaiting_confirm') return s;
+ // What it is waiting ON, not just that it waits: a driver cannot answer a
+ // question it has to go digging through the transcript to read.
+ const t = await window.estella.agent.transcript();
+ const last = [...t].reverse().find((e) => e.type === 'awaiting_confirm');
+ return { ...s, pending: last?.request ?? null };
+ })` },
+ { name: 'agent_confirm', effect: 'irreversible', driverOnly: true,
+ description: "Answer the built-in agent's pending confirmation (agent_status reports it as `pending` while phase is 'awaiting_confirm'). `answer`: 'once' this call, 'turn' every call of that tool for the rest of the run, 'no' declined. `declined` strikes out lines of a previewed batch by index; the rest still runs.",
+ schema: obj({
+ answer: { type: 'string' }, callId: { type: 'string' },
+ declined: { type: 'array', items: { type: 'number' } },
+ }, ['answer']),
+ js: (i) => `(async () => {
+ let callId = ${JSON.stringify(i.callId ?? null)};
+ if (!callId) {
+ const t = await window.estella.agent.transcript();
+ callId = [...t].reverse().find((e) => e.type === 'awaiting_confirm')?.request?.callId ?? null;
+ }
+ if (!callId) return { ok: false, error: 'the agent is not waiting on a confirmation' };
+ await window.estella.agent.confirm(callId, ${JSON.stringify(i.answer)}, ${JSON.stringify(i.declined ?? undefined)});
+ return { ok: true, callId };
+ })()` },
+ { name: 'agent_transcript', driverOnly: true,
+ description: "The built-in agent's conversation as events (text deltas, tool calls, results, errors) — what it did and what came back. Returns `{ total, from, events }`: `tail` (default 40) is how many of the most recent to send, and `total` is how many exist, so a POLLING driver can tell what it missed rather than counting a rolling window and silently going blind.",
+ schema: obj({ tail: { type: 'number' } }),
+ js: (i) => `window.estella.agent.transcript().then((t) => {
+ const tail = ${Number(i.tail) > 0 ? Number(i.tail) : 40};
+ const from = Math.max(0, t.length - tail);
+ return { total: t.length, from, events: t.slice(from) };
+ })` },
{ name: 'play_probe', effect: 'irreversible',
description: "Evaluate JS inside the RUNNING play realm and return the result — the gameplay probe. window.__estellaPlay = { app, getComponent } for state reads; to drive gameplay input, dispatch KeyboardEvents on DOCUMENT (the engine listens there, not on window): document.dispatchEvent(new KeyboardEvent('keydown', {code:'ArrowRight'})). frame picks the realm in multiplayer previews (0 = host).",
schema: obj({ code: { type: 'string' }, frame: { type: 'number' } }, ['code']),
diff --git a/desktop/tests/agent-plugin-tools.test.ts b/desktop/tests/agent-plugin-tools.test.ts
index 7e0bab6f8..c69d7182c 100644
--- a/desktop/tests/agent-plugin-tools.test.ts
+++ b/desktop/tests/agent-plugin-tools.test.ts
@@ -28,6 +28,17 @@ describe('the catalog a session is given', () => {
expect(agentTools().some((t) => t.name === 'get_scene_tree')).toBe(true);
});
+ // The catalog also carries doors for an external DRIVER — a harness running
+ // the editor's agent as its subject. Handing those to the agent would let it
+ // message itself: a loop with a bill attached, and nothing it could learn
+ // that way is true.
+ it('withholds the driver-only doors from the agent itself', () => {
+ const names = agentTools().map((t) => t.name);
+ for (const door of ['agent_send', 'agent_status', 'agent_confirm', 'agent_transcript']) {
+ expect(names).not.toContain(door);
+ }
+ });
+
it('appends what plugins offered, after the built-ins', () => {
const merged = agentTools([tool()]);
expect(merged).toHaveLength(agentTools().length + 1);