Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): the agent turn loop, with the editor's policy in it

esenginec45c0d7ae068
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.ts
new file mode 100644
index 000000000..baa2fcb3b
--- /dev/null
+++ b/desktop/electron/agent/kernel.ts
@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file kernel.ts
+ * @brief One agent turn: ask the model, run what it asks for, feed the results
+ * back, until it stops asking. The editor policy lives here — the tool
+ * tiers, the undo checkpoint, and the verification reflex — so a
+ * provider stays a wire format and nothing more.
+ *
+ * Tool calls go through the SAME catalog and the SAME `runTool` dispatch the MCP
+ * fronts use, against the same driver. A built-in agent that reached the editor
+ * some other way would be a second definition of what an agent can do, and the
+ * two would drift; there is one.
+ */
+import type {
+ AgentEvent, CatalogTool, KernelDeps, ToolCall, ToolOutcome,
+} from './types';
+// Plain .mjs, shared with the MCP fronts — esbuild bundles it into main.
+// @ts-expect-error untyped shared module
+import { TOOLS, runTool, mutates, irreversible } from '../../shared/toolCatalog.mjs';
+
+const catalog = TOOLS as CatalogTool[];
+const byName = new Map(catalog.map((t) => [t.name, t]));
+
+/** The tools an in-editor agent may call. Everything: the coarse
+ * ESTELLA_MCP_ALLOW_WRITES door exists because a REMOTE client has nobody to
+ * ask, and this one has the user right there (see the confirm gate below). */
+export const agentTools = (): readonly CatalogTool[] => catalog;
+
+/** Why `tool` needs saying out loud, phrased for the person who will read it. */
+function confirmReason(tool: CatalogTool): string {
+ return tool.name === 'run_editor_command' || tool.name === 'play_probe'
+ ? 'runs code the agent wrote, so its effect is whatever that code does'
+ : 'writes outside the scene, and Undo cannot take it back';
+}
+
+/** Short enough to render as one transcript row; the model gets the full text. */
+function summarize(text: string): string {
+ const flat = text.replace(/\s+/g, ' ').trim();
+ return flat.length > 160 ? `${flat.slice(0, 157)}…` : flat;
+}
+
+/**
+ * MCP wraps a result as content blocks because that is its wire format. The
+ * provider wants text, and an image block wants to stay an image — flatten,
+ * keeping the distinction the model actually acts on.
+ */
+function toOutcome(id: string, result: {
+ content: Array<{ type: string; text?: string; data?: string }>;
+ isError?: boolean;
+}): ToolOutcome {
+ const parts = result.content.map((c) => (c.type === 'image' ? '[image]' : c.text ?? ''));
+ return { id, content: parts.join('\n') || 'ok', isError: result.isError === true };
+}
+
+/**
+ * Run one turn to completion.
+ *
+ * The turn is bracketed by an undo checkpoint rather than a transaction: a
+ * transaction held across model latency would lock the editor for seconds and
+ * leave a dangling one on a crash, while a checkpoint lets each tool call stay
+ * its own ordinary undo step AND the whole turn revert in one gesture
+ * (EditorHistory.mark).
+ *
+ * @returns the checkpoint, so the caller can offer that revert.
+ */
+export async function runTurn(
+ deps: KernelDeps,
+ text: string,
+ context: string | null,
+ signal: AbortSignal,
+): Promise<{ mark: unknown; steps: number }> {
+ const { driver, session, emit } = deps;
+ const mark = await driver('mark', []);
+ emit({ type: 'turn_start' });
+
+ let reason: Extract<AgentEvent, { type: 'turn_end' }>['reason'] = 'end_turn';
+ try {
+ if (context) session.pushContext(context);
+ session.pushUser(text);
+
+ // Bounded so a model that keeps calling tools cannot spin forever. Reaching
+ // it is reported as an ordinary end of turn — the work so far stands, and
+ // the person can say "keep going".
+ for (let round = 0; round < MAX_ROUNDS; round++) {
+ const calls: ToolCall[] = [];
+ let refused = false;
+
+ for await (const ev of session.step(signal)) {
+ if (ev.type === 'tool_call') calls.push(ev.call);
+ if (ev.type === 'stop' && ev.reason === 'refusal') refused = true;
+ emit(ev);
+ }
+ if (refused) { reason = 'refusal'; break; }
+ if (calls.length === 0) break;
+
+ const outcomes: ToolOutcome[] = [];
+ let wrote = false;
+ for (const call of calls) {
+ if (signal.aborted) break;
+ const { outcome, mutated } = await execute(deps, call);
+ outcomes.push(outcome);
+ wrote ||= mutated;
+ }
+ if (signal.aborted) break;
+
+ // The verification reflex. get_diagnostics is what the editor itself
+ // flags in the Details panel, and its own description calls an empty list
+ // the done signal — so after a batch of edits the agent is TOLD what it
+ // broke rather than left to remember to look. Context, not a tool result:
+ // a tool result has to answer a call the model made.
+ if (wrote) {
+ const issues = await diagnostics(deps);
+ if (issues) session.pushContext(issues);
+ }
+ session.pushToolResults(outcomes);
+ }
+ } catch (e) {
+ reason = signal.aborted ? 'aborted' : 'error';
+ if (!signal.aborted) emit({ type: 'error', message: (e as Error)?.message ?? String(e) });
+ }
+ if (signal.aborted) reason = 'aborted';
+
+ const steps = Number(await driver('stepsSince', [mark]).catch(() => 0)) || 0;
+ emit({ type: 'turn_end', steps, reason });
+ return { mark, steps };
+}
+
+const MAX_ROUNDS = 48;
+
+/** Run one call: gate it, dispatch it, and report both ways. */
+async function execute(
+ deps: KernelDeps,
+ call: ToolCall,
+): Promise<{ outcome: ToolOutcome; mutated: boolean }> {
+ const { emit, driver, confirm } = deps;
+ const tool = byName.get(call.name);
+ if (!tool) {
+ return {
+ outcome: { id: call.id, content: `no such tool: ${call.name}`, isError: true },
+ mutated: false,
+ };
+ }
+
+ const effect = tool.effect ?? 'read';
+ emit({ type: 'tool_start', call, effect });
+
+ // The one gate. `read` and `undoable` run unasked — the turn's checkpoint is
+ // the approval, and a prompt per setField would make the agent unusable.
+ // `irreversible` is where undo stops being an answer, so it is where the
+ // person has to be.
+ if (irreversible(tool)) {
+ const request = { callId: call.id, tool: call.name, reason: confirmReason(tool), input: call.input };
+ emit({ type: 'awaiting_confirm', request });
+ if (!await confirm(request)) {
+ const content = `the user declined to run ${call.name}. Do not retry it; `
+ + 'continue with what you can do without it, or say what you need.';
+ emit({ type: 'tool_end', id: call.id, ok: false, summary: 'declined' });
+ return { outcome: { id: call.id, content, isError: false }, mutated: false };
+ }
+ }
+
+ const result = await runTool(tool, driver, call.input, true) as {
+ content: Array<{ type: string; text?: string; data?: string }>;
+ isError?: boolean;
+ };
+ const outcome = toOutcome(call.id, result);
+ emit({ type: 'tool_end', id: call.id, ok: !outcome.isError, summary: summarize(outcome.content) });
+ // A failed call changed nothing worth re-verifying.
+ return { outcome, mutated: mutates(tool) && !outcome.isError };
+}
+
+/** The scene's outstanding validation issues as one line of context, or null
+ * when there are none (silence is the "still clean" signal). */
+async function diagnostics(deps: KernelDeps): Promise<string | null> {
+ try {
+ const issues = await deps.driver('getDiagnostics', []) as Array<{
+ entityName: string; component: string; field?: string; problem: string; detail: string;
+ }>;
+ const errors = issues.filter((i) => i.problem !== 'notice');
+ if (errors.length === 0) return null;
+ const lines = errors.slice(0, 12).map((i) => `- ${i.entityName}: ${i.detail}`);
+ const more = errors.length > lines.length ? `\n- …and ${errors.length - lines.length} more` : '';
+ return `The editor flags ${errors.length} problem(s) with the scene after those edits:\n`
+ + `${lines.join('\n')}${more}\nFix them before reporting the work as done.`;
+ } catch {
+ // A host without a scene open (or mid-reload) is not a turn-ending problem.
+ return null;
+ }
+}
diff --git a/desktop/electron/agent/types.ts b/desktop/electron/agent/types.ts
new file mode 100644
index 000000000..0a17158dc
--- /dev/null
+++ b/desktop/electron/agent/types.ts
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file types.ts
+ * @brief The contract between the agent kernel and a model provider, and the
+ * events the kernel streams out to the editor.
+ *
+ * The split is deliberate and is what keeps a second provider cheap: the
+ * PROVIDER owns its own conversation history in whatever shape its API wants
+ * (Anthropic requires thinking blocks echoed back unchanged, and a message array
+ * whose prefix must stay byte-identical to stay cached — neither survives being
+ * normalised into a neutral shape and rebuilt), while the KERNEL owns the loop,
+ * the permission tiers, the undo checkpoint and the verification reflex. Nothing
+ * about a vendor reaches the kernel, and no editor policy reaches a provider.
+ */
+import type { HistoryMark } from '../../src/engine/EditorHistory';
+
+/** A tool as the catalog declares it (the .mjs registry is untyped). */
+export interface CatalogTool {
+ name: string;
+ description: string;
+ schema: Record<string, unknown>;
+ effect?: 'read' | 'undoable' | 'irreversible';
+}
+
+/** One call the model wants made. */
+export interface ToolCall {
+ id: string;
+ name: string;
+ input: Record<string, unknown>;
+}
+
+/** Its outcome, in the shape a provider appends to its history. */
+export interface ToolOutcome {
+ id: string;
+ /** Text or an image the model reads back. */
+ content: string;
+ isError: boolean;
+}
+
+/** What a provider emits while ONE model call runs. */
+export type StepEvent =
+ | { type: 'text'; delta: string }
+ | { type: 'thinking'; delta: string }
+ | { type: 'tool_call'; call: ToolCall }
+ /** Why the model stopped. `refusal` is a provider-level decline, not an error. */
+ | { type: 'stop'; reason: 'end_turn' | 'tool_use' | 'refusal' | 'max_tokens' }
+ | { type: 'usage'; inputTokens: number; outputTokens: number };
+
+/**
+ * One conversation. Stateful on purpose — see the file header. The kernel
+ * appends, then asks for a step; it never reads the history back.
+ */
+export interface AgentSession {
+ /** The person's turn. */
+ pushUser(text: string): void;
+ /**
+ * Operator context for the turns from here on — what the editor is showing,
+ * what the diagnostics say. Kept OUT of the system prompt: that sits at the
+ * front of the cached prefix, and rewriting it every turn would re-bill the
+ * whole conversation.
+ */
+ pushContext(text: string): void;
+ /** Results for the calls the last step asked for, all of them, in one go. */
+ pushToolResults(outcomes: readonly ToolOutcome[]): void;
+ /** Run one model call. */
+ step(signal: AbortSignal): AsyncIterable<StepEvent>;
+}
+
+export interface AgentProvider {
+ readonly id: string;
+ /** The model this provider will run, for the transcript header. */
+ readonly model: string;
+ createSession(opts: { system: string; tools: readonly CatalogTool[] }): AgentSession;
+}
+
+/** A tool the user has to say yes to before it runs (see `effect`). */
+export interface ConfirmRequest {
+ callId: string;
+ tool: string;
+ /** Why it needs asking, already phrased for a person. */
+ reason: string;
+ input: Record<string, unknown>;
+}
+
+/** What the editor renders. A superset of what the provider emitted. */
+export type AgentEvent =
+ | StepEvent
+ | { type: 'turn_start' }
+ | { type: 'tool_start'; call: ToolCall; effect: NonNullable<CatalogTool['effect']> }
+ | { type: 'tool_end'; id: string; ok: boolean; summary: string }
+ | { type: 'awaiting_confirm'; request: ConfirmRequest }
+ /** The turn is over. `steps` is what a single Undo would take back, and 0 is
+ * the signal not to offer one. */
+ | { type: 'turn_end'; steps: number; reason: 'end_turn' | 'aborted' | 'error' | 'refusal' }
+ | { type: 'error'; message: string };
+
+/** Everything the kernel needs from outside, so it runs under a test with none
+ * of Electron present. */
+export interface KernelDeps {
+ /** The one driver every consumer shares (electron/surfaceDriver.ts). */
+ driver: {
+ (method: string, args?: readonly unknown[], root?: string): Promise<unknown>;
+ js(code: string): Promise<unknown>;
+ op(op: string, input?: Record<string, unknown>): Promise<unknown>;
+ };
+ session: AgentSession;
+ /** Ask the person. Resolving false means "declined", which the model is told. */
+ confirm(request: ConfirmRequest): Promise<boolean>;
+ emit(event: AgentEvent): void;
+}
+
+export type { HistoryMark };
diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.ts
new file mode 100644
index 000000000..ef3d508bb
--- /dev/null
+++ b/desktop/tests/agent-kernel.test.ts
@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file The agent turn loop: what runs unasked, what has to be confirmed, what
+ * the model is told afterwards, and what a turn leaves behind for the
+ * Undo affordance. Pure TS — a fake provider and a fake driver, no
+ * Electron and no network.
+ */
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { runTurn } from '../electron/agent/kernel';
+import type { AgentEvent, AgentSession, StepEvent, ToolCall, ToolOutcome } from '../electron/agent/types';
+
+/** A provider that replays scripted steps and records what it was told. */
+function fakeSession(steps: StepEvent[][]): AgentSession & {
+ context: string[]; user: string[]; results: ToolOutcome[][];
+} {
+ let at = 0;
+ const context: string[] = [];
+ const user: string[] = [];
+ const results: ToolOutcome[][] = [];
+ return {
+ context, user, results,
+ pushUser: (t) => { user.push(t); },
+ pushContext: (t) => { context.push(t); },
+ pushToolResults: (o) => { results.push([...o]); },
+ step: async function* () {
+ for (const ev of steps[at] ?? [{ type: 'stop', reason: 'end_turn' }]) yield ev;
+ at++;
+ },
+ };
+}
+
+const call = (name: string, input: Record<string, unknown> = {}): ToolCall =>
+ ({ id: `c${name}`, name, input });
+const asks = (...calls: ToolCall[]): StepEvent[] =>
+ [...calls.map((c) => ({ type: 'tool_call' as const, call: c })), { type: 'stop' as const, reason: 'tool_use' as const }];
+const ends = (): StepEvent[] => [{ type: 'stop', reason: 'end_turn' }];
+
+describe('the agent turn', () => {
+ let events: AgentEvent[];
+ let confirm: ReturnType<typeof vi.fn>;
+ let driver: ReturnType<typeof vi.fn> & { js: unknown; op: unknown };
+ let stepsSince: number;
+ let diagnostics: unknown[];
+
+ const deps = (session: AgentSession) => ({
+ driver: driver as never, session, confirm: confirm as never,
+ emit: (e: AgentEvent) => { events.push(e); },
+ });
+ const kinds = () => events.map((e) => e.type);
+ const ran = (name: string) => events.some((e) => e.type === 'tool_start' && e.call.name === name);
+
+ beforeEach(() => {
+ events = [];
+ stepsSince = 0;
+ diagnostics = [];
+ confirm = vi.fn(async () => true);
+ driver = vi.fn(async (method: string) => {
+ if (method === 'mark') return { seq: 7 };
+ if (method === 'stepsSince') return stepsSince;
+ if (method === 'getDiagnostics') return diagnostics;
+ return null;
+ }) as never;
+ (driver as { js: unknown }).js = vi.fn(async () => null);
+ (driver as { op: unknown }).op = vi.fn(async () => null);
+ });
+
+ it('brackets the turn with a checkpoint and reports what Undo would take back', async () => {
+ stepsSince = 3;
+ const s = fakeSession([ends()]);
+ const out = await runTurn(deps(s), 'hi', null, new AbortController().signal);
+ expect(out.mark).toEqual({ seq: 7 });
+ expect(out.steps).toBe(3);
+ expect(events.at(0)).toEqual({ type: 'turn_start' });
+ expect(events.at(-1)).toEqual({ type: 'turn_end', steps: 3, reason: 'end_turn' });
+ });
+
+ it('runs an undoable edit without asking — the checkpoint is the approval', async () => {
+ const s = fakeSession([asks(call('add_entity')), ends()]);
+ await runTurn(deps(s), 'add one', null, new AbortController().signal);
+ expect(confirm).not.toHaveBeenCalled();
+ expect(ran('add_entity')).toBe(true);
+ expect(kinds()).toContain('tool_end');
+ });
+
+ it('asks before anything Undo cannot take back', async () => {
+ const s = fakeSession([asks(call('save_scene')), ends()]);
+ await runTurn(deps(s), 'save it', null, new AbortController().signal);
+ expect(confirm).toHaveBeenCalledTimes(1);
+ expect(confirm.mock.calls[0][0]).toMatchObject({ tool: 'save_scene' });
+ expect(kinds()).toContain('awaiting_confirm');
+ });
+
+ it('a decline is told to the model, not raised as a failure', async () => {
+ confirm.mockResolvedValue(false);
+ const s = fakeSession([asks(call('export_game', { platform: 'web' })), ends()]);
+ await runTurn(deps(s), 'ship it', null, new AbortController().signal);
+
+ const [outcomes] = s.results;
+ expect(outcomes[0].isError).toBe(false); // a refusal is an answer, not an error
+ expect(outcomes[0].content).toContain('declined');
+ expect(outcomes[0].content).toContain('Do not retry');
+ // …and the tool never reached the editor.
+ expect(driver).not.toHaveBeenCalledWith('exportGame', expect.anything(), expect.anything());
+ });
+
+ it('feeds every result of a round back in ONE push, so parallel calls stay together', async () => {
+ const s = fakeSession([asks(call('add_entity'), call('get_scene_tree')), ends()]);
+ await runTurn(deps(s), 'do both', null, new AbortController().signal);
+ expect(s.results).toHaveLength(1);
+ expect(s.results[0]).toHaveLength(2);
+ });
+
+ it('tells the model what the editor flags after it writes', async () => {
+ diagnostics = [{
+ entityName: 'Hero', component: 'Sprite', field: 'texture',
+ problem: 'required-empty', detail: 'Sprite.texture is required but empty',
+ }];
+ const s = fakeSession([asks(call('add_entity')), ends()]);
+ await runTurn(deps(s), 'add a hero', null, new AbortController().signal);
+ expect(s.context.join('\n')).toContain('Sprite.texture is required but empty');
+ });
+
+ it('says nothing when the scene is clean — silence is the done signal', async () => {
+ const s = fakeSession([asks(call('add_entity')), ends()]);
+ await runTurn(deps(s), 'add', null, new AbortController().signal);
+ expect(s.context).toHaveLength(0);
+ });
+
+ it('does not re-verify after a read-only round', async () => {
+ const s = fakeSession([asks(call('get_scene_tree')), ends()]);
+ await runTurn(deps(s), 'look', null, new AbortController().signal);
+ expect(driver).not.toHaveBeenCalledWith('getDiagnostics', expect.anything());
+ });
+
+ it('answers an unknown tool instead of throwing the turn away', async () => {
+ const s = fakeSession([asks(call('teleport')), ends()]);
+ await runTurn(deps(s), 'go', null, new AbortController().signal);
+ expect(s.results[0][0]).toMatchObject({ isError: true });
+ expect(s.results[0][0].content).toContain('no such tool');
+ expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'end_turn' });
+ });
+
+ it('stops on abort and still reports the checkpoint', async () => {
+ stepsSince = 2;
+ const ac = new AbortController();
+ const s = fakeSession([asks(call('add_entity'), call('add_entity')), ends()]);
+ confirm.mockImplementation(async () => true);
+ driver.mockImplementation(async (method: string) => {
+ if (method === 'mark') return { seq: 7 };
+ if (method === 'stepsSince') return stepsSince;
+ if (method === 'getDiagnostics') return diagnostics;
+ ac.abort();
+ return null;
+ });
+ await runTurn(deps(s), 'lots', null, ac.signal);
+ expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'aborted', steps: 2 });
+ });
+
+ it('carries a provider refusal out as its own reason', async () => {
+ const s = fakeSession([[{ type: 'stop', reason: 'refusal' }]]);
+ await runTurn(deps(s), 'no', null, new AbortController().signal);
+ expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'refusal' });
+ });
+
+ it('puts editor context ahead of the user turn, and only when there is some', async () => {
+ const s = fakeSession([ends()]);
+ await runTurn(deps(s), 'hi', 'The open document is main.esscene.', new AbortController().signal);
+ expect(s.context[0]).toContain('main.esscene');
+ expect(s.user).toEqual(['hi']);
+ });
+});