A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(editor): the agent turn loop, with the editor's policy in it
esenginec45c0d7ae068
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.tsnew file mode 100644index 000000000..baa2fcb3b--- /dev/null+++ b/desktop/electron/agent/kernel.ts@@ -0,0 +1,190 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file kernel.ts+ * @brief One agent turn: ask the model, run what it asks for, feed the results+ * back, until it stops asking. The editor policy lives here — the tool+ * tiers, the undo checkpoint, and the verification reflex — so a+ * provider stays a wire format and nothing more.+ *+ * Tool calls go through the SAME catalog and the SAME `runTool` dispatch the MCP+ * fronts use, against the same driver. A built-in agent that reached the editor+ * some other way would be a second definition of what an agent can do, and the+ * two would drift; there is one.+ */+import type {+ AgentEvent, CatalogTool, KernelDeps, ToolCall, ToolOutcome,+} from './types';+// Plain .mjs, shared with the MCP fronts — esbuild bundles it into main.+// @ts-expect-error untyped shared module+import { TOOLS, runTool, mutates, irreversible } from '../../shared/toolCatalog.mjs';++const catalog = TOOLS as CatalogTool[];+const byName = new Map(catalog.map((t) => [t.name, t]));++/** The tools an in-editor agent may call. Everything: the coarse+ * ESTELLA_MCP_ALLOW_WRITES door exists because a REMOTE client has nobody to+ * ask, and this one has the user right there (see the confirm gate below). */+export const agentTools = (): readonly CatalogTool[] => catalog;++/** Why `tool` needs saying out loud, phrased for the person who will read it. */+function confirmReason(tool: CatalogTool): string {+ return tool.name === 'run_editor_command' || tool.name === 'play_probe'+ ? 'runs code the agent wrote, so its effect is whatever that code does'+ : 'writes outside the scene, and Undo cannot take it back';+}++/** Short enough to render as one transcript row; the model gets the full text. */+function summarize(text: string): string {+ const flat = text.replace(/\s+/g, ' ').trim();+ return flat.length > 160 ? `${flat.slice(0, 157)}…` : flat;+}++/**+ * MCP wraps a result as content blocks because that is its wire format. The+ * provider wants text, and an image block wants to stay an image — flatten,+ * keeping the distinction the model actually acts on.+ */+function toOutcome(id: string, result: {+ content: Array<{ type: string; text?: string; data?: string }>;+ isError?: boolean;+}): ToolOutcome {+ const parts = result.content.map((c) => (c.type === 'image' ? '[image]' : c.text ?? ''));+ return { id, content: parts.join('\n') || 'ok', isError: result.isError === true };+}++/**+ * Run one turn to completion.+ *+ * The turn is bracketed by an undo checkpoint rather than a transaction: a+ * transaction held across model latency would lock the editor for seconds and+ * leave a dangling one on a crash, while a checkpoint lets each tool call stay+ * its own ordinary undo step AND the whole turn revert in one gesture+ * (EditorHistory.mark).+ *+ * @returns the checkpoint, so the caller can offer that revert.+ */+export async function runTurn(+ deps: KernelDeps,+ text: string,+ context: string | null,+ signal: AbortSignal,+): Promise<{ mark: unknown; steps: number }> {+ const { driver, session, emit } = deps;+ const mark = await driver('mark', []);+ emit({ type: 'turn_start' });++ let reason: Extract<AgentEvent, { type: 'turn_end' }>['reason'] = 'end_turn';+ try {+ if (context) session.pushContext(context);+ session.pushUser(text);++ // Bounded so a model that keeps calling tools cannot spin forever. Reaching+ // it is reported as an ordinary end of turn — the work so far stands, and+ // the person can say "keep going".+ for (let round = 0; round < MAX_ROUNDS; round++) {+ const calls: ToolCall[] = [];+ let refused = false;++ for await (const ev of session.step(signal)) {+ if (ev.type === 'tool_call') calls.push(ev.call);+ if (ev.type === 'stop' && ev.reason === 'refusal') refused = true;+ emit(ev);+ }+ if (refused) { reason = 'refusal'; break; }+ if (calls.length === 0) break;++ const outcomes: ToolOutcome[] = [];+ let wrote = false;+ for (const call of calls) {+ if (signal.aborted) break;+ const { outcome, mutated } = await execute(deps, call);+ outcomes.push(outcome);+ wrote ||= mutated;+ }+ if (signal.aborted) break;++ // The verification reflex. get_diagnostics is what the editor itself+ // flags in the Details panel, and its own description calls an empty list+ // the done signal — so after a batch of edits the agent is TOLD what it+ // broke rather than left to remember to look. Context, not a tool result:+ // a tool result has to answer a call the model made.+ if (wrote) {+ const issues = await diagnostics(deps);+ if (issues) session.pushContext(issues);+ }+ session.pushToolResults(outcomes);+ }+ } catch (e) {+ reason = signal.aborted ? 'aborted' : 'error';+ if (!signal.aborted) emit({ type: 'error', message: (e as Error)?.message ?? String(e) });+ }+ if (signal.aborted) reason = 'aborted';++ const steps = Number(await driver('stepsSince', [mark]).catch(() => 0)) || 0;+ emit({ type: 'turn_end', steps, reason });+ return { mark, steps };+}++const MAX_ROUNDS = 48;++/** Run one call: gate it, dispatch it, and report both ways. */+async function execute(+ deps: KernelDeps,+ call: ToolCall,+): Promise<{ outcome: ToolOutcome; mutated: boolean }> {+ const { emit, driver, confirm } = deps;+ const tool = byName.get(call.name);+ if (!tool) {+ return {+ outcome: { id: call.id, content: `no such tool: ${call.name}`, isError: true },+ mutated: false,+ };+ }++ const effect = tool.effect ?? 'read';+ emit({ type: 'tool_start', call, effect });++ // The one gate. `read` and `undoable` run unasked — the turn's checkpoint is+ // the approval, and a prompt per setField would make the agent unusable.+ // `irreversible` is where undo stops being an answer, so it is where the+ // person has to be.+ if (irreversible(tool)) {+ const request = { callId: call.id, tool: call.name, reason: confirmReason(tool), input: call.input };+ emit({ type: 'awaiting_confirm', request });+ if (!await confirm(request)) {+ const content = `the user declined to run ${call.name}. Do not retry it; `+ + 'continue with what you can do without it, or say what you need.';+ emit({ type: 'tool_end', id: call.id, ok: false, summary: 'declined' });+ return { outcome: { id: call.id, content, isError: false }, mutated: false };+ }+ }++ const result = await runTool(tool, driver, call.input, true) as {+ content: Array<{ type: string; text?: string; data?: string }>;+ isError?: boolean;+ };+ const outcome = toOutcome(call.id, result);+ emit({ type: 'tool_end', id: call.id, ok: !outcome.isError, summary: summarize(outcome.content) });+ // A failed call changed nothing worth re-verifying.+ return { outcome, mutated: mutates(tool) && !outcome.isError };+}++/** The scene's outstanding validation issues as one line of context, or null+ * when there are none (silence is the "still clean" signal). */+async function diagnostics(deps: KernelDeps): Promise<string | null> {+ try {+ const issues = await deps.driver('getDiagnostics', []) as Array<{+ entityName: string; component: string; field?: string; problem: string; detail: string;+ }>;+ const errors = issues.filter((i) => i.problem !== 'notice');+ if (errors.length === 0) return null;+ const lines = errors.slice(0, 12).map((i) => `- ${i.entityName}: ${i.detail}`);+ const more = errors.length > lines.length ? `\n- …and ${errors.length - lines.length} more` : '';+ return `The editor flags ${errors.length} problem(s) with the scene after those edits:\n`+ + `${lines.join('\n')}${more}\nFix them before reporting the work as done.`;+ } catch {+ // A host without a scene open (or mid-reload) is not a turn-ending problem.+ return null;+ }+}diff --git a/desktop/electron/agent/types.ts b/desktop/electron/agent/types.tsnew file mode 100644index 000000000..0a17158dc--- /dev/null+++ b/desktop/electron/agent/types.ts@@ -0,0 +1,113 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file types.ts+ * @brief The contract between the agent kernel and a model provider, and the+ * events the kernel streams out to the editor.+ *+ * The split is deliberate and is what keeps a second provider cheap: the+ * PROVIDER owns its own conversation history in whatever shape its API wants+ * (Anthropic requires thinking blocks echoed back unchanged, and a message array+ * whose prefix must stay byte-identical to stay cached — neither survives being+ * normalised into a neutral shape and rebuilt), while the KERNEL owns the loop,+ * the permission tiers, the undo checkpoint and the verification reflex. Nothing+ * about a vendor reaches the kernel, and no editor policy reaches a provider.+ */+import type { HistoryMark } from '../../src/engine/EditorHistory';++/** A tool as the catalog declares it (the .mjs registry is untyped). */+export interface CatalogTool {+ name: string;+ description: string;+ schema: Record<string, unknown>;+ effect?: 'read' | 'undoable' | 'irreversible';+}++/** One call the model wants made. */+export interface ToolCall {+ id: string;+ name: string;+ input: Record<string, unknown>;+}++/** Its outcome, in the shape a provider appends to its history. */+export interface ToolOutcome {+ id: string;+ /** Text or an image the model reads back. */+ content: string;+ isError: boolean;+}++/** What a provider emits while ONE model call runs. */+export type StepEvent =+ | { type: 'text'; delta: string }+ | { type: 'thinking'; delta: string }+ | { type: 'tool_call'; call: ToolCall }+ /** Why the model stopped. `refusal` is a provider-level decline, not an error. */+ | { type: 'stop'; reason: 'end_turn' | 'tool_use' | 'refusal' | 'max_tokens' }+ | { type: 'usage'; inputTokens: number; outputTokens: number };++/**+ * One conversation. Stateful on purpose — see the file header. The kernel+ * appends, then asks for a step; it never reads the history back.+ */+export interface AgentSession {+ /** The person's turn. */+ pushUser(text: string): void;+ /**+ * Operator context for the turns from here on — what the editor is showing,+ * what the diagnostics say. Kept OUT of the system prompt: that sits at the+ * front of the cached prefix, and rewriting it every turn would re-bill the+ * whole conversation.+ */+ pushContext(text: string): void;+ /** Results for the calls the last step asked for, all of them, in one go. */+ pushToolResults(outcomes: readonly ToolOutcome[]): void;+ /** Run one model call. */+ step(signal: AbortSignal): AsyncIterable<StepEvent>;+}++export interface AgentProvider {+ readonly id: string;+ /** The model this provider will run, for the transcript header. */+ readonly model: string;+ createSession(opts: { system: string; tools: readonly CatalogTool[] }): AgentSession;+}++/** A tool the user has to say yes to before it runs (see `effect`). */+export interface ConfirmRequest {+ callId: string;+ tool: string;+ /** Why it needs asking, already phrased for a person. */+ reason: string;+ input: Record<string, unknown>;+}++/** What the editor renders. A superset of what the provider emitted. */+export type AgentEvent =+ | StepEvent+ | { type: 'turn_start' }+ | { type: 'tool_start'; call: ToolCall; effect: NonNullable<CatalogTool['effect']> }+ | { type: 'tool_end'; id: string; ok: boolean; summary: string }+ | { type: 'awaiting_confirm'; request: ConfirmRequest }+ /** The turn is over. `steps` is what a single Undo would take back, and 0 is+ * the signal not to offer one. */+ | { type: 'turn_end'; steps: number; reason: 'end_turn' | 'aborted' | 'error' | 'refusal' }+ | { type: 'error'; message: string };++/** Everything the kernel needs from outside, so it runs under a test with none+ * of Electron present. */+export interface KernelDeps {+ /** The one driver every consumer shares (electron/surfaceDriver.ts). */+ driver: {+ (method: string, args?: readonly unknown[], root?: string): Promise<unknown>;+ js(code: string): Promise<unknown>;+ op(op: string, input?: Record<string, unknown>): Promise<unknown>;+ };+ session: AgentSession;+ /** Ask the person. Resolving false means "declined", which the model is told. */+ confirm(request: ConfirmRequest): Promise<boolean>;+ emit(event: AgentEvent): void;+}++export type { HistoryMark };diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.tsnew file mode 100644index 000000000..ef3d508bb--- /dev/null+++ b/desktop/tests/agent-kernel.test.ts@@ -0,0 +1,172 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file The agent turn loop: what runs unasked, what has to be confirmed, what+ * the model is told afterwards, and what a turn leaves behind for the+ * Undo affordance. Pure TS — a fake provider and a fake driver, no+ * Electron and no network.+ */+import { describe, it, expect, beforeEach, vi } from 'vitest';+import { runTurn } from '../electron/agent/kernel';+import type { AgentEvent, AgentSession, StepEvent, ToolCall, ToolOutcome } from '../electron/agent/types';++/** A provider that replays scripted steps and records what it was told. */+function fakeSession(steps: StepEvent[][]): AgentSession & {+ context: string[]; user: string[]; results: ToolOutcome[][];+} {+ let at = 0;+ const context: string[] = [];+ const user: string[] = [];+ const results: ToolOutcome[][] = [];+ return {+ context, user, results,+ pushUser: (t) => { user.push(t); },+ pushContext: (t) => { context.push(t); },+ pushToolResults: (o) => { results.push([...o]); },+ step: async function* () {+ for (const ev of steps[at] ?? [{ type: 'stop', reason: 'end_turn' }]) yield ev;+ at++;+ },+ };+}++const call = (name: string, input: Record<string, unknown> = {}): ToolCall =>+ ({ id: `c${name}`, name, input });+const asks = (...calls: ToolCall[]): StepEvent[] =>+ [...calls.map((c) => ({ type: 'tool_call' as const, call: c })), { type: 'stop' as const, reason: 'tool_use' as const }];+const ends = (): StepEvent[] => [{ type: 'stop', reason: 'end_turn' }];++describe('the agent turn', () => {+ let events: AgentEvent[];+ let confirm: ReturnType<typeof vi.fn>;+ let driver: ReturnType<typeof vi.fn> & { js: unknown; op: unknown };+ let stepsSince: number;+ let diagnostics: unknown[];++ const deps = (session: AgentSession) => ({+ driver: driver as never, session, confirm: confirm as never,+ emit: (e: AgentEvent) => { events.push(e); },+ });+ const kinds = () => events.map((e) => e.type);+ const ran = (name: string) => events.some((e) => e.type === 'tool_start' && e.call.name === name);++ beforeEach(() => {+ events = [];+ stepsSince = 0;+ diagnostics = [];+ confirm = vi.fn(async () => true);+ driver = vi.fn(async (method: string) => {+ if (method === 'mark') return { seq: 7 };+ if (method === 'stepsSince') return stepsSince;+ if (method === 'getDiagnostics') return diagnostics;+ return null;+ }) as never;+ (driver as { js: unknown }).js = vi.fn(async () => null);+ (driver as { op: unknown }).op = vi.fn(async () => null);+ });++ it('brackets the turn with a checkpoint and reports what Undo would take back', async () => {+ stepsSince = 3;+ const s = fakeSession([ends()]);+ const out = await runTurn(deps(s), 'hi', null, new AbortController().signal);+ expect(out.mark).toEqual({ seq: 7 });+ expect(out.steps).toBe(3);+ expect(events.at(0)).toEqual({ type: 'turn_start' });+ expect(events.at(-1)).toEqual({ type: 'turn_end', steps: 3, reason: 'end_turn' });+ });++ it('runs an undoable edit without asking — the checkpoint is the approval', async () => {+ const s = fakeSession([asks(call('add_entity')), ends()]);+ await runTurn(deps(s), 'add one', null, new AbortController().signal);+ expect(confirm).not.toHaveBeenCalled();+ expect(ran('add_entity')).toBe(true);+ expect(kinds()).toContain('tool_end');+ });++ it('asks before anything Undo cannot take back', async () => {+ const s = fakeSession([asks(call('save_scene')), ends()]);+ await runTurn(deps(s), 'save it', null, new AbortController().signal);+ expect(confirm).toHaveBeenCalledTimes(1);+ expect(confirm.mock.calls[0][0]).toMatchObject({ tool: 'save_scene' });+ expect(kinds()).toContain('awaiting_confirm');+ });++ it('a decline is told to the model, not raised as a failure', async () => {+ confirm.mockResolvedValue(false);+ const s = fakeSession([asks(call('export_game', { platform: 'web' })), ends()]);+ await runTurn(deps(s), 'ship it', null, new AbortController().signal);++ const [outcomes] = s.results;+ expect(outcomes[0].isError).toBe(false); // a refusal is an answer, not an error+ expect(outcomes[0].content).toContain('declined');+ expect(outcomes[0].content).toContain('Do not retry');+ // …and the tool never reached the editor.+ expect(driver).not.toHaveBeenCalledWith('exportGame', expect.anything(), expect.anything());+ });++ it('feeds every result of a round back in ONE push, so parallel calls stay together', async () => {+ const s = fakeSession([asks(call('add_entity'), call('get_scene_tree')), ends()]);+ await runTurn(deps(s), 'do both', null, new AbortController().signal);+ expect(s.results).toHaveLength(1);+ expect(s.results[0]).toHaveLength(2);+ });++ it('tells the model what the editor flags after it writes', async () => {+ diagnostics = [{+ entityName: 'Hero', component: 'Sprite', field: 'texture',+ problem: 'required-empty', detail: 'Sprite.texture is required but empty',+ }];+ const s = fakeSession([asks(call('add_entity')), ends()]);+ await runTurn(deps(s), 'add a hero', null, new AbortController().signal);+ expect(s.context.join('\n')).toContain('Sprite.texture is required but empty');+ });++ it('says nothing when the scene is clean — silence is the done signal', async () => {+ const s = fakeSession([asks(call('add_entity')), ends()]);+ await runTurn(deps(s), 'add', null, new AbortController().signal);+ expect(s.context).toHaveLength(0);+ });++ it('does not re-verify after a read-only round', async () => {+ const s = fakeSession([asks(call('get_scene_tree')), ends()]);+ await runTurn(deps(s), 'look', null, new AbortController().signal);+ expect(driver).not.toHaveBeenCalledWith('getDiagnostics', expect.anything());+ });++ it('answers an unknown tool instead of throwing the turn away', async () => {+ const s = fakeSession([asks(call('teleport')), ends()]);+ await runTurn(deps(s), 'go', null, new AbortController().signal);+ expect(s.results[0][0]).toMatchObject({ isError: true });+ expect(s.results[0][0].content).toContain('no such tool');+ expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'end_turn' });+ });++ it('stops on abort and still reports the checkpoint', async () => {+ stepsSince = 2;+ const ac = new AbortController();+ const s = fakeSession([asks(call('add_entity'), call('add_entity')), ends()]);+ confirm.mockImplementation(async () => true);+ driver.mockImplementation(async (method: string) => {+ if (method === 'mark') return { seq: 7 };+ if (method === 'stepsSince') return stepsSince;+ if (method === 'getDiagnostics') return diagnostics;+ ac.abort();+ return null;+ });+ await runTurn(deps(s), 'lots', null, ac.signal);+ expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'aborted', steps: 2 });+ });++ it('carries a provider refusal out as its own reason', async () => {+ const s = fakeSession([[{ type: 'stop', reason: 'refusal' }]]);+ await runTurn(deps(s), 'no', null, new AbortController().signal);+ expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'refusal' });+ });++ it('puts editor context ahead of the user turn, and only when there is some', async () => {+ const s = fakeSession([ends()]);+ await runTurn(deps(s), 'hi', 'The open document is main.esscene.', new AbortController().signal);+ expect(s.context[0]).toContain('main.esscene');+ expect(s.user).toEqual(['hi']);+ });+});