A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(editor): the agent notices you editing, knows the room, and knows if it is blind
esengineeea9547d8574
diff --git a/desktop/electron/agent/anthropic.ts b/desktop/electron/agent/anthropic.tsindex 96867bccd..3210dbc99 100644--- a/desktop/electron/agent/anthropic.ts+++ b/desktop/electron/agent/anthropic.ts@@ -522,6 +522,8 @@ export function createAnthropicProvider(options: AnthropicOptions): AgentProvidereturn {id: dialect === 'anthropic' ? 'anthropic' : `compatible:${baseURL ?? ''}`,model,+ // The same condition toolResultContent substitutes on — one fact, one place.+ acceptsImages: dialect === 'anthropic',createSession: ({ system, tools }) => new AnthropicSession(client,{ model, effort, dialect, contextWindow: options.contextWindow || DEFAULT_CONTEXT_WINDOW },diff --git a/desktop/electron/agent/host.ts b/desktop/electron/agent/host.tsindex 80bb92505..bdaf85223 100644--- a/desktop/electron/agent/host.ts+++ b/desktop/electron/agent/host.ts@@ -100,6 +100,7 @@ export interface AgentHost {export function createAgentHost(deps: AgentHostDeps): AgentHost {let session: AgentSession | null = null;let model: string | null = null;+ let acceptsImages = true;let phase: AgentPhase = 'idle';let error: string | null = null;let running: AbortController | null = null;@@ -165,6 +166,7 @@ export function createAgentHost(deps: AgentHostDeps): AgentHost {const provider = deps.provider();session = provider.createSession({ system: SYSTEM_PROMPT, tools: agentTools() });model = provider.model;+ acceptsImages = provider.acceptsImages;} catch (e) {error = (e as Error)?.message ?? String(e);pushStatus();@@ -190,7 +192,7 @@ export function createAgentHost(deps: AgentHostDeps): AgentHost {// would be a second definition of "what the agent knows".const context = await editorContext(deps.driver).catch(() => null);await runTurn(- { driver: deps.driver, session: session!, model: model ?? '', confirm, emit },+ { driver: deps.driver, session: session!, model: model ?? '', acceptsImages, confirm, emit },text,context,controller.signal,diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.tsindex e61685937..ac05a45cf 100644--- a/desktop/electron/agent/kernel.ts+++ b/desktop/electron/agent/kernel.ts@@ -117,7 +117,7 @@ export async function runTurn(context: string | null,signal: AbortSignal,): Promise<{ mark: unknown; steps: number }> {- const { driver, session, emit, model } = deps;+ const { driver, session, emit, model, acceptsImages } = deps;const mark = await driver('mark', []);// Read before pushUser opens it: this is the index that turn will occupy.emit({ type: 'turn_start', prompt: text, model, index: session.turnIndex });@@ -127,14 +127,41 @@ export async function runTurn(// an "always" that outlives the run is a permission switch nobody remembers// flipping, which is the thing the gate exists to prevent.const allowed = new Set<string>();+ // Undo steps this turn is responsible for, so a jump beyond it is the person.+ let ours = 0;try {if (context) session.pushContext(context);+ if (!acceptsImages) {+ session.pushContext(+ 'This endpoint cannot carry images, so capture_viewport and the other screenshot '+ + 'tools will not let you SEE anything — their frames cannot reach you. Verify with '+ + 'get_diagnostics and by reading fields back instead of by looking.',+ );+ }session.pushUser(text);// Bounded so a model that keeps calling tools cannot spin forever. Reaching// it is reported as an ordinary end of turn — the work so far stands, and// the person can say "keep going".for (let round = 0; round < MAX_ROUNDS; round++) {+ // The editor is not frozen while a turn runs: the person can drag an+ // entity while the model is thinking. Whatever the agent read before that+ // is now possibly stale, and it has no way to notice — an edit made from+ // the old reading silently overwrites the one just made by hand. Steps+ // that appeared since our own work is enough to say SO; what changed is+ // the scene's to answer, and telling it to re-read beats guessing.+ if (round > 0) {+ const now = await undoSteps(deps, mark);+ if (now > ours) {+ session.pushContext(+ `While you were working, the user edited the scene themselves (${now - ours} undo `+ + 'steps beyond your own). Anything you read before now may be out of date — '+ + 're-read what you are about to change rather than acting on it.',+ );+ ours = now;+ }+ }+const calls: ToolCall[] = [];let refused = false;@@ -177,6 +204,8 @@ export async function runTurn(if (issues) session.pushContext(issues);}session.pushToolResults(outcomes);+ // Everything on the stack now is this turn's own doing.+ ours = await undoSteps(deps, mark);}} catch (e) {reason = signal.aborted ? 'aborted' : 'error';@@ -184,13 +213,18 @@ export async function runTurn(}if (signal.aborted) reason = 'aborted';- const steps = Number(await driver('stepsSince', [mark]).catch(() => 0)) || 0;+ const steps = await undoSteps(deps, mark);emit({ type: 'turn_end', steps, mark, reason });return { mark, steps };}const MAX_ROUNDS = 48;+/** Undo steps recorded since `mark`, or 0 when the editor cannot say. */+async function undoSteps(deps: KernelDeps, mark: unknown): Promise<number> {+ return Number(await deps.driver('stepsSince', [mark]).catch(() => 0)) || 0;+}+/*** The batch minus the lines the person struck out, or null when nothing is left.*diff --git a/desktop/electron/agent/prompt.ts b/desktop/electron/agent/prompt.tsindex 12854c9ac..15ec8beb9 100644--- a/desktop/electron/agent/prompt.ts+++ b/desktop/electron/agent/prompt.ts@@ -31,7 +31,7 @@ Working style:- Look before you edit. get_scene_tree, get_inspector and get_document cost one round trip and are the difference between editing the right entity and editing a plausible one.- You can see the result: capture_viewport renders what the user is looking at. Use it when the question is visual — layout, whether something is on screen, why the scene looks wrong.-- Prefer one apply_scene_ops over many small calls when building something: it is one undo step for the user and one round trip for you.+- Prefer one apply_scene_ops over many small calls when building something: it is one undo step for the user and one round trip for you, and it is the ONE write the user sees BEFORE it lands — they get a line-by-line preview and can strike parts of it out. A dozen set_field calls give them no such chance, so build with a batch and save the single writes for genuinely single changes.- Units are the editor's: 1 world unit = 1 design pixel. Y is up in the world, and UI is laid out top-down.- Say what you did in a sentence or two. The user watched it happen — a summary of every call is noise. Speak the user's language: answer in the language they wrote to you in.`;@@ -58,6 +58,24 @@ export async function editorContext(driver: SurfaceDriver): Promise<string | nul}} catch { /* no project open yet */ }+ try {+ // What the tools MEAN depends on the mode — a tilemap task is about cells,+ // a UI one about anchors — and every UI coordinate is relative to the design+ // resolution. An agent told neither has to guess, and guessing the unit+ // convention is how positions come out an order of magnitude off.+ const setup = await driver.js('window.__estellaEditor.editorSetup()') as {+ mode: string; designResolution: { width: number; height: number } | null;+ } | null;+ if (setup?.mode) lines.push(`Editor mode: ${setup.mode}.`);+ if (setup?.designResolution) {+ const { width, height } = setup.designResolution;+ lines.push(+ `The project's design resolution is ${width}×${height}. One world unit is one design `+ + 'pixel, so UI coordinates are in that frame.',+ );+ }+ } catch { /* older window, or no project */ }+try {const ids = await driver('getSelectionIds', []) as number[];if (ids?.length) {diff --git a/desktop/electron/agent/types.ts b/desktop/electron/agent/types.tsindex 1bfb33a68..efbe0285d 100644--- a/desktop/electron/agent/types.ts+++ b/desktop/electron/agent/types.ts@@ -111,6 +111,15 @@ export interface AgentProvider {readonly id: string;/** The model this provider will run, for the transcript header. */readonly model: string;+ /**+ * Whether a rendered frame can reach the model at all.+ *+ * A gateway that speaks the core format may still refuse image blocks, and+ * the agent has no way to find out except by spending a call on a screenshot+ * it will be told it cannot see. Stated once, in the turn's context, so it+ * can plan around being blind instead of discovering it.+ */+ readonly acceptsImages: boolean;createSession(opts: { system: string; tools: readonly CatalogTool[] }): AgentSession;}@@ -207,6 +216,8 @@ export interface KernelDeps {/** What is behind `session`, for the transcript. The session abstraction has* no name for itself, and the host is where that name is known. */model: string;+ /** See {@link AgentProvider.acceptsImages}. */+ acceptsImages: boolean;/** Ask the person. `no` is declined, which the model is told about. */confirm(request: ConfirmRequest): Promise<ConfirmDecision>;emit(event: AgentEvent): void;diff --git a/desktop/src/main.tsx b/desktop/src/main.tsxindex e444b3fe7..3d4d9d62c 100644--- a/desktop/src/main.tsx+++ b/desktop/src/main.tsx@@ -70,6 +70,7 @@ import { initBackgroundThrottle } from './engine/backgroundThrottle';import { mcpStatus, subscribeMcp } from './store/McpStore';import { attachAgentBridge, agentDriving, subscribeAgent } from './store/AgentStore';import { guardAutomationHook } from './engine/automationGate';+import { activeMode } from './mode/activeMode';// Register the built-in settings (side effect) and replay persisted ones.import './settings';import { applySettings } from './store/settingsStore';@@ -103,6 +104,22 @@ function documentState(): { kind: 'scene' | 'prefab'; path: string | null; name:return { kind: 'scene', path: st?.currentScene ?? null, name: st?.name ?? null, dirty };}+/**+ * What the editor is set up as, for the agent's per-turn context.+ *+ * The mode changes what the tools MEAN — a tilemap-mode task is about cells, a+ * UI-mode one about anchors — and the design resolution is the frame every UI+ * coordinate is relative to. An agent told neither guesses, and guessing the+ * unit convention is how coordinates come out an order of magnitude off.+ */+function editorSetup(): { mode: string; designResolution: { width: number; height: number } | null } {+ const res = ProjectStore.designResolution?.();+ return {+ mode: activeMode().id,+ designResolution: res ? { width: res.width, height: res.height } : null,+ };+}+/** Open a scene and resolve once it is ADOPTED and readable, so a driver never has to* poll get_scene_tree itself. Shared by the openScene and openAsset doors. */async function openSceneAwaited(rel: string): Promise<void> {@@ -385,6 +402,7 @@ function buildEditorAutomation(): unknown {* different document), and no way to know whether `save` writes a scene or an asset.*/documentState,+ editorSetup,/** Leave Prefab Mode — the banner's "Back to Scene". Refuses on unsaved prefab* changes unless `discardChanges` (see openAsset). */exitPrefabMode: async (discardChanges = false) => {diff --git a/desktop/tests/agent-host.test.ts b/desktop/tests/agent-host.test.tsindex 5ce173603..1d34972f6 100644--- a/desktop/tests/agent-host.test.ts+++ b/desktop/tests/agent-host.test.ts@@ -36,7 +36,7 @@ function fakeProvider(steps: StepEvent[][]): AgentProvider & { session: AgentSesat++;},};- return { id: 'fake', model: 'fake-model', session, createSession: () => session };+ return { id: 'fake', model: 'fake-model', acceptsImages: true, session, createSession: () => session };}describe('the agent host', () => {diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.tsindex 0e997f8a7..a4cfbc00b 100644--- a/desktop/tests/agent-kernel.test.ts+++ b/desktop/tests/agent-kernel.test.ts@@ -47,7 +47,7 @@ describe('the agent turn', () => {let diagnostics: unknown[];const deps = (session: AgentSession) => ({- driver: driver as never, session, model: 'fake-model', confirm: confirm as never,+ driver: driver as never, session, model: 'fake-model', acceptsImages: true, confirm: confirm as never,emit: (e: AgentEvent) => { events.push(e); },});const kinds = () => events.map((e) => e.type);@@ -363,4 +363,50 @@ describe('the agent turn', () => {expect(confirm).toHaveBeenCalledTimes(1);});});++ // The editor is not frozen while a turn runs. An edit the agent makes from a+ // reading taken before the person dragged something silently overwrites them,+ // and nothing else in the loop would notice.+ describe('the person editing mid-turn', () => {+ it('says so, once, between rounds', async () => {+ const s = fakeSession([asks(call('add_entity')), asks(call('get_stats')), ends()]);+ // The stack grows between the agent's own work and the next round.+ let reported = 0;+ driver = vi.fn(async (method: string) => {+ if (method === 'mark') return { seq: 7 };+ if (method === 'stepsSince') { reported += 1; return reported; }+ if (method === 'getDiagnostics') return diagnostics;+ return null;+ }) as never;+ (driver as { js: unknown }).js = vi.fn(async () => null);+ (driver as { op: unknown }).op = vi.fn(async () => null);++ await runTurn(deps(s), 'add one', null, new AbortController().signal);+ expect(s.context.some((c) => /edited the scene themselves/i.test(c))).toBe(true);+ expect(s.context.some((c) => /re-read/i.test(c))).toBe(true);+ });++ it('stays quiet when every step is the agent\'s own', async () => {+ stepsSince = 3;+ const s = fakeSession([asks(call('add_entity')), asks(call('get_stats')), ends()]);+ await runTurn(deps(s), 'add one', null, new AbortController().signal);+ expect(s.context.some((c) => /edited the scene themselves/i.test(c))).toBe(false);+ });+ });++ // A gateway that speaks the core format may still refuse image blocks. The+ // agent has no way to find that out except by spending a call on a screenshot+ // it will then be told it cannot see.+ it('tells a blind endpoint\'s agent not to rely on looking', async () => {+ const s = fakeSession([ends()]);+ await runTurn({ ...deps(s), acceptsImages: false }, 'how does it look', null, new AbortController().signal);+ expect(s.context.some((c) => /cannot carry images/i.test(c))).toBe(true);+ expect(s.context.some((c) => /get_diagnostics/.test(c))).toBe(true);+ });++ it('says nothing about it where images do cross', async () => {+ const s = fakeSession([ends()]);+ await runTurn(deps(s), 'how does it look', null, new AbortController().signal);+ expect(s.context.some((c) => /cannot carry images/i.test(c))).toBe(false);+ });});