Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): the agent notices you editing, knows the room, and knows if it is blind

esengineeea9547d8574
diff --git a/desktop/electron/agent/anthropic.ts b/desktop/electron/agent/anthropic.ts
index 96867bccd..3210dbc99 100644
--- a/desktop/electron/agent/anthropic.ts
+++ b/desktop/electron/agent/anthropic.ts
@@ -522,6 +522,8 @@ export function createAnthropicProvider(options: AnthropicOptions): AgentProvide
return {
id: dialect === 'anthropic' ? 'anthropic' : `compatible:${baseURL ?? ''}`,
model,
+ // The same condition toolResultContent substitutes on — one fact, one place.
+ acceptsImages: dialect === 'anthropic',
createSession: ({ system, tools }) => new AnthropicSession(
client,
{ model, effort, dialect, contextWindow: options.contextWindow || DEFAULT_CONTEXT_WINDOW },
diff --git a/desktop/electron/agent/host.ts b/desktop/electron/agent/host.ts
index 80bb92505..bdaf85223 100644
--- a/desktop/electron/agent/host.ts
+++ b/desktop/electron/agent/host.ts
@@ -100,6 +100,7 @@ export interface AgentHost {
export function createAgentHost(deps: AgentHostDeps): AgentHost {
let session: AgentSession | null = null;
let model: string | null = null;
+ let acceptsImages = true;
let phase: AgentPhase = 'idle';
let error: string | null = null;
let running: AbortController | null = null;
@@ -165,6 +166,7 @@ export function createAgentHost(deps: AgentHostDeps): AgentHost {
const provider = deps.provider();
session = provider.createSession({ system: SYSTEM_PROMPT, tools: agentTools() });
model = provider.model;
+ acceptsImages = provider.acceptsImages;
} catch (e) {
error = (e as Error)?.message ?? String(e);
pushStatus();
@@ -190,7 +192,7 @@ export function createAgentHost(deps: AgentHostDeps): AgentHost {
// would be a second definition of "what the agent knows".
const context = await editorContext(deps.driver).catch(() => null);
await runTurn(
- { driver: deps.driver, session: session!, model: model ?? '', confirm, emit },
+ { driver: deps.driver, session: session!, model: model ?? '', acceptsImages, confirm, emit },
text,
context,
controller.signal,
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.ts
index e61685937..ac05a45cf 100644
--- a/desktop/electron/agent/kernel.ts
+++ b/desktop/electron/agent/kernel.ts
@@ -117,7 +117,7 @@ export async function runTurn(
context: string | null,
signal: AbortSignal,
): Promise<{ mark: unknown; steps: number }> {
- const { driver, session, emit, model } = deps;
+ const { driver, session, emit, model, acceptsImages } = deps;
const mark = await driver('mark', []);
// Read before pushUser opens it: this is the index that turn will occupy.
emit({ type: 'turn_start', prompt: text, model, index: session.turnIndex });
@@ -127,14 +127,41 @@ export async function runTurn(
// an "always" that outlives the run is a permission switch nobody remembers
// flipping, which is the thing the gate exists to prevent.
const allowed = new Set<string>();
+ // Undo steps this turn is responsible for, so a jump beyond it is the person.
+ let ours = 0;
try {
if (context) session.pushContext(context);
+ if (!acceptsImages) {
+ session.pushContext(
+ 'This endpoint cannot carry images, so capture_viewport and the other screenshot '
+ + 'tools will not let you SEE anything — their frames cannot reach you. Verify with '
+ + 'get_diagnostics and by reading fields back instead of by looking.',
+ );
+ }
session.pushUser(text);
// Bounded so a model that keeps calling tools cannot spin forever. Reaching
// it is reported as an ordinary end of turn — the work so far stands, and
// the person can say "keep going".
for (let round = 0; round < MAX_ROUNDS; round++) {
+ // The editor is not frozen while a turn runs: the person can drag an
+ // entity while the model is thinking. Whatever the agent read before that
+ // is now possibly stale, and it has no way to notice — an edit made from
+ // the old reading silently overwrites the one just made by hand. Steps
+ // that appeared since our own work is enough to say SO; what changed is
+ // the scene's to answer, and telling it to re-read beats guessing.
+ if (round > 0) {
+ const now = await undoSteps(deps, mark);
+ if (now > ours) {
+ session.pushContext(
+ `While you were working, the user edited the scene themselves (${now - ours} undo `
+ + 'steps beyond your own). Anything you read before now may be out of date — '
+ + 're-read what you are about to change rather than acting on it.',
+ );
+ ours = now;
+ }
+ }
+
const calls: ToolCall[] = [];
let refused = false;
@@ -177,6 +204,8 @@ export async function runTurn(
if (issues) session.pushContext(issues);
}
session.pushToolResults(outcomes);
+ // Everything on the stack now is this turn's own doing.
+ ours = await undoSteps(deps, mark);
}
} catch (e) {
reason = signal.aborted ? 'aborted' : 'error';
@@ -184,13 +213,18 @@ export async function runTurn(
}
if (signal.aborted) reason = 'aborted';
- const steps = Number(await driver('stepsSince', [mark]).catch(() => 0)) || 0;
+ const steps = await undoSteps(deps, mark);
emit({ type: 'turn_end', steps, mark, reason });
return { mark, steps };
}
const MAX_ROUNDS = 48;
+/** Undo steps recorded since `mark`, or 0 when the editor cannot say. */
+async function undoSteps(deps: KernelDeps, mark: unknown): Promise<number> {
+ return Number(await deps.driver('stepsSince', [mark]).catch(() => 0)) || 0;
+}
+
/**
* The batch minus the lines the person struck out, or null when nothing is left.
*
diff --git a/desktop/electron/agent/prompt.ts b/desktop/electron/agent/prompt.ts
index 12854c9ac..15ec8beb9 100644
--- a/desktop/electron/agent/prompt.ts
+++ b/desktop/electron/agent/prompt.ts
@@ -31,7 +31,7 @@ Working style:
- Look before you edit. get_scene_tree, get_inspector and get_document cost one round trip and are the difference between editing the right entity and editing a plausible one.
- You can see the result: capture_viewport renders what the user is looking at. Use it when the question is visual — layout, whether something is on screen, why the scene looks wrong.
-- Prefer one apply_scene_ops over many small calls when building something: it is one undo step for the user and one round trip for you.
+- Prefer one apply_scene_ops over many small calls when building something: it is one undo step for the user and one round trip for you, and it is the ONE write the user sees BEFORE it lands — they get a line-by-line preview and can strike parts of it out. A dozen set_field calls give them no such chance, so build with a batch and save the single writes for genuinely single changes.
- Units are the editor's: 1 world unit = 1 design pixel. Y is up in the world, and UI is laid out top-down.
- Say what you did in a sentence or two. The user watched it happen — a summary of every call is noise. Speak the user's language: answer in the language they wrote to you in.`;
@@ -58,6 +58,24 @@ export async function editorContext(driver: SurfaceDriver): Promise<string | nul
}
} catch { /* no project open yet */ }
+ try {
+ // What the tools MEAN depends on the mode — a tilemap task is about cells,
+ // a UI one about anchors — and every UI coordinate is relative to the design
+ // resolution. An agent told neither has to guess, and guessing the unit
+ // convention is how positions come out an order of magnitude off.
+ const setup = await driver.js('window.__estellaEditor.editorSetup()') as {
+ mode: string; designResolution: { width: number; height: number } | null;
+ } | null;
+ if (setup?.mode) lines.push(`Editor mode: ${setup.mode}.`);
+ if (setup?.designResolution) {
+ const { width, height } = setup.designResolution;
+ lines.push(
+ `The project's design resolution is ${width}×${height}. One world unit is one design `
+ + 'pixel, so UI coordinates are in that frame.',
+ );
+ }
+ } catch { /* older window, or no project */ }
+
try {
const ids = await driver('getSelectionIds', []) as number[];
if (ids?.length) {
diff --git a/desktop/electron/agent/types.ts b/desktop/electron/agent/types.ts
index 1bfb33a68..efbe0285d 100644
--- a/desktop/electron/agent/types.ts
+++ b/desktop/electron/agent/types.ts
@@ -111,6 +111,15 @@ export interface AgentProvider {
readonly id: string;
/** The model this provider will run, for the transcript header. */
readonly model: string;
+ /**
+ * Whether a rendered frame can reach the model at all.
+ *
+ * A gateway that speaks the core format may still refuse image blocks, and
+ * the agent has no way to find out except by spending a call on a screenshot
+ * it will be told it cannot see. Stated once, in the turn's context, so it
+ * can plan around being blind instead of discovering it.
+ */
+ readonly acceptsImages: boolean;
createSession(opts: { system: string; tools: readonly CatalogTool[] }): AgentSession;
}
@@ -207,6 +216,8 @@ export interface KernelDeps {
/** What is behind `session`, for the transcript. The session abstraction has
* no name for itself, and the host is where that name is known. */
model: string;
+ /** See {@link AgentProvider.acceptsImages}. */
+ acceptsImages: boolean;
/** Ask the person. `no` is declined, which the model is told about. */
confirm(request: ConfirmRequest): Promise<ConfirmDecision>;
emit(event: AgentEvent): void;
diff --git a/desktop/src/main.tsx b/desktop/src/main.tsx
index e444b3fe7..3d4d9d62c 100644
--- a/desktop/src/main.tsx
+++ b/desktop/src/main.tsx
@@ -70,6 +70,7 @@ import { initBackgroundThrottle } from './engine/backgroundThrottle';
import { mcpStatus, subscribeMcp } from './store/McpStore';
import { attachAgentBridge, agentDriving, subscribeAgent } from './store/AgentStore';
import { guardAutomationHook } from './engine/automationGate';
+import { activeMode } from './mode/activeMode';
// Register the built-in settings (side effect) and replay persisted ones.
import './settings';
import { applySettings } from './store/settingsStore';
@@ -103,6 +104,22 @@ function documentState(): { kind: 'scene' | 'prefab'; path: string | null; name:
return { kind: 'scene', path: st?.currentScene ?? null, name: st?.name ?? null, dirty };
}
+/**
+ * What the editor is set up as, for the agent's per-turn context.
+ *
+ * The mode changes what the tools MEAN — a tilemap-mode task is about cells, a
+ * UI-mode one about anchors — and the design resolution is the frame every UI
+ * coordinate is relative to. An agent told neither guesses, and guessing the
+ * unit convention is how coordinates come out an order of magnitude off.
+ */
+function editorSetup(): { mode: string; designResolution: { width: number; height: number } | null } {
+ const res = ProjectStore.designResolution?.();
+ return {
+ mode: activeMode().id,
+ designResolution: res ? { width: res.width, height: res.height } : null,
+ };
+}
+
/** Open a scene and resolve once it is ADOPTED and readable, so a driver never has to
* poll get_scene_tree itself. Shared by the openScene and openAsset doors. */
async function openSceneAwaited(rel: string): Promise<void> {
@@ -385,6 +402,7 @@ function buildEditorAutomation(): unknown {
* different document), and no way to know whether `save` writes a scene or an asset.
*/
documentState,
+ editorSetup,
/** Leave Prefab Mode — the banner's "Back to Scene". Refuses on unsaved prefab
* changes unless `discardChanges` (see openAsset). */
exitPrefabMode: async (discardChanges = false) => {
diff --git a/desktop/tests/agent-host.test.ts b/desktop/tests/agent-host.test.ts
index 5ce173603..1d34972f6 100644
--- a/desktop/tests/agent-host.test.ts
+++ b/desktop/tests/agent-host.test.ts
@@ -36,7 +36,7 @@ function fakeProvider(steps: StepEvent[][]): AgentProvider & { session: AgentSes
at++;
},
};
- return { id: 'fake', model: 'fake-model', session, createSession: () => session };
+ return { id: 'fake', model: 'fake-model', acceptsImages: true, session, createSession: () => session };
}
describe('the agent host', () => {
diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.ts
index 0e997f8a7..a4cfbc00b 100644
--- a/desktop/tests/agent-kernel.test.ts
+++ b/desktop/tests/agent-kernel.test.ts
@@ -47,7 +47,7 @@ describe('the agent turn', () => {
let diagnostics: unknown[];
const deps = (session: AgentSession) => ({
- driver: driver as never, session, model: 'fake-model', confirm: confirm as never,
+ driver: driver as never, session, model: 'fake-model', acceptsImages: true, confirm: confirm as never,
emit: (e: AgentEvent) => { events.push(e); },
});
const kinds = () => events.map((e) => e.type);
@@ -363,4 +363,50 @@ describe('the agent turn', () => {
expect(confirm).toHaveBeenCalledTimes(1);
});
});
+
+ // The editor is not frozen while a turn runs. An edit the agent makes from a
+ // reading taken before the person dragged something silently overwrites them,
+ // and nothing else in the loop would notice.
+ describe('the person editing mid-turn', () => {
+ it('says so, once, between rounds', async () => {
+ const s = fakeSession([asks(call('add_entity')), asks(call('get_stats')), ends()]);
+ // The stack grows between the agent's own work and the next round.
+ let reported = 0;
+ driver = vi.fn(async (method: string) => {
+ if (method === 'mark') return { seq: 7 };
+ if (method === 'stepsSince') { reported += 1; return reported; }
+ if (method === 'getDiagnostics') return diagnostics;
+ return null;
+ }) as never;
+ (driver as { js: unknown }).js = vi.fn(async () => null);
+ (driver as { op: unknown }).op = vi.fn(async () => null);
+
+ await runTurn(deps(s), 'add one', null, new AbortController().signal);
+ expect(s.context.some((c) => /edited the scene themselves/i.test(c))).toBe(true);
+ expect(s.context.some((c) => /re-read/i.test(c))).toBe(true);
+ });
+
+ it('stays quiet when every step is the agent\'s own', async () => {
+ stepsSince = 3;
+ const s = fakeSession([asks(call('add_entity')), asks(call('get_stats')), ends()]);
+ await runTurn(deps(s), 'add one', null, new AbortController().signal);
+ expect(s.context.some((c) => /edited the scene themselves/i.test(c))).toBe(false);
+ });
+ });
+
+ // A gateway that speaks the core format may still refuse image blocks. The
+ // agent has no way to find that out except by spending a call on a screenshot
+ // it will then be told it cannot see.
+ it('tells a blind endpoint\'s agent not to rely on looking', async () => {
+ const s = fakeSession([ends()]);
+ await runTurn({ ...deps(s), acceptsImages: false }, 'how does it look', null, new AbortController().signal);
+ expect(s.context.some((c) => /cannot carry images/i.test(c))).toBe(true);
+ expect(s.context.some((c) => /get_diagnostics/.test(c))).toBe(true);
+ });
+
+ it('says nothing about it where images do cross', async () => {
+ const s = fakeSession([ends()]);
+ await runTurn(deps(s), 'how does it look', null, new AbortController().signal);
+ expect(s.context.some((c) => /cannot carry images/i.test(c))).toBe(false);
+ });
});