Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(agent): a turn looks at what it built before it reports it

esengine8d3a57b341ef
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.ts
index 7f718cd46..983a4ce22 100644
--- a/desktop/electron/agent/kernel.ts
+++ b/desktop/electron/agent/kernel.ts
@@ -193,6 +193,11 @@ export async function runTurn(
session.pushUser(text, images);
// Bounded so a model that keeps calling tools cannot spin forever.
+ // Turn-level, not round-level: "did this turn build anything" and "did it
+ // ever look" are questions about the whole turn.
+ let builtSomething = false;
+ let sawPixels = false;
+ let askedToLook = false;
let round = 0;
for (; round < MAX_ROUNDS; round++) {
// The editor is not frozen while a turn runs: the person can drag an
@@ -213,6 +218,16 @@ export async function runTurn(
}
}
+ // Running out of room is something the model can act on, but only if it
+ // learns before the last round rather than at it.
+ if (round === MAX_ROUNDS - ROUNDS_WARNING) {
+ session.pushContext(
+ `You have ${ROUNDS_WARNING} tool rounds left in this turn before it is cut off. `
+ + 'Land what you have: finish the step you are on, leave the project in a working '
+ + 'state, and tell the user where things stand and what is left.',
+ );
+ }
+
const calls: ToolCall[] = [];
let refused = false;
@@ -222,7 +237,25 @@ export async function runTurn(
emit(ev);
}
if (refused) { reason = 'refusal'; break; }
- if (calls.length === 0) break;
+ // No calls means it is about to answer. If it built something and never
+ // once looked at it, that answer is a report on work nobody has seen —
+ // which is how a board half off camera gets delivered as a finished game,
+ // with every write compiling and every diagnostic clean. Asked ONCE per
+ // turn: a second ask would be an argument, and the model may have a reason.
+ if (calls.length === 0) {
+ if (builtSomething && !sawPixels && !askedToLook) {
+ askedToLook = true;
+ session.pushContext(
+ 'You changed the scene this turn and have not looked at it once. Diagnostics only '
+ + 'cover what the editor can name — whether the content is ON CAMERA, whether it '
+ + 'reads, whether it is where you meant, are all things only the picture answers. '
+ + 'capture_viewport now; if it is something that has to be PLAYED, toggle_play, '
+ + 'drive it with play_input and screenshot that. Then fix what you see, or report.',
+ );
+ continue;
+ }
+ break;
+ }
// Reads have no side effects and no order among themselves, so a run of
// them goes out together — three lookups become one wait instead of three.
@@ -241,6 +274,10 @@ export async function runTurn(
outcomes.push(done.outcome);
wrote ||= done.mutated;
}
+ for (const c of calls.slice(i, i + batch.length)) {
+ if (LOOKING_TOOLS.has(c.name)) sawPixels = true;
+ }
+ builtSomething ||= wrote;
i += batch.length;
}
if (signal.aborted) break;
@@ -274,7 +311,26 @@ export async function runTurn(
return { mark, steps };
}
-const MAX_ROUNDS = 48;
+/**
+ * The backstop against a turn that never ends — a model retrying a call that
+ * always fails would otherwise spend a balance before anyone noticed.
+ *
+ * Held high because the failure it prevents is rare and the one it CAUSES is
+ * not: building a small game takes some seventy tool calls, and at 48 both
+ * dogfood runs were cut off mid-build. A turn that stops here has spent real
+ * money to deliver half a thing.
+ */
+const MAX_ROUNDS = 128;
+
+/**
+ * How many rounds from the cap the model is told it is running out.
+ *
+ * The cap on its own truncates: the round where it would have said "the board is
+ * built, here is how to play it" is the round it never gets. Warned, it can land
+ * the work it has — which is the difference between an unfinished turn and an
+ * unfinished turn nobody explained.
+ */
+const ROUNDS_WARNING = 8;
/** Undo steps recorded since `mark`, or 0 when the editor cannot say. */
async function undoSteps(deps: KernelDeps, mark: unknown): Promise<number> {
@@ -416,6 +472,10 @@ async function execute(
return { outcome, mutated: mutates(tool) && !outcome.isError };
}
+/** The tools that put PIXELS in front of the model — the only answer to "does it
+ * look right", which no diagnostic can give. */
+const LOOKING_TOOLS = new Set(['capture_viewport', 'screenshot']);
+
/** The scene's outstanding validation issues as one line of context, or null
* when there are none (silence is the "still clean" signal). */
async function diagnostics(deps: KernelDeps): Promise<string | null> {
diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.ts
index 63d063eb2..4bece8666 100644
--- a/desktop/tests/agent-kernel.test.ts
+++ b/desktop/tests/agent-kernel.test.ts
@@ -143,7 +143,10 @@ describe('the agent turn', () => {
});
it('says nothing when the scene is clean — silence is the done signal', async () => {
- const s = fakeSession([asks(call('add_entity')), ends()]);
+ // A clean sweep adds no diagnostics line. (The turn is still asked to LOOK
+ // once, which is a different reflex with its own cases below — this one is
+ // about what the DIAGNOSTICS say, so it looks first and leaves that quiet.)
+ const s = fakeSession([asks(call('add_entity')), asks(call('capture_viewport')), ends()]);
await runTurn(deps(s), 'add', null, new AbortController().signal);
expect(s.context).toHaveLength(0);
});
@@ -188,7 +191,9 @@ describe('the agent turn', () => {
// an end_turn here looks exactly like a run that finished, and the sentence
// that would have told you otherwise is the one it never got to write.
it('says the step budget ran out rather than reporting a finished run', async () => {
- const s = fakeSession(Array.from({ length: 60 }, () => asks(call('get_scene_tree'))));
+ // Enough steps to outlast any cap: writing the cap's number here is how a
+ // test starts failing for the raise it was meant to be indifferent to.
+ const s = fakeSession(Array.from({ length: 400 }, () => asks(call('get_scene_tree'))));
await runTurn(deps(s), 'build the whole game', null, new AbortController().signal);
expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'max_rounds' });
});
@@ -285,7 +290,12 @@ describe('the agent turn', () => {
// the whole reason it is not persisted.
it('expires with the run', async () => {
confirm.mockResolvedValue({ answer: 'turn' });
- const s = fakeSession([asks(call('save_scene')), ends(), asks(call('save_scene')), ends()]);
+ // capture_viewport in each turn: a turn that wrote and never looked spends
+ // an extra round being asked to, which would shift the scripted steps.
+ const s = fakeSession([
+ asks(call('save_scene')), asks(call('capture_viewport')), ends(),
+ asks(call('save_scene')), asks(call('capture_viewport')), ends(),
+ ]);
await runTurn(deps(s), 'save', null, new AbortController().signal);
await runTurn(deps(s), 'save again', null, new AbortController().signal);
expect(confirm).toHaveBeenCalledTimes(2);
@@ -510,3 +520,92 @@ describe('the agent turn', () => {
});
});
});
+
+describe('looking before reporting', () => {
+ let events: AgentEvent[];
+ let driver: ReturnType<typeof vi.fn> & { js: unknown; op: unknown };
+
+ const deps = (session: AgentSession) => ({
+ driver: driver as never, session, model: 'fake-model', acceptsImages: true,
+ confirm: (async () => ({ answer: 'once' })) as never,
+ emit: (e: AgentEvent) => { events.push(e); },
+ });
+
+ beforeEach(() => {
+ events = [];
+ driver = vi.fn(async (method: string) => {
+ if (method === 'mark') return { seq: 1 };
+ if (method === 'stepsSince') return 0;
+ if (method === 'getDiagnostics') return [];
+ return null;
+ }) as never;
+ (driver as { js: unknown }).js = vi.fn(async () => null);
+ (driver as { op: unknown }).op = vi.fn(async () => null);
+ });
+
+ it('sends a turn that built something but never looked back for a look', async () => {
+ // The gomoku turn: 71 calls, every write compiling, every diagnostic clean,
+ // and the board half off camera — because nothing in the loop ever put a
+ // picture in front of it.
+ const s = fakeSession([asks(call('add_entity', { name: 'Board' })), ends(), ends()]);
+ await runTurn(deps(s), 'build me a board', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('capture_viewport'))).toBe(true);
+ });
+
+ it('says nothing to a turn that already looked', async () => {
+ const s = fakeSession([
+ asks(call('add_entity', { name: 'Board' })),
+ asks(call('capture_viewport')),
+ ends(),
+ ]);
+ await runTurn(deps(s), 'build me a board', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('capture_viewport now'))).toBe(false);
+ });
+
+ it('says nothing to a turn that only read', async () => {
+ // Answering a question is not building something, and a question does not
+ // need a screenshot to be answered honestly.
+ const s = fakeSession([asks(call('get_scene_tree')), ends()]);
+ await runTurn(deps(s), 'what is in the scene?', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('capture_viewport'))).toBe(false);
+ });
+
+ it('asks once, not every round', async () => {
+ // A second ask is an argument. The model may have a reason not to look, and
+ // a loop that insists cannot be ended by the model at all.
+ const s = fakeSession([asks(call('add_entity', { name: 'B' })), ends(), ends(), ends()]);
+ await runTurn(deps(s), 'build', null, new AbortController().signal);
+ expect(s.context.filter((c) => c.includes('capture_viewport')).length).toBe(1);
+ });
+});
+
+describe('running out of rounds', () => {
+ it('warns before the cap so the turn can be landed, not truncated', async () => {
+ // The cap used to arrive without notice: the round in which it would have
+ // summarised the work is the one it never got, so an unfinished turn and a
+ // finished one looked identical to whoever was reading.
+ const events: AgentEvent[] = [];
+ const driver = vi.fn(async (method: string) => {
+ if (method === 'mark') return { seq: 1 };
+ if (method === 'stepsSince') return 0;
+ if (method === 'getDiagnostics') return [];
+ return null;
+ }) as ReturnType<typeof vi.fn> & { js: unknown; op: unknown };
+ (driver as { js: unknown }).js = vi.fn(async () => null);
+ (driver as { op: unknown }).op = vi.fn(async () => null);
+
+ // Never stops asking: the only way to reach the cap.
+ const forever: StepEvent[][] = Array.from({ length: 200 }, () => asks(call('get_scene_tree')));
+ const s = fakeSession(forever);
+ const out = await runTurn({
+ driver: driver as never, session: s, model: 'm', acceptsImages: true,
+ confirm: (async () => ({ answer: 'once' })) as never,
+ emit: (e: AgentEvent) => { events.push(e); },
+ }, 'go', null, new AbortController().signal);
+
+ expect(out).toBeTruthy();
+ expect(events.some((e) => e.type === 'turn_end' && e.reason === 'max_rounds')).toBe(true);
+ const warnings = s.context.filter((c) => c.includes('rounds left'));
+ expect(warnings).toHaveLength(1);
+ });
+});