A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(agent): a turn looks at what it built before it reports it
esengine8d3a57b341ef
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.tsindex 7f718cd46..983a4ce22 100644--- a/desktop/electron/agent/kernel.ts+++ b/desktop/electron/agent/kernel.ts@@ -193,6 +193,11 @@ export async function runTurn(session.pushUser(text, images);// Bounded so a model that keeps calling tools cannot spin forever.+ // Turn-level, not round-level: "did this turn build anything" and "did it+ // ever look" are questions about the whole turn.+ let builtSomething = false;+ let sawPixels = false;+ let askedToLook = false;let round = 0;for (; round < MAX_ROUNDS; round++) {// The editor is not frozen while a turn runs: the person can drag an@@ -213,6 +218,16 @@ export async function runTurn(}}+ // Running out of room is something the model can act on, but only if it+ // learns before the last round rather than at it.+ if (round === MAX_ROUNDS - ROUNDS_WARNING) {+ session.pushContext(+ `You have ${ROUNDS_WARNING} tool rounds left in this turn before it is cut off. `+ + 'Land what you have: finish the step you are on, leave the project in a working '+ + 'state, and tell the user where things stand and what is left.',+ );+ }+const calls: ToolCall[] = [];let refused = false;@@ -222,7 +237,25 @@ export async function runTurn(emit(ev);}if (refused) { reason = 'refusal'; break; }- if (calls.length === 0) break;+ // No calls means it is about to answer. If it built something and never+ // once looked at it, that answer is a report on work nobody has seen —+ // which is how a board half off camera gets delivered as a finished game,+ // with every write compiling and every diagnostic clean. Asked ONCE per+ // turn: a second ask would be an argument, and the model may have a reason.+ if (calls.length === 0) {+ if (builtSomething && !sawPixels && !askedToLook) {+ askedToLook = true;+ session.pushContext(+ 'You changed the scene this turn and have not looked at it once. Diagnostics only '+ + 'cover what the editor can name — whether the content is ON CAMERA, whether it '+ + 'reads, whether it is where you meant, are all things only the picture answers. '+ + 'capture_viewport now; if it is something that has to be PLAYED, toggle_play, '+ + 'drive it with play_input and screenshot that. Then fix what you see, or report.',+ );+ continue;+ }+ break;+ }// Reads have no side effects and no order among themselves, so a run of// them goes out together — three lookups become one wait instead of three.@@ -241,6 +274,10 @@ export async function runTurn(outcomes.push(done.outcome);wrote ||= done.mutated;}+ for (const c of calls.slice(i, i + batch.length)) {+ if (LOOKING_TOOLS.has(c.name)) sawPixels = true;+ }+ builtSomething ||= wrote;i += batch.length;}if (signal.aborted) break;@@ -274,7 +311,26 @@ export async function runTurn(return { mark, steps };}-const MAX_ROUNDS = 48;+/**+ * The backstop against a turn that never ends — a model retrying a call that+ * always fails would otherwise spend a balance before anyone noticed.+ *+ * Held high because the failure it prevents is rare and the one it CAUSES is+ * not: building a small game takes some seventy tool calls, and at 48 both+ * dogfood runs were cut off mid-build. A turn that stops here has spent real+ * money to deliver half a thing.+ */+const MAX_ROUNDS = 128;++/**+ * How many rounds from the cap the model is told it is running out.+ *+ * The cap on its own truncates: the round where it would have said "the board is+ * built, here is how to play it" is the round it never gets. Warned, it can land+ * the work it has — which is the difference between an unfinished turn and an+ * unfinished turn nobody explained.+ */+const ROUNDS_WARNING = 8;/** Undo steps recorded since `mark`, or 0 when the editor cannot say. */async function undoSteps(deps: KernelDeps, mark: unknown): Promise<number> {@@ -416,6 +472,10 @@ async function execute(return { outcome, mutated: mutates(tool) && !outcome.isError };}+/** The tools that put PIXELS in front of the model — the only answer to "does it+ * look right", which no diagnostic can give. */+const LOOKING_TOOLS = new Set(['capture_viewport', 'screenshot']);+/** The scene's outstanding validation issues as one line of context, or null* when there are none (silence is the "still clean" signal). */async function diagnostics(deps: KernelDeps): Promise<string | null> {diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.tsindex 63d063eb2..4bece8666 100644--- a/desktop/tests/agent-kernel.test.ts+++ b/desktop/tests/agent-kernel.test.ts@@ -143,7 +143,10 @@ describe('the agent turn', () => {});it('says nothing when the scene is clean — silence is the done signal', async () => {- const s = fakeSession([asks(call('add_entity')), ends()]);+ // A clean sweep adds no diagnostics line. (The turn is still asked to LOOK+ // once, which is a different reflex with its own cases below — this one is+ // about what the DIAGNOSTICS say, so it looks first and leaves that quiet.)+ const s = fakeSession([asks(call('add_entity')), asks(call('capture_viewport')), ends()]);await runTurn(deps(s), 'add', null, new AbortController().signal);expect(s.context).toHaveLength(0);});@@ -188,7 +191,9 @@ describe('the agent turn', () => {// an end_turn here looks exactly like a run that finished, and the sentence// that would have told you otherwise is the one it never got to write.it('says the step budget ran out rather than reporting a finished run', async () => {- const s = fakeSession(Array.from({ length: 60 }, () => asks(call('get_scene_tree'))));+ // Enough steps to outlast any cap: writing the cap's number here is how a+ // test starts failing for the raise it was meant to be indifferent to.+ const s = fakeSession(Array.from({ length: 400 }, () => asks(call('get_scene_tree'))));await runTurn(deps(s), 'build the whole game', null, new AbortController().signal);expect(events.at(-1)).toMatchObject({ type: 'turn_end', reason: 'max_rounds' });});@@ -285,7 +290,12 @@ describe('the agent turn', () => {// the whole reason it is not persisted.it('expires with the run', async () => {confirm.mockResolvedValue({ answer: 'turn' });- const s = fakeSession([asks(call('save_scene')), ends(), asks(call('save_scene')), ends()]);+ // capture_viewport in each turn: a turn that wrote and never looked spends+ // an extra round being asked to, which would shift the scripted steps.+ const s = fakeSession([+ asks(call('save_scene')), asks(call('capture_viewport')), ends(),+ asks(call('save_scene')), asks(call('capture_viewport')), ends(),+ ]);await runTurn(deps(s), 'save', null, new AbortController().signal);await runTurn(deps(s), 'save again', null, new AbortController().signal);expect(confirm).toHaveBeenCalledTimes(2);@@ -510,3 +520,92 @@ describe('the agent turn', () => {});});});++describe('looking before reporting', () => {+ let events: AgentEvent[];+ let driver: ReturnType<typeof vi.fn> & { js: unknown; op: unknown };++ const deps = (session: AgentSession) => ({+ driver: driver as never, session, model: 'fake-model', acceptsImages: true,+ confirm: (async () => ({ answer: 'once' })) as never,+ emit: (e: AgentEvent) => { events.push(e); },+ });++ beforeEach(() => {+ events = [];+ driver = vi.fn(async (method: string) => {+ if (method === 'mark') return { seq: 1 };+ if (method === 'stepsSince') return 0;+ if (method === 'getDiagnostics') return [];+ return null;+ }) as never;+ (driver as { js: unknown }).js = vi.fn(async () => null);+ (driver as { op: unknown }).op = vi.fn(async () => null);+ });++ it('sends a turn that built something but never looked back for a look', async () => {+ // The gomoku turn: 71 calls, every write compiling, every diagnostic clean,+ // and the board half off camera — because nothing in the loop ever put a+ // picture in front of it.+ const s = fakeSession([asks(call('add_entity', { name: 'Board' })), ends(), ends()]);+ await runTurn(deps(s), 'build me a board', null, new AbortController().signal);+ expect(s.context.some((c) => c.includes('capture_viewport'))).toBe(true);+ });++ it('says nothing to a turn that already looked', async () => {+ const s = fakeSession([+ asks(call('add_entity', { name: 'Board' })),+ asks(call('capture_viewport')),+ ends(),+ ]);+ await runTurn(deps(s), 'build me a board', null, new AbortController().signal);+ expect(s.context.some((c) => c.includes('capture_viewport now'))).toBe(false);+ });++ it('says nothing to a turn that only read', async () => {+ // Answering a question is not building something, and a question does not+ // need a screenshot to be answered honestly.+ const s = fakeSession([asks(call('get_scene_tree')), ends()]);+ await runTurn(deps(s), 'what is in the scene?', null, new AbortController().signal);+ expect(s.context.some((c) => c.includes('capture_viewport'))).toBe(false);+ });++ it('asks once, not every round', async () => {+ // A second ask is an argument. The model may have a reason not to look, and+ // a loop that insists cannot be ended by the model at all.+ const s = fakeSession([asks(call('add_entity', { name: 'B' })), ends(), ends(), ends()]);+ await runTurn(deps(s), 'build', null, new AbortController().signal);+ expect(s.context.filter((c) => c.includes('capture_viewport')).length).toBe(1);+ });+});++describe('running out of rounds', () => {+ it('warns before the cap so the turn can be landed, not truncated', async () => {+ // The cap used to arrive without notice: the round in which it would have+ // summarised the work is the one it never got, so an unfinished turn and a+ // finished one looked identical to whoever was reading.+ const events: AgentEvent[] = [];+ const driver = vi.fn(async (method: string) => {+ if (method === 'mark') return { seq: 1 };+ if (method === 'stepsSince') return 0;+ if (method === 'getDiagnostics') return [];+ return null;+ }) as ReturnType<typeof vi.fn> & { js: unknown; op: unknown };+ (driver as { js: unknown }).js = vi.fn(async () => null);+ (driver as { op: unknown }).op = vi.fn(async () => null);++ // Never stops asking: the only way to reach the cap.+ const forever: StepEvent[][] = Array.from({ length: 200 }, () => asks(call('get_scene_tree')));+ const s = fakeSession(forever);+ const out = await runTurn({+ driver: driver as never, session: s, model: 'm', acceptsImages: true,+ confirm: (async () => ({ answer: 'once' })) as never,+ emit: (e: AgentEvent) => { events.push(e); },+ }, 'go', null, new AbortController().signal);++ expect(out).toBeTruthy();+ expect(events.some((e) => e.type === 'turn_end' && e.reason === 'max_rounds')).toBe(true);+ const warnings = s.context.filter((c) => c.includes('rounds left'));+ expect(warnings).toHaveLength(1);+ });+});