Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(agent): the reflex that checks the work was switched off by looking first

esenginea98f7a186684
diff --git a/desktop/electron/agent/kernel.ts b/desktop/electron/agent/kernel.ts
index 70fc34bea..40f4ae9bd 100644
--- a/desktop/electron/agent/kernel.ts
+++ b/desktop/electron/agent/kernel.ts
@@ -206,6 +206,10 @@ export async function runTurn(
let builtSomething = false;
let sawPixels = false;
let askedToLook = false;
+ // How often each identical failing call has been re-issued, and the ones
+ // that just crossed the line — see REPEAT_LIMIT.
+ const failing = new Map<string, number>();
+ const stuck: string[] = [];
let round = 0;
for (; round < MAX_ROUNDS; round++) {
// The editor is not frozen while a turn runs: the person can drag an
@@ -314,18 +318,36 @@ export async function runTurn(
const batch = end - i > 1
? await Promise.all(calls.slice(i, end).map((c) => execute(deps, c, allowed, signal, tx !== null)))
: [await execute(deps, calls[i], allowed, signal, tx !== null)];
+ const slice = calls.slice(i, i + batch.length);
for (const done of batch) {
outcomes.push(done.outcome);
wrote ||= done.mutated;
}
- for (const c of calls.slice(i, i + batch.length)) {
- if (LOOKING_TOOLS.has(c.name)) sawPixels = true;
+ // A look counts only AFTER the write it checks. "Did you look at some
+ // point" is satisfied by the prompt's own "look before you edit", which
+ // switched the reflex off for every round that followed.
+ if (batch.some((d) => d.mutated)) sawPixels = false;
+ for (let k = 0; k < slice.length; k++) {
+ if (LOOKING_TOOLS.has(slice[k].name)) sawPixels = true;
+ if (batch[k].outcome.isError) stuckOn(failing, slice[k], stuck);
}
builtSomething ||= wrote;
i += batch.length;
}
if (signal.aborted) break;
+ // Told once per signature, and only after it has failed enough times to
+ // be a pattern: the round cap is a spend limit, and a model re-issuing a
+ // call that keeps failing reaches it without ever being told why.
+ if (stuck.length) {
+ session.pushContext(
+ `${stuck.join(' and ')} has now failed ${REPEAT_LIMIT} times with the same arguments. `
+ + 'Repeating it will keep failing. Read the error, then do something different — a '
+ + 'different tool, different arguments, or tell the user what you need and why.',
+ );
+ stuck.length = 0;
+ }
+
// The verification reflex. get_diagnostics is what the editor itself
// flags in the Details panel, and its own description calls an empty list
// the done signal — so after a batch of edits the agent is TOLD what it
@@ -386,6 +408,24 @@ const STEP_RETRIES = 2;
*/
const ROUNDS_WARNING = 8;
+/**
+ * Failures of the SAME call with the SAME arguments before it is called a loop.
+ *
+ * Three, because two is a retry — a tool that failed on a race, a scene that had
+ * not finished loading — and telling a model off for retrying once is noise it
+ * learns to skip past.
+ */
+const REPEAT_LIMIT = 3;
+
+/** Count one failure, and name the tool on the attempt that makes it a pattern.
+ * Only that attempt: a fourth telling is an argument, not information. */
+function stuckOn(failing: Map<string, number>, call: ToolCall, stuck: string[]): void {
+ const signature = `${call.name}:${JSON.stringify(call.input)}`;
+ const n = (failing.get(signature) ?? 0) + 1;
+ failing.set(signature, n);
+ if (n === REPEAT_LIMIT) stuck.push(call.name);
+}
+
/** Undo steps recorded since `mark`, or 0 when the editor cannot say. */
async function undoSteps(deps: KernelDeps, mark: unknown): Promise<number> {
return Number(await deps.driver('stepsSince', [mark]).catch(() => 0)) || 0;
diff --git a/desktop/tests/agent-kernel.test.ts b/desktop/tests/agent-kernel.test.ts
index 8ca55e994..0cf0763fa 100644
--- a/desktop/tests/agent-kernel.test.ts
+++ b/desktop/tests/agent-kernel.test.ts
@@ -665,6 +665,101 @@ describe('looking before reporting', () => {
await runTurn(deps(s), 'build', null, new AbortController().signal);
expect(s.context.filter((c) => c.includes('capture_viewport')).length).toBe(1);
});
+
+ // The prompt tells it to look BEFORE it edits, so a reflex asking "did you
+ // look at any point" is switched off by the behaviour it asks for: one glance
+ // at the scene in the first round covered forty rounds of building after it.
+ it('does not count a look taken before the edit', async () => {
+ const s = fakeSession([
+ asks(call('capture_viewport')),
+ asks(call('add_entity')),
+ ends(), ends(),
+ ]);
+ await runTurn(deps(s), 'build me a board', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('capture_viewport now'))).toBe(true);
+ });
+
+ it('counts a look taken after the edit', async () => {
+ const s = fakeSession([
+ asks(call('capture_viewport')),
+ asks(call('add_entity')),
+ asks(call('capture_viewport')),
+ ends(),
+ ]);
+ await runTurn(deps(s), 'build me a board', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('capture_viewport now'))).toBe(false);
+ });
+
+ // Looking and then editing again is back to unverified: the picture is of the
+ // scene before the last change.
+ it('stops counting an earlier look once something else is written', async () => {
+ const s = fakeSession([
+ asks(call('add_entity')),
+ asks(call('capture_viewport')),
+ asks(call('add_entity')),
+ ends(), ends(),
+ ]);
+ await runTurn(deps(s), 'build me a board', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('capture_viewport now'))).toBe(true);
+ });
+});
+
+// The round cap is a spend limit, not a signal: a model re-issuing a call that
+// keeps failing reaches 128 rounds without anyone telling it why.
+describe('a call that keeps failing the same way', () => {
+ let events: AgentEvent[];
+ let driver: ReturnType<typeof vi.fn> & { js: unknown; op: unknown };
+
+ const deps = (session: AgentSession) => ({
+ driver: driver as never, session, model: 'fake-model', acceptsImages: true,
+ confirm: (async () => ({ answer: 'once' })) as never,
+ emit: (e: AgentEvent) => { events.push(e); },
+ });
+
+ beforeEach(() => {
+ events = [];
+ driver = vi.fn(async (method: string) => {
+ if (method === 'mark') return { seq: 0 };
+ if (method === 'stepsSince') return 0;
+ if (method === 'getDiagnostics') return [];
+ throw new Error('the entity does not exist');
+ }) as never;
+ (driver as { js: unknown }).js = vi.fn(async () => { throw new Error('the entity does not exist'); });
+ (driver as { op: unknown }).op = vi.fn(async () => null);
+ });
+
+ const same = () => asks(call('set_field', { id: 3, component: 'Transform', field: 'x', value: 1 }));
+
+ it('is named once it has failed three times with the same arguments', async () => {
+ const s = fakeSession([same(), same(), same(), ends()]);
+ await runTurn(deps(s), 'move it', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('set_field') && c.includes('3 times'))).toBe(true);
+ });
+
+ it('says nothing about two — that is a retry, not a loop', async () => {
+ const s = fakeSession([same(), same(), ends()]);
+ await runTurn(deps(s), 'move it', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('3 times'))).toBe(false);
+ });
+
+ it('says it once, however many more times it is repeated', async () => {
+ const s = fakeSession([same(), same(), same(), same(), same(), ends()]);
+ await runTurn(deps(s), 'move it', null, new AbortController().signal);
+ expect(s.context.filter((c) => c.includes('3 times'))).toHaveLength(1);
+ });
+
+ // Different arguments are a model TRYING something, which is the behaviour
+ // the nudge exists to provoke — counting them together would punish it.
+ it('counts each set of arguments on its own', async () => {
+ const s = fakeSession([
+ asks(call('set_field', { id: 3, field: 'x' })),
+ asks(call('set_field', { id: 4, field: 'x' })),
+ asks(call('set_field', { id: 5, field: 'x' })),
+ ends(),
+ ]);
+ await runTurn(deps(s), 'move them', null, new AbortController().signal);
+ expect(s.context.some((c) => c.includes('3 times'))).toBe(false);
+ });
});
describe('running out of rounds', () => {