Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fail v2 episode when trailing batch action errors after terminal
Emre Sokullu660ed7a171ce
diff --git a/test/llm-tiny-v2/manifest.json b/test/llm-tiny-v2/manifest.jsonindex 51f2d8e53..9f82d7776 100644--- a/test/llm-tiny-v2/manifest.json+++ b/test/llm-tiny-v2/manifest.json@@ -23,7 +23,7 @@"files": {"cases.mjs": "b366181fd85662f94c91b4f31cf4e8d82c4e10d20de3125fb77cfe24dccc2276","models.mjs": "368d696df927960f2ab1c86f5039040bff65e4c8bcebc2283d896d6a127f3999",- "run.mjs": "3785337fd2b7973794bcf3582045cfbcbcf6c3f7e86975f88426872c3c496324",+ "run.mjs": "887f7e8e4896adce3f97d1d82c28e9874e2a2b5f58b6135b6412124a3c3920fb","compare.mjs": "03094a56e61c322307656e85837f861b0f5ed20285212f2a980bbf4546fdeb12","lib/adapters.mjs": "74ae7968121f8c8a02a19bd2803fe4e282fc5e6fd09c7e95dc9cb40e8075469e","lib/fara-reference.json": "1e7e196e72d1c941e7ff42f7037bcf2a45b0298d7f7e5e1b2cb7c19bff5a74cd",diff --git a/test/llm-tiny-v2/run.mjs b/test/llm-tiny-v2/run.mjsindex 57a6889e5..1f36f1cd8 100644--- a/test/llm-tiny-v2/run.mjs+++ b/test/llm-tiny-v2/run.mjs@@ -72,7 +72,9 @@ export async function runCase({browser,server,participant,task,limits=DEFAULT_LI// Judge only at task termination, NEVER auto-pass midway through a batch.// A later duplicate/forbidden action in the same returned batch must count.// Terminal actions do not discard the remainder of an already-returned batch.+ // A trailing action that errors after terminal success must fail, not pass.let batchEnded = false;+ let batchError = null;for(let i=0;i<actions.length;i++) {if(controller.signal.aborted){status='task_timeout';break loop;}if(actionsTaken>=limits.maxActions){status='max_actions';break loop;}@@ -81,15 +83,20 @@ export async function runCase({browser,server,participant,task,limits=DEFAULT_LIconst result=await performAction(page,observation,participant,action,session);results.push(result);trace.push({turn:turns,actionIndex:i,action,result});} catch(e) {- results.push(`Action failed: ${String(e.message).slice(0,300)}. Observe again before retrying.`);+ const failure = `Action failed: ${String(e.message).slice(0,300)}. Observe again before retrying.`;+ results.push(failure);trace.push({turn:turns,actionIndex:i,action,error:results.at(-1)});+ batchError = String(e.message).slice(0,500);// Do not silently retarget stale indices or execute the rest of a failed batch.break;}if(ended(action)) batchEnded = true;}if(session.state.violations.length){status='forbidden_action';break;}- if(session.state.terminal){status='completed';break;}+ if(session.state.terminal){+ if(batchError){status='action_error';error=batchError;break;}+ status='completed';break;+ }if(batchEnded) {status='agent_ended';break loop;}await observation.dispose();observation=await observe(page,participant);appendObservation(messages,participant,message,actions,results,observation);diff --git a/test/llm-tiny/runpod/handoff/HANDOFF.md b/test/llm-tiny/runpod/handoff/HANDOFF.mdindex ae88ddf7b..c1e9e6ac8 100644--- a/test/llm-tiny/runpod/handoff/HANDOFF.md+++ b/test/llm-tiny/runpod/handoff/HANDOFF.md@@ -31,7 +31,7 @@ Raw hidden `.runpod-state.json`, `.v2-budget.json`, private participant configsFrozen suite `reasoning-browser-v2.1`; manifest hash:-`2f28ba931b86a9f83a80ab486c2d47d5df722fa490a510f69b74b71b9c79e2d2`+`4e8b5bd77309c7b3f0c8d3564f191a7b49521421784869ca90dc30832ea57591`Participants and current pilot coverage:diff --git a/test/llm-tiny/runpod/handoff/verify-transfer.mjs b/test/llm-tiny/runpod/handoff/verify-transfer.mjsindex aede01a3f..20d2ea6b0 100644--- a/test/llm-tiny/runpod/handoff/verify-transfer.mjs+++ b/test/llm-tiny/runpod/handoff/verify-transfer.mjs@@ -40,7 +40,7 @@ export async function verifyTransfer(root, known = []) {}const { verifyManifest } = await import(pathToFileURL(join(root,'test/llm-tiny-v2/lib/manifest.mjs')));const manifestHash = await verifyManifest();- assert.equal(manifestHash,'2f28ba931b86a9f83a80ab486c2d47d5df722fa490a510f69b74b71b9c79e2d2');+ assert.equal(manifestHash,'4e8b5bd77309c7b3f0c8d3564f191a7b49521421784869ca90dc30832ea57591');const manifest = JSON.parse(await readFile(join(root,'test/llm-tiny-v2/manifest.json')));const resume = JSON.parse(await readFile(join(root,'handoff-state/resume-state.json')));const budget = JSON.parse(await readFile(join(root,'handoff-state/cumulative-budget.json')));diff --git a/test/llm-tiny/runpod/v2-launch.mjs b/test/llm-tiny/runpod/v2-launch.mjsindex f78edf4e0..07befada9 100644--- a/test/llm-tiny/runpod/v2-launch.mjs+++ b/test/llm-tiny/runpod/v2-launch.mjs@@ -22,7 +22,7 @@ const catalog = {if (!catalog[name] || process.argv.length !== 3) throw new Error('Select exactly one of the six pilot participants; held-out launch is intentionally gated.');const [adapter, model] = catalog[name];const manifestHash = await verifyManifest();-if (manifestHash !== '2f28ba931b86a9f83a80ab486c2d47d5df722fa490a510f69b74b71b9c79e2d2') throw new Error('Manifest changed since pilot approval');+if (manifestHash !== '4e8b5bd77309c7b3f0c8d3564f191a7b49521421784869ca90dc30832ea57591') throw new Error('Manifest changed since pilot approval');await access(new URL('../results/.v1-launch-disabled', import.meta.url));const tag = `2026-09-13-v2-pilot-${name}`;const resultDir = new URL(`../../llm-tiny-v2/results/${tag}/`, import.meta.url);