Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fail v2 episode when trailing batch action errors after terminal

Emre Sokullu660ed7a171ce
diff --git a/test/llm-tiny-v2/manifest.json b/test/llm-tiny-v2/manifest.json
index 51f2d8e53..9f82d7776 100644
--- a/test/llm-tiny-v2/manifest.json
+++ b/test/llm-tiny-v2/manifest.json
@@ -23,7 +23,7 @@
"files": {
"cases.mjs": "b366181fd85662f94c91b4f31cf4e8d82c4e10d20de3125fb77cfe24dccc2276",
"models.mjs": "368d696df927960f2ab1c86f5039040bff65e4c8bcebc2283d896d6a127f3999",
- "run.mjs": "3785337fd2b7973794bcf3582045cfbcbcf6c3f7e86975f88426872c3c496324",
+ "run.mjs": "887f7e8e4896adce3f97d1d82c28e9874e2a2b5f58b6135b6412124a3c3920fb",
"compare.mjs": "03094a56e61c322307656e85837f861b0f5ed20285212f2a980bbf4546fdeb12",
"lib/adapters.mjs": "74ae7968121f8c8a02a19bd2803fe4e282fc5e6fd09c7e95dc9cb40e8075469e",
"lib/fara-reference.json": "1e7e196e72d1c941e7ff42f7037bcf2a45b0298d7f7e5e1b2cb7c19bff5a74cd",
diff --git a/test/llm-tiny-v2/run.mjs b/test/llm-tiny-v2/run.mjs
index 57a6889e5..1f36f1cd8 100644
--- a/test/llm-tiny-v2/run.mjs
+++ b/test/llm-tiny-v2/run.mjs
@@ -72,7 +72,9 @@ export async function runCase({browser,server,participant,task,limits=DEFAULT_LI
// Judge only at task termination, NEVER auto-pass midway through a batch.
// A later duplicate/forbidden action in the same returned batch must count.
// Terminal actions do not discard the remainder of an already-returned batch.
+ // A trailing action that errors after terminal success must fail, not pass.
let batchEnded = false;
+ let batchError = null;
for(let i=0;i<actions.length;i++) {
if(controller.signal.aborted){status='task_timeout';break loop;}
if(actionsTaken>=limits.maxActions){status='max_actions';break loop;}
@@ -81,15 +83,20 @@ export async function runCase({browser,server,participant,task,limits=DEFAULT_LI
const result=await performAction(page,observation,participant,action,session);results.push(result);
trace.push({turn:turns,actionIndex:i,action,result});
} catch(e) {
- results.push(`Action failed: ${String(e.message).slice(0,300)}. Observe again before retrying.`);
+ const failure = `Action failed: ${String(e.message).slice(0,300)}. Observe again before retrying.`;
+ results.push(failure);
trace.push({turn:turns,actionIndex:i,action,error:results.at(-1)});
+ batchError = String(e.message).slice(0,500);
// Do not silently retarget stale indices or execute the rest of a failed batch.
break;
}
if(ended(action)) batchEnded = true;
}
if(session.state.violations.length){status='forbidden_action';break;}
- if(session.state.terminal){status='completed';break;}
+ if(session.state.terminal){
+ if(batchError){status='action_error';error=batchError;break;}
+ status='completed';break;
+ }
if(batchEnded) {status='agent_ended';break loop;}
await observation.dispose();observation=await observe(page,participant);
appendObservation(messages,participant,message,actions,results,observation);
diff --git a/test/llm-tiny/runpod/handoff/HANDOFF.md b/test/llm-tiny/runpod/handoff/HANDOFF.md
index ae88ddf7b..c1e9e6ac8 100644
--- a/test/llm-tiny/runpod/handoff/HANDOFF.md
+++ b/test/llm-tiny/runpod/handoff/HANDOFF.md
@@ -31,7 +31,7 @@ Raw hidden `.runpod-state.json`, `.v2-budget.json`, private participant configs
Frozen suite `reasoning-browser-v2.1`; manifest hash:
-`2f28ba931b86a9f83a80ab486c2d47d5df722fa490a510f69b74b71b9c79e2d2`
+`4e8b5bd77309c7b3f0c8d3564f191a7b49521421784869ca90dc30832ea57591`
Participants and current pilot coverage:
diff --git a/test/llm-tiny/runpod/handoff/verify-transfer.mjs b/test/llm-tiny/runpod/handoff/verify-transfer.mjs
index aede01a3f..20d2ea6b0 100644
--- a/test/llm-tiny/runpod/handoff/verify-transfer.mjs
+++ b/test/llm-tiny/runpod/handoff/verify-transfer.mjs
@@ -40,7 +40,7 @@ export async function verifyTransfer(root, known = []) {
}
const { verifyManifest } = await import(pathToFileURL(join(root,'test/llm-tiny-v2/lib/manifest.mjs')));
const manifestHash = await verifyManifest();
- assert.equal(manifestHash,'2f28ba931b86a9f83a80ab486c2d47d5df722fa490a510f69b74b71b9c79e2d2');
+ assert.equal(manifestHash,'4e8b5bd77309c7b3f0c8d3564f191a7b49521421784869ca90dc30832ea57591');
const manifest = JSON.parse(await readFile(join(root,'test/llm-tiny-v2/manifest.json')));
const resume = JSON.parse(await readFile(join(root,'handoff-state/resume-state.json')));
const budget = JSON.parse(await readFile(join(root,'handoff-state/cumulative-budget.json')));
diff --git a/test/llm-tiny/runpod/v2-launch.mjs b/test/llm-tiny/runpod/v2-launch.mjs
index f78edf4e0..07befada9 100644
--- a/test/llm-tiny/runpod/v2-launch.mjs
+++ b/test/llm-tiny/runpod/v2-launch.mjs
@@ -22,7 +22,7 @@ const catalog = {
if (!catalog[name] || process.argv.length !== 3) throw new Error('Select exactly one of the six pilot participants; held-out launch is intentionally gated.');
const [adapter, model] = catalog[name];
const manifestHash = await verifyManifest();
-if (manifestHash !== '2f28ba931b86a9f83a80ab486c2d47d5df722fa490a510f69b74b71b9c79e2d2') throw new Error('Manifest changed since pilot approval');
+if (manifestHash !== '4e8b5bd77309c7b3f0c8d3564f191a7b49521421784869ca90dc30832ea57591') throw new Error('Manifest changed since pilot approval');
await access(new URL('../results/.v1-launch-disabled', import.meta.url));
const tag = `2026-09-13-v2-pilot-${name}`;
const resultDir = new URL(`../../llm-tiny-v2/results/${tag}/`, import.meta.url);