Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: match production batch guidance order in evaluation payloads

Barack Sokullu3ea28fdd3fa9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b657f9cb6..9d2ff0186 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ This changelog was generated from the repository Git history and release tags. V
- `browser_batch` runs predictable browser actions in order in one model round trip on Chrome and Firefox. Mid allows up to 6 actions and Full up to 10; Dev inherits its provider tier. Each action keeps its normal permission and verification checks, and failures or submits stop the remaining work for a fresh turn. A final page read can follow navigation. Ask and Compact keep their existing tool surface; an internal `browserBatchEnabled` switch disables batching.
### Fixed
+- Evaluation payloads place batch guidance immediately after the base Act prompt, matching production ordering before Dev guidance, site adapters, and loaded skills while preserving frozen snapshots.
- Build batch guidance from the current tab's run provider, and grade proposed batches only after the complete wrapper and all child arguments pass production validation against the evaluated tool surface.
- Browser batches use the run's selected provider tier for action limits and continuation rules, and wait briefly for delayed navigation before deciding whether another child can run. Navigation that remains unsettled stops the remaining actions for verification.
- Keep each retained browser batch result inside its original untrusted-content boundaries when trimming context. Rejected batches no longer overwrite a native sibling's execution receipt when its call ID resembles a generated batch child ID.
diff --git a/test/browser-batch-exposure.mjs b/test/browser-batch-exposure.mjs
index 2dd70d901..f044103ff 100644
--- a/test/browser-batch-exposure.mjs
+++ b/test/browser-batch-exposure.mjs
@@ -1,6 +1,8 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { test } from 'node:test';
+import { buildPayload, BROWSERS as PAYLOAD_BROWSERS, loadFrozenBaseline, getFrozenSnapshot } from './llm/lib/build-payload.mjs';
+import { buildScenarioPayload } from './llm/lib/scenario-payload.mjs';
for (const build of ['chrome', 'firefox']) {
const { getToolsForMode, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_BATCHING } = await import(`../src/${build}/src/agent/tools.js`);
@@ -98,4 +100,49 @@ for (const build of ['chrome', 'firefox']) {
assert.match(text, /'tool\.browser_batch':\s*["'][^"']+["']/u, file);
}
});
+
+ test(`${build}: both evaluation builders place batch guidance immediately after the base prompt`, () => {
+ const browser = PAYLOAD_BROWSERS[build];
+ const prompts = { full: SYSTEM_PROMPT_ACT, mid: SYSTEM_PROMPT_ACT_MID, compact: SYSTEM_PROMPT_ACT_COMPACT };
+ for (const mode of ['act', 'dev', 'ask']) for (const tier of Object.keys(prompts)) {
+ if (mode === 'dev' && tier === 'compact') continue;
+ const options = { tier, browser: build, profile: { enabled: true, text: 'Fixture profile.' } };
+ const payloads = [
+ buildPayload({ mode, user: 'Fill the known fields.' }, options),
+ buildScenarioPayload({ mode, browser: build, seed: [], packagedSkillIds: ['cms-wix'], activeSkillIds: ['cms-wix'] }, options),
+ ];
+ const agent = new Agent({ getActive: () => ({ promptTier: tier }) });
+ const production = agent._buildSystemPrompt(mode);
+ for (const payload of payloads) {
+ const prompt = payload.messages[0].content;
+ const hasBatch = payload.tools.some(tool => tool.function.name === 'browser_batch');
+ assert.equal(prompt.includes(SYSTEM_PROMPT_BATCHING), hasBatch);
+ if (!hasBatch) continue;
+ assert.ok(prompt.startsWith(`${prompts[tier]}\n\n${SYSTEM_PROMPT_BATCHING}`));
+ assert.ok(prompt.indexOf(SYSTEM_PROMPT_BATCHING) < prompt.indexOf(browser.UNIVERSAL_PREAMBLE.trim()));
+ assert.ok(production.indexOf(SYSTEM_PROMPT_BATCHING) < production.indexOf(browser.UNIVERSAL_PREAMBLE.trim()));
+ if (mode === 'dev') {
+ const appendix = browser.SYSTEM_PROMPT_DEV_APPENDIX.trim();
+ assert.ok(prompt.indexOf(SYSTEM_PROMPT_BATCHING) < prompt.indexOf(appendix));
+ assert.ok(production.indexOf(SYSTEM_PROMPT_BATCHING) < production.indexOf(appendix));
+ }
+ assert.equal(prompt.split(SYSTEM_PROMPT_BATCHING).length - 1, 1);
+ }
+ }
+ });
+
+ test(`${build}: prompt-order changes leave frozen prompts and tools untouched`, () => {
+ loadFrozenBaseline(new URL('./llm/freeze/baseline-2026-05-23.json', import.meta.url));
+ try {
+ const frozen = getFrozenSnapshot();
+ const options = { tier: 'compact', browser: build, profile: { enabled: true, text: 'Must not be appended.' } };
+ for (const payload of [
+ buildPayload({ mode: 'act', user: 'Fixture.' }, options),
+ buildScenarioPayload({ mode: 'act', browser: build, seed: [], packagedSkillIds: ['missing-skill-must-not-load'] }, options),
+ ]) {
+ assert.equal(payload.messages[0].content, frozen.systemContent);
+ assert.equal(payload.tools, frozen.tools);
+ }
+ } finally { loadFrozenBaseline(null); }
+ });
}
diff --git a/test/llm/README.md b/test/llm/README.md
index 8ceefa89b..20efac6ed 100644
--- a/test/llm/README.md
+++ b/test/llm/README.md
@@ -439,7 +439,9 @@ navigating to a known search URL followed by a read, and avoiding a batch when
the next choice depends on unseen results. The positive cases expect batching;
their prose rubrics distinguish safe individual calls from the preferred
grouped emission. Both payload builders add batching guidance only when the
-selected tools advertise it; frozen payloads remain unchanged.
+ selected tools advertise it, immediately after the base Act prompt and before
+ Dev guidance, universal adapters, and loaded skills, matching production order;
+ frozen payloads remain unchanged.
These cases measure next-call emission, not execution speed. Compare steps and
wall-clock time separately using real browser traces with `browserBatchEnabled`
diff --git a/test/llm/lib/build-payload.mjs b/test/llm/lib/build-payload.mjs
index c81e33054..b2e5be31c 100644
--- a/test/llm/lib/build-payload.mjs
+++ b/test/llm/lib/build-payload.mjs
@@ -118,9 +118,12 @@ function getActPromptForTier(browser, tier) {
return browser.SYSTEM_PROMPT_ACT;
}
-export function getSystemPromptForMode(browser, mode, tier) {
+export function getSystemPromptForMode(browser, mode, tier, tools = []) {
if (mode === 'ask') return browser.SYSTEM_PROMPT_ASK;
let prompt = getActPromptForTier(browser, tier);
+ if (tools.some(tool => tool.function?.name === 'browser_batch')) {
+ prompt += `\n\n${browser.SYSTEM_PROMPT_BATCHING}`;
+ }
if (mode === 'dev') {
prompt += `\n\n${browser.SYSTEM_PROMPT_DEV_APPENDIX.trim()}`;
}
@@ -189,6 +192,18 @@ export function buildPayload(caseRec, opts = {}) {
const title = caseRec.tab?.title || '';
const useSiteAdapters = opts.useSiteAdapters !== false;
const strictSecretMode = !!opts.strictSecretMode;
+ if (!FROZEN_BASELINE) assertRunnableModeTier(mode, tier);
+
+ // ── tools ────────────────────────────────────────────────────────────
+ // FREEZE MODE: use the snapshot's tools verbatim. The strictSecretMode,
+ // tier, and mode options have no effect on a frozen baseline by design.
+ const tools = FROZEN_BASELINE
+ ? FROZEN_BASELINE.tools
+ : browser.getToolsForMode(mode, {
+ strictSecretMode,
+ tier,
+ researchEscalationEnabled: opts.researchEscalationEnabled === true,
+ });
// ── system message ───────────────────────────────────────────────────
// FREEZE MODE: skip ALL system-prompt assembly (incl. adapters, profile,
@@ -200,9 +215,8 @@ export function buildPayload(caseRec, opts = {}) {
if (FROZEN_BASELINE) {
systemContent = FROZEN_BASELINE.systemContent;
} else {
- assertRunnableModeTier(mode, tier);
// ACT and DEV modes honor tier (full / mid / compact). ASK mode has one prompt.
- systemContent = getSystemPromptForMode(browser, mode, tier);
+ systemContent = getSystemPromptForMode(browser, mode, tier, tools);
if (useSiteAdapters) {
systemContent += `\n\n${browser.UNIVERSAL_PREAMBLE.trim()}`;
}
@@ -233,21 +247,6 @@ export function buildPayload(caseRec, opts = {}) {
const userContent = contextLine + caseRec.user;
- // ── tools ────────────────────────────────────────────────────────────
- // FREEZE MODE: use the snapshot's tools verbatim. The strictSecretMode,
- // tier, and mode options have no effect on a frozen baseline by design.
- const tools = FROZEN_BASELINE
- ? FROZEN_BASELINE.tools
- : browser.getToolsForMode(mode, {
- strictSecretMode,
- tier,
- researchEscalationEnabled: opts.researchEscalationEnabled === true,
- });
-
- if (!FROZEN_BASELINE && tools.some(tool => tool.function?.name === 'browser_batch')) {
- systemContent += `\n\n${browser.SYSTEM_PROMPT_BATCHING}`;
- }
-
return {
messages: [
{ role: 'system', content: systemContent },
diff --git a/test/llm/lib/scenario-payload.mjs b/test/llm/lib/scenario-payload.mjs
index d14650b24..660f8d359 100644
--- a/test/llm/lib/scenario-payload.mjs
+++ b/test/llm/lib/scenario-payload.mjs
@@ -118,24 +118,14 @@ export function buildScenarioPayload(scenario, opts = {}) {
const strictSecretMode = opts.strictSecretMode ?? (scenario.strictSecretMode === true);
const unprotected = !!opts.unprotected;
- // FREEZE MODE: snapshot wins. Site adapters / tier are ignored β€” whatever
- // was active at capture time is what we replay. Multi-turn seeds still
- // flow through verbatim.
+ // Freeze preserves both the captured prompt and its tool surface verbatim.
const FROZEN = getFrozenSnapshot();
- let systemContent;
- if (FROZEN) {
- systemContent = FROZEN.systemContent;
- } else {
- assertRunnableModeTier(mode, tier);
- systemContent = getSystemPromptForMode(browser, mode, tier);
- if (useSiteAdapters) {
- systemContent += `\n\n${browser.UNIVERSAL_PREAMBLE.trim()}`;
- }
- }
+ if (!FROZEN) assertRunnableModeTier(mode, tier);
// Fixture-owned enabled/active IDs reproduce on-demand package loading. Never
// infer activation from seed/tool text; Compact uses the production exclusion.
let skillLoaderTool;
+ let skillsPrompt = '';
if (!FROZEN && scenario.packagedSkillIds?.length) {
const skills = scenario.browser === 'firefox' ? firefoxSkills : chromeSkills;
const enabled = scenario.packagedSkillIds.map(id => {
@@ -147,7 +137,7 @@ export function buildScenarioPayload(scenario, opts = {}) {
const context = { mode, tier, activeSkillIds: scenario.activeSkillIds || [] };
skillLoaderTool = skills.buildSkillLoaderDefinition(enabled, context);
const loaded = skills.buildCustomSkillsPrompt(enabled, context);
- if (loaded) systemContent += `\n\n${loaded}`;
+ if (loaded) skillsPrompt = loaded;
}
const tools = FROZEN
@@ -159,8 +149,12 @@ export function buildScenarioPayload(scenario, opts = {}) {
researchEscalationEnabled: opts.researchEscalationEnabled === true,
});
- if (!FROZEN && tools.some(tool => tool.function?.name === 'browser_batch')) {
- systemContent += `\n\n${browser.SYSTEM_PROMPT_BATCHING}`;
+ let systemContent = FROZEN
+ ? FROZEN.systemContent
+ : getSystemPromptForMode(browser, mode, tier, tools);
+ if (!FROZEN) {
+ if (useSiteAdapters) systemContent += `\n\n${browser.UNIVERSAL_PREAMBLE.trim()}`;
+ if (skillsPrompt) systemContent += `\n\n${skillsPrompt}`;
}
// ABLATION: remove both layers of injection defense.