Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: retain verified type_text values within browser batches
Barack Sokullu8bfec6b04348
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9d2ff0186..d4eec3296 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,7 @@ This changelog was generated from the repository Git history and release tags. V- `browser_batch` runs predictable browser actions in order in one model round trip on Chrome and Firefox. Mid allows up to 6 actions and Full up to 10; Dev inherits its provider tier. Each action keeps its normal permission and verification checks, and failures or submits stop the remaining work for a fresh turn. A final page read can follow navigation. Ask and Compact keep their existing tool surface; an internal `browserBatchEnabled` switch disables batching.### Fixed+- Verified `type_text` writes now carry their exact clear/append value into the same batch's original field certificates, allowing later known fields to run without accepting unrelated page changes.- Evaluation payloads place batch guidance immediately after the base Act prompt, matching production ordering before Dev guidance, site adapters, and loaded skills while preserving frozen snapshots.- Build batch guidance from the current tab's run provider, and grade proposed batches only after the complete wrapper and all child arguments pass production validation against the evaluated tool surface.- Browser batches use the run's selected provider tier for action limits and continuation rules, and wait briefly for delayed navigation before deciding whether another child can run. Navigation that remains unsettled stops the remaining actions for verification.diff --git a/docs/agent-tools.md b/docs/agent-tools.mdindex 488a14318..2d3871108 100644--- a/docs/agent-tools.md+++ b/docs/agent-tools.md@@ -160,7 +160,7 @@ Navigation also stops unless only trailing observations remain. Mid's usualsingle-action boundary is suspended inside a batch but still applies after it.Coordinate clicks after an earlier mutation are rejected during validation.-Verified `set_field`/`type_ax` edits to native input and textarea controls can+Verified `set_field`/`type_ax`/`type_text` edits to native input and textarea controls cancontinue within the same batch using updated evidence for those exact values.Element identities, labels, surrounding form/recipient bindings, other controlvalues and user-interaction checks remain fixed. A sibling call or anotherdiff --git a/docs/prompt-injection-defense.md b/docs/prompt-injection-defense.mdindex 267ffa428..fb1b4d9d0 100644--- a/docs/prompt-injection-defense.md+++ b/docs/prompt-injection-defense.md@@ -137,8 +137,8 @@ fresh submit checks, before dispatch. Validate the whole batch before runningthe first action; neither allowlist membership nor a valid earlier actionauthorizes a later malformed or unavailable tool.-The page monitor permits a narrow continuation after verified `set_field` or-`type_ax` input into a native input/textarea: only the completed control's value+The page monitor permits a narrow continuation after verified `set_field`,+`type_ax`, or `type_text` input into a native input/textarea: only the completed control's valueevidence changes, bound to the original snapshot and the same parent batch.Node identities, labels, ancestors, form/recipient identity, all other valuesand human-interaction authority remain checked. Other batches and top-leveldiff --git a/src/chrome/ARCHITECTURE.md b/src/chrome/ARCHITECTURE.mdindex e2eb876c9..858c3df5d 100644--- a/src/chrome/ARCHITECTURE.md+++ b/src/chrome/ARCHITECTURE.md@@ -422,7 +422,7 @@ between actions of the same batch. Navigation-prone actions use the existingbounded settling helpers before the next action.The page monitor can derive same-batch continuation evidence after verified-native input/textarea edits by `set_field` or `type_ax`. It updates only the+native input/textarea edits by `set_field`, `type_ax`, or `type_text`. It updates only thecompleted control's value while retaining original nodes, labels, ancestors,form/recipient bindings, collateral values and human authority. Other batchesand top-level siblings cannot borrow this evidence. Select/contenteditable anddiff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex de6d2e6ee..9bfa37310 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -1564,7 +1564,7 @@ export function installPageFeedback(Agent) {if (run && terminal) terminalStreak = run.passiveSupersessionStreak;const result = await execute.call(this, tabId, name, args, onUpdate, executionContext);noDispatch = result?.noDispatch === true || result?.dispatched === false;- commitBrowserBatchInput = !!batchId && modelBound && ['set_field', 'type_ax'].includes(name)+ commitBrowserBatchInput = !!batchId && modelBound && ['set_field', 'type_ax', 'type_text'].includes(name)&& result?.success === true && result.verified === true && !noDispatch && !result.error&& !result.outcomeUnknown && !result.mutationMayHaveOccurred && !args?.submit&& this._pageFeedbackRuns?.get(tabId) === run && run.modelState === modelStatediff --git a/src/firefox/ARCHITECTURE.md b/src/firefox/ARCHITECTURE.mdindex ea1e61437..959160fa7 100644--- a/src/firefox/ARCHITECTURE.md+++ b/src/firefox/ARCHITECTURE.md@@ -289,7 +289,7 @@ between actions of the same batch. Navigation-prone actions use the existingbounded settling helpers before the next action.The page monitor can derive same-batch continuation evidence after verified-native input/textarea edits by `set_field` or `type_ax`. It updates only the+native input/textarea edits by `set_field`, `type_ax`, or `type_text`. It updates only thecompleted control's value while retaining original nodes, labels, ancestors,form/recipient bindings, collateral values and human authority. Other batchesand top-level siblings cannot borrow this evidence. Select/contenteditable anddiff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex de6d2e6ee..9bfa37310 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -1564,7 +1564,7 @@ export function installPageFeedback(Agent) {if (run && terminal) terminalStreak = run.passiveSupersessionStreak;const result = await execute.call(this, tabId, name, args, onUpdate, executionContext);noDispatch = result?.noDispatch === true || result?.dispatched === false;- commitBrowserBatchInput = !!batchId && modelBound && ['set_field', 'type_ax'].includes(name)+ commitBrowserBatchInput = !!batchId && modelBound && ['set_field', 'type_ax', 'type_text'].includes(name)&& result?.success === true && result.verified === true && !noDispatch && !result.error&& !result.outcomeUnknown && !result.mutationMayHaveOccurred && !args?.submit&& this._pageFeedbackRuns?.get(tabId) === run && run.modelState === modelStatediff --git a/test/browser-batch-feedback.mjs b/test/browser-batch-feedback.mjsindex 9e30a7a57..a9702d61d 100644--- a/test/browser-batch-feedback.mjs+++ b/test/browser-batch-feedback.mjs@@ -5,7 +5,7 @@ import { chromium, firefox } from 'playwright';const RUN = 'batch-feedback-run';const html = `<!doctype html><main><h1>Account details</h1><form aria-label="Account details">-<label>Name <input id="name"></label><label>Email <input id="email"></label>+<label>Name <input id="name" value="Existing "></label><label>Email <input id="email"></label><label>Note <textarea id="note"></textarea></label><button type="button">Save details</button></form></main><aside id="counter">0</aside><button id="human">Human control</button>`;@@ -51,16 +51,16 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {const validate = (page, selector = '#email', batchId = 'parent') => page.evaluate(({ selector, batchId }) => send('page_monitor_validate_model', {tool: 'set_field', selector, snapshotToken: snapshot.snapshotToken, batchId,}), { selector, batchId });- const write = (page, { seal = true, selector = '#name', text = 'Ada', mutation = null } = {}) => page.evaluate(async ({ seal, selector, text, mutation }) => {- const params = { tool: 'set_field', selector, operationId: `op-${selector}`, batchId: 'parent',+ const write = (page, { seal = true, selector = '#name', text = 'Ada', mutation = null, tool = 'set_field', clear = true } = {}) => page.evaluate(async ({ seal, selector, text, mutation, tool, clear }) => {+ const params = { tool, selector, operationId: `op-${selector}`, batchId: 'parent',expectedModelSnapshot: snapshot.snapshotToken, allowPassiveRebase: true };const preparation = await send('page_monitor_prepare', params);- const action = preparation.modelBindingValid ? await send('type', { selector, text, clear: true }) : null;+ const action = preparation.modelBindingValid ? await send('type', { selector, text, clear }) : null;if (mutation) (0, eval)(`(${mutation})`)();const finish = await send('page_monitor_finish', { operationId: params.operationId, batchId: 'parent',- commitBrowserBatchInput: seal, batchInputText: text, batchInputClear: true });+ commitBrowserBatchInput: seal, batchInputText: text, batchInputClear: clear });return { preparation, action, finish, value: document.querySelector(selector).value };- }, { seal, selector, text, mutation: mutation?.toString() || null });+ }, { seal, selector, text, mutation: mutation?.toString() || null, tool, clear });try {await t.test('two owned field writes retain the original next field and sibling top-level calls stay fenced', async () => {const { page, context } = await fixture();@@ -83,6 +83,21 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {try { await write(page, { seal: false }); assert.equal((await validate(page)).ready, false); }finally { await context.close(); }});+ for (const clear of [true, false]) await t.test(`selector type_text ${clear ? 'replacement' : 'append'} retains only its own verified field value`, async () => {+ const { page, context } = await fixture();+ try {+ const first = await write(page, { tool: 'type_text', clear });+ assert.equal(first.preparation.modelBindingValid, true, JSON.stringify(first));+ assert.equal(first.action?.success, true, JSON.stringify(first));+ assert.equal(first.value, clear ? 'Ada' : 'Existing Ada');+ assert.equal(first.finish.batchInputSealed, true, JSON.stringify(first));+ assert.equal((await validate(page)).ready, true);+ assert.equal((await validate(page, '#email', 'another-batch')).ready, false);+ const second = await write(page, { tool: 'type_text', selector: '#email', text: '[email protected]', clear: true });+ assert.equal(second.finish.batchInputSealed, true, JSON.stringify(second));+ assert.equal((await validate(page, '#note')).ready, true);+ } finally { await context.close(); }+ });for (const [name, mutation] of [['collateral value', () => { document.querySelector('#note').value = 'Unseen collateral value'; }],['form identity', () => { document.querySelector('form').setAttribute('aria-label', 'Another account'); }],diff --git a/test/browser-batch-runtime.mjs b/test/browser-batch-runtime.mjsindex bc26b3959..b717f402d 100644--- a/test/browser-batch-runtime.mjs+++ b/test/browser-batch-runtime.mjs@@ -31,6 +31,7 @@ for (const build of ['chrome', 'firefox']) {const { getToolsForMode } = await import(`../src/${build}/src/agent/tools.js`);const { AnthropicProvider } = await import(`../src/${build}/src/providers/anthropic.js`);const { OpenAICompatibleProvider } = await import(`../src/${build}/src/providers/openai.js`);+ const { installPageFeedback } = await import(`../src/${build}/src/agent/page-feedback.js`);function setup(calls, tier = 'full') {const provider = { name: 'test', model: 'test', promptTier: tier, contextWindow: 128000, supportsVision: false };const agent = new Agent({ getActive: () => provider, getProvider: () => provider, getVisionProvider: async () => null });@@ -91,6 +92,52 @@ for (const build of ['chrome', 'firefox']) {assert.equal(h.updates.filter(event => event.type === 'tool_result' && event.data.name === 'browser_batch').length, 1);});+ test(`${build}: only verified batched type_text dispatches hand off exact clear semantics`, async () => {+ for (const scenario of [+ { clear: true, result: { success: true, verified: true }, seal: true },+ { clear: false, result: { success: true, verified: true }, seal: true },+ { result: { success: true, verified: true }, seal: true },+ { clear: true, result: { success: true, verified: false }, seal: false },+ { clear: true, result: { success: true, verified: true, noDispatch: true }, seal: false },+ { clear: true, result: { success: true, verified: true, outcomeUnknown: true }, seal: false },+ { clear: true, result: { success: true, verified: true }, seal: false, native: true },+ ]) {+ class FixtureAgent {+ static STATE_CHANGE_TOOLS = new Set(['type_text']);+ isRunning() { return true; }+ _checkAbort() { return false; }+ async executeTool() { return scenario.result; }+ }+ installPageFeedback(FixtureAgent);+ const agent = new FixtureAgent();+ const originalMessage = api.tabs.sendMessage;+ let finish;+ api.tabs.sendMessage = async (_tab, message) => {+ if (message.action === 'page_monitor_prepare') return { ready: true, modelBindingValid: true };+ if (message.action === 'page_monitor_finish') finish = message.params;+ return {};+ };+ try {+ await agent._beginPageFeedbackRun(1993, 'interactive');+ agent._pageFeedbackRuns.get(1993).modelState = { actionBinding: { snapshotToken: 'original-snapshot' } };+ agent._validatePageFeedbackAction = async () => true;+ agent._pageFeedbackStateCurrent = () => true;+ agent._pageFeedbackBlocksCall = () => false;+ await agent.executeTool(1993, 'type_text', { selector: '#name', text: 'Ada', clear: scenario.clear }, () => {},+ scenario.native ? {} : { _browserBatch: { parentId: 'parent' } });+ assert.equal(finish?.commitBrowserBatchInput === true, scenario.seal, JSON.stringify(scenario));+ if (scenario.seal) {+ assert.equal(finish.batchId, 'parent');+ assert.equal(finish.batchInputText, 'Ada');+ assert.equal(finish.batchInputClear, scenario.clear === true);+ }+ } finally {+ agent._finishPageFeedbackRun(1993);+ api.tabs.sendMessage = originalMessage;+ }+ }+ });+for (const scenario of [{ tier: 'full', active: 'mid', count: 7, dispatched: 7 },{ tier: 'mid', active: 'full', count: 7, dispatched: 0 },