Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Keep finished Act answers through automatic page churn
Claudeb998d4957289
diff --git a/src/chrome/src/agent/page-feedback-policy.js b/src/chrome/src/agent/page-feedback-policy.jsindex 810517e1e..d42eb3466 100644--- a/src/chrome/src/agent/page-feedback-policy.js+++ b/src/chrome/src/agent/page-feedback-policy.js@@ -226,9 +226,9 @@ export function isPassivePageFeedback(events) {}// Automatic feedback is an observed page consequence with no evidence of a-// human gesture. Every 'user' event, main-frame navigation, a top-level-// viewport change of unknown origin and base/style/link updates keep the-// strict fence. Focus moved to an <iframe> is page-script focus: Chromium and+// human gesture. Every 'user' event, main-frame navigation, any top-level+// viewport change (the monitor reports those as unknown) and base/style/link+// updates keep the strict fence. Focus moved to an <iframe> is page-script focus: Chromium and// Firefox fire no parent focusin for a click or Tab inside a frame (a Tab// reaches the parent monitor as user keydown activity).const AUTOMATIC_SOURCES = new Map([['dom', ['page', 'unknown']], ['scroll', ['page', 'unknown']],@@ -237,7 +237,7 @@ export function isAutomaticPageFeedbackEvent(event) {if (!AUTOMATIC_SOURCES.get(event?.kind)?.includes(event.source)|| BASE_STYLE_LINK.test(typeof event.target === 'string' ? event.target : '')) return false;const childFrame = Number.isSafeInteger(event.frameId) && event.frameId > 0;- return event.kind === 'navigation' || (event.kind === 'resize' && event.source === 'unknown') ? childFrame : true;+ return event.kind === 'navigation' || event.kind === 'resize' ? childFrame : true;}export function isAutomaticPageFeedback(events) {@@ -364,7 +364,8 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {if (!calls?.length) return false;const passive = isPassivePageFeedback(events);const calculation = isPageIndependentCalculationCompletion(calls, state, events);- if (!passive && !calculation && !isAutomaticPageFeedback(events)) return false;+ const automatic = !passive && isAutomaticPageFeedback(events);+ if (!passive && !calculation && !automatic) return false;const interactive = !passive && events.some(event => ['activity', 'input'].includes(event?.kind));return calls.every(call => {let args;@@ -372,9 +373,12 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {if (!args || typeof args !== 'object' || Array.isArray(args)) return false;const policy = pageFeedbackCallPolicy(call.function?.name, args, state, currentPage, events);if (policy.terminal && calls.length !== 1) return false;- // A terminal answer keeps its established fences (passive churn or an owned- // literal calculation); Ask scope and the completion verifier own the rest.- if (policy.terminal) return passive || calculation;+ // An Act completion survives automatic churn (ad frames, page focus,+ // unattributed scroll or DOM): superseding it on every tick would end a+ // finished task as page_unstable. Its batch still runs the completion+ // verifier on fresh evidence, and user input still supersedes it. Ask+ // answers keep their own scope classification and passive-only fence.+ if (policy.terminal) return passive || calculation || (automatic && state?.runMode !== 'ask');if (interactive && policy.kind === 'observe' && !PAGE_ACTIVITY_READS.has(call.function?.name)) return false;// A target comparison only certifies the snapshot just read. Further DOM// changes require local revalidation; fresh readers/media have their own fences.diff --git a/src/firefox/src/agent/page-feedback-policy.js b/src/firefox/src/agent/page-feedback-policy.jsindex 810517e1e..d42eb3466 100644--- a/src/firefox/src/agent/page-feedback-policy.js+++ b/src/firefox/src/agent/page-feedback-policy.js@@ -226,9 +226,9 @@ export function isPassivePageFeedback(events) {}// Automatic feedback is an observed page consequence with no evidence of a-// human gesture. Every 'user' event, main-frame navigation, a top-level-// viewport change of unknown origin and base/style/link updates keep the-// strict fence. Focus moved to an <iframe> is page-script focus: Chromium and+// human gesture. Every 'user' event, main-frame navigation, any top-level+// viewport change (the monitor reports those as unknown) and base/style/link+// updates keep the strict fence. Focus moved to an <iframe> is page-script focus: Chromium and// Firefox fire no parent focusin for a click or Tab inside a frame (a Tab// reaches the parent monitor as user keydown activity).const AUTOMATIC_SOURCES = new Map([['dom', ['page', 'unknown']], ['scroll', ['page', 'unknown']],@@ -237,7 +237,7 @@ export function isAutomaticPageFeedbackEvent(event) {if (!AUTOMATIC_SOURCES.get(event?.kind)?.includes(event.source)|| BASE_STYLE_LINK.test(typeof event.target === 'string' ? event.target : '')) return false;const childFrame = Number.isSafeInteger(event.frameId) && event.frameId > 0;- return event.kind === 'navigation' || (event.kind === 'resize' && event.source === 'unknown') ? childFrame : true;+ return event.kind === 'navigation' || event.kind === 'resize' ? childFrame : true;}export function isAutomaticPageFeedback(events) {@@ -364,7 +364,8 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {if (!calls?.length) return false;const passive = isPassivePageFeedback(events);const calculation = isPageIndependentCalculationCompletion(calls, state, events);- if (!passive && !calculation && !isAutomaticPageFeedback(events)) return false;+ const automatic = !passive && isAutomaticPageFeedback(events);+ if (!passive && !calculation && !automatic) return false;const interactive = !passive && events.some(event => ['activity', 'input'].includes(event?.kind));return calls.every(call => {let args;@@ -372,9 +373,12 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {if (!args || typeof args !== 'object' || Array.isArray(args)) return false;const policy = pageFeedbackCallPolicy(call.function?.name, args, state, currentPage, events);if (policy.terminal && calls.length !== 1) return false;- // A terminal answer keeps its established fences (passive churn or an owned- // literal calculation); Ask scope and the completion verifier own the rest.- if (policy.terminal) return passive || calculation;+ // An Act completion survives automatic churn (ad frames, page focus,+ // unattributed scroll or DOM): superseding it on every tick would end a+ // finished task as page_unstable. Its batch still runs the completion+ // verifier on fresh evidence, and user input still supersedes it. Ask+ // answers keep their own scope classification and passive-only fence.+ if (policy.terminal) return passive || calculation || (automatic && state?.runMode !== 'ask');if (interactive && policy.kind === 'observe' && !PAGE_ACTIVITY_READS.has(call.function?.name)) return false;// A target comparison only certifies the snapshot just read. Further DOM// changes require local revalidation; fresh readers/media have their own fences.diff --git a/test/feedback-automatic-churn.mjs b/test/feedback-automatic-churn.mjsindex 165c8906b..a850c599f 100644--- a/test/feedback-automatic-churn.mjs+++ b/test/feedback-automatic-churn.mjs@@ -121,7 +121,7 @@ for (const build of ['chrome', 'firefox']) {assert.equal(policy.isAutomaticPageFeedback([]), false);});- test(`${build}: automatic churn retains reads and independent tools but keeps unsafe, legacy non-DOM and terminal fences`, () => {+ test(`${build}: automatic churn retains reads, independent tools and Act completions but keeps unsafe and legacy non-DOM fences`, () => {const page = { success: true, pageContent: TREE };const state = { url: PAGE_URL, page };const keep = (name, args, events, current = page, scope = state) =>@@ -134,7 +134,16 @@ for (const build of ['chrome', 'firefox']) {assert.equal(keep('calculate', { expression: '2 + 2' }, events), true);assert.equal(keep('click', { x: 10, y: 20 }, events), false, 'Coordinates stay fenced');assert.equal(keep('execute_js', { code: 'save()' }, events), false, 'Scripts stay fenced');- assert.equal(keep('done', { summary: 'Done', outcome: 'success' }, events), false, 'Terminal answers keep passive-only retention');+ // An Act completion survives automatic churn; its batch verifies it.+ assert.equal(keep('done', { summary: 'Done', outcome: 'success' }, events), true, 'Act completion survives automatic churn');+ // Ask answers keep their own scope classification and passive-only fence.+ assert.equal(keep('done', { summary: 'The price is 12' }, events, page, { ...state, runMode: 'ask' }), false,+ 'Ask answers keep passive-only retention');+ // A user gesture during inference still supersedes the completion.+ assert.equal(keep('done', { summary: 'Done', outcome: 'success' },+ [...events, { kind: 'click', source: 'user', frameId: 0, target: 'button#other' }]), false, 'User input supersedes a completion');+ assert.equal(keep('done', { summary: 'Done', outcome: 'success' },+ [...events, { kind: 'navigation', source: 'unknown', frameId: 0 }]), false, 'Main-frame navigation supersedes a completion');}assert.equal(keep('chat_observe', {}, focus), false, 'Read preparation stays fenced after page focus changes');assert.equal(keep('chat_observe', {}, scroll), true);@@ -217,6 +226,40 @@ for (const build of ['chrome', 'firefox']) {});}+ for (const streaming of [false, true]) {+ test(`${build}: ${streaming ? 'stream' : 'chat'} a finished task under continuous ad-frame churn ends with its answer`, async () => {+ const tab = nextTab++, dispatched = [];+ let agent, frames, requests = 0, terminal = null;+ pages.set(tab, {});+ agent = makeAgent(Agent, async () => {+ requests++;+ // The ad frame ticks during every inference, as on many real pages.+ frames.child({ kind: 'dom', source: 'page', target: 'div#ad' });+ frames.main({ kind: 'scroll', source: 'unknown', target: 'div#carousel', viewport: { x: 100 * requests, y: 0 } });+ return requests === 1 ? call('add-1', 'click_ax', { ref_id: 'ref_add' })+ : call('finished-' + requests, 'done', { summary: 'Added to cart', outcome: 'success' });+ });+ agent.maxSteps = 8;+ agent._endTraceRun = async (_tab, _run, status) => { terminal = status; };+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => { frames = bindFrames(agent, tab); return { role: 'user', content }; };+ agent._executeToolWithLifecycle = async (_tab, tool, args) => {+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: TREE };+ dispatched.push(tool);+ return tool === 'done' ? { done: true, success: true, summary: args.summary, outcome: args.outcome }+ : { success: true, dispatched: true, verified: true };+ };+ try {+ const options = { detachedRequestId: 'finished-' + tab, askStreamingEnabled: false };+ const result = streaming ? await agent.processMessageStream(tab, 'Add the blue shoes', () => {}, 'act', options)+ : await agent.processMessage(tab, 'Add the blue shoes', () => {}, 'act', [], options);+ assert.equal(result, 'Added to cart');+ assert.equal(requests, 2, 'Automatic churn must not send a finished answer back to the model');+ assert.deepEqual(dispatched, ['click_ax', 'done']);+ assert.notEqual(terminal, 'page_unstable');+ } finally { pages.delete(tab); }+ });+ }+for (const [kind, target] of [['click', 'button#other'], ['input', 'input#q'], ['activity', 'textarea#note']]) {test(`${build}: a genuine user ${kind === 'activity' ? 'keypress' : kind} during inference still supersedes and waits for idle`, async () => {const tab = nextTab++, dispatched = [], starts = [];@@ -346,6 +389,20 @@ for (const build of ['chrome', 'firefox']) {frames.child({ kind: 'dom', source: 'page', target: 'div#ad' });return { done: true, success: true, summary: args.summary };};+ // Automatic ad-frame churn alone no longer supersedes the completion.+ assert.equal((await agent.executeTool(tab, 'done', args, () => {}, {})).done, true);+ agent._resetPageFeedbackProgress(tab);+ assert.equal(await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: [doneCall] }), false,+ 'An Act completion survives automatic churn');+ // An unattributed top-level viewport change is strict but not user input:+ // each superseded completion counts toward the bounded budget.+ agent._executeToolWithLifecycle = async (_tab, tool) => {+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: TREE };+ assert.equal(tool, 'done');+ assert.equal((await agent.executeTool(tab, 'get_accessibility_tree', { maxChars: 5500 })).success, true);+ frames.main({ kind: 'resize', source: 'unknown', target: 'viewport', viewport: { width: 1000 + run.revision, height: 700 } });+ return { done: true, success: true, summary: args.summary };+ };for (let attempt = 1; attempt <= 5; attempt++) {assert.equal((await agent.executeTool(tab, 'done', args, () => {}, {})).done, true);agent._resetPageFeedbackProgress(tab); // the batch's successful-dispatch hookdiff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex e048ccf17..26db28eef 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -130,9 +130,12 @@ for (const build of ['chrome', 'firefox']) {});}+ // An Act completion also survives automatic churn (unknown-attribution and+ // child-frame DOM); its batch still runs the completion verifier.+ const completionSurvives = change => ['feed', 'partial', 'unknown', 'iframe'].includes(change);for (const change of ['feed', 'user', 'unknown', 'iframe', 'navigation', 'viewport', 'style', 'failed_read','partial', 'missing_outcome', 'malformed', 'mixed', 'duplicate', 'pending']) {- test(`${build}: explicit completion ${['feed', 'partial'].includes(change) ? 'survives' : 'rejects'} ${change} feedback`, async () => {+ test(`${build}: explicit completion ${completionSurvives(change) ? 'survives' : 'rejects'} ${change} feedback`, async () => {const agent = setup(Agent), tab = nextTab++;let page = { success: true, pageContent: 'article "Published WebBrain post" [ref_1]' };agent.executeTool = async () => page;@@ -156,7 +159,7 @@ for (const build of ['chrome', 'firefox']) {const call = { function: { name: 'done', arguments: change === 'malformed' ? '{' : JSON.stringify(args) } };const calls = change === 'duplicate' ? [call, call] : change === 'mixed'? [call, { function: { name: 'click_ax', arguments: '{"ref_id":"ref_1"}' } }] : [call];- assert.equal(await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: calls }), !['feed', 'partial'].includes(change));+ assert.equal(await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: calls }), !completionSurvives(change));assert.equal(agent._activeTaskBinding(messages).text, 'Publish a WebBrain post');} finally { agent._releaseRunEntry(tab); }});@@ -1359,10 +1362,12 @@ for (const build of ['chrome', 'firefox']) {assert.equal(note.kind, 'note'); assert.equal(note.data.note, 'page_feedback_decision'); assert.equal(note.data.step, 0);const details = note.data.extra;assert.deepEqual(note.metadata.extra.context, details.context, 'Safe causal details survive the actual metadata-only trace projection');- assert.equal(superseded, change !== 'passive', `Diagnostic fields must not change existing retention decisions: ${change}`);+ // An Act completion survives passive and automatic page activity.+ const retained = ['passive', 'page_focus', 'page_input', 'child'].includes(change);+ assert.equal(superseded, !retained, `Diagnostic fields must not change existing retention decisions: ${change}`);assert.equal(details.reason, change === 'passive' ? 'passive_dom': ['page_focus', 'page_input', 'child'].includes(change) ? 'automatic_page' : 'intervention_or_document');- assert.equal(details.disposition, change === 'passive' ? 'retained' : 'superseded');+ assert.equal(details.disposition, retained ? 'retained' : 'superseded');assert.equal(stateChecks, ['document', 'run'].includes(change) ? 0 : 1, 'Preserve the original state-check short circuit and call count');assert.deepEqual(details.context.bindingAtRefresh, { documentMatches: change !== 'document', currentRun: change !== 'run',stateCurrent: ['document', 'run'].includes(change) ? null : change !== 'steering',@@ -1531,9 +1536,10 @@ for (const build of ['chrome', 'firefox']) {const calls = [{ function: { name, arguments: change === 'malformed' ? '{' : JSON.stringify(args) } }];if (change === 'mixed') calls.push({ function: { name: 'type_text', arguments: '{"text":"new value"}' } });const superseded = await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: calls });- // Page input, child-frame focus and iframe focus are automatic; pure readers- // and independent tools stay current while action fences remain.- assert.equal(superseded, !['focus', 'pending_focus', 'page_input', 'child', 'frame_focus', 'calculate'].includes(change), change);+ // Page input, child-frame focus and iframe focus are automatic; pure readers,+ // independent tools and Act completions stay current while action fences remain.+ assert.equal(superseded, !['focus', 'pending_focus', 'page_input', 'child', 'frame_focus', 'calculate',+ 'success_done', 'partial_done', 'failed_done'].includes(change), change);} finally { api.webNavigation = navigation; agent._releaseRunEntry(tab); }}});diff --git a/test/pure-calculation-tools.mjs b/test/pure-calculation-tools.mjsindex 4a484d983..043c595f4 100644--- a/test/pure-calculation-tools.mjs+++ b/test/pure-calculation-tools.mjs@@ -498,7 +498,10 @@ for (const build of ['chrome', 'firefox']) {const blocked = await agent._applyPendingPageFeedback(tab, messages, callback,{ responseToolCalls: call('actual-terminal-check', 'done', { summary: answer, outcome: 'success' }).toolCalls });assert.equal(callbackObserved, true);- assert.equal(blocked, change !== 'none');+ // An Act completion survives automatic page activity (page focus,+ // synthetic input, focus moved to an iframe), with or without a+ // receipt. Authority, document, user and unknown changes still block.+ assert.equal(blocked, !['none', 'no_receipt', 'page_input', 'iframe_focus'].includes(change));assert.equal(messages.filter(message => message.role === 'tool').length, 0,'Retention is not a synthesized tool result or terminal completion.');} finally {