Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Keep finished Act answers through automatic page churn

Claudeb998d4957289
diff --git a/src/chrome/src/agent/page-feedback-policy.js b/src/chrome/src/agent/page-feedback-policy.js
index 810517e1e..d42eb3466 100644
--- a/src/chrome/src/agent/page-feedback-policy.js
+++ b/src/chrome/src/agent/page-feedback-policy.js
@@ -226,9 +226,9 @@ export function isPassivePageFeedback(events) {
}
// Automatic feedback is an observed page consequence with no evidence of a
-// human gesture. Every 'user' event, main-frame navigation, a top-level
-// viewport change of unknown origin and base/style/link updates keep the
-// strict fence. Focus moved to an <iframe> is page-script focus: Chromium and
+// human gesture. Every 'user' event, main-frame navigation, any top-level
+// viewport change (the monitor reports those as unknown) and base/style/link
+// updates keep the strict fence. Focus moved to an <iframe> is page-script focus: Chromium and
// Firefox fire no parent focusin for a click or Tab inside a frame (a Tab
// reaches the parent monitor as user keydown activity).
const AUTOMATIC_SOURCES = new Map([['dom', ['page', 'unknown']], ['scroll', ['page', 'unknown']],
@@ -237,7 +237,7 @@ export function isAutomaticPageFeedbackEvent(event) {
if (!AUTOMATIC_SOURCES.get(event?.kind)?.includes(event.source)
|| BASE_STYLE_LINK.test(typeof event.target === 'string' ? event.target : '')) return false;
const childFrame = Number.isSafeInteger(event.frameId) && event.frameId > 0;
- return event.kind === 'navigation' || (event.kind === 'resize' && event.source === 'unknown') ? childFrame : true;
+ return event.kind === 'navigation' || event.kind === 'resize' ? childFrame : true;
}
export function isAutomaticPageFeedback(events) {
@@ -364,7 +364,8 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {
if (!calls?.length) return false;
const passive = isPassivePageFeedback(events);
const calculation = isPageIndependentCalculationCompletion(calls, state, events);
- if (!passive && !calculation && !isAutomaticPageFeedback(events)) return false;
+ const automatic = !passive && isAutomaticPageFeedback(events);
+ if (!passive && !calculation && !automatic) return false;
const interactive = !passive && events.some(event => ['activity', 'input'].includes(event?.kind));
return calls.every(call => {
let args;
@@ -372,9 +373,12 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {
if (!args || typeof args !== 'object' || Array.isArray(args)) return false;
const policy = pageFeedbackCallPolicy(call.function?.name, args, state, currentPage, events);
if (policy.terminal && calls.length !== 1) return false;
- // A terminal answer keeps its established fences (passive churn or an owned
- // literal calculation); Ask scope and the completion verifier own the rest.
- if (policy.terminal) return passive || calculation;
+ // An Act completion survives automatic churn (ad frames, page focus,
+ // unattributed scroll or DOM): superseding it on every tick would end a
+ // finished task as page_unstable. Its batch still runs the completion
+ // verifier on fresh evidence, and user input still supersedes it. Ask
+ // answers keep their own scope classification and passive-only fence.
+ if (policy.terminal) return passive || calculation || (automatic && state?.runMode !== 'ask');
if (interactive && policy.kind === 'observe' && !PAGE_ACTIVITY_READS.has(call.function?.name)) return false;
// A target comparison only certifies the snapshot just read. Further DOM
// changes require local revalidation; fresh readers/media have their own fences.
diff --git a/src/firefox/src/agent/page-feedback-policy.js b/src/firefox/src/agent/page-feedback-policy.js
index 810517e1e..d42eb3466 100644
--- a/src/firefox/src/agent/page-feedback-policy.js
+++ b/src/firefox/src/agent/page-feedback-policy.js
@@ -226,9 +226,9 @@ export function isPassivePageFeedback(events) {
}
// Automatic feedback is an observed page consequence with no evidence of a
-// human gesture. Every 'user' event, main-frame navigation, a top-level
-// viewport change of unknown origin and base/style/link updates keep the
-// strict fence. Focus moved to an <iframe> is page-script focus: Chromium and
+// human gesture. Every 'user' event, main-frame navigation, any top-level
+// viewport change (the monitor reports those as unknown) and base/style/link
+// updates keep the strict fence. Focus moved to an <iframe> is page-script focus: Chromium and
// Firefox fire no parent focusin for a click or Tab inside a frame (a Tab
// reaches the parent monitor as user keydown activity).
const AUTOMATIC_SOURCES = new Map([['dom', ['page', 'unknown']], ['scroll', ['page', 'unknown']],
@@ -237,7 +237,7 @@ export function isAutomaticPageFeedbackEvent(event) {
if (!AUTOMATIC_SOURCES.get(event?.kind)?.includes(event.source)
|| BASE_STYLE_LINK.test(typeof event.target === 'string' ? event.target : '')) return false;
const childFrame = Number.isSafeInteger(event.frameId) && event.frameId > 0;
- return event.kind === 'navigation' || (event.kind === 'resize' && event.source === 'unknown') ? childFrame : true;
+ return event.kind === 'navigation' || event.kind === 'resize' ? childFrame : true;
}
export function isAutomaticPageFeedback(events) {
@@ -364,7 +364,8 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {
if (!calls?.length) return false;
const passive = isPassivePageFeedback(events);
const calculation = isPageIndependentCalculationCompletion(calls, state, events);
- if (!passive && !calculation && !isAutomaticPageFeedback(events)) return false;
+ const automatic = !passive && isAutomaticPageFeedback(events);
+ if (!passive && !calculation && !automatic) return false;
const interactive = !passive && events.some(event => ['activity', 'input'].includes(event?.kind));
return calls.every(call => {
let args;
@@ -372,9 +373,12 @@ export function canRetainPageFeedbackCalls(calls, state, currentPage, events, {
if (!args || typeof args !== 'object' || Array.isArray(args)) return false;
const policy = pageFeedbackCallPolicy(call.function?.name, args, state, currentPage, events);
if (policy.terminal && calls.length !== 1) return false;
- // A terminal answer keeps its established fences (passive churn or an owned
- // literal calculation); Ask scope and the completion verifier own the rest.
- if (policy.terminal) return passive || calculation;
+ // An Act completion survives automatic churn (ad frames, page focus,
+ // unattributed scroll or DOM): superseding it on every tick would end a
+ // finished task as page_unstable. Its batch still runs the completion
+ // verifier on fresh evidence, and user input still supersedes it. Ask
+ // answers keep their own scope classification and passive-only fence.
+ if (policy.terminal) return passive || calculation || (automatic && state?.runMode !== 'ask');
if (interactive && policy.kind === 'observe' && !PAGE_ACTIVITY_READS.has(call.function?.name)) return false;
// A target comparison only certifies the snapshot just read. Further DOM
// changes require local revalidation; fresh readers/media have their own fences.
diff --git a/test/feedback-automatic-churn.mjs b/test/feedback-automatic-churn.mjs
index 165c8906b..a850c599f 100644
--- a/test/feedback-automatic-churn.mjs
+++ b/test/feedback-automatic-churn.mjs
@@ -121,7 +121,7 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(policy.isAutomaticPageFeedback([]), false);
});
- test(`${build}: automatic churn retains reads and independent tools but keeps unsafe, legacy non-DOM and terminal fences`, () => {
+ test(`${build}: automatic churn retains reads, independent tools and Act completions but keeps unsafe and legacy non-DOM fences`, () => {
const page = { success: true, pageContent: TREE };
const state = { url: PAGE_URL, page };
const keep = (name, args, events, current = page, scope = state) =>
@@ -134,7 +134,16 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(keep('calculate', { expression: '2 + 2' }, events), true);
assert.equal(keep('click', { x: 10, y: 20 }, events), false, 'Coordinates stay fenced');
assert.equal(keep('execute_js', { code: 'save()' }, events), false, 'Scripts stay fenced');
- assert.equal(keep('done', { summary: 'Done', outcome: 'success' }, events), false, 'Terminal answers keep passive-only retention');
+ // An Act completion survives automatic churn; its batch verifies it.
+ assert.equal(keep('done', { summary: 'Done', outcome: 'success' }, events), true, 'Act completion survives automatic churn');
+ // Ask answers keep their own scope classification and passive-only fence.
+ assert.equal(keep('done', { summary: 'The price is 12' }, events, page, { ...state, runMode: 'ask' }), false,
+ 'Ask answers keep passive-only retention');
+ // A user gesture during inference still supersedes the completion.
+ assert.equal(keep('done', { summary: 'Done', outcome: 'success' },
+ [...events, { kind: 'click', source: 'user', frameId: 0, target: 'button#other' }]), false, 'User input supersedes a completion');
+ assert.equal(keep('done', { summary: 'Done', outcome: 'success' },
+ [...events, { kind: 'navigation', source: 'unknown', frameId: 0 }]), false, 'Main-frame navigation supersedes a completion');
}
assert.equal(keep('chat_observe', {}, focus), false, 'Read preparation stays fenced after page focus changes');
assert.equal(keep('chat_observe', {}, scroll), true);
@@ -217,6 +226,40 @@ for (const build of ['chrome', 'firefox']) {
});
}
+ for (const streaming of [false, true]) {
+ test(`${build}: ${streaming ? 'stream' : 'chat'} a finished task under continuous ad-frame churn ends with its answer`, async () => {
+ const tab = nextTab++, dispatched = [];
+ let agent, frames, requests = 0, terminal = null;
+ pages.set(tab, {});
+ agent = makeAgent(Agent, async () => {
+ requests++;
+ // The ad frame ticks during every inference, as on many real pages.
+ frames.child({ kind: 'dom', source: 'page', target: 'div#ad' });
+ frames.main({ kind: 'scroll', source: 'unknown', target: 'div#carousel', viewport: { x: 100 * requests, y: 0 } });
+ return requests === 1 ? call('add-1', 'click_ax', { ref_id: 'ref_add' })
+ : call('finished-' + requests, 'done', { summary: 'Added to cart', outcome: 'success' });
+ });
+ agent.maxSteps = 8;
+ agent._endTraceRun = async (_tab, _run, status) => { terminal = status; };
+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => { frames = bindFrames(agent, tab); return { role: 'user', content }; };
+ agent._executeToolWithLifecycle = async (_tab, tool, args) => {
+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: TREE };
+ dispatched.push(tool);
+ return tool === 'done' ? { done: true, success: true, summary: args.summary, outcome: args.outcome }
+ : { success: true, dispatched: true, verified: true };
+ };
+ try {
+ const options = { detachedRequestId: 'finished-' + tab, askStreamingEnabled: false };
+ const result = streaming ? await agent.processMessageStream(tab, 'Add the blue shoes', () => {}, 'act', options)
+ : await agent.processMessage(tab, 'Add the blue shoes', () => {}, 'act', [], options);
+ assert.equal(result, 'Added to cart');
+ assert.equal(requests, 2, 'Automatic churn must not send a finished answer back to the model');
+ assert.deepEqual(dispatched, ['click_ax', 'done']);
+ assert.notEqual(terminal, 'page_unstable');
+ } finally { pages.delete(tab); }
+ });
+ }
+
for (const [kind, target] of [['click', 'button#other'], ['input', 'input#q'], ['activity', 'textarea#note']]) {
test(`${build}: a genuine user ${kind === 'activity' ? 'keypress' : kind} during inference still supersedes and waits for idle`, async () => {
const tab = nextTab++, dispatched = [], starts = [];
@@ -346,6 +389,20 @@ for (const build of ['chrome', 'firefox']) {
frames.child({ kind: 'dom', source: 'page', target: 'div#ad' });
return { done: true, success: true, summary: args.summary };
};
+ // Automatic ad-frame churn alone no longer supersedes the completion.
+ assert.equal((await agent.executeTool(tab, 'done', args, () => {}, {})).done, true);
+ agent._resetPageFeedbackProgress(tab);
+ assert.equal(await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: [doneCall] }), false,
+ 'An Act completion survives automatic churn');
+ // An unattributed top-level viewport change is strict but not user input:
+ // each superseded completion counts toward the bounded budget.
+ agent._executeToolWithLifecycle = async (_tab, tool) => {
+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: TREE };
+ assert.equal(tool, 'done');
+ assert.equal((await agent.executeTool(tab, 'get_accessibility_tree', { maxChars: 5500 })).success, true);
+ frames.main({ kind: 'resize', source: 'unknown', target: 'viewport', viewport: { width: 1000 + run.revision, height: 700 } });
+ return { done: true, success: true, summary: args.summary };
+ };
for (let attempt = 1; attempt <= 5; attempt++) {
assert.equal((await agent.executeTool(tab, 'done', args, () => {}, {})).done, true);
agent._resetPageFeedbackProgress(tab); // the batch's successful-dispatch hook
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index e048ccf17..26db28eef 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -130,9 +130,12 @@ for (const build of ['chrome', 'firefox']) {
});
}
+ // An Act completion also survives automatic churn (unknown-attribution and
+ // child-frame DOM); its batch still runs the completion verifier.
+ const completionSurvives = change => ['feed', 'partial', 'unknown', 'iframe'].includes(change);
for (const change of ['feed', 'user', 'unknown', 'iframe', 'navigation', 'viewport', 'style', 'failed_read',
'partial', 'missing_outcome', 'malformed', 'mixed', 'duplicate', 'pending']) {
- test(`${build}: explicit completion ${['feed', 'partial'].includes(change) ? 'survives' : 'rejects'} ${change} feedback`, async () => {
+ test(`${build}: explicit completion ${completionSurvives(change) ? 'survives' : 'rejects'} ${change} feedback`, async () => {
const agent = setup(Agent), tab = nextTab++;
let page = { success: true, pageContent: 'article "Published WebBrain post" [ref_1]' };
agent.executeTool = async () => page;
@@ -156,7 +159,7 @@ for (const build of ['chrome', 'firefox']) {
const call = { function: { name: 'done', arguments: change === 'malformed' ? '{' : JSON.stringify(args) } };
const calls = change === 'duplicate' ? [call, call] : change === 'mixed'
? [call, { function: { name: 'click_ax', arguments: '{"ref_id":"ref_1"}' } }] : [call];
- assert.equal(await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: calls }), !['feed', 'partial'].includes(change));
+ assert.equal(await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: calls }), !completionSurvives(change));
assert.equal(agent._activeTaskBinding(messages).text, 'Publish a WebBrain post');
} finally { agent._releaseRunEntry(tab); }
});
@@ -1359,10 +1362,12 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(note.kind, 'note'); assert.equal(note.data.note, 'page_feedback_decision'); assert.equal(note.data.step, 0);
const details = note.data.extra;
assert.deepEqual(note.metadata.extra.context, details.context, 'Safe causal details survive the actual metadata-only trace projection');
- assert.equal(superseded, change !== 'passive', `Diagnostic fields must not change existing retention decisions: ${change}`);
+ // An Act completion survives passive and automatic page activity.
+ const retained = ['passive', 'page_focus', 'page_input', 'child'].includes(change);
+ assert.equal(superseded, !retained, `Diagnostic fields must not change existing retention decisions: ${change}`);
assert.equal(details.reason, change === 'passive' ? 'passive_dom'
: ['page_focus', 'page_input', 'child'].includes(change) ? 'automatic_page' : 'intervention_or_document');
- assert.equal(details.disposition, change === 'passive' ? 'retained' : 'superseded');
+ assert.equal(details.disposition, retained ? 'retained' : 'superseded');
assert.equal(stateChecks, ['document', 'run'].includes(change) ? 0 : 1, 'Preserve the original state-check short circuit and call count');
assert.deepEqual(details.context.bindingAtRefresh, { documentMatches: change !== 'document', currentRun: change !== 'run',
stateCurrent: ['document', 'run'].includes(change) ? null : change !== 'steering',
@@ -1531,9 +1536,10 @@ for (const build of ['chrome', 'firefox']) {
const calls = [{ function: { name, arguments: change === 'malformed' ? '{' : JSON.stringify(args) } }];
if (change === 'mixed') calls.push({ function: { name: 'type_text', arguments: '{"text":"new value"}' } });
const superseded = await agent._applyPendingPageFeedback(tab, messages, () => {}, { responseToolCalls: calls });
- // Page input, child-frame focus and iframe focus are automatic; pure readers
- // and independent tools stay current while action fences remain.
- assert.equal(superseded, !['focus', 'pending_focus', 'page_input', 'child', 'frame_focus', 'calculate'].includes(change), change);
+ // Page input, child-frame focus and iframe focus are automatic; pure readers,
+ // independent tools and Act completions stay current while action fences remain.
+ assert.equal(superseded, !['focus', 'pending_focus', 'page_input', 'child', 'frame_focus', 'calculate',
+ 'success_done', 'partial_done', 'failed_done'].includes(change), change);
} finally { api.webNavigation = navigation; agent._releaseRunEntry(tab); }
}
});
diff --git a/test/pure-calculation-tools.mjs b/test/pure-calculation-tools.mjs
index 4a484d983..043c595f4 100644
--- a/test/pure-calculation-tools.mjs
+++ b/test/pure-calculation-tools.mjs
@@ -498,7 +498,10 @@ for (const build of ['chrome', 'firefox']) {
const blocked = await agent._applyPendingPageFeedback(tab, messages, callback,
{ responseToolCalls: call('actual-terminal-check', 'done', { summary: answer, outcome: 'success' }).toolCalls });
assert.equal(callbackObserved, true);
- assert.equal(blocked, change !== 'none');
+ // An Act completion survives automatic page activity (page focus,
+ // synthetic input, focus moved to an iframe), with or without a
+ // receipt. Authority, document, user and unknown changes still block.
+ assert.equal(blocked, !['none', 'no_receipt', 'page_input', 'iframe_focus'].includes(change));
assert.equal(messages.filter(message => message.role === 'tool').length, 0,
'Retention is not a synthesized tool result or terminal completion.');
} finally {