Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix page feedback review findings

Barack Sokullu077b0492235e
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index d0d5c4c2c..6fe711fbf 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -191,11 +191,23 @@ export const pageFeedbackMethods = {
const explicit = ['typed', 'auto_bookmark', 'generated', 'keyword', 'keyword_generated'].includes(details.transitionType)
|| qualifiers.includes('from_address_bar');
const navigation = run.navigation;
+ const redirect = qualifiers.some(q => /redirect$/.test(q));
+ const sameNavigation = !navigation?.redirectChain
+ ? (!navigation?.url || navigation.url === details.url || redirect)
+ : redirect && !!navigation.documentId && navigation.documentId === details.documentId;
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
- && (!navigation.url || navigation.url === details.url || qualifiers.some(q => /redirect$/.test(q)));
+ && sameNavigation;
+ // A completed click/navigation consumes its marker. Retain it only for a
+ // redirect explicitly tied to the same committed browser document.
+ if (navigation?.frameId === frameId) {
+ if (agentNavigation && redirect && details.documentId) {
+ run.navigation = { ...navigation, at: Date.now(), url: details.url || navigation.url,
+ redirectChain: true, documentId: details.documentId };
+ } else run.navigation = null;
+ }
if (agentNavigation) return;
- const source = explicit || (!navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';
+ const source = explicit || (!run.navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';
if (source === 'user') run.lastUserAt = Date.now();
run.lastActivityAt = Date.now();
this._queuePageFeedback(details.tabId, { kind: 'navigation', source, frameId,
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index b946ddb9a..3c6b2d926 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -142,6 +142,11 @@
return op;
};
for (const op of operations.values()) {
+ // Preparation can focus only the field that was already identified for
+ // this input action. A page script moving focus elsewhere stays visible.
+ if (kind === 'focus' && op.focusEligible && op.focusTarget === el && lastUserAt <= op.userAt) {
+ const found = match(op); if (found) return found;
+ }
if (!op.dispatched) continue;
if (lastUserAt > op.userAt) continue;
// Moving agent focus commits the previous field's native change event.
@@ -172,6 +177,11 @@
textSignatures.set(node, { text, signature: value });
return value;
}
+ function deepActiveElement() {
+ let el = document.activeElement;
+ while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;
+ return el;
+ }
function signature(el) {
if (ignored(el)) return '';
const shown = visible(el);
@@ -288,7 +298,15 @@
if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
- if (++measured > 300) break;
+ if (++measured > 300) {
+ // The remaining records are intentionally uninspected. Invalidate the
+ // prepared page conservatively so a later visible target cannot vanish
+ // silently at the end of a large, same-task update.
+ changed = true;
+ if (source !== 'user') source = agentTurn && lastUserAt <= agentTurn.userAt ? 'agent' : 'unknown';
+ target ||= targetName(el);
+ break;
+ }
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
if (record.type === 'childList') {
@@ -392,6 +410,15 @@
if (!event.isTrusted || expected('click', el, event)) return;
interact('click', el);
});
+ listen(document, 'focusin', event => {
+ const el = elementFor(event);
+ if (!event.isTrusted || ignored(el) || deepActiveElement() !== el || expected('focus', el, event)) return;
+ if (agentTurn && lastUserAt <= agentTurn.userAt) return;
+ // The pointer/key that led to a same-turn focus is already reported.
+ if (userTurn) return;
+ revision++; publishRevision();
+ send({ kind: 'activity', source: 'page', target: targetName(el) });
+ });
for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {
const el = elementFor(event);
if (ignored(el) || expected('input', el, event)
@@ -483,14 +510,18 @@
function prepare(params) {
if (!active || params.runToken !== runToken) return;
prune();
- operations.set(params.operationId, { ...params, target: resolveTarget(params), kinds: new Set(),
+ const target = resolveTarget(params);
+ const focusEligible = kindFor(params.tool) === 'input';
+ const focusTarget = target || (focusEligible ? deepActiveElement() : null);
+ operations.set(params.operationId, { ...params, target: target || focusTarget, focusTarget, focusEligible, kinds: new Set(),
until: Date.now() + 30000, dispatched: false, userAt: lastUserAt, preparedRevision: revision });
}
function dispatch(params) {
if (!active || (params.runToken && params.runToken !== runToken)) return;
const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };
op.blurTarget ??= document.activeElement;
- op.target = resolveTarget(params) || (params.kind === 'input' ? document.activeElement : op.target);
+ op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);
+ if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;
op.x = params.x; op.y = params.y;
if (!params.release || !op.seenEvents) op.seenEvents = new Set();
if (!params.release) op.eventTypes = Array.isArray(params.eventTypes) ? new Set(params.eventTypes) : null;
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index d0d5c4c2c..6fe711fbf 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -191,11 +191,23 @@ export const pageFeedbackMethods = {
const explicit = ['typed', 'auto_bookmark', 'generated', 'keyword', 'keyword_generated'].includes(details.transitionType)
|| qualifiers.includes('from_address_bar');
const navigation = run.navigation;
+ const redirect = qualifiers.some(q => /redirect$/.test(q));
+ const sameNavigation = !navigation?.redirectChain
+ ? (!navigation?.url || navigation.url === details.url || redirect)
+ : redirect && !!navigation.documentId && navigation.documentId === details.documentId;
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
- && (!navigation.url || navigation.url === details.url || qualifiers.some(q => /redirect$/.test(q)));
+ && sameNavigation;
+ // A completed click/navigation consumes its marker. Retain it only for a
+ // redirect explicitly tied to the same committed browser document.
+ if (navigation?.frameId === frameId) {
+ if (agentNavigation && redirect && details.documentId) {
+ run.navigation = { ...navigation, at: Date.now(), url: details.url || navigation.url,
+ redirectChain: true, documentId: details.documentId };
+ } else run.navigation = null;
+ }
if (agentNavigation) return;
- const source = explicit || (!navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';
+ const source = explicit || (!run.navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';
if (source === 'user') run.lastUserAt = Date.now();
run.lastActivityAt = Date.now();
this._queuePageFeedback(details.tabId, { kind: 'navigation', source, frameId,
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index b946ddb9a..3c6b2d926 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -142,6 +142,11 @@
return op;
};
for (const op of operations.values()) {
+ // Preparation can focus only the field that was already identified for
+ // this input action. A page script moving focus elsewhere stays visible.
+ if (kind === 'focus' && op.focusEligible && op.focusTarget === el && lastUserAt <= op.userAt) {
+ const found = match(op); if (found) return found;
+ }
if (!op.dispatched) continue;
if (lastUserAt > op.userAt) continue;
// Moving agent focus commits the previous field's native change event.
@@ -172,6 +177,11 @@
textSignatures.set(node, { text, signature: value });
return value;
}
+ function deepActiveElement() {
+ let el = document.activeElement;
+ while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;
+ return el;
+ }
function signature(el) {
if (ignored(el)) return '';
const shown = visible(el);
@@ -288,7 +298,15 @@
if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
- if (++measured > 300) break;
+ if (++measured > 300) {
+ // The remaining records are intentionally uninspected. Invalidate the
+ // prepared page conservatively so a later visible target cannot vanish
+ // silently at the end of a large, same-task update.
+ changed = true;
+ if (source !== 'user') source = agentTurn && lastUserAt <= agentTurn.userAt ? 'agent' : 'unknown';
+ target ||= targetName(el);
+ break;
+ }
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
if (record.type === 'childList') {
@@ -392,6 +410,15 @@
if (!event.isTrusted || expected('click', el, event)) return;
interact('click', el);
});
+ listen(document, 'focusin', event => {
+ const el = elementFor(event);
+ if (!event.isTrusted || ignored(el) || deepActiveElement() !== el || expected('focus', el, event)) return;
+ if (agentTurn && lastUserAt <= agentTurn.userAt) return;
+ // The pointer/key that led to a same-turn focus is already reported.
+ if (userTurn) return;
+ revision++; publishRevision();
+ send({ kind: 'activity', source: 'page', target: targetName(el) });
+ });
for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {
const el = elementFor(event);
if (ignored(el) || expected('input', el, event)
@@ -483,14 +510,18 @@
function prepare(params) {
if (!active || params.runToken !== runToken) return;
prune();
- operations.set(params.operationId, { ...params, target: resolveTarget(params), kinds: new Set(),
+ const target = resolveTarget(params);
+ const focusEligible = kindFor(params.tool) === 'input';
+ const focusTarget = target || (focusEligible ? deepActiveElement() : null);
+ operations.set(params.operationId, { ...params, target: target || focusTarget, focusTarget, focusEligible, kinds: new Set(),
until: Date.now() + 30000, dispatched: false, userAt: lastUserAt, preparedRevision: revision });
}
function dispatch(params) {
if (!active || (params.runToken && params.runToken !== runToken)) return;
const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };
op.blurTarget ??= document.activeElement;
- op.target = resolveTarget(params) || (params.kind === 'input' ? document.activeElement : op.target);
+ op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);
+ if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;
op.x = params.x; op.y = params.y;
if (!params.release || !op.seenEvents) op.seenEvents = new Set();
if (!params.release) op.eventTypes = Array.isArray(params.eventTypes) ? new Set(params.eventTypes) : null;
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 5966d0e43..de0905ec6 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -467,6 +467,38 @@ for (const build of ['chrome', 'firefox']) {
} finally { agent._releaseRunEntry(tab); }
});
+ test(`${build}: click navigation markers expire after one event and redirects stay document-scoped`, async () => {
+ const agent = setup(Agent), tab = nextTab++;
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ let binding = bind(agent, tab);
+ binding.send({ kind: 'activity', source: 'agent', operation: 'click' });
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'agent-document',
+ url: 'https://example.com/destination', transitionType: 'link' }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'The click-caused commit is correlated');
+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null, 'The click marker is consumed after commit');
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'timer-reload',
+ url: 'https://example.com/destination', transitionType: 'reload' }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'A later reload to the same URL is still visible');
+ assert.equal([...agent._pageFeedbackRuns.get(tab).events.values()].at(-1).source, 'unknown');
+
+ agent._pageFeedbackRuns.get(tab).events.clear();
+ binding = bind(agent, tab, 0, 'timer-reload', 'content-timer-reload');
+ binding.send({ kind: 'activity', source: 'agent', operation: 'click' });
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',
+ url: 'https://example.com/final', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'An identified redirect is part of the click navigation');
+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirected-document');
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',
+ url: 'https://example.com/final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'Redirect continuation stays on its exact document');
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'later-navigation',
+ url: 'https://example.com/automatic', transitionType: 'reload' }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'A later navigation outside the redirect chain is visible');
+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null);
+ } finally { agent._releaseRunEntry(tab); }
+ });
+
test(`${build}: content feedback uses authenticated sender identity; composer delivery never gates monitoring`, async () => {
const agent = setup(Agent), tab = nextTab++;
await agent._claimRunEntry(tab, 'cloud', { cloudRun: true });
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 771be9428..4d61714b0 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -404,6 +404,85 @@ test('Firefox native preparation keeps same-target user edits and sends no stale
});
for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
+ test(`${build}: page focus theft fences a selectorless input while the identified agent target stays quiet`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ document.getElementById('field').focus();
+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');
+ });
+ await page.waitForTimeout(200);
+ await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'selectorless-type', tool: 'type_text' });
+ deliver('page_monitor_dispatch', { operationId: 'selectorless-type', kind: 'input', fenceOnly: true });
+ document.getElementById('secret').focus();
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'activity'
+ && event.source === 'page' && event.target === 'input#secret'), null, { timeout: 1000 });
+ const observed = await page.evaluate(() => feedback);
+ assert.equal(JSON.stringify(observed).includes('password-never-recorded'), false);
+ const blocked = await page.evaluate(() => new Promise(resolve => {
+ deliver('page_monitor_dispatch', { operationId: 'selectorless-type', kind: 'input' });
+ setTimeout(() => resolve(lastMonitorResponse), 0);
+ }));
+ assert.equal(blocked.pageFeedbackPending, true);
+
+ await page.evaluate(() => {
+ monitorEnabled = false; deliver('page_monitor_state'); document.getElementById('field').focus();
+ monitorEnabled = true; deliver('page_monitor_state');
+ });
+ await page.waitForTimeout(100);
+ const accepted = await page.evaluate(() => {
+ feedback = []; deliver('page_monitor_prepare', { operationId: 'agent-focus', tool: 'type_text', selector: '#field' });
+ deliver('page_monitor_dispatch', { operationId: 'agent-focus', kind: 'input', fenceOnly: true });
+ document.getElementById('field').blur(); document.getElementById('field').focus();
+ return lastMonitorResponse;
+ });
+ assert.equal(accepted.ready, true);
+ await page.waitForTimeout(200);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'The active operation may focus its exact prepared target without looping');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: mutation batches over the inspection cap invalidate pending actions`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const hidden = document.createElement('div'); hidden.style.display = 'none';
+ hidden.innerHTML = '<span></span>'.repeat(320); document.body.prepend(hidden);
+ });
+ await page.waitForTimeout(200);
+ const result = await page.evaluate(async () => {
+ feedback = [];
+ const finish = __wbPageMonitor.beginContentAction('click', { selector: '#agent', _bidiPrepare: true });
+ for (const [index, node] of [...document.querySelectorAll('body > div[style*="display: none"] span')].entries())
+ node.setAttribute('aria-label', `hidden ${index}`);
+ document.getElementById('status').textContent = 'Visible tail mutation';
+ await Promise.resolve();
+ let code; try { __wbPageMonitor.beforeLocalDispatch(); } catch (error) { code = error.code; } finally { finish(); }
+ return { code, events: feedback };
+ });
+ assert.equal(result.code, 'page_feedback_pending');
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.source === 'unknown'), null, { timeout: 1000 });
+ assert.equal(JSON.stringify(result.events).includes('Visible tail mutation'), false);
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Visible tail mutation'), false);
+
+ await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'agent-large-write', tool: 'execute_js' });
+ deliver('page_monitor_dispatch', { operationId: 'agent-large-write', kind: 'dom' });
+ document.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', { detail: JSON.stringify(lastMonitorResponse.guard) }));
+ for (const [index, node] of [...document.querySelectorAll('body > div[style*="display: none"] span')].entries())
+ node.setAttribute('aria-label', `agent hidden ${index}`);
+ document.getElementById('status').textContent = 'Agent visible tail mutation';
+ });
+ await page.waitForTimeout(200);
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'dom' && event.source !== 'agent'), false,
+ 'Overflow in a marked agent write keeps the event attributable instead of creating page feedback');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: a pointer gesture ending on owned UI releases the background idle gate`, async () => {
const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser, tab = 945;
const host = Object.assign({ isRunning: () => true, _checkAbort: () => false, _runAbortSignal: () => null,