Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix page feedback review findings
Barack Sokullu077b0492235e
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex d0d5c4c2c..6fe711fbf 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -191,11 +191,23 @@ export const pageFeedbackMethods = {const explicit = ['typed', 'auto_bookmark', 'generated', 'keyword', 'keyword_generated'].includes(details.transitionType)|| qualifiers.includes('from_address_bar');const navigation = run.navigation;+ const redirect = qualifiers.some(q => /redirect$/.test(q));+ const sameNavigation = !navigation?.redirectChain+ ? (!navigation?.url || navigation.url === details.url || redirect)+ : redirect && !!navigation.documentId && navigation.documentId === details.documentId;const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)- && (!navigation.url || navigation.url === details.url || qualifiers.some(q => /redirect$/.test(q)));+ && sameNavigation;+ // A completed click/navigation consumes its marker. Retain it only for a+ // redirect explicitly tied to the same committed browser document.+ if (navigation?.frameId === frameId) {+ if (agentNavigation && redirect && details.documentId) {+ run.navigation = { ...navigation, at: Date.now(), url: details.url || navigation.url,+ redirectChain: true, documentId: details.documentId };+ } else run.navigation = null;+ }if (agentNavigation) return;- const source = explicit || (!navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';+ const source = explicit || (!run.navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';if (source === 'user') run.lastUserAt = Date.now();run.lastActivityAt = Date.now();this._queuePageFeedback(details.tabId, { kind: 'navigation', source, frameId,diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex b946ddb9a..3c6b2d926 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -142,6 +142,11 @@return op;};for (const op of operations.values()) {+ // Preparation can focus only the field that was already identified for+ // this input action. A page script moving focus elsewhere stays visible.+ if (kind === 'focus' && op.focusEligible && op.focusTarget === el && lastUserAt <= op.userAt) {+ const found = match(op); if (found) return found;+ }if (!op.dispatched) continue;if (lastUserAt > op.userAt) continue;// Moving agent focus commits the previous field's native change event.@@ -172,6 +177,11 @@textSignatures.set(node, { text, signature: value });return value;}+ function deepActiveElement() {+ let el = document.activeElement;+ while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;+ return el;+ }function signature(el) {if (ignored(el)) return '';const shown = visible(el);@@ -288,7 +298,15 @@if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;- if (++measured > 300) break;+ if (++measured > 300) {+ // The remaining records are intentionally uninspected. Invalidate the+ // prepared page conservatively so a later visible target cannot vanish+ // silently at the end of a large, same-task update.+ changed = true;+ if (source !== 'user') source = agentTurn && lastUserAt <= agentTurn.userAt ? 'agent' : 'unknown';+ target ||= targetName(el);+ break;+ }let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));if (record.type === 'childList') {@@ -392,6 +410,15 @@if (!event.isTrusted || expected('click', el, event)) return;interact('click', el);});+ listen(document, 'focusin', event => {+ const el = elementFor(event);+ if (!event.isTrusted || ignored(el) || deepActiveElement() !== el || expected('focus', el, event)) return;+ if (agentTurn && lastUserAt <= agentTurn.userAt) return;+ // The pointer/key that led to a same-turn focus is already reported.+ if (userTurn) return;+ revision++; publishRevision();+ send({ kind: 'activity', source: 'page', target: targetName(el) });+ });for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {const el = elementFor(event);if (ignored(el) || expected('input', el, event)@@ -483,14 +510,18 @@function prepare(params) {if (!active || params.runToken !== runToken) return;prune();- operations.set(params.operationId, { ...params, target: resolveTarget(params), kinds: new Set(),+ const target = resolveTarget(params);+ const focusEligible = kindFor(params.tool) === 'input';+ const focusTarget = target || (focusEligible ? deepActiveElement() : null);+ operations.set(params.operationId, { ...params, target: target || focusTarget, focusTarget, focusEligible, kinds: new Set(),until: Date.now() + 30000, dispatched: false, userAt: lastUserAt, preparedRevision: revision });}function dispatch(params) {if (!active || (params.runToken && params.runToken !== runToken)) return;const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };op.blurTarget ??= document.activeElement;- op.target = resolveTarget(params) || (params.kind === 'input' ? document.activeElement : op.target);+ op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);+ if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;op.x = params.x; op.y = params.y;if (!params.release || !op.seenEvents) op.seenEvents = new Set();if (!params.release) op.eventTypes = Array.isArray(params.eventTypes) ? new Set(params.eventTypes) : null;diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex d0d5c4c2c..6fe711fbf 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -191,11 +191,23 @@ export const pageFeedbackMethods = {const explicit = ['typed', 'auto_bookmark', 'generated', 'keyword', 'keyword_generated'].includes(details.transitionType)|| qualifiers.includes('from_address_bar');const navigation = run.navigation;+ const redirect = qualifiers.some(q => /redirect$/.test(q));+ const sameNavigation = !navigation?.redirectChain+ ? (!navigation?.url || navigation.url === details.url || redirect)+ : redirect && !!navigation.documentId && navigation.documentId === details.documentId;const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)- && (!navigation.url || navigation.url === details.url || qualifiers.some(q => /redirect$/.test(q)));+ && sameNavigation;+ // A completed click/navigation consumes its marker. Retain it only for a+ // redirect explicitly tied to the same committed browser document.+ if (navigation?.frameId === frameId) {+ if (agentNavigation && redirect && details.documentId) {+ run.navigation = { ...navigation, at: Date.now(), url: details.url || navigation.url,+ redirectChain: true, documentId: details.documentId };+ } else run.navigation = null;+ }if (agentNavigation) return;- const source = explicit || (!navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';+ const source = explicit || (!run.navigation && (Date.now() - run.lastUserAt < 2000 || qualifiers.includes('forward_back'))) ? 'user' : 'unknown';if (source === 'user') run.lastUserAt = Date.now();run.lastActivityAt = Date.now();this._queuePageFeedback(details.tabId, { kind: 'navigation', source, frameId,diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex b946ddb9a..3c6b2d926 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -142,6 +142,11 @@return op;};for (const op of operations.values()) {+ // Preparation can focus only the field that was already identified for+ // this input action. A page script moving focus elsewhere stays visible.+ if (kind === 'focus' && op.focusEligible && op.focusTarget === el && lastUserAt <= op.userAt) {+ const found = match(op); if (found) return found;+ }if (!op.dispatched) continue;if (lastUserAt > op.userAt) continue;// Moving agent focus commits the previous field's native change event.@@ -172,6 +177,11 @@textSignatures.set(node, { text, signature: value });return value;}+ function deepActiveElement() {+ let el = document.activeElement;+ while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;+ return el;+ }function signature(el) {if (ignored(el)) return '';const shown = visible(el);@@ -288,7 +298,15 @@if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;- if (++measured > 300) break;+ if (++measured > 300) {+ // The remaining records are intentionally uninspected. Invalidate the+ // prepared page conservatively so a later visible target cannot vanish+ // silently at the end of a large, same-task update.+ changed = true;+ if (source !== 'user') source = agentTurn && lastUserAt <= agentTurn.userAt ? 'agent' : 'unknown';+ target ||= targetName(el);+ break;+ }let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));if (record.type === 'childList') {@@ -392,6 +410,15 @@if (!event.isTrusted || expected('click', el, event)) return;interact('click', el);});+ listen(document, 'focusin', event => {+ const el = elementFor(event);+ if (!event.isTrusted || ignored(el) || deepActiveElement() !== el || expected('focus', el, event)) return;+ if (agentTurn && lastUserAt <= agentTurn.userAt) return;+ // The pointer/key that led to a same-turn focus is already reported.+ if (userTurn) return;+ revision++; publishRevision();+ send({ kind: 'activity', source: 'page', target: targetName(el) });+ });for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {const el = elementFor(event);if (ignored(el) || expected('input', el, event)@@ -483,14 +510,18 @@function prepare(params) {if (!active || params.runToken !== runToken) return;prune();- operations.set(params.operationId, { ...params, target: resolveTarget(params), kinds: new Set(),+ const target = resolveTarget(params);+ const focusEligible = kindFor(params.tool) === 'input';+ const focusTarget = target || (focusEligible ? deepActiveElement() : null);+ operations.set(params.operationId, { ...params, target: target || focusTarget, focusTarget, focusEligible, kinds: new Set(),until: Date.now() + 30000, dispatched: false, userAt: lastUserAt, preparedRevision: revision });}function dispatch(params) {if (!active || (params.runToken && params.runToken !== runToken)) return;const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };op.blurTarget ??= document.activeElement;- op.target = resolveTarget(params) || (params.kind === 'input' ? document.activeElement : op.target);+ op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);+ if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;op.x = params.x; op.y = params.y;if (!params.release || !op.seenEvents) op.seenEvents = new Set();if (!params.release) op.eventTypes = Array.isArray(params.eventTypes) ? new Set(params.eventTypes) : null;diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 5966d0e43..de0905ec6 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -467,6 +467,38 @@ for (const build of ['chrome', 'firefox']) {} finally { agent._releaseRunEntry(tab); }});+ test(`${build}: click navigation markers expire after one event and redirects stay document-scoped`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ await agent._claimRunEntry(tab, 'interactive');+ try {+ let binding = bind(agent, tab);+ binding.send({ kind: 'activity', source: 'agent', operation: 'click' });+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'agent-document',+ url: 'https://example.com/destination', transitionType: 'link' }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'The click-caused commit is correlated');+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null, 'The click marker is consumed after commit');+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'timer-reload',+ url: 'https://example.com/destination', transitionType: 'reload' }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'A later reload to the same URL is still visible');+ assert.equal([...agent._pageFeedbackRuns.get(tab).events.values()].at(-1).source, 'unknown');++ agent._pageFeedbackRuns.get(tab).events.clear();+ binding = bind(agent, tab, 0, 'timer-reload', 'content-timer-reload');+ binding.send({ kind: 'activity', source: 'agent', operation: 'click' });+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',+ url: 'https://example.com/final', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'An identified redirect is part of the click navigation');+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirected-document');+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',+ url: 'https://example.com/final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'Redirect continuation stays on its exact document');+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'later-navigation',+ url: 'https://example.com/automatic', transitionType: 'reload' }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'A later navigation outside the redirect chain is visible');+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null);+ } finally { agent._releaseRunEntry(tab); }+ });+test(`${build}: content feedback uses authenticated sender identity; composer delivery never gates monitoring`, async () => {const agent = setup(Agent), tab = nextTab++;await agent._claimRunEntry(tab, 'cloud', { cloudRun: true });diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 771be9428..4d61714b0 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -404,6 +404,85 @@ test('Firefox native preparation keeps same-target user edits and sends no stale});for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {+ test(`${build}: page focus theft fences a selectorless input while the identified agent target stays quiet`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ document.getElementById('field').focus();+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');+ });+ await page.waitForTimeout(200);+ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'selectorless-type', tool: 'type_text' });+ deliver('page_monitor_dispatch', { operationId: 'selectorless-type', kind: 'input', fenceOnly: true });+ document.getElementById('secret').focus();+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'activity'+ && event.source === 'page' && event.target === 'input#secret'), null, { timeout: 1000 });+ const observed = await page.evaluate(() => feedback);+ assert.equal(JSON.stringify(observed).includes('password-never-recorded'), false);+ const blocked = await page.evaluate(() => new Promise(resolve => {+ deliver('page_monitor_dispatch', { operationId: 'selectorless-type', kind: 'input' });+ setTimeout(() => resolve(lastMonitorResponse), 0);+ }));+ assert.equal(blocked.pageFeedbackPending, true);++ await page.evaluate(() => {+ monitorEnabled = false; deliver('page_monitor_state'); document.getElementById('field').focus();+ monitorEnabled = true; deliver('page_monitor_state');+ });+ await page.waitForTimeout(100);+ const accepted = await page.evaluate(() => {+ feedback = []; deliver('page_monitor_prepare', { operationId: 'agent-focus', tool: 'type_text', selector: '#field' });+ deliver('page_monitor_dispatch', { operationId: 'agent-focus', kind: 'input', fenceOnly: true });+ document.getElementById('field').blur(); document.getElementById('field').focus();+ return lastMonitorResponse;+ });+ assert.equal(accepted.ready, true);+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'The active operation may focus its exact prepared target without looping');+ } finally { await browser.close(); }+ });++ test(`${build}: mutation batches over the inspection cap invalidate pending actions`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const hidden = document.createElement('div'); hidden.style.display = 'none';+ hidden.innerHTML = '<span></span>'.repeat(320); document.body.prepend(hidden);+ });+ await page.waitForTimeout(200);+ const result = await page.evaluate(async () => {+ feedback = [];+ const finish = __wbPageMonitor.beginContentAction('click', { selector: '#agent', _bidiPrepare: true });+ for (const [index, node] of [...document.querySelectorAll('body > div[style*="display: none"] span')].entries())+ node.setAttribute('aria-label', `hidden ${index}`);+ document.getElementById('status').textContent = 'Visible tail mutation';+ await Promise.resolve();+ let code; try { __wbPageMonitor.beforeLocalDispatch(); } catch (error) { code = error.code; } finally { finish(); }+ return { code, events: feedback };+ });+ assert.equal(result.code, 'page_feedback_pending');+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.source === 'unknown'), null, { timeout: 1000 });+ assert.equal(JSON.stringify(result.events).includes('Visible tail mutation'), false);+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Visible tail mutation'), false);++ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'agent-large-write', tool: 'execute_js' });+ deliver('page_monitor_dispatch', { operationId: 'agent-large-write', kind: 'dom' });+ document.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', { detail: JSON.stringify(lastMonitorResponse.guard) }));+ for (const [index, node] of [...document.querySelectorAll('body > div[style*="display: none"] span')].entries())+ node.setAttribute('aria-label', `agent hidden ${index}`);+ document.getElementById('status').textContent = 'Agent visible tail mutation';+ });+ await page.waitForTimeout(200);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'dom' && event.source !== 'agent'), false,+ 'Overflow in a marked agent write keeps the event attributable instead of creating page feedback');+ } finally { await browser.close(); }+ });+test(`${build}: a pointer gesture ending on owned UI releases the background idle gate`, async () => {const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser, tab = 945;const host = Object.assign({ isRunning: () => true, _checkAbort: () => false, _runAbortSignal: () => null,