Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: scope navigation correlation to dispatched frames
Barack Sokulluf361ceb8c538
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.mdindex 8d37dd9ae..31692cb59 100644--- a/docs/page-feedback-monitor.md+++ b/docs/page-feedback-monitor.md@@ -71,6 +71,10 @@ Click navigation correlation starts with matching page input, rather than thepreparation handshake. Proven undispatched actions release their navigationmarker. Notice IDs capture the consumed batch revision before the asynchronouspage read, so navigation arriving during that read keeps its own persisted note.+Navigation correlation stays within the dispatched frame; a child link explicitly+targeting the top frame can correlate that target without hiding sibling changes.+Firefox arms direct navigation after URL validation and the unsaved-changes probe,+immediately before the native transport or tab update.## Verificationdiff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex 9232949ab..13e6a1738 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -75,6 +75,7 @@ export const pageFeedbackMethods = {dispatched: details => {if (details.kind === 'navigate') {run.navigation = { at: Date.now(), url: details.url || '', kind: details.kind, history: details.history === true,+ frameId: Number(details.frameId) || 0,operationId: dispatchOwners.get(tabId)?.operationId || '' };}} });@@ -130,6 +131,7 @@ export const pageFeedbackMethods = {frame.seq = seq;if (feedback.source === 'agent') {if (feedback.operation === 'click') run.navigation = { at: Date.now(), url: '', kind: 'click',+ frameId: feedback.navigationTarget === '_top' ? 0 : frameId,operationId: dispatchOwners.get(tabId)?.operationId || '' };return { accepted: true };}@@ -168,7 +170,7 @@ export const pageFeedbackMethods = {const explicit = ['typed', 'auto_bookmark', 'generated', 'keyword', 'keyword_generated'].includes(details.transitionType)|| qualifiers.includes('from_address_bar');const navigation = run.navigation;- const agentNavigation = !explicit && navigation && Date.now() - navigation.at < 10000+ const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)&& (!navigation.url || navigation.url === details.url || qualifiers.some(q => /redirect$/.test(q)));if (agentNavigation) return;diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex aaffa0687..c5fc8beed 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -263,7 +263,12 @@const marker = { userAt: op.userAt };agentTurn = marker;if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {- send({ kind: 'activity', source: 'agent', operation: 'click' });+ const link = event.composedPath().find(node => node instanceof Element && node.matches('a[href],area[href]'));+ const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')+ || (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();+ send({ kind: 'activity', source: 'agent', operation: 'click',+ ...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)+ ? { navigationTarget: '_top' } : {}) });}// Native listeners have microtask checkpoints between callbacks. Keep this// exact input's attribution through its page handlers, until the next task.diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex c72f34e4b..81f4ed089 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -31181,9 +31181,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}async _executeToolImpl(tabId, name, args, onUpdate = null, executionContext = null) {- if (['navigate', 'go_back', 'go_forward', 'execute_js', 'inject_css', 'remove_injected_css'].includes(name)) {- await beforePageAgentDispatch(browser, tabId, { kind: ['navigate', 'go_back', 'go_forward'].includes(name) ? 'navigate' : 'dom',- url: name === 'navigate' ? args?.url : undefined });+ if (['execute_js', 'inject_css', 'remove_injected_css'].includes(name)) {+ await beforePageAgentDispatch(browser, tabId, { kind: 'dom' });}const dispatchContext = executionContext && typeof executionContext === 'object'? executionContext@@ -31916,9 +31915,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtry {if (firefoxBidi.runs.has(tabId)) await firefoxBidi.perform(tabId, 'navigate', { url: rawUrl });- else await browser.tabs.update(tabId, { url: rawUrl });+ else {+ await beforePageAgentDispatch(browser, tabId, { kind: 'navigate', url: rawUrl });+ await browser.tabs.update(tabId, { url: rawUrl });+ }} catch (e) {removeNavigationListener();+ if (e?.code === 'page_feedback_pending') throw e;return {success: false,dispatched: false,diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex 9232949ab..13e6a1738 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -75,6 +75,7 @@ export const pageFeedbackMethods = {dispatched: details => {if (details.kind === 'navigate') {run.navigation = { at: Date.now(), url: details.url || '', kind: details.kind, history: details.history === true,+ frameId: Number(details.frameId) || 0,operationId: dispatchOwners.get(tabId)?.operationId || '' };}} });@@ -130,6 +131,7 @@ export const pageFeedbackMethods = {frame.seq = seq;if (feedback.source === 'agent') {if (feedback.operation === 'click') run.navigation = { at: Date.now(), url: '', kind: 'click',+ frameId: feedback.navigationTarget === '_top' ? 0 : frameId,operationId: dispatchOwners.get(tabId)?.operationId || '' };return { accepted: true };}@@ -168,7 +170,7 @@ export const pageFeedbackMethods = {const explicit = ['typed', 'auto_bookmark', 'generated', 'keyword', 'keyword_generated'].includes(details.transitionType)|| qualifiers.includes('from_address_bar');const navigation = run.navigation;- const agentNavigation = !explicit && navigation && Date.now() - navigation.at < 10000+ const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)&& (!navigation.url || navigation.url === details.url || qualifiers.some(q => /redirect$/.test(q)));if (agentNavigation) return;diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex aaffa0687..c5fc8beed 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -263,7 +263,12 @@const marker = { userAt: op.userAt };agentTurn = marker;if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {- send({ kind: 'activity', source: 'agent', operation: 'click' });+ const link = event.composedPath().find(node => node instanceof Element && node.matches('a[href],area[href]'));+ const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')+ || (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();+ send({ kind: 'activity', source: 'agent', operation: 'click',+ ...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)+ ? { navigationTarget: '_top' } : {}) });}// Native listeners have microtask checkpoints between callbacks. Keep this// exact input's attribution through its page handlers, until the next task.diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 86db7b08e..58c39a540 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -134,6 +134,78 @@ for (const build of ['chrome', 'firefox']) {} finally { agent._releaseRunEntry(tab); }});+ for (const tool of ['navigate', 'go_back', 'go_forward']) {+ test(`${build}: ${tool} does not hide navigation during the unsaved-changes probe`, async () => {+ const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();+ delete agent.executeTool;+ allowBatchPreparation(agent);+ const destination = 'https://example.com/destination';+ agent._probeUnsavedChanges = async () => {+ entered.resolve(); await release.promise;+ return { success: false, dispatched: false, noDispatch: true, error: 'Unsaved fields block navigation' };+ };+ await agent._claimRunEntry(tab, 'interactive');+ try {+ const action = agent.executeTool(tab, tool, { url: destination });+ await entered.promise;+ agent.observePageNavigation({ tabId: tab, frameId: 0, url: destination, transitionType: 'reload' }, 'history');+ const queuedDuringPreparation = agent._hasPendingPageFeedback(tab);+ release.resolve();+ assert.equal((await action).noDispatch, true);+ assert.equal(queuedDuringPreparation, true, 'Only a dispatched navigation may hide its own browser event');+ } finally { release.resolve(); agent._releaseRunEntry(tab); }+ });+ }++ test(`${build}: a child-frame click only correlates compatible frame navigation`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ await agent._claimRunEntry(tab, 'interactive');+ try {+ const child = bind(agent, tab, 2, 'child', 'child-token');+ child.send({ kind: 'activity', source: 'agent', operation: 'click' });+ for (const frameId of [0, 3, 2]) agent.observePageNavigation({ tabId: tab, frameId,+ url: `https://example.com/frame-${frameId}`, transitionType: 'link' }, 'history');+ const events = [...agent._pageFeedbackRuns.get(tab).events.values()];+ assert.deepEqual(events.map(event => event.frameId), [0, 3], 'Main and sibling changes must survive child-click correlation');+ agent._pageFeedbackRuns.get(tab).events.clear();+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top' });+ agent.observePageNavigation({ tabId: tab, frameId: 0, url: 'https://example.com/top', transitionType: 'link' }, 'history');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'An explicit _top click can correlate top-frame navigation');+ agent.observePageNavigation({ tabId: tab, frameId: 3, url: 'https://example.com/sibling', transitionType: 'link' }, 'history');+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'A declared top target must still preserve sibling changes');+ } finally { agent._releaseRunEntry(tab); }+ });++ if (build === 'firefox') for (const transport of ['content', 'bidi']) {+ test(`firefox: ${transport} navigation skips dispatch after feedback during an allowed probe`, async () => {+ const { firefoxBidi } = await import('../src/firefox/src/bidi/client.js');+ const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();+ delete agent.executeTool;+ allowBatchPreparation(agent);+ let dispatches = 0;+ const originalUpdate = api.tabs.update, originalRequest = firefoxBidi.request;+ api.tabs.update = async () => { dispatches++; return {}; };+ firefoxBidi.request = async () => { dispatches++; return { success: true }; };+ if (transport === 'bidi') firefoxBidi.runs.set(tab, { runId: 'native-navigation', bound: true });+ agent._probeUnsavedChanges = async () => { entered.resolve(); await release.promise; return null; };+ await agent._claimRunEntry(tab, 'interactive');+ try {+ const destination = 'https://example.com/destination';+ const action = agent.executeTool(tab, 'navigate', { url: destination });+ await entered.promise;+ agent.observePageNavigation({ tabId: tab, frameId: 0, url: destination, transitionType: 'reload' }, 'history');+ release.resolve();+ const result = await action;+ assert.equal(dispatches, 0);+ assert.equal(result.noDispatch, true);+ assert.equal(result.pageFeedbackPending, true);+ } finally {+ release.resolve(); agent._releaseRunEntry(tab);+ firefoxBidi.runs.delete(tab); firefoxBidi.request = originalRequest; api.tabs.update = originalUpdate;+ }+ });+ }+for (const kind of ['interactive', 'cloud', 'scheduled', 'workflow']) {test(`${build}: ${kind} feedback refreshes the page without creating a trusted text correction`, async () => {const agent = setup(Agent), tab = nextTab++, updates = [];diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 99c1ebb66..5808d67b7 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -250,6 +250,22 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {await child.locator('#human').click();const tokens = await Promise.all([page.evaluate(() => feedback[0]?.documentToken), child.evaluate(() => feedback[0]?.documentToken)]);assert.ok(tokens[0] && tokens[1] && tokens[0] !== tokens[1]);+ await child.evaluate(() => {+ const link = document.createElement('a'); link.id = 'top-link'; link.href = 'https://monitor.test/destination';+ link.textContent = 'Top link'; link.addEventListener('click', event => event.preventDefault()); document.body.append(link);+ });+ await page.waitForTimeout(200);+ for (const target of ['_top', '_parent']) {+ await child.evaluate(target => {+ document.getElementById('top-link').target = target;+ deliver('page_monitor_prepare', { operationId: `link-${target}`, tool: 'click', selector: '#top-link' });+ deliver('page_monitor_dispatch', { operationId: `link-${target}`, kind: 'click', selector: '#top-link' });+ feedback = [];+ }, target);+ await child.locator('#top-link').click();+ assert.ok((await child.evaluate(() => feedback)).some(event => event.source === 'agent' && event.navigationTarget === '_top'),+ `${target} navigation must carry the compatible top-frame target`);+ }await page.waitForTimeout(200);await page.evaluate(() => {feedback = [];