Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
attribute capture scroll and correlate click navigation
Barack Sokullufdb72199c6b7
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex 1a65b3f22..9bc9d9190 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -51,7 +51,7 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {}owner.operationFrames.get(owner.operationId)?.add(Number(details.frameId) || 0);const { prepareMonitor = false, ...dispatchDetails } = details;- const operationId = owner.operationId || token();+ const operationId = dispatchDetails.operationId || owner.operationId || token();const monitorParams = { ...dispatchDetails, runToken: owner.runToken, operationId };let acknowledgement;let preparationAcknowledgement;@@ -178,9 +178,19 @@ export const pageFeedbackMethods = {|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };frame.seq = seq;if (feedback.source === 'agent') {- if (feedback.operation === 'click') run.navigation = { at: Date.now(), url: '', kind: 'click',- frameId: feedback.navigationTarget === '_top' ? 0 : frameId,- operationId: dispatchOwners.get(tabId)?.operationId || '' };+ if (feedback.operation === 'click') {+ let url = '';+ try {+ const destination = new URL(String(feedback.navigationUrl || ''));+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password+ && destination.href.length <= 2000) {+ url = destination.href;+ }+ } catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }+ run.navigation = url ? { at: Date.now(), url, kind: 'click',+ frameId: feedback.navigationTarget === '_top' ? 0 : frameId,+ operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;+ }return { accepted: true };}const item = { kind: feedback.kind, source: feedback.source, frameId,@@ -222,9 +232,10 @@ export const pageFeedbackMethods = {|| qualifiers.includes('from_address_bar');const navigation = run.navigation;const redirect = qualifiers.some(q => /redirect$/.test(q));- const sameNavigation = !navigation?.redirectChain- ? (!navigation?.url || navigation.url === details.url || redirect)- : redirect && !!navigation.documentId && navigation.documentId === details.documentId;+ const sameNavigation = navigation?.redirectChain+ ? redirect && !!navigation.documentId && navigation.documentId === details.documentId+ : (!!navigation?.url && navigation.url === details.url)+ || (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)&& sameNavigation;diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex a75db5200..649ee3647 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -2346,6 +2346,32 @@ export class CDPClient {* captureBounds:{x:number,y:number,width:number,height:number}* }>} Capture bounds are CSS pixels in top-page coordinates.*/+ async _scrollForFullPageCapture(tabId, x, y, captureState) {+ const guard = await beforePageAgentDispatch(globalThis.chrome, tabId, {+ kind: 'scroll', scrollIntoView: true, fenceOnly: true,+ ...(captureState.operationId ? { operationId: captureState.operationId } : {}),+ });+ if (guard?.operationId) captureState.operationId = guard.operationId;+ if (guard?.runToken) captureState.runToken = guard.runToken;+ const result = await this.evaluateFunction(tabId, `function (pageGuard, x, y) {+ ${SELECTOR_SCROLL_DISPATCH_SOURCE}+ if (!beforeSelectorScroll(document.documentElement, pageGuard)) return { scrolled: false, pageFeedbackPending: true };+ window.scrollTo(x, y);+ return { scrolled: true };+ }`, [guard || null, x, y]);+ const value = result?.result?.value;+ throwIfPageFeedbackPending(value);+ if (value?.scrolled !== true) throw new Error('Could not scroll the page for full-page capture');+ }++ async _finishFullPageCaptureScroll(tabId, captureState) {+ if (!captureState.runToken || !captureState.operationId || !hasPageAgentDispatchOwner(tabId)) return;+ try {+ await globalThis.chrome?.tabs?.sendMessage(tabId, { target: 'content', action: 'page_monitor_finish',+ params: { runToken: captureState.runToken, operationId: captureState.operationId } }, { frameId: 0 });+ } catch { /* The page may have navigated or the run may have ended. */ }+ }+async captureFullPageScreenshot(tabId, options = {}) {await this.sendCommand(tabId, 'Page.enable');const metrics = await this.sendCommand(tabId, 'Page.getLayoutMetrics');@@ -2374,6 +2400,7 @@ export class CDPClient {let contentGrowths = 0;let captureBoundsFrozen = false;let infiniteScrollWarningAdded = false;+ const captureState = {};const addInfiniteScrollWarning = () => {if (infiniteScrollWarningAdded) return;infiniteScrollWarningAdded = true;@@ -2420,7 +2447,7 @@ export class CDPClient {) {const bottomY = contentY + Math.max(0, contentHeight - tileHeight);const targetY = Math.min(contentY + discoveryOffsetY, bottomY);- await this.evaluate(tabId, `window.scrollTo(${contentX}, ${targetY})`);+ await this._scrollForFullPageCapture(tabId, contentX, targetY, captureState);await new Promise(resolve => setTimeout(resolve, FULL_PAGE_SCROLL_SETTLE_MS));const grew = updateContentBounds(await this.sendCommand(tabId, 'Page.getLayoutMetrics'));const updatedBottomY = contentY + Math.max(0, contentHeight - tileHeight);@@ -2446,7 +2473,7 @@ export class CDPClient {}const clipX = contentX + x;const clipY = contentY + y;- await this.evaluate(tabId, `window.scrollTo(${clipX}, ${clipY})`);+ await this._scrollForFullPageCapture(tabId, clipX, clipY, captureState);await new Promise(resolve => setTimeout(resolve, FULL_PAGE_SCROLL_SETTLE_MS));// The page can still grow during the capture pass. Expand the loop// bounds before sizing this tile so a newly moved footer is included,@@ -2503,7 +2530,12 @@ export class CDPClient {},};} finally {- await this.evaluate(tabId, `window.scrollTo(${originalScrollX}, ${originalScrollY})`).catch(() => {});+ try {+ if (!captureState.runToken || hasPageAgentDispatchOwner(tabId)) {+ await this._scrollForFullPageCapture(tabId, originalScrollX, originalScrollY, captureState);+ }+ } catch { /* Preserve the user's new position if the page changed during capture. */ }+ await this._finishFullPageCaptureScroll(tabId, captureState);}}diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 7404455ad..1f14b73bd 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -742,10 +742,20 @@const marker = { userAt: op.userAt };agentTurn = marker;if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {- const link = event.composedPath().find(node => node instanceof Element && node.matches('a[href],area[href]'));+ const path = event.composedPath();+ const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));+ const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();+ let navigationUrl = '';+ try {+ const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');+ const destination = new URL(rawUrl, document.baseURI);+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password+ && destination.href.length <= 2000) navigationUrl = destination.href;+ } catch { /* Non-web and malformed targets are not navigation correlations. */ }send({ kind: 'activity', source: 'agent', operation: 'click',+ ...(navigationUrl ? { navigationUrl } : {}),...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)? { navigationTarget: '_top' } : {}) });}diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex 1a65b3f22..9bc9d9190 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -51,7 +51,7 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {}owner.operationFrames.get(owner.operationId)?.add(Number(details.frameId) || 0);const { prepareMonitor = false, ...dispatchDetails } = details;- const operationId = owner.operationId || token();+ const operationId = dispatchDetails.operationId || owner.operationId || token();const monitorParams = { ...dispatchDetails, runToken: owner.runToken, operationId };let acknowledgement;let preparationAcknowledgement;@@ -178,9 +178,19 @@ export const pageFeedbackMethods = {|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };frame.seq = seq;if (feedback.source === 'agent') {- if (feedback.operation === 'click') run.navigation = { at: Date.now(), url: '', kind: 'click',- frameId: feedback.navigationTarget === '_top' ? 0 : frameId,- operationId: dispatchOwners.get(tabId)?.operationId || '' };+ if (feedback.operation === 'click') {+ let url = '';+ try {+ const destination = new URL(String(feedback.navigationUrl || ''));+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password+ && destination.href.length <= 2000) {+ url = destination.href;+ }+ } catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }+ run.navigation = url ? { at: Date.now(), url, kind: 'click',+ frameId: feedback.navigationTarget === '_top' ? 0 : frameId,+ operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;+ }return { accepted: true };}const item = { kind: feedback.kind, source: feedback.source, frameId,@@ -222,9 +232,10 @@ export const pageFeedbackMethods = {|| qualifiers.includes('from_address_bar');const navigation = run.navigation;const redirect = qualifiers.some(q => /redirect$/.test(q));- const sameNavigation = !navigation?.redirectChain- ? (!navigation?.url || navigation.url === details.url || redirect)- : redirect && !!navigation.documentId && navigation.documentId === details.documentId;+ const sameNavigation = navigation?.redirectChain+ ? redirect && !!navigation.documentId && navigation.documentId === details.documentId+ : (!!navigation?.url && navigation.url === details.url)+ || (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)&& sameNavigation;diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 7404455ad..1f14b73bd 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -742,10 +742,20 @@const marker = { userAt: op.userAt };agentTurn = marker;if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {- const link = event.composedPath().find(node => node instanceof Element && node.matches('a[href],area[href]'));+ const path = event.composedPath();+ const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));+ const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();+ let navigationUrl = '';+ try {+ const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');+ const destination = new URL(rawUrl, document.baseURI);+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password+ && destination.href.length <= 2000) navigationUrl = destination.href;+ } catch { /* Non-web and malformed targets are not navigation correlations. */ }send({ kind: 'activity', source: 'agent', operation: 'click',+ ...(navigationUrl ? { navigationUrl } : {}),...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)? { navigationTarget: '_top' } : {}) });}diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 263fc8d19..48ecbfbff 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -411,13 +411,27 @@ for (const build of ['chrome', 'firefox']) {for (const frameId of [0, 3, 2]) agent.observePageNavigation({ tabId: tab, frameId,url: `https://example.com/frame-${frameId}`, transitionType: 'link' }, 'history');const events = [...agent._pageFeedbackRuns.get(tab).events.values()];- assert.deepEqual(events.map(event => event.frameId), [0, 3], 'Main and sibling changes must survive child-click correlation');+ assert.deepEqual(events.map(event => event.frameId), [0, 3, 2], 'A click without a known destination cannot suppress any navigation');agent._pageFeedbackRuns.get(tab).events.clear();- child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top' });+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top',+ navigationUrl: 'https://example.com/top' });agent.observePageNavigation({ tabId: tab, frameId: 0, url: 'https://example.com/top', transitionType: 'link' }, 'history');- assert.equal(agent._hasPendingPageFeedback(tab), false, 'An explicit _top click can correlate top-frame navigation');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'An explicit _top click can correlate its exact top-frame destination');agent.observePageNavigation({ tabId: tab, frameId: 3, url: 'https://example.com/sibling', transitionType: 'link' }, 'history');assert.equal(agent._hasPendingPageFeedback(tab), true, 'A declared top target must still preserve sibling changes');+ agent._pageFeedbackRuns.get(tab).events.clear();+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/expected' });+ agent.observePageNavigation({ tabId: tab, frameId: 2, url: 'https://example.com/unrelated', transitionType: 'link' }, 'history');+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'A same-frame navigation to a different URL remains visible');+ agent._pageFeedbackRuns.get(tab).events.clear();+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/expected' });+ agent.observePageNavigation({ tabId: tab, frameId: 2, url: 'https://example.com/unrelated-redirect', transitionType: 'link',+ transitionQualifiers: ['server_redirect'] }, 'history');+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'Redirect qualifiers cannot turn an unrelated destination into an agent navigation');+ agent._pageFeedbackRuns.get(tab).events.clear();+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top', navigationUrl: 'javascript:alert(1)' });+ agent.observePageNavigation({ tabId: tab, frameId: 0, url: 'https://example.com/safe', transitionType: 'link' }, 'history');+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'Non-web click targets cannot correlate browser navigation');} finally { agent._releaseRunEntry(tab); }});@@ -704,7 +718,7 @@ for (const build of ['chrome', 'firefox']) {await agent._claimRunEntry(tab, 'interactive');try {let binding = bind(agent, tab);- binding.send({ kind: 'activity', source: 'agent', operation: 'click' });+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/destination' });agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'agent-document',url: 'https://example.com/destination', transitionType: 'link' }, 'committed');assert.equal(agent._hasPendingPageFeedback(tab), false, 'The click-caused commit is correlated');@@ -719,9 +733,17 @@ for (const build of ['chrome', 'firefox']) {binding.send({ kind: 'activity', source: 'agent', operation: 'click' });agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',url: 'https://example.com/final', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');- assert.equal(agent._hasPendingPageFeedback(tab), false, 'An identified redirect is part of the click navigation');- assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirected-document');- agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'An uncorrelated redirect remains visible even after an agent click');+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null);++ agent._pageFeedbackRuns.get(tab).events.clear();+ binding = bind(agent, tab, 0, 'redirected-document', 'content-redirected');+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/redirect-start' });+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',+ url: 'https://example.com/redirect-start', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'An exact click destination can start a redirect chain');+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirect-chain-document');+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',url: 'https://example.com/final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');assert.equal(agent._hasPendingPageFeedback(tab), false, 'Redirect continuation stays on its exact document');agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'later-navigation',diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 97e820ac0..e9e1544c5 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -123,6 +123,52 @@ test('Chrome AX preparation attributes actual scrolling without claiming field i} finally { await browser.close(); }});+test('Chrome full-page capture marks temporary scrolling without suppressing a later page scroll', async () => {+ const { browser, page } = await fixture(chromium, 'chrome');+ const tabId = 88001;+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;+ const monitorApi = { tabs: {+ get: async id => ({ id, url: 'https://monitor.test/start' }),+ sendMessage: async (_tab, message) => {+ if (message?.target !== 'content') return { ready: true };+ await page.evaluate(({ action, params }) => deliver(action, params), { action: message.action, params: message.params || {} });+ return page.evaluate(() => window.lastMonitorResponse || { ready: true });+ },+ }, webNavigation: { getAllFrames: async () => [{ frameId: 0 }] } };+ globalThis.chrome = monitorApi;+ globalThis.browser = monitorApi;+ const agent = Object.assign({ _pageFeedbackRuns: new Map(), isRunning: () => true, _checkAbort: () => false,+ _hasPendingPageFeedback(tab) { return (this._pageFeedbackRuns.get(tab)?.events.size || 0) > 0; } }, pageFeedbackMethods);+ let session;+ try {+ await agent._beginPageFeedbackRun(tabId, 'interactive');+ const run = agent._pageFeedbackRuns.get(tabId);+ await page.evaluate(token => { window.monitorRunToken = token; deliver('page_monitor_state'); }, run.token);+ await page.waitForTimeout(30);+ const client = new CDPClient();+ session = await page.context().newCDPSession(page);+ client.sendCommand = (_tab, method, params) => session.send(method, params);+ const captureState = {};+ await page.evaluate(() => { feedback = []; });+ await client._scrollForFullPageCapture(tabId, 0, 600, captureState);+ await client._finishFullPageCaptureScroll(tabId, captureState);+ await page.waitForTimeout(250);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'scroll'), false,+ 'The CDP scroll used by full-page capture must be attributed to the active agent run');++ await page.evaluate(() => { feedback = []; window.scrollTo(0, 900); });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'scroll'));+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'scroll' && event.source !== 'agent'),+ 'A later unmarked page scroll must remain visible to the monitor');+ } finally {+ if (session) await session.detach().catch(() => {});+ if (agent._pageFeedbackRuns.has(tabId)) agent._finishPageFeedbackRun(tabId);+ if (previousChrome === undefined) delete globalThis.chrome; else globalThis.chrome = previousChrome;+ if (previousBrowser === undefined) delete globalThis.browser; else globalThis.browser = previousBrowser;+ await browser.close();+ }+});+test('Chrome: unrelated layout shifts during an agent operation stay observable', async () => {const { browser, page } = await fixture(chromium, 'chrome');try {@@ -1232,8 +1278,9 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {feedback = [];}, { target, id });await child.locator(`#${id}`).click();- assert.ok((await child.evaluate(() => feedback)).some(event => event.source === 'agent' && event.navigationTarget === '_top'),- `${target} navigation must carry the compatible top-frame target`);+ assert.ok((await child.evaluate(() => feedback)).some(event => event.source === 'agent'+ && event.navigationTarget === '_top' && event.navigationUrl === 'https://monitor.test/destination'),+ `${target} navigation must carry its safe destination and compatible top-frame target`);}await page.waitForTimeout(200);await page.evaluate(() => {