Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

attribute capture scroll and correlate click navigation

Barack Sokullufdb72199c6b7
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index 1a65b3f22..9bc9d9190 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -51,7 +51,7 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {
}
owner.operationFrames.get(owner.operationId)?.add(Number(details.frameId) || 0);
const { prepareMonitor = false, ...dispatchDetails } = details;
- const operationId = owner.operationId || token();
+ const operationId = dispatchDetails.operationId || owner.operationId || token();
const monitorParams = { ...dispatchDetails, runToken: owner.runToken, operationId };
let acknowledgement;
let preparationAcknowledgement;
@@ -178,9 +178,19 @@ export const pageFeedbackMethods = {
|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };
frame.seq = seq;
if (feedback.source === 'agent') {
- if (feedback.operation === 'click') run.navigation = { at: Date.now(), url: '', kind: 'click',
- frameId: feedback.navigationTarget === '_top' ? 0 : frameId,
- operationId: dispatchOwners.get(tabId)?.operationId || '' };
+ if (feedback.operation === 'click') {
+ let url = '';
+ try {
+ const destination = new URL(String(feedback.navigationUrl || ''));
+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password
+ && destination.href.length <= 2000) {
+ url = destination.href;
+ }
+ } catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }
+ run.navigation = url ? { at: Date.now(), url, kind: 'click',
+ frameId: feedback.navigationTarget === '_top' ? 0 : frameId,
+ operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;
+ }
return { accepted: true };
}
const item = { kind: feedback.kind, source: feedback.source, frameId,
@@ -222,9 +232,10 @@ export const pageFeedbackMethods = {
|| qualifiers.includes('from_address_bar');
const navigation = run.navigation;
const redirect = qualifiers.some(q => /redirect$/.test(q));
- const sameNavigation = !navigation?.redirectChain
- ? (!navigation?.url || navigation.url === details.url || redirect)
- : redirect && !!navigation.documentId && navigation.documentId === details.documentId;
+ const sameNavigation = navigation?.redirectChain
+ ? redirect && !!navigation.documentId && navigation.documentId === details.documentId
+ : (!!navigation?.url && navigation.url === details.url)
+ || (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
&& sameNavigation;
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.js
index a75db5200..649ee3647 100644
--- a/src/chrome/src/cdp/cdp-client.js
+++ b/src/chrome/src/cdp/cdp-client.js
@@ -2346,6 +2346,32 @@ export class CDPClient {
* captureBounds:{x:number,y:number,width:number,height:number}
* }>} Capture bounds are CSS pixels in top-page coordinates.
*/
+ async _scrollForFullPageCapture(tabId, x, y, captureState) {
+ const guard = await beforePageAgentDispatch(globalThis.chrome, tabId, {
+ kind: 'scroll', scrollIntoView: true, fenceOnly: true,
+ ...(captureState.operationId ? { operationId: captureState.operationId } : {}),
+ });
+ if (guard?.operationId) captureState.operationId = guard.operationId;
+ if (guard?.runToken) captureState.runToken = guard.runToken;
+ const result = await this.evaluateFunction(tabId, `function (pageGuard, x, y) {
+ ${SELECTOR_SCROLL_DISPATCH_SOURCE}
+ if (!beforeSelectorScroll(document.documentElement, pageGuard)) return { scrolled: false, pageFeedbackPending: true };
+ window.scrollTo(x, y);
+ return { scrolled: true };
+ }`, [guard || null, x, y]);
+ const value = result?.result?.value;
+ throwIfPageFeedbackPending(value);
+ if (value?.scrolled !== true) throw new Error('Could not scroll the page for full-page capture');
+ }
+
+ async _finishFullPageCaptureScroll(tabId, captureState) {
+ if (!captureState.runToken || !captureState.operationId || !hasPageAgentDispatchOwner(tabId)) return;
+ try {
+ await globalThis.chrome?.tabs?.sendMessage(tabId, { target: 'content', action: 'page_monitor_finish',
+ params: { runToken: captureState.runToken, operationId: captureState.operationId } }, { frameId: 0 });
+ } catch { /* The page may have navigated or the run may have ended. */ }
+ }
+
async captureFullPageScreenshot(tabId, options = {}) {
await this.sendCommand(tabId, 'Page.enable');
const metrics = await this.sendCommand(tabId, 'Page.getLayoutMetrics');
@@ -2374,6 +2400,7 @@ export class CDPClient {
let contentGrowths = 0;
let captureBoundsFrozen = false;
let infiniteScrollWarningAdded = false;
+ const captureState = {};
const addInfiniteScrollWarning = () => {
if (infiniteScrollWarningAdded) return;
infiniteScrollWarningAdded = true;
@@ -2420,7 +2447,7 @@ export class CDPClient {
) {
const bottomY = contentY + Math.max(0, contentHeight - tileHeight);
const targetY = Math.min(contentY + discoveryOffsetY, bottomY);
- await this.evaluate(tabId, `window.scrollTo(${contentX}, ${targetY})`);
+ await this._scrollForFullPageCapture(tabId, contentX, targetY, captureState);
await new Promise(resolve => setTimeout(resolve, FULL_PAGE_SCROLL_SETTLE_MS));
const grew = updateContentBounds(await this.sendCommand(tabId, 'Page.getLayoutMetrics'));
const updatedBottomY = contentY + Math.max(0, contentHeight - tileHeight);
@@ -2446,7 +2473,7 @@ export class CDPClient {
}
const clipX = contentX + x;
const clipY = contentY + y;
- await this.evaluate(tabId, `window.scrollTo(${clipX}, ${clipY})`);
+ await this._scrollForFullPageCapture(tabId, clipX, clipY, captureState);
await new Promise(resolve => setTimeout(resolve, FULL_PAGE_SCROLL_SETTLE_MS));
// The page can still grow during the capture pass. Expand the loop
// bounds before sizing this tile so a newly moved footer is included,
@@ -2503,7 +2530,12 @@ export class CDPClient {
},
};
} finally {
- await this.evaluate(tabId, `window.scrollTo(${originalScrollX}, ${originalScrollY})`).catch(() => {});
+ try {
+ if (!captureState.runToken || hasPageAgentDispatchOwner(tabId)) {
+ await this._scrollForFullPageCapture(tabId, originalScrollX, originalScrollY, captureState);
+ }
+ } catch { /* Preserve the user's new position if the page changed during capture. */ }
+ await this._finishFullPageCaptureScroll(tabId, captureState);
}
}
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 7404455ad..1f14b73bd 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -742,10 +742,20 @@
const marker = { userAt: op.userAt };
agentTurn = marker;
if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {
- const link = event.composedPath().find(node => node instanceof Element && node.matches('a[href],area[href]'));
+ const path = event.composedPath();
+ const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));
+ const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);
const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')
|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();
+ let navigationUrl = '';
+ try {
+ const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');
+ const destination = new URL(rawUrl, document.baseURI);
+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password
+ && destination.href.length <= 2000) navigationUrl = destination.href;
+ } catch { /* Non-web and malformed targets are not navigation correlations. */ }
send({ kind: 'activity', source: 'agent', operation: 'click',
+ ...(navigationUrl ? { navigationUrl } : {}),
...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)
? { navigationTarget: '_top' } : {}) });
}
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index 1a65b3f22..9bc9d9190 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -51,7 +51,7 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {
}
owner.operationFrames.get(owner.operationId)?.add(Number(details.frameId) || 0);
const { prepareMonitor = false, ...dispatchDetails } = details;
- const operationId = owner.operationId || token();
+ const operationId = dispatchDetails.operationId || owner.operationId || token();
const monitorParams = { ...dispatchDetails, runToken: owner.runToken, operationId };
let acknowledgement;
let preparationAcknowledgement;
@@ -178,9 +178,19 @@ export const pageFeedbackMethods = {
|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };
frame.seq = seq;
if (feedback.source === 'agent') {
- if (feedback.operation === 'click') run.navigation = { at: Date.now(), url: '', kind: 'click',
- frameId: feedback.navigationTarget === '_top' ? 0 : frameId,
- operationId: dispatchOwners.get(tabId)?.operationId || '' };
+ if (feedback.operation === 'click') {
+ let url = '';
+ try {
+ const destination = new URL(String(feedback.navigationUrl || ''));
+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password
+ && destination.href.length <= 2000) {
+ url = destination.href;
+ }
+ } catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }
+ run.navigation = url ? { at: Date.now(), url, kind: 'click',
+ frameId: feedback.navigationTarget === '_top' ? 0 : frameId,
+ operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;
+ }
return { accepted: true };
}
const item = { kind: feedback.kind, source: feedback.source, frameId,
@@ -222,9 +232,10 @@ export const pageFeedbackMethods = {
|| qualifiers.includes('from_address_bar');
const navigation = run.navigation;
const redirect = qualifiers.some(q => /redirect$/.test(q));
- const sameNavigation = !navigation?.redirectChain
- ? (!navigation?.url || navigation.url === details.url || redirect)
- : redirect && !!navigation.documentId && navigation.documentId === details.documentId;
+ const sameNavigation = navigation?.redirectChain
+ ? redirect && !!navigation.documentId && navigation.documentId === details.documentId
+ : (!!navigation?.url && navigation.url === details.url)
+ || (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
&& sameNavigation;
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 7404455ad..1f14b73bd 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -742,10 +742,20 @@
const marker = { userAt: op.userAt };
agentTurn = marker;
if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {
- const link = event.composedPath().find(node => node instanceof Element && node.matches('a[href],area[href]'));
+ const path = event.composedPath();
+ const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));
+ const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);
const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')
|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();
+ let navigationUrl = '';
+ try {
+ const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');
+ const destination = new URL(rawUrl, document.baseURI);
+ if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password
+ && destination.href.length <= 2000) navigationUrl = destination.href;
+ } catch { /* Non-web and malformed targets are not navigation correlations. */ }
send({ kind: 'activity', source: 'agent', operation: 'click',
+ ...(navigationUrl ? { navigationUrl } : {}),
...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)
? { navigationTarget: '_top' } : {}) });
}
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 263fc8d19..48ecbfbff 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -411,13 +411,27 @@ for (const build of ['chrome', 'firefox']) {
for (const frameId of [0, 3, 2]) agent.observePageNavigation({ tabId: tab, frameId,
url: `https://example.com/frame-${frameId}`, transitionType: 'link' }, 'history');
const events = [...agent._pageFeedbackRuns.get(tab).events.values()];
- assert.deepEqual(events.map(event => event.frameId), [0, 3], 'Main and sibling changes must survive child-click correlation');
+ assert.deepEqual(events.map(event => event.frameId), [0, 3, 2], 'A click without a known destination cannot suppress any navigation');
agent._pageFeedbackRuns.get(tab).events.clear();
- child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top' });
+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top',
+ navigationUrl: 'https://example.com/top' });
agent.observePageNavigation({ tabId: tab, frameId: 0, url: 'https://example.com/top', transitionType: 'link' }, 'history');
- assert.equal(agent._hasPendingPageFeedback(tab), false, 'An explicit _top click can correlate top-frame navigation');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'An explicit _top click can correlate its exact top-frame destination');
agent.observePageNavigation({ tabId: tab, frameId: 3, url: 'https://example.com/sibling', transitionType: 'link' }, 'history');
assert.equal(agent._hasPendingPageFeedback(tab), true, 'A declared top target must still preserve sibling changes');
+ agent._pageFeedbackRuns.get(tab).events.clear();
+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/expected' });
+ agent.observePageNavigation({ tabId: tab, frameId: 2, url: 'https://example.com/unrelated', transitionType: 'link' }, 'history');
+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'A same-frame navigation to a different URL remains visible');
+ agent._pageFeedbackRuns.get(tab).events.clear();
+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/expected' });
+ agent.observePageNavigation({ tabId: tab, frameId: 2, url: 'https://example.com/unrelated-redirect', transitionType: 'link',
+ transitionQualifiers: ['server_redirect'] }, 'history');
+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'Redirect qualifiers cannot turn an unrelated destination into an agent navigation');
+ agent._pageFeedbackRuns.get(tab).events.clear();
+ child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top', navigationUrl: 'javascript:alert(1)' });
+ agent.observePageNavigation({ tabId: tab, frameId: 0, url: 'https://example.com/safe', transitionType: 'link' }, 'history');
+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'Non-web click targets cannot correlate browser navigation');
} finally { agent._releaseRunEntry(tab); }
});
@@ -704,7 +718,7 @@ for (const build of ['chrome', 'firefox']) {
await agent._claimRunEntry(tab, 'interactive');
try {
let binding = bind(agent, tab);
- binding.send({ kind: 'activity', source: 'agent', operation: 'click' });
+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/destination' });
agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'agent-document',
url: 'https://example.com/destination', transitionType: 'link' }, 'committed');
assert.equal(agent._hasPendingPageFeedback(tab), false, 'The click-caused commit is correlated');
@@ -719,9 +733,17 @@ for (const build of ['chrome', 'firefox']) {
binding.send({ kind: 'activity', source: 'agent', operation: 'click' });
agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',
url: 'https://example.com/final', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');
- assert.equal(agent._hasPendingPageFeedback(tab), false, 'An identified redirect is part of the click navigation');
- assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirected-document');
- agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirected-document',
+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'An uncorrelated redirect remains visible even after an agent click');
+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null);
+
+ agent._pageFeedbackRuns.get(tab).events.clear();
+ binding = bind(agent, tab, 0, 'redirected-document', 'content-redirected');
+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/redirect-start' });
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',
+ url: 'https://example.com/redirect-start', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'An exact click destination can start a redirect chain');
+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirect-chain-document');
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',
url: 'https://example.com/final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');
assert.equal(agent._hasPendingPageFeedback(tab), false, 'Redirect continuation stays on its exact document');
agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'later-navigation',
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 97e820ac0..e9e1544c5 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -123,6 +123,52 @@ test('Chrome AX preparation attributes actual scrolling without claiming field i
} finally { await browser.close(); }
});
+test('Chrome full-page capture marks temporary scrolling without suppressing a later page scroll', async () => {
+ const { browser, page } = await fixture(chromium, 'chrome');
+ const tabId = 88001;
+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;
+ const monitorApi = { tabs: {
+ get: async id => ({ id, url: 'https://monitor.test/start' }),
+ sendMessage: async (_tab, message) => {
+ if (message?.target !== 'content') return { ready: true };
+ await page.evaluate(({ action, params }) => deliver(action, params), { action: message.action, params: message.params || {} });
+ return page.evaluate(() => window.lastMonitorResponse || { ready: true });
+ },
+ }, webNavigation: { getAllFrames: async () => [{ frameId: 0 }] } };
+ globalThis.chrome = monitorApi;
+ globalThis.browser = monitorApi;
+ const agent = Object.assign({ _pageFeedbackRuns: new Map(), isRunning: () => true, _checkAbort: () => false,
+ _hasPendingPageFeedback(tab) { return (this._pageFeedbackRuns.get(tab)?.events.size || 0) > 0; } }, pageFeedbackMethods);
+ let session;
+ try {
+ await agent._beginPageFeedbackRun(tabId, 'interactive');
+ const run = agent._pageFeedbackRuns.get(tabId);
+ await page.evaluate(token => { window.monitorRunToken = token; deliver('page_monitor_state'); }, run.token);
+ await page.waitForTimeout(30);
+ const client = new CDPClient();
+ session = await page.context().newCDPSession(page);
+ client.sendCommand = (_tab, method, params) => session.send(method, params);
+ const captureState = {};
+ await page.evaluate(() => { feedback = []; });
+ await client._scrollForFullPageCapture(tabId, 0, 600, captureState);
+ await client._finishFullPageCaptureScroll(tabId, captureState);
+ await page.waitForTimeout(250);
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'scroll'), false,
+ 'The CDP scroll used by full-page capture must be attributed to the active agent run');
+
+ await page.evaluate(() => { feedback = []; window.scrollTo(0, 900); });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'scroll'));
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'scroll' && event.source !== 'agent'),
+ 'A later unmarked page scroll must remain visible to the monitor');
+ } finally {
+ if (session) await session.detach().catch(() => {});
+ if (agent._pageFeedbackRuns.has(tabId)) agent._finishPageFeedbackRun(tabId);
+ if (previousChrome === undefined) delete globalThis.chrome; else globalThis.chrome = previousChrome;
+ if (previousBrowser === undefined) delete globalThis.browser; else globalThis.browser = previousBrowser;
+ await browser.close();
+ }
+});
+
test('Chrome: unrelated layout shifts during an agent operation stay observable', async () => {
const { browser, page } = await fixture(chromium, 'chrome');
try {
@@ -1232,8 +1278,9 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
feedback = [];
}, { target, id });
await child.locator(`#${id}`).click();
- assert.ok((await child.evaluate(() => feedback)).some(event => event.source === 'agent' && event.navigationTarget === '_top'),
- `${target} navigation must carry the compatible top-frame target`);
+ assert.ok((await child.evaluate(() => feedback)).some(event => event.source === 'agent'
+ && event.navigationTarget === '_top' && event.navigationUrl === 'https://monitor.test/destination'),
+ `${target} navigation must carry its safe destination and compatible top-frame target`);
}
await page.waitForTimeout(200);
await page.evaluate(() => {