Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix redirect correlation and bounded page monitoring

Barack Sokullu22b4eb53805c
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index 9bc9d9190..a1e3d1750 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -232,9 +232,11 @@ export const pageFeedbackMethods = {
|| qualifiers.includes('from_address_bar');
const navigation = run.navigation;
const redirect = qualifiers.some(q => /redirect$/.test(q));
+ const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect
+ && !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);
const sameNavigation = navigation?.redirectChain
? redirect && !!navigation.documentId && navigation.documentId === details.documentId
- : (!!navigation?.url && navigation.url === details.url)
+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect
|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index d4a14139c..df58ae9bf 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -16,7 +16,8 @@
const operations = new Map();
const agentLayoutHistory = [];
const nativeTargets = new Set();
- const liveControls = new Set();
+ // Weak references keep disconnected fields from outliving the DOM until their batch is sampled.
+ const controlReferences = new Set();
let signatures = new WeakMap();
let controlSignatures = new WeakMap();
let textSignatures = new WeakMap();
@@ -24,7 +25,7 @@
let roots = new WeakSet();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;
- let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;
+ let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
let lastViewport = '';
let lastFeedbackDelivery = Promise.resolve();
@@ -352,8 +353,8 @@
controlValueFingerprint(el)]);
}
function trackControl(el) {
- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;
- liveControls.add(el);
+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || controlSignatures.has(el)) return;
+ controlReferences.add(new WeakRef(el));
controlSignatures.set(el, controlState(el));
}
function refreshControl(el) {
@@ -362,7 +363,8 @@
controlSignatures.set(el, controlState(el));
}
if (el.tagName === 'SELECT') {
- for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {
+ for (const option of el.options || []) {
+ trackControl(option);
controlSignatures.set(option, controlState(option));
signatures.set(option, signature(option));
}
@@ -407,16 +409,19 @@
}
function sampleFormControls(...targets) {
const extraTargets = targets.flat().filter(el => el instanceof Element);
- if (!active || (!liveControls.size && !extraTargets.length)) return;
- const controls = [...liveControls];
- if (!controls.length || controlCursor >= controls.length) controlCursor = 0;
- const batchSize = Math.min(200, controls.length);
- const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);
- if (controls.length) controlCursor = (controlCursor + batchSize) % controls.length;
- const sampled = new Set(batch);
+ if (!active || (!controlReferences.size && !extraTargets.length)) return;
+ const sampled = new Set();
const records = [];
- for (const el of batch) {
- if (!el.isConnected) { liveControls.delete(el); continue; }
+ let inspected = 0;
+ while (inspected < 200) {
+ controlIterator ||= controlReferences.values();
+ const next = controlIterator.next();
+ if (next.done) { controlIterator = null; break; }
+ inspected++;
+ const reference = next.value;
+ const el = reference.deref();
+ if (!el || !el.isConnected) { controlReferences.delete(reference); continue; }
+ sampled.add(el);
if (!ignored(el)) records.push({ type: 'control', target: el });
}
for (const el of extraTargets) {
@@ -847,7 +852,16 @@
}
listen(document, 'scroll', event => {
const el = event.target === document ? document.documentElement : event.target;
- if (ignored(el) || expected('scroll', el)) return;
+ if (ignored(el)) return;
+ const agentScroll = expected('scroll', el);
+ if (agentScroll) {
+ // Page scroll handlers can synchronously append lazy content. Carry the
+ // exact scroll's attribution through the MutationObserver checkpoint.
+ const marker = { userAt: agentScroll.userAt };
+ agentTurn = marker;
+ setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
+ return;
+ }
const source = Date.now() - lastUserAt < 1000 ? 'user' : 'unknown';
const viewport = { x: el === document.documentElement ? scrollX : el.scrollLeft,
y: el === document.documentElement ? scrollY : el.scrollTop };
@@ -872,7 +886,7 @@
}
function stop() {
active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;
- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;
+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null;
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
el.removeAttribute?.('data-webbrain-native-action');
@@ -885,7 +899,7 @@
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
agentLayoutHistory.length = 0;
- roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();
+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;
}
async function requestState() {
const generation = ++requestGeneration;
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index 9bc9d9190..a1e3d1750 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -232,9 +232,11 @@ export const pageFeedbackMethods = {
|| qualifiers.includes('from_address_bar');
const navigation = run.navigation;
const redirect = qualifiers.some(q => /redirect$/.test(q));
+ const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect
+ && !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);
const sameNavigation = navigation?.redirectChain
? redirect && !!navigation.documentId && navigation.documentId === details.documentId
- : (!!navigation?.url && navigation.url === details.url)
+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect
|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index d4a14139c..df58ae9bf 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -16,7 +16,8 @@
const operations = new Map();
const agentLayoutHistory = [];
const nativeTargets = new Set();
- const liveControls = new Set();
+ // Weak references keep disconnected fields from outliving the DOM until their batch is sampled.
+ const controlReferences = new Set();
let signatures = new WeakMap();
let controlSignatures = new WeakMap();
let textSignatures = new WeakMap();
@@ -24,7 +25,7 @@
let roots = new WeakSet();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;
- let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;
+ let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
let lastViewport = '';
let lastFeedbackDelivery = Promise.resolve();
@@ -352,8 +353,8 @@
controlValueFingerprint(el)]);
}
function trackControl(el) {
- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;
- liveControls.add(el);
+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || controlSignatures.has(el)) return;
+ controlReferences.add(new WeakRef(el));
controlSignatures.set(el, controlState(el));
}
function refreshControl(el) {
@@ -362,7 +363,8 @@
controlSignatures.set(el, controlState(el));
}
if (el.tagName === 'SELECT') {
- for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {
+ for (const option of el.options || []) {
+ trackControl(option);
controlSignatures.set(option, controlState(option));
signatures.set(option, signature(option));
}
@@ -407,16 +409,19 @@
}
function sampleFormControls(...targets) {
const extraTargets = targets.flat().filter(el => el instanceof Element);
- if (!active || (!liveControls.size && !extraTargets.length)) return;
- const controls = [...liveControls];
- if (!controls.length || controlCursor >= controls.length) controlCursor = 0;
- const batchSize = Math.min(200, controls.length);
- const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);
- if (controls.length) controlCursor = (controlCursor + batchSize) % controls.length;
- const sampled = new Set(batch);
+ if (!active || (!controlReferences.size && !extraTargets.length)) return;
+ const sampled = new Set();
const records = [];
- for (const el of batch) {
- if (!el.isConnected) { liveControls.delete(el); continue; }
+ let inspected = 0;
+ while (inspected < 200) {
+ controlIterator ||= controlReferences.values();
+ const next = controlIterator.next();
+ if (next.done) { controlIterator = null; break; }
+ inspected++;
+ const reference = next.value;
+ const el = reference.deref();
+ if (!el || !el.isConnected) { controlReferences.delete(reference); continue; }
+ sampled.add(el);
if (!ignored(el)) records.push({ type: 'control', target: el });
}
for (const el of extraTargets) {
@@ -847,7 +852,16 @@
}
listen(document, 'scroll', event => {
const el = event.target === document ? document.documentElement : event.target;
- if (ignored(el) || expected('scroll', el)) return;
+ if (ignored(el)) return;
+ const agentScroll = expected('scroll', el);
+ if (agentScroll) {
+ // Page scroll handlers can synchronously append lazy content. Carry the
+ // exact scroll's attribution through the MutationObserver checkpoint.
+ const marker = { userAt: agentScroll.userAt };
+ agentTurn = marker;
+ setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
+ return;
+ }
const source = Date.now() - lastUserAt < 1000 ? 'user' : 'unknown';
const viewport = { x: el === document.documentElement ? scrollX : el.scrollLeft,
y: el === document.documentElement ? scrollY : el.scrollTop };
@@ -872,7 +886,7 @@
}
function stop() {
active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;
- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;
+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null;
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
el.removeAttribute?.('data-webbrain-native-action');
@@ -885,7 +899,7 @@
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
agentLayoutHistory.length = 0;
- roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();
+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;
}
async function requestState() {
const generation = ++requestGeneration;
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 48ecbfbff..75434dcb8 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -425,9 +425,9 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(agent._hasPendingPageFeedback(tab), true, 'A same-frame navigation to a different URL remains visible');
agent._pageFeedbackRuns.get(tab).events.clear();
child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/expected' });
- agent.observePageNavigation({ tabId: tab, frameId: 2, url: 'https://example.com/unrelated-redirect', transitionType: 'link',
- transitionQualifiers: ['server_redirect'] }, 'history');
- assert.equal(agent._hasPendingPageFeedback(tab), true, 'Redirect qualifiers cannot turn an unrelated destination into an agent navigation');
+ agent.observePageNavigation({ tabId: tab, frameId: 2, documentId: 'child-redirect',
+ url: 'https://example.com/unrelated-redirect', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'The first same-frame redirect may commit at a URL different from the armed destination');
agent._pageFeedbackRuns.get(tab).events.clear();
child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top', navigationUrl: 'javascript:alert(1)' });
agent.observePageNavigation({ tabId: tab, frameId: 0, url: 'https://example.com/safe', transitionType: 'link' }, 'history');
@@ -740,8 +740,8 @@ for (const build of ['chrome', 'firefox']) {
binding = bind(agent, tab, 0, 'redirected-document', 'content-redirected');
binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/redirect-start' });
agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',
- url: 'https://example.com/redirect-start', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');
- assert.equal(agent._hasPendingPageFeedback(tab), false, 'An exact click destination can start a redirect chain');
+ url: 'https://example.com/first-redirect-target', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'A click destination may commit at its first redirect target');
assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirect-chain-document');
agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',
url: 'https://example.com/final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');
@@ -753,6 +753,23 @@ for (const build of ['chrome', 'firefox']) {
} finally { agent._releaseRunEntry(tab); }
});
+ test(`${build}: first redirect of an armed URL navigation stays agent-attributed`, async () => {
+ const agent = setup(Agent), tab = nextTab++;
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ await agent._beginPageFeedbackRun(tab, 'interactive');
+ bind(agent, tab);
+ await beforePageAgentDispatch(api, tab, { kind: 'navigate', url: 'https://example.com/redirect-start' });
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'first-redirect-document',
+ url: 'https://example.com/redirect-final', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'The first redirect may commit at a URL different from the armed destination');
+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'first-redirect-document');
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'first-redirect-document',
+ url: 'https://example.com/redirect-final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'Redirect continuation remains bound to the first committed document');
+ } finally { agent._finishPageFeedbackRun(tab); agent._releaseRunEntry(tab); }
+ });
+
test(`${build}: content feedback uses authenticated sender identity; composer delivery never gates monitoring`, async () => {
const agent = setup(Agent), tab = nextTab++;
await agent._claimRunEntry(tab, 'cloud', { cloudRun: true });
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index da1fa78da..d6e881465 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -18,11 +18,18 @@ body { margin: 0; } button,input { margin: 12px; }
<script>document.getElementById('agent').addEventListener('click', () => { document.getElementById('status').textContent = 'Agent changed this'; });</script>`;
async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false, omitEmptyFrameMonitor = false,
- capturePointerHandlers = false } = {}) {
+ capturePointerHandlers = false, instrumentWeakRefDeref = false } = {}) {
const browser = await engine.launch({ headless: true, ...(siteIsolation ? { args: ['--site-per-process'] } : {}) });
const context = await browser.newContext();
await context.route('https://monitor.test/**', route => route.fulfill({ contentType: 'text/html', body: html }));
await context.addInitScript(token => { window.monitorRunToken = token; }, runToken);
+ if (instrumentWeakRefDeref) await context.addInitScript(() => {
+ const NativeWeakRef = globalThis.WeakRef;
+ globalThis.__controlReferenceDerefs = 0;
+ globalThis.WeakRef = class CountedWeakRef extends NativeWeakRef {
+ deref() { globalThis.__controlReferenceDerefs++; return super.deref(); }
+ };
+ });
if (capturePointerHandlers) await context.addInitScript(() => {
window.monitorPointerHandlers = {};
const add = EventTarget.prototype.addEventListener;
@@ -149,12 +156,20 @@ test('Chrome full-page capture marks temporary scrolling without suppressing a l
session = await page.context().newCDPSession(page);
client.sendCommand = (_tab, method, params) => session.send(method, params);
const captureState = {};
- await page.evaluate(() => { feedback = []; });
+ await page.evaluate(() => {
+ feedback = [];
+ document.addEventListener('scroll', () => {
+ const loaded = document.createElement('button'); loaded.id = 'capture-lazy-content';
+ loaded.style.cssText = 'position:fixed;top:0;left:0'; loaded.textContent = 'Loaded'; document.body.append(loaded);
+ }, { once: true });
+ });
await client._scrollForFullPageCapture(tabId, 0, 600, captureState);
await client._finishFullPageCaptureScroll(tabId, captureState);
await page.waitForTimeout(250);
- assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'scroll'), false,
- 'The CDP scroll used by full-page capture must be attributed to the active agent run');
+ const captureFeedback = await page.evaluate(() => feedback);
+ assert.ok(await page.locator('#capture-lazy-content').count(), 'The trusted page scroll handler appended lazy content');
+ assert.equal(captureFeedback.length, 0,
+ 'The capture scroll and its synchronous lazy-load mutation must be attributed to the active agent run');
await page.evaluate(() => { feedback = []; window.scrollTo(0, 900); });
await page.waitForFunction(() => feedback.some(event => event.kind === 'scroll'));
@@ -2053,6 +2068,27 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: control polling dereferences only one bounded registry batch`, async () => {
+ const { browser, page } = await fixture(engine, build, { instrumentWeakRefDeref: true });
+ try {
+ await page.evaluate(() => {
+ const controls = document.createElement('div');
+ for (let index = 0; index < 900; index++) {
+ const input = document.createElement('input'); input.id = `bounded-control-${index}`; controls.append(input);
+ }
+ document.body.append(controls);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));
+ const derefs = await page.evaluate(() => {
+ __controlReferenceDerefs = 0;
+ deliver('page_monitor_prepare', { operationId: 'bounded-control-sample', tool: 'click', selector: '#agent' });
+ deliver('page_monitor_dispatch', { operationId: 'bounded-control-sample', kind: 'click', selector: '#agent', fenceOnly: true });
+ return __controlReferenceDerefs;
+ });
+ assert.ok(derefs > 0 && derefs <= 200, `One sample should dereference at most 200 controls, got ${derefs}`);
+ } finally { await browser.close(); }
+ });
+
for (const action of ['click', 'type']) {
test(`${build}: ${action} preparation preserves same-target human input`, async () => {
const { browser, page } = await fixture(engine, build);