Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix redirect correlation and bounded page monitoring
Barack Sokullu22b4eb53805c
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex 9bc9d9190..a1e3d1750 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -232,9 +232,11 @@ export const pageFeedbackMethods = {|| qualifiers.includes('from_address_bar');const navigation = run.navigation;const redirect = qualifiers.some(q => /redirect$/.test(q));+ const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect+ && !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);const sameNavigation = navigation?.redirectChain? redirect && !!navigation.documentId && navigation.documentId === details.documentId- : (!!navigation?.url && navigation.url === details.url)+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex d4a14139c..df58ae9bf 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -16,7 +16,8 @@const operations = new Map();const agentLayoutHistory = [];const nativeTargets = new Set();- const liveControls = new Set();+ // Weak references keep disconnected fields from outliving the DOM until their batch is sampled.+ const controlReferences = new Set();let signatures = new WeakMap();let controlSignatures = new WeakMap();let textSignatures = new WeakMap();@@ -24,7 +25,7 @@let roots = new WeakSet();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;- let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;+ let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;let requestGeneration = 0, unreported = 0, pendingDOM = null;let lastViewport = '';let lastFeedbackDelivery = Promise.resolve();@@ -352,8 +353,8 @@controlValueFingerprint(el)]);}function trackControl(el) {- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;- liveControls.add(el);+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || controlSignatures.has(el)) return;+ controlReferences.add(new WeakRef(el));controlSignatures.set(el, controlState(el));}function refreshControl(el) {@@ -362,7 +363,8 @@controlSignatures.set(el, controlState(el));}if (el.tagName === 'SELECT') {- for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {+ for (const option of el.options || []) {+ trackControl(option);controlSignatures.set(option, controlState(option));signatures.set(option, signature(option));}@@ -407,16 +409,19 @@}function sampleFormControls(...targets) {const extraTargets = targets.flat().filter(el => el instanceof Element);- if (!active || (!liveControls.size && !extraTargets.length)) return;- const controls = [...liveControls];- if (!controls.length || controlCursor >= controls.length) controlCursor = 0;- const batchSize = Math.min(200, controls.length);- const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);- if (controls.length) controlCursor = (controlCursor + batchSize) % controls.length;- const sampled = new Set(batch);+ if (!active || (!controlReferences.size && !extraTargets.length)) return;+ const sampled = new Set();const records = [];- for (const el of batch) {- if (!el.isConnected) { liveControls.delete(el); continue; }+ let inspected = 0;+ while (inspected < 200) {+ controlIterator ||= controlReferences.values();+ const next = controlIterator.next();+ if (next.done) { controlIterator = null; break; }+ inspected++;+ const reference = next.value;+ const el = reference.deref();+ if (!el || !el.isConnected) { controlReferences.delete(reference); continue; }+ sampled.add(el);if (!ignored(el)) records.push({ type: 'control', target: el });}for (const el of extraTargets) {@@ -847,7 +852,16 @@}listen(document, 'scroll', event => {const el = event.target === document ? document.documentElement : event.target;- if (ignored(el) || expected('scroll', el)) return;+ if (ignored(el)) return;+ const agentScroll = expected('scroll', el);+ if (agentScroll) {+ // Page scroll handlers can synchronously append lazy content. Carry the+ // exact scroll's attribution through the MutationObserver checkpoint.+ const marker = { userAt: agentScroll.userAt };+ agentTurn = marker;+ setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);+ return;+ }const source = Date.now() - lastUserAt < 1000 ? 'user' : 'unknown';const viewport = { x: el === document.documentElement ? scrollX : el.scrollLeft,y: el === document.documentElement ? scrollY : el.scrollTop };@@ -872,7 +886,7 @@}function stop() {active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null;document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {el.removeAttribute?.('data-webbrain-native-action');@@ -885,7 +899,7 @@listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;agentLayoutHistory.length = 0;- roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;}async function requestState() {const generation = ++requestGeneration;diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex 9bc9d9190..a1e3d1750 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -232,9 +232,11 @@ export const pageFeedbackMethods = {|| qualifiers.includes('from_address_bar');const navigation = run.navigation;const redirect = qualifiers.some(q => /redirect$/.test(q));+ const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect+ && !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);const sameNavigation = navigation?.redirectChain? redirect && !!navigation.documentId && navigation.documentId === details.documentId- : (!!navigation?.url && navigation.url === details.url)+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex d4a14139c..df58ae9bf 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -16,7 +16,8 @@const operations = new Map();const agentLayoutHistory = [];const nativeTargets = new Set();- const liveControls = new Set();+ // Weak references keep disconnected fields from outliving the DOM until their batch is sampled.+ const controlReferences = new Set();let signatures = new WeakMap();let controlSignatures = new WeakMap();let textSignatures = new WeakMap();@@ -24,7 +25,7 @@let roots = new WeakSet();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;- let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;+ let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;let requestGeneration = 0, unreported = 0, pendingDOM = null;let lastViewport = '';let lastFeedbackDelivery = Promise.resolve();@@ -352,8 +353,8 @@controlValueFingerprint(el)]);}function trackControl(el) {- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;- liveControls.add(el);+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || controlSignatures.has(el)) return;+ controlReferences.add(new WeakRef(el));controlSignatures.set(el, controlState(el));}function refreshControl(el) {@@ -362,7 +363,8 @@controlSignatures.set(el, controlState(el));}if (el.tagName === 'SELECT') {- for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {+ for (const option of el.options || []) {+ trackControl(option);controlSignatures.set(option, controlState(option));signatures.set(option, signature(option));}@@ -407,16 +409,19 @@}function sampleFormControls(...targets) {const extraTargets = targets.flat().filter(el => el instanceof Element);- if (!active || (!liveControls.size && !extraTargets.length)) return;- const controls = [...liveControls];- if (!controls.length || controlCursor >= controls.length) controlCursor = 0;- const batchSize = Math.min(200, controls.length);- const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);- if (controls.length) controlCursor = (controlCursor + batchSize) % controls.length;- const sampled = new Set(batch);+ if (!active || (!controlReferences.size && !extraTargets.length)) return;+ const sampled = new Set();const records = [];- for (const el of batch) {- if (!el.isConnected) { liveControls.delete(el); continue; }+ let inspected = 0;+ while (inspected < 200) {+ controlIterator ||= controlReferences.values();+ const next = controlIterator.next();+ if (next.done) { controlIterator = null; break; }+ inspected++;+ const reference = next.value;+ const el = reference.deref();+ if (!el || !el.isConnected) { controlReferences.delete(reference); continue; }+ sampled.add(el);if (!ignored(el)) records.push({ type: 'control', target: el });}for (const el of extraTargets) {@@ -847,7 +852,16 @@}listen(document, 'scroll', event => {const el = event.target === document ? document.documentElement : event.target;- if (ignored(el) || expected('scroll', el)) return;+ if (ignored(el)) return;+ const agentScroll = expected('scroll', el);+ if (agentScroll) {+ // Page scroll handlers can synchronously append lazy content. Carry the+ // exact scroll's attribution through the MutationObserver checkpoint.+ const marker = { userAt: agentScroll.userAt };+ agentTurn = marker;+ setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);+ return;+ }const source = Date.now() - lastUserAt < 1000 ? 'user' : 'unknown';const viewport = { x: el === document.documentElement ? scrollX : el.scrollLeft,y: el === document.documentElement ? scrollY : el.scrollTop };@@ -872,7 +886,7 @@}function stop() {active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null;document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {el.removeAttribute?.('data-webbrain-native-action');@@ -885,7 +899,7 @@listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;agentLayoutHistory.length = 0;- roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;}async function requestState() {const generation = ++requestGeneration;diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 48ecbfbff..75434dcb8 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -425,9 +425,9 @@ for (const build of ['chrome', 'firefox']) {assert.equal(agent._hasPendingPageFeedback(tab), true, 'A same-frame navigation to a different URL remains visible');agent._pageFeedbackRuns.get(tab).events.clear();child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/expected' });- agent.observePageNavigation({ tabId: tab, frameId: 2, url: 'https://example.com/unrelated-redirect', transitionType: 'link',- transitionQualifiers: ['server_redirect'] }, 'history');- assert.equal(agent._hasPendingPageFeedback(tab), true, 'Redirect qualifiers cannot turn an unrelated destination into an agent navigation');+ agent.observePageNavigation({ tabId: tab, frameId: 2, documentId: 'child-redirect',+ url: 'https://example.com/unrelated-redirect', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'The first same-frame redirect may commit at a URL different from the armed destination');agent._pageFeedbackRuns.get(tab).events.clear();child.send({ kind: 'activity', source: 'agent', operation: 'click', navigationTarget: '_top', navigationUrl: 'javascript:alert(1)' });agent.observePageNavigation({ tabId: tab, frameId: 0, url: 'https://example.com/safe', transitionType: 'link' }, 'history');@@ -740,8 +740,8 @@ for (const build of ['chrome', 'firefox']) {binding = bind(agent, tab, 0, 'redirected-document', 'content-redirected');binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/redirect-start' });agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',- url: 'https://example.com/redirect-start', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');- assert.equal(agent._hasPendingPageFeedback(tab), false, 'An exact click destination can start a redirect chain');+ url: 'https://example.com/first-redirect-target', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'A click destination may commit at its first redirect target');assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'redirect-chain-document');agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'redirect-chain-document',url: 'https://example.com/final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');@@ -753,6 +753,23 @@ for (const build of ['chrome', 'firefox']) {} finally { agent._releaseRunEntry(tab); }});+ test(`${build}: first redirect of an armed URL navigation stays agent-attributed`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ await agent._claimRunEntry(tab, 'interactive');+ try {+ await agent._beginPageFeedbackRun(tab, 'interactive');+ bind(agent, tab);+ await beforePageAgentDispatch(api, tab, { kind: 'navigate', url: 'https://example.com/redirect-start' });+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'first-redirect-document',+ url: 'https://example.com/redirect-final', transitionType: 'link', transitionQualifiers: ['server_redirect'] }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'The first redirect may commit at a URL different from the armed destination');+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation.documentId, 'first-redirect-document');+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'first-redirect-document',+ url: 'https://example.com/redirect-final-two', transitionType: 'link', transitionQualifiers: ['client_redirect'] }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'Redirect continuation remains bound to the first committed document');+ } finally { agent._finishPageFeedbackRun(tab); agent._releaseRunEntry(tab); }+ });+test(`${build}: content feedback uses authenticated sender identity; composer delivery never gates monitoring`, async () => {const agent = setup(Agent), tab = nextTab++;await agent._claimRunEntry(tab, 'cloud', { cloudRun: true });diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex da1fa78da..d6e881465 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -18,11 +18,18 @@ body { margin: 0; } button,input { margin: 12px; }<script>document.getElementById('agent').addEventListener('click', () => { document.getElementById('status').textContent = 'Agent changed this'; });</script>`;async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false, omitEmptyFrameMonitor = false,- capturePointerHandlers = false } = {}) {+ capturePointerHandlers = false, instrumentWeakRefDeref = false } = {}) {const browser = await engine.launch({ headless: true, ...(siteIsolation ? { args: ['--site-per-process'] } : {}) });const context = await browser.newContext();await context.route('https://monitor.test/**', route => route.fulfill({ contentType: 'text/html', body: html }));await context.addInitScript(token => { window.monitorRunToken = token; }, runToken);+ if (instrumentWeakRefDeref) await context.addInitScript(() => {+ const NativeWeakRef = globalThis.WeakRef;+ globalThis.__controlReferenceDerefs = 0;+ globalThis.WeakRef = class CountedWeakRef extends NativeWeakRef {+ deref() { globalThis.__controlReferenceDerefs++; return super.deref(); }+ };+ });if (capturePointerHandlers) await context.addInitScript(() => {window.monitorPointerHandlers = {};const add = EventTarget.prototype.addEventListener;@@ -149,12 +156,20 @@ test('Chrome full-page capture marks temporary scrolling without suppressing a lsession = await page.context().newCDPSession(page);client.sendCommand = (_tab, method, params) => session.send(method, params);const captureState = {};- await page.evaluate(() => { feedback = []; });+ await page.evaluate(() => {+ feedback = [];+ document.addEventListener('scroll', () => {+ const loaded = document.createElement('button'); loaded.id = 'capture-lazy-content';+ loaded.style.cssText = 'position:fixed;top:0;left:0'; loaded.textContent = 'Loaded'; document.body.append(loaded);+ }, { once: true });+ });await client._scrollForFullPageCapture(tabId, 0, 600, captureState);await client._finishFullPageCaptureScroll(tabId, captureState);await page.waitForTimeout(250);- assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'scroll'), false,- 'The CDP scroll used by full-page capture must be attributed to the active agent run');+ const captureFeedback = await page.evaluate(() => feedback);+ assert.ok(await page.locator('#capture-lazy-content').count(), 'The trusted page scroll handler appended lazy content');+ assert.equal(captureFeedback.length, 0,+ 'The capture scroll and its synchronous lazy-load mutation must be attributed to the active agent run');await page.evaluate(() => { feedback = []; window.scrollTo(0, 900); });await page.waitForFunction(() => feedback.some(event => event.kind === 'scroll'));@@ -2053,6 +2068,27 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: control polling dereferences only one bounded registry batch`, async () => {+ const { browser, page } = await fixture(engine, build, { instrumentWeakRefDeref: true });+ try {+ await page.evaluate(() => {+ const controls = document.createElement('div');+ for (let index = 0; index < 900; index++) {+ const input = document.createElement('input'); input.id = `bounded-control-${index}`; controls.append(input);+ }+ document.body.append(controls);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));+ const derefs = await page.evaluate(() => {+ __controlReferenceDerefs = 0;+ deliver('page_monitor_prepare', { operationId: 'bounded-control-sample', tool: 'click', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'bounded-control-sample', kind: 'click', selector: '#agent', fenceOnly: true });+ return __controlReferenceDerefs;+ });+ assert.ok(derefs > 0 && derefs <= 200, `One sample should dereference at most 200 controls, got ${derefs}`);+ } finally { await browser.close(); }+ });+for (const action of ['click', 'type']) {test(`${build}: ${action} preparation preserves same-target human input`, async () => {const { browser, page } = await fixture(engine, build);