Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Bound child frame gestures and guard iframe clicks
Barack Sokulluc38dde55c702
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 0bac6d0e3..baef661f3 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -38187,11 +38187,6 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaitry {const el = document.querySelectorAll(sel)[matchIndex];if (!el) return { ok: false, url: location.href, reason: 'not-found-after-census' };- if (el.tagName !== 'SELECT') el.scrollIntoView({ block: 'center', inline: 'center' });- const rect = el.getBoundingClientRect();- const cx = rect.left + rect.width / 2;- const cy = rect.top + rect.height / 2;- const opts = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy, button: 0 };if (monitorGuard?.operationId) {if (typeof window.__wbPageMonitor?.activatePreparedDispatch !== 'function') {return { ok: false, url: location.href, reason: 'page-monitor-unavailable' };@@ -38200,6 +38195,18 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaioperationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,});}+ if (el.tagName !== 'SELECT') {+ const scroll = () => el.scrollIntoView({ block: 'center', inline: 'center' });+ if (monitorGuard?.operationId && window.__wbPageMonitor?.withPreparedDispatch)+ window.__wbPageMonitor.withPreparedDispatch(monitorGuard.operationId, scroll);+ else scroll();+ }+ if (!el.isConnected || document.querySelectorAll(sel)[matchIndex] !== el)+ return { ok: false, url: location.href, reason: 'target-changed-after-scroll' };+ const rect = el.getBoundingClientRect();+ const cx = rect.left + rect.width / 2;+ const cy = rect.top + rect.height / 2;+ const opts = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy, button: 0 };const dispatch = () => {targetDispatched = true;try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex 4a269972b..4fd466317 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -1,11 +1,36 @@/** Run-owned browser observations. Keep the Firefox copy byte-identical. */export const PAGE_FEEDBACK_IDLE_MS = 1000;const EVENT_LIMIT = 32;+const PAGE_GESTURE_LEASE_MS = 15000;const dispatchOwners = new Map();const clean = (value, limit = 240) => String(value ?? '').replace(/[\u0000-\u001f]/g, ' ').slice(0, limit);const token = () => globalThis.crypto.randomUUID();const apiFor = () => globalThis.browser || globalThis.chrome;+function clearGestureLease(run, frameId) {+ const lease = run.gestureLeases?.get(frameId);+ if (lease) clearTimeout(lease.timer);+ run.gestureLeases?.delete(frameId);+ run.gestures.delete(frameId);+}++function clearGestureLeases(run) {+ for (const frameId of run.gestureLeases?.keys() || []) clearGestureLease(run, frameId);+ run.gestures.clear();+}++function renewGestureLease(run, frameId, duration) {+ clearGestureLease(run, frameId);+ const lease = { timer: null };+ run.gestures.add(frameId);+ run.gestureLeases.set(frameId, lease);+ lease.timer = setTimeout(() => {+ if (run.gestureLeases.get(frameId) !== lease) return;+ run.gestureLeases.delete(frameId);+ run.gestures.delete(frameId);+ }, duration);+}+async function notifyPageMonitorFrames(api, tabId, message, knownFrames = [], onFrame = null) {if (!api?.tabs?.sendMessage) return;const frameIds = new Set([0, ...knownFrames]);@@ -124,7 +149,8 @@ export const pageFeedbackMethods = {async _beginPageFeedbackRun(tabId, kind) {this._pageFeedbackRuns ??= new Map();const run = { token: token(), kind, events: new Map(), frames: new Map(), documents: new Map(),- revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), navigation: null, url: '', onUpdate: null };+ revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), gestureLeases: new Map(),+ navigation: null, url: '', onUpdate: null };this._pageFeedbackRuns.set(tabId, run);dispatchOwners.set(tabId, { runToken: run.token, operationId: '', operationFrames: new Map(),pending: () => this._hasPendingPageFeedback(tabId),@@ -172,6 +198,7 @@ export const pageFeedbackMethods = {id: `${run.token}:${run.revision}`, kinds: ['navigation'], source: navigation.source,navigation: true, before: navigation.before, after: navigation.after,}); } catch { /* UI delivery cannot retain run ownership. */ }+ clearGestureLeases(run);this._pageFeedbackRuns.delete(tabId);if (dispatchOwners.get(tabId)?.runToken === run.token) dispatchOwners.delete(tabId);void notifyPageMonitorFrames(apiFor(), tabId, { target: 'content', action: 'page_monitor_state',@@ -188,7 +215,7 @@ export const pageFeedbackMethods = {if (document?.id && sender.documentId !== document.id) return { active: false };if (document && !document.id && document.url && sender.url?.split('#')[0] !== document.url.split('#')[0]) return { active: false };const previous = run.frames.get(frameId);- if (previous && previous.token !== documentToken) run.gestures.delete(frameId);+ if (previous && previous.token !== documentToken) clearGestureLease(run, frameId);if (!previous || previous.token !== documentToken) {run.frames.set(frameId, { token: documentToken, id: sender.documentId || '', seq: 0,name: clean(frameName, 256) });@@ -256,8 +283,9 @@ export const pageFeedbackMethods = {if (feedback.source === 'user') {// DOM callbacks describe consequences; only physical activity extends the idle gate.if (feedback.kind !== 'dom') run.lastUserAt = Date.now();- if (feedback.interacting === true) run.gestures.add(frameId);- else if (feedback.interacting === false) run.gestures.delete(frameId);+ if (feedback.interacting === true) renewGestureLease(run, frameId,+ Math.max(1, Number(this._pageFeedbackGestureLeaseMs) || PAGE_GESTURE_LEASE_MS));+ else if (feedback.interacting === false) clearGestureLease(run, frameId);// A physical interaction supersedes an expected agent navigation.run.navigation = null;}@@ -273,11 +301,11 @@ export const pageFeedbackMethods = {const frameId = Number(details.frameId) || 0;if (type === 'committed') {run.frames.delete(frameId);- run.gestures.delete(frameId);+ clearGestureLease(run, frameId);run.documents.set(frameId, { id: details.documentId || '', url: details.url || '',parentFrameId: Number.isSafeInteger(details.parentFrameId) ? details.parentFrameId : undefined });if (frameId === 0) {- run.frames.clear(); run.gestures.clear(); run.documents.clear();+ run.frames.clear(); clearGestureLeases(run); run.documents.clear();run.documents.set(0, { id: details.documentId || '', url: details.url || '', parentFrameId: -1 });}}diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex 4a269972b..4fd466317 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -1,11 +1,36 @@/** Run-owned browser observations. Keep the Firefox copy byte-identical. */export const PAGE_FEEDBACK_IDLE_MS = 1000;const EVENT_LIMIT = 32;+const PAGE_GESTURE_LEASE_MS = 15000;const dispatchOwners = new Map();const clean = (value, limit = 240) => String(value ?? '').replace(/[\u0000-\u001f]/g, ' ').slice(0, limit);const token = () => globalThis.crypto.randomUUID();const apiFor = () => globalThis.browser || globalThis.chrome;+function clearGestureLease(run, frameId) {+ const lease = run.gestureLeases?.get(frameId);+ if (lease) clearTimeout(lease.timer);+ run.gestureLeases?.delete(frameId);+ run.gestures.delete(frameId);+}++function clearGestureLeases(run) {+ for (const frameId of run.gestureLeases?.keys() || []) clearGestureLease(run, frameId);+ run.gestures.clear();+}++function renewGestureLease(run, frameId, duration) {+ clearGestureLease(run, frameId);+ const lease = { timer: null };+ run.gestures.add(frameId);+ run.gestureLeases.set(frameId, lease);+ lease.timer = setTimeout(() => {+ if (run.gestureLeases.get(frameId) !== lease) return;+ run.gestureLeases.delete(frameId);+ run.gestures.delete(frameId);+ }, duration);+}+async function notifyPageMonitorFrames(api, tabId, message, knownFrames = [], onFrame = null) {if (!api?.tabs?.sendMessage) return;const frameIds = new Set([0, ...knownFrames]);@@ -124,7 +149,8 @@ export const pageFeedbackMethods = {async _beginPageFeedbackRun(tabId, kind) {this._pageFeedbackRuns ??= new Map();const run = { token: token(), kind, events: new Map(), frames: new Map(), documents: new Map(),- revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), navigation: null, url: '', onUpdate: null };+ revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), gestureLeases: new Map(),+ navigation: null, url: '', onUpdate: null };this._pageFeedbackRuns.set(tabId, run);dispatchOwners.set(tabId, { runToken: run.token, operationId: '', operationFrames: new Map(),pending: () => this._hasPendingPageFeedback(tabId),@@ -172,6 +198,7 @@ export const pageFeedbackMethods = {id: `${run.token}:${run.revision}`, kinds: ['navigation'], source: navigation.source,navigation: true, before: navigation.before, after: navigation.after,}); } catch { /* UI delivery cannot retain run ownership. */ }+ clearGestureLeases(run);this._pageFeedbackRuns.delete(tabId);if (dispatchOwners.get(tabId)?.runToken === run.token) dispatchOwners.delete(tabId);void notifyPageMonitorFrames(apiFor(), tabId, { target: 'content', action: 'page_monitor_state',@@ -188,7 +215,7 @@ export const pageFeedbackMethods = {if (document?.id && sender.documentId !== document.id) return { active: false };if (document && !document.id && document.url && sender.url?.split('#')[0] !== document.url.split('#')[0]) return { active: false };const previous = run.frames.get(frameId);- if (previous && previous.token !== documentToken) run.gestures.delete(frameId);+ if (previous && previous.token !== documentToken) clearGestureLease(run, frameId);if (!previous || previous.token !== documentToken) {run.frames.set(frameId, { token: documentToken, id: sender.documentId || '', seq: 0,name: clean(frameName, 256) });@@ -256,8 +283,9 @@ export const pageFeedbackMethods = {if (feedback.source === 'user') {// DOM callbacks describe consequences; only physical activity extends the idle gate.if (feedback.kind !== 'dom') run.lastUserAt = Date.now();- if (feedback.interacting === true) run.gestures.add(frameId);- else if (feedback.interacting === false) run.gestures.delete(frameId);+ if (feedback.interacting === true) renewGestureLease(run, frameId,+ Math.max(1, Number(this._pageFeedbackGestureLeaseMs) || PAGE_GESTURE_LEASE_MS));+ else if (feedback.interacting === false) clearGestureLease(run, frameId);// A physical interaction supersedes an expected agent navigation.run.navigation = null;}@@ -273,11 +301,11 @@ export const pageFeedbackMethods = {const frameId = Number(details.frameId) || 0;if (type === 'committed') {run.frames.delete(frameId);- run.gestures.delete(frameId);+ clearGestureLease(run, frameId);run.documents.set(frameId, { id: details.documentId || '', url: details.url || '',parentFrameId: Number.isSafeInteger(details.parentFrameId) ? details.parentFrameId : undefined });if (frameId === 0) {- run.frames.clear(); run.gestures.clear(); run.documents.clear();+ run.frames.clear(); clearGestureLeases(run); run.documents.clear();run.documents.set(0, { id: details.documentId || '', url: details.url || '', parentFrameId: -1 });}}diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 374ec9b50..d2d638a43 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -224,6 +224,24 @@ for (const build of ['chrome', 'firefox']) {assert.equal(agent._pageFeedbackRuns.size, 0);});+ test(`${build}: detached-frame gestures release the idle gate after their lease expires`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ agent._pageFeedbackGestureLeaseMs = 20;+ await agent._claimRunEntry(tab, 'interactive');+ try {+ const child = bind(agent, tab, 2, 'detached-child', 'detached-token');+ assert.equal(child.send({ kind: 'activity', interacting: true }).accepted, true);+ const run = agent._pageFeedbackRuns.get(tab);+ assert.equal(run.gestures.has(2), true);+ // Removing a frame does not always produce a committed navigation or a+ // final interacting:false message from the destroyed document.+ run.frames.delete(2); run.documents.delete(2);+ await agent._waitForPageFeedbackIdle(tab);+ assert.equal(run.gestures.has(2), false);+ assert.equal(run.gestureLeases.has(2), false);+ } finally { agent._releaseRunEntry(tab); }+ });+test(`${build}: run startup awaits every accessible frame and cleanup stops all of them`, async () => {const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();const states = new Map([[0, false], [2, false]]);diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 61af6516f..d924d4224 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -895,6 +895,46 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ if (build === 'chrome') test('Chrome iframe fallback validates a stale target before scrolling it', async () => {+ const { browser, page } = await fixture(chromium, 'chrome');+ try {+ const source = read('chrome', 'agent/agent.js');+ const marker = 'func: (sel, matchIndex, monitorGuard) => {';+ const start = source.indexOf(marker);+ const endMarker = '\n },\n args: [selector, requestedMatchIndex, dispatchGuard],';+ const end = source.indexOf(endMarker, start);+ assert.ok(start >= 0 && end > start, 'Find the content-script-unavailable iframe click fallback');+ const declaration = source.slice(start + 'func: '.length, end + '\n }'.length);+ await page.evaluate(code => { window.__iframeFallback = eval(`(${code})`); }, declaration);+ await page.evaluate(() => {+ const old = document.createElement('button'); old.id = 'stale-iframe-target'; old.textContent = 'Old target';+ document.body.append(old);+ });+ await page.waitForTimeout(150);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'stale-iframe-click', tool: 'click', selector: '#stale-iframe-target' });+ deliver('page_monitor_dispatch', { operationId: 'stale-iframe-click', kind: 'click',+ selector: '#stale-iframe-target', fenceOnly: true });+ return lastMonitorResponse.guard;+ });+ assert.equal(guard.operationId, 'stale-iframe-click');+ const outcome = await page.evaluate(monitorGuard => {+ document.getElementById('stale-iframe-target').remove();+ const replacement = document.createElement('button');+ replacement.id = 'stale-iframe-target'; replacement.textContent = 'Replacement';+ window.scrollAttempts = 0;+ replacement.scrollIntoView = () => { window.scrollAttempts++; };+ document.body.append(replacement);+ const result = window.__iframeFallback('#stale-iframe-target', 0, monitorGuard);+ return { scrollAttempts: window.scrollAttempts, result };+ }, guard);+ assert.equal(outcome.scrollAttempts, 0, 'A replacement must be rejected before it can trigger page scrolling');+ assert.equal(outcome.result.dispatched, false);+ assert.match(outcome.result.error, /changed during action preparation/i);+ } finally { await browser.close(); }+ });+test(`${build}: large ancestor style changes stop descendant sampling at its cap`, async () => {const { browser, page } = await fixture(engine, build);try {