Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Bound child frame gestures and guard iframe clicks

Barack Sokulluc38dde55c702
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 0bac6d0e3..baef661f3 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -38187,11 +38187,6 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
try {
const el = document.querySelectorAll(sel)[matchIndex];
if (!el) return { ok: false, url: location.href, reason: 'not-found-after-census' };
- if (el.tagName !== 'SELECT') el.scrollIntoView({ block: 'center', inline: 'center' });
- const rect = el.getBoundingClientRect();
- const cx = rect.left + rect.width / 2;
- const cy = rect.top + rect.height / 2;
- const opts = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy, button: 0 };
if (monitorGuard?.operationId) {
if (typeof window.__wbPageMonitor?.activatePreparedDispatch !== 'function') {
return { ok: false, url: location.href, reason: 'page-monitor-unavailable' };
@@ -38200,6 +38195,18 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
operationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,
});
}
+ if (el.tagName !== 'SELECT') {
+ const scroll = () => el.scrollIntoView({ block: 'center', inline: 'center' });
+ if (monitorGuard?.operationId && window.__wbPageMonitor?.withPreparedDispatch)
+ window.__wbPageMonitor.withPreparedDispatch(monitorGuard.operationId, scroll);
+ else scroll();
+ }
+ if (!el.isConnected || document.querySelectorAll(sel)[matchIndex] !== el)
+ return { ok: false, url: location.href, reason: 'target-changed-after-scroll' };
+ const rect = el.getBoundingClientRect();
+ const cx = rect.left + rect.width / 2;
+ const cy = rect.top + rect.height / 2;
+ const opts = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy, button: 0 };
const dispatch = () => {
targetDispatched = true;
try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index 4a269972b..4fd466317 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -1,11 +1,36 @@
/** Run-owned browser observations. Keep the Firefox copy byte-identical. */
export const PAGE_FEEDBACK_IDLE_MS = 1000;
const EVENT_LIMIT = 32;
+const PAGE_GESTURE_LEASE_MS = 15000;
const dispatchOwners = new Map();
const clean = (value, limit = 240) => String(value ?? '').replace(/[\u0000-\u001f]/g, ' ').slice(0, limit);
const token = () => globalThis.crypto.randomUUID();
const apiFor = () => globalThis.browser || globalThis.chrome;
+function clearGestureLease(run, frameId) {
+ const lease = run.gestureLeases?.get(frameId);
+ if (lease) clearTimeout(lease.timer);
+ run.gestureLeases?.delete(frameId);
+ run.gestures.delete(frameId);
+}
+
+function clearGestureLeases(run) {
+ for (const frameId of run.gestureLeases?.keys() || []) clearGestureLease(run, frameId);
+ run.gestures.clear();
+}
+
+function renewGestureLease(run, frameId, duration) {
+ clearGestureLease(run, frameId);
+ const lease = { timer: null };
+ run.gestures.add(frameId);
+ run.gestureLeases.set(frameId, lease);
+ lease.timer = setTimeout(() => {
+ if (run.gestureLeases.get(frameId) !== lease) return;
+ run.gestureLeases.delete(frameId);
+ run.gestures.delete(frameId);
+ }, duration);
+}
+
async function notifyPageMonitorFrames(api, tabId, message, knownFrames = [], onFrame = null) {
if (!api?.tabs?.sendMessage) return;
const frameIds = new Set([0, ...knownFrames]);
@@ -124,7 +149,8 @@ export const pageFeedbackMethods = {
async _beginPageFeedbackRun(tabId, kind) {
this._pageFeedbackRuns ??= new Map();
const run = { token: token(), kind, events: new Map(), frames: new Map(), documents: new Map(),
- revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), navigation: null, url: '', onUpdate: null };
+ revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), gestureLeases: new Map(),
+ navigation: null, url: '', onUpdate: null };
this._pageFeedbackRuns.set(tabId, run);
dispatchOwners.set(tabId, { runToken: run.token, operationId: '', operationFrames: new Map(),
pending: () => this._hasPendingPageFeedback(tabId),
@@ -172,6 +198,7 @@ export const pageFeedbackMethods = {
id: `${run.token}:${run.revision}`, kinds: ['navigation'], source: navigation.source,
navigation: true, before: navigation.before, after: navigation.after,
}); } catch { /* UI delivery cannot retain run ownership. */ }
+ clearGestureLeases(run);
this._pageFeedbackRuns.delete(tabId);
if (dispatchOwners.get(tabId)?.runToken === run.token) dispatchOwners.delete(tabId);
void notifyPageMonitorFrames(apiFor(), tabId, { target: 'content', action: 'page_monitor_state',
@@ -188,7 +215,7 @@ export const pageFeedbackMethods = {
if (document?.id && sender.documentId !== document.id) return { active: false };
if (document && !document.id && document.url && sender.url?.split('#')[0] !== document.url.split('#')[0]) return { active: false };
const previous = run.frames.get(frameId);
- if (previous && previous.token !== documentToken) run.gestures.delete(frameId);
+ if (previous && previous.token !== documentToken) clearGestureLease(run, frameId);
if (!previous || previous.token !== documentToken) {
run.frames.set(frameId, { token: documentToken, id: sender.documentId || '', seq: 0,
name: clean(frameName, 256) });
@@ -256,8 +283,9 @@ export const pageFeedbackMethods = {
if (feedback.source === 'user') {
// DOM callbacks describe consequences; only physical activity extends the idle gate.
if (feedback.kind !== 'dom') run.lastUserAt = Date.now();
- if (feedback.interacting === true) run.gestures.add(frameId);
- else if (feedback.interacting === false) run.gestures.delete(frameId);
+ if (feedback.interacting === true) renewGestureLease(run, frameId,
+ Math.max(1, Number(this._pageFeedbackGestureLeaseMs) || PAGE_GESTURE_LEASE_MS));
+ else if (feedback.interacting === false) clearGestureLease(run, frameId);
// A physical interaction supersedes an expected agent navigation.
run.navigation = null;
}
@@ -273,11 +301,11 @@ export const pageFeedbackMethods = {
const frameId = Number(details.frameId) || 0;
if (type === 'committed') {
run.frames.delete(frameId);
- run.gestures.delete(frameId);
+ clearGestureLease(run, frameId);
run.documents.set(frameId, { id: details.documentId || '', url: details.url || '',
parentFrameId: Number.isSafeInteger(details.parentFrameId) ? details.parentFrameId : undefined });
if (frameId === 0) {
- run.frames.clear(); run.gestures.clear(); run.documents.clear();
+ run.frames.clear(); clearGestureLeases(run); run.documents.clear();
run.documents.set(0, { id: details.documentId || '', url: details.url || '', parentFrameId: -1 });
}
}
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index 4a269972b..4fd466317 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -1,11 +1,36 @@
/** Run-owned browser observations. Keep the Firefox copy byte-identical. */
export const PAGE_FEEDBACK_IDLE_MS = 1000;
const EVENT_LIMIT = 32;
+const PAGE_GESTURE_LEASE_MS = 15000;
const dispatchOwners = new Map();
const clean = (value, limit = 240) => String(value ?? '').replace(/[\u0000-\u001f]/g, ' ').slice(0, limit);
const token = () => globalThis.crypto.randomUUID();
const apiFor = () => globalThis.browser || globalThis.chrome;
+function clearGestureLease(run, frameId) {
+ const lease = run.gestureLeases?.get(frameId);
+ if (lease) clearTimeout(lease.timer);
+ run.gestureLeases?.delete(frameId);
+ run.gestures.delete(frameId);
+}
+
+function clearGestureLeases(run) {
+ for (const frameId of run.gestureLeases?.keys() || []) clearGestureLease(run, frameId);
+ run.gestures.clear();
+}
+
+function renewGestureLease(run, frameId, duration) {
+ clearGestureLease(run, frameId);
+ const lease = { timer: null };
+ run.gestures.add(frameId);
+ run.gestureLeases.set(frameId, lease);
+ lease.timer = setTimeout(() => {
+ if (run.gestureLeases.get(frameId) !== lease) return;
+ run.gestureLeases.delete(frameId);
+ run.gestures.delete(frameId);
+ }, duration);
+}
+
async function notifyPageMonitorFrames(api, tabId, message, knownFrames = [], onFrame = null) {
if (!api?.tabs?.sendMessage) return;
const frameIds = new Set([0, ...knownFrames]);
@@ -124,7 +149,8 @@ export const pageFeedbackMethods = {
async _beginPageFeedbackRun(tabId, kind) {
this._pageFeedbackRuns ??= new Map();
const run = { token: token(), kind, events: new Map(), frames: new Map(), documents: new Map(),
- revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), navigation: null, url: '', onUpdate: null };
+ revision: 0, lastUserAt: 0, lastActivityAt: 0, gestures: new Set(), gestureLeases: new Map(),
+ navigation: null, url: '', onUpdate: null };
this._pageFeedbackRuns.set(tabId, run);
dispatchOwners.set(tabId, { runToken: run.token, operationId: '', operationFrames: new Map(),
pending: () => this._hasPendingPageFeedback(tabId),
@@ -172,6 +198,7 @@ export const pageFeedbackMethods = {
id: `${run.token}:${run.revision}`, kinds: ['navigation'], source: navigation.source,
navigation: true, before: navigation.before, after: navigation.after,
}); } catch { /* UI delivery cannot retain run ownership. */ }
+ clearGestureLeases(run);
this._pageFeedbackRuns.delete(tabId);
if (dispatchOwners.get(tabId)?.runToken === run.token) dispatchOwners.delete(tabId);
void notifyPageMonitorFrames(apiFor(), tabId, { target: 'content', action: 'page_monitor_state',
@@ -188,7 +215,7 @@ export const pageFeedbackMethods = {
if (document?.id && sender.documentId !== document.id) return { active: false };
if (document && !document.id && document.url && sender.url?.split('#')[0] !== document.url.split('#')[0]) return { active: false };
const previous = run.frames.get(frameId);
- if (previous && previous.token !== documentToken) run.gestures.delete(frameId);
+ if (previous && previous.token !== documentToken) clearGestureLease(run, frameId);
if (!previous || previous.token !== documentToken) {
run.frames.set(frameId, { token: documentToken, id: sender.documentId || '', seq: 0,
name: clean(frameName, 256) });
@@ -256,8 +283,9 @@ export const pageFeedbackMethods = {
if (feedback.source === 'user') {
// DOM callbacks describe consequences; only physical activity extends the idle gate.
if (feedback.kind !== 'dom') run.lastUserAt = Date.now();
- if (feedback.interacting === true) run.gestures.add(frameId);
- else if (feedback.interacting === false) run.gestures.delete(frameId);
+ if (feedback.interacting === true) renewGestureLease(run, frameId,
+ Math.max(1, Number(this._pageFeedbackGestureLeaseMs) || PAGE_GESTURE_LEASE_MS));
+ else if (feedback.interacting === false) clearGestureLease(run, frameId);
// A physical interaction supersedes an expected agent navigation.
run.navigation = null;
}
@@ -273,11 +301,11 @@ export const pageFeedbackMethods = {
const frameId = Number(details.frameId) || 0;
if (type === 'committed') {
run.frames.delete(frameId);
- run.gestures.delete(frameId);
+ clearGestureLease(run, frameId);
run.documents.set(frameId, { id: details.documentId || '', url: details.url || '',
parentFrameId: Number.isSafeInteger(details.parentFrameId) ? details.parentFrameId : undefined });
if (frameId === 0) {
- run.frames.clear(); run.gestures.clear(); run.documents.clear();
+ run.frames.clear(); clearGestureLeases(run); run.documents.clear();
run.documents.set(0, { id: details.documentId || '', url: details.url || '', parentFrameId: -1 });
}
}
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 374ec9b50..d2d638a43 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -224,6 +224,24 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(agent._pageFeedbackRuns.size, 0);
});
+ test(`${build}: detached-frame gestures release the idle gate after their lease expires`, async () => {
+ const agent = setup(Agent), tab = nextTab++;
+ agent._pageFeedbackGestureLeaseMs = 20;
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ const child = bind(agent, tab, 2, 'detached-child', 'detached-token');
+ assert.equal(child.send({ kind: 'activity', interacting: true }).accepted, true);
+ const run = agent._pageFeedbackRuns.get(tab);
+ assert.equal(run.gestures.has(2), true);
+ // Removing a frame does not always produce a committed navigation or a
+ // final interacting:false message from the destroyed document.
+ run.frames.delete(2); run.documents.delete(2);
+ await agent._waitForPageFeedbackIdle(tab);
+ assert.equal(run.gestures.has(2), false);
+ assert.equal(run.gestureLeases.has(2), false);
+ } finally { agent._releaseRunEntry(tab); }
+ });
+
test(`${build}: run startup awaits every accessible frame and cleanup stops all of them`, async () => {
const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();
const states = new Map([[0, false], [2, false]]);
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 61af6516f..d924d4224 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -895,6 +895,46 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ if (build === 'chrome') test('Chrome iframe fallback validates a stale target before scrolling it', async () => {
+ const { browser, page } = await fixture(chromium, 'chrome');
+ try {
+ const source = read('chrome', 'agent/agent.js');
+ const marker = 'func: (sel, matchIndex, monitorGuard) => {';
+ const start = source.indexOf(marker);
+ const endMarker = '\n },\n args: [selector, requestedMatchIndex, dispatchGuard],';
+ const end = source.indexOf(endMarker, start);
+ assert.ok(start >= 0 && end > start, 'Find the content-script-unavailable iframe click fallback');
+ const declaration = source.slice(start + 'func: '.length, end + '\n }'.length);
+ await page.evaluate(code => { window.__iframeFallback = eval(`(${code})`); }, declaration);
+ await page.evaluate(() => {
+ const old = document.createElement('button'); old.id = 'stale-iframe-target'; old.textContent = 'Old target';
+ document.body.append(old);
+ });
+ await page.waitForTimeout(150);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'stale-iframe-click', tool: 'click', selector: '#stale-iframe-target' });
+ deliver('page_monitor_dispatch', { operationId: 'stale-iframe-click', kind: 'click',
+ selector: '#stale-iframe-target', fenceOnly: true });
+ return lastMonitorResponse.guard;
+ });
+ assert.equal(guard.operationId, 'stale-iframe-click');
+ const outcome = await page.evaluate(monitorGuard => {
+ document.getElementById('stale-iframe-target').remove();
+ const replacement = document.createElement('button');
+ replacement.id = 'stale-iframe-target'; replacement.textContent = 'Replacement';
+ window.scrollAttempts = 0;
+ replacement.scrollIntoView = () => { window.scrollAttempts++; };
+ document.body.append(replacement);
+ const result = window.__iframeFallback('#stale-iframe-target', 0, monitorGuard);
+ return { scrollAttempts: window.scrollAttempts, result };
+ }, guard);
+ assert.equal(outcome.scrollAttempts, 0, 'A replacement must be rejected before it can trigger page scrolling');
+ assert.equal(outcome.result.dispatched, false);
+ assert.match(outcome.result.error, /changed during action preparation/i);
+ } finally { await browser.close(); }
+ });
+
test(`${build}: large ancestor style changes stop descendant sampling at its cap`, async () => {
const { browser, page } = await fixture(engine, build);
try {