Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix page feedback for base and animated changes

Barack Sokulluf49f51442e5e
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.js
index 649ee3647..0e15ac716 100644
--- a/src/chrome/src/cdp/cdp-client.js
+++ b/src/chrome/src/cdp/cdp-client.js
@@ -446,8 +446,10 @@ export class CDPClient {
...(Number.isFinite(params.x) ? { x: params.x, y: params.y } : {}),
release,
...(method === 'Input.insertText' ? { eventTypes: ['beforeinput', 'input', 'change'] } : {}),
- ...(method === 'Input.dispatchMouseEvent' ? { eventTypes: params.type === 'mouseMoved' ? ['pointermove']
- : params.type === 'mouseWheel' ? ['wheel'] : ['pointerdown', 'click', 'input', 'change'] } : {}),
+ ...(method === 'Input.dispatchMouseEvent' ? { eventTypes: params.type === 'mouseMoved'
+ ? ['pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove']
+ : params.type === 'mousePressed' ? ['pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointerdown', 'mousedown', 'click', 'input', 'change']
+ : params.type === 'mouseWheel' ? ['wheel'] : ['pointerup', 'mouseup', 'click', 'input', 'change'] } : {}),
...target,
});
}
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index a0d43bd2f..4e255193a 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -23,6 +23,7 @@
let textSignatures = new WeakMap();
let popoverTurns = new WeakMap();
let roots = new WeakSet();
+ let animationAttribution = new WeakMap();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;
let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;
@@ -469,6 +470,8 @@
'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget',
'aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
+ for (const name of ['animationend', 'animationcancel', 'transitionend', 'transitioncancel'])
+ listen(root, name, checkSettledAnimation);
listen(root, 'beforetoggle', event => {
const el = elementFor(event);
if (ignored(el) || !el.hasAttribute('popover') || !('popover' in el)) return;
@@ -571,6 +574,39 @@
}
return stylesheets;
}
+ function baseNavigationMutation(record, el) {
+ if (record.type === 'attributes' && el?.tagName === 'BASE' && ['href', 'target'].includes(record.attributeName))
+ return record.oldValue !== el.getAttribute(record.attributeName);
+ return record.type === 'childList' && [...record.addedNodes, ...record.removedNodes]
+ .some(node => node.nodeType === 1 && node.tagName === 'BASE');
+ }
+ function runningAnimations(el) {
+ try {
+ const animations = el?.getAnimations?.();
+ return (Array.isArray(animations) ? animations : []).filter(animation => animation.playState === 'running');
+ } catch { return []; }
+ }
+ function deferAnimationCheck(record, el) {
+ const animations = runningAnimations(el);
+ if (!animations.length) return false;
+ const agentUserAt = Number.isFinite(record.agentUserAt) ? record.agentUserAt
+ : agentTurn && lastUserAt <= agentTurn.userAt ? agentTurn.userAt : undefined;
+ if (Number.isFinite(agentUserAt)) for (const animation of animations) {
+ const target = animation.effect?.target;
+ animationAttribution.set(target instanceof Element ? target : el, agentUserAt);
+ }
+ return true;
+ }
+ function checkSettledAnimation(event) {
+ const el = event.target;
+ if (!active || !(el instanceof Element) || ignored(el)) return;
+ const finiteAnimationRunning = runningAnimations(el).some(animation =>
+ animation.effect?.getComputedTiming?.().iterations !== Infinity);
+ if (finiteAnimationRunning) return;
+ const agentUserAt = animationAttribution.get(el);
+ animationAttribution.delete(el);
+ onMutations([{ type: 'animation-settled', target: el, ...(Number.isFinite(agentUserAt) ? { agentUserAt } : {}) }]);
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -591,7 +627,7 @@
for (const record of records) {
if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;
if (ignored(record.target)) continue;
- if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
+ if (deferAnimationCheck(record, record.target)) continue;
if (record.type === 'attributes' && record.attributeName === 'selected'
&& record.oldValue !== record.target.getAttribute('selected')) {
const select = record.target.closest?.('select');
@@ -615,6 +651,7 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
+ if (baseNavigationMutation(record, el)) { noteChange(document.documentElement, record.agentUserAt); continue; }
const changedStylesheets = stylesheetMutation(record);
if (changedStylesheets.length) {
if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);
@@ -628,7 +665,8 @@
// while observing independent page-script changes to editable text.
if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
- && el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
+ && deferAnimationCheck(record, el)) continue;
+ if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;
if (++measured > 300) {
// The remaining records are intentionally uninspected. Invalidate the
// prepared page conservatively so a later visible target cannot vanish
@@ -804,7 +842,8 @@
for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {
if (event.isTrusted) expected('click', elementFor(event), event, true);
});
- for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);
+ for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove', 'pointerdown', 'mousedown', 'pointerup', 'mouseup'])
+ listen(document, name, noteAgentTurn);
listen(document, 'pointerdown', event => {
const el = elementFor(event);
if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;
@@ -931,6 +970,7 @@
clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
+ animationAttribution = new WeakMap();
agentLayoutHistory.length = 0;
roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;
}
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index a0d43bd2f..4e255193a 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -23,6 +23,7 @@
let textSignatures = new WeakMap();
let popoverTurns = new WeakMap();
let roots = new WeakSet();
+ let animationAttribution = new WeakMap();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;
let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;
@@ -469,6 +470,8 @@
'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget',
'aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
+ for (const name of ['animationend', 'animationcancel', 'transitionend', 'transitioncancel'])
+ listen(root, name, checkSettledAnimation);
listen(root, 'beforetoggle', event => {
const el = elementFor(event);
if (ignored(el) || !el.hasAttribute('popover') || !('popover' in el)) return;
@@ -571,6 +574,39 @@
}
return stylesheets;
}
+ function baseNavigationMutation(record, el) {
+ if (record.type === 'attributes' && el?.tagName === 'BASE' && ['href', 'target'].includes(record.attributeName))
+ return record.oldValue !== el.getAttribute(record.attributeName);
+ return record.type === 'childList' && [...record.addedNodes, ...record.removedNodes]
+ .some(node => node.nodeType === 1 && node.tagName === 'BASE');
+ }
+ function runningAnimations(el) {
+ try {
+ const animations = el?.getAnimations?.();
+ return (Array.isArray(animations) ? animations : []).filter(animation => animation.playState === 'running');
+ } catch { return []; }
+ }
+ function deferAnimationCheck(record, el) {
+ const animations = runningAnimations(el);
+ if (!animations.length) return false;
+ const agentUserAt = Number.isFinite(record.agentUserAt) ? record.agentUserAt
+ : agentTurn && lastUserAt <= agentTurn.userAt ? agentTurn.userAt : undefined;
+ if (Number.isFinite(agentUserAt)) for (const animation of animations) {
+ const target = animation.effect?.target;
+ animationAttribution.set(target instanceof Element ? target : el, agentUserAt);
+ }
+ return true;
+ }
+ function checkSettledAnimation(event) {
+ const el = event.target;
+ if (!active || !(el instanceof Element) || ignored(el)) return;
+ const finiteAnimationRunning = runningAnimations(el).some(animation =>
+ animation.effect?.getComputedTiming?.().iterations !== Infinity);
+ if (finiteAnimationRunning) return;
+ const agentUserAt = animationAttribution.get(el);
+ animationAttribution.delete(el);
+ onMutations([{ type: 'animation-settled', target: el, ...(Number.isFinite(agentUserAt) ? { agentUserAt } : {}) }]);
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -591,7 +627,7 @@
for (const record of records) {
if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;
if (ignored(record.target)) continue;
- if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
+ if (deferAnimationCheck(record, record.target)) continue;
if (record.type === 'attributes' && record.attributeName === 'selected'
&& record.oldValue !== record.target.getAttribute('selected')) {
const select = record.target.closest?.('select');
@@ -615,6 +651,7 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
+ if (baseNavigationMutation(record, el)) { noteChange(document.documentElement, record.agentUserAt); continue; }
const changedStylesheets = stylesheetMutation(record);
if (changedStylesheets.length) {
if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);
@@ -628,7 +665,8 @@
// while observing independent page-script changes to editable text.
if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
- && el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
+ && deferAnimationCheck(record, el)) continue;
+ if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;
if (++measured > 300) {
// The remaining records are intentionally uninspected. Invalidate the
// prepared page conservatively so a later visible target cannot vanish
@@ -804,7 +842,8 @@
for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {
if (event.isTrusted) expected('click', elementFor(event), event, true);
});
- for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);
+ for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove', 'pointerdown', 'mousedown', 'pointerup', 'mouseup'])
+ listen(document, name, noteAgentTurn);
listen(document, 'pointerdown', event => {
const el = elementFor(event);
if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;
@@ -931,6 +970,7 @@
clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
+ animationAttribution = new WeakMap();
agentLayoutHistory.length = 0;
roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;
}
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 32345079f..f71102148 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -420,12 +420,20 @@ for (const siteIsolation of [false, true]) {
}
rootSession = await context.newCDPSession(page);
const client = new CDPClient(); client.sessions.set(tab, { attached: true });
+ await child.evaluate(() => document.getElementById('agent').addEventListener('pointerover', () => {
+ document.getElementById('status').textContent = 'Hover changed this';
+ }, { once: true }));
+ const hover = await child.locator('#agent').boundingBox();
+ await client.dispatchMouseEvent(tab, 'mouseMoved', hover.x + hover.width / 2, hover.y + hover.height / 2);
+ await child.waitForFunction(() => document.getElementById('status').textContent === 'Hover changed this');
+ await child.waitForTimeout(200);
+ assert.deepEqual(await child.evaluate(() => feedback), [], 'CDP pointer-entry handlers must remain agent-attributed');
const rect = await child.locator('#field').boundingBox();
await client.dispatchMouseEvent(tab, 'mousePressed', rect.x + 8, rect.y + 8);
await client.dispatchMouseEvent(tab, 'mouseReleased', rect.x + 8, rect.y + 8);
await client.sendCommand(tab, 'Input.insertText', { text: 'agent' });
assert.equal(await child.locator('#field').inputValue(), 'agent');
- assert.deepEqual(registrations, [2, 2, 2]);
+ assert.deepEqual(registrations, [2, 2, 2, 2]);
const container = await child.locator('#container').boundingBox();
await client.sendCommand(tab, 'Input.dispatchMouseEvent', { type: 'mouseWheel', x: container.x + 20, y: container.y + 20,
deltaX: 0, deltaY: 100 });
@@ -855,6 +863,96 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: base URL changes invalidate prepared relative navigation`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const base = document.createElement('base'); base.id = 'page-base'; base.href = '/before/';
+ const link = document.createElement('a'); link.id = 'relative-link'; link.href = 'next'; link.textContent = 'Continue';
+ document.head.append(base); document.body.append(link);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));
+ await page.waitForTimeout(180);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'relative-navigation', tool: 'click', selector: '#relative-link' });
+ deliver('page_monitor_dispatch', { operationId: 'relative-navigation', kind: 'click', selector: '#relative-link', fenceOnly: true });
+ return lastMonitorResponse.guard;
+ });
+ await page.locator('#page-base').evaluate(el => el.setAttribute('href', '/after/'));
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'html'), null, { timeout: 1000 });
+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,
+ })), guard);
+ assert.equal(accepted, false, 'Changing the base URL must stale relative link destinations');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: visibility signatures are rechecked after animations and transitions settle`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const style = document.createElement('style');
+ style.textContent = `@keyframes hide-target { to { opacity: 0; } }
+ #animated-target { opacity: 1; }
+ #animated-target.hide { animation: hide-target 120ms linear forwards; }
+ #transition-target { opacity: 1; transition: opacity 120ms linear; }
+ #transition-target.hide { opacity: 0; }
+ #mixed-transition { opacity: 1; transition: opacity 400ms linear; }
+ #mixed-container.hide #mixed-hidden { visibility: hidden; }
+ #mixed-container.hide #mixed-transition { opacity: 0; }`;
+ const animated = document.createElement('button'); animated.id = 'animated-target'; animated.textContent = 'Animated target';
+ const transitioned = document.createElement('button'); transitioned.id = 'transition-target'; transitioned.textContent = 'Transition target';
+ const mixed = document.createElement('div'); mixed.id = 'mixed-container';
+ mixed.innerHTML = '<button id="mixed-hidden">Hidden sibling</button><button id="mixed-transition">Animated sibling</button>';
+ const shadowHost = document.createElement('div'); shadowHost.id = 'shadow-host';
+ const shadow = shadowHost.attachShadow({ mode: 'open' });
+ shadow.innerHTML = `<style>@keyframes shadow-hide { to { opacity: 0; } } #shadow-target.hide { animation: shadow-hide 120ms linear forwards; }</style>
+ <button id="shadow-target">Shadow target</button>`;
+ document.head.append(style); document.body.append(animated, transitioned, mixed, shadowHost);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));
+ await page.waitForTimeout(180);
+ for (const id of ['animated-target', 'transition-target']) {
+ const guard = await page.evaluate(targetId => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: targetId, tool: 'click', selector: `#${targetId}` });
+ deliver('page_monitor_dispatch', { operationId: targetId, kind: 'click', selector: `#${targetId}`, fenceOnly: true });
+ return lastMonitorResponse.guard;
+ }, id);
+ await page.locator(`#${id}`).evaluate(el => el.classList.add('hide'));
+ await page.waitForFunction(targetId => feedback.some(event => event.kind === 'dom' && event.target === `button#${targetId}`), id,
+ { timeout: 1500 });
+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,
+ })), guard);
+ assert.equal(accepted, false, `${id} must invalidate the prepared click after its final hidden state`);
+ }
+ await page.evaluate(() => {
+ feedback = [];
+ document.getElementById('shadow-host').shadowRoot.getElementById('shadow-target').classList.add('hide');
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button#shadow-target'), null,
+ { timeout: 1500 });
+
+ const mixedGuard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'mixed-animation', tool: 'click', selector: '#mixed-hidden' });
+ deliver('page_monitor_dispatch', { operationId: 'mixed-animation', kind: 'click', selector: '#mixed-hidden', fenceOnly: true });
+ return lastMonitorResponse.guard;
+ });
+ await page.locator('#mixed-container').evaluate(el => el.classList.add('hide'));
+ await page.waitForFunction(revision => Number(document.documentElement.getAttribute('data-webbrain-page-revision').split(':').at(-1)) > revision,
+ mixedGuard.revision, { timeout: 500 });
+ const mixedAccepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,
+ })), mixedGuard);
+ assert.equal(mixedAccepted, false, 'A transition on one descendant must not defer checks for hidden siblings');
+ await page.waitForFunction(revision => Number(document.documentElement.getAttribute('data-webbrain-page-revision').split(':').at(-1)) > revision + 1,
+ mixedGuard.revision, { timeout: 1500 });
+ } finally { await browser.close(); }
+ });
+
test(`${build}: page focus theft fences a selectorless input while the identified agent target stays quiet`, async () => {
const { browser, page } = await fixture(engine, build);
try {