Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix page feedback for base and animated changes
Barack Sokulluf49f51442e5e
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex 649ee3647..0e15ac716 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -446,8 +446,10 @@ export class CDPClient {...(Number.isFinite(params.x) ? { x: params.x, y: params.y } : {}),release,...(method === 'Input.insertText' ? { eventTypes: ['beforeinput', 'input', 'change'] } : {}),- ...(method === 'Input.dispatchMouseEvent' ? { eventTypes: params.type === 'mouseMoved' ? ['pointermove']- : params.type === 'mouseWheel' ? ['wheel'] : ['pointerdown', 'click', 'input', 'change'] } : {}),+ ...(method === 'Input.dispatchMouseEvent' ? { eventTypes: params.type === 'mouseMoved'+ ? ['pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove']+ : params.type === 'mousePressed' ? ['pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointerdown', 'mousedown', 'click', 'input', 'change']+ : params.type === 'mouseWheel' ? ['wheel'] : ['pointerup', 'mouseup', 'click', 'input', 'change'] } : {}),...target,});}diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex a0d43bd2f..4e255193a 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -23,6 +23,7 @@let textSignatures = new WeakMap();let popoverTurns = new WeakMap();let roots = new WeakSet();+ let animationAttribution = new WeakMap();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;@@ -469,6 +470,8 @@'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget','aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });+ for (const name of ['animationend', 'animationcancel', 'transitionend', 'transitioncancel'])+ listen(root, name, checkSettledAnimation);listen(root, 'beforetoggle', event => {const el = elementFor(event);if (ignored(el) || !el.hasAttribute('popover') || !('popover' in el)) return;@@ -571,6 +574,39 @@}return stylesheets;}+ function baseNavigationMutation(record, el) {+ if (record.type === 'attributes' && el?.tagName === 'BASE' && ['href', 'target'].includes(record.attributeName))+ return record.oldValue !== el.getAttribute(record.attributeName);+ return record.type === 'childList' && [...record.addedNodes, ...record.removedNodes]+ .some(node => node.nodeType === 1 && node.tagName === 'BASE');+ }+ function runningAnimations(el) {+ try {+ const animations = el?.getAnimations?.();+ return (Array.isArray(animations) ? animations : []).filter(animation => animation.playState === 'running');+ } catch { return []; }+ }+ function deferAnimationCheck(record, el) {+ const animations = runningAnimations(el);+ if (!animations.length) return false;+ const agentUserAt = Number.isFinite(record.agentUserAt) ? record.agentUserAt+ : agentTurn && lastUserAt <= agentTurn.userAt ? agentTurn.userAt : undefined;+ if (Number.isFinite(agentUserAt)) for (const animation of animations) {+ const target = animation.effect?.target;+ animationAttribution.set(target instanceof Element ? target : el, agentUserAt);+ }+ return true;+ }+ function checkSettledAnimation(event) {+ const el = event.target;+ if (!active || !(el instanceof Element) || ignored(el)) return;+ const finiteAnimationRunning = runningAnimations(el).some(animation =>+ animation.effect?.getComputedTiming?.().iterations !== Infinity);+ if (finiteAnimationRunning) return;+ const agentUserAt = animationAttribution.get(el);+ animationAttribution.delete(el);+ onMutations([{ type: 'animation-settled', target: el, ...(Number.isFinite(agentUserAt) ? { agentUserAt } : {}) }]);+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -591,7 +627,7 @@for (const record of records) {if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;if (ignored(record.target)) continue;- if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ if (deferAnimationCheck(record, record.target)) continue;if (record.type === 'attributes' && record.attributeName === 'selected'&& record.oldValue !== record.target.getAttribute('selected')) {const select = record.target.closest?.('select');@@ -615,6 +651,7 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;+ if (baseNavigationMutation(record, el)) { noteChange(document.documentElement, record.agentUserAt); continue; }const changedStylesheets = stylesheetMutation(record);if (changedStylesheets.length) {if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);@@ -628,7 +665,8 @@// while observing independent page-script changes to editable text.if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)- && el.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ && deferAnimationCheck(record, el)) continue;+ if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;if (++measured > 300) {// The remaining records are intentionally uninspected. Invalidate the// prepared page conservatively so a later visible target cannot vanish@@ -804,7 +842,8 @@for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {if (event.isTrusted) expected('click', elementFor(event), event, true);});- for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);+ for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove', 'pointerdown', 'mousedown', 'pointerup', 'mouseup'])+ listen(document, name, noteAgentTurn);listen(document, 'pointerdown', event => {const el = elementFor(event);if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;@@ -931,6 +970,7 @@clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;+ animationAttribution = new WeakMap();agentLayoutHistory.length = 0;roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;}diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex a0d43bd2f..4e255193a 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -23,6 +23,7 @@let textSignatures = new WeakMap();let popoverTurns = new WeakMap();let roots = new WeakSet();+ let animationAttribution = new WeakMap();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;let pointerHeld = false, composing = false, localOperation = null, controlIterator = null;@@ -469,6 +470,8 @@'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget','aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });+ for (const name of ['animationend', 'animationcancel', 'transitionend', 'transitioncancel'])+ listen(root, name, checkSettledAnimation);listen(root, 'beforetoggle', event => {const el = elementFor(event);if (ignored(el) || !el.hasAttribute('popover') || !('popover' in el)) return;@@ -571,6 +574,39 @@}return stylesheets;}+ function baseNavigationMutation(record, el) {+ if (record.type === 'attributes' && el?.tagName === 'BASE' && ['href', 'target'].includes(record.attributeName))+ return record.oldValue !== el.getAttribute(record.attributeName);+ return record.type === 'childList' && [...record.addedNodes, ...record.removedNodes]+ .some(node => node.nodeType === 1 && node.tagName === 'BASE');+ }+ function runningAnimations(el) {+ try {+ const animations = el?.getAnimations?.();+ return (Array.isArray(animations) ? animations : []).filter(animation => animation.playState === 'running');+ } catch { return []; }+ }+ function deferAnimationCheck(record, el) {+ const animations = runningAnimations(el);+ if (!animations.length) return false;+ const agentUserAt = Number.isFinite(record.agentUserAt) ? record.agentUserAt+ : agentTurn && lastUserAt <= agentTurn.userAt ? agentTurn.userAt : undefined;+ if (Number.isFinite(agentUserAt)) for (const animation of animations) {+ const target = animation.effect?.target;+ animationAttribution.set(target instanceof Element ? target : el, agentUserAt);+ }+ return true;+ }+ function checkSettledAnimation(event) {+ const el = event.target;+ if (!active || !(el instanceof Element) || ignored(el)) return;+ const finiteAnimationRunning = runningAnimations(el).some(animation =>+ animation.effect?.getComputedTiming?.().iterations !== Infinity);+ if (finiteAnimationRunning) return;+ const agentUserAt = animationAttribution.get(el);+ animationAttribution.delete(el);+ onMutations([{ type: 'animation-settled', target: el, ...(Number.isFinite(agentUserAt) ? { agentUserAt } : {}) }]);+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -591,7 +627,7 @@for (const record of records) {if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;if (ignored(record.target)) continue;- if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ if (deferAnimationCheck(record, record.target)) continue;if (record.type === 'attributes' && record.attributeName === 'selected'&& record.oldValue !== record.target.getAttribute('selected')) {const select = record.target.closest?.('select');@@ -615,6 +651,7 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;+ if (baseNavigationMutation(record, el)) { noteChange(document.documentElement, record.agentUserAt); continue; }const changedStylesheets = stylesheetMutation(record);if (changedStylesheets.length) {if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);@@ -628,7 +665,8 @@// while observing independent page-script changes to editable text.if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)- && el.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ && deferAnimationCheck(record, el)) continue;+ if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;if (++measured > 300) {// The remaining records are intentionally uninspected. Invalidate the// prepared page conservatively so a later visible target cannot vanish@@ -804,7 +842,8 @@for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {if (event.isTrusted) expected('click', elementFor(event), event, true);});- for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);+ for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove', 'pointerdown', 'mousedown', 'pointerup', 'mouseup'])+ listen(document, name, noteAgentTurn);listen(document, 'pointerdown', event => {const el = elementFor(event);if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;@@ -931,6 +970,7 @@clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;+ animationAttribution = new WeakMap();agentLayoutHistory.length = 0;roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); controlReferences.clear(); controlIterator = null;}diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 32345079f..f71102148 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -420,12 +420,20 @@ for (const siteIsolation of [false, true]) {}rootSession = await context.newCDPSession(page);const client = new CDPClient(); client.sessions.set(tab, { attached: true });+ await child.evaluate(() => document.getElementById('agent').addEventListener('pointerover', () => {+ document.getElementById('status').textContent = 'Hover changed this';+ }, { once: true }));+ const hover = await child.locator('#agent').boundingBox();+ await client.dispatchMouseEvent(tab, 'mouseMoved', hover.x + hover.width / 2, hover.y + hover.height / 2);+ await child.waitForFunction(() => document.getElementById('status').textContent === 'Hover changed this');+ await child.waitForTimeout(200);+ assert.deepEqual(await child.evaluate(() => feedback), [], 'CDP pointer-entry handlers must remain agent-attributed');const rect = await child.locator('#field').boundingBox();await client.dispatchMouseEvent(tab, 'mousePressed', rect.x + 8, rect.y + 8);await client.dispatchMouseEvent(tab, 'mouseReleased', rect.x + 8, rect.y + 8);await client.sendCommand(tab, 'Input.insertText', { text: 'agent' });assert.equal(await child.locator('#field').inputValue(), 'agent');- assert.deepEqual(registrations, [2, 2, 2]);+ assert.deepEqual(registrations, [2, 2, 2, 2]);const container = await child.locator('#container').boundingBox();await client.sendCommand(tab, 'Input.dispatchMouseEvent', { type: 'mouseWheel', x: container.x + 20, y: container.y + 20,deltaX: 0, deltaY: 100 });@@ -855,6 +863,96 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: base URL changes invalidate prepared relative navigation`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const base = document.createElement('base'); base.id = 'page-base'; base.href = '/before/';+ const link = document.createElement('a'); link.id = 'relative-link'; link.href = 'next'; link.textContent = 'Continue';+ document.head.append(base); document.body.append(link);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));+ await page.waitForTimeout(180);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'relative-navigation', tool: 'click', selector: '#relative-link' });+ deliver('page_monitor_dispatch', { operationId: 'relative-navigation', kind: 'click', selector: '#relative-link', fenceOnly: true });+ return lastMonitorResponse.guard;+ });+ await page.locator('#page-base').evaluate(el => el.setAttribute('href', '/after/'));+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'html'), null, { timeout: 1000 });+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,+ })), guard);+ assert.equal(accepted, false, 'Changing the base URL must stale relative link destinations');+ } finally { await browser.close(); }+ });++ test(`${build}: visibility signatures are rechecked after animations and transitions settle`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const style = document.createElement('style');+ style.textContent = `@keyframes hide-target { to { opacity: 0; } }+ #animated-target { opacity: 1; }+ #animated-target.hide { animation: hide-target 120ms linear forwards; }+ #transition-target { opacity: 1; transition: opacity 120ms linear; }+ #transition-target.hide { opacity: 0; }+ #mixed-transition { opacity: 1; transition: opacity 400ms linear; }+ #mixed-container.hide #mixed-hidden { visibility: hidden; }+ #mixed-container.hide #mixed-transition { opacity: 0; }`;+ const animated = document.createElement('button'); animated.id = 'animated-target'; animated.textContent = 'Animated target';+ const transitioned = document.createElement('button'); transitioned.id = 'transition-target'; transitioned.textContent = 'Transition target';+ const mixed = document.createElement('div'); mixed.id = 'mixed-container';+ mixed.innerHTML = '<button id="mixed-hidden">Hidden sibling</button><button id="mixed-transition">Animated sibling</button>';+ const shadowHost = document.createElement('div'); shadowHost.id = 'shadow-host';+ const shadow = shadowHost.attachShadow({ mode: 'open' });+ shadow.innerHTML = `<style>@keyframes shadow-hide { to { opacity: 0; } } #shadow-target.hide { animation: shadow-hide 120ms linear forwards; }</style>+ <button id="shadow-target">Shadow target</button>`;+ document.head.append(style); document.body.append(animated, transitioned, mixed, shadowHost);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));+ await page.waitForTimeout(180);+ for (const id of ['animated-target', 'transition-target']) {+ const guard = await page.evaluate(targetId => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: targetId, tool: 'click', selector: `#${targetId}` });+ deliver('page_monitor_dispatch', { operationId: targetId, kind: 'click', selector: `#${targetId}`, fenceOnly: true });+ return lastMonitorResponse.guard;+ }, id);+ await page.locator(`#${id}`).evaluate(el => el.classList.add('hide'));+ await page.waitForFunction(targetId => feedback.some(event => event.kind === 'dom' && event.target === `button#${targetId}`), id,+ { timeout: 1500 });+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,+ })), guard);+ assert.equal(accepted, false, `${id} must invalidate the prepared click after its final hidden state`);+ }+ await page.evaluate(() => {+ feedback = [];+ document.getElementById('shadow-host').shadowRoot.getElementById('shadow-target').classList.add('hide');+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button#shadow-target'), null,+ { timeout: 1500 });++ const mixedGuard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'mixed-animation', tool: 'click', selector: '#mixed-hidden' });+ deliver('page_monitor_dispatch', { operationId: 'mixed-animation', kind: 'click', selector: '#mixed-hidden', fenceOnly: true });+ return lastMonitorResponse.guard;+ });+ await page.locator('#mixed-container').evaluate(el => el.classList.add('hide'));+ await page.waitForFunction(revision => Number(document.documentElement.getAttribute('data-webbrain-page-revision').split(':').at(-1)) > revision,+ mixedGuard.revision, { timeout: 500 });+ const mixedAccepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,+ })), mixedGuard);+ assert.equal(mixedAccepted, false, 'A transition on one descendant must not defer checks for hidden siblings');+ await page.waitForFunction(revision => Number(document.documentElement.getAttribute('data-webbrain-page-revision').split(':').at(-1)) > revision + 1,+ mixedGuard.revision, { timeout: 1500 });+ } finally { await browser.close(); }+ });+test(`${build}: page focus theft fences a selectorless input while the identified agent target stays quiet`, async () => {const { browser, page } = await fixture(engine, build);try {