Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: fence selector scrolling and observe viewport state

Barack Sokullu8de947a11fab
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.md
index 9ff279731..1aa723fe0 100644
--- a/docs/page-feedback-monitor.md
+++ b/docs/page-feedback-monitor.md
@@ -10,6 +10,9 @@ accessibility tree and continues the original task. Pointer gestures and IME
composition keep the gate closed until they finish. External scroll/navigation
with uncertain attribution also get the idle delay; automatic DOM observations
do not extend it. Stop interrupts the wait.
+Window and visual-viewport size, zoom and offset changes invalidate prepared
+coordinates. Top-level viewport activity waits for geometry to settle; embedded
+frame size changes are page layout observations and do not extend the user gate.
`composerDeliveryMode` only controls typed chat messages (Queue/Steer). Automatic
page feedback uses its own bounded queue and cannot promote a queued message to
@@ -34,6 +37,10 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behavior
a shortcut's modifier and letter, as well as before each later key. A different physical intervention
breaks the expectation, including a later click on the same target. Uncertain
effects remain `unknown`; `isTrusted` alone does not establish human origin.
+- Chrome selector resolution checks a fence without claiming scroll during its
+ asynchronous search or retries. Only a resolved node's actual `scrollIntoView`
+ dispatch activates attribution, with a live page revision check in both open
+ and closed shadow-root paths. Read-only selector queries do not arm scrolling.
- Meaningful visible content, control state, visibility and geometry changes are
coalesced. Visible subtree identity changes invalidate targets even when the
replacement has identical text and geometry. CSS animation and extension
@@ -44,6 +51,8 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behavior
shadow roots on existing hosts before their initial rendering; shadow-root
child-list mutations are also monitored. Inaccessible frames and closed shadow roots retain the
existing browser-access limitations.
+ Existing dialog/details `open` and subtree `inert` state changes are observed
+ even when no children, classes or styles change.
- Event payloads contain bounded target identifiers and scroll positions, never
typed keys, field values or selected text. The subsequent normal page read uses
the existing untrusted-content and secret-handling rules.
@@ -107,5 +116,10 @@ the user edits the same field.
Frame lifecycle regressions cover delayed child acknowledgements and inaccessible
frames. Browser fixtures also cover page-owned marker attributes and the real
extension indicators across monitor replacement.
+Real CDP regressions pause selector resolution during runtime enablement, missing
+target traversal and queued evaluation, then verify external scroll feedback and
+the final dispatch fence. Open and closed shadow-root agent scrolling remain
+attributed. Geometry fixtures also cover modal/details/inert transitions and
+viewport resizing during coordinate preparation and after cancellation.
The monitor regressions are also included in `npm run test:runtime-lifecycle`.
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index 34595e0fb..d0d5c4c2c 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -144,7 +144,7 @@ export const pageFeedbackMethods = {
}
const seq = Number(feedback.seq);
if (!Number.isSafeInteger(seq) || seq <= frame.seq) return { accepted: false, reason: 'duplicate-observation' };
- if (!['click', 'input', 'selection', 'scroll', 'dom', 'activity'].includes(feedback.kind)
+ if (!['click', 'input', 'selection', 'scroll', 'resize', 'dom', 'activity'].includes(feedback.kind)
|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };
frame.seq = seq;
if (feedback.source === 'agent') {
@@ -156,6 +156,8 @@ export const pageFeedbackMethods = {
const item = { kind: feedback.kind, source: feedback.source, frameId,
revision: Math.max(0, Number(feedback.revision) || 0), target: clean(feedback.target, 180) };
if (feedback.kind === 'scroll') item.viewport = { x: Number(feedback.viewport?.x) || 0, y: Number(feedback.viewport?.y) || 0 };
+ if (feedback.kind === 'resize') item.viewport = Object.fromEntries(['width', 'height', 'visualWidth', 'visualHeight', 'scale', 'offsetX', 'offsetY']
+ .map(key => [key, Number.isFinite(feedback.viewport?.[key]) ? feedback.viewport[key] : 0]));
if (feedback.source === 'user') {
// DOM callbacks describe consequences; only physical activity extends the idle gate.
if (feedback.kind !== 'dom') run.lastUserAt = Date.now();
@@ -164,7 +166,8 @@ export const pageFeedbackMethods = {
// A physical interaction supersedes an expected agent navigation.
run.navigation = null;
}
- if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll')) run.lastActivityAt = Date.now();
+ if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll'
+ || (feedback.kind === 'resize' && feedback.source !== 'page'))) run.lastActivityAt = Date.now();
this._queuePageFeedback(tabId, item);
return { accepted: true };
},
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.js
index 8043b778b..cb2c6d09e 100644
--- a/src/chrome/src/cdp/cdp-client.js
+++ b/src/chrome/src/cdp/cdp-client.js
@@ -5,7 +5,24 @@
*/
import { combineImages } from './image-utils.js';
-import { beforePageAgentDispatch, hasPageAgentDispatchOwner } from '../agent/page-feedback.js';
+import { beforePageAgentDispatch, hasPageAgentDispatchOwner, pageFeedbackPendingResult } from '../agent/page-feedback.js';
+
+const SELECTOR_SCROLL_DISPATCH_SOURCE = `
+ const beforeSelectorScroll = (element, guard) => {
+ if (!guard) return true;
+ const doc = element.ownerDocument;
+ if (!element.isConnected || doc.documentElement?.getAttribute('data-webbrain-page-revision') !== guard.documentToken + ':' + guard.revision) return false;
+ return element.dispatchEvent(new doc.defaultView.CustomEvent('webbrain-agent-scroll-dispatch', {
+ bubbles: true, composed: true, cancelable: true, detail: JSON.stringify(guard),
+ }));
+ };
+`;
+
+function throwIfPageFeedbackPending(result) {
+ if (!result?.pageFeedbackPending) return;
+ const error = new Error(pageFeedbackPendingResult().error);
+ error.code = 'page_feedback_pending'; throw error;
+}
function readProseMirrorText(el) {
if (!el?.isContentEditable || !el.classList?.contains('ProseMirror')) return null;
@@ -3688,7 +3705,6 @@ export class CDPClient {
let lastResult = null;
for (let i = 0; i <= retries; i++) {
deadline.throwIfExpired();
- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'scroll', selector, scrollIntoView: true });
const result = await this._resolveSelectorOnce(tabId, selector, options);
deadline.throwIfExpired();
// Found and usable → done.
@@ -3969,12 +3985,19 @@ export class CDPClient {
await this.sendCommand(tabId, 'Runtime.enable');
deadline.throwIfExpired();
+ const scrollGuard = scrollRequested ? await beforePageAgentDispatch(globalThis.chrome, tabId, {
+ kind: 'scroll', selector, scrollIntoView: true, fenceOnly: true,
+ }) : null;
+ deadline.throwIfExpired();
+
const selectorJSON = JSON.stringify(selector);
const requireUnique = options?.requireUnique === true;
// ---- Strategy 1: JS walker (open shadow roots) ----
const jsExpr = `
(() => {
+ ${SELECTOR_SCROLL_DISPATCH_SOURCE}
+ const pageGuard = ${JSON.stringify(scrollGuard || null)};
const sel = ${selectorJSON};
const requireUnique = ${requireUnique};
const scrollRequested = ${scrollRequested};
@@ -4026,6 +4049,7 @@ export class CDPClient {
if (deadlineExpired()) return { found: false, deadlineExpired: true };
if (scrollRequested && found.tagName !== 'SELECT') {
if (deadlineExpired()) return { found: false, deadlineExpired: true };
+ if (!beforeSelectorScroll(found, pageGuard)) return { found: false, pageFeedbackPending: true };
try { found.scrollIntoView({ block: 'center', inline: 'center' }); } catch (e) {}
}
const r = found.getBoundingClientRect();
@@ -4059,6 +4083,7 @@ export class CDPClient {
const jsRes = await this.evaluate(tabId, jsExpr);
deadline.throwIfExpired();
const jsInfo = jsRes?.result?.value;
+ throwIfPageFeedbackPending(jsInfo);
if (jsInfo?.deadlineExpired) deadline.throwIfExpired(true);
if (jsInfo?.error) return jsInfo;
if (jsInfo?.found) {
@@ -4180,7 +4205,7 @@ export class CDPClient {
// deadline when a frozen renderer resumes, so deadline-bound callers use
// a guarded Runtime function on the exact closed-shadow node instead.
if (scrollRequested) {
- if (deadline.deadlineAt > 0) {
+ if (deadline.deadlineAt > 0 || hasPageAgentDispatchOwner(tabId)) {
let scrollObjectId = null;
try {
deadline.throwIfExpired();
@@ -4188,12 +4213,19 @@ export class CDPClient {
deadline.throwIfExpired();
scrollObjectId = resolved?.object?.objectId || null;
if (scrollObjectId) {
+ const target = hasPageAgentDispatchOwner(tabId) ? await this._pageAgentObjectTarget(tabId, scrollObjectId) : {};
+ const guard = await beforePageAgentDispatch(globalThis.chrome, tabId, {
+ kind: 'scroll', scrollIntoView: true, fenceOnly: true, ...target,
+ });
+ deadline.throwIfExpired();
const scrolled = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {
objectId: scrollObjectId,
returnByValue: true,
- functionDeclaration: `function (actionDeadlineAt) {
+ functionDeclaration: `function (actionDeadlineAt, pageGuard) {
+ ${SELECTOR_SCROLL_DISPATCH_SOURCE}
const deadlineExpired = () => actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt;
if (deadlineExpired()) return { scrolled: false, deadlineExpired: true };
+ if (!beforeSelectorScroll(this, pageGuard)) return { scrolled: false, pageFeedbackPending: true };
try {
this.scrollIntoView({ block: 'center', inline: 'center' });
} catch {
@@ -4201,9 +4233,10 @@ export class CDPClient {
}
return { scrolled: true };
}`,
- arguments: [{ value: deadline.deadlineAt }],
+ arguments: [{ value: deadline.deadlineAt }, { value: guard || null }],
});
deadline.throwIfExpired();
+ throwIfPageFeedbackPending(scrolled?.result?.value);
if (scrolled?.result?.value?.deadlineExpired) deadline.throwIfExpired(true);
}
} finally {
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index e24812b5a..eb4be100c 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -12,6 +12,7 @@
let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;
let pointerHeld = false, composing = false, localOperation = null;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
+ let lastViewport = '';
let lastFeedbackDelivery = Promise.resolve();
let agentTurn = null, userTurn = null, lastUserTarget = null;
let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();
@@ -154,6 +155,7 @@
.map(node => node.textContent).join('').slice(0, 200);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
+ el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
}
@@ -170,7 +172,7 @@
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true,
attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',
- 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'class', 'style'] });
+ 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });
seed(root, budget);
}
function subtreeChanged(record, el) {
@@ -200,7 +202,7 @@
let changed = false, source = 'page', target = '';
const changes = [...records];
for (const record of records) {
- if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;
+ if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName)) continue;
if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
const descendants = new Set([
...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),
@@ -247,8 +249,8 @@
pendingDOM = { kind: 'dom', source, target };
domTimer = setTimeout(() => { domTimer = null; const observation = pendingDOM; pendingDOM = null; send(observation); }, 150);
}
- function listen(target, name, handler) {
- target.addEventListener(name, handler, { capture: true, passive: true });
+ function listen(target, name, handler, passive = true) {
+ target.addEventListener(name, handler, { capture: true, passive });
listeners.push(() => target.removeEventListener(name, handler, true));
}
function start(state) {
@@ -274,6 +276,18 @@
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
} catch { /* Only a current prepared operation can mark a DOM write. */ }
});
+ listen(document, 'webbrain-agent-scroll-dispatch', event => {
+ try {
+ const guard = JSON.parse(String(event.detail));
+ if (guard.documentToken !== documentToken || guard.revision !== revision || !operations.has(guard.operationId)
+ || domTimer || unreported) { event.preventDefault(); return; }
+ const el = elementFor(event);
+ dispatch({ operationId: guard.operationId, kind: 'scroll', scrollIntoView: true, runToken });
+ const op = operations.get(guard.operationId);
+ op.target = el; op.scrollAncestors.clear();
+ for (let node = el?.parentElement || el?.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) op.scrollAncestors.add(node);
+ } catch { event.preventDefault(); }
+ }, false);
const noteAgentTurn = event => {
const el = elementFor(event);
const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);
@@ -339,6 +353,21 @@
const el = elementFor(event);
if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);
});
+ const viewport = () => ({ width: innerWidth, height: innerHeight,
+ visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,
+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });
+ lastViewport = JSON.stringify(viewport());
+ const onResize = () => {
+ const current = viewport(), key = JSON.stringify(current);
+ if (!active || key === lastViewport) return;
+ lastViewport = key; revision++;
+ send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });
+ };
+ listen(window, 'resize', onResize);
+ if (window.visualViewport) {
+ listen(window.visualViewport, 'resize', onResize);
+ listen(window.visualViewport, 'scroll', onResize);
+ }
listen(document, 'scroll', event => {
const el = event.target === document ? document.documentElement : event.target;
if (ignored(el) || expected('scroll', el)) return;
@@ -366,7 +395,7 @@
}
function stop() {
active = false; runToken = ''; pointerHeld = false; composing = false;
- lastUserAt = 0; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();
+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
el.removeAttribute?.('data-webbrain-native-action');
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index 34595e0fb..d0d5c4c2c 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -144,7 +144,7 @@ export const pageFeedbackMethods = {
}
const seq = Number(feedback.seq);
if (!Number.isSafeInteger(seq) || seq <= frame.seq) return { accepted: false, reason: 'duplicate-observation' };
- if (!['click', 'input', 'selection', 'scroll', 'dom', 'activity'].includes(feedback.kind)
+ if (!['click', 'input', 'selection', 'scroll', 'resize', 'dom', 'activity'].includes(feedback.kind)
|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };
frame.seq = seq;
if (feedback.source === 'agent') {
@@ -156,6 +156,8 @@ export const pageFeedbackMethods = {
const item = { kind: feedback.kind, source: feedback.source, frameId,
revision: Math.max(0, Number(feedback.revision) || 0), target: clean(feedback.target, 180) };
if (feedback.kind === 'scroll') item.viewport = { x: Number(feedback.viewport?.x) || 0, y: Number(feedback.viewport?.y) || 0 };
+ if (feedback.kind === 'resize') item.viewport = Object.fromEntries(['width', 'height', 'visualWidth', 'visualHeight', 'scale', 'offsetX', 'offsetY']
+ .map(key => [key, Number.isFinite(feedback.viewport?.[key]) ? feedback.viewport[key] : 0]));
if (feedback.source === 'user') {
// DOM callbacks describe consequences; only physical activity extends the idle gate.
if (feedback.kind !== 'dom') run.lastUserAt = Date.now();
@@ -164,7 +166,8 @@ export const pageFeedbackMethods = {
// A physical interaction supersedes an expected agent navigation.
run.navigation = null;
}
- if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll')) run.lastActivityAt = Date.now();
+ if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll'
+ || (feedback.kind === 'resize' && feedback.source !== 'page'))) run.lastActivityAt = Date.now();
this._queuePageFeedback(tabId, item);
return { accepted: true };
},
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index e24812b5a..eb4be100c 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -12,6 +12,7 @@
let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;
let pointerHeld = false, composing = false, localOperation = null;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
+ let lastViewport = '';
let lastFeedbackDelivery = Promise.resolve();
let agentTurn = null, userTurn = null, lastUserTarget = null;
let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();
@@ -154,6 +155,7 @@
.map(node => node.textContent).join('').slice(0, 200);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
+ el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
}
@@ -170,7 +172,7 @@
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true,
attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',
- 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'class', 'style'] });
+ 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });
seed(root, budget);
}
function subtreeChanged(record, el) {
@@ -200,7 +202,7 @@
let changed = false, source = 'page', target = '';
const changes = [...records];
for (const record of records) {
- if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;
+ if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName)) continue;
if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
const descendants = new Set([
...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),
@@ -247,8 +249,8 @@
pendingDOM = { kind: 'dom', source, target };
domTimer = setTimeout(() => { domTimer = null; const observation = pendingDOM; pendingDOM = null; send(observation); }, 150);
}
- function listen(target, name, handler) {
- target.addEventListener(name, handler, { capture: true, passive: true });
+ function listen(target, name, handler, passive = true) {
+ target.addEventListener(name, handler, { capture: true, passive });
listeners.push(() => target.removeEventListener(name, handler, true));
}
function start(state) {
@@ -274,6 +276,18 @@
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
} catch { /* Only a current prepared operation can mark a DOM write. */ }
});
+ listen(document, 'webbrain-agent-scroll-dispatch', event => {
+ try {
+ const guard = JSON.parse(String(event.detail));
+ if (guard.documentToken !== documentToken || guard.revision !== revision || !operations.has(guard.operationId)
+ || domTimer || unreported) { event.preventDefault(); return; }
+ const el = elementFor(event);
+ dispatch({ operationId: guard.operationId, kind: 'scroll', scrollIntoView: true, runToken });
+ const op = operations.get(guard.operationId);
+ op.target = el; op.scrollAncestors.clear();
+ for (let node = el?.parentElement || el?.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) op.scrollAncestors.add(node);
+ } catch { event.preventDefault(); }
+ }, false);
const noteAgentTurn = event => {
const el = elementFor(event);
const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);
@@ -339,6 +353,21 @@
const el = elementFor(event);
if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);
});
+ const viewport = () => ({ width: innerWidth, height: innerHeight,
+ visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,
+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });
+ lastViewport = JSON.stringify(viewport());
+ const onResize = () => {
+ const current = viewport(), key = JSON.stringify(current);
+ if (!active || key === lastViewport) return;
+ lastViewport = key; revision++;
+ send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });
+ };
+ listen(window, 'resize', onResize);
+ if (window.visualViewport) {
+ listen(window.visualViewport, 'resize', onResize);
+ listen(window.visualViewport, 'scroll', onResize);
+ }
listen(document, 'scroll', event => {
const el = event.target === document ? document.documentElement : event.target;
if (ignored(el) || expected('scroll', el)) return;
@@ -366,7 +395,7 @@
}
function stop() {
active = false; runToken = ''; pointerHeld = false; composing = false;
- lastUserAt = 0; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();
+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
el.removeAttribute?.('data-webbrain-native-action');
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 4f9879522..7d8176be3 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -146,6 +146,25 @@ for (const build of ['chrome', 'firefox']) {
} finally { agent._releaseRunEntry(tab); }
});
+ test(`${build}: viewport resize invalidates coordinates and waits for geometry to settle`, async () => {
+ const agent = setup(Agent), tab = nextTab++;
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ const binding = bind(agent, tab);
+ agent.screenshotCaptures.set(tab, { captureId: 'stale-coordinates' });
+ agent._jevSessions = new Map([[tab, { disabled: false, queue: ['prepared'], snapshot: {} }]]);
+ assert.equal(binding.send({ kind: 'resize', source: 'unknown', target: 'viewport', viewport: { width: 900, height: 600, scale: 1 } }).accepted, true);
+ assert.equal(agent.screenshotCaptures.has(tab), false);
+ assert.equal(agent._jevSessions.get(tab).disabled, true);
+ assert.deepEqual(agent._jevSessions.get(tab).queue, []);
+ assert.equal(agent._pageFeedbackRuns.get(tab).lastUserAt, 0, 'Resize alone does not prove physical user input');
+ assert.ok(agent._pageFeedbackRuns.get(tab).lastActivityAt > 0, 'Unknown geometry activity must settle before replanning');
+ agent._pageFeedbackRuns.get(tab).lastActivityAt = 0;
+ binding.send({ kind: 'resize', source: 'page', target: 'viewport', viewport: { width: 500, height: 200 } });
+ assert.equal(agent._pageFeedbackRuns.get(tab).lastActivityAt, 0, 'Page layout changes to embedded frame dimensions must not extend the user idle gate');
+ } finally { agent._releaseRunEntry(tab); }
+ });
+
test(`${build}: navigation arriving during a page read gets its own feedback ID`, async () => {
const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();
const updates = [], messages = [];
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index d92a49c85..7a7521d9f 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -65,6 +65,96 @@ async function installContentEditableFallback(page, build) {
});
}
+test('Chrome selector scrolling claims input only at the actual page mutation', { timeout: 30000 }, async () => {
+ const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser, tab = 912;
+ const host = Object.assign({ isRunning: () => true, _checkAbort: () => false }, pageFeedbackMethods);
+ let browser, page, session, pauseMethod = '', entered, resume;
+ const registrations = [];
+ const api = { runtime: {}, tabs: {
+ get: async () => ({ url: 'https://monitor.test/start' }),
+ sendMessage: async (_tab, msg) => {
+ if (!page) return {};
+ if (msg.action === 'page_monitor_dispatch') registrations.push(msg.params);
+ return page.evaluate(msg => new Promise(resolve => messageListeners.forEach(fn => fn(msg, {}, resolve))), msg);
+ },
+ }, debugger: { sendCommand(_source, method, params, callback) {
+ void (async () => {
+ if (method === pauseMethod) {
+ pauseMethod = ''; entered(); await new Promise(resolve => { resume = resolve; });
+ }
+ return await session.send(method, params);
+ })().then(callback, error => { api.runtime.lastError = { message: error.message }; callback(); delete api.runtime.lastError; });
+ } } };
+ globalThis.chrome = api; delete globalThis.browser;
+ try {
+ await host._beginPageFeedbackRun(tab, 'interactive');
+ ({ browser, page } = await fixture(chromium, 'chrome', { runToken: host._pageFeedbackRuns.get(tab).token }));
+ const documentToken = await page.evaluate(() => document.documentElement.getAttribute('data-webbrain-page-revision').split(':')[0]);
+ host.pageMonitorState({ tab: { id: tab }, frameId: 0 }, documentToken);
+ await page.exposeFunction('feedbackToHost', feedback => host.observePageFeedback({ tab: { id: tab }, frameId: 0 }, feedback));
+ await page.exposeFunction('monitorStateFromHost', token => host.pageMonitorState({ tab: { id: tab }, frameId: 0 }, token));
+ await page.evaluate(() => {
+ const original = chrome.runtime.sendMessage;
+ chrome.runtime.sendMessage = async message => {
+ if (message.action === 'get_page_monitor_state') return await monitorStateFromHost(message.documentToken);
+ const response = await original(message);
+ return message.action === 'page_feedback' ? await feedbackToHost(message.feedback) : response;
+ };
+ });
+ session = await page.context().newCDPSession(page);
+ const client = new CDPClient(); client.sessions.set(tab, { attached: true });
+ const reset = async () => {
+ host._finishPageFeedbackRun(tab);
+ await page.waitForTimeout(20);
+ await page.evaluate(() => window.scrollTo(0, 0));
+ await page.waitForTimeout(100);
+ await host._beginPageFeedbackRun(tab, 'interactive');
+ registrations.length = 0;
+ await page.evaluate(() => { feedback = []; });
+ };
+ for (const scenario of [
+ { method: 'Runtime.enable', selector: '#agent', options: { scroll: false, retries: 0 }, readOnly: true },
+ { method: 'DOM.getDocument', selector: '#missing', options: { retries: 1, delayMs: 0 } },
+ { method: 'Runtime.evaluate', selector: '#agent', options: { retries: 0 } },
+ ]) {
+ await reset(); pauseMethod = scenario.method;
+ const paused = new Promise(resolve => { entered = resolve; });
+ const resolving = client.resolveSelector(tab, scenario.selector, scenario.options)
+ .then(value => ({ value }), error => ({ code: error.code }));
+ await paused;
+ await page.evaluate(() => window.scrollTo(0, 600));
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'scroll'), null, { timeout: 1000 });
+ assert.ok(registrations.every(mark => mark.fenceOnly === true), 'A resolver cannot claim scroll while a DOM search is pending');
+ resume(); resume = null;
+ const result = await resolving;
+ if (scenario.readOnly) {
+ assert.equal(result.value.found, true); assert.equal(registrations.length, 0);
+ } else assert.equal(result.code, 'page_feedback_pending', `${scenario.method} must preserve the external intervention`);
+ assert.equal(await page.evaluate(() => scrollY), 600, 'A stale resolver must not scroll back to its target');
+ }
+ await reset();
+ await page.evaluate(() => {
+ for (const mode of ['open', 'closed']) {
+ const el = document.createElement('div');
+ el.attachShadow({ mode }).innerHTML = `<button id="deep-${mode}">Deep target</button>`;
+ document.body.append(el);
+ }
+ });
+ await page.waitForTimeout(200);
+ for (const mode of ['open', 'closed']) {
+ await reset();
+ const result = await client.resolveSelector(tab, `#deep-${mode}`, { retries: 0 });
+ assert.equal(result.found, true); assert.equal(result.inViewport, true);
+ await page.waitForTimeout(200);
+ assert.ok(await page.evaluate(() => scrollY > 1000), `${mode} target must actually scroll into view`);
+ assert.deepEqual(await page.evaluate(() => feedback), [], `${mode} agent scroll must not trigger a feedback loop`);
+ }
+ } finally {
+ resume?.(); host._finishPageFeedbackRun(tab);
+ await browser?.close(); globalThis.chrome = savedChrome; globalThis.browser = savedBrowser;
+ }
+});
+
for (const siteIsolation of [false, true]) {
test(`Chrome CDP input reaches the receiving iframe monitor (site isolation: ${siteIsolation})`, { timeout: 30000 }, async () => {
const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser;
@@ -376,6 +466,69 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: existing dialogs, details and inert surfaces invalidate prepared actions`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const surfaces = document.createElement('div');
+ surfaces.innerHTML = '<dialog id="modal"><button>Modal target</button></dialog><details id="details" style="height:100px"><summary>Summary</summary><button>Existing details target</button></details><div id="inert-surface"><button>Existing inert target</button></div>';
+ document.body.prepend(surfaces);
+ });
+ await page.waitForTimeout(200);
+ for (const state of ['modal-open', 'modal-close', 'details-open', 'details-close', 'inert-on', 'inert-off']) {
+ const response = await page.evaluate(async state => {
+ feedback = []; deliver('page_monitor_prepare', { operationId: state, tool: 'click', selector: '#agent' });
+ if (state === 'modal-open') document.getElementById('modal').showModal();
+ if (state === 'modal-close') document.getElementById('modal').close();
+ if (state.startsWith('details')) document.getElementById('details').open = state.endsWith('open');
+ if (state.startsWith('inert')) document.getElementById('inert-surface').inert = state.endsWith('on');
+ await Promise.resolve();
+ deliver('page_monitor_dispatch', { operationId: state, kind: 'click', selector: '#agent' });
+ await new Promise(resolve => setTimeout(resolve, 0));
+ return lastMonitorResponse;
+ }, state);
+ assert.equal(response.pageFeedbackPending, true, `${state} must invalidate the prepared click`);
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom'), `${state} must refresh page context`);
+ }
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: viewport resize fences prepared coordinates and stops with the run`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => { document.getElementById('agent').style.cssText = 'position:absolute;left:50vw;top:0'; });
+ await page.waitForTimeout(200);
+ const before = await page.locator('#agent').boundingBox();
+ await page.evaluate(() => { feedback = []; deliver('page_monitor_prepare', { operationId: 'resize-click', tool: 'click', selector: '#agent' }); });
+ await page.setViewportSize({ width: 900, height: 600 });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'resize'), null, { timeout: 1000 });
+ const after = await page.locator('#agent').boundingBox();
+ assert.notEqual(before.x, after.x);
+ const response = await page.evaluate(async () => {
+ deliver('page_monitor_dispatch', { operationId: 'resize-click', kind: 'click', x: 650, y: 20 });
+ await new Promise(resolve => setTimeout(resolve, 0)); return lastMonitorResponse;
+ });
+ assert.equal(response.pageFeedbackPending, true);
+ const observation = await page.evaluate(() => feedback.find(event => event.kind === 'resize'));
+ assert.equal(observation.source, 'unknown'); assert.equal(observation.viewport.width, 900);
+ const frameNavigation = page.waitForEvent('framenavigated', { predicate: frame => frame.url() === 'https://monitor.test/frame' });
+ await page.evaluate(() => {
+ const frame = document.createElement('iframe'); frame.src = 'https://monitor.test/frame';
+ frame.style.cssText = 'width:400px;height:250px'; document.body.prepend(frame);
+ });
+ const child = await frameNavigation;
+ await child.waitForFunction(() => window.__wbPageMonitor && document.documentElement.hasAttribute('data-webbrain-page-revision'));
+ await child.evaluate(() => { feedback = []; });
+ await page.evaluate(() => { document.querySelector('iframe').style.width = '450px'; });
+ await child.waitForFunction(() => feedback.some(event => event.kind === 'resize'), null, { timeout: 1000 });
+ assert.equal(await child.evaluate(() => feedback.find(event => event.kind === 'resize').source), 'page', 'Embedded frame dimensions are page layout changes');
+ await page.evaluate(() => { monitorEnabled = false; deliver('page_monitor_state'); feedback = []; });
+ await page.setViewportSize({ width: 1000, height: 700 });
+ await page.waitForTimeout(200);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Stop removes viewport listeners');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: contenteditable fallback attributes its cancellable gate and native edits`, async () => {
const { browser, page } = await fixture(engine, build);
try {
diff --git a/test/run.js b/test/run.js
index 6c1336632..1ba3d03bc 100644
--- a/test/run.js
+++ b/test/run.js
@@ -58006,7 +58006,7 @@ test('CDP selector resolution rejects queued open- and closed-shadow scrolls aft
if (method === 'DOM.resolveNode') return { object: { objectId: 'late-closed-shadow-target' } };
if (method === 'Runtime.callFunctionOn') {
assert.match(params.functionDeclaration, /deadlineExpired[\s\S]*scrollIntoView/);
- assert.deepEqual(params.arguments, [{ value: deadlineAt }]);
+ assert.deepEqual(params.arguments, [{ value: deadlineAt }, { value: null }]);
return { result: { value: { scrolled: false, deadlineExpired: true } } };
}
return {};