Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: fence selector scrolling and observe viewport state
Barack Sokullu8de947a11fab
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.mdindex 9ff279731..1aa723fe0 100644--- a/docs/page-feedback-monitor.md+++ b/docs/page-feedback-monitor.md@@ -10,6 +10,9 @@ accessibility tree and continues the original task. Pointer gestures and IMEcomposition keep the gate closed until they finish. External scroll/navigationwith uncertain attribution also get the idle delay; automatic DOM observationsdo not extend it. Stop interrupts the wait.+Window and visual-viewport size, zoom and offset changes invalidate prepared+coordinates. Top-level viewport activity waits for geometry to settle; embedded+frame size changes are page layout observations and do not extend the user gate.`composerDeliveryMode` only controls typed chat messages (Queue/Steer). Automaticpage feedback uses its own bounded queue and cannot promote a queued message to@@ -34,6 +37,10 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behaviora shortcut's modifier and letter, as well as before each later key. A different physical interventionbreaks the expectation, including a later click on the same target. Uncertaineffects remain `unknown`; `isTrusted` alone does not establish human origin.+- Chrome selector resolution checks a fence without claiming scroll during its+ asynchronous search or retries. Only a resolved node's actual `scrollIntoView`+ dispatch activates attribution, with a live page revision check in both open+ and closed shadow-root paths. Read-only selector queries do not arm scrolling.- Meaningful visible content, control state, visibility and geometry changes arecoalesced. Visible subtree identity changes invalidate targets even when thereplacement has identical text and geometry. CSS animation and extension@@ -44,6 +51,8 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behaviorshadow roots on existing hosts before their initial rendering; shadow-rootchild-list mutations are also monitored. Inaccessible frames and closed shadow roots retain theexisting browser-access limitations.+ Existing dialog/details `open` and subtree `inert` state changes are observed+ even when no children, classes or styles change.- Event payloads contain bounded target identifiers and scroll positions, nevertyped keys, field values or selected text. The subsequent normal page read usesthe existing untrusted-content and secret-handling rules.@@ -107,5 +116,10 @@ the user edits the same field.Frame lifecycle regressions cover delayed child acknowledgements and inaccessibleframes. Browser fixtures also cover page-owned marker attributes and the realextension indicators across monitor replacement.+Real CDP regressions pause selector resolution during runtime enablement, missing+target traversal and queued evaluation, then verify external scroll feedback and+the final dispatch fence. Open and closed shadow-root agent scrolling remain+attributed. Geometry fixtures also cover modal/details/inert transitions and+viewport resizing during coordinate preparation and after cancellation.The monitor regressions are also included in `npm run test:runtime-lifecycle`.diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex 34595e0fb..d0d5c4c2c 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -144,7 +144,7 @@ export const pageFeedbackMethods = {}const seq = Number(feedback.seq);if (!Number.isSafeInteger(seq) || seq <= frame.seq) return { accepted: false, reason: 'duplicate-observation' };- if (!['click', 'input', 'selection', 'scroll', 'dom', 'activity'].includes(feedback.kind)+ if (!['click', 'input', 'selection', 'scroll', 'resize', 'dom', 'activity'].includes(feedback.kind)|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };frame.seq = seq;if (feedback.source === 'agent') {@@ -156,6 +156,8 @@ export const pageFeedbackMethods = {const item = { kind: feedback.kind, source: feedback.source, frameId,revision: Math.max(0, Number(feedback.revision) || 0), target: clean(feedback.target, 180) };if (feedback.kind === 'scroll') item.viewport = { x: Number(feedback.viewport?.x) || 0, y: Number(feedback.viewport?.y) || 0 };+ if (feedback.kind === 'resize') item.viewport = Object.fromEntries(['width', 'height', 'visualWidth', 'visualHeight', 'scale', 'offsetX', 'offsetY']+ .map(key => [key, Number.isFinite(feedback.viewport?.[key]) ? feedback.viewport[key] : 0]));if (feedback.source === 'user') {// DOM callbacks describe consequences; only physical activity extends the idle gate.if (feedback.kind !== 'dom') run.lastUserAt = Date.now();@@ -164,7 +166,8 @@ export const pageFeedbackMethods = {// A physical interaction supersedes an expected agent navigation.run.navigation = null;}- if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll')) run.lastActivityAt = Date.now();+ if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll'+ || (feedback.kind === 'resize' && feedback.source !== 'page'))) run.lastActivityAt = Date.now();this._queuePageFeedback(tabId, item);return { accepted: true };},diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex 8043b778b..cb2c6d09e 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -5,7 +5,24 @@*/import { combineImages } from './image-utils.js';-import { beforePageAgentDispatch, hasPageAgentDispatchOwner } from '../agent/page-feedback.js';+import { beforePageAgentDispatch, hasPageAgentDispatchOwner, pageFeedbackPendingResult } from '../agent/page-feedback.js';++const SELECTOR_SCROLL_DISPATCH_SOURCE = `+ const beforeSelectorScroll = (element, guard) => {+ if (!guard) return true;+ const doc = element.ownerDocument;+ if (!element.isConnected || doc.documentElement?.getAttribute('data-webbrain-page-revision') !== guard.documentToken + ':' + guard.revision) return false;+ return element.dispatchEvent(new doc.defaultView.CustomEvent('webbrain-agent-scroll-dispatch', {+ bubbles: true, composed: true, cancelable: true, detail: JSON.stringify(guard),+ }));+ };+`;++function throwIfPageFeedbackPending(result) {+ if (!result?.pageFeedbackPending) return;+ const error = new Error(pageFeedbackPendingResult().error);+ error.code = 'page_feedback_pending'; throw error;+}function readProseMirrorText(el) {if (!el?.isContentEditable || !el.classList?.contains('ProseMirror')) return null;@@ -3688,7 +3705,6 @@ export class CDPClient {let lastResult = null;for (let i = 0; i <= retries; i++) {deadline.throwIfExpired();- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'scroll', selector, scrollIntoView: true });const result = await this._resolveSelectorOnce(tabId, selector, options);deadline.throwIfExpired();// Found and usable → done.@@ -3969,12 +3985,19 @@ export class CDPClient {await this.sendCommand(tabId, 'Runtime.enable');deadline.throwIfExpired();+ const scrollGuard = scrollRequested ? await beforePageAgentDispatch(globalThis.chrome, tabId, {+ kind: 'scroll', selector, scrollIntoView: true, fenceOnly: true,+ }) : null;+ deadline.throwIfExpired();+const selectorJSON = JSON.stringify(selector);const requireUnique = options?.requireUnique === true;// ---- Strategy 1: JS walker (open shadow roots) ----const jsExpr = `(() => {+ ${SELECTOR_SCROLL_DISPATCH_SOURCE}+ const pageGuard = ${JSON.stringify(scrollGuard || null)};const sel = ${selectorJSON};const requireUnique = ${requireUnique};const scrollRequested = ${scrollRequested};@@ -4026,6 +4049,7 @@ export class CDPClient {if (deadlineExpired()) return { found: false, deadlineExpired: true };if (scrollRequested && found.tagName !== 'SELECT') {if (deadlineExpired()) return { found: false, deadlineExpired: true };+ if (!beforeSelectorScroll(found, pageGuard)) return { found: false, pageFeedbackPending: true };try { found.scrollIntoView({ block: 'center', inline: 'center' }); } catch (e) {}}const r = found.getBoundingClientRect();@@ -4059,6 +4083,7 @@ export class CDPClient {const jsRes = await this.evaluate(tabId, jsExpr);deadline.throwIfExpired();const jsInfo = jsRes?.result?.value;+ throwIfPageFeedbackPending(jsInfo);if (jsInfo?.deadlineExpired) deadline.throwIfExpired(true);if (jsInfo?.error) return jsInfo;if (jsInfo?.found) {@@ -4180,7 +4205,7 @@ export class CDPClient {// deadline when a frozen renderer resumes, so deadline-bound callers use// a guarded Runtime function on the exact closed-shadow node instead.if (scrollRequested) {- if (deadline.deadlineAt > 0) {+ if (deadline.deadlineAt > 0 || hasPageAgentDispatchOwner(tabId)) {let scrollObjectId = null;try {deadline.throwIfExpired();@@ -4188,12 +4213,19 @@ export class CDPClient {deadline.throwIfExpired();scrollObjectId = resolved?.object?.objectId || null;if (scrollObjectId) {+ const target = hasPageAgentDispatchOwner(tabId) ? await this._pageAgentObjectTarget(tabId, scrollObjectId) : {};+ const guard = await beforePageAgentDispatch(globalThis.chrome, tabId, {+ kind: 'scroll', scrollIntoView: true, fenceOnly: true, ...target,+ });+ deadline.throwIfExpired();const scrolled = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {objectId: scrollObjectId,returnByValue: true,- functionDeclaration: `function (actionDeadlineAt) {+ functionDeclaration: `function (actionDeadlineAt, pageGuard) {+ ${SELECTOR_SCROLL_DISPATCH_SOURCE}const deadlineExpired = () => actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt;if (deadlineExpired()) return { scrolled: false, deadlineExpired: true };+ if (!beforeSelectorScroll(this, pageGuard)) return { scrolled: false, pageFeedbackPending: true };try {this.scrollIntoView({ block: 'center', inline: 'center' });} catch {@@ -4201,9 +4233,10 @@ export class CDPClient {}return { scrolled: true };}`,- arguments: [{ value: deadline.deadlineAt }],+ arguments: [{ value: deadline.deadlineAt }, { value: guard || null }],});deadline.throwIfExpired();+ throwIfPageFeedbackPending(scrolled?.result?.value);if (scrolled?.result?.value?.deadlineExpired) deadline.throwIfExpired(true);}} finally {diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex e24812b5a..eb4be100c 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -12,6 +12,7 @@let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;let pointerHeld = false, composing = false, localOperation = null;let requestGeneration = 0, unreported = 0, pendingDOM = null;+ let lastViewport = '';let lastFeedbackDelivery = Promise.resolve();let agentTurn = null, userTurn = null, lastUserTarget = null;let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();@@ -154,6 +155,7 @@.map(node => node.textContent).join('').slice(0, 200);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),+ el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);}@@ -170,7 +172,7 @@roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',- 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'class', 'style'] });+ 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });seed(root, budget);}function subtreeChanged(record, el) {@@ -200,7 +202,7 @@let changed = false, source = 'page', target = '';const changes = [...records];for (const record of records) {- if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;+ if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;const descendants = new Set([...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),@@ -247,8 +249,8 @@pendingDOM = { kind: 'dom', source, target };domTimer = setTimeout(() => { domTimer = null; const observation = pendingDOM; pendingDOM = null; send(observation); }, 150);}- function listen(target, name, handler) {- target.addEventListener(name, handler, { capture: true, passive: true });+ function listen(target, name, handler, passive = true) {+ target.addEventListener(name, handler, { capture: true, passive });listeners.push(() => target.removeEventListener(name, handler, true));}function start(state) {@@ -274,6 +276,18 @@setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);} catch { /* Only a current prepared operation can mark a DOM write. */ }});+ listen(document, 'webbrain-agent-scroll-dispatch', event => {+ try {+ const guard = JSON.parse(String(event.detail));+ if (guard.documentToken !== documentToken || guard.revision !== revision || !operations.has(guard.operationId)+ || domTimer || unreported) { event.preventDefault(); return; }+ const el = elementFor(event);+ dispatch({ operationId: guard.operationId, kind: 'scroll', scrollIntoView: true, runToken });+ const op = operations.get(guard.operationId);+ op.target = el; op.scrollAncestors.clear();+ for (let node = el?.parentElement || el?.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) op.scrollAncestors.add(node);+ } catch { event.preventDefault(); }+ }, false);const noteAgentTurn = event => {const el = elementFor(event);const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);@@ -339,6 +353,21 @@const el = elementFor(event);if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);});+ const viewport = () => ({ width: innerWidth, height: innerHeight,+ visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });+ lastViewport = JSON.stringify(viewport());+ const onResize = () => {+ const current = viewport(), key = JSON.stringify(current);+ if (!active || key === lastViewport) return;+ lastViewport = key; revision++;+ send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });+ };+ listen(window, 'resize', onResize);+ if (window.visualViewport) {+ listen(window.visualViewport, 'resize', onResize);+ listen(window.visualViewport, 'scroll', onResize);+ }listen(document, 'scroll', event => {const el = event.target === document ? document.documentElement : event.target;if (ignored(el) || expected('scroll', el)) return;@@ -366,7 +395,7 @@}function stop() {active = false; runToken = ''; pointerHeld = false; composing = false;- lastUserAt = 0; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {el.removeAttribute?.('data-webbrain-native-action');diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex 34595e0fb..d0d5c4c2c 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -144,7 +144,7 @@ export const pageFeedbackMethods = {}const seq = Number(feedback.seq);if (!Number.isSafeInteger(seq) || seq <= frame.seq) return { accepted: false, reason: 'duplicate-observation' };- if (!['click', 'input', 'selection', 'scroll', 'dom', 'activity'].includes(feedback.kind)+ if (!['click', 'input', 'selection', 'scroll', 'resize', 'dom', 'activity'].includes(feedback.kind)|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };frame.seq = seq;if (feedback.source === 'agent') {@@ -156,6 +156,8 @@ export const pageFeedbackMethods = {const item = { kind: feedback.kind, source: feedback.source, frameId,revision: Math.max(0, Number(feedback.revision) || 0), target: clean(feedback.target, 180) };if (feedback.kind === 'scroll') item.viewport = { x: Number(feedback.viewport?.x) || 0, y: Number(feedback.viewport?.y) || 0 };+ if (feedback.kind === 'resize') item.viewport = Object.fromEntries(['width', 'height', 'visualWidth', 'visualHeight', 'scale', 'offsetX', 'offsetY']+ .map(key => [key, Number.isFinite(feedback.viewport?.[key]) ? feedback.viewport[key] : 0]));if (feedback.source === 'user') {// DOM callbacks describe consequences; only physical activity extends the idle gate.if (feedback.kind !== 'dom') run.lastUserAt = Date.now();@@ -164,7 +166,8 @@ export const pageFeedbackMethods = {// A physical interaction supersedes an expected agent navigation.run.navigation = null;}- if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll')) run.lastActivityAt = Date.now();+ if (feedback.kind !== 'dom' && (feedback.source === 'user' || feedback.kind === 'scroll'+ || (feedback.kind === 'resize' && feedback.source !== 'page'))) run.lastActivityAt = Date.now();this._queuePageFeedback(tabId, item);return { accepted: true };},diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex e24812b5a..eb4be100c 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -12,6 +12,7 @@let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;let pointerHeld = false, composing = false, localOperation = null;let requestGeneration = 0, unreported = 0, pendingDOM = null;+ let lastViewport = '';let lastFeedbackDelivery = Promise.resolve();let agentTurn = null, userTurn = null, lastUserTarget = null;let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();@@ -154,6 +155,7 @@.map(node => node.textContent).join('').slice(0, 200);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),+ el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);}@@ -170,7 +172,7 @@roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',- 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'class', 'style'] });+ 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });seed(root, budget);}function subtreeChanged(record, el) {@@ -200,7 +202,7 @@let changed = false, source = 'page', target = '';const changes = [...records];for (const record of records) {- if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;+ if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;const descendants = new Set([...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),@@ -247,8 +249,8 @@pendingDOM = { kind: 'dom', source, target };domTimer = setTimeout(() => { domTimer = null; const observation = pendingDOM; pendingDOM = null; send(observation); }, 150);}- function listen(target, name, handler) {- target.addEventListener(name, handler, { capture: true, passive: true });+ function listen(target, name, handler, passive = true) {+ target.addEventListener(name, handler, { capture: true, passive });listeners.push(() => target.removeEventListener(name, handler, true));}function start(state) {@@ -274,6 +276,18 @@setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);} catch { /* Only a current prepared operation can mark a DOM write. */ }});+ listen(document, 'webbrain-agent-scroll-dispatch', event => {+ try {+ const guard = JSON.parse(String(event.detail));+ if (guard.documentToken !== documentToken || guard.revision !== revision || !operations.has(guard.operationId)+ || domTimer || unreported) { event.preventDefault(); return; }+ const el = elementFor(event);+ dispatch({ operationId: guard.operationId, kind: 'scroll', scrollIntoView: true, runToken });+ const op = operations.get(guard.operationId);+ op.target = el; op.scrollAncestors.clear();+ for (let node = el?.parentElement || el?.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) op.scrollAncestors.add(node);+ } catch { event.preventDefault(); }+ }, false);const noteAgentTurn = event => {const el = elementFor(event);const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);@@ -339,6 +353,21 @@const el = elementFor(event);if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);});+ const viewport = () => ({ width: innerWidth, height: innerHeight,+ visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });+ lastViewport = JSON.stringify(viewport());+ const onResize = () => {+ const current = viewport(), key = JSON.stringify(current);+ if (!active || key === lastViewport) return;+ lastViewport = key; revision++;+ send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });+ };+ listen(window, 'resize', onResize);+ if (window.visualViewport) {+ listen(window.visualViewport, 'resize', onResize);+ listen(window.visualViewport, 'scroll', onResize);+ }listen(document, 'scroll', event => {const el = event.target === document ? document.documentElement : event.target;if (ignored(el) || expected('scroll', el)) return;@@ -366,7 +395,7 @@}function stop() {active = false; runToken = ''; pointerHeld = false; composing = false;- lastUserAt = 0; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {el.removeAttribute?.('data-webbrain-native-action');diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 4f9879522..7d8176be3 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -146,6 +146,25 @@ for (const build of ['chrome', 'firefox']) {} finally { agent._releaseRunEntry(tab); }});+ test(`${build}: viewport resize invalidates coordinates and waits for geometry to settle`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ await agent._claimRunEntry(tab, 'interactive');+ try {+ const binding = bind(agent, tab);+ agent.screenshotCaptures.set(tab, { captureId: 'stale-coordinates' });+ agent._jevSessions = new Map([[tab, { disabled: false, queue: ['prepared'], snapshot: {} }]]);+ assert.equal(binding.send({ kind: 'resize', source: 'unknown', target: 'viewport', viewport: { width: 900, height: 600, scale: 1 } }).accepted, true);+ assert.equal(agent.screenshotCaptures.has(tab), false);+ assert.equal(agent._jevSessions.get(tab).disabled, true);+ assert.deepEqual(agent._jevSessions.get(tab).queue, []);+ assert.equal(agent._pageFeedbackRuns.get(tab).lastUserAt, 0, 'Resize alone does not prove physical user input');+ assert.ok(agent._pageFeedbackRuns.get(tab).lastActivityAt > 0, 'Unknown geometry activity must settle before replanning');+ agent._pageFeedbackRuns.get(tab).lastActivityAt = 0;+ binding.send({ kind: 'resize', source: 'page', target: 'viewport', viewport: { width: 500, height: 200 } });+ assert.equal(agent._pageFeedbackRuns.get(tab).lastActivityAt, 0, 'Page layout changes to embedded frame dimensions must not extend the user idle gate');+ } finally { agent._releaseRunEntry(tab); }+ });+test(`${build}: navigation arriving during a page read gets its own feedback ID`, async () => {const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();const updates = [], messages = [];diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex d92a49c85..7a7521d9f 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -65,6 +65,96 @@ async function installContentEditableFallback(page, build) {});}+test('Chrome selector scrolling claims input only at the actual page mutation', { timeout: 30000 }, async () => {+ const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser, tab = 912;+ const host = Object.assign({ isRunning: () => true, _checkAbort: () => false }, pageFeedbackMethods);+ let browser, page, session, pauseMethod = '', entered, resume;+ const registrations = [];+ const api = { runtime: {}, tabs: {+ get: async () => ({ url: 'https://monitor.test/start' }),+ sendMessage: async (_tab, msg) => {+ if (!page) return {};+ if (msg.action === 'page_monitor_dispatch') registrations.push(msg.params);+ return page.evaluate(msg => new Promise(resolve => messageListeners.forEach(fn => fn(msg, {}, resolve))), msg);+ },+ }, debugger: { sendCommand(_source, method, params, callback) {+ void (async () => {+ if (method === pauseMethod) {+ pauseMethod = ''; entered(); await new Promise(resolve => { resume = resolve; });+ }+ return await session.send(method, params);+ })().then(callback, error => { api.runtime.lastError = { message: error.message }; callback(); delete api.runtime.lastError; });+ } } };+ globalThis.chrome = api; delete globalThis.browser;+ try {+ await host._beginPageFeedbackRun(tab, 'interactive');+ ({ browser, page } = await fixture(chromium, 'chrome', { runToken: host._pageFeedbackRuns.get(tab).token }));+ const documentToken = await page.evaluate(() => document.documentElement.getAttribute('data-webbrain-page-revision').split(':')[0]);+ host.pageMonitorState({ tab: { id: tab }, frameId: 0 }, documentToken);+ await page.exposeFunction('feedbackToHost', feedback => host.observePageFeedback({ tab: { id: tab }, frameId: 0 }, feedback));+ await page.exposeFunction('monitorStateFromHost', token => host.pageMonitorState({ tab: { id: tab }, frameId: 0 }, token));+ await page.evaluate(() => {+ const original = chrome.runtime.sendMessage;+ chrome.runtime.sendMessage = async message => {+ if (message.action === 'get_page_monitor_state') return await monitorStateFromHost(message.documentToken);+ const response = await original(message);+ return message.action === 'page_feedback' ? await feedbackToHost(message.feedback) : response;+ };+ });+ session = await page.context().newCDPSession(page);+ const client = new CDPClient(); client.sessions.set(tab, { attached: true });+ const reset = async () => {+ host._finishPageFeedbackRun(tab);+ await page.waitForTimeout(20);+ await page.evaluate(() => window.scrollTo(0, 0));+ await page.waitForTimeout(100);+ await host._beginPageFeedbackRun(tab, 'interactive');+ registrations.length = 0;+ await page.evaluate(() => { feedback = []; });+ };+ for (const scenario of [+ { method: 'Runtime.enable', selector: '#agent', options: { scroll: false, retries: 0 }, readOnly: true },+ { method: 'DOM.getDocument', selector: '#missing', options: { retries: 1, delayMs: 0 } },+ { method: 'Runtime.evaluate', selector: '#agent', options: { retries: 0 } },+ ]) {+ await reset(); pauseMethod = scenario.method;+ const paused = new Promise(resolve => { entered = resolve; });+ const resolving = client.resolveSelector(tab, scenario.selector, scenario.options)+ .then(value => ({ value }), error => ({ code: error.code }));+ await paused;+ await page.evaluate(() => window.scrollTo(0, 600));+ await page.waitForFunction(() => feedback.some(event => event.kind === 'scroll'), null, { timeout: 1000 });+ assert.ok(registrations.every(mark => mark.fenceOnly === true), 'A resolver cannot claim scroll while a DOM search is pending');+ resume(); resume = null;+ const result = await resolving;+ if (scenario.readOnly) {+ assert.equal(result.value.found, true); assert.equal(registrations.length, 0);+ } else assert.equal(result.code, 'page_feedback_pending', `${scenario.method} must preserve the external intervention`);+ assert.equal(await page.evaluate(() => scrollY), 600, 'A stale resolver must not scroll back to its target');+ }+ await reset();+ await page.evaluate(() => {+ for (const mode of ['open', 'closed']) {+ const el = document.createElement('div');+ el.attachShadow({ mode }).innerHTML = `<button id="deep-${mode}">Deep target</button>`;+ document.body.append(el);+ }+ });+ await page.waitForTimeout(200);+ for (const mode of ['open', 'closed']) {+ await reset();+ const result = await client.resolveSelector(tab, `#deep-${mode}`, { retries: 0 });+ assert.equal(result.found, true); assert.equal(result.inViewport, true);+ await page.waitForTimeout(200);+ assert.ok(await page.evaluate(() => scrollY > 1000), `${mode} target must actually scroll into view`);+ assert.deepEqual(await page.evaluate(() => feedback), [], `${mode} agent scroll must not trigger a feedback loop`);+ }+ } finally {+ resume?.(); host._finishPageFeedbackRun(tab);+ await browser?.close(); globalThis.chrome = savedChrome; globalThis.browser = savedBrowser;+ }+});+for (const siteIsolation of [false, true]) {test(`Chrome CDP input reaches the receiving iframe monitor (site isolation: ${siteIsolation})`, { timeout: 30000 }, async () => {const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser;@@ -376,6 +466,69 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: existing dialogs, details and inert surfaces invalidate prepared actions`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const surfaces = document.createElement('div');+ surfaces.innerHTML = '<dialog id="modal"><button>Modal target</button></dialog><details id="details" style="height:100px"><summary>Summary</summary><button>Existing details target</button></details><div id="inert-surface"><button>Existing inert target</button></div>';+ document.body.prepend(surfaces);+ });+ await page.waitForTimeout(200);+ for (const state of ['modal-open', 'modal-close', 'details-open', 'details-close', 'inert-on', 'inert-off']) {+ const response = await page.evaluate(async state => {+ feedback = []; deliver('page_monitor_prepare', { operationId: state, tool: 'click', selector: '#agent' });+ if (state === 'modal-open') document.getElementById('modal').showModal();+ if (state === 'modal-close') document.getElementById('modal').close();+ if (state.startsWith('details')) document.getElementById('details').open = state.endsWith('open');+ if (state.startsWith('inert')) document.getElementById('inert-surface').inert = state.endsWith('on');+ await Promise.resolve();+ deliver('page_monitor_dispatch', { operationId: state, kind: 'click', selector: '#agent' });+ await new Promise(resolve => setTimeout(resolve, 0));+ return lastMonitorResponse;+ }, state);+ assert.equal(response.pageFeedbackPending, true, `${state} must invalidate the prepared click`);+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom'), `${state} must refresh page context`);+ }+ } finally { await browser.close(); }+ });++ test(`${build}: viewport resize fences prepared coordinates and stops with the run`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => { document.getElementById('agent').style.cssText = 'position:absolute;left:50vw;top:0'; });+ await page.waitForTimeout(200);+ const before = await page.locator('#agent').boundingBox();+ await page.evaluate(() => { feedback = []; deliver('page_monitor_prepare', { operationId: 'resize-click', tool: 'click', selector: '#agent' }); });+ await page.setViewportSize({ width: 900, height: 600 });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'resize'), null, { timeout: 1000 });+ const after = await page.locator('#agent').boundingBox();+ assert.notEqual(before.x, after.x);+ const response = await page.evaluate(async () => {+ deliver('page_monitor_dispatch', { operationId: 'resize-click', kind: 'click', x: 650, y: 20 });+ await new Promise(resolve => setTimeout(resolve, 0)); return lastMonitorResponse;+ });+ assert.equal(response.pageFeedbackPending, true);+ const observation = await page.evaluate(() => feedback.find(event => event.kind === 'resize'));+ assert.equal(observation.source, 'unknown'); assert.equal(observation.viewport.width, 900);+ const frameNavigation = page.waitForEvent('framenavigated', { predicate: frame => frame.url() === 'https://monitor.test/frame' });+ await page.evaluate(() => {+ const frame = document.createElement('iframe'); frame.src = 'https://monitor.test/frame';+ frame.style.cssText = 'width:400px;height:250px'; document.body.prepend(frame);+ });+ const child = await frameNavigation;+ await child.waitForFunction(() => window.__wbPageMonitor && document.documentElement.hasAttribute('data-webbrain-page-revision'));+ await child.evaluate(() => { feedback = []; });+ await page.evaluate(() => { document.querySelector('iframe').style.width = '450px'; });+ await child.waitForFunction(() => feedback.some(event => event.kind === 'resize'), null, { timeout: 1000 });+ assert.equal(await child.evaluate(() => feedback.find(event => event.kind === 'resize').source), 'page', 'Embedded frame dimensions are page layout changes');+ await page.evaluate(() => { monitorEnabled = false; deliver('page_monitor_state'); feedback = []; });+ await page.setViewportSize({ width: 1000, height: 700 });+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Stop removes viewport listeners');+ } finally { await browser.close(); }+ });+test(`${build}: contenteditable fallback attributes its cancellable gate and native edits`, async () => {const { browser, page } = await fixture(engine, build);try {diff --git a/test/run.js b/test/run.jsindex 6c1336632..1ba3d03bc 100644--- a/test/run.js+++ b/test/run.js@@ -58006,7 +58006,7 @@ test('CDP selector resolution rejects queued open- and closed-shadow scrolls aftif (method === 'DOM.resolveNode') return { object: { objectId: 'late-closed-shadow-target' } };if (method === 'Runtime.callFunctionOn') {assert.match(params.functionDeclaration, /deadlineExpired[\s\S]*scrollIntoView/);- assert.deepEqual(params.arguments, [{ value: deadlineAt }]);+ assert.deepEqual(params.arguments, [{ value: deadlineAt }, { value: null }]);return { result: { value: { scrolled: false, deadlineExpired: true } } };}return {};