Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: attribute preparation scrolls and recover empty Firefox frames

Barack Sokullu00f3472d18cf
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.md
index 1aa723fe0..0b1f69d22 100644
--- a/docs/page-feedback-monitor.md
+++ b/docs/page-feedback-monitor.md
@@ -28,6 +28,9 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behavior
acknowledgement; cleanup sends the owning run token to all frames. Restricted
frames do not prevent a run, and frame enumeration failures retain broadcast
delivery.
+ Firefox also registers the monitor and MAIN-world shadow hook at document end
+ to cover inherited `about:blank`/`about:srcdoc` frames skipped at document start.
+ This supplemental pass preserves an existing live monitor and its expectations.
- Native CDP/BiDi and content actions register their target, input operation and
actual dispatch before changing the page. Target and native-event phase
matching suppress their observed effects. CDP resolves the receiving document
@@ -41,6 +44,11 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behavior
asynchronous search or retries. Only a resolved node's actual `scrollIntoView`
dispatch activates attribution, with a live page revision check in both open
and closed shadow-root paths. Read-only selector queries do not arm scrolling.
+- AX rect/field preparation and content/BiDi visibility helpers register the
+ exact node immediately before each actual scroll. Multiple scrolled ancestors
+ remain attributed within the operation; preparation scrolling never claims
+ input events on the field. Pending feedback is propagated through scroll
+ fallbacks rather than permitting an extra viewport mutation.
- Meaningful visible content, control state, visibility and geometry changes are
coalesced. Visible subtree identity changes invalidate targets even when the
replacement has identical text and geometry. CSS animation and extension
@@ -121,5 +129,10 @@ target traversal and queued evaluation, then verify external scroll feedback and
the final dispatch fence. Open and closed shadow-root agent scrolling remain
attributed. Geometry fixtures also cover modal/details/inert transitions and
viewport resizing during coordinate preparation and after cancellation.
+Preparation-scroll fixtures cover the real AX handlers, nested scroll containers,
+same-field human typing and an intervening DOM change. Firefox frame fixtures
+model the document-start omission, apply the manifest's document-end scripts,
+and verify user/DOM/late-shadow feedback in blank and srcdoc frames while keeping
+an existing monitor intact. These remain mocked extension registration fixtures.
The monitor regressions are also included in `npm run test:runtime-lifecycle`.
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 910a6ec51..5af8580d2 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -425,15 +425,20 @@
const fullyVisible = _isFullyVisibleForInteraction(el, insets);
const coveredByFixed = fullyVisible && _isCoveredByFixedNonModalSurface(el, view);
if (!fullyVisible || coveredByFixed) {
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
const rAfter = el.getBoundingClientRect();
if (rAfter.top < insets.top) {
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
view.scrollBy({ top: rAfter.top - insets.top - 16, behavior: 'instant' });
} else if (rAfter.bottom > view.innerHeight - insets.bottom) {
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
view.scrollBy({ top: rAfter.bottom - (view.innerHeight - insets.bottom) + 16, behavior: 'instant' });
}
}
- } catch {
+ } catch (error) {
+ if (error?.code === 'page_feedback_pending') throw error;
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
}
}
@@ -7146,7 +7151,7 @@
);
}
if (!_isFullyVisibleForInteraction(el)) {
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
}
try { el.focus({ preventScroll: true }); } catch {}
const rect = el.getBoundingClientRect();
@@ -7534,7 +7539,7 @@
return failure(`set_checked only supports native input[type="checkbox"] controls; ${ref_id} resolved to ${tag || 'unknown'}${inputType ? `[type="${inputType}"]` : ''}.`);
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -7684,7 +7689,7 @@
return failure(`ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids β€” do NOT guess ref numbers or invent placeholders.`, { suggestions });
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -7874,7 +7879,7 @@
return failure(`ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids β€” do NOT guess ref numbers or invent placeholders.`, { suggestions });
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -8129,6 +8134,7 @@
if (!el || !el.isConnected) return { success: false, error: `ref_id ${ref_id} is stale` };
const typeable = el.isContentEditable || el.tagName === 'INPUT' || el.tagName === 'TEXTAREA';
if (!typeable) return { success: false, error: `ref_id ${ref_id} is not a text field` };
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
try { el.focus({ preventScroll: true }); } catch {}
if (el.isContentEditable) {
@@ -8317,7 +8323,7 @@
: '';
return { success: false, error: `ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids.`, suggestions };
}
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
showAgentWorkingTarget(el, 'ax_resolve_rect');
let r = el.getBoundingClientRect();
const cx = r.left + r.width / 2;
@@ -8467,7 +8473,9 @@
// BOTH rects against that frame is what drag_drop wants.
// Source may end up partly off-screen if the two are far
// apart vertically β€” flagged via inViewport on the return.
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: fromEl });
try { fromEl.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: toEl });
try { toEl.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
showAgentWorkingTarget(toEl, 'ax_resolve_two_rects');
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index eb4be100c..7b3b6e4b5 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -1,5 +1,9 @@
/** Value-free, run-scoped page monitoring. Keep the Firefox copy byte-identical. */
(() => {
+ if (window.__wbPageMonitorRecoveryOnly) {
+ delete window.__wbPageMonitorRecoveryOnly;
+ if (window.__wbPageMonitor && !window.__wbPageMonitor.disposed) return;
+ }
window.__wbPageMonitor?.dispose?.();
const api = globalThis.browser || globalThis.chrome;
const documentToken = crypto.randomUUID();
@@ -74,7 +78,7 @@
send({ kind, source: 'user', target: targetName(el), interacting: pointerHeld || composing, ...extra });
};
const resolveTarget = params => {
- let el = params.ref_id ? window.__wb_ax_lookup?.(params.ref_id) : null;
+ let el = params.element instanceof Element ? params.element : params.ref_id ? window.__wb_ax_lookup?.(params.ref_id) : null;
if (params.nativeTarget) {
const find = root => {
const marked = root.querySelector(`[data-webbrain-native-target="${CSS.escape(params.nativeTarget)}"]`);
@@ -432,9 +436,10 @@
op.focused = params.kind === 'input'; op.kind = params.kind;
op.nativeWheel = params.nativeWheel === true;
if (!params.release) op.navigationCandidate = params.kind === 'click' && params.navigationCandidate !== false;
- op.scrollAncestors = new Set();
+ op.scrollAncestors = params.addScrollTarget ? op.scrollAncestors || new Set() : new Set();
if (params.scrollIntoView || params.nativeWheel || ['input', 'click'].includes(params.kind)) {
- for (let node = op.target?.parentElement; node; node = node.parentElement) op.scrollAncestors.add(node);
+ if (params.addScrollTarget && op.target) op.scrollAncestors.add(op.target);
+ for (let node = op.target?.parentElement || op.target?.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) op.scrollAncestors.add(node);
}
op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);
op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);
@@ -461,7 +466,7 @@
localOperation = previous;
};
}
- function beforeLocalDispatch({ preparation = false } = {}) {
+ function beforeLocalDispatch({ preparation = false, kind, target } = {}) {
if (!active || !localOperation) return;
if (domTimer || unreported || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
@@ -472,7 +477,10 @@
const marker = { userAt: lastUserAt }; agentTurn = marker;
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
}
- dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate,
+ dispatch({ ...localOperation, kind: kind || localOperation.kind, element: target,
+ scrollIntoView: localOperation.scrollIntoView || (kind === 'scroll' && !!target),
+ addScrollTarget: kind === 'scroll' && !!target,
+ navigationCandidate: !preparation && localOperation.navigationCandidate,
fenceOnly: preparation, runToken });
}
function withLocalDispatch(callback) {
@@ -518,6 +526,7 @@
};
api.runtime.onMessage.addListener(onMessage);
window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, registerDecoration,
+ get disposed() { return disposed; },
dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };
void requestState();
// A document restored from BFCache needs a fresh run/document handshake.
diff --git a/src/firefox/manifest.json b/src/firefox/manifest.json
index ec73e04b9..669ca6907 100644
--- a/src/firefox/manifest.json
+++ b/src/firefox/manifest.json
@@ -51,6 +51,21 @@
"all_frames": true,
"match_about_blank": true
},
+ {
+ "matches": ["<all_urls>"],
+ "js": ["src/content/page-monitor-recovery.js", "src/content/page-monitor.js"],
+ "run_at": "document_end",
+ "all_frames": true,
+ "match_about_blank": true
+ },
+ {
+ "matches": ["<all_urls>"],
+ "js": ["src/content/page-monitor-shadow.js"],
+ "run_at": "document_end",
+ "world": "MAIN",
+ "all_frames": true,
+ "match_about_blank": true
+ },
{
"matches": ["<all_urls>"],
"js": ["src/content/file-picker-guard-loader.js"],
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index c8baed163..b233b728f 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -684,15 +684,20 @@
const fullyVisible = _isFullyVisibleForInteraction(el, insets);
const coveredByFixed = fullyVisible && _isCoveredByFixedNonModalSurface(el, view);
if (!fullyVisible || coveredByFixed) {
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
const rAfter = el.getBoundingClientRect();
if (rAfter.top < insets.top) {
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
view.scrollBy({ top: rAfter.top - insets.top - 16, behavior: 'instant' });
} else if (rAfter.bottom > view.innerHeight - insets.bottom) {
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
view.scrollBy({ top: rAfter.bottom - (view.innerHeight - insets.bottom) + 16, behavior: 'instant' });
}
}
- } catch {
+ } catch (error) {
+ if (error?.code === 'page_feedback_pending') throw error;
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: el });
try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
}
}
@@ -6574,7 +6579,7 @@
return failure(`set_checked only supports native input[type="checkbox"] controls; ${ref_id} resolved to ${tag || 'unknown'}${inputType ? `[type="${inputType}"]` : ''}.`);
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -6694,7 +6699,7 @@
return failure(`ref_id ${ref_id} not found. Re-read the accessibility tree to get fresh ids.`, { suggestions });
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -6872,7 +6877,7 @@
const el = window.__wb_ax_lookup(ref_id);
if (!el) return failure(`ref_id ${ref_id} not found. Re-read the accessibility tree.`);
if (actionDeadlineExpired()) return deadlineFailure();
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -7263,7 +7268,7 @@
return { success: false, error: `ref_id ${ref_id} not found.`, suggestions };
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { _scrollElementIntoClearView(el); } catch {}
+ try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
const r = el.getBoundingClientRect();
const cx = r.left + r.width / 2;
diff --git a/src/firefox/src/content/page-monitor-recovery.js b/src/firefox/src/content/page-monitor-recovery.js
new file mode 100644
index 000000000..2a918d823
--- /dev/null
+++ b/src/firefox/src/content/page-monitor-recovery.js
@@ -0,0 +1,2 @@
+/** The following monitor script only fills Firefox's empty-frame startup gap. */
+window.__wbPageMonitorRecoveryOnly = true;
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index eb4be100c..7b3b6e4b5 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -1,5 +1,9 @@
/** Value-free, run-scoped page monitoring. Keep the Firefox copy byte-identical. */
(() => {
+ if (window.__wbPageMonitorRecoveryOnly) {
+ delete window.__wbPageMonitorRecoveryOnly;
+ if (window.__wbPageMonitor && !window.__wbPageMonitor.disposed) return;
+ }
window.__wbPageMonitor?.dispose?.();
const api = globalThis.browser || globalThis.chrome;
const documentToken = crypto.randomUUID();
@@ -74,7 +78,7 @@
send({ kind, source: 'user', target: targetName(el), interacting: pointerHeld || composing, ...extra });
};
const resolveTarget = params => {
- let el = params.ref_id ? window.__wb_ax_lookup?.(params.ref_id) : null;
+ let el = params.element instanceof Element ? params.element : params.ref_id ? window.__wb_ax_lookup?.(params.ref_id) : null;
if (params.nativeTarget) {
const find = root => {
const marked = root.querySelector(`[data-webbrain-native-target="${CSS.escape(params.nativeTarget)}"]`);
@@ -432,9 +436,10 @@
op.focused = params.kind === 'input'; op.kind = params.kind;
op.nativeWheel = params.nativeWheel === true;
if (!params.release) op.navigationCandidate = params.kind === 'click' && params.navigationCandidate !== false;
- op.scrollAncestors = new Set();
+ op.scrollAncestors = params.addScrollTarget ? op.scrollAncestors || new Set() : new Set();
if (params.scrollIntoView || params.nativeWheel || ['input', 'click'].includes(params.kind)) {
- for (let node = op.target?.parentElement; node; node = node.parentElement) op.scrollAncestors.add(node);
+ if (params.addScrollTarget && op.target) op.scrollAncestors.add(op.target);
+ for (let node = op.target?.parentElement || op.target?.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) op.scrollAncestors.add(node);
}
op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);
op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);
@@ -461,7 +466,7 @@
localOperation = previous;
};
}
- function beforeLocalDispatch({ preparation = false } = {}) {
+ function beforeLocalDispatch({ preparation = false, kind, target } = {}) {
if (!active || !localOperation) return;
if (domTimer || unreported || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
@@ -472,7 +477,10 @@
const marker = { userAt: lastUserAt }; agentTurn = marker;
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
}
- dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate,
+ dispatch({ ...localOperation, kind: kind || localOperation.kind, element: target,
+ scrollIntoView: localOperation.scrollIntoView || (kind === 'scroll' && !!target),
+ addScrollTarget: kind === 'scroll' && !!target,
+ navigationCandidate: !preparation && localOperation.navigationCandidate,
fenceOnly: preparation, runToken });
}
function withLocalDispatch(callback) {
@@ -518,6 +526,7 @@
};
api.runtime.onMessage.addListener(onMessage);
window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, registerDecoration,
+ get disposed() { return disposed; },
dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };
void requestState();
// A document restored from BFCache needs a fresh run/document handshake.
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 7d8176be3..593c391f9 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -592,6 +592,16 @@ test('shared page monitoring modules stay identical between browser builds', ()
}
});
+test('Firefox supplements empty-frame monitoring after document start', () => {
+ const manifest = JSON.parse(fs.readFileSync(new URL('../src/firefox/manifest.json', import.meta.url), 'utf8'));
+ const monitor = manifest.content_scripts.find(entry => entry.run_at === 'document_end' && entry.js.includes('src/content/page-monitor.js'));
+ assert.ok(monitor); assert.equal(monitor.all_frames, true); assert.equal(monitor.match_about_blank, true);
+ assert.deepEqual(monitor.js, ['src/content/page-monitor-recovery.js', 'src/content/page-monitor.js']);
+ const shadow = manifest.content_scripts.find(entry => entry.run_at === 'document_end' && entry.world === 'MAIN'
+ && entry.js.includes('src/content/page-monitor-shadow.js'));
+ assert.ok(shadow); assert.equal(shadow.all_frames, true); assert.equal(shadow.match_about_blank, true);
+});
+
test('both manifests signal shadow attachment from MAIN in all monitored frames', () => {
for (const build of ['chrome', 'firefox']) {
const manifest = JSON.parse(fs.readFileSync(new URL(`../src/${build}/manifest.json`, import.meta.url), 'utf8'));
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 7a7521d9f..95983d587 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -17,7 +17,7 @@ body { margin: 0; } button,input { margin: 12px; }
<div id="moving">Animated</div><div style="height:3000px"></div>
<script>document.getElementById('agent').addEventListener('click', () => { document.getElementById('status').textContent = 'Agent changed this'; });</script>`;
-async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false } = {}) {
+async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false, omitEmptyFrameMonitor = false } = {}) {
const browser = await engine.launch({ headless: true, ...(siteIsolation ? { args: ['--site-per-process'] } : {}) });
const context = await browser.newContext();
await context.route('https://monitor.test/**', route => route.fulfill({ contentType: 'text/html', body: html }));
@@ -39,8 +39,10 @@ async function fixture(engine, build, { runToken = 'test-run', siteIsolation = f
params: { runToken: monitorRunToken, ...params }, active: action === 'page_monitor_state' ? monitorEnabled : undefined,
runToken: monitorRunToken }, {}, value => { window.lastMonitorResponse = value; }));
});
- await context.addInitScript({ content: read(build, 'content/page-monitor-shadow.js') });
- await context.addInitScript({ content: read(build, 'content/page-monitor.js') });
+ const initialScript = file => omitEmptyFrameMonitor
+ ? `if (window === window.top) { ${read(build, file)} }` : read(build, file);
+ await context.addInitScript({ content: initialScript('content/page-monitor-shadow.js') });
+ await context.addInitScript({ content: initialScript('content/page-monitor.js') });
const page = await context.newPage();
await page.goto('https://monitor.test/start');
await page.waitForTimeout(200);
@@ -48,6 +50,69 @@ async function fixture(engine, build, { runToken = 'test-run', siteIsolation = f
return { browser, context, page };
}
+async function installScrollHelper(page, build) {
+ const source = read(build, 'content/content.js');
+ const helper = source.match(/^ function _scrollElementIntoClearView\([\s\S]*?^ }/m)?.[0];
+ assert.ok(helper);
+ await page.addScriptTag({ content: `window._scrollElementIntoClearView = ${helper};` });
+ await page.evaluate(() => Object.assign(window, {
+ _isAlreadyVisibleInFixedSurface: () => false, _getViewportDockedInsets: () => ({ top: 0, bottom: 0 }),
+ _isFullyVisibleForInteraction: () => false, _isCoveredByFixedNonModalSurface: () => false,
+ showAgentWorkingTarget: () => {}, _fieldMeta: () => ({}),
+ _axFallbackStaticAssessment: () => ({ tag: 'button', role: 'button', name: 'Target' }),
+ }));
+}
+
+test('Chrome AX preparation attributes actual scrolling without claiming field input', async () => {
+ const { browser, page } = await fixture(chromium, 'chrome');
+ try {
+ await installScrollHelper(page, 'chrome');
+ const source = read('chrome', 'content/content.js');
+ const entries = ['ax_resolve_rect', 'ax_resolve_two_rects', 'ax_prepare_field_for_trusted_type'].map(action => {
+ const start = source.indexOf(`'${action}': () => {`), end = source.indexOf('\n },', start);
+ assert.ok(start > 0 && end > start); return source.slice(start, end + 8);
+ });
+ await page.addScriptTag({ content: `window.axHelpers = { ${entries.join(',\n')} };` });
+ await page.evaluate(() => {
+ for (let i = 1; i <= 3; i++) {
+ const box = document.createElement('div'); box.id = `scroll-box-${i}`;
+ box.style.cssText = 'overflow:auto;height:100px;width:300px';
+ box.innerHTML = `<${i === 3 ? 'input' : 'button'} id="ref_${i}" style="margin-top:900px">Target</${i === 3 ? 'input' : 'button'}>`;
+ document.body.append(box);
+ }
+ window.__wb_ax_lookup = ref => document.getElementById(ref);
+ });
+ await page.waitForTimeout(200);
+ for (const [action, params] of [
+ ['ax_resolve_rect', { ref_id: 'ref_1' }],
+ ['ax_resolve_two_rects', { fromRefId: 'ref_1', toRefId: 'ref_2' }],
+ ['ax_prepare_field_for_trusted_type', { ref_id: 'ref_3' }],
+ ]) {
+ await page.evaluate(() => {
+ monitorEnabled = false; deliver('page_monitor_state'); window.scrollTo(0, 0);
+ for (const el of document.querySelectorAll('[id^="scroll-box-"]')) el.scrollTop = 0;
+ });
+ await page.waitForTimeout(50);
+ await page.evaluate(() => { monitorEnabled = true; deliver('page_monitor_state'); });
+ await page.waitForTimeout(20);
+ const result = await page.evaluate(({ action, params }) => {
+ feedback = []; window.msg = { params };
+ const finish = __wbPageMonitor.beginContentAction(action, params);
+ try { return axHelpers[action](); } finally { finish(); }
+ }, { action, params });
+ assert.equal(result.success, true, JSON.stringify(result));
+ await page.waitForTimeout(80);
+ assert.deepEqual(await page.evaluate(() => feedback), [], `${action} must not block the native action after scroll settling`);
+ assert.ok(await page.evaluate(() => [...document.querySelectorAll('[id^="scroll-box-"]')].some(el => el.scrollTop > 0)));
+ if (action === 'ax_resolve_two_rects') assert.ok(await page.evaluate(() => [1, 2].every(i => document.getElementById(`scroll-box-${i}`).scrollTop > 0)));
+ if (action === 'ax_prepare_field_for_trusted_type') {
+ await page.keyboard.press('x');
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'input' && event.source === 'user'), null, { timeout: 1000 });
+ }
+ }
+ } finally { await browser.close(); }
+});
+
async function installContentEditableFallback(page, build) {
const insertion = read(build, 'content/content.js').match(/^ async function _insertContentEditableText\([\s\S]*?^ }/m)?.[0];
assert.ok(insertion, 'Exercise the real contenteditable fallback with the real monitor');
@@ -65,6 +130,52 @@ async function installContentEditableFallback(page, build) {
});
}
+test('Firefox late registration covers blank/srcdoc frames and preserves an existing monitor', async () => {
+ const { browser, page } = await fixture(firefox, 'firefox', { omitEmptyFrameMonitor: true });
+ try {
+ const manifest = JSON.parse(fs.readFileSync(new URL('../src/firefox/manifest.json', import.meta.url), 'utf8'));
+ const late = manifest.content_scripts.find(entry => entry.run_at !== 'document_start'
+ && entry.js.includes('src/content/page-monitor.js') && entry.all_frames && entry.match_about_blank);
+ const lateShadow = manifest.content_scripts.find(entry => entry.run_at !== 'document_start' && entry.world === 'MAIN'
+ && entry.js.includes('src/content/page-monitor-shadow.js') && entry.all_frames && entry.match_about_blank);
+ assert.ok(late && lateShadow, 'Empty Firefox frames need a later monitor and shadow-hook registration');
+ const recovery = late.js.map(file => read('firefox', file.replace(/^src\//, ''))).join('\n');
+ await page.evaluate(() => { window.earlyMonitor = __wbPageMonitor; window.earlyFence = document.documentElement.getAttribute('data-webbrain-page-revision'); });
+ await page.addScriptTag({ content: recovery });
+ assert.equal(await page.evaluate(() => __wbPageMonitor === earlyMonitor && document.documentElement.getAttribute('data-webbrain-page-revision') === earlyFence), true,
+ 'The supplemental pass must not replace a live monitor or its dispatch expectations');
+ await page.evaluate(() => {
+ const blank = document.createElement('iframe'); blank.id = 'blank-frame'; document.body.prepend(blank);
+ const srcdoc = document.createElement('iframe'); srcdoc.id = 'srcdoc-frame';
+ srcdoc.srcdoc = '<input id="inside"><p id="inside-status">Ready</p>'; document.body.prepend(srcdoc);
+ });
+ for (const [id, url] of [['blank-frame', 'about:blank'], ['srcdoc-frame', 'about:srcdoc']]) {
+ const handle = await page.locator(`#${id}`).elementHandle(), frame = await handle.contentFrame();
+ await frame.waitForFunction(url => location.href === url && window.chrome?.runtime && document.body, url);
+ assert.equal(await frame.evaluate(() => !!window.__wbPageMonitor), false, 'Model the Firefox document_start omission');
+ if (url === 'about:blank') await frame.evaluate(() => { document.body.innerHTML = '<input id="inside"><p id="inside-status">Ready</p>'; });
+ await frame.addScriptTag({ content: read('firefox', 'content/page-monitor-shadow.js') });
+ await frame.addScriptTag({ content: recovery });
+ await frame.waitForFunction(() => __wbPageMonitor && document.documentElement.hasAttribute('data-webbrain-page-revision'));
+ await frame.locator('#inside').fill('Private input');
+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'input' && event.source === 'user'));
+ assert.ok(!(JSON.stringify(await frame.evaluate(() => feedback))).includes('Private input'));
+ await frame.evaluate(() => { document.getElementById('inside-status').textContent = 'Page changed'; });
+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'dom'));
+ await frame.evaluate(() => {
+ const host = document.createElement('div'); host.id = 'late-frame-host'; document.body.append(host);
+ });
+ await frame.waitForTimeout(200);
+ await frame.evaluate(() => {
+ feedback = []; document.getElementById('late-frame-host').attachShadow({ mode: 'open' }).innerHTML = '<button id="late-frame-target">New target</button>';
+ });
+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'div#late-frame-host'), null, { timeout: 1000 });
+ await frame.evaluate(() => { feedback = []; document.getElementById('late-frame-host').shadowRoot.querySelector('button').textContent = 'Updated target'; });
+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button#late-frame-target'), null, { timeout: 1000 });
+ }
+ } finally { await browser.close(); }
+});
+
test('Chrome selector scrolling claims input only at the actual page mutation', { timeout: 30000 }, async () => {
const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser, tab = 912;
const host = Object.assign({ isRunning: () => true, _checkAbort: () => false }, pageFeedbackMethods);
@@ -466,6 +577,43 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: preparation scrolling preserves same-target input and rejects intervening DOM changes`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await installScrollHelper(page, build);
+ await page.evaluate(() => {
+ const box = document.createElement('div'); box.id = 'prep-scroll-box'; box.style.cssText = 'overflow:auto;height:100px;width:300px';
+ box.innerHTML = '<input id="prepared-field" style="margin-top:900px">'; document.body.append(box);
+ window.__wb_ax_lookup = ref => document.getElementById(ref);
+ });
+ await page.waitForTimeout(200);
+ await page.evaluate(() => {
+ feedback = [];
+ const finish = __wbPageMonitor.beginContentAction('type_ax', { ref_id: 'prepared-field', _bidiPrepare: { action: 'type' } });
+ try { _scrollElementIntoClearView(document.getElementById('prepared-field')); } finally { finish(); }
+ });
+ await page.waitForTimeout(80);
+ assert.ok(await page.evaluate(() => document.getElementById('prep-scroll-box').scrollTop > 0));
+ assert.deepEqual(await page.evaluate(() => feedback), []);
+ await page.evaluate(() => document.getElementById('prepared-field').focus({ preventScroll: true }));
+ await page.keyboard.press('x');
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'input' && event.source === 'user'), null, { timeout: 1000 });
+ await page.evaluate(() => { monitorEnabled = false; deliver('page_monitor_state'); document.getElementById('prep-scroll-box').scrollTop = 0; window.scrollTo(0, 0); });
+ await page.waitForTimeout(50);
+ await page.evaluate(() => { monitorEnabled = true; deliver('page_monitor_state'); });
+ await page.waitForTimeout(20);
+ const result = await page.evaluate(async () => {
+ const finish = __wbPageMonitor.beginContentAction('type_ax', { ref_id: 'prepared-field', _bidiPrepare: { action: 'type' } });
+ document.getElementById('status').textContent = 'Intervening page change'; await Promise.resolve();
+ try { _scrollElementIntoClearView(document.getElementById('prepared-field')); return { dispatched: true }; }
+ catch (error) { return { code: error.code, dispatched: error.dispatched }; }
+ finally { finish(); }
+ });
+ assert.deepEqual(result, { code: 'page_feedback_pending', dispatched: false });
+ assert.equal(await page.evaluate(() => document.getElementById('prep-scroll-box').scrollTop), 0, 'A rejected helper must not scroll through its fallback');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: existing dialogs, details and inert surfaces invalidate prepared actions`, async () => {
const { browser, page } = await fixture(engine, build);
try {