Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Validate actual click coordinates during relaxed Retry

Barack Sokullu15d8ab694e9f
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 6f9f255df..e02b06da5 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -1686,7 +1686,16 @@
return ['characterData', 'childList'].includes(record.type)
&& (node === binding.target || binding.target.contains(node));
}
- function relaxedActionability(target, { hit = false, input = false } = {}) {
+ function nativeReceiverBelongsTo(target, receiver) {
+ // Decorative descendants can receive their control's input; another
+ // activator inside that control cannot borrow the original certificate.
+ for (let node = receiver, depth = 0; node && depth < 256; node = bindingParent(node), depth++) {
+ if (node === target) return true;
+ if (node.matches(modelControlSelector + ',[tabindex]')) return false;
+ }
+ return false;
+ }
+ function relaxedActionability(target, { hit = false, input = false, point = null } = {}) {
if (!(target instanceof Element) || !target.isConnected || target.ownerDocument !== document || !visible(target)
|| !target.matches(modelControlSelector) || target.disabled === true || (input && target.readOnly === true)) return false;
for (let node = target; node; node = bindingParent(node)) {
@@ -1698,7 +1707,9 @@
const rect = target.getBoundingClientRect();
if (!(rect.width > 0 && rect.height > 0)) return false;
if (!hit) return true;
- const receiver = deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
+ const receiver = point ? deepElementFromPoint(point.x, point.y)
+ : deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
+ if (point) return nativeReceiverBelongsTo(target, receiver);
return !!receiver && (receiver === target || target.contains(receiver));
}
function modelBindingResult(params = {}) {
@@ -2068,10 +2079,17 @@
&& resolveTarget(params) !== op.preparedBinding.target) {
revokeRelaxedGrant(); return 'target_changed';
}
+ // Native markers bind a node, not the mouse point. Passive movement may
+ // leave that node intact while the captured command now hits another one.
+ const point = op?.preparedBinding?.relaxed && params.kind === 'click' && params.pointerPreparationOnly !== true
+ && (params.x !== undefined || params.y !== undefined) ? { x: params.x, y: params.y } : null;
+ if (point && (!Number.isFinite(point.x) || !Number.isFinite(point.y))) {
+ revokeRelaxedGrant(); return 'target_hit_changed';
+ }
if (op?.preparedBinding?.relaxed && ['click', 'input'].includes(params.kind)
&& params.pointerPreparationOnly !== true
- && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input' })) {
- revokeRelaxedGrant(); return 'target_not_actionable';
+ && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input', point })) {
+ revokeRelaxedGrant(); return point ? 'target_hit_changed' : 'target_not_actionable';
}
if (!params.nativeTarget || op?.preparedBinding?.model !== true) return '';
// Resolve the actual transport marker alone; a stale marker cannot fall
@@ -2080,14 +2098,7 @@
if (!receiver) return 'target_unresolved';
const target = op.tabFocusRebound === true && receiver === op.focusTarget
? op.focusTarget : op.preparedBinding.target;
- // Button text/SVG and editor paragraphs can receive their owner's input.
- // A separate activator, including one inside a shadow root, cannot.
- let node = receiver;
- for (let depth = 0; node && depth < 256; depth++, node = bindingParent(node)) {
- if (node === target) return '';
- if (node.matches(modelControlSelector + ',[tabindex]')) return 'target_changed';
- }
- return 'target_changed';
+ return nativeReceiverBelongsTo(target, receiver) ? '' : 'target_changed';
}
function rebasePassivePreparation(op, params = {}) {
const relaxed = op?.preparedBinding?.relaxed === true && currentRelaxedGrant(op.preparedBinding.grantToken);
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 6f9f255df..e02b06da5 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -1686,7 +1686,16 @@
return ['characterData', 'childList'].includes(record.type)
&& (node === binding.target || binding.target.contains(node));
}
- function relaxedActionability(target, { hit = false, input = false } = {}) {
+ function nativeReceiverBelongsTo(target, receiver) {
+ // Decorative descendants can receive their control's input; another
+ // activator inside that control cannot borrow the original certificate.
+ for (let node = receiver, depth = 0; node && depth < 256; node = bindingParent(node), depth++) {
+ if (node === target) return true;
+ if (node.matches(modelControlSelector + ',[tabindex]')) return false;
+ }
+ return false;
+ }
+ function relaxedActionability(target, { hit = false, input = false, point = null } = {}) {
if (!(target instanceof Element) || !target.isConnected || target.ownerDocument !== document || !visible(target)
|| !target.matches(modelControlSelector) || target.disabled === true || (input && target.readOnly === true)) return false;
for (let node = target; node; node = bindingParent(node)) {
@@ -1698,7 +1707,9 @@
const rect = target.getBoundingClientRect();
if (!(rect.width > 0 && rect.height > 0)) return false;
if (!hit) return true;
- const receiver = deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
+ const receiver = point ? deepElementFromPoint(point.x, point.y)
+ : deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
+ if (point) return nativeReceiverBelongsTo(target, receiver);
return !!receiver && (receiver === target || target.contains(receiver));
}
function modelBindingResult(params = {}) {
@@ -2068,10 +2079,17 @@
&& resolveTarget(params) !== op.preparedBinding.target) {
revokeRelaxedGrant(); return 'target_changed';
}
+ // Native markers bind a node, not the mouse point. Passive movement may
+ // leave that node intact while the captured command now hits another one.
+ const point = op?.preparedBinding?.relaxed && params.kind === 'click' && params.pointerPreparationOnly !== true
+ && (params.x !== undefined || params.y !== undefined) ? { x: params.x, y: params.y } : null;
+ if (point && (!Number.isFinite(point.x) || !Number.isFinite(point.y))) {
+ revokeRelaxedGrant(); return 'target_hit_changed';
+ }
if (op?.preparedBinding?.relaxed && ['click', 'input'].includes(params.kind)
&& params.pointerPreparationOnly !== true
- && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input' })) {
- revokeRelaxedGrant(); return 'target_not_actionable';
+ && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input', point })) {
+ revokeRelaxedGrant(); return point ? 'target_hit_changed' : 'target_not_actionable';
}
if (!params.nativeTarget || op?.preparedBinding?.model !== true) return '';
// Resolve the actual transport marker alone; a stale marker cannot fall
@@ -2080,14 +2098,7 @@
if (!receiver) return 'target_unresolved';
const target = op.tabFocusRebound === true && receiver === op.focusTarget
? op.focusTarget : op.preparedBinding.target;
- // Button text/SVG and editor paragraphs can receive their owner's input.
- // A separate activator, including one inside a shadow root, cannot.
- let node = receiver;
- for (let depth = 0; node && depth < 256; depth++, node = bindingParent(node)) {
- if (node === target) return '';
- if (node.matches(modelControlSelector + ',[tabindex]')) return 'target_changed';
- }
- return 'target_changed';
+ return nativeReceiverBelongsTo(target, receiver) ? '' : 'target_changed';
}
function rebasePassivePreparation(op, params = {}) {
const relaxed = op?.preparedBinding?.relaxed === true && currentRelaxedGrant(op.preparedBinding.grantToken);
diff --git a/test/page-monitor-relaxed-retry.mjs b/test/page-monitor-relaxed-retry.mjs
index e3e4b9b26..c8e42a8f7 100644
--- a/test/page-monitor-relaxed-retry.mjs
+++ b/test/page-monitor-relaxed-retry.mjs
@@ -118,6 +118,63 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: relaxed native clicks validate the supplied point after passive target movement`, { timeout: 60000 }, async t => {
+ const browser = await engine.launch({ headless: true });
+ try {
+ for (const scenario of [
+ { name: 'an unchanged off-center point passes', moved: false, freshPoint: false, explicitRef: false, accepted: true },
+ { name: 'a stale native marker cannot approve a point over another button', moved: true, freshPoint: false, explicitRef: false, accepted: false },
+ { name: 'an explicit original ref cannot conceal stale coordinates', moved: true, freshPoint: false, explicitRef: true, accepted: false },
+ { name: 'a freshly resolved point follows the unchanged moved target', moved: true, freshPoint: true, explicitRef: false, accepted: true },
+ ]) await t.test(scenario.name, async () => {
+ const f = await fixture(browser, build);
+ try {
+ await f.page.evaluate(() => {
+ window.qaWrongClicks = 0;
+ human.addEventListener('click', () => qaWrongClicks++);
+ });
+ const snapshot = await capture(f.page);
+ assert.equal(snapshot.monitoringRelaxed, true);
+ assert.equal((await prepare(f.page, snapshot)).modelBindingValid, true);
+ const native = await f.page.evaluate(() => {
+ const rect = send.getBoundingClientRect();
+ // Match the transport race: the receiver marker and revision are
+ // captured before page layout changes, separately from dispatch.
+ send.setAttribute('data-webbrain-native-target', 'qa-native-point-target');
+ const fence = document.documentElement.getAttribute('data-webbrain-page-revision');
+ const split = fence.lastIndexOf(':');
+ return { nativeTarget: 'qa-native-point-target', ref_id: __wb_ax_ref(send),
+ documentToken: fence.slice(0, split), documentRevision: Number(fence.slice(split + 1)),
+ x: rect.left + 5, y: rect.top + 5,
+ rect: { left: rect.left, top: rect.top, width: rect.width, height: rect.height } };
+ });
+ if (scenario.moved) await f.page.evaluate(rect => {
+ send.style.transform = 'translateY(120px)';
+ human.style.cssText = `position:fixed;left:${rect.left}px;top:${rect.top}px;width:${rect.width}px;height:${rect.height}px;margin:0`;
+ }, native.rect);
+ const point = scenario.freshPoint ? await f.page.evaluate(() => {
+ const rect = send.getBoundingClientRect();
+ return { x: rect.left + 5, y: rect.top + 5 };
+ }) : { x: native.x, y: native.y };
+ assert.equal(await f.page.evaluate(point => document.elementFromPoint(point.x, point.y)?.id, point),
+ scenario.accepted ? 'send' : 'human');
+ const receipt = await f.page.evaluate(({ native, point, explicitRef }) => qaMessage('page_monitor_dispatch', {
+ operationId: 'owned-retry-action', kind: 'click', nativeTarget: native.nativeTarget,
+ documentToken: native.documentToken, documentRevision: native.documentRevision, ...point,
+ ...(explicitRef ? { ref_id: native.ref_id } : {}),
+ eventTypes: ['pointerdown', 'mousedown', 'click'],
+ }), { native, point, explicitRef: scenario.explicitRef });
+ // Simulate the transport only executing input after its fence agrees.
+ if (receipt.ready) await f.page.mouse.click(point.x, point.y);
+ assert.equal(receipt.ready, scenario.accepted, JSON.stringify(receipt));
+ if (!scenario.accepted) assert.equal(receipt.preparationVeto.predicate, 'target_hit_changed');
+ assert.equal(await f.page.evaluate(() => qaWrongClicks), 0);
+ assert.equal(await f.page.evaluate(() => qaClicks), scenario.accepted ? 1 : 0);
+ } finally { await f.context.close(); }
+ });
+ } finally { await browser.close(); }
+ });
+
test(`${build}: no private state grant, mismatched grant or old observation preserves ordinary monitoring`, { timeout: 120000 }, async t => {
const browser = await engine.launch({ headless: true });
try {