Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Validate actual click coordinates during relaxed Retry
Barack Sokullu15d8ab694e9f
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 6f9f255df..e02b06da5 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -1686,7 +1686,16 @@return ['characterData', 'childList'].includes(record.type)&& (node === binding.target || binding.target.contains(node));}- function relaxedActionability(target, { hit = false, input = false } = {}) {+ function nativeReceiverBelongsTo(target, receiver) {+ // Decorative descendants can receive their control's input; another+ // activator inside that control cannot borrow the original certificate.+ for (let node = receiver, depth = 0; node && depth < 256; node = bindingParent(node), depth++) {+ if (node === target) return true;+ if (node.matches(modelControlSelector + ',[tabindex]')) return false;+ }+ return false;+ }+ function relaxedActionability(target, { hit = false, input = false, point = null } = {}) {if (!(target instanceof Element) || !target.isConnected || target.ownerDocument !== document || !visible(target)|| !target.matches(modelControlSelector) || target.disabled === true || (input && target.readOnly === true)) return false;for (let node = target; node; node = bindingParent(node)) {@@ -1698,7 +1707,9 @@const rect = target.getBoundingClientRect();if (!(rect.width > 0 && rect.height > 0)) return false;if (!hit) return true;- const receiver = deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);+ const receiver = point ? deepElementFromPoint(point.x, point.y)+ : deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);+ if (point) return nativeReceiverBelongsTo(target, receiver);return !!receiver && (receiver === target || target.contains(receiver));}function modelBindingResult(params = {}) {@@ -2068,10 +2079,17 @@&& resolveTarget(params) !== op.preparedBinding.target) {revokeRelaxedGrant(); return 'target_changed';}+ // Native markers bind a node, not the mouse point. Passive movement may+ // leave that node intact while the captured command now hits another one.+ const point = op?.preparedBinding?.relaxed && params.kind === 'click' && params.pointerPreparationOnly !== true+ && (params.x !== undefined || params.y !== undefined) ? { x: params.x, y: params.y } : null;+ if (point && (!Number.isFinite(point.x) || !Number.isFinite(point.y))) {+ revokeRelaxedGrant(); return 'target_hit_changed';+ }if (op?.preparedBinding?.relaxed && ['click', 'input'].includes(params.kind)&& params.pointerPreparationOnly !== true- && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input' })) {- revokeRelaxedGrant(); return 'target_not_actionable';+ && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input', point })) {+ revokeRelaxedGrant(); return point ? 'target_hit_changed' : 'target_not_actionable';}if (!params.nativeTarget || op?.preparedBinding?.model !== true) return '';// Resolve the actual transport marker alone; a stale marker cannot fall@@ -2080,14 +2098,7 @@if (!receiver) return 'target_unresolved';const target = op.tabFocusRebound === true && receiver === op.focusTarget? op.focusTarget : op.preparedBinding.target;- // Button text/SVG and editor paragraphs can receive their owner's input.- // A separate activator, including one inside a shadow root, cannot.- let node = receiver;- for (let depth = 0; node && depth < 256; depth++, node = bindingParent(node)) {- if (node === target) return '';- if (node.matches(modelControlSelector + ',[tabindex]')) return 'target_changed';- }- return 'target_changed';+ return nativeReceiverBelongsTo(target, receiver) ? '' : 'target_changed';}function rebasePassivePreparation(op, params = {}) {const relaxed = op?.preparedBinding?.relaxed === true && currentRelaxedGrant(op.preparedBinding.grantToken);diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 6f9f255df..e02b06da5 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -1686,7 +1686,16 @@return ['characterData', 'childList'].includes(record.type)&& (node === binding.target || binding.target.contains(node));}- function relaxedActionability(target, { hit = false, input = false } = {}) {+ function nativeReceiverBelongsTo(target, receiver) {+ // Decorative descendants can receive their control's input; another+ // activator inside that control cannot borrow the original certificate.+ for (let node = receiver, depth = 0; node && depth < 256; node = bindingParent(node), depth++) {+ if (node === target) return true;+ if (node.matches(modelControlSelector + ',[tabindex]')) return false;+ }+ return false;+ }+ function relaxedActionability(target, { hit = false, input = false, point = null } = {}) {if (!(target instanceof Element) || !target.isConnected || target.ownerDocument !== document || !visible(target)|| !target.matches(modelControlSelector) || target.disabled === true || (input && target.readOnly === true)) return false;for (let node = target; node; node = bindingParent(node)) {@@ -1698,7 +1707,9 @@const rect = target.getBoundingClientRect();if (!(rect.width > 0 && rect.height > 0)) return false;if (!hit) return true;- const receiver = deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);+ const receiver = point ? deepElementFromPoint(point.x, point.y)+ : deepElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);+ if (point) return nativeReceiverBelongsTo(target, receiver);return !!receiver && (receiver === target || target.contains(receiver));}function modelBindingResult(params = {}) {@@ -2068,10 +2079,17 @@&& resolveTarget(params) !== op.preparedBinding.target) {revokeRelaxedGrant(); return 'target_changed';}+ // Native markers bind a node, not the mouse point. Passive movement may+ // leave that node intact while the captured command now hits another one.+ const point = op?.preparedBinding?.relaxed && params.kind === 'click' && params.pointerPreparationOnly !== true+ && (params.x !== undefined || params.y !== undefined) ? { x: params.x, y: params.y } : null;+ if (point && (!Number.isFinite(point.x) || !Number.isFinite(point.y))) {+ revokeRelaxedGrant(); return 'target_hit_changed';+ }if (op?.preparedBinding?.relaxed && ['click', 'input'].includes(params.kind)&& params.pointerPreparationOnly !== true- && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input' })) {- revokeRelaxedGrant(); return 'target_not_actionable';+ && !relaxedActionability(op.preparedBinding.target, { hit: true, input: params.kind === 'input', point })) {+ revokeRelaxedGrant(); return point ? 'target_hit_changed' : 'target_not_actionable';}if (!params.nativeTarget || op?.preparedBinding?.model !== true) return '';// Resolve the actual transport marker alone; a stale marker cannot fall@@ -2080,14 +2098,7 @@if (!receiver) return 'target_unresolved';const target = op.tabFocusRebound === true && receiver === op.focusTarget? op.focusTarget : op.preparedBinding.target;- // Button text/SVG and editor paragraphs can receive their owner's input.- // A separate activator, including one inside a shadow root, cannot.- let node = receiver;- for (let depth = 0; node && depth < 256; depth++, node = bindingParent(node)) {- if (node === target) return '';- if (node.matches(modelControlSelector + ',[tabindex]')) return 'target_changed';- }- return 'target_changed';+ return nativeReceiverBelongsTo(target, receiver) ? '' : 'target_changed';}function rebasePassivePreparation(op, params = {}) {const relaxed = op?.preparedBinding?.relaxed === true && currentRelaxedGrant(op.preparedBinding.grantToken);diff --git a/test/page-monitor-relaxed-retry.mjs b/test/page-monitor-relaxed-retry.mjsindex e3e4b9b26..c8e42a8f7 100644--- a/test/page-monitor-relaxed-retry.mjs+++ b/test/page-monitor-relaxed-retry.mjs@@ -118,6 +118,63 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: relaxed native clicks validate the supplied point after passive target movement`, { timeout: 60000 }, async t => {+ const browser = await engine.launch({ headless: true });+ try {+ for (const scenario of [+ { name: 'an unchanged off-center point passes', moved: false, freshPoint: false, explicitRef: false, accepted: true },+ { name: 'a stale native marker cannot approve a point over another button', moved: true, freshPoint: false, explicitRef: false, accepted: false },+ { name: 'an explicit original ref cannot conceal stale coordinates', moved: true, freshPoint: false, explicitRef: true, accepted: false },+ { name: 'a freshly resolved point follows the unchanged moved target', moved: true, freshPoint: true, explicitRef: false, accepted: true },+ ]) await t.test(scenario.name, async () => {+ const f = await fixture(browser, build);+ try {+ await f.page.evaluate(() => {+ window.qaWrongClicks = 0;+ human.addEventListener('click', () => qaWrongClicks++);+ });+ const snapshot = await capture(f.page);+ assert.equal(snapshot.monitoringRelaxed, true);+ assert.equal((await prepare(f.page, snapshot)).modelBindingValid, true);+ const native = await f.page.evaluate(() => {+ const rect = send.getBoundingClientRect();+ // Match the transport race: the receiver marker and revision are+ // captured before page layout changes, separately from dispatch.+ send.setAttribute('data-webbrain-native-target', 'qa-native-point-target');+ const fence = document.documentElement.getAttribute('data-webbrain-page-revision');+ const split = fence.lastIndexOf(':');+ return { nativeTarget: 'qa-native-point-target', ref_id: __wb_ax_ref(send),+ documentToken: fence.slice(0, split), documentRevision: Number(fence.slice(split + 1)),+ x: rect.left + 5, y: rect.top + 5,+ rect: { left: rect.left, top: rect.top, width: rect.width, height: rect.height } };+ });+ if (scenario.moved) await f.page.evaluate(rect => {+ send.style.transform = 'translateY(120px)';+ human.style.cssText = `position:fixed;left:${rect.left}px;top:${rect.top}px;width:${rect.width}px;height:${rect.height}px;margin:0`;+ }, native.rect);+ const point = scenario.freshPoint ? await f.page.evaluate(() => {+ const rect = send.getBoundingClientRect();+ return { x: rect.left + 5, y: rect.top + 5 };+ }) : { x: native.x, y: native.y };+ assert.equal(await f.page.evaluate(point => document.elementFromPoint(point.x, point.y)?.id, point),+ scenario.accepted ? 'send' : 'human');+ const receipt = await f.page.evaluate(({ native, point, explicitRef }) => qaMessage('page_monitor_dispatch', {+ operationId: 'owned-retry-action', kind: 'click', nativeTarget: native.nativeTarget,+ documentToken: native.documentToken, documentRevision: native.documentRevision, ...point,+ ...(explicitRef ? { ref_id: native.ref_id } : {}),+ eventTypes: ['pointerdown', 'mousedown', 'click'],+ }), { native, point, explicitRef: scenario.explicitRef });+ // Simulate the transport only executing input after its fence agrees.+ if (receipt.ready) await f.page.mouse.click(point.x, point.y);+ assert.equal(receipt.ready, scenario.accepted, JSON.stringify(receipt));+ if (!scenario.accepted) assert.equal(receipt.preparationVeto.predicate, 'target_hit_changed');+ assert.equal(await f.page.evaluate(() => qaWrongClicks), 0);+ assert.equal(await f.page.evaluate(() => qaClicks), scenario.accepted ? 1 : 0);+ } finally { await f.context.close(); }+ });+ } finally { await browser.close(); }+ });+test(`${build}: no private state grant, mismatched grant or old observation preserves ordinary monitoring`, { timeout: 120000 }, async t => {const browser = await engine.launch({ headless: true });try {