Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fix text click guard target parity
Emre Sokullu524c96edde95
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 1187d03e1..797fa7542 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -1614,7 +1614,15 @@if (!_isInteractive(actionable) || !_isEligibleTextClickTarget(actionable)) continue;if (!controls.has(actionable)) controls.set(actionable, match);}- return controls.size === 1 ? [...controls.values()][0] : null;+ if (controls.size !== 1) return null;+ const [actionable, match] = controls.entries().next().value;+ // A visible caption wins only over passive text. A distinct interactive+ // element with the same accessible name remains an explicit ambiguity.+ for (const candidate of matches) {+ const owner = _resolveInteractiveAncestor(candidate.e) || candidate.e;+ if (owner !== actionable && _isInteractive(owner)) return null;+ }+ return match;}const _textCandidateCache = new WeakMap();@@ -5708,12 +5716,18 @@// interactive match over passive labels. Multiple matches of the// same tier must still stop at this match tier.if (matches.length > 1) {- const eligibleMatches = matches.filter(({ e }) => _isEligibleTextClickTarget(e));- if (eligibleMatches.length) matches = eligibleMatches;const primaryMatch = _preferredPrimaryTextClickMatch(matches);if (primaryMatch) matches = [primaryMatch];const interactiveMatches = matches.filter(({ e }) => _isInteractive(e));if (interactiveMatches.length === 1) matches = interactiveMatches;+ else {+ const hitTargetPool = (interactiveMatches.length > 1 ? interactiveMatches : matches)+ .filter(({ e }) => _isEligibleTextClickTarget(e));+ if (hitTargetPool.length === 1) matches = hitTargetPool;+ else if (hitTargetPool.length > 1 && hitTargetPool.length < matches.length) {+ matches = hitTargetPool;+ }+ }}if (matches.length === 1) target = _resolveInteractiveAncestor(matches[0].e);if (matches.length) break;@@ -6254,7 +6268,7 @@const linkedInComposeSurface = params.adapterName === 'linkedin' && (/^start a post$/i.test(actionLabel)|| /^\/(?:messaging(?:\/|$)|sharing\/compose(?:\/|$))/.test(location.pathname)- || !!_composedClosestElement(control, '.share-box,[class*="msg-"],[class*="messaging"],[id*="messaging"],[data-messaging],[data-test-messaging],[role="log"]')+ || !!_composedClosestElement(control, `.share-box,${_MESSAGING_SURFACE_SELECTOR}`)|| (composeDialog && Array.from(composeDialog.querySelectorAll('[contenteditable="true"],textarea')).some(visible)));const linkedInUnverified = linkedInComposeSurface && (diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex f5e1a9c06..7257611f8 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -1578,7 +1578,15 @@if (!_isInteractive(actionable) || !_isEligibleTextClickTarget(actionable)) continue;if (!controls.has(actionable)) controls.set(actionable, match);}- return controls.size === 1 ? [...controls.values()][0] : null;+ if (controls.size !== 1) return null;+ const [actionable, match] = controls.entries().next().value;+ // A visible caption wins only over passive text. A distinct interactive+ // element with the same accessible name remains an explicit ambiguity.+ for (const candidate of matches) {+ const owner = _resolveInteractiveAncestor(candidate.e) || candidate.e;+ if (owner !== actionable && _isInteractive(owner)) return null;+ }+ return match;}const _textCandidateCache = new WeakMap();@@ -4796,12 +4804,18 @@// interactive match over passive labels. Multiple matches of the// same tier must still stop at this match tier.if (matches.length > 1) {- const eligibleMatches = matches.filter(({ e }) => _isEligibleTextClickTarget(e));- if (eligibleMatches.length) matches = eligibleMatches;const primaryMatch = _preferredPrimaryTextClickMatch(matches);if (primaryMatch) matches = [primaryMatch];const interactiveMatches = matches.filter(({ e }) => _isInteractive(e));if (interactiveMatches.length === 1) matches = interactiveMatches;+ else {+ const hitTargetPool = (interactiveMatches.length > 1 ? interactiveMatches : matches)+ .filter(({ e }) => _isEligibleTextClickTarget(e));+ if (hitTargetPool.length === 1) matches = hitTargetPool;+ else if (hitTargetPool.length > 1 && hitTargetPool.length < matches.length) {+ matches = hitTargetPool;+ }+ }}if (matches.length === 1) target = _resolveInteractiveAncestor(matches[0].e);if (matches.length) break;diff --git a/test/fixtures/interaction-review-regressions.mjs b/test/fixtures/interaction-review-regressions.mjsindex 4517fb38c..698a56278 100644--- a/test/fixtures/interaction-review-regressions.mjs+++ b/test/fixtures/interaction-review-regressions.mjs@@ -51,6 +51,35 @@ export function registerInteractionReviewRegressions({}});+ register(`${kind}: text preflight cannot approve a passive match over an ineligible send control`, async page => {+ await setupContentHtml(page, `<!doctype html>+ <button id="send" aria-disabled="true" aria-label="Send message">Continue</button>+ <div id="label" tabindex="0">Continue</div>`, kind);+ await page.evaluate(() => {+ window.fixtureClicks = [];+ document.querySelector('#send').addEventListener('click', () => window.fixtureClicks.push('send'));+ document.querySelector('#label').addEventListener('click', () => window.fixtureClicks.push('label'));+ });+ const args = { text: 'Continue', textMatch: 'exact' };+ const probe = await call(page, 'probe_message_recipient_guard', { tool: 'click', args, adapterName: 'gmail' });+ assert.equal(probe.messageSend, null, JSON.stringify(probe));+ assert.equal(probe.conclusive, false, JSON.stringify(probe));+ assert.equal((await call(page, 'click', args)).success, true);+ assert.deepEqual(await page.evaluate(() => window.fixtureClicks), ['send']);+ });++ register(`${kind}: visible text does not override an interactive accessible-name match`, async page => {+ await setupContentHtml(page, `<!doctype html>+ <a id="link" href="/follow">Follow</a>+ <button id="button" aria-label="Follow">★</button>`, kind);+ const args = { text: 'Follow', textMatch: 'exact' };+ const probe = await call(page, 'probe_message_recipient_guard', { tool: 'click', args, adapterName: 'gmail' });+ assert.equal(probe.conclusive, false, JSON.stringify(probe));+ const clicked = await call(page, 'click', args);+ assert.equal(clicked.success, false, JSON.stringify(clicked));+ assert.match(clicked.error, /ambiguous/i);+ });+register(`${kind}: LinkedIn ordinary browsing remains available without a composer`, async page => {await page.route('**/*', route => route.fulfill({ contentType: 'text/html', body: '<!doctype html>' }));await page.goto('https://www.linkedin.com/feed/');@@ -120,12 +149,15 @@ export function registerInteractionReviewRegressions({const probe = await call(page, 'probe_message_recipient_guard', {tool: 'click', args, adapterName: 'gmail',});- assert.equal(probe.messageSend, false, JSON.stringify({ variant, probe }));- assert.equal(probe.conclusive, true, JSON.stringify({ variant, probe }));+ assert.notEqual(probe.messageSend, true, JSON.stringify({ variant, probe }));const clicked = await call(page, 'click', args);+ if (variant === 'enabled') {+ assert.equal(clicked.success, false, JSON.stringify({ variant, clicked }));+ assert.match(clicked.error, /ambiguous/i);+ continue;+ }assert.equal(clicked.success, true, JSON.stringify({ variant, clicked }));- assert.deepEqual(await page.evaluate(() => window.fixtureClicks),- [variant === 'enabled' ? 'primary' : 'glyph'], variant);+ assert.deepEqual(await page.evaluate(() => window.fixtureClicks), ['glyph'], variant);}});