Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix text click guard target parity

Emre Sokullu524c96edde95
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 1187d03e1..797fa7542 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -1614,7 +1614,15 @@
if (!_isInteractive(actionable) || !_isEligibleTextClickTarget(actionable)) continue;
if (!controls.has(actionable)) controls.set(actionable, match);
}
- return controls.size === 1 ? [...controls.values()][0] : null;
+ if (controls.size !== 1) return null;
+ const [actionable, match] = controls.entries().next().value;
+ // A visible caption wins only over passive text. A distinct interactive
+ // element with the same accessible name remains an explicit ambiguity.
+ for (const candidate of matches) {
+ const owner = _resolveInteractiveAncestor(candidate.e) || candidate.e;
+ if (owner !== actionable && _isInteractive(owner)) return null;
+ }
+ return match;
}
const _textCandidateCache = new WeakMap();
@@ -5708,12 +5716,18 @@
// interactive match over passive labels. Multiple matches of the
// same tier must still stop at this match tier.
if (matches.length > 1) {
- const eligibleMatches = matches.filter(({ e }) => _isEligibleTextClickTarget(e));
- if (eligibleMatches.length) matches = eligibleMatches;
const primaryMatch = _preferredPrimaryTextClickMatch(matches);
if (primaryMatch) matches = [primaryMatch];
const interactiveMatches = matches.filter(({ e }) => _isInteractive(e));
if (interactiveMatches.length === 1) matches = interactiveMatches;
+ else {
+ const hitTargetPool = (interactiveMatches.length > 1 ? interactiveMatches : matches)
+ .filter(({ e }) => _isEligibleTextClickTarget(e));
+ if (hitTargetPool.length === 1) matches = hitTargetPool;
+ else if (hitTargetPool.length > 1 && hitTargetPool.length < matches.length) {
+ matches = hitTargetPool;
+ }
+ }
}
if (matches.length === 1) target = _resolveInteractiveAncestor(matches[0].e);
if (matches.length) break;
@@ -6254,7 +6268,7 @@
const linkedInComposeSurface = params.adapterName === 'linkedin' && (
/^start a post$/i.test(actionLabel)
|| /^\/(?:messaging(?:\/|$)|sharing\/compose(?:\/|$))/.test(location.pathname)
- || !!_composedClosestElement(control, '.share-box,[class*="msg-"],[class*="messaging"],[id*="messaging"],[data-messaging],[data-test-messaging],[role="log"]')
+ || !!_composedClosestElement(control, `.share-box,${_MESSAGING_SURFACE_SELECTOR}`)
|| (composeDialog && Array.from(composeDialog.querySelectorAll('[contenteditable="true"],textarea')).some(visible))
);
const linkedInUnverified = linkedInComposeSurface && (
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index f5e1a9c06..7257611f8 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -1578,7 +1578,15 @@
if (!_isInteractive(actionable) || !_isEligibleTextClickTarget(actionable)) continue;
if (!controls.has(actionable)) controls.set(actionable, match);
}
- return controls.size === 1 ? [...controls.values()][0] : null;
+ if (controls.size !== 1) return null;
+ const [actionable, match] = controls.entries().next().value;
+ // A visible caption wins only over passive text. A distinct interactive
+ // element with the same accessible name remains an explicit ambiguity.
+ for (const candidate of matches) {
+ const owner = _resolveInteractiveAncestor(candidate.e) || candidate.e;
+ if (owner !== actionable && _isInteractive(owner)) return null;
+ }
+ return match;
}
const _textCandidateCache = new WeakMap();
@@ -4796,12 +4804,18 @@
// interactive match over passive labels. Multiple matches of the
// same tier must still stop at this match tier.
if (matches.length > 1) {
- const eligibleMatches = matches.filter(({ e }) => _isEligibleTextClickTarget(e));
- if (eligibleMatches.length) matches = eligibleMatches;
const primaryMatch = _preferredPrimaryTextClickMatch(matches);
if (primaryMatch) matches = [primaryMatch];
const interactiveMatches = matches.filter(({ e }) => _isInteractive(e));
if (interactiveMatches.length === 1) matches = interactiveMatches;
+ else {
+ const hitTargetPool = (interactiveMatches.length > 1 ? interactiveMatches : matches)
+ .filter(({ e }) => _isEligibleTextClickTarget(e));
+ if (hitTargetPool.length === 1) matches = hitTargetPool;
+ else if (hitTargetPool.length > 1 && hitTargetPool.length < matches.length) {
+ matches = hitTargetPool;
+ }
+ }
}
if (matches.length === 1) target = _resolveInteractiveAncestor(matches[0].e);
if (matches.length) break;
diff --git a/test/fixtures/interaction-review-regressions.mjs b/test/fixtures/interaction-review-regressions.mjs
index 4517fb38c..698a56278 100644
--- a/test/fixtures/interaction-review-regressions.mjs
+++ b/test/fixtures/interaction-review-regressions.mjs
@@ -51,6 +51,35 @@ export function registerInteractionReviewRegressions({
}
});
+ register(`${kind}: text preflight cannot approve a passive match over an ineligible send control`, async page => {
+ await setupContentHtml(page, `<!doctype html>
+ <button id="send" aria-disabled="true" aria-label="Send message">Continue</button>
+ <div id="label" tabindex="0">Continue</div>`, kind);
+ await page.evaluate(() => {
+ window.fixtureClicks = [];
+ document.querySelector('#send').addEventListener('click', () => window.fixtureClicks.push('send'));
+ document.querySelector('#label').addEventListener('click', () => window.fixtureClicks.push('label'));
+ });
+ const args = { text: 'Continue', textMatch: 'exact' };
+ const probe = await call(page, 'probe_message_recipient_guard', { tool: 'click', args, adapterName: 'gmail' });
+ assert.equal(probe.messageSend, null, JSON.stringify(probe));
+ assert.equal(probe.conclusive, false, JSON.stringify(probe));
+ assert.equal((await call(page, 'click', args)).success, true);
+ assert.deepEqual(await page.evaluate(() => window.fixtureClicks), ['send']);
+ });
+
+ register(`${kind}: visible text does not override an interactive accessible-name match`, async page => {
+ await setupContentHtml(page, `<!doctype html>
+ <a id="link" href="/follow">Follow</a>
+ <button id="button" aria-label="Follow">★</button>`, kind);
+ const args = { text: 'Follow', textMatch: 'exact' };
+ const probe = await call(page, 'probe_message_recipient_guard', { tool: 'click', args, adapterName: 'gmail' });
+ assert.equal(probe.conclusive, false, JSON.stringify(probe));
+ const clicked = await call(page, 'click', args);
+ assert.equal(clicked.success, false, JSON.stringify(clicked));
+ assert.match(clicked.error, /ambiguous/i);
+ });
+
register(`${kind}: LinkedIn ordinary browsing remains available without a composer`, async page => {
await page.route('**/*', route => route.fulfill({ contentType: 'text/html', body: '<!doctype html>' }));
await page.goto('https://www.linkedin.com/feed/');
@@ -120,12 +149,15 @@ export function registerInteractionReviewRegressions({
const probe = await call(page, 'probe_message_recipient_guard', {
tool: 'click', args, adapterName: 'gmail',
});
- assert.equal(probe.messageSend, false, JSON.stringify({ variant, probe }));
- assert.equal(probe.conclusive, true, JSON.stringify({ variant, probe }));
+ assert.notEqual(probe.messageSend, true, JSON.stringify({ variant, probe }));
const clicked = await call(page, 'click', args);
+ if (variant === 'enabled') {
+ assert.equal(clicked.success, false, JSON.stringify({ variant, clicked }));
+ assert.match(clicked.error, /ambiguous/i);
+ continue;
+ }
assert.equal(clicked.success, true, JSON.stringify({ variant, clicked }));
- assert.deepEqual(await page.evaluate(() => window.fixtureClicks),
- [variant === 'enabled' ? 'primary' : 'glyph'], variant);
+ assert.deepEqual(await page.evaluate(() => window.fixtureClicks), ['glyph'], variant);
}
});