Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Limit the grid-cell trusted retry to WhatsApp's chat list

Claude17954fe03bd2
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 9a1869338..75903ebc5 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -1241,10 +1241,13 @@
const unsafeAncestor = el?.parentElement?.closest?.(
'button,a,input,select,textarea,label,form,[role="button"],[role="link"],[onclick],[data-action]'
);
- // A grid cell is the clickable row of a grid-based list. WhatsApp's chat
- // list is one, and a script-dispatched click on it does not open the
- // chat, so without the trusted retry the agent could not open a chat.
- const safeRole = !role || ['generic', 'listitem', 'row', 'gridcell'].includes(role);
+ // WhatsApp's chat list is a grid whose cells a script-dispatched click does
+ // not open, so without the trusted retry the agent could not open a chat.
+ // Any other grid (a spreadsheet, a calendar) keeps its cells synthetic-only:
+ // a second, trusted click there can turn a selection into an edit.
+ const whatsappChatCell = role === 'gridcell' && location.hostname === 'web.whatsapp.com'
+ && !!el?.closest?.('[data-testid="chat-list"]');
+ const safeRole = !role || ['generic', 'listitem', 'row'].includes(role) || whatsappChatCell;
let blockedReason = '';
if (isEditable) blockedReason = 'editable targets must not be auto-clicked twice';
else if (tag === 'select') blockedReason = 'native select controls must use keyboard selection';
diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs
index 8ef019c10..7d66cc630 100644
--- a/test/fixtures/run.mjs
+++ b/test/fixtures/run.mjs
@@ -87,9 +87,16 @@ const stubFirefoxBrowser = `
};
`;
-async function setup(page, fixture) {
+async function setup(page, fixture, { asHost = '' } = {}) {
await page.addInitScript(stubChrome);
- await page.goto(fixtureUrl(fixture));
+ if (asHost) {
+ // Serve the fixture as if from another host, without touching the network.
+ const body = await readFile(path.join(__dirname, fixture), 'utf-8');
+ await page.route(`${asHost}/**`, route => route.fulfill({ contentType: 'text/html', body }));
+ await page.goto(`${asHost}/fixture`);
+ } else {
+ await page.goto(fixtureUrl(fixture));
+ }
const axSrc = await readFile(accessibilityTreeJsPath, 'utf-8');
await page.addScriptTag({ content: axSrc });
await page.addScriptTag({ content: await readFile(toolbarHeuristicJsPath, 'utf-8') });
@@ -5900,9 +5907,22 @@ test('click_ax: Agent.executeTool keeps synthetic-first behavior and uses truste
// From a trace: WhatsApp's chat list rows are grid cells that a
// script-dispatched click does not open, and the trusted retry skipped every
// role but generic, listitem and row, so the agent could not open the chat.
-test('click_ax: a grid cell row that ignores the synthetic click gets the trusted CDP retry', async (page) => {
+// Other grids (spreadsheets, calendars) keep their cells synthetic-only: a
+// second, trusted click there can turn a selection into an edit.
+test('click_ax: a grid cell outside WhatsApp\'s chat list is never retried with a trusted click', async (page) => {
await setup(page, 'trusted-click-fallback.html');
const tree = await call(page, 'get_accessibility_tree', { filter: 'all', maxDepth: 10, maxChars: 30000 });
+ const cellMatch = String(tree?.pageContent || '').match(/gridcell "Grid chat row" \[(ref_\d+)\]/);
+ if (!cellMatch) throw new Error('expected the grid cell in the AX tree');
+ const result = await call(page, 'ax_resolve_rect', { ref_id: cellMatch[1], forClickFallback: true });
+ if (result?.fallbackEligible !== false || !/role "gridcell" is not eligible/.test(result.fallbackBlockedReason || '')) {
+ throw new Error(`a grid cell on another site must stay synthetic-only: ${JSON.stringify(result)}`);
+ }
+});
+
+test('click_ax: a grid cell row that ignores the synthetic click gets the trusted CDP retry', async (page) => {
+ await setup(page, 'trusted-click-fallback.html', { asHost: 'https://web.whatsapp.com' });
+ const tree = await call(page, 'get_accessibility_tree', { filter: 'all', maxDepth: 10, maxChars: 30000 });
const content = String(tree?.pageContent || '');
const cellMatch = content.match(/gridcell "Grid chat row" \[(ref_\d+)\]/);
const selectedMatch = content.match(/gridcell "Selected grid chat row" \[(ref_\d+)\]/);
diff --git a/test/fixtures/trusted-click-fallback.html b/test/fixtures/trusted-click-fallback.html
index b80a4a305..3be106c2f 100644
--- a/test/fixtures/trusted-click-fallback.html
+++ b/test/fixtures/trusted-click-fallback.html
@@ -197,7 +197,7 @@
<div id="opacity-row" class="row" role="listitem" tabindex="0">Opacity row</div>
<div id="pointer-row" class="row" role="listitem" tabindex="0" style="pointer-events:none">Pointer disabled row</div>
<div id="zero-row" role="listitem" tabindex="0">Zero row</div>
- <div id="grid-chat-list" role="grid" aria-label="Grid chat list">
+ <div id="grid-chat-list" role="grid" aria-label="Grid chat list" data-testid="chat-list">
<div role="row"><div id="grid-chat-cell" class="row" role="gridcell" tabindex="-1">Grid chat row</div></div>
<div role="row"><div id="grid-selected-cell" class="row" role="gridcell" tabindex="-1" aria-selected="true">Selected grid chat row</div></div>
</div>