Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Limit the grid-cell trusted retry to WhatsApp's chat list
Claude17954fe03bd2
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 9a1869338..75903ebc5 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -1241,10 +1241,13 @@const unsafeAncestor = el?.parentElement?.closest?.('button,a,input,select,textarea,label,form,[role="button"],[role="link"],[onclick],[data-action]');- // A grid cell is the clickable row of a grid-based list. WhatsApp's chat- // list is one, and a script-dispatched click on it does not open the- // chat, so without the trusted retry the agent could not open a chat.- const safeRole = !role || ['generic', 'listitem', 'row', 'gridcell'].includes(role);+ // WhatsApp's chat list is a grid whose cells a script-dispatched click does+ // not open, so without the trusted retry the agent could not open a chat.+ // Any other grid (a spreadsheet, a calendar) keeps its cells synthetic-only:+ // a second, trusted click there can turn a selection into an edit.+ const whatsappChatCell = role === 'gridcell' && location.hostname === 'web.whatsapp.com'+ && !!el?.closest?.('[data-testid="chat-list"]');+ const safeRole = !role || ['generic', 'listitem', 'row'].includes(role) || whatsappChatCell;let blockedReason = '';if (isEditable) blockedReason = 'editable targets must not be auto-clicked twice';else if (tag === 'select') blockedReason = 'native select controls must use keyboard selection';diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjsindex 8ef019c10..7d66cc630 100644--- a/test/fixtures/run.mjs+++ b/test/fixtures/run.mjs@@ -87,9 +87,16 @@ const stubFirefoxBrowser = `};`;-async function setup(page, fixture) {+async function setup(page, fixture, { asHost = '' } = {}) {await page.addInitScript(stubChrome);- await page.goto(fixtureUrl(fixture));+ if (asHost) {+ // Serve the fixture as if from another host, without touching the network.+ const body = await readFile(path.join(__dirname, fixture), 'utf-8');+ await page.route(`${asHost}/**`, route => route.fulfill({ contentType: 'text/html', body }));+ await page.goto(`${asHost}/fixture`);+ } else {+ await page.goto(fixtureUrl(fixture));+ }const axSrc = await readFile(accessibilityTreeJsPath, 'utf-8');await page.addScriptTag({ content: axSrc });await page.addScriptTag({ content: await readFile(toolbarHeuristicJsPath, 'utf-8') });@@ -5900,9 +5907,22 @@ test('click_ax: Agent.executeTool keeps synthetic-first behavior and uses truste// From a trace: WhatsApp's chat list rows are grid cells that a// script-dispatched click does not open, and the trusted retry skipped every// role but generic, listitem and row, so the agent could not open the chat.-test('click_ax: a grid cell row that ignores the synthetic click gets the trusted CDP retry', async (page) => {+// Other grids (spreadsheets, calendars) keep their cells synthetic-only: a+// second, trusted click there can turn a selection into an edit.+test('click_ax: a grid cell outside WhatsApp\'s chat list is never retried with a trusted click', async (page) => {await setup(page, 'trusted-click-fallback.html');const tree = await call(page, 'get_accessibility_tree', { filter: 'all', maxDepth: 10, maxChars: 30000 });+ const cellMatch = String(tree?.pageContent || '').match(/gridcell "Grid chat row" \[(ref_\d+)\]/);+ if (!cellMatch) throw new Error('expected the grid cell in the AX tree');+ const result = await call(page, 'ax_resolve_rect', { ref_id: cellMatch[1], forClickFallback: true });+ if (result?.fallbackEligible !== false || !/role "gridcell" is not eligible/.test(result.fallbackBlockedReason || '')) {+ throw new Error(`a grid cell on another site must stay synthetic-only: ${JSON.stringify(result)}`);+ }+});++test('click_ax: a grid cell row that ignores the synthetic click gets the trusted CDP retry', async (page) => {+ await setup(page, 'trusted-click-fallback.html', { asHost: 'https://web.whatsapp.com' });+ const tree = await call(page, 'get_accessibility_tree', { filter: 'all', maxDepth: 10, maxChars: 30000 });const content = String(tree?.pageContent || '');const cellMatch = content.match(/gridcell "Grid chat row" \[(ref_\d+)\]/);const selectedMatch = content.match(/gridcell "Selected grid chat row" \[(ref_\d+)\]/);diff --git a/test/fixtures/trusted-click-fallback.html b/test/fixtures/trusted-click-fallback.htmlindex b80a4a305..3be106c2f 100644--- a/test/fixtures/trusted-click-fallback.html+++ b/test/fixtures/trusted-click-fallback.html@@ -197,7 +197,7 @@<div id="opacity-row" class="row" role="listitem" tabindex="0">Opacity row</div><div id="pointer-row" class="row" role="listitem" tabindex="0" style="pointer-events:none">Pointer disabled row</div><div id="zero-row" role="listitem" tabindex="0">Zero row</div>- <div id="grid-chat-list" role="grid" aria-label="Grid chat list">+ <div id="grid-chat-list" role="grid" aria-label="Grid chat list" data-testid="chat-list"><div role="row"><div id="grid-chat-cell" class="row" role="gridcell" tabindex="-1">Grid chat row</div></div><div role="row"><div id="grid-selected-cell" class="row" role="gridcell" tabindex="-1" aria-selected="true">Selected grid chat row</div></div></div>