Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Let a post shown on its own page own its buttons, and each reply its own
Claude78d3b15c5b1d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5520a43ad..5296aef54 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ This changelog was generated from the repository Git history and release tags. V### Fixed- Chrome: when a click on a plain page element shows no effect, WebBrain clicks it once more with a real mouse click only if a model, reading the element's name in any language, does not see it sending, buying, deleting, liking, following or downloading something. The word lists in 17 languages that decided this are gone: "Fiók törlése" or "Radera konto" are now kept from a second click like "Delete account" was, while names like "Order history" or "Share price" are no longer treated as actions. A name the model cannot read keeps the retry, as any unlisted name did before.+- A click on a button of a Mastodon post shown on its own page, such as its "…" menu, or on a reply around it, is no longer thrown away and asked for again because the page or a reply changed while WebBrain was deciding. A post the page names as one item for screen readers now owns its own buttons, as posts in a timeline already do, while a change to that post, or another post taking its place, still stops the click.- When a task ends, a search box with an "Ara", "Suchen" or "Buscar" button no longer counts as a form left unsent. Whether such a button only searches is read by the model in any language, instead of looking for the English words "search", "filter" or "go", which also matched words like "Google".- A click on a Mastodon post, or on any timeline that marks each post as an item of a feed, is no longer thrown away and asked for again because another post's time or content changed while WebBrain was deciding. Each post now owns its own buttons, and a timestamp ticking over ("2 dk" to "3 dk") is not a change. A change to the clicked post itself, such as its author or its own buttons, still stops the click.- What a clicked button does is now read by a model in any language, instead of being matched against an English word list. A form's "Kaydet", "Devam" or "送信" is now checked for a validation error like "Save" or "Continue" was, and a quick second click on "Gönder" or "Pagar" is held as a possible duplicate like "Send" or "Pay" was, while "Ekle" or "Filtrele" stays repeatable. Each caption is read once and only after a click that page structure does not already explain, so ordinary clicks get no extra wait. The duplicate-click guard, which the security notes listed as missing in Firefox, works the same in both builds.diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.mdindex 24e2a6862..ada2adc6c 100644--- a/docs/page-feedback-monitor.md+++ b/docs/page-feedback-monitor.md@@ -242,6 +242,18 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behaviorcontrol is bound by its `datetime` value rather than its rendered text, so arelative timestamp ticking over ("2 min" to "3 min") is not a change ofcontext, while a changed `datetime` still is.+- Inside an unnamed region, or outside any entity, the nearest focusable item+ the page names for assistive technology owns the controls inside it: an+ element in the tab order (`tabindex` 0 or more), with no role or+ `role="group"`, named by its own `aria-label` or a complete+ `aria-labelledby`, that links somewhere (its permalink or author), holds no+ field or editor, and stays within 256 elements and 64 controls. This is how+ Mastodon marks a post shown on its own page and each reply around it. The+ item's name, controls, images and attributes stay bound, so another post+ taking its place or a change to its own controls invalidates the decision,+ while changes in the replies around it do not. Forms, landmarks, toolbars,+ menus, grids, other entities, unnamed or unfocusable wrappers and items that+ link nowhere gain nothing from this rule.- Noninteractive `aria-hidden` icon labels inside an exact control remainprivately bound without requiring model-visible dialog identity from an AXhidden node. The bounded predicate is rechecked for every hidden descendant;diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex efb3ef358..e9c3e94d2 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -1341,6 +1341,40 @@}return null;}+ // A focusable item the page names for assistive technology (tabindex with+ // its own aria-label, or a complete aria-labelledby), as Mastodon marks a+ // post shown on its own page and each reply around it. It is the nearest+ // such ancestor below the given boundary; widgets, fields, composers,+ // forms, landmarks and other entities are never items, and an item must+ // link somewhere (its permalink or author) and stay bounded.+ function namedFocusableItem(action, boundary) {+ const stop = 'html,body,main,[role="main"],form,[role="form"],aside,[role="complementary"],'+ + entityOwnerSelector + ',' + navigationOwnerSelector+ + ',[role="toolbar"],[role="menu"],[role="menubar"],[role="listbox"],[role="grid"],[role="tree"],[role="treegrid"],[role="tabpanel"]';+ for (let node = bindingParent(action), depth = 0; node && node !== boundary && depth < 12; node = bindingParent(node), depth++) {+ if (!(node instanceof Element) || node.matches(stop)) return null;+ if (!node.hasAttribute('tabindex') || !(node.tabIndex >= 0)+ || node.matches('a,button,input,select,textarea,summary,label,option,[contenteditable]:not([contenteditable="false"])')) continue;+ const role = String(node.getAttribute('role') || '').trim().toLowerCase();+ if (role && role !== 'group') continue;+ let named = !!String(node.getAttribute('aria-label') || '').trim();+ if (!named) {+ const ids = String(node.getAttribute('aria-labelledby') || '').trim().split(/\s+/).filter(Boolean);+ const labels = ids.length ? modelLabels(node) : null;+ named = !!labels && ids.every(id => labels.some(label => label.node.id === id))+ && labels.some(label => String(window.__wb_ax_name?.(label.node) || label.node.textContent || '').trim());+ }+ if (!named) continue;+ if (node.querySelectorAll('*').length > 256 || node.querySelectorAll(modelControlSelector).length > 64+ || node.querySelector('input:not([type="hidden"]),textarea,select,[contenteditable]:not([contenteditable="false"]),[role="textbox"]')) return null;+ const linked = [...node.querySelectorAll('a[href]')].some(link => {+ const href = String(link.getAttribute('href') || '').trim();+ return visible(link) && !!href && !href.startsWith('#') && !/^javascript:/i.test(href);+ });+ return linked ? node : null;+ }+ return null;+ }function actionFootprint(target, readonlyPreparation = null) {if (readonlyPreparation && !readonlyPreparationMatches(readonlyPreparation, target)) return null;const structuralFile = target instanceof Element && target.tagName === 'INPUT' && target.type === 'file';@@ -1371,6 +1405,14 @@return !!labels && ids.every(id => labels.some(label => label.node.id === id))&& labels.some(label => ids.includes(label.node.id) && String(window.__wb_ax_name?.(label.node) || label.node.textContent || '').trim());};+ // Inside an unnamed region, or outside any entity, a named focusable item+ // owns its controls, so a change in the items around it is not its context.+ let namedItemOwner = false;+ if (!conversation && !readonlyPreparation && !form && !structuralFile+ && (!owner || (owner.matches('[role="region"]') && !owner.matches(modelEntityIdentitySelector)))) {+ const item = namedFocusableItem(action, owner);+ if (item) { owner = item; namedItemOwner = true; }+ }if (!owner && !conversation && !readonlyPreparation) {const semanticScope = form || action.closest('section,[role="form"]');if (namedSemanticScope(semanticScope)) owner = semanticScope;@@ -1560,7 +1602,8 @@// items of the feed are not this item's context.const feedItem = owner.matches('article[aria-posinset],[role="article"][aria-posinset]')&& !!owner.parentElement?.closest('[role="feed"]');- meaningfulOwner ||= headings.length > 0 || identities.length > 0 || owner.matches(modelEntityIdentitySelector) || feedItem;+ meaningfulOwner ||= headings.length > 0 || identities.length > 0 || owner.matches(modelEntityIdentitySelector) || feedItem+ || namedItemOwner;if (meaningfulOwner) {const images = [...owner.querySelectorAll('img')].filter(node => belongs(node, navigationOwner)&& !node.closest('input,textarea,[contenteditable]:not([contenteditable="false"])')diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex efb3ef358..e9c3e94d2 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -1341,6 +1341,40 @@}return null;}+ // A focusable item the page names for assistive technology (tabindex with+ // its own aria-label, or a complete aria-labelledby), as Mastodon marks a+ // post shown on its own page and each reply around it. It is the nearest+ // such ancestor below the given boundary; widgets, fields, composers,+ // forms, landmarks and other entities are never items, and an item must+ // link somewhere (its permalink or author) and stay bounded.+ function namedFocusableItem(action, boundary) {+ const stop = 'html,body,main,[role="main"],form,[role="form"],aside,[role="complementary"],'+ + entityOwnerSelector + ',' + navigationOwnerSelector+ + ',[role="toolbar"],[role="menu"],[role="menubar"],[role="listbox"],[role="grid"],[role="tree"],[role="treegrid"],[role="tabpanel"]';+ for (let node = bindingParent(action), depth = 0; node && node !== boundary && depth < 12; node = bindingParent(node), depth++) {+ if (!(node instanceof Element) || node.matches(stop)) return null;+ if (!node.hasAttribute('tabindex') || !(node.tabIndex >= 0)+ || node.matches('a,button,input,select,textarea,summary,label,option,[contenteditable]:not([contenteditable="false"])')) continue;+ const role = String(node.getAttribute('role') || '').trim().toLowerCase();+ if (role && role !== 'group') continue;+ let named = !!String(node.getAttribute('aria-label') || '').trim();+ if (!named) {+ const ids = String(node.getAttribute('aria-labelledby') || '').trim().split(/\s+/).filter(Boolean);+ const labels = ids.length ? modelLabels(node) : null;+ named = !!labels && ids.every(id => labels.some(label => label.node.id === id))+ && labels.some(label => String(window.__wb_ax_name?.(label.node) || label.node.textContent || '').trim());+ }+ if (!named) continue;+ if (node.querySelectorAll('*').length > 256 || node.querySelectorAll(modelControlSelector).length > 64+ || node.querySelector('input:not([type="hidden"]),textarea,select,[contenteditable]:not([contenteditable="false"]),[role="textbox"]')) return null;+ const linked = [...node.querySelectorAll('a[href]')].some(link => {+ const href = String(link.getAttribute('href') || '').trim();+ return visible(link) && !!href && !href.startsWith('#') && !/^javascript:/i.test(href);+ });+ return linked ? node : null;+ }+ return null;+ }function actionFootprint(target, readonlyPreparation = null) {if (readonlyPreparation && !readonlyPreparationMatches(readonlyPreparation, target)) return null;const structuralFile = target instanceof Element && target.tagName === 'INPUT' && target.type === 'file';@@ -1371,6 +1405,14 @@return !!labels && ids.every(id => labels.some(label => label.node.id === id))&& labels.some(label => ids.includes(label.node.id) && String(window.__wb_ax_name?.(label.node) || label.node.textContent || '').trim());};+ // Inside an unnamed region, or outside any entity, a named focusable item+ // owns its controls, so a change in the items around it is not its context.+ let namedItemOwner = false;+ if (!conversation && !readonlyPreparation && !form && !structuralFile+ && (!owner || (owner.matches('[role="region"]') && !owner.matches(modelEntityIdentitySelector)))) {+ const item = namedFocusableItem(action, owner);+ if (item) { owner = item; namedItemOwner = true; }+ }if (!owner && !conversation && !readonlyPreparation) {const semanticScope = form || action.closest('section,[role="form"]');if (namedSemanticScope(semanticScope)) owner = semanticScope;@@ -1560,7 +1602,8 @@// items of the feed are not this item's context.const feedItem = owner.matches('article[aria-posinset],[role="article"][aria-posinset]')&& !!owner.parentElement?.closest('[role="feed"]');- meaningfulOwner ||= headings.length > 0 || identities.length > 0 || owner.matches(modelEntityIdentitySelector) || feedItem;+ meaningfulOwner ||= headings.length > 0 || identities.length > 0 || owner.matches(modelEntityIdentitySelector) || feedItem+ || namedItemOwner;if (meaningfulOwner) {const images = [...owner.querySelectorAll('img')].filter(node => belongs(node, navigationOwner)&& !node.closest('input,textarea,[contenteditable]:not([contenteditable="false"])')diff --git a/test/page-monitor-feed-items.mjs b/test/page-monitor-feed-items.mjsindex 544332176..4d1b54a4b 100644--- a/test/page-monitor-feed-items.mjs+++ b/test/page-monitor-feed-items.mjs@@ -3,6 +3,8 @@// controls inside it. A model decision to click one of them stays certified// while other items of the feed change, and a relative timestamp ticking over// is not a change of context; a change of the item's own controls still is.+// A post shown on its own page, and each reply around it, is a focusable item+// the page names for assistive technology; it owns its controls the same way.import assert from 'node:assert/strict';import fs from 'node:fs';import { test } from 'node:test';@@ -18,7 +20,30 @@ const post = (id, time, { feedItem = true } = {}) => `<article data-id="${id}" $<button type="button" aria-label="Daha fazla" id="more${id}">…</button></div></div></div></article>`;-async function open(engine, build, body) {+// Mastodon 4.4 renders a post's own page as an unnamed region holding the+// post's focusable, screen-reader-named wrapper (the post and its action bar)+// between the posts it replies to and the replies below it.+const reply = (id, name, time, { label = true, tabindex = true, link = true } = {}) => `<div class="status__wrapper focusable"+ ${tabindex ? 'tabindex="0"' : ''} ${label ? `aria-label="${name}, yanıt ${id}, 14 Eki 12:3${id}, ${name.toLowerCase()}"` : ''}>+ <div class="status" data-id="${id}">${link ? `<a id="author${id}" href="/@${name.toLowerCase()}">${name}</a>` : `<span id="author${id}">${name}</span>`}+ <span class="status__relative-time"><time id="time${id}" datetime="2026-10-14T09:3${id}:00.000Z">${time}</time></span>+ <div class="status__content"><p>yanıt ${id}</p></div>+ <div class="status__action-bar"><button type="button" aria-label="Yanıtla">R</button>+ <button type="button" aria-label="Favori" id="fav${id}">0</button><button type="button" aria-label="Daha fazla" id="more${id}">…</button></div>+ </div></div>`;+const detail = (wrapper = 'tabindex="0" aria-label="Emre, gönderi 1, 14 Eki 12:31, emre"') => `<div class="column">+ <div class="column-header__wrapper"><h1 class="column-header"><button type="button">Geri</button></h1></div>+ <div role="region"><div class="scrollable">${reply(0, 'Ali', '1 sa')}+ <div class="focusable detailed-status__wrapper" id="detail" ${wrapper}>+ <div><div class="detailed-status"><a class="detailed-status__display-name" id="author1" href="/@emre"><span>Emre</span></a>+ <div class="status__content"><p>gönderi 1</p></div>+ <div class="detailed-status__meta"><a class="detailed-status__datetime" href="/@emre/1"><time datetime="2026-10-14T09:31:00.000Z">14 Eki 2026 12:31</time></a>+ <a href="/@emre/1/favourites" id="favs1">2 beğeni</a></div></div></div>+ <div class="detailed-status__action-bar"><button type="button" aria-label="Yanıtla">R</button>+ <button type="button" aria-label="Daha fazla" id="more1">…</button></div>+ </div>${reply(2, 'Mert', '9 dk')}${reply(3, 'Can', '8 dk')}</div></div></div>`;++async function open(engine, build, body, targetId = 'more1') {const browser = await engine.launch({ headless: true });const context = await browser.newContext({ viewport: { width: 1000, height: 800 } });await context.route('https://feed.test/**', route => route.fulfill({ contentType: 'text/html', body: `<!doctype html><title>Feed</title>${body}` }));@@ -50,7 +75,7 @@ async function open(engine, build, body) {new Promise(resolve => setTimeout(() => resolve({ ready: false, code: 'monitor_response_missing' }), 2000))]), { action, params });const snapshot = await message('page_monitor_capture_model', { includeTree: true });assert.equal(snapshot.ready, true, JSON.stringify(snapshot));- const ref = await page.evaluate(() => window.__wb_ax_ref?.(document.getElementById('more1')));+ const ref = await page.evaluate(id => window.__wb_ax_ref?.(document.getElementById(id)), targetId);assert.match(String(ref), /^ref_/);const validate = () => message('page_monitor_validate_model', { snapshotToken: snapshot.snapshotToken, tool: 'click_ax', ref_id: ref });return { browser, page, validate };@@ -86,6 +111,65 @@ for (const build of ['chrome', 'firefox']) {} finally { await f.browser.close(); }});+ test(`${build} on ${engineName}: a post on its own page owns its controls`, async () => {+ const f = await open(engine, build, detail());+ try {+ assert.equal((await f.validate()).ready, true, 'The post\'s own More control is certified');+ // A reply below it changes: its time, author and counter.+ await f.page.evaluate(() => {+ document.getElementById('time2').textContent = '10 dk'; document.getElementById('author2').textContent = 'Mehmet';+ document.getElementById('fav2').textContent = '1';+ });+ assert.equal((await f.validate()).ready, true, 'A change in a reply keeps the decision');+ // The post's own controls change.+ await f.page.evaluate(() => { document.getElementById('favs1').textContent = '3 beğeni'; });+ const counted = await f.validate();+ assert.equal(counted.ready, false);+ assert.equal(counted.reason, 'semantic_binding_changed');+ } finally { await f.browser.close(); }+ });++ test(`${build} on ${engineName}: a reply beside the post owns its own controls`, async () => {+ const f = await open(engine, build, detail(), 'more2');+ try {+ assert.equal((await f.validate()).ready, true);+ await f.page.evaluate(() => { document.getElementById('favs1').textContent = '3 beğeni'; document.getElementById('fav3').textContent = '4'; });+ assert.equal((await f.validate()).ready, true, 'The post and another reply changing keep the decision');+ // Another post takes this reply's place under the same node.+ await f.page.evaluate(() => {+ const wrapper = document.getElementById('more2').closest('.status__wrapper');+ wrapper.setAttribute('aria-label', 'Deniz, yanıt 4, 14 Eki 12:34, deniz');+ });+ const replaced = await f.validate();+ assert.equal(replaced.ready, false);+ assert.equal(replaced.reason, 'semantic_binding_changed');+ } finally { await f.browser.close(); }+ });++ test(`${build} on ${engineName}: an unnamed, unfocusable, unlinked or widget wrapper owns nothing`, async () => {+ const cases = [+ ['unnamed', detail('tabindex="0"')],+ ['unfocusable', detail('aria-label="Emre, gönderi 1, 14 Eki 12:31, emre"')],+ ['removed from the tab order', detail('tabindex="-1" aria-label="Emre, gönderi 1, 14 Eki 12:31, emre"')],+ ['a toolbar', detail('tabindex="0" role="toolbar" aria-label="Emre, gönderi 1, 14 Eki 12:31, emre"')],+ ['named by a missing element', detail('tabindex="0" aria-labelledby="missing-name"')],+ ];+ for (const [name, body] of cases) {+ const f = await open(engine, build, body);+ try {+ const result = await f.validate();+ assert.equal(result.ready, false, name);+ assert.equal(result.certificationFailure, 'owner_unresolved', name);+ } finally { await f.browser.close(); }+ }+ const unlinked = await open(engine, build, `<main><div role="region">${reply(5, 'Ece', '3 dk', { link: false })}${reply(6, 'Su', '4 dk', { link: false })}</div></main>`, 'more5');+ try {+ const result = await unlinked.validate();+ assert.equal(result.ready, false, 'An item that links nowhere');+ assert.equal(result.certificationFailure, 'owner_unresolved');+ } finally { await unlinked.browser.close(); }+ });+test(`${build} on ${engineName}: an article outside a feed, or without its place in one, owns nothing new`, async () => {for (const body of [`<main>${post(1, '2 dk')}${post(2, '9 dk')}</main>`,`<main><div role="feed">${post(1, '2 dk', { feedItem: false })}${post(2, '9 dk', { feedItem: false })}</div></main>`]) {