Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
track shadow text and select state changes
Barack Sokullu842f11551412
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex eae77576e..01c5fdd19 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -228,6 +228,8 @@el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-pressed'),el.getAttribute('popover'), popoverOpen(el),+ el.selected === true, el.defaultSelected === true,+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);@@ -325,9 +327,14 @@};const changes = [...records];for (const record of records) {- if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName))) continue;+ if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;if (ignored(record.target)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ if (record.type === 'attributes' && record.attributeName === 'selected'+ && record.oldValue !== record.target.getAttribute('selected')) {+ const select = record.target.closest?.('select');+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);+ }const sample = sampleDescendants(record.target);// A bounded layout sample cannot prove that inherited visibility left a// large subtree unchanged. Invalidate conservatively on an actual state@@ -344,7 +351,8 @@let measured = 0;const seedBudget = { remaining: 600 };for (const record of changes) {- const el = record.target.nodeType === 1 ? record.target : record.target.host || record.target.parentElement;+ const el = record.target.nodeType === 1 ? record.target+ : record.target.host || record.target.parentElement || record.target.getRootNode?.().host;if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex eae77576e..01c5fdd19 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -228,6 +228,8 @@el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-pressed'),el.getAttribute('popover'), popoverOpen(el),+ el.selected === true, el.defaultSelected === true,+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);@@ -325,9 +327,14 @@};const changes = [...records];for (const record of records) {- if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName))) continue;+ if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;if (ignored(record.target)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ if (record.type === 'attributes' && record.attributeName === 'selected'+ && record.oldValue !== record.target.getAttribute('selected')) {+ const select = record.target.closest?.('select');+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);+ }const sample = sampleDescendants(record.target);// A bounded layout sample cannot prove that inherited visibility left a// large subtree unchanged. Invalidate conservatively on an actual state@@ -344,7 +351,8 @@let measured = 0;const seedBudget = { remaining: 600 };for (const record of changes) {- const el = record.target.nodeType === 1 ? record.target : record.target.host || record.target.parentElement;+ const el = record.target.nodeType === 1 ? record.target+ : record.target.host || record.target.parentElement || record.target.getRootNode?.().host;if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 32e4fdb53..dc25d35bf 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -526,6 +526,58 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: direct shadow-root text changes reach the host feedback target`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const host = document.createElement('div'); host.id = 'shadow-text-host';+ host.attachShadow({ mode: 'open' }).append(document.createTextNode('Initial result'));+ document.body.append(host);+ });+ await page.waitForTimeout(150);+ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'shadow-text', tool: 'execute_js' });+ deliver('page_monitor_dispatch', { operationId: 'shadow-text', kind: 'dom', fenceOnly: true });+ document.getElementById('shadow-text-host').shadowRoot.firstChild.data = 'Updated result without an element wrapper';+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'+ && event.source === 'page' && event.target === 'div#shadow-text-host'), null, { timeout: 1000 });+ const blocked = await page.evaluate(() => new Promise(resolve => {+ deliver('page_monitor_dispatch', { operationId: 'shadow-text', kind: 'dom' });+ setTimeout(() => resolve(lastMonitorResponse), 0);+ }));+ assert.equal(blocked.pageFeedbackPending, true, 'The text-node mutation must invalidate the prepared revision');+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated result'), false,+ 'Feedback identifies the host without copying page text');+ } finally { await browser.close(); }+ });++ test(`${build}: native option selection mutations invalidate prepared state`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const select = document.createElement('select'); select.id = 'state-select';+ select.innerHTML = '<option>First</option><option id="second-option">Second</option>';+ document.body.append(select);+ });+ await page.waitForTimeout(150);+ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'select-state', tool: 'click', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'select-state', kind: 'click', selector: '#agent', fenceOnly: true });+ document.getElementById('second-option').setAttribute('selected', '');+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'+ && event.source === 'page' && event.target === 'select#state-select'), null, { timeout: 1000 });+ const blocked = await page.evaluate(() => new Promise(resolve => {+ deliver('page_monitor_dispatch', { operationId: 'select-state', kind: 'click', selector: '#agent' });+ setTimeout(() => resolve(lastMonitorResponse), 0);+ }));+ assert.equal(blocked.pageFeedbackPending, true, 'Changing defaultSelected must not leave the observed select state current');+ } finally { await browser.close(); }+ });+test(`${build}: opacity-only visibility changes invalidate prepared actions`, async () => {const { browser, page } = await fixture(engine, build);try {