Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

track shadow text and select state changes

Barack Sokullu842f11551412
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index eae77576e..01c5fdd19 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -228,6 +228,8 @@
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-pressed'),
el.getAttribute('popover'), popoverOpen(el),
+ el.selected === true, el.defaultSelected === true,
+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,
el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
@@ -325,9 +327,14 @@
};
const changes = [...records];
for (const record of records) {
- if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName))) continue;
+ if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;
if (ignored(record.target)) continue;
if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
+ if (record.type === 'attributes' && record.attributeName === 'selected'
+ && record.oldValue !== record.target.getAttribute('selected')) {
+ const select = record.target.closest?.('select');
+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);
+ }
const sample = sampleDescendants(record.target);
// A bounded layout sample cannot prove that inherited visibility left a
// large subtree unchanged. Invalidate conservatively on an actual state
@@ -344,7 +351,8 @@
let measured = 0;
const seedBudget = { remaining: 600 };
for (const record of changes) {
- const el = record.target.nodeType === 1 ? record.target : record.target.host || record.target.parentElement;
+ const el = record.target.nodeType === 1 ? record.target
+ : record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index eae77576e..01c5fdd19 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -228,6 +228,8 @@
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-pressed'),
el.getAttribute('popover'), popoverOpen(el),
+ el.selected === true, el.defaultSelected === true,
+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,
el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
@@ -325,9 +327,14 @@
};
const changes = [...records];
for (const record of records) {
- if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName))) continue;
+ if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert', 'selected'].includes(record.attributeName))) continue;
if (ignored(record.target)) continue;
if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
+ if (record.type === 'attributes' && record.attributeName === 'selected'
+ && record.oldValue !== record.target.getAttribute('selected')) {
+ const select = record.target.closest?.('select');
+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);
+ }
const sample = sampleDescendants(record.target);
// A bounded layout sample cannot prove that inherited visibility left a
// large subtree unchanged. Invalidate conservatively on an actual state
@@ -344,7 +351,8 @@
let measured = 0;
const seedBudget = { remaining: 600 };
for (const record of changes) {
- const el = record.target.nodeType === 1 ? record.target : record.target.host || record.target.parentElement;
+ const el = record.target.nodeType === 1 ? record.target
+ : record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 32e4fdb53..dc25d35bf 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -526,6 +526,58 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: direct shadow-root text changes reach the host feedback target`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const host = document.createElement('div'); host.id = 'shadow-text-host';
+ host.attachShadow({ mode: 'open' }).append(document.createTextNode('Initial result'));
+ document.body.append(host);
+ });
+ await page.waitForTimeout(150);
+ await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'shadow-text', tool: 'execute_js' });
+ deliver('page_monitor_dispatch', { operationId: 'shadow-text', kind: 'dom', fenceOnly: true });
+ document.getElementById('shadow-text-host').shadowRoot.firstChild.data = 'Updated result without an element wrapper';
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'
+ && event.source === 'page' && event.target === 'div#shadow-text-host'), null, { timeout: 1000 });
+ const blocked = await page.evaluate(() => new Promise(resolve => {
+ deliver('page_monitor_dispatch', { operationId: 'shadow-text', kind: 'dom' });
+ setTimeout(() => resolve(lastMonitorResponse), 0);
+ }));
+ assert.equal(blocked.pageFeedbackPending, true, 'The text-node mutation must invalidate the prepared revision');
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated result'), false,
+ 'Feedback identifies the host without copying page text');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: native option selection mutations invalidate prepared state`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const select = document.createElement('select'); select.id = 'state-select';
+ select.innerHTML = '<option>First</option><option id="second-option">Second</option>';
+ document.body.append(select);
+ });
+ await page.waitForTimeout(150);
+ await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'select-state', tool: 'click', selector: '#agent' });
+ deliver('page_monitor_dispatch', { operationId: 'select-state', kind: 'click', selector: '#agent', fenceOnly: true });
+ document.getElementById('second-option').setAttribute('selected', '');
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'
+ && event.source === 'page' && event.target === 'select#state-select'), null, { timeout: 1000 });
+ const blocked = await page.evaluate(() => new Promise(resolve => {
+ deliver('page_monitor_dispatch', { operationId: 'select-state', kind: 'click', selector: '#agent' });
+ setTimeout(() => resolve(lastMonitorResponse), 0);
+ }));
+ assert.equal(blocked.pageFeedbackPending, true, 'Changing defaultSelected must not leave the observed select state current');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: opacity-only visibility changes invalidate prepared actions`, async () => {
const { browser, page } = await fixture(engine, build);
try {