Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: observe full text and aria-pressed state changes

Barack Sokullu19bd7d5b72b6
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.md
index d2cd5cccd..fd909192b 100644
--- a/docs/page-feedback-monitor.md
+++ b/docs/page-feedback-monitor.md
@@ -144,5 +144,9 @@ monitor and MAIN shadow-hook registrations. Fixtures model these registrations
and verify private-input, DOM and late-shadow feedback inside those documents.
Document and local-action tokens use cryptographic random bytes when the document
does not expose the secure-context-only randomUUID API.
+Text signatures cover the whole non-editable text with a cached fixed-size
+fingerprint, so middle/suffix edits and appends beyond the first 200 characters
+invalidate prepared actions even when geometry stays fixed. Accessibility-state
+regressions also cover aria-pressed-only toggle changes and their agent attribution.
The monitor regressions are also included in `npm run test:runtime-lifecycle`.
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index c0eafbe92..dded97dd4 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -15,6 +15,7 @@
const operations = new Map();
const nativeTargets = new Set();
let signatures = new WeakMap();
+ let textSignatures = new WeakMap();
let roots = new WeakSet();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;
@@ -155,14 +156,30 @@
}
return null;
}
+ function textSignature(node) {
+ const text = node.textContent || '', cached = textSignatures.get(node);
+ if (cached?.text === text) return cached.signature;
+ // Keep the fingerprint fixed-size while covering the entire text, including
+ // equal-length middle/suffix edits. Unchanged nodes avoid rehashing on layout reads.
+ let hash = 0, power = 1;
+ for (let i = 0; i < text.length; i++) {
+ hash = (Math.imul(hash, 16777619) + text.charCodeAt(i)) >>> 0;
+ power = Math.imul(power, 16777619) >>> 0;
+ }
+ const value = [text.length, hash, power];
+ textSignatures.set(node, { text, signature: value });
+ return value;
+ }
function signature(el) {
if (ignored(el)) return '';
const shown = visible(el);
const rect = shown ? el.getBoundingClientRect() : {};
const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3)
- .map(node => node.textContent).join('').slice(0, 200);
+ .map(textSignature).reduce(([size, hash], [length, part, power]) =>
+ [size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
+ el.getAttribute('aria-pressed'),
el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
@@ -179,7 +196,7 @@
if (!root || roots.has(root) || !observer) return;
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',
+ attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });
seed(root, budget);
}
@@ -424,7 +441,7 @@
clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
- roots = new WeakSet(); signatures = new WeakMap();
+ roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap();
}
async function requestState() {
const generation = ++requestGeneration;
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index c0eafbe92..dded97dd4 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -15,6 +15,7 @@
const operations = new Map();
const nativeTargets = new Set();
let signatures = new WeakMap();
+ let textSignatures = new WeakMap();
let roots = new WeakSet();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;
@@ -155,14 +156,30 @@
}
return null;
}
+ function textSignature(node) {
+ const text = node.textContent || '', cached = textSignatures.get(node);
+ if (cached?.text === text) return cached.signature;
+ // Keep the fingerprint fixed-size while covering the entire text, including
+ // equal-length middle/suffix edits. Unchanged nodes avoid rehashing on layout reads.
+ let hash = 0, power = 1;
+ for (let i = 0; i < text.length; i++) {
+ hash = (Math.imul(hash, 16777619) + text.charCodeAt(i)) >>> 0;
+ power = Math.imul(power, 16777619) >>> 0;
+ }
+ const value = [text.length, hash, power];
+ textSignatures.set(node, { text, signature: value });
+ return value;
+ }
function signature(el) {
if (ignored(el)) return '';
const shown = visible(el);
const rect = shown ? el.getBoundingClientRect() : {};
const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3)
- .map(node => node.textContent).join('').slice(0, 200);
+ .map(textSignature).reduce(([size, hash], [length, part, power]) =>
+ [size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
+ el.getAttribute('aria-pressed'),
el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
@@ -179,7 +196,7 @@
if (!root || roots.has(root) || !observer) return;
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',
+ attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });
seed(root, budget);
}
@@ -424,7 +441,7 @@
clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
- roots = new WeakSet(); signatures = new WeakMap();
+ roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap();
}
async function requestState() {
const generation = ++requestGeneration;
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index af610ead0..7caad8b04 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -683,10 +683,15 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
await page.evaluate(() => document.getElementById('prepared-field').focus({ preventScroll: true }));
await page.keyboard.press('x');
await page.waitForFunction(() => feedback.some(event => event.kind === 'input' && event.source === 'user'), null, { timeout: 1000 });
- await page.evaluate(() => { monitorEnabled = false; deliver('page_monitor_state'); document.getElementById('prep-scroll-box').scrollTop = 0; window.scrollTo(0, 0); });
+ await page.evaluate(() => {
+ monitorEnabled = false; deliver('page_monitor_state'); document.activeElement?.blur();
+ document.getElementById('prep-scroll-box').scrollTop = 0; window.scrollTo(0, 0);
+ });
await page.waitForTimeout(50);
await page.evaluate(() => { monitorEnabled = true; deliver('page_monitor_state'); });
await page.waitForTimeout(20);
+ assert.equal(await page.evaluate(() => document.getElementById('prep-scroll-box').scrollTop), 0,
+ 'Keep the target offscreen before testing a rejected preparation, without browser caret scrolling');
const result = await page.evaluate(async () => {
const finish = __wbPageMonitor.beginContentAction('type_ax', { ref_id: 'prepared-field', _bidiPrepare: { action: 'type' } });
document.getElementById('status').textContent = 'Intervening page change'; await Promise.resolve();
@@ -1066,6 +1071,80 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: long text edits beyond the initial prefix invalidate prepared actions`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const output = document.createElement('p'); output.id = 'long-output';
+ output.style.cssText = 'width:800px;height:160px;font:12px/14px monospace;word-break:break-all';
+ output.textContent = 'A'.repeat(260) + ' middle OLD ' + 'B'.repeat(260) + ' suffix OLD';
+ document.body.prepend(output);
+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');
+ });
+ await page.waitForTimeout(200);
+ for (const edit of ['middle', 'suffix', 'append']) {
+ const result = await page.evaluate(async edit => {
+ feedback = []; const output = document.getElementById('long-output'), before = output.getBoundingClientRect();
+ const prefix = output.textContent.slice(0, 200);
+ const finish = __wbPageMonitor.beginContentAction('click', { selector: '#agent', _bidiPrepare: true });
+ if (edit === 'middle') output.firstChild.data = output.firstChild.data.replace('middle OLD', 'middle NEW');
+ else if (edit === 'suffix') output.textContent = output.textContent.replace('suffix OLD', 'suffix NEW');
+ else output.firstChild.appendData(' additional output');
+ await Promise.resolve();
+ let code; try { __wbPageMonitor.beforeLocalDispatch(); } catch (error) { code = error.code; } finally { finish(); }
+ const after = output.getBoundingClientRect();
+ return { code, sameGeometry: before.width === after.width && before.height === after.height,
+ samePrefix: prefix === output.textContent.slice(0, 200) };
+ }, edit);
+ assert.deepEqual(result, { code: 'page_feedback_pending', sameGeometry: true, samePrefix: true });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ }
+ await page.evaluate(() => { feedback = []; const output = document.getElementById('long-output'); output.textContent = output.textContent; });
+ await page.waitForTimeout(200);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Replacing an identical text node must remain coalesced');
+ await page.evaluate(() => {
+ feedback = []; document.getElementById('long-output').firstChild.splitText(270);
+ });
+ await page.waitForTimeout(200);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Splitting unchanged text must retain the same content signature');
+ await page.evaluate(() => {
+ feedback = []; const output = document.getElementById('long-output'); output.normalize(); output.append(document.createTextNode(''));
+ });
+ await page.waitForTimeout(200);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Normalizing or adding empty text must not produce feedback');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: aria-pressed-only toggle changes invalidate preparation and preserve agent attribution`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const toggle = document.getElementById('human'); toggle.setAttribute('aria-pressed', 'false');
+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');
+ });
+ await page.waitForTimeout(200);
+ for (const state of ['true', 'mixed', 'false']) {
+ const code = await page.evaluate(async state => {
+ feedback = []; const finish = __wbPageMonitor.beginContentAction('click', { selector: '#human', _bidiPrepare: true });
+ document.getElementById('human').setAttribute('aria-pressed', state);
+ await Promise.resolve();
+ try { __wbPageMonitor.beforeLocalDispatch(); return null; }
+ catch (error) { return error.code; }
+ finally { finish(); }
+ }, state);
+ assert.equal(code, 'page_feedback_pending');
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button#human'), null, { timeout: 1000 });
+ }
+ await page.evaluate(() => {
+ feedback = []; const finish = __wbPageMonitor.beginContentAction('execute_js');
+ try { __wbPageMonitor.beforeLocalDispatch(); document.getElementById('human').setAttribute('aria-pressed', 'true'); }
+ finally { finish(); }
+ });
+ await page.waitForTimeout(200);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'A marked synchronous toggle must not feed back into its own run');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: upgrading an existing host observes its newly attached shadow root`, async () => {
const { browser, page } = await fixture(engine, build);
try {