Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: observe full text and aria-pressed state changes
Barack Sokullu19bd7d5b72b6
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.mdindex d2cd5cccd..fd909192b 100644--- a/docs/page-feedback-monitor.md+++ b/docs/page-feedback-monitor.md@@ -144,5 +144,9 @@ monitor and MAIN shadow-hook registrations. Fixtures model these registrationsand verify private-input, DOM and late-shadow feedback inside those documents.Document and local-action tokens use cryptographic random bytes when the documentdoes not expose the secure-context-only randomUUID API.+Text signatures cover the whole non-editable text with a cached fixed-size+fingerprint, so middle/suffix edits and appends beyond the first 200 characters+invalidate prepared actions even when geometry stays fixed. Accessibility-state+regressions also cover aria-pressed-only toggle changes and their agent attribution.The monitor regressions are also included in `npm run test:runtime-lifecycle`.diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex c0eafbe92..dded97dd4 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -15,6 +15,7 @@const operations = new Map();const nativeTargets = new Set();let signatures = new WeakMap();+ let textSignatures = new WeakMap();let roots = new WeakSet();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;@@ -155,14 +156,30 @@}return null;}+ function textSignature(node) {+ const text = node.textContent || '', cached = textSignatures.get(node);+ if (cached?.text === text) return cached.signature;+ // Keep the fingerprint fixed-size while covering the entire text, including+ // equal-length middle/suffix edits. Unchanged nodes avoid rehashing on layout reads.+ let hash = 0, power = 1;+ for (let i = 0; i < text.length; i++) {+ hash = (Math.imul(hash, 16777619) + text.charCodeAt(i)) >>> 0;+ power = Math.imul(power, 16777619) >>> 0;+ }+ const value = [text.length, hash, power];+ textSignatures.set(node, { text, signature: value });+ return value;+ }function signature(el) {if (ignored(el)) return '';const shown = visible(el);const rect = shown ? el.getBoundingClientRect() : {};const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3)- .map(node => node.textContent).join('').slice(0, 200);+ .map(textSignature).reduce(([size, hash], [length, part, power]) =>+ [size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),+ el.getAttribute('aria-pressed'),el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);@@ -179,7 +196,7 @@if (!root || roots.has(root) || !observer) return;roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',+ attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });seed(root, budget);}@@ -424,7 +441,7 @@clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;- roots = new WeakSet(); signatures = new WeakMap();+ roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap();}async function requestState() {const generation = ++requestGeneration;diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex c0eafbe92..dded97dd4 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -15,6 +15,7 @@const operations = new Map();const nativeTargets = new Set();let signatures = new WeakMap();+ let textSignatures = new WeakMap();let roots = new WeakSet();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;let observer = null, domTimer = null, scrollTimer = null, lastUserAt = 0;@@ -155,14 +156,30 @@}return null;}+ function textSignature(node) {+ const text = node.textContent || '', cached = textSignatures.get(node);+ if (cached?.text === text) return cached.signature;+ // Keep the fingerprint fixed-size while covering the entire text, including+ // equal-length middle/suffix edits. Unchanged nodes avoid rehashing on layout reads.+ let hash = 0, power = 1;+ for (let i = 0; i < text.length; i++) {+ hash = (Math.imul(hash, 16777619) + text.charCodeAt(i)) >>> 0;+ power = Math.imul(power, 16777619) >>> 0;+ }+ const value = [text.length, hash, power];+ textSignatures.set(node, { text, signature: value });+ return value;+ }function signature(el) {if (ignored(el)) return '';const shown = visible(el);const rect = shown ? el.getBoundingClientRect() : {};const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3)- .map(node => node.textContent).join('').slice(0, 200);+ .map(textSignature).reduce(([size, hash], [length, part, power]) =>+ [size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),+ el.getAttribute('aria-pressed'),el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);@@ -179,7 +196,7 @@if (!root || roots.has(root) || !observer) return;roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled',+ attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });seed(root, budget);}@@ -424,7 +441,7 @@clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;- roots = new WeakSet(); signatures = new WeakMap();+ roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap();}async function requestState() {const generation = ++requestGeneration;diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex af610ead0..7caad8b04 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -683,10 +683,15 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {await page.evaluate(() => document.getElementById('prepared-field').focus({ preventScroll: true }));await page.keyboard.press('x');await page.waitForFunction(() => feedback.some(event => event.kind === 'input' && event.source === 'user'), null, { timeout: 1000 });- await page.evaluate(() => { monitorEnabled = false; deliver('page_monitor_state'); document.getElementById('prep-scroll-box').scrollTop = 0; window.scrollTo(0, 0); });+ await page.evaluate(() => {+ monitorEnabled = false; deliver('page_monitor_state'); document.activeElement?.blur();+ document.getElementById('prep-scroll-box').scrollTop = 0; window.scrollTo(0, 0);+ });await page.waitForTimeout(50);await page.evaluate(() => { monitorEnabled = true; deliver('page_monitor_state'); });await page.waitForTimeout(20);+ assert.equal(await page.evaluate(() => document.getElementById('prep-scroll-box').scrollTop), 0,+ 'Keep the target offscreen before testing a rejected preparation, without browser caret scrolling');const result = await page.evaluate(async () => {const finish = __wbPageMonitor.beginContentAction('type_ax', { ref_id: 'prepared-field', _bidiPrepare: { action: 'type' } });document.getElementById('status').textContent = 'Intervening page change'; await Promise.resolve();@@ -1066,6 +1071,80 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: long text edits beyond the initial prefix invalidate prepared actions`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const output = document.createElement('p'); output.id = 'long-output';+ output.style.cssText = 'width:800px;height:160px;font:12px/14px monospace;word-break:break-all';+ output.textContent = 'A'.repeat(260) + ' middle OLD ' + 'B'.repeat(260) + ' suffix OLD';+ document.body.prepend(output);+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');+ });+ await page.waitForTimeout(200);+ for (const edit of ['middle', 'suffix', 'append']) {+ const result = await page.evaluate(async edit => {+ feedback = []; const output = document.getElementById('long-output'), before = output.getBoundingClientRect();+ const prefix = output.textContent.slice(0, 200);+ const finish = __wbPageMonitor.beginContentAction('click', { selector: '#agent', _bidiPrepare: true });+ if (edit === 'middle') output.firstChild.data = output.firstChild.data.replace('middle OLD', 'middle NEW');+ else if (edit === 'suffix') output.textContent = output.textContent.replace('suffix OLD', 'suffix NEW');+ else output.firstChild.appendData(' additional output');+ await Promise.resolve();+ let code; try { __wbPageMonitor.beforeLocalDispatch(); } catch (error) { code = error.code; } finally { finish(); }+ const after = output.getBoundingClientRect();+ return { code, sameGeometry: before.width === after.width && before.height === after.height,+ samePrefix: prefix === output.textContent.slice(0, 200) };+ }, edit);+ assert.deepEqual(result, { code: 'page_feedback_pending', sameGeometry: true, samePrefix: true });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ }+ await page.evaluate(() => { feedback = []; const output = document.getElementById('long-output'); output.textContent = output.textContent; });+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Replacing an identical text node must remain coalesced');+ await page.evaluate(() => {+ feedback = []; document.getElementById('long-output').firstChild.splitText(270);+ });+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Splitting unchanged text must retain the same content signature');+ await page.evaluate(() => {+ feedback = []; const output = document.getElementById('long-output'); output.normalize(); output.append(document.createTextNode(''));+ });+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Normalizing or adding empty text must not produce feedback');+ } finally { await browser.close(); }+ });++ test(`${build}: aria-pressed-only toggle changes invalidate preparation and preserve agent attribution`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const toggle = document.getElementById('human'); toggle.setAttribute('aria-pressed', 'false');+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');+ });+ await page.waitForTimeout(200);+ for (const state of ['true', 'mixed', 'false']) {+ const code = await page.evaluate(async state => {+ feedback = []; const finish = __wbPageMonitor.beginContentAction('click', { selector: '#human', _bidiPrepare: true });+ document.getElementById('human').setAttribute('aria-pressed', state);+ await Promise.resolve();+ try { __wbPageMonitor.beforeLocalDispatch(); return null; }+ catch (error) { return error.code; }+ finally { finish(); }+ }, state);+ assert.equal(code, 'page_feedback_pending');+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button#human'), null, { timeout: 1000 });+ }+ await page.evaluate(() => {+ feedback = []; const finish = __wbPageMonitor.beginContentAction('execute_js');+ try { __wbPageMonitor.beforeLocalDispatch(); document.getElementById('human').setAttribute('aria-pressed', 'true'); }+ finally { finish(); }+ });+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'A marked synchronous toggle must not feed back into its own run');+ } finally { await browser.close(); }+ });+test(`${build}: upgrading an existing host observes its newly attached shadow root`, async () => {const { browser, page } = await fixture(engine, build);try {