Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Track editable text and field metadata changes
Barack Sokullu687f6fc4e624
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 964ac15ee..2c89ef6d8 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -360,6 +360,7 @@[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);const controlValue = controlValueFingerprint(el);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),+ el.getAttribute('name'), el.getAttribute('placeholder'), el.getAttribute('aria-required'), el.getAttribute('aria-readonly'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),el.getAttribute('aria-pressed'),@@ -437,7 +438,8 @@if (!root || roots.has(root) || !observer) return;roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',+ attributeFilter: ['role', 'aria-label', 'name', 'placeholder', 'aria-required', 'aria-readonly',+ 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget','aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });listen(root, 'beforetoggle', event => {@@ -534,7 +536,12 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;- if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;+ const editableCharacterData = record.type === 'characterData' && editable(el);+ if (ignored(el)) continue;+ // Native input already reports user edits, and agent input's synchronous+ // DOM writes are attributed to its dispatch. Keep those paths deduplicated+ // while observing independent page-script changes to editable text.+ if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;if (++measured > 300) {@@ -555,6 +562,7 @@}let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)+ || editableCharacterData|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));if (record.type === 'childList') {identityChanged = subtreeChanged(record, el);diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 964ac15ee..2c89ef6d8 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -360,6 +360,7 @@[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);const controlValue = controlValueFingerprint(el);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),+ el.getAttribute('name'), el.getAttribute('placeholder'), el.getAttribute('aria-required'), el.getAttribute('aria-readonly'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),el.getAttribute('aria-pressed'),@@ -437,7 +438,8 @@if (!root || roots.has(root) || !observer) return;roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',+ attributeFilter: ['role', 'aria-label', 'name', 'placeholder', 'aria-required', 'aria-readonly',+ 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget','aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });listen(root, 'beforetoggle', event => {@@ -534,7 +536,12 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;- if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;+ const editableCharacterData = record.type === 'characterData' && editable(el);+ if (ignored(el)) continue;+ // Native input already reports user edits, and agent input's synchronous+ // DOM writes are attributed to its dispatch. Keep those paths deduplicated+ // while observing independent page-script changes to editable text.+ if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;if (++measured > 300) {@@ -555,6 +562,7 @@}let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)+ || editableCharacterData|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));if (record.type === 'childList') {identityChanged = subtreeChanged(record, el);diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 73b243574..c1cd3c02b 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -1532,6 +1532,62 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: programmatic contenteditable text updates produce page feedback`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const editor = document.createElement('div');+ editor.id = 'programmatic-editor'; editor.contentEditable = 'true'; editor.setAttribute('role', 'textbox');+ editor.textContent = 'Initial text'; document.body.append(editor);+ });+ await page.waitForTimeout(180);+ await page.evaluate(() => {+ feedback = [];+ document.getElementById('programmatic-editor').firstChild.data = 'Page updated this editor';+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ const observations = await page.evaluate(() => feedback);+ assert.ok(observations.some(event => event.source === 'page' && event.target.startsWith('div#programmatic-editor')));+ assert.equal(JSON.stringify(observations).includes('Page updated this editor'), false,+ 'Editable content must invalidate state without being included in feedback');+ } finally { await browser.close(); }+ });++ test(`${build}: accessibility field metadata mutations invalidate prepared input`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const input = document.createElement('input'); input.id = 'metadata-field';+ input.name = 'original-name'; input.placeholder = 'Original placeholder'; input.setAttribute('aria-required', 'false');+ input.setAttribute('aria-readonly', 'false'); document.body.append(input);+ });+ await page.waitForTimeout(180);+ const changes = [+ ['name', 'updated-name'], ['placeholder', 'Updated placeholder'],+ ['aria-required', 'true'], ['aria-readonly', 'true'],+ ];+ for (const [attribute, value] of changes) {+ const guard = await page.evaluate(attribute => {+ feedback = [];+ const operationId = 'metadata-' + attribute;+ deliver('page_monitor_prepare', { operationId, tool: 'type_text', selector: '#metadata-field' });+ deliver('page_monitor_dispatch', { operationId, kind: 'input', selector: '#metadata-field', fenceOnly: true });+ return lastMonitorResponse.guard;+ }, attribute);+ await page.locator('#metadata-field').evaluate((element, [name, next]) => element.setAttribute(name, next), [attribute, value]);+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'input#metadata-field'), null, { timeout: 1000 });+ const ready = await page.evaluate(value => {+ deliver('page_monitor_validate', { ...value, kind: 'input' });+ return lastMonitorResponse.ready;+ }, guard);+ assert.equal(ready, false, `${attribute} changes must invalidate an existing input guard`);+ const serialized = await page.evaluate(() => JSON.stringify(feedback));+ assert.equal(serialized.includes(value), false, 'Field metadata values must not be included in feedback');+ await page.waitForTimeout(180);+ }+ } finally { await browser.close(); }+ });+test(`${build}: button type changes invalidate a prepared click`, async () => {const { browser, page } = await fixture(engine, build);try {