Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Track editable text and field metadata changes

Barack Sokullu687f6fc4e624
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 964ac15ee..2c89ef6d8 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -360,6 +360,7 @@
[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);
const controlValue = controlValueFingerprint(el);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
+ el.getAttribute('name'), el.getAttribute('placeholder'), el.getAttribute('aria-required'), el.getAttribute('aria-readonly'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),
el.getAttribute('aria-pressed'),
@@ -437,7 +438,8 @@
if (!root || roots.has(root) || !observer) return;
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
+ attributeFilter: ['role', 'aria-label', 'name', 'placeholder', 'aria-required', 'aria-readonly',
+ 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
'aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
listen(root, 'beforetoggle', event => {
@@ -534,7 +536,12 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
- if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;
+ const editableCharacterData = record.type === 'characterData' && editable(el);
+ if (ignored(el)) continue;
+ // Native input already reports user edits, and agent input's synchronous
+ // DOM writes are attributed to its dispatch. Keep those paths deduplicated
+ // while observing independent page-script changes to editable text.
+ if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
if (++measured > 300) {
@@ -555,6 +562,7 @@
}
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
+ || editableCharacterData
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
if (record.type === 'childList') {
identityChanged = subtreeChanged(record, el);
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 964ac15ee..2c89ef6d8 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -360,6 +360,7 @@
[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);
const controlValue = controlValueFingerprint(el);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
+ el.getAttribute('name'), el.getAttribute('placeholder'), el.getAttribute('aria-required'), el.getAttribute('aria-readonly'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),
el.getAttribute('aria-pressed'),
@@ -437,7 +438,8 @@
if (!root || roots.has(root) || !observer) return;
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
- attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
+ attributeFilter: ['role', 'aria-label', 'name', 'placeholder', 'aria-required', 'aria-readonly',
+ 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
'aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
listen(root, 'beforetoggle', event => {
@@ -534,7 +536,12 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
- if (ignored(el) || (record.type === 'characterData' && editable(el))) continue;
+ const editableCharacterData = record.type === 'characterData' && editable(el);
+ if (ignored(el)) continue;
+ // Native input already reports user edits, and agent input's synchronous
+ // DOM writes are attributed to its dispatch. Keep those paths deduplicated
+ // while observing independent page-script changes to editable text.
+ if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
if (++measured > 300) {
@@ -555,6 +562,7 @@
}
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
+ || editableCharacterData
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
if (record.type === 'childList') {
identityChanged = subtreeChanged(record, el);
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 73b243574..c1cd3c02b 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -1532,6 +1532,62 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: programmatic contenteditable text updates produce page feedback`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const editor = document.createElement('div');
+ editor.id = 'programmatic-editor'; editor.contentEditable = 'true'; editor.setAttribute('role', 'textbox');
+ editor.textContent = 'Initial text'; document.body.append(editor);
+ });
+ await page.waitForTimeout(180);
+ await page.evaluate(() => {
+ feedback = [];
+ document.getElementById('programmatic-editor').firstChild.data = 'Page updated this editor';
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ const observations = await page.evaluate(() => feedback);
+ assert.ok(observations.some(event => event.source === 'page' && event.target.startsWith('div#programmatic-editor')));
+ assert.equal(JSON.stringify(observations).includes('Page updated this editor'), false,
+ 'Editable content must invalidate state without being included in feedback');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: accessibility field metadata mutations invalidate prepared input`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const input = document.createElement('input'); input.id = 'metadata-field';
+ input.name = 'original-name'; input.placeholder = 'Original placeholder'; input.setAttribute('aria-required', 'false');
+ input.setAttribute('aria-readonly', 'false'); document.body.append(input);
+ });
+ await page.waitForTimeout(180);
+ const changes = [
+ ['name', 'updated-name'], ['placeholder', 'Updated placeholder'],
+ ['aria-required', 'true'], ['aria-readonly', 'true'],
+ ];
+ for (const [attribute, value] of changes) {
+ const guard = await page.evaluate(attribute => {
+ feedback = [];
+ const operationId = 'metadata-' + attribute;
+ deliver('page_monitor_prepare', { operationId, tool: 'type_text', selector: '#metadata-field' });
+ deliver('page_monitor_dispatch', { operationId, kind: 'input', selector: '#metadata-field', fenceOnly: true });
+ return lastMonitorResponse.guard;
+ }, attribute);
+ await page.locator('#metadata-field').evaluate((element, [name, next]) => element.setAttribute(name, next), [attribute, value]);
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'input#metadata-field'), null, { timeout: 1000 });
+ const ready = await page.evaluate(value => {
+ deliver('page_monitor_validate', { ...value, kind: 'input' });
+ return lastMonitorResponse.ready;
+ }, guard);
+ assert.equal(ready, false, `${attribute} changes must invalidate an existing input guard`);
+ const serialized = await page.evaluate(() => JSON.stringify(feedback));
+ assert.equal(serialized.includes(value), false, 'Field metadata values must not be included in feedback');
+ await page.waitForTimeout(180);
+ }
+ } finally { await browser.close(); }
+ });
+
test(`${build}: button type changes invalidate a prepared click`, async () => {
const { browser, page } = await fixture(engine, build);
try {