Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

cover all form state and hidden native labels

Barack Sokullu9e6f314d730c
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 49cf26b38..7404455ad 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -352,7 +352,7 @@
controlValueFingerprint(el)]);
}
function trackControl(el) {
- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;
+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;
liveControls.add(el);
controlSignatures.set(el, controlState(el));
}
@@ -398,7 +398,7 @@
function seed(root, budget = { remaining: 600 }) {
const nodes = root.querySelectorAll?.('*') || [];
for (const el of nodes) {
- // Form-state tracking has its own cap, separate from bounded layout reads.
+ // Keep a baseline for every control; periodic reads rotate through them in bounded batches.
trackControl(el);
if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }
// Root discovery is cheap and must not share the layout-measurement cap.
@@ -529,6 +529,20 @@
}
return false;
}
+ function hasVisibleNativeLabelConsumer(el, previousFor = '') {
+ let label = null;
+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {
+ if (node.tagName === 'LABEL') { label = node; break; }
+ }
+ if (!label) return false;
+ const visibleControl = control => control instanceof Element && !ignored(control) && visible(control);
+ if (visibleControl(label.control)) return true;
+ if (label !== el || !previousFor) return false;
+ const root = label.getRootNode?.();
+ const priorControl = root?.getElementById?.(previousFor)
+ || (root === document ? document.getElementById(previousFor) : null);
+ return visibleControl(priorControl);
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -598,9 +612,13 @@
if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
continue;
}
+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);
+ const nativeLabelAssociationMutation = record.type === 'attributes'
+ && record.attributeName === 'for' && el.tagName === 'LABEL';
const hiddenAccessibleNameChanged = !visible(el)
- && ['characterData', 'childList'].includes(record.type)
- && hasVisibleAriaLabelledbyConsumer(el);
+ && ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))
+ || ((accessibleNameContentMutation || nativeLabelAssociationMutation)
+ && hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
|| editableTextMutation || hiddenAccessibleNameChanged
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 49cf26b38..7404455ad 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -352,7 +352,7 @@
controlValueFingerprint(el)]);
}
function trackControl(el) {
- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;
+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;
liveControls.add(el);
controlSignatures.set(el, controlState(el));
}
@@ -398,7 +398,7 @@
function seed(root, budget = { remaining: 600 }) {
const nodes = root.querySelectorAll?.('*') || [];
for (const el of nodes) {
- // Form-state tracking has its own cap, separate from bounded layout reads.
+ // Keep a baseline for every control; periodic reads rotate through them in bounded batches.
trackControl(el);
if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }
// Root discovery is cheap and must not share the layout-measurement cap.
@@ -529,6 +529,20 @@
}
return false;
}
+ function hasVisibleNativeLabelConsumer(el, previousFor = '') {
+ let label = null;
+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {
+ if (node.tagName === 'LABEL') { label = node; break; }
+ }
+ if (!label) return false;
+ const visibleControl = control => control instanceof Element && !ignored(control) && visible(control);
+ if (visibleControl(label.control)) return true;
+ if (label !== el || !previousFor) return false;
+ const root = label.getRootNode?.();
+ const priorControl = root?.getElementById?.(previousFor)
+ || (root === document ? document.getElementById(previousFor) : null);
+ return visibleControl(priorControl);
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -598,9 +612,13 @@
if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
continue;
}
+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);
+ const nativeLabelAssociationMutation = record.type === 'attributes'
+ && record.attributeName === 'for' && el.tagName === 'LABEL';
const hiddenAccessibleNameChanged = !visible(el)
- && ['characterData', 'childList'].includes(record.type)
- && hasVisibleAriaLabelledbyConsumer(el);
+ && ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))
+ || ((accessibleNameContentMutation || nativeLabelAssociationMutation)
+ && hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
|| editableTextMutation || hiddenAccessibleNameChanged
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 91c452a43..97e820ac0 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -1808,6 +1808,68 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: hidden native label changes invalidate visible control clicks`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const form = document.createElement('form');
+ form.innerHTML = '<label id="hidden-native-label" for="native-field" style="display:none">Before</label>'
+ + '<input id="native-field"><input id="other-field">';
+ document.body.append(form);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));
+ await page.waitForTimeout(180);
+
+ const prepare = operationId => page.evaluate(id => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: id, tool: 'click', selector: '#native-field' });
+ deliver('page_monitor_dispatch', { operationId: id, kind: 'click', selector: '#native-field' });
+ return lastMonitorResponse.guard;
+ }, operationId);
+ const assertStale = guard => page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'focus' }), cancelable: true,
+ })), guard);
+
+ const textGuard = await prepare('native-label-text');
+ await page.locator('#hidden-native-label').evaluate(label => { label.firstChild.data = 'Updated hidden name'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'label#hidden-native-label'));
+ assert.equal(await assertStale(textGuard), false, 'Changing hidden native label text must stale its visible control guard');
+
+ const associationGuard = await prepare('native-label-for');
+ await page.locator('#hidden-native-label').evaluate(label => label.setAttribute('for', 'other-field'));
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'label#hidden-native-label'));
+ assert.equal(await assertStale(associationGuard), false, 'Reassigning a hidden native label must stale its previous control guard');
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated hidden name'), false,
+ 'Hidden label text must not be copied into feedback');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: controls after a large select remain in rotating form-state coverage`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const form = document.createElement('form');
+ const select = document.createElement('select');
+ select.id = 'large-select';
+ select.innerHTML = Array.from({ length: 610 }, (_, index) => `<option value="${index}">Option ${index}</option>`).join('');
+ const input = document.createElement('input');
+ input.id = 'late-form-control';
+ input.value = 'before';
+ form.append(select, input);
+ document.body.append(form);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));
+ await page.waitForTimeout(180);
+ await page.evaluate(() => {
+ feedback = [];
+ document.getElementById('late-form-control').value = 'sensitive-after';
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'input#late-form-control'), null, { timeout: 5000 });
+ const serialized = JSON.stringify(await page.evaluate(() => feedback));
+ assert.equal(serialized.includes('sensitive-after'), false, 'Detected property values must stay private');
+ } finally { await browser.close(); }
+ });
+
for (const action of ['click', 'type']) {
test(`${build}: ${action} preparation preserves same-target human input`, async () => {
const { browser, page } = await fixture(engine, build);