Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
cover all form state and hidden native labels
Barack Sokullu9e6f314d730c
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 49cf26b38..7404455ad 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -352,7 +352,7 @@controlValueFingerprint(el)]);}function trackControl(el) {- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;liveControls.add(el);controlSignatures.set(el, controlState(el));}@@ -398,7 +398,7 @@function seed(root, budget = { remaining: 600 }) {const nodes = root.querySelectorAll?.('*') || [];for (const el of nodes) {- // Form-state tracking has its own cap, separate from bounded layout reads.+ // Keep a baseline for every control; periodic reads rotate through them in bounded batches.trackControl(el);if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }// Root discovery is cheap and must not share the layout-measurement cap.@@ -529,6 +529,20 @@}return false;}+ function hasVisibleNativeLabelConsumer(el, previousFor = '') {+ let label = null;+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {+ if (node.tagName === 'LABEL') { label = node; break; }+ }+ if (!label) return false;+ const visibleControl = control => control instanceof Element && !ignored(control) && visible(control);+ if (visibleControl(label.control)) return true;+ if (label !== el || !previousFor) return false;+ const root = label.getRootNode?.();+ const priorControl = root?.getElementById?.(previousFor)+ || (root === document ? document.getElementById(previousFor) : null);+ return visibleControl(priorControl);+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -598,9 +612,13 @@if (nextControl !== previousControl) noteChange(el, record.agentUserAt);continue;}+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);+ const nativeLabelAssociationMutation = record.type === 'attributes'+ && record.attributeName === 'for' && el.tagName === 'LABEL';const hiddenAccessibleNameChanged = !visible(el)- && ['characterData', 'childList'].includes(record.type)- && hasVisibleAriaLabelledbyConsumer(el);+ && ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))+ || ((accessibleNameContentMutation || nativeLabelAssociationMutation)+ && hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)|| editableTextMutation || hiddenAccessibleNameChangeddiff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 49cf26b38..7404455ad 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -352,7 +352,7 @@controlValueFingerprint(el)]);}function trackControl(el) {- if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el)) return;liveControls.add(el);controlSignatures.set(el, controlState(el));}@@ -398,7 +398,7 @@function seed(root, budget = { remaining: 600 }) {const nodes = root.querySelectorAll?.('*') || [];for (const el of nodes) {- // Form-state tracking has its own cap, separate from bounded layout reads.+ // Keep a baseline for every control; periodic reads rotate through them in bounded batches.trackControl(el);if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }// Root discovery is cheap and must not share the layout-measurement cap.@@ -529,6 +529,20 @@}return false;}+ function hasVisibleNativeLabelConsumer(el, previousFor = '') {+ let label = null;+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {+ if (node.tagName === 'LABEL') { label = node; break; }+ }+ if (!label) return false;+ const visibleControl = control => control instanceof Element && !ignored(control) && visible(control);+ if (visibleControl(label.control)) return true;+ if (label !== el || !previousFor) return false;+ const root = label.getRootNode?.();+ const priorControl = root?.getElementById?.(previousFor)+ || (root === document ? document.getElementById(previousFor) : null);+ return visibleControl(priorControl);+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -598,9 +612,13 @@if (nextControl !== previousControl) noteChange(el, record.agentUserAt);continue;}+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);+ const nativeLabelAssociationMutation = record.type === 'attributes'+ && record.attributeName === 'for' && el.tagName === 'LABEL';const hiddenAccessibleNameChanged = !visible(el)- && ['characterData', 'childList'].includes(record.type)- && hasVisibleAriaLabelledbyConsumer(el);+ && ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))+ || ((accessibleNameContentMutation || nativeLabelAssociationMutation)+ && hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)|| editableTextMutation || hiddenAccessibleNameChangeddiff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 91c452a43..97e820ac0 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -1808,6 +1808,68 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: hidden native label changes invalidate visible control clicks`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const form = document.createElement('form');+ form.innerHTML = '<label id="hidden-native-label" for="native-field" style="display:none">Before</label>'+ + '<input id="native-field"><input id="other-field">';+ document.body.append(form);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));+ await page.waitForTimeout(180);++ const prepare = operationId => page.evaluate(id => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: id, tool: 'click', selector: '#native-field' });+ deliver('page_monitor_dispatch', { operationId: id, kind: 'click', selector: '#native-field' });+ return lastMonitorResponse.guard;+ }, operationId);+ const assertStale = guard => page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'focus' }), cancelable: true,+ })), guard);++ const textGuard = await prepare('native-label-text');+ await page.locator('#hidden-native-label').evaluate(label => { label.firstChild.data = 'Updated hidden name'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'label#hidden-native-label'));+ assert.equal(await assertStale(textGuard), false, 'Changing hidden native label text must stale its visible control guard');++ const associationGuard = await prepare('native-label-for');+ await page.locator('#hidden-native-label').evaluate(label => label.setAttribute('for', 'other-field'));+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'label#hidden-native-label'));+ assert.equal(await assertStale(associationGuard), false, 'Reassigning a hidden native label must stale its previous control guard');+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated hidden name'), false,+ 'Hidden label text must not be copied into feedback');+ } finally { await browser.close(); }+ });++ test(`${build}: controls after a large select remain in rotating form-state coverage`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const form = document.createElement('form');+ const select = document.createElement('select');+ select.id = 'large-select';+ select.innerHTML = Array.from({ length: 610 }, (_, index) => `<option value="${index}">Option ${index}</option>`).join('');+ const input = document.createElement('input');+ input.id = 'late-form-control';+ input.value = 'before';+ form.append(select, input);+ document.body.append(form);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));+ await page.waitForTimeout(180);+ await page.evaluate(() => {+ feedback = [];+ document.getElementById('late-form-control').value = 'sensitive-after';+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'input#late-form-control'), null, { timeout: 5000 });+ const serialized = JSON.stringify(await page.evaluate(() => feedback));+ assert.equal(serialized.includes('sensitive-after'), false, 'Detected property values must stay private');+ } finally { await browser.close(); }+ });+for (const action of ['click', 'type']) {test(`${build}: ${action} preparation preserves same-target human input`, async () => {const { browser, page } = await fixture(engine, build);