Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: monitor focus movement and large related-frame changes
Barack Sokulluc596c5cd0976
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.mdindex 0b1f69d22..d2cd5cccd 100644--- a/docs/page-feedback-monitor.md+++ b/docs/page-feedback-monitor.md@@ -118,7 +118,12 @@ notes. Browser extension APIs are mocked in these fixtures; they are not a livecompanion-session integration test.Visibility regressions include fixed-size ancestors changing class or CSS custom-properties to reveal or hide non-interactive content. Native preparation fixtures+properties to reveal or hide non-interactive content past the layout sampling and+signature seed limits. An actual inherited-state attribute change on an oversized+subtree conservatively invalidates prepared actions, while keeping layout reads+bounded; identical assignments, known animations and owned indicators are filtered.+Tab and Shift+Tab focus movement also invalidate preparation without recording keys.+Native preparation fixturesalso exercise the real companion code across an asynchronous target lookup whilethe user edits the same field.Frame lifecycle regressions cover delayed child acknowledgements and inaccessible@@ -134,5 +139,10 @@ same-field human typing and an intervening DOM change. Firefox frame fixturesmodel the document-start omission, apply the manifest's document-end scripts,and verify user/DOM/late-shadow feedback in blank and srcdoc frames while keepingan existing monitor intact. These remain mocked extension registration fixtures.+Both manifests enable origin fallback for related data/blob documents in the+monitor and MAIN shadow-hook registrations. Fixtures model these registrations+and verify private-input, DOM and late-shadow feedback inside those documents.+Document and local-action tokens use cryptographic random bytes when the document+does not expose the secure-context-only randomUUID API.The monitor regressions are also included in `npm run test:runtime-lifecycle`.diff --git a/src/chrome/manifest.json b/src/chrome/manifest.jsonindex 762aaddbf..7dec42e7e 100644--- a/src/chrome/manifest.json+++ b/src/chrome/manifest.json@@ -51,7 +51,8 @@"js": ["src/content/page-monitor.js"],"run_at": "document_start","all_frames": true,- "match_about_blank": true+ "match_about_blank": true,+ "match_origin_as_fallback": true},{"matches": ["<all_urls>"],@@ -59,7 +60,8 @@"run_at": "document_start","world": "MAIN","all_frames": true,- "match_about_blank": true+ "match_about_blank": true,+ "match_origin_as_fallback": true},{"matches": ["<all_urls>"],diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 7b3b6e4b5..c0eafbe92 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -6,7 +6,11 @@}window.__wbPageMonitor?.dispose?.();const api = globalThis.browser || globalThis.chrome;- const documentToken = crypto.randomUUID();+ // randomUUID is secure-context-only; related data frames and HTTP pages+ // still provide getRandomValues for unpredictable document/action tokens.+ const randomToken = () => crypto.randomUUID?.()+ || Array.from(crypto.getRandomValues(new Uint8Array(16)), byte => byte.toString(16).padStart(2, '0')).join('');+ const documentToken = randomToken();const listeners = [];const operations = new Map();const nativeTargets = new Set();@@ -174,7 +178,7 @@function observeRoot(root, budget) {if (!root || roots.has(root) || !observer) return;roots.add(root);- observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true,+ observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled','aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });seed(root, budget);@@ -204,13 +208,29 @@function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';+ const noteChange = el => {+ // Attribute the marked input/write's synchronous handlers to that dispatch.+ // Later DOM writes stay observable, including writes on the same target.+ if (agentTurn && lastUserAt <= agentTurn.userAt) return;+ changed = true;+ target ||= targetName(el);+ if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';+ else if (source !== 'user' && [...operations.values()].some(op => op.dispatched)) source = 'unknown';+ };const changes = [...records];for (const record of records) {if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName)) continue;+ if (ignored(record.target)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ const interactive = record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [];+ const all = record.target.querySelectorAll?.('*') || [];+ // A bounded layout sample cannot prove that inherited visibility left a+ // large subtree unchanged. Invalidate conservatively on an actual state+ // change, before the measurement cap can omit a late affected descendant.+ if (all.length > 100 && record.oldValue !== record.target.getAttribute(record.attributeName)) noteChange(record.target);const descendants = new Set([- ...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),- ...[...(record.target.querySelectorAll?.('*') || [])].slice(0, 100),+ ...Array.prototype.slice.call(interactive, 0, 100),+ ...Array.prototype.slice.call(all, 0, 100),]);for (const el of descendants)changes.push({ type: 'layout', target: el });@@ -239,13 +259,7 @@const next = signature(el), previous = signatures.get(el);signatures.set(el, next);if (!identityChanged && (next === previous || (!visible(el) && !previous?.startsWith('[true')))) continue;- // Attribute the marked input/write's synchronous handlers to that dispatch.- // Later DOM writes stay observable, including writes on the same target.- if (agentTurn && lastUserAt <= agentTurn.userAt) continue;- changed = true;- target ||= targetName(el);- if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';- else if ([...operations.values()].some(op => op.dispatched)) source = 'unknown';+ noteChange(el);}if (!changed) return;revision++; publishRevision();@@ -349,9 +363,9 @@listen(document, 'keydown', event => {const el = elementFor(event);if (!event.isTrusted || expected('input', el, event)) return;- // Key names are only used locally to identify scrolling, never transmitted.+ // Key names identify scrolling and focus movement locally, never transmitted.if (editable(el)) interact('activity', el);- else if (['ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);+ else if (['Tab', 'ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);});for (const name of ['wheel', 'touchmove']) listen(document, name, event => {const el = elementFor(event);@@ -451,7 +465,7 @@'ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);function beginContentAction(action, params = {}) {if (!active || !localMutations.has(action)) return () => {};- const operationId = `local-${crypto.randomUUID()}`;+ const operationId = `local-${randomToken()}`;const previous = localOperation;prepare({ selector: params.selector, ref_id: params.ref_id, x: params.x, y: params.y,textMatch: action === 'click' ? params.text : undefined, tool: action, operationId, runToken });diff --git a/src/firefox/manifest.json b/src/firefox/manifest.jsonindex 669ca6907..07e1cb3ad 100644--- a/src/firefox/manifest.json+++ b/src/firefox/manifest.json@@ -41,7 +41,8 @@"js": ["src/content/page-monitor.js"],"run_at": "document_start","all_frames": true,- "match_about_blank": true+ "match_about_blank": true,+ "match_origin_as_fallback": true},{"matches": ["<all_urls>"],@@ -49,14 +50,16 @@"run_at": "document_start","world": "MAIN","all_frames": true,- "match_about_blank": true+ "match_about_blank": true,+ "match_origin_as_fallback": true},{"matches": ["<all_urls>"],"js": ["src/content/page-monitor-recovery.js", "src/content/page-monitor.js"],"run_at": "document_end","all_frames": true,- "match_about_blank": true+ "match_about_blank": true,+ "match_origin_as_fallback": true},{"matches": ["<all_urls>"],@@ -64,7 +67,8 @@"run_at": "document_end","world": "MAIN","all_frames": true,- "match_about_blank": true+ "match_about_blank": true,+ "match_origin_as_fallback": true},{"matches": ["<all_urls>"],diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 7b3b6e4b5..c0eafbe92 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -6,7 +6,11 @@}window.__wbPageMonitor?.dispose?.();const api = globalThis.browser || globalThis.chrome;- const documentToken = crypto.randomUUID();+ // randomUUID is secure-context-only; related data frames and HTTP pages+ // still provide getRandomValues for unpredictable document/action tokens.+ const randomToken = () => crypto.randomUUID?.()+ || Array.from(crypto.getRandomValues(new Uint8Array(16)), byte => byte.toString(16).padStart(2, '0')).join('');+ const documentToken = randomToken();const listeners = [];const operations = new Map();const nativeTargets = new Set();@@ -174,7 +178,7 @@function observeRoot(root, budget) {if (!root || roots.has(root) || !observer) return;roots.add(root);- observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true,+ observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-disabled','aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'inert', 'class', 'style'] });seed(root, budget);@@ -204,13 +208,29 @@function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';+ const noteChange = el => {+ // Attribute the marked input/write's synchronous handlers to that dispatch.+ // Later DOM writes stay observable, including writes on the same target.+ if (agentTurn && lastUserAt <= agentTurn.userAt) return;+ changed = true;+ target ||= targetName(el);+ if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';+ else if (source !== 'user' && [...operations.values()].some(op => op.dispatched)) source = 'unknown';+ };const changes = [...records];for (const record of records) {if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName)) continue;+ if (ignored(record.target)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;+ const interactive = record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [];+ const all = record.target.querySelectorAll?.('*') || [];+ // A bounded layout sample cannot prove that inherited visibility left a+ // large subtree unchanged. Invalidate conservatively on an actual state+ // change, before the measurement cap can omit a late affected descendant.+ if (all.length > 100 && record.oldValue !== record.target.getAttribute(record.attributeName)) noteChange(record.target);const descendants = new Set([- ...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),- ...[...(record.target.querySelectorAll?.('*') || [])].slice(0, 100),+ ...Array.prototype.slice.call(interactive, 0, 100),+ ...Array.prototype.slice.call(all, 0, 100),]);for (const el of descendants)changes.push({ type: 'layout', target: el });@@ -239,13 +259,7 @@const next = signature(el), previous = signatures.get(el);signatures.set(el, next);if (!identityChanged && (next === previous || (!visible(el) && !previous?.startsWith('[true')))) continue;- // Attribute the marked input/write's synchronous handlers to that dispatch.- // Later DOM writes stay observable, including writes on the same target.- if (agentTurn && lastUserAt <= agentTurn.userAt) continue;- changed = true;- target ||= targetName(el);- if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';- else if ([...operations.values()].some(op => op.dispatched)) source = 'unknown';+ noteChange(el);}if (!changed) return;revision++; publishRevision();@@ -349,9 +363,9 @@listen(document, 'keydown', event => {const el = elementFor(event);if (!event.isTrusted || expected('input', el, event)) return;- // Key names are only used locally to identify scrolling, never transmitted.+ // Key names identify scrolling and focus movement locally, never transmitted.if (editable(el)) interact('activity', el);- else if (['ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);+ else if (['Tab', 'ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);});for (const name of ['wheel', 'touchmove']) listen(document, name, event => {const el = elementFor(event);@@ -451,7 +465,7 @@'ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);function beginContentAction(action, params = {}) {if (!active || !localMutations.has(action)) return () => {};- const operationId = `local-${crypto.randomUUID()}`;+ const operationId = `local-${randomToken()}`;const previous = localOperation;prepare({ selector: params.selector, ref_id: params.ref_id, x: params.x, y: params.y,textMatch: action === 'click' ? params.text : undefined, tool: action, operationId, runToken });diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 593c391f9..5966d0e43 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -613,6 +613,15 @@ test('both manifests signal shadow attachment from MAIN in all monitored frames'}});+test('monitor registrations include related-origin frames in both execution worlds', () => {+ for (const build of ['chrome', 'firefox']) {+ const manifest = JSON.parse(fs.readFileSync(new URL(`../src/${build}/manifest.json`, import.meta.url), 'utf8'));+ const entries = manifest.content_scripts.filter(entry => entry.js.some(file => /\/page-monitor(?:-shadow)?\.js$/.test(file)));+ assert.equal(entries.length, build === 'firefox' ? 4 : 2);+ for (const entry of entries) assert.ok(entry.all_frames && entry.match_about_blank && entry.match_origin_as_fallback);+ }+});+for (const path of ['local', 'memory']) {for (const intervention of ['preparation', 'handshake', 'none']) {test(`Chrome ${path} upload honors feedback arriving during ${intervention}`, async () => {diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 95983d587..af610ead0 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -404,6 +404,91 @@ test('Firefox native preparation keeps same-target user edits and sends no stale});for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {+ test(`${build}: related data/blob frames receive monitor and MAIN shadow hook`, async () => {+ const { browser, page } = await fixture(engine, build, { omitEmptyFrameMonitor: true });+ try {+ const manifest = JSON.parse(fs.readFileSync(new URL(`../src/${build}/manifest.json`, import.meta.url), 'utf8'));+ const entries = manifest.content_scripts.filter(entry => entry.run_at === 'document_start'+ && entry.js.some(file => /\/page-monitor(?:-shadow)?\.js$/.test(file)));+ assert.equal(entries.length, 2);+ for (const entry of entries) assert.ok(entry.all_frames && entry.match_origin_as_fallback,+ 'Related opaque-origin documents need origin fallback for both execution worlds');+ await page.evaluate(() => {+ const markup = '<input id="inside"><p id="inside-status">Ready</p><div id="late-host"></div>';+ for (const [id, src] of [['data-frame', `data:text/html,${encodeURIComponent(markup)}`],+ ['blob-frame', URL.createObjectURL(new Blob([markup], { type: 'text/html' }))]]) {+ const frame = document.createElement('iframe'); frame.id = id; frame.src = src; document.body.prepend(frame);+ }+ });+ for (const [id, scheme] of [['data-frame', 'data:'], ['blob-frame', 'blob:']]) {+ const frame = await (await page.locator(`#${id}`).elementHandle()).contentFrame();+ await frame.waitForFunction(scheme => location.href.startsWith(scheme) && window.chrome?.runtime && document.getElementById('inside'), scheme);+ assert.equal(await frame.evaluate(() => !!window.__wbPageMonitor), false);+ // Model the manifest's related-origin registration, not Playwright's unconditional init-script injection.+ for (const entry of entries) for (const file of entry.js.filter(file => /\/page-monitor(?:-shadow)?\.js$/.test(file)))+ await frame.addScriptTag({ content: read(build, file.replace(/^src\//, '')) });+ await frame.waitForFunction(() => document.documentElement.hasAttribute('data-webbrain-page-revision'));+ const preparation = await frame.evaluate(() => {+ const finish = __wbPageMonitor.beginContentAction('type', { selector: '#inside', _bidiPrepare: true });+ try { __wbPageMonitor.beforeLocalDispatch(); return true; } finally { finish(); }+ });+ assert.equal(preparation, true, 'Document and local action tokens must work in related origins');+ await frame.waitForTimeout(220);+ await frame.locator('#inside').fill('Private frame input');+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'input' && event.source === 'user'));+ assert.ok(!JSON.stringify(await frame.evaluate(() => feedback)).includes('Private frame input'));+ await frame.evaluate(() => { feedback = []; document.getElementById('inside-status').textContent = 'Updated'; });+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'dom'));+ await frame.evaluate(() => { feedback = []; document.getElementById('late-host').attachShadow({ mode: 'open' }).innerHTML = '<button>Target</button>'; });+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'div#late-host'));+ await frame.evaluate(() => { feedback = []; document.getElementById('late-host').shadowRoot.querySelector('button').textContent = 'Changed'; });+ await frame.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button'));+ }+ } finally { await browser.close(); }+ });++ test(`${build}: Tab focus movement invalidates preparation without recording keys`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ for (const key of ['Tab', 'Shift+Tab']) {+ await page.locator('#human').focus();+ const before = await page.evaluate(() => {+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');+ return document.activeElement.id;+ });+ await page.waitForTimeout(40);+ await page.evaluate(() => {+ feedback = []; window.finishPreparation = __wbPageMonitor.beginContentAction('type', { _bidiPrepare: true });+ });+ await page.keyboard.press(key);+ assert.notEqual(await page.evaluate(() => document.activeElement.id), before);+ const code = await page.evaluate(() => {+ try { __wbPageMonitor.beforeLocalDispatch(); return null; }+ catch (error) { return error.code; }+ finally { finishPreparation(); }+ });+ assert.equal(code, 'page_feedback_pending');+ const events = await page.evaluate(() => feedback);+ assert.ok(events.some(event => event.kind === 'activity' && event.source === 'user'));+ assert.ok(!JSON.stringify(events).includes('Tab'));+ }+ await page.locator('#human').focus();+ await page.evaluate(() => {+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');+ });+ await page.waitForTimeout(40);+ await page.evaluate(() => {+ feedback = []; deliver('page_monitor_prepare', { operationId: 'agent-tab', tool: 'press_keys', selector: '#human' });+ deliver('page_monitor_dispatch', { operationId: 'agent-tab', kind: 'input', selector: '#human', fenceOnly: true });+ document.getElementById('human').setAttribute('data-webbrain-native-action', JSON.stringify({ ...lastMonitorResponse.guard, kind: 'input', sequence: 1 }));+ });+ await page.keyboard.press('Tab');+ await page.waitForTimeout(80);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,+ 'An attributed native Tab must not steer its own run');+ } finally { await browser.close(); }+ });+test(`${build}: monitor captures user input without values, suppresses agent input and keeps concurrent user input`, async () => {const { browser, page } = await fixture(engine, build);try {@@ -898,6 +983,56 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: large ancestor visibility changes invalidate actions beyond the sampling budget`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const style = document.createElement('style');+ style.textContent = '#late-result {display:var(--late-display,block)} .closed #late-result {display:none}'; document.head.append(style);+ const region = document.createElement('div'); region.id = 'large-region'; region.className = 'closed';+ region.style.cssText = 'width:200px;height:80px';+ region.innerHTML = '<span style="display:none">Earlier unchanged node</span>'.repeat(650) + '<div id="late-result">Late visible result</div>';+ document.body.prepend(region);+ });+ for (const attribute of ['class', 'style']) {+ await page.evaluate(attribute => {+ const region = document.getElementById('large-region'); region.className = attribute === 'class' ? 'closed' : '';+ region.style.setProperty('--late-display', attribute === 'style' ? 'none' : 'block');+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');+ }, attribute);+ await page.waitForTimeout(200);+ for (const shown of [true, false]) {+ const result = await page.evaluate(async ({ attribute, shown }) => {+ const region = document.getElementById('large-region'), before = region.getBoundingClientRect();+ feedback = []; const finish = __wbPageMonitor.beginContentAction('click', { selector: '#agent', _bidiPrepare: true });+ if (attribute === 'class') region.className = shown ? '' : 'closed';+ else region.style.setProperty('--late-display', shown ? 'block' : 'none');+ await Promise.resolve();+ let code; try { __wbPageMonitor.beforeLocalDispatch(); } catch (error) { code = error.code; } finally { finish(); }+ const after = region.getBoundingClientRect();+ return { code, sameGeometry: before.width === after.width && before.height === after.height };+ }, { attribute, shown });+ assert.deepEqual(result, { code: 'page_feedback_pending', sameGeometry: true });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ assert.equal(await page.locator('#late-result').isVisible(), shown);+ await page.evaluate(attribute => {+ feedback = []; const region = document.getElementById('large-region');+ region.setAttribute(attribute, region.getAttribute(attribute));+ }, attribute);+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Repeated identical attributes must not trigger the conservative fallback');+ }+ }+ await page.evaluate(() => {+ feedback = []; const finish = __wbPageMonitor.beginContentAction('execute_js');+ try { __wbPageMonitor.beforeLocalDispatch(); document.getElementById('large-region').className = 'agent-theme'; }+ finally { finish(); }+ });+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'A synchronous attributed ancestor write must not steer itself');+ } finally { await browser.close(); }+ });+test(`${build}: same-shape subtree replacements invalidate prepared actions`, async () => {const { browser, page } = await fixture(engine, build);try {