Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
attribute local form events and shadow focus
Barack Sokullu8a27bc169636
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex 127a895d3..6e73ebfc4 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -293,7 +293,13 @@ export class BidiSession {const marker = createNativeActionMarker(guard, kind, ++nativeSequence);const checked = await this.call(match, `(el, marker, kind, action) => {if (!el.isConnected) return false;- const target = kind === 'input' && action !== 'upload' ? el.getRootNode().activeElement || el : el;+ const deepActiveElement = () => {+ let active = document.activeElement;+ while (active?.shadowRoot?.activeElement) active = active.shadowRoot.activeElement;+ return active;+ };+ const target = kind === 'input' && action !== 'upload' ? deepActiveElement() || el : el;+ if (!target?.isConnected) return false;target.setAttribute('data-webbrain-native-action', marker);return true;}`, [{ type: 'string', value: marker }, { type: 'string', value: kind }, { type: 'string', value: action }]);@@ -319,7 +325,12 @@ export class BidiSession {try {await this.call(match, `(el, marker, action) => {const kind = JSON.parse(marker).kind;- const targets = kind === 'input' && action !== 'upload' ? [el, el.getRootNode().activeElement] : [el];+ const deepActiveElement = () => {+ let active = document.activeElement;+ while (active?.shadowRoot?.activeElement) active = active.shadowRoot.activeElement;+ return active;+ };+ const targets = kind === 'input' && action !== 'upload' ? [el, deepActiveElement()] : [el];for (const target of new Set(targets))if (target?.getAttribute('data-webbrain-native-action') === marker) target.removeAttribute('data-webbrain-native-action');}`, [{ type: 'string', value: marker }, { type: 'string', value: action }]);diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 82bc8de28..4e8cbd650 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -2293,6 +2293,13 @@return _typeTextInner(params, actionDeadlineExpired);}+ function withLocalInputDispatch(callback) {+ const monitor = window.__wbPageMonitor;+ if (typeof monitor?.withLocalDispatch === 'function') return monitor.withLocalDispatch(callback);+ monitor?.beforeLocalDispatch();+ return callback();+ }+async function _insertContentEditableText(el, text, clear, actionDeadlineExpired = () => false) {const doc = el.ownerDocument;let dispatched = false;@@ -2540,44 +2547,40 @@}const nativeSetter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- if (nativeSetter) nativeSetter.call(el, match.value);- else el.value = match.value;- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('input', { bubbles: true }));- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('change', { bubbles: true }));+ const dispatchResult = withLocalInputDispatch(() => {+ dispatched = true;+ if (nativeSetter) nativeSetter.call(el, match.value);+ else el.value = match.value;+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('input', { bubbles: true }));+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('change', { bubbles: true }));+ });+ if (dispatchResult) return dispatchResult;await new Promise(resolve => setTimeout(resolve, 30));if (actionDeadlineExpired()) return deadlineFailure();return { success: true, ...(el.isConnected && el.value === match.value ? { verified: true } : {}), method: 'select', value: el.value };}- if (params.clear) {- if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- el.value = '';- }-const proto = el instanceof HTMLTextAreaElement? HTMLTextAreaElement.prototype: HTMLInputElement.prototype;const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- if (nativeInputValueSetter) {- nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);- } else {- el.value = (params.clear ? '' : (el.value || '')) + params.text;- }-- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('input', { bubbles: true }));- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('change', { bubbles: true }));+ const dispatchResult = withLocalInputDispatch(() => {+ dispatched = true;+ if (nativeInputValueSetter) {+ nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);+ } else {+ el.value = (params.clear ? '' : (el.value || '')) + params.text;+ }+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('input', { bubbles: true }));+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('change', { bubbles: true }));+ });+ if (dispatchResult) return dispatchResult;const verified = await verifyValue(el, typedText, params.clear === true, beforeValue);if (actionDeadlineExpired()) return deadlineFailure();diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex f14334bc0..c27fe0392 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -2609,6 +2609,13 @@return _typeTextInner(params, actionDeadlineExpired);}+ function withLocalInputDispatch(callback) {+ const monitor = window.__wbPageMonitor;+ if (typeof monitor?.withLocalDispatch === 'function') return monitor.withLocalDispatch(callback);+ monitor?.beforeLocalDispatch();+ return callback();+ }+async function _insertContentEditableText(el, text, clear, actionDeadlineExpired = () => false) {const doc = el.ownerDocument;let dispatched = false;@@ -2859,26 +2866,21 @@}const nativeSetter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- if (nativeSetter) nativeSetter.call(el, match.value);- else el.value = match.value;- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('input', { bubbles: true }));- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('change', { bubbles: true }));+ const dispatchResult = withLocalInputDispatch(() => {+ dispatched = true;+ if (nativeSetter) nativeSetter.call(el, match.value);+ else el.value = match.value;+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('input', { bubbles: true }));+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('change', { bubbles: true }));+ });+ if (dispatchResult) return dispatchResult;await new Promise(resolve => setTimeout(resolve, 30));if (actionDeadlineExpired()) return deadlineFailure();return { success: true, ...(el.isConnected && el.value === match.value ? { verified: true } : {}), method: 'select', value: el.value };}- if (params.clear) {- if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- el.value = '';- }-// Use input events for React/Vue compatibilityconst proto = el instanceof HTMLTextAreaElement? HTMLTextAreaElement.prototype@@ -2886,18 +2888,19 @@const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- if (nativeInputValueSetter) {- nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);- } else {- el.value = (params.clear ? '' : (el.value || '')) + params.text;- }-- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('input', { bubbles: true }));- if (actionDeadlineExpired()) return deadlineFailure();- el.dispatchEvent(new Event('change', { bubbles: true }));+ const dispatchResult = withLocalInputDispatch(() => {+ dispatched = true;+ if (nativeInputValueSetter) {+ nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);+ } else {+ el.value = (params.clear ? '' : (el.value || '')) + params.text;+ }+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('input', { bubbles: true }));+ if (actionDeadlineExpired()) return deadlineFailure();+ el.dispatchEvent(new Event('change', { bubbles: true }));+ });+ if (dispatchResult) return dispatchResult;const verified = await verifyValue(el, typedText, params.clear === true, beforeValue);if (actionDeadlineExpired()) return deadlineFailure();diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex e9e1544c5..997f562c5 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -478,7 +478,116 @@ test('Firefox native preparation keeps same-target user edits and sends no stale}});+test('Firefox repeated native Tab marks the deeply focused control across shadow boundaries', async () => {+ const { page, browser } = await fixture(firefox, 'firefox');+ const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser;+ const { pageFeedbackMethods: methods } = await import('../src/firefox/src/agent/page-feedback.js');+ const tab = 961, runId = crypto.randomUUID(), token = crypto.randomUUID(), operationId = 'shadow-tab-operation', session = new BidiSession();+ const host = { ...methods, isRunning: () => true, _checkAbort: () => false };+ const api = { tabs: {+ get: async () => ({ url: page.url() }),+ sendMessage: async (_tab, message) => page.evaluate(message => {+ deliver(message.action, message.params || {}); return lastMonitorResponse;+ }, message),+ }, webNavigation: { getAllFrames: async () => [{ frameId: 0 }] } };+ globalThis.chrome = api; globalThis.browser = api;+ session.runs.set(runId, { context: 'tab' });+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'shadow-field' } });+ session.call = async (_match, declaration, args = []) => ({ result: { value: await page.evaluate(({ declaration, args }) => {+ const el = document.getElementById('tab-shadow-host').shadowRoot.getElementById('shadow-field');+ return (0, eval)(`(${declaration})`)(el, ...args.map(arg => arg.value));+ }, { declaration, args }) } });+ session.send = async method => { if (method === 'input.performActions') await page.keyboard.press('Tab'); return {}; };+ try {+ await page.evaluate(() => {+ const shadowHost = document.createElement('div'); shadowHost.id = 'tab-shadow-host';+ shadowHost.style.cssText = 'position:fixed;left:10px;top:120px';+ shadowHost.attachShadow({ mode: 'open' }).innerHTML = '<input id="shadow-field">';+ const first = document.createElement('input'); first.id = 'outside-shadow-first';+ first.style.cssText = 'position:fixed;left:10px;top:160px';+ const second = document.createElement('input'); second.id = 'outside-shadow-second';+ second.style.cssText = 'position:fixed;left:10px;top:200px';+ document.body.append(shadowHost, first, second);+ });+ await page.waitForTimeout(180);+ await host._beginPageFeedbackRun(tab, 'interactive');+ const runToken = host._pageFeedbackRuns.get(tab).token;+ await page.evaluate(token => {+ monitorEnabled = false; deliver('page_monitor_state');+ monitorRunToken = token; monitorEnabled = true; deliver('page_monitor_state');+ }, runToken);+ await page.waitForTimeout(100);+ const documentToken = await page.evaluate(() => document.documentElement.getAttribute('data-webbrain-page-revision').split(':')[0]);+ host.pageMonitorState({ tab: { id: tab }, frameId: 0 }, documentToken);+ await page.evaluate(operationId => {+ feedback = [];+ const element = document.getElementById('tab-shadow-host').shadowRoot.getElementById('shadow-field');+ const params = { operationId, tool: 'press_keys', element };+ deliver('page_monitor_prepare', params);+ deliver('page_monitor_dispatch', { ...params, kind: 'input', fenceOnly: true });+ }, operationId);+ const guard = await page.evaluate(() => lastMonitorResponse.guard);+ assert.ok(guard?.nativeSecret, 'the prepared action should carry the monitor capability');+ await page.evaluate(token => document.getElementById('tab-shadow-host').shadowRoot.getElementById('shadow-field')+ .setAttribute('data-webbrain-bidi', token), token);+ await page.locator('#tab-shadow-host').evaluate(hostElement => hostElement.shadowRoot.getElementById('shadow-field').focus());++ const result = await session.perform(runId, 'key', { token, key: 'Tab', repeat: 2, pageFeedbackGuard: guard },+ async (_id, received, kind, rebindFocus) => (await api.tabs.sendMessage(tab, { target: 'content', action: 'page_monitor_validate',+ params: { ...received, kind, rebindFocus } }, { frameId: 0 }))?.ready === true);+ assert.equal(result.success, true);+ assert.equal(await page.evaluate(() => document.activeElement.id), 'outside-shadow-second',+ 'both native Tab presses should follow the focus across the shadow boundary');+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,+ 'rebound native input must not be reported as a user intervention');+ assert.equal(await page.evaluate(() => [...document.querySelectorAll('[data-webbrain-native-action]'),+ ...document.getElementById('tab-shadow-host').shadowRoot.querySelectorAll('[data-webbrain-native-action]')].length), 0,+ 'consumed native markers must be removed');+ } finally {+ await session.close();+ if (host._pageFeedbackRuns.has(tab)) host._finishPageFeedbackRun(tab);+ if (savedChrome === undefined) delete globalThis.chrome; else globalThis.chrome = savedChrome;+ if (savedBrowser === undefined) delete globalThis.browser; else globalThis.browser = savedBrowser;+ await browser.close();+ }+});+for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {+ test(`${build}: synthetic local form input stays attributed to its dispatch`, async () => {+ const source = read(build, 'content/content.js');+ const typeStart = source.indexOf('async function _typeTextInner(');+ const findTextStart = source.indexOf('\n function findText(', typeStart);+ assert.ok(typeStart >= 0 && findTextStart > typeStart);+ assert.equal((source.slice(typeStart, findTextStart).match(/withLocalInputDispatch\(\(\) => \{/g) || []).length, 2,+ `${build}: both native form and select setters must include their synthetic events in the local dispatch`);+ const helper = source.match(/^ function withLocalInputDispatch\(callback\) \{[\s\S]*?^ \}/m)?.[0];+ assert.ok(helper, `${build}: local input events must use the production monitor boundary`);++ const { browser, page } = await fixture(engine, build);+ try {+ await page.addScriptTag({ content: `window.withLocalInputDispatch = ${helper};` });+ for (const [selector, tag, value] of [['#field', 'input', 'agent-synthetic-value'], ['#select', 'select', 'B']]) {+ await page.evaluate(() => { feedback = []; });+ await page.evaluate(({ selector, tag, value }) => {+ const field = document.querySelector(selector);+ const finish = __wbPageMonitor.beginContentAction('type', { selector });+ try {+ withLocalInputDispatch(() => {+ const prototype = tag === 'select' ? HTMLSelectElement.prototype : HTMLInputElement.prototype;+ Object.getOwnPropertyDescriptor(prototype, 'value').set.call(field, value);+ field.dispatchEvent(new Event('input', { bubbles: true }));+ field.dispatchEvent(new Event('change', { bubbles: true }));+ });+ } finally { finish(); }+ }, { selector, tag, value });+ await page.waitForTimeout(220);+ const events = await page.evaluate(() => feedback);+ assert.equal(events.some(event => event.source !== 'agent'), false, `${build}: synthetic ${tag} events must not become page feedback`);+ assert.equal(JSON.stringify(events).includes(value), false, `${build}: page feedback must not include form values`);+ }+ } finally { await browser.close(); }+ });+test(`${build}: click-driven programmatic focus checks the monitor fence before focusin`, () => {const source = read(build, 'content/content.js');const clickStart = source.indexOf('function clickElement(params, actionDeadlineExpired = () => false)');