Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Attribute local synthetic clicks to agent dispatch
Barack Sokullubcedd0ef4c65
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 4e8cbd650..79a2b20b5 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -2225,9 +2225,10 @@if (actionDeadlineExpired()) return deadlineFailure();const clickedRect = rememberInteractionPoint(el, 'click');if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());+ const filePickerGuard = withLocalPageDispatch(() => {+ dispatched = true;+ return clickWithoutNativeFilePicker(() => el.click());+ });if (filePickerGuard.blocked) {return {...filePickerBlockedResponse(filePickerGuard.blocked, params.text || el.innerText?.trim() || ''),@@ -2293,7 +2294,7 @@return _typeTextInner(params, actionDeadlineExpired);}- function withLocalInputDispatch(callback) {+ function withLocalPageDispatch(callback) {const monitor = window.__wbPageMonitor;if (typeof monitor?.withLocalDispatch === 'function') return monitor.withLocalDispatch(callback);monitor?.beforeLocalDispatch();@@ -2547,7 +2548,7 @@}const nativeSetter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- const dispatchResult = withLocalInputDispatch(() => {+ const dispatchResult = withLocalPageDispatch(() => {dispatched = true;if (nativeSetter) nativeSetter.call(el, match.value);else el.value = match.value;@@ -2568,7 +2569,7 @@const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- const dispatchResult = withLocalInputDispatch(() => {+ const dispatchResult = withLocalPageDispatch(() => {dispatched = true;if (nativeInputValueSetter) {nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);@@ -7377,10 +7378,11 @@{ deadlineExpired: true, retryable: true },);}- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;const syntheticClickDispatchStartedAt = performance.now();- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());+ const filePickerGuard = withLocalPageDispatch(() => {+ dispatched = true;+ return clickWithoutNativeFilePicker(() => el.click());+ });const syntheticClickDispatchMs = Math.max(0, performance.now() - syntheticClickDispatchStartedAt);const fallbackStateAfterImmediate = _axFallbackState(el);if (filePickerGuard.blocked) {@@ -7683,9 +7685,10 @@};}if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- el.click();+ withLocalPageDispatch(() => {+ dispatched = true;+ el.click();+ });if (actionDeadlineExpired()) return deadlineFailure();const checkedAfter = !!el.checked;const success = checkedAfter === checked;diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex c27fe0392..4c874b7ab 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -2535,9 +2535,10 @@...(coordinateClick ? { point: { x: Math.round(params.x), y: Math.round(params.y) } } : {}),_filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };}- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());+ const filePickerGuard = withLocalPageDispatch(() => {+ dispatched = true;+ return clickWithoutNativeFilePicker(() => el.click());+ });if (filePickerGuard.blocked) {return {...filePickerBlockedResponse(filePickerGuard.blocked, params.text || el.innerText?.trim() || ''),@@ -2609,7 +2610,7 @@return _typeTextInner(params, actionDeadlineExpired);}- function withLocalInputDispatch(callback) {+ function withLocalPageDispatch(callback) {const monitor = window.__wbPageMonitor;if (typeof monitor?.withLocalDispatch === 'function') return monitor.withLocalDispatch(callback);monitor?.beforeLocalDispatch();@@ -2866,7 +2867,7 @@}const nativeSetter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- const dispatchResult = withLocalInputDispatch(() => {+ const dispatchResult = withLocalPageDispatch(() => {dispatched = true;if (nativeSetter) nativeSetter.call(el, match.value);else el.value = match.value;@@ -2888,7 +2889,7 @@const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;if (actionDeadlineExpired()) return deadlineFailure();- const dispatchResult = withLocalInputDispatch(() => {+ const dispatchResult = withLocalPageDispatch(() => {dispatched = true;if (nativeInputValueSetter) {nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);@@ -6467,11 +6468,12 @@_filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId,};}- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;const syntheticClickStartedAt = Date.now();const syntheticClickDispatchStartedAt = performance.now();- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());+ const filePickerGuard = withLocalPageDispatch(() => {+ dispatched = true;+ return clickWithoutNativeFilePicker(() => el.click());+ });const syntheticClickDispatchMs = Math.max(0, performance.now() - syntheticClickDispatchStartedAt);if (filePickerGuard.blocked) {return failure(@@ -6690,9 +6692,10 @@};}if (actionDeadlineExpired()) return deadlineFailure();- window.__wbPageMonitor?.beforeLocalDispatch();- dispatched = true;- el.click();+ withLocalPageDispatch(() => {+ dispatched = true;+ el.click();+ });if (actionDeadlineExpired()) return deadlineFailure();await new Promise(resolve => setTimeout(resolve, SET_FIELD_VERIFY_DELAY_MS));if (actionDeadlineExpired()) return deadlineFailure();diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 78a4ca257..222f89a9e 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -585,21 +585,21 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {const typeStart = source.indexOf('async function _typeTextInner(');const findTextStart = source.indexOf('\n function findText(', typeStart);assert.ok(typeStart >= 0 && findTextStart > typeStart);- assert.equal((source.slice(typeStart, findTextStart).match(/withLocalInputDispatch\(\(\) => \{/g) || []).length, 2,+ assert.equal((source.slice(typeStart, findTextStart).match(/withLocalPageDispatch\(\(\) => \{/g) || []).length, 2,`${build}: both native form and select setters must include their synthetic events in the local dispatch`);- const helper = source.match(/^ function withLocalInputDispatch\(callback\) \{[\s\S]*?^ \}/m)?.[0];+ const helper = source.match(/^ function withLocalPageDispatch\(callback\) \{[\s\S]*?^ \}/m)?.[0];assert.ok(helper, `${build}: local input events must use the production monitor boundary`);const { browser, page } = await fixture(engine, build);try {- await page.addScriptTag({ content: `window.withLocalInputDispatch = ${helper};` });+ await page.addScriptTag({ content: `window.withLocalPageDispatch = ${helper};` });for (const [selector, tag, value] of [['#field', 'input', 'agent-synthetic-value'], ['#select', 'select', 'B']]) {await page.evaluate(() => { feedback = []; });await page.evaluate(({ selector, tag, value }) => {const field = document.querySelector(selector);const finish = __wbPageMonitor.beginContentAction('type', { selector });try {- withLocalInputDispatch(() => {+ withLocalPageDispatch(() => {const prototype = tag === 'select' ? HTMLSelectElement.prototype : HTMLInputElement.prototype;Object.getOwnPropertyDescriptor(prototype, 'value').set.call(field, value);field.dispatchEvent(new Event('input', { bubbles: true }));@@ -615,6 +615,52 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: local synthetic clicks attribute checkbox changes and navigation`, async () => {+ const source = read(build, 'content/content.js');+ const clickStart = source.indexOf('function clickElement(params, actionDeadlineExpired = () => false)');+ const typeStart = source.indexOf('function typeText(params, actionDeadlineExpired = () => false)', clickStart);+ const clickAxStart = source.indexOf("'click_ax':");+ const checkedStart = source.indexOf("'set_checked':", clickAxStart);+ const typeAxStart = source.indexOf("'type_ax':", checkedStart);+ assert.ok(clickStart >= 0 && typeStart > clickStart && clickAxStart >= 0 && checkedStart > clickAxStart && typeAxStart > checkedStart);+ for (const body of [source.slice(clickStart, typeStart), source.slice(clickAxStart, checkedStart), source.slice(checkedStart, typeAxStart)]) {+ assert.match(body, /withLocalPageDispatch\(\(\) => \{[\s\S]*?el\.click\(\)/,+ `${build}: every synthetic click path must keep the monitor's local dispatch scope active`);+ }+ const helper = source.match(/^ function withLocalPageDispatch\(callback\) \{[\s\S]*?^ \}/m)?.[0];+ assert.ok(helper, `${build}: synthetic clicks must use the production local dispatch boundary`);++ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ document.body.insertAdjacentHTML('beforeend', '<input id="local-click-checkbox" type="checkbox"><input id="local-checked-checkbox" type="checkbox"><a id="local-click-link" href="#local-click-destination">Continue</a>');+ });+ await page.waitForTimeout(160);+ await page.addScriptTag({ content: `window.withLocalPageDispatch = ${helper};` });+ const events = await page.evaluate(() => {+ feedback = [];+ const actions = [+ ['click', document.getElementById('local-click-link')],+ ['click_ax', document.getElementById('local-click-checkbox')],+ ['set_checked', document.getElementById('local-checked-checkbox')],+ ];+ for (const [action, target] of actions) {+ const finish = __wbPageMonitor.beginContentAction(action, { selector: '#' + target.id });+ try { withLocalPageDispatch(() => target.click()); }+ finally { finish(); }+ }+ return new Promise(resolve => setTimeout(() => resolve(feedback), 150));+ });+ assert.ok(events.some(event => event.source === 'agent' && event.operation === 'click'+ && event.navigationUrl === 'https://monitor.test/start#local-click-destination'),+ `${build}: a synthetic link click must arm its agent navigation`);+ assert.equal(events.some(event => event.source !== 'agent'), false,+ `${build}: synthetic checkbox changes must not be reported as page/user feedback`);+ assert.equal(await page.locator('#local-click-checkbox').isChecked(), true);+ assert.equal(await page.locator('#local-checked-checkbox').isChecked(), true);+ } finally { await browser.close(); }+ });+test(`${build}: click-driven programmatic focus checks the monitor fence before focusin`, () => {const source = read(build, 'content/content.js');const clickStart = source.indexOf('function clickElement(params, actionDeadlineExpired = () => false)');