Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Attribute local synthetic clicks to agent dispatch

Barack Sokullubcedd0ef4c65
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 4e8cbd650..79a2b20b5 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -2225,9 +2225,10 @@
if (actionDeadlineExpired()) return deadlineFailure();
const clickedRect = rememberInteractionPoint(el, 'click');
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
- dispatched = true;
- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());
+ const filePickerGuard = withLocalPageDispatch(() => {
+ dispatched = true;
+ return clickWithoutNativeFilePicker(() => el.click());
+ });
if (filePickerGuard.blocked) {
return {
...filePickerBlockedResponse(filePickerGuard.blocked, params.text || el.innerText?.trim() || ''),
@@ -2293,7 +2294,7 @@
return _typeTextInner(params, actionDeadlineExpired);
}
- function withLocalInputDispatch(callback) {
+ function withLocalPageDispatch(callback) {
const monitor = window.__wbPageMonitor;
if (typeof monitor?.withLocalDispatch === 'function') return monitor.withLocalDispatch(callback);
monitor?.beforeLocalDispatch();
@@ -2547,7 +2548,7 @@
}
const nativeSetter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set;
if (actionDeadlineExpired()) return deadlineFailure();
- const dispatchResult = withLocalInputDispatch(() => {
+ const dispatchResult = withLocalPageDispatch(() => {
dispatched = true;
if (nativeSetter) nativeSetter.call(el, match.value);
else el.value = match.value;
@@ -2568,7 +2569,7 @@
const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
if (actionDeadlineExpired()) return deadlineFailure();
- const dispatchResult = withLocalInputDispatch(() => {
+ const dispatchResult = withLocalPageDispatch(() => {
dispatched = true;
if (nativeInputValueSetter) {
nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);
@@ -7377,10 +7378,11 @@
{ deadlineExpired: true, retryable: true },
);
}
- window.__wbPageMonitor?.beforeLocalDispatch();
- dispatched = true;
const syntheticClickDispatchStartedAt = performance.now();
- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());
+ const filePickerGuard = withLocalPageDispatch(() => {
+ dispatched = true;
+ return clickWithoutNativeFilePicker(() => el.click());
+ });
const syntheticClickDispatchMs = Math.max(0, performance.now() - syntheticClickDispatchStartedAt);
const fallbackStateAfterImmediate = _axFallbackState(el);
if (filePickerGuard.blocked) {
@@ -7683,9 +7685,10 @@
};
}
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
- dispatched = true;
- el.click();
+ withLocalPageDispatch(() => {
+ dispatched = true;
+ el.click();
+ });
if (actionDeadlineExpired()) return deadlineFailure();
const checkedAfter = !!el.checked;
const success = checkedAfter === checked;
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index c27fe0392..4c874b7ab 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -2535,9 +2535,10 @@
...(coordinateClick ? { point: { x: Math.round(params.x), y: Math.round(params.y) } } : {}),
_filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };
}
- window.__wbPageMonitor?.beforeLocalDispatch();
- dispatched = true;
- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());
+ const filePickerGuard = withLocalPageDispatch(() => {
+ dispatched = true;
+ return clickWithoutNativeFilePicker(() => el.click());
+ });
if (filePickerGuard.blocked) {
return {
...filePickerBlockedResponse(filePickerGuard.blocked, params.text || el.innerText?.trim() || ''),
@@ -2609,7 +2610,7 @@
return _typeTextInner(params, actionDeadlineExpired);
}
- function withLocalInputDispatch(callback) {
+ function withLocalPageDispatch(callback) {
const monitor = window.__wbPageMonitor;
if (typeof monitor?.withLocalDispatch === 'function') return monitor.withLocalDispatch(callback);
monitor?.beforeLocalDispatch();
@@ -2866,7 +2867,7 @@
}
const nativeSetter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set;
if (actionDeadlineExpired()) return deadlineFailure();
- const dispatchResult = withLocalInputDispatch(() => {
+ const dispatchResult = withLocalPageDispatch(() => {
dispatched = true;
if (nativeSetter) nativeSetter.call(el, match.value);
else el.value = match.value;
@@ -2888,7 +2889,7 @@
const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
if (actionDeadlineExpired()) return deadlineFailure();
- const dispatchResult = withLocalInputDispatch(() => {
+ const dispatchResult = withLocalPageDispatch(() => {
dispatched = true;
if (nativeInputValueSetter) {
nativeInputValueSetter.call(el, (params.clear ? '' : (el.value || '')) + params.text);
@@ -6467,11 +6468,12 @@
_filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId,
};
}
- window.__wbPageMonitor?.beforeLocalDispatch();
- dispatched = true;
const syntheticClickStartedAt = Date.now();
const syntheticClickDispatchStartedAt = performance.now();
- const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());
+ const filePickerGuard = withLocalPageDispatch(() => {
+ dispatched = true;
+ return clickWithoutNativeFilePicker(() => el.click());
+ });
const syntheticClickDispatchMs = Math.max(0, performance.now() - syntheticClickDispatchStartedAt);
if (filePickerGuard.blocked) {
return failure(
@@ -6690,9 +6692,10 @@
};
}
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
- dispatched = true;
- el.click();
+ withLocalPageDispatch(() => {
+ dispatched = true;
+ el.click();
+ });
if (actionDeadlineExpired()) return deadlineFailure();
await new Promise(resolve => setTimeout(resolve, SET_FIELD_VERIFY_DELAY_MS));
if (actionDeadlineExpired()) return deadlineFailure();
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 78a4ca257..222f89a9e 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -585,21 +585,21 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
const typeStart = source.indexOf('async function _typeTextInner(');
const findTextStart = source.indexOf('\n function findText(', typeStart);
assert.ok(typeStart >= 0 && findTextStart > typeStart);
- assert.equal((source.slice(typeStart, findTextStart).match(/withLocalInputDispatch\(\(\) => \{/g) || []).length, 2,
+ assert.equal((source.slice(typeStart, findTextStart).match(/withLocalPageDispatch\(\(\) => \{/g) || []).length, 2,
`${build}: both native form and select setters must include their synthetic events in the local dispatch`);
- const helper = source.match(/^ function withLocalInputDispatch\(callback\) \{[\s\S]*?^ \}/m)?.[0];
+ const helper = source.match(/^ function withLocalPageDispatch\(callback\) \{[\s\S]*?^ \}/m)?.[0];
assert.ok(helper, `${build}: local input events must use the production monitor boundary`);
const { browser, page } = await fixture(engine, build);
try {
- await page.addScriptTag({ content: `window.withLocalInputDispatch = ${helper};` });
+ await page.addScriptTag({ content: `window.withLocalPageDispatch = ${helper};` });
for (const [selector, tag, value] of [['#field', 'input', 'agent-synthetic-value'], ['#select', 'select', 'B']]) {
await page.evaluate(() => { feedback = []; });
await page.evaluate(({ selector, tag, value }) => {
const field = document.querySelector(selector);
const finish = __wbPageMonitor.beginContentAction('type', { selector });
try {
- withLocalInputDispatch(() => {
+ withLocalPageDispatch(() => {
const prototype = tag === 'select' ? HTMLSelectElement.prototype : HTMLInputElement.prototype;
Object.getOwnPropertyDescriptor(prototype, 'value').set.call(field, value);
field.dispatchEvent(new Event('input', { bubbles: true }));
@@ -615,6 +615,52 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: local synthetic clicks attribute checkbox changes and navigation`, async () => {
+ const source = read(build, 'content/content.js');
+ const clickStart = source.indexOf('function clickElement(params, actionDeadlineExpired = () => false)');
+ const typeStart = source.indexOf('function typeText(params, actionDeadlineExpired = () => false)', clickStart);
+ const clickAxStart = source.indexOf("'click_ax':");
+ const checkedStart = source.indexOf("'set_checked':", clickAxStart);
+ const typeAxStart = source.indexOf("'type_ax':", checkedStart);
+ assert.ok(clickStart >= 0 && typeStart > clickStart && clickAxStart >= 0 && checkedStart > clickAxStart && typeAxStart > checkedStart);
+ for (const body of [source.slice(clickStart, typeStart), source.slice(clickAxStart, checkedStart), source.slice(checkedStart, typeAxStart)]) {
+ assert.match(body, /withLocalPageDispatch\(\(\) => \{[\s\S]*?el\.click\(\)/,
+ `${build}: every synthetic click path must keep the monitor's local dispatch scope active`);
+ }
+ const helper = source.match(/^ function withLocalPageDispatch\(callback\) \{[\s\S]*?^ \}/m)?.[0];
+ assert.ok(helper, `${build}: synthetic clicks must use the production local dispatch boundary`);
+
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ document.body.insertAdjacentHTML('beforeend', '<input id="local-click-checkbox" type="checkbox"><input id="local-checked-checkbox" type="checkbox"><a id="local-click-link" href="#local-click-destination">Continue</a>');
+ });
+ await page.waitForTimeout(160);
+ await page.addScriptTag({ content: `window.withLocalPageDispatch = ${helper};` });
+ const events = await page.evaluate(() => {
+ feedback = [];
+ const actions = [
+ ['click', document.getElementById('local-click-link')],
+ ['click_ax', document.getElementById('local-click-checkbox')],
+ ['set_checked', document.getElementById('local-checked-checkbox')],
+ ];
+ for (const [action, target] of actions) {
+ const finish = __wbPageMonitor.beginContentAction(action, { selector: '#' + target.id });
+ try { withLocalPageDispatch(() => target.click()); }
+ finally { finish(); }
+ }
+ return new Promise(resolve => setTimeout(() => resolve(feedback), 150));
+ });
+ assert.ok(events.some(event => event.source === 'agent' && event.operation === 'click'
+ && event.navigationUrl === 'https://monitor.test/start#local-click-destination'),
+ `${build}: a synthetic link click must arm its agent navigation`);
+ assert.equal(events.some(event => event.source !== 'agent'), false,
+ `${build}: synthetic checkbox changes must not be reported as page/user feedback`);
+ assert.equal(await page.locator('#local-click-checkbox').isChecked(), true);
+ assert.equal(await page.locator('#local-checked-checkbox').isChecked(), true);
+ } finally { await browser.close(); }
+ });
+
test(`${build}: click-driven programmatic focus checks the monitor fence before focusin`, () => {
const source = read(build, 'content/content.js');
const clickStart = source.indexOf('function clickElement(params, actionDeadlineExpired = () => false)');