Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

perf: wake Firefox checkable observations on events

alectimison-makerf2240ddf8a94
diff --git a/.github/workflows/webmcp.yml b/.github/workflows/webmcp.yml
index 51b2227d9..8a8d25b54 100644
--- a/.github/workflows/webmcp.yml
+++ b/.github/workflows/webmcp.yml
@@ -42,9 +42,12 @@ jobs:
node-version: 22
cache: npm
- - run: npm ci
+ - name: Install dependencies
+ id: dependencies
+ run: npm ci
- name: Report Chrome version
+ id: chrome_version
run: google-chrome --version
- name: Run PDF MIME handler regression
@@ -53,6 +56,7 @@ jobs:
run: npm run test:pdf-mime-handler
- name: Run real Chrome WebMCP smoke test
+ if: ${{ !cancelled() && steps.dependencies.outcome == 'success' && steps.chrome_version.outcome == 'success' }}
env:
WEBMCP_CHROME_PATH: /usr/bin/google-chrome
run: npm run test:webmcp
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index c9d12972f..7199ebacc 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -2,6 +2,17 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, basename } from 'node:path';
+function unpackRemoteObjectValue(value) {
+ if (value?.type !== 'object' || !Array.isArray(value.value)) return value;
+ return Object.fromEntries(value.value.flatMap(entry => {
+ if (!Array.isArray(entry) || entry.length < 2 || typeof entry[0] !== 'string') return [];
+ const [key, property] = entry;
+ return [[key, property && typeof property === 'object' && Object.hasOwn(property, 'value')
+ ? property.value
+ : undefined]];
+ }));
+}
+
export class BidiSession {
constructor(WebSocketImpl = WebSocket) {
this.WebSocketImpl = WebSocketImpl;
@@ -102,6 +113,52 @@ export class BidiSession {
return this.send('script.callFunction', { target: { context: match.context }, functionDeclaration,
arguments: [{ sharedId: match.node.sharedId }, ...args], awaitPromise: true });
}
+ async _waitForCheckableState(match, desiredChecked, timeoutMs = 80) {
+ const result = await this.call(match, `async (el, desired, timeout) => {
+ const read = () => el.isConnected && (el.type === "checkbox" || el.type === "radio")
+ ? !!el.checked
+ : null;
+ const initial = read();
+ if (initial === desired || initial === null) return { checkedAfter: initial, elapsedMs: 0 };
+ const startedAt = performance.now();
+ return await new Promise(resolve => {
+ let timer = null;
+ let settled = false;
+ const cleanup = () => {
+ el.removeEventListener('input', check);
+ el.removeEventListener('change', check);
+ clearTimeout(timer);
+ };
+ const finish = checkedAfter => {
+ if (settled) return;
+ settled = true;
+ cleanup();
+ resolve({ checkedAfter, elapsedMs: Math.max(0, Math.round(performance.now() - startedAt)) });
+ };
+ const check = () => {
+ const checkedAfter = read();
+ if (checkedAfter === desired || checkedAfter === null) finish(checkedAfter);
+ };
+ el.addEventListener('input', check);
+ el.addEventListener('change', check);
+ timer = setTimeout(() => finish(read()), Math.max(0, timeout));
+ check();
+ });
+ }`, [
+ { type: 'boolean', value: desiredChecked },
+ { type: 'number', value: timeoutMs },
+ ]);
+ const observation = unpackRemoteObjectValue(result.result?.type ? result.result : result.result?.value);
+ if (observation && typeof observation === 'object') {
+ return {
+ checkedAfter: typeof observation.checkedAfter === 'boolean' ? observation.checkedAfter : null,
+ elapsedMs: Number.isFinite(Number(observation.elapsedMs)) ? Math.max(0, Number(observation.elapsedMs)) : timeoutMs,
+ };
+ }
+ // A malformed page response stays inconclusive and keeps the bounded
+ // observation budget visible to the caller.
+ return { checkedAfter: null, elapsedMs: timeoutMs };
+ }
async closeRun(id) {
this.closedRuns.add(id);
if (this.closedRuns.size > 4096) this.closedRuns.delete(this.closedRuns.values().next().value);
@@ -223,12 +280,12 @@ export class BidiSession {
finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }
assertLive();
if (checkable) {
- await new Promise(resolve => setTimeout(resolve, 80));
+ const observation = await this._waitForCheckableState(match, checkable.desiredChecked, 80);
assertLive();
- const checked = await this.call(match, '(el) => el.isConnected && (el.type === "checkbox" || el.type === "radio") ? !!el.checked : null');
- const checkedAfter = checked.result?.value;
+ const checkedAfter = observation.checkedAfter;
if (typeof checkedAfter !== 'boolean') {
return { success: false, dispatched: true, outcomeUnknown: true, retryable: false,
+ _checkableObservationMs: observation.elapsedMs,
error: 'Checkable target changed after trusted click; inspect the page before retrying.' };
}
const stateMatchesDesired = checkedAfter === checkable.desiredChecked;
@@ -242,6 +299,7 @@ export class BidiSession {
desiredChecked: checkable.desiredChecked,
checkboxIdentity: checkable.checkboxIdentity,
checkboxState: { identity: checkable.checkboxIdentity, desiredChecked: checkable.desiredChecked, actualChecked: checkedAfter },
+ _checkableObservationMs: observation.elapsedMs,
...(stateMatchesDesired ? { observedEffects: ['checked_state'] } : {
noProgress: true,
error: checkable.inputType === 'checkbox'
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 0cdfb6627..eaf550f60 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -11008,6 +11008,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
onUpdate,
{
completionBatchStartState,
+ traceStep: step,
promptTier,
dispatchBinding: pipelineToolbarPreflight.probe?.dispatchBinding || null,
...messageRecipientExecutionContext,
@@ -12256,9 +12257,51 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
}
- async _dispatchClickAx(tabId, args, axScope = null, dispatchBinding = null, messageRecipientContext = {}, upstreamAbortSignal = null, dispatchState = { started: false }) {
+ _recordClickAxTiming(tabId, step, timing, result) {
+ const runId = this.currentRunId.get(tabId);
+ if (!runId) return;
+ let outcome = 'inconclusive';
+ if (timing.outcomeUnknown === true) outcome = 'unknown';
+ else if (!timing.syntheticDispatched && timing.trustedInputEvents === 0) outcome = 'pre_dispatch';
+ else if (result?.noProgress === true) outcome = 'no_progress';
+ else if (result?.success !== true) outcome = 'failed';
+ else if (result?.verified === true) outcome = 'verified';
+ const extra = {
+ preflightMs: timing.preflightMs,
+ syntheticDispatchMs: timing.syntheticDispatchMs,
+ syntheticResponseMs: timing.syntheticResponseMs,
+ postClickObservationMs: timing.postClickObservationMs,
+ fallbackPreparationMs: 0,
+ fallbackDispatchMs: 0,
+ trustedObservationMs: 0,
+ clickPathMs: timing.clickPathMs,
+ trustedInputEvents: timing.trustedInputEvents,
+ syntheticDispatched: timing.syntheticDispatched,
+ trustedFallbackAttempted: false,
+ outcomeUnknown: timing.outcomeUnknown === true,
+ safetyVeto: false,
+ duplicateFallbackBlocked: false,
+ outcome,
+ };
try {
- return await this._withContentActionDeadline(
+ const pending = trace.recordNote(runId, Number.isInteger(step) ? step : null, 'click_ax_timing', extra);
+ if (pending?.catch) pending.catch(() => {});
+ } catch {}
+ }
+
+ async _dispatchClickAx(tabId, args, axScope = null, dispatchBinding = null, messageRecipientContext = {}, upstreamAbortSignal = null, dispatchState = { started: false }, traceStep = null) {
+ const timing = {
+ startedAtEpoch: Date.now(),
+ preflightMs: 0,
+ syntheticDispatchMs: 0,
+ syntheticResponseMs: 0,
+ postClickObservationMs: 0,
+ trustedInputEvents: 0,
+ syntheticDispatched: false,
+ };
+ let result = null;
+ try {
+ result = await this._withContentActionDeadline(
deadlineAbortSignal => this._dispatchClickAxImpl(
tabId,
args,
@@ -12268,20 +12311,29 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
deadlineAbortSignal,
upstreamAbortSignal,
dispatchState,
+ timing,
),
'click_ax',
);
+ return result;
} catch (error) {
if (error?.code !== 'content_action_timeout') throw error;
- if (dispatchState.started) return this._contentActionTimeoutResult('click_ax', error);
- return {
- success: false,
- dispatched: false,
- noDispatch: true,
- outcomeUnknown: false,
- retryable: true,
- error: `${error.message} No click was sent because click preparation did not finish. Re-observe the page before retrying.`,
- };
+ result = dispatchState.started
+ ? this._contentActionTimeoutResult('click_ax', error)
+ : {
+ success: false,
+ dispatched: false,
+ noDispatch: true,
+ outcomeUnknown: false,
+ retryable: true,
+ error: `${error.message} No click was sent because click preparation did not finish. Re-observe the page before retrying.`,
+ };
+ return result;
+ } finally {
+ timing.clickPathMs = Math.max(0, Date.now() - timing.startedAtEpoch);
+ timing.outcomeUnknown = result?.outcomeUnknown === true
+ || (dispatchState.started && result?.dispatched !== false && result?.noDispatch !== true && result?.success !== true);
+ this._recordClickAxTiming(tabId, traceStep, timing, result);
}
}
@@ -12294,6 +12346,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
deadlineAbortSignal = null,
upstreamAbortSignal = null,
dispatchState = { started: false },
+ timing = null,
) {
const throwIfAborted = () => {
this._throwIfAborted(upstreamAbortSignal);
@@ -12326,6 +12379,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
.sort((a, b) => a - b)[0] || 0;
const send = () => {
throwIfAborted();
+ if (timing) timing.preflightMs = Math.max(0, Date.now() - timing.startedAtEpoch);
dispatchState.started = true;
return firefoxBidi.sendContent(tabId, {
target: 'content',
@@ -12351,6 +12405,35 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const deadlineResult = provenNoDispatchDeadline(response);
if (deadlineResult) return deadlineResult;
throwIfAborted();
+ const syntheticClickStartedAt = Number(response?._syntheticClickStartedAt);
+ const syntheticClickDispatchMs = Number(response?._syntheticClickDispatchMs);
+ if (timing && Number.isFinite(syntheticClickStartedAt) && syntheticClickStartedAt > 0) {
+ timing.preflightMs = Math.max(0, syntheticClickStartedAt - timing.startedAtEpoch);
+ timing.syntheticDispatchMs = Number.isFinite(syntheticClickDispatchMs)
+ ? Math.max(0, syntheticClickDispatchMs)
+ : 0;
+ timing.syntheticResponseMs = Math.max(
+ 0,
+ Date.now() - syntheticClickStartedAt - timing.syntheticDispatchMs,
+ );
+ timing.syntheticDispatched = true;
+ }
+ const checkableObservationMs = Number(response?._checkableObservationMs);
+ if (timing && Number.isFinite(checkableObservationMs)) {
+ timing.postClickObservationMs = Math.max(0, checkableObservationMs);
+ }
+ if (response && Object.prototype.hasOwnProperty.call(response, '_syntheticClickStartedAt')) {
+ delete response._syntheticClickStartedAt;
+ }
+ if (response && Object.prototype.hasOwnProperty.call(response, '_syntheticClickDispatchMs')) {
+ delete response._syntheticClickDispatchMs;
+ }
+ if (response && Object.prototype.hasOwnProperty.call(response, '_checkableObservationMs')) {
+ delete response._checkableObservationMs;
+ }
+ if (timing && response?.method === 'firefox-bidi' && response?.dispatched === true) {
+ timing.trustedInputEvents = 3;
+ }
response = await this._settleContentFilePickerGuard(tabId, response);
throwIfAborted();
if (response?.documentToken && (
@@ -33386,6 +33469,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
{ messageRecipientGuardRequired, messageRecipientDispatchBinding },
contentPipelineAbortSignal,
contentPipelineDispatchState,
+ dispatchContext.traceStep,
);
}
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 6708b81cc..039c79327 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -5337,7 +5337,10 @@
};
}
dispatched = true;
+ const syntheticClickStartedAt = Date.now();
+ const syntheticClickDispatchStartedAt = performance.now();
const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());
+ const syntheticClickDispatchMs = Math.max(0, performance.now() - syntheticClickDispatchStartedAt);
if (filePickerGuard.blocked) {
return failure(
filePickerBlockedResponse(filePickerGuard.blocked, targetName || '').error,
@@ -5362,6 +5365,8 @@
method: 'click_ax',
ref_id,
tag,
+ _syntheticClickStartedAt: syntheticClickStartedAt,
+ _syntheticClickDispatchMs: syntheticClickDispatchMs,
rect: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) },
...(targetContext ? { targetContext } : {}),
...(filePickerGuard.guardId ? { _filePickerGuardId: filePickerGuard.guardId } : {}),
diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjs
index 460a88a3e..a87264ace 100644
--- a/test/firefox-bidi-e2e.mjs
+++ b/test/firefox-bidi-e2e.mjs
@@ -76,7 +76,7 @@ try {
assert.deepEqual(pointPrepared.point,{x:22,y:210});
await session.perform(runId,'click',{token:pointToken,url,point:pointPrepared.point});
assert.deepEqual(JSON.parse((await evaluate('JSON.stringify(window.clickedPoint)')).value),{x:22,y:210,trusted:true});
- await evaluate(`document.body.insertAdjacentHTML('beforeend', '<input id="checkable" type="checkbox"><input id="blocked-checkable" type="checkbox">'); document.querySelector('#blocked-checkable').onclick=event=>event.preventDefault();`);
+ await evaluate(`document.body.insertAdjacentHTML('beforeend', '<input id="checkable" type="checkbox"><input id="blocked-checkable" type="checkbox"><input id="delayed-checkable" type="checkbox">'); document.querySelector('#blocked-checkable').onclick=event=>event.preventDefault(); document.querySelector('#delayed-checkable').addEventListener('click',event=>{event.preventDefault();setTimeout(()=>{const input=document.querySelector('#delayed-checkable');input.checked=true;input.dispatchEvent(new Event('input',{bubbles:true}));input.dispatchEvent(new Event('change',{bubbles:true}));},25);},{once:true});`);
const checked = await session.perform(runId, 'click', {
...await mark('#checkable'),
checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'checkable' },
@@ -84,6 +84,15 @@ try {
assert.equal(checked.success, true);
assert.equal(checked.verified, true);
assert.equal(checked.checkedAfter, true);
+ assert.equal(checked._checkableObservationMs, 0, 'already-observed check state should not wait out the old 80ms delay');
+ const delayedChecked = await session.perform(runId, 'click', {
+ ...await mark('#delayed-checkable'),
+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'delayed-checkable' },
+ });
+ assert.equal(delayedChecked.success, true);
+ assert.equal(delayedChecked.verified, true);
+ assert.ok(delayedChecked._checkableObservationMs >= 10, 'the test should wait for the delayed state event');
+ assert.ok(delayedChecked._checkableObservationMs < 80, 'the change event should wake the observer before the timeout');
const blockedCheck = await session.perform(runId, 'click', {
...await mark('#blocked-checkable'),
checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'blocked-checkable' },
@@ -197,7 +206,7 @@ try {
await new Promise(resolve => setTimeout(resolve, 100));
assert.equal((await evaluate('document.querySelector("#text").value')).value, afterStop);
await assert.rejects(session.perform(resumed, 'click', await mark('#click')), /stopped|disconnected/);
- console.log('PASS: Firefox connection, dialogs, uploads, literal newlines, focus guards, interrupted typing, deferred blank-tab binding, and packaged extension integration');
+ console.log(`PASS: Firefox connection, dialogs, uploads, literal newlines, focus guards, interrupted typing, deferred blank-tab binding, and packaged extension integration; checkable state event woke in ${delayedChecked._checkableObservationMs}ms (80ms cap)`);
} finally {
await session?.close().catch(() => {});
firefox.kill('SIGTERM');
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index bad2c0312..9a24f4c02 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -221,7 +221,9 @@ test('trusted checkable clicks report a prevented state transition', async () =>
const session = new BidiSession(); const runId = id(); const sent = [];
session.runs.set(runId, { context: 'tab' });
session.locate = async () => ({ context: 'tab', node: { sharedId: 'check' } });
- session.call = async (_match, fn) => ({ result: { value: fn.includes('checkbox" || el.type') ? false : true } });
+ session.call = async (_match, fn) => ({ result: { value: fn.includes('checkbox" || el.type')
+ ? { checkedAfter: false, elapsedMs: 80 }
+ : true } });
session.send = async (method, params) => { sent.push({ method, params }); return {}; };
const result = await session.perform(runId, 'click', {
checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'check' },
@@ -233,6 +235,27 @@ test('trusted checkable clicks report a prevented state transition', async () =>
assert.equal(result.checkedAfter, false);
assert.equal(result.checkedChanged, false);
});
+test('trusted checkable clicks wait on state events and report observation time', async () => {
+ const session = new BidiSession(); const runId = id(); const scripts = [];
+ session.runs.set(runId, { context: 'tab' });
+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'check' } });
+ session.call = async (_match, fn) => {
+ scripts.push(fn);
+ return scripts.length === 1
+ ? { result: { value: true } }
+ : { result: { value: { checkedAfter: true, elapsedMs: 12 } } };
+ };
+ session.send = async () => ({});
+ const result = await session.perform(runId, 'click', {
+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'check' },
+ });
+ assert.equal(result.success, true);
+ assert.equal(result.verified, true);
+ assert.equal(result.checkedAfter, true);
+ assert.equal(result._checkableObservationMs, 12);
+ assert.match(scripts[1], /addEventListener\(['"]change['"], check\)/);
+ assert.match(scripts[1], /setTimeout\(/, 'a bounded timeout must remain as the no-event fallback');
+});
test('trusted target validation descends through open shadow roots', async () => {
const session = new BidiSession(); const runId = id(); let validation;
session.runs.set(runId, { context: 'tab' });
diff --git a/test/run.js b/test/run.js
index 1e76b4f90..f0abaf4ee 100644
--- a/test/run.js
+++ b/test/run.js
@@ -20839,6 +20839,9 @@ test('public click_ax preserves the complete pre-extraction dispatch and post-pr
documentToken: 'fresh-document-token',
refScopeUrl: 'https://example.test/fresh-route',
routeChanged: true,
+ _syntheticClickStartedAt: Date.now(),
+ _syntheticClickDispatchMs: 1.25,
+ _checkableObservationMs: 12,
};
const tabs = {
get: async () => ({ url: 'https://example.test/fresh-route' }),
@@ -20918,6 +20921,11 @@ test('public click_ax preserves the complete pre-extraction dispatch and post-pr
const result = await agent.executeTool(tabId, 'click_ax', { ref_id: 'ref_907' });
assert.equal(result.documentToken, 'fresh-document-token', `${label}: click_ax response token must remain public`);
assert.equal(result.refScopeUrl, 'https://example.test/fresh-route', `${label}: click_ax scope URL must remain public`);
+ if (label === 'firefox') {
+ assert.equal(Object.hasOwn(result, '_syntheticClickStartedAt'), false, 'firefox: private click clock leaked to the tool result');
+ assert.equal(Object.hasOwn(result, '_syntheticClickDispatchMs'), false, 'firefox: private dispatch clock leaked to the tool result');
+ assert.equal(Object.hasOwn(result, '_checkableObservationMs'), false, 'firefox: private observation clock leaked to the tool result');
+ }
assert.deepEqual(agent._lastAxScopes.get(tabId), {
documentToken: 'fresh-document-token',
pageUrl: 'https://example.test/fresh-route',
@@ -110799,6 +110807,13 @@ test('content-script actions have a bounded unknown-outcome timeout', async () =
assert.match(fallbackDeadlineSource, /this\._withContentActionDeadline\([\s\S]*this\._maybeFallbackClickAxWithCdpImpl\([\s\S]*'click_ax'/, 'chrome: click_ax fallback pipeline bypasses the deadline');
assert.match(fallbackDeadlineSource, /abortSignal =>[\s\S]*baseline,[\s\S]*abortSignal/, 'chrome: click_ax fallback deadline does not propagate cancellation');
} else {
+ const clickTimingTraceStart = source.indexOf(' _recordClickAxTiming(');
+ const clickTimingTraceEnd = source.indexOf(' async _dispatchClickAx(', clickTimingTraceStart);
+ assert.ok(clickTimingTraceStart >= 0 && clickTimingTraceEnd > clickTimingTraceStart, 'firefox: click_ax timing trace recorder missing');
+ assert.match(source.slice(clickTimingTraceStart, clickTimingTraceEnd), /trace\.recordNote\(runId,[\s\S]*'click_ax_timing'/, 'firefox: click_ax timings are not recorded as a trace note');
+ assert.match(toolPipelineSource, /traceStep: step,/, 'firefox: click_ax trace notes do not receive the current tool step');
+ assert.match(clickAxSource, /_checkableObservationMs[\s\S]*delete response\._checkableObservationMs/, 'firefox: private state-observation timing can escape in the tool result');
+ assert.match(contentClickAx, /_syntheticClickStartedAt:[\s\S]*_syntheticClickDispatchMs:/, 'firefox: synthetic click boundary timing is missing');
assert.match(source, /runContentActionStage\([\s\S]*this\._keyProgressSnapshot\(tabId, abortSignal\)/, 'firefox: pre-dispatch Arrow snapshot bypasses the pipeline deadline');
assert.match(source, /runContentActionStage\([\s\S]*this\._verifyProvisionalKeyProgress\([\s\S]*abortSignal/, 'firefox: post-dispatch Arrow verification bypasses the pipeline deadline');
const uploadStart = source.indexOf('const buildInjectCode = actionDeadlineAt =>');