Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
perf: wake Firefox checkable observations on events
alectimison-makerf2240ddf8a94
diff --git a/.github/workflows/webmcp.yml b/.github/workflows/webmcp.ymlindex 51b2227d9..8a8d25b54 100644--- a/.github/workflows/webmcp.yml+++ b/.github/workflows/webmcp.yml@@ -42,9 +42,12 @@ jobs:node-version: 22cache: npm- - run: npm ci+ - name: Install dependencies+ id: dependencies+ run: npm ci- name: Report Chrome version+ id: chrome_versionrun: google-chrome --version- name: Run PDF MIME handler regression@@ -53,6 +56,7 @@ jobs:run: npm run test:pdf-mime-handler- name: Run real Chrome WebMCP smoke test+ if: ${{ !cancelled() && steps.dependencies.outcome == 'success' && steps.chrome_version.outcome == 'success' }}env:WEBMCP_CHROME_PATH: /usr/bin/google-chromerun: npm run test:webmcpdiff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex c9d12972f..7199ebacc 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -2,6 +2,17 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join, basename } from 'node:path';+function unpackRemoteObjectValue(value) {+ if (value?.type !== 'object' || !Array.isArray(value.value)) return value;+ return Object.fromEntries(value.value.flatMap(entry => {+ if (!Array.isArray(entry) || entry.length < 2 || typeof entry[0] !== 'string') return [];+ const [key, property] = entry;+ return [[key, property && typeof property === 'object' && Object.hasOwn(property, 'value')+ ? property.value+ : undefined]];+ }));+}+export class BidiSession {constructor(WebSocketImpl = WebSocket) {this.WebSocketImpl = WebSocketImpl;@@ -102,6 +113,52 @@ export class BidiSession {return this.send('script.callFunction', { target: { context: match.context }, functionDeclaration,arguments: [{ sharedId: match.node.sharedId }, ...args], awaitPromise: true });}+ async _waitForCheckableState(match, desiredChecked, timeoutMs = 80) {+ const result = await this.call(match, `async (el, desired, timeout) => {+ const read = () => el.isConnected && (el.type === "checkbox" || el.type === "radio")+ ? !!el.checked+ : null;+ const initial = read();+ if (initial === desired || initial === null) return { checkedAfter: initial, elapsedMs: 0 };+ const startedAt = performance.now();+ return await new Promise(resolve => {+ let timer = null;+ let settled = false;+ const cleanup = () => {+ el.removeEventListener('input', check);+ el.removeEventListener('change', check);+ clearTimeout(timer);+ };+ const finish = checkedAfter => {+ if (settled) return;+ settled = true;+ cleanup();+ resolve({ checkedAfter, elapsedMs: Math.max(0, Math.round(performance.now() - startedAt)) });+ };+ const check = () => {+ const checkedAfter = read();+ if (checkedAfter === desired || checkedAfter === null) finish(checkedAfter);+ };+ el.addEventListener('input', check);+ el.addEventListener('change', check);+ timer = setTimeout(() => finish(read()), Math.max(0, timeout));+ check();+ });+ }`, [+ { type: 'boolean', value: desiredChecked },+ { type: 'number', value: timeoutMs },+ ]);+ const observation = unpackRemoteObjectValue(result.result?.type ? result.result : result.result?.value);+ if (observation && typeof observation === 'object') {+ return {+ checkedAfter: typeof observation.checkedAfter === 'boolean' ? observation.checkedAfter : null,+ elapsedMs: Number.isFinite(Number(observation.elapsedMs)) ? Math.max(0, Number(observation.elapsedMs)) : timeoutMs,+ };+ }+ // A malformed page response stays inconclusive and keeps the bounded+ // observation budget visible to the caller.+ return { checkedAfter: null, elapsedMs: timeoutMs };+ }async closeRun(id) {this.closedRuns.add(id);if (this.closedRuns.size > 4096) this.closedRuns.delete(this.closedRuns.values().next().value);@@ -223,12 +280,12 @@ export class BidiSession {finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }assertLive();if (checkable) {- await new Promise(resolve => setTimeout(resolve, 80));+ const observation = await this._waitForCheckableState(match, checkable.desiredChecked, 80);assertLive();- const checked = await this.call(match, '(el) => el.isConnected && (el.type === "checkbox" || el.type === "radio") ? !!el.checked : null');- const checkedAfter = checked.result?.value;+ const checkedAfter = observation.checkedAfter;if (typeof checkedAfter !== 'boolean') {return { success: false, dispatched: true, outcomeUnknown: true, retryable: false,+ _checkableObservationMs: observation.elapsedMs,error: 'Checkable target changed after trusted click; inspect the page before retrying.' };}const stateMatchesDesired = checkedAfter === checkable.desiredChecked;@@ -242,6 +299,7 @@ export class BidiSession {desiredChecked: checkable.desiredChecked,checkboxIdentity: checkable.checkboxIdentity,checkboxState: { identity: checkable.checkboxIdentity, desiredChecked: checkable.desiredChecked, actualChecked: checkedAfter },+ _checkableObservationMs: observation.elapsedMs,...(stateMatchesDesired ? { observedEffects: ['checked_state'] } : {noProgress: true,error: checkable.inputType === 'checkbox'diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 0cdfb6627..eaf550f60 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -11008,6 +11008,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout donUpdate,{completionBatchStartState,+ traceStep: step,promptTier,dispatchBinding: pipelineToolbarPreflight.probe?.dispatchBinding || null,...messageRecipientExecutionContext,@@ -12256,9 +12257,51 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}}- async _dispatchClickAx(tabId, args, axScope = null, dispatchBinding = null, messageRecipientContext = {}, upstreamAbortSignal = null, dispatchState = { started: false }) {+ _recordClickAxTiming(tabId, step, timing, result) {+ const runId = this.currentRunId.get(tabId);+ if (!runId) return;+ let outcome = 'inconclusive';+ if (timing.outcomeUnknown === true) outcome = 'unknown';+ else if (!timing.syntheticDispatched && timing.trustedInputEvents === 0) outcome = 'pre_dispatch';+ else if (result?.noProgress === true) outcome = 'no_progress';+ else if (result?.success !== true) outcome = 'failed';+ else if (result?.verified === true) outcome = 'verified';+ const extra = {+ preflightMs: timing.preflightMs,+ syntheticDispatchMs: timing.syntheticDispatchMs,+ syntheticResponseMs: timing.syntheticResponseMs,+ postClickObservationMs: timing.postClickObservationMs,+ fallbackPreparationMs: 0,+ fallbackDispatchMs: 0,+ trustedObservationMs: 0,+ clickPathMs: timing.clickPathMs,+ trustedInputEvents: timing.trustedInputEvents,+ syntheticDispatched: timing.syntheticDispatched,+ trustedFallbackAttempted: false,+ outcomeUnknown: timing.outcomeUnknown === true,+ safetyVeto: false,+ duplicateFallbackBlocked: false,+ outcome,+ };try {- return await this._withContentActionDeadline(+ const pending = trace.recordNote(runId, Number.isInteger(step) ? step : null, 'click_ax_timing', extra);+ if (pending?.catch) pending.catch(() => {});+ } catch {}+ }++ async _dispatchClickAx(tabId, args, axScope = null, dispatchBinding = null, messageRecipientContext = {}, upstreamAbortSignal = null, dispatchState = { started: false }, traceStep = null) {+ const timing = {+ startedAtEpoch: Date.now(),+ preflightMs: 0,+ syntheticDispatchMs: 0,+ syntheticResponseMs: 0,+ postClickObservationMs: 0,+ trustedInputEvents: 0,+ syntheticDispatched: false,+ };+ let result = null;+ try {+ result = await this._withContentActionDeadline(deadlineAbortSignal => this._dispatchClickAxImpl(tabId,args,@@ -12268,20 +12311,29 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout ddeadlineAbortSignal,upstreamAbortSignal,dispatchState,+ timing,),'click_ax',);+ return result;} catch (error) {if (error?.code !== 'content_action_timeout') throw error;- if (dispatchState.started) return this._contentActionTimeoutResult('click_ax', error);- return {- success: false,- dispatched: false,- noDispatch: true,- outcomeUnknown: false,- retryable: true,- error: `${error.message} No click was sent because click preparation did not finish. Re-observe the page before retrying.`,- };+ result = dispatchState.started+ ? this._contentActionTimeoutResult('click_ax', error)+ : {+ success: false,+ dispatched: false,+ noDispatch: true,+ outcomeUnknown: false,+ retryable: true,+ error: `${error.message} No click was sent because click preparation did not finish. Re-observe the page before retrying.`,+ };+ return result;+ } finally {+ timing.clickPathMs = Math.max(0, Date.now() - timing.startedAtEpoch);+ timing.outcomeUnknown = result?.outcomeUnknown === true+ || (dispatchState.started && result?.dispatched !== false && result?.noDispatch !== true && result?.success !== true);+ this._recordClickAxTiming(tabId, traceStep, timing, result);}}@@ -12294,6 +12346,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout ddeadlineAbortSignal = null,upstreamAbortSignal = null,dispatchState = { started: false },+ timing = null,) {const throwIfAborted = () => {this._throwIfAborted(upstreamAbortSignal);@@ -12326,6 +12379,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d.sort((a, b) => a - b)[0] || 0;const send = () => {throwIfAborted();+ if (timing) timing.preflightMs = Math.max(0, Date.now() - timing.startedAtEpoch);dispatchState.started = true;return firefoxBidi.sendContent(tabId, {target: 'content',@@ -12351,6 +12405,35 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst deadlineResult = provenNoDispatchDeadline(response);if (deadlineResult) return deadlineResult;throwIfAborted();+ const syntheticClickStartedAt = Number(response?._syntheticClickStartedAt);+ const syntheticClickDispatchMs = Number(response?._syntheticClickDispatchMs);+ if (timing && Number.isFinite(syntheticClickStartedAt) && syntheticClickStartedAt > 0) {+ timing.preflightMs = Math.max(0, syntheticClickStartedAt - timing.startedAtEpoch);+ timing.syntheticDispatchMs = Number.isFinite(syntheticClickDispatchMs)+ ? Math.max(0, syntheticClickDispatchMs)+ : 0;+ timing.syntheticResponseMs = Math.max(+ 0,+ Date.now() - syntheticClickStartedAt - timing.syntheticDispatchMs,+ );+ timing.syntheticDispatched = true;+ }+ const checkableObservationMs = Number(response?._checkableObservationMs);+ if (timing && Number.isFinite(checkableObservationMs)) {+ timing.postClickObservationMs = Math.max(0, checkableObservationMs);+ }+ if (response && Object.prototype.hasOwnProperty.call(response, '_syntheticClickStartedAt')) {+ delete response._syntheticClickStartedAt;+ }+ if (response && Object.prototype.hasOwnProperty.call(response, '_syntheticClickDispatchMs')) {+ delete response._syntheticClickDispatchMs;+ }+ if (response && Object.prototype.hasOwnProperty.call(response, '_checkableObservationMs')) {+ delete response._checkableObservationMs;+ }+ if (timing && response?.method === 'firefox-bidi' && response?.dispatched === true) {+ timing.trustedInputEvents = 3;+ }response = await this._settleContentFilePickerGuard(tabId, response);throwIfAborted();if (response?.documentToken && (@@ -33386,6 +33469,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d{ messageRecipientGuardRequired, messageRecipientDispatchBinding },contentPipelineAbortSignal,contentPipelineDispatchState,+ dispatchContext.traceStep,);}diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 6708b81cc..039c79327 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -5337,7 +5337,10 @@};}dispatched = true;+ const syntheticClickStartedAt = Date.now();+ const syntheticClickDispatchStartedAt = performance.now();const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());+ const syntheticClickDispatchMs = Math.max(0, performance.now() - syntheticClickDispatchStartedAt);if (filePickerGuard.blocked) {return failure(filePickerBlockedResponse(filePickerGuard.blocked, targetName || '').error,@@ -5362,6 +5365,8 @@method: 'click_ax',ref_id,tag,+ _syntheticClickStartedAt: syntheticClickStartedAt,+ _syntheticClickDispatchMs: syntheticClickDispatchMs,rect: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) },...(targetContext ? { targetContext } : {}),...(filePickerGuard.guardId ? { _filePickerGuardId: filePickerGuard.guardId } : {}),diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjsindex 460a88a3e..a87264ace 100644--- a/test/firefox-bidi-e2e.mjs+++ b/test/firefox-bidi-e2e.mjs@@ -76,7 +76,7 @@ try {assert.deepEqual(pointPrepared.point,{x:22,y:210});await session.perform(runId,'click',{token:pointToken,url,point:pointPrepared.point});assert.deepEqual(JSON.parse((await evaluate('JSON.stringify(window.clickedPoint)')).value),{x:22,y:210,trusted:true});- await evaluate(`document.body.insertAdjacentHTML('beforeend', '<input id="checkable" type="checkbox"><input id="blocked-checkable" type="checkbox">'); document.querySelector('#blocked-checkable').onclick=event=>event.preventDefault();`);+ await evaluate(`document.body.insertAdjacentHTML('beforeend', '<input id="checkable" type="checkbox"><input id="blocked-checkable" type="checkbox"><input id="delayed-checkable" type="checkbox">'); document.querySelector('#blocked-checkable').onclick=event=>event.preventDefault(); document.querySelector('#delayed-checkable').addEventListener('click',event=>{event.preventDefault();setTimeout(()=>{const input=document.querySelector('#delayed-checkable');input.checked=true;input.dispatchEvent(new Event('input',{bubbles:true}));input.dispatchEvent(new Event('change',{bubbles:true}));},25);},{once:true});`);const checked = await session.perform(runId, 'click', {...await mark('#checkable'),checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'checkable' },@@ -84,6 +84,15 @@ try {assert.equal(checked.success, true);assert.equal(checked.verified, true);assert.equal(checked.checkedAfter, true);+ assert.equal(checked._checkableObservationMs, 0, 'already-observed check state should not wait out the old 80ms delay');+ const delayedChecked = await session.perform(runId, 'click', {+ ...await mark('#delayed-checkable'),+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'delayed-checkable' },+ });+ assert.equal(delayedChecked.success, true);+ assert.equal(delayedChecked.verified, true);+ assert.ok(delayedChecked._checkableObservationMs >= 10, 'the test should wait for the delayed state event');+ assert.ok(delayedChecked._checkableObservationMs < 80, 'the change event should wake the observer before the timeout');const blockedCheck = await session.perform(runId, 'click', {...await mark('#blocked-checkable'),checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'blocked-checkable' },@@ -197,7 +206,7 @@ try {await new Promise(resolve => setTimeout(resolve, 100));assert.equal((await evaluate('document.querySelector("#text").value')).value, afterStop);await assert.rejects(session.perform(resumed, 'click', await mark('#click')), /stopped|disconnected/);- console.log('PASS: Firefox connection, dialogs, uploads, literal newlines, focus guards, interrupted typing, deferred blank-tab binding, and packaged extension integration');+ console.log(`PASS: Firefox connection, dialogs, uploads, literal newlines, focus guards, interrupted typing, deferred blank-tab binding, and packaged extension integration; checkable state event woke in ${delayedChecked._checkableObservationMs}ms (80ms cap)`);} finally {await session?.close().catch(() => {});firefox.kill('SIGTERM');diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex bad2c0312..9a24f4c02 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -221,7 +221,9 @@ test('trusted checkable clicks report a prevented state transition', async () =>const session = new BidiSession(); const runId = id(); const sent = [];session.runs.set(runId, { context: 'tab' });session.locate = async () => ({ context: 'tab', node: { sharedId: 'check' } });- session.call = async (_match, fn) => ({ result: { value: fn.includes('checkbox" || el.type') ? false : true } });+ session.call = async (_match, fn) => ({ result: { value: fn.includes('checkbox" || el.type')+ ? { checkedAfter: false, elapsedMs: 80 }+ : true } });session.send = async (method, params) => { sent.push({ method, params }); return {}; };const result = await session.perform(runId, 'click', {checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'check' },@@ -233,6 +235,27 @@ test('trusted checkable clicks report a prevented state transition', async () =>assert.equal(result.checkedAfter, false);assert.equal(result.checkedChanged, false);});+test('trusted checkable clicks wait on state events and report observation time', async () => {+ const session = new BidiSession(); const runId = id(); const scripts = [];+ session.runs.set(runId, { context: 'tab' });+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'check' } });+ session.call = async (_match, fn) => {+ scripts.push(fn);+ return scripts.length === 1+ ? { result: { value: true } }+ : { result: { value: { checkedAfter: true, elapsedMs: 12 } } };+ };+ session.send = async () => ({});+ const result = await session.perform(runId, 'click', {+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'check' },+ });+ assert.equal(result.success, true);+ assert.equal(result.verified, true);+ assert.equal(result.checkedAfter, true);+ assert.equal(result._checkableObservationMs, 12);+ assert.match(scripts[1], /addEventListener\(['"]change['"], check\)/);+ assert.match(scripts[1], /setTimeout\(/, 'a bounded timeout must remain as the no-event fallback');+});test('trusted target validation descends through open shadow roots', async () => {const session = new BidiSession(); const runId = id(); let validation;session.runs.set(runId, { context: 'tab' });diff --git a/test/run.js b/test/run.jsindex 1e76b4f90..f0abaf4ee 100644--- a/test/run.js+++ b/test/run.js@@ -20839,6 +20839,9 @@ test('public click_ax preserves the complete pre-extraction dispatch and post-prdocumentToken: 'fresh-document-token',refScopeUrl: 'https://example.test/fresh-route',routeChanged: true,+ _syntheticClickStartedAt: Date.now(),+ _syntheticClickDispatchMs: 1.25,+ _checkableObservationMs: 12,};const tabs = {get: async () => ({ url: 'https://example.test/fresh-route' }),@@ -20918,6 +20921,11 @@ test('public click_ax preserves the complete pre-extraction dispatch and post-prconst result = await agent.executeTool(tabId, 'click_ax', { ref_id: 'ref_907' });assert.equal(result.documentToken, 'fresh-document-token', `${label}: click_ax response token must remain public`);assert.equal(result.refScopeUrl, 'https://example.test/fresh-route', `${label}: click_ax scope URL must remain public`);+ if (label === 'firefox') {+ assert.equal(Object.hasOwn(result, '_syntheticClickStartedAt'), false, 'firefox: private click clock leaked to the tool result');+ assert.equal(Object.hasOwn(result, '_syntheticClickDispatchMs'), false, 'firefox: private dispatch clock leaked to the tool result');+ assert.equal(Object.hasOwn(result, '_checkableObservationMs'), false, 'firefox: private observation clock leaked to the tool result');+ }assert.deepEqual(agent._lastAxScopes.get(tabId), {documentToken: 'fresh-document-token',pageUrl: 'https://example.test/fresh-route',@@ -110799,6 +110807,13 @@ test('content-script actions have a bounded unknown-outcome timeout', async () =assert.match(fallbackDeadlineSource, /this\._withContentActionDeadline\([\s\S]*this\._maybeFallbackClickAxWithCdpImpl\([\s\S]*'click_ax'/, 'chrome: click_ax fallback pipeline bypasses the deadline');assert.match(fallbackDeadlineSource, /abortSignal =>[\s\S]*baseline,[\s\S]*abortSignal/, 'chrome: click_ax fallback deadline does not propagate cancellation');} else {+ const clickTimingTraceStart = source.indexOf(' _recordClickAxTiming(');+ const clickTimingTraceEnd = source.indexOf(' async _dispatchClickAx(', clickTimingTraceStart);+ assert.ok(clickTimingTraceStart >= 0 && clickTimingTraceEnd > clickTimingTraceStart, 'firefox: click_ax timing trace recorder missing');+ assert.match(source.slice(clickTimingTraceStart, clickTimingTraceEnd), /trace\.recordNote\(runId,[\s\S]*'click_ax_timing'/, 'firefox: click_ax timings are not recorded as a trace note');+ assert.match(toolPipelineSource, /traceStep: step,/, 'firefox: click_ax trace notes do not receive the current tool step');+ assert.match(clickAxSource, /_checkableObservationMs[\s\S]*delete response\._checkableObservationMs/, 'firefox: private state-observation timing can escape in the tool result');+ assert.match(contentClickAx, /_syntheticClickStartedAt:[\s\S]*_syntheticClickDispatchMs:/, 'firefox: synthetic click boundary timing is missing');assert.match(source, /runContentActionStage\([\s\S]*this\._keyProgressSnapshot\(tabId, abortSignal\)/, 'firefox: pre-dispatch Arrow snapshot bypasses the pipeline deadline');assert.match(source, /runContentActionStage\([\s\S]*this\._verifyProvisionalKeyProgress\([\s\S]*abortSignal/, 'firefox: post-dispatch Arrow verification bypasses the pipeline deadline');const uploadStart = source.indexOf('const buildInjectCode = actionDeadlineAt =>');