Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
// Delivery regression: hover, drag_drop and set_checked mark dispatch at the
// input transport boundary, never before their fences.
// - Chrome: the trusted CDP command that is the tool's first consequential
// input (hover: the pointer move; drag_drop and set_checked: the press, the
// no-button move before it being pointer preparation as for every click).
// - Firefox: the native BiDi request after its preparation probe and
// page-feedback fence, or the content message for content-script tools.
// A refusal before that boundary reports noDispatch; a failure after it
// reports dispatched with an unknown outcome, and nothing is resent.
//
// [mocked transport] The real Agent of each build. Chrome runs the real
// cdpClient.sendCommand: its pre-protocol check (_beforeCommand, where the
// page-feedback fence runs) is wrapped to refuse, and chrome.debugger is a
// recorder that can lose an acknowledgement after the command arrived.
// Firefox runs the real FirefoxBidiClient with a mocked native request and
// content messaging; its page-feedback fences use the real page-feedback run.
import assert from 'node:assert/strict';
import { test } from 'node:test';
const area = { get: async () => ({}), set: async () => {}, remove: async () => {} };
const api = {
storage: { local: area, session: area },
runtime: { getURL: value => `extension://test/${value}`, getManifest: () => ({ version: 'test' }), lastError: null },
tabs: { get: async () => ({ id: 17, url: 'https://boundary.test/start' }), query: async () => [{ id: 17 }],
sendMessage: async () => ({}), update: async () => ({}) },
scripting: { executeScript: async () => [{}] },
webNavigation: { getAllFrames: async () => [{ frameId: 0, documentId: 'boundary-document', url: 'https://boundary.test/start' }] },
debugger: { sendCommand: () => { throw new Error('debugger recorder not installed'); },
onEvent: { addListener() {}, removeListener() {} }, onDetach: { addListener() {}, removeListener() {} } },
};
globalThis.chrome = api;
globalThis.browser = api;
const { Agent: ChromeAgent } = await import('../src/chrome/src/agent/agent.js');
const { cdpClient } = await import('../src/chrome/src/cdp/cdp-client.js');
const { Agent: FirefoxAgent } = await import('../src/firefox/src/agent/agent.js');
const { firefoxBidi } = await import('../src/firefox/src/bidi/client.js');
const pageFeedbackVeto = () => Object.assign(
new Error('The browser changed during action preparation. Re-observe the page before acting.'),
{ code: 'page_feedback_pending' },
);
const notAttached = () => new Error('Not attached to tab 17');
const lostAck = 'Detached while handling command.';
// ββ Chrome βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
const original = { attach: cdpClient.attach, beforeCommand: cdpClient._beforeCommand,
resolveSelector: cdpClient.resolveSelector, arm: cdpClient.armFileInputClickGuard, consume: cdpClient.consumeFileInputClickGuard };
// `refuse(params, index)` rejects an Input command before the protocol call
// (the fence or the session check); `lose(params, index)` loses the
// acknowledgement of an Input command that reached the page. Returns the
// Input commands that reached chrome.debugger.
function chromeTransport({ refuse = () => null, lose = () => false } = {}) {
const delivered = [];
let index = 0;
cdpClient.sessions.set(17, { tabId: 17, attached: true });
cdpClient.attach = async () => {};
cdpClient.armFileInputClickGuard = async () => {};
cdpClient.consumeFileInputClickGuard = async () => ({ blocked: false });
cdpClient._beforeCommand = async function (tabId, method, params, sessionId) {
if (method === 'Input.dispatchMouseEvent') {
const refusal = refuse(params, index);
if (refusal) { index++; throw refusal(); }
}
return original.beforeCommand.call(this, tabId, method, params, sessionId);
};
api.debugger.sendCommand = (_target, method, params, callback) => {
let lost = false;
if (method === 'Input.dispatchMouseEvent') {
delivered.push({ type: params.type, x: params.x, y: params.y, buttons: params.buttons });
lost = lose(params, index++);
}
Promise.resolve().then(() => {
if (lost) api.runtime.lastError = { message: lostAck };
try { callback({}); } finally { api.runtime.lastError = null; }
});
};
return delivered;
}
function restoreChrome() {
Object.assign(cdpClient, { attach: original.attach, _beforeCommand: original.beforeCommand,
resolveSelector: original.resolveSelector, armFileInputClickGuard: original.arm, consumeFileInputClickGuard: original.consume });
cdpClient.sessions.delete(17);
api.debugger.sendCommand = () => { throw new Error('debugger recorder not installed'); };
api.tabs.sendMessage = async () => ({});
}
function chromeAgent() {
const agent = new ChromeAgent({ getActive: () => ({ model: 'boundary-test' }) });
agent._currentUrl = async () => 'https://boundary.test/start';
agent._clickProgressSnapshot = async () => '';
agent._annotateClickProgress = async (_t, _n, _a, response) => response;
agent._recordInteractionRect = () => {};
agent._isPdfTab = async () => false;
agent._getDevDocumentIdentity = async () => ({ documentId: 'boundary-document', pageUrl: 'https://boundary.test/start' });
return agent;
}
function chromeContent(extra = {}) {
const messages = [];
api.tabs.sendMessage = async (_tab, message) => {
messages.push(message.action);
if (message.action === 'ax_resolve_rect') {
return { success: true, ref_id: 'ref_1', tag: 'button', name: 'Menu', x: 60, y: 40, inViewport: true, rect: { x: 20, y: 20, w: 80, h: 40 } };
}
if (message.action === 'ax_resolve_two_rects') {
return { success: true, from: { ref_id: 'ref_1', x: 60, y: 40, inViewport: true, name: 'Card A' },
to: { ref_id: 'ref_2', x: 60, y: 240, inViewport: true, name: 'Lane B' } };
}
if (message.action === 'set_checked') return extra.setChecked?.(message.params) ?? { success: true };
return { success: true };
};
return messages;
}
const presses = delivered => delivered.filter(event => event.type === 'mousePressed');
const run = (agent, name, args) => {
const state = { started: false };
return agent.executeTool(17, name, args, null, { _contentActionDispatchState: state }).then(result => ({ result, state }));
};
function assertNotDispatched({ result, state }, label) {
assert.equal(result.success, false, `${label}: ${JSON.stringify(result)}`);
assert.equal(result.dispatched, false, `${label}: ${JSON.stringify(result)}`);
assert.equal(result.noDispatch, true, label);
assert.notEqual(result.outcomeUnknown, true, label);
assert.equal(state.started, false, `${label}: the call never entered dispatch`);
}
function assertUncertain({ result, state }, label) {
assert.equal(result.success, false, `${label}: ${JSON.stringify(result)}`);
assert.equal(result.dispatched, true, `${label}: ${JSON.stringify(result)}`);
assert.notEqual(result.noDispatch, true, label);
assert.equal(result.outcomeUnknown, true, label);
assert.equal(result.retryable, false, label);
assert.equal(state.started, true, label);
}
test('Chrome hover: dispatch starts at the first pointer move\'s transport boundary', async t => {
await t.test('a fence veto before the first move is not dispatched', async () => {
const delivered = chromeTransport({ refuse: () => pageFeedbackVeto });
chromeContent();
try {
const outcome = await run(chromeAgent(), 'hover', { ref_id: 'ref_1' });
assertNotDispatched(outcome, 'hover fence veto');
assert.equal(outcome.result.pageFeedbackPending, true);
assert.deepEqual(delivered, [], 'no pointer input reached the page');
} finally { restoreChrome(); }
});
await t.test('a transport refusal before the protocol call is not dispatched', async () => {
const delivered = chromeTransport({ refuse: () => notAttached });
chromeContent();
try {
assertNotDispatched(await run(chromeAgent(), 'hover', { ref_id: 'ref_1' }), 'hover transport refusal');
assert.deepEqual(delivered, []);
} finally { restoreChrome(); }
});
await t.test('a lost acknowledgement after the first move is dispatched with an unknown outcome and not resent', async () => {
const delivered = chromeTransport({ lose: (_params, index) => index === 1 });
chromeContent();
try {
assertUncertain(await run(chromeAgent(), 'hover', { ref_id: 'ref_1' }), 'hover lost acknowledgement');
assert.deepEqual(delivered.map(event => event.type), ['mouseMoved', 'mouseMoved'], 'each move was sent once');
} finally { restoreChrome(); }
});
await t.test('a refusal of the second move after the first was sent stays uncertain', async () => {
const delivered = chromeTransport({ refuse: (_params, index) => index === 1 ? pageFeedbackVeto : null });
chromeContent();
try {
const outcome = await run(chromeAgent(), 'hover', { ref_id: 'ref_1' });
assertUncertain(outcome, 'hover second-move veto');
assert.deepEqual(delivered.map(event => event.type), ['mouseMoved']);
} finally { restoreChrome(); }
});
await t.test('a normal hover sends two moves once and is dispatched', async () => {
const delivered = chromeTransport();
chromeContent();
try {
const { result, state } = await run(chromeAgent(), 'hover', { ref_id: 'ref_1' });
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(state.started, true);
assert.deepEqual(delivered.map(event => [event.type, event.x, event.y]), [['mouseMoved', 50, 40], ['mouseMoved', 60, 40]]);
} finally { restoreChrome(); }
});
});
test('Chrome drag_drop: dispatch starts when the press passes its fences', async t => {
for (const [label, refuse] of [
['a fence veto on the pointer move before the press', (_params, index) => index === 0 ? pageFeedbackVeto : null],
['a fence veto on the press', params => params.type === 'mousePressed' ? pageFeedbackVeto : null],
]) await t.test(`${label} is not dispatched`, async () => {
const delivered = chromeTransport({ refuse });
chromeContent();
try {
const outcome = await run(chromeAgent(), 'drag_drop', { fromRefId: 'ref_1', toRefId: 'ref_2', steps: 3 });
assertNotDispatched(outcome, label);
assert.equal(outcome.result.pageFeedbackPending, true);
assert.equal(presses(delivered).length, 0, 'no press reached the page');
} finally { restoreChrome(); }
});
await t.test('a transport refusal before the press protocol call is not dispatched', async () => {
const delivered = chromeTransport({ refuse: params => params.type === 'mousePressed' ? notAttached : null });
chromeContent();
try {
assertNotDispatched(await run(chromeAgent(), 'drag_drop', { fromRefId: 'ref_1', toRefId: 'ref_2', steps: 3 }), 'drag transport refusal');
assert.equal(presses(delivered).length, 0);
} finally { restoreChrome(); }
});
for (const [label, lose] of [
['a lost press acknowledgement', params => params.type === 'mousePressed'],
['a lost waypoint acknowledgement after the press', params => params.type === 'mouseMoved' && params.buttons === 1 && params.x >= 0],
]) await t.test(`${label} is dispatched with an unknown outcome, released, and never resent`, async () => {
const delivered = chromeTransport({ lose });
chromeContent();
try {
assertUncertain(await run(chromeAgent(), 'drag_drop', { fromRefId: 'ref_1', toRefId: 'ref_2', steps: 3 }), label);
assert.equal(presses(delivered).length, 1, 'exactly one press');
assert.equal(delivered.at(-1).type, 'mouseReleased', 'the possibly held button was released');
assert.equal(delivered.filter(event => event.type === 'mouseMoved' && event.buttons === 1 && event.x >= 0).length,
label.includes('waypoint') ? 1 : 0, 'no waypoint was resent');
} finally { restoreChrome(); }
});
await t.test('a normal drag presses once and is dispatched', async () => {
const delivered = chromeTransport();
chromeContent();
try {
const { result, state } = await run(chromeAgent(), 'drag_drop', { fromRefId: 'ref_1', toRefId: 'ref_2', steps: 3 });
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(state.started, true);
assert.deepEqual(delivered.map(event => event.type),
['mouseMoved', 'mousePressed', 'mouseMoved', 'mouseMoved', 'mouseMoved', 'mouseReleased']);
} finally { restoreChrome(); }
});
});
test('Chrome set_checked: the trusted click is dispatched at its press transport boundary', async t => {
const checkbox = { setChecked: params => params.cleanupMarker
? { success: true, method: 'set_checked', tag: 'input', type: 'checkbox', checkboxIdentity: 'id:agree', checkedBefore: false, checkedAfter: false }
: { success: true, dispatched: false, noDispatch: true, needsTrustedClick: true, marker: 'boundary-marker',
trustedSelector: '[data-webbrain-set-checked-target="boundary-marker"]', method: 'set_checked', tag: 'input', type: 'checkbox',
checkboxIdentity: 'id:agree', checkedBefore: false, checkedAfter: false, desiredChecked: true, rect: { x: 20, y: 20, w: 16, h: 16 } } };
const resolveCheckbox = () => {
cdpClient.resolveSelector = async () => ({ inViewport: true, hitOk: true, x: 28, y: 28, width: 16, height: 16, tag: 'INPUT', text: '' });
};
for (const [label, refuse] of [
['a fence veto on the press', params => params.type === 'mousePressed' ? pageFeedbackVeto : null],
['a transport refusal before the press protocol call', params => params.type === 'mousePressed' ? notAttached : null],
]) await t.test(`${label} is not dispatched`, async () => {
const delivered = chromeTransport({ refuse });
const messages = chromeContent(checkbox);
resolveCheckbox();
try {
const outcome = await run(chromeAgent(), 'set_checked', { ref_id: 'ref_3', checked: true });
assertNotDispatched(outcome, label);
assert.equal(presses(delivered).length, 0, 'no press reached the page');
assert.deepEqual(messages, ['set_checked', 'set_checked'], 'probe and read-only verification only');
} finally { restoreChrome(); }
});
await t.test('a lost release acknowledgement after the press is dispatched with an unknown outcome and never resent', async () => {
const delivered = chromeTransport({ lose: params => params.type === 'mouseReleased' && params.x >= 0 });
chromeContent(checkbox);
resolveCheckbox();
try {
assertUncertain(await run(chromeAgent(), 'set_checked', { ref_id: 'ref_3', checked: true }), 'set_checked lost release');
assert.equal(presses(delivered).length, 1, 'exactly one trusted press, no fallback click');
} finally { restoreChrome(); }
});
});
// ββ Firefox ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
function firefoxAgent() {
const agent = new FirefoxAgent({ getActive: () => ({ model: 'boundary-test' }) });
agent._runningTabs.add(17);
agent._currentUrl = async () => 'https://boundary.test/start';
agent._isPdfTab = async () => false;
agent._injectCoreContentScripts = async () => {};
return agent;
}
// `monitor(action, params)` answers page-monitor fences; `content(message)`
// answers content messages (or throws a browser messaging error).
function firefoxMessaging({ monitor = () => ({ ready: true }), content = () => ({ success: true }) } = {}) {
const contentMessages = [];
api.tabs.sendMessage = async (_tab, message) => {
if (message.action?.startsWith('page_monitor_')) return monitor(message.action, message.params || {});
contentMessages.push({ action: message.action, prepare: !!message.params?._bidiPrepare });
return content(message);
};
return contentMessages;
}
function bidiRun({ request } = {}) {
const requests = [];
firefoxBidi.runs.set(17, { runId: 'boundary-run', bound: true });
firefoxBidi.request = async (command, args) => { requests.push({ command, action: args.action }); return request?.(command, args) ?? { success: true, dispatched: true, method: 'firefox-bidi' }; };
return requests;
}
function restoreFirefox() {
firefoxBidi.runs.delete(17);
delete firefoxBidi.request;
api.tabs.sendMessage = async () => ({});
}
const bidiPrepared = message => message.params?._bidiPrepare
? { bidiPrepared: true, success: false, dispatched: false, noDispatch: true, url: 'https://boundary.test/start', rect: { x: 20, y: 20, w: 80, h: 40 } }
: { success: true, method: 'synthetic-hover' };
async function withFeedbackRun(agent, body) {
await agent._beginPageFeedbackRun(17, 'interactive');
try { return await body(); } finally { agent._finishPageFeedbackRun(17); }
}
test('Firefox hover through BiDi: dispatch starts when the native request is posted', async t => {
await t.test('a fence veto at the native request is not dispatched', async () => {
let prepared = false;
const contentMessages = firefoxMessaging({ content: bidiPrepared, monitor: (action, params) => {
if (action === 'page_monitor_prepare' && params.fenceOnly) { prepared = true; return { ready: true, modelBindingValid: true }; }
// The native request's own fence (after its preparation probe) refuses.
return action === 'page_monitor_dispatch' && prepared
? { ready: false, pageFeedbackPending: true, preparationVeto: { predicate: 'geometry_changed' } } : { ready: true };
} });
const requests = bidiRun();
const agent = firefoxAgent();
try {
const outcome = await withFeedbackRun(agent, () => run(agent, 'hover', { ref_id: 'ref_1' }));
assertNotDispatched(outcome, 'BiDi hover fence veto');
assert.equal(outcome.result.pageFeedbackPending, true);
assert.deepEqual(requests, [], 'no native input request was posted');
assert.deepEqual(contentMessages, [{ action: 'hover', prepare: true }], 'only the read-only preparation probe');
} finally { restoreFirefox(); }
});
await t.test('a native port that cannot accept the request is not dispatched', async () => {
firefoxMessaging({ content: bidiPrepared });
firefoxBidi.runs.set(17, { runId: 'boundary-run', bound: true });
const previous = firefoxBidi.apiOverride;
firefoxBidi.apiOverride = { ...api, runtime: { connectNative: () => ({
onMessage: { addListener() {} }, onDisconnect: { addListener() {} }, disconnect() {},
postMessage() { throw new Error('Attempt to postMessage on disconnected port'); },
}) } };
try {
assertNotDispatched(await run(firefoxAgent(), 'hover', { ref_id: 'ref_1' }), 'BiDi hover port refusal');
} finally { firefoxBidi.apiOverride = previous; firefoxBidi.port = null; restoreFirefox(); }
});
await t.test('a companion refusal before any input is not dispatched', async () => {
firefoxMessaging({ content: bidiPrepared });
bidiRun({ request: () => { throw Object.assign(new Error('Page changed during native preparation; no further input sent'), {
code: 'page_feedback_pending', dispatchState: { dispatched: false, noDispatch: true, outcomeUnknown: false, retryable: true } }); } });
try {
const outcome = await run(firefoxAgent(), 'hover', { ref_id: 'ref_1' });
assertNotDispatched(outcome, 'BiDi hover companion refusal');
assert.equal(outcome.result.pageFeedbackPending, true);
} finally { restoreFirefox(); }
});
await t.test('a lost native reply after the request was posted is dispatched with an unknown outcome and never resent', async () => {
const contentMessages = firefoxMessaging({ content: bidiPrepared });
const requests = bidiRun({ request: () => { throw new Error('Firefox companion disconnected; do not retry an uncertain action'); } });
try {
assertUncertain(await run(firefoxAgent(), 'hover', { ref_id: 'ref_1' }), 'BiDi hover lost reply');
assert.deepEqual(requests.map(request => request.action), ['hover'], 'one native request');
assert.equal(contentMessages.filter(message => !message.prepare).length, 0, 'no synthetic content-script fallback');
} finally { restoreFirefox(); }
});
await t.test('a delivered native hover is dispatched', async () => {
firefoxMessaging({ content: bidiPrepared });
const requests = bidiRun();
try {
const { result, state } = await run(firefoxAgent(), 'hover', { ref_id: 'ref_1' });
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(state.started, true);
assert.equal(requests.length, 1);
} finally { restoreFirefox(); }
});
});
test('Firefox content-script hover, drag_drop and set_checked: dispatch starts at the content message', async t => {
const lostResponse = "Actor 'Conduits' destroyed before query 'RuntimeMessage' was resolved";
const missingReceiver = 'Could not establish connection. Receiving end does not exist.';
for (const [name, args] of [
['hover', { ref_id: 'ref_1' }],
['drag_drop', { fromRefId: 'ref_1', toRefId: 'ref_2' }],
['set_checked', { ref_id: 'ref_3', checked: true }],
]) {
await t.test(`${name}: a fence veto before the content message is not dispatched`, async () => {
const contentMessages = firefoxMessaging({ monitor: action => action === 'page_monitor_dispatch'
? { ready: false, pageFeedbackPending: true, preparationVeto: { predicate: 'pending_feedback' } } : { ready: true } });
const agent = firefoxAgent();
try {
const outcome = await withFeedbackRun(agent, () => run(agent, name, args));
assertNotDispatched(outcome, `${name} fence veto`);
assert.deepEqual(contentMessages, [], 'no content message was sent');
} finally { restoreFirefox(); }
});
await t.test(`${name}: a content-side fence veto is not dispatched`, async () => {
firefoxMessaging({ content: () => ({ success: false, dispatched: false, noDispatch: true, pageFeedbackPending: true,
error: `${name} failed: Browser changed during action preparation. Re-observe before acting.` }) });
try {
assertNotDispatched(await run(firefoxAgent(), name, args), `${name} content veto`);
} finally { restoreFirefox(); }
});
await t.test(`${name}: a missing receiver is not dispatched`, async () => {
const contentMessages = firefoxMessaging({ content: () => { throw new Error(missingReceiver); } });
try {
assertNotDispatched(await run(firefoxAgent(), name, args), `${name} missing receiver`);
assert.equal(contentMessages.length, 2, 'one reinjected attempt; nothing was delivered');
} finally { restoreFirefox(); }
});
await t.test(`${name}: a lost content response is dispatched with an unknown outcome and never resent`, async () => {
const contentMessages = firefoxMessaging({ content: () => { throw new Error(lostResponse); } });
try {
assertUncertain(await run(firefoxAgent(), name, args), `${name} lost response`);
assert.equal(contentMessages.length, 1, 'the content action was sent once');
} finally { restoreFirefox(); }
});
}
});