Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { test } from 'node:test';
import { chromium, firefox } from 'playwright';
import { Agent as ChromeAgent } from '../src/chrome/src/agent/agent.js';
import { Agent as FirefoxAgent } from '../src/firefox/src/agent/agent.js';
import { guardProbeWithReadings, stubPageTextReadings } from './semantic-decision-stub.mjs';
// These local pages exercise the real content observer and one-use dispatch
// validator. Every navigation is intercepted; no external message is sent.
const sendArgs = { selector: '#send' };
const phone = '+90 555 000 00 01';
const call = (page, action, params = {}) => page.evaluate(message => new Promise(resolve => {
for (const handler of window.fixtureHandlers) handler(message, {}, resolve);
}), { target: 'content', action, params });
async function fixture(kind, engine, Agent, options = {}) {
const browser = await engine.launch({ headless: true });
const page = await browser.newPage({ viewport: { width: 1280, height: 800 } });
await page.route('**/*', route => route.fulfill({ contentType: 'text/html', body: '<!doctype html>' }));
await page.goto(options.url || 'https://web.whatsapp.com/');
await page.setContent(`<!doctype html><style>
body{margin:0;font:16px sans-serif}aside{position:absolute;left:0;top:0;width:300px;height:790px;overflow:auto}
#main{position:absolute;left:320px;top:0;width:940px;height:790px}
#main>header{height:70px;display:flex;align-items:center;gap:30px;padding-left:70px}
#contact{width:278px;height:20px}#messages{height:580px;overflow:auto}
footer{display:flex;gap:15px;align-items:center}#body{width:550px;min-height:32px}
button{min-height:28px}#alien-header{position:absolute;left:330px;top:80px}
</style><aside><header><h1>WhatsApp</h1></header><button>Anne</button><h2>Sidebar recipient</h2></aside>
<main id="main"><header><img alt="" width="40" height="40">
<div id="contact" role="button" tabindex="0"><span title="Anne">Anne</span></div>
<button id="call" aria-label="Voice call">Call</button><button aria-label="Search">Search</button>
</header><section id="messages"><div>Anne</div><button>Other person</button></section>
<footer><button id="attach">Attach</button>
<div id="body" contenteditable="true" role="textbox" aria-label="Type a message to ${phone}">Selam</div>
<input id="file" type="file" accept="image/*" hidden><button id="send" aria-label="Send">Send</button>
</footer></main>`);
await page.evaluate(() => {
window.fixtureHandlers = [];
const runtime = { onMessage: { addListener(fn) { window.fixtureHandlers.push(fn); } }, getURL: p => p };
window.chrome = { runtime }; window.browser = { runtime };
window.sent = 0;
document.querySelector('#send').addEventListener('click', () => window.sent++);
});
for (const file of ['file-upload-policy.js', 'rich-text-toolbar-heuristic.js', 'accessibility-tree.js', 'content.js']) {
await page.addScriptTag({ content: await readFile(path.resolve('src', kind, 'src/content', file), 'utf8') });
}
const agent = new Agent({ getActive: () => ({ supportsVision: false }) });
agent._currentUrl = async () => page.url();
// Control names (Call, Search, Send) are read by the fixtures' model oracle.
stubPageTextReadings(agent);
agent._messageRecipientContentProbe = (_, params) => call(page, 'probe_message_recipient_guard', params);
agent._planExecutionGuards.set(71, { messaging: { target_kind: 'named', recipients: ['anne'] },
requiresStateChange: true, requiresSubmission: true });
const probe = (tool = 'click', args = sendArgs) => guardProbeWithReadings(params => call(page, 'probe_message_recipient_guard', params), {
adapterName: 'generic-messaging', tool, args, bindDispatch: true, expectedRecipients: ['anne'],
});
const bind = async (tool = 'click', args = sendArgs) => {
const context = {};
const blocked = await agent._messageRecipientGuardBlock(71, tool, args, page.url(), context);
return { blocked, context };
};
return { browser, page, agent, probe, bind };
}
for (const [kind, engine, Agent] of [['chrome', chromium, ChromeAgent], ['firefox', firefox, FirefoxAgent]]) {
test(`${kind}: WhatsApp contact button permits the named chat without a clarification`, async () => {
const f = await fixture(kind, engine, Agent);
try {
const observed = await f.probe();
assert.equal(observed.success, true);
assert.equal(observed.conclusive, true);
assert.ok(observed.strongRecipientCandidates.some(recipient => recipient.identity === 'Anne'), JSON.stringify(observed));
assert.equal(observed.strongRecipientCandidates.length, 1, 'sidebar, app title and toolbar must not become recipients');
assert.ok(observed.observedRecipientCandidates[0].aliases.includes(phone), 'the authored destination must stay tied to the observed contact');
const { blocked, context } = await f.bind();
assert.equal(blocked, null, JSON.stringify(blocked));
assert.equal(context.messageRecipientGuardRequired, true);
assert.ok(context.messageRecipientDispatchBinding?.token);
const result = await call(f.page, 'click', { ...sendArgs, ...context });
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(await f.page.evaluate(() => window.sent), 1);
assert.equal(f.agent._planExecutionGuards.get(71).pendingRecipientAuthorization, undefined);
} finally { await f.browser.close(); }
});
test(`${kind}: one real recipient approval persists for repeated sends`, async () => {
const f = await fixture(kind, engine, Agent);
try {
f.agent._planExecutionGuards.get(71).messaging = { target_kind: 'named', recipients: ['Different recipient'] };
const first = await f.bind();
assert.equal(first.blocked?.reasonCode, 'active_recipient_unverified', JSON.stringify(first.blocked));
// WebBrain's own card, answered once with its fixed Send choice.
const cards = [];
const approval = await f.agent._resolveMessageRecipientConfirmation(71, first.blocked, (status, details) => {
if (status !== 'clarify') return;
cards.push(details);
f.agent._settleClarification(f.agent._pendingClarifications.get(71).get(details.clarifyId), { answer: 'once', source: 'option' });
}, () => f.bind().then(next => next.blocked), { allowPrompt: true });
assert.equal(approval, null, JSON.stringify(approval));
assert.equal(cards.length, 1);
assert.deepEqual(cards[0].recipientConfirmation.recipients, [{ identity: 'Anne', role: 'to' }]);
for (let attempt = 0; attempt < 2; attempt++) {
const next = await f.bind();
assert.equal(next.blocked, null, JSON.stringify(next.blocked));
const result = await call(f.page, 'click', { ...sendArgs, ...next.context });
assert.equal(result.success, true, JSON.stringify(result));
}
assert.equal(await f.page.evaluate(() => window.sent), 2);
assert.equal(cards.length, 1, 'The same person is not asked about again in this task');
} finally { await f.browser.close(); }
});
test(`${kind}: the scoped header button pattern also works on a generic chat site`, async () => {
const f = await fixture(kind, engine, Agent, { url: 'https://example.test/chat/rooms/one' });
try {
const { blocked, context } = await f.bind();
assert.equal(blocked, null, JSON.stringify(blocked));
const result = await call(f.page, 'click', { ...sendArgs, ...context });
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(await f.page.evaluate(() => window.sent), 1);
} finally { await f.browser.close(); }
});
for (const drift of ['recipient', 'same-name phone', 'contact replacement', 'composer replacement', 'body']) {
test(`${kind}: ${drift} change blocks the previously bound send`, async () => {
const f = await fixture(kind, engine, Agent);
try {
const { blocked, context } = await f.bind();
assert.equal(blocked, null, JSON.stringify(blocked));
await f.page.evaluate(drift => {
if (drift === 'recipient') document.querySelector('#contact span').textContent = 'Different recipient';
if (drift === 'same-name phone') document.querySelector('#body').setAttribute('aria-label', 'Type a message to +90 555 000 00 02');
if (drift === 'contact replacement') {
const old = document.querySelector('#contact'); old.replaceWith(old.cloneNode(true));
}
if (drift === 'composer replacement') {
const old = document.querySelector('#body'); old.replaceWith(old.cloneNode(true));
}
if (drift === 'body') document.querySelector('#body').textContent = 'Different message';
}, drift);
const result = await call(f.page, 'click', { ...sendArgs, ...context });
assert.equal(result.success, false, JSON.stringify(result));
assert.equal(result.messageRecipientGuard, true, JSON.stringify(result));
assert.equal(result.dispatched, false);
assert.equal(await f.page.evaluate(() => window.sent), 0);
} finally { await f.browser.close(); }
});
}
test(`${kind}: unrelated message updates preserve the current recipient proof`, async () => {
const f = await fixture(kind, engine, Agent);
try {
const { blocked, context } = await f.bind();
assert.equal(blocked, null, JSON.stringify(blocked));
await f.page.evaluate(() => {
document.querySelector('#messages').append('new unrelated incoming message');
document.querySelector('aside button').textContent = 'Sidebar updated';
});
const result = await call(f.page, 'click', { ...sendArgs, ...context });
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(await f.page.evaluate(() => window.sent), 1);
} finally { await f.browser.close(); }
});
for (const missing of ['contact missing', 'contact hidden', 'ambiguous contacts', 'message impersonation', 'nested toolbar', 'foreign owner']) {
test(`${kind}: ${missing} cannot authorize a send or stage another approval`, async () => {
const f = await fixture(kind, engine, Agent);
try {
await f.page.evaluate(missing => {
const contact = document.querySelector('#contact');
if (missing === 'contact hidden') contact.hidden = true;
if (missing === 'contact missing' || missing === 'message impersonation') contact.remove();
if (missing === 'ambiguous contacts') {
const second = contact.cloneNode(true); second.id = 'another-contact';
second.querySelector('span').textContent = 'Other recipient'; contact.after(second);
}
if (missing === 'message impersonation') {
document.querySelector('#main > header').remove();
document.querySelector('#messages').innerHTML = '<header><div role="button">Anne</div></header>';
}
if (missing === 'nested toolbar') {
const toolbar = document.createElement('div'); toolbar.setAttribute('role', 'toolbar');
contact.replaceWith(toolbar); toolbar.append(contact);
}
if (missing === 'foreign owner') {
const owner = document.createElement('section'); owner.id = 'alien-header';
contact.closest('header').replaceWith(owner); owner.append(contact);
}
}, missing);
const { blocked } = await f.bind();
assert.equal(blocked?.success, false, JSON.stringify(blocked));
assert.equal(blocked?.noDispatch, true);
assert.equal(blocked?.reasonCode, 'recipient_identity_unavailable', JSON.stringify(blocked));
assert.equal(blocked?.recipientAuthorizationRequired, false);
assert.equal(blocked.recipientConfirmation, undefined, 'A page without a verifiable recipient offers no card');
assert.doesNotMatch(blocked.error, /clarify\(|Ask the user to authorize/);
// The recipient refusal above ran in the default mode. Strict checks
// only for the clarify refusal below: by default that quality check
// advises and the question would wait for an answer.
f.agent.strictGuards = true;
const updates = [];
const repeatedApproval = await f.agent._executeToolImpl(71, 'clarify', {
purpose: 'message_recipient', question: 'Send to Anne instead?', options: ['Anne', 'Cancel'],
}, status => updates.push(status));
assert.equal(repeatedApproval.success, false);
assert.equal(repeatedApproval.reasonCode, 'recipient_identity_unavailable');
assert.deepEqual(updates, [], 'technical failures must not present an unusable approval question');
assert.equal(f.agent._pendingClarifications.has(71), false);
if (missing === 'contact missing') {
// Any other question is just a question: whatever is picked, it
// cannot supply the missing page identity or approve the send.
f.agent.clarifyTimeoutSec = -1;
f.agent._recordClarificationAuthorization = async () => true;
for (const [question, options, answer] of [['Send to Anne instead?', ['Anne', 'Cancel'], 'Anne'],
['Which image size do you prefer?', ['Original', 'Small'], 'Original'],
['Send to Anne?', ['Send all remaining messages without asking again', 'Cancel'], 'Send all remaining messages without asking again']]) {
const asked = await f.agent._executeToolImpl(71, 'clarify', { question, options }, (status, details) => {
if (status !== 'clarify') return;
updates.push(status);
f.agent._settleClarification(f.agent._pendingClarifications.get(71).get(details.clarifyId), { answer, source: 'option' });
});
assert.equal(asked.success, true, question);
assert.equal(asked.recipientBinding, undefined, question);
assert.equal((await f.bind()).blocked?.reasonCode, 'recipient_identity_unavailable', question);
}
assert.equal(f.agent._planExecutionGuards.get(71).messageRecipientApprovedAll, undefined);
assert.equal(typeof f.agent._bindClarifiedMessageRecipient, 'undefined', 'No answer wording is read for consent');
}
assert.equal(await f.page.evaluate(() => window.sent), 0);
} finally { await f.browser.close(); }
});
}
test(`${kind}: a fresh observed contact clears the technical blocker`, async () => {
const f = await fixture(kind, engine, Agent);
try {
await f.page.evaluate(() => { document.querySelector('#contact').hidden = true; });
assert.equal((await f.bind()).blocked?.reasonCode, 'recipient_identity_unavailable');
await f.page.evaluate(() => { document.querySelector('#contact').hidden = false; });
const { blocked, context } = await f.bind();
assert.equal(blocked, null, JSON.stringify(blocked));
assert.equal(f.agent._planExecutionGuards.get(71).messageRecipientTechnicalBlock, undefined);
const result = await call(f.page, 'click', { ...sendArgs, ...context });
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(await f.page.evaluate(() => window.sent), 1);
} finally { await f.browser.close(); }
});
test(`${kind}: missing delivery evidence preserves consent and never asks to approve again`, async () => {
const f = await fixture(kind, engine, Agent);
try {
const guard = f.agent._planExecutionGuards.get(71);
guard.approvedRecipients = ['anne'];
const realProbe = f.agent._messageRecipientContentProbe;
for (const [field, value, reasonCode] of [
['messageBody', '', 'message_body_unverified'],
['messageBodyBaselineCount', null, 'message_body_baseline_unavailable'],
['messageBodyBaselineCount', '', 'message_body_baseline_unavailable'],
['messageRecipientDispatchBinding', undefined, 'recipient_dispatch_binding_unavailable'],
]) {
f.agent._messageRecipientContentProbe = async (...args) => ({ ...await realProbe(...args), [field]: value });
// The recipient refusal runs in the default mode; Strict checks only
// for the clarify refusal after it (a quality check that advises by
// default, when the question would wait for an answer).
f.agent.strictGuards = false;
const { blocked } = await f.bind();
assert.equal(blocked?.reasonCode, reasonCode, JSON.stringify(blocked));
assert.equal(blocked.recipientAuthorizationRequired, false);
assert.deepEqual(guard.approvedRecipients, ['anne']);
assert.equal(blocked.recipientConfirmation, undefined, 'Missing delivery evidence is not put to the user again');
f.agent.strictGuards = true;
const updates = [];
const confirmation = await f.agent._executeToolImpl(71, 'clarify', {
purpose: 'message_recipient', question: 'Send to Anne?', options: ['Anne', 'Cancel'],
}, status => updates.push(status));
assert.equal(confirmation.success, false);
assert.equal(confirmation.reasonCode, reasonCode);
assert.deepEqual(updates, []);
}
f.agent._messageRecipientContentProbe = realProbe;
f.agent.strictGuards = false;
assert.equal((await f.bind()).blocked, null);
assert.equal(guard.messageRecipientTechnicalBlock, undefined);
assert.deepEqual(guard.approvedRecipients, ['anne']);
assert.equal(await f.page.evaluate(() => window.sent), 0);
} finally { await f.browser.close(); }
});
test(`${kind}: protected screenshot attachment binds the WhatsApp contact`, async () => {
const f = await fixture(kind, engine, Agent);
try {
const { blocked, context } = await f.bind('upload_file', { selector: '#file', downloadId: 2027 });
assert.equal(blocked, null, JSON.stringify(blocked));
const result = await call(f.page, 'attach_message_recipient_bound_upload', {
selector: '#file', ...context, uploadSourceKey: context.messageRecipientUploadSourceKey,
filename: 'fixture.png', mimeType: 'image/png', base64: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aS3kAAAAASUVORK5CYII=',
});
assert.equal(result.success, true, JSON.stringify(result));
assert.equal(result.verified, false);
assert.equal(result.remoteStateVerified, false);
assert.equal(await f.page.evaluate(() => document.querySelector('#file').files.length), 1);
assert.equal(await f.page.evaluate(() => window.sent), 0);
} finally { await f.browser.close(); }
});
}