Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
import { chromium, firefox } from 'playwright';
import { createServer } from 'node:http';
import { readFile, mkdir } from 'node:fs/promises';
import { resolve, extname, sep } from 'node:path';
import assert from 'node:assert/strict';
const root = resolve('.');
const output = process.env.MEDIA_UI_OUTPUT || '/tmp/webbrain-generative-media-ui';
await mkdir(output, { recursive: true });
const server = createServer(async (req, res) => {
try {
const file = resolve(root, '.' + new URL(req.url, 'http://localhost').pathname);
if (!file.startsWith(root + sep)) throw new Error('outside root');
res.setHeader('Content-Type', ({ '.js': 'text/javascript', '.mjs': 'text/javascript', '.html': 'text/html', '.css': 'text/css', '.svg': 'image/svg+xml' })[extname(file)] || 'text/plain');
res.end(await readFile(file));
} catch { res.statusCode = 404; res.end(); }
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const origin = `http://127.0.0.1:${server.address().port}`;
const workflow = JSON.stringify({ '6': { class_type: 'CLIPTextEncode', inputs: { text: '{{prompt}}' } }, '9': { class_type: 'SaveImage', inputs: { images: ['6', 0] } } }, null, 2);
try {
for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
const backgroundSource = await readFile(`src/${build}/src/background.js`, 'utf8');
const sendStart = backgroundSource.indexOf('function sendAgentUpdate(');
const sendSource = backgroundSource.slice(sendStart, backgroundSource.indexOf('\nfunction ', sendStart + 1));
const panelSource = await readFile(`src/${build}/src/ui/sidepanel.js`, 'utf8');
const handleStart = panelSource.indexOf('function handleAgentUpdateMessage(');
const handleEnd = panelSource.slice(handleStart).search(/\n(?:chrome|browser)\.runtime\.onMessage\.addListener/);
assert.ok(handleStart >= 0 && handleEnd > 0);
const handleSource = panelSource.slice(handleStart, handleStart + handleEnd);
const browser = await engine.launch();
try {
for (const lang of ['en', 'tr']) for (const width of [390, 1280]) {
const page = await browser.newPage({ viewport: { width, height: 1100 } });
const errors = []; page.on('pageerror', error => errors.push(error.message));
await page.addInitScript(({ build, lang }) => {
localStorage.setItem('wbLocale', lang);
const data = JSON.parse(localStorage.getItem('media-test-store') || 'null') || { wbLocale: lang, imageGenModel: { apiKey: 'legacy-key', model: 'fal-ai/flux/schnell' } };
window.testStore = data; window.testRequests = [];
const storage = {
async get(keys) { return keys == null ? { ...data } : Object.fromEntries((Array.isArray(keys) ? keys : typeof keys === 'string' ? [keys] : Object.keys(keys)).map(key => [key, data[key]])); },
async set(values) { Object.assign(data, values); localStorage.setItem('media-test-store', JSON.stringify(data)); },
async remove(keys) { for (const key of Array.isArray(keys) ? keys : [keys]) delete data[key]; localStorage.setItem('media-test-store', JSON.stringify(data)); },
};
window.chrome = window.browser = {
storage: { local: storage, onChanged: { addListener() {} } },
runtime: { getURL: path => `${location.origin}/src/${build}/${path}`, getManifest: () => ({ version: '39.1.1' }), onMessage: { addListener() {} }, sendMessage(msg, callback) {
testRequests.push(msg);
const result = msg.action === 'get_providers' ? { providers: {}, active: '' }
: msg.action === 'test_image_gen_provider' ? { ok: true, model: data.imageGenModel.model || 'ComfyUI workflow' } : {};
callback?.(result); return Promise.resolve(result);
} },
commands: { getAll: async () => [] }, tabs: { create: async () => ({}) },
};
}, { build, lang });
await page.goto(`${origin}/src/${build}/src/ui/providers.html#generative`);
await page.waitForFunction(() => document.querySelector('#image-gen-api-key').value === 'legacy-key');
assert.equal(await page.locator('#image-gen-heading').innerText(), lang === 'en' ? 'Generative Media' : 'Üretken medya');
const provider = page.locator('#image-gen-provider');
assert.equal(await provider.inputValue(), 'fal', 'legacy config defaults to fal.ai');
assert.deepEqual(await provider.locator('option').evaluateAll(options => options.map(option => option.value)), ['fal', 'openrouter', 'comfyrouter', 'comfyui']);
await provider.selectOption('openrouter');
assert.equal(await page.locator('#image-gen-api-key').inputValue(), '', 'keys do not cross providers');
await page.locator('#image-gen-api-key').fill('synthetic-or-key');
await page.locator('#image-gen-model').fill('google/gemini-2.5-flash-image');
await page.locator('#btn-save-image-gen').click();
await page.waitForFunction(() => testStore.imageGenModel?.provider === 'openrouter');
await page.reload();
await page.waitForFunction(() => document.querySelector('#image-gen-provider').value === 'openrouter');
assert.equal(await page.locator('#image-gen-api-key').inputValue(), 'synthetic-or-key');
await page.locator('#btn-test-image-gen').click();
await page.waitForFunction(() => testRequests.some(request => request.action === 'test_image_gen_provider'));
await provider.selectOption('comfyrouter');
assert.equal(await page.locator('#image-gen-parameters-field').isVisible(), true);
await page.locator('#image-gen-api-key').fill('synthetic-comfy-key');
await page.locator('#image-gen-model').fill('bfl/flux-2-pro');
await page.locator('#image-gen-parameters').fill('{bad json');
await page.locator('#btn-save-image-gen').click();
await page.waitForFunction(() => document.querySelector('#test-image-gen').classList.contains('fail'));
assert.equal(await page.evaluate(() => testStore.imageGenModel.provider), 'openrouter', 'invalid input preserves the saved configuration');
await page.locator('#image-gen-parameters').fill('{"prompt":"{{prompt}}","width":1024}');
await page.locator('#btn-save-image-gen').click();
await page.waitForFunction(() => testStore.imageGenModel?.provider === 'comfyrouter');
await provider.selectOption('openrouter');
assert.equal(await page.locator('#image-gen-api-key').inputValue(), 'synthetic-or-key', 'unsaved draft retained');
await provider.selectOption('comfyui');
assert.equal(await page.locator('#image-gen-key-field').isVisible(), false);
assert.equal(await page.locator('#image-gen-model-field').isVisible(), false);
assert.equal(await page.locator('#image-gen-url-field').isVisible(), true);
assert.equal(await page.locator('#image-gen-parameters-field').isVisible(), false);
assert.equal(await page.locator('#image-gen-base-url').inputValue(), 'http://127.0.0.1:8188');
await page.locator('#image-gen-workflow').fill(workflow);
await page.locator('#btn-save-image-gen').click();
await page.waitForFunction(() => testStore.imageGenModel?.provider === 'comfyui');
assert.equal(await page.evaluate(() => testStore.imageGenModel.apiKey), undefined);
await page.reload();
await page.waitForFunction(() => document.querySelector('#image-gen-provider').value === 'comfyui');
assert.equal(await page.locator('#image-gen-workflow').inputValue(), workflow);
await page.locator('#image-gen-heading').scrollIntoViewIfNeeded();
await page.screenshot({ animations: 'disabled', path: `${output}/${build}-${lang}-${width}-comfyui-configured.png` });
await page.locator('#btn-test-image-gen').click();
await page.waitForFunction(() => document.querySelector('#test-image-gen').classList.contains('ok'));
await page.locator('#btn-clear-image-gen').click();
await page.waitForFunction(() => testStore.imageGenModel == null);
assert.equal(await page.locator('#image-gen-workflow').inputValue(), '');
for (const value of ['fal', 'openrouter', 'comfyrouter', 'comfyui']) {
await provider.selectOption(value);
assert.equal(await page.locator('#image-gen-api-key').inputValue(), '');
await page.locator('#image-gen-heading').scrollIntoViewIfNeeded();
await page.screenshot({ animations: 'disabled', path: `${output}/${build}-${lang}-${width}-${value}.png` });
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true, `${value}: fits viewport`);
}
// Exercise the real journal/broadcast/panel handler rather than handing
// a raw result straight to the renderer (which hid the original bug).
const transport = await page.evaluate(async ({ build, sendSource, handleSource }) => {
const { generateImage } = await import(`/src/${build}/src/agent/generative-media.js`);
const { appendGeneratedMedia } = await import(`/src/${build}/src/ui/generated-media-view.js`);
const { RunUiJournal, compactRunUiSnapshotForPersist, RUN_UI_PERSIST_RETRY_BUDGET } = await import(`/src/${build}/src/run-ui-journal.js`);
const { loadGeneratedMedia } = await import(`/src/${build}/src/generated-media-store.js`);
const canvas = document.createElement('canvas'); canvas.width = canvas.height = 512;
const context = canvas.getContext('2d'); const pixels = context.createImageData(512, 512);
for (let i = 0; i < pixels.data.length; i += 65536) crypto.getRandomValues(pixels.data.subarray(i, i + 65536));
context.putImageData(pixels, 0, 0);
const dataUrl = canvas.toDataURL('image/png');
const result = await generateImage({ prompt: 'test image' }, {
config: { provider: 'openrouter', apiKey: 'synthetic-key', model: 'image-model' },
fetchImpl: async url => new Response(JSON.stringify({ data: url.endsWith('/models')
? [{ id: 'image-model' }] : [{ b64_json: dataUrl.split(',')[1], media_type: 'image/png' }] }), { headers: { 'Content-Type': 'application/json' } }),
});
if (!result.success) throw new Error(result.error);
const div = document.createElement('div'); div.id = 'media-test-preview'; document.querySelector('#image-gen-card').appendChild(div);
div.innerHTML = '<div class="message assistant" data-run-request-id="media-run"><div class="message-content"></div></div>';
const handler = new Function('appendGeneratedMedia', 'context', `
const { currentAssistantEl, messagesEl, currentTabId, verboseMode } = context;
const t = key => key;
const clearedConversationRunRequestIds = new Set();
const localRunRequestIds = new Map();
const ensureCurrentRunAssistant = () => currentAssistantEl;
const markLastStepDone = () => {};
const appendVerboseToolResult = () => {};
const scrollToBottom = () => {};
const schedulePersist = () => context.persistCalls++;
return (${handleSource});
`)(appendGeneratedMedia, { messagesEl: div, currentAssistantEl: div.firstElementChild, currentTabId: 11, verboseMode: false, persistCalls: 0 });
const journal = new RunUiJournal(); journal.begin(11, 'media-run');
const sent = [];
const api = { runtime: { sendMessage: async message => { sent.push(message); handler(message); } } };
const send = new Function('recordRunUiEvent', 'agent', 'chrome', 'browser', `return (${sendSource});`)(
(...args) => journal.record(...args), { currentRunId: new Map() }, api, api,
);
send(11, 'media-run', 'tool_result', { name: 'generate_image', result });
// Hosted videos must also reach the media element, not their thumbnail.
send(11, 'media-run', 'tool_result', { name: 'generate_image', result: { success: true, provider: 'comfyrouter', url: 'https://cdn.example/final.mp4' } });
appendGeneratedMedia(div, { success: true, url: 'javascript:alert(1)' }, key => key);
const snapshot = compactRunUiSnapshotForPersist(journal.get(11), { tight: true });
localStorage.setItem('media-replay-snapshot', JSON.stringify(snapshot));
localStorage.setItem('media-asset-id', result.mediaId);
const blob = await loadGeneratedMedia(result.mediaId);
return { sentHandle: sent[0].data.result.mediaId, mediaId: result.mediaId, url: result.url, blobSize: blob.size, snapshotSize: JSON.stringify(snapshot).length, budget: RUN_UI_PERSIST_RETRY_BUDGET, discarded: snapshot.discardedBeforeSeq };
}, { build, sendSource, handleSource });
assert.equal(transport.sentHandle, transport.mediaId);
assert.equal(transport.url, undefined, 'inline bytes travel as an asset handle');
assert.ok(transport.blobSize > 512 * 1024, 'real inline image exceeds journal budget');
assert.ok(transport.snapshotSize < transport.budget);
assert.equal(transport.discarded, 0);
await page.waitForFunction(() => document.querySelector('#media-test-preview img')?.naturalWidth === 512);
assert.equal(await page.locator('#media-test-preview a[download]').getAttribute('download'), 'webbrain-media.png');
assert.equal(await page.locator('#media-test-preview video').getAttribute('src'), 'https://cdn.example/final.mp4');
await page.evaluate(() => localStorage.setItem('media-chat-html', document.querySelector('#media-test-preview').innerHTML));
await page.reload();
await page.evaluate(async build => {
const { appendGeneratedMedia, restoreGeneratedMedia } = await import(`/src/${build}/src/ui/generated-media-view.js`);
const { RunUiJournal } = await import(`/src/${build}/src/run-ui-journal.js`);
const restored = new RunUiJournal(); restored.restore(11, JSON.parse(localStorage.getItem('media-replay-snapshot')));
const div = document.createElement('div'); div.id = 'media-replay-preview'; document.querySelector('#image-gen-card').appendChild(div);
for (const event of restored.get(11).events) await appendGeneratedMedia(div, event.data.result, key => key);
const chat = document.createElement('div'); chat.id = 'media-restored-chat'; document.querySelector('#image-gen-card').appendChild(chat);
chat.innerHTML = localStorage.getItem('media-chat-html');
await restoreGeneratedMedia(chat, key => key);
}, build);
await page.waitForFunction(() => document.querySelector('#media-replay-preview img')?.naturalWidth === 512 && document.querySelector('#media-restored-chat img')?.naturalWidth === 512);
assert.equal(await page.locator('#media-restored-chat .generated-media[data-media-id]').getAttribute('data-media-id'), transport.mediaId);
assert.equal(await page.locator('#media-replay-preview a[download]').getAttribute('download'), 'webbrain-media.png');
assert.equal(await page.locator('#media-replay-preview video').count(), 1);
assert.deepEqual(errors, []);
console.log(`${build} ${lang} ${width}: settings, live journal delivery, large inline media, replay and restored chat passed`);
await page.close();
}
} finally { await browser.close(); }
}
} finally { await new Promise(resolve => server.close(resolve)); }