Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
// Real Agent stream loop with the real one-shot local vision fallback (browser
// APIs, vision route and local describer mocked), both builds. When page
// feedback is applied at the start of the retry step, the fallback must be
// kept (never resend the image the provider rejected) and must carry the new
// browser observation instead of the superseded one.
import assert from 'node:assert/strict';
import { test } from 'node:test';
const PAGE_URL = 'https://example.com/photo/1';
const IMAGE = 'data:image/png;base64,iVBORw0KGgo=';
const area = { get: async () => ({}), set: async () => {}, remove: async () => {} };
const api = {
storage: { local: area, session: area },
runtime: { getURL: value => `extension://test/${value}`, sendMessage: async () => ({}) },
tabs: { get: async id => ({ id, url: PAGE_URL, title: 'Photo' }), sendMessage: async () => ({ ready: true }) },
webNavigation: { getAllFrames: async () => [{ frameId: 0, parentFrameId: -1, documentId: 'document-1', url: PAGE_URL }] },
scripting: { executeScript: async () => [{ result: null }] },
};
globalThis.chrome = api;
globalThis.browser = api;
let nextTab = 8700;
const hasImage = messages => messages.some(message => Array.isArray(message.content)
&& message.content.some(block => block?.type === 'image_url'));
const text = messages => JSON.stringify(messages);
for (const build of ['chrome', 'firefox']) {
const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
// OpenAI-compatible providers (Ollama, LM Studio, ...) report the HTTP
// status only as httpStatus; the rejection must still reach the fallback.
for (const [feedback, statusField] of [[false, 'status'], [true, 'status'], [false, 'httpStatus']]) {
const variant = statusField === 'httpStatus'
? 'for an OpenAI-compatible rejection carrying only httpStatus'
: feedback ? 'and adds the new page observation' : 'without page feedback';
test(`${build}: stream keeps the one-shot vision fallback ${variant}`, async () => {
const tab = nextTab++, requests = [], updates = [];
let agent, send;
const next = async messages => {
requests.push(structuredClone(messages));
if (requests.length === 1) {
assert.equal(hasImage(messages), true, 'fixture: the first request carries the image');
if (feedback) send({}); // an ordinary page update while the request fails
const error = new Error('Invalid request: image input is not supported by this model.');
error[statusField] = 400;
throw error;
}
return { content: 'The capture shows the photo settings page.' };
};
const provider = {
name: 'vision fallback test', model: 'test', promptTier: 'full', contextWindow: 128000,
supportsTools: true, supportsVision: true,
chat: messages => messages[0]?.content?.startsWith?.('You classify whether one proposed terminal WebBrain Ask answer')
? Promise.resolve({ content: '{"pageIndependent":false}' }) : next(messages),
async *chatStream(messages) {
const response = await next(messages);
if (response.content) yield { type: 'text', content: response.content };
yield { type: 'done' };
},
};
const local = { name: 'local vision', config: { model: 'lfm' } };
agent = new Agent({ getActive: () => provider, getProvider: () => provider, getVisionProvider: async () => null,
getLocalVisionFallbackProvider: async () => local });
Object.assign(agent, {
_hydrate: async () => {}, _persist: () => {}, _persistNow: async () => ({ ok: true }), _persistSubmittedTurn: async () => {},
_startTraceRun: async () => null, _endTraceRun: async () => {}, _beginReadCompleteness: async () => null,
_manageContext: async () => {}, _checkCostAllowance: async () => null, _recordCostUsage: async () => null,
_currentUrl: async () => PAGE_URL, _getTabUrlTitle: async () => ({ tabUrl: PAGE_URL, tabTitle: 'Photo' }),
_maybeEmitAskModeHandoff: async () => {}, _maybeReinjectAdapter: async () => {}, _shouldAutoScreenshot: () => false,
_pageFeedbackIdleMs: 0,
_resolveVisionRoute: async (_tab, active) => ({ route: 'active_raw', provider: active, rawImage: true }),
_describeScreenshot: async () => ({ text: 'local description of the capture', model: 'lfm' }),
});
agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => {
const from = { tab: { id: tab }, frameId: 0, documentId: 'document-1', url: PAGE_URL };
const state = agent.pageMonitorState(from, 'content-1');
let seq = 0;
send = value => agent.observePageFeedback(from, { ...state, seq: ++seq, revision: seq, kind: 'dom', source: 'page', target: 'span#views', ...value });
return { role: 'user', content: [{ type: 'text', text: content }, { type: 'image_url', image_url: { url: IMAGE } }] };
};
agent._executeToolWithLifecycle = async () => ({ success: true, pageContent: 'main [ref_0]\n img "Photo" [ref_1]' });
const result = await agent.processMessageStream(tab, 'Describe the attached capture', (type, data) => updates.push({ type, data }),
'ask', { detachedRequestId: 'vision-' + tab, askStreamingEnabled: false });
assert.equal(result, 'The capture shows the photo settings page.');
assert.equal(requests.length, 2);
assert.ok(updates.some(update => update.data?.code === 'vision_local_fallback_retry'));
const retry = requests[1];
assert.equal(hasImage(retry), false, 'The rejected image is never resent');
assert.match(text(retry), /local description of the capture/);
if (feedback) {
assert.equal((text(retry).match(/BROWSER STATE UPDATE/g) || []).length, 1, 'The applied page observation reaches the retry exactly once');
}
});
}
}