Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
function getPath(value, dottedPath) {
return String(dottedPath).split('.').reduce((current, key) => current?.[key], value);
}
function checkValue(actual, check) {
if (Object.hasOwn(check, 'equals')) return actual === check.equals;
if (Object.hasOwn(check, 'contains')) {
if (Array.isArray(actual)) return actual.includes(check.contains);
return String(actual ?? '').toLowerCase().includes(String(check.contains).toLowerCase());
}
if (Object.hasOwn(check, 'matches')) return new RegExp(check.matches, 'i').test(String(actual ?? ''));
if (check.truthy) return Boolean(actual);
return actual !== undefined;
}
function getFinalUrl(run, trace) {
return run?.final_url || run?.finalUrl || trace?.run?.final_url || trace?.run?.finalUrl || '';
}
function toolCalls(run, trace) {
const calls = [];
const pending = [];
for (const update of trace?.run?.updates || run?.updates || []) {
const name = update.data?.name || update.data?.tool || '';
if (update.type === 'tool_call') {
const call = {
name,
args: update.data?.args || update.data?.arguments || {},
result: null,
};
calls.push(call);
pending.push(call);
} else if (update.type === 'tool_result') {
const pendingIndex = pending.findIndex((call) => call.name === name);
if (pendingIndex >= 0) {
pending[pendingIndex].result = update.data?.result || {};
pending.splice(pendingIndex, 1);
}
}
}
return calls;
}
function successfulToolRequest(call) {
if (call?.result?.success !== true) return false;
if (call.name !== 'fetch_url') return true;
const status = Number(call.result.status);
return Number.isInteger(status) && status >= 200 && status < 300;
}
export function inferStuckAt({
run,
trace,
setupError,
scheduledError,
postStartError,
artifactError,
cleanupErrors = [],
checks,
}) {
if (setupError) return 'setup';
if (scheduledError) return 'scheduled_execution';
if (postStartError) return run?.status === 'completed' ? 'verification' : 'execution';
if (!run) return 'run_start';
if (run.status === 'needs_user_input') return 'user_handoff';
const updates = trace?.run?.updates || run.updates || [];
const toolNames = updates
.filter((update) => update.type === 'tool_result' || update.type === 'tool_call')
.map((update) => update.data?.name || update.data?.tool || '');
if (run.status !== 'completed') {
if (!toolNames.length) return 'planning';
if (!getFinalUrl(run, trace)) return 'navigation';
return 'execution';
}
if (cleanupErrors.length) return 'cleanup';
if (checks.some((check) => !check.passed && !['artifact:video', 'cleanup'].includes(check.id))) return 'verification';
if (artifactError) return 'artifact_capture';
return null;
}
export function gradeScenario({
scenario,
run,
trace,
remoteState,
scheduledState,
setupError,
scheduledError,
postStartError,
artifactError,
cleanupErrors = [],
captureRequired = false,
}) {
const checks = [];
const add = (id, label, weight, passed, evidence = '') => {
checks.push({ id, label, weight, passed: Boolean(passed), evidence: String(evidence || '') });
};
add(
'run_completed',
'Cloud run completed',
20,
run?.status === 'completed',
run?.status || scheduledError?.message || postStartError?.message || setupError?.message || 'run unavailable',
);
if (scenario.verify?.mode) {
const actualMode = run?.mode || trace?.run?.mode || 'act';
add('mode', `Run used ${scenario.verify.mode} mode`, 10, actualMode === scenario.verify.mode, actualMode);
}
const calls = toolCalls(run, trace);
for (const skillId of scenario.verify?.skills || []) {
const candidates = calls.filter((call) => call.name === 'load_skill' && call.args?.skill_id === skillId);
const loaded = candidates.some((call) => call.result?.success === true);
add(
`skill:${skillId}`,
`Loaded ${skillId}`,
10,
loaded,
loaded ? skillId : candidates.length ? 'failed or missing result' : 'not observed',
);
}
for (const toolName of scenario.verify?.tools || []) {
const candidates = calls.filter((call) => call.name === toolName);
const used = candidates.some((call) => call.result?.success === true);
add(
`tool:${toolName}`,
`Completed ${toolName}`,
10,
used,
used ? 'success' : candidates.length ? 'failed or missing result' : 'not observed',
);
}
for (const toolName of scenario.verify?.successfulTools || []) {
const completed = calls.find((call) => call.name === toolName && call.result?.success === true);
const attempted = calls.findLast((call) => call.name === toolName);
add(
`tool_success:${toolName}`,
`Completed ${toolName}`,
10,
Boolean(completed),
completed ? 'success' : attempted ? 'failed or missing result' : 'not observed',
);
}
for (const expected of scenario.verify?.toolCounts || []) {
const candidates = calls.filter((call) => (
call.name === expected.tool
&& (expected.successful === false || call.result?.success === true)
));
const count = candidates.length;
const passed = Object.hasOwn(expected, 'equals')
? count === expected.equals
: count >= (expected.min || 1);
add(
`tool_count:${expected.tool}`,
expected.label || `Completed ${expected.tool} ${expected.equals ?? expected.min ?? 1} time(s)`,
expected.weight || 10,
passed,
`${count} observed`,
);
}
for (const expected of scenario.verify?.toolResults || []) {
const call = calls.findLast((candidate) => (
candidate.name === expected.tool && candidate.result?.success === true
));
const actual = call ? getPath(call.result, expected.path) : undefined;
add(
`tool_result:${expected.tool}:${expected.path}`,
expected.label || `${expected.tool} result field ${expected.path}`,
expected.weight || 10,
Boolean(call) && checkValue(actual, expected),
actual === undefined ? 'not observed' : JSON.stringify(actual),
);
}
for (const expected of scenario.verify?.toolRequests || []) {
const candidates = calls.filter((call) => (
call.name === expected.tool
&& (!expected.origin || call.args?.url_origin === expected.origin)
&& (!expected.method || String(call.args?.method || 'GET').toUpperCase() === expected.method.toUpperCase())
));
const matched = candidates.find(successfulToolRequest);
const attempted = candidates.at(-1);
const target = [expected.method, expected.origin].filter(Boolean).join(' ');
const attemptedStatus = Number(attempted?.result?.status);
const evidence = matched
? `${target} -> HTTP ${matched.result.status}`
: attempted
? `request did not succeed${Number.isInteger(attemptedStatus) ? ` (HTTP ${attemptedStatus})` : ''}`
: 'not observed';
add(
`tool_request:${expected.tool}:${target}`,
expected.label || `Observed ${expected.tool} request to ${target}`,
expected.weight || 10,
Boolean(matched),
evidence,
);
}
for (const toolName of scenario.verify?.forbiddenTools || []) {
const used = calls.some((call) => call.name === toolName);
add(`tool_forbidden:${toolName}`, `Did not use ${toolName}`, 5, !used, used ? 'observed' : 'absent');
}
for (const expected of scenario.verify?.result || []) {
const actual = getPath(run?.result, expected.path);
add(
`result:${expected.path}`,
expected.label || `Result field ${expected.path}`,
expected.weight || 10,
checkValue(actual, expected),
JSON.stringify(actual),
);
}
const events = remoteState?.events || [];
for (const expected of scenario.verify?.events || []) {
const matches = events.filter((event) => event.type === expected.type);
add(
`event:${expected.type}`,
expected.label || `Gnippets event ${expected.type}`,
expected.weight || 15,
matches.length >= (expected.min || 1),
matches.map((event) => event.detail).join(' | ') || 'event absent',
);
}
const scheduledExpectation = scenario.verify?.scheduledJobs;
if (scheduledExpectation) {
const jobs = scheduledState?.jobs || [];
const scheduledCalls = calls.filter((call) => (
call.name === 'schedule_task' && call.result?.success === true
));
const expectedCount = scheduledExpectation.count || 1;
add(
'scheduled_jobs:count',
`Observed ${expectedCount} scheduled job outcome(s)`,
scheduledExpectation.countWeight || 15,
jobs.length === expectedCount,
`${jobs.length} observed`,
);
if (scheduledExpectation.status) {
const matching = jobs.filter(job => job.status === scheduledExpectation.status).length;
add(
'scheduled_jobs:status',
`All scheduled jobs reached ${scheduledExpectation.status}`,
scheduledExpectation.statusWeight || 20,
jobs.length === expectedCount && matching === expectedCount,
`${matching}/${expectedCount}`,
);
}
if (scheduledExpectation.lastOutcome) {
const matching = jobs.filter(job => job.lastOutcome === scheduledExpectation.lastOutcome).length;
add(
'scheduled_jobs:outcome',
`All scheduled jobs reported ${scheduledExpectation.lastOutcome}`,
scheduledExpectation.outcomeWeight || 25,
jobs.length === expectedCount && matching === expectedCount,
`${matching}/${expectedCount}`,
);
}
const definitions = scheduledExpectation.definitions || [];
for (const [index, expected] of definitions.entries()) {
const call = scheduledCalls[index];
const args = call?.args || {};
const jobId = call?.result?.jobId || call?.result?.existingJobId || '';
const prompt = String(args.prompt || '');
const definitionPassed = Boolean(call)
&& (!expected.title || args.title === expected.title)
&& (!expected.mode || args.mode === expected.mode)
&& (!expected.scheduleType || args.schedule?.type === expected.scheduleType)
&& (!Object.hasOwn(expected, 'afterSeconds')
|| Number(args.schedule?.after_seconds) === expected.afterSeconds)
&& (!expected.targetType || args.target?.type === expected.targetType)
&& (!expected.targetUrl || args.target?.url === expected.targetUrl)
&& (expected.promptContains || []).every((value) => prompt.includes(value))
&& (expected.promptExcludes || []).every((value) => !prompt.includes(value));
add(
`scheduled_jobs:definition:${index}`,
expected.label || `Scheduled job ${index + 1} matched its requested definition`,
expected.weight || 15,
definitionPassed,
definitionPassed ? `${args.title} -> ${args.target?.url}` : 'definition mismatch',
);
if (expected.resultPath) {
const resultJobId = getPath(run?.result, expected.resultPath);
const stateJobId = jobs[index]?.id;
const idsMatch = Boolean(jobId)
&& resultJobId === jobId
&& stateJobId === jobId;
add(
`scheduled_jobs:id:${expected.resultPath}`,
`Scheduled job ID matches result field ${expected.resultPath}`,
expected.idWeight || 10,
idsMatch,
idsMatch ? jobId : 'ID mismatch',
);
}
}
if (scheduledExpectation.distinctPrompts) {
const prompts = scheduledCalls.map((call) => String(call.args?.prompt || ''));
add(
'scheduled_jobs:distinct_prompts',
'Scheduled jobs use distinct non-empty prompts',
scheduledExpectation.distinctPromptsWeight || 10,
prompts.length === expectedCount
&& prompts.every(Boolean)
&& new Set(prompts).size === expectedCount,
`${new Set(prompts.filter(Boolean)).size}/${expectedCount} distinct`,
);
}
}
if (scenario.verify?.finalUrlHost) {
let host = '';
try { host = new URL(getFinalUrl(run, trace)).hostname; } catch {}
add('final_url', 'Finished on the expected host', 10, host === scenario.verify.finalUrlHost, host);
}
if (captureRequired) {
add(
'artifact:video',
'Run video synchronized',
10,
!artifactError,
artifactError?.message || 'video.webm',
);
}
add(
'cleanup',
'Ephemeral resources cleaned up',
10,
cleanupErrors.length === 0,
cleanupErrors.map((error) => error.message).join(' | ') || 'browser and fixture removed',
);
const available = checks.reduce((sum, check) => sum + check.weight, 0);
const earned = checks.filter((check) => check.passed).reduce((sum, check) => sum + check.weight, 0);
const score = available ? Math.round((earned / available) * 100) : 0;
const requiredPassed = checks.every((check) => check.passed);
return {
scenario_id: scenario.id,
passed: requiredPassed && !setupError && !scheduledError && !postStartError,
score,
earned,
available,
stuck_at: inferStuckAt({
run,
trace,
setupError,
scheduledError,
postStartError,
artifactError,
cleanupErrors,
checks,
}),
checks,
error: scheduledError?.message || postStartError?.message || setupError?.message || run?.error || '',
artifact_warning: artifactError?.message || '',
};
}
export function renderSummary(results, metadata = {}) {
const passed = results.filter((result) => result.grade.passed).length;
const lines = [
'# WebBrain Cloud E2E report',
'',
`- Started: ${metadata.startedAt || 'unknown'}`,
`- Finished: ${metadata.finishedAt || 'unknown'}`,
`- Pack: ${metadata.pack || 'all'}`,
`- Passed: ${passed}/${results.length}`,
'',
'| Scenario | Result | Score | Stuck at |',
'|---|---:|---:|---|',
...results.map(({ scenario, grade }) => (
`| ${scenario.title} | ${grade.passed ? 'PASS' : 'FAIL'} | ${grade.score} | ${grade.stuck_at || '—'} |`
)),
'',
];
for (const { scenario, grade } of results) {
lines.push(`## ${scenario.title}`, '');
for (const check of grade.checks) {
lines.push(`- ${check.passed ? '✓' : '✗'} ${check.label} (${check.weight})${check.evidence ? ` — ${check.evidence}` : ''}`);
}
if (grade.error) lines.push(`- Error: ${grade.error}`);
if (grade.artifact_warning) lines.push(`- Artifact warning: ${grade.artifact_warning}`);
lines.push('');
}
return lines.join('\n');
}