Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

12.8 KB
Raw Blame
#!/usr/bin/env node
import { mkdir,readFile,writeFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { resolve,join } from 'node:path';
import { chromium } from 'playwright';
import { buildCases,SEEDS } from './cases.mjs';
import { manifest,verifyManifest,DEFAULT_LIMITS } from './lib/manifest.mjs';
import { createParticipant,makeInitialMessages,requestForTurn,parseAdapterActions,trimScreenshotHistory,chatCompletionsUrl,ADAPTER_METADATA } from './lib/adapters.mjs';
import { startFixtureServer } from './lib/fixture.mjs';
import { scoreTask,aggregate } from './lib/scoring.mjs';
import { observe,performAction,appendObservation,ended,isolatedPage } from './lib/browser.mjs';

const HERE=fileURLToPath(new URL('.',import.meta.url));
export function parseArgs(argv) {
  const allowed=new Set(['help','validate-only','allow-paid','config','adapter','model','base','api-key-env','name','split','seeds','only','tag','max-turns','max-actions','task-timeout','request-timeout','max-tokens']);
  const flags=new Set(['help','validate-only','allow-paid']),out={};
  for(let i=0;i<argv.length;i++){
    const key=argv[i].replace(/^--/,'');
    if(!argv[i].startsWith('--')||!allowed.has(key))throw new Error(`Unknown option ${argv[i]}`);
    if(flags.has(key))out[key]=true;
    else {if(!argv[i+1]||argv[i+1].startsWith('--'))throw new Error(`Missing value for --${key}`);out[key]=argv[++i];}
  }
  return out;
}
function positive(value,fallback) {const n=value===undefined?fallback:Number(value);if(!Number.isSafeInteger(n)||n<=0)throw new Error('Limits must be positive integers');return n;}
export function selectCases(args) {
  const seeds=args.seeds?args.seeds.split(',').map(Number):SEEDS;
  if(seeds.some(s=>!SEEDS.includes(s)))throw new Error(`Use frozen seeds ${SEEDS.join(',')}`);
  const cases=buildCases({split:args.split||'heldout',seeds});
  if(!args.only)return cases;
  const wanted=new Set(args.only.split(',')),selected=cases.filter(c=>wanted.has(c.taskId)||wanted.has(c.id));
  for(const id of wanted)if(!selected.some(c=>c.id===id||c.taskId===id))throw new Error(`Unknown selected task ${id}`);
  return selected;
}
export function limitsFrom(args) {
  return {maxTurns:positive(args['max-turns'],DEFAULT_LIMITS.maxTurns),maxActions:positive(args['max-actions'],DEFAULT_LIMITS.maxActions),taskTimeoutMs:positive(args['task-timeout'],DEFAULT_LIMITS.taskTimeoutMs),requestTimeoutMs:positive(args['request-timeout'],DEFAULT_LIMITS.requestTimeoutMs),maxTokens:positive(args['max-tokens'],DEFAULT_LIMITS.maxTokens)};
}

async function endpointRequest(participant,body,signal) {
  const headers={'content-type':'application/json'};if(participant.apiKey)headers.authorization=`Bearer ${participant.apiKey}`;
  const started=Date.now();
  const response=await fetch(chatCompletionsUrl(participant.base),{method:'POST',headers,body:JSON.stringify(body),signal});
  if(!response.ok){await response.body?.cancel();throw new Error(`Endpoint HTTP ${response.status}`);}
  const result=await response.json();
  return {response:result,latencyMs:Date.now()-started};
}

export async function runCase({browser,server,participant,task,limits=DEFAULT_LIMITS,request=endpointRequest,onScreenshot=async()=>{}}) {
  const session=server.create(task),started=Date.now(),controller=new AbortController();
  let context,observation,status='max_turns',error=null,actionsTaken=0,turns=0;
  const trace=[],usage={prompt:0,completion:0,total:0,costUsd:0};
  const timer=setTimeout(()=>{controller.abort();context?.close().catch(()=>{});},limits.taskTimeoutMs);
  try {
    context=await isolatedPage(browser,session);
    const {page}=context;observation=await observe(page,participant);
    const messages=makeInitialMessages(participant,task.task,observation);
    loop:for(turns=1;turns<=limits.maxTurns;turns++) {
      if(controller.signal.aborted){status='task_timeout';break;}
      if(actionsTaken>=limits.maxActions){status='max_actions';break;}
      if(observation.screenshot)await onScreenshot(turns,observation.screenshot);
      trimScreenshotHistory(messages,3);
      const body=requestForTurn({...participant,maxTokens:limits.maxTokens},messages);
      let response,latencyMs;
      try {({response,latencyMs}=await request(participant,body,AbortSignal.any([controller.signal,AbortSignal.timeout(limits.requestTimeoutMs)])));}
      catch(e){usage.costUsd=null;status=controller.signal.aborted?'task_timeout':'endpoint_error';error=String(e.message).replaceAll(participant.apiKey||'\0','[redacted]');trace.push({turn:turns,status,error});break;}
      const u=response.usage||{};
      usage.prompt+=Number(u.prompt_tokens)||0;usage.completion+=Number(u.completion_tokens)||0;usage.total+=Number(u.total_tokens)||0;
      if(typeof u.cost==='number'&&Number.isFinite(u.cost)&&u.cost>=0){if(usage.costUsd!==null)usage.costUsd+=u.cost;}else usage.costUsd=null;
      const message=response.choices?.[0]?.message||{},actions=parseAdapterActions(participant.adapter,message),results=[];
      trace.push({turn:turns,latencyMs,response:message,usage:u});
      if(!actions.length){status='unparseable';break;}
      // Judge only at task termination, NEVER auto-pass midway through a batch.
      // A later duplicate/forbidden action in the same returned batch must count.
      // Terminal actions do not discard the remainder of an already-returned batch.
      // A trailing action that errors after terminal success must fail, not pass.
      let batchEnded = false;
      let batchError = null;
      for(let i=0;i<actions.length;i++) {
        if(controller.signal.aborted){status='task_timeout';break loop;}
        if(actionsTaken>=limits.maxActions){status='max_actions';break loop;}
        const action=actions[i];actionsTaken++;
        try {
          const result=await performAction(page,observation,participant,action,session);results.push(result);
          trace.push({turn:turns,actionIndex:i,action,result});
        } catch(e) {
          const failure = `Action failed: ${String(e.message).slice(0,300)}. Observe again before retrying.`;
          results.push(failure);
          trace.push({turn:turns,actionIndex:i,action,error:results.at(-1)});
          batchError = String(e.message).slice(0,500);
          // Do not silently retarget stale indices or execute the rest of a failed batch.
          break;
        }
        if(ended(action)) batchEnded = true;
      }
      if(session.state.violations.length){status='forbidden_action';break;}
      if(session.state.terminal){
        if(batchError){status='action_error';error=batchError;break;}
        status='completed';break;
      }
      if(batchEnded) {status='agent_ended';break loop;}
      await observation.dispose();observation=await observe(page,participant);
      appendObservation(messages,participant,message,actions,results,observation);
    }
    if(!controller.signal.aborted&&!page.isClosed())await onScreenshot('final',await page.screenshot());
  } catch(e) {status=controller.signal.aborted?'task_timeout':'harness_error';error=String(e.message).replaceAll(participant.apiKey||'\0','[redacted]').slice(0,500);}
  finally {clearTimeout(timer);await observation?.dispose().catch(()=>{});await context?.close().catch(()=>{});session.dispose();}
  const score=scoreTask(task,session.state);
  // Operational failures and exhausted limits are failures even with correct partial UI state.
  const success=score.success&&['completed','agent_ended'].includes(status);
  return {id:task.id,taskId:task.taskId,category:task.category,split:task.split,seed:task.seed,success,status:success?'passed':status,error,score,turns:Math.min(turns,limits.maxTurns),actions:actionsTaken,latencyMs:Date.now()-started,usage,state:session.state,trace};
}

export async function main(argv=process.argv.slice(2)) {
  const args=parseArgs(argv);
  if(args.help){console.log(`LLM tiny v2 — isolated reasoning-heavy browser benchmark

Safe local validation (no model calls):
  node test/llm-tiny-v2/run.mjs --validate-only
  node --test --test-concurrency=1 test/llm-tiny-v2/*.test.mjs test/llm-tiny-v2/lib/*.test.mjs

Live execution requires separate budget approval and explicit --allow-paid:
  node test/llm-tiny-v2/run.mjs --allow-paid --config /private/participants.json --split pilot --seeds 17 --tag pilot-01

Single participant: --adapter openai|compass|browser-use|fara --model NAME --base URL --api-key-env ENV
Selection: --split pilot|heldout --seeds 17,43,89 --only TASK_IDS --tag UNIQUE_NAME
Limits (identical for every participant): --max-turns 80 --max-actions 120
  --task-timeout 900000 --request-timeout 90000 --max-tokens 4096
No provider provisioning, background monitors, or changes to test/llm-tiny.
`);return;}
  const manifestHash=await verifyManifest(),cases=selectCases(args),limits=limitsFrom(args);
  if(args['validate-only']){const data=await manifest();console.log(`Validated frozen ${data.version}: ${data.cases.length} variants (60 held-out + 10 pilot tasks, three seeds). Selected ${cases.length}. No endpoints called.`);return;}
  if(!args['allow-paid'])throw new Error('No model calls made. Live execution requires --allow-paid and separate approval within the remaining hard spending cap.');
  const inputs=args.config?JSON.parse(await readFile(resolve(args.config),'utf8')).participants:[{name:args.name,adapter:args.adapter,model:args.model,base:args.base,apiKeyEnv:args['api-key-env']}];
  if(!Array.isArray(inputs)||!inputs.length)throw new Error('Config requires participants');
  const participants=inputs.map(input=>{
    if(input.apiKey)throw new Error('Use apiKeyEnv, not an inline API key');
    if(input.apiKeyEnv&&!process.env[input.apiKeyEnv])throw new Error(`Missing credential environment variable ${input.apiKeyEnv}`);
    if(input.extraBody&&/"(?:api_key|apiKey|authorization|password|secret|access_token)"\s*:/i.test(JSON.stringify(input.extraBody)))throw new Error('Do not put credentials in extraBody');
    if(input.noScreenshot)throw new Error('Text-only baseline changes the protocol; use the native Compass adapter for text-only runs');
    const p=createParticipant({...input,timeoutMs:limits.requestTimeoutMs,maxTokens:limits.maxTokens});
    if(!input.name)p.name=p.name.replace(/[^a-zA-Z0-9._-]+/g,'-');
    const url=new URL(p.base);
    if(!['http:','https:'].includes(url.protocol)||url.username||url.password||url.search||url.hash)throw new Error('Endpoint must be HTTP(S) with no embedded secrets, query, or fragment');
    // Bearer credentials must never travel over plaintext to a remote host:
    // HTTP is only allowed for loopback endpoints (local inference).
    if(url.protocol==='http:'&&!['localhost','127.0.0.1','::1'].includes(url.hostname.toLowerCase()))throw new Error('Remote endpoints must use HTTPS');
    if(!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(p.name))throw new Error('Set a simple unique participant name');
    return p;
  });
  if(new Set(participants.map(p=>p.name)).size!==participants.length)throw new Error('Participant names must be unique');
  const tag=args.tag||new Date().toISOString().replace(/[:.]/g,'-');
  if(!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(tag))throw new Error('Invalid result tag');
  const root=join(HERE,'results');await mkdir(root,{recursive:true});
  const out=join(root,tag);await mkdir(out); // Exclusive: an existing run can never be overwritten.
  const protocol={manifestHash,limits,split:args.split||'heldout',caseIds:cases.map(c=>c.id),screenshotHistory:3};
  await writeFile(join(out,'protocol.json'),JSON.stringify(protocol,null,2));
  const server=await startFixtureServer();let browser;
  try {
    browser=await chromium.launch({headless:true});
    for(const participant of participants) {
      const dir=join(out,participant.name);await mkdir(dir);const results=[];
      const {messages:unusedMessages,tools:unusedTools,...requestSettings}=requestForTurn(participant,[]);
      const metadata={name:participant.name,model:participant.model,adapter:participant.adapter,endpoint:chatCompletionsUrl(participant.base),requestSettings,...ADAPTER_METADATA[participant.adapter]};
      await writeFile(join(dir,'participant.json'),JSON.stringify(metadata,null,2));
      for(const task of cases) {
        const result=await runCase({browser,server,participant,task,limits,onScreenshot:(step,buffer)=>writeFile(join(dir,`${task.id}-${step}.png`),buffer)});
        results.push(result);await writeFile(join(dir,`${task.id}.json`),JSON.stringify(result,null,2));
        const summary={...metadata,protocol,...aggregate(results),complete:results.length===cases.length,results};
        await writeFile(join(dir,'summary.json'),JSON.stringify(summary,null,2));
        console.log(`${participant.name} ${task.id} ${result.status} (${result.actions} actions)`);
      }
    }
  } finally {await browser?.close();await server.close();}
  console.log(`Results: ${out}`);
}

if(process.argv[1]&&resolve(process.argv[1])===fileURLToPath(import.meta.url))main().catch(()=>{console.error('Execution failed. Re-run with safe local debugging to inspect details.');process.exitCode=1;});