Open-source AI browser agent for Chrome and Firefox (monorepo) đź§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
#!/usr/bin/env node
// Read-only aggregation of matching v2 results; no v1 scores enter the report.
import { readFile, writeFile, mkdir } from 'node:fs/promises';
import { compare } from '../../llm-tiny-v2/compare.mjs';
import { verifyManifest } from '../../llm-tiny-v2/lib/manifest.mjs';
const root = new URL('../../llm-tiny-v2/results/', import.meta.url);
const names = ['browser-use', 'deepseek', 'fara-4b', 'fara-9b', 'fara-27b', 'compass'];
const hash = await verifyManifest(), summaries = [];
for (const name of names) {
try {
const s = JSON.parse(await readFile(new URL(`2026-09-13-v2-pilot-${name}/${name}/summary.json`, root)));
if (s.protocol.manifestHash !== hash || s.protocol.split !== 'pilot' || s.results.some(r => r.seed !== 17)) throw new Error(`Nonmatching protocol: ${name}`);
summaries.push(s);
} catch (e) { if (e.code !== 'ENOENT') throw e; }
}
const state = JSON.parse(await readFile(new URL('../results/2026-09-13-six-model-matrix/.runpod-state.json', import.meta.url)));
const budget = JSON.parse(await readFile(new URL('../results/2026-09-13-six-model-matrix/.v2-budget.json', import.meta.url)));
const now = Date.now();
const computeAccrued = state.pods.reduce((n,p) => n + ((p.deletedAt || now)-p.createdAt)/3600000*p.budgetRate,0);
const computeThroughExpiry = state.pods.reduce((n,p) => n + ((p.deletedAt || Math.max(now,p.deadline*1000))-p.createdAt)/3600000*p.budgetRate,0);
const remaining = 30-computeAccrued-budget.apiChargedOrReservedUsd;
const afterCommitted = 30-computeThroughExpiry-budget.apiChargedOrReservedUsd-budget.cleanupReserveUsd;
const money = n => `$${n.toFixed(3)}`;
const out = new URL('2026-09-13-v2-comparison/',root);
let cleanup;
try { cleanup = JSON.parse(await readFile(new URL('cleanup-verification.json', out))); }
catch (e) { if (e.code !== 'ENOENT') throw e; }
let md = `# Six-participant v2 pilot\n\nUpdated ${new Date(now).toISOString()}. Frozen manifest: \`${hash}\`. Ten pilot tasks per participant, seed 17, unchanged limits. Earlier v1 scores are not comparable and are excluded.\n\n`;
if (cleanup) md += `## Autonomous handoff blocked; protected shutdown complete\n\nNo CPU instance or remote archive was created, and no cloud coordinator owns this run. The benchmark will not continue when the laptop closes. RunPod removed its automatic stop/terminate flags because the backend accepted but did not enforce their deadlines. The proposed restoration remains open and draft as of ${cleanup.at}. A process watchdog is not a provider-enforced total spending ceiling, so the original hard-cap prerequisite failed before provisioning. [RunPod removal](https://github.com/runpod/runpodctl/pull/330), [pending restoration](https://github.com/runpod/runpodctl/pull/331).\n\nThe last Fara 4B task, \`pilot-workflow-02-s17\`, finished and checkpointed before the local coordinator exited. All 106 dispatched Fara 4B requests have saved responses; none were pending. No next episode launched. Model loading had already finished. Existing safeguards stayed active through archive verification and Pod deletion. Both v1 and deployment-side v2 launches are fenced; cleanup and read-only inspection remain available.\n\nNext uncompleted coverage is Fara 4B recovery and visual, then Fara 9B, Fara 27B and Compass. These are **not scheduled to run**. Browser Use and DeepSeek pilot results are preserved for reuse. No held-out episodes ran. The held-out gate remains all six reviewed pilots plus safe funding for 180 episodes per model, 1,080 total; no reduced protocol is substituted.\n\n`;
if (summaries.length === 6 && summaries.every(s=>s.complete)) md += compare([summaries.find(s=>s.name==='deepseek'),...summaries.filter(s=>s.name!=='deepseek')]);
else {
md += 'Coverage is incomplete: 28/60 required pilot episodes, only 2/6 complete participants. There is no matching six-model comparison. Rates below use completed episodes only, never unrun tasks as failures. Latency is episode wall time; actions are attempted individual actions, not model turns.\n\n| Participant | Episodes completed | Strict passes | Median seconds | p95 seconds | Median actions | Total attempted actions |\n|---|---:|---:|---:|---:|---:|---:|\n';
for (const name of names) {
const s=summaries.find(s=>s.name===name);
md += `| ${name} | ${s?s.cases:0}/10 | ${s?`${s.passed}/${s.cases} (${(100*s.successRate).toFixed(1)}%)`:'not run'} | ${s?(s.medianLatencyMs/1000).toFixed(3):'—'} | ${s?(s.p95LatencyMs/1000).toFixed(3):'—'} | ${s?s.medianActions:'—'} | ${s?s.results.reduce((n,r)=>n+r.actions,0):'—'} |\n`;
}
md += '\n| Category | '+names.join(' | ')+' |\n|---|'+names.map(()=>'---:').join('|')+'|\n';
for (const cat of ['planning','calculation','evidence','workflow','recovery','visual']) md += `| ${cat} | ${names.map(n=>{const c=summaries.find(s=>s.name===n)?.categories[cat];return c?`${c.passed}/${c.cases} (${(100*c.successRate).toFixed(0)}%)`:'—';}).join(' | ')} |\n`;
}
md += `\n## Original shared $30 budget\n\n- Conservative RunPod compute/storage accrued: ${money(computeAccrued)}. Includes every download, setup, inference, idle and cleanup second from creation until deletion, including earlier v1 hosting. The $1.65/hour budget rate exceeds the $1.59/hour GPU quote and includes a storage allowance.\n- All recorded API charges plus unresolved request reservations: ${money(budget.apiChargedOrReservedUsd)}. This includes $0.029319759 incurred before v2 and $0.031821054 for the matching DeepSeek pilot.\n- Total conservative accrued: ${money(computeAccrued+budget.apiChargedOrReservedUsd)}. Remaining: ${money(remaining)}.\n- Active resource/runtime commitments: ${money(computeThroughExpiry-computeAccrued)}; cleanup reserve ${money(budget.cleanupReserveUsd)}; future deployment planning reserve ${money(budget.futureComputeReserveUsd)}. ${cleanup?'All future reservations were released after confirmed cleanup. No requests are pending; the $2 API policy allowance is not an active obligation.':''}\n- CPU setup, new storage and cloud retention incurred under this handoff: $0. No resource was provisioned.\n- ${cleanup?'RunPod deletion and an empty live Pod inventory are independently confirmed. Its billing feed had not settled; an empty billing response is not zero spend.':'Existing deployment expiries are unchanged.'} These are conservative estimates, not a settled invoice or a provider-enforced spending cap.\n- GPU costs cover a shared lifetime across model downloads and both suite versions, not an attributable per-model inference invoice. No unsupported per-model allocation is invented.\n`;
md += '\n## Pilot interpretation caveats\n\nThe conference-space task uses the heading “Base/daily dollars” and wording that can be read as a fee per session or both amounts per session, while the frozen answer charges the base once. This is a task-wording ambiguity, not clean evidence of an arithmetic deficit. Evidence tasks request a “source ID” but label documents only by title; numeric control indices are not accepted by the exact-string grader. Their task text explicitly authorizes submitting the form to record the decision, as the scorer requires. Answer-only termination and wrong submitted values must be distinguished from broken action execution. Preserve the frozen strict scores and flag these ambiguities; do not silently change answers, drop tasks, or re-freeze.\n';
md += '\nThe two Fara 4B workflow failures are exact-case formatting failures: both completed every other check (5/6 each) but submitted `Draft` where the frozen answer is `draft`. These remain strict failures; they are not failures to calculate totals or follow the multi-stage workflow.\n';
md += '\n## Operational failures\n\n';
for (const s of summaries) {
const errors=s.results.filter(r=>r.error||r.trace.some(t=>t.error));
md += `- ${s.name}: ${s.results.filter(r=>r.error).length} episode-level errors, ${errors.length} episodes with recorded action/trace errors, ${s.results.filter(r=>r.status==='unparseable').length} unparseable outputs.\n`;
}
md += '\nDeepSeek\'s three recorded action errors were navigation-invalidated handles in batched actions. It re-observed and passed all three affected episodes; these are recovered control errors, not failed episodes or a model endpoint outage. No endpoint failures or unparseable outputs occurred in the 28 saved episodes.\n\nFara attempted browser Find shortcuts during evidence-02. An offline check found neither Control+F nor Meta+F opened browser Find in this headless Mac Chromium surface; subsequent typing went into the page input. This is an adapter/browser-interface limitation even though the key dispatch itself did not throw. It is not evidence of a reasoning failure, and no frozen adapter change or paid rerun was made.\n\n### Failed-episode review\n\nClassifications describe observed traces and can overlap; all scores remain frozen.\n\n| Participant / task | Observed result | Classification |\n|---|---|---|\n| Browser Use / planning-02 | K36 instead of frozen K46 | Wrong total with task-wording ambiguity |\n| Browser Use / evidence-01 | Numeric control index 4 instead of document title | Source-ID format ambiguity |\n| Browser Use / evidence-02 | Correct No / HOLD in final answer, no form submission | Completion/submission wording mismatch |\n| Browser Use / workflow-01 | Did not read Brief; invented reference, owner, quantity and total | Model omitted required evidence and submitted unsupported values |\n| DeepSeek / planning-02 | K108 instead of frozen K46 | Alternative interpretation of ambiguous base/daily wording |\n| Fara 4B / planning-01 | K195 instead of cheaper L185 | Model selection/optimization error |\n| Fara 4B / planning-02 | L66 | Wrong selection/total; conference wording caveat also applies |\n| Fara 4B / calculation-02 | Submitted 121.20 | Incorrect calculated value |\n| Fara 4B / evidence-01 | Answered No instead of Yes, ended without submission | Evidence-reading error plus submission caveat |\n| Fara 4B / evidence-02 | Submitted empty fields; attempted browser Find | Browser-control limitation plus incomplete submission |\n| Fara 4B / workflow-01 | All fields correct except Draft vs draft | Exact-case formatting only |\n| Fara 4B / workflow-02 | All fields correct except Draft vs draft | Exact-case formatting only |\n';
await mkdir(out,{recursive:true});
if (cleanup) md += `\n## Cleanup, preservation and access\n\nBenchmark Pod \`${cleanup.benchmarkPodId}\` was deleted at ${cleanup.podDeletedAt}; its original deadline was ${cleanup.originalDeadline}, never extended. Independent checks at ${cleanup.at}: Pod GET returned 404; Pod listing returned 200 with zero Pods. No active benchmark resource or future expiry remains. The local guard exited only after the deletion was recorded. Unrelated storage was left untouched.\n\nBefore deleting the Pod, a private local archive was restored and every one of its 1,207 files was checksum-verified; its restored frozen manifest also verified. The archive includes the 28 completed v2 episodes, traces/screenshots/usage/errors, hosting metadata, pinned sources/revisions and preserved old v1 results. A scan found no known benchmark credentials. Local source results were not deleted. The archive is a pre-cleanup snapshot; this report, budget and cleanup receipt are retained alongside it.\n\n- [Private local archive](../2026-09-13-private-handoff-snapshot.tar.gz) — mode 0600, SHA-256 \`03cf98a3791ad7123b438e8e33ff021b64a1dfaecd6acc84753428853b14d960\`.\n- [Archive verification](archive-verification.json), [cleanup verification](cleanup-verification.json), [budget](budget.json), [runtime](runtime.json).\n- Status and retrieval are local files, not a public URL or a cloud dashboard. RunPod's authenticated console can independently confirm Pod deletion. No CPU status service or remote download was created.\n- Local copies have no RunPod retention cost and remain until the user deletes them. Cloud retention duration is still unchosen. A CPU container alone is not a durable archive after its deletion; independent storage would require an explicit bounded retention decision and safe billing controls.\n\n## Evaluation and autonomy verification\n\nFrozen version reasoning-browser-v2.1; manifest unchanged. Limits remain 80 turns, 120 attempted actions, 900-second episodes, 90-second requests, 4,096 output tokens, 1440×900 viewport and latest three screenshots. Adapter-native prompts and settings remain unchanged. Five Hugging Face revisions and the serving image are pinned in the saved metadata. DeepSeek uses the specified OpenRouter model ID, not an independently pinned upstream weight revision. The GPU service had a 16,384-token context ceiling; this hosting limit remains disclosed.\n\nLocal runtime: Node 22.22.3, Playwright 1.59.1, Chromium 147.0.7727.15, macOS arm64; vLLM 0.19.1 on the GPU. All 235 offline harness tests passed before shutdown. The manifest verified again from the restored archive. No Linux CPU rendering comparison or cloud inference took place; there are no environments silently mixed in these pilot results.\n\nCloud-owned progress, cloud restart/resume, distributed duplicate prevention, and independent cloud budget/deadline failure tests are **not verified** because no cloud coordinator was deployed. Local launch fences are tested without network calls; those are not proof of cloud autonomy. On this hard-cap failure the run remained ownerless, launched nothing further and entered protected shutdown. It will not auto-resume or provision a replacement. This report is not authorization to close the laptop expecting continued execution.\n`;
if (summaries.length === 6 && summaries.every(s=>s.complete)) {
const counts={planning:45,calculation:45,evidence:30,workflow:30,recovery:15,visual:15};
const projections=summaries.map(s=>{
let ms=0;
for(const [category,count] of Object.entries(counts)) {
const rows=s.results.filter(r=>r.category===category);
// Held-out workflows have six stages; pilot workflows have four.
ms+=rows.reduce((n,r)=>n+r.latencyMs,0)/rows.length*count*(category==='workflow'?1.5:1);
}
return {name:s.name,episodes:180,pointHours:ms/3600000,safetyHours:2*ms/3600000,
pointComputeUsd:s.adapter==='openai'?0:ms/3600000*1.65,
safetyComputeUsd:s.adapter==='openai'?0:2*ms/3600000*1.65,
apiPointUsd:s.adapter==='openai'?s.reportedApiCostUsd*18:null};
});
await writeFile(new URL('heldout-planning-estimate.json',out),JSON.stringify({note:'Planning only, not authorization. Category-weighted pilot extrapolation with 6/4 workflow adjustment and 2x runtime allowance. Assumes one A100 per GPU participant; startup, idle, current expenditure, API, and cleanup must be added. Independent Pod lifetime checks are still required.',projections},null,2)+'\n');
md+='\n## Held-out planning estimate (not a launch authorization)\n\nThe full frozen held-out protocol is 180 episodes per participant, 1,080 total. Estimates below weight the pilot by held-out category counts and scale workflow time by 6/4 stages. The safety column doubles execution time; loading, idle time, prior spending, API usage and cleanup are additional. Native hosting must be checked independently.\n\n| Participant | Point execution hours | 2x execution hours | 2x compute estimate at one A100 |\n|---|---:|---:|---:|\n';
for(const p of projections)md+=`| ${p.name} | ${p.pointHours.toFixed(2)} | ${p.safetyHours.toFixed(2)} | ${money(p.safetyComputeUsd)} |\n`;
}
try {
const billing=JSON.parse(await readFile(new URL('billing-snapshot.json',out)));
md+=`\n## Later billing update\n\nAt ${billing.at}, RunPod began returning ${money(billing.reportedPartialUsd)} in charges, covering ${billing.reportedTimeBilledMs} ms of the ${billing.resourceWallClockLifetimeMs} ms resource lifetime. This supersedes the earlier empty-feed observation, but is not a confirmed final invoice. The full conservative compute/storage estimate is retained; the reported partial amount overlaps it and is not added twice. See [billing snapshot](billing-snapshot.json).\n`;
} catch(e) { if(e.code!=='ENOENT')throw e; }
await writeFile(new URL('report.md',out),md);
await writeFile(new URL('budget.json',out),JSON.stringify({at:new Date(now).toISOString(),capUsd:30,computeAccruedUsd:computeAccrued,computeThroughExpiryUsd:computeThroughExpiry,apiChargedOrReservedUsd:budget.apiChargedOrReservedUsd,totalAccruedUsd:computeAccrued+budget.apiChargedOrReservedUsd,remainingAccruedUsd:remaining,remainingAfterActiveExpiryAndCleanupUsd:afterCommitted,activeComputeCommitmentUsd:computeThroughExpiry-computeAccrued,cleanupReserveUsd:budget.cleanupReserveUsd,futureComputePlanningReserveUsd:budget.futureComputeReserveUsd,activeFutureApiCommitmentUsd:budget.activeFutureApiCommitmentUsd??null,settledRunpodInvoice:false,autonomousHandoffBlocked:budget.handoffBlocked??false},null,2)+'\n');
console.log(JSON.stringify({report:new URL('report.md',out).pathname,completeParticipants:summaries.filter(s=>s.complete).length,remainingAccruedUsd:remaining,remainingAfterActiveExpiryAndCleanupUsd:afterCommitted}));