Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
#!/usr/bin/env node
// Offline, auditable comparison against the pinned August full-tier cohort.
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { readFileSync, readdirSync, writeFileSync } from 'node:fs';
import { dirname, join, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { deepEqual } from './lib/score.mjs';
const here = dirname(fileURLToPath(import.meta.url)), root = join(here, '../..');
const out = join(here, 'analysis/2026-10-06-ling31-flash');
const read = path => JSON.parse(readFileSync(path, 'utf8'));
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
const repoPath = path => relative(root, path).replaceAll('\\', '/');
const replayPath = join(here, 'freeze/full-replay-7182c21f.json');
const replay = read(replayPath), rubrics = read(join(out, 'rubrics.json'));
const historical = read(join(here, 'results/2026-08-02-full-suite-consensus.json'));
assert.equal(replay.meta.sourceCommit, rubrics.sourceCommit);
const resultDirs = readdirSync(join(here, 'results'));
const labels = {
'deepseek/deepseek-v4-flash-0731': 'DeepSeek V4 Flash 0731',
'tencent/hy3': 'Tencent HY3', 'z-ai/glm-5.2': 'GLM-5.2',
'poolside/laguna-xs-2.1': 'Poolside Laguna XS 2.1',
'inclusionai/ling-3.1-flash': 'Ling 3.1 Flash',
'minimax/minimax-m3': 'MiniMax M3',
};
const selected = new Set(Object.keys(labels).filter(model => model !== 'minimax/minimax-m3'));
const ids = Object.keys(replay.firstTurn).sort();
assert.equal(ids.length, 100);
const stable = value => value === null || typeof value !== 'object' ? value
: Array.isArray(value) ? value.map(stable)
: Object.fromEntries(Object.keys(value).sort().map(key => [key, stable(value[key])]));
const signature = call => JSON.stringify(stable(call || null));
// Validate precisely the JSON Schema keywords present in this archived tool set.
// Unknown validation keywords fail closed so future schema changes cannot slip by.
const keywords = new Set(['type', 'properties', 'required', 'items', 'enum', 'minimum', 'maxLength', 'description']);
function checkSchema(schema) {
for (const key of Object.keys(schema)) assert(keywords.has(key), `Unsupported schema keyword: ${key}`);
Object.values(schema.properties || {}).forEach(checkSchema);
if (schema.items) checkSchema(schema.items);
}
for (const tools of Object.values(replay.tools)) tools.forEach(tool => checkSchema(tool.function.parameters));
function validate(value, schema, path = '$') {
const errors = [];
const types = { object: v => v !== null && typeof v === 'object' && !Array.isArray(v),
array: Array.isArray, string: v => typeof v === 'string', boolean: v => typeof v === 'boolean',
number: v => typeof v === 'number' && Number.isFinite(v), integer: Number.isInteger };
if (schema.type && !types[schema.type]?.(value)) return [`${path}: expected ${schema.type}`];
if (schema.enum && !schema.enum.some(option => deepEqual(option, value))) errors.push(`${path}: invalid enum`);
if (schema.minimum !== undefined && value < schema.minimum) errors.push(`${path}: below minimum`);
if (schema.maxLength !== undefined && [...value].length > schema.maxLength) errors.push(`${path}: maxLength exceeded`);
for (const key of schema.required || []) if (!Object.hasOwn(value, key)) errors.push(`${path}.${key}: missing`);
for (const [key, child] of Object.entries(schema.properties || {})) {
if (Object.hasOwn(value, key)) errors.push(...validate(value[key], child, `${path}.${key}`));
}
if (schema.items) value.forEach((item, index) => errors.push(...validate(item, schema.items, `${path}[${index}]`)));
return errors;
}
const sources = historical.results.map(row => {
const resultDir = row.resultDir || resultDirs.find(dir => dir.startsWith('2026-08-02-')
&& !dir.includes('-7182') && dir.endsWith(`_chrome_${row.model.replace(/[^\w.-]+/g, '_')}`));
assert(resultDir, `Missing historical results: ${row.model}`);
return { model: row.model, resultDir, historical: row, date: '2026-08-02' };
});
const fresh = { model: 'inclusionai/ling-3.1-flash', date: '2026-10-06', fresh: true,
resultDir: '2026-10-06-openrouter-ling31-flash-full-7182_chrome_inclusionai_ling-3.1-flash' };
const audit = [];
function load(source, { caseIds = ids, recordAudit = true } = {}) {
const dir = join(here, 'results', source.resultDir), summary = read(join(dir, 'summary.json'));
assert.equal(readdirSync(dir).filter(file => /^\d{3}\.json$/.test(file)).length, 100);
assert.equal(summary.model, source.model);
assert.equal(summary.cases, 100);
assert.equal(summary.tier, 'full');
assert.equal(summary.freeze, null);
assert.equal(summary.modeOverride, null);
assert.equal(summary.reasoningEffort, null);
assert.equal(summary.chatTemplateCompat, 'off');
assert.equal(summary.structuredToolsSent, true);
if (source.fresh) {
assert.equal(summary.replay.sha256, hash(readFileSync(replayPath)));
assert.equal(summary.replay.sourceCommit, replay.meta.sourceCommit);
assert.equal(summary.saveRequest, true);
}
const row = { model: source.model, label: labels[source.model] || source.model, date: source.date,
source: repoPath(dir), fresh: !!source.fresh, cases: caseIds.length, errors: 0, responses: 0,
emittedToolCalls: 0, nativeToolCalls: 0, schemaValidToolCalls: 0, idealToolMatches: 0, exactIdealMatches: 0,
promptTokens: 0, completionTokens: 0, cachedPromptTokens: 0, reasoningTokens: 0,
costRecords: 0, usageCost: 0, lengthLimited: 0, contentFallbackCalls: 0,
accessibilityTreeFirstCalls: 0, attempts: 0, providers: {}, byTool: {}, schemaErrors: [] };
const records = [], latencies = [];
for (const id of caseIds) {
const path = join(dir, `${id}.json`), bytes = readFileSync(path), record = JSON.parse(bytes);
const archived = rubrics.cases[id].question, saved = replay.firstTurn[id];
assert.equal(record.id, id);
assert.equal(record.user, archived.user);
assert.equal(record.mode, archived.mode);
assert.deepEqual(record.tab, archived.tab);
if (source.fresh) {
assert.deepEqual(record.request, { model: source.model, temperature: saved.temperature,
max_tokens: saved.max_tokens, messages: saved.messages, tools: replay.tools[saved.mode] }, `Request drift ${id}`);
assert(!/^HTTP 40[13]:/.test(record.error || ''), 'Authentication failures cannot be scored.');
assert(record.error || record.response?.choices?.length, `Missing raw API response ${id}`);
if (record.response) {
assert.equal(record.response.model, source.model, `Unexpected response model ${id}`);
assert.deepEqual(record.usage, record.response.usage);
}
}
row.attempts += record.attempts || 1;
if (record.error) row.errors++;
else { row.responses++; latencies.push(record.latencyMs); }
const call = record.firstToolCall || null, ideal = rubrics.cases[id].expected.idealFirstToolCall;
let schemaErrors = [];
if (call) {
row.emittedToolCalls++;
if (record.toolCallSource === 'tool_calls') row.nativeToolCalls++;
if (source.fresh) {
const native = record.response?.choices?.[0]?.message?.tool_calls?.[0];
if (record.toolCallSource === 'tool_calls') {
assert(native, `Missing native call ${id}`);
try {
const args = typeof native.function.arguments === 'string' ? JSON.parse(native.function.arguments) : native.function.arguments;
assert.deepEqual(call, { name: native.function.name, args });
} catch (error) {
if (!(error instanceof SyntaxError)) throw error;
schemaErrors.push('Malformed native arguments JSON');
}
}
}
const tool = replay.tools[saved.mode].find(tool => tool.function.name === call.name);
schemaErrors.push(...(tool ? validate(call.args, tool.function.parameters) : ['Unknown tool name']));
if (!schemaErrors.length) row.schemaValidToolCalls++;
else row.schemaErrors.push({ id, call, errors: schemaErrors });
if (!record.error && call.name === ideal.name) {
row.idealToolMatches++;
if (deepEqual(call.args, ideal.args)) row.exactIdealMatches++;
}
if (call.name === 'get_accessibility_tree') row.accessibilityTreeFirstCalls++;
}
const toolName = call?.name || (record.error ? '(error)' : '(no-tool)');
row.byTool[toolName] = (row.byTool[toolName] || 0) + 1;
if (record.finishReason === 'length') row.lengthLimited++;
if (record.toolCallSource === 'content_fallback') row.contentFallbackCalls++;
if (typeof record.usage?.cost === 'number') { row.costRecords++; row.usageCost += record.usage.cost; }
row.promptTokens += record.usage?.prompt_tokens || 0;
row.cachedPromptTokens += record.usage?.prompt_tokens_details?.cached_tokens || 0;
row.completionTokens += record.usage?.completion_tokens || 0;
row.reasoningTokens += record.usage?.completion_tokens_details?.reasoning_tokens || 0;
if (record.response?.provider) row.providers[record.response.provider] = (row.providers[record.response.provider] || 0) + 1;
records.push({ id, call, name: toolName, signature: record.error ? `error:${source.model}:${id}` : signature(call) });
if (recordAudit) audit.push({ model: source.model, id, path: repoPath(path), sha256: hash(bytes),
requestSha256: record.request ? hash(JSON.stringify(record.request)) : null,
rubricSha256: replay.meta.sourceHashes[`test/llm/expected/${id}.json`] });
}
if (caseIds.length === 100) {
assert.equal(row.errors, summary.errors);
assert.equal(row.emittedToolCalls, summary.withToolCall);
}
latencies.sort((a, b) => a - b);
row.medianLatencyMs = latencies.length ? (latencies[Math.floor((latencies.length - 1) / 2)] + latencies[Math.floor(latencies.length / 2)]) / 2 : null;
row.p95LatencyMs = latencies.length ? latencies[Math.ceil(latencies.length * 0.95) - 1] : null;
row.cachedPromptSharePct = row.promptTokens ? row.cachedPromptTokens / row.promptTokens * 100 : null;
if (source.historical && caseIds.length === 100) for (const metric of ['emittedToolCalls', 'schemaValidToolCalls', 'idealToolMatches', 'exactIdealMatches']) {
assert.equal(row[metric], source.historical[metric], `Historical ${metric} regression: ${source.model}`);
}
return { source, row, records };
}
function consensus(cohort) {
const caseCount = cohort[0].records.length;
assert(caseCount > 0);
cohort.forEach(entry => assert.equal(entry.records.length, caseCount));
return cohort.map(entry => {
let exact = 0, names = 0;
for (const peer of cohort.filter(peer => peer !== entry)) {
for (let i = 0; i < caseCount; i++) {
if (entry.records[i].signature === peer.records[i].signature) exact++;
if (entry.records[i].name === peer.records[i].name) names++;
}
}
return { ...entry.row, exactActionMatches: exact, toolNameMatches: names,
comparisons: caseCount * (cohort.length - 1),
exactActionConsensusPct: +(exact * 100 / (caseCount * (cohort.length - 1))).toFixed(1),
toolNameConsensusPct: +(names * 100 / (caseCount * (cohort.length - 1))).toFixed(1) };
}).sort((a, b) => b.exactActionMatches - a.exactActionMatches || b.toolNameMatches - a.toolNameMatches)
.map((row, index) => ({ rank: index + 1, ...row }));
}
const oldCohort = sources.map(source => load(source)), reproduced = consensus(oldCohort);
for (const row of reproduced) {
const old = historical.results.find(old => old.model === row.model);
assert.equal(row.exactActionConsensusPct, old.exactActionConsensusPct, `Historical exact consensus: ${row.model}`);
assert.equal(row.toolNameConsensusPct, old.toolNameConsensusPct, `Historical tool consensus: ${row.model}`);
}
writeFileSync(join(out, 'historical-regression.json'), JSON.stringify({ sourceCommit: replay.meta.sourceCommit,
validatedModels: 13, cases: 1300, note: 'All saved exact/tool consensus and emitted/valid/ideal counts reproduced from raw case files.',
results: reproduced }, null, 2) + '\n');
if (process.argv.includes('--historical-only')) {
console.log('Historical 13-model consensus, schema validity, and ideal counts reproduced.');
} else {
const cohort = [...oldCohort, load(fresh)];
const ling = cohort.at(-1);
const campaign = read(join(out, 'campaign.json'));
const finalDir = join(here, 'results', fresh.resultDir);
const recoveries = [campaign.recovery, ...(campaign.additionalRecoveries || [])];
const recoveryRecords = recoveries.flatMap(pass => pass.selectedIds.map(id =>
read(join(pass.archiveDir ? join(out, pass.archiveDir) : finalDir, `${id}.json`))));
const firstPassRecords = ids.map(id => read(join(out, 'first-pass', `${id}.json`)));
for (const record of firstPassRecords) {
const saved = replay.firstTurn[record.id];
assert.deepEqual(record.request, { model: fresh.model, temperature: saved.temperature,
max_tokens: saved.max_tokens, messages: saved.messages, tools: replay.tools[saved.mode] });
}
const initial = read(join(out, 'preflight/001-initial-429.json'));
const retried = read(join(out, 'preflight/001-retried-429.json'));
const diagnostics = readdirSync(join(out, 'preflight')).filter(file => file.endsWith('.json'))
.map(file => read(join(out, 'preflight', file))).filter(record => record.diagnosticOnly);
const count429 = record => (record.retryErrors || []).filter(error => error.error.startsWith('HTTP 429:')).length
+ (record.error?.startsWith('HTTP 429:') ? 1 : 0);
const diagnosticRequests = initial.attempts + retried.attempts + diagnostics.length;
const benchmarkRequests = firstPassRecords.reduce((sum, record) => sum + record.attempts, 0)
+ recoveryRecords.reduce((sum, record) => sum + record.attempts, 0);
const campaignMetrics = { firstPassResponses: firstPassRecords.filter(record => !record.error).length,
firstPassErrors: firstPassRecords.filter(record => record.error).length,
recoveryCaseSelections: recoveryRecords.length,
uniqueRecoveryCases: new Set(recoveries.flatMap(pass => pass.selectedIds)).size,
benchmarkRequests, diagnosticRequests,
totalApiRequests: benchmarkRequests + diagnosticRequests,
http429Responses: [...firstPassRecords, ...recoveryRecords, initial, retried].reduce((sum, record) => sum + count429(record), 0)
+ diagnostics.filter(record => record.status === 429).length,
recoveryWallTimeMs: recoveries.reduce((sum, pass) => sum + read(join(pass.archiveDir ? join(out, pass.archiveDir) : finalDir, 'summary.json')).totalLatencyMs, 0),
firstPassWallTimeMs: read(join(out, 'first-pass/summary.json')).totalLatencyMs,
recoveryPasses: recoveries.length,
concurrency: campaign.recovery.concurrency, retryDelayMs: campaign.recovery.retryDelayMs,
retryMax: campaign.recovery.retryMax,
latencyNote: 'Inference latency uses each successful final HTTP attempt, including network time but excluding failed attempts and backoff. Wall time is reported separately.',
};
for (const folder of ['first-pass', 'preflight', ...recoveries.map(pass => pass.archiveDir).filter(Boolean)]) {
for (const file of readdirSync(join(out, folder)).filter(file => file.endsWith('.json'))) {
const path = join(out, folder, file), bytes = readFileSync(path);
audit.push({ track: folder, path: repoPath(path), sha256: hash(bytes) });
}
}
const complete = ling.row.responses === 100 && ling.row.errors === 0;
// A rejected request is availability evidence, not a model action to rank.
// Retain the unchanged historical pool when the new run lacks full coverage.
const unavailable = { rank: null, ...ling.row, exactActionMatches: null, toolNameMatches: null,
comparisons: null, exactActionConsensusPct: null, toolNameConsensusPct: null,
idealToolMatches: null, exactIdealMatches: null, medianLatencyMs: null, p95LatencyMs: null,
usageCost: null, cachedPromptSharePct: null };
const results = complete ? consensus(cohort) : [...reproduced, unavailable];
const availableIds = ids.filter(id => !read(join(finalDir, `${id}.json`)).error);
const matchedSubset = !complete && availableIds.length ? {
cases: availableIds.length, caseIds: availableIds,
comparisonsPerModel: availableIds.length * 13,
note: 'Availability-selected, non-random subset. Every model is rescored on exactly these same cases with all 14 peers in the pool. These figures cannot be compared numerically with the original full-suite percentages. Cost and latency also use only the matched cases.',
results: consensus([...sources, fresh].map(source => load(source, { caseIds: availableIds, recordAudit: false }))),
} : null;
const pairs = oldCohort.map(peer => {
let exact = 0, names = 0;
for (let i = 0; i < 100; i++) {
if (ling.records[i].signature === peer.records[i].signature) exact++;
if (ling.records[i].name === peer.records[i].name) names++;
}
return { model: peer.row.model, exactActionMatches: complete ? exact : null,
toolNameMatches: complete ? names : null, cases: complete ? 100 : null };
});
const report = { date: fresh.date, sourceCommit: replay.meta.sourceCommit, browser: 'chrome', tier: 'full',
casesPerModel: 100, models: 14, historicalModels: 13, freshModels: 1,
freshRequests: benchmarkRequests, campaign: campaignMetrics,
status: complete ? 'complete' : 'inference-incomplete', consensusModels: complete ? 14 : 13,
freshResponses: ling.row.responses, freshErrors: ling.row.errors,
replayPath: repoPath(replayPath), replaySha256: hash(readFileSync(replayPath)), toolCounts: { act: 48 },
method: complete
? 'Leave-one-out exact action (recursive sorted keys only; all values preserved) and tool-name agreement. No-tool matches no-tool; errors never match. Each model has 13 peers and 1300 comparisons. Original 13-model scores reproduced before adding Ling.'
: 'Ling lacks 100 inference responses and is excluded from consensus. Its quality, inference latency, and cost metrics are N/A. The original 13-model pool and 1200 comparisons per historical model are retained unchanged; all historical scores reproduced from raw cases.',
caveat: 'Historical bodies were omitted; Ling inputs are reconstructed from the pinned source. Provider conditions and test dates differ. Consensus is not ground truth and this is first-action-only text input.',
results, textOnlyComparison: results.filter(row => selected.has(row.model)),
additionalReference: results.find(row => row.model === 'minimax/minimax-m3'),
matchedSubset, lingPairwise: pairs };
writeFileSync(join(out, 'comparison.json'), JSON.stringify(report, null, 2) + '\n');
writeFileSync(join(out, 'audit.json'), JSON.stringify({ replaySha256: report.replaySha256, cases: audit }, null, 2) + '\n');
const pct = value => value == null ? 'N/A' : value.toFixed(1) + '%';
const seconds = value => value == null ? 'N/A' : (value / 1000).toFixed(2) + 's';
const cost = value => value == null ? 'N/A' : '$' + value.toFixed(3);
const table = rows => '| Model | Exact consensus | Tool consensus | Valid / emitted | Ideal tool | Exact ideal | Median | p95 | Cost |\n'
+ '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n'
+ rows.map(row => `| ${row.label} | ${pct(row.exactActionConsensusPct)} | ${pct(row.toolNameConsensusPct)} | ${row.schemaValidToolCalls} / ${row.emittedToolCalls} | ${row.idealToolMatches ?? 'N/A'} | ${row.exactIdealMatches ?? 'N/A'} | ${seconds(row.medianLatencyMs)} | ${seconds(row.p95LatencyMs)} | ${cost(row.usageCost)} |`).join('\n');
writeFileSync(join(out, 'report.md'), `# Ling 3.1 Flash: pinned full-tier comparison\n\n${report.method}\n\n${report.caveat}\n\n${table(report.textOnlyComparison)}\n\nFull cohort (${report.consensusModels} scored models):\n\n${table(results)}\n`
+ (matchedSubset ? `\n## Matched available cases (${matchedSubset.cases})\n\n${matchedSubset.note}\n\n${table(matchedSubset.results.filter(row => selected.has(row.model) || row.model === 'minimax/minimax-m3'))}\n` : ''));
console.log(JSON.stringify(report.textOnlyComparison, null, 2));
}