Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
#!/usr/bin/env node
/**
* Relevance benchmark for offline retrieval.
*
* The sibling benchmark-offline-rag.mjs measures how fast the index builds and
* answers; this one measures whether the right passage comes back at all. It
* runs the shipped query builder and the shipped bm25 search SQL against the
* fixture corpus, so a change to either shows up here before it reaches anyone
* relying on the archive with no network to check against.
*
* Usage: node scripts/benchmark-offline-relevance.mjs [--json] [--verbose]
* Exits non-zero when a metric falls below the floor recorded in FLOORS.
*/
import { readFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import path from 'node:path';
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const asJson = process.argv.includes('--json');
const verbose = process.argv.includes('--verbose');
const sqlite3InitModule = (await import(new URL('../src/chrome/vendor/sqlite/index.mjs', import.meta.url))).default;
const indexRuntime = await import(new URL('../src/chrome/src/agent/offline-rag-index.js', import.meta.url));
const ragRuntime = await import(new URL('../src/chrome/src/agent/offline-rag.js', import.meta.url));
const { RELAXED_RETRY_THRESHOLD } = await import(new URL('../src/chrome/src/agent/offline-retrieval.js', import.meta.url));
const { MAX_LEXICAL_CANDIDATES_PER_SOURCE } = await import(new URL('../src/chrome/src/agent/offline-rag.js', import.meta.url));
const { RELEVANCE_CORPUS, RELEVANCE_QUERIES } = await import(
new URL('../test/fixtures/offline-relevance-corpus.mjs', import.meta.url)
);
// Regression floors for the harder ~250-passage fixture. Measured on this
// corpus after age-cohort ranking: recall@1 55.4%, recall@5 87.5%, MRR 0.685
// (keyword 57.1/96.4/0.738, question 67.9/96.4/0.792, inflection 39.3/78.6/0.536,
// typo 35.7/82.1/0.539, fragment 57.1/85.7/0.708, non-english 75.0/85.7/0.798).
// The 22-passage fixture was recall@1 93.8%, recall@5 100%, MRR 0.964; that drop
// is a harder fixture, not worse retrieval. Do not raise floors back to the
// 22-passage numbers without shrinking the corpus.
const FLOORS = { 'recall@1': 0.53, 'recall@5': 0.85, mrr: 0.66 };
const corpusIds = new Set(RELEVANCE_CORPUS.map(doc => doc.id));
if (corpusIds.size !== RELEVANCE_CORPUS.length) {
throw new Error('offline-relevance-corpus: duplicate passage id');
}
for (const item of RELEVANCE_QUERIES) {
if (!corpusIds.has(item.expect)) {
throw new Error(`offline-relevance-corpus: query "${item.q}" expects missing id ${item.expect}`);
}
}
const previousLocation = globalThis.location;
globalThis.location = { href: 'https://offline.invalid/?opfs-disable&opfs-wl-disable' };
let sqlite3;
try {
sqlite3 = await sqlite3InitModule({
wasmBinary: await readFile(path.join(root, 'src/chrome/vendor/sqlite/sqlite3.wasm')),
print: () => {},
printErr: () => {},
});
} finally {
if (previousLocation === undefined) delete globalThis.location;
else globalThis.location = previousLocation;
}
sqlite3.config.log = () => {};
const db = new sqlite3.oo1.DB(':memory:', 'c');
db.exec(indexRuntime.EMERGENCY_FTS_SCHEMA_SQL);
const insert = db.prepare(indexRuntime.EMERGENCY_FTS_INSERT_SQL);
db.exec('BEGIN');
try {
for (const [ordinal, doc] of RELEVANCE_CORPUS.entries()) {
const searchTerms = ragRuntime.tokenizeForLexicalSearch(
`${doc.title}\n${doc.collection}\n${doc.locator}\n${doc.text}`,
{ language: doc.language },
).join(' ');
insert.bind([
`${doc.id}:0`, doc.id, 'relevance-fixture', doc.title, doc.language,
doc.collection, 'fixture', 'CC0', doc.locator, doc.text, searchTerms,
'a'.repeat(64), Math.ceil(doc.text.length / 4), ordinal,
`webbrain-reader://emergency-box/${doc.id}?passage=${encodeURIComponent(`${doc.id}:0`)}`,
]).stepReset().clearBindings();
}
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
} finally {
insert.finalize();
}
db.exec("INSERT INTO passages(passages) VALUES('optimize')");
function runMatch(match, limit) {
if (!match) return [];
const statement = db.prepare(indexRuntime.EMERGENCY_FTS_SEARCH_SQL);
const documents = [];
try {
statement.bind([match, limit]);
while (statement.step()) documents.push(statement.get({}).documentId);
} finally {
statement.finalize();
}
return documents;
}
function searchPass(match, queryText, limit) {
return indexRuntime.preferMatchingAgeCohort(
runMatch(match, MAX_LEXICAL_CANDIDATES_PER_SOURCE).map(id => {
const doc = RELEVANCE_CORPUS.find(item => item.id === id);
return {
documentId: id,
title: doc?.title || '',
text: doc?.text || '',
language: doc?.language || 'eng',
collection: doc?.collection || '',
sourceKind: doc?.sourceKind || 'emergency-box',
};
}),
queryText,
).map(hit => hit.documentId).slice(0, limit);
}
// Mirrors searchEmergencyLexical: exact first, relaxed only when exact is thin,
// exact results always ranked above relaxed ones. Age-cohort rerank runs inside
// each pass so a relaxed infant hit cannot leapfrog an exact adult hit.
function search(queryText, limit = 5) {
const exact = searchPass(indexRuntime.buildFts5Query(queryText), queryText, limit);
if (exact.length >= RELAXED_RETRY_THRESHOLD) return exact;
const relaxed = searchPass(indexRuntime.buildFts5Query(queryText, { relax: true }), queryText, limit);
const merged = [...exact];
for (const id of relaxed) if (!merged.includes(id)) merged.push(id);
return merged.slice(0, limit);
}
const results = RELEVANCE_QUERIES.map(item => {
const ranked = search(item.q);
const rank = ranked.indexOf(item.expect);
return { ...item, ranked, rank: rank < 0 ? null : rank + 1 };
});
function summarize(rows) {
const total = rows.length || 1;
return {
n: rows.length,
'recall@1': rows.filter(row => row.rank === 1).length / total,
'recall@5': rows.filter(row => row.rank !== null).length / total,
mrr: rows.reduce((sum, row) => sum + (row.rank ? 1 / row.rank : 0), 0) / total,
};
}
const overall = summarize(results);
const byKind = {};
for (const kind of [...new Set(RELEVANCE_QUERIES.map(item => item.kind))]) {
byKind[kind] = summarize(results.filter(row => row.kind === kind));
}
if (asJson) {
console.log(JSON.stringify({ overall, byKind, results }, null, 2));
} else {
const pct = value => `${(value * 100).toFixed(1)}%`;
console.log(`Offline retrieval relevance: ${RELEVANCE_CORPUS.length} passages, ${results.length} queries\n`);
console.log(`${'query class'.padEnd(14)} ${'n'.padStart(3)} ${'recall@1'.padStart(8)} ${'recall@5'.padStart(8)} ${'MRR'.padStart(6)}`);
for (const [kind, stats] of Object.entries(byKind)) {
console.log(`${kind.padEnd(14)} ${String(stats.n).padStart(3)} ${pct(stats['recall@1']).padStart(8)} ${pct(stats['recall@5']).padStart(8)} ${stats.mrr.toFixed(3).padStart(6)}`);
}
console.log(`${'OVERALL'.padEnd(14)} ${String(overall.n).padStart(3)} ${pct(overall['recall@1']).padStart(8)} ${pct(overall['recall@5']).padStart(8)} ${overall.mrr.toFixed(3).padStart(6)}`);
const missed = results.filter(row => row.rank === null);
if (missed.length) {
console.log(`\nnot found in top 5 (${missed.length}):`);
for (const row of missed) console.log(` [${row.kind}] "${row.q}" -> wanted ${row.expect}, got ${row.ranked.slice(0, 3).join(', ') || '(nothing)'}`);
}
if (verbose) {
const demoted = results.filter(row => row.rank !== null && row.rank > 1);
if (demoted.length) {
console.log(`\nfound but not first (${demoted.length}):`);
for (const row of demoted) console.log(` [${row.kind}] "${row.q}" -> ${row.expect} at rank ${row.rank} (top: ${row.ranked[0]})`);
}
}
}
db.close();
const failures = Object.entries(FLOORS).filter(([metric, floor]) => overall[metric] < floor);
if (failures.length) {
for (const [metric, floor] of failures) {
console.error(`\nFAIL ${metric} ${(overall[metric] * 100).toFixed(1)}% is below the ${(floor * 100).toFixed(1)}% floor.`);
}
process.exit(1);
}