Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

14.6 KB
Raw Blame
#!/usr/bin/env node
// Fact-check accuracy benchmark runner.
//
// Runs tests/fixtures/fact-check-benchmark.json through the REAL fact-check chain
// (POST /api/x402/fact-check) and scores accuracy overall, per verdict class, and
// per difficulty. Writes data/_generated/fact-check-benchmark.json so the public
// /fact-check accuracy page can render real, checkable numbers.
//
// The endpoint is paid ($0.10/claim), so the runner needs an access path. Two
// exist, both handled by api/_lib/x402/access-control.js:
//   • INTERNAL_API_KEY: the internal service key, sent as `X-API-Key`. This is
//     the path the published runs use; the same value is set on the three-ws-api
//     Cloud Run service. See docs/fact-check.md for rotation.
//   • FACT_CHECK_BYPASS_TOKEN: an OAuth bearer carrying the `x402:bypass`
//     scope, sent as `Authorization: Bearer`. Use it when you want a scoped,
//     per-user credential instead of the service key.
// With neither, the free lane (3 checks/day per IP) covers only the first few
// claims and the rest 402: so the runner EXITS with a clear message naming what
// is missing and writes NOTHING. It never fabricates scores.
//
// Usage:
//   node --env-file=.env scripts/fact-check-benchmark.mjs
//   FACT_CHECK_BYPASS_TOKEN=… node scripts/fact-check-benchmark.mjs
//   FACT_CHECK_ENDPOINT=https://three.ws/api/x402/fact-check node scripts/fact-check-benchmark.mjs
//
// In-process mode (how the published numbers are produced): imports
// api/x402/fact-check.js#_checkClaim directly and runs the REAL chain (live
// search + live LLM) with no HTTP or payment layer in between. The Redis cache
// is disabled for the run so every claim exercises the live chain instead of a
// stale cached verdict:
//   node --env-file=.env scripts/fact-check-benchmark.mjs --in-process
//
// In-process runs are paced (FACT_CHECK_BENCH_SPACING_MS, default 6000) and
// retry a failed claim once, because 80 unspaced LLM turns trip the free lanes'
// per-minute limits and score the runner's own throttling as errored claims.
//
// Publishing (`--publish`) additionally writes the run to the DB, which is what
// the live /api/fact-check-benchmark endpoint reads first, so a re-run reaches
// the public page without a deploy. Without the flag the run only updates the
// committed file, which takes effect on the next deploy.
//
// The scoring core (scoreResults / validateFixture) is pure, lives in
// api/_lib/fact-check-benchmark.js so the endpoint and the scheduled re-run share
// it, and is unit-tested in tests/api/fact-check-benchmark.test.js.

import { readFile, writeFile, mkdir } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

import {
	MAX_ERROR_RATE,
	buildReport,
	degradationOf,
	degradedReason,
	isDegraded,
	savePublishedRun,
	scoreResults,
	validateFixture,
} from '../api/_lib/fact-check-benchmark.js';

const HERE = dirname(fileURLToPath(import.meta.url));
const REPO = join(HERE, '..');
const FIXTURE = join(REPO, 'tests/fixtures/fact-check-benchmark.json');
const OUT_DIR = join(REPO, 'data/_generated');
const OUT_FILE = join(OUT_DIR, 'fact-check-benchmark.json');

// Re-exported for anything still importing the pure core from the script path.
export { scoreResults, validateFixture };

function refuseIfDegraded(score) {
	if (!isDegraded(score)) return;
	console.error(
		`\nRefusing to publish: ${degradedReason(score)}\n` +
			`(ceiling: ${Math.round(MAX_ERROR_RATE * 100)}% errored claims.)\n` +
			'Fix the chain (the "[llm] chain exhausted" warn line names every rung that ' +
			'failed) and re-run.\n' +
			'Nothing was written — the accuracy page keeps rendering its honest state.',
	);
	process.exit(1);
}

// Write the scored run: always the committed file (the deploy-time fallback and
// the seed for a fresh environment), plus the DB row when --publish is passed so
// the live page picks it up immediately.
async function writeReport(report, { publish }) {
	await mkdir(OUT_DIR, { recursive: true });
	await writeFile(OUT_FILE, JSON.stringify(report, null, 2) + '\n');
	console.log(`Wrote ${OUT_FILE}`);
	if (!publish) {
		console.log('Not published to the DB (pass --publish to update the live page now).');
		return;
	}
	await savePublishedRun(report);
	console.log('Published to the DB: GET /api/fact-check-benchmark serves this run now.');
}

// ── Live chain call ──────────────────────────────────────────────────────────

// Build the bypass headers once. Both paths are checked by
// api/_lib/x402/access-control.js; sending both is harmless (the service key is
// tried first) and lets a caller hold either credential.
function bypassHeaders({ apiKey, bearer }) {
	const headers = {};
	if (apiKey) headers['x-api-key'] = apiKey;
	if (bearer) headers.authorization = `Bearer ${bearer}`;
	return headers;
}

async function checkOne(endpoint, access, claim) {
	const headers = { 'content-type': 'application/json', ...bypassHeaders(access) };
	const res = await fetch(endpoint, {
		method: 'POST',
		headers,
		// 'medium' matches parseFactCheckBody's own default in api/x402/fact-check.js —
		// spelled out explicitly rather than relying on an unrecognized value falling
		// through to it (the endpoint only accepts high|medium|low; anything else,
		// including the previous 'normal' here, silently resolves to 'medium' anyway).
		body: JSON.stringify({ claim, strictness: 'medium' }),
		signal: AbortSignal.timeout(60_000),
	});
	if (res.status === 402) {
		const e = new Error('payment_required'); e.paymentRequired = true; throw e;
	}
	if (!res.ok) {
		const e = new Error(`chain returned ${res.status}`); e.status = res.status; throw e;
	}
	const data = await res.json();
	const payload = data?.result ?? data;
	const degradation = degradationOf(payload);
	if (degradation) {
		const e = new Error(`degraded check: ${degradation}`); e.degraded = true; throw e;
	}
	return payload?.verdict ?? null;
}

// One live check with a bounded retry. A single 502 or edge timeout on a 40-claim
// run costs 2.5 percentage points of "accuracy" that is really a transport blip,
// and four of them trip the degraded-run refusal, so a transient failure is
// retried once rather than being scored as a wrong verdict. A 402 is NOT
// transient (no credential will appear between attempts) and fails immediately.
async function checkOneWithRetry(endpoint, access, claim) {
	try {
		return await checkOne(endpoint, access, claim);
	} catch (err) {
		if (err.paymentRequired || err.status === 400 || err.status === 403) throw err;
		await new Promise((r) => setTimeout(r, 3_000));
		return checkOne(endpoint, access, claim);
	}
}

async function main() {
	const raw = await readFile(FIXTURE, 'utf8');
	const fixture = JSON.parse(raw);
	const claims = validateFixture(fixture);
	console.log(`Loaded ${claims.length} benchmark claims (validated).`);

	const publish = process.argv.includes('--publish');
	const inProcess =
		process.argv.includes('--in-process') || process.env.FACT_CHECK_INPROCESS === '1';
	if (inProcess) return mainInProcess(claims, fixture, { publish });

	const endpoint = process.env.FACT_CHECK_ENDPOINT || 'https://three.ws/api/x402/fact-check';
	const access = {
		apiKey: process.env.FACT_CHECK_API_KEY || process.env.INTERNAL_API_KEY || '',
		bearer: process.env.FACT_CHECK_BYPASS_TOKEN || '',
	};
	const hasBypass = Boolean(access.apiKey || access.bearer);

	// Probe reachability before spending a full run. A 402 without any bypass
	// credential means the run can't proceed without payment, exit clearly,
	// write nothing.
	if (!hasBypass) {
		try {
			await checkOne(endpoint, access, claims[0].claim);
		} catch (err) {
			if (err.paymentRequired) {
				console.error(
					'\nCannot run the benchmark: the fact-check endpoint requires payment and no ' +
						'bypass credential was provided.\n' +
						'Set INTERNAL_API_KEY (the service key, sent as X-API-Key) or ' +
						'FACT_CHECK_BYPASS_TOKEN (an x402:bypass-scoped bearer), optionally with ' +
						'FACT_CHECK_ENDPOINT, and re-run.\n' +
						'No scores were written — the accuracy page will render its honest "not yet run" state.',
				);
				process.exit(1);
			}
			console.error(`\nCannot reach the fact-check chain at ${endpoint}: ${err.message}`);
			process.exit(1);
		}
	}

	console.log(
		`Running ${claims.length} claims through ${endpoint} ` +
			`(bypass: ${access.apiKey ? 'X-API-Key' : access.bearer ? 'Bearer x402:bypass' : 'none, free lane only'}) …`,
	);
	const results = [];
	for (const [i, c] of claims.entries()) {
		let actual = null;
		try {
			actual = await checkOneWithRetry(endpoint, access, c.claim);
		} catch (err) {
			console.warn(`  [${i + 1}/${claims.length}] error: ${err.message}`);
		}
		const ok = actual === c.expected_verdict;
		console.log(`  [${i + 1}/${claims.length}] ${ok ? 'PASS' : 'MISS'} expected=${c.expected_verdict} actual=${actual ?? 'ERR'} :: ${c.claim.slice(0, 60)}`);
		results.push({ claim: c.claim, expected_verdict: c.expected_verdict, difficulty: c.difficulty, actual_verdict: actual });
	}

	const score = scoreResults(results);
	refuseIfDegraded(score);
	const report = buildReport({ score, endpoint, fixture, claimCount: claims.length });
	console.log(`\nOverall accuracy: ${score.accuracy_pct}%  (${score.correct}/${score.total}, ${score.errors} errors)`);
	await writeReport(report, { publish });
}

// ── In-process mode ──────────────────────────────────────────────────────────
// Runs the real chain by importing the endpoint's exported _checkClaim — the
// same code path a paying caller hits, minus the HTTP/x402 wrapper. Used to
// produce the published data/_generated numbers so the benchmark measures the
// product's verdict quality, not payment plumbing.

// Seconds to wait between claims. A 40-claim run is 80 LLM turns (query
// generation + stance extraction per claim); fired back to back they arrive
// far inside the free lanes' per-minute allowances, and the chain answers the
// first few claims and then 429s on every rung at once. That is the benchmark
// throttling itself, and it scores as an errored claim, so a fast run walks
// straight into the degraded-run refusal while the chain is perfectly healthy
// (observed 2026-09-09: 7/40 errors, every one a 429 sweep across all three
// groq rungs and all five openrouter keys, minutes after the same chain
// answered a probe in 225ms). Pace the run instead. Tunable for a machine with
// paid lanes that does not need the wait.
function claimSpacingMs() {
	const raw = Number(process.env.FACT_CHECK_BENCH_SPACING_MS);
	return Number.isFinite(raw) && raw >= 0 ? raw : 6_000;
}

// One in-process check with the same bounded retry the remote path has had all
// along (see checkOneWithRetry): a lane that 429s or stalls on one attempt is a
// transient failure, not a wrong verdict, and four of them trip the degraded
// refusal. The backoff is longer than the remote path's because the failure
// being ridden out here is a per-minute rate-limit window, not an edge blip.
async function checkClaimWithRetry(checkClaim, claim) {
	try {
		return await checkClaim(claim);
	} catch (err) {
		await new Promise((r) => setTimeout(r, 15_000));
		return checkClaim(claim);
	}
}

async function mainInProcess(claims, fixture, { publish } = {}) {
	// Disable the shared Redis verdict cache for the run: a benchmark that reads
	// week-old cached verdicts measures the cache, not the chain.
	delete process.env.UPSTASH_REDIS_REST_URL;
	delete process.env.UPSTASH_REDIS_REST_TOKEN;
	delete process.env.three_KV_REST_API_URL;
	delete process.env.three_KV_REST_API_TOKEN;
	delete process.env.KV_REST_API_URL;
	delete process.env.KV_REST_API_TOKEN;

	const { _checkClaim } = await import('../api/x402/fact-check.js');
	const endpoint = 'in-process:api/x402/fact-check.js#_checkClaim (real chain, no HTTP/x402 payment layer)';
	console.log(`Running ${claims.length} claims in-process (live search + live LLM, cache disabled) …`);

	// One claim through the real chain, raising on a degraded result so the retry
	// above treats a half-answered check the same as an outright failure.
	const checkOnce = async (claim) => {
		const r = await _checkClaim(claim, 'medium', null);
		const degradation = degradationOf(r);
		if (degradation) throw new Error(`degraded check: ${degradation}`);
		return r;
	};

	const spacingMs = claimSpacingMs();
	if (spacingMs > 0) console.log(`Pacing ${spacingMs}ms between claims to stay inside the free lanes' rate limits.`);

	const results = [];
	const details = [];
	for (const [i, c] of claims.entries()) {
		if (i > 0 && spacingMs > 0) await new Promise((r) => setTimeout(r, spacingMs));
		let actual = null;
		let detail = null;
		try {
			const r = await checkClaimWithRetry(checkOnce, c.claim);
			actual = r?.verdict ?? null;
			detail = {
				confidence: r?.confidence ?? null,
				sources: (r?.sources || []).map((s) => ({ url: s.url, stance: s.stance, weight: s.weight })),
			};
		} catch (err) {
			console.warn(`  [${i + 1}/${claims.length}] error: ${err.message}`);
		}
		const ok = actual === c.expected_verdict;
		console.log(`  [${i + 1}/${claims.length}] ${ok ? 'PASS' : 'MISS'} expected=${c.expected_verdict} actual=${actual ?? 'ERR'} :: ${c.claim.slice(0, 60)}`);
		results.push({ claim: c.claim, expected_verdict: c.expected_verdict, difficulty: c.difficulty, actual_verdict: actual });
		details.push({ claim: c.claim, expected: c.expected_verdict, actual, ...detail });
	}

	const score = scoreResults(results);
	// Per-claim detail for diagnosis (not published; scratch aid for whoever is
	// tuning the chain). Written next to nothing public, and written BEFORE the
	// degradation guard: a refused run is exactly the run someone needs the
	// per-source stances for, and refuseIfDegraded exits the process, so writing
	// this after it threw away the evidence for every diagnosis it demanded.
	if (process.env.FACT_CHECK_DETAIL_FILE) {
		await mkdir(dirname(process.env.FACT_CHECK_DETAIL_FILE), { recursive: true });
		await writeFile(process.env.FACT_CHECK_DETAIL_FILE, JSON.stringify(details, null, 2) + '\n');
	}
	refuseIfDegraded(score);
	const report = buildReport({ score, endpoint, fixture, claimCount: claims.length });
	console.log(`\nOverall accuracy: ${score.accuracy_pct}%  (${score.correct}/${score.total}, ${score.errors} errors)`);
	await writeReport(report, { publish });
}

// Only run main() when invoked directly, not when imported by the test.
if (process.argv[1] && process.argv[1].endsWith('fact-check-benchmark.mjs')) {
	main().catch((err) => {
		console.error('benchmark failed:', err);
		process.exit(1);
	});
}