Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(server): read Quality's checks against AI citations, for the maintainer to calibrate the weights (CE-055, part a) (#1502)

Cihan1057dac86053
diff --git a/server/src/lib/quality-calibration.js b/server/src/lib/quality-calibration.js
new file mode 100644
index 000000000..c78ef3927
--- /dev/null
+++ b/server/src/lib/quality-calibration.js
@@ -0,0 +1,282 @@
+import { z } from 'zod';
+
+/**
+ * Quality's weights against AI citations (Content Editor Quality spec §11,
+ * Phase 3, CE-055): "check whether pages with higher scores get more AI
+ * citations … Adjust the weights from the result. Until then, the
+ * 'Ansvisor estimate' label stays."
+ *
+ * It reads rows a maintainer exports, one page each: its type's profile,
+ * each check's share of its points (0–1) by score, the AI citations the
+ * page got over the same window, and optionally a group (a salted hash of
+ * its brand: correlations are read within each, so a big brand's pages
+ * don't stand for a good score). It gives back aggregates only:
+ * - per profile, each score's rank correlation (Spearman's rho) with the
+ * citations, the page's score worked out as the web's weighed() does,
+ * with the weights in use and with the suggested ones (in-sample);
+ * - per check, the pages that had it, its rho and its 95% interval, and
+ * whether the interval leaves 0 out (only such a check is read);
+ * - suggested weights: half the weights in use, half the read checks'
+ * points shared by their positive rho; never 0 for a check weighed
+ * today (dropping one is the maintainer's call), each score's total kept.
+ * Nothing here reads a database or the network, and nothing of a page (its
+ * URL, its brand, its group) is in what it returns: a row's other fields
+ * are dropped.
+ */
+
+export const QUALITY_PROFILES = ['article', 'product', 'category', 'feature', 'comparison'];
+export const QUALITY_SCORES = ['eeat', 'trust', 'readability'];
+
+/** The fewest pages a check's correlation is read from: fewer say nothing. */
+export const MIN_PAGES = 30;
+/** How much of a suggestion the correlations make: the rest is the weights in use. */
+export const BLEND = 0.5;
+/** The two-sided 95% normal quantile. */
+const Z95 = 1.96;
+
+const checkName = z.string().regex(/^[a-z][A-Za-z]{0,39}$/);
+const shares = z.record(checkName, z.number().min(0).max(1));
+
+/** One page's row: its profile, its checks' shares by score, its AI citations, its group. */
+export const calibrationRow = z.object({
+ profile: z.enum(QUALITY_PROFILES),
+ checks: z.object({
+ eeat: shares.optional(),
+ trust: shares.optional(),
+ readability: shares.optional(),
+ }),
+ // Only their ranks count: per day or per prompt is as good as a count.
+ citations: z.number().min(0),
+ group: z.string().min(1).max(128).optional(),
+});
+
+/** The weights in use, as PROFILE_WEIGHTS gives them: profile → score → check → points. */
+export const calibrationWeights = z.partialRecord(
+ z.enum(QUALITY_PROFILES),
+ z.object({
+ eeat: z.record(checkName, z.number().int().min(0)),
+ trust: z.record(checkName, z.number().int().min(0)),
+ readability: z.record(checkName, z.number().int().min(0)),
+ }),
+);
+
+/** A text without the byte-order mark some editors start a file with. */
+export const withoutBom = (text) => String(text).replace(/^\uFEFF/, '');
+
+/**
+ * The rows of a JSON-lines text: each line a page's row; a blank line is
+ * skipped, and a line that isn't one is counted, never kept or printed.
+ */
+export function parseRows(text) {
+ const rows = [];
+ let invalid = 0;
+ for (const line of withoutBom(text).split(/\r?\n/)) {
+ if (!line.trim()) continue;
+ let value;
+ try {
+ value = JSON.parse(line);
+ } catch {
+ invalid += 1;
+ continue;
+ }
+ const row = calibrationRow.safeParse(value);
+ if (row.success) rows.push(row.data);
+ else invalid += 1;
+ }
+ return { rows, invalid };
+}
+
+/** Each value's rank, 1 for the smallest, ties sharing their average rank. */
+export function ranks(values) {
+ const order = values.map((value, i) => ({ value, i })).sort((a, b) => a.value - b.value);
+ const out = new Array(values.length);
+ for (let start = 0; start < order.length; ) {
+ let end = start;
+ while (end + 1 < order.length && order[end + 1].value === order[start].value) end += 1;
+ const rank = (start + end) / 2 + 1;
+ for (let k = start; k <= end; k += 1) out[order[k].i] = rank;
+ start = end + 1;
+ }
+ return out;
+}
+
+/**
+ * Spearman's rank correlation of two equal-length lists: -1 to 1, rounded
+ * to three places; null below three pairs, or when either list is all one
+ * value (it says nothing about the other).
+ */
+export function spearman(xs, ys) {
+ if (xs.length !== ys.length || xs.length < 3) return null;
+ const rx = ranks(xs);
+ const ry = ranks(ys);
+ const mean = (rx.length + 1) / 2;
+ let cov = 0;
+ let vx = 0;
+ let vy = 0;
+ for (let i = 0; i < rx.length; i += 1) {
+ cov += (rx[i] - mean) * (ry[i] - mean);
+ vx += (rx[i] - mean) ** 2;
+ vy += (ry[i] - mean) ** 2;
+ }
+ if (vx === 0 || vy === 0) return null;
+ return Math.round((cov / Math.sqrt(vx * vy)) * 1000) / 1000;
+}
+
+/**
+ * A correlation's 95% interval from `n` pairs (Fisher's z), and whether it
+ * leaves 0 out: only then does it say more than noise. Null below four pairs.
+ */
+export function interval(rho, n) {
+ if (rho === null || n < 4) return null;
+ const z = Math.atanh(Math.max(-0.999999, Math.min(0.999999, rho)));
+ const half = Z95 / Math.sqrt(n - 3);
+ const round = (v) => Math.round(Math.tanh(v) * 1000) / 1000;
+ const low = round(z - half);
+ const high = round(z + half);
+ return { low, high, read: low > 0 || high < 0 };
+}
+
+/**
+ * A correlation between a value and the citations over pages, read within
+ * each group where pages have one (each brand's own correlation, weighed by
+ * its pages: a big brand's pages, cited more whatever their checks, aren't
+ * a good score), else over them all; null where nothing can be read.
+ * Returns the pages it was read from too, for the interval.
+ */
+export function groupedSpearman(pairs) {
+ if (!pairs.some((p) => p.group !== undefined)) {
+ return {
+ rho: spearman(
+ pairs.map((p) => p.x),
+ pairs.map((p) => p.y),
+ ),
+ pages: pairs.length,
+ };
+ }
+ const groups = new Map();
+ for (const p of pairs) {
+ const key = p.group ?? '';
+ if (!groups.has(key)) groups.set(key, []);
+ groups.get(key).push(p);
+ }
+ let sum = 0;
+ let pages = 0;
+ for (const members of groups.values()) {
+ const rho = spearman(
+ members.map((p) => p.x),
+ members.map((p) => p.y),
+ );
+ if (rho === null) continue;
+ sum += rho * members.length;
+ pages += members.length;
+ }
+ return { rho: pages ? Math.round((sum / pages) * 1000) / 1000 : null, pages };
+}
+
+/** A page's score as the web's weighed() gives it: its checks' shares by their points, 0–100; null with none. */
+export function scoreOf(checks, weights) {
+ let points = 0;
+ let max = 0;
+ for (const [check, share] of Object.entries(checks ?? {})) {
+ const weight = weights[check] ?? 0;
+ if (weight <= 0) continue;
+ points += share * weight;
+ max += weight;
+ }
+ return max > 0 ? Math.round((points / max) * 100) : null;
+}
+
+/**
+ * Integer points that add up to `total`, as near as can be to the values
+ * given (the largest remainders round up), none under its floor.
+ */
+function toPoints(values, total, floors = {}) {
+ const keys = Object.keys(values);
+ const out = Object.fromEntries(
+ keys.map((k) => [k, Math.max(floors[k] ?? 0, Math.floor(values[k]))]),
+ );
+ let left = total - keys.reduce((s, k) => s + out[k], 0);
+ const order = [...keys].sort((a, b) => (values[b] % 1) - (values[a] % 1));
+ for (const k of order) {
+ if (left <= 0) break;
+ out[k] += 1;
+ left -= 1;
+ }
+ // Floors that took more than their share give it back from the largest.
+ for (const k of [...keys].sort((a, b) => out[b] - out[a])) {
+ while (left < 0 && out[k] > (floors[k] ?? 0)) {
+ out[k] -= 1;
+ left += 1;
+ }
+ }
+ return out;
+}
+
+/**
+ * A score's weights as its checks' correlations suggest: the read checks'
+ * points (a check read from enough pages, its interval leaving 0 out),
+ * half kept as they are, half shared by positive rho; a check not read
+ * keeps its points, and none weighed today goes under 1. The score's total
+ * stays as it was. With no positive read check, the weights stay as they are.
+ */
+export function suggestWeights(current, stats, blend = BLEND) {
+ const read = Object.keys(current).filter(
+ (check) => current[check] > 0 && stats[check]?.interval?.read === true,
+ );
+ const positive = Object.fromEntries(read.map((c) => [c, Math.max(0, stats[c].rho)]));
+ const sum = Object.values(positive).reduce((a, b) => a + b, 0);
+ if (!read.length || sum === 0) return { ...current };
+ const budget = read.reduce((total, c) => total + current[c], 0);
+ const values = Object.fromEntries(
+ read.map((c) => [c, (1 - blend) * current[c] + blend * (positive[c] / sum) * budget]),
+ );
+ const floors = Object.fromEntries(read.map((c) => [c, 1]));
+ return { ...current, ...toPoints(values, budget, floors) };
+}
+
+/**
+ * The calibration: per profile, its pages; per score, its correlation with
+ * the citations (with the weights in use and the suggested ones) and per
+ * check, the pages with it, its correlation and interval (none when fewer
+ * than `minPages` pages had it); with the weights in use, the ones suggested.
+ */
+export function calibrate(rows, { minPages = MIN_PAGES, weights = null } = {}) {
+ const profiles = {};
+ for (const profile of QUALITY_PROFILES) {
+ const pages = rows.filter((row) => row.profile === profile);
+ if (!pages.length) continue;
+ const scores = {};
+ for (const score of QUALITY_SCORES) {
+ const checks = new Set(pages.flatMap((row) => Object.keys(row.checks[score] ?? {})));
+ const stats = {};
+ for (const check of [...checks].sort()) {
+ const pairs = pages.flatMap((row) =>
+ row.checks[score]?.[check] !== undefined
+ ? [{ x: row.checks[score][check], y: row.citations, group: row.group }]
+ : [],
+ );
+ const read = pairs.length >= minPages ? groupedSpearman(pairs) : null;
+ const rho = read ? read.rho : null;
+ stats[check] = { pages: pairs.length, rho, interval: interval(rho, read?.pages ?? 0) };
+ }
+ const current = weights?.[profile]?.[score] ?? null;
+ const suggested = current ? suggestWeights(current, stats) : null;
+ // The score itself against the citations: the spec's question.
+ const scored = (w) => {
+ const pairs = pages.flatMap((row) => {
+ const value = scoreOf(row.checks[score], w);
+ return value === null ? [] : [{ x: value, y: row.citations, group: row.group }];
+ });
+ return pairs.length >= minPages ? groupedSpearman(pairs).rho : null;
+ };
+ scores[score] = {
+ checks: stats,
+ ...(current
+ ? { rho: scored(current), suggested, rhoSuggestedInSample: scored(suggested) }
+ : {}),
+ };
+ }
+ profiles[profile] = { pages: pages.length, scores };
+ }
+ return { pages: rows.length, minPages, profiles };
+}
diff --git a/server/src/lib/quality-calibration.test.js b/server/src/lib/quality-calibration.test.js
new file mode 100644
index 000000000..0e36e64a7
--- /dev/null
+++ b/server/src/lib/quality-calibration.test.js
@@ -0,0 +1,231 @@
+import { describe, expect, it } from 'vitest';
+import {
+ calibrate,
+ calibrationWeights,
+ interval,
+ parseRows,
+ ranks,
+ scoreOf,
+ spearman,
+ suggestWeights,
+} from './quality-calibration.js';
+
+/** Quality's weights against AI citations (Quality spec §11, CE-055). */
+
+/** A seeded random number generator: the same "random" pages every run. */
+function seeded(seed) {
+ let s = seed;
+ return () => {
+ s = (s * 1664525 + 1013904223) % 4294967296;
+ return s / 4294967296;
+ };
+}
+
+describe('ranks, spearman and interval', () => {
+ it('ranks ties by their average, and correlates by rank', () => {
+ expect(ranks([10, 20, 20, 5])).toEqual([2, 3.5, 3.5, 1]);
+ expect(spearman([1, 2, 3, 4], [10, 20, 30, 400])).toBe(1);
+ expect(spearman([1, 2, 3, 4], [4, 3, 2, 1])).toBe(-1);
+ // Too few pairs, or a list of one value: nothing to say.
+ expect(spearman([1, 2], [1, 2])).toBeNull();
+ expect(spearman([1, 1, 1], [1, 2, 3])).toBeNull();
+ expect(spearman([1, 2, 3], [1, 2])).toBeNull();
+ });
+
+ it('reads a correlation only where its 95% interval leaves 0 out', () => {
+ // 30 pages: about 0.36 is where noise ends.
+ expect(interval(0.156, 30)).toMatchObject({ read: false });
+ expect(interval(0.5, 30)).toMatchObject({ read: true });
+ expect(interval(-0.5, 30)).toMatchObject({ read: true });
+ const { low, high } = interval(0.5, 30);
+ expect(low).toBeLessThan(0.5);
+ expect(high).toBeGreaterThan(0.5);
+ expect(interval(null, 30)).toBeNull();
+ expect(interval(0.9, 3)).toBeNull();
+ });
+});
+
+describe('parseRows', () => {
+ it('keeps the rows as their profile, checks, citations and group, and counts the lines that aren’t rows', () => {
+ const text = [
+ // A byte-order mark before the first row doesn't cost it.
+ `\uFEFF${JSON.stringify({
+ url: 'https://acme.com/blog/x',
+ brand: 'Acme',
+ profile: 'article',
+ checks: { trust: { claimsSourced: 0.5 } },
+ citations: 3,
+ group: 'g1',
+ })}`,
+ '',
+ 'not json',
+ JSON.stringify({ profile: 'blog', checks: {}, citations: 1 }),
+ JSON.stringify({ profile: 'article', checks: { eeat: { byline: 2 } }, citations: 1 }),
+ // A check named like a URL isn't one.
+ JSON.stringify({ profile: 'article', checks: { eeat: { 'acme.com/x': 1 } }, citations: 1 }),
+ // Citations per day are as good as counts: only their ranks count.
+ JSON.stringify({ profile: 'article', checks: {}, citations: 0.25 }),
+ ].join('\r\n');
+ const { rows, invalid } = parseRows(text);
+ expect(rows).toEqual([
+ { profile: 'article', checks: { trust: { claimsSourced: 0.5 } }, citations: 3, group: 'g1' },
+ { profile: 'article', checks: {}, citations: 0.25 },
+ ]);
+ expect(invalid).toBe(4);
+ });
+});
+
+describe('calibrate', () => {
+ // Pages whose sourced claims go with their citations, and whose links don't.
+ const pages = Array.from({ length: 40 }, (_, i) => ({
+ profile: 'article',
+ checks: {
+ trust: { claimsSourced: i / 39, noBrokenLinks: 1 - i / 39, expertQuote: 1 },
+ eeat: { byline: i < 20 ? 0 : 1 },
+ },
+ citations: i,
+ }));
+ const weights = {
+ article: {
+ trust: { claimsSourced: 30, noBrokenLinks: 10, expertQuote: 8, brandFacts: 12 },
+ eeat: { byline: 50, authorBio: 50 },
+ readability: {},
+ },
+ };
+
+ it('gives each check’s correlation and interval, from enough pages only, and nothing of a page', () => {
+ const text = pages
+ .map((page, i) =>
+ JSON.stringify({ ...page, url: `https://acme.com/p/${i}`, brand: 'Acme', group: 'acme' }),
+ )
+ .join('\n');
+ const report = calibrate(parseRows(text).rows);
+ expect(report.pages).toBe(40);
+ const trust = report.profiles.article.scores.trust.checks;
+ expect(trust.claimsSourced).toMatchObject({ pages: 40, rho: 1, interval: { read: true } });
+ expect(trust.noBrokenLinks).toMatchObject({ rho: -1, interval: { read: true } });
+ // One value on every page: no correlation.
+ expect(trust.expertQuote.rho).toBeNull();
+ // Fewer pages than the minimum: none read.
+ expect(
+ calibrate(pages, { minPages: 50 }).profiles.article.scores.trust.checks.claimsSourced,
+ ).toEqual({ pages: 40, rho: null, interval: null });
+ // No page, brand or group in what it gives back.
+ expect(JSON.stringify(report)).not.toMatch(/acme|https?:/i);
+ });
+
+ it('answers the spec’s question: how each score goes with the citations', () => {
+ const { trust } = calibrate(pages, { weights }).profiles.article.scores;
+ expect(trust.rho).toBeGreaterThan(0.8);
+ expect(trust.rhoSuggestedInSample).toBeGreaterThan(0.8);
+ expect(
+ scoreOf(
+ { claimsSourced: 0.5, noBrokenLinks: 1 },
+ { claimsSourced: 30, noBrokenLinks: 10, x: 0 },
+ ),
+ ).toBe(63);
+ expect(scoreOf({}, { claimsSourced: 30 })).toBeNull();
+ });
+
+ it('compares citations within a page’s group: a big brand’s pages aren’t a good score', () => {
+ // Within each brand, more sourced claims go with more citations; across
+ // them, the big brand's pages have fewer sourced claims and far more citations.
+ const rows = [
+ ...Array.from({ length: 20 }, (_, i) => ({
+ profile: 'article',
+ checks: { trust: { claimsSourced: i / 40 } },
+ citations: 1000 + i,
+ group: 'big',
+ })),
+ ...Array.from({ length: 20 }, (_, i) => ({
+ profile: 'article',
+ checks: { trust: { claimsSourced: 0.5 + i / 40 } },
+ citations: i,
+ group: 'small',
+ })),
+ ];
+ const rho = (list) => calibrate(list).profiles.article.scores.trust.checks.claimsSourced.rho;
+ expect(rho(rows)).toBeGreaterThan(0.9);
+ expect(rho(rows.map(({ group: _group, ...row }) => row))).toBeLessThan(0);
+ });
+
+ it('suggests weights from the weights in use, the score’s total kept', () => {
+ const { suggested } = calibrate(pages, { weights }).profiles.article.scores.trust;
+ // Sourced claims take half of what working links had; brand facts, not
+ // read, and the expert quote, of one value, keep theirs; none goes to 0.
+ expect(suggested).toEqual({
+ claimsSourced: 35,
+ noBrokenLinks: 5,
+ expertQuote: 8,
+ brandFacts: 12,
+ });
+ expect(Object.values(suggested).every((points) => points > 0)).toBe(true);
+ expect(Object.values(suggested).reduce((a, b) => a + b, 0)).toBe(60);
+ // No weights given: no suggestion.
+ expect(calibrate(pages).profiles.article.scores.trust).not.toHaveProperty('suggested');
+ });
+
+ it('leaves the weights as they are on pages whose checks have nothing to do with their citations', () => {
+ const random = seeded(7);
+ const noise = Array.from({ length: 30 }, () => ({
+ profile: 'product',
+ checks: {
+ eeat: { experience: random(), originalData: random(), visualProof: random() },
+ },
+ citations: Math.floor(random() * 20),
+ }));
+ const current = { experience: 20, originalData: 20, visualProof: 60 };
+ const report = calibrate(noise, {
+ weights: { product: { eeat: current, trust: {}, readability: {} } },
+ });
+ expect(report.profiles.product.scores.eeat.suggested).toEqual(current);
+ });
+});
+
+describe('suggestWeights', () => {
+ const read = (rho) => ({ pages: 40, rho, interval: interval(rho, 40) });
+
+ it('keeps half the weights in use and shares half by positive correlation, none under 1', () => {
+ expect(
+ suggestWeights(
+ { a: 20, b: 20, c: 10 },
+ { a: read(0.6), b: read(0.4), c: { pages: 5, rho: null } },
+ ),
+ ).toEqual({ a: 22, b: 18, c: 10 });
+ // A check that goes with fewer citations keeps a point at least: dropping it is the maintainer's call.
+ expect(suggestWeights({ a: 20, b: 20 }, { a: read(0.5), b: read(-0.5) })).toEqual({
+ a: 30,
+ b: 10,
+ });
+ expect(suggestWeights({ a: 2, b: 1 }, { a: read(0.9), b: read(-0.9) })).toEqual({ a: 2, b: 1 });
+ // No positive read check: as they are.
+ expect(suggestWeights({ a: 20, b: 20 }, { a: read(-0.5), b: read(0.1) })).toEqual({
+ a: 20,
+ b: 20,
+ });
+ // A check weighed 0 stays out.
+ expect(suggestWeights({ a: 0, b: 30 }, { a: read(0.9), b: read(0.5) })).toEqual({
+ a: 0,
+ b: 30,
+ });
+ });
+
+ it('rounds to whole points, the total exact', () => {
+ const out = suggestWeights({ a: 4, b: 3, c: 3 }, { a: read(0.5), b: read(0.5), c: read(0.5) });
+ expect(Object.values(out).reduce((x, y) => x + y, 0)).toBe(10);
+ expect(Object.values(out).every(Number.isInteger)).toBe(true);
+ });
+});
+
+describe('calibrationWeights', () => {
+ it('reads PROFILE_WEIGHTS, and not points written as text or a wrapped export', () => {
+ const scores = { eeat: { byline: 100 }, trust: { claimsSourced: 100 }, readability: {} };
+ expect(calibrationWeights.safeParse({ article: scores }).success).toBe(true);
+ expect(
+ calibrationWeights.safeParse({ article: { ...scores, eeat: { byline: '100' } } }).success,
+ ).toBe(false);
+ expect(calibrationWeights.safeParse({ PROFILE_WEIGHTS: { article: scores } }).success).toBe(
+ false,
+ );
+ });
+});
diff --git a/server/src/scripts/calibrate-quality-weights.js b/server/src/scripts/calibrate-quality-weights.js
new file mode 100644
index 000000000..f6c7f7873
--- /dev/null
+++ b/server/src/scripts/calibrate-quality-weights.js
@@ -0,0 +1,119 @@
+/**
+ * Quality's weights against AI citations (Content Editor Quality spec §11,
+ * CE-055): reads a JSON-lines file of pages a maintainer exported, one page
+ * a line, and prints, per profile and score, how the score goes with the AI
+ * citations the pages got, and per check, its pages, rank correlation and
+ * 95% interval; given the weights in use, the ones the correlations suggest.
+ * See src/lib/quality-calibration.js for what a row holds and how a
+ * suggestion is made: they are a starting point for the maintainer, never
+ * applied by themselves.
+ *
+ * Export the pages over one window, live for all of it, each with a group
+ * (a salted hash of its brand: correlations are read within each brand).
+ * weights.json is web/src/lib/content/quality-profiles.ts's PROFILE_WEIGHTS
+ * as JSON (JSON.stringify(PROFILE_WEIGHTS), logged from a test, say).
+ *
+ * Run from server/:
+ * node src/scripts/calibrate-quality-weights.js <pages.jsonl> [--weights <weights.json>] [--min 30] [--out <report.json>]
+ *
+ * --weights the weights in use (profile → score → check → points)
+ * --min the fewest pages a check's correlation is read from (30; 3 at least)
+ * --out also write the report as JSON
+ *
+ * Reads no database and no network, and prints and writes aggregates only:
+ * a row's fields other than its profile, checks, citations and group are
+ * dropped, the group is never printed, and a line that isn't a row is
+ * counted, never printed. Exits 1 when no row could be read. Isn't run in CI.
+ */
+import { readFileSync, writeFileSync } from 'fs';
+import {
+ MIN_PAGES,
+ QUALITY_SCORES,
+ calibrate,
+ calibrationWeights,
+ parseRows,
+ withoutBom,
+} from '../lib/quality-calibration.js';
+
+const args = process.argv.slice(2);
+const option = (name) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
+const fail = (message) => {
+ console.error(message);
+ process.exit(1);
+};
+const file = args[0] && !args[0].startsWith('--') ? args[0] : null;
+if (!file) {
+ fail(
+ 'Usage: node src/scripts/calibrate-quality-weights.js <pages.jsonl> [--weights <weights.json>] [--min 30] [--out <report.json>]',
+ );
+}
+const minArg = option('--min');
+const minPages = minArg === null ? MIN_PAGES : Number(minArg);
+if (!Number.isInteger(minPages) || minPages < 3)
+ fail('--min takes a whole number of pages, 3 at least.');
+const out = option('--out');
+
+let weights = null;
+const weightsFile = option('--weights');
+if (weightsFile) {
+ let raw;
+ try {
+ raw = JSON.parse(withoutBom(readFileSync(weightsFile, 'utf8')));
+ } catch {
+ fail(`${weightsFile} isn't JSON.`);
+ }
+ const parsed = calibrationWeights.safeParse(raw);
+ if (!parsed.success || !Object.keys(parsed.data).length) {
+ fail(
+ `${weightsFile} isn't PROFILE_WEIGHTS: profile → { eeat, trust, readability } → check → whole points.`,
+ );
+ }
+ weights = parsed.data;
+ for (const [profile, scores] of Object.entries(weights)) {
+ for (const score of QUALITY_SCORES) {
+ const total = Object.values(scores[score]).reduce((a, b) => a + b, 0);
+ if (total !== 100) console.warn(`Warning: ${profile} ${score} adds up to ${total}, not 100.`);
+ }
+ }
+}
+
+const { rows, invalid } = parseRows(readFileSync(file, 'utf8'));
+console.log(
+ `${rows.length} pages read${invalid ? `, ${invalid} ${invalid === 1 ? 'line' : 'lines'} skipped` : ''}.`,
+);
+if (!rows.length) fail('No page could be read.');
+const report = calibrate(rows, { minPages, weights });
+
+const show = (rho) => (rho === null || rho === undefined ? '—' : rho.toFixed(3));
+for (const [profile, { pages, scores }] of Object.entries(report.profiles)) {
+ console.log(`\n${profile} · ${pages} pages`);
+ for (const score of QUALITY_SCORES) {
+ const { checks, rho, suggested, rhoSuggestedInSample } = scores[score];
+ if (weights?.[profile]) {
+ console.log(
+ ` ${score} score: rho ${show(rho)} with the weights in use, ${show(rhoSuggestedInSample)} with the suggested (in-sample)`,
+ );
+ }
+ for (const [check, stat] of Object.entries(checks)) {
+ const now = weights?.[profile]?.[score]?.[check];
+ const next = suggested?.[check];
+ const range = stat.interval
+ ? `[${stat.interval.low.toFixed(2)}, ${stat.interval.high.toFixed(2)}]${stat.interval.read ? ' *' : ''}`
+ : '';
+ console.log(
+ ` ${score}.${check}`.padEnd(32),
+ `${stat.pages} pages`.padEnd(11),
+ `rho ${show(stat.rho)}`.padEnd(11),
+ range.padEnd(18),
+ now === undefined ? '' : `${now} → ${next}`,
+ );
+ }
+ }
+}
+console.log(
+ '\n* the 95% interval leaves 0 out: only these checks move a suggestion. Suggestions keep half the weights in use and never take a weighed check to 0: weigh them with judgment before changing PROFILE_WEIGHTS.',
+);
+if (out) {
+ writeFileSync(out, `${JSON.stringify(report, null, 2)}\n`);
+ console.log(`\nReport written to ${out}.`);
+}