Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
43.2 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Executed-test identity ledger (X2).
 *
 * Reads the receipts every CI lane writes (scripts/lib/test-env.sh
 * `receipts_init`: one `<id>.receipt` + `<id>.files` + Bun JUnit report per bun
 * invocation; verify writes one testcase per check) and accounts for every
 * executed test by identity: (lane, file, test name, backend arm). Each
 * identity also carries the database backend its name pins (`testBackend`):
 * a test registered per backend (`testBackends()`) names it, and a move of
 * such a test may change arm, because the backend, not the lane's arm, is
 * what it exercises.
 *
 * One side (head only) prints the per-lane executed/skipped/failed table and
 * every completeness problem. Two sides (base and head) also list each base
 * identity that no longer executes in head. A drop passes only when
 * docs/test-audit/2026-10-04/expected-deltas.tsv declares it (retirement,
 * move, pass-to-skip); otherwise the tool prints the exact row to declare it.
 * New tests need no declaration.
 *
 * Completeness (any problem means `incomplete`, which fails):
 *   - an assigned file with no valid JUnit from any attempt (a killed shard
 *     that no rescue re-ran, a truncated report); a later attempt (rerun or
 *     rescue) supersedes an earlier one for the files it re-ran;
 *   - a receipt from an earlier run attempt than the run's latest;
 *   - a missing shard, a lane that executed zero tests (unless the lane was
 *     declared empty), or a base artifact missing from head without a
 *     `job` row.
 *
 * Usage:
 *   bun scripts/ci-executed-counts.ts --head-run <id>[,<id>] [--base-run <id>[,<id>]]
 *   bun scripts/ci-executed-counts.ts --head-dir <dir> [--base-dir <dir>]
 * Options: --deltas <tsv>, --summary <file> (appends Markdown, e.g. the step
 * summary), --json <file>, --report-only (always exit 0), --fail-on-additions
 * (stability check between baseline runs), --expect-attempt <run>:<n>,
 * --missing-keys K1,K2 (count skipped tests in files that read those keys),
 * --repo owner/name.
 * Exit: 0 pass, 1 drop/incomplete, 2 usage or download error.
 */
import { existsSync, mkdtempSync, readdirSync, readFileSync, statSync, appendFileSync, writeFileSync } from 'node:fs';
import { basename, join } from 'node:path';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';

export const DEFAULT_DELTAS = 'docs/test-audit/2026-10-04/expected-deltas.tsv';
const DOCS = 'Docs: docs/TESTING.md#executed-test-receipts';

export type Status = 'pass' | 'fail' | 'skip' | 'todo';
export interface TestCase { file: string; test: string; status: Status }

export class JUnitError extends Error {}

const ENTITIES: Record<string, string> = { lt: '<', gt: '>', amp: '&', quot: '"', apos: "'" };
function decode(text: string): string {
  return text.replace(/&(#x[0-9a-fA-F]+|#[0-9]+|[a-z]+);/g, (whole, ref: string) => {
    if (ref[0] === '#') {
      const code = ref[1] === 'x' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);
      return Number.isFinite(code) ? String.fromCodePoint(code) : whole;
    }
    return ENTITIES[ref] ?? whole;
  });
}

/** Unstable fragments (temp paths, UUIDs) must not split one test into two identities. Timestamps stay: test names use fixed dates on purpose. */
export function canonicalTestName(name: string): string {
  return name
    .replace(/\/(?:private\/)?(?:tmp|var\/folders)\/[^\s'"`),]+/g, '<tmp>')
    .replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, '<uuid>');
}

/**
 * Strict parser for the JUnit subset Bun (and run-verify-parallel.sh) writes.
 * Anything malformed or cut short throws: a truncated report is never read as
 * a shorter test list.
 */
export function parseJUnit(xml: string): TestCase[] {
  const token = /<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<\?[\s\S]*?\?>|<(\/?)([A-Za-z_][\w:.-]*)((?:\s+[\w:.-]+\s*=\s*"[^"]*")*)\s*(\/?)>|[^<]+/y;
  const attrRe = /([\w:.-]+)\s*=\s*"([^"]*)"/g;
  const stack: { tag: string; attrs: Record<string, string> }[] = [];
  const cases: TestCase[] = [];
  let current: { file: string; test: string; status: Status } | undefined;
  let rootTests: number | undefined;
  let closedRoot = false;
  const open = (tag: string, attrs: Record<string, string>) => {
    if (tag === 'testcase') {
      const suites = stack.filter(s => s.tag === 'testsuite');
      const file = suites[0]?.attrs.file ?? attrs.file ?? suites[0]?.attrs.name ?? '';
      const describe = suites.slice(1).map(s => s.attrs.name ?? '');
      if (!attrs.name) throw new JUnitError('testcase without a name');
      current = { file, test: [...describe, attrs.name].join(' > '), status: 'pass' };
    } else if (current && tag === 'skipped') {
      if (current.status !== 'fail') current.status = attrs.message === 'TODO' ? 'todo' : 'skip';
    } else if (current && (tag === 'failure' || tag === 'error')) {
      current.status = 'fail';
    }
  };
  const close = (tag: string) => {
    if (tag === 'testcase' && current) {
      cases.push({ ...current, test: canonicalTestName(current.test) });
      current = undefined;
    }
    if (tag === 'testsuites' && stack.length === 0) closedRoot = true;
  };
  let index = 0;
  while (index < xml.length) {
    token.lastIndex = index;
    const match = token.exec(xml);
    if (!match) throw new JUnitError(`malformed or truncated report near byte ${index}`);
    index = token.lastIndex;
    const [whole, slash, tag, rawAttrs, selfClose] = match;
    if (!tag) {
      if (whole.startsWith('<') || stack.length || closedRoot || !whole.trim()) continue;
      throw new JUnitError('text outside the report root');
    }
    if (closedRoot) throw new JUnitError('content after the closing </testsuites>');
    if (slash) {
      const top = stack.pop();
      if (!top || top.tag !== tag) throw new JUnitError(`unbalanced </${tag}>`);
      close(tag);
      continue;
    }
    const attrs: Record<string, string> = {};
    for (const a of (rawAttrs ?? '').matchAll(attrRe)) attrs[a[1]] = decode(a[2]);
    if (stack.length === 0) {
      if (tag !== 'testsuites') throw new JUnitError(`report root is <${tag}>, expected <testsuites>`);
      if (attrs.tests !== undefined) rootTests = Number(attrs.tests);
    }
    open(tag, attrs);
    if (selfClose) close(tag);
    else stack.push({ tag, attrs });
  }
  if (!closedRoot || stack.length) throw new JUnitError('report ends before </testsuites> (truncated)');
  if (rootTests !== undefined && rootTests !== cases.length) {
    throw new JUnitError(`report declares ${rootTests} tests but lists ${cases.length}`);
  }
  return cases;
}

const RANK: Record<string, number> = { primary: 0, rerun: 1, rescue: 2 };

export interface Receipt {
  id: string;
  artifact: string;
  lane: string;
  kind: string;
  shard?: number;
  of?: number;
  arm: string;
  runId: string;
  runAttempt?: number;
  sha: string;
  started: number;
  exit?: number;
  empty: boolean;
  declaredEmpty: boolean;
  files: string[];
  junit: 'ok' | 'missing' | 'truncated' | 'not-expected';
  junitError?: string;
  cases: TestCase[];
}

function normalizePath(path: string, root: string): string {
  let p = path.trim();
  if (root && p.startsWith(`${root}/`)) p = p.slice(root.length + 1);
  return p.replace(/^\.\//, '');
}

function readMeta(text: string): Record<string, string> {
  const meta: Record<string, string> = {};
  for (const line of text.split('\n')) {
    const eq = line.indexOf('=');
    if (eq > 0) meta[line.slice(0, eq)] = line.slice(eq + 1);
  }
  return meta;
}

const optionalInt = (value: string | undefined): number | undefined =>
  value !== undefined && /^\d+$/.test(value) ? Number(value) : undefined;

export function loadReceipt(dir: string, id: string, artifact: string): Receipt {
  const meta = readMeta(readFileSync(join(dir, `${id}.receipt`), 'utf8'));
  const root = meta.root ?? '';
  const filesPath = join(dir, `${id}.files`);
  const files = existsSync(filesPath)
    ? readFileSync(filesPath, 'utf8').split('\n').filter(Boolean).map(f => normalizePath(f, root))
    : [];
  const empty = meta.empty === '1' || meta.declared_empty === '1';
  const receipt: Receipt = {
    id, artifact, files, empty,
    lane: meta.lane || 'unknown',
    kind: meta.kind || 'primary',
    shard: optionalInt(meta.shard),
    of: optionalInt(meta.of),
    arm: meta.arm ?? '',
    runId: meta.run_id ?? '',
    runAttempt: optionalInt(meta.run_attempt),
    sha: meta.sha ?? '',
    started: optionalInt(meta.started) ?? 0,
    exit: optionalInt(meta.exit),
    declaredEmpty: meta.declared_empty === '1',
    junit: empty ? 'not-expected' : 'missing',
    cases: [],
  };
  const junitPath = join(dir, `${id}.junit.xml`);
  if (!empty && existsSync(junitPath)) {
    try {
      receipt.cases = parseJUnit(readFileSync(junitPath, 'utf8')).map(c => ({ ...c, file: normalizePath(c.file, root) }));
      receipt.junit = 'ok';
    } catch (error) {
      receipt.junit = 'truncated';
      receipt.junitError = error instanceof Error ? error.message : String(error);
    }
  }
  return receipt;
}

/** A directory of receipts is one artifact; a directory of such directories is several. */
export function loadReceiptDir(dir: string): Receipt[] {
  const ids = (d: string) => readdirSync(d).filter(n => n.endsWith('.receipt')).map(n => n.slice(0, -'.receipt'.length)).sort();
  const own = ids(dir);
  if (own.length) return own.map(id => loadReceipt(dir, id, basename(dir)));
  const receipts: Receipt[] = [];
  for (const name of readdirSync(dir).sort()) {
    const sub = join(dir, name);
    if (!statSync(sub).isDirectory()) continue;
    for (const id of ids(sub)) receipts.push(loadReceipt(sub, id, name));
  }
  return receipts;
}

export interface Identity { lane: string; file: string; test: string; arm: string; backend: TestBackend; status: Status; artifact: string; kind: string }
export interface Issue { kind: string; lane?: string; artifact?: string; file?: string; detail: string }
export interface LaneStats { lane: string; files: number; executed: number; passed: number; failed: number; skipped: number; superseded: number; declaredEmpty: boolean }
export interface Side {
  label: string;
  artifacts: Set<string>;
  identities: Map<string, Identity>;
  issues: Issue[];
  duplicates: { key: string; count: number }[];
  overlaps: string[];
  lanes: Map<string, LaneStats>;
  shas: Set<string>;
}

export const identityKey = (lane: string, file: string, test: string, arm: string) => [lane, file, test, arm].join('\u001f');

/**
 * The database backend a test name pins: `pglite` or `postgres` when its
 * top-level describe names exactly one of them as a word (testBackends()
 * suites prefix or suffix it, e.g. "postgres: semantic writes …" or "…
 * (pglite)"), else when the whole name does; otherwise '' (the name does not
 * say, or names both). A hyphenated compound such as postgres-unit-arms is
 * not a backend.
 */
export type TestBackend = 'pglite' | 'postgres' | '';
export function testBackend(test: string): TestBackend {
  const pins = (text: string): TestBackend => {
    const pglite = /(?<![\w-])pglite(?![\w-])/i.test(text);
    const postgres = /(?<![\w-])postgres(?:ql)?(?![\w-])/i.test(text);
    return pglite === postgres ? '' : pglite ? 'pglite' : 'postgres';
  };
  return pins(test.split(' > ')[0]!) || pins(test);
}
const executed = (status: Status | undefined) => status === 'pass' || status === 'fail';

function describeAttempt(r: Receipt): string {
  const state = r.junit === 'truncated' ? `truncated JUnit (${r.junitError})` : r.junit === 'missing' ? 'no JUnit report' : r.junit;
  return `${r.artifact}/${r.id} [${r.kind}${r.exit === undefined ? ', interrupted' : `, exit ${r.exit}`}]: ${state}`;
}

/**
 * `latestBy` 'run' (the comparison) keeps only receipts from the run's latest attempt. 'job' keeps each artifact's
 * newest attempt: a re-run of failed jobs overwrites only their artifacts, so the others still hold the receipts of
 * the attempt in which their job last ran, and the side counts the whole suite (#8001).
 */
export function buildSide(label: string, all: Receipt[], expectedAttempts = new Map<string, number>(), latestBy: 'run' | 'job' = 'run'): Side {
  const side: Side = { label, artifacts: new Set(all.map(r => r.artifact)), identities: new Map(), issues: [], duplicates: [], overlaps: [], lanes: new Map(), shas: new Set() };
  const scope = (r: Receipt) => (latestBy === 'run' ? r.runId : `${r.runId}\u001f${r.artifact}`);
  const latest = new Map(latestBy === 'run' ? expectedAttempts : []);
  for (const r of all) {
    if (r.runId && r.runAttempt !== undefined && r.runAttempt > (latest.get(scope(r)) ?? 0)) latest.set(scope(r), r.runAttempt);
  }
  const receipts = all.filter(r => {
    const want = r.runId ? latest.get(scope(r)) : undefined;
    if (want !== undefined && r.runAttempt !== undefined && r.runAttempt < want) {
      side.issues.push({ kind: 'prior-attempt', lane: r.lane, artifact: r.artifact, detail: `${r.artifact}/${r.id} comes from attempt ${r.runAttempt} of run ${r.runId}; the run's latest attempt is ${want}. Re-run all jobs, not only failed ones, so every lane reports from one attempt.` });
      return false;
    }
    if (r.sha) side.shas.add(r.sha);
    return true;
  });
  const byLane = new Map<string, Receipt[]>();
  for (const r of receipts) byLane.set(r.lane, [...(byLane.get(r.lane) ?? []), r]);
  const duplicateCounts = new Map<string, number>();
  for (const [lane, laneReceipts] of [...byLane].sort(([a], [b]) => a.localeCompare(b))) {
    const stats: LaneStats = { lane, files: 0, executed: 0, passed: 0, failed: 0, skipped: 0, superseded: 0, declaredEmpty: laneReceipts.some(r => r.declaredEmpty) };
    side.lanes.set(lane, stats);
    const sharded = laneReceipts.filter(r => r.of !== undefined);
    const ofs = new Set(sharded.map(r => r.of));
    if (ofs.size > 1) side.issues.push({ kind: 'missing-shard', lane, detail: `lane ${lane} mixes shard totals (${[...ofs].join(', ')})` });
    for (const of of ofs) {
      const present = new Set(sharded.filter(r => r.of === of).map(r => r.shard));
      for (let shard = 1; shard <= (of ?? 0); shard++) {
        if (!present.has(shard)) side.issues.push({ kind: 'missing-shard', lane, detail: `lane ${lane}: shard ${shard} of ${of} left no receipt (its job did not run, or its "Upload receipts" step found nothing)` });
      }
    }
    const byFile = new Map<string, Receipt[]>();
    for (const r of laneReceipts) {
      if (r.empty) continue;
      const files = new Set([...r.files, ...r.cases.map(c => c.file)]);
      for (const file of files) {
        const key = `${file}\u001f${r.arm}`;
        byFile.set(key, [...(byFile.get(key) ?? []), r]);
      }
    }
    stats.files = new Set([...byFile.keys()].map(k => k.split('\u001f')[0])).size;
    for (const [fileArm, attempts] of [...byFile].sort(([a], [b]) => a.localeCompare(b))) {
      const [file, arm] = fileArm.split('\u001f');
      const where = arm ? `${file} [${arm}]` : file;
      const valid = attempts.filter(r => r.junit === 'ok');
      if (!valid.length) {
        side.issues.push({ kind: 'incomplete', lane, file, artifact: attempts[0].artifact, detail: `lane ${lane}: ${where} has no valid receipt from any attempt (${attempts.map(describeAttempt).join('; ')})` });
        continue;
      }
      const primaries = valid.filter(r => (RANK[r.kind] ?? 0) === 0);
      if (primaries.length > 1) side.overlaps.push(`lane ${lane}: ${where} ran in ${primaries.length} primary invocations (${primaries.map(r => `${r.artifact}/${r.id}`).join(', ')})`);
      const winner = valid.reduce((best, r) => {
        const order = (x: Receipt) => [RANK[x.kind] ?? 0, x.started];
        const [br, bs] = order(best);
        const [rr, rs] = order(r);
        return rr > br || (rr === br && (rs > bs || (rs === bs && r.id > best.id))) ? r : best;
      });
      if ((RANK[winner.kind] ?? 0) > 0) stats.superseded++;
      // Repeated names inside one file (Bun's JUnit keeps test.each titles as
      // the unformatted "%s" template) stay distinct by occurrence order.
      const seen = new Map<string, number>();
      for (const c of winner.cases) {
        if (c.file !== file) continue;
        const n = (seen.get(c.test) ?? 0) + 1;
        seen.set(c.test, n);
        const test = n === 1 ? c.test : `${c.test} [#${n}]`;
        if (n > 1) duplicateCounts.set(identityKey(lane, file, c.test, winner.arm), n);
        side.identities.set(identityKey(lane, file, test, winner.arm), { lane, file, test, arm: winner.arm, backend: testBackend(c.test), status: c.status, artifact: winner.artifact, kind: winner.kind });
      }
    }
  }
  for (const id of side.identities.values()) {
    const stats = side.lanes.get(id.lane)!;
    if (id.status === 'pass') stats.passed++;
    else if (id.status === 'fail') stats.failed++;
    else stats.skipped++;
    if (executed(id.status)) stats.executed++;
  }
  for (const stats of side.lanes.values()) {
    if (stats.executed === 0 && !stats.declaredEmpty && stats.files > 0) {
      side.issues.push({ kind: 'empty-lane', lane: stats.lane, detail: `lane ${stats.lane} executed 0 tests across ${stats.files} file(s)` });
    }
  }
  side.duplicates = [...duplicateCounts].map(([key, count]) => ({ key, count }));
  return side;
}

export type DeltaKind = 'retire' | 'move' | 'skip' | 'job';
export interface DeltaRow { line: number; kind: DeltaKind; lane: string; file: string; test: string; arm: string; newLane: string; newFile: string; newTest: string; reason: string; evidence: string }
export const DELTA_COLUMNS = ['kind', 'lane', 'file', 'test', 'arm', 'new_lane', 'new_file', 'new_test', 'reason', 'evidence'];

const unescapeField = (value: string) => value.replace(/\\(t|n|\\)/g, (_, c: string) => (c === 't' ? '\t' : c === 'n' ? '\n' : '\\'));
export const escapeField = (value: string) => value.replace(/\\/g, '\\\\').replace(/\t/g, '\\t').replace(/\n/g, '\\n');
const isTodo = (value: string) => !value.trim() || /^TODO\b/i.test(value.trim());

export function parseDeltas(text: string): { rows: DeltaRow[]; errors: string[] } {
  const rows: DeltaRow[] = [];
  const errors: string[] = [];
  let header = false;
  text.split('\n').forEach((raw, i) => {
    const line = i + 1;
    if (!raw.trim() || raw.startsWith('#')) return;
    const cells = raw.split('\t');
    if (!header) {
      if (cells.join('\t') !== DELTA_COLUMNS.join('\t')) errors.push(`line ${line}: header must be ${DELTA_COLUMNS.join(' ')}`);
      header = true;
      return;
    }
    if (cells.length !== DELTA_COLUMNS.length) {
      errors.push(`line ${line}: ${cells.length} columns, expected ${DELTA_COLUMNS.length} (tab-separated)`);
      return;
    }
    const [kind, lane, file, test, arm, newLane, newFile, newTest, reason, evidence] = cells.map(unescapeField);
    const row: DeltaRow = { line, kind: kind as DeltaKind, lane, file, test, arm, newLane, newFile, newTest, reason, evidence };
    if (!['retire', 'move', 'skip', 'job'].includes(kind)) errors.push(`line ${line}: kind "${kind}" is not retire, move, skip or job`);
    else if (isTodo(reason)) errors.push(`line ${line}: reason is empty or still TODO`);
    else if (kind === 'job' ? !lane : !lane || lane === '*' || !file || file === '*' || !test) errors.push(`line ${line}: ${kind === 'job' ? 'job rows name the artifact in the lane column' : 'lane and file must be exact; test is a name or *'}`);
    else if (kind === 'retire' && isTodo(evidence)) errors.push(`line ${line}: a retirement needs its Retiring-a-test evidence (docs/test-audit/.../implementation/<lane>.md#...)`);
    else if (kind === 'move' && !newLane && !newFile && !newTest) errors.push(`line ${line}: a move names new_lane, new_file or new_test`);
    else rows.push(row);
  });
  if (!header && text.trim()) errors.push('missing header row');
  return { rows, errors };
}

function globMatch(pattern: string, value: string): boolean {
  const parts = pattern.split('*');
  if (parts.length === 1) return pattern === value;
  const first = parts[0];
  const last = parts[parts.length - 1];
  if (!value.startsWith(first) || value.length < first.length + last.length || !value.endsWith(last)) return false;
  let at = first.length;
  const end = value.length - last.length;
  for (const part of parts.slice(1, -1)) {
    const found = value.indexOf(part, at);
    if (found === -1 || found + part.length > end) return false;
    at = found + part.length;
  }
  return true;
}

function rowMatches(row: DeltaRow, id: Identity): boolean {
  return row.kind !== 'job' && row.lane === id.lane && row.file === id.file
    && (row.test === '*' || row.test === id.test) && (row.arm === '*' || row.arm === id.arm);
}

export interface Drop { identity: Identity; head: Status | 'absent'; verdict: 'undeclared' | 'retired' | 'moved' | 'skipped' | 'lane-declared-empty' | 'move-target-missing' | 'skip-mismatch'; row?: DeltaRow; detail?: string }
export interface Comparison {
  ok: boolean;
  base?: Side;
  head: Side;
  drops: Drop[];
  additions: Identity[];
  issues: Issue[];
  deltaErrors: string[];
  unusedRows: DeltaRow[];
  suggestions: string[];
}

export function deltaRow(kind: DeltaKind, id: Pick<Identity, 'lane' | 'file' | 'test' | 'arm'>, target: { lane?: string; file?: string; test?: string } = {}): string {
  const evidence = kind === 'retire' ? 'TODO: docs/test-audit/2026-10-04/implementation/<lane>.md#<retiring-a-test row>' : '';
  return [kind, id.lane, id.file, id.test, id.arm, target.lane ?? '', target.file ?? '', target.test ?? '', 'TODO: why this test no longer runs here', evidence].map(escapeField).join('\t');
}

export function compare(head: Side, base: Side | undefined, deltas: { rows: DeltaRow[]; errors: string[] }, opts: { failOnAdditions?: boolean } = {}): Comparison {
  const result: Comparison = { ok: true, base, head, drops: [], additions: [], issues: [...(base?.issues.map(i => ({ ...i, detail: `base: ${i.detail}` })) ?? []), ...head.issues.map(i => ({ ...i, detail: base ? `head: ${i.detail}` : i.detail }))], deltaErrors: deltas.errors, unusedRows: [], suggestions: [] };
  const used = new Set<DeltaRow>();
  const headAt = new Map<string, Identity[]>();
  for (const h of head.identities.values()) {
    const at = identityKey(h.lane, h.file, h.test, '');
    headAt.set(at, [...(headAt.get(at) ?? []), h]);
  }
  const sideOf = (detail: string) => (base && detail.startsWith('base: ') ? base : head);
  result.issues = result.issues.filter(issue => {
    if (issue.kind !== 'empty-lane') return true;
    const ids = [...sideOf(issue.detail).identities.values()].filter(id => id.lane === issue.lane);
    const rows = ids.map(id => deltas.rows.find(r => r.kind === 'skip' && r.lane === id.lane && r.file === id.file && r.test === '*'));
    if (!ids.length || rows.some(r => !r)) return true;
    for (const r of rows) used.add(r!);
    return false;
  });
  if (base) {
    for (const artifact of base.artifacts) {
      if (head.artifacts.has(artifact)) continue;
      const row = deltas.rows.find(r => r.kind === 'job' && globMatch(r.lane, artifact));
      if (row) used.add(row);
      else result.issues.push({ kind: 'missing-artifact', artifact, detail: `artifact ${artifact} is in base but not in head: its job did not run or uploaded nothing. Declare a removed or renamed job with a "job" row naming the artifact.` });
    }
    for (const id of base.identities.values()) {
      if (!executed(id.status)) continue;
      const now = head.identities.get(identityKey(id.lane, id.file, id.test, id.arm));
      if (executed(now?.status)) continue;
      const drop: Drop = { identity: id, head: now?.status ?? 'absent', verdict: 'undeclared' };
      const row = deltas.rows.find(r => rowMatches(r, id));
      if (row) {
        used.add(row);
        drop.row = row;
        if (row.kind === 'retire') drop.verdict = 'retired';
        else if (row.kind === 'skip') drop.verdict = now && !executed(now.status) ? 'skipped' : 'skip-mismatch';
        else {
          const [lane, file, test] = [row.newLane || id.lane, row.newFile || id.file, row.test === '*' ? id.test : row.newTest || id.test];
          drop.verdict = (headAt.get(identityKey(lane, file, test, '')) ?? []).some(h => movedTo(id, h)) ? 'moved' : 'move-target-missing';
          if (drop.verdict === 'move-target-missing') drop.detail = `declared move target ${[lane, file, test, id.arm].join(' | ')} did not execute in head${id.backend ? ` under any arm on the ${id.backend} backend` : ''}`;
        }
      } else if (head.lanes.get(id.lane)?.declaredEmpty) {
        drop.verdict = 'lane-declared-empty';
      }
      result.drops.push(drop);
    }
    const baseExecuted = new Set([...base.identities].filter(([, id]) => executed(id.status)).map(([key]) => key));
    result.additions = [...head.identities].filter(([key, id]) => executed(id.status) && !baseExecuted.has(key)).map(([, id]) => id);
    result.suggestions = suggestRows(result.drops.filter(d => d.verdict === 'undeclared'), base, head);
  }
  result.unusedRows = deltas.rows.filter(r => !used.has(r));
  const failing = result.drops.some(d => d.verdict === 'undeclared' || d.verdict === 'move-target-missing' || d.verdict === 'skip-mismatch');
  result.ok = !failing && !result.issues.length && !result.deltaErrors.length && !(opts.failOnAdditions && result.additions.length);
  return result;
}

/**
 * Whether head identity `to` executes what base identity `from` did: the same
 * arm, or another arm when the test's name pins its backend. A PostgreSQL-arm
 * cell run with --postgres-only runs only the PostgreSQL side of testBackends()
 * suites, so their PGLite tests leave that lane (arm postgres) for the unit and serial
 * lanes (no arm) and still run on PGLite there. A PostgreSQL test may change
 * arm only to another database arm: a lane with no arm has no database.
 */
const movedTo = (from: Identity, to: Identity) =>
  executed(to.status) && (to.arm === from.arm
    || (from.backend !== '' && to.backend === from.backend && (from.backend === 'pglite' || to.arm !== '')));

/** The exact TSV rows that would declare each undeclared drop: a move when the test runs elsewhere in head, else a retirement. */
export function suggestRows(drops: Drop[], base: Side, head: Side): string[] {
  const rows: string[] = [];
  const headByTest = new Map<string, Identity[]>();
  for (const id of head.identities.values()) if (executed(id.status)) headByTest.set(id.test, [...(headByTest.get(id.test) ?? []), id]);
  const groups = new Map<string, Drop[]>();
  for (const d of drops) {
    const key = [d.identity.lane, d.identity.file, d.identity.arm].join('\u001f');
    groups.set(key, [...(groups.get(key) ?? []), d]);
  }
  for (const group of groups.values()) {
    const { lane, file, arm } = group[0].identity;
    const baseExecuted = [...base.identities.values()].filter(id => id.lane === lane && id.file === file && id.arm === arm && executed(id.status)).length;
    const headFileRuns = [...head.identities.values()].some(id => id.lane === lane && id.file === file && id.arm === arm && executed(id.status));
    const elsewhere = (d: Drop) => (headByTest.get(d.identity.test) ?? []).find(h => (h.lane !== lane || h.file !== file) && movedTo(d.identity, h));
    const backend = group[0].identity.backend;
    const backendBase = [...base.identities.values()].filter(id => id.lane === lane && id.file === file && id.arm === arm && id.backend === backend && executed(id.status)).length;
    const targets = new Set(group.map(d => { const h = elsewhere(d); return h ? `${h.lane}\u001f${h.file}` : ''; }));
    if (backend && group.length > 1 && group.length === backendBase && group.every(d => d.identity.backend === backend) && targets.size === 1 && !targets.has('')) {
      const [newLane, newFile] = [...targets][0]!.split('\u001f');
      rows.push(deltaRow('move', { lane, file, test: '*', arm }, { lane: newLane === lane ? '' : newLane, file: newFile === file ? '' : newFile }));
      continue;
    }
    if (group.length === baseExecuted && !headFileRuns && group.length > 1) {
      const votes = new Map<string, number>();
      for (const d of group) {
        const h = elsewhere(d);
        if (h) votes.set(`${h.lane}\u001f${h.file}`, (votes.get(`${h.lane}\u001f${h.file}`) ?? 0) + 1);
      }
      const best = [...votes].sort((a, b) => b[1] - a[1])[0];
      if (best && best[1] * 2 >= group.length) {
        const [newLane, newFile] = best[0].split('\u001f');
        rows.push(deltaRow('move', { lane, file, test: '*', arm }, { lane: newLane === lane ? '' : newLane, file: newFile === file ? '' : newFile }));
      } else {
        rows.push(deltaRow('retire', { lane, file, test: '*', arm }));
      }
      continue;
    }
    for (const d of group) {
      const h = elsewhere(d);
      if (h) rows.push(deltaRow('move', d.identity, { lane: h.lane === lane ? '' : h.lane, file: h.file === file ? '' : h.file }));
      else if (d.head === 'skip' || d.head === 'todo') rows.push(deltaRow('skip', d.identity));
      else rows.push(deltaRow('retire', d.identity));
    }
  }
  return rows;
}

/** A9: skipped tests in files that read a provider key the run did not have. */
export function missingKeySkips(head: Side, keys: string[], root = '.'): { tests: number; files: string[] } {
  const files = new Map<string, number>();
  const reads = new Map<string, boolean>();
  for (const id of head.identities.values()) {
    if (executed(id.status) || id.file === 'verify') continue;
    if (!reads.has(id.file)) {
      let source = '';
      try { source = readFileSync(join(root, id.file), 'utf8'); } catch { /* deleted or out-of-tree file */ }
      reads.set(id.file, keys.some(k => source.includes(k)));
    }
    if (reads.get(id.file)) files.set(id.file, (files.get(id.file) ?? 0) + 1);
  }
  return { tests: [...files.values()].reduce((a, b) => a + b, 0), files: [...files.keys()].sort() };
}

const md = (value: string, max = 160) => {
  const text = value.length > max ? `${value.slice(0, max - 1)}…` : value;
  return text.replace(/[\r\n]+/g, ' ').replace(/([|`*_<>[\]\\])/g, '\\$1').replace(/@/g, '@\u200b');
};
const LIST_CAP = 200;

export function render(result: Comparison, extra: string[] = []): string {
  const out: string[] = [];
  const { base, head } = result;
  const verdict = result.ok ? 'PASS' : result.issues.length ? 'INCOMPLETE (fails)' : 'FAIL';
  out.push('## Executed-test identities (X2)', '');
  out.push(base ? `Base: ${md(base.label)} · Head: ${md(head.label)} · **${verdict}**` : `Head: ${md(head.label)} · **${verdict}**`, '');
  const lanes = [...new Set([...(base?.lanes.keys() ?? []), ...head.lanes.keys()])].sort();
  if (base) {
    out.push('| Lane | Base executed | Head executed | Head skipped | Head failed | Undeclared drops | Declared drops | Added |', '|---|---:|---:|---:|---:|---:|---:|---:|');
    for (const lane of lanes) {
      const b = base.lanes.get(lane);
      const h = head.lanes.get(lane);
      const drops = result.drops.filter(d => d.identity.lane === lane);
      const bad = drops.filter(d => d.verdict === 'undeclared' || d.verdict === 'move-target-missing' || d.verdict === 'skip-mismatch').length;
      out.push(`| ${md(lane)} | ${b?.executed ?? 0} | ${h?.executed ?? 0} | ${h?.skipped ?? 0} | ${h?.failed ?? 0} | ${bad} | ${drops.length - bad} | ${result.additions.filter(a => a.lane === lane).length} |`);
    }
  } else {
    out.push('| Lane | Files | Executed | Passed | Failed | Skipped | Superseded by rerun/rescue |', '|---|---:|---:|---:|---:|---:|---:|');
    for (const lane of lanes) {
      const h = head.lanes.get(lane)!;
      out.push(`| ${md(lane)}${h.declaredEmpty ? ' (declared empty)' : ''} | ${h.files} | ${h.executed} | ${h.passed} | ${h.failed} | ${h.skipped} | ${h.superseded} |`);
    }
  }
  out.push('');
  const byBackend = lanes.map(lane => {
    const ids = [...head.identities.values()].filter(id => id.lane === lane && executed(id.status));
    return { lane, pglite: ids.filter(id => id.backend === 'pglite').length, postgres: ids.filter(id => id.backend === 'postgres').length, unnamed: ids.filter(id => !id.backend).length };
  }).filter(b => b.pglite || b.postgres);
  if (byBackend.length) {
    out.push('Head executed by backend (the backend a test name pins; testBackends() suites name it):', '', '| Lane | PGLite | PostgreSQL | Name pins none |', '|---|---:|---:|---:|');
    for (const b of byBackend) out.push(`| ${md(b.lane)} | ${b.pglite} | ${b.postgres} | ${b.unnamed} |`);
    out.push('');
  }
  out.push(...extra);
  if (result.issues.length) {
    out.push(`### Incomplete (${result.issues.length})`, '', 'Why: a lane without a complete receipt cannot prove which tests ran, so the comparison fails closed.', 'Fix: rerun every job of the run (not only failed jobs) so each lane uploads a receipt from one attempt; a killed shard needs its rescue or a rerun.', DOCS, '');
    for (const issue of result.issues.slice(0, LIST_CAP)) out.push(`- ${md(issue.detail, 400)}`);
    if (result.issues.length > LIST_CAP) out.push(`- … ${result.issues.length - LIST_CAP} more (see --json)`);
    out.push('');
  }
  if (result.deltaErrors.length) {
    out.push(`### expected-deltas.tsv errors (${result.deltaErrors.length})`, '');
    for (const e of result.deltaErrors) out.push(`- ${md(e, 300)}`);
    out.push('');
  }
  const bad = result.drops.filter(d => d.verdict === 'undeclared' || d.verdict === 'move-target-missing' || d.verdict === 'skip-mismatch');
  if (bad.length) {
    out.push(`### Dropped identities (${bad.length})`, '', 'Why: these base tests no longer execute in head and expected-deltas.tsv does not account for them.', `Fix: restore the tests, or add the rows below to ${DEFAULT_DELTAS} with a real reason (retirements also need their Retiring-a-test evidence).`, DOCS, '');
    for (const d of bad.slice(0, LIST_CAP)) out.push(`- ${md(d.identity.lane)} · ${md(d.identity.file)} · ${md(d.identity.test)}${d.identity.arm ? ` · ${md(d.identity.arm)}` : ''} — head: ${d.head}${d.verdict === 'undeclared' ? '' : ` (${d.verdict}${d.detail ? `: ${md(d.detail, 200)}` : ''})`}`);
    if (bad.length > LIST_CAP) out.push(`- … ${bad.length - LIST_CAP} more (see --json)`);
    if (result.suggestions.length) out.push('', 'Rows to declare them:', '', '```tsv', ...result.suggestions.slice(0, LIST_CAP), '```');
    out.push('');
  }
  const declared = result.drops.length - bad.length;
  if (declared) out.push(`Declared drops: ${declared} (${['retired', 'moved', 'skipped', 'lane-declared-empty'].map(v => `${v} ${result.drops.filter(d => d.verdict === v).length}`).join(', ')}).`, '');
  if (base) out.push(`Added identities (new tests need no declaration): ${result.additions.length}.`, '');
  if (head.overlaps.length) {
    out.push(`Files that ran in more than one primary invocation in head (sharding overlap; the latest attempt counts): ${head.overlaps.length}.`, '');
    for (const o of head.overlaps.slice(0, 20)) out.push(`- ${md(o, 300)}`);
    out.push('');
  }
  if (head.duplicates.length) out.push(`Repeated test names in head (numbered by occurrence, e.g. test.each titles): ${head.duplicates.length}.`, '');
  if (result.unusedRows.length) out.push(`expected-deltas.tsv rows that matched nothing: ${result.unusedRows.map(r => `line ${r.line}`).join(', ')}.`, '');
  if (head.shas.size > 1) out.push(`Head receipts come from ${head.shas.size} commits: ${[...head.shas].map(s => s.slice(0, 12)).join(', ')}.`, '');
  return `${out.join('\n')}\n`;
}

export function toJSON(result: Comparison): unknown {
  const side = (s: Side) => ({ label: s.label, artifacts: [...s.artifacts].sort(), shas: [...s.shas], lanes: [...s.lanes.values()], identities: [...s.identities.values()], duplicates: s.duplicates.length, overlaps: s.overlaps });
  return {
    ok: result.ok,
    base: result.base ? side(result.base) : null,
    head: side(result.head),
    issues: result.issues,
    deltaErrors: result.deltaErrors,
    drops: result.drops.map(d => ({ ...d.identity, head: d.head, verdict: d.verdict, row: d.row?.line, detail: d.detail })),
    additions: result.additions,
    suggestions: result.suggestions,
    unusedRows: result.unusedRows.map(r => r.line),
  };
}

class UsageError extends Error {}

function gh(args: string[]): string {
  const r = spawnSync('gh', args, { encoding: 'utf8' });
  if (r.error || r.status !== 0) {
    throw new UsageError(`gh ${args.join(' ')} failed: ${(r.stderr || r.error?.message || '').trim()}\nWhy: run artifacts are read through the GitHub CLI.\nFix: gh auth status (a token with actions:read), and check the run id exists in the repo.`);
  }
  return r.stdout;
}

/** Download a run's receipts-* artifacts; returns the receipts and the run's latest attempt. */
export function downloadRun(repo: string, runId: string, into: string): { receipts: Receipt[]; attempt: number; label: string } {
  if (!/^\d+$/.test(runId)) throw new UsageError(`run id "${runId}" is not numeric`);
  const run = JSON.parse(gh(['api', `repos/${repo}/actions/runs/${runId}`, '--jq', '{attempt: .run_attempt, sha: .head_sha, status: .status, name: .name, event: .event}']));
  if (run.status !== 'completed') throw new UsageError(`run ${runId} is ${run.status}\nWhy: a running workflow has not uploaded every receipt yet.\nFix: wait for it to finish (gh run watch ${runId} --repo ${repo}).`);
  const dir = join(into, runId);
  gh(['run', 'download', runId, '--repo', repo, '--pattern', 'receipts-*', '--dir', dir]);
  const receipts = existsSync(dir) ? loadReceiptDir(dir) : [];
  if (!receipts.length) throw new UsageError(`run ${runId} has no receipts-* artifacts\nWhy: the run predates executed-test receipts, or its upload steps did not run.\nFix: re-run the workflow on a commit whose test.yml/e2e.yml upload receipts.`);
  return { receipts, attempt: Number(run.attempt), label: `${run.name} #${runId} (${run.event}, ${String(run.sha).slice(0, 12)}, attempt ${run.attempt})` };
}

interface Args { baseRuns: string[]; headRuns: string[]; baseDirs: string[]; headDirs: string[]; deltas?: string; summary?: string; json?: string; reportOnly: boolean; failOnAdditions: boolean; expect: Map<string, number>; missingKeys?: string[]; repo: string }

export function parseArgs(argv: string[]): Args {
  const args: Args = { baseRuns: [], headRuns: [], baseDirs: [], headDirs: [], reportOnly: false, failOnAdditions: false, expect: new Map(), repo: process.env.GITHUB_REPOSITORY || 'garrytan/gbrain' };
  for (let i = 0; i < argv.length; i++) {
    const flag = argv[i];
    const value = () => {
      const v = argv[++i];
      if (v === undefined || v.startsWith('--')) throw new UsageError(`${flag} needs a value`);
      return v;
    };
    const list = () => value().split(',').map(s => s.trim()).filter(Boolean);
    switch (flag) {
      case '--base-run': args.baseRuns.push(...list()); break;
      case '--head-run': args.headRuns.push(...list()); break;
      case '--base-dir': args.baseDirs.push(value()); break;
      case '--head-dir': args.headDirs.push(value()); break;
      case '--deltas': args.deltas = value(); break;
      case '--summary': args.summary = value(); break;
      case '--json': args.json = value(); break;
      case '--repo': args.repo = value(); break;
      case '--report-only': args.reportOnly = true; break;
      case '--fail-on-additions': args.failOnAdditions = true; break;
      case '--missing-keys': args.missingKeys = list(); break;
      case '--expect-attempt': {
        const m = /^(\d+):(\d+)$/.exec(value());
        if (!m) throw new UsageError('--expect-attempt takes <run-id>:<attempt>');
        args.expect.set(m[1], Number(m[2]));
        break;
      }
      case '--help': case '-h': throw new UsageError('help');
      default: throw new UsageError(`unknown option ${flag}`);
    }
  }
  if (!args.headRuns.length && !args.headDirs.length) throw new UsageError('pass --head-run <id> or --head-dir <dir>');
  return args;
}

function loadSide(runs: string[], dirs: string[], args: Args, into: string): { side: Side; receipts: Receipt[] } | undefined {
  if (!runs.length && !dirs.length) return undefined;
  const receipts: Receipt[] = [];
  const expect = new Map(args.expect);
  const labels: string[] = [];
  for (const run of runs) {
    const got = downloadRun(args.repo, run, into);
    receipts.push(...got.receipts);
    expect.set(run, got.attempt);
    labels.push(got.label);
  }
  for (const dir of dirs) {
    if (!existsSync(dir)) throw new UsageError(`receipt directory ${dir} does not exist`);
    receipts.push(...loadReceiptDir(dir));
    labels.push(dir);
  }
  const seen = new Map<string, string>();
  for (const r of receipts) {
    const key = `${r.artifact}/${r.id}`;
    const source = r.runId || 'local';
    if (seen.has(key) && seen.get(key) !== source) throw new UsageError(`artifact ${r.artifact} appears in two runs of one side; pass one test.yml run and one e2e.yml run per side`);
    seen.set(key, source);
  }
  return { side: buildSide(labels.join(' + '), receipts, expect), receipts };
}

export function main(argv: string[]): number {
  let args: Args;
  try {
    args = parseArgs(argv);
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    if (message !== 'help') console.error(`ci-executed-counts: ${message}`);
    console.error('usage: bun scripts/ci-executed-counts.ts (--head-run <id>[,<id>] | --head-dir <dir>) [--base-run <id>[,<id>] | --base-dir <dir>] [--deltas <tsv>] [--summary <file>] [--json <file>] [--report-only] [--fail-on-additions] [--expect-attempt <run>:<n>] [--missing-keys K1,K2] [--repo owner/name]');
    return message === 'help' ? 0 : 2;
  }
  try {
    const scratch = mkdtempSync(join(tmpdir(), 'gbrain-receipts-'));
    const loaded = loadSide(args.headRuns, args.headDirs, args, scratch)!;
    const head = loaded.side;
    const base = loadSide(args.baseRuns, args.baseDirs, args, scratch)?.side;
    const deltasPath = args.deltas ?? (existsSync(DEFAULT_DELTAS) ? DEFAULT_DELTAS : undefined);
    const deltas = deltasPath ? parseDeltas(readFileSync(deltasPath, 'utf8')) : { rows: [], errors: [] };
    const result = compare(head, base, deltas, { failOnAdditions: args.failOnAdditions });
    const extra: string[] = [];
    if (args.missingKeys) {
      const keys = args.missingKeys;
      if (keys.length) {
        const skipped = missingKeySkips(head, keys);
        const listed = skipped.files.slice(0, 20).map(f => md(f)).join(', ') + (skipped.files.length > 20 ? ', …' : '');
        extra.push(`**${skipped.tests} tests skipped for missing keys**: skipped tests in ${skipped.files.length} file(s) that read ${keys.join(' or ')}, which at least one lane ran without${skipped.files.length ? ` (${listed})` : ''}.`, '');
      } else {
        extra.push('**0 tests skipped for missing keys** (every provider key was set).', '');
      }
    }
    const text = render(result, extra);
    process.stdout.write(text);
    if (args.summary) appendFileSync(args.summary, text);
    if (args.json) {
      // The run this ledger was written for, and the head's executed counts with each job at its newest attempt
      // (scripts/ci-reuse.ts --gate reads both, #8001).
      const [run] = args.expect.size === 1 ? [...args.expect] : [];
      const jobs = [...buildSide(head.label, loaded.receipts, new Map(), 'job').lanes.values()];
      const attempts = [...new Set(loaded.receipts.flatMap(r => (r.runAttempt === undefined ? [] : [r.runAttempt])))].sort((a, b) => a - b);
      writeFileSync(args.json, `${JSON.stringify({
        ...(toJSON(result) as object),
        run: run ? { id: Number(run[0]), attempt: run[1] } : null,
        job_latest: { executed: jobs.reduce((n, l) => n + l.executed, 0), failed: jobs.reduce((n, l) => n + l.failed, 0), attempts },
      }, null, 2)}\n`);
    }
    return result.ok || args.reportOnly ? 0 : 1;
  } catch (error) {
    console.error(`ci-executed-counts: ${error instanceof Error ? error.message : String(error)}`);
    console.error(DOCS);
    return 2;
  }
}

if (import.meta.main) process.exitCode = main(process.argv.slice(2));