Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.3 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Test-intent classifier: separates STRUCTURAL suites (assert on repo source/
 * doc TEXT — guards against code shape) from BEHAVIORAL suites (execute
 * product code). Output is a committed TSV (scripts/structural-suites.tsv)
 * kept fresh by scripts/check-structural-manifest.sh in `bun run verify`, and
 * rendered next to coverage numbers so the headline test count stops
 * conflating the two.
 *
 * Method (suite-level, content-based — filename heuristics measured 22/23
 * false-positive on `*audit*`):
 *   1. Per test file, find repo-anchored source-read DETECTORS:
 *      - readFileSync / Bun.file whose argument window names a repo path
 *        (src/, scripts/, docs/, .github/, llms*, CLAUDE.md and friends) and
 *        not a tmpdir
 *      - execSync/spawnSync windows that grep/scan repo sources or invoke
 *        scripts/check-*.sh
 *      - the doctor-source and per-surface source helpers
 *        (test/helpers/doctor-source.ts, test/helpers/source-surface.ts),
 *        which exist precisely to feed structural guards
 *   2. Bind each detector to constants it initializes; attribute a describe()
 *      suite as structural when its region contains a detector or references
 *      a bound constant. Hits outside any describe attribute to the suites
 *      that reference the binding, else to a '(file-level)' pseudo-suite.
 *   3. Anything with detectors but no attributable suite lands in the
 *      `unknown` bucket — surfaced in reporting, never silently dropped.
 *
 * This is an APPROXIMATE inventory by design; the TSV freshness check makes
 * drift loud, and misclassifications are fixed by editing the detector list
 * here (never by hand-editing the TSV).
 *
 * Usage:
 *   bun scripts/classify-tests.ts            # rewrite scripts/structural-suites.tsv
 *   bun scripts/classify-tests.ts --check    # exit 1 if the committed TSV is stale
 *   bun scripts/classify-tests.ts --summary  # print counts only
 */
import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs';
import { join, relative } from 'node:path';

const REPO_ROOT = join(import.meta.dir, '..');
const OUT_TSV = join(REPO_ROOT, 'scripts', 'structural-suites.tsv');

const REPO_ANCHORS = [
  /['"`]\.?\.?\/?src\//,
  /['"]src['"]/, // join(..., 'src', ...)
  /['"`]\.?\.?\/?scripts\//,
  /['"]scripts['"]/,
  /['"`]\.?\.?\/?docs\//,
  /['"`]\.?\.?\/?\.github\//,
  /llms(?:-full)?\.txt/,
  /CLAUDE\.md|AGENTS\.md|TESTING\.md|CONTRIBUTING\.md|README\.md|CHANGELOG\.md/,
  /['"`]\.?\.?\/?skills\//,
  /package\.json['"`]/,
];
const TMP_ANCHORS = /tmpdir|mkdtemp|TMPDIR|os\.tmp|TMP_|tmpPath|tempDir|testDir|workDir|FIXTURES?_/i;

export interface SuiteRow {
  file: string;
  suite: string;
  cases: number;
  detector: string;
}
export interface FileResult {
  rows: SuiteRow[];
  unknown: boolean; // detectors present but nothing attributable
}

interface Detector {
  line: number;
  kind: string;
  binding: string | null;
}

/** Window of a line plus the next few — read args often span lines. */
function windowAt(lines: string[], i: number, span = 4): string {
  return lines.slice(i, i + span).join('\n');
}

function isRepoAnchored(win: string): boolean {
  if (TMP_ANCHORS.test(win)) return false;
  const code = win.replace(/\bimport\s*\(\s*(['"`])[^'"`]*\1\s*\)/g, '');
  return REPO_ANCHORS.some((r) => r.test(code));
}

export function classifyFile(relPath: string, content: string): FileResult {
  const lines = content.split('\n');
  const detectors: Detector[] = [];

  for (let i = 0; i < lines.length; i++) {
    const line = lines[i];
    if (/^\s*(\/\/|\*|\/\*)/.test(line)) continue; // comments
    let kind: string | null = null;
    if (/\b(?:readFileSync|readFile)\s*\(/.test(line)) kind = 'readFileSync';
    else if (/\bBun\.file\s*\(/.test(line)) kind = 'bun-file';
    else if (/\b(?:execSync|spawnSync|execFileSync)\s*\(/.test(line)) kind = 'exec-scan';
    else if (/\bdoctor(?:File)?Source\s*\(/.test(line)) kind = 'doctor-source-helper';
    else if (/\bsurface(?:File)?Source\s*\(/.test(line)) kind = 'source-surface-helper';
    if (!kind) continue;

    const win = windowAt(lines, i);
    if (kind === 'exec-scan') {
      // Only exec calls that scan repo sources / run repo guards are structural.
      // Anchor check is looser than the read-path one: shell commands name
      // repo dirs bare (`grep ... src/`), not as quoted path prefixes.
      const scansRepo = /(^|[\s'"`=])(src|scripts|docs)\//.test(win) && !TMP_ANCHORS.test(win);
      if (!(/grep|rg\s|scripts\/check-|--include=.*\.ts/.test(win) && scansRepo)) continue;
    } else if (kind !== 'doctor-source-helper' && kind !== 'source-surface-helper' && !isRepoAnchored(win)) {
      continue;
    }

    const bind = line.match(/(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=/);
    detectors.push({ line: i, kind, binding: bind ? bind[1] : null });
  }

  if (detectors.length === 0) return { rows: [], unknown: false };

  // describe regions: [start, nextDescribeStart)
  const describes: { line: number; title: string }[] = [];
  for (let i = 0; i < lines.length; i++) {
    const m = lines[i].match(/(?:^|\s)describe(?:\.\w+)?\s*\(\s*(['"`])((?:\\.|(?!\1).)*)\1/);
    if (m) describes.push({ line: i, title: m[2] });
  }

  const bindings = detectors.map((d) => d.binding).filter((b): b is string => b !== null);
  const rows: SuiteRow[] = [];
  let attributedAny = false;

  const caseCount = (start: number, end: number): number => {
    let n = 0;
    for (let i = start; i < end; i++) {
      if (/(?:^|\s)(?:test|it)(?:\.\w+)?\s*\(\s*['"`]/.test(lines[i])) n++;
    }
    return n;
  };

  for (let d = 0; d < describes.length; d++) {
    const start = describes[d].line;
    const end = d + 1 < describes.length ? describes[d + 1].line : lines.length;
    const kinds = new Set<string>();
    for (const det of detectors) {
      if (det.line >= start && det.line < end) kinds.add(det.kind);
    }
    if (bindings.length > 0) {
      const region = lines.slice(start, end).join('\n');
      for (const b of bindings) {
        // Binding names may contain regex metachars (e.g. `$SRC`) — escape
        // them or the interpolated RegExp misparses (a `$` reads as an
        // end-anchor and the binding can never match). `\b` anchors are also
        // wrong for `$`-prefixed names ($ is not a word char, so \b$ never
        // matches after `(`), so bound the name with explicit
        // non-identifier-char checks instead.
        const escaped = b.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
        if (new RegExp(`(?:^|[^\\w$])${escaped}(?![\\w$])`).test(region)) {
          const det = detectors.find((x) => x.binding === b);
          if (det) kinds.add(det.kind);
        }
      }
    }
    if (kinds.size > 0) {
      rows.push({
        file: relPath,
        suite: describes[d].title,
        cases: caseCount(start, end),
        detector: [...kinds].sort().join(','),
      });
      attributedAny = true;
    }
  }

  if (!attributedAny) {
    // Detectors exist but no describe claimed them: file-level tests, or
    // bindings only used outside describes.
    const total = caseCount(0, lines.length);
    if (total > 0) {
      rows.push({
        file: relPath,
        suite: '(file-level)',
        cases: total,
        detector: [...new Set(detectors.map((d) => d.kind))].sort().join(','),
      });
      return { rows, unknown: false };
    }
    return { rows: [], unknown: true };
  }

  return { rows, unknown: false };
}

function listTestFiles(dir: string): string[] {
  const out: string[] = [];
  for (const entry of readdirSync(dir).sort()) {
    const full = join(dir, entry);
    if (statSync(full).isDirectory()) {
      if (entry === 'fixtures' || entry === 'node_modules') continue;
      out.push(...listTestFiles(full));
    } else if (entry.endsWith('.test.ts')) {
      out.push(full);
    }
  }
  return out;
}

export function generate(): { tsv: string; suites: number; cases: number; files: number; unknown: string[] } {
  const testDir = join(REPO_ROOT, 'test');
  const rows: SuiteRow[] = [];
  const unknown: string[] = [];
  for (const f of listTestFiles(testDir)) {
    const rel = relative(REPO_ROOT, f);
    const res = classifyFile(rel, readFileSync(f, 'utf-8'));
    rows.push(...res.rows);
    if (res.unknown) unknown.push(rel);
  }
  // Codepoint compare, NOT localeCompare: the committed TSV is byte-diffed
  // by check-structural-manifest.sh, and localeCompare ordering shifts
  // across ICU versions/locales — a flake source, not a real change.
  const codepointCompare = (x: string, y: string): number => (x < y ? -1 : x > y ? 1 : 0);
  rows.sort((a, b) => (a.file === b.file ? codepointCompare(a.suite, b.suite) : codepointCompare(a.file, b.file)));
  const header = [
    '# Structural test suites — suites whose assertions read repo source/doc text.',
    '# GENERATED by scripts/classify-tests.ts; freshness-checked in verify.',
    '# Fix misclassifications in the classifier, never by hand-editing rows.',
    '# Columns: file\tsuite\tcases\tdetector',
  ].join('\n');
  const body = rows.map((r) => `${r.file}\t${r.suite}\t${r.cases}\t${r.detector}`).join('\n');
  const unknownBlock = unknown.length
    ? `\n# unknown (detectors present, no attributable suite):\n${unknown.map((u) => `# unknown\t${u}`).join('\n')}`
    : '';
  return {
    tsv: `${header}\n${body}${unknownBlock}\n`,
    suites: rows.length,
    cases: rows.reduce((s, r) => s + r.cases, 0),
    files: new Set(rows.map((r) => r.file)).size,
    unknown,
  };
}

if (import.meta.main) {
  const mode = process.argv[2] ?? '';
  const res = generate();
  const summary = `structural suites: ${res.suites} · cases: ${res.cases} · files: ${res.files} · unknown: ${res.unknown.length}`;
  if (mode === '--check') {
    let committed = '';
    try {
      committed = readFileSync(OUT_TSV, 'utf-8');
    } catch {
      console.error(`FAIL: ${relative(REPO_ROOT, OUT_TSV)} missing. Run: bun scripts/classify-tests.ts`);
      process.exit(1);
    }
    if (committed !== res.tsv) {
      console.error('FAIL: scripts/structural-suites.tsv is stale (test suites changed shape).');
      console.error('      Regenerate and commit: bun scripts/classify-tests.ts');
      process.exit(1);
    }
    console.log(`OK: structural-suites.tsv fresh (${summary})`);
  } else if (mode === '--summary') {
    console.log(summary);
  } else {
    writeFileSync(OUT_TSV, res.tsv);
    console.log(`wrote scripts/structural-suites.tsv (${summary})`);
  }
}