Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
4.6 KB
Raw Blame
import { journeyFiles, filesUnder, testName } from "../bin/test-files.mjs";
// Run with --write-baseline to replace the baseline with every currently violating spec.
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
import { relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const evalsRoot = fileURLToPath(new URL("..", import.meta.url));
const specsDirectory = resolve(evalsRoot, "specs");
const baselinePath = resolve(specsDirectory, "boundary-ratchet.baseline.json");

function importedSpecifiers(source) {
  return [...source.matchAll(/(?:\bimport\s+(?:[^"'`;]*?\s+from\s+)?|\bexport\s+[^"'`;]*?\s+from\s+)["']([^"']+)["']/g)]
    .map((match) => match[1]);
}

export function classifySpec(source) {
  const imports = importedSpecifiers(source);
  const productSource = /^\.\.\/\.\.\/(?:apps|packages|ee)\//;
  return {
    importsProductSource: imports.some((specifier) => productSource.test(specifier))
      || /\bnew\s+URL\(\s*["']\.\.\/\.\.\/(?:apps|packages|ee)\/[^"']*["']\s*,\s*import\.meta\.url\s*\)/.test(source),
    importsNodeFs: imports.some((specifier) => /^(?:node:)?fs(?:\/promises)?$/.test(specifier)),
    importsChildProcess: imports.some((specifier) => /^(?:node:)?child_process$/.test(specifier)),
    crossesBoundary: imports.some((specifier) => /^(?:\.\.\/)?\.\.\/worlds\/|^@omnirush\/world$/.test(specifier))
      || /(?<!\.)\b(?:app|chrome|server|inviteMember|faultProxy|startCaptureEfficiencyLab)\s*\(|\bspec\.world\s*\(/.test(source),
  };
}

export function violations(file, classification) {
  const reasons = [];
  if (classification.importsProductSource) reasons.push(`${file}: imports product source (../../apps/...) — unit tests belong next to the module they test, not in evals/specs`);
  if (classification.importsNodeFs && !classification.crossesBoundary) reasons.push(`${file}: reads the filesystem (node:fs) — a spec observes the product, not the repository`);
  if (classification.importsChildProcess && !classification.crossesBoundary) reasons.push(`${file}: spawns processes (node:child_process) — wrapping another test runner is not evidence`);
  if (!classification.crossesBoundary) reasons.push(`${file}: never crosses a product boundary (app()/chrome()/server()/spec.world()) — see write-a-spec`);
  return reasons;
}

export function compareBaseline(files, baseline) {
  const errors = [];
  const warnings = [];
  const grandfathered = new Set(baseline);
  for (const [file, classification] of Object.entries(files)) {
    const reasons = violations(file, classification);
    if (grandfathered.has(file)) {
      if (reasons.length === 0) warnings.push(`${file}: is clean; remove it from boundary-ratchet.baseline.json`);
    } else {
      errors.push(...reasons);
    }
  }
  for (const file of baseline) {
    if (!(file in files)) errors.push(`${file}: baseline is stale; file no longer exists — remove it`);
  }
  return { errors, warnings };
}

export function scanSpecs(directory = specsDirectory) {
  const files = {};
  for (const path of directory === specsDirectory ? journeyFiles(/\.test\.ts$/) : filesUnder(directory, /\.test\.ts$/)) {
    const file = directory === specsDirectory ? testName(path) : relative(directory, path);
    files[file] = classifySpec(readFileSync(path, "utf8"));
  }
  return files;
}

function readBaseline() {
  const value = JSON.parse(readFileSync(baselinePath, "utf8"));
  if (!Array.isArray(value) || value.some((file) => typeof file !== "string" || !file.endsWith(".test.ts"))) {
    throw new Error("Boundary ratchet baseline must be an array of spec filenames.");
  }
  return value;
}

export function main() {
  const files = scanSpecs();
  if (process.argv.includes("--write-baseline")) {
    const baseline = Object.entries(files)
      .filter(([file, classification]) => violations(file, classification).length > 0)
      .map(([file]) => file);
    writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`);
    console.log(`spec-boundary-ratchet: wrote ${baseline.length} baseline entries`);
    return;
  }
  const { errors, warnings } = compareBaseline(files, readBaseline());
  for (const warning of warnings) console.warn(`WARNING: ${warning}`);
  if (errors.length > 0) {
    console.error(`spec-boundary-ratchet failed:\n- ${errors.join("\n- ")}\nFix existing debt only by shrinking evals/specs/boundary-ratchet.baseline.json; new specs must cross a real product boundary.`);
    process.exitCode = 1;
  } else {
    console.log(`spec-boundary-ratchet: ${Object.keys(files).length} specs checked`);
  }
}

const invokedDirectly = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (invokedDirectly) main();