Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

6.9 KB
Raw Blame
/**
 * Proof that guardrails fire, read from the record `loop-drill --record` writes.
 *
 * Every other signal in the audit asks whether a file exists. That is how a
 * repo of empty files used to score 100/L3. loop-drill runs each guardrail
 * against a seeded fault and a benign case; this module reads what it found,
 * so the audit can tell a guardrail that works from one that is only present.
 *
 * A guardrail's results only count while they still describe the repo:
 *
 *   gate      the record's sha256 of gate.yaml must match the current file.
 *             Gate drills are deterministic, so they never expire otherwise.
 *   breaker   drills loop-context's defaults, which the repo does not edit.
 *   others    (verifier, injection, ...) run real commands against code that
 *             keeps changing, so they expire after PROOF_MAX_AGE_MS.
 */

import { createHash } from 'node:crypto';
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { fileExists } from '@cobusgreyling/readiness-core';

export const PROOF_FILE = 'loop-drill.json';
export const PROOF_SCHEMA = 1;

/** Canary results (verifier, injection) older than this no longer count. */
export const PROOF_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;

/** Guardrails whose results depend only on their recorded input, not on time. */
const TIMELESS = new Set(['gate', 'breaker']);

type Outcome = 'passed' | 'failed' | 'skipped';
type Direction = 'sensitivity' | 'specificity';

export interface RecordedResult {
  id: string;
  direction: Direction;
  outcome: Outcome;
  detail?: string;
}

export interface RecordedGuardrail {
  recordedAt: string;
  /** What was drilled: for the gate, the policy file and its fingerprint. */
  input?: { file?: string; sha256?: string | null; command?: string };
  results: RecordedResult[];
}

export interface DrillRecord {
  schema: number;
  guardrails: Record<string, RecordedGuardrail>;
}

export interface ProofSignals {
  /** loop-drill.json exists at the root. */
  present: boolean;
  /** Why the record could not be used, when present but unreadable. */
  error?: string;
  /** Current, no failures, and a passing drill in both directions. */
  proven: string[];
  /** Current, and at least one drill failed: the guardrail did not fire. */
  failed: string[];
  /** Current, but every drill was skipped, so nothing was tested. */
  untested: string[];
  /** Recorded against a different gate.yaml, or too long ago to count. */
  stale: string[];
  /** Ids of failed drills, for the report. */
  failures: string[];
  /** Skip reasons for untested guardrails, for the report. */
  skipReasons: Record<string, string>;
}

/**
 * sha256 of a policy file with line endings normalised, so a checkout with
 * core.autocrlf fingerprints the same as the Linux runner that recorded it.
 * loop-drill computes the same value; both test suites pin the same vector.
 */
export function fingerprint(text: string): string {
  return createHash('sha256').update(text.replace(/\r\n/g, '\n'), 'utf8').digest('hex');
}

function emptyProof(present: boolean, error?: string): ProofSignals {
  return { present, error, proven: [], failed: [], untested: [], stale: [], failures: [], skipReasons: {} };
}

const OUTCOMES = new Set<Outcome>(['passed', 'failed', 'skipped']);
const DIRECTIONS = new Set<Direction>(['sensitivity', 'specificity']);

function isResult(value: unknown): value is RecordedResult {
  const r = value as RecordedResult | null;
  return (
    !!r &&
    typeof r.id === 'string' &&
    OUTCOMES.has(r.outcome) &&
    DIRECTIONS.has(r.direction) &&
    (r.detail === undefined || typeof r.detail === 'string')
  );
}

/** Validate the parsed record. Returns an error message, or null when usable. */
export function validateRecord(value: unknown): string | null {
  const record = value as DrillRecord | null;
  if (!record || typeof record !== 'object') return 'expected a JSON object';
  if (record.schema !== PROOF_SCHEMA) return `unsupported schema ${String(record.schema)} (expected ${PROOF_SCHEMA})`;
  if (!record.guardrails || typeof record.guardrails !== 'object') return 'missing "guardrails"';
  for (const [name, g] of Object.entries(record.guardrails)) {
    if (!g || typeof g.recordedAt !== 'string' || Number.isNaN(new Date(g.recordedAt).getTime())) {
      return `guardrail "${name}" has no valid recordedAt`;
    }
    if (!Array.isArray(g.results) || !g.results.every(isResult)) {
      return `guardrail "${name}" has malformed results`;
    }
  }
  return null;
}

async function currentGateFingerprint(root: string): Promise<string | null> {
  const p = path.join(root, 'gate.yaml');
  if (!(await fileExists(p))) return null;
  try {
    return fingerprint(await readFile(p, 'utf8'));
  } catch {
    return null;
  }
}

/** Does this guardrail's record still describe the repo as it is now? */
function isCurrent(name: string, g: RecordedGuardrail, gateSha: string | null, now: number): boolean {
  if (name === 'gate') {
    return g.input?.file === 'gate.yaml' && !!g.input.sha256 && g.input.sha256 === gateSha;
  }
  if (TIMELESS.has(name)) return true;
  const age = now - new Date(g.recordedAt).getTime();
  return age <= PROOF_MAX_AGE_MS && age >= -60_000;
}

/**
 * Proven means the guardrail caught a seeded fault *and* let a benign case
 * through, with nothing failing. One direction is easy to fake: a denylist of
 * "**" catches every fault, and a verifier that rejects everything catches
 * every mutant. loop-drill always drills both; this insists on both.
 */
function classify(results: RecordedResult[]): 'proven' | 'failed' | 'untested' {
  if (results.some((r) => r.outcome === 'failed')) return 'failed';
  const passed = results.filter((r) => r.outcome === 'passed');
  const both = passed.some((r) => r.direction === 'sensitivity') && passed.some((r) => r.direction === 'specificity');
  return both ? 'proven' : 'untested';
}

export async function readProof(root: string, now = Date.now()): Promise<ProofSignals> {
  const p = path.join(root, PROOF_FILE);
  if (!(await fileExists(p))) return emptyProof(false);

  let parsed: unknown;
  try {
    parsed = JSON.parse(await readFile(p, 'utf8'));
  } catch (err) {
    return emptyProof(true, `not valid JSON (${(err as Error).message})`);
  }
  const error = validateRecord(parsed);
  if (error) return emptyProof(true, error);

  const record = parsed as DrillRecord;
  const gateSha = await currentGateFingerprint(root);
  const proof = emptyProof(true);

  for (const [name, g] of Object.entries(record.guardrails).sort(([a], [b]) => a.localeCompare(b))) {
    if (!isCurrent(name, g, gateSha, now)) {
      proof.stale.push(name);
      continue;
    }
    const verdict = classify(g.results);
    proof[verdict].push(name);
    if (verdict === 'failed') {
      proof.failures.push(...g.results.filter((r) => r.outcome === 'failed').map((r) => r.id));
    }
    if (verdict === 'untested') {
      const reason = g.results.find((r) => r.outcome === 'skipped' && r.detail)?.detail;
      if (reason) proof.skipReasons[name] = reason;
    }
  }
  return proof;
}