Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
/**
* Proof that guardrails fire, read from the record `loop-drill --record` writes.
*
* Every other signal in the audit asks whether a file exists. That is how a
* repo of empty files used to score 100/L3. loop-drill runs each guardrail
* against a seeded fault and a benign case; this module reads what it found,
* so the audit can tell a guardrail that works from one that is only present.
*
* A guardrail's results only count while they still describe the repo:
*
* gate the record's sha256 of gate.yaml must match the current file.
* Gate drills are deterministic, so they never expire otherwise.
* breaker drills loop-context's defaults, which the repo does not edit.
* others (verifier, injection, ...) run real commands against code that
* keeps changing, so they expire after PROOF_MAX_AGE_MS.
*/
import { createHash } from 'node:crypto';
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { fileExists } from '@cobusgreyling/readiness-core';
export const PROOF_FILE = 'loop-drill.json';
export const PROOF_SCHEMA = 1;
/** Canary results (verifier, injection) older than this no longer count. */
export const PROOF_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
/** Guardrails whose results depend only on their recorded input, not on time. */
const TIMELESS = new Set(['gate', 'breaker']);
type Outcome = 'passed' | 'failed' | 'skipped';
type Direction = 'sensitivity' | 'specificity';
export interface RecordedResult {
id: string;
direction: Direction;
outcome: Outcome;
detail?: string;
}
export interface RecordedGuardrail {
recordedAt: string;
/** What was drilled: for the gate, the policy file and its fingerprint. */
input?: { file?: string; sha256?: string | null; command?: string };
results: RecordedResult[];
}
export interface DrillRecord {
schema: number;
guardrails: Record<string, RecordedGuardrail>;
}
export interface ProofSignals {
/** loop-drill.json exists at the root. */
present: boolean;
/** Why the record could not be used, when present but unreadable. */
error?: string;
/** Current, no failures, and a passing drill in both directions. */
proven: string[];
/** Current, and at least one drill failed: the guardrail did not fire. */
failed: string[];
/** Current, but every drill was skipped, so nothing was tested. */
untested: string[];
/** Recorded against a different gate.yaml, or too long ago to count. */
stale: string[];
/** Ids of failed drills, for the report. */
failures: string[];
/** Skip reasons for untested guardrails, for the report. */
skipReasons: Record<string, string>;
}
/**
* sha256 of a policy file with line endings normalised, so a checkout with
* core.autocrlf fingerprints the same as the Linux runner that recorded it.
* loop-drill computes the same value; both test suites pin the same vector.
*/
export function fingerprint(text: string): string {
return createHash('sha256').update(text.replace(/\r\n/g, '\n'), 'utf8').digest('hex');
}
function emptyProof(present: boolean, error?: string): ProofSignals {
return { present, error, proven: [], failed: [], untested: [], stale: [], failures: [], skipReasons: {} };
}
const OUTCOMES = new Set<Outcome>(['passed', 'failed', 'skipped']);
const DIRECTIONS = new Set<Direction>(['sensitivity', 'specificity']);
function isResult(value: unknown): value is RecordedResult {
const r = value as RecordedResult | null;
return (
!!r &&
typeof r.id === 'string' &&
OUTCOMES.has(r.outcome) &&
DIRECTIONS.has(r.direction) &&
(r.detail === undefined || typeof r.detail === 'string')
);
}
/** Validate the parsed record. Returns an error message, or null when usable. */
export function validateRecord(value: unknown): string | null {
const record = value as DrillRecord | null;
if (!record || typeof record !== 'object') return 'expected a JSON object';
if (record.schema !== PROOF_SCHEMA) return `unsupported schema ${String(record.schema)} (expected ${PROOF_SCHEMA})`;
if (!record.guardrails || typeof record.guardrails !== 'object') return 'missing "guardrails"';
for (const [name, g] of Object.entries(record.guardrails)) {
if (!g || typeof g.recordedAt !== 'string' || Number.isNaN(new Date(g.recordedAt).getTime())) {
return `guardrail "${name}" has no valid recordedAt`;
}
if (!Array.isArray(g.results) || !g.results.every(isResult)) {
return `guardrail "${name}" has malformed results`;
}
}
return null;
}
async function currentGateFingerprint(root: string): Promise<string | null> {
const p = path.join(root, 'gate.yaml');
if (!(await fileExists(p))) return null;
try {
return fingerprint(await readFile(p, 'utf8'));
} catch {
return null;
}
}
/** Does this guardrail's record still describe the repo as it is now? */
function isCurrent(name: string, g: RecordedGuardrail, gateSha: string | null, now: number): boolean {
if (name === 'gate') {
return g.input?.file === 'gate.yaml' && !!g.input.sha256 && g.input.sha256 === gateSha;
}
if (TIMELESS.has(name)) return true;
const age = now - new Date(g.recordedAt).getTime();
return age <= PROOF_MAX_AGE_MS && age >= -60_000;
}
/**
* Proven means the guardrail caught a seeded fault *and* let a benign case
* through, with nothing failing. One direction is easy to fake: a denylist of
* "**" catches every fault, and a verifier that rejects everything catches
* every mutant. loop-drill always drills both; this insists on both.
*/
function classify(results: RecordedResult[]): 'proven' | 'failed' | 'untested' {
if (results.some((r) => r.outcome === 'failed')) return 'failed';
const passed = results.filter((r) => r.outcome === 'passed');
const both = passed.some((r) => r.direction === 'sensitivity') && passed.some((r) => r.direction === 'specificity');
return both ? 'proven' : 'untested';
}
export async function readProof(root: string, now = Date.now()): Promise<ProofSignals> {
const p = path.join(root, PROOF_FILE);
if (!(await fileExists(p))) return emptyProof(false);
let parsed: unknown;
try {
parsed = JSON.parse(await readFile(p, 'utf8'));
} catch (err) {
return emptyProof(true, `not valid JSON (${(err as Error).message})`);
}
const error = validateRecord(parsed);
if (error) return emptyProof(true, error);
const record = parsed as DrillRecord;
const gateSha = await currentGateFingerprint(root);
const proof = emptyProof(true);
for (const [name, g] of Object.entries(record.guardrails).sort(([a], [b]) => a.localeCompare(b))) {
if (!isCurrent(name, g, gateSha, now)) {
proof.stale.push(name);
continue;
}
const verdict = classify(g.results);
proof[verdict].push(name);
if (verdict === 'failed') {
proof.failures.push(...g.results.filter((r) => r.outcome === 'failed').map((r) => r.id));
}
if (verdict === 'untested') {
const reason = g.results.find((r) => r.outcome === 'skipped' && r.detail)?.detail;
if (reason) proof.skipReasons[name] = reason;
}
}
return proof;
}