Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

19.5 KB
Raw Blame
/**
 * Loop Sync - Detect and sync drift between Loop configuration files
 * 
 * This module detects:
 * 1. STATE.md ↔ LOOP.md consistency
 * 2. Skills version updates
 * 3. Missing required files
 * 4. Configuration drift from starters
 */

import { readFile, writeFile, access, mkdir } from 'node:fs/promises';
import { readdir, stat } from 'node:fs/promises';
import path from 'node:path';
import {
  comparePolicies,
  extractGateDenylist,
  extractSafetyDenylist,
  findUndocumentedLimits,
  hasDrift,
} from './policy-drift.js';

export interface DriftReport {
  score: number;           // 0-100 consistency score
  level: 'healthy' | 'warning' | 'critical';
  issues: DriftIssue[];
  suggestions: string[];
  timestamp: string;
}

export interface DriftIssue {
  type: 'missing' | 'outdated' | 'inconsistent' | 'orphaned';
  file: string;
  message: string;
  severity: 'error' | 'warning' | 'info';
  suggestion?: string;
}

export interface SyncOptions {
  targetDir: string;
  autoFix: boolean;
  dryRun: boolean;
  verbose: boolean;
  help?: boolean;
  json?: boolean;
}

/**
 * Check if a file exists
 */
async function fileExists(filePath: string): Promise<boolean> {
  try {
    await access(filePath);
    return true;
  } catch {
    return false;
  }
}

/**
 * Read file content
 */
async function readFileContent(filePath: string): Promise<string | null> {
  try {
    return await readFile(filePath, 'utf8');
  } catch {
    return null;
  }
}

/**
 * Extract frontmatter from markdown files
 */
export function extractFrontmatter(content: string): { frontmatter: Record<string, string>; body: string } {
  const frontmatter: Record<string, string> = {};
  let body = content;
  
  if (content.startsWith('---\n') || content.startsWith('---\r\n')) {
    const endIndex = content.indexOf('\n---', 3);
    if (endIndex !== -1 && (endIndex + 4 === content.length || content[endIndex + 4] === '\n' || content[endIndex + 4] === '\r')) {
      const fmContent = content.slice(3, endIndex);
      const closingEnd = content.indexOf('\n', endIndex + 4);
      body = closingEnd === -1 ? content.slice(endIndex + 1) : content.slice(closingEnd + 1);
      
      for (const line of fmContent.split(/\r?\n/)) {
        const colonIndex = line.indexOf(':');
        if (colonIndex !== -1) {
          const key = line.slice(0, colonIndex).trim();
          const value = line.slice(colonIndex + 1).trim();
          frontmatter[key] = value;
        }
      }
    }
  }
  
  return { frontmatter, body };
}

/**
 * Extract patterns from LOOP.md
 */
function extractLoopPatterns(loopContent: string): string[] {
  const patterns: string[] = [];
  
  // Look for pattern references
  const patternRegex = /pattern[s]?[\s]*[:=][\s]*([\w\-]+)/gi;
  let match;
  while ((match = patternRegex.exec(loopContent)) !== null) {
    patterns.push(match[1]);
  }
  
  return patterns;
}

/**
 * Extract state file references from LOOP.md
 */
function extractStateFiles(loopContent: string): string[] {
  const stateFiles: string[] = [];
  
  // Look for STATE.md or other state file references
  const stateRegex = /(?:state[s]?[\s]*[:=][\s]*|update[\s]+)([\w\-]+\.md)/gi;
  let match;
  while ((match = stateRegex.exec(loopContent)) !== null) {
    stateFiles.push(match[1]);
  }
  
  return stateFiles;
}

/**
 * Compare two markdown files for consistency
 */
function compareMarkdownFiles(
  file1Content: string,
  file2Content: string,
  threshold: number = 0.5
): { similarity: number; differences: string[] } {
  const differences: string[] = [];
  
  // Extract headings from both files
  const headings1 = file1Content.match(/^## .+$/gm) || [];
  const headings2 = file2Content.match(/^## .+$/gm) || [];
  
  // Check heading consistency
  const headingSet1 = new Set(headings1.map(h => h.toLowerCase()));
  const headingSet2 = new Set(headings2.map(h => h.toLowerCase()));
  
  const allHeadings = new Set([...headingSet1, ...headingSet2]);
  let matchingHeadings = 0;
  
  for (const heading of allHeadings) {
    if (headingSet1.has(heading) && headingSet2.has(heading)) {
      matchingHeadings++;
    } else if (!headingSet1.has(heading)) {
      differences.push(`Missing heading in file 1: ${heading}`);
    } else {
      differences.push(`Missing heading in file 2: ${heading}`);
    }
  }
  
  const similarity = allHeadings.size === 0 ? 1 : matchingHeadings / allHeadings.size;
  
  return { similarity, differences };
}

/**
 * Required files whose content is pattern-agnostic enough to scaffold a
 * correct, minimal default without knowing which of loop-init's patterns
 * (daily-triage, pr-babysitter, ci-sweeper, ...) the project actually uses.
 * STATE.md, gate.yaml, loop-budget.md, and loop-run-log.md all have exactly
 * this shape upstream (see templates/*.template) -- LOOP.md and AGENTS.md
 * don't, since their content *is* the pattern-specific goal and workflow, so
 * auto-fix intentionally leaves those two as report-only rather than
 * fabricating wrong content.
 *
 * These are inlined rather than read from templates/*.template because
 * loop-sync ships as a standalone npm package (files: ["dist"]) that runs
 * against arbitrary target directories -- a path relative to this package's
 * own install location wouldn't resolve once installed outside this
 * monorepo checkout.
 */
const AUTO_FIXABLE_FILES = new Set(['STATE.md', 'gate.yaml', 'loop-budget.md', 'loop-run-log.md']);

function projectNameFrom(targetDir: string): string {
  return path.basename(path.resolve(targetDir)) || 'this project';
}

function scaffoldContent(file: string, projectName: string): string {
  switch (file) {
    case 'STATE.md':
      return `# Loop State — ${projectName}

Last run: (set by loop on each run)

## High Priority (loop is acting or waiting on human)

<!-- Format:
- [ ] ID — one-line description
  Loop action: what the loop did last
  Human decision: (if any)
-->

## Watch List

<!-- Items to monitor but not act on yet -->

## Recent Noise (ignored this run)

<!-- Brief list — helps tune triage skill -->

---
Run log: (timestamp) | findings | actions | escalations
`;
    case 'gate.yaml':
      return `# Machine-readable twin of your project's safety policy. Adjust denylist
# and autoMergeAllowlist for your project's own sensitive paths.
version: 1

denylist:
  - "**/.env"
  - "**/.env.*"
  - "**/secrets/**"
  - "**/credentials/**"
  - "**/*_key*"
  - "**/*_secret*"
  - "**/.terraform/**"
  - "**/k8s/production/**"
  - "**/migrations/**"
  - "**/auth/**"
  - "**/payments/**"
  - "**/billing/**"

# Escalate instead of auto-merging when a change touches more than this many files.
maxFiles: 10

# --action auto-merge only proceeds if every changed path matches one of these.
autoMergeAllowlist:
  - "docs/**"
  - "**/*.md"
`;
    case 'loop-budget.md':
      return `# Loop Budget — ${projectName}

## Daily limits

| Loop | Max runs/day | Max tokens/day | Max sub-agent spawns/run |
|------|--------------|----------------|--------------------------|
| Daily Triage | 2 | 100k | 0 (L1) / 2 (L2) |
| PR Babysitter | 288 | 2M | 3 |
| CI Sweeper | 96 | 1M | 3 |
| Dependency Sweeper | 4 | 500k | 3 |
| Post-Merge Cleanup | 1 | 200k | 2 |

## On budget exceed

1. Pause all schedulers (\`scheduler_delete\` or disable automations)
2. Append event to \`loop-run-log.md\`
3. Notify human (Slack / issue / STATE.md High Priority)

## Kill switch

- Command or issue label: \`loop-pause-all\`
- Resume only after human clears the flag in STATE.md
`;
    case 'loop-run-log.md':
      return `# Loop Run Log — ${projectName}

Append one entry per run. Prune entries older than 30 days.

## Format

\`\`\`json
{
  "run_id": "2026-06-09T08:15:00Z",
  "pattern": "daily-triage",
  "duration_s": 45,
  "items_found": 4,
  "actions_taken": 1,
  "escalations": 0,
  "tokens_estimate": 52000,
  "outcome": "report-only | fix-proposed | escalated | no-op"
}
\`\`\`

## Recent Runs

<!-- Loop appends below this line -->
`;
    default:
      throw new Error(`No auto-fix scaffold defined for ${file}.`);
  }
}

/**
 * Scan skills directory for version information
 */
async function scanSkillsDirectory(targetDir: string): Promise<Map<string, string>> {
  const skillsVersions = new Map<string, string>();
  const skillDirs = [
    path.join(targetDir, 'skills'),
    path.join(targetDir, '.grok', 'skills'),
    path.join(targetDir, '.claude', 'skills'),
    path.join(targetDir, '.codex', 'skills'),
  ];

  for (const skillsDir of skillDirs) {
    if (!await fileExists(skillsDir)) continue;

    try {
      const entries = await readdir(skillsDir);
      for (const entry of entries) {
        const skillPath = path.join(skillsDir, entry);
        const statResult = await stat(skillPath);

        if (statResult.isDirectory()) {
          const skillMd = path.join(skillPath, 'SKILL.md');
          const content = await readFileContent(skillMd);

          if (content) {
            const { frontmatter } = extractFrontmatter(content);
            skillsVersions.set(entry, frontmatter.version || 'unknown');
          }
        }
      }
    } catch {
      // Ignore unreadable dirs
    }
  }

  return skillsVersions;
}

/**
 * Main sync function
 */
export async function runSync(options: SyncOptions): Promise<DriftReport> {
  const { targetDir, autoFix, dryRun, verbose } = options;
  const issues: DriftIssue[] = [];
  const suggestions: string[] = [];

  // Define required files
  const requiredFiles = [
    'STATE.md',
    'LOOP.md',
    'AGENTS.md',
    'gate.yaml',
    'loop-budget.md',
    'loop-run-log.md',
  ];

  // Check for missing required files
  for (const file of requiredFiles) {
    const filePath = path.join(targetDir, file);
    if (!await fileExists(filePath)) {
      const fixable = AUTO_FIXABLE_FILES.has(file);
      if (autoFix && fixable) {
        if (!dryRun) {
          await mkdir(path.dirname(filePath), { recursive: true });
          await writeFile(filePath, scaffoldContent(file, projectNameFrom(targetDir)));
        }
        issues.push({
          type: 'missing',
          file,
          message: dryRun
            ? `${file} is missing — would scaffold a minimal default (--dry-run, no changes made)`
            : `${file} was missing — scaffolded a minimal default`,
          severity: 'info',
          suggestion: 'Review the generated content and adjust it for your project.',
        });
        continue;
      }
      issues.push({
        type: 'missing',
        file,
        message: `${file} is missing`,
        severity: 'error',
        suggestion: fixable
          ? `Run with --auto-fix to scaffold a minimal default, or 'npx @cobusgreyling/loop-init . --pattern daily-triage' for pattern-specific content`
          : `Run 'npx @cobusgreyling/loop-init . --pattern daily-triage' to scaffold required files`,
      });
    }
  }
  
  // Check STATE.md ↔ LOOP.md consistency
  const statePath = path.join(targetDir, 'STATE.md');
  const loopPath = path.join(targetDir, 'LOOP.md');
  
  if (await fileExists(statePath) && await fileExists(loopPath)) {
    const stateContent = await readFileContent(statePath);
    const loopContent = await readFileContent(loopPath);
    
    if (stateContent && loopContent) {
      // Extract patterns from LOOP.md
      const patterns = extractLoopPatterns(loopContent);
      
      // Extract state files from LOOP.md
      const stateFiles = extractStateFiles(loopContent);
      
      // Check if STATE.md is referenced in LOOP.md
      if (!stateFiles.includes('STATE.md')) {
        if (autoFix) {
          if (!dryRun) {
            const separator = /\n\n$/.test(loopContent) ? '' : loopContent.endsWith('\n') ? '\n' : '\n\n';
            await writeFile(loopPath, `${loopContent}${separator}Update STATE.md after each run.\n`);
          }
          issues.push({
            type: 'inconsistent',
            file: 'LOOP.md',
            message: dryRun
              ? 'LOOP.md does not reference STATE.md — would append a reference (--dry-run, no changes made)'
              : 'LOOP.md did not reference STATE.md — appended a reference',
            severity: 'info',
            suggestion: 'Review the appended line and reword it if you prefer.',
          });
        } else {
          issues.push({
            type: 'inconsistent',
            file: 'LOOP.md',
            message: 'LOOP.md does not reference STATE.md',
            severity: 'warning',
            suggestion: 'Add STATE.md to the state files list in LOOP.md, or run with --auto-fix',
          });
        }
      }
      
      // Compare structural similarity
      const { similarity, differences } = compareMarkdownFiles(stateContent, loopContent, 0.5);
      
      if (similarity < 0.3 && differences.length > 0) {
        issues.push({
          type: 'inconsistent',
          file: 'STATE.md ↔ LOOP.md',
          message: 'Low structural similarity between STATE.md and LOOP.md',
          severity: 'warning',
          suggestion: 'Review both files for consistency',
        });
        
        if (verbose) {
          for (const diff of differences.slice(0, 3)) {
            issues.push({
              type: 'inconsistent',
              file: 'STATE.md ↔ LOOP.md',
              message: diff,
              severity: 'info',
            });
          }
        }
      }
    }
  }
  
  // Check gate.yaml ↔ docs/safety.md policy parity. gate.yaml is the
  // machine-readable twin of safety.md's Path Denylist; if they drift,
  // loop-gate enforces something other than the documented policy.
  await checkPolicyDrift(targetDir, issues);

  // Scan skills for version information
  const skillsVersions = await scanSkillsDirectory(targetDir);
  
  const hasSkillsDir = await fileExists(path.join(targetDir, 'skills'))
    || await fileExists(path.join(targetDir, '.grok', 'skills'))
    || await fileExists(path.join(targetDir, '.claude', 'skills'))
    || await fileExists(path.join(targetDir, '.codex', 'skills'));

  if (skillsVersions.size === 0 && hasSkillsDir) {
    suggestions.push('No skills found. Run loop-init to scaffold skills.');
  }
  
  // Calculate score
  let score = 100;
  for (const issue of issues) {
    if (issue.severity === 'error') {
      score -= 20;
    } else if (issue.severity === 'warning') {
      score -= 10;
    } else {
      score -= 1;
    }
  }
  score = Math.max(0, Math.min(100, score));
  
  // Determine level
  let level: 'healthy' | 'warning' | 'critical' = 'healthy';
  if (score < 40) {
    level = 'critical';
  } else if (score < 70) {
    level = 'warning';
  }
  
  // Add suggestions based on issues
  if (issues.some(i => i.type === 'missing')) {
    suggestions.push('Run loop-init to scaffold missing files');
  }
  
  if (issues.some(i => i.type === 'inconsistent')) {
    suggestions.push('Review STATE.md and LOOP.md for consistency');
  }
  
  return {
    score,
    level,
    issues,
    suggestions,
    timestamp: new Date().toISOString(),
  };
}

/**
 * Format report for CLI output
 */
export function formatReport(report: DriftReport): string {
  const lines: string[] = [];
  
  lines.push('');
  lines.push('Loop Sync Report');
  lines.push('══════════════════════════════════════════════════');
  lines.push(`Score: ${report.score}/100 (${report.level})`);
  lines.push('');
  
  if (report.issues.length === 0) {
    lines.push('✅ No issues detected. Configuration is consistent.');
  } else {
    lines.push(`Found ${report.issues.length} issue(s):`);
    lines.push('');
    
    const errors = report.issues.filter(i => i.severity === 'error');
    const warnings = report.issues.filter(i => i.severity === 'warning');
    const infos = report.issues.filter(i => i.severity === 'info');
    
    if (errors.length > 0) {
      lines.push('❌ Errors:');
      for (const issue of errors) {
        lines.push(`   - ${issue.file}: ${issue.message}`);
      }
      lines.push('');
    }
    
    if (warnings.length > 0) {
      lines.push('⚠️ Warnings:');
      for (const issue of warnings) {
        lines.push(`   - ${issue.file}: ${issue.message}`);
      }
      lines.push('');
    }
    
    if (infos.length > 0) {
      lines.push('ℹ️ Information:');
      for (const issue of infos) {
        lines.push(`   - ${issue.file}: ${issue.message}`);
      }
      lines.push('');
    }
  }
  
  if (report.suggestions.length > 0) {
    lines.push('💡 Suggestions:');
    for (const suggestion of report.suggestions) {
      lines.push(`   - ${suggestion}`);
    }
  }
  
  return lines.join('\n');
}
/**
 * Compare docs/safety.md's Path Denylist against gate.yaml and record any
 * divergence. Read-only by design: which side is correct is a human call, so
 * this is never auto-fixed even under --auto-fix.
 */
async function checkPolicyDrift(targetDir: string, issues: DriftIssue[]): Promise<void> {
  const gatePath = path.join(targetDir, 'gate.yaml');
  const safetyPath = path.join(targetDir, 'docs', 'safety.md');

  const gateRaw = await readFileContent(gatePath);
  const safetyRaw = await readFileContent(safetyPath);

  // Missing gate.yaml is already reported by the required-files check, and a
  // project with no docs/safety.md has no prose policy to drift from.
  if (!gateRaw || !safetyRaw) return;

  const gate = extractGateDenylist(gateRaw);
  const safety = extractSafetyDenylist(safetyRaw);

  if (!gate) {
    // loop-gate reports malformed gate files in detail; just point at it.
    issues.push({
      type: 'inconsistent',
      file: 'gate.yaml',
      message: 'gate.yaml has no readable denylist — cannot verify it matches docs/safety.md',
      severity: 'warning',
      suggestion: "Run 'npx @cobusgreyling/loop-gate check --action commit --paths .' to see the parse error",
    });
    return;
  }

  if (!safety) {
    issues.push({
      type: 'inconsistent',
      file: 'docs/safety.md',
      message: 'docs/safety.md has no "Path Denylist" code block to compare against gate.yaml',
      severity: 'info',
      suggestion: 'Document the denylist in docs/safety.md so the enforced policy has a prose twin',
    });
    return;
  }

  const drift = comparePolicies(safety, gate, findUndocumentedLimits(gateRaw, safetyRaw));
  if (!hasDrift(drift)) return;

  // Documented-but-unenforced is the dangerous direction: readers trust
  // safety.md while loop-gate silently allows the path.
  if (drift.missingInGate.length > 0) {
    issues.push({
      type: 'inconsistent',
      file: 'gate.yaml ↔ docs/safety.md',
      message: `${drift.missingInGate.length} denylist ${plural(drift.missingInGate.length, 'path')} documented in docs/safety.md but not enforced by gate.yaml: ${drift.missingInGate.join(', ')}`,
      severity: 'error',
      suggestion: 'Add these entries to gate.yaml\'s denylist, or remove them from docs/safety.md if intentional',
    });
  }

  if (drift.missingInDoc.length > 0) {
    issues.push({
      type: 'inconsistent',
      file: 'gate.yaml ↔ docs/safety.md',
      message: `${drift.missingInDoc.length} denylist ${plural(drift.missingInDoc.length, 'path')} enforced by gate.yaml but undocumented in docs/safety.md: ${drift.missingInDoc.join(', ')}`,
      severity: 'warning',
      suggestion: 'Add these entries to the Path Denylist block in docs/safety.md so humans see what is blocked',
    });
  }

  if (drift.undocumentedLimits.length > 0) {
    issues.push({
      type: 'inconsistent',
      file: 'gate.yaml ↔ docs/safety.md',
      message: `gate.yaml sets ${drift.undocumentedLimits.join(', ')} but docs/safety.md never states this limit`,
      severity: 'info',
      suggestion: 'Document the limit in docs/safety.md so the escalation threshold is discoverable',
    });
  }
}

function plural(n: number, word: string): string {
  return n === 1 ? word : `${word}s`;
}