Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

10.7 KB
Raw Blame
export type ReadinessLevel = 'L1' | 'L2' | 'L3';

export const VALID_READINESS_LEVELS: ReadinessLevel[] = ['L1', 'L2', 'L3'];

export function assertValidLevel(level: string): asserts level is ReadinessLevel {
  if (!VALID_READINESS_LEVELS.includes(level as ReadinessLevel)) {
    throw new Error(`Invalid level: ${level}. Valid: ${VALID_READINESS_LEVELS.join(', ')}`);
  }
}

export interface PatternCost {
  tokens_noop: number;
  tokens_report: number;
  tokens_action: number;
  /** Fraction (0.0-1.0) of report/action tokens that are stable, repeated
   *  content (STATE.md, skills, system prompt) eligible for prompt caching.
   *  Optional — patterns without it are treated as fully variable (0). */
  stable_fraction?: number;
  suggested_daily_cap: number;
  early_exit_required: boolean;
}

export interface RegistryPattern {
  id: string;
  name: string;
  cadence: string;
  token_cost: string;
  cost: PatternCost;
}

export interface RegistryDoc {
  patterns: RegistryPattern[];
}

export interface Orchestration {
  /** Normalized mode string, e.g. 'single', 'maker-checker', 'parallel:3', 'debate:2'. */
  mode: string;
  /** Multiplier applied to the action-path token cost. */
  multiplier: number;
}

/**
 * Parse an orchestration spec into a multiplier on the action path.
 *
 * The action path (implementer + verifier work) is where multi-agent
 * orchestration lands; the no-op scan and single triage pass are unaffected.
 *
 *   single         1x   one implementer pass, self-checked (default)
 *   maker-checker  2x   implementer + an independent verifier pass
 *   parallel:N     N+1  N candidate agents fan out, plus a merge/arbiter pass
 *   debate:R       1+R  one proposer plus R critique-and-revise rounds
 */
export function parseOrchestration(spec: string | undefined): Orchestration {
  const s = (spec ?? 'single').trim().toLowerCase();
  if (s === '' || s === 'single') return { mode: 'single', multiplier: 1 };
  if (s === 'maker-checker') return { mode: 'maker-checker', multiplier: 2 };

  const parallel = s.match(/^parallel:(\d+)$/);
  if (parallel) {
    const n = Number(parallel[1]);
    if (n < 2) throw new Error(`Invalid orchestration: parallel:N needs N>=2 (got ${n}).`);
    return { mode: `parallel:${n}`, multiplier: n + 1 };
  }

  const debate = s.match(/^debate:(\d+)$/);
  if (debate) {
    const r = Number(debate[1]);
    if (r < 1) throw new Error(`Invalid orchestration: debate:R needs R>=1 (got ${r}).`);
    return { mode: `debate:${r}`, multiplier: 1 + r };
  }

  throw new Error(
    `Invalid orchestration: ${spec}. Valid: single, maker-checker, parallel:N, debate:R`,
  );
}

export interface EstimateInput {
  pattern: RegistryPattern;
  cadence?: string;
  level: ReadinessLevel;
  conservative?: boolean;
  orchestration?: string;
  withCaching?: boolean;
}

export interface EstimateResult {
  patternId: string;
  patternName: string;
  cadence: string;
  level: ReadinessLevel;
  runsPerDay: number;
  tokenCostTier: string;
  suggestedDailyCap: number;
  earlyExitRequired: boolean;
  orchestration: Orchestration;
  scenarios: {
    noop: { tokensPerRun: number; tokensPerDay: number };
    report: { tokensPerRun: number; tokensPerDay: number };
    action: { tokensPerRun: number; tokensPerDay: number };
    realistic: { tokensPerRun: number; tokensPerDay: number; assumptions: string };
    caching?: { tokensPerRun: number; tokensPerDay: number; savingsPercent: number };
  };
  warnings: string[];
}

const INTERVAL_MS: Record<string, number> = {
  m: 60_000,
  h: 3_600_000,
  d: 86_400_000,
};

export function parseInterval(token: string): number {
  const m = token.match(/^(\d+)([mhd])$/);
  if (!m) throw new Error(`Invalid cadence interval: ${token}`);
  const unit = m[2] as keyof typeof INTERVAL_MS;
  const value = Number(m[1]);
  if (value <= 0) {
    throw new Error(`Invalid cadence interval: ${token}. The interval value must be greater than zero.`);
  }
  return value * INTERVAL_MS[unit];
}

/** Runs per day for a single interval like 15m or 1d. */
export function runsPerDayForInterval(interval: string): number {
  const ms = parseInterval(interval);
  return Math.floor(86_400_000 / ms);
}

/**
 * Parse pattern cadence (e.g. 5m-15m, 1d-2h) into runs/day.
 * conservative=true picks the slower cadence in a range.
 */
export function cadenceToRunsPerDay(cadence: string, conservative = false): number {
  const parts = cadence.split('-').map((p) => p.trim());
  if (parts.length === 1) return runsPerDayForInterval(parts[0]);

  const runs = parts.map(runsPerDayForInterval);
  return conservative ? Math.min(...runs) : Math.max(...runs);
}

function realisticMix(level: ReadinessLevel, earlyExitRequired: boolean): {
  noop: number;
  report: number;
  action: number;
  assumptions: string;
} {
  if (level === 'L1') {
    return {
      noop: earlyExitRequired ? 0.9 : 0.6,
      report: earlyExitRequired ? 0.1 : 0.4,
      action: 0,
      assumptions: earlyExitRequired
        ? 'L1: 90% early-exit, 10% full triage'
        : 'L1: 60% no-op, 40% full triage',
    };
  }
  if (level === 'L2') {
    return {
      noop: earlyExitRequired ? 0.85 : 0.5,
      report: earlyExitRequired ? 0.1 : 0.3,
      action: earlyExitRequired ? 0.05 : 0.2,
      assumptions: earlyExitRequired
        ? 'L2: 85% early-exit, 10% triage, 5% implementer+verifier'
        : 'L2: 50% no-op, 30% triage, 20% action',
    };
  }
  return {
    noop: 0.4,
    report: 0.35,
    action: 0.25,
    assumptions: 'L3: 40% no-op, 35% triage, 25% action (unattended — monitor closely)',
  };
}

function formatTokens(n: number): string {
  if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
  if (n >= 1_000) return `${Math.round(n / 1_000)}k`;
  return String(n);
}

// Anthropic prompt caching: cache reads bill at roughly 10% of base input
// token cost. Only the stable portion of a run's tokens (STATE.md, skills,
// system prompt — unchanged since the last cache write) benefit; the
// variable portion (fresh scan results, diffs) still costs full price.
const CACHE_READ_DISCOUNT = 0.1;

function applyCaching(tokensPerRun: number, stableFraction: number): number {
  const stable = tokensPerRun * stableFraction;
  const variable = tokensPerRun - stable;
  return Math.round(variable + stable * CACHE_READ_DISCOUNT);
}

export function estimateCost(input: EstimateInput): EstimateResult {
  assertValidLevel(input.level);
  const cadence = input.cadence ?? input.pattern.cadence;
  const runsPerDay = cadenceToRunsPerDay(cadence, input.conservative);
  const { cost, token_cost: tokenCostTier } = input.pattern;
  const mix = realisticMix(input.level, cost.early_exit_required);
  const orchestration = parseOrchestration(input.orchestration);

  // Orchestration overhead lands on the action path only — the no-op scan and
  // the single triage pass do not fan out.
  const actionPerRun = Math.round(cost.tokens_action * orchestration.multiplier);

  const noopDay = cost.tokens_noop * runsPerDay;
  const reportDay = cost.tokens_report * runsPerDay;
  const actionDay = actionPerRun * runsPerDay;

  const realisticPerRun =
    cost.tokens_noop * mix.noop +
    cost.tokens_report * mix.report +
    actionPerRun * mix.action;
  const realisticDay = Math.round(realisticPerRun * runsPerDay);

  let caching: EstimateResult['scenarios']['caching'];
  if (input.withCaching && cost.stable_fraction) {
    const cachedPerRun = applyCaching(realisticPerRun, cost.stable_fraction);
    const cachedDay = Math.round(cachedPerRun * runsPerDay);
    const savingsPercent = Math.round((1 - cachedPerRun / realisticPerRun) * 100);
    caching = { tokensPerRun: cachedPerRun, tokensPerDay: cachedDay, savingsPercent };
  }

  const warnings: string[] = [];
  if (cost.early_exit_required) {
    warnings.push('Early-exit triage is required — empty watchlist should exit in <5k tokens.');
  }
  if (actionDay > cost.suggested_daily_cap) {
    warnings.push(
      `Worst case (action every run) exceeds suggested cap (${formatTokens(cost.suggested_daily_cap)}/day).`,
    );
  }
  if (realisticDay > cost.suggested_daily_cap) {
    warnings.push(`Realistic estimate exceeds suggested daily cap — slow cadence or tighten scope.`);
  }
  if (runsPerDay >= 96) {
    warnings.push(`High cadence (${runsPerDay} runs/day) — verify early-exit is working.`);
  }
  if (orchestration.multiplier > 2) {
    warnings.push(
      `Orchestration ${orchestration.mode} multiplies action cost ${orchestration.multiplier}x — confirm the fan-out or debate depth is justified.`,
    );
  }

  return {
    patternId: input.pattern.id,
    patternName: input.pattern.name,
    cadence,
    level: input.level,
    runsPerDay,
    tokenCostTier,
    suggestedDailyCap: cost.suggested_daily_cap,
    earlyExitRequired: cost.early_exit_required,
    orchestration,
    scenarios: {
      noop: { tokensPerRun: cost.tokens_noop, tokensPerDay: noopDay },
      report: { tokensPerRun: cost.tokens_report, tokensPerDay: reportDay },
      action: { tokensPerRun: actionPerRun, tokensPerDay: actionDay },
      realistic: {
        tokensPerRun: Math.round(realisticPerRun),
        tokensPerDay: realisticDay,
        assumptions: mix.assumptions,
      },
      caching,
    },
    warnings,
  };
}

export function formatEstimateHuman(r: EstimateResult): string {
  const lines: string[] = [];
  lines.push('');
  lines.push(`Loop Cost Estimate — ${r.patternName} (${r.patternId})`);
  lines.push('═'.repeat(50));
  lines.push(`Cadence: ${r.cadence}  →  ${r.runsPerDay} runs/day`);
  lines.push(`Level: ${r.level}  ·  Registry tier: ${r.tokenCostTier}`);
  if (r.orchestration.multiplier > 1) {
    lines.push(`Orchestration: ${r.orchestration.mode}  ·  action x${r.orchestration.multiplier}`);
  }
  lines.push(`Suggested daily cap: ${formatTokens(r.suggestedDailyCap)} tokens`);
  lines.push('');
  lines.push('Daily token estimates:');
  lines.push(`  Early-exit / no-op:  ${formatTokens(r.scenarios.noop.tokensPerDay)}  (${formatTokens(r.scenarios.noop.tokensPerRun)}/run)`);
  lines.push(`  Full triage:         ${formatTokens(r.scenarios.report.tokensPerDay)}  (${formatTokens(r.scenarios.report.tokensPerRun)}/run)`);
  lines.push(`  Action every run:    ${formatTokens(r.scenarios.action.tokensPerDay)}  (${formatTokens(r.scenarios.action.tokensPerRun)}/run)`);
  lines.push(`  Realistic blend:     ${formatTokens(r.scenarios.realistic.tokensPerDay)}  (${r.scenarios.realistic.assumptions})`);
  if (r.scenarios.caching) {
    lines.push(`  With prompt caching: ${formatTokens(r.scenarios.caching.tokensPerDay)}  (${r.scenarios.caching.savingsPercent}% reduction vs. realistic blend)`);
  }
  if (r.warnings.length) {
    lines.push('');
    lines.push('Warnings:');
    for (const w of r.warnings) lines.push(`  ! ${w}`);
  }
  lines.push('');
  lines.push('Docs: docs/operating-loops.md · Scaffold: npx @cobusgreyling/loop-init');
  lines.push('');
  return lines.join('\n');
}