Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
feat(loop-cost): model multi-agent orchestration cost on the action path (#256)
JonyT Agenticc142dfd56eee
diff --git a/tools/loop-cost/README.md b/tools/loop-cost/README.mdindex c383cc7..4c4aa28 100644--- a/tools/loop-cost/README.md+++ b/tools/loop-cost/README.md@@ -27,6 +27,7 @@ npm test| `--pattern` | Pattern id (see `--list`) || `--cadence` | Override cadence (e.g. `15m`, `1d`) || `--level` | `L1`, `L2`, or `L3` (default `L1`) |+| `--orchestration` | Multi-agent action cost: `single` (default), `maker-checker`, `parallel:N`, `debate:R` || `--conservative` | Use slower cadence from ranges || `--json` | Machine-readable output |@@ -41,6 +42,29 @@ Each estimate includes:Pair with `loop-budget.md` (scaffolded by `loop-init`) and `loop-audit` cost observability checks.+## Orchestration cost++The scenarios above assume one implementer pass. A loop that fans out to+multiple agents costs more on the **action path** (the no-op scan and single+triage pass are unaffected). `--orchestration` applies a multiplier so the+estimate — and the realistic per-run cap that feeds the circuit breaker —+reflects the real shape of the run:++| Mode | Multiplier | Models |+|------|-----------|--------|+| `single` (default) | 1x | One implementer pass, self-checked |+| `maker-checker` | 2x | Implementer + an independent verifier pass (the L2+ default) |+| `parallel:N` | N+1 | N candidate agents fan out, plus a merge/arbiter pass |+| `debate:R` | 1+R | One proposer plus R critique-and-revise rounds |++```bash+npx @cobusgreyling/loop-cost --pattern ci-sweeper --level L2 --orchestration maker-checker+npx @cobusgreyling/loop-cost --pattern ci-sweeper --level L2 --orchestration parallel:3+```++Multipliers above 2x emit a warning — deep fan-out or debate is easy to enable+and expensive to run unattended.+## Feed the circuit breaker`--json` output resolves a realistic per-run token cap fordiff --git a/tools/loop-cost/dist/cli.js b/tools/loop-cost/dist/cli.jsindex 6d324d6..1d12976 100755--- a/tools/loop-cost/dist/cli.js+++ b/tools/loop-cost/dist/cli.js@@ -3,7 +3,7 @@ import { readFile, access } from 'node:fs/promises';import path from 'node:path';import { fileURLToPath } from 'node:url';import yaml from 'yaml';-import { assertValidLevel, estimateCost, formatEstimateHuman, } from './estimator.js';+import { assertValidLevel, estimateCost, formatEstimateHuman, parseOrchestration, } from './estimator.js';const __dirname = path.dirname(fileURLToPath(import.meta.url));const PACKAGE_ROOT = path.resolve(__dirname, '..');function parseArgs(argv) {@@ -13,6 +13,7 @@ function parseArgs(argv) {let conservative = false;let json = false;let list = false;+ let orchestration = 'single';for (let i = 0; i < argv.length; i++) {const a = argv[i];if (a === '--pattern' || a === '-p')@@ -21,6 +22,8 @@ function parseArgs(argv) {cadence = argv[++i];else if (a === '--level' || a === '-l')level = argv[++i];+ else if (a === '--orchestration' || a === '-o')+ orchestration = argv[++i];else if (a === '--conservative')conservative = true;else if (a === '--json')@@ -30,7 +33,7 @@ function parseArgs(argv) {else if (a === '--help' || a === '-h')return { help: true };}- return { help: false, pattern, cadence, level, conservative, json, list };+ return { help: false, pattern, cadence, level, conservative, json, list, orchestration };}async function loadRegistry() {const candidates = [@@ -63,6 +66,9 @@ Options:-p, --pattern <id> Pattern id (default: daily-triage)-c, --cadence <spec> Override cadence (e.g. 15m, 1d, 5m-15m)-l, --level <L1|L2|L3> Readiness level (default: L1)+ -o, --orchestration <mode>+ Multi-agent action cost: single (default),+ maker-checker, parallel:N, debate:R--conservative Use slower cadence from ranges (e.g. 15m not 5m)--json Machine-readable output--list List pattern ids@@ -70,6 +76,7 @@ Options:Examples:loop-cost --pattern ci-sweeper --cadence 15m --level L2+ loop-cost --pattern ci-sweeper --level L2 --orchestration maker-checkerloop-cost --pattern daily-triage --level L1 --jsonloop-cost --list`);@@ -89,6 +96,7 @@ Examples:}try {assertValidLevel(args.level);+ parseOrchestration(args.orchestration);}catch (err) {const msg = err instanceof Error ? err.message : String(err);@@ -104,6 +112,7 @@ Examples:cadence: args.cadence,level: args.level,conservative: args.conservative,+ orchestration: args.orchestration,});if (args.json)console.log(JSON.stringify(result, null, 2));diff --git a/tools/loop-cost/dist/estimator.d.ts b/tools/loop-cost/dist/estimator.d.tsindex b5da5e7..8011c28 100644--- a/tools/loop-cost/dist/estimator.d.ts+++ b/tools/loop-cost/dist/estimator.d.ts@@ -18,11 +18,30 @@ export interface RegistryPattern {export interface RegistryDoc {patterns: RegistryPattern[];}+export interface Orchestration {+ /** Normalized mode string, e.g. 'single', 'maker-checker', 'parallel:3', 'debate:2'. */+ mode: string;+ /** Multiplier applied to the action-path token cost. */+ multiplier: number;+}+/**+ * Parse an orchestration spec into a multiplier on the action path.+ *+ * The action path (implementer + verifier work) is where multi-agent+ * orchestration lands; the no-op scan and single triage pass are unaffected.+ *+ * single 1x one implementer pass, self-checked (default)+ * maker-checker 2x implementer + an independent verifier pass+ * parallel:N N+1 N candidate agents fan out, plus a merge/arbiter pass+ * debate:R 1+R one proposer plus R critique-and-revise rounds+ */+export declare function parseOrchestration(spec: string | undefined): Orchestration;export interface EstimateInput {pattern: RegistryPattern;cadence?: string;level: ReadinessLevel;conservative?: boolean;+ orchestration?: string;}export interface EstimateResult {patternId: string;@@ -33,6 +52,7 @@ export interface EstimateResult {tokenCostTier: string;suggestedDailyCap: number;earlyExitRequired: boolean;+ orchestration: Orchestration;scenarios: {noop: {tokensPerRun: number;diff --git a/tools/loop-cost/dist/estimator.js b/tools/loop-cost/dist/estimator.jsindex 18988c4..24809df 100644--- a/tools/loop-cost/dist/estimator.js+++ b/tools/loop-cost/dist/estimator.js@@ -4,6 +4,39 @@ export function assertValidLevel(level) {throw new Error(`Invalid level: ${level}. Valid: ${VALID_READINESS_LEVELS.join(', ')}`);}}+/**+ * Parse an orchestration spec into a multiplier on the action path.+ *+ * The action path (implementer + verifier work) is where multi-agent+ * orchestration lands; the no-op scan and single triage pass are unaffected.+ *+ * single 1x one implementer pass, self-checked (default)+ * maker-checker 2x implementer + an independent verifier pass+ * parallel:N N+1 N candidate agents fan out, plus a merge/arbiter pass+ * debate:R 1+R one proposer plus R critique-and-revise rounds+ */+export function parseOrchestration(spec) {+ const s = (spec ?? 'single').trim().toLowerCase();+ if (s === '' || s === 'single')+ return { mode: 'single', multiplier: 1 };+ if (s === 'maker-checker')+ return { mode: 'maker-checker', multiplier: 2 };+ const parallel = s.match(/^parallel:(\d+)$/);+ if (parallel) {+ const n = Number(parallel[1]);+ if (n < 2)+ throw new Error(`Invalid orchestration: parallel:N needs N>=2 (got ${n}).`);+ return { mode: `parallel:${n}`, multiplier: n + 1 };+ }+ const debate = s.match(/^debate:(\d+)$/);+ if (debate) {+ const r = Number(debate[1]);+ if (r < 1)+ throw new Error(`Invalid orchestration: debate:R needs R>=1 (got ${r}).`);+ return { mode: `debate:${r}`, multiplier: 1 + r };+ }+ throw new Error(`Invalid orchestration: ${spec}. Valid: single, maker-checker, parallel:N, debate:R`);+}const INTERVAL_MS = {m: 60_000,h: 3_600_000,@@ -73,12 +106,16 @@ export function estimateCost(input) {const runsPerDay = cadenceToRunsPerDay(cadence, input.conservative);const { cost, token_cost: tokenCostTier } = input.pattern;const mix = realisticMix(input.level, cost.early_exit_required);+ const orchestration = parseOrchestration(input.orchestration);+ // Orchestration overhead lands on the action path only — the no-op scan and+ // the single triage pass do not fan out.+ const actionPerRun = Math.round(cost.tokens_action * orchestration.multiplier);const noopDay = cost.tokens_noop * runsPerDay;const reportDay = cost.tokens_report * runsPerDay;- const actionDay = cost.tokens_action * runsPerDay;+ const actionDay = actionPerRun * runsPerDay;const realisticPerRun = cost.tokens_noop * mix.noop +cost.tokens_report * mix.report +- cost.tokens_action * mix.action;+ actionPerRun * mix.action;const realisticDay = Math.round(realisticPerRun * runsPerDay);const warnings = [];if (cost.early_exit_required) {@@ -93,6 +130,9 @@ export function estimateCost(input) {if (runsPerDay >= 96) {warnings.push(`High cadence (${runsPerDay} runs/day) — verify early-exit is working.`);}+ if (orchestration.multiplier > 2) {+ warnings.push(`Orchestration ${orchestration.mode} multiplies action cost ${orchestration.multiplier}x — confirm the fan-out or debate depth is justified.`);+ }return {patternId: input.pattern.id,patternName: input.pattern.name,@@ -102,10 +142,11 @@ export function estimateCost(input) {tokenCostTier,suggestedDailyCap: cost.suggested_daily_cap,earlyExitRequired: cost.early_exit_required,+ orchestration,scenarios: {noop: { tokensPerRun: cost.tokens_noop, tokensPerDay: noopDay },report: { tokensPerRun: cost.tokens_report, tokensPerDay: reportDay },- action: { tokensPerRun: cost.tokens_action, tokensPerDay: actionDay },+ action: { tokensPerRun: actionPerRun, tokensPerDay: actionDay },realistic: {tokensPerRun: Math.round(realisticPerRun),tokensPerDay: realisticDay,@@ -122,6 +163,9 @@ export function formatEstimateHuman(r) {lines.push('═'.repeat(50));lines.push(`Cadence: ${r.cadence} → ${r.runsPerDay} runs/day`);lines.push(`Level: ${r.level} · Registry tier: ${r.tokenCostTier}`);+ if (r.orchestration.multiplier > 1) {+ lines.push(`Orchestration: ${r.orchestration.mode} · action x${r.orchestration.multiplier}`);+ }lines.push(`Suggested daily cap: ${formatTokens(r.suggestedDailyCap)} tokens`);lines.push('');lines.push('Daily token estimates:');diff --git a/tools/loop-cost/package.json b/tools/loop-cost/package.jsonindex a56e4a7..2f1ebe1 100644--- a/tools/loop-cost/package.json+++ b/tools/loop-cost/package.json@@ -1,6 +1,6 @@{"name": "@cobusgreyling/loop-cost",- "version": "1.0.3",+ "version": "1.1.0","description": "Estimate token budgets and daily costs for loop engineering patterns. By cadence, readiness level (L1/L2/L3), and coding agent tool.","type": "module","bin": {diff --git a/tools/loop-cost/src/cli.ts b/tools/loop-cost/src/cli.tsindex d7e6fe7..c40c164 100644--- a/tools/loop-cost/src/cli.ts+++ b/tools/loop-cost/src/cli.ts@@ -7,6 +7,7 @@ import {assertValidLevel,estimateCost,formatEstimateHuman,+ parseOrchestration,type ReadinessLevel,type RegistryDoc,} from './estimator.js';@@ -21,19 +22,21 @@ function parseArgs(argv: string[]) {let conservative = false;let json = false;let list = false;+ let orchestration = 'single';for (let i = 0; i < argv.length; i++) {const a = argv[i];if (a === '--pattern' || a === '-p') pattern = argv[++i];else if (a === '--cadence' || a === '-c') cadence = argv[++i];else if (a === '--level' || a === '-l') level = argv[++i] as ReadinessLevel;+ else if (a === '--orchestration' || a === '-o') orchestration = argv[++i];else if (a === '--conservative') conservative = true;else if (a === '--json') json = true;else if (a === '--list') list = true;else if (a === '--help' || a === '-h') return { help: true as const };}- return { help: false as const, pattern, cadence, level, conservative, json, list };+ return { help: false as const, pattern, cadence, level, conservative, json, list, orchestration };}async function loadRegistry(): Promise<RegistryDoc> {@@ -68,6 +71,9 @@ Options:-p, --pattern <id> Pattern id (default: daily-triage)-c, --cadence <spec> Override cadence (e.g. 15m, 1d, 5m-15m)-l, --level <L1|L2|L3> Readiness level (default: L1)+ -o, --orchestration <mode>+ Multi-agent action cost: single (default),+ maker-checker, parallel:N, debate:R--conservative Use slower cadence from ranges (e.g. 15m not 5m)--json Machine-readable output--list List pattern ids@@ -75,6 +81,7 @@ Options:Examples:loop-cost --pattern ci-sweeper --cadence 15m --level L2+ loop-cost --pattern ci-sweeper --level L2 --orchestration maker-checkerloop-cost --pattern daily-triage --level L1 --jsonloop-cost --list`);@@ -98,6 +105,7 @@ Examples:try {assertValidLevel(args.level);+ parseOrchestration(args.orchestration);} catch (err: unknown) {const msg = err instanceof Error ? err.message : String(err);console.error(msg);@@ -114,6 +122,7 @@ Examples:cadence: args.cadence,level: args.level,conservative: args.conservative,+ orchestration: args.orchestration,});if (args.json) console.log(JSON.stringify(result, null, 2));diff --git a/tools/loop-cost/src/estimator.ts b/tools/loop-cost/src/estimator.tsindex 7814f03..39129f3 100644--- a/tools/loop-cost/src/estimator.ts+++ b/tools/loop-cost/src/estimator.ts@@ -28,11 +28,54 @@ export interface RegistryDoc {patterns: RegistryPattern[];}+export interface Orchestration {+ /** Normalized mode string, e.g. 'single', 'maker-checker', 'parallel:3', 'debate:2'. */+ mode: string;+ /** Multiplier applied to the action-path token cost. */+ multiplier: number;+}++/**+ * Parse an orchestration spec into a multiplier on the action path.+ *+ * The action path (implementer + verifier work) is where multi-agent+ * orchestration lands; the no-op scan and single triage pass are unaffected.+ *+ * single 1x one implementer pass, self-checked (default)+ * maker-checker 2x implementer + an independent verifier pass+ * parallel:N N+1 N candidate agents fan out, plus a merge/arbiter pass+ * debate:R 1+R one proposer plus R critique-and-revise rounds+ */+export function parseOrchestration(spec: string | undefined): Orchestration {+ const s = (spec ?? 'single').trim().toLowerCase();+ if (s === '' || s === 'single') return { mode: 'single', multiplier: 1 };+ if (s === 'maker-checker') return { mode: 'maker-checker', multiplier: 2 };++ const parallel = s.match(/^parallel:(\d+)$/);+ if (parallel) {+ const n = Number(parallel[1]);+ if (n < 2) throw new Error(`Invalid orchestration: parallel:N needs N>=2 (got ${n}).`);+ return { mode: `parallel:${n}`, multiplier: n + 1 };+ }++ const debate = s.match(/^debate:(\d+)$/);+ if (debate) {+ const r = Number(debate[1]);+ if (r < 1) throw new Error(`Invalid orchestration: debate:R needs R>=1 (got ${r}).`);+ return { mode: `debate:${r}`, multiplier: 1 + r };+ }++ throw new Error(+ `Invalid orchestration: ${spec}. Valid: single, maker-checker, parallel:N, debate:R`,+ );+}+export interface EstimateInput {pattern: RegistryPattern;cadence?: string;level: ReadinessLevel;conservative?: boolean;+ orchestration?: string;}export interface EstimateResult {@@ -44,6 +87,7 @@ export interface EstimateResult {tokenCostTier: string;suggestedDailyCap: number;earlyExitRequired: boolean;+ orchestration: Orchestration;scenarios: {noop: { tokensPerRun: number; tokensPerDay: number };report: { tokensPerRun: number; tokensPerDay: number };@@ -130,15 +174,20 @@ export function estimateCost(input: EstimateInput): EstimateResult {const runsPerDay = cadenceToRunsPerDay(cadence, input.conservative);const { cost, token_cost: tokenCostTier } = input.pattern;const mix = realisticMix(input.level, cost.early_exit_required);+ const orchestration = parseOrchestration(input.orchestration);++ // Orchestration overhead lands on the action path only — the no-op scan and+ // the single triage pass do not fan out.+ const actionPerRun = Math.round(cost.tokens_action * orchestration.multiplier);const noopDay = cost.tokens_noop * runsPerDay;const reportDay = cost.tokens_report * runsPerDay;- const actionDay = cost.tokens_action * runsPerDay;+ const actionDay = actionPerRun * runsPerDay;const realisticPerRun =cost.tokens_noop * mix.noop +cost.tokens_report * mix.report +- cost.tokens_action * mix.action;+ actionPerRun * mix.action;const realisticDay = Math.round(realisticPerRun * runsPerDay);const warnings: string[] = [];@@ -156,6 +205,11 @@ export function estimateCost(input: EstimateInput): EstimateResult {if (runsPerDay >= 96) {warnings.push(`High cadence (${runsPerDay} runs/day) — verify early-exit is working.`);}+ if (orchestration.multiplier > 2) {+ warnings.push(+ `Orchestration ${orchestration.mode} multiplies action cost ${orchestration.multiplier}x — confirm the fan-out or debate depth is justified.`,+ );+ }return {patternId: input.pattern.id,@@ -166,10 +220,11 @@ export function estimateCost(input: EstimateInput): EstimateResult {tokenCostTier,suggestedDailyCap: cost.suggested_daily_cap,earlyExitRequired: cost.early_exit_required,+ orchestration,scenarios: {noop: { tokensPerRun: cost.tokens_noop, tokensPerDay: noopDay },report: { tokensPerRun: cost.tokens_report, tokensPerDay: reportDay },- action: { tokensPerRun: cost.tokens_action, tokensPerDay: actionDay },+ action: { tokensPerRun: actionPerRun, tokensPerDay: actionDay },realistic: {tokensPerRun: Math.round(realisticPerRun),tokensPerDay: realisticDay,@@ -187,6 +242,9 @@ export function formatEstimateHuman(r: EstimateResult): string {lines.push('═'.repeat(50));lines.push(`Cadence: ${r.cadence} → ${r.runsPerDay} runs/day`);lines.push(`Level: ${r.level} · Registry tier: ${r.tokenCostTier}`);+ if (r.orchestration.multiplier > 1) {+ lines.push(`Orchestration: ${r.orchestration.mode} · action x${r.orchestration.multiplier}`);+ }lines.push(`Suggested daily cap: ${formatTokens(r.suggestedDailyCap)} tokens`);lines.push('');lines.push('Daily token estimates:');diff --git a/tools/loop-cost/test/estimator.test.mjs b/tools/loop-cost/test/estimator.test.mjsindex d1fce3a..cd65a86 100644--- a/tools/loop-cost/test/estimator.test.mjs+++ b/tools/loop-cost/test/estimator.test.mjs@@ -5,6 +5,7 @@ import {cadenceToRunsPerDay,runsPerDayForInterval,estimateCost,+ parseOrchestration,} from '../dist/estimator.js';const CI_SWEEPER = {@@ -53,6 +54,56 @@ test('assertValidLevel: rejects unknown level', () => {assert.throws(() => assertValidLevel('garbage'), /Invalid level/);});+test('parseOrchestration: modes map to expected multipliers', () => {+ assert.equal(parseOrchestration(undefined).multiplier, 1);+ assert.equal(parseOrchestration('single').multiplier, 1);+ assert.equal(parseOrchestration('maker-checker').multiplier, 2);+ assert.equal(parseOrchestration('parallel:3').multiplier, 4);+ assert.equal(parseOrchestration('debate:2').multiplier, 3);+});++test('parseOrchestration: rejects bad specs', () => {+ assert.throws(() => parseOrchestration('parallel:1'), /parallel/);+ assert.throws(() => parseOrchestration('debate:0'), /debate/);+ assert.throws(() => parseOrchestration('garbage'), /Invalid orchestration/);+});++test('estimateCost: orchestration scales only the action path', () => {+ const base = estimateCost({ pattern: CI_SWEEPER, cadence: '15m', level: 'L2' });+ const mc = estimateCost({+ pattern: CI_SWEEPER,+ cadence: '15m',+ level: 'L2',+ orchestration: 'maker-checker',+ });+ assert.equal(mc.orchestration.mode, 'maker-checker');+ assert.equal(mc.orchestration.multiplier, 2);+ // action doubles; no-op and full-triage scans are untouched.+ assert.equal(mc.scenarios.action.tokensPerRun, base.scenarios.action.tokensPerRun * 2);+ assert.equal(mc.scenarios.noop.tokensPerRun, base.scenarios.noop.tokensPerRun);+ assert.equal(mc.scenarios.report.tokensPerRun, base.scenarios.report.tokensPerRun);+ // realistic blend rises because its action component is scaled.+ assert.ok(mc.scenarios.realistic.tokensPerDay > base.scenarios.realistic.tokensPerDay);+});++test('estimateCost: default single leaves action unchanged and adds no warning', () => {+ const r = estimateCost({ pattern: CI_SWEEPER, cadence: '15m', level: 'L2' });+ assert.equal(r.orchestration.mode, 'single');+ assert.equal(r.scenarios.action.tokensPerRun, CI_SWEEPER.cost.tokens_action);+ assert.ok(!r.warnings.some((w) => /Orchestration/.test(w)));+});++test('estimateCost: deep fan-out warns', () => {+ const r = estimateCost({+ pattern: CI_SWEEPER,+ cadence: '15m',+ level: 'L2',+ orchestration: 'parallel:4',+ });+ assert.equal(r.orchestration.multiplier, 5);+ assert.ok(r.warnings.some((w) => /Orchestration/.test(w)));+});+test('estimateCost: daily-triage 1d L1 is cheap', () => {const r = estimateCost({pattern: {