Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

feat(loop-cost): model multi-agent orchestration cost on the action path (#256)

JonyT Agenticc142dfd56eee
diff --git a/tools/loop-cost/README.md b/tools/loop-cost/README.md
index c383cc7..4c4aa28 100644
--- a/tools/loop-cost/README.md
+++ b/tools/loop-cost/README.md
@@ -27,6 +27,7 @@ npm test
| `--pattern` | Pattern id (see `--list`) |
| `--cadence` | Override cadence (e.g. `15m`, `1d`) |
| `--level` | `L1`, `L2`, or `L3` (default `L1`) |
+| `--orchestration` | Multi-agent action cost: `single` (default), `maker-checker`, `parallel:N`, `debate:R` |
| `--conservative` | Use slower cadence from ranges |
| `--json` | Machine-readable output |
@@ -41,6 +42,29 @@ Each estimate includes:
Pair with `loop-budget.md` (scaffolded by `loop-init`) and `loop-audit` cost observability checks.
+## Orchestration cost
+
+The scenarios above assume one implementer pass. A loop that fans out to
+multiple agents costs more on the **action path** (the no-op scan and single
+triage pass are unaffected). `--orchestration` applies a multiplier so the
+estimate — and the realistic per-run cap that feeds the circuit breaker —
+reflects the real shape of the run:
+
+| Mode | Multiplier | Models |
+|------|-----------|--------|
+| `single` (default) | 1x | One implementer pass, self-checked |
+| `maker-checker` | 2x | Implementer + an independent verifier pass (the L2+ default) |
+| `parallel:N` | N+1 | N candidate agents fan out, plus a merge/arbiter pass |
+| `debate:R` | 1+R | One proposer plus R critique-and-revise rounds |
+
+```bash
+npx @cobusgreyling/loop-cost --pattern ci-sweeper --level L2 --orchestration maker-checker
+npx @cobusgreyling/loop-cost --pattern ci-sweeper --level L2 --orchestration parallel:3
+```
+
+Multipliers above 2x emit a warning — deep fan-out or debate is easy to enable
+and expensive to run unattended.
+
## Feed the circuit breaker
`--json` output resolves a realistic per-run token cap for
diff --git a/tools/loop-cost/dist/cli.js b/tools/loop-cost/dist/cli.js
index 6d324d6..1d12976 100755
--- a/tools/loop-cost/dist/cli.js
+++ b/tools/loop-cost/dist/cli.js
@@ -3,7 +3,7 @@ import { readFile, access } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import yaml from 'yaml';
-import { assertValidLevel, estimateCost, formatEstimateHuman, } from './estimator.js';
+import { assertValidLevel, estimateCost, formatEstimateHuman, parseOrchestration, } from './estimator.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const PACKAGE_ROOT = path.resolve(__dirname, '..');
function parseArgs(argv) {
@@ -13,6 +13,7 @@ function parseArgs(argv) {
let conservative = false;
let json = false;
let list = false;
+ let orchestration = 'single';
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === '--pattern' || a === '-p')
@@ -21,6 +22,8 @@ function parseArgs(argv) {
cadence = argv[++i];
else if (a === '--level' || a === '-l')
level = argv[++i];
+ else if (a === '--orchestration' || a === '-o')
+ orchestration = argv[++i];
else if (a === '--conservative')
conservative = true;
else if (a === '--json')
@@ -30,7 +33,7 @@ function parseArgs(argv) {
else if (a === '--help' || a === '-h')
return { help: true };
}
- return { help: false, pattern, cadence, level, conservative, json, list };
+ return { help: false, pattern, cadence, level, conservative, json, list, orchestration };
}
async function loadRegistry() {
const candidates = [
@@ -63,6 +66,9 @@ Options:
-p, --pattern <id> Pattern id (default: daily-triage)
-c, --cadence <spec> Override cadence (e.g. 15m, 1d, 5m-15m)
-l, --level <L1|L2|L3> Readiness level (default: L1)
+ -o, --orchestration <mode>
+ Multi-agent action cost: single (default),
+ maker-checker, parallel:N, debate:R
--conservative Use slower cadence from ranges (e.g. 15m not 5m)
--json Machine-readable output
--list List pattern ids
@@ -70,6 +76,7 @@ Options:
Examples:
loop-cost --pattern ci-sweeper --cadence 15m --level L2
+ loop-cost --pattern ci-sweeper --level L2 --orchestration maker-checker
loop-cost --pattern daily-triage --level L1 --json
loop-cost --list
`);
@@ -89,6 +96,7 @@ Examples:
}
try {
assertValidLevel(args.level);
+ parseOrchestration(args.orchestration);
}
catch (err) {
const msg = err instanceof Error ? err.message : String(err);
@@ -104,6 +112,7 @@ Examples:
cadence: args.cadence,
level: args.level,
conservative: args.conservative,
+ orchestration: args.orchestration,
});
if (args.json)
console.log(JSON.stringify(result, null, 2));
diff --git a/tools/loop-cost/dist/estimator.d.ts b/tools/loop-cost/dist/estimator.d.ts
index b5da5e7..8011c28 100644
--- a/tools/loop-cost/dist/estimator.d.ts
+++ b/tools/loop-cost/dist/estimator.d.ts
@@ -18,11 +18,30 @@ export interface RegistryPattern {
export interface RegistryDoc {
patterns: RegistryPattern[];
}
+export interface Orchestration {
+ /** Normalized mode string, e.g. 'single', 'maker-checker', 'parallel:3', 'debate:2'. */
+ mode: string;
+ /** Multiplier applied to the action-path token cost. */
+ multiplier: number;
+}
+/**
+ * Parse an orchestration spec into a multiplier on the action path.
+ *
+ * The action path (implementer + verifier work) is where multi-agent
+ * orchestration lands; the no-op scan and single triage pass are unaffected.
+ *
+ * single 1x one implementer pass, self-checked (default)
+ * maker-checker 2x implementer + an independent verifier pass
+ * parallel:N N+1 N candidate agents fan out, plus a merge/arbiter pass
+ * debate:R 1+R one proposer plus R critique-and-revise rounds
+ */
+export declare function parseOrchestration(spec: string | undefined): Orchestration;
export interface EstimateInput {
pattern: RegistryPattern;
cadence?: string;
level: ReadinessLevel;
conservative?: boolean;
+ orchestration?: string;
}
export interface EstimateResult {
patternId: string;
@@ -33,6 +52,7 @@ export interface EstimateResult {
tokenCostTier: string;
suggestedDailyCap: number;
earlyExitRequired: boolean;
+ orchestration: Orchestration;
scenarios: {
noop: {
tokensPerRun: number;
diff --git a/tools/loop-cost/dist/estimator.js b/tools/loop-cost/dist/estimator.js
index 18988c4..24809df 100644
--- a/tools/loop-cost/dist/estimator.js
+++ b/tools/loop-cost/dist/estimator.js
@@ -4,6 +4,39 @@ export function assertValidLevel(level) {
throw new Error(`Invalid level: ${level}. Valid: ${VALID_READINESS_LEVELS.join(', ')}`);
}
}
+/**
+ * Parse an orchestration spec into a multiplier on the action path.
+ *
+ * The action path (implementer + verifier work) is where multi-agent
+ * orchestration lands; the no-op scan and single triage pass are unaffected.
+ *
+ * single 1x one implementer pass, self-checked (default)
+ * maker-checker 2x implementer + an independent verifier pass
+ * parallel:N N+1 N candidate agents fan out, plus a merge/arbiter pass
+ * debate:R 1+R one proposer plus R critique-and-revise rounds
+ */
+export function parseOrchestration(spec) {
+ const s = (spec ?? 'single').trim().toLowerCase();
+ if (s === '' || s === 'single')
+ return { mode: 'single', multiplier: 1 };
+ if (s === 'maker-checker')
+ return { mode: 'maker-checker', multiplier: 2 };
+ const parallel = s.match(/^parallel:(\d+)$/);
+ if (parallel) {
+ const n = Number(parallel[1]);
+ if (n < 2)
+ throw new Error(`Invalid orchestration: parallel:N needs N>=2 (got ${n}).`);
+ return { mode: `parallel:${n}`, multiplier: n + 1 };
+ }
+ const debate = s.match(/^debate:(\d+)$/);
+ if (debate) {
+ const r = Number(debate[1]);
+ if (r < 1)
+ throw new Error(`Invalid orchestration: debate:R needs R>=1 (got ${r}).`);
+ return { mode: `debate:${r}`, multiplier: 1 + r };
+ }
+ throw new Error(`Invalid orchestration: ${spec}. Valid: single, maker-checker, parallel:N, debate:R`);
+}
const INTERVAL_MS = {
m: 60_000,
h: 3_600_000,
@@ -73,12 +106,16 @@ export function estimateCost(input) {
const runsPerDay = cadenceToRunsPerDay(cadence, input.conservative);
const { cost, token_cost: tokenCostTier } = input.pattern;
const mix = realisticMix(input.level, cost.early_exit_required);
+ const orchestration = parseOrchestration(input.orchestration);
+ // Orchestration overhead lands on the action path only — the no-op scan and
+ // the single triage pass do not fan out.
+ const actionPerRun = Math.round(cost.tokens_action * orchestration.multiplier);
const noopDay = cost.tokens_noop * runsPerDay;
const reportDay = cost.tokens_report * runsPerDay;
- const actionDay = cost.tokens_action * runsPerDay;
+ const actionDay = actionPerRun * runsPerDay;
const realisticPerRun = cost.tokens_noop * mix.noop +
cost.tokens_report * mix.report +
- cost.tokens_action * mix.action;
+ actionPerRun * mix.action;
const realisticDay = Math.round(realisticPerRun * runsPerDay);
const warnings = [];
if (cost.early_exit_required) {
@@ -93,6 +130,9 @@ export function estimateCost(input) {
if (runsPerDay >= 96) {
warnings.push(`High cadence (${runsPerDay} runs/day) — verify early-exit is working.`);
}
+ if (orchestration.multiplier > 2) {
+ warnings.push(`Orchestration ${orchestration.mode} multiplies action cost ${orchestration.multiplier}x — confirm the fan-out or debate depth is justified.`);
+ }
return {
patternId: input.pattern.id,
patternName: input.pattern.name,
@@ -102,10 +142,11 @@ export function estimateCost(input) {
tokenCostTier,
suggestedDailyCap: cost.suggested_daily_cap,
earlyExitRequired: cost.early_exit_required,
+ orchestration,
scenarios: {
noop: { tokensPerRun: cost.tokens_noop, tokensPerDay: noopDay },
report: { tokensPerRun: cost.tokens_report, tokensPerDay: reportDay },
- action: { tokensPerRun: cost.tokens_action, tokensPerDay: actionDay },
+ action: { tokensPerRun: actionPerRun, tokensPerDay: actionDay },
realistic: {
tokensPerRun: Math.round(realisticPerRun),
tokensPerDay: realisticDay,
@@ -122,6 +163,9 @@ export function formatEstimateHuman(r) {
lines.push('═'.repeat(50));
lines.push(`Cadence: ${r.cadence} → ${r.runsPerDay} runs/day`);
lines.push(`Level: ${r.level} · Registry tier: ${r.tokenCostTier}`);
+ if (r.orchestration.multiplier > 1) {
+ lines.push(`Orchestration: ${r.orchestration.mode} · action x${r.orchestration.multiplier}`);
+ }
lines.push(`Suggested daily cap: ${formatTokens(r.suggestedDailyCap)} tokens`);
lines.push('');
lines.push('Daily token estimates:');
diff --git a/tools/loop-cost/package.json b/tools/loop-cost/package.json
index a56e4a7..2f1ebe1 100644
--- a/tools/loop-cost/package.json
+++ b/tools/loop-cost/package.json
@@ -1,6 +1,6 @@
{
"name": "@cobusgreyling/loop-cost",
- "version": "1.0.3",
+ "version": "1.1.0",
"description": "Estimate token budgets and daily costs for loop engineering patterns. By cadence, readiness level (L1/L2/L3), and coding agent tool.",
"type": "module",
"bin": {
diff --git a/tools/loop-cost/src/cli.ts b/tools/loop-cost/src/cli.ts
index d7e6fe7..c40c164 100644
--- a/tools/loop-cost/src/cli.ts
+++ b/tools/loop-cost/src/cli.ts
@@ -7,6 +7,7 @@ import {
assertValidLevel,
estimateCost,
formatEstimateHuman,
+ parseOrchestration,
type ReadinessLevel,
type RegistryDoc,
} from './estimator.js';
@@ -21,19 +22,21 @@ function parseArgs(argv: string[]) {
let conservative = false;
let json = false;
let list = false;
+ let orchestration = 'single';
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === '--pattern' || a === '-p') pattern = argv[++i];
else if (a === '--cadence' || a === '-c') cadence = argv[++i];
else if (a === '--level' || a === '-l') level = argv[++i] as ReadinessLevel;
+ else if (a === '--orchestration' || a === '-o') orchestration = argv[++i];
else if (a === '--conservative') conservative = true;
else if (a === '--json') json = true;
else if (a === '--list') list = true;
else if (a === '--help' || a === '-h') return { help: true as const };
}
- return { help: false as const, pattern, cadence, level, conservative, json, list };
+ return { help: false as const, pattern, cadence, level, conservative, json, list, orchestration };
}
async function loadRegistry(): Promise<RegistryDoc> {
@@ -68,6 +71,9 @@ Options:
-p, --pattern <id> Pattern id (default: daily-triage)
-c, --cadence <spec> Override cadence (e.g. 15m, 1d, 5m-15m)
-l, --level <L1|L2|L3> Readiness level (default: L1)
+ -o, --orchestration <mode>
+ Multi-agent action cost: single (default),
+ maker-checker, parallel:N, debate:R
--conservative Use slower cadence from ranges (e.g. 15m not 5m)
--json Machine-readable output
--list List pattern ids
@@ -75,6 +81,7 @@ Options:
Examples:
loop-cost --pattern ci-sweeper --cadence 15m --level L2
+ loop-cost --pattern ci-sweeper --level L2 --orchestration maker-checker
loop-cost --pattern daily-triage --level L1 --json
loop-cost --list
`);
@@ -98,6 +105,7 @@ Examples:
try {
assertValidLevel(args.level);
+ parseOrchestration(args.orchestration);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
console.error(msg);
@@ -114,6 +122,7 @@ Examples:
cadence: args.cadence,
level: args.level,
conservative: args.conservative,
+ orchestration: args.orchestration,
});
if (args.json) console.log(JSON.stringify(result, null, 2));
diff --git a/tools/loop-cost/src/estimator.ts b/tools/loop-cost/src/estimator.ts
index 7814f03..39129f3 100644
--- a/tools/loop-cost/src/estimator.ts
+++ b/tools/loop-cost/src/estimator.ts
@@ -28,11 +28,54 @@ export interface RegistryDoc {
patterns: RegistryPattern[];
}
+export interface Orchestration {
+ /** Normalized mode string, e.g. 'single', 'maker-checker', 'parallel:3', 'debate:2'. */
+ mode: string;
+ /** Multiplier applied to the action-path token cost. */
+ multiplier: number;
+}
+
+/**
+ * Parse an orchestration spec into a multiplier on the action path.
+ *
+ * The action path (implementer + verifier work) is where multi-agent
+ * orchestration lands; the no-op scan and single triage pass are unaffected.
+ *
+ * single 1x one implementer pass, self-checked (default)
+ * maker-checker 2x implementer + an independent verifier pass
+ * parallel:N N+1 N candidate agents fan out, plus a merge/arbiter pass
+ * debate:R 1+R one proposer plus R critique-and-revise rounds
+ */
+export function parseOrchestration(spec: string | undefined): Orchestration {
+ const s = (spec ?? 'single').trim().toLowerCase();
+ if (s === '' || s === 'single') return { mode: 'single', multiplier: 1 };
+ if (s === 'maker-checker') return { mode: 'maker-checker', multiplier: 2 };
+
+ const parallel = s.match(/^parallel:(\d+)$/);
+ if (parallel) {
+ const n = Number(parallel[1]);
+ if (n < 2) throw new Error(`Invalid orchestration: parallel:N needs N>=2 (got ${n}).`);
+ return { mode: `parallel:${n}`, multiplier: n + 1 };
+ }
+
+ const debate = s.match(/^debate:(\d+)$/);
+ if (debate) {
+ const r = Number(debate[1]);
+ if (r < 1) throw new Error(`Invalid orchestration: debate:R needs R>=1 (got ${r}).`);
+ return { mode: `debate:${r}`, multiplier: 1 + r };
+ }
+
+ throw new Error(
+ `Invalid orchestration: ${spec}. Valid: single, maker-checker, parallel:N, debate:R`,
+ );
+}
+
export interface EstimateInput {
pattern: RegistryPattern;
cadence?: string;
level: ReadinessLevel;
conservative?: boolean;
+ orchestration?: string;
}
export interface EstimateResult {
@@ -44,6 +87,7 @@ export interface EstimateResult {
tokenCostTier: string;
suggestedDailyCap: number;
earlyExitRequired: boolean;
+ orchestration: Orchestration;
scenarios: {
noop: { tokensPerRun: number; tokensPerDay: number };
report: { tokensPerRun: number; tokensPerDay: number };
@@ -130,15 +174,20 @@ export function estimateCost(input: EstimateInput): EstimateResult {
const runsPerDay = cadenceToRunsPerDay(cadence, input.conservative);
const { cost, token_cost: tokenCostTier } = input.pattern;
const mix = realisticMix(input.level, cost.early_exit_required);
+ const orchestration = parseOrchestration(input.orchestration);
+
+ // Orchestration overhead lands on the action path only — the no-op scan and
+ // the single triage pass do not fan out.
+ const actionPerRun = Math.round(cost.tokens_action * orchestration.multiplier);
const noopDay = cost.tokens_noop * runsPerDay;
const reportDay = cost.tokens_report * runsPerDay;
- const actionDay = cost.tokens_action * runsPerDay;
+ const actionDay = actionPerRun * runsPerDay;
const realisticPerRun =
cost.tokens_noop * mix.noop +
cost.tokens_report * mix.report +
- cost.tokens_action * mix.action;
+ actionPerRun * mix.action;
const realisticDay = Math.round(realisticPerRun * runsPerDay);
const warnings: string[] = [];
@@ -156,6 +205,11 @@ export function estimateCost(input: EstimateInput): EstimateResult {
if (runsPerDay >= 96) {
warnings.push(`High cadence (${runsPerDay} runs/day) — verify early-exit is working.`);
}
+ if (orchestration.multiplier > 2) {
+ warnings.push(
+ `Orchestration ${orchestration.mode} multiplies action cost ${orchestration.multiplier}x — confirm the fan-out or debate depth is justified.`,
+ );
+ }
return {
patternId: input.pattern.id,
@@ -166,10 +220,11 @@ export function estimateCost(input: EstimateInput): EstimateResult {
tokenCostTier,
suggestedDailyCap: cost.suggested_daily_cap,
earlyExitRequired: cost.early_exit_required,
+ orchestration,
scenarios: {
noop: { tokensPerRun: cost.tokens_noop, tokensPerDay: noopDay },
report: { tokensPerRun: cost.tokens_report, tokensPerDay: reportDay },
- action: { tokensPerRun: cost.tokens_action, tokensPerDay: actionDay },
+ action: { tokensPerRun: actionPerRun, tokensPerDay: actionDay },
realistic: {
tokensPerRun: Math.round(realisticPerRun),
tokensPerDay: realisticDay,
@@ -187,6 +242,9 @@ export function formatEstimateHuman(r: EstimateResult): string {
lines.push('═'.repeat(50));
lines.push(`Cadence: ${r.cadence} → ${r.runsPerDay} runs/day`);
lines.push(`Level: ${r.level} · Registry tier: ${r.tokenCostTier}`);
+ if (r.orchestration.multiplier > 1) {
+ lines.push(`Orchestration: ${r.orchestration.mode} · action x${r.orchestration.multiplier}`);
+ }
lines.push(`Suggested daily cap: ${formatTokens(r.suggestedDailyCap)} tokens`);
lines.push('');
lines.push('Daily token estimates:');
diff --git a/tools/loop-cost/test/estimator.test.mjs b/tools/loop-cost/test/estimator.test.mjs
index d1fce3a..cd65a86 100644
--- a/tools/loop-cost/test/estimator.test.mjs
+++ b/tools/loop-cost/test/estimator.test.mjs
@@ -5,6 +5,7 @@ import {
cadenceToRunsPerDay,
runsPerDayForInterval,
estimateCost,
+ parseOrchestration,
} from '../dist/estimator.js';
const CI_SWEEPER = {
@@ -53,6 +54,56 @@ test('assertValidLevel: rejects unknown level', () => {
assert.throws(() => assertValidLevel('garbage'), /Invalid level/);
});
+test('parseOrchestration: modes map to expected multipliers', () => {
+ assert.equal(parseOrchestration(undefined).multiplier, 1);
+ assert.equal(parseOrchestration('single').multiplier, 1);
+ assert.equal(parseOrchestration('maker-checker').multiplier, 2);
+ assert.equal(parseOrchestration('parallel:3').multiplier, 4);
+ assert.equal(parseOrchestration('debate:2').multiplier, 3);
+});
+
+test('parseOrchestration: rejects bad specs', () => {
+ assert.throws(() => parseOrchestration('parallel:1'), /parallel/);
+ assert.throws(() => parseOrchestration('debate:0'), /debate/);
+ assert.throws(() => parseOrchestration('garbage'), /Invalid orchestration/);
+});
+
+test('estimateCost: orchestration scales only the action path', () => {
+ const base = estimateCost({ pattern: CI_SWEEPER, cadence: '15m', level: 'L2' });
+ const mc = estimateCost({
+ pattern: CI_SWEEPER,
+ cadence: '15m',
+ level: 'L2',
+ orchestration: 'maker-checker',
+ });
+ assert.equal(mc.orchestration.mode, 'maker-checker');
+ assert.equal(mc.orchestration.multiplier, 2);
+ // action doubles; no-op and full-triage scans are untouched.
+ assert.equal(mc.scenarios.action.tokensPerRun, base.scenarios.action.tokensPerRun * 2);
+ assert.equal(mc.scenarios.noop.tokensPerRun, base.scenarios.noop.tokensPerRun);
+ assert.equal(mc.scenarios.report.tokensPerRun, base.scenarios.report.tokensPerRun);
+ // realistic blend rises because its action component is scaled.
+ assert.ok(mc.scenarios.realistic.tokensPerDay > base.scenarios.realistic.tokensPerDay);
+});
+
+test('estimateCost: default single leaves action unchanged and adds no warning', () => {
+ const r = estimateCost({ pattern: CI_SWEEPER, cadence: '15m', level: 'L2' });
+ assert.equal(r.orchestration.mode, 'single');
+ assert.equal(r.scenarios.action.tokensPerRun, CI_SWEEPER.cost.tokens_action);
+ assert.ok(!r.warnings.some((w) => /Orchestration/.test(w)));
+});
+
+test('estimateCost: deep fan-out warns', () => {
+ const r = estimateCost({
+ pattern: CI_SWEEPER,
+ cadence: '15m',
+ level: 'L2',
+ orchestration: 'parallel:4',
+ });
+ assert.equal(r.orchestration.multiplier, 5);
+ assert.ok(r.warnings.some((w) => /Orchestration/.test(w)));
+});
+
test('estimateCost: daily-triage 1d L1 is cheap', () => {
const r = estimateCost({
pattern: {