Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

fix(loop-metrics): stop successRatePct from going negative on multi-escalation runs (#537)

JonyT Agenticb37ca9ea348e
diff --git a/tools/loop-metrics/dist/metrics.js b/tools/loop-metrics/dist/metrics.js
index 2b41b2b..aa4797d 100644
--- a/tools/loop-metrics/dist/metrics.js
+++ b/tools/loop-metrics/dist/metrics.js
@@ -48,14 +48,25 @@ export function aggregateMetrics(entries) {
let totalDurationS = 0;
let totalActionsTaken = 0;
let totalEscalations = 0;
+ let runsWithoutEscalation = 0;
for (const entry of entries) {
totalTokens += entry.tokens_estimate || 0;
totalDurationS += entry.duration_s || 0;
totalActionsTaken += entry.actions_taken || 0;
totalEscalations += entry.escalations || 0;
+ if (!entry.escalations)
+ runsWithoutEscalation++;
}
const totalRuns = entries.length;
- const successRatePct = totalRuns > 0 ? ((totalRuns - totalEscalations) / totalRuns) * 100 : 0;
+ // totalEscalations is a sum of each run's escalation *count* (a run can
+ // log more than one, e.g. several items escalated in one triage pass) --
+ // subtracting it from totalRuns (a count of runs) conflates events with
+ // runs and can go negative for a real, ordinary run log (this repo's own
+ // loop-run-log.md has entries with escalations: 4 and escalations: 5).
+ // Success rate is "what fraction of runs didn't escalate at all", which
+ // stays correctly bounded to [0, 100] regardless of how many escalations
+ // any single run logged.
+ const successRatePct = totalRuns > 0 ? (runsWithoutEscalation / totalRuns) * 100 : 0;
const avgDurationS = totalRuns > 0 ? totalDurationS / totalRuns : 0;
// Simple heuristic: Each successful action is worth +10, each escalation is -5.
const roiScore = (totalActionsTaken * 10) - (totalEscalations * 5);
diff --git a/tools/loop-metrics/src/metrics.ts b/tools/loop-metrics/src/metrics.ts
index 14afcad..b4448a5 100644
--- a/tools/loop-metrics/src/metrics.ts
+++ b/tools/loop-metrics/src/metrics.ts
@@ -77,16 +77,26 @@ export function aggregateMetrics(entries: RunEntry[]): MetricsDashboard {
let totalDurationS = 0;
let totalActionsTaken = 0;
let totalEscalations = 0;
+ let runsWithoutEscalation = 0;
for (const entry of entries) {
totalTokens += entry.tokens_estimate || 0;
totalDurationS += entry.duration_s || 0;
totalActionsTaken += entry.actions_taken || 0;
totalEscalations += entry.escalations || 0;
+ if (!entry.escalations) runsWithoutEscalation++;
}
const totalRuns = entries.length;
- const successRatePct = totalRuns > 0 ? ((totalRuns - totalEscalations) / totalRuns) * 100 : 0;
+ // totalEscalations is a sum of each run's escalation *count* (a run can
+ // log more than one, e.g. several items escalated in one triage pass) --
+ // subtracting it from totalRuns (a count of runs) conflates events with
+ // runs and can go negative for a real, ordinary run log (this repo's own
+ // loop-run-log.md has entries with escalations: 4 and escalations: 5).
+ // Success rate is "what fraction of runs didn't escalate at all", which
+ // stays correctly bounded to [0, 100] regardless of how many escalations
+ // any single run logged.
+ const successRatePct = totalRuns > 0 ? (runsWithoutEscalation / totalRuns) * 100 : 0;
const avgDurationS = totalRuns > 0 ? totalDurationS / totalRuns : 0;
// Simple heuristic: Each successful action is worth +10, each escalation is -5.
diff --git a/tools/loop-metrics/test/metrics.test.mjs b/tools/loop-metrics/test/metrics.test.mjs
index 056d793..a331dba 100644
--- a/tools/loop-metrics/test/metrics.test.mjs
+++ b/tools/loop-metrics/test/metrics.test.mjs
@@ -17,6 +17,24 @@ test('loop-metrics filters and aggregates', () => {
assert.strictEqual(metrics.totalActionsTaken, 3);
assert.strictEqual(metrics.totalEscalations, 1);
assert.strictEqual(metrics.roiScore, (3 * 10) - (1 * 5)); // 25
+ assert.strictEqual(metrics.successRatePct, 50); // 1 of 2 runs had no escalation
+});
+
+test('successRatePct stays within [0, 100] when a single run logs more than one escalation', () => {
+ // A run can escalate several items in one pass (this repo's own
+ // loop-run-log.md has real entries with escalations: 4 and 5), so
+ // totalEscalations (a sum of per-run counts) can exceed totalRuns.
+ // successRatePct must still reflect "runs that didn't escalate", not go
+ // negative from subtracting an event count as if it were a run count.
+ const entries = [
+ { run_id: 'a', pattern: 'ci-sweeper', duration_s: 1, items_found: 5, actions_taken: 1, escalations: 5, tokens_estimate: 1000, outcome: 'escalated' },
+ { run_id: 'b', pattern: 'ci-sweeper', duration_s: 1, items_found: 0, actions_taken: 0, escalations: 0, tokens_estimate: 1000, outcome: 'report-only' },
+ ];
+
+ const metrics = aggregateMetrics(entries);
+ assert.strictEqual(metrics.totalEscalations, 5);
+ assert.strictEqual(metrics.successRatePct, 50); // 1 of 2 runs had no escalation
+ assert.ok(metrics.successRatePct >= 0 && metrics.successRatePct <= 100);
});
test('filterEntries keeps entries with unparseable run_id when a timeframe is set', () => {