Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

fix(benchmarks): report successful repair rounds and exact coverage

tt-a1i5418162bda5d
diff --git a/archify/test/repair-rounds.test.mjs b/archify/test/repair-rounds.test.mjs
index 459d344c..5b7dadb5 100644
--- a/archify/test/repair-rounds.test.mjs
+++ b/archify/test/repair-rounds.test.mjs
@@ -139,8 +139,8 @@ test('report aggregates disclosure, late discovery, and token cost across receip
benchmark: 'repair-rounds',
caseId: 'late-case',
diagramType: 'architecture',
- command: 'finalize',
- repairMode: 'oracle',
+ command: 'validate',
+ repairMode: 'rules',
defects: [
{ class: 'node-long-label', detected: true, firstSeenRound: 1, firstSeenGate: 'validate', late: false },
{ class: 'title-overflow', detected: true, firstSeenRound: 2, firstSeenGate: 'browser-check', late: true },
@@ -196,3 +196,57 @@ test('report rejects duplicate and foreign receipts', () => {
assert.equal(foreign.status, 2);
assert.equal(JSON.parse(foreign.stdout).error.code, 'INVALID_RESULT');
});
+
+function reportRow(caseId, overrides = {}) {
+ return {
+ schemaVersion: 1, benchmark: 'repair-rounds', caseId, diagramType: 'workflow',
+ command: 'validate', repairMode: 'rules', defects: [], rounds: [],
+ passed: true, stalled: false,
+ totals: { rounds: 2, unactionable: 0, tokens: { estimate: 10, receipts: 5 } },
+ ...overrides,
+ };
+}
+
+function aggregateRows(name, rows, extraArgs = []) {
+ const file = path.join(tmp, `${name}.jsonl`);
+ fs.writeFileSync(file, rows.map((row) => JSON.stringify(row)).join('\n'));
+ return run(['report', '--results', file, ...extraArgs]);
+}
+
+test('rounds to pass excludes stalled and exhausted cases', () => {
+ const failed = [
+ reportRow('stalled', { passed: false, stalled: true, totals: { rounds: 1, unactionable: 1, tokens: { estimate: 10, receipts: 5 } } }),
+ reportRow('exhausted', { passed: false }),
+ ];
+ const result = aggregateRows('outcomes', [reportRow('success'), ...failed]);
+ assert.equal(result.status, 0);
+ const { overall } = JSON.parse(result.stdout);
+ assert.deepEqual(overall.roundsToPass, { mean: 2, median: 2, max: 2 });
+ assert.equal(overall.passed, 1);
+ assert.equal(overall.stalled, 1);
+ assert.equal(overall.exhausted, 1);
+ const noPass = aggregateRows('no-pass', failed);
+ assert.equal(noPass.status, 0);
+ assert.deepEqual(JSON.parse(noPass.stdout).overall.roundsToPass, { mean: null, median: null, max: null });
+});
+
+test('report rejects mixed modes instead of counting one case twice toward coverage', () => {
+ const manifestFile = writeJson('coverage-manifest.json', { id: 'coverage', cases: [{ id: 'A' }, { id: 'B' }] });
+ for (const override of [{ repairMode: 'oracle' }, { command: 'finalize' }]) {
+ const result = aggregateRows('mixed', [reportRow('A'), reportRow('A', override)], ['--manifest', manifestFile]);
+ assert.equal(result.status, 2);
+ assert.equal(JSON.parse(result.stdout).error.code, 'MIXED_MODES');
+ }
+ for (const [name, rows, complete, missing] of [
+ ['missing', [reportRow('A')], false, ['B']],
+ ['foreign-case', [reportRow('A'), reportRow('C')], false, ['B']],
+ ['complete', [reportRow('A'), reportRow('B')], true, []],
+ ]) {
+ const result = aggregateRows(name, rows, ['--manifest', manifestFile]);
+ assert.equal(result.status, 0);
+ const report = JSON.parse(result.stdout);
+ assert.equal(report.coverage.complete, complete);
+ assert.equal(report.evidenceEligible, complete);
+ assert.deepEqual(report.coverage.missing, missing);
+ }
+});
diff --git a/benchmarks/repair-rounds/README.md b/benchmarks/repair-rounds/README.md
index acb01f34..a48c27e5 100644
--- a/benchmarks/repair-rounds/README.md
+++ b/benchmarks/repair-rounds/README.md
@@ -8,7 +8,7 @@ It is a measurement harness, not a model leaderboard, and it contains no human o
| Metric | Definition |
|---|---|
-| Rounds to pass | `finalize`/`validate` invocations needed before the mutated candidate passes |
+| Rounds to pass | `finalize`/`validate` invocations for successful cases only; `null` when none pass (stalled and round-cap-exhausted cases are counted separately) |
| First-round disclosure rate | injected defects surfaced by round 1 ÷ injected defects |
| First-detection stage | the gate that first reports each defect (`validate`, `deliver`, `check`, `browser-check`, or `undetected`) |
| Late discovery rate | detected defects first surfaced at `check` or `browser-check` ÷ detected defects |
@@ -46,6 +46,8 @@ node benchmarks/repair-rounds/benchmark.mjs report --results results.jsonl --man
`run` sets `ARCHIFY_UPDATE_CHECK_DISABLED=1` so receipts are not perturbed by the update check. `browser-check` needs local Chrome; without it the late-discovery cases report the gate truthfully as skipped rather than passed.
+Reports require one command and repair mode per input file; mixed configurations are rejected. Coverage compares unique case IDs against the manifest, so repeated cases cannot fill missing cases.
+
## Repair modes
- `rules` (default): the repairer sees only the receipt — `subject` paths, `evidence`, `supportedFixes`, and pixel figures inside `layout/constraint` messages. It models the weakest honest agent: repairs derived purely from what the tool reports.
diff --git a/benchmarks/repair-rounds/benchmark.mjs b/benchmarks/repair-rounds/benchmark.mjs
index 9e0b2f0a..4dd4bb6d 100644
--- a/benchmarks/repair-rounds/benchmark.mjs
+++ b/benchmarks/repair-rounds/benchmark.mjs
@@ -367,16 +367,17 @@ function summarize(manifest, results) {
const diagnostics = results.flatMap((result) => (result.rounds || []).flatMap((round) => round.diagnostics));
const actionable = diagnostics.filter((d) => d.hasSubjectDetail && d.hasEvidence && d.hasSupportedFixes);
const unactionable = results.reduce((sum, result) => sum + result.totals.unactionable, 0);
- const roundCounts = results.map((result) => result.totals.rounds).sort((a, b) => a - b);
+ const roundCounts = results.filter((result) => result.passed).map((result) => result.totals.rounds).sort((a, b) => a - b);
const tokenTotals = results.map((result) => result.totals.tokens.estimate);
return {
cases: results.length,
passed: results.filter((result) => result.passed).length,
stalled: results.filter((result) => result.stalled).length,
+ exhausted: results.filter((result) => !result.passed && !result.stalled).length,
roundsToPass: {
- mean: roundCounts.length ? roundCounts.reduce((a, b) => a + b, 0) / roundCounts.length : 0,
- median: roundCounts.length ? roundCounts[Math.floor(roundCounts.length / 2)] : 0,
- max: roundCounts.length ? Math.max(...roundCounts) : 0,
+ mean: roundCounts.length ? roundCounts.reduce((a, b) => a + b, 0) / roundCounts.length : null,
+ median: roundCounts.length ? roundCounts[Math.floor(roundCounts.length / 2)] : null,
+ max: roundCounts.length ? Math.max(...roundCounts) : null,
},
disclosure: {
defects: defects.length,
@@ -424,9 +425,13 @@ function report(args) {
.filter(Boolean)
.map((line) => JSON.parse(line));
for (const result of results) validateResult(result);
+ const modes = new Set(results.map((result) => `${result.command}:${result.repairMode}`));
+ if (modes.size > 1) {
+ throw new BenchmarkError('MIXED_MODES', 'report requires one command and repair mode; report each configuration separately');
+ }
const seen = new Set();
for (const result of results) {
- const key = `${result.caseId}${result.command}${result.repairMode}`;
+ const key = result.caseId;
if (seen.has(key)) {
throw new BenchmarkError('DUPLICATE_RESULT', `duplicate result for ${result.caseId}`);
}
@@ -438,7 +443,7 @@ function report(args) {
expected: expected.size,
present: new Set(results.map((result) => result.caseId)).size,
missing: [...expected].filter((id) => !results.some((result) => result.caseId === id)).sort(),
- complete: results.length === expected.size && results.every((result) => expected.has(result.caseId)),
+ complete: seen.size === expected.size && [...expected].every((id) => seen.has(id)),
}
: null;
process.stdout.write(`${JSON.stringify({