Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.
HTML agent-skillsai-agentsarchitecture-diagramclaude-codeclaude-skillscodexcoding-agentsdeepseek-harnessdeveloper-toolsdiagramsdiagrams-as-codedsh-pluginflowchartllmmermaidopencodesequence-diagramsoftware-architecturesystem-designvisualization
fix(benchmarks): report successful repair rounds and exact coverage
tt-a1i5418162bda5d
diff --git a/archify/test/repair-rounds.test.mjs b/archify/test/repair-rounds.test.mjsindex 459d344c..5b7dadb5 100644--- a/archify/test/repair-rounds.test.mjs+++ b/archify/test/repair-rounds.test.mjs@@ -139,8 +139,8 @@ test('report aggregates disclosure, late discovery, and token cost across receipbenchmark: 'repair-rounds',caseId: 'late-case',diagramType: 'architecture',- command: 'finalize',- repairMode: 'oracle',+ command: 'validate',+ repairMode: 'rules',defects: [{ class: 'node-long-label', detected: true, firstSeenRound: 1, firstSeenGate: 'validate', late: false },{ class: 'title-overflow', detected: true, firstSeenRound: 2, firstSeenGate: 'browser-check', late: true },@@ -196,3 +196,57 @@ test('report rejects duplicate and foreign receipts', () => {assert.equal(foreign.status, 2);assert.equal(JSON.parse(foreign.stdout).error.code, 'INVALID_RESULT');});++function reportRow(caseId, overrides = {}) {+ return {+ schemaVersion: 1, benchmark: 'repair-rounds', caseId, diagramType: 'workflow',+ command: 'validate', repairMode: 'rules', defects: [], rounds: [],+ passed: true, stalled: false,+ totals: { rounds: 2, unactionable: 0, tokens: { estimate: 10, receipts: 5 } },+ ...overrides,+ };+}++function aggregateRows(name, rows, extraArgs = []) {+ const file = path.join(tmp, `${name}.jsonl`);+ fs.writeFileSync(file, rows.map((row) => JSON.stringify(row)).join('\n'));+ return run(['report', '--results', file, ...extraArgs]);+}++test('rounds to pass excludes stalled and exhausted cases', () => {+ const failed = [+ reportRow('stalled', { passed: false, stalled: true, totals: { rounds: 1, unactionable: 1, tokens: { estimate: 10, receipts: 5 } } }),+ reportRow('exhausted', { passed: false }),+ ];+ const result = aggregateRows('outcomes', [reportRow('success'), ...failed]);+ assert.equal(result.status, 0);+ const { overall } = JSON.parse(result.stdout);+ assert.deepEqual(overall.roundsToPass, { mean: 2, median: 2, max: 2 });+ assert.equal(overall.passed, 1);+ assert.equal(overall.stalled, 1);+ assert.equal(overall.exhausted, 1);+ const noPass = aggregateRows('no-pass', failed);+ assert.equal(noPass.status, 0);+ assert.deepEqual(JSON.parse(noPass.stdout).overall.roundsToPass, { mean: null, median: null, max: null });+});++test('report rejects mixed modes instead of counting one case twice toward coverage', () => {+ const manifestFile = writeJson('coverage-manifest.json', { id: 'coverage', cases: [{ id: 'A' }, { id: 'B' }] });+ for (const override of [{ repairMode: 'oracle' }, { command: 'finalize' }]) {+ const result = aggregateRows('mixed', [reportRow('A'), reportRow('A', override)], ['--manifest', manifestFile]);+ assert.equal(result.status, 2);+ assert.equal(JSON.parse(result.stdout).error.code, 'MIXED_MODES');+ }+ for (const [name, rows, complete, missing] of [+ ['missing', [reportRow('A')], false, ['B']],+ ['foreign-case', [reportRow('A'), reportRow('C')], false, ['B']],+ ['complete', [reportRow('A'), reportRow('B')], true, []],+ ]) {+ const result = aggregateRows(name, rows, ['--manifest', manifestFile]);+ assert.equal(result.status, 0);+ const report = JSON.parse(result.stdout);+ assert.equal(report.coverage.complete, complete);+ assert.equal(report.evidenceEligible, complete);+ assert.deepEqual(report.coverage.missing, missing);+ }+});diff --git a/benchmarks/repair-rounds/README.md b/benchmarks/repair-rounds/README.mdindex acb01f34..a48c27e5 100644--- a/benchmarks/repair-rounds/README.md+++ b/benchmarks/repair-rounds/README.md@@ -8,7 +8,7 @@ It is a measurement harness, not a model leaderboard, and it contains no human o| Metric | Definition ||---|---|-| Rounds to pass | `finalize`/`validate` invocations needed before the mutated candidate passes |+| Rounds to pass | `finalize`/`validate` invocations for successful cases only; `null` when none pass (stalled and round-cap-exhausted cases are counted separately) || First-round disclosure rate | injected defects surfaced by round 1 ÷ injected defects || First-detection stage | the gate that first reports each defect (`validate`, `deliver`, `check`, `browser-check`, or `undetected`) || Late discovery rate | detected defects first surfaced at `check` or `browser-check` ÷ detected defects |@@ -46,6 +46,8 @@ node benchmarks/repair-rounds/benchmark.mjs report --results results.jsonl --man`run` sets `ARCHIFY_UPDATE_CHECK_DISABLED=1` so receipts are not perturbed by the update check. `browser-check` needs local Chrome; without it the late-discovery cases report the gate truthfully as skipped rather than passed.+Reports require one command and repair mode per input file; mixed configurations are rejected. Coverage compares unique case IDs against the manifest, so repeated cases cannot fill missing cases.+## Repair modes- `rules` (default): the repairer sees only the receipt — `subject` paths, `evidence`, `supportedFixes`, and pixel figures inside `layout/constraint` messages. It models the weakest honest agent: repairs derived purely from what the tool reports.diff --git a/benchmarks/repair-rounds/benchmark.mjs b/benchmarks/repair-rounds/benchmark.mjsindex 9e0b2f0a..4dd4bb6d 100644--- a/benchmarks/repair-rounds/benchmark.mjs+++ b/benchmarks/repair-rounds/benchmark.mjs@@ -367,16 +367,17 @@ function summarize(manifest, results) {const diagnostics = results.flatMap((result) => (result.rounds || []).flatMap((round) => round.diagnostics));const actionable = diagnostics.filter((d) => d.hasSubjectDetail && d.hasEvidence && d.hasSupportedFixes);const unactionable = results.reduce((sum, result) => sum + result.totals.unactionable, 0);- const roundCounts = results.map((result) => result.totals.rounds).sort((a, b) => a - b);+ const roundCounts = results.filter((result) => result.passed).map((result) => result.totals.rounds).sort((a, b) => a - b);const tokenTotals = results.map((result) => result.totals.tokens.estimate);return {cases: results.length,passed: results.filter((result) => result.passed).length,stalled: results.filter((result) => result.stalled).length,+ exhausted: results.filter((result) => !result.passed && !result.stalled).length,roundsToPass: {- mean: roundCounts.length ? roundCounts.reduce((a, b) => a + b, 0) / roundCounts.length : 0,- median: roundCounts.length ? roundCounts[Math.floor(roundCounts.length / 2)] : 0,- max: roundCounts.length ? Math.max(...roundCounts) : 0,+ mean: roundCounts.length ? roundCounts.reduce((a, b) => a + b, 0) / roundCounts.length : null,+ median: roundCounts.length ? roundCounts[Math.floor(roundCounts.length / 2)] : null,+ max: roundCounts.length ? Math.max(...roundCounts) : null,},disclosure: {defects: defects.length,@@ -424,9 +425,13 @@ function report(args) {.filter(Boolean).map((line) => JSON.parse(line));for (const result of results) validateResult(result);+ const modes = new Set(results.map((result) => `${result.command}:${result.repairMode}`));+ if (modes.size > 1) {+ throw new BenchmarkError('MIXED_MODES', 'report requires one command and repair mode; report each configuration separately');+ }const seen = new Set();for (const result of results) {- const key = `${result.caseId}${result.command}${result.repairMode}`;+ const key = result.caseId;if (seen.has(key)) {throw new BenchmarkError('DUPLICATE_RESULT', `duplicate result for ${result.caseId}`);}@@ -438,7 +443,7 @@ function report(args) {expected: expected.size,present: new Set(results.map((result) => result.caseId)).size,missing: [...expected].filter((id) => !results.some((result) => result.caseId === id)).sort(),- complete: results.length === expected.size && results.every((result) => expected.has(result.caseId)),+ complete: seen.size === expected.size && [...expected].every((id) => seen.has(id)),}: null;process.stdout.write(`${JSON.stringify({