Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

feat: repair-rounds tests and protocol documentation

momoyeyufd036b9b2ccb
diff --git a/archify/test/repair-rounds.test.mjs b/archify/test/repair-rounds.test.mjs
new file mode 100644
index 00000000..df5bb1c5
--- /dev/null
+++ b/archify/test/repair-rounds.test.mjs
@@ -0,0 +1,197 @@
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const skillRoot = path.resolve(here, '..');
+const repoRoot = path.resolve(skillRoot, '..');
+const benchmark = path.join(repoRoot, 'benchmarks/repair-rounds/benchmark.mjs');
+const manifest = path.join(repoRoot, 'benchmarks/repair-rounds/manifest.json');
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'archify-repair-rounds-'));
+const chromeAvailable = Boolean(process.env.ARCHIFY_CHROME)
+ || fs.existsSync('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome');
+
+function run(args) {
+ return spawnSync(process.execPath, [benchmark, ...args], {
+ cwd: repoRoot,
+ encoding: 'utf8',
+ env: { ...process.env, ARCHIFY_UPDATE_CHECK_DISABLED: '1' },
+ });
+}
+
+function writeJson(name, value) {
+ const file = path.join(tmp, name);
+ fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+ return file;
+}
+
+function miniManifest(id, type, defects) {
+ const fixture = path.join(skillRoot, 'examples', {
+ architecture: 'web-app.architecture.json',
+ workflow: 'agent-tool-call.workflow.json',
+ sequence: 'cache-miss-request.sequence.json',
+ dataflow: 'product-analytics.dataflow.json',
+ lifecycle: 'deployment-release.lifecycle.json',
+ }[type]);
+ return writeJson(`${id}.manifest.json`, {
+ schema_version: 1,
+ id,
+ purpose: 'feedback-efficiency',
+ evidence_eligible: false,
+ fixtures: { [type]: fixture },
+ cases: [{ id, type, defects }],
+ });
+}
+
+test('check accepts the checked-in suite end to end', () => {
+ const result = run(['check', '--manifest', manifest]);
+ assert.equal(result.status, 0, result.stderr || result.stdout);
+ const receipt = JSON.parse(result.stdout);
+ assert.equal(receipt.schemaVersion, 1);
+ assert.equal(receipt.benchmark, 'repair-rounds');
+ assert.equal(receipt.caseCount, 41);
+ assert.equal(receipt.evidenceEligible, false);
+ for (const entry of receipt.cases) {
+ assert.deepEqual(entry.problems, [], entry.caseId);
+ }
+});
+
+test('check reports an unknown defect class instead of crashing', () => {
+ const bad = miniManifest('bad-defect', 'architecture', [{ class: 'does-not-exist' }]);
+ const result = run(['check', '--manifest', bad]);
+ assert.equal(result.status, 1, result.stderr || result.stdout);
+ const receipt = JSON.parse(result.stdout);
+ assert.match(receipt.cases[0].problems[0], /unknown defect class/);
+});
+
+test('rules repair converges a schema defect reported with a path and evidence', () => {
+ const file = miniManifest('flow-missing-output', 'workflow', [{ class: 'meta-missing-output' }]);
+ const result = run(['run', '--manifest', file, '--command', 'validate']);
+ assert.equal(result.status, 0, result.stderr || result.stdout);
+ const receipt = JSON.parse(result.stdout.split('\n')[0]);
+ assert.equal(receipt.passed, true);
+ assert.equal(receipt.totals.rounds, 2);
+ assert.deepEqual(receipt.defects.map((d) => d.firstSeenGate), ['validate:render']);
+ assert.equal(receipt.rounds[0].diagnostics[0].code, 'schema/required');
+ assert.equal(receipt.rounds[0].repairs.applied.length, 1);
+ assert.equal(receipt.totals.tokens.context > 0, true);
+ assert.equal(receipt.totals.tokens.receipts > 0, true);
+});
+
+test('a staged second defect surfaces only after the first is repaired', () => {
+ const file = miniManifest('arch-staged', 'architecture', [
+ { class: 'meta-missing-output' },
+ { class: 'node-long-label' },
+ ]);
+ const result = run(['run', '--manifest', file, '--command', 'validate', '--repair', 'oracle']);
+ assert.equal(result.status, 0, result.stderr || result.stdout);
+ const receipt = JSON.parse(result.stdout.split('\n')[0]);
+ assert.equal(receipt.passed, true);
+ assert.equal(receipt.totals.rounds, 3);
+ const [schema, label] = receipt.defects;
+ assert.equal(schema.class, 'meta-missing-output');
+ assert.equal(schema.firstSeenRound, 1);
+ assert.equal(label.class, 'node-long-label');
+ assert.equal(label.firstSeenRound, 2);
+});
+
+test('diagnostics the rules cannot map are counted as unactionable and stall the loop', () => {
+ const file = miniManifest('arch-duplicate-id', 'architecture', [{ class: 'node-duplicate-id' }]);
+ const result = run(['run', '--manifest', file, '--command', 'validate']);
+ assert.equal(result.status, 0, result.stderr || result.stdout);
+ const receipt = JSON.parse(result.stdout.split('\n')[0]);
+ assert.equal(receipt.stalled, true);
+ assert.equal(receipt.passed, false);
+ assert.equal(receipt.totals.unactionable > 0, true);
+});
+
+test('a header overflow is detected late, only at the browser-check gate', { skip: !chromeAvailable }, () => {
+ const file = miniManifest('arch-title-overflow', 'architecture', [{ class: 'title-overflow' }]);
+ const result = run(['run', '--manifest', file, '--command', 'finalize', '--repair', 'oracle']);
+ assert.equal(result.status, 0, result.stderr || result.stdout);
+ const receipt = JSON.parse(result.stdout.split('\n')[0]);
+ assert.equal(receipt.passed, true);
+ assert.equal(receipt.defects[0].firstSeenGate, 'browser-check');
+ assert.equal(receipt.defects[0].late, true);
+});
+
+test('report aggregates disclosure, late discovery, and token cost across receipts', () => {
+ const receipts = [
+ {
+ schemaVersion: 1,
+ benchmark: 'repair-rounds',
+ caseId: 'early-case',
+ diagramType: 'workflow',
+ command: 'validate',
+ repairMode: 'rules',
+ defects: [{ class: 'meta-missing-output', detected: true, firstSeenRound: 1, firstSeenGate: 'validate', late: false }],
+ rounds: [{ diagnostics: [{ code: 'schema/required', hasSubjectDetail: true, hasEvidence: true, hasSupportedFixes: true }] }],
+ passed: true,
+ stalled: false,
+ totals: { rounds: 2, diagnostics: 1, unactionable: 0, tokens: { estimate: 40000, receipts: 900 }, wallMs: 500 },
+ },
+ {
+ schemaVersion: 1,
+ benchmark: 'repair-rounds',
+ caseId: 'late-case',
+ diagramType: 'architecture',
+ command: 'finalize',
+ repairMode: 'oracle',
+ defects: [
+ { class: 'node-long-label', detected: true, firstSeenRound: 1, firstSeenGate: 'validate', late: false },
+ { class: 'title-overflow', detected: true, firstSeenRound: 2, firstSeenGate: 'browser-check', late: true },
+ { class: 'subtitle-overflow', detected: false, firstSeenRound: null, firstSeenGate: null, late: false },
+ ],
+ rounds: [{ diagnostics: [{ code: 'viewer/viewport-overflow', hasSubjectDetail: true, hasEvidence: true, hasSupportedFixes: true }] }],
+ passed: true,
+ stalled: false,
+ totals: { rounds: 3, diagnostics: 1, unactionable: 0, tokens: { estimate: 80000, receipts: 6000 }, wallMs: 4000 },
+ },
+ ];
+ const resultsFile = path.join(tmp, 'repair-results.jsonl');
+ fs.writeFileSync(resultsFile, `${receipts.map((row) => JSON.stringify(row)).join('\n')}\n`);
+
+ const result = run(['report', '--results', resultsFile]);
+ assert.equal(result.status, 0, result.stderr || result.stdout);
+ const report = JSON.parse(result.stdout);
+ assert.equal(report.overall.cases, 2);
+ assert.equal(report.overall.passed, 2);
+ assert.deepEqual(report.overall.roundsToPass, { mean: 2.5, median: 3, max: 3 });
+ assert.equal(report.overall.disclosure.defects, 4);
+ assert.equal(report.overall.disclosure.detected, 3);
+ assert.equal(report.overall.disclosure.undetected, 1);
+ assert.equal(report.overall.disclosure.firstRoundDisclosureRate, 0.5);
+ assert.equal(report.overall.disclosure.lateDiscoveryRate, 1 / 3);
+ assert.equal(report.overall.disclosure.firstSeenGate['browser-check'], 1);
+ assert.equal(report.overall.tokens.meanEstimatePerCase, 60000);
+});
+
+test('report rejects duplicate and foreign receipts', () => {
+ const receipt = {
+ schemaVersion: 1,
+ benchmark: 'repair-rounds',
+ caseId: 'dup',
+ diagramType: 'workflow',
+ command: 'validate',
+ repairMode: 'rules',
+ defects: [],
+ rounds: [],
+ passed: true,
+ totals: { rounds: 1, diagnostics: 0, unactionable: 0, tokens: { estimate: 1, receipts: 1 }, wallMs: 1 },
+ };
+ const dupFile = path.join(tmp, 'dup-results.jsonl');
+ fs.writeFileSync(dupFile, `${JSON.stringify(receipt)}\n${JSON.stringify(receipt)}\n`);
+ const dup = run(['report', '--results', dupFile]);
+ assert.equal(dup.status, 2);
+ assert.equal(JSON.parse(dup.stdout).error.code, 'DUPLICATE_RESULT');
+
+ const foreignFile = path.join(tmp, 'foreign-results.jsonl');
+ fs.writeFileSync(foreignFile, `${JSON.stringify({ schemaVersion: 1, benchmark: 'ordinary-model-floor' })}\n`);
+ const foreign = run(['report', '--results', foreignFile]);
+ assert.equal(foreign.status, 2);
+ assert.equal(JSON.parse(foreign.stdout).error.code, 'INVALID_RESULT');
+});
diff --git a/benchmarks/repair-rounds/README.md b/benchmarks/repair-rounds/README.md
new file mode 100644
index 00000000..acb01f34
--- /dev/null
+++ b/benchmarks/repair-rounds/README.md
@@ -0,0 +1,64 @@
+# Repair Rounds v1
+
+This benchmark measures a narrow product question: **how efficiently do Archify receipts let a caller converge to a passing artifact?** It complements `ordinary-model-floor` (first-pass usability) and the `authoring-cost` studies (real agent time and token spend) by isolating the part of authoring cost that lives inside the feedback loop — how many validation rounds a defect costs, how early each defect is surfaced, and how much text each round forces the caller to read.
+
+It is a measurement harness, not a model leaderboard, and it contains no human or model judgment. Everything is driven by deterministic defect injection against the checked-in example fixtures and the real Archify CLI.
+
+## What it measures
+
+| Metric | Definition |
+|---|---|
+| Rounds to pass | `finalize`/`validate` invocations needed before the mutated candidate passes |
+| First-round disclosure rate | injected defects surfaced by round 1 ÷ injected defects |
+| First-detection stage | the gate that first reports each defect (`validate`, `deliver`, `check`, `browser-check`, or `undetected`) |
+| Late discovery rate | detected defects first surfaced at `check` or `browser-check` ÷ detected defects |
+| Actionability | repairs a rules-only repairer could derive from diagnostics alone; `unactionable` receipts are counted honestly |
+| Observable token cost | a deterministic lexical estimate over the required-reading set, each round's receipt, and each round's candidate |
+
+A defect reported only after earlier gates passed means at least one extra authoring round in the real loop; `viewer/viewport-overflow` is the canonical example. Diagnostics that name no subject, carry no evidence, and suggest no fix produce `unactionable` repairs rather than silent passes.
+
+## Defect classes
+
+`defects.mjs` defines eight deterministic mutations: `meta-missing-output`, `node-invalid-type`, `node-long-label`, `node-duplicate-id`, `edge-dangling-target`, `viewbox-oversized` (architecture), `title-overflow`, and `subtitle-overflow`. Each declares the diagnostic codes it is expected to surface per diagram type, which is how the harness attributes a diagnostic to a defect inside a combined case. The two header-overflow classes use unbreakable strings so the defect can only be measured by real browser layout.
+
+`manifest.json` holds the suite: a single-defect probe per class per applicable type, plus one mixed-defect case per type whose classes have disjoint expected codes so every attribution stays unambiguous.
+
+## Commands
+
+```bash
+# Suite integrity: fixtures pass a clean validate, every defect injects a real change.
+node benchmarks/repair-rounds/benchmark.mjs check --manifest benchmarks/repair-rounds/manifest.json
+
+# Run the suite. Default: finalize + rules repair. Emits one JSONL receipt per case.
+node benchmarks/repair-rounds/benchmark.mjs run --manifest benchmarks/repair-rounds/manifest.json > results.jsonl
+
+# Subsets and modes:
+# --case <id> repeat to select cases
+# --repair oracle restore each attributed defect exactly (upper bound)
+# --command validate never touch the browser; late classes report undetected
+# --browser-never alias for --command validate
+# --max-rounds N repair cap per case (default 8)
+# --work-dir <dir> keep candidates, artifacts, and receipts for inspection
+
+# Aggregate a results file against the manifest.
+node benchmarks/repair-rounds/benchmark.mjs report --results results.jsonl --manifest benchmarks/repair-rounds/manifest.json
+```
+
+`run` sets `ARCHIFY_UPDATE_CHECK_DISABLED=1` so receipts are not perturbed by the update check. `browser-check` needs local Chrome; without it the late-discovery cases report the gate truthfully as skipped rather than passed.
+
+## Repair modes
+
+- `rules` (default): the repairer sees only the receipt — `subject` paths, `evidence`, `supportedFixes`, and pixel figures inside `layout/constraint` messages. It models the weakest honest agent: repairs derived purely from what the tool reports.
+- `oracle`: each attributed defect is restored exactly. This upper bound isolates staging (how early defects surface) from actionability (whether the receipt suffices to fix them).
+
+## Metrics provenance
+
+The suite borrows three objective ideas and implements them without their original subjective or model-based parts: multi-turn feedback efficiency (ConvCodeWorld, ICLR 2025), failure attribution by pipeline stage (in the spirit of HAL, ICLR 2026), and observable token accounting (TALE, ACL 2025 Findings). The token estimate is a fixed lexical counter, not a provider tokenizer — it exists to compare configurations, not to price a model.
+
+## Non-goals
+
+- No human or VLM/LLM scoring; no screenshot similarity.
+- No model, agent, or provider invocation — the harness only runs the CLI.
+- No renderer, validator, schema, or SKILL.md changes; the suite only observes them.
+- No latency claims from `wallMs`: timing is recorded context, not a gate.
+- No end-to-end authoring-cost claims: repair-guide reads are approximated by the documented failure-reading set, not measured from a live agent transcript.