Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.
HTML agent-skillsai-agentsarchitecture-diagramclaude-codeclaude-skillscodexcoding-agentsdeepseek-harnessdeveloper-toolsdiagramsdiagrams-as-codedsh-pluginflowchartllmmermaidopencodesequence-diagramsoftware-architecturesystem-designvisualization
feat: repair-rounds tests and protocol documentation
momoyeyufd036b9b2ccb
diff --git a/archify/test/repair-rounds.test.mjs b/archify/test/repair-rounds.test.mjsnew file mode 100644index 00000000..df5bb1c5--- /dev/null+++ b/archify/test/repair-rounds.test.mjs@@ -0,0 +1,197 @@+import assert from 'node:assert/strict';+import { spawnSync } from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import test from 'node:test';+import { fileURLToPath } from 'node:url';++const here = path.dirname(fileURLToPath(import.meta.url));+const skillRoot = path.resolve(here, '..');+const repoRoot = path.resolve(skillRoot, '..');+const benchmark = path.join(repoRoot, 'benchmarks/repair-rounds/benchmark.mjs');+const manifest = path.join(repoRoot, 'benchmarks/repair-rounds/manifest.json');+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'archify-repair-rounds-'));+const chromeAvailable = Boolean(process.env.ARCHIFY_CHROME)+ || fs.existsSync('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome');++function run(args) {+ return spawnSync(process.execPath, [benchmark, ...args], {+ cwd: repoRoot,+ encoding: 'utf8',+ env: { ...process.env, ARCHIFY_UPDATE_CHECK_DISABLED: '1' },+ });+}++function writeJson(name, value) {+ const file = path.join(tmp, name);+ fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);+ return file;+}++function miniManifest(id, type, defects) {+ const fixture = path.join(skillRoot, 'examples', {+ architecture: 'web-app.architecture.json',+ workflow: 'agent-tool-call.workflow.json',+ sequence: 'cache-miss-request.sequence.json',+ dataflow: 'product-analytics.dataflow.json',+ lifecycle: 'deployment-release.lifecycle.json',+ }[type]);+ return writeJson(`${id}.manifest.json`, {+ schema_version: 1,+ id,+ purpose: 'feedback-efficiency',+ evidence_eligible: false,+ fixtures: { [type]: fixture },+ cases: [{ id, type, defects }],+ });+}++test('check accepts the checked-in suite end to end', () => {+ const result = run(['check', '--manifest', manifest]);+ assert.equal(result.status, 0, result.stderr || result.stdout);+ const receipt = JSON.parse(result.stdout);+ assert.equal(receipt.schemaVersion, 1);+ assert.equal(receipt.benchmark, 'repair-rounds');+ assert.equal(receipt.caseCount, 41);+ assert.equal(receipt.evidenceEligible, false);+ for (const entry of receipt.cases) {+ assert.deepEqual(entry.problems, [], entry.caseId);+ }+});++test('check reports an unknown defect class instead of crashing', () => {+ const bad = miniManifest('bad-defect', 'architecture', [{ class: 'does-not-exist' }]);+ const result = run(['check', '--manifest', bad]);+ assert.equal(result.status, 1, result.stderr || result.stdout);+ const receipt = JSON.parse(result.stdout);+ assert.match(receipt.cases[0].problems[0], /unknown defect class/);+});++test('rules repair converges a schema defect reported with a path and evidence', () => {+ const file = miniManifest('flow-missing-output', 'workflow', [{ class: 'meta-missing-output' }]);+ const result = run(['run', '--manifest', file, '--command', 'validate']);+ assert.equal(result.status, 0, result.stderr || result.stdout);+ const receipt = JSON.parse(result.stdout.split('\n')[0]);+ assert.equal(receipt.passed, true);+ assert.equal(receipt.totals.rounds, 2);+ assert.deepEqual(receipt.defects.map((d) => d.firstSeenGate), ['validate:render']);+ assert.equal(receipt.rounds[0].diagnostics[0].code, 'schema/required');+ assert.equal(receipt.rounds[0].repairs.applied.length, 1);+ assert.equal(receipt.totals.tokens.context > 0, true);+ assert.equal(receipt.totals.tokens.receipts > 0, true);+});++test('a staged second defect surfaces only after the first is repaired', () => {+ const file = miniManifest('arch-staged', 'architecture', [+ { class: 'meta-missing-output' },+ { class: 'node-long-label' },+ ]);+ const result = run(['run', '--manifest', file, '--command', 'validate', '--repair', 'oracle']);+ assert.equal(result.status, 0, result.stderr || result.stdout);+ const receipt = JSON.parse(result.stdout.split('\n')[0]);+ assert.equal(receipt.passed, true);+ assert.equal(receipt.totals.rounds, 3);+ const [schema, label] = receipt.defects;+ assert.equal(schema.class, 'meta-missing-output');+ assert.equal(schema.firstSeenRound, 1);+ assert.equal(label.class, 'node-long-label');+ assert.equal(label.firstSeenRound, 2);+});++test('diagnostics the rules cannot map are counted as unactionable and stall the loop', () => {+ const file = miniManifest('arch-duplicate-id', 'architecture', [{ class: 'node-duplicate-id' }]);+ const result = run(['run', '--manifest', file, '--command', 'validate']);+ assert.equal(result.status, 0, result.stderr || result.stdout);+ const receipt = JSON.parse(result.stdout.split('\n')[0]);+ assert.equal(receipt.stalled, true);+ assert.equal(receipt.passed, false);+ assert.equal(receipt.totals.unactionable > 0, true);+});++test('a header overflow is detected late, only at the browser-check gate', { skip: !chromeAvailable }, () => {+ const file = miniManifest('arch-title-overflow', 'architecture', [{ class: 'title-overflow' }]);+ const result = run(['run', '--manifest', file, '--command', 'finalize', '--repair', 'oracle']);+ assert.equal(result.status, 0, result.stderr || result.stdout);+ const receipt = JSON.parse(result.stdout.split('\n')[0]);+ assert.equal(receipt.passed, true);+ assert.equal(receipt.defects[0].firstSeenGate, 'browser-check');+ assert.equal(receipt.defects[0].late, true);+});++test('report aggregates disclosure, late discovery, and token cost across receipts', () => {+ const receipts = [+ {+ schemaVersion: 1,+ benchmark: 'repair-rounds',+ caseId: 'early-case',+ diagramType: 'workflow',+ command: 'validate',+ repairMode: 'rules',+ defects: [{ class: 'meta-missing-output', detected: true, firstSeenRound: 1, firstSeenGate: 'validate', late: false }],+ rounds: [{ diagnostics: [{ code: 'schema/required', hasSubjectDetail: true, hasEvidence: true, hasSupportedFixes: true }] }],+ passed: true,+ stalled: false,+ totals: { rounds: 2, diagnostics: 1, unactionable: 0, tokens: { estimate: 40000, receipts: 900 }, wallMs: 500 },+ },+ {+ schemaVersion: 1,+ benchmark: 'repair-rounds',+ caseId: 'late-case',+ diagramType: 'architecture',+ command: 'finalize',+ repairMode: 'oracle',+ defects: [+ { class: 'node-long-label', detected: true, firstSeenRound: 1, firstSeenGate: 'validate', late: false },+ { class: 'title-overflow', detected: true, firstSeenRound: 2, firstSeenGate: 'browser-check', late: true },+ { class: 'subtitle-overflow', detected: false, firstSeenRound: null, firstSeenGate: null, late: false },+ ],+ rounds: [{ diagnostics: [{ code: 'viewer/viewport-overflow', hasSubjectDetail: true, hasEvidence: true, hasSupportedFixes: true }] }],+ passed: true,+ stalled: false,+ totals: { rounds: 3, diagnostics: 1, unactionable: 0, tokens: { estimate: 80000, receipts: 6000 }, wallMs: 4000 },+ },+ ];+ const resultsFile = path.join(tmp, 'repair-results.jsonl');+ fs.writeFileSync(resultsFile, `${receipts.map((row) => JSON.stringify(row)).join('\n')}\n`);++ const result = run(['report', '--results', resultsFile]);+ assert.equal(result.status, 0, result.stderr || result.stdout);+ const report = JSON.parse(result.stdout);+ assert.equal(report.overall.cases, 2);+ assert.equal(report.overall.passed, 2);+ assert.deepEqual(report.overall.roundsToPass, { mean: 2.5, median: 3, max: 3 });+ assert.equal(report.overall.disclosure.defects, 4);+ assert.equal(report.overall.disclosure.detected, 3);+ assert.equal(report.overall.disclosure.undetected, 1);+ assert.equal(report.overall.disclosure.firstRoundDisclosureRate, 0.5);+ assert.equal(report.overall.disclosure.lateDiscoveryRate, 1 / 3);+ assert.equal(report.overall.disclosure.firstSeenGate['browser-check'], 1);+ assert.equal(report.overall.tokens.meanEstimatePerCase, 60000);+});++test('report rejects duplicate and foreign receipts', () => {+ const receipt = {+ schemaVersion: 1,+ benchmark: 'repair-rounds',+ caseId: 'dup',+ diagramType: 'workflow',+ command: 'validate',+ repairMode: 'rules',+ defects: [],+ rounds: [],+ passed: true,+ totals: { rounds: 1, diagnostics: 0, unactionable: 0, tokens: { estimate: 1, receipts: 1 }, wallMs: 1 },+ };+ const dupFile = path.join(tmp, 'dup-results.jsonl');+ fs.writeFileSync(dupFile, `${JSON.stringify(receipt)}\n${JSON.stringify(receipt)}\n`);+ const dup = run(['report', '--results', dupFile]);+ assert.equal(dup.status, 2);+ assert.equal(JSON.parse(dup.stdout).error.code, 'DUPLICATE_RESULT');++ const foreignFile = path.join(tmp, 'foreign-results.jsonl');+ fs.writeFileSync(foreignFile, `${JSON.stringify({ schemaVersion: 1, benchmark: 'ordinary-model-floor' })}\n`);+ const foreign = run(['report', '--results', foreignFile]);+ assert.equal(foreign.status, 2);+ assert.equal(JSON.parse(foreign.stdout).error.code, 'INVALID_RESULT');+});diff --git a/benchmarks/repair-rounds/README.md b/benchmarks/repair-rounds/README.mdnew file mode 100644index 00000000..acb01f34--- /dev/null+++ b/benchmarks/repair-rounds/README.md@@ -0,0 +1,64 @@+# Repair Rounds v1++This benchmark measures a narrow product question: **how efficiently do Archify receipts let a caller converge to a passing artifact?** It complements `ordinary-model-floor` (first-pass usability) and the `authoring-cost` studies (real agent time and token spend) by isolating the part of authoring cost that lives inside the feedback loop — how many validation rounds a defect costs, how early each defect is surfaced, and how much text each round forces the caller to read.++It is a measurement harness, not a model leaderboard, and it contains no human or model judgment. Everything is driven by deterministic defect injection against the checked-in example fixtures and the real Archify CLI.++## What it measures++| Metric | Definition |+|---|---|+| Rounds to pass | `finalize`/`validate` invocations needed before the mutated candidate passes |+| First-round disclosure rate | injected defects surfaced by round 1 ÷ injected defects |+| First-detection stage | the gate that first reports each defect (`validate`, `deliver`, `check`, `browser-check`, or `undetected`) |+| Late discovery rate | detected defects first surfaced at `check` or `browser-check` ÷ detected defects |+| Actionability | repairs a rules-only repairer could derive from diagnostics alone; `unactionable` receipts are counted honestly |+| Observable token cost | a deterministic lexical estimate over the required-reading set, each round's receipt, and each round's candidate |++A defect reported only after earlier gates passed means at least one extra authoring round in the real loop; `viewer/viewport-overflow` is the canonical example. Diagnostics that name no subject, carry no evidence, and suggest no fix produce `unactionable` repairs rather than silent passes.++## Defect classes++`defects.mjs` defines eight deterministic mutations: `meta-missing-output`, `node-invalid-type`, `node-long-label`, `node-duplicate-id`, `edge-dangling-target`, `viewbox-oversized` (architecture), `title-overflow`, and `subtitle-overflow`. Each declares the diagnostic codes it is expected to surface per diagram type, which is how the harness attributes a diagnostic to a defect inside a combined case. The two header-overflow classes use unbreakable strings so the defect can only be measured by real browser layout.++`manifest.json` holds the suite: a single-defect probe per class per applicable type, plus one mixed-defect case per type whose classes have disjoint expected codes so every attribution stays unambiguous.++## Commands++```bash+# Suite integrity: fixtures pass a clean validate, every defect injects a real change.+node benchmarks/repair-rounds/benchmark.mjs check --manifest benchmarks/repair-rounds/manifest.json++# Run the suite. Default: finalize + rules repair. Emits one JSONL receipt per case.+node benchmarks/repair-rounds/benchmark.mjs run --manifest benchmarks/repair-rounds/manifest.json > results.jsonl++# Subsets and modes:+# --case <id> repeat to select cases+# --repair oracle restore each attributed defect exactly (upper bound)+# --command validate never touch the browser; late classes report undetected+# --browser-never alias for --command validate+# --max-rounds N repair cap per case (default 8)+# --work-dir <dir> keep candidates, artifacts, and receipts for inspection++# Aggregate a results file against the manifest.+node benchmarks/repair-rounds/benchmark.mjs report --results results.jsonl --manifest benchmarks/repair-rounds/manifest.json+```++`run` sets `ARCHIFY_UPDATE_CHECK_DISABLED=1` so receipts are not perturbed by the update check. `browser-check` needs local Chrome; without it the late-discovery cases report the gate truthfully as skipped rather than passed.++## Repair modes++- `rules` (default): the repairer sees only the receipt — `subject` paths, `evidence`, `supportedFixes`, and pixel figures inside `layout/constraint` messages. It models the weakest honest agent: repairs derived purely from what the tool reports.+- `oracle`: each attributed defect is restored exactly. This upper bound isolates staging (how early defects surface) from actionability (whether the receipt suffices to fix them).++## Metrics provenance++The suite borrows three objective ideas and implements them without their original subjective or model-based parts: multi-turn feedback efficiency (ConvCodeWorld, ICLR 2025), failure attribution by pipeline stage (in the spirit of HAL, ICLR 2026), and observable token accounting (TALE, ACL 2025 Findings). The token estimate is a fixed lexical counter, not a provider tokenizer — it exists to compare configurations, not to price a model.++## Non-goals++- No human or VLM/LLM scoring; no screenshot similarity.+- No model, agent, or provider invocation — the harness only runs the CLI.+- No renderer, validator, schema, or SKILL.md changes; the suite only observes them.+- No latency claims from `wallMs`: timing is recorded context, not a gate.+- No end-to-end authoring-cost claims: repair-guide reads are approximated by the documented failure-reading set, not measured from a live agent transcript.