Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

feat: defect catalog and token meter for repair-rounds benchmark

momoyeyu7266d7cc5da3
diff --git a/benchmarks/repair-rounds/defects.mjs b/benchmarks/repair-rounds/defects.mjs
new file mode 100644
index 00000000..a1fb3f57
--- /dev/null
+++ b/benchmarks/repair-rounds/defects.mjs
@@ -0,0 +1,156 @@
+// Defect catalog for the repair-rounds benchmark. Each class mutates a valid
+// fixture deterministically, declares the diagnostic codes it is expected to
+// surface, and carries an oracle repair that restores the fixture exactly.
+
+export const COLLECTIONS = {
+ architecture: { nodes: 'components', edges: 'connections' },
+ workflow: { nodes: 'nodes', edges: 'edges' },
+ sequence: { nodes: 'participants', edges: 'messages' },
+ dataflow: { nodes: 'nodes', edges: 'flows' },
+ lifecycle: { nodes: 'states', edges: 'transitions' },
+};
+
+const ALL_TYPES = Object.keys(COLLECTIONS);
+
+function clone(value) {
+ return JSON.parse(JSON.stringify(value));
+}
+
+function nodeAt(doc, type, index) {
+ const nodes = doc[COLLECTIONS[type].nodes] || [];
+ const at = index ?? Math.min(1, nodes.length - 1);
+ return nodes[at];
+}
+
+function edgeAt(doc, type, index) {
+ const edges = doc[COLLECTIONS[type].edges] || [];
+ return edges[index ?? 0];
+}
+
+function restore(record) {
+ return (doc) => {
+ for (const { pointer, value, absent } of record.restores) {
+ const parts = pointer.split('/').filter(Boolean).map((part) => part.replace(/~1/g, '/').replace(/~0/g, '~'));
+ let target = doc;
+ for (const part of parts.slice(0, -1)) target = target[part];
+ const key = parts[parts.length - 1];
+ if (absent) delete target[key];
+ else target[key] = clone(value);
+ }
+ };
+}
+
+function capture(doc, pointer) {
+ const parts = pointer.split('/').filter(Boolean).map((part) => part.replace(/~1/g, '/').replace(/~0/g, '~'));
+ let target = doc;
+ for (const part of parts.slice(0, -1)) target = target?.[part];
+ const key = parts[parts.length - 1];
+ const absent = target === undefined || target === null || !(key in target);
+ return { pointer, value: absent ? undefined : clone(target[key]), absent };
+}
+
+const LONG_LABEL = 'Quarterly Cross-Regional Compliance Reconciliation And Settlement Orchestration Service';
+const LONG_TITLE = 'Untrusted Extremely Long Diagram Title '.repeat(11).trim();
+const LONG_SUBTITLE = 'A deliberately overlong subtitle that inflates the rendered header far beyond any reasonable desktop viewport width '.repeat(4).trim();
+
+export const DEFECTS = {
+ 'meta-missing-output': {
+ appliesTo: ALL_TYPES,
+ expectedCodes: { default: ['schema/required'] },
+ inject(doc) {
+ const restores = [capture(doc, '/meta/output')];
+ delete doc.meta.output;
+ return { restores };
+ },
+ },
+ 'node-invalid-type': {
+ appliesTo: ALL_TYPES,
+ expectedCodes: { default: ['schema/enum', 'schema/const', 'schema/anyOf'] },
+ inject(doc, { type, nodeIndex } = {}) {
+ const node = nodeAt(doc, type, nodeIndex);
+ const restores = [capture(doc, `/${COLLECTIONS[type].nodes}/${doc[COLLECTIONS[type].nodes].indexOf(node)}/type`)];
+ node.type = 'benchmark-bogus-type';
+ return { restores };
+ },
+ },
+ 'node-long-label': {
+ appliesTo: ALL_TYPES,
+ expectedCodes: { default: ['layout/constraint'] },
+ inject(doc, { type, nodeIndex } = {}) {
+ const nodes = doc[COLLECTIONS[type].nodes];
+ const node = nodeAt(doc, type, nodeIndex);
+ const restores = [capture(doc, `/${COLLECTIONS[type].nodes}/${nodes.indexOf(node)}/label`)];
+ node.label = LONG_LABEL;
+ return { restores };
+ },
+ },
+ 'node-duplicate-id': {
+ appliesTo: ALL_TYPES,
+ expectedCodes: {
+ workflow: ['workflow/duplicate-node-id'],
+ default: ['layout/constraint', 'clean-flow/edge-through-node', 'clean-flow/endpoint-side-direction', 'composition/proper-crossing', 'composition/label-route-clearance'],
+ },
+ inject(doc, { type, nodeIndex } = {}) {
+ const nodes = doc[COLLECTIONS[type].nodes];
+ const node = nodeAt(doc, type, nodeIndex);
+ const restores = [capture(doc, `/${COLLECTIONS[type].nodes}/${nodes.indexOf(node)}/id`)];
+ node.id = nodes[0].id;
+ return { restores };
+ },
+ },
+ 'edge-dangling-target': {
+ appliesTo: ALL_TYPES,
+ expectedCodes: {
+ workflow: ['workflow/unknown-edge-endpoint'],
+ default: ['layout/constraint', 'sequence/unknown-endpoint', 'dataflow/unknown-endpoint', 'lifecycle/unknown-endpoint'],
+ },
+ inject(doc, { type, edgeIndex } = {}) {
+ const edges = doc[COLLECTIONS[type].edges];
+ const edge = edgeAt(doc, type, edgeIndex);
+ const restores = [capture(doc, `/${COLLECTIONS[type].edges}/${edges.indexOf(edge)}/to`)];
+ edge.to = 'benchmark-ghost-node';
+ return { restores };
+ },
+ },
+ 'viewbox-oversized': {
+ appliesTo: ['architecture'],
+ expectedCodes: { default: ['composition/desktop-readability'] },
+ inject(doc) {
+ const restores = [capture(doc, '/meta/viewBox')];
+ doc.meta.viewBox = [2400, 1200];
+ return { restores };
+ },
+ },
+ 'title-overflow': {
+ appliesTo: ALL_TYPES,
+ expectedCodes: { default: ['viewer/viewport-overflow'] },
+ inject(doc) {
+ const restores = [capture(doc, '/meta/title')];
+ doc.meta.title = LONG_TITLE;
+ return { restores };
+ },
+ },
+ 'subtitle-overflow': {
+ appliesTo: ALL_TYPES,
+ expectedCodes: { default: ['viewer/viewport-overflow'] },
+ inject(doc) {
+ const restores = [capture(doc, '/meta/subtitle')];
+ doc.meta.subtitle = LONG_SUBTITLE;
+ return { restores };
+ },
+ },
+};
+
+export function defectEntry(classId, type) {
+ const entry = DEFECTS[classId];
+ if (!entry) throw new Error(`unknown defect class "${classId}"`);
+ if (!entry.appliesTo.includes(type)) {
+ throw new Error(`defect class "${classId}" does not apply to ${type}`);
+ }
+ return {
+ class: classId,
+ expectedCodes: entry.expectedCodes[type] || entry.expectedCodes.default,
+ inject: (doc, options) => entry.inject(doc, options),
+ oracleRepair: (doc, record) => restore(record)(doc),
+ };
+}
diff --git a/benchmarks/repair-rounds/tokens.mjs b/benchmarks/repair-rounds/tokens.mjs
new file mode 100644
index 00000000..10e0ec27
--- /dev/null
+++ b/benchmarks/repair-rounds/tokens.mjs
@@ -0,0 +1,53 @@
+// Deterministic token accounting for the repair-rounds benchmark.
+// Dependency-free lexical estimate: word/number runs, individual punctuation,
+// and whitespace runs. Not a model tokenizer; it is a stable comparison unit.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+export function estimateTokens(text) {
+ const matches = String(text).match(/[\p{L}\p{N}_]+|[^\s\p{L}\p{N}_]|\s+/gu);
+ return matches ? matches.length : 0;
+}
+
+// Required reading per SKILL.md step 2 and the Type router:
+// every type reads SKILL.md, authoring defaults, its schema, and its example;
+// sequence/dataflow/lifecycle additionally read the common schema.
+// On failure the agent reads the delivery contract; measured field or
+// geometry failures also pull in the authoring contract.
+const COMMON_READS = ['SKILL.md', 'references/authoring-defaults.md'];
+const MODE_SCHEMA = (type) => `schemas/${type === 'dataflow' ? 'dataflow' : type}.schema.json`;
+const MODE_EXAMPLE = {
+ architecture: 'examples/web-app.architecture.json',
+ workflow: 'examples/agent-tool-call.workflow.json',
+ sequence: 'examples/cache-miss-request.sequence.json',
+ dataflow: 'examples/product-analytics.dataflow.json',
+ lifecycle: 'examples/deployment-release.lifecycle.json',
+};
+const NEEDS_COMMON_SCHEMA = new Set(['sequence', 'dataflow', 'lifecycle']);
+const FAILURE_READS = ['references/delivery-contract.md', 'references/authoring-contract.md'];
+
+export function initialReadingPaths(type) {
+ const paths = [...COMMON_READS, MODE_SCHEMA(type), MODE_EXAMPLE[type]];
+ if (NEEDS_COMMON_SCHEMA.has(type)) paths.splice(2, 0, 'schemas/common.schema.json');
+ return paths;
+}
+
+export function failureReadingPaths() {
+ return [...FAILURE_READS];
+}
+
+export function measureFiles(skillRoot, relativePaths) {
+ const files = [];
+ let bytes = 0;
+ let tokens = 0;
+ for (const relative of relativePaths) {
+ const file = path.join(skillRoot, relative);
+ const source = fs.readFileSync(file, 'utf8');
+ const entry = { path: relative, bytes: Buffer.byteLength(source), tokens: estimateTokens(source) };
+ bytes += entry.bytes;
+ tokens += entry.tokens;
+ files.push(entry);
+ }
+ return { files, bytes, tokens };
+}