Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.
HTML agent-skillsai-agentsarchitecture-diagramclaude-codeclaude-skillscodexcoding-agentsdeepseek-harnessdeveloper-toolsdiagramsdiagrams-as-codedsh-pluginflowchartllmmermaidopencodesequence-diagramsoftware-architecturesystem-designvisualization
feat: defect catalog and token meter for repair-rounds benchmark
momoyeyu7266d7cc5da3
diff --git a/benchmarks/repair-rounds/defects.mjs b/benchmarks/repair-rounds/defects.mjsnew file mode 100644index 00000000..a1fb3f57--- /dev/null+++ b/benchmarks/repair-rounds/defects.mjs@@ -0,0 +1,156 @@+// Defect catalog for the repair-rounds benchmark. Each class mutates a valid+// fixture deterministically, declares the diagnostic codes it is expected to+// surface, and carries an oracle repair that restores the fixture exactly.++export const COLLECTIONS = {+ architecture: { nodes: 'components', edges: 'connections' },+ workflow: { nodes: 'nodes', edges: 'edges' },+ sequence: { nodes: 'participants', edges: 'messages' },+ dataflow: { nodes: 'nodes', edges: 'flows' },+ lifecycle: { nodes: 'states', edges: 'transitions' },+};++const ALL_TYPES = Object.keys(COLLECTIONS);++function clone(value) {+ return JSON.parse(JSON.stringify(value));+}++function nodeAt(doc, type, index) {+ const nodes = doc[COLLECTIONS[type].nodes] || [];+ const at = index ?? Math.min(1, nodes.length - 1);+ return nodes[at];+}++function edgeAt(doc, type, index) {+ const edges = doc[COLLECTIONS[type].edges] || [];+ return edges[index ?? 0];+}++function restore(record) {+ return (doc) => {+ for (const { pointer, value, absent } of record.restores) {+ const parts = pointer.split('/').filter(Boolean).map((part) => part.replace(/~1/g, '/').replace(/~0/g, '~'));+ let target = doc;+ for (const part of parts.slice(0, -1)) target = target[part];+ const key = parts[parts.length - 1];+ if (absent) delete target[key];+ else target[key] = clone(value);+ }+ };+}++function capture(doc, pointer) {+ const parts = pointer.split('/').filter(Boolean).map((part) => part.replace(/~1/g, '/').replace(/~0/g, '~'));+ let target = doc;+ for (const part of parts.slice(0, -1)) target = target?.[part];+ const key = parts[parts.length - 1];+ const absent = target === undefined || target === null || !(key in target);+ return { pointer, value: absent ? undefined : clone(target[key]), absent };+}++const LONG_LABEL = 'Quarterly Cross-Regional Compliance Reconciliation And Settlement Orchestration Service';+const LONG_TITLE = 'Untrusted Extremely Long Diagram Title '.repeat(11).trim();+const LONG_SUBTITLE = 'A deliberately overlong subtitle that inflates the rendered header far beyond any reasonable desktop viewport width '.repeat(4).trim();++export const DEFECTS = {+ 'meta-missing-output': {+ appliesTo: ALL_TYPES,+ expectedCodes: { default: ['schema/required'] },+ inject(doc) {+ const restores = [capture(doc, '/meta/output')];+ delete doc.meta.output;+ return { restores };+ },+ },+ 'node-invalid-type': {+ appliesTo: ALL_TYPES,+ expectedCodes: { default: ['schema/enum', 'schema/const', 'schema/anyOf'] },+ inject(doc, { type, nodeIndex } = {}) {+ const node = nodeAt(doc, type, nodeIndex);+ const restores = [capture(doc, `/${COLLECTIONS[type].nodes}/${doc[COLLECTIONS[type].nodes].indexOf(node)}/type`)];+ node.type = 'benchmark-bogus-type';+ return { restores };+ },+ },+ 'node-long-label': {+ appliesTo: ALL_TYPES,+ expectedCodes: { default: ['layout/constraint'] },+ inject(doc, { type, nodeIndex } = {}) {+ const nodes = doc[COLLECTIONS[type].nodes];+ const node = nodeAt(doc, type, nodeIndex);+ const restores = [capture(doc, `/${COLLECTIONS[type].nodes}/${nodes.indexOf(node)}/label`)];+ node.label = LONG_LABEL;+ return { restores };+ },+ },+ 'node-duplicate-id': {+ appliesTo: ALL_TYPES,+ expectedCodes: {+ workflow: ['workflow/duplicate-node-id'],+ default: ['layout/constraint', 'clean-flow/edge-through-node', 'clean-flow/endpoint-side-direction', 'composition/proper-crossing', 'composition/label-route-clearance'],+ },+ inject(doc, { type, nodeIndex } = {}) {+ const nodes = doc[COLLECTIONS[type].nodes];+ const node = nodeAt(doc, type, nodeIndex);+ const restores = [capture(doc, `/${COLLECTIONS[type].nodes}/${nodes.indexOf(node)}/id`)];+ node.id = nodes[0].id;+ return { restores };+ },+ },+ 'edge-dangling-target': {+ appliesTo: ALL_TYPES,+ expectedCodes: {+ workflow: ['workflow/unknown-edge-endpoint'],+ default: ['layout/constraint', 'sequence/unknown-endpoint', 'dataflow/unknown-endpoint', 'lifecycle/unknown-endpoint'],+ },+ inject(doc, { type, edgeIndex } = {}) {+ const edges = doc[COLLECTIONS[type].edges];+ const edge = edgeAt(doc, type, edgeIndex);+ const restores = [capture(doc, `/${COLLECTIONS[type].edges}/${edges.indexOf(edge)}/to`)];+ edge.to = 'benchmark-ghost-node';+ return { restores };+ },+ },+ 'viewbox-oversized': {+ appliesTo: ['architecture'],+ expectedCodes: { default: ['composition/desktop-readability'] },+ inject(doc) {+ const restores = [capture(doc, '/meta/viewBox')];+ doc.meta.viewBox = [2400, 1200];+ return { restores };+ },+ },+ 'title-overflow': {+ appliesTo: ALL_TYPES,+ expectedCodes: { default: ['viewer/viewport-overflow'] },+ inject(doc) {+ const restores = [capture(doc, '/meta/title')];+ doc.meta.title = LONG_TITLE;+ return { restores };+ },+ },+ 'subtitle-overflow': {+ appliesTo: ALL_TYPES,+ expectedCodes: { default: ['viewer/viewport-overflow'] },+ inject(doc) {+ const restores = [capture(doc, '/meta/subtitle')];+ doc.meta.subtitle = LONG_SUBTITLE;+ return { restores };+ },+ },+};++export function defectEntry(classId, type) {+ const entry = DEFECTS[classId];+ if (!entry) throw new Error(`unknown defect class "${classId}"`);+ if (!entry.appliesTo.includes(type)) {+ throw new Error(`defect class "${classId}" does not apply to ${type}`);+ }+ return {+ class: classId,+ expectedCodes: entry.expectedCodes[type] || entry.expectedCodes.default,+ inject: (doc, options) => entry.inject(doc, options),+ oracleRepair: (doc, record) => restore(record)(doc),+ };+}diff --git a/benchmarks/repair-rounds/tokens.mjs b/benchmarks/repair-rounds/tokens.mjsnew file mode 100644index 00000000..10e0ec27--- /dev/null+++ b/benchmarks/repair-rounds/tokens.mjs@@ -0,0 +1,53 @@+// Deterministic token accounting for the repair-rounds benchmark.+// Dependency-free lexical estimate: word/number runs, individual punctuation,+// and whitespace runs. Not a model tokenizer; it is a stable comparison unit.++import fs from 'node:fs';+import path from 'node:path';++export function estimateTokens(text) {+ const matches = String(text).match(/[\p{L}\p{N}_]+|[^\s\p{L}\p{N}_]|\s+/gu);+ return matches ? matches.length : 0;+}++// Required reading per SKILL.md step 2 and the Type router:+// every type reads SKILL.md, authoring defaults, its schema, and its example;+// sequence/dataflow/lifecycle additionally read the common schema.+// On failure the agent reads the delivery contract; measured field or+// geometry failures also pull in the authoring contract.+const COMMON_READS = ['SKILL.md', 'references/authoring-defaults.md'];+const MODE_SCHEMA = (type) => `schemas/${type === 'dataflow' ? 'dataflow' : type}.schema.json`;+const MODE_EXAMPLE = {+ architecture: 'examples/web-app.architecture.json',+ workflow: 'examples/agent-tool-call.workflow.json',+ sequence: 'examples/cache-miss-request.sequence.json',+ dataflow: 'examples/product-analytics.dataflow.json',+ lifecycle: 'examples/deployment-release.lifecycle.json',+};+const NEEDS_COMMON_SCHEMA = new Set(['sequence', 'dataflow', 'lifecycle']);+const FAILURE_READS = ['references/delivery-contract.md', 'references/authoring-contract.md'];++export function initialReadingPaths(type) {+ const paths = [...COMMON_READS, MODE_SCHEMA(type), MODE_EXAMPLE[type]];+ if (NEEDS_COMMON_SCHEMA.has(type)) paths.splice(2, 0, 'schemas/common.schema.json');+ return paths;+}++export function failureReadingPaths() {+ return [...FAILURE_READS];+}++export function measureFiles(skillRoot, relativePaths) {+ const files = [];+ let bytes = 0;+ let tokens = 0;+ for (const relative of relativePaths) {+ const file = path.join(skillRoot, relative);+ const source = fs.readFileSync(file, 'utf8');+ const entry = { path: relative, bytes: Buffer.byteLength(source), tokens: estimateTokens(source) };+ bytes += entry.bytes;+ tokens += entry.tokens;+ files.push(entry);+ }+ return { files, bytes, tokens };+}