Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.
HTML agent-skillsai-agentsarchitecture-diagramclaude-codeclaude-skillscodexcoding-agentsdeepseek-harnessdeveloper-toolsdiagramsdiagrams-as-codedsh-pluginflowchartllmmermaidopencodesequence-diagramsoftware-architecturesystem-designvisualization
fix(benchmark): reject reused semantic node bindings (#466)
Jaime Hernández109501fb516c
diff --git a/benchmarks/ordinary-model-floor/benchmark.mjs b/benchmarks/ordinary-model-floor/benchmark.mjsindex 89afa0fd..01f81d30 100644--- a/benchmarks/ordinary-model-floor/benchmark.mjs+++ b/benchmarks/ordinary-model-floor/benchmark.mjs@@ -132,6 +132,7 @@ function evaluateSemantic(benchmarkCase, candidate) {const bindings = {};const ambiguousNodes = [];const mismatchedNodes = [];+ const identitiesByNode = new Map();for (const required of requirements.nodes || []) {const identity = required.key || required.id;const binding = bindRequiredNode(required, nodesById, nodes);@@ -147,6 +148,10 @@ function evaluateSemantic(benchmarkCase, candidate) {continue;}if (typeof required.key === 'string') bindings[required.key] = actual.id;+ if (typeof identity === 'string') {+ if (!identitiesByNode.has(actual.id)) identitiesByNode.set(actual.id, new Set());+ identitiesByNode.get(actual.id).add(identity);+ }for (const field of ['type', 'label', 'sublabel', 'tag']) {if (required[field] !== undefined && actual[field] !== required[field]) {mismatchedNodes.push({@@ -175,6 +180,18 @@ function evaluateSemantic(benchmarkCase, candidate) {});}}+ // Resolve first so every conflicting identity is diagnosed, regardless of requirement order.+ for (const [nodeId, identities] of identitiesByNode) {+ if (identities.size < 2) continue;+ for (const identity of identities) {+ mismatchedNodes.push({+ id: identity,+ field: 'binding',+ expected: 'distinct candidate node',+ actual: nodeId,+ });+ }+ }const missingRelationships = (requirements.relationships || []).filter((relationship) => {const resolved = {...relationship,diff --git a/test/ordinary-model-floor.test.mjs b/test/ordinary-model-floor.test.mjsindex dd71260a..4e104dc1 100644--- a/test/ordinary-model-floor.test.mjs+++ b/test/ordinary-model-floor.test.mjs@@ -188,6 +188,83 @@ test('benchmark checks accepted node labels even when an authored identity match}});+test('benchmark rejects one candidate node bound to distinct required identities', () => {+ const caseSource = JSON.parse(fs.readFileSync(path.join(repoRoot, 'benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json'), 'utf8'));+ const runFile = writeJson('reused-binding.run.json', {+ schema_version: 1, case_id: caseSource.id,+ agent: 'fixture-agent', model: 'fixture-model', attempt: 1,+ // Synthetic review metadata isolates the semantic gate; no visual review occurred.+ visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects: [] },+ });+ for (const storageId of ['storage', 'cache']) {+ const candidate = writeJson('reused-binding.architecture.json', {+ schema_version: 1, diagram_type: 'architecture',+ meta: { title: 'Distinct storage requirements', output: 'binding.html', quality_profile: 'showcase' },+ components: [+ { id: 'users', type: 'external', label: 'Users', pos: [40, 180], size: [220, 64] },+ { id: 'cdn', type: 'cloud', label: 'CloudFront', pos: [400, 180], size: [220, 64] },+ { id: 'api', type: 'backend', label: 'API Server', pos: [760, 180], size: [220, 64] },+ { id: storageId, type: 'database', label: 'Redis PostgreSQL', pos: [1120, 180], size: [220, 64] },+ ],+ connections: [+ { from: 'users', to: 'cdn', label: 'HTTPS' },+ { from: 'cdn', to: 'api' },+ { from: 'api', to: storageId, label: 'read-through SQL' },+ ],+ });+ for (const identityField of ['key', 'id']) {+ for (const reverse of [false, true]) {+ const caseData = structuredClone(caseSource);+ const cache = caseData.requirements.nodes.find((node) => node.key === 'cache');+ if (identityField === 'id') {+ // Exercise an explicit ID sharing its node with an alias-bound requirement.+ delete cache.key;+ cache.id = storageId;+ caseData.requirements.relationships.find((edge) => edge.to === 'cache').to = storageId;+ }+ if (reverse) caseData.requirements.nodes.reverse();+ const caseFile = writeJson('reused-binding.case.json', caseData);+ const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);+ const receipt = JSON.parse(result.stdout);+ assert.equal(receipt.gates.validation.ok, true);+ assert.equal(receipt.gates.validation.checksPassed, 9);+ assert.deepEqual(receipt.gates.semantic.missingNodeIds, []);+ assert.deepEqual(receipt.gates.semantic.missingRelationships, []);+ assert.equal(receipt.gates.semantic.ok, false);+ assert.equal(result.status, 1, result.stderr || result.stdout);+ const identities = [identityField === 'id' ? storageId : 'cache', 'db'].sort();+ assert.deepEqual(receipt.gates.semantic.mismatchedNodes.slice().sort((a, b) => a.id.localeCompare(b.id)),+ identities.map((id) => ({ id, field: 'binding', expected: 'distinct candidate node', actual: storageId })));+ assert.equal(receipt.firstPassUsable, false);+ const resultsFile = path.join(tmp, 'reused-binding.results.jsonl');+ fs.writeFileSync(resultsFile, `${JSON.stringify(receipt)}\n`);+ const reportResult = run(['report', '--results', resultsFile]);+ assert.equal(reportResult.status, 0, reportResult.stderr || reportResult.stdout);+ const report = JSON.parse(reportResult.stdout);+ assert.deepEqual(report.overall.failureClusters, { semantic: 1, validation: 0, visualReview: 0, operational: 0 });+ }+ }+ }+});++test('benchmark preserves repeated constraints on the same required identity', () => {+ const caseFile = writeJson('same-identity.case.json', {+ schema_version: 1, id: 'same-identity', diagram_type: 'architecture', quality_profile: 'showcase',+ requirements: { nodes: [{ id: 'cache', type: 'database' }, { key: 'cache', labels: ['Redis'] }] },+ });+ const runFile = writeJson('same-identity.run.json', {+ schema_version: 1, case_id: 'same-identity', agent: 'fixture-agent', model: 'fixture-model', attempt: 1,+ visual_review: { status: 'skipped', defects: [] },+ });+ const result = run(['verify', '--case', caseFile, '--candidate', path.join(skillRoot, 'examples/web-app.architecture.json'), '--run', runFile]);+ const receipt = JSON.parse(result.stdout);+ assert.equal(receipt.gates.semantic.ok, true);+ assert.deepEqual(receipt.gates.semantic.mismatchedNodes, []);+ assert.equal(receipt.gates.validation.ok, true);+ assert.equal(receipt.firstPassUsable, false, 'an omitted visual review is still not a pass');+ assert.equal(result.status, 1);+});+test('benchmark never accepts a visual pass without an identified reviewer', () => {const caseFile = writeJson('unreviewed.case.json', {schema_version: 1,