Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

fix(benchmark): reject reused semantic node bindings (#466)

Jaime Hernández109501fb516c
diff --git a/benchmarks/ordinary-model-floor/benchmark.mjs b/benchmarks/ordinary-model-floor/benchmark.mjs
index 89afa0fd..01f81d30 100644
--- a/benchmarks/ordinary-model-floor/benchmark.mjs
+++ b/benchmarks/ordinary-model-floor/benchmark.mjs
@@ -132,6 +132,7 @@ function evaluateSemantic(benchmarkCase, candidate) {
const bindings = {};
const ambiguousNodes = [];
const mismatchedNodes = [];
+ const identitiesByNode = new Map();
for (const required of requirements.nodes || []) {
const identity = required.key || required.id;
const binding = bindRequiredNode(required, nodesById, nodes);
@@ -147,6 +148,10 @@ function evaluateSemantic(benchmarkCase, candidate) {
continue;
}
if (typeof required.key === 'string') bindings[required.key] = actual.id;
+ if (typeof identity === 'string') {
+ if (!identitiesByNode.has(actual.id)) identitiesByNode.set(actual.id, new Set());
+ identitiesByNode.get(actual.id).add(identity);
+ }
for (const field of ['type', 'label', 'sublabel', 'tag']) {
if (required[field] !== undefined && actual[field] !== required[field]) {
mismatchedNodes.push({
@@ -175,6 +180,18 @@ function evaluateSemantic(benchmarkCase, candidate) {
});
}
}
+ // Resolve first so every conflicting identity is diagnosed, regardless of requirement order.
+ for (const [nodeId, identities] of identitiesByNode) {
+ if (identities.size < 2) continue;
+ for (const identity of identities) {
+ mismatchedNodes.push({
+ id: identity,
+ field: 'binding',
+ expected: 'distinct candidate node',
+ actual: nodeId,
+ });
+ }
+ }
const missingRelationships = (requirements.relationships || []).filter((relationship) => {
const resolved = {
...relationship,
diff --git a/test/ordinary-model-floor.test.mjs b/test/ordinary-model-floor.test.mjs
index dd71260a..4e104dc1 100644
--- a/test/ordinary-model-floor.test.mjs
+++ b/test/ordinary-model-floor.test.mjs
@@ -188,6 +188,83 @@ test('benchmark checks accepted node labels even when an authored identity match
}
});
+test('benchmark rejects one candidate node bound to distinct required identities', () => {
+ const caseSource = JSON.parse(fs.readFileSync(path.join(repoRoot, 'benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json'), 'utf8'));
+ const runFile = writeJson('reused-binding.run.json', {
+ schema_version: 1, case_id: caseSource.id,
+ agent: 'fixture-agent', model: 'fixture-model', attempt: 1,
+ // Synthetic review metadata isolates the semantic gate; no visual review occurred.
+ visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects: [] },
+ });
+ for (const storageId of ['storage', 'cache']) {
+ const candidate = writeJson('reused-binding.architecture.json', {
+ schema_version: 1, diagram_type: 'architecture',
+ meta: { title: 'Distinct storage requirements', output: 'binding.html', quality_profile: 'showcase' },
+ components: [
+ { id: 'users', type: 'external', label: 'Users', pos: [40, 180], size: [220, 64] },
+ { id: 'cdn', type: 'cloud', label: 'CloudFront', pos: [400, 180], size: [220, 64] },
+ { id: 'api', type: 'backend', label: 'API Server', pos: [760, 180], size: [220, 64] },
+ { id: storageId, type: 'database', label: 'Redis PostgreSQL', pos: [1120, 180], size: [220, 64] },
+ ],
+ connections: [
+ { from: 'users', to: 'cdn', label: 'HTTPS' },
+ { from: 'cdn', to: 'api' },
+ { from: 'api', to: storageId, label: 'read-through SQL' },
+ ],
+ });
+ for (const identityField of ['key', 'id']) {
+ for (const reverse of [false, true]) {
+ const caseData = structuredClone(caseSource);
+ const cache = caseData.requirements.nodes.find((node) => node.key === 'cache');
+ if (identityField === 'id') {
+ // Exercise an explicit ID sharing its node with an alias-bound requirement.
+ delete cache.key;
+ cache.id = storageId;
+ caseData.requirements.relationships.find((edge) => edge.to === 'cache').to = storageId;
+ }
+ if (reverse) caseData.requirements.nodes.reverse();
+ const caseFile = writeJson('reused-binding.case.json', caseData);
+ const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);
+ const receipt = JSON.parse(result.stdout);
+ assert.equal(receipt.gates.validation.ok, true);
+ assert.equal(receipt.gates.validation.checksPassed, 9);
+ assert.deepEqual(receipt.gates.semantic.missingNodeIds, []);
+ assert.deepEqual(receipt.gates.semantic.missingRelationships, []);
+ assert.equal(receipt.gates.semantic.ok, false);
+ assert.equal(result.status, 1, result.stderr || result.stdout);
+ const identities = [identityField === 'id' ? storageId : 'cache', 'db'].sort();
+ assert.deepEqual(receipt.gates.semantic.mismatchedNodes.slice().sort((a, b) => a.id.localeCompare(b.id)),
+ identities.map((id) => ({ id, field: 'binding', expected: 'distinct candidate node', actual: storageId })));
+ assert.equal(receipt.firstPassUsable, false);
+ const resultsFile = path.join(tmp, 'reused-binding.results.jsonl');
+ fs.writeFileSync(resultsFile, `${JSON.stringify(receipt)}\n`);
+ const reportResult = run(['report', '--results', resultsFile]);
+ assert.equal(reportResult.status, 0, reportResult.stderr || reportResult.stdout);
+ const report = JSON.parse(reportResult.stdout);
+ assert.deepEqual(report.overall.failureClusters, { semantic: 1, validation: 0, visualReview: 0, operational: 0 });
+ }
+ }
+ }
+});
+
+test('benchmark preserves repeated constraints on the same required identity', () => {
+ const caseFile = writeJson('same-identity.case.json', {
+ schema_version: 1, id: 'same-identity', diagram_type: 'architecture', quality_profile: 'showcase',
+ requirements: { nodes: [{ id: 'cache', type: 'database' }, { key: 'cache', labels: ['Redis'] }] },
+ });
+ const runFile = writeJson('same-identity.run.json', {
+ schema_version: 1, case_id: 'same-identity', agent: 'fixture-agent', model: 'fixture-model', attempt: 1,
+ visual_review: { status: 'skipped', defects: [] },
+ });
+ const result = run(['verify', '--case', caseFile, '--candidate', path.join(skillRoot, 'examples/web-app.architecture.json'), '--run', runFile]);
+ const receipt = JSON.parse(result.stdout);
+ assert.equal(receipt.gates.semantic.ok, true);
+ assert.deepEqual(receipt.gates.semantic.mismatchedNodes, []);
+ assert.equal(receipt.gates.validation.ok, true);
+ assert.equal(receipt.firstPassUsable, false, 'an omitted visual review is still not a pass');
+ assert.equal(result.status, 1);
+});
+
test('benchmark never accepts a visual pass without an identified reviewer', () => {
const caseFile = writeJson('unreviewed.case.json', {
schema_version: 1,