Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

fix: calibrate ordinary-model semantic equivalence

tt-a1i0cccd0eadf77
diff --git a/archify/test/ordinary-model-floor.test.mjs b/archify/test/ordinary-model-floor.test.mjs
index a6deb013..2bea0d15 100644
--- a/archify/test/ordinary-model-floor.test.mjs
+++ b/archify/test/ordinary-model-floor.test.mjs
@@ -25,6 +25,33 @@ function run(args) {
});
}
+function renameCandidateIds(candidate, mapping) {
+ for (const collection of ['components', 'nodes', 'participants', 'states']) {
+ for (const node of candidate[collection] || []) {
+ node.id = mapping.get(node.id) || node.id;
+ }
+ }
+ for (const collection of ['connections', 'edges', 'messages', 'flows', 'transitions']) {
+ for (const relationship of candidate[collection] || []) {
+ relationship.from = mapping.get(relationship.from) || relationship.from;
+ relationship.to = mapping.get(relationship.to) || relationship.to;
+ }
+ }
+ for (const activation of candidate.activations || []) {
+ activation.participant = mapping.get(activation.participant) || activation.participant;
+ }
+ if (Array.isArray(candidate.mainPath)) {
+ candidate.mainPath = candidate.mainPath.map((id) => mapping.get(id) || id);
+ }
+ for (const boundary of candidate.boundaries || []) {
+ boundary.wraps = boundary.wraps.map((id) => mapping.get(id) || id);
+ }
+ for (const view of candidate.meta?.views || []) {
+ view.focus = view.focus.map((id) => mapping.get(id) || id);
+ }
+ return candidate;
+}
+
test('benchmark verifies one first-pass architecture candidate through semantic, renderer, and visual-review gates', () => {
const caseFile = writeJson('web-runtime.case.json', {
schema_version: 1,
@@ -445,6 +472,134 @@ test('benchmark semantic requirements bind by accepted technical labels instead
assert.equal(receipt.firstPassUsable, true);
});
+test('checked-in cases accept equivalent ordinary-model vocabulary without weakening required topology', () => {
+ const suiteRoot = path.join(repoRoot, 'benchmarks/ordinary-model-floor');
+ const fixtures = [
+ {
+ type: 'workflow',
+ example: 'agent-tool-call.workflow.json',
+ caseFile: 'agent-tool-call.workflow.case.json',
+ mapping: new Map([
+ ['planner', 'task-planner'], ['router', 'risk-router'], ['approval', 'consent-check'],
+ ['tool', 'tool-runner'], ['blocked', 'request-blocked'], ['external', 'service-provider'],
+ ]),
+ mutate(candidate) {
+ Object.assign(candidate.nodes.find((node) => node.id === 'risk-router'), { label: 'Router', type: 'security' });
+ candidate.nodes.find((node) => node.id === 'consent-check').label = 'Human Approval';
+ candidate.nodes.find((node) => node.id === 'tool-runner').label = 'Tool Executor';
+ candidate.nodes.find((node) => node.id === 'request-blocked').label = 'Denied';
+ candidate.nodes.find((node) => node.id === 'service-provider').label = 'Provider API';
+ },
+ },
+ {
+ type: 'sequence',
+ example: 'cache-miss-request.sequence.json',
+ caseFile: 'cache-miss.sequence.case.json',
+ mapping: new Map([
+ ['web', 'browser-tab'], ['api', 'dashboard-api'], ['auth', 'token-guard'],
+ ['redis', 'response-cache'], ['db', 'account-store'],
+ ]),
+ mutate(candidate) {
+ Object.assign(candidate.participants.find((node) => node.id === 'browser-tab'), { label: 'Browser', type: 'external' });
+ candidate.participants.find((node) => node.id === 'dashboard-api').label = 'Dashboard API';
+ candidate.participants.find((node) => node.id === 'token-guard').label = 'JWT Guard';
+ candidate.messages.find((message) => message.id === 'verify-jwt').label = 'validate JWT';
+ candidate.messages.find((message) => message.id === 'cache-read').label = 'GET dashboard key';
+ candidate.messages.find((message) => message.id === 'profile-query').label = 'SELECT profile + metrics';
+ candidate.messages.find((message) => message.id === 'cache-write').label = 'SET assembled result';
+ },
+ },
+ {
+ type: 'dataflow',
+ example: 'product-analytics.dataflow.json',
+ caseFile: 'product-analytics.dataflow.case.json',
+ mapping: new Map([
+ ['edge', 'ingest-gateway'], ['consent', 'consent-filter'], ['stream', 'event-stream'],
+ ['pii', 'identity-vault'], ['warehouse', 'facts-warehouse'], ['dashboard', 'metric-dashboards'],
+ ]),
+ mutate(candidate) {
+ candidate.nodes.find((node) => node.id === 'ingest-gateway').label = 'Ingestion Gateway';
+ candidate.nodes.find((node) => node.id === 'consent-filter').label = 'Consent Filter';
+ candidate.nodes.find((node) => node.id === 'identity-vault').label = 'Identity Vault';
+ candidate.nodes.find((node) => node.id === 'facts-warehouse').label = 'Facts Warehouse';
+ candidate.nodes.find((node) => node.id === 'metric-dashboards').label = 'Metric Dashboards';
+ candidate.flows.find((flow) => flow.id === 'consent-enrichment').label = 'identity claims';
+ candidate.flows.find((flow) => flow.id === 'identity-map').label = 'consented identity';
+ candidate.flows.find((flow) => flow.id === 'metrics-query').label = 'KPI reads';
+ },
+ },
+ {
+ type: 'lifecycle',
+ example: 'agent-run.lifecycle.json',
+ caseFile: 'agent-run.lifecycle.case.json',
+ mapping: new Map([
+ ['queued', 'run-queued'], ['executing', 'run-executing'], ['reviewing', 'run-reviewing'],
+ ['approval', 'approval-wait'], ['blocked', 'input-wait'], ['cancelled', 'user-cancelled'],
+ ['expired', 'run-expired'],
+ ]),
+ mutate(candidate) {
+ candidate.states.find((node) => node.id === 'approval-wait').label = 'Awaiting Approval';
+ candidate.states.find((node) => node.id === 'input-wait').label = 'Awaiting Input';
+ },
+ },
+ ];
+
+ for (const fixture of fixtures) {
+ const source = JSON.parse(fs.readFileSync(path.join(skillRoot, 'examples', fixture.example), 'utf8'));
+ const candidate = renameCandidateIds(source, fixture.mapping);
+ fixture.mutate(candidate);
+ const candidateFile = writeJson(`calibrated-${fixture.type}.json`, candidate);
+ const benchmarkCase = JSON.parse(fs.readFileSync(
+ path.join(suiteRoot, 'cases', fixture.caseFile),
+ 'utf8',
+ ));
+ const runFile = writeJson(`calibrated-${fixture.type}.run.json`, {
+ schema_version: 1,
+ case_id: benchmarkCase.id,
+ agent: 'ordinary-agent',
+ model: 'ordinary-model',
+ attempt: 1,
+ visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects: [] },
+ });
+
+ const result = run([
+ 'verify', '--case', path.join(suiteRoot, 'cases', fixture.caseFile),
+ '--candidate', candidateFile, '--run', runFile,
+ ]);
+
+ assert.equal(result.status, 0, `${fixture.type}: ${result.stderr || result.stdout}`);
+ assert.equal(JSON.parse(result.stdout).gates.semantic.ok, true, fixture.type);
+
+ if (fixture.type === 'workflow') {
+ const wrongRoleCandidate = structuredClone(candidate);
+ wrongRoleCandidate.nodes.find((node) => node.id === 'risk-router').type = 'external';
+ const wrongRoleFile = writeJson('calibrated-workflow-wrong-role.json', wrongRoleCandidate);
+ const wrongRole = run([
+ 'verify', '--case', path.join(suiteRoot, 'cases', fixture.caseFile),
+ '--candidate', wrongRoleFile, '--run', runFile,
+ ]);
+ assert.equal(wrongRole.status, 1, wrongRole.stderr || wrongRole.stdout);
+ assert.deepEqual(JSON.parse(wrongRole.stdout).gates.semantic.mismatchedNodes, [{
+ id: 'router',
+ field: 'type',
+ expected: ['backend', 'security'],
+ actual: 'external',
+ }]);
+
+ candidate.edges = candidate.edges.filter(
+ (edge) => !(edge.from === 'consent-check' && edge.to === 'tool-runner'),
+ );
+ const brokenCandidate = writeJson('calibrated-workflow-missing-route.json', candidate);
+ const broken = run([
+ 'verify', '--case', path.join(suiteRoot, 'cases', fixture.caseFile),
+ '--candidate', brokenCandidate, '--run', runFile,
+ ]);
+ assert.equal(broken.status, 1, broken.stderr || broken.stdout);
+ assert.equal(JSON.parse(broken.stdout).gates.semantic.missingRelationships.length, 1);
+ }
+ }
+});
+
test('checked-in benchmark suite covers all five diagram types without presenting reference fixtures as model evidence', () => {
const manifest = path.join(repoRoot, 'benchmarks/ordinary-model-floor/manifest.json');
diff --git a/benchmarks/ordinary-model-floor/README.md b/benchmarks/ordinary-model-floor/README.md
index 1229cf7b..e3075980 100644
--- a/benchmarks/ordinary-model-floor/README.md
+++ b/benchmarks/ordinary-model-floor/README.md
@@ -12,7 +12,7 @@ A renderer-valid but semantically wrong diagram is a failure. A visually pleasin
## Suite
-`manifest.json` contains five bounded tasks: architecture, workflow, sequence, data flow, and lifecycle. Each case declares semantic keys, accepted technical labels, required roles, and required relationships. The model remains free to choose internal IDs and layout.
+`manifest.json` contains five bounded tasks: architecture, workflow, sequence, data flow, and lifecycle. Each case declares semantic keys, accepted technical labels, accepted visual-role types where more than one rendering is legitimate, and required relationships. The model remains free to choose internal IDs and layout. Vocabulary aliases never replace topology: every required node must bind once and every required relationship must still exist in the declared direction.
The checked-in reference fixtures only prove that the suite and verifier are wired correctly. **Reference fixtures are not benchmark evidence** and must not be published as model results.
diff --git a/benchmarks/ordinary-model-floor/benchmark.mjs b/benchmarks/ordinary-model-floor/benchmark.mjs
index f449e1e2..36a65f9e 100644
--- a/benchmarks/ordinary-model-floor/benchmark.mjs
+++ b/benchmarks/ordinary-model-floor/benchmark.mjs
@@ -144,6 +144,14 @@ function evaluateSemantic(benchmarkCase, candidate) {
});
}
}
+ if (Array.isArray(required.types) && !required.types.includes(actual.type)) {
+ mismatchedNodes.push({
+ id: identity,
+ field: 'type',
+ expected: required.types,
+ actual: actual.type ?? null,
+ });
+ }
}
const missingRelationships = (requirements.relationships || []).filter((relationship) => {
const resolved = {
diff --git a/benchmarks/ordinary-model-floor/cases/agent-run.lifecycle.case.json b/benchmarks/ordinary-model-floor/cases/agent-run.lifecycle.case.json
index 7aae3ccf..89e8f295 100644
--- a/benchmarks/ordinary-model-floor/cases/agent-run.lifecycle.case.json
+++ b/benchmarks/ordinary-model-floor/cases/agent-run.lifecycle.case.json
@@ -8,8 +8,8 @@
{ "key": "queued", "labels": ["Queued", "Accepted"], "type": "start" },
{ "key": "executing", "labels": ["Executing", "Running"], "type": "active" },
{ "key": "reviewing", "labels": ["Reviewing", "Quality Review"], "type": "decision" },
- { "key": "approval", "labels": ["Needs Approval", "Waiting for Approval"], "type": "waiting" },
- { "key": "blocked", "labels": ["Blocked", "Waiting for Input"], "type": "waiting" },
+ { "key": "approval", "labels": ["Needs Approval", "Waiting for Approval", "Awaiting Approval"], "type": "waiting" },
+ { "key": "blocked", "labels": ["Blocked", "Waiting for Input", "Awaiting Input", "Blocked for Input"], "type": "waiting" },
{ "key": "cancelled", "labels": ["Cancelled", "Canceled"], "type": "failure" },
{ "key": "expired", "labels": ["Expired", "Timed Out"], "type": "failure" }
],
diff --git a/benchmarks/ordinary-model-floor/cases/agent-tool-call.workflow.case.json b/benchmarks/ordinary-model-floor/cases/agent-tool-call.workflow.case.json
index d80269d7..a9ebc464 100644
--- a/benchmarks/ordinary-model-floor/cases/agent-tool-call.workflow.case.json
+++ b/benchmarks/ordinary-model-floor/cases/agent-tool-call.workflow.case.json
@@ -6,9 +6,9 @@
"requirements": {
"nodes": [
{ "key": "planner", "labels": ["Agent Planner", "Planner"], "type": "backend" },
- { "key": "router", "labels": ["Tool Router", "Router"], "type": "backend" },
+ { "key": "router", "labels": ["Tool Router", "Router"], "types": ["backend", "security"] },
{ "key": "approval", "labels": ["Approval Gate", "Human Approval"], "type": "security" },
- { "key": "tool", "labels": ["Tool Call", "Tool Execution"], "type": "messagebus" },
+ { "key": "tool", "labels": ["Tool Call", "Tool Execution", "Tool Executor", "Tool Runner"], "type": "messagebus" },
{ "key": "blocked", "labels": ["Blocked", "Denied"], "type": "security" },
{ "key": "external", "labels": ["External API", "Provider API"], "type": "cloud" }
],
diff --git a/benchmarks/ordinary-model-floor/cases/cache-miss.sequence.case.json b/benchmarks/ordinary-model-floor/cases/cache-miss.sequence.case.json
index 5412bab8..8c8b738c 100644
--- a/benchmarks/ordinary-model-floor/cases/cache-miss.sequence.case.json
+++ b/benchmarks/ordinary-model-floor/cases/cache-miss.sequence.case.json
@@ -5,18 +5,18 @@
"quality_profile": "showcase",
"requirements": {
"nodes": [
- { "key": "web", "labels": ["Web App", "Browser"], "type": "frontend" },
+ { "key": "web", "labels": ["Web App", "Browser"], "types": ["frontend", "external"] },
{ "key": "api", "labels": ["API", "API Server"], "type": "backend" },
- { "key": "auth", "labels": ["Auth", "Authentication"], "type": "security" },
+ { "key": "auth", "labels": ["Auth", "Authentication", "Identity", "JWT Guard", "Token Guard"], "type": "security" },
{ "key": "redis", "labels": ["Redis", "Cache"], "type": "database" },
{ "key": "db", "labels": ["Postgres", "PostgreSQL", "Database"], "type": "database" }
],
"relationships": [
- { "from": "api", "to": "auth", "labels": ["verify JWT", "validate token"] },
- { "from": "api", "to": "redis", "labels": ["read cache", "cache get"] },
+ { "from": "api", "to": "auth", "labels": ["verify JWT", "validate JWT", "validate token", "verify token"] },
+ { "from": "api", "to": "redis", "labels": ["read cache", "cache get", "GET"] },
{ "from": "redis", "to": "api", "labels": ["miss", "cache miss"] },
- { "from": "api", "to": "db", "labels": ["query profile + metrics", "query database"] },
- { "from": "api", "to": "redis", "labels": ["set cache", "cache set"] }
+ { "from": "api", "to": "db", "labels": ["query profile + metrics", "query database", "SELECT"] },
+ { "from": "api", "to": "redis", "labels": ["set cache", "cache set", "write cache", "SET"] }
]
}
}
diff --git a/benchmarks/ordinary-model-floor/cases/product-analytics.dataflow.case.json b/benchmarks/ordinary-model-floor/cases/product-analytics.dataflow.case.json
index e528a868..084ce8d2 100644
--- a/benchmarks/ordinary-model-floor/cases/product-analytics.dataflow.case.json
+++ b/benchmarks/ordinary-model-floor/cases/product-analytics.dataflow.case.json
@@ -5,19 +5,19 @@
"quality_profile": "showcase",
"requirements": {
"nodes": [
- { "key": "edge", "labels": ["Edge API", "Ingestion API"], "type": "cloud" },
- { "key": "consent", "labels": ["Consent Gate", "Consent Check"], "type": "security" },
+ { "key": "edge", "labels": ["Edge API", "Ingestion API", "Ingestion Gateway", "Ingest Gateway"], "type": "cloud" },
+ { "key": "consent", "labels": ["Consent Gate", "Consent Check", "Consent Filter"], "type": "security" },
{ "key": "stream", "labels": ["Event Stream", "Event Bus"], "type": "messagebus" },
{ "key": "pii", "labels": ["PII Vault", "Identity Vault"], "type": "security" },
{ "key": "warehouse", "labels": ["Warehouse", "Data Warehouse"], "type": "database" },
{ "key": "dashboard", "labels": ["Dashboards", "Analytics Dashboard"], "type": "backend" }
],
"relationships": [
- { "from": "edge", "to": "consent", "labels": ["identity + consent", "consent check"] },
+ { "from": "edge", "to": "consent", "labels": ["identity + consent", "consent check", "identity claims"] },
{ "from": "edge", "to": "stream", "labels": ["accepted events", "validated events"] },
- { "from": "consent", "to": "pii", "labels": ["identity map", "approved identity"] },
+ { "from": "consent", "to": "pii", "labels": ["identity map", "approved identity", "consented identity"] },
{ "from": "stream", "to": "warehouse", "labels": ["normalized facts", "normalized events"] },
- { "from": "warehouse", "to": "dashboard", "labels": ["metrics SQL", "analytics query"] }
+ { "from": "warehouse", "to": "dashboard", "labels": ["metrics SQL", "analytics query", "KPI reads"] }
]
}
}