Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

51.4 KB
Raw Blame
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { fileURLToPath } from 'node:url';

const here = path.dirname(fileURLToPath(import.meta.url));
const skillRoot = path.resolve(here, '..', 'archify');
const repoRoot = path.resolve(skillRoot, '..');
const benchmark = path.join(repoRoot, 'benchmarks/ordinary-model-floor/benchmark.mjs');
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'archify-ordinary-model-floor-'));

function writeJson(name, value) {
  const file = path.join(tmp, name);
  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
  return file;
}

function run(args) {
  return spawnSync(process.execPath, [benchmark, ...args], {
    cwd: repoRoot,
    encoding: 'utf8',
  });
}

function renameCandidateIds(candidate, mapping) {
  for (const collection of ['components', 'nodes', 'participants', 'states']) {
    for (const node of candidate[collection] || []) {
      node.id = mapping.get(node.id) || node.id;
    }
  }
  for (const collection of ['connections', 'edges', 'messages', 'flows', 'transitions']) {
    for (const relationship of candidate[collection] || []) {
      relationship.from = mapping.get(relationship.from) || relationship.from;
      relationship.to = mapping.get(relationship.to) || relationship.to;
    }
  }
  for (const activation of candidate.activations || []) {
    activation.participant = mapping.get(activation.participant) || activation.participant;
  }
  if (Array.isArray(candidate.mainPath)) {
    candidate.mainPath = candidate.mainPath.map((id) => mapping.get(id) || id);
  }
  for (const boundary of candidate.boundaries || []) {
    boundary.wraps = boundary.wraps.map((id) => mapping.get(id) || id);
  }
  return candidate;
}

test('benchmark verifies one first-pass architecture candidate through semantic, renderer, and visual-review gates', () => {
  const caseFile = writeJson('web-runtime.case.json', {
    schema_version: 1,
    id: 'web-runtime-architecture',
    diagram_type: 'architecture',
    quality_profile: 'showcase',
    requirements: {
      node_ids: ['users', 'cdn', 'lb', 'api', 'db'],
      relationships: [
        { from: 'users', to: 'cdn' },
        { from: 'cdn', to: 'lb' },
        { from: 'lb', to: 'api' },
        { from: 'api', to: 'db' },
      ],
    },
  });
  const runFile = writeJson('web-runtime.run.json', {
    schema_version: 1,
    case_id: 'web-runtime-architecture',
    agent: 'fixture-agent',
    model: 'fixture-model',
    attempt: 1,
    visual_review: {
      status: 'passed',
      reviewer: 'fixture-reviewer',
      defects: [],
    },
  });
  const candidate = path.join(skillRoot, 'examples/web-app.architecture.json');

  const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);

  assert.equal(result.status, 0, result.stderr || result.stdout);
  assert.equal(result.stderr, '');
  const receipt = JSON.parse(result.stdout);
  assert.equal(receipt.schemaVersion, 1);
  assert.equal(receipt.benchmark, 'ordinary-model-floor');
  assert.equal(receipt.caseId, 'web-runtime-architecture');
  assert.deepEqual(receipt.run, {
    agent: 'fixture-agent',
    model: 'fixture-model',
    attempt: 1,
  });
  assert.equal(receipt.gates.semantic.ok, true);
  assert.deepEqual(receipt.gates.semantic.missingNodeIds, []);
  assert.deepEqual(receipt.gates.semantic.missingRelationships, []);
  assert.equal(receipt.gates.validation.ok, true);
  assert.equal(receipt.gates.validation.checksPassed, 9);
  assert.deepEqual(receipt.gates.validation.composition, { errors: 0, warnings: 0 });
  assert.deepEqual(receipt.gates.visualReview, {
    status: 'passed',
    reviewer: 'fixture-reviewer',
    defects: [],
  });
  assert.equal(receipt.firstPassUsable, true);
});

test('benchmark rejects invalid run identities and preserves reportable valid identities', () => {
  const caseFile = path.join(repoRoot, 'benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json');
  const candidate = path.join(skillRoot, 'examples/web-app.architecture.json');
  const validRun = {
    schema_version: 1, case_id: 'web-runtime-architecture', attempt: 1,
    agent: ' fixture-agent ', model: ' fixture-model ',
    // Synthetic review metadata isolates identity validation; no visual review occurred.
    visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects: [] },
  };
  for (const field of ['agent', 'model']) {
    for (const value of [undefined, null, '', ' \t\n ', 42, false, {}, []]) {
      const runFile = writeJson('invalid-identity.run.json', { ...validRun, [field]: value });
      const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);
      assert.equal(result.status, 2, `${field}=${JSON.stringify(value)}: ${result.stderr || result.stdout}`);
      assert.deepEqual(JSON.parse(result.stdout).error, {
        code: 'INVALID_RUN', message: 'run agent and model must be non-empty strings',
      });
    }
  }
  const runFile = writeJson('valid-identity.run.json', validRun);
  const verified = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);
  assert.equal(verified.status, 0, verified.stderr || verified.stdout);
  const receipt = JSON.parse(verified.stdout);
  assert.equal(receipt.firstPassUsable, true);
  assert.deepEqual(receipt.run, { agent: validRun.agent, model: validRun.model, attempt: 1 });
  const resultsFile = path.join(tmp, 'valid-identity.results.jsonl');
  fs.writeFileSync(resultsFile, `${JSON.stringify(receipt)}\n`);
  const reported = run(['report', '--results', resultsFile]);
  assert.equal(reported.status, 0, reported.stderr || reported.stdout);
  const report = JSON.parse(reported.stdout);
  assert.equal(report.overall.firstPassUsableRate, 1);
  assert.equal(report.byConfiguration[0].agent, validRun.agent);
  assert.equal(report.byConfiguration[0].model, validRun.model);
});

test('benchmark rejects a renderer-valid candidate that changes required technical roles or relationship labels', () => {
  const source = JSON.parse(fs.readFileSync(path.join(skillRoot, 'examples/web-app.architecture.json'), 'utf8'));
  source.components.find((component) => component.id === 'cache').type = 'frontend';
  source.connections.find((connection) => connection.from === 'api' && connection.to === 'db').label = 'HTTP';
  const candidate = writeJson('semantic-drift.architecture.json', source);
  const caseFile = writeJson('semantic-drift.case.json', {
    schema_version: 1,
    id: 'semantic-drift-architecture',
    diagram_type: 'architecture',
    quality_profile: 'showcase',
    requirements: {
      nodes: [
        { id: 'cache', type: 'database' },
        { id: 'db', type: 'database' },
      ],
      relationships: [
        { from: 'api', to: 'cache', label: 'read-through' },
        { from: 'api', to: 'db', label: 'SQL' },
      ],
    },
  });
  const runFile = writeJson('semantic-drift.run.json', {
    schema_version: 1,
    case_id: 'semantic-drift-architecture',
    agent: 'fixture-agent',
    model: 'fixture-model',
    attempt: 1,
    visual_review: {
      status: 'passed',
      reviewer: 'fixture-reviewer',
      defects: [],
    },
  });

  const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);

  assert.equal(result.status, 1, result.stderr || result.stdout);
  assert.equal(result.stderr, '');
  const receipt = JSON.parse(result.stdout);
  assert.equal(receipt.gates.validation.ok, true, 'the deterministic renderer should still accept this controlled drift');
  assert.equal(receipt.gates.semantic.ok, false);
  assert.deepEqual(receipt.gates.semantic.mismatchedNodes, [
    { id: 'cache', field: 'type', expected: 'database', actual: 'frontend' },
  ]);
  assert.deepEqual(receipt.gates.semantic.missingRelationships, [
    { from: 'api', to: 'db', label: 'SQL' },
  ]);
  assert.equal(receipt.firstPassUsable, false);
});

test('benchmark checks accepted node labels even when an authored identity matches', () => {
  for (const identityField of ['key', 'id']) {
    const caseData = JSON.parse(fs.readFileSync(path.join(repoRoot, 'benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json'), 'utf8'));
    const requirement = caseData.requirements.nodes.find((node) => node.key === 'cache');
    delete requirement.key;
    requirement[identityField] = 'cache';
    const caseFile = writeJson(`node-label-${identityField}.case.json`, caseData);
    for (const label of ['Redis Cache', 'MySQL']) {
      const source = JSON.parse(fs.readFileSync(path.join(skillRoot, 'examples/web-app.architecture.json'), 'utf8'));
      source.components.find((node) => node.id === 'cache').label = label;
      const candidate = writeJson(`node-label-${identityField}.architecture.json`, source);
      const runFile = writeJson(`node-label-${identityField}.run.json`, {
        schema_version: 1, case_id: caseData.id,
        agent: 'fixture-agent', model: 'fixture-model', attempt: 1,
        visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects: [] },
      });
      const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);
      const accepted = label === 'Redis Cache';
      assert.equal(result.status, accepted ? 0 : 1, result.stderr || result.stdout);
      const receipt = JSON.parse(result.stdout);
      assert.equal(receipt.gates.validation.ok, true);
      assert.equal(receipt.gates.semantic.ok, accepted);
      assert.deepEqual(receipt.gates.semantic.missingNodeIds, []);
      assert.deepEqual(receipt.gates.semantic.missingRelationships, []);
      assert.deepEqual(receipt.gates.semantic.mismatchedNodes, accepted ? [] : [
        { id: 'cache', field: 'label', expected: requirement.labels, actual: 'MySQL' },
      ]);
      assert.equal(receipt.firstPassUsable, accepted);
    }
  }
});

test('benchmark rejects one candidate node bound to distinct required identities', () => {
  const caseSource = JSON.parse(fs.readFileSync(path.join(repoRoot, 'benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json'), 'utf8'));
  const runFile = writeJson('reused-binding.run.json', {
    schema_version: 1, case_id: caseSource.id,
    agent: 'fixture-agent', model: 'fixture-model', attempt: 1,
    // Synthetic review metadata isolates the semantic gate; no visual review occurred.
    visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects: [] },
  });
  for (const storageId of ['storage', 'cache']) {
    const candidate = writeJson('reused-binding.architecture.json', {
      schema_version: 1, diagram_type: 'architecture',
      meta: { title: 'Distinct storage requirements', output: 'binding.html', quality_profile: 'showcase' },
      components: [
        { id: 'users', type: 'external', label: 'Users', pos: [40, 180], size: [220, 64] },
        { id: 'cdn', type: 'cloud', label: 'CloudFront', pos: [400, 180], size: [220, 64] },
        { id: 'api', type: 'backend', label: 'API Server', pos: [760, 180], size: [220, 64] },
        { id: storageId, type: 'database', label: 'Redis PostgreSQL', pos: [1120, 180], size: [220, 64] },
      ],
      connections: [
        { from: 'users', to: 'cdn', label: 'HTTPS' },
        { from: 'cdn', to: 'api' },
        { from: 'api', to: storageId, label: 'read-through SQL' },
      ],
    });
    for (const identityField of ['key', 'id']) {
      for (const reverse of [false, true]) {
        const caseData = structuredClone(caseSource);
        const cache = caseData.requirements.nodes.find((node) => node.key === 'cache');
        if (identityField === 'id') {
          // Exercise an explicit ID sharing its node with an alias-bound requirement.
          delete cache.key;
          cache.id = storageId;
          caseData.requirements.relationships.find((edge) => edge.to === 'cache').to = storageId;
        }
        if (reverse) caseData.requirements.nodes.reverse();
        const caseFile = writeJson('reused-binding.case.json', caseData);
        const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);
        const receipt = JSON.parse(result.stdout);
        assert.equal(receipt.gates.validation.ok, true);
        assert.equal(receipt.gates.validation.checksPassed, 9);
        assert.deepEqual(receipt.gates.semantic.missingNodeIds, []);
        assert.deepEqual(receipt.gates.semantic.missingRelationships, []);
        assert.equal(receipt.gates.semantic.ok, false);
        assert.equal(result.status, 1, result.stderr || result.stdout);
        const identities = [identityField === 'id' ? storageId : 'cache', 'db'].sort();
        assert.deepEqual(receipt.gates.semantic.mismatchedNodes.slice().sort((a, b) => a.id.localeCompare(b.id)),
          identities.map((id) => ({ id, field: 'binding', expected: 'distinct candidate node', actual: storageId })));
        assert.equal(receipt.firstPassUsable, false);
        const resultsFile = path.join(tmp, 'reused-binding.results.jsonl');
        fs.writeFileSync(resultsFile, `${JSON.stringify(receipt)}\n`);
        const reportResult = run(['report', '--results', resultsFile]);
        assert.equal(reportResult.status, 0, reportResult.stderr || reportResult.stdout);
        const report = JSON.parse(reportResult.stdout);
        assert.deepEqual(report.overall.failureClusters, { semantic: 1, validation: 0, visualReview: 0, operational: 0 });
      }
    }
  }
});

test('benchmark preserves repeated constraints on the same required identity', () => {
  const caseFile = writeJson('same-identity.case.json', {
    schema_version: 1, id: 'same-identity', diagram_type: 'architecture', quality_profile: 'showcase',
    requirements: { nodes: [{ id: 'cache', type: 'database' }, { key: 'cache', labels: ['Redis'] }] },
  });
  const runFile = writeJson('same-identity.run.json', {
    schema_version: 1, case_id: 'same-identity', agent: 'fixture-agent', model: 'fixture-model', attempt: 1,
    visual_review: { status: 'skipped', defects: [] },
  });
  const result = run(['verify', '--case', caseFile, '--candidate', path.join(skillRoot, 'examples/web-app.architecture.json'), '--run', runFile]);
  const receipt = JSON.parse(result.stdout);
  assert.equal(receipt.gates.semantic.ok, true);
  assert.deepEqual(receipt.gates.semantic.mismatchedNodes, []);
  assert.equal(receipt.gates.validation.ok, true);
  assert.equal(receipt.firstPassUsable, false, 'an omitted visual review is still not a pass');
  assert.equal(result.status, 1);
});

test('benchmark never accepts a visual pass without an identified reviewer', () => {
  const caseFile = writeJson('unreviewed.case.json', {
    schema_version: 1,
    id: 'unreviewed-architecture',
    diagram_type: 'architecture',
    quality_profile: 'showcase',
    requirements: {
      node_ids: ['users', 'api', 'db'],
      relationships: [{ from: 'api', to: 'db' }],
    },
  });
  const runFile = writeJson('unreviewed.run.json', {
    schema_version: 1,
    case_id: 'unreviewed-architecture',
    agent: 'fixture-agent',
    model: 'fixture-model',
    attempt: 1,
    visual_review: {
      status: 'passed',
      reviewer: '',
      defects: [],
    },
  });
  const candidate = path.join(skillRoot, 'examples/web-app.architecture.json');

  const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);

  assert.equal(result.status, 1, result.stderr || result.stdout);
  assert.equal(result.stderr, '');
  const receipt = JSON.parse(result.stdout);
  assert.equal(receipt.gates.semantic.ok, true);
  assert.equal(receipt.gates.validation.ok, true);
  assert.deepEqual(receipt.gates.visualReview, {
    status: 'invalid',
    reviewer: null,
    defects: [],
    reason: 'passed visual review requires a non-empty reviewer identity',
  });
  assert.equal(receipt.firstPassUsable, false);
});

test('benchmark never upgrades missing or malformed visual defects to a clean pass', () => {
  const caseFile = path.join(repoRoot, 'benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json');
  const candidate = path.join(skillRoot, 'examples/web-app.architecture.json');
  for (const [index, defects] of [undefined, null, 'clipping', { clipping: true }, false, 0, [], ['clipping']].entries()) {
    const runFile = writeJson(`visual-defects-${index}.run.json`, {
      schema_version: 1,
      case_id: 'web-runtime-architecture',
      agent: 'fixture-agent',
      model: 'fixture-model',
      attempt: 1,
      visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects },
    });
    const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);
    const clean = Array.isArray(defects) && defects.length === 0;
    assert.equal(result.status, clean ? 0 : 1, result.stderr || result.stdout);
    const receipt = JSON.parse(result.stdout);
    assert.equal(receipt.gates.semantic.ok, true);
    assert.equal(receipt.gates.validation.ok, true);
    assert.equal(receipt.firstPassUsable, clean);
    assert.equal(receipt.gates.visualReview.status, Array.isArray(defects) ? 'passed' : 'invalid');
    if (Array.isArray(defects)) {
      assert.deepEqual(receipt.gates.visualReview.defects, defects);
    } else {
      assert.equal(receipt.gates.visualReview.reason, 'passed visual review requires an explicit defects array');
    }
  }
});

test('benchmark applies the same semantic and delivery seam to workflow, sequence, data-flow, and lifecycle candidates', () => {
  const cases = [
    {
      type: 'workflow',
      example: 'agent-tool-call.workflow.json',
      nodes: [{ id: 'approval', type: 'security' }, { id: 'tool', type: 'messagebus' }],
      relationships: [{ from: 'router', to: 'approval', label: 'needs approval?' }],
    },
    {
      type: 'sequence',
      example: 'cache-miss-request.sequence.json',
      nodes: [{ id: 'redis', type: 'database' }, { id: 'db', type: 'database' }],
      relationships: [{ from: 'redis', to: 'api', label: 'miss' }],
    },
    {
      type: 'dataflow',
      example: 'product-analytics.dataflow.json',
      nodes: [{ id: 'consent', type: 'security' }, { id: 'pii', type: 'security' }],
      relationships: [{ from: 'consent', to: 'pii', label: 'identity map' }],
    },
    {
      type: 'lifecycle',
      example: 'agent-run.lifecycle.json',
      nodes: [{ id: 'approval', type: 'waiting' }, { id: 'cancelled', type: 'failure' }],
      relationships: [{ from: 'approval', to: 'cancelled', variant: 'security' }],
    },
  ];

  for (const item of cases) {
    const caseId = `${item.type}-representative`;
    const caseFile = writeJson(`${caseId}.case.json`, {
      schema_version: 1,
      id: caseId,
      diagram_type: item.type,
      quality_profile: 'showcase',
      requirements: {
        nodes: item.nodes,
        relationships: item.relationships,
      },
    });
    const runFile = writeJson(`${caseId}.run.json`, {
      schema_version: 1,
      case_id: caseId,
      agent: 'fixture-agent',
      model: 'fixture-model',
      attempt: 1,
      visual_review: {
        status: 'passed',
        reviewer: 'fixture-reviewer',
        defects: [],
      },
    });
    const candidate = path.join(skillRoot, 'examples', item.example);

    const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);

    assert.equal(result.status, 0, `${item.type}: ${result.stderr || result.stdout}`);
    const receipt = JSON.parse(result.stdout);
    assert.equal(receipt.gates.semantic.ok, true, item.type);
    assert.equal(receipt.gates.validation.ok, true, item.type);
    assert.equal(receipt.firstPassUsable, true, item.type);
  }
});

test('benchmark report separates first-pass usable rate from semantic, validation, and visual-review failures by configuration', () => {
  const resultsFile = path.join(tmp, 'benchmark-results.jsonl');
  const rows = [
    {
      schemaVersion: 1,
      benchmark: 'ordinary-model-floor',
      caseId: 'architecture-runtime',
      run: { agent: 'codex', model: 'strong', attempt: 1 },
      gates: {
        semantic: { ok: true },
        validation: { ok: true },
        visualReview: { status: 'passed', reviewer: 'reviewer', defects: [] },
      },
      firstPassUsable: true,
    },
    {
      schemaVersion: 1,
      benchmark: 'ordinary-model-floor',
      caseId: 'sequence-cache-miss',
      run: { agent: 'codex', model: 'strong', attempt: 1 },
      gates: {
        semantic: { ok: true },
        validation: { ok: true },
        visualReview: { status: 'passed', reviewer: 'reviewer', defects: [] },
      },
      firstPassUsable: true,
    },
    {
      schemaVersion: 1,
      benchmark: 'ordinary-model-floor',
      caseId: 'architecture-runtime',
      run: { agent: 'opencode', model: 'ordinary', attempt: 1 },
      gates: {
        semantic: { ok: false },
        validation: { ok: true },
        visualReview: { status: 'passed', reviewer: 'reviewer', defects: [] },
      },
      firstPassUsable: false,
    },
    {
      schemaVersion: 1,
      benchmark: 'ordinary-model-floor',
      caseId: 'sequence-cache-miss',
      run: { agent: 'opencode', model: 'ordinary', attempt: 1 },
      gates: {
        semantic: { ok: true },
        validation: { ok: false },
        visualReview: { status: 'skipped', reviewer: null, defects: [] },
      },
      firstPassUsable: false,
    },
  ];
  fs.writeFileSync(resultsFile, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`);

  const result = run(['report', '--results', resultsFile]);

  assert.equal(result.status, 0, result.stderr || result.stdout);
  assert.equal(result.stderr, '');
  const report = JSON.parse(result.stdout);
  assert.equal(report.schemaVersion, 1);
  assert.equal(report.benchmark, 'ordinary-model-floor');
  assert.deepEqual(report.overall, {
    runs: 4,
    firstPassUsable: 2,
    firstPassUsableRate: 0.5,
    failureClusters: {
      semantic: 1,
      validation: 1,
      visualReview: 1,
      operational: 0,
    },
  });
  assert.deepEqual(report.byConfiguration, [
    {
      agent: 'codex',
      model: 'strong',
      runs: 2,
      firstPassUsable: 2,
      firstPassUsableRate: 1,
      failureClusters: { semantic: 0, validation: 0, visualReview: 0, operational: 0 },
    },
    {
      agent: 'opencode',
      model: 'ordinary',
      runs: 2,
      firstPassUsable: 0,
      firstPassUsableRate: 0,
      failureClusters: { semantic: 1, validation: 1, visualReview: 1, operational: 0 },
    },
  ]);
});

test('benchmark records a timeout without a candidate as a complete first-pass failure', () => {
  const caseFile = writeJson('timeout.case.json', {
    schema_version: 1,
    id: 'timeout-architecture',
    diagram_type: 'architecture',
    requirements: { nodes: [] },
  });
  const runFile = writeJson('timeout.run.json', {
    schema_version: 1,
    case_id: 'timeout-architecture',
    agent: 'pi',
    model: 'glm-5.2-low',
    attempt: 1,
  });

  const recorded = run([
    'record-failure',
    '--case', caseFile,
    '--run', runFile,
    '--failure', 'timeout',
  ]);

  assert.equal(recorded.status, 1, recorded.stderr || recorded.stdout);
  assert.equal(recorded.stderr, '');
  const failureReceipt = JSON.parse(recorded.stdout);
  assert.deepEqual(failureReceipt.operational, { status: 'failed', reason: 'timeout' });
  assert.equal(failureReceipt.gates.semantic.status, 'not_run');
  assert.equal(failureReceipt.gates.validation.status, 'not_run');
  assert.equal(failureReceipt.gates.visualReview.status, 'skipped');
  assert.equal(failureReceipt.firstPassUsable, false);

  const resultsFile = path.join(tmp, 'timeout-results.jsonl');
  fs.writeFileSync(resultsFile, `${JSON.stringify(failureReceipt)}\n`);
  const manifestFile = writeJson('timeout.manifest.json', {
    id: 'timeout-suite',
    cases: [{ case: caseFile }],
  });
  const reported = run(['report', '--results', resultsFile, '--manifest', manifestFile]);

  assert.equal(reported.status, 0, reported.stderr || reported.stdout);
  const report = JSON.parse(reported.stdout);
  assert.equal(report.evidenceEligible, true);
  assert.deepEqual(report.overall.failureClusters, {
    semantic: 0,
    validation: 0,
    visualReview: 0,
    operational: 1,
  });
  assert.equal(report.overall.firstPassUsableRate, 0);
});

test('benchmark semantic requirements bind by accepted technical labels instead of forcing model-authored internal IDs', () => {
  const source = JSON.parse(fs.readFileSync(path.join(skillRoot, 'examples/web-app.architecture.json'), 'utf8'));
  source.components.find((component) => component.id === 'users').label = 'Browser Users';
  const rename = new Map([
    ['users', 'browser-users-v1'],
    ['api', 'service-api-v1'],
    ['cache', 'redis-cache-v1'],
  ]);
  for (const component of source.components) component.id = rename.get(component.id) || component.id;
  for (const connection of source.connections) {
    connection.from = rename.get(connection.from) || connection.from;
    connection.to = rename.get(connection.to) || connection.to;
  }
  source.connections.find(
    (connection) => connection.from === 'service-api-v1' && connection.to === 'redis-cache-v1',
  ).label = 'cache read-through GET / SET';
  for (const boundary of source.boundaries || []) {
    boundary.wraps = boundary.wraps.map((id) => rename.get(id) || id);
  }
  const candidate = writeJson('semantic-aliases.architecture.json', source);
  const caseFile = writeJson('semantic-aliases.case.json', {
    schema_version: 1,
    id: 'semantic-aliases-architecture',
    diagram_type: 'architecture',
    quality_profile: 'showcase',
    requirements: {
      nodes: [
        { key: 'users', labels: ['Users'], type: 'external' },
        { key: 'api', labels: ['API', 'API Server'], type: 'backend' },
        { key: 'cache', labels: ['Redis', 'Redis Cache'], type: 'database' },
      ],
      relationships: [
        { from: 'api', to: 'cache', labels: ['read-through', 'cache read'] },
      ],
    },
  });
  const runFile = writeJson('semantic-aliases.run.json', {
    schema_version: 1,
    case_id: 'semantic-aliases-architecture',
    agent: 'fixture-agent',
    model: 'fixture-model',
    attempt: 1,
    visual_review: {
      status: 'passed',
      reviewer: 'fixture-reviewer',
      defects: [],
    },
  });

  const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);

  assert.equal(result.status, 0, result.stderr || result.stdout);
  const receipt = JSON.parse(result.stdout);
  assert.equal(receipt.gates.semantic.ok, true);
  assert.deepEqual(receipt.gates.semantic.bindings, {
    users: 'browser-users-v1',
    api: 'service-api-v1',
    cache: 'redis-cache-v1',
  });
  assert.equal(receipt.firstPassUsable, true);
});

test('checked-in cases accept equivalent ordinary-model vocabulary without weakening required topology', () => {
  const suiteRoot = path.join(repoRoot, 'benchmarks/ordinary-model-floor');
  const fixtures = [
    {
      type: 'architecture',
      example: 'web-app.architecture.json',
      caseFile: 'web-runtime.architecture.case.json',
      mapping: new Map([
        ['users', 'browser-clients'], ['cdn', 'cdn-edge'], ['api', 'app-api'],
        ['cache', 'redis-store'], ['db', 'postgres-primary'],
      ]),
      mutate(candidate) {
        candidate.components.find((node) => node.id === 'browser-clients').label = 'Browser Clients';
        candidate.components.find((node) => node.id === 'cdn-edge').label = 'CDN Edge';
        candidate.components.find((node) => node.id === 'app-api').label = 'App API';
      },
    },
    {
      type: 'workflow',
      example: 'agent-tool-call.workflow.json',
      caseFile: 'agent-tool-call.workflow.case.json',
      mapping: new Map([
        ['planner', 'task-planner'], ['router', 'risk-router'], ['approval', 'consent-check'],
        ['tool', 'tool-runner'], ['blocked', 'request-blocked'], ['external', 'service-provider'],
      ]),
      mutate(candidate) {
        candidate.nodes.find((node) => node.id === 'task-planner').label = 'Intent Planner';
        Object.assign(candidate.nodes.find((node) => node.id === 'risk-router'), { label: 'Route Decision', type: 'security' });
        candidate.nodes.find((node) => node.id === 'consent-check').label = 'Approval';
        candidate.nodes.find((node) => node.id === 'tool-runner').label = 'Tool Dispatch';
        candidate.nodes.find((node) => node.id === 'request-blocked').label = 'Held';
        candidate.nodes.find((node) => node.id === 'service-provider').label = 'Remote API';
        candidate.edges.find((edge) => edge.from === 'risk-router' && edge.to === 'consent-check').label = 'requires approval?';
      },
    },
    {
      type: 'sequence',
      example: 'cache-miss-request.sequence.json',
      caseFile: 'cache-miss.sequence.case.json',
      mapping: new Map([
        ['web', 'browser-tab'], ['api', 'dashboard-api'], ['auth', 'token-guard'],
        ['redis', 'response-cache'], ['db', 'account-store'],
      ]),
      mutate(candidate) {
        Object.assign(candidate.participants.find((node) => node.id === 'browser-tab'), { label: 'Browser', type: 'external' });
        candidate.participants.find((node) => node.id === 'dashboard-api').label = 'Dashboard API';
        candidate.participants.find((node) => node.id === 'token-guard').label = 'JWT Guard';
        candidate.messages.find((message) => message.id === 'verify-jwt').label = 'verify Bearer JWT';
        candidate.messages.find((message) => message.id === 'cache-read').label = 'GET dashboard key';
        candidate.messages.find((message) => message.id === 'profile-query').label = 'SELECT profile + metrics';
        candidate.messages.find((message) => message.id === 'cache-write').label = 'SETEX profile 300';
      },
    },
    {
      type: 'dataflow',
      example: 'product-analytics.dataflow.json',
      caseFile: 'product-analytics.dataflow.case.json',
      mapping: new Map([
        ['edge', 'edge-collector'], ['consent', 'consent-policy'], ['stream', 'event-stream'],
        ['pii', 'identity-vault'], ['warehouse', 'facts-warehouse'], ['dashboard', 'metric-dashboards'],
      ]),
      mutate(candidate) {
        candidate.nodes.find((node) => node.id === 'edge-collector').label = 'Edge Ingestion';
        candidate.nodes.find((node) => node.id === 'consent-policy').label = 'Consent Policy';
        candidate.nodes.find((node) => node.id === 'identity-vault').label = 'Identity Vault';
        candidate.nodes.find((node) => node.id === 'facts-warehouse').label = 'Analytics Warehouse';
        candidate.nodes.find((node) => node.id === 'metric-dashboards').label = 'Analytics UI';
        candidate.flows.find((flow) => flow.id === 'consent-enrichment').label = 'identity context';
        candidate.flows.find((flow) => flow.id === 'accepted-events').label = 'telemetry';
        candidate.flows.find((flow) => flow.id === 'identity-map').label = 'encrypted identity';
        candidate.flows.find((flow) => flow.id === 'metrics-query').label = 'metrics query';
      },
    },
    {
      type: 'lifecycle',
      example: 'agent-run.lifecycle.json',
      caseFile: 'agent-run.lifecycle.case.json',
      mapping: new Map([
        ['queued', 'run-queued'], ['executing', 'run-executing'], ['reviewing', 'run-reviewing'],
        ['approval', 'approval-wait'], ['blocked', 'input-wait'], ['cancelled', 'user-cancelled'],
        ['expired', 'run-expired'],
      ]),
      mutate(candidate) {
        candidate.states.find((node) => node.id === 'approval-wait').label = 'Approval Pending';
        candidate.states.find((node) => node.id === 'input-wait').label = 'Pending Input';
        candidate.states.push({
          id: 'fatal-failure',
          type: 'failure',
          label: 'Failed',
          sublabel: 'budget exhausted',
          tag: 'terminal',
        });
      },
    },
  ];

  for (const fixture of fixtures) {
    const source = JSON.parse(fs.readFileSync(path.join(skillRoot, 'examples', fixture.example), 'utf8'));
    const candidate = renameCandidateIds(source, fixture.mapping);
    fixture.mutate(candidate);
    const candidateFile = writeJson(`calibrated-${fixture.type}.json`, candidate);
    const benchmarkCase = JSON.parse(fs.readFileSync(
      path.join(suiteRoot, 'cases', fixture.caseFile),
      'utf8',
    ));
    const runFile = writeJson(`calibrated-${fixture.type}.run.json`, {
      schema_version: 1,
      case_id: benchmarkCase.id,
      agent: 'ordinary-agent',
      model: 'ordinary-model',
      attempt: 1,
      visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects: [] },
    });

    const result = run([
      'verify', '--case', path.join(suiteRoot, 'cases', fixture.caseFile),
      '--candidate', candidateFile, '--run', runFile,
    ]);

    assert.equal(result.status, 0, `${fixture.type}: ${result.stderr || result.stdout}`);
    assert.equal(JSON.parse(result.stdout).gates.semantic.ok, true, fixture.type);

    if (fixture.type === 'workflow') {
      const wrongRoleCandidate = structuredClone(candidate);
      wrongRoleCandidate.nodes.find((node) => node.id === 'risk-router').type = 'external';
      const wrongRoleFile = writeJson('calibrated-workflow-wrong-role.json', wrongRoleCandidate);
      const wrongRole = run([
        'verify', '--case', path.join(suiteRoot, 'cases', fixture.caseFile),
        '--candidate', wrongRoleFile, '--run', runFile,
      ]);
      assert.equal(wrongRole.status, 1, wrongRole.stderr || wrongRole.stdout);
      assert.deepEqual(JSON.parse(wrongRole.stdout).gates.semantic.mismatchedNodes, [{
        id: 'router',
        field: 'type',
        expected: ['backend', 'security'],
        actual: 'external',
      }]);

      candidate.edges = candidate.edges.filter(
        (edge) => !(edge.from === 'consent-check' && edge.to === 'tool-runner'),
      );
      const brokenCandidate = writeJson('calibrated-workflow-missing-route.json', candidate);
      const broken = run([
        'verify', '--case', path.join(suiteRoot, 'cases', fixture.caseFile),
        '--candidate', brokenCandidate, '--run', runFile,
      ]);
      assert.equal(broken.status, 1, broken.stderr || broken.stdout);
      assert.equal(JSON.parse(broken.stdout).gates.semantic.missingRelationships.length, 1);
    }
  }
});

test('checked-in benchmark suite covers all five diagram types without presenting reference fixtures as model evidence', () => {
  const manifest = path.join(repoRoot, 'benchmarks/ordinary-model-floor/manifest.json');

  const result = run(['check', '--manifest', manifest]);

  assert.equal(result.status, 0, result.stderr || result.stdout);
  assert.equal(result.stderr, '');
  const receipt = JSON.parse(result.stdout);
  assert.equal(receipt.schemaVersion, 1);
  assert.equal(receipt.benchmark, 'ordinary-model-floor');
  assert.equal(receipt.suiteId, 'ordinary-model-floor-v1');
  assert.equal(receipt.purpose, 'suite-integrity');
  assert.equal(receipt.evidenceEligible, false);
  assert.equal(receipt.caseCount, 5);
  assert.deepEqual(receipt.diagramTypes, [
    'architecture',
    'dataflow',
    'lifecycle',
    'sequence',
    'workflow',
  ]);
  assert.equal(receipt.cases.length, 5);
  for (const item of receipt.cases) {
    assert.equal(item.promptOk, true, item.caseId);
    assert.equal(item.semanticOk, true, item.caseId);
    assert.equal(item.validationOk, true, item.caseId);
  }
});

test('checked-in prompts permit bundled CLI repair while retaining external validation authority', () => {
  const suiteRoot = path.join(repoRoot, 'benchmarks/ordinary-model-floor');
  const manifest = JSON.parse(fs.readFileSync(path.join(suiteRoot, 'manifest.json'), 'utf8'));

  for (const entry of manifest.cases) {
    const prompt = fs.readFileSync(path.join(suiteRoot, entry.prompt), 'utf8');
    assert.match(
      prompt,
      /Use the bundled Archify CLI to validate and repair the candidate when shell access is available\./,
      entry.prompt,
    );
    assert.match(
      prompt,
      /The external harness will independently validate the frozen candidate\./,
      entry.prompt,
    );
  }
});

test('suite integrity rejects a long prompt that omits the attempt-1 file contract', () => {
  const sourceSuite = path.join(repoRoot, 'benchmarks/ordinary-model-floor');
  const incompletePrompt = path.join(tmp, 'incomplete-benchmark-prompt.md');
  fs.writeFileSync(
    incompletePrompt,
    `# Plausible but incomplete prompt\n\n${'Describe the requested system accurately. '.repeat(12)}`,
  );
  const manifest = JSON.parse(fs.readFileSync(path.join(sourceSuite, 'manifest.json'), 'utf8'));
  manifest.cases = manifest.cases.map((entry, index) => ({
    ...entry,
    case: path.resolve(sourceSuite, entry.case),
    prompt: index === 0 ? incompletePrompt : path.resolve(sourceSuite, entry.prompt),
    reference_fixture: path.resolve(sourceSuite, entry.reference_fixture),
  }));
  const manifestFile = writeJson('prompt-contract.manifest.json', manifest);

  const result = run(['check', '--manifest', manifestFile]);

  assert.equal(result.status, 1, result.stderr || result.stdout);
  assert.equal(result.stderr, '');
  const receipt = JSON.parse(result.stdout);
  assert.equal(receipt.cases.find((item) => item.caseId === 'web-runtime-architecture').promptOk, false);
});

test('benchmark fails closed with machine-readable errors for malformed JSON and mismatched run identity', () => {
  const caseFile = writeJson('identity.case.json', {
    schema_version: 1,
    id: 'identity-case',
    diagram_type: 'architecture',
    requirements: { node_ids: ['api'] },
  });
  const candidate = path.join(skillRoot, 'examples/web-app.architecture.json');
  const mismatchedRun = writeJson('identity.run.json', {
    schema_version: 1,
    case_id: 'different-case',
    agent: 'fixture-agent',
    model: 'fixture-model',
    attempt: 1,
    visual_review: { status: 'skipped', reviewer: null, defects: [] },
  });

  const mismatch = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', mismatchedRun]);

  assert.equal(mismatch.status, 2, mismatch.stderr || mismatch.stdout);
  assert.equal(mismatch.stderr, '');
  assert.deepEqual(JSON.parse(mismatch.stdout), {
    schemaVersion: 1,
    benchmark: 'ordinary-model-floor',
    error: {
      code: 'RUN_CASE_MISMATCH',
      message: 'run case_id "different-case" does not match benchmark case "identity-case"',
    },
  });

  const malformedCandidate = path.join(tmp, 'malformed.architecture.json');
  fs.writeFileSync(malformedCandidate, '{ definitely not JSON\n');
  const validRun = writeJson('valid-identity.run.json', {
    schema_version: 1,
    case_id: 'identity-case',
    agent: 'fixture-agent',
    model: 'fixture-model',
    attempt: 1,
    visual_review: { status: 'skipped', reviewer: null, defects: [] },
  });

  const malformed = run(['verify', '--case', caseFile, '--candidate', malformedCandidate, '--run', validRun]);

  assert.equal(malformed.status, 2, malformed.stderr || malformed.stdout);
  assert.equal(malformed.stderr, '');
  const malformedReceipt = JSON.parse(malformed.stdout);
  assert.equal(malformedReceipt.error.code, 'INVALID_JSON');
  assert.match(malformedReceipt.error.message, /malformed\.architecture\.json/);
  assert.doesNotMatch(malformed.stdout, /SyntaxError|at JSON\.parse/);
});

test('benchmark report marks only a complete first-pass matrix as evidence and rejects duplicate runs', () => {
  const manifestFile = path.join(repoRoot, 'benchmarks/ordinary-model-floor/manifest.json');
  const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8'));
  const rows = manifest.cases.map((entry) => {
    const benchmarkCase = JSON.parse(fs.readFileSync(
      path.resolve(path.dirname(manifestFile), entry.case),
      'utf8',
    ));
    return {
      schemaVersion: 1,
      benchmark: 'ordinary-model-floor',
      caseId: benchmarkCase.id,
      run: { agent: 'fixture-agent', model: 'ordinary-model', attempt: 1 },
      gates: {
        semantic: { ok: true },
        validation: { ok: true },
        visualReview: { status: 'passed', reviewer: 'human-reviewer', defects: [] },
      },
      firstPassUsable: true,
    };
  });
  const completeResults = path.join(tmp, 'complete-results.jsonl');
  fs.writeFileSync(completeResults, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`);

  const complete = run(['report', '--results', completeResults, '--manifest', manifestFile]);

  assert.equal(complete.status, 0, complete.stderr || complete.stdout);
  const report = JSON.parse(complete.stdout);
  assert.equal(report.suiteId, 'ordinary-model-floor-v1');
  assert.equal(report.evidenceEligible, true);
  assert.deepEqual(report.coverage, [{
    agent: 'fixture-agent',
    model: 'ordinary-model',
    expected: 5,
    present: 5,
    missingCaseIds: [],
    unexpectedCaseIds: [],
    complete: true,
  }]);

  const duplicateResults = path.join(tmp, 'duplicate-results.jsonl');
  fs.writeFileSync(
    duplicateResults,
    `${[...rows, rows[0]].map((row) => JSON.stringify(row)).join('\n')}\n`,
  );

  const duplicate = run(['report', '--results', duplicateResults, '--manifest', manifestFile]);

  assert.equal(duplicate.status, 2, duplicate.stderr || duplicate.stdout);
  assert.equal(duplicate.stderr, '');
  assert.equal(JSON.parse(duplicate.stdout).error.code, 'DUPLICATE_RESULT');
});

test('benchmark documentation locks the fair-run and truthful-evidence contract', () => {
  const readme = fs.readFileSync(
    path.join(repoRoot, 'benchmarks/ordinary-model-floor/README.md'),
    'utf8',
  );

  for (const required of [
    'firstPassUsable',
    'same prompt',
    'same repository commit',
    'packaged skill root',
    'model-visible working tree',
    'bundled Archify CLI',
    'independently revalidates',
    'attempt 1',
    'no post-hoc edits',
    'Reference fixtures are not benchmark evidence',
    '`record-failure`',
    '`timeout`',
    '`no_candidate`',
    '`provider_error`',
    '`passed`',
    '`failed`',
    '`skipped`',
    'check --manifest',
    'verify --case',
    'report --results',
  ]) {
    assert.match(readme, new RegExp(required.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')), required);
  }
});

test('packaged skill keeps first-draft rules inline and repair contracts behind reachable triggers', () => {
  const read = (file) => fs.readFileSync(path.join(skillRoot, file), 'utf8');
  const skill = read('SKILL.md');
  const defaults = read('references/authoring-defaults.md');
  const authoring = read('references/authoring-contract.md');
  const delivery = read('references/delivery-contract.md');
  const repository = read('references/repository-authoring.md');
  const fastPath = skill.slice(skill.indexOf('## Native diagram authoring'), skill.indexOf('## Update awareness'));
  assert.match(skill, /## Existing candidate handoff[\s\S]*run `finalize` first as one CLI invocation/);
  assert.match(fastPath, /exact schema and example paths in the Type router without listing their directories/);
  assert.match(fastPath, /references\/authoring-defaults\.md/);
  assert.match(fastPath, /bounded batch separate from project documents and complete schemas/);
  assert.match(fastPath, /recover any missing section before writing/);
  assert.match(fastPath, /write the complete candidate directly without planning coordinates in prose/);
  assert.match(fastPath, /Set `meta\.quality_profile` to `"showcase"`/);
  assert.match(fastPath, /successful first drafts need no separate pre-validation/);
  assert.match(fastPath, /Keep the candidate unchanged while the command runs/);
  assert.match(fastPath, /finalize <type> <candidate\.json> <output\.html> --repo-root <repo-root> --quality showcase --json/);
  assert.match(fastPath, /standalone commands only for a separate request or focused failure diagnosis/);
  assert.match(fastPath, /A non-zero exit is never success/);
  assert.match(fastPath, /references\/delivery-contract\.md#failed-finalize-and-candidate-repair/);
  const repair = delivery.match(/## Failed finalize and candidate repair[\s\S]*?(?=\n## |$)/)?.[0] ?? '';
  assert.match(repair, /candidateFrozen: true/);
  assert.match(repair, /--candidate-sha256/);
  assert.match(repair, /fresh `--out-dir/);
  assert.match(repair, /all nine checks, zero composition errors, and zero warnings/);
  assert.match(repair, /two focused repairs[\s\S]*one evidence-based retry/);
  assert.match(defaults, /default to a system overview/);
  assert.match(defaults, /There is no node, edge, source, card, or boundary quota/);
  assert.match(defaults, /Start with automatic routes and endpoint sides/);
  assert.match(defaults, /Pin a side only for a necessary branch, return, or supplied geometry/);
  assert.match(defaults, /Before writing positions[\s\S]*6\.5px × ASCII units \+ 21px/);
  assert.match(defaults, /9px[\s\S]*5\.4px × text units \+ 8px/);
  assert.match(defaults, /card alone cannot qualify an otherwise unconditional arrow/);
  assert.match(repository, /local-only[\s\S]*SSH origin, unsupported forge/);
  assert.match(repository, /actual write or execution\s+site and the conditions/);
  for (const instructions of [skill, defaults]) {
    assert.doesNotMatch(instructions, /(?:at most|no more than|maximum of|cap(?:ped)? at|limit(?:ed)? to)\s+\d+\s+(?:nodes?|components?|relationships?)/i);
  }
  assert.match(defaults, /recoverable failure needs a real transition back/);
  assert.match(authoring, /componentType/);
  assert.match(authoring, /clear gap between boxes, not center distance/i);
  assert.match(read('references/viewer-runtime.md'), /Direct Relationship Pin/);
});

test('dated three-model evidence retains every frozen attempt-1 candidate and truthful gate result', () => {
  const evidence = JSON.parse(fs.readFileSync(path.join(
    repoRoot,
    'benchmarks/ordinary-model-floor/results/2026-07-26-pi-three-models.json',
  ), 'utf8'));

  assert.equal(evidence.generation.repositoryCommit, '66414c7d2366d16a70c9e7282836e416b7917d51');
  assert.equal(evidence.generation.packageSha256, '1f32354a466ec10c21f56346634ce66e170b6e0d23306cc2e9a9cbb68283f05a');
  assert.equal(evidence.generation.attempt, 1);
  assert.equal(evidence.report.evidenceEligible, true);
  assert.deepEqual(evidence.report.overall, {
    runs: 15,
    firstPassUsable: 10,
    firstPassUsableRate: 2 / 3,
    failureClusters: { semantic: 0, validation: 5, visualReview: 5, operational: 0 },
  });
  assert.equal(evidence.runs.length, 15);

  const identities = new Set();
  for (const entry of evidence.runs) {
    identities.add(`${entry.agent}\0${entry.model}\0${entry.caseId}`);
    assert.equal(entry.run.attempt, 1, entry.caseId);
    assert.equal(entry.run.case_id, entry.caseId, entry.caseId);
    assert.equal(entry.receipt.caseId, entry.caseId, entry.caseId);
    assert.equal(entry.receipt.gates.semantic.ok, true, entry.caseId);
    assert.equal(entry.candidate.schema_version, 1, entry.caseId);
    assert.ok(entry.candidate.diagram_type, entry.caseId);
  }
  assert.equal(identities.size, 15);
});

test('post-fix evidence keeps the complete matrix and reports the no-uplift comparison truthfully', () => {
  const evidence = JSON.parse(fs.readFileSync(path.join(
    repoRoot,
    'benchmarks/ordinary-model-floor/results/2026-07-26-pi-three-models-postfix.json',
  ), 'utf8'));

  assert.equal(evidence.generation.repositoryCommit, '2dce766ab19ff5871020828eb85d770745f1069d');
  assert.equal(evidence.generation.packageSha256, 'cc34ab9484ca84e43fce9fce3de612b11c525c6dd5c797646645c6f35954b24e');
  assert.equal(evidence.report.evidenceEligible, true);
  assert.deepEqual(evidence.report.overall, {
    runs: 15,
    firstPassUsable: 8,
    firstPassUsableRate: 8 / 15,
    failureClusters: { semantic: 2, validation: 6, visualReview: 7, operational: 0 },
  });
  assert.equal(evidence.comparison.baselineReverified.firstPassUsable, 8);
  assert.equal(evidence.comparison.postFix.firstPassUsable, 8);
  assert.match(evidence.comparison.outcome, /no measured overall uplift/i);

  const identities = new Set();
  for (const entry of evidence.runs) {
    identities.add(`${entry.agent}\0${entry.model}\0${entry.caseId}`);
    assert.equal(entry.run.attempt, 1, entry.caseId);
    assert.equal(entry.receipt.caseId, entry.caseId, entry.caseId);
    assert.equal(entry.candidate.schema_version, 1, entry.caseId);
    assert.ok(entry.transcript.length > 0, entry.caseId);
  }
  assert.equal(identities.size, 15);

  const qwenArchitecture = evidence.runs.find(
    (entry) => entry.model === 'codewiz-anthropic/qwen3.7-plus'
      && entry.caseId === 'web-runtime-architecture',
  );
  assert.equal(qwenArchitecture.receipt.firstPassUsable, true);
  assert.equal(qwenArchitecture.run.visual_review.reviewer, 'codex-browser-visual-audit-2026-07-26-route-fix');

  const minimaxLifecycle = evidence.runs.find(
    (entry) => entry.model === 'codewiz-anthropic/minimax-m3'
      && entry.caseId === 'agent-run-lifecycle',
  );
  assert.equal(minimaxLifecycle.receipt.gates.semantic.ok, false);
  assert.equal(minimaxLifecycle.receipt.gates.visualReview.status, 'failed');
  assert.ok(minimaxLifecycle.receipt.gates.visualReview.defects.includes(
    'recoverable-failure-has-no-retry-transition-to-execution',
  ));
});

test('quality-first evidence preserves the complete matrix and the measured lifecycle gain without overstating uplift', () => {
  const evidence = JSON.parse(fs.readFileSync(path.join(
    repoRoot,
    'benchmarks/ordinary-model-floor/results/2026-07-26-pi-three-models-quality-first.json',
  ), 'utf8'));

  assert.equal(evidence.generation.repositoryCommit, '7eef4db36a97d04da74a9cb1d1bc3f735058c074');
  assert.equal(evidence.generation.packageSha256, '92135b360ee1502080dac8f2eea6258bb8fa0aa7f9a59cba119b02233f797593');
  assert.equal(evidence.generation.timeLimitSeconds, null);
  assert.match(evidence.generation.latencyPolicy, /not a quality failure/i);
  assert.equal(evidence.generation.processRecovery, undefined);
  assert.ok(evidence.verification.calibratedAliases.includes('Admitted'));
  assert.equal(evidence.report.evidenceEligible, true);
  assert.deepEqual(evidence.report.overall, {
    runs: 15,
    firstPassUsable: 8,
    firstPassUsableRate: 8 / 15,
    failureClusters: { semantic: 2, validation: 5, visualReview: 7, operational: 0 },
  });
  assert.equal(evidence.comparison.baselineReverified.firstPassUsable, 8);
  assert.equal(evidence.comparison.postFixReverified.firstPassUsable, 8);
  assert.equal(evidence.comparison.qualityFirst.firstPassUsable, 8);
  assert.equal(evidence.comparison.qualityFirst.firstPassUsableByCase['agent-run-lifecycle'], 1);
  assert.equal(evidence.comparison.qualityFirst.firstPassUsableByCase['web-runtime-architecture'], 3);
  assert.match(evidence.comparison.outcome, /no measured overall uplift/i);

  const identities = new Set();
  for (const entry of evidence.runs) {
    identities.add(`${entry.agent}\0${entry.model}\0${entry.caseId}`);
    assert.equal(entry.run.attempt, 1, entry.caseId);
    assert.equal(entry.receipt.caseId, entry.caseId, entry.caseId);
    assert.equal(entry.candidate.schema_version, 1, entry.caseId);
    assert.ok(entry.transcript.length > 0, entry.caseId);
  }
  assert.equal(identities.size, 15);

  const minimaxLifecycle = evidence.runs.find(
    (entry) => entry.model === 'codewiz-anthropic/minimax-m3'
      && entry.caseId === 'agent-run-lifecycle',
  );
  assert.equal(minimaxLifecycle.receipt.firstPassUsable, true);
  assert.equal(minimaxLifecycle.receipt.gates.validation.checksPassed, 9);
  assert.ok(minimaxLifecycle.candidate.states.some(
    (state) => state.id === 'admitted' && state.label === 'Admitted',
  ));
  assert.equal(
    minimaxLifecycle.run.visual_review.reviewer,
    'codex-browser-visual-audit-2026-07-27-quality-first-clean-rerun',
  );
  assert.equal(
    minimaxLifecycle.receipt.gates.visualReview.reviewer,
    minimaxLifecycle.run.visual_review.reviewer,
  );

  const deepseekWorkflow = evidence.runs.find(
    (entry) => entry.model === 'seal/deepseek-v4-flash'
      && entry.caseId === 'agent-tool-call-workflow',
  );
  assert.equal(deepseekWorkflow.receipt.firstPassUsable, false);
  assert.ok(deepseekWorkflow.receipt.gates.visualReview.defects.includes(
    'card-claims-retry-loop-without-authored-return-edge',
  ));
});

process.on('exit', () => fs.rmSync(tmp, { recursive: true, force: true }));