Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

feat: count benchmark operational failures

tt-a1ibd16237e6bbe
diff --git a/archify/test/ordinary-model-floor.test.mjs b/archify/test/ordinary-model-floor.test.mjs
index f025bc43..38d14be7 100644
--- a/archify/test/ordinary-model-floor.test.mjs
+++ b/archify/test/ordinary-model-floor.test.mjs
@@ -306,6 +306,7 @@ test('benchmark report separates first-pass usable rate from semantic, validatio
semantic: 1,
validation: 1,
visualReview: 1,
+ operational: 0,
},
});
assert.deepEqual(report.byConfiguration, [
@@ -315,7 +316,7 @@ test('benchmark report separates first-pass usable rate from semantic, validatio
runs: 2,
firstPassUsable: 2,
firstPassUsableRate: 1,
- failureClusters: { semantic: 0, validation: 0, visualReview: 0 },
+ failureClusters: { semantic: 0, validation: 0, visualReview: 0, operational: 0 },
},
{
agent: 'opencode',
@@ -323,11 +324,62 @@ test('benchmark report separates first-pass usable rate from semantic, validatio
runs: 2,
firstPassUsable: 0,
firstPassUsableRate: 0,
- failureClusters: { semantic: 1, validation: 1, visualReview: 1 },
+ failureClusters: { semantic: 1, validation: 1, visualReview: 1, operational: 0 },
},
]);
});
+test('benchmark records a timeout without a candidate as a complete first-pass failure', () => {
+ const caseFile = writeJson('timeout.case.json', {
+ schema_version: 1,
+ id: 'timeout-architecture',
+ diagram_type: 'architecture',
+ requirements: { nodes: [] },
+ });
+ const runFile = writeJson('timeout.run.json', {
+ schema_version: 1,
+ case_id: 'timeout-architecture',
+ agent: 'pi',
+ model: 'glm-5.2-low',
+ attempt: 1,
+ });
+
+ const recorded = run([
+ 'record-failure',
+ '--case', caseFile,
+ '--run', runFile,
+ '--failure', 'timeout',
+ ]);
+
+ assert.equal(recorded.status, 1, recorded.stderr || recorded.stdout);
+ assert.equal(recorded.stderr, '');
+ const failureReceipt = JSON.parse(recorded.stdout);
+ assert.deepEqual(failureReceipt.operational, { status: 'failed', reason: 'timeout' });
+ assert.equal(failureReceipt.gates.semantic.status, 'not_run');
+ assert.equal(failureReceipt.gates.validation.status, 'not_run');
+ assert.equal(failureReceipt.gates.visualReview.status, 'skipped');
+ assert.equal(failureReceipt.firstPassUsable, false);
+
+ const resultsFile = path.join(tmp, 'timeout-results.jsonl');
+ fs.writeFileSync(resultsFile, `${JSON.stringify(failureReceipt)}\n`);
+ const manifestFile = writeJson('timeout.manifest.json', {
+ id: 'timeout-suite',
+ cases: [{ case: caseFile }],
+ });
+ const reported = run(['report', '--results', resultsFile, '--manifest', manifestFile]);
+
+ assert.equal(reported.status, 0, reported.stderr || reported.stdout);
+ const report = JSON.parse(reported.stdout);
+ assert.equal(report.evidenceEligible, true);
+ assert.deepEqual(report.overall.failureClusters, {
+ semantic: 0,
+ validation: 0,
+ visualReview: 0,
+ operational: 1,
+ });
+ assert.equal(report.overall.firstPassUsableRate, 0);
+});
+
test('benchmark semantic requirements bind by accepted technical labels instead of forcing model-authored internal IDs', () => {
const source = JSON.parse(fs.readFileSync(path.join(skillRoot, 'examples/web-app.architecture.json'), 'utf8'));
const rename = new Map([
@@ -556,6 +608,10 @@ test('benchmark documentation locks the fair-run and truthful-evidence contract'
'attempt 1',
'no post-hoc edits',
'Reference fixtures are not benchmark evidence',
+ '`record-failure`',
+ '`timeout`',
+ '`no_candidate`',
+ '`provider_error`',
'`passed`',
'`failed`',
'`skipped`',
diff --git a/benchmarks/ordinary-model-floor/README.md b/benchmarks/ordinary-model-floor/README.md
index 40d4c6cf..bf5f364c 100644
--- a/benchmarks/ordinary-model-floor/README.md
+++ b/benchmarks/ordinary-model-floor/README.md
@@ -57,6 +57,14 @@ node benchmarks/ordinary-model-floor/benchmark.mjs verify --case benchmarks/ordi
The command writes one machine-readable receipt to stdout. Exit code `0` means first-pass usable, `1` means the candidate failed one or more gates, and `2` means the inputs or benchmark invocation were invalid.
+If the agent invocation ends without a candidate, preserve the provider's raw operational record and use `record-failure` to emit a first-pass failure receipt instead of dropping the run:
+
+```bash
+node benchmarks/ordinary-model-floor/benchmark.mjs record-failure --case benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json --run /path/to/run.json --failure timeout
+```
+
+The allow-listed reasons are `timeout`, `no_candidate`, and `provider_error`. These receipts count toward complete matrix coverage and the operational failure cluster, but semantic, validation, and visual-review gates remain truthfully `not_run` or `skipped`. Never turn an absent candidate into a fabricated invalid JSON file.
+
## Visual review
Review the final browser artifact or canonical raster, not the source JSON alone:
@@ -75,6 +83,6 @@ Store one verifier receipt per line in a JSONL file, then aggregate it against t
node benchmarks/ordinary-model-floor/benchmark.mjs report --results /path/to/results.jsonl --manifest benchmarks/ordinary-model-floor/manifest.json
```
-The report separates semantic, deterministic-validation, and visual-review failure clusters. `evidenceEligible` is true only when every agent/model configuration has exactly one valid attempt 1 receipt for every manifest case. The report does not prove that an external transcript is authentic; retain the raw prompts, candidate files, repository commit, and reviewer evidence alongside any published claim.
+The report separates operational, semantic, deterministic-validation, and visual-review failure clusters. `evidenceEligible` is true only when every agent/model configuration has exactly one valid attempt 1 receipt for every manifest case. The report does not prove that an external transcript is authentic; retain the raw prompts, candidate files, repository commit, and reviewer evidence alongside any published claim.
No model result or leaderboard is checked in until the corresponding runs and visual reviews have actually happened.
diff --git a/benchmarks/ordinary-model-floor/benchmark.mjs b/benchmarks/ordinary-model-floor/benchmark.mjs
index fcaae082..aaea89ed 100644
--- a/benchmarks/ordinary-model-floor/benchmark.mjs
+++ b/benchmarks/ordinary-model-floor/benchmark.mjs
@@ -22,6 +22,7 @@ const PROMPT_CONTRACT_PHRASES = [
'Do not copy a checked-in example.',
'Do not claim that validation passed; the external harness will validate the frozen file.',
];
+const OPERATIONAL_FAILURES = new Set(['timeout', 'no_candidate', 'provider_error']);
class BenchmarkError extends Error {
constructor(code, message) {
@@ -227,6 +228,7 @@ function verify(args) {
schemaVersion: 1,
benchmark: 'ordinary-model-floor',
caseId: benchmarkCase.id,
+ operational: { status: 'candidate_produced' },
run: {
agent: run.agent,
model: run.model,
@@ -245,6 +247,61 @@ function verify(args) {
process.exitCode = firstPassUsable ? 0 : 1;
}
+function recordFailure(args) {
+ const caseFile = option(args, '--case');
+ const runFile = option(args, '--run');
+ const reason = option(args, '--failure');
+ if (!caseFile || !runFile || !reason) {
+ throw new Error('record-failure requires --case, --run, and --failure');
+ }
+ if (!OPERATIONAL_FAILURES.has(reason)) {
+ throw new BenchmarkError(
+ 'INVALID_OPERATIONAL_FAILURE',
+ `failure must be one of ${[...OPERATIONAL_FAILURES].join(', ')}`,
+ );
+ }
+
+ const benchmarkCase = readJson(caseFile);
+ const run = readJson(runFile);
+ if (run.case_id !== benchmarkCase.id) {
+ throw new BenchmarkError(
+ 'RUN_CASE_MISMATCH',
+ `run case_id "${run.case_id}" does not match benchmark case "${benchmarkCase.id}"`,
+ );
+ }
+ if (typeof run.agent !== 'string' || run.agent.trim() === ''
+ || typeof run.model !== 'string' || run.model.trim() === '') {
+ throw new BenchmarkError('INVALID_RUN', 'run agent and model must be non-empty strings');
+ }
+ if (run.attempt !== 1) {
+ throw new BenchmarkError('NON_FIRST_PASS_RESULT', 'operational failures must record attempt 1');
+ }
+
+ process.stdout.write(`${JSON.stringify({
+ schemaVersion: 1,
+ benchmark: 'ordinary-model-floor',
+ caseId: benchmarkCase.id,
+ operational: { status: 'failed', reason },
+ run: {
+ agent: run.agent,
+ model: run.model,
+ attempt: run.attempt,
+ },
+ gates: {
+ semantic: { ok: false, status: 'not_run' },
+ validation: { ok: false, status: 'not_run' },
+ visualReview: {
+ status: 'skipped',
+ reviewer: null,
+ defects: [],
+ reason: 'candidate was not produced',
+ },
+ },
+ firstPassUsable: false,
+ }, null, 2)}\n`);
+ process.exitCode = 1;
+}
+
function check(args) {
const manifestFile = option(args, '--manifest');
if (!manifestFile) throw new Error('check requires --manifest');
@@ -296,6 +353,7 @@ function newSummary() {
semantic: 0,
validation: 0,
visualReview: 0,
+ operational: 0,
},
};
}
@@ -303,11 +361,15 @@ function newSummary() {
function addResult(summary, result) {
summary.runs += 1;
if (result.firstPassUsable === true) summary.firstPassUsable += 1;
- if (result.gates?.semantic?.ok !== true) summary.failureClusters.semantic += 1;
- if (result.gates?.validation?.ok !== true) summary.failureClusters.validation += 1;
- if (result.gates?.visualReview?.status !== 'passed'
- || (result.gates?.visualReview?.defects || []).length > 0) {
- summary.failureClusters.visualReview += 1;
+ if (result.operational?.status === 'failed') {
+ summary.failureClusters.operational += 1;
+ } else {
+ if (result.gates?.semantic?.ok !== true) summary.failureClusters.semantic += 1;
+ if (result.gates?.validation?.ok !== true) summary.failureClusters.validation += 1;
+ if (result.gates?.visualReview?.status !== 'passed'
+ || (result.gates?.visualReview?.defects || []).length > 0) {
+ summary.failureClusters.visualReview += 1;
+ }
}
summary.firstPassUsableRate = summary.runs === 0
? 0
@@ -331,8 +393,18 @@ function validateResult(result) {
`result for ${result.run.agent}/${result.run.model}/${result.caseId} is not attempt 1`,
);
}
+ const operationalFailed = result.operational?.status === 'failed';
+ if (result.operational !== undefined
+ && result.operational.status !== 'candidate_produced'
+ && !operationalFailed) {
+ throw new BenchmarkError('INVALID_RESULT', 'result operational status is invalid');
+ }
+ if (operationalFailed && !OPERATIONAL_FAILURES.has(result.operational.reason)) {
+ throw new BenchmarkError('INVALID_RESULT', 'result operational failure reason is invalid');
+ }
const visual = result.gates?.visualReview;
- const derivedFirstPassUsable = result.gates?.semantic?.ok === true
+ const derivedFirstPassUsable = !operationalFailed
+ && result.gates?.semantic?.ok === true
&& result.gates?.validation?.ok === true
&& visual?.status === 'passed'
&& typeof visual.reviewer === 'string'
@@ -434,6 +506,8 @@ try {
const [command, ...args] = process.argv.slice(2);
if (command === 'verify') {
verify(args);
+ } else if (command === 'record-failure') {
+ recordFailure(args);
} else if (command === 'check') {
check(args);
} else if (command === 'report') {
@@ -441,7 +515,7 @@ try {
} else {
throw new BenchmarkError(
'USAGE',
- 'usage: benchmark.mjs <verify|check|report> [options]',
+ 'usage: benchmark.mjs <verify|record-failure|check|report> [options]',
);
}
} catch (error) {