Skip to content

Turn any idea, plan, or codebase into a beautiful interactive diagram. An agent skill for Claude Code, Codex, and more.

fix(benchmark): reject malformed visual defect lists before passing runs (#402)

nbt0d15010a2d49f
diff --git a/archify/test/ordinary-model-floor.test.mjs b/archify/test/ordinary-model-floor.test.mjs
index 89b3e0d7..eab980ad 100644
--- a/archify/test/ordinary-model-floor.test.mjs
+++ b/archify/test/ordinary-model-floor.test.mjs
@@ -200,6 +200,34 @@ test('benchmark never accepts a visual pass without an identified reviewer', ()
assert.equal(receipt.firstPassUsable, false);
});
+test('benchmark never upgrades missing or malformed visual defects to a clean pass', () => {
+ const caseFile = path.join(repoRoot, 'benchmarks/ordinary-model-floor/cases/web-runtime.architecture.case.json');
+ const candidate = path.join(skillRoot, 'examples/web-app.architecture.json');
+ for (const [index, defects] of [undefined, null, 'clipping', { clipping: true }, false, 0, [], ['clipping']].entries()) {
+ const runFile = writeJson(`visual-defects-${index}.run.json`, {
+ schema_version: 1,
+ case_id: 'web-runtime-architecture',
+ agent: 'fixture-agent',
+ model: 'fixture-model',
+ attempt: 1,
+ visual_review: { status: 'passed', reviewer: 'fixture-reviewer', defects },
+ });
+ const result = run(['verify', '--case', caseFile, '--candidate', candidate, '--run', runFile]);
+ const clean = Array.isArray(defects) && defects.length === 0;
+ assert.equal(result.status, clean ? 0 : 1, result.stderr || result.stdout);
+ const receipt = JSON.parse(result.stdout);
+ assert.equal(receipt.gates.semantic.ok, true);
+ assert.equal(receipt.gates.validation.ok, true);
+ assert.equal(receipt.firstPassUsable, clean);
+ assert.equal(receipt.gates.visualReview.status, Array.isArray(defects) ? 'passed' : 'invalid');
+ if (Array.isArray(defects)) {
+ assert.deepEqual(receipt.gates.visualReview.defects, defects);
+ } else {
+ assert.equal(receipt.gates.visualReview.reason, 'passed visual review requires an explicit defects array');
+ }
+ }
+});
+
test('benchmark applies the same semantic and delivery seam to workflow, sequence, data-flow, and lifecycle candidates', () => {
const cases = [
{
diff --git a/benchmarks/ordinary-model-floor/benchmark.mjs b/benchmarks/ordinary-model-floor/benchmark.mjs
index 1c866857..ec47764c 100644
--- a/benchmarks/ordinary-model-floor/benchmark.mjs
+++ b/benchmarks/ordinary-model-floor/benchmark.mjs
@@ -249,6 +249,13 @@ function verify(args) {
reason: 'passed visual review requires a non-empty reviewer identity',
};
}
+ if (visualReview.status === 'passed' && !Array.isArray(run.visual_review?.defects)) {
+ visualReview = {
+ ...visualReview,
+ status: 'invalid',
+ reason: 'passed visual review requires an explicit defects array',
+ };
+ }
const firstPassUsable = run.attempt === 1
&& semantic.ok
&& validation.ok