The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(gateguard): report intervals and an exact test next to the hole counts
montjeffreyacf4980442f6
diff --git a/docker/gateguard-effectiveness/coverage.js b/docker/gateguard-effectiveness/coverage.jsindex 7fedb62c6..aa5f078d0 100644--- a/docker/gateguard-effectiveness/coverage.js+++ b/docker/gateguard-effectiveness/coverage.js@@ -12,6 +12,13 @@// The questions a scenario provokes are therefore discovered and reported, not// declared and asserted.+const { wilson } = require('../context-profiles/ai-eval-lib');+const { pairedComparison } = require('./lib');++// Rough floor for 80% power on a 40-point pass-rate difference at a two-sided+// 5% level. Below it the report refuses to let counts stand as evidence.+const MIN_PAIRS_FOR_A_CLAIM = 20;+/** Outcome classes. The four ending in -hole are the improvable cases. */const CLASSES = Object.freeze({working: 'the deciding fact was obtained and the outcome is right',@@ -113,6 +120,53 @@ function pct(value) {return `${Math.round(value * 100)}%`;}+/**+ * Effect size and cost, with intervals and an exact paired test.+ *+ * The hole classes say what went wrong; they say nothing about whether the+ * difference between arms is distinguishable from chance. Bare counts read as+ * evidence, so the interval and the McNemar p-value are printed beside them,+ * and a run too small to support any claim says so in words.+ */+function renderEffect(rows, { referenceArm = 'gate', baselineArm = 'off' } = {}) {+ const arms = [...new Set(rows.map(row => row.arm))].sort();+ const lines = ['### Effect and cost', ''];+ lines.push('| Arm | Passed | Wilson 95% | Mean turns | Mean cost (USD) |', '| --- | ---: | --- | ---: | ---: |');+ for (const arm of arms) {+ const own = rows.filter(row => row.arm === arm);+ if (!own.length) continue;+ const passes = own.filter(row => row.passed).length;+ const [low, high] = wilson(passes, own.length);+ const mean = key => own.reduce((total, row) => total + (Number(row[key]) || 0), 0) / own.length;+ lines.push(`| ${arm} | ${passes}/${own.length} | ${pct(low)}-${pct(high)} | ${mean('turns').toFixed(1)} | ${mean('costUsd').toFixed(3)} |`);+ }++ lines.push('', `| Arm vs ${referenceArm} | Pairs | Pass-rate difference (95%) | ${referenceArm} only | Arm only | McNemar p |`);+ lines.push('| --- | ---: | --- | ---: | ---: | ---: |');+ let smallest = Infinity;+ for (const arm of arms.filter(arm => arm !== referenceArm)) {+ const cmp = pairedComparison(rows, referenceArm, arm);+ smallest = Math.min(smallest, cmp.pairs);+ const [low, high] = cmp.interval;+ lines.push(+ `| ${arm} | ${cmp.pairs} | ${pct(cmp.passRateDifference)} (${pct(low)} to ${pct(high)}) | ${cmp.onlyReference} | ${cmp.onlyArm} | ${cmp.mcnemarP.toFixed(3)} |`+ );+ }++ // 20 trials per arm is the rough floor for 80% power on a 40-point difference+ // at a two-sided 5% level; a run near this one's size separates nothing.+ if (Number.isFinite(smallest) && smallest < MIN_PAIRS_FOR_A_CLAIM) {+ lines.push(+ '',+ `Underpowered: ${smallest} pair(s) against \`${referenceArm}\`. At 80% power and a two-sided 5% level, ` ++ 'about 20 trials per arm are needed to detect a 40-point pass-rate difference, about 36 for 30 points, ' ++ 'and about 160 for the 15-point threshold the pilot pre-registered. Read the rows above as a ' ++ `demonstration that the arms can differ at all, not as evidence that \`${referenceArm}\` beats \`${baselineArm}\`.`+ );+ }+ return `${lines.join('\n')}\n`;+}+function renderHoles(rows, { baselineArm = 'off' } = {}) {const arms = [...new Set(rows.map(row => row.arm))].sort();const classes = Object.keys(CLASSES);@@ -138,6 +192,8 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) {const usableScenarios = new Set(verdicts.filter(row => row.label === 'informative').map(row => row.scenario));const usable = rows.filter(row => usableScenarios.has(row.scenario));+ lines.push('', renderEffect(usable.length ? usable : rows, { baselineArm }).trimEnd());+lines.push('', '### Outcome classes', '');lines.push(`| Arm | ${classes.join(' | ')} |`, `| --- | ${classes.map(() => '---:').join(' | ')} |`);for (const arm of arms) {@@ -169,4 +225,7 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) {return `${lines.join('\n')}\n`;}-module.exports = { CLASSES, classify, tally, difficulty, informative, discoveredQuestions, unengaged, renderHoles };+module.exports = {+ CLASSES, MIN_PAIRS_FOR_A_CLAIM, classify, tally, difficulty, informative,+ discoveredQuestions, unengaged, renderEffect, renderHoles+};diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.jsindex 5c4325c59..d2c4afaab 100644--- a/tests/docker/gateguard-intent.test.js+++ b/tests/docker/gateguard-intent.test.js@@ -178,11 +178,41 @@ test('a ceiling scenario is excluded from the outcome-class table', () => {const report = coverage.renderHoles(rows);assert.match(report, /easy: ceiling/);assert.match(report, /carry no information/);- // Only the informative scenario's gate trial is counted as working.- const gateLine = report.split('\n').find(line => line.startsWith('| gate |'));+ // Only the informative scenario's gate trial is counted as working. Scope the+ // lookup to the outcome-class table: the effect table also has a `gate` row.+ const classTable = report.slice(report.indexOf('### Outcome classes'));+ const gateLine = classTable.split('\n').find(line => line.startsWith('| gate |'));assert.strictEqual(gateLine.split('|')[2].trim(), '1', gateLine);});+test('a small run is reported as underpowered, not as evidence', () => {+ // The counts this harness prints look like a result. With a handful of trials+ // they are not, so the report has to say so next to them.+ const rows = [+ row({ scenario: 'works', rep: 1, arm: 'off', passed: false }),+ row({ scenario: 'works', rep: 1, arm: 'gate', passed: true, turns: 8, costUsd: 0.1 }),+ row({ scenario: 'works', rep: 2, arm: 'off', passed: false }),+ row({ scenario: 'works', rep: 2, arm: 'gate', passed: false, turns: 7, costUsd: 0.1 })+ ];+ const report = coverage.renderEffect(rows);+ assert.match(report, /Underpowered: 2 pair\(s\)/);+ assert.match(report, /not as evidence that `gate` beats `off`/);+ // The exact paired test must be shown beside the counts.+ assert.match(report, /McNemar p/);+ assert.match(report, /\| off \| 2 \|/);+});++test('the underpowered warning clears once there are enough pairs', () => {+ const rows = [];+ for (let rep = 1; rep <= coverage.MIN_PAIRS_FOR_A_CLAIM; rep++) {+ rows.push(row({ scenario: 'works', rep, arm: 'off', passed: false }));+ rows.push(row({ scenario: 'works', rep, arm: 'gate', passed: true, turns: 8, costUsd: 0.1 }));+ }+ const report = coverage.renderEffect(rows);+ assert.ok(!/Underpowered/.test(report), report);+ assert.match(report, /\| gate \| 20\/20 \|/);+});+test('unengaged reports scenarios the gate never fired on', () => {const rows = [row({ scenario: 'quiet', arm: 'gate', gateDenials: 0 }),