Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(gateguard): report intervals and an exact test next to the hole counts

montjeffreyacf4980442f6
diff --git a/docker/gateguard-effectiveness/coverage.js b/docker/gateguard-effectiveness/coverage.js
index 7fedb62c6..aa5f078d0 100644
--- a/docker/gateguard-effectiveness/coverage.js
+++ b/docker/gateguard-effectiveness/coverage.js
@@ -12,6 +12,13 @@
// The questions a scenario provokes are therefore discovered and reported, not
// declared and asserted.
+const { wilson } = require('../context-profiles/ai-eval-lib');
+const { pairedComparison } = require('./lib');
+
+// Rough floor for 80% power on a 40-point pass-rate difference at a two-sided
+// 5% level. Below it the report refuses to let counts stand as evidence.
+const MIN_PAIRS_FOR_A_CLAIM = 20;
+
/** Outcome classes. The four ending in -hole are the improvable cases. */
const CLASSES = Object.freeze({
working: 'the deciding fact was obtained and the outcome is right',
@@ -113,6 +120,53 @@ function pct(value) {
return `${Math.round(value * 100)}%`;
}
+/**
+ * Effect size and cost, with intervals and an exact paired test.
+ *
+ * The hole classes say what went wrong; they say nothing about whether the
+ * difference between arms is distinguishable from chance. Bare counts read as
+ * evidence, so the interval and the McNemar p-value are printed beside them,
+ * and a run too small to support any claim says so in words.
+ */
+function renderEffect(rows, { referenceArm = 'gate', baselineArm = 'off' } = {}) {
+ const arms = [...new Set(rows.map(row => row.arm))].sort();
+ const lines = ['### Effect and cost', ''];
+ lines.push('| Arm | Passed | Wilson 95% | Mean turns | Mean cost (USD) |', '| --- | ---: | --- | ---: | ---: |');
+ for (const arm of arms) {
+ const own = rows.filter(row => row.arm === arm);
+ if (!own.length) continue;
+ const passes = own.filter(row => row.passed).length;
+ const [low, high] = wilson(passes, own.length);
+ const mean = key => own.reduce((total, row) => total + (Number(row[key]) || 0), 0) / own.length;
+ lines.push(`| ${arm} | ${passes}/${own.length} | ${pct(low)}-${pct(high)} | ${mean('turns').toFixed(1)} | ${mean('costUsd').toFixed(3)} |`);
+ }
+
+ lines.push('', `| Arm vs ${referenceArm} | Pairs | Pass-rate difference (95%) | ${referenceArm} only | Arm only | McNemar p |`);
+ lines.push('| --- | ---: | --- | ---: | ---: | ---: |');
+ let smallest = Infinity;
+ for (const arm of arms.filter(arm => arm !== referenceArm)) {
+ const cmp = pairedComparison(rows, referenceArm, arm);
+ smallest = Math.min(smallest, cmp.pairs);
+ const [low, high] = cmp.interval;
+ lines.push(
+ `| ${arm} | ${cmp.pairs} | ${pct(cmp.passRateDifference)} (${pct(low)} to ${pct(high)}) | ${cmp.onlyReference} | ${cmp.onlyArm} | ${cmp.mcnemarP.toFixed(3)} |`
+ );
+ }
+
+ // 20 trials per arm is the rough floor for 80% power on a 40-point difference
+ // at a two-sided 5% level; a run near this one's size separates nothing.
+ if (Number.isFinite(smallest) && smallest < MIN_PAIRS_FOR_A_CLAIM) {
+ lines.push(
+ '',
+ `Underpowered: ${smallest} pair(s) against \`${referenceArm}\`. At 80% power and a two-sided 5% level, ` +
+ 'about 20 trials per arm are needed to detect a 40-point pass-rate difference, about 36 for 30 points, ' +
+ 'and about 160 for the 15-point threshold the pilot pre-registered. Read the rows above as a ' +
+ `demonstration that the arms can differ at all, not as evidence that \`${referenceArm}\` beats \`${baselineArm}\`.`
+ );
+ }
+ return `${lines.join('\n')}\n`;
+}
+
function renderHoles(rows, { baselineArm = 'off' } = {}) {
const arms = [...new Set(rows.map(row => row.arm))].sort();
const classes = Object.keys(CLASSES);
@@ -138,6 +192,8 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) {
const usableScenarios = new Set(verdicts.filter(row => row.label === 'informative').map(row => row.scenario));
const usable = rows.filter(row => usableScenarios.has(row.scenario));
+ lines.push('', renderEffect(usable.length ? usable : rows, { baselineArm }).trimEnd());
+
lines.push('', '### Outcome classes', '');
lines.push(`| Arm | ${classes.join(' | ')} |`, `| --- | ${classes.map(() => '---:').join(' | ')} |`);
for (const arm of arms) {
@@ -169,4 +225,7 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) {
return `${lines.join('\n')}\n`;
}
-module.exports = { CLASSES, classify, tally, difficulty, informative, discoveredQuestions, unengaged, renderHoles };
+module.exports = {
+ CLASSES, MIN_PAIRS_FOR_A_CLAIM, classify, tally, difficulty, informative,
+ discoveredQuestions, unengaged, renderEffect, renderHoles
+};
diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js
index 5c4325c59..d2c4afaab 100644
--- a/tests/docker/gateguard-intent.test.js
+++ b/tests/docker/gateguard-intent.test.js
@@ -178,11 +178,41 @@ test('a ceiling scenario is excluded from the outcome-class table', () => {
const report = coverage.renderHoles(rows);
assert.match(report, /easy: ceiling/);
assert.match(report, /carry no information/);
- // Only the informative scenario's gate trial is counted as working.
- const gateLine = report.split('\n').find(line => line.startsWith('| gate |'));
+ // Only the informative scenario's gate trial is counted as working. Scope the
+ // lookup to the outcome-class table: the effect table also has a `gate` row.
+ const classTable = report.slice(report.indexOf('### Outcome classes'));
+ const gateLine = classTable.split('\n').find(line => line.startsWith('| gate |'));
assert.strictEqual(gateLine.split('|')[2].trim(), '1', gateLine);
});
+test('a small run is reported as underpowered, not as evidence', () => {
+ // The counts this harness prints look like a result. With a handful of trials
+ // they are not, so the report has to say so next to them.
+ const rows = [
+ row({ scenario: 'works', rep: 1, arm: 'off', passed: false }),
+ row({ scenario: 'works', rep: 1, arm: 'gate', passed: true, turns: 8, costUsd: 0.1 }),
+ row({ scenario: 'works', rep: 2, arm: 'off', passed: false }),
+ row({ scenario: 'works', rep: 2, arm: 'gate', passed: false, turns: 7, costUsd: 0.1 })
+ ];
+ const report = coverage.renderEffect(rows);
+ assert.match(report, /Underpowered: 2 pair\(s\)/);
+ assert.match(report, /not as evidence that `gate` beats `off`/);
+ // The exact paired test must be shown beside the counts.
+ assert.match(report, /McNemar p/);
+ assert.match(report, /\| off \| 2 \|/);
+});
+
+test('the underpowered warning clears once there are enough pairs', () => {
+ const rows = [];
+ for (let rep = 1; rep <= coverage.MIN_PAIRS_FOR_A_CLAIM; rep++) {
+ rows.push(row({ scenario: 'works', rep, arm: 'off', passed: false }));
+ rows.push(row({ scenario: 'works', rep, arm: 'gate', passed: true, turns: 8, costUsd: 0.1 }));
+ }
+ const report = coverage.renderEffect(rows);
+ assert.ok(!/Underpowered/.test(report), report);
+ assert.match(report, /\| gate \| 20\/20 \|/);
+});
+
test('unengaged reports scenarios the gate never fired on', () => {
const rows = [
row({ scenario: 'quiet', arm: 'gate', gateDenials: 0 }),