The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(gateguard): make harness graders check content and count only valid trials
montjeffrey6a7996aa7052
diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.mdindex 646446118..33e703384 100644--- a/docker/gateguard-effectiveness/README.md+++ b/docker/gateguard-effectiveness/README.md@@ -33,6 +33,14 @@ scenario and harness inputs and hashes the output artifacts. Verify an outputdirectory with `--verify`; this detects later file changes but is not acryptographic signature or proof of who ran the experiment.+`--check-graders`, and every run before it bills a session, requires each+grader to score `reference` 1 and the start workspace, the `naive` trap and every+`decoy-*` overlay below 1. A decoy has the right shape and the wrong content, so+a grader that checks only the form fails the check. Both runners count only valid+trials: an attempt that ended in a provider error or timeout stays in+`results.jsonl` with its own transcript, is reported apart, and is retried on+resume. A gated trial whose hook never ran stops the run and is not recorded.+```bashnode docker/gateguard-effectiveness/run-intent.js --check-gradersnode docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-rundiff --git a/docker/gateguard-effectiveness/coverage.js b/docker/gateguard-effectiveness/coverage.jsindex 45b4896f7..a2f267210 100644--- a/docker/gateguard-effectiveness/coverage.js+++ b/docker/gateguard-effectiveness/coverage.js@@ -13,17 +13,13 @@// declared and asserted.const { wilson } = require('../context-profiles/ai-eval-lib');-const { pairedComparison } = require('./lib');+const { pairedComparison, isValidTrial } = require('./lib');// Rough floor for 80% power on a 40-point pass-rate difference at a two-sided// 5% level. Below it the report refuses to let counts stand as evidence.const MIN_PAIRS_FOR_A_CLAIM = 20;const MAX_INVALID_TRIAL_FRACTION = 0.1;-function isValidTrial(row) {- return !row.providerError && !row.timedOut && !row.judgeFailed;-}-/** Outcome classes. The four ending in -hole are the improvable cases. */const CLASSES = Object.freeze({working: 'the deciding fact was obtained and the outcome is right',diff --git a/docker/gateguard-effectiveness/intent-eval.js b/docker/gateguard-effectiveness/intent-eval.jsindex 01a3ff529..67aafa1f6 100644--- a/docker/gateguard-effectiveness/intent-eval.js+++ b/docker/gateguard-effectiveness/intent-eval.js@@ -62,16 +62,33 @@ function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) {return scenarios.filter(scenario => ids.includes(scenario.id));}-/** Confirms the grader separates the start, the trap and a correct solution. */+/**+ * Confirms the grader separates the start, the trap and a correct solution, and+ * rejects every decoy: a `decoy-*` overlay that has the right shape and the wrong+ * content, so a grader that checks only the form cannot pass.+ */function graderIsSound(scenario, workRoot) {+ const decoys = fs.readdirSync(scenario.root, { withFileTypes: true })+ .filter(entry => entry.isDirectory() && entry.name.startsWith('decoy-'))+ .map(entry => entry.name)+ .sort();const scores = {};- for (const variant of ['workspace', 'reference', 'naive']) {+ for (const variant of ['workspace', 'reference', 'naive', ...decoys]) {const dest = path.join(workRoot, `check-${scenario.id}-${variant}`);fs.rmSync(dest, { recursive: true, force: true });prepareWorkspace(scenario, dest, variant === 'workspace' ? null : variant);scores[variant] = grade(scenario, dest).score;}- return { scores, sound: scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 };+ const sound = scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 && decoys.every(decoy => scores[decoy] < 1);+ return { scores, sound };+}++/** One line of grader scores: start, trap, reference, then each decoy. */+function formatGraderScores(scores) {+ const { workspace, naive, reference, ...decoys } = scores;+ return [`start=${workspace}`, `trap=${naive}`, `reference=${reference}`]+ .concat(Object.entries(decoys).map(([name, score]) => `${name}=${score}`))+ .join(' ');}/** One trial: fresh workspace, a conversation with the simulated user, then the hidden grader. */@@ -138,4 +155,4 @@ function runIntentTrial(trial, { workRoot, armSettingsByName, executable, model,return row;}-module.exports = { loadIntentScenarios, graderIsSound, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS };+module.exports = { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS };diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.jsindex 75b6982c8..330d0876f 100644--- a/docker/gateguard-effectiveness/lib.js+++ b/docker/gateguard-effectiveness/lib.js@@ -374,8 +374,20 @@ function pairedComparison(rows, reference, arm, { rounds = 4000, seed = 0x9e3779};}+/** A trial counts only when the provider, the clock and the judge all completed. */+function isValidTrial(row) {+ return !row.providerError && !row.timedOut && !row.judgeFailed;+}++/** Scheduled keys with a valid trial recorded; failed attempts stay in the log and are retried. */+function completedTrialKeys(rows) {+ return new Set(rows.filter(isValidTrial).map(row => row.key));+}++// Totals cover valid trials only; failed attempts are counted apart as providerErrors.function armSummary(rows, arm) {- const own = rows.filter(row => row.arm === arm);+ const attempts = rows.filter(row => row.arm === arm);+ const own = attempts.filter(isValidTrial);const passes = own.filter(row => row.passed).length;return {arm,@@ -390,12 +402,12 @@ function armSummary(rows, arm) {medianWallMs: median(own.map(row => row.wallMs)),medianDenials: median(own.map(row => row.gateDenials)),meanEvidenceRecall: own.length ? own.reduce((sum, row) => sum + row.evidenceRecall, 0) / own.length : null,- providerErrors: own.filter(row => row.providerError || row.timedOut).length+ providerErrors: attempts.filter(row => !isValidTrial(row)).length};}function summarize(rows, arms, reference = 'gate') {- const valid = rows.filter(row => !row.providerError && !row.timedOut);+ const valid = rows.filter(isValidTrial);const scenarios = [...new Set(valid.map(row => row.scenario))].sort();return {arms: arms.map(arm => armSummary(rows, arm)),@@ -451,6 +463,8 @@ module.exports = {runTrial,binomialTwoSided,pairedComparison,+ isValidTrial,+ completedTrialKeys,summarize,renderMarkdown};diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.jsindex b828af9cb..0fd0f07fa 100644--- a/docker/gateguard-effectiveness/run-intent.js+++ b/docker/gateguard-effectiveness/run-intent.js@@ -13,9 +13,9 @@ const crypto = require('crypto');const os = require('os');const path = require('path');const { spawnSync } = require('child_process');-const { schedule } = require('./lib');+const { schedule, completedTrialKeys } = require('./lib');const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');-const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');+const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');const { renderHoles, difficulty, isValidTrial } = require('./coverage');const evidence = require('./evidence');@@ -118,10 +118,6 @@ function writeAtomic(file, contents) {}}-function completedTrialKeys(rows) {- return new Set(rows.filter(isValidTrial).map(row => row.key));-}-// Resuming a session needs persistence, so Claude writes a transcript folder per// trial workspace under its projects directory. A per-trial CLAUDE_CONFIG_DIR// would isolate those but also hides the credentials, so they are removed here@@ -208,10 +204,10 @@ function checkGraders(scenarios) {for (const scenario of scenarios) {const { scores, sound } = graderIsSound(scenario, workRoot);const mark = sound ? 'ok ' : 'BAD ';- process.stdout.write(`${mark} ${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}\n`);+ process.stdout.write(`${mark} ${scenario.id}: ${formatGraderScores(scores)}\n`);if (!sound) bad++;}- if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference`);+ if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference, or pass a decoy`);} finally {fs.rmSync(workRoot, { recursive: true, force: true });}@@ -295,7 +291,7 @@ function main() {for (const scenario of scenarios) {const { scores, sound } = graderIsSound(scenario, workRoot);if (!sound) {- throw new Error(`${scenario.id}: grader scores start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}; fix it before spending sessions`);+ throw new Error(`${scenario.id}: grader scores ${formatGraderScores(scores)}; fix it before spending sessions`);}}const treeRoots = {};diff --git a/docker/gateguard-effectiveness/run.js b/docker/gateguard-effectiveness/run.jsindex 8824b4a5c..69b62c638 100644--- a/docker/gateguard-effectiveness/run.js+++ b/docker/gateguard-effectiveness/run.js@@ -7,7 +7,7 @@ const os = require('os');const path = require('path');const { spawnSync } = require('child_process');const { ARMS, DEFAULT_ARMS, resolveRef, materializeTree, armSettings } = require('./arms');-const { loadScenarios, schedule, runTrial, summarize, renderMarkdown } = require('./lib');+const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys } = require('./lib');const REPO_ROOT = path.join(__dirname, '..', '..');@@ -167,7 +167,8 @@ function main() {const treeRoots = {};for (const [name, sha] of Object.entries(trees)) treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name));const armSettingsByName = Object.fromEntries(options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)]));- const done = new Set(rows.map(row => row.key));+ // Failed attempts stay in results.jsonl for audit; a key is done only once a valid trial is recorded.+ const done = completedTrialKeys(rows);const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key));process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);if (options.dryRun) {@@ -192,7 +193,10 @@ function main() {timeoutMs: options.timeoutMin * 60000,execute});- fs.writeFileSync(path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`), transcript);+ // One transcript per attempt, so a retry never overwrites the failed attempt's record.+ row.attemptId = crypto.randomUUID();+ row.transcript = `transcripts/${trial.key.replace(/\//g, '__')}__${row.attemptId}.jsonl`;+ fs.writeFileSync(path.join(options.out, row.transcript), transcript);fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjsindex c010c214c..37129807c 100644--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs@@ -1,20 +1,33 @@'use strict';-// Scores against the hidden dialect: semicolons, CRLF, every field quoted.+// Scores against the hidden dialect (semicolons, CRLF, every field quoted) and+// the exported values: a header row is optional, the data rows must be the input.const path = require('path');+const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }];+const CHECKS = 4;++function valuesMatch(lines) {+ const parsed = lines.map(line => line.slice(1, -1).split('";"'));+ const header = Object.keys(ROWS[0]);+ const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === JSON.stringify(header) ? parsed.slice(1) : parsed;+ return JSON.stringify(data) === JSON.stringify(ROWS.map(row => Object.values(row)));+}+let checks = [];try {const { exportCsv } = require(path.resolve('src/export.js'));if (typeof exportCsv !== 'function') throw new Error('no exportCsv');- const out = exportCsv([{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]);+ const out = exportCsv(ROWS);if (typeof out !== 'string') throw new Error('not a string');const lines = out.split('\r\n').filter(Boolean);+ const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));checks.push(out.includes('\r\n'));checks.push(lines.length >= 2 && lines.every(line => line.includes(';')));- checks.push(lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)));+ checks.push(quoted);+ checks.push(quoted && valuesMatch(lines));} catch (_) {- checks = [false, false, false];+ checks = new Array(CHECKS).fill(false);}-console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / 3 })}`);+console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / CHECKS })}`);diff --git a/docs/gateguard/question-effectiveness.md b/docs/gateguard/question-effectiveness.mdindex 3135ebca5..7ac2d7355 100644--- a/docs/gateguard/question-effectiveness.md+++ b/docs/gateguard/question-effectiveness.md@@ -103,8 +103,11 @@ Per trial (`results.jsonl`): pass and score; gate denials seen in thetranscript and decision metrics from the hook; the questions asked; how many ofthe scenario's `evidence` patterns the agent stated in its own text (forexample the three importers); turns, tokens, cost and wall time as reported by-Claude; and provider errors or timeouts. The full transcript of each trial is-kept in `transcripts/`.+Claude; and provider errors or timeouts. The full transcript of each attempt is+kept in `transcripts/`. An attempt that ends in a provider error or timeout+stays in `results.jsonl` but is not a trial: summaries count valid trials only,+report the failed attempts apart, and a resumed run retries their keys. A gated+trial whose hook never ran is not recorded at all; the run stops instead.## Analysis@@ -253,9 +256,14 @@ arms — every trial passing or every trial failing. Trap strength (how often thungated arm passes) is reported separately and does not decide usability, sincethe ungated arm failing is what a working trap looks like.-Graders are checked before any session is billed: the start workspace and the-`naive` overlay must both score below 1 and `reference` must score 1, or the run-stops.+Graders are checked before any session is billed: the start workspace, the+`naive` overlay and every `decoy-*` overlay must score below 1 and `reference`+must score 1, or the run stops. A decoy has the right shape and the wrong+content. `csv-export-delimiter` has two (a constant in the right dialect, and+the column names instead of the values), and its grader now checks the+exported values as well as the dialect. Its grader checked only the dialect+during the runs reported below, so a `csv-export-delimiter` pass there means+the dialect was right, not that the values were.### Hidden-intent results