The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix: integrate truthful GateGuard harness grading and retry evidence
haelyra344c1fc3fa73
diff --cc docker/gateguard-effectiveness/README.mdindex 646446118,33e703384..c8886fea0--- a/docker/gateguard-effectiveness/README.md+++ b/docker/gateguard-effectiveness/README.md@@@ -33,6 -33,14 +33,14 @@@ scenario and harness inputs and hashesdirectory with `--verify`; this detects later file changes but is not acryptographic signature or proof of who ran the experiment.-`--check-graders`, and every run before it bills a session, requires each++`run-intent.js --check-graders`, and each intent run before it bills a session, requires each+ grader to score `reference` 1 and the start workspace, the `naive` trap and every+ `decoy-*` overlay below 1. A decoy has the right shape and the wrong content, so+ a grader that checks only the form fails the check. Both runners count only valid+ trials: an attempt that ended in a provider error or timeout stays in+ `results.jsonl` with its own transcript, is reported apart, and is retried on+ resume. A gated trial whose hook never ran stops the run and is not recorded.+```bashnode docker/gateguard-effectiveness/run-intent.js --check-gradersnode docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-rundiff --cc docker/gateguard-effectiveness/lib.jsindex 75b6982c8,330d0876f..b166741f3--- a/docker/gateguard-effectiveness/lib.js+++ b/docker/gateguard-effectiveness/lib.js@@@ -374,8 -374,20 +374,20 @@@ function pairedComparison(rows, referen};}+ /** A trial counts only when the provider, the clock and the judge all completed. */+ function isValidTrial(row) {+ return !row.providerError && !row.timedOut && !row.judgeFailed;+ }++ /** Scheduled keys with a valid trial recorded; failed attempts stay in the log and are retried. */+ function completedTrialKeys(rows) {+ return new Set(rows.filter(isValidTrial).map(row => row.key));+ }+-// Totals cover valid trials only; failed attempts are counted apart as providerErrors.++// Totals cover valid trials only; provider failures and timeouts are counted apart.function armSummary(rows, arm) {- const own = rows.filter(row => row.arm === arm);+ const attempts = rows.filter(row => row.arm === arm);+ const own = attempts.filter(isValidTrial);const passes = own.filter(row => row.passed).length;return {arm,@@@ -390,7 -402,7 +402,7 @@@medianWallMs: median(own.map(row => row.wallMs)),medianDenials: median(own.map(row => row.gateDenials)),meanEvidenceRecall: own.length ? own.reduce((sum, row) => sum + row.evidenceRecall, 0) / own.length : null,- providerErrors: own.filter(row => row.providerError || row.timedOut).length- providerErrors: attempts.filter(row => !isValidTrial(row)).length++ providerErrors: attempts.filter(row => row.providerError || row.timedOut).length};}diff --cc docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjsindex c010c214c,37129807c..852fa0ca0--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs@@@ -8,13 -19,15 +19,16 @@@ let checks = []try {const { exportCsv } = require(path.resolve('src/export.js'));if (typeof exportCsv !== 'function') throw new Error('no exportCsv');- const out = exportCsv([{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]);+ const out = exportCsv(ROWS);if (typeof out !== 'string') throw new Error('not a string');-- const lines = out.split('\r\n').filter(Boolean);++ const lines = out.split('\r\n');++ if (lines.at(-1) === '') lines.pop();+ const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));checks.push(out.includes('\r\n'));checks.push(lines.length >= 2 && lines.every(line => line.includes(';')));- checks.push(lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)));+ checks.push(quoted);+ checks.push(quoted && valuesMatch(lines));} catch (_) {- checks = [false, false, false];+ checks = new Array(CHECKS).fill(false);}- console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / 3 })}`);+ console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / CHECKS })}`);diff --cc tests/docker/gateguard-effectiveness.test.jsindex 143844b0e,c18663116..68a5834af--- a/tests/docker/gateguard-effectiveness.test.js+++ b/tests/docker/gateguard-effectiveness.test.js@@@ -317,7 -317,83 +317,84 @@@ if (gnuTarDir)});}+ test('pilot summaries count valid trials only and report provider errors apart', () => {+ const row = (arm, rep, passed, extra = {}) => ({+ key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,+ inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false, ...extra+ });+ const rows = [+ row('gate', 1, true),+ row('gate', 2, false, { providerError: true }),+ row('gate', 2, true),+ row('gate', 3, false, { timedOut: true }),++ row('gate', 4, false, { judgeFailed: true }),+ row('off', 1, true)+ ];+ const gate = lib.summarize(rows, ['gate', 'off']).arms.find(arm => arm.arm === 'gate');+ assert.strictEqual(gate.trials, 2, 'a provider error or timeout is not a trial');+ assert.strictEqual(gate.passes, 2);+ assert.strictEqual(gate.meanScore, 1);+ assert.strictEqual(gate.providerErrors, 2);+ assert.deepStrictEqual([...lib.completedTrialKeys(rows)].sort(), ['s/gate/1', 's/gate/2', 's/off/1']);+ });++ /** The pilot runner driven end to end with fake-claude.js as the provider. */+ function syntheticPilotRunner(root, scenario, arms) {+ const out = path.join(root, 'out');+ const preload = path.join(root, 'preload.js');+ fs.writeFileSync(preload, `+ const cp = require('child_process');+ const originalSpawn = cp.spawnSync;+ cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'+ ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)+ : originalSpawn(file, args, options);+ `);+ const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];+ const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };+ const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });+ const results = () => {+ const file = path.join(out, 'results.jsonl');+ return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];+ };+ return { out, run, results };+ }+if (gitAvailable()) {+ test('a pilot trial that ended in a provider error is retried on resume', () => {+ const root = tempDir('gg-eff-retry-');+ try {+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');+ const { out, run, results } = syntheticPilotRunner(root, scenario, 'off');+ const failed = run({ FAKE_CLAUDE_PROVIDER_ERROR: '1' });+ assert.strictEqual(failed.status, 0, failed.stderr);+ assert.deepStrictEqual(results().map(row => row.providerError), [true]);+ const retried = run({});+ assert.strictEqual(retried.status, 0, retried.stderr);+ assert.ok(retried.stderr.includes('1 trials to run'), retried.stderr);+ const rows = results();+ assert.deepStrictEqual(rows.map(row => row.providerError), [true, false], 'the failed attempt stays for audit');+ assert.strictEqual(rows[1].passed, true);+ assert.notStrictEqual(rows[0].transcript, rows[1].transcript, 'each attempt keeps its own transcript');+ const { summary } = JSON.parse(fs.readFileSync(path.join(out, 'summary.json'), 'utf8'));+ const off = summary.arms.find(arm => arm.arm === 'off');+ assert.deepStrictEqual([off.trials, off.passes, off.providerErrors], [1, 1, 1]);+ const again = run({});+ assert.ok(again.stderr.includes('0 trials to run'), again.stderr);+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+ });++ test('the pilot runner stops before recording a gated trial whose hook never ran', () => {+ const root = tempDir('gg-eff-hookless-');+ try {+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');+ const { run, results } = syntheticPilotRunner(root, scenario, 'gate');+ const result = run({ FAKE_CLAUDE_SKIP_HOOK: '1' });+ assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');+ assert.match(result.stderr, /the gate did not run although the agent edited files/);+ assert.deepStrictEqual(results(), [], 'the hookless trial is not recorded as data');+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+ });+test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => {const root = tempDir('gg-eff-cli-');const out = path.join(root, 'out');diff --cc tests/docker/gateguard-intent.test.jsindex 1982ebb64,a0ea86f99..5616c156c--- a/tests/docker/gateguard-intent.test.js+++ b/tests/docker/gateguard-intent.test.js@@@ -177,6 -177,33 +177,58 @@@ test('every grader separates the start}});+ test('the CSV grader rejects the right dialect around the wrong values', () => {+ const lib = require(path.join(DIR, 'lib'));+ const workRoot = tempDir('gg-decoy-');- const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');- const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort();- assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']);- for (const decoy of decoys) {- const dest = path.join(workRoot, decoy);- lib.prepareWorkspace(csv, dest, decoy);- assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`);- }- const { scores, sound } = intentEval.graderIsSound(csv, workRoot);- assert.ok(sound, JSON.stringify(scores));- for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`);++ try {++ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');++ const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort();++ assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']);++ for (const decoy of decoys) {++ const dest = path.join(workRoot, decoy);++ lib.prepareWorkspace(csv, dest, decoy);++ assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`);++ }++ const { scores, sound } = intentEval.graderIsSound(csv, workRoot);++ assert.ok(sound, JSON.stringify(scores));++ for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`);++ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }+ });++ test('a grader that passes one of its decoys is not sound', () => {+ const workRoot = tempDir('gg-decoy-pass-');- const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');- const root = path.join(workRoot, 'scenario');- fs.cpSync(csv.root, root, { recursive: true });- fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true });- const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot);- assert.strictEqual(scores['decoy-reference'], 1);- assert.strictEqual(sound, false);++ try {++ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');++ const root = path.join(workRoot, 'scenario');++ fs.cpSync(csv.root, root, { recursive: true });++ fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true });++ const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot);++ assert.strictEqual(scores['decoy-reference'], 1);++ assert.strictEqual(sound, false);++ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }++});++++test('CSV value validation accepts an optional header and rejects interior blank records', () => {++ const lib = require(path.join(DIR, 'lib'));++ const workRoot = tempDir('gg-csv-records-');++ try {++ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');++ const dest = path.join(workRoot, 'workspace');++ lib.prepareWorkspace(csv, dest);++ for (const [output, expected] of [++ ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n', 1],++ ['"name";"city"\r\n"Ada";"Paris"\r\n"Bo";"Rome"', 1],++ ['"Ada";"Paris"\r\n\r\n"Bo";"Rome"\r\n', 0],++ ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n\r\n', 0]++ ]) {++ fs.writeFileSync(path.join(dest, 'src/export.js'), `exports.exportCsv = () => ${JSON.stringify(output)};\n`);++ const score = lib.grade(csv, dest).score;++ if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(output));++ else assert.ok(score < 1, JSON.stringify(output));++ }++ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }+ });+// --- behaviour is classified into the right hole ---test('classify maps each behaviour to its hole', () => {