The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
test(gateguard): pin grader decoys, pilot retries and the hookless-trial abort
montjeffrey64d155ae1d59
diff --git a/docker/gateguard-effectiveness/fake-claude.js b/docker/gateguard-effectiveness/fake-claude.jsindex 41979941f..60ef40433 100644--- a/docker/gateguard-effectiveness/fake-claude.js+++ b/docker/gateguard-effectiveness/fake-claude.js@@ -39,9 +39,11 @@ const input = fs.existsSync(target)const toolName = 'old_string' in input ? 'Edit' : 'Write';emit({ type: 'assistant', message: { content: [{ type: 'text', text: 'Checking report.js, invoice.js and csv.js first.' }, { type: 'tool_use', id: 't1', name: toolName, input }] } });+// FAKE_CLAUDE_SKIP_HOOK=1 edits without running the hook (a broken hook setup);+// FAKE_CLAUDE_PROVIDER_ERROR=1 ends the session in a non-authentication provider error.let denial = '';const group = (settings.hooks && settings.hooks.PreToolUse || []).find(entry => new RegExp(`^(${entry.matcher})$`).test(toolName));-if (group) {+if (group && process.env.FAKE_CLAUDE_SKIP_HOOK !== '1') {const env = { ...process.env, CLAUDE_PROJECT_DIR: process.cwd() };const payload = { session_id: 'fake', cwd: process.cwd(), hook_event_name: 'PreToolUse', tool_name: toolName, tool_input: input, tool_use_id: 't1' };const result = spawnSync(group.hooks[0].command, { shell: true, input: JSON.stringify(payload), env, encoding: 'utf8' });@@ -50,4 +52,8 @@ if (group) {}emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't1', is_error: Boolean(denial), content: denial || 'ok' }] } });copyTree(overlay, process.cwd());+if (process.env.FAKE_CLAUDE_PROVIDER_ERROR === '1') {+ emit({ type: 'result', is_error: true, result: 'API Error: 529 overloaded', num_turns: 1, duration_ms: 100, total_cost_usd: 0, usage: {}, modelUsage: {} });+ process.exit(0);+}emit({ type: 'result', is_error: false, result: 'done', num_turns: 3, duration_ms: 1200, total_cost_usd: 0.01, usage: { input_tokens: 100, cache_creation_input_tokens: 50, cache_read_input_tokens: 400, output_tokens: 80 }, modelUsage: { 'fake-model': {} } });diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.jsnew file mode 100644index 000000000..bd6ab13da--- /dev/null+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js@@ -0,0 +1,8 @@+'use strict';++/** Decoy: a constant in the right dialect that ignores its input. */+function exportCsv() {+ return '"x";"y"\r\n"x";"y"\r\n';+}++module.exports = { exportCsv };diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.jsnew file mode 100644index 000000000..9fe5c9cce--- /dev/null+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js@@ -0,0 +1,8 @@+'use strict';++/** Decoy: the right dialect around the wrong fields, the column names instead of the values. */+function exportCsv(rows) {+ return rows.map(row => Object.keys(row).map(key => `"${key}"`).join(';')).join('\r\n') + '\r\n';+}++module.exports = { exportCsv };diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.jsindex 143844b0e..c18663116 100644--- a/tests/docker/gateguard-effectiveness.test.js+++ b/tests/docker/gateguard-effectiveness.test.js@@ -317,7 +317,83 @@ if (gnuTarDir) {});}+test('pilot summaries count valid trials only and report provider errors apart', () => {+ const row = (arm, rep, passed, extra = {}) => ({+ key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,+ inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false, ...extra+ });+ const rows = [+ row('gate', 1, true),+ row('gate', 2, false, { providerError: true }),+ row('gate', 2, true),+ row('gate', 3, false, { timedOut: true }),+ row('off', 1, true)+ ];+ const gate = lib.summarize(rows, ['gate', 'off']).arms.find(arm => arm.arm === 'gate');+ assert.strictEqual(gate.trials, 2, 'a provider error or timeout is not a trial');+ assert.strictEqual(gate.passes, 2);+ assert.strictEqual(gate.meanScore, 1);+ assert.strictEqual(gate.providerErrors, 2);+ assert.deepStrictEqual([...lib.completedTrialKeys(rows)].sort(), ['s/gate/1', 's/gate/2', 's/off/1']);+});++/** The pilot runner driven end to end with fake-claude.js as the provider. */+function syntheticPilotRunner(root, scenario, arms) {+ const out = path.join(root, 'out');+ const preload = path.join(root, 'preload.js');+ fs.writeFileSync(preload, `+ const cp = require('child_process');+ const originalSpawn = cp.spawnSync;+ cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'+ ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)+ : originalSpawn(file, args, options);+ `);+ const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];+ const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };+ const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });+ const results = () => {+ const file = path.join(out, 'results.jsonl');+ return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];+ };+ return { out, run, results };+}+if (gitAvailable()) {+ test('a pilot trial that ended in a provider error is retried on resume', () => {+ const root = tempDir('gg-eff-retry-');+ try {+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');+ const { out, run, results } = syntheticPilotRunner(root, scenario, 'off');+ const failed = run({ FAKE_CLAUDE_PROVIDER_ERROR: '1' });+ assert.strictEqual(failed.status, 0, failed.stderr);+ assert.deepStrictEqual(results().map(row => row.providerError), [true]);+ const retried = run({});+ assert.strictEqual(retried.status, 0, retried.stderr);+ assert.ok(retried.stderr.includes('1 trials to run'), retried.stderr);+ const rows = results();+ assert.deepStrictEqual(rows.map(row => row.providerError), [true, false], 'the failed attempt stays for audit');+ assert.strictEqual(rows[1].passed, true);+ assert.notStrictEqual(rows[0].transcript, rows[1].transcript, 'each attempt keeps its own transcript');+ const { summary } = JSON.parse(fs.readFileSync(path.join(out, 'summary.json'), 'utf8'));+ const off = summary.arms.find(arm => arm.arm === 'off');+ assert.deepStrictEqual([off.trials, off.passes, off.providerErrors], [1, 1, 1]);+ const again = run({});+ assert.ok(again.stderr.includes('0 trials to run'), again.stderr);+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+ });++ test('the pilot runner stops before recording a gated trial whose hook never ran', () => {+ const root = tempDir('gg-eff-hookless-');+ try {+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');+ const { run, results } = syntheticPilotRunner(root, scenario, 'gate');+ const result = run({ FAKE_CLAUDE_SKIP_HOOK: '1' });+ assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');+ assert.match(result.stderr, /the gate did not run although the agent edited files/);+ assert.deepStrictEqual(results(), [], 'the hookless trial is not recorded as data');+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+ });+test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => {const root = tempDir('gg-eff-cli-');const out = path.join(root, 'out');diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.jsindex 1982ebb64..a0ea86f99 100644--- a/tests/docker/gateguard-intent.test.js+++ b/tests/docker/gateguard-intent.test.js@@ -177,6 +177,33 @@ test('every grader separates the start, the trap and a correct solution', () =>}});+test('the CSV grader rejects the right dialect around the wrong values', () => {+ const lib = require(path.join(DIR, 'lib'));+ const workRoot = tempDir('gg-decoy-');+ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');+ const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort();+ assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']);+ for (const decoy of decoys) {+ const dest = path.join(workRoot, decoy);+ lib.prepareWorkspace(csv, dest, decoy);+ assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`);+ }+ const { scores, sound } = intentEval.graderIsSound(csv, workRoot);+ assert.ok(sound, JSON.stringify(scores));+ for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`);+});++test('a grader that passes one of its decoys is not sound', () => {+ const workRoot = tempDir('gg-decoy-pass-');+ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');+ const root = path.join(workRoot, 'scenario');+ fs.cpSync(csv.root, root, { recursive: true });+ fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true });+ const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot);+ assert.strictEqual(scores['decoy-reference'], 1);+ assert.strictEqual(sound, false);+});+// --- behaviour is classified into the right hole ---test('classify maps each behaviour to its hole', () => {@@ -515,50 +542,74 @@ test('session cleanup preserves concurrent and pre-existing folders under custom}});-test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));+/**+ * A committed copy of the harness whose provider is fake-claude.js, so the actual+ * runner can be driven end to end. `fakeEnv` reaches the fake provider only.+ */+function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) {const repo = path.join(root, 'repo');const copied = path.join(repo, 'docker', 'gateguard-effectiveness');const out = path.join(root, 'out');const preload = path.join(root, 'preload.js');const calls = path.join(root, 'calls.log');const config = path.join(root, 'claude-config');+ fs.mkdirSync(repo);+ fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });+ fs.cpSync(DIR, copied, { recursive: true });+ fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });+ fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));+ fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));+ const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',+ NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),+ GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: '[email protected]', GIT_COMMITTER_EMAIL: '[email protected]' };+ for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {+ const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });+ assert.strictEqual(result.status, 0, result.stderr);+ }+ const scenario = intentEval.loadIntentScenarios()[0];+ const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');+ fs.writeFileSync(preload, `+ const cp = require('child_process');+ const fs = require('fs');+ const path = require('path');+ const spawn = cp.spawnSync;+ cp.spawnSync = (file, args, options) => {+ if (file !== 'ecc-intent-fake') return spawn(file, args, options);+ fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');+ if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);+ const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');+ const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');+ for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {+ fs.mkdirSync(path.join(projects, name), { recursive: true });+ fs.writeFileSync(path.join(projects, name, 'sentinel'), name);+ }+ const fakeEnv = { ...options.env, ...${JSON.stringify(fakeEnv)}, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} };+ return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: fakeEnv });+ };+ `);+ const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', arms, '--scenarios', scenario.id, '--reps', '1'];+ const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });+ return { copied, out, calls, config, scenario, run };+}++test('actual runner stops before recording a gated trial whose hook never ran', () => {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-'));try {- fs.mkdirSync(repo);- fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });- fs.cpSync(DIR, copied, { recursive: true });- fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });- fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));- fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));- const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',- NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),- GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: '[email protected]', GIT_COMMITTER_EMAIL: '[email protected]' };- for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {- const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });- assert.strictEqual(result.status, 0, result.stderr);- }- const scenario = intentEval.loadIntentScenarios()[0];- const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');- fs.writeFileSync(preload, `- const cp = require('child_process');- const fs = require('fs');- const path = require('path');- const spawn = cp.spawnSync;- cp.spawnSync = (file, args, options) => {- if (file !== 'ecc-intent-fake') return spawn(file, args, options);- fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');- if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);- const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');- const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');- for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {- fs.mkdirSync(path.join(projects, name), { recursive: true });- fs.writeFileSync(path.join(projects, name, 'sentinel'), name);- }- return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: { ...options.env, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} } });- };- `);- const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', 'off,gate', '--scenarios', scenario.id, '--reps', '1'];- const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });+ // The runner needs a comparator arm; the ungated off trial is valid either way.+ const { out, run } = syntheticIntentRunner(root, { fakeEnv: { FAKE_CLAUDE_SKIP_HOOK: '1' } });+ const result = run();+ assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');+ assert.match(result.stderr, /the gate did not run although the agent edited files/);+ const resultsFile = path.join(out, 'results.jsonl');+ const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : [];+ assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data');+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+});++test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));+ try {+ const { copied, out, calls, config, run } = syntheticIntentRunner(root);const first = run();assert.strictEqual(first.status, 0, first.stderr);const meta = JSON.parse(fs.readFileSync(path.join(out, 'meta.json'), 'utf8'));