Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(eval): preserve session ownership and frozen intent inputs

haelyra19a1b62f35de
diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md
index 46b3213b5..e4aed748a 100644
--- a/docker/gateguard-effectiveness/README.md
+++ b/docker/gateguard-effectiveness/README.md
@@ -38,8 +38,16 @@ node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize
node docker/gateguard-effectiveness/run-intent.js --out gg-intent --verify
```
-Runs sharing an output directory are serialized by `.run.lock`; concurrent workers
+The one-shot `run.js` runner serializes an output directory with `.run.lock`; concurrent workers
must use separate `--out` directories. Normal completion and errors release the
lock. After a killed process, confirm that its worker has stopped before removing
the stale lock and resuming. Metadata uses private exclusive temporary creation
and atomic rename so a destination symlink cannot redirect metadata writes.
+
+Hidden-intent trials retain the selected Claude configuration directory and pass
+`--claude` to both the agent and simulated user. Session cleanup removes only new
+project folders matching workspaces observed inside that trial's temporary root;
+pre-existing folders and concurrent runs remain. Shared project-name overrides
+remain untouched because they do not establish exclusive trial ownership.
+`arm-patch.js` is included in frozen harness fingerprints, so changes to ablations
+require a new output directory rather than continuing an existing experiment.
diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.js
index 5ea37e4b8..75b6982c8 100644
--- a/docker/gateguard-effectiveness/lib.js
+++ b/docker/gateguard-effectiveness/lib.js
@@ -11,7 +11,7 @@ const { ARMS, TRIAL_FILE, trialPatch } = require('./arms');
const SCENARIO_DIR = path.join(__dirname, 'scenarios');
const DOT_PREFIX = 'dot-';
const AUTONOMY_NOTE = 'Work autonomously in this repository and do not ask clarifying questions.';
-const PASSTHROUGH_ENV = new Set(['CLAUDE_CODE_OAUTH_TOKEN', 'CLAUDE_CODE_GIT_BASH_PATH']);
+const PASSTHROUGH_ENV = new Set(['CLAUDE_CODE_OAUTH_TOKEN', 'CLAUDE_CODE_GIT_BASH_PATH', 'CLAUDE_CONFIG_DIR']);
const SCRUBBED_ENV = /^(CLAUDE_|ECC_|GATEGUARD_)/;
// --- scenarios ---
diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.js
index 4bdfe90f8..b828af9cb 100644
--- a/docker/gateguard-effectiveness/run-intent.js
+++ b/docker/gateguard-effectiveness/run-intent.js
@@ -28,6 +28,7 @@ const USAGE = [
' --model <model> Claude model under test (required unless --dry-run or --summarize)',
' --allow-real-provider run real Claude sessions (each trial is several billed turns)',
' --judge-model <model> model playing the user (default: haiku)',
+ ' --claude <executable> CLI executable for both agent and simulated user',
` --arms <a,b,...> ${SUPPORTED_ARMS.join(', ')} (default: off,gate,placebo)`,
' --scenarios <a,b,...> scenario ids (default: all)',
' --reps <n> repetitions per scenario and arm (default: 3)',
@@ -126,23 +127,38 @@ function completedTrialKeys(rows) {
// would isolate those but also hides the credentials, so they are removed here
// instead. Only folders this run created, named after its own temp work root,
// are touched.
-function sessionsDir() {
- const home = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude');
+function sessionsDir(env = process.env) {
+ const home = env.CLAUDE_CONFIG_DIR || path.join(env.HOME || env.USERPROFILE || os.homedir(), '.claude');
return path.join(home, 'projects');
}
-function listSessions() {
+function listSessions(env = process.env) {
try {
- return new Set(fs.readdirSync(sessionsDir()));
+ return new Set(fs.readdirSync(sessionsDir(env)));
} catch (_) {
return new Set();
}
}
-function removeTrialSessions(before) {
- const dir = sessionsDir();
- for (const entry of listSessions()) {
- if (before.has(entry) || !entry.includes('gateguard-intent')) continue;
+function projectSessionName(workspace) {
+ const resolved = path.resolve(workspace);
+ const encoded = resolved.replace(/[^A-Za-z0-9]/g, '-');
+ if (encoded.length <= 200) return encoded;
+ // Match the official Claude Agent SDK0.3.296 project storage key, including
+ // its long-path suffix. Shared provider project-name overrides are not owned.
+ let hash = 0;
+ for (let index = 0; index < resolved.length; index += 1) {
+ hash = ((hash << 5) - hash + resolved.charCodeAt(index)) | 0;
+ }
+ return `${encoded.slice(0, 200)}-${Math.abs(hash).toString(36)}`;
+}
+
+function removeTrialSessions(before, ownedWorkspaces, env = process.env) {
+ if (env.CLAUDE_CODE_PROJECT_DIR_NAME) return;
+ const dir = sessionsDir(env);
+ const ownedNames = new Set([...ownedWorkspaces].map(projectSessionName));
+ for (const entry of listSessions(env)) {
+ if (before.has(entry) || !ownedNames.has(entry)) continue;
fs.rmSync(path.join(dir, entry), { recursive: true, force: true });
}
}
@@ -245,7 +261,7 @@ function main() {
seed: options.seed, configuration,
scenarioFingerprints: Object.fromEntries(scenarios.map(scenario => [scenario.id, evidence.hashDirectory(scenario.root)])),
sourceFingerprints: Object.fromEntries([
- 'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'evidence.js',
+ 'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'arm-patch.js', 'evidence.js',
'../context-profiles/ai-eval-lib.js'
].map(name => [name, evidence.hashFile(path.resolve(__dirname, name))]))
};
@@ -303,22 +319,35 @@ function main() {
for (const [index, trial] of trials.entries()) {
const sessionsBefore = listSessions();
const transcript = [];
+ const ownedWorkspaces = new Set();
const execute = (file, args, spawnOptions) => {
+ if (spawnOptions.cwd) {
+ const cwd = path.resolve(spawnOptions.cwd);
+ const relative = path.relative(workRoot, cwd);
+ if (relative && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) {
+ ownedWorkspaces.add(cwd);
+ }
+ }
const result = spawnSync(file, args, spawnOptions);
if (result.stdout) transcript.push(result.stdout);
return result;
};
- const row = runIntentTrial(trial, {
- workRoot,
- armSettingsByName,
- executable: options.claude,
- model: options.model,
- judgeModel: options.judgeModel,
- maxTurns: options.maxTurns,
- timeoutMs: options.timeoutMin * 60000,
- userTurns: options.userTurns,
- execute
- });
+ let row;
+ try {
+ row = runIntentTrial(trial, {
+ workRoot,
+ armSettingsByName,
+ executable: options.claude,
+ model: options.model,
+ judgeModel: options.judgeModel,
+ maxTurns: options.maxTurns,
+ timeoutMs: options.timeoutMin * 60000,
+ userTurns: options.userTurns,
+ execute
+ });
+ } finally {
+ removeTrialSessions(sessionsBefore, ownedWorkspaces);
+ }
const attemptId = crypto.randomUUID();
const transcriptName = `${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`;
row.attemptId = attemptId;
@@ -329,7 +358,6 @@ function main() {
);
row.transcriptSha256 = evidence.hashFile(path.join(options.out, row.transcript));
fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
- removeTrialSessions(sessionsBefore);
// An unauthenticated or hookless run must not be recorded: graded as data it
// looks like a finding (the first such trial was filed as a coverage hole).
if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
@@ -363,4 +391,4 @@ if (require.main === module) {
}
}
-module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic };
+module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic, sessionsDir, listSessions, projectSessionName, removeTrialSessions };
diff --git a/docker/gateguard-effectiveness/session.js b/docker/gateguard-effectiveness/session.js
index ed96dc01e..0dc1e53f5 100644
--- a/docker/gateguard-effectiveness/session.js
+++ b/docker/gateguard-effectiveness/session.js
@@ -128,7 +128,7 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option
exchange.push({ role: 'agent', text: message });
if (turn === userTurns) break;
- const reply = judge(intent, message, { model: judgeModel, execute });
+ const reply = judge(intent, message, { executable, model: judgeModel, execute });
judgeFailed = judgeFailed || reply.judgeFailed;
if (!reply.isQuestion) break;
asked = true;
diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js
index 00cad5264..1982ebb64 100644
--- a/tests/docker/gateguard-intent.test.js
+++ b/tests/docker/gateguard-intent.test.js
@@ -17,6 +17,7 @@ const assert = require('assert');
const fs = require('fs');
const os = require('os');
const path = require('path');
+const { spawnSync } = require('child_process');
const ROOT = path.join(__dirname, '..', '..');
const DIR = path.join(ROOT, 'docker', 'gateguard-effectiveness');
@@ -24,7 +25,7 @@ const userSim = require(path.join(DIR, 'user-sim'));
const session = require(path.join(DIR, 'session'));
const coverage = require(path.join(DIR, 'coverage'));
const intentEval = require(path.join(DIR, 'intent-eval'));
-const { parseArgs, completedTrialKeys, writeAtomic } = require(path.join(DIR, 'run-intent'));
+const { parseArgs, completedTrialKeys, writeAtomic, sessionsDir, listSessions, projectSessionName, removeTrialSessions } = require(path.join(DIR, 'run-intent'));
const evidence = require(path.join(DIR, 'evidence'));
let passed = 0;
@@ -437,21 +438,170 @@ test('a timeout on an earlier turn is not cleared by a later clean turn', () =>
assert.strictEqual(result.timedOut, true);
});
+test('a selected custom executable runs both agent and simulated user with bare claude absent', () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-executable-'));
+ const fake = path.join(dir, 'fake.js');
+ const log = path.join(dir, 'calls.jsonl');
+ try {
+ fs.writeFileSync(fake, `
+ const fs = require('fs');
+ const args = process.argv.slice(2);
+ const model = args[args.indexOf('--model') + 1];
+ const agent = args.includes('--output-format');
+ fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify({ agent, model }) + '\\n');
+ if (agent) {
+ const text = args.includes('--resume') ? 'done' : 'how long is the window?';
+ console.log(JSON.stringify({ type: 'assistant', message: { content: [{ type: 'text', text }] } }));
+ console.log(JSON.stringify({ type: 'result', is_error: false, num_turns: 1, usage: {} }));
+ } else {
+ const text = fs.readFileSync(0, 'utf8');
+ const isQuestion = text.includes('how long is the window?');
+ console.log(JSON.stringify({ isQuestion, reply: isQuestion ? '400 days' : 'done', disclosed: isQuestion ? ['window'] : [] }));
+ }
+ `);
+ const execute = (file, args, options) => spawnSync(file, [fake, ...args], {
+ ...options, env: { ...options.env, PATH: '', HOME: dir, USERPROFILE: dir }
+ });
+ const missing = spawnSync('claude', ['--version'], { env: { ...process.env, PATH: '' }, encoding: 'utf8' });
+ assert.ok(missing.error && missing.error.code === 'ENOENT', 'bare claude is unavailable');
+ const result = session.runConversation(
+ { cwd: dir, stateDir: dir, settingsPath: path.join(dir, 'settings.json'), prompt: 'p', intent: INTENT },
+ { executable: process.execPath, model: 'selected-agent', judgeModel: 'selected-judge', maxTurns: 2, timeoutMs: 10000, userTurns: 2, execute }
+ );
+ assert.strictEqual(result.judgeFailed, false);
+ assert.strictEqual(result.providerError, false);
+ assert.strictEqual(result.userTurns, 1);
+ assert.deepStrictEqual(fs.readFileSync(log, 'utf8').trim().split('\n').map(line => JSON.parse(line)), [
+ { agent: true, model: 'selected-agent' }, { agent: false, model: 'selected-judge' },
+ { agent: true, model: 'selected-agent' }, { agent: false, model: 'selected-judge' }
+ ]);
+ } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('session cleanup preserves concurrent and pre-existing folders under custom config and HOME', () => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-cleanup-'));
+ const saved = { HOME: process.env.HOME, CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR };
+ try {
+ for (const env of [{ HOME: path.join(root, 'home') }, { HOME: path.join(root, 'home'), CLAUDE_CONFIG_DIR: path.join(root, 'config') }]) {
+ process.env.HOME = env.HOME;
+ if (env.CLAUDE_CONFIG_DIR) process.env.CLAUDE_CONFIG_DIR = env.CLAUDE_CONFIG_DIR;
+ else delete process.env.CLAUDE_CONFIG_DIR;
+ const dir = sessionsDir(env);
+ fs.mkdirSync(dir, { recursive: true });
+ const workRoot = path.join(root, 'gateguard-intent-OWNED');
+ const workspace = path.join(workRoot, 'date__gate__1', 'repo');
+ const own = projectSessionName(workspace);
+ const preexisting = projectSessionName(path.join(workRoot, 'date__off__1', 'repo'));
+ const concurrent = projectSessionName(path.join(root, 'gateguard-intent-OTHER', 'date__gate__1', 'repo'));
+ const make = name => { fs.mkdirSync(path.join(dir, name)); fs.writeFileSync(path.join(dir, name, 'sentinel'), name); };
+ make(preexisting);
+ const before = listSessions(env);
+ for (const name of [own, concurrent, `${own}-other`]) make(name);
+ removeTrialSessions(before, new Set([workspace, path.join(workRoot, 'date__off__1', 'repo')]), env);
+ assert.ok(!fs.existsSync(path.join(dir, own)), 'exact owned folder removed');
+ for (const name of [preexisting, concurrent, `${own}-other`]) {
+ assert.strictEqual(fs.readFileSync(path.join(dir, name, 'sentinel'), 'utf8'), name);
+ }
+ make(own);
+ removeTrialSessions(before, new Set([workspace]), { ...env, CLAUDE_CODE_PROJECT_DIR_NAME: own });
+ assert.strictEqual(fs.readFileSync(path.join(dir, own, 'sentinel'), 'utf8'), own, 'shared provider name override stays untouched');
+ }
+ } finally {
+ for (const [key, value] of Object.entries(saved)) {
+ if (value === undefined) delete process.env[key];
+ else process.env[key] = value;
+ }
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));
+ const repo = path.join(root, 'repo');
+ const copied = path.join(repo, 'docker', 'gateguard-effectiveness');
+ const out = path.join(root, 'out');
+ const preload = path.join(root, 'preload.js');
+ const calls = path.join(root, 'calls.log');
+ const config = path.join(root, 'claude-config');
+ try {
+ fs.mkdirSync(repo);
+ fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });
+ fs.cpSync(DIR, copied, { recursive: true });
+ fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });
+ fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));
+ fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));
+ const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',
+ NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),
+ GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: '[email protected]', GIT_COMMITTER_EMAIL: '[email protected]' };
+ for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {
+ const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });
+ assert.strictEqual(result.status, 0, result.stderr);
+ }
+ const scenario = intentEval.loadIntentScenarios()[0];
+ const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');
+ fs.writeFileSync(preload, `
+ const cp = require('child_process');
+ const fs = require('fs');
+ const path = require('path');
+ const spawn = cp.spawnSync;
+ cp.spawnSync = (file, args, options) => {
+ if (file !== 'ecc-intent-fake') return spawn(file, args, options);
+ fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');
+ if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);
+ const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');
+ const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');
+ for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {
+ fs.mkdirSync(path.join(projects, name), { recursive: true });
+ fs.writeFileSync(path.join(projects, name, 'sentinel'), name);
+ }
+ return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: { ...options.env, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} } });
+ };
+ `);
+ const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', 'off,gate', '--scenarios', scenario.id, '--reps', '1'];
+ const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });
+ const first = run();
+ assert.strictEqual(first.status, 0, first.stderr);
+ const meta = JSON.parse(fs.readFileSync(path.join(out, 'meta.json'), 'utf8'));
+ const patch = path.join(copied, 'arm-patch.js');
+ assert.strictEqual(meta.sourceFingerprints['arm-patch.js'], evidence.hashFile(patch));
+ assert.deepStrictEqual(fs.readdirSync(path.join(config, 'projects')), ['concurrent-gateguard-intent-OTHER'], 'concurrent folder survives exact cleanup');
+ const results = fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8');
+ const callsBefore = fs.readFileSync(calls, 'utf8');
+ const resumed = run();
+ assert.strictEqual(resumed.status, 0, resumed.stderr);
+ assert.ok(resumed.stderr.includes('0 trials to run'));
+ assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore);
+ fs.appendFileSync(patch, '\n// Changed ablation implementation must invalidate resume.\n');
+ const changed = run();
+ assert.notStrictEqual(changed.status, 0);
+ assert.match(changed.stderr, /different experiment configuration/);
+ assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore, 'changed input rejected before provider dispatch');
+ assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
+ } finally { fs.rmSync(root, { recursive: true, force: true }); }
+});
+
test('the config directory is never isolated, because that hides the credentials', () => {
// A per-trial CLAUDE_CONFIG_DIR looks like the right way to stop persisted
// sessions piling up, but it makes every turn return "Not logged in", and the
// trial is then graded as a coverage hole. Guard against reintroducing it.
- let seen = 'unset';
- const execute = (file, args, options) => {
- seen = options.env.CLAUDE_CONFIG_DIR;
- return streamFor('done');
- };
- const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false });
- session.runConversation(
- { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
- { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge }
- );
- assert.strictEqual(seen, process.env.CLAUDE_CONFIG_DIR);
+ const previous = process.env.CLAUDE_CONFIG_DIR;
+ process.env.CLAUDE_CONFIG_DIR = path.join(os.tmpdir(), 'explicit-claude-config-fixture');
+ try {
+ let seen = 'unset';
+ const execute = (file, args, options) => {
+ seen = options.env.CLAUDE_CONFIG_DIR;
+ return streamFor('done');
+ };
+ const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false });
+ session.runConversation(
+ { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
+ { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge }
+ );
+ assert.strictEqual(seen, process.env.CLAUDE_CONFIG_DIR);
+ } finally {
+ if (previous === undefined) delete process.env.CLAUDE_CONFIG_DIR;
+ else process.env.CLAUDE_CONFIG_DIR = previous;
+ }
});
test('a missing or errored result is a provider error, not a trial', () => {