Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix: integrate truthful GateGuard harness grading and retry evidence

haelyra344c1fc3fa73
diff --cc docker/gateguard-effectiveness/README.md
index 646446118,33e703384..c8886fea0
--- a/docker/gateguard-effectiveness/README.md
+++ b/docker/gateguard-effectiveness/README.md
@@@ -33,6 -33,14 +33,14 @@@ scenario and harness inputs and hashes
directory with `--verify`; this detects later file changes but is not a
cryptographic signature or proof of who ran the experiment.
-`--check-graders`, and every run before it bills a session, requires each
++`run-intent.js --check-graders`, and each intent run before it bills a session, requires each
+ grader to score `reference` 1 and the start workspace, the `naive` trap and every
+ `decoy-*` overlay below 1. A decoy has the right shape and the wrong content, so
+ a grader that checks only the form fails the check. Both runners count only valid
+ trials: an attempt that ended in a provider error or timeout stays in
+ `results.jsonl` with its own transcript, is reported apart, and is retried on
+ resume. A gated trial whose hook never ran stops the run and is not recorded.
+
```bash
node docker/gateguard-effectiveness/run-intent.js --check-graders
node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run
diff --cc docker/gateguard-effectiveness/lib.js
index 75b6982c8,330d0876f..b166741f3
--- a/docker/gateguard-effectiveness/lib.js
+++ b/docker/gateguard-effectiveness/lib.js
@@@ -374,8 -374,20 +374,20 @@@ function pairedComparison(rows, referen
};
}
+ /** A trial counts only when the provider, the clock and the judge all completed. */
+ function isValidTrial(row) {
+ return !row.providerError && !row.timedOut && !row.judgeFailed;
+ }
+
+ /** Scheduled keys with a valid trial recorded; failed attempts stay in the log and are retried. */
+ function completedTrialKeys(rows) {
+ return new Set(rows.filter(isValidTrial).map(row => row.key));
+ }
+
-// Totals cover valid trials only; failed attempts are counted apart as providerErrors.
++// Totals cover valid trials only; provider failures and timeouts are counted apart.
function armSummary(rows, arm) {
- const own = rows.filter(row => row.arm === arm);
+ const attempts = rows.filter(row => row.arm === arm);
+ const own = attempts.filter(isValidTrial);
const passes = own.filter(row => row.passed).length;
return {
arm,
@@@ -390,7 -402,7 +402,7 @@@
medianWallMs: median(own.map(row => row.wallMs)),
medianDenials: median(own.map(row => row.gateDenials)),
meanEvidenceRecall: own.length ? own.reduce((sum, row) => sum + row.evidenceRecall, 0) / own.length : null,
- providerErrors: own.filter(row => row.providerError || row.timedOut).length
- providerErrors: attempts.filter(row => !isValidTrial(row)).length
++ providerErrors: attempts.filter(row => row.providerError || row.timedOut).length
};
}
diff --cc docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
index c010c214c,37129807c..852fa0ca0
--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
@@@ -8,13 -19,15 +19,16 @@@ let checks = []
try {
const { exportCsv } = require(path.resolve('src/export.js'));
if (typeof exportCsv !== 'function') throw new Error('no exportCsv');
- const out = exportCsv([{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]);
+ const out = exportCsv(ROWS);
if (typeof out !== 'string') throw new Error('not a string');
-- const lines = out.split('\r\n').filter(Boolean);
++ const lines = out.split('\r\n');
++ if (lines.at(-1) === '') lines.pop();
+ const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));
checks.push(out.includes('\r\n'));
checks.push(lines.length >= 2 && lines.every(line => line.includes(';')));
- checks.push(lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)));
+ checks.push(quoted);
+ checks.push(quoted && valuesMatch(lines));
} catch (_) {
- checks = [false, false, false];
+ checks = new Array(CHECKS).fill(false);
}
- console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / 3 })}`);
+ console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / CHECKS })}`);
diff --cc tests/docker/gateguard-effectiveness.test.js
index 143844b0e,c18663116..68a5834af
--- a/tests/docker/gateguard-effectiveness.test.js
+++ b/tests/docker/gateguard-effectiveness.test.js
@@@ -317,7 -317,83 +317,84 @@@ if (gnuTarDir)
});
}
+ test('pilot summaries count valid trials only and report provider errors apart', () => {
+ const row = (arm, rep, passed, extra = {}) => ({
+ key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,
+ inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false, ...extra
+ });
+ const rows = [
+ row('gate', 1, true),
+ row('gate', 2, false, { providerError: true }),
+ row('gate', 2, true),
+ row('gate', 3, false, { timedOut: true }),
++ row('gate', 4, false, { judgeFailed: true }),
+ row('off', 1, true)
+ ];
+ const gate = lib.summarize(rows, ['gate', 'off']).arms.find(arm => arm.arm === 'gate');
+ assert.strictEqual(gate.trials, 2, 'a provider error or timeout is not a trial');
+ assert.strictEqual(gate.passes, 2);
+ assert.strictEqual(gate.meanScore, 1);
+ assert.strictEqual(gate.providerErrors, 2);
+ assert.deepStrictEqual([...lib.completedTrialKeys(rows)].sort(), ['s/gate/1', 's/gate/2', 's/off/1']);
+ });
+
+ /** The pilot runner driven end to end with fake-claude.js as the provider. */
+ function syntheticPilotRunner(root, scenario, arms) {
+ const out = path.join(root, 'out');
+ const preload = path.join(root, 'preload.js');
+ fs.writeFileSync(preload, `
+ const cp = require('child_process');
+ const originalSpawn = cp.spawnSync;
+ cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'
+ ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)
+ : originalSpawn(file, args, options);
+ `);
+ const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];
+ const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };
+ const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });
+ const results = () => {
+ const file = path.join(out, 'results.jsonl');
+ return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
+ };
+ return { out, run, results };
+ }
+
if (gitAvailable()) {
+ test('a pilot trial that ended in a provider error is retried on resume', () => {
+ const root = tempDir('gg-eff-retry-');
+ try {
+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+ const { out, run, results } = syntheticPilotRunner(root, scenario, 'off');
+ const failed = run({ FAKE_CLAUDE_PROVIDER_ERROR: '1' });
+ assert.strictEqual(failed.status, 0, failed.stderr);
+ assert.deepStrictEqual(results().map(row => row.providerError), [true]);
+ const retried = run({});
+ assert.strictEqual(retried.status, 0, retried.stderr);
+ assert.ok(retried.stderr.includes('1 trials to run'), retried.stderr);
+ const rows = results();
+ assert.deepStrictEqual(rows.map(row => row.providerError), [true, false], 'the failed attempt stays for audit');
+ assert.strictEqual(rows[1].passed, true);
+ assert.notStrictEqual(rows[0].transcript, rows[1].transcript, 'each attempt keeps its own transcript');
+ const { summary } = JSON.parse(fs.readFileSync(path.join(out, 'summary.json'), 'utf8'));
+ const off = summary.arms.find(arm => arm.arm === 'off');
+ assert.deepStrictEqual([off.trials, off.passes, off.providerErrors], [1, 1, 1]);
+ const again = run({});
+ assert.ok(again.stderr.includes('0 trials to run'), again.stderr);
+ } finally { fs.rmSync(root, { recursive: true, force: true }); }
+ });
+
+ test('the pilot runner stops before recording a gated trial whose hook never ran', () => {
+ const root = tempDir('gg-eff-hookless-');
+ try {
+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+ const { run, results } = syntheticPilotRunner(root, scenario, 'gate');
+ const result = run({ FAKE_CLAUDE_SKIP_HOOK: '1' });
+ assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');
+ assert.match(result.stderr, /the gate did not run although the agent edited files/);
+ assert.deepStrictEqual(results(), [], 'the hookless trial is not recorded as data');
+ } finally { fs.rmSync(root, { recursive: true, force: true }); }
+ });
+
test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => {
const root = tempDir('gg-eff-cli-');
const out = path.join(root, 'out');
diff --cc tests/docker/gateguard-intent.test.js
index 1982ebb64,a0ea86f99..5616c156c
--- a/tests/docker/gateguard-intent.test.js
+++ b/tests/docker/gateguard-intent.test.js
@@@ -177,6 -177,33 +177,58 @@@ test('every grader separates the start
}
});
+ test('the CSV grader rejects the right dialect around the wrong values', () => {
+ const lib = require(path.join(DIR, 'lib'));
+ const workRoot = tempDir('gg-decoy-');
- const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
- const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort();
- assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']);
- for (const decoy of decoys) {
- const dest = path.join(workRoot, decoy);
- lib.prepareWorkspace(csv, dest, decoy);
- assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`);
- }
- const { scores, sound } = intentEval.graderIsSound(csv, workRoot);
- assert.ok(sound, JSON.stringify(scores));
- for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`);
++ try {
++ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
++ const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort();
++ assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']);
++ for (const decoy of decoys) {
++ const dest = path.join(workRoot, decoy);
++ lib.prepareWorkspace(csv, dest, decoy);
++ assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`);
++ }
++ const { scores, sound } = intentEval.graderIsSound(csv, workRoot);
++ assert.ok(sound, JSON.stringify(scores));
++ for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`);
++ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+ });
+
+ test('a grader that passes one of its decoys is not sound', () => {
+ const workRoot = tempDir('gg-decoy-pass-');
- const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
- const root = path.join(workRoot, 'scenario');
- fs.cpSync(csv.root, root, { recursive: true });
- fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true });
- const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot);
- assert.strictEqual(scores['decoy-reference'], 1);
- assert.strictEqual(sound, false);
++ try {
++ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
++ const root = path.join(workRoot, 'scenario');
++ fs.cpSync(csv.root, root, { recursive: true });
++ fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true });
++ const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot);
++ assert.strictEqual(scores['decoy-reference'], 1);
++ assert.strictEqual(sound, false);
++ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
++});
++
++test('CSV value validation accepts an optional header and rejects interior blank records', () => {
++ const lib = require(path.join(DIR, 'lib'));
++ const workRoot = tempDir('gg-csv-records-');
++ try {
++ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
++ const dest = path.join(workRoot, 'workspace');
++ lib.prepareWorkspace(csv, dest);
++ for (const [output, expected] of [
++ ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n', 1],
++ ['"name";"city"\r\n"Ada";"Paris"\r\n"Bo";"Rome"', 1],
++ ['"Ada";"Paris"\r\n\r\n"Bo";"Rome"\r\n', 0],
++ ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n\r\n', 0]
++ ]) {
++ fs.writeFileSync(path.join(dest, 'src/export.js'), `exports.exportCsv = () => ${JSON.stringify(output)};\n`);
++ const score = lib.grade(csv, dest).score;
++ if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(output));
++ else assert.ok(score < 1, JSON.stringify(output));
++ }
++ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+ });
+
// --- behaviour is classified into the right hole ---
test('classify maps each behaviour to its hole', () => {