Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(gateguard): a throwing hook no longer ends the fresh-process replay

montjeffrey3a4a574609a7
diff --git a/scripts/dev/gateguard-eval.js b/scripts/dev/gateguard-eval.js
index 927272469..1500b8693 100644
--- a/scripts/dev/gateguard-eval.js
+++ b/scripts/dev/gateguard-eval.js
@@ -309,13 +309,22 @@ async function runCorpus(hookFile, scenarios) {
// --- cold latency ---
// see docs/gateguard/evaluation.md#latency
+// A hook that throws is recorded the way workerMain records it - the raw payload
+// stands in for the result, so the step reads as an allow - rather than ending the
+// run. Otherwise one throwing step aborted the whole replay and no report was written.
const COLD_PROBE = [
"const { performance } = require('perf_hooks');",
'const started = performance.now();',
- 'const hook = require(process.argv[1]);',
- 'const result = hook.run(process.argv[2]);',
+ 'let result;',
+ 'let error = null;',
+ 'try {',
+ ' result = require(process.argv[1]).run(process.argv[2]);',
+ '} catch (thrown) {',
+ ' error = String(thrown && thrown.message ? thrown.message : thrown);',
+ ' result = process.argv[2];',
+ '}',
'const ms = performance.now() - started;',
- 'process.stdout.write(JSON.stringify({ ms, result: result === undefined ? null : result }));'
+ 'process.stdout.write(JSON.stringify({ ms, result: result === undefined ? null : result, error }));'
].join('\n');
/** Replay every scenario with one fresh Node process per step; returns per-step require+run latency and decisions. */
@@ -331,15 +340,27 @@ function measureColdLatency(hookFile, scenarios) {
fs.appendFileSync(prepared.transcript, step.records.map(record => `${JSON.stringify(record)}\n`).join(''), 'utf8');
}
const raw = JSON.stringify(step.payload);
- const stdout = execFileSync(process.execPath, ['-e', COLD_PROBE, hookFile, raw], {
- env: prepared.env,
- encoding: 'utf8',
- maxBuffer: 16 * 1024 * 1024,
- stdio: ['ignore', 'pipe', 'ignore']
+ let probe;
+ try {
+ probe = JSON.parse(execFileSync(process.execPath, ['-e', COLD_PROBE, hookFile, raw], {
+ env: prepared.env,
+ encoding: 'utf8',
+ maxBuffer: 16 * 1024 * 1024,
+ stdio: ['ignore', 'pipe', 'ignore']
+ }));
+ } catch (failed) {
+ // The process died or wrote nothing parseable, so there is no latency to
+ // report; the step still counts, as an allow, matching a throwing hook.
+ probe = { ms: null, result: raw, error: String(failed && failed.message ? failed.message : failed) };
+ }
+ const output = hookOutput(probe.result);
+ steps.push({
+ scenario: scenario.name,
+ step: step.id,
+ latencyMs: probe.ms,
+ decision: output.decision === 'deny' ? 'deny' : 'allow',
+ error: probe.error || null
});
- const { ms, result } = JSON.parse(stdout);
- const output = hookOutput(result);
- steps.push({ scenario: scenario.name, step: step.id, latencyMs: ms, decision: output.decision === 'deny' ? 'deny' : 'allow' });
}
}
return steps;
@@ -349,7 +370,8 @@ function measureColdLatency(hookFile, scenarios) {
}
function coldSummary(coldSteps, warmSteps) {
- const sorted = coldSteps.map(step => step.latencyMs).sort((a, b) => a - b);
+ // A step whose process died has no latency; it still counts toward disagreements.
+ const sorted = coldSteps.map(step => step.latencyMs).filter(Number.isFinite).sort((a, b) => a - b);
const warmDecisions = new Map(warmSteps.map(step => [`${step.scenario}/${step.step}`, step.decision]));
const disagreements = coldSteps.filter(step => warmDecisions.get(`${step.scenario}/${step.step}`) !== step.decision).length;
return {
diff --git a/tests/scripts/gateguard-eval.test.js b/tests/scripts/gateguard-eval.test.js
index 593cd87b2..dce12aab7 100644
--- a/tests/scripts/gateguard-eval.test.js
+++ b/tests/scripts/gateguard-eval.test.js
@@ -188,6 +188,32 @@ async function coldTests() {
test('fresh processes decide every step as the worker does', () => {
assert.deepStrictEqual(cold.map(step => step.decision), warm.steps.map(step => step.decision));
});
+
+ // A hook that throws used to end the fresh-process pass, so neither the report
+ // nor the SARIF output was written for a run that had hit one bad step.
+ const throwingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-eval-throw-'));
+ try {
+ const throwingHook = path.join(throwingDir, 'throwing-hook.js');
+ fs.writeFileSync(throwingHook, "'use strict';\nmodule.exports = { run() { throw new Error('boom'); } };\n");
+ const warmThrow = summarize(await runCorpus(throwingHook, scenarios));
+ let coldThrow = null;
+ let escaped = null;
+ try {
+ coldThrow = measureColdLatency(throwingHook, scenarios);
+ } catch (error) {
+ escaped = error;
+ }
+ test('a hook that throws does not end the fresh-process pass', () => {
+ assert.strictEqual(escaped, null, escaped && escaped.message);
+ assert.strictEqual(coldThrow.length, scenarios[0].steps.length);
+ assert.ok(coldThrow.every(step => step.error === 'boom'), JSON.stringify(coldThrow.map(step => step.error)));
+ });
+ test('a throwing step is decided as the worker decides it, so it is no disagreement', () => {
+ assert.deepStrictEqual(coldThrow.map(step => step.decision), warmThrow.steps.map(step => step.decision));
+ });
+ } finally {
+ fs.rmSync(throwingDir, { recursive: true, force: true });
+ }
} finally {
fs.rmSync(corpusDir, { recursive: true, force: true });
}