The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(gateguard): a throwing hook no longer ends the fresh-process replay
montjeffrey3a4a574609a7
diff --git a/scripts/dev/gateguard-eval.js b/scripts/dev/gateguard-eval.jsindex 927272469..1500b8693 100644--- a/scripts/dev/gateguard-eval.js+++ b/scripts/dev/gateguard-eval.js@@ -309,13 +309,22 @@ async function runCorpus(hookFile, scenarios) {// --- cold latency ---// see docs/gateguard/evaluation.md#latency+// A hook that throws is recorded the way workerMain records it - the raw payload+// stands in for the result, so the step reads as an allow - rather than ending the+// run. Otherwise one throwing step aborted the whole replay and no report was written.const COLD_PROBE = ["const { performance } = require('perf_hooks');",'const started = performance.now();',- 'const hook = require(process.argv[1]);',- 'const result = hook.run(process.argv[2]);',+ 'let result;',+ 'let error = null;',+ 'try {',+ ' result = require(process.argv[1]).run(process.argv[2]);',+ '} catch (thrown) {',+ ' error = String(thrown && thrown.message ? thrown.message : thrown);',+ ' result = process.argv[2];',+ '}','const ms = performance.now() - started;',- 'process.stdout.write(JSON.stringify({ ms, result: result === undefined ? null : result }));'+ 'process.stdout.write(JSON.stringify({ ms, result: result === undefined ? null : result, error }));'].join('\n');/** Replay every scenario with one fresh Node process per step; returns per-step require+run latency and decisions. */@@ -331,15 +340,27 @@ function measureColdLatency(hookFile, scenarios) {fs.appendFileSync(prepared.transcript, step.records.map(record => `${JSON.stringify(record)}\n`).join(''), 'utf8');}const raw = JSON.stringify(step.payload);- const stdout = execFileSync(process.execPath, ['-e', COLD_PROBE, hookFile, raw], {- env: prepared.env,- encoding: 'utf8',- maxBuffer: 16 * 1024 * 1024,- stdio: ['ignore', 'pipe', 'ignore']+ let probe;+ try {+ probe = JSON.parse(execFileSync(process.execPath, ['-e', COLD_PROBE, hookFile, raw], {+ env: prepared.env,+ encoding: 'utf8',+ maxBuffer: 16 * 1024 * 1024,+ stdio: ['ignore', 'pipe', 'ignore']+ }));+ } catch (failed) {+ // The process died or wrote nothing parseable, so there is no latency to+ // report; the step still counts, as an allow, matching a throwing hook.+ probe = { ms: null, result: raw, error: String(failed && failed.message ? failed.message : failed) };+ }+ const output = hookOutput(probe.result);+ steps.push({+ scenario: scenario.name,+ step: step.id,+ latencyMs: probe.ms,+ decision: output.decision === 'deny' ? 'deny' : 'allow',+ error: probe.error || null});- const { ms, result } = JSON.parse(stdout);- const output = hookOutput(result);- steps.push({ scenario: scenario.name, step: step.id, latencyMs: ms, decision: output.decision === 'deny' ? 'deny' : 'allow' });}}return steps;@@ -349,7 +370,8 @@ function measureColdLatency(hookFile, scenarios) {}function coldSummary(coldSteps, warmSteps) {- const sorted = coldSteps.map(step => step.latencyMs).sort((a, b) => a - b);+ // A step whose process died has no latency; it still counts toward disagreements.+ const sorted = coldSteps.map(step => step.latencyMs).filter(Number.isFinite).sort((a, b) => a - b);const warmDecisions = new Map(warmSteps.map(step => [`${step.scenario}/${step.step}`, step.decision]));const disagreements = coldSteps.filter(step => warmDecisions.get(`${step.scenario}/${step.step}`) !== step.decision).length;return {diff --git a/tests/scripts/gateguard-eval.test.js b/tests/scripts/gateguard-eval.test.jsindex 593cd87b2..dce12aab7 100644--- a/tests/scripts/gateguard-eval.test.js+++ b/tests/scripts/gateguard-eval.test.js@@ -188,6 +188,32 @@ async function coldTests() {test('fresh processes decide every step as the worker does', () => {assert.deepStrictEqual(cold.map(step => step.decision), warm.steps.map(step => step.decision));});++ // A hook that throws used to end the fresh-process pass, so neither the report+ // nor the SARIF output was written for a run that had hit one bad step.+ const throwingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-eval-throw-'));+ try {+ const throwingHook = path.join(throwingDir, 'throwing-hook.js');+ fs.writeFileSync(throwingHook, "'use strict';\nmodule.exports = { run() { throw new Error('boom'); } };\n");+ const warmThrow = summarize(await runCorpus(throwingHook, scenarios));+ let coldThrow = null;+ let escaped = null;+ try {+ coldThrow = measureColdLatency(throwingHook, scenarios);+ } catch (error) {+ escaped = error;+ }+ test('a hook that throws does not end the fresh-process pass', () => {+ assert.strictEqual(escaped, null, escaped && escaped.message);+ assert.strictEqual(coldThrow.length, scenarios[0].steps.length);+ assert.ok(coldThrow.every(step => step.error === 'boom'), JSON.stringify(coldThrow.map(step => step.error)));+ });+ test('a throwing step is decided as the worker decides it, so it is no disagreement', () => {+ assert.deepStrictEqual(coldThrow.map(step => step.decision), warmThrow.steps.map(step => step.decision));+ });+ } finally {+ fs.rmSync(throwingDir, { recursive: true, force: true });+ }} finally {fs.rmSync(corpusDir, { recursive: true, force: true });}