Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(gateguard): report fresh-process failures and fail the gate on them

montjeffrey6dbf6e6883ec
diff --git a/scripts/dev/gateguard-eval.js b/scripts/dev/gateguard-eval.js
index 1500b8693..261595600 100644
--- a/scripts/dev/gateguard-eval.js
+++ b/scripts/dev/gateguard-eval.js
@@ -370,14 +370,21 @@ function measureColdLatency(hookFile, scenarios) {
}
function coldSummary(coldSteps, warmSteps) {
- // A step whose process died has no latency; it still counts toward disagreements.
+ // A step whose process died has no latency; it still counts toward disagreements
+ // and errors. Failures are counted on their own: a failed step is recorded as an
+ // allow, so where the worker also allowed it, it is no disagreement and would
+ // otherwise vanish from the report and the exit status.
const sorted = coldSteps.map(step => step.latencyMs).filter(Number.isFinite).sort((a, b) => a - b);
const warmDecisions = new Map(warmSteps.map(step => [`${step.scenario}/${step.step}`, step.decision]));
const disagreements = coldSteps.filter(step => warmDecisions.get(`${step.scenario}/${step.step}`) !== step.decision).length;
+ // With no measured step there is no latency to report; 0 would read as a fast hook.
+ const at = fraction => (sorted.length ? Number(percentile(sorted, fraction).toFixed(3)) : null);
return {
- p50Ms: Number(percentile(sorted, 0.5).toFixed(3)),
- p90Ms: Number(percentile(sorted, 0.9).toFixed(3)),
- p95Ms: Number(percentile(sorted, 0.95).toFixed(3)),
+ p50Ms: at(0.5),
+ p90Ms: at(0.9),
+ p95Ms: at(0.95),
+ samples: sorted.length,
+ errors: coldSteps.filter(step => step.error).length,
disagreements
};
}
@@ -507,6 +514,12 @@ function summarize(runs, table = questionTextTable()) {
// --- reporting ---
+/** A fresh-process percentile, or why there is none: not run, or no step measured. */
+function coldMs(s, key) {
+ if (!s.cold) return 'not run';
+ return s.cold[key] === null ? 'no valid sample' : s.cold[key].toFixed(2);
+}
+
const METRIC_ROWS = [
['Steps', s => s.totals.steps],
['Denials', s => s.totals.denials],
@@ -520,9 +533,9 @@ const METRIC_ROWS = [
['Allows with a credit note', s => s.totals.allowsByKind.credit],
['Allows with a sibling note', s => s.totals.allowsByKind.sibling],
['Allows with a trivial-edit note', s => s.totals.allowsByKind.trivial],
- ['Hook latency p50, fresh process (ms)', s => (s.cold ? s.cold.p50Ms.toFixed(2) : 'not run')],
- ['Hook latency p90, fresh process (ms)', s => (s.cold ? s.cold.p90Ms.toFixed(2) : 'not run')],
- ['Hook latency p95, fresh process (ms)', s => (s.cold ? s.cold.p95Ms.toFixed(2) : 'not run')],
+ ['Hook latency p50, fresh process (ms)', s => coldMs(s, 'p50Ms')],
+ ['Hook latency p90, fresh process (ms)', s => coldMs(s, 'p90Ms')],
+ ['Hook latency p95, fresh process (ms)', s => coldMs(s, 'p95Ms')],
['run() latency p50, warm (ms)', s => s.latency.p50Ms.toFixed(2)],
['run() latency p90, warm (ms)', s => s.latency.p90Ms.toFixed(2)],
['run() latency p95, warm (ms)', s => s.latency.p95Ms.toFixed(2)]
@@ -563,6 +576,9 @@ function renderMarkdown(report) {
if (cold && cold.disagreements > 0) {
lines.push('', `${report.hooks[0].label}: ${cold.disagreements} step(s) decided differently in a fresh process.`);
}
+ if (cold && cold.errors > 0) {
+ lines.push('', `${report.hooks[0].label}: ${cold.errors} step(s) failed in a fresh process.`);
+ }
return `${lines.join('\n')}\n`;
}
@@ -732,11 +748,18 @@ async function main() {
const report = await evaluate(options);
process.stdout.write(options.format === 'json' ? `${JSON.stringify(report, null, 2)}\n` : renderMarkdown(report));
if (options.sarif) fs.writeFileSync(options.sarif, renderSarif(report), 'utf8');
+ if (gateFails(report)) process.exitCode = 1;
+}
+
+/**
+ * True when the working tree must not pass: a bypass, a mismatch, an explicit allow,
+ * a hook error in either pass, or a step a fresh process decided differently.
+ */
+function gateFails(report) {
const working = report.hooks[0].summary.totals;
- const coldDisagreements = report.hooks[0].summary.cold ? report.hooks[0].summary.cold.disagreements : 0;
- if (working.mustDenyBypasses > 0 || working.mismatches > 0 || working.explicitAllows > 0 || working.errors > 0 || coldDisagreements > 0) {
- process.exitCode = 1;
- }
+ const cold = report.hooks[0].summary.cold || {};
+ return working.mustDenyBypasses > 0 || working.mismatches > 0 || working.explicitAllows > 0 ||
+ working.errors > 0 || cold.disagreements > 0 || cold.errors > 0;
}
if (!isMainThread && workerData && workerData.gateguardEval) {
@@ -748,4 +771,7 @@ if (!isMainThread && workerData && workerData.gateguardEval) {
});
}
-module.exports = { loadCorpus, materializeHook, runCorpus, measureColdLatency, summarize, questionsAsked, renderMarkdown, renderSarif, parseArgs, evaluate };
+module.exports = {
+ loadCorpus, materializeHook, runCorpus, measureColdLatency, coldSummary, summarize, questionsAsked,
+ renderMarkdown, renderSarif, parseArgs, evaluate, gateFails
+};
diff --git a/tests/scripts/gateguard-eval.test.js b/tests/scripts/gateguard-eval.test.js
index dce12aab7..690c888e8 100644
--- a/tests/scripts/gateguard-eval.test.js
+++ b/tests/scripts/gateguard-eval.test.js
@@ -9,7 +9,9 @@ const fs = require('fs');
const os = require('os');
const path = require('path');
const { pathToFileURL } = require('url');
-const { parseArgs, renderSarif, loadCorpus, runCorpus, summarize, measureColdLatency } = require('../../scripts/dev/gateguard-eval');
+const {
+ parseArgs, renderSarif, renderMarkdown, loadCorpus, runCorpus, summarize, measureColdLatency, coldSummary, gateFails
+} = require('../../scripts/dev/gateguard-eval');
let passed = 0;
let failed = 0;
@@ -170,6 +172,46 @@ test('--no-cold turns off the fresh-process pass', () => {
assert.strictEqual(parseArgs(['--no-cold']).cold, false);
});
+// A fresh-process failure is recorded as an allow. Where the worker also allowed the
+// step it is no disagreement, so before failures were counted on their own it left
+// no trace in the report, the latency or the exit status.
+test('a fresh-process failure on a step the worker allows is still counted', () => {
+ const cold = coldSummary(
+ [{ scenario: 's', step: 'a', latencyMs: null, decision: 'allow', error: 'killed' }],
+ [{ scenario: 's', step: 'a', decision: 'allow' }]
+ );
+ assert.strictEqual(cold.disagreements, 0);
+ assert.strictEqual(cold.errors, 1);
+ assert.strictEqual(cold.samples, 0);
+ assert.strictEqual(cold.p50Ms, null, 'no measured step must not read as a 0 ms latency');
+});
+
+test('a fresh-process failure fails the gate when nothing else does', () => {
+ const clean = report([step()]);
+ clean.hooks[0].summary.cold = { p50Ms: 1, p90Ms: 1, p95Ms: 1, samples: 1, errors: 0, disagreements: 0 };
+ assert.strictEqual(gateFails(clean), false);
+ const failed = report([step()]);
+ failed.hooks[0].summary.cold = { p50Ms: null, p90Ms: null, p95Ms: null, samples: 0, errors: 1, disagreements: 0 };
+ assert.strictEqual(gateFails(failed), true);
+});
+
+test('the report names fresh-process failures and invents no latency', () => {
+ // summarize([]) is the real summary shape with every count at zero, so this keeps
+ // working as metrics are added, which a hand-built totals object would not.
+ const working = { ...summarize([]), cold: { p50Ms: null, p90Ms: null, p95Ms: null, samples: 0, errors: 1, disagreements: 0 } };
+ const failed = {
+ corpus: { scenarios: 0, steps: 0 },
+ hooks: [
+ { label: 'working tree', ref: null, summary: working },
+ { label: 'upstream/main', ref: 'upstream/main', summary: summarize([]) }
+ ]
+ };
+ const text = renderMarkdown(failed);
+ assert.match(text, /1 step\(s\) failed in a fresh process/);
+ assert.match(text, /no valid sample/);
+ assert.ok(!/fresh process \(ms\)[^\n]*\b0\.00\b/.test(text), 'a latency was reported with no measured step');
+});
+
async function coldTests() {
const corpusDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-eval-corpus-'));
try {
@@ -211,6 +253,9 @@ async function coldTests() {
test('a throwing step is decided as the worker decides it, so it is no disagreement', () => {
assert.deepStrictEqual(coldThrow.map(step => step.decision), warmThrow.steps.map(step => step.decision));
});
+ test('every throwing step is reported as a fresh-process failure', () => {
+ assert.strictEqual(coldSummary(coldThrow, warmThrow.steps).errors, coldThrow.length);
+ });
} finally {
fs.rmSync(throwingDir, { recursive: true, force: true });
}