The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
feat(gateguard): write scenario gate results as SARIF
Jeffrey Montoya216fe89c36fb
diff --git a/docs/gateguard/evaluation.md b/docs/gateguard/evaluation.mdindex b5476ceb3..63028bb8e 100644--- a/docs/gateguard/evaluation.md+++ b/docs/gateguard/evaluation.md@@ -190,6 +190,7 @@ git fetch upstream mainnode scripts/dev/gateguard-eval.js --markdownnode scripts/dev/gateguard-eval.js --markdown --baseline upstream/main --baseline 58a4a0d0node scripts/dev/gateguard-eval.js --json > gateguard-eval.json+node scripts/dev/gateguard-eval.js --baseline upstream/main --sarif gateguard-eval.sarifnode tests/hooks/gateguard-scenarios.test.js```@@ -198,6 +199,10 @@ node tests/hooks/gateguard-scenarios.test.jsagainst it. `--corpus <dir>` points at anotherscenario directory. The script exits non-zero when the working tree has amismatch, a must-deny bypass, an explicit `allow` decision or a thrown error.+`--sarif <file>` also writes those working-tree failures as SARIF 2.1.0+(bypasses, explicit allows and errors as `error`, other mismatches as+`warning`) with each hook's totals in the run properties. Timing is left out,+so the file is identical across reruns of the same tree and corpus.## Corpus formatdiff --git a/scripts/dev/gateguard-eval.js b/scripts/dev/gateguard-eval.jsindex 629d588d7..8bb74b0e2 100644--- a/scripts/dev/gateguard-eval.js+++ b/scripts/dev/gateguard-eval.js@@ -396,6 +396,7 @@ function summarize(runs, table = questionTextTable()) {if (result.error) totals.errors += 1;steps.push({scenario: run.scenario.name,+ file: run.scenario.file,step: step.id,expect: step.expect,decision: result.decision,@@ -407,6 +408,7 @@ function summarize(runs, table = questionTextTable()) {irrelevant,missed,latencyMs: Number(result.latencyMs.toFixed(3)),+ ...(result.explicitAllow ? { explicitAllow: true } : {}),...(result.error ? { error: result.error } : {})});});@@ -474,27 +476,121 @@ function renderMarkdown(report) {return `${lines.join('\n')}\n`;}+const SARIF_RULES = Object.freeze([+ {+ id: 'gateguard/must-deny-bypass',+ level: 'error',+ name: 'MustDenyBypass',+ text: 'A step that must be denied (sensitive target, bypass attempt, or mutating first command) was allowed.'+ },+ {+ id: 'gateguard/explicit-allow',+ level: 'error',+ name: 'ExplicitAllow',+ text: 'The hook emitted permissionDecision "allow", which skips the host permission prompt.'+ },+ {+ id: 'gateguard/hook-error',+ level: 'error',+ name: 'HookError',+ text: 'The hook threw or produced unparseable output for a step.'+ },+ {+ id: 'gateguard/expectation-mismatch',+ level: 'warning',+ name: 'ExpectationMismatch',+ text: 'The hook decision differs from the decision the scenario expects.'+ }+]);++function sarifFindings(step) {+ const findings = [];+ if (step.mustDeny && step.decision !== 'deny') findings.push('gateguard/must-deny-bypass');+ if (step.explicitAllow) findings.push('gateguard/explicit-allow');+ if (step.error) findings.push('gateguard/hook-error');+ if (step.decision !== step.expect && findings.length === 0) findings.push('gateguard/expectation-mismatch');+ return findings;+}++function sarifTotals(summary) {+ const { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios } = summary.totals;+ return { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios };+}++/** SARIF 2.1.0 log of working-tree gate failures, with per-hook totals; timing is omitted so output is reproducible. */+function renderSarif(report) {+ const working = report.hooks[0].summary;+ const levels = new Map(SARIF_RULES.map(rule => [rule.id, rule.level]));+ const results = [];+ for (const step of working.steps) {+ for (const ruleId of sarifFindings(step)) {+ results.push({+ ruleId,+ level: levels.get(ruleId),+ message: { text: `${step.scenario} / ${step.step}: expected ${step.expect}, got ${step.decision} (${step.kind})` },+ locations: [+ {+ physicalLocation: {+ artifactLocation: { uri: `tests/fixtures/gateguard-scenarios/${step.file}`, uriBaseId: '%SRCROOT%' }+ }+ }+ ]+ });+ }+ }+ const log = {+ version: '2.1.0',+ $schema: 'https://json.schemastore.org/sarif-2.1.0.json',+ runs: [+ {+ tool: {+ driver: {+ name: 'gateguard-eval',+ informationUri: 'https://github.com/affaan-m/ECC/blob/main/docs/gateguard/evaluation.md',+ rules: SARIF_RULES.map(rule => ({+ id: rule.id,+ name: rule.name,+ shortDescription: { text: rule.text },+ defaultConfiguration: { level: rule.level }+ }))+ }+ },+ automationDetails: { id: 'gateguard/scenario-corpus' },+ invocations: [{ executionSuccessful: true }],+ properties: {+ corpus: report.corpus,+ hooks: report.hooks.map(hook => ({ label: hook.label, totals: sarifTotals(hook.summary) }))+ },+ results+ }+ ]+ };+ return `${JSON.stringify(log, null, 2)}\n`;+}+// --- CLI ---const USAGE = [- 'Usage: node scripts/dev/gateguard-eval.js [--markdown | --json] [--baseline <ref>]... [--corpus <dir>]',+ 'Usage: node scripts/dev/gateguard-eval.js [--markdown | --json] [--baseline <ref>]... [--corpus <dir>] [--sarif <file>]','',' --baseline <ref> git ref whose hook is compared with the working tree (repeatable;',` default ${DEFAULT_BASELINE})`,' --corpus <dir> scenario directory (default tests/fixtures/gateguard-scenarios)',' --markdown print markdown tables (default)',- ' --json print the full report as JSON'+ ' --json print the full report as JSON',+ ' --sarif <file> also write working-tree gate failures as SARIF 2.1.0'].join('\n');function parseArgs(argv) {- const options = { format: 'markdown', baselines: [], corpus: DEFAULT_CORPUS_DIR, help: false };+ const options = { format: 'markdown', baselines: [], corpus: DEFAULT_CORPUS_DIR, sarif: null, help: false };for (let i = 0; i < argv.length; i++) {const arg = argv[i];if (arg === '--markdown') options.format = 'markdown';else if (arg === '--json') options.format = 'json';else if (arg === '--help' || arg === '-h') options.help = true;- else if ((arg === '--baseline' || arg === '--corpus') && i + 1 < argv.length) {+ else if ((arg === '--baseline' || arg === '--corpus' || arg === '--sarif') && i + 1 < argv.length) {if (arg === '--baseline') options.baselines.push(argv[++i]);+ else if (arg === '--sarif') options.sarif = path.resolve(argv[++i]);else options.corpus = path.resolve(argv[++i]);} else fail(`unknown or incomplete argument: ${arg}\n\n${USAGE}`);}@@ -533,6 +629,7 @@ async function main() {}const report = await evaluate(options);process.stdout.write(options.format === 'json' ? `${JSON.stringify(report, null, 2)}\n` : renderMarkdown(report));+ if (options.sarif) fs.writeFileSync(options.sarif, renderSarif(report), 'utf8');const working = report.hooks[0].summary.totals;if (working.mustDenyBypasses > 0 || working.mismatches > 0 || working.explicitAllows > 0 || working.errors > 0) {process.exitCode = 1;@@ -548,4 +645,4 @@ if (!isMainThread && workerData && workerData.gateguardEval) {});}-module.exports = { loadCorpus, materializeHook, runCorpus, summarize, questionsAsked, renderMarkdown, evaluate };+module.exports = { loadCorpus, materializeHook, runCorpus, summarize, questionsAsked, renderMarkdown, renderSarif, parseArgs, evaluate };diff --git a/tests/scripts/gateguard-eval.test.js b/tests/scripts/gateguard-eval.test.jsnew file mode 100644index 000000000..1c986709f--- /dev/null+++ b/tests/scripts/gateguard-eval.test.js@@ -0,0 +1,153 @@+/**+ * Tests for scripts/dev/gateguard-eval.js argument parsing and SARIF output.+ */++'use strict';++const assert = require('assert');+const path = require('path');+const { parseArgs, renderSarif } = require('../../scripts/dev/gateguard-eval');++let passed = 0;+let failed = 0;++function test(name, fn) {+ try {+ fn();+ console.log(` ✓ ${name}`);+ passed++;+ } catch (error) {+ console.log(` ✗ ${name}`);+ console.log(` Error: ${error.message}`);+ failed++;+ }+}++function step(overrides) {+ return {+ scenario: 'sensitive-targets',+ file: '09-sensitive-targets.json',+ step: 'edit-env',+ expect: 'deny',+ decision: 'deny',+ kind: 'deny',+ mustDeny: true,+ redundant: false,+ ...overrides+ };+}++function summary(steps, totals = {}) {+ return {+ totals: { steps: steps.length, denials: 0, mustDenyBypasses: 0, mismatches: 0, explicitAllows: 0, errors: 0, skippedScenarios: 0, ...totals },+ latency: { p50Ms: 1.234, p95Ms: 5.678 },+ perScenario: [],+ steps+ };+}++function report(workingSteps, baselineTotals = { mustDenyBypasses: 8 }) {+ return {+ corpus: { scenarios: 1, steps: workingSteps.length },+ hooks: [+ { label: 'working tree', ref: null, summary: summary(workingSteps) },+ { label: 'upstream/main', ref: 'upstream/main', summary: summary([], baselineTotals) }+ ]+ };+}++function results(log) {+ return log.runs[0].results;+}++console.log('\n=== Testing gateguard-eval ===\n');++test('--sarif takes a file path and keeps the other options', () => {+ const options = parseArgs(['--json', '--sarif', 'out/gate.sarif', '--baseline', 'main']);+ assert.strictEqual(options.format, 'json');+ assert.strictEqual(options.sarif, path.resolve('out/gate.sarif'));+ assert.deepStrictEqual(options.baselines, ['main']);+});++test('--sarif without a value is rejected', () => {+ assert.throws(() => parseArgs(['--sarif']), /unknown or incomplete argument: --sarif/);+});++test('a clean working tree produces a valid SARIF log with no results', () => {+ const log = JSON.parse(renderSarif(report([step({}), step({ step: 'read-only', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass' })])));+ assert.strictEqual(log.version, '2.1.0');+ assert.strictEqual(log.runs.length, 1);+ assert.strictEqual(log.runs[0].tool.driver.name, 'gateguard-eval');+ assert.deepStrictEqual(results(log), []);+});++test('every rule declares a default level', () => {+ const rules = JSON.parse(renderSarif(report([]))).runs[0].tool.driver.rules;+ assert.deepStrictEqual(+ rules.map(rule => [rule.id, rule.defaultConfiguration.level]),+ [+ ['gateguard/must-deny-bypass', 'error'],+ ['gateguard/explicit-allow', 'error'],+ ['gateguard/hook-error', 'error'],+ ['gateguard/expectation-mismatch', 'warning']+ ]+ );+});++test('a must-deny step that was allowed is an error on its scenario file', () => {+ const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'credit' })])));+ assert.strictEqual(results(log).length, 1);+ const [result] = results(log);+ assert.strictEqual(result.ruleId, 'gateguard/must-deny-bypass');+ assert.strictEqual(result.level, 'error');+ assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uri, 'tests/fixtures/gateguard-scenarios/09-sensitive-targets.json');+ assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uriBaseId, '%SRCROOT%');+ assert.match(result.message.text, /sensitive-targets \/ edit-env: expected deny, got allow \(credit\)/);+});++test('an explicit allow and a hook error are reported as errors', () => {+ const log = JSON.parse(+ renderSarif(+ report([+ step({ step: 'a', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass', explicitAllow: true }),+ step({ step: 'b', mustDeny: false, error: 'boom' })+ ])+ )+ );+ assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [+ ['gateguard/explicit-allow', 'error'],+ ['gateguard/hook-error', 'error']+ ]);+});++test('a wrong decision that is not a bypass is a warning', () => {+ const log = JSON.parse(renderSarif(report([step({ mustDeny: false, expect: 'allow', decision: 'deny', kind: 'deny' })])));+ assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [['gateguard/expectation-mismatch', 'warning']]);+});++test('a bypass is not reported a second time as a mismatch', () => {+ const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'pass' })])));+ assert.deepStrictEqual(results(log).map(result => result.ruleId), ['gateguard/must-deny-bypass']);+});++test('baseline totals are recorded without their failures becoming results', () => {+ const log = JSON.parse(renderSarif(report([step({})])));+ const hooks = log.runs[0].properties.hooks;+ assert.deepStrictEqual(hooks.map(hook => [hook.label, hook.totals.mustDenyBypasses]), [+ ['working tree', 0],+ ['upstream/main', 8]+ ]);+ assert.deepStrictEqual(results(log), []);+});++test('output leaves out timing so reruns are byte-identical', () => {+ const text = renderSarif(report([step({ latencyMs: 3.21 })]));+ assert.ok(!/latency|p50|p95|Time/i.test(text), 'timing leaked into SARIF');+ const slower = report([step({ latencyMs: 9.87 })]);+ slower.hooks[0].summary.latency = { p50Ms: 9, p95Ms: 99 };+ assert.strictEqual(renderSarif(slower), text);+});++console.log(`\nPassed: ${passed}`);+console.log(`Failed: ${failed}`);+process.exitCode = failed > 0 ? 1 : 0;