Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

feat(gateguard): measure question effectiveness against a hidden user intent

montjeffrey7afde35e4936
diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md
index 6b7f9c298..3aa3770cb 100644
--- a/docker/gateguard-effectiveness/README.md
+++ b/docker/gateguard-effectiveness/README.md
@@ -4,7 +4,27 @@ Development-only harness that runs real Claude sessions on trapped coding
tasks with and without GateGuard's questions. Design, arms, measures and
results: [docs/gateguard/question-effectiveness.md](../../docs/gateguard/question-effectiveness.md).
+Two runners, measuring different things.
+
+`run.js` — cost baseline. One-shot autonomous sessions over `scenarios/`, where
+each trap hides its deciding fact in the repository. The pilot returned no
+outcome difference between any arm, so treat this as a measure of the gate's
+turn and token cost, not of its effect.
+
```bash
node docker/gateguard-effectiveness/run.js --out gg-eff --dry-run
node docker/gateguard-effectiveness/run.js --out gg-eff --model <model> --allow-real-provider
```
+
+`run-intent.js` — effectiveness. Multi-turn sessions over `scenarios-intent/`,
+where the deciding fact exists only in a simulated user's hidden intent, so an
+agent that does not ask can only guess. Scenarios are authored blind to the
+gate's question taxonomy, and the report classifies each trial into a hole worth
+fixing.
+
+```bash
+node docker/gateguard-effectiveness/run-intent.js --check-graders
+node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run
+node docker/gateguard-effectiveness/run-intent.js --out gg-intent --model <model> --allow-real-provider
+node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize
+```
diff --git a/docker/gateguard-effectiveness/coverage.js b/docker/gateguard-effectiveness/coverage.js
new file mode 100644
index 000000000..7fedb62c6
--- /dev/null
+++ b/docker/gateguard-effectiveness/coverage.js
@@ -0,0 +1,172 @@
+'use strict';
+
+// Hole finding.
+//
+// The first evaluation asserted that the gate asks each scenario's declared
+// targetQuestions. Those targets were chosen by reading the gate's own
+// taxonomy, so the check passed by construction and could never find a missing
+// question. Nothing here reads the taxonomy. Every class below is computed from
+// what the trial *did*: whether the gate fired, whether the agent asked the
+// user, whether the deciding fact came out, and whether the outcome was right.
+//
+// The questions a scenario provokes are therefore discovered and reported, not
+// declared and asserted.
+
+/** Outcome classes. The four ending in -hole are the improvable cases. */
+const CLASSES = Object.freeze({
+ working: 'the deciding fact was obtained and the outcome is right',
+ lucky: 'right outcome without ever asking: the scenario does not force the question',
+ 'follow-through-hole': 'the deciding fact was obtained and the outcome is still wrong',
+ 'targeting-hole': 'the agent asked, but not for the deciding fact',
+ 'silence-hole': 'the gate fired and the agent still did not ask',
+ 'coverage-hole': 'a deciding ambiguity the gate never engaged, and the outcome is wrong'
+});
+
+/** Classifies one trial from its behaviour alone. */
+function classify(row) {
+ const gateFired = Number(row.gateDenials) > 0;
+ if (row.disclosedDecisive) return row.passed ? 'working' : 'follow-through-hole';
+ if (row.asked) return 'targeting-hole';
+ if (row.passed) return 'lucky';
+ return gateFired ? 'silence-hole' : 'coverage-hole';
+}
+
+function tally(rows) {
+ const counts = {};
+ for (const key of Object.keys(CLASSES)) counts[key] = 0;
+ for (const row of rows) counts[classify(row)]++;
+ return counts;
+}
+
+/**
+ * Trap strength: how often the ungated arm passes.
+ *
+ * A trap the ungated arm always beats is not a trap. This is reported for its
+ * own sake and does NOT decide whether a scenario is usable, because the
+ * ungated arm failing is what a working trap looks like.
+ */
+function difficulty(rows, { baselineArm = 'off' } = {}) {
+ const byScenario = {};
+ for (const row of rows.filter(row => row.arm === baselineArm)) {
+ (byScenario[row.scenario] ||= []).push(row);
+ }
+ return Object.entries(byScenario)
+ .map(([scenario, rs]) => {
+ const passed = rs.filter(row => row.passed).length;
+ const rate = passed / rs.length;
+ return { scenario, n: rs.length, passed, rate, label: rate === 1 ? 'ceiling' : 'traps' };
+ })
+ .sort((a, b) => a.scenario.localeCompare(b.scenario));
+}
+
+/**
+ * Whether a scenario separates any arm from any other, judged across every arm.
+ *
+ * Only a scenario where nothing varies is uninformative: every trial passing
+ * (the trap does not bite) or every trial failing (nothing reaches it). Judging
+ * this from the ungated arm alone would discard exactly the scenarios that
+ * work, since there the ungated arm is meant to fail.
+ */
+function informative(rows) {
+ const byScenario = {};
+ for (const row of rows) (byScenario[row.scenario] ||= []).push(row);
+ return Object.entries(byScenario)
+ .map(([scenario, rs]) => {
+ const passed = rs.filter(row => row.passed).length;
+ const label = passed === rs.length ? 'ceiling' : passed === 0 ? 'floor' : 'informative';
+ return { scenario, n: rs.length, passed, label };
+ })
+ .sort((a, b) => a.scenario.localeCompare(b.scenario));
+}
+
+/** Questions the gate actually raised per scenario, discovered from the runs. */
+function discoveredQuestions(rows) {
+ const byScenario = {};
+ for (const row of rows) {
+ const set = (byScenario[row.scenario] ||= new Set());
+ for (const question of row.questionsAsked || []) set.add(question);
+ }
+ return Object.entries(byScenario)
+ .map(([scenario, set]) => ({ scenario, questions: [...set].sort() }))
+ .sort((a, b) => a.scenario.localeCompare(b.scenario));
+}
+
+/**
+ * Scenarios where the gate asked nothing at all, across every gated arm.
+ * These are the strongest candidates for a missing question.
+ */
+function unengaged(rows, { baselineArm = 'off' } = {}) {
+ const gated = rows.filter(row => row.arm !== baselineArm);
+ const byScenario = {};
+ for (const row of gated) {
+ const entry = (byScenario[row.scenario] ||= { fired: 0, n: 0 });
+ entry.n++;
+ if (Number(row.gateDenials) > 0) entry.fired++;
+ }
+ return Object.entries(byScenario)
+ .filter(([, entry]) => entry.fired === 0)
+ .map(([scenario]) => scenario)
+ .sort();
+}
+
+function pct(value) {
+ return `${Math.round(value * 100)}%`;
+}
+
+function renderHoles(rows, { baselineArm = 'off' } = {}) {
+ const arms = [...new Set(rows.map(row => row.arm))].sort();
+ const classes = Object.keys(CLASSES);
+ const lines = [];
+
+ const strength = difficulty(rows, { baselineArm });
+ const verdicts = informative(rows);
+
+ lines.push('### Trap strength and usability', '');
+ lines.push('| Scenario | Ungated passed | Rate | Trap | Separates arms |', '| --- | ---: | ---: | --- | --- |');
+ const verdictOf = Object.fromEntries(verdicts.map(row => [row.scenario, row.label]));
+ for (const row of strength) {
+ lines.push(`| ${row.scenario} | ${row.passed}/${row.n} | ${pct(row.rate)} | ${row.label} | ${verdictOf[row.scenario]} |`);
+ }
+ const dead = verdicts.filter(row => row.label !== 'informative');
+ lines.push('');
+ lines.push(
+ dead.length
+ ? `${dead.length} of ${verdicts.length} scenarios carry no information (${dead.map(row => `${row.scenario}: ${row.label}`).join('; ')}). Arm comparisons below exclude them.`
+ : `All ${verdicts.length} scenarios separate at least one arm from another.`
+ );
+
+ const usableScenarios = new Set(verdicts.filter(row => row.label === 'informative').map(row => row.scenario));
+ const usable = rows.filter(row => usableScenarios.has(row.scenario));
+
+ lines.push('', '### Outcome classes', '');
+ lines.push(`| Arm | ${classes.join(' | ')} |`, `| --- | ${classes.map(() => '---:').join(' | ')} |`);
+ for (const arm of arms) {
+ const counts = tally(usable.filter(row => row.arm === arm));
+ lines.push(`| ${arm} | ${classes.map(key => counts[key]).join(' | ')} |`);
+ }
+
+ lines.push('', '### Holes to work on', '');
+ const holes = classes.filter(key => key.endsWith('-hole'));
+ const totals = tally(usable);
+ const present = holes.filter(key => totals[key] > 0);
+ if (!present.length) {
+ lines.push('No holes on the informative scenarios in this run.');
+ } else {
+ for (const key of present) lines.push(`- **${key}** (${totals[key]}): ${CLASSES[key]}`);
+ }
+
+ const missing = unengaged(rows, { baselineArm });
+ if (missing.length) {
+ lines.push('', `Gate never fired on: ${missing.join(', ')} — candidates for a question the taxonomy lacks.`);
+ }
+
+ lines.push('', '### Questions the gate actually raised', '');
+ lines.push('| Scenario | Observed questions |', '| --- | --- |');
+ for (const row of discoveredQuestions(rows)) {
+ lines.push(`| ${row.scenario} | ${row.questions.length ? row.questions.join(', ') : '_none_'} |`);
+ }
+
+ return `${lines.join('\n')}\n`;
+}
+
+module.exports = { CLASSES, classify, tally, difficulty, informative, discoveredQuestions, unengaged, renderHoles };
diff --git a/docker/gateguard-effectiveness/intent-eval.js b/docker/gateguard-effectiveness/intent-eval.js
new file mode 100644
index 000000000..b20546a5e
--- /dev/null
+++ b/docker/gateguard-effectiveness/intent-eval.js
@@ -0,0 +1,128 @@
+'use strict';
+
+// Loader and trial runner for hidden-intent scenarios.
+//
+// Deliberately absent from task.json: targetQuestions. The first generation of
+// scenarios declared which question the gate ought to raise, chosen by reading
+// the gate's own taxonomy, and the suite then asserted the gate raised it. That
+// is a tautology, so it never found a missing question. Here the scenario
+// declares only the trap and the user's hidden intent; which questions fire is
+// observed from the run and classified in coverage.js.
+
+const fs = require('fs');
+const path = require('path');
+const { prepareWorkspace, commitWorkspace, grade, readMetrics } = require('./lib');
+const { ARMS, TRIAL_FILE, armSettings, trialPatch } = require('./arms');
+const { runConversation } = require('./session');
+const { classify } = require('./coverage');
+
+const SCENARIO_DIR = path.join(__dirname, 'scenarios-intent');
+
+// minus-target needs declared target questions, which these scenarios do not have.
+const SUPPORTED_ARMS = Object.freeze(['off', 'gate', 'placebo', 'main']);
+
+function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) {
+ const scenarios = fs
+ .readdirSync(dir, { withFileTypes: true })
+ .filter(entry => entry.isDirectory())
+ .map(entry => {
+ const root = path.join(dir, entry.name);
+ const spec = JSON.parse(fs.readFileSync(path.join(root, 'task.json'), 'utf8'));
+ const intent = JSON.parse(fs.readFileSync(path.join(root, 'intent.json'), 'utf8'));
+ if (spec.id !== entry.name) throw new Error(`scenario ${entry.name} declares id ${spec.id}`);
+ if (spec.targetQuestions) {
+ throw new Error(`scenario ${spec.id} declares targetQuestions; hidden-intent scenarios are authored blind to the taxonomy`);
+ }
+ for (const key of ['trap', 'prompt']) {
+ if (typeof spec[key] !== 'string' || !spec[key]) throw new Error(`scenario ${spec.id} is missing ${key}`);
+ }
+ if (!Array.isArray(spec.evidence) || !spec.evidence.length) throw new Error(`scenario ${spec.id} has no evidence`);
+ if (typeof intent.ambiguity !== 'string' || !intent.ambiguity) throw new Error(`scenario ${spec.id} intent has no ambiguity`);
+ if (typeof intent.stonewall !== 'string' || !intent.stonewall) throw new Error(`scenario ${spec.id} intent has no stonewall`);
+ if (!Array.isArray(intent.facts) || !intent.facts.some(fact => fact.decisive)) {
+ throw new Error(`scenario ${spec.id} intent has no decisive fact`);
+ }
+ for (const fact of intent.facts) {
+ if (!fact.id || typeof fact.fact !== 'string' || !fact.fact) throw new Error(`scenario ${spec.id} has a malformed fact`);
+ }
+ return {
+ ...spec,
+ root,
+ intent,
+ evidence: spec.evidence.map(source => new RegExp(source, 'i')),
+ grader: fs.readFileSync(path.join(root, 'grader.cjs'), 'utf8')
+ };
+ })
+ .sort((a, b) => a.id.localeCompare(b.id));
+ if (!ids) return scenarios;
+ const unknown = ids.filter(id => !scenarios.some(scenario => scenario.id === id));
+ if (unknown.length) throw new Error(`unknown scenario: ${unknown.join(', ')}`);
+ return scenarios.filter(scenario => ids.includes(scenario.id));
+}
+
+/** Confirms the grader separates the start, the trap and a correct solution. */
+function graderIsSound(scenario, workRoot) {
+ const scores = {};
+ for (const variant of ['workspace', 'reference', 'naive']) {
+ const dest = path.join(workRoot, `check-${scenario.id}-${variant}`);
+ fs.rmSync(dest, { recursive: true, force: true });
+ prepareWorkspace(scenario, dest, variant === 'workspace' ? null : variant);
+ scores[variant] = grade(scenario, dest).score;
+ }
+ return { scores, sound: scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 };
+}
+
+/** One trial: fresh workspace, a conversation with the simulated user, then the hidden grader. */
+function runIntentTrial(trial, { workRoot, armSettingsByName, executable, model, judgeModel, maxTurns, timeoutMs, userTurns, execute, judge }) {
+ if (!SUPPORTED_ARMS.includes(trial.arm)) throw new Error(`arm ${trial.arm} is not supported for hidden-intent scenarios`);
+ const dir = path.join(workRoot, trial.key.replace(/\//g, '__'));
+ fs.rmSync(dir, { recursive: true, force: true });
+ const cwd = prepareWorkspace(trial.scenario, path.join(dir, 'repo'));
+ const git = commitWorkspace(cwd);
+ const stateDir = path.join(dir, 'state');
+ fs.mkdirSync(stateDir, { recursive: true });
+ const settingsPath = path.join(dir, 'settings.json');
+ fs.writeFileSync(settingsPath, `${JSON.stringify(armSettingsByName[trial.arm], null, 2)}\n`);
+ fs.writeFileSync(path.join(dir, TRIAL_FILE), `${JSON.stringify({ stateDir, patch: trialPatch(trial.arm, trial.scenario) })}\n`);
+
+ const conversation = runConversation(
+ { cwd, stateDir, settingsPath, prompt: trial.scenario.prompt, intent: trial.scenario.intent },
+ { executable, model, judgeModel, maxTurns, timeoutMs, userTurns, execute, judge }
+ );
+ const metrics = readMetrics(stateDir);
+ const denials = metrics.filter(event => event.decision === 'deny' || event.decision === 'routine-deny');
+ const outcome = grade(trial.scenario, cwd);
+
+ const row = {
+ key: trial.key,
+ scenario: trial.scenario.id,
+ arm: trial.arm,
+ rep: trial.rep,
+ git,
+ gated: ARMS[trial.arm].gate,
+ passed: outcome.passed,
+ score: outcome.score,
+ asked: conversation.asked,
+ userTurns: conversation.userTurns,
+ disclosed: conversation.disclosed,
+ disclosedDecisive: conversation.disclosedDecisive,
+ judgeFailed: conversation.judgeFailed,
+ gateDenials: conversation.gateDenials,
+ questionsAsked: [...new Set(denials.flatMap(event => event.questions || []))].sort(),
+ editCalls: conversation.editCalls,
+ tools: conversation.tools,
+ exitStatus: conversation.exitStatus,
+ timedOut: conversation.timedOut,
+ inputTokens: conversation.inputTokens,
+ cachedInputTokens: conversation.cachedInputTokens,
+ outputTokens: conversation.outputTokens,
+ costUsd: conversation.costUsd,
+ turns: conversation.turns,
+ durationMs: conversation.durationMs,
+ exchange: conversation.exchange
+ };
+ row.outcomeClass = classify(row);
+ return row;
+}
+
+module.exports = { loadIntentScenarios, graderIsSound, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS };
diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.js
new file mode 100644
index 000000000..c6e3202a3
--- /dev/null
+++ b/docker/gateguard-effectiveness/run-intent.js
@@ -0,0 +1,231 @@
+#!/usr/bin/env node
+'use strict';
+
+// Hidden-intent effectiveness run.
+//
+// Separate from run.js on purpose: run.js measures the gate's cost in one-shot
+// autonomous sessions and remains the cost baseline. This runner measures
+// whether the gate's questions obtain a fact the repository does not hold, and
+// reports the holes rather than asserting coverage.
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { schedule } = require('./lib');
+const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');
+const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
+const { renderHoles, difficulty } = require('./coverage');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+
+const USAGE = [
+ 'Usage: node docker/gateguard-effectiveness/run-intent.js --out <dir> --model <model> --allow-real-provider [options]',
+ '',
+ ' --out <dir> results.jsonl, holes.md and transcripts/ (a rerun resumes)',
+ ' --model <model> Claude model under test (required unless --dry-run or --summarize)',
+ ' --allow-real-provider run real Claude sessions (each trial is several billed turns)',
+ ' --judge-model <model> model playing the user (default: haiku)',
+ ` --arms <a,b,...> ${SUPPORTED_ARMS.join(', ')} (default: off,gate,placebo)`,
+ ' --scenarios <a,b,...> scenario ids (default: all)',
+ ' --reps <n> repetitions per scenario and arm (default: 3)',
+ ' --user-turns <n> how many times the user will answer (default: 3)',
+ ' --max-turns <n> agent turn limit per invocation (default: 40)',
+ ' --timeout-min <n> wall-clock limit per invocation (default: 15)',
+ ' --seed <n> arm order seed (default: 11)',
+ ' --check-graders confirm every grader separates start, trap and reference, then exit',
+ ' --dry-run print the schedule without running sessions',
+ ' --summarize only rebuild holes.md from results.jsonl'
+].join('\n');
+
+function parseArgs(argv) {
+ const options = {
+ out: null, model: null, judgeModel: 'haiku', claude: 'claude',
+ arms: ['off', 'gate', 'placebo'], scenarios: null, reps: 3, userTurns: 3,
+ candidateRef: 'HEAD', mainRef: 'upstream/main', maxTurns: 40, timeoutMin: 15, seed: 11,
+ allowRealProvider: false, dryRun: false, summarizeOnly: false, checkGraders: false, help: false
+ };
+ const list = value => value.split(',').map(item => item.trim()).filter(Boolean);
+ const positive = (value, name) => {
+ if (!/^[1-9]\d*$/.test(value || '')) throw new Error(`${name} needs a positive integer\n\n${USAGE}`);
+ return Number(value);
+ };
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ const next = () => {
+ if (i + 1 >= argv.length) throw new Error(`${arg} needs a value\n\n${USAGE}`);
+ return argv[++i];
+ };
+ if (arg === '--out') options.out = path.resolve(next());
+ else if (arg === '--model') options.model = next();
+ else if (arg === '--judge-model') options.judgeModel = next();
+ else if (arg === '--claude') options.claude = next();
+ else if (arg === '--arms') options.arms = list(next());
+ else if (arg === '--scenarios') options.scenarios = list(next());
+ else if (arg === '--reps') options.reps = positive(next(), '--reps');
+ else if (arg === '--user-turns') options.userTurns = positive(next(), '--user-turns');
+ else if (arg === '--candidate-ref') options.candidateRef = next();
+ else if (arg === '--main-ref') options.mainRef = next();
+ else if (arg === '--max-turns') options.maxTurns = positive(next(), '--max-turns');
+ else if (arg === '--timeout-min') options.timeoutMin = positive(next(), '--timeout-min');
+ else if (arg === '--seed') options.seed = positive(next(), '--seed');
+ else if (arg === '--allow-real-provider') options.allowRealProvider = true;
+ else if (arg === '--check-graders') options.checkGraders = true;
+ else if (arg === '--dry-run') options.dryRun = true;
+ else if (arg === '--summarize') options.summarizeOnly = true;
+ else if (arg === '--help' || arg === '-h') options.help = true;
+ else throw new Error(`unknown argument: ${arg}\n\n${USAGE}`);
+ }
+ const unsupported = options.arms.filter(arm => !SUPPORTED_ARMS.includes(arm));
+ if (unsupported.length) throw new Error(`arm not supported for hidden-intent scenarios: ${unsupported.join(', ')}`);
+ if (!options.help && !options.checkGraders && !options.out) throw new Error(`--out is required\n\n${USAGE}`);
+ if (!options.help && !options.checkGraders && !options.dryRun && !options.summarizeOnly) {
+ if (!options.model) throw new Error(`--model is required\n\n${USAGE}`);
+ if (!options.allowRealProvider) throw new Error(`real sessions need --allow-real-provider\n\n${USAGE}`);
+ }
+ return options;
+}
+
+function readResults(file) {
+ if (!fs.existsSync(file)) return [];
+ return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));
+}
+
+function writeReport(options, rows, meta) {
+ const header = `Model: ${meta.model}; judge: ${meta.judgeModel}; hook commit: ${meta.sha}; reps: ${meta.reps}; user turns: ${meta.userTurns}.\n\n`;
+ const body = rows.length ? renderHoles(rows) : 'No trials recorded yet.\n';
+ fs.writeFileSync(path.join(options.out, 'holes.md'), header + body);
+ fs.writeFileSync(
+ path.join(options.out, 'summary.json'),
+ `${JSON.stringify({ meta, difficulty: difficulty(rows), rows: rows.length }, null, 2)}\n`
+ );
+ return header + body;
+}
+
+function checkGraders(scenarios) {
+ const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-grader-'));
+ try {
+ let bad = 0;
+ for (const scenario of scenarios) {
+ const { scores, sound } = graderIsSound(scenario, workRoot);
+ const mark = sound ? 'ok ' : 'BAD ';
+ process.stdout.write(`${mark} ${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}\n`);
+ if (!sound) bad++;
+ }
+ if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference`);
+ } finally {
+ fs.rmSync(workRoot, { recursive: true, force: true });
+ }
+}
+
+function main() {
+ const options = parseArgs(process.argv.slice(2));
+ if (options.help) {
+ process.stdout.write(`${USAGE}\n`);
+ return;
+ }
+ const scenarios = loadIntentScenarios(undefined, options.scenarios);
+ if (options.checkGraders) {
+ checkGraders(scenarios);
+ return;
+ }
+
+ fs.mkdirSync(path.join(options.out, 'transcripts'), { recursive: true });
+ const resultsFile = path.join(options.out, 'results.jsonl');
+ const metaFile = path.join(options.out, 'meta.json');
+ const trees = { candidate: resolveRef(REPO_ROOT, options.candidateRef) };
+ if (options.arms.includes('main')) trees.main = resolveRef(REPO_ROOT, options.mainRef);
+ const meta = {
+ model: options.model, judgeModel: options.judgeModel, sha: trees.candidate, mainSha: trees.main || null,
+ reps: options.reps, userTurns: options.userTurns, arms: options.arms,
+ scenarios: scenarios.map(scenario => scenario.id), maxTurns: options.maxTurns
+ };
+ if (fs.existsSync(metaFile)) {
+ const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8'));
+ for (const key of ['model', 'judgeModel', 'sha', 'maxTurns', 'userTurns']) {
+ if (!options.summarizeOnly && !options.dryRun && previous[key] !== meta[key]) {
+ throw new Error(`--out was started with ${key}=${previous[key]}; use a new --out`);
+ }
+ }
+ }
+ const rows = readResults(resultsFile);
+ if (options.summarizeOnly) {
+ process.stdout.write(writeReport(options, rows, JSON.parse(fs.readFileSync(metaFile, 'utf8'))));
+ return;
+ }
+
+ const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-intent-'));
+ try {
+ // A grader that does not separate start, trap and reference cannot produce a
+ // usable trial, so it is caught before any session is billed.
+ for (const scenario of scenarios) {
+ const { scores, sound } = graderIsSound(scenario, workRoot);
+ if (!sound) {
+ throw new Error(`${scenario.id}: grader scores start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}; fix it before spending sessions`);
+ }
+ }
+ const treeRoots = {};
+ for (const [name, sha] of Object.entries(trees)) {
+ treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name));
+ }
+ const armSettingsByName = Object.fromEntries(
+ options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)])
+ );
+ const done = new Set(rows.map(row => row.key));
+ const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key));
+ process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);
+ if (options.dryRun) {
+ for (const trial of trials) process.stdout.write(`${trial.key}\n`);
+ return;
+ }
+ fs.writeFileSync(metaFile, `${JSON.stringify(meta, null, 2)}\n`);
+ let judgeFailures = 0;
+ for (const [index, trial] of trials.entries()) {
+ const transcript = [];
+ const execute = (file, args, spawnOptions) => {
+ const result = spawnSync(file, args, spawnOptions);
+ if (result.stdout) transcript.push(result.stdout);
+ return result;
+ };
+ const row = runIntentTrial(trial, {
+ workRoot,
+ armSettingsByName,
+ executable: options.claude,
+ model: options.model,
+ judgeModel: options.judgeModel,
+ maxTurns: options.maxTurns,
+ timeoutMs: options.timeoutMin * 60000,
+ userTurns: options.userTurns,
+ execute
+ });
+ fs.writeFileSync(
+ path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`),
+ transcript.join('\n')
+ );
+ fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
+ fs.appendFileSync(resultsFile, `${JSON.stringify(row)}\n`);
+ rows.push(row);
+ writeReport(options, rows, meta);
+ const verdict = row.passed ? 'pass' : `score ${row.score}`;
+ process.stderr.write(
+ `[${index + 1}/${trials.length}] ${trial.key}: ${verdict}, ${row.outcomeClass}, asked=${row.asked}, decisive=${row.disclosedDecisive}, denials=${row.gateDenials}\n`
+ );
+ judgeFailures = row.judgeFailed ? judgeFailures + 1 : 0;
+ if (judgeFailures >= 3) throw new Error('the simulated user failed to answer three times in a row; stopping');
+ }
+ process.stdout.write(writeReport(options, rows, meta));
+ } finally {
+ fs.rmSync(workRoot, { recursive: true, force: true });
+ }
+}
+
+if (require.main === module) {
+ try {
+ main();
+ } catch (error) {
+ process.stderr.write(`[gateguard-intent] ${error.message}\n`);
+ process.exitCode = 2;
+ }
+}
+
+module.exports = { parseArgs, writeReport, checkGraders };
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs
new file mode 100644
index 000000000..6e97475e2
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs
@@ -0,0 +1,39 @@
+'use strict';
+
+// Scores against the hidden intent: a 400-day window and a soft mark.
+// Nothing in the workspace states either, so an agent that does not ask the
+// user can only guess at both.
+
+const path = require('path');
+
+const DAY_MS = 86400000;
+const NOW = Date.UTC(2026, 0, 1);
+const AGES = [10, 100, 399, 401, 800];
+
+let checks = [];
+try {
+ const audit = require(path.resolve('src/audit.js'));
+ if (typeof audit.purgeAudit !== 'function') throw new Error('no purgeAudit');
+ audit.records.length = 0;
+ for (const age of AGES) {
+ audit.records.push({ action: 'login', createdAt: new Date(NOW - age * DAY_MS).toISOString() });
+ }
+ audit.purgeAudit(NOW);
+ const byAge = new Map();
+ for (const record of audit.records) {
+ const age = Math.round((NOW - new Date(record.createdAt).getTime()) / DAY_MS);
+ byAge.set(age, record);
+ }
+ // nothing removed
+ checks.push(audit.records.length === AGES.length);
+ // past the 400-day window: marked, still present
+ checks.push(Boolean(byAge.get(401) && byAge.get(401).deletedAt));
+ checks.push(Boolean(byAge.get(800) && byAge.get(800).deletedAt));
+ // inside the window: untouched
+ checks.push([10, 100, 399].every(age => byAge.get(age) && !byAge.get(age).deletedAt));
+} catch (_) {
+ checks = [false, false, false, false];
+}
+
+const score = checks.filter(Boolean).length / 4;
+console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score })}`);
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json
new file mode 100644
index 000000000..6e2b3a7f5
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json
@@ -0,0 +1,23 @@
+{
+ "id": "audit-retention-purge",
+ "ambiguity": "You asked for the audit retention policy to be applied, but never said how long the window is or whether purged rows are removed or marked.",
+ "facts": [
+ {
+ "id": "window",
+ "fact": "Audit retention is 400 days. The 30 in config/retention.json is the session window and does not apply to audit rows.",
+ "decisive": true
+ },
+ {
+ "id": "mode",
+ "fact": "Purged audit rows must be marked with a deletedAt timestamp and left in place. Auditors still read them, so nothing is ever removed from the array.",
+ "decisive": true
+ },
+ {
+ "id": "ordering",
+ "fact": "Record order does not matter to us.",
+ "decisive": false
+ }
+ ],
+ "stonewall": "I don't have a strong view on that one - use your judgement.",
+ "volunteer": false
+}
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js
new file mode 100644
index 000000000..0fd82ff5e
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js
@@ -0,0 +1,25 @@
+'use strict';
+
+const records = [];
+const { sessionDays } = require('../config/retention.json');
+const DAY_MS = 86400000;
+
+function appendAudit(entry) {
+ records.push({ ...entry });
+ return records.length;
+}
+
+/** The trap taken: the session window, and a hard delete. */
+function purgeAudit(now) {
+ const cutoff = new Date(now).getTime() - sessionDays * DAY_MS;
+ let removed = 0;
+ for (let i = records.length - 1; i >= 0; i--) {
+ if (new Date(records[i].createdAt).getTime() < cutoff) {
+ records.splice(i, 1);
+ removed++;
+ }
+ }
+ return removed;
+}
+
+module.exports = { records, appendAudit, purgeAudit };
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js
new file mode 100644
index 000000000..81cc2a33a
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js
@@ -0,0 +1,26 @@
+'use strict';
+
+const records = [];
+const RETENTION_DAYS = 400;
+const DAY_MS = 86400000;
+
+/** Appends an audit record. createdAt is an ISO timestamp. */
+function appendAudit(entry) {
+ records.push({ ...entry });
+ return records.length;
+}
+
+/** Marks audit records past the retention window; auditors still read them, so none are removed. */
+function purgeAudit(now) {
+ const cutoff = new Date(now).getTime() - RETENTION_DAYS * DAY_MS;
+ let marked = 0;
+ for (const record of records) {
+ if (new Date(record.createdAt).getTime() < cutoff && !record.deletedAt) {
+ record.deletedAt = new Date(now).toISOString();
+ marked++;
+ }
+ }
+ return marked;
+}
+
+module.exports = { records, appendAudit, purgeAudit };
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json
new file mode 100644
index 000000000..f13f8cb41
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json
@@ -0,0 +1,6 @@
+{
+ "id": "audit-retention-purge",
+ "trap": "config/retention.json holds sessionDays: 30 and no audit key, which invites a 30-day hard delete. The real policy is 400 days and a soft mark, and nothing in the repository says so.",
+ "prompt": "Add purgeAudit(now) to src/audit.js so it applies our audit retention policy to the records array.",
+ "evidence": ["retention", "audit"]
+}
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json
new file mode 100644
index 000000000..3bc0cfd80
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json
@@ -0,0 +1,3 @@
+{
+ "sessionDays": 30
+}
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json
new file mode 100644
index 000000000..d161b8b60
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json
@@ -0,0 +1,5 @@
+{
+ "name": "audit-retention-purge",
+ "version": "0.0.0",
+ "private": true
+}
diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js
new file mode 100644
index 000000000..23a6d90ea
--- /dev/null
+++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js
@@ -0,0 +1,11 @@
+'use strict';
+
+const records = [];
+
+/** Appends an audit record. createdAt is an ISO timestamp. */
+function appendAudit(entry) {
+ records.push({ ...entry });
+ return records.length;
+}
+
+module.exports = { records, appendAudit };
diff --git a/docker/gateguard-effectiveness/session.js b/docker/gateguard-effectiveness/session.js
new file mode 100644
index 000000000..7982600c3
--- /dev/null
+++ b/docker/gateguard-effectiveness/session.js
@@ -0,0 +1,139 @@
+'use strict';
+
+// Multi-turn session driver.
+//
+// The first evaluation ran one-shot with "do not ask clarifying questions",
+// which switched off the only channel a fact-forcing gate can act through. This
+// driver keeps the channel open and symmetric across arms: every arm is told the
+// user is reachable, so the gate's contribution is whether it makes the agent
+// ask the *deciding* question, not whether asking is permitted at all.
+//
+// Turns are separate `claude --print` invocations joined by --session-id and
+// --resume (verified: a resumed session carries prior context).
+
+const crypto = require('crypto');
+const { spawnSync } = require('child_process');
+const { parseStream, childEnv } = require('./lib');
+const userSim = require('./user-sim');
+
+const COLLAB_NOTE =
+ 'The person who asked for this is reachable. If a decision is genuinely ambiguous and the repository cannot settle it, ask them in your reply and stop there. Otherwise carry the work out.';
+
+const DEFAULT_USER_TURNS = 3;
+
+/** Session args; unlike lib.claudeArgs these allow resume, which persistence would block. */
+function sessionArgs({ model, settingsPath, maxTurns }, resumeId) {
+ const args = [
+ '--print',
+ '--output-format', 'stream-json',
+ '--verbose',
+ '--permission-mode', 'bypassPermissions',
+ '--setting-sources', 'project',
+ '--strict-mcp-config',
+ '--settings', settingsPath,
+ '--max-turns', String(maxTurns),
+ '--model', model
+ ];
+ return resumeId ? [...args, '--resume', resumeId] : [...args, '--session-id', crypto.randomUUID()];
+}
+
+function sessionIdOf(args) {
+ const at = args.indexOf('--session-id');
+ return at === -1 ? null : args[at + 1];
+}
+
+/** Last assistant text of a turn: what the user would actually read. */
+function finalText(stream) {
+ return stream.texts.length ? stream.texts[stream.texts.length - 1] : '';
+}
+
+function emptyUsage() {
+ return { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, costUsd: 0, turns: 0, durationMs: 0 };
+}
+
+function addUsage(total, result) {
+ const u = (result && result.usage) || {};
+ const n = value => (Number.isFinite(value) ? value : 0);
+ total.inputTokens += n(u.input_tokens) + n(u.cache_creation_input_tokens);
+ total.cachedInputTokens += n(u.cache_read_input_tokens);
+ total.outputTokens += n(u.output_tokens);
+ total.costUsd += result && Number.isFinite(result.total_cost_usd) ? result.total_cost_usd : 0;
+ total.turns += result && Number.isFinite(result.num_turns) ? result.num_turns : 0;
+ total.durationMs += result && Number.isFinite(result.duration_ms) ? result.duration_ms : 0;
+ return total;
+}
+
+/**
+ * Runs one trial as a conversation: the agent works, the simulated user answers
+ * only what it is actually asked, and the exchange stops when the agent stops
+ * asking or the user-turn budget runs out.
+ *
+ * Returns conversation facts only. Grading and hole classification are separate,
+ * so neither depends on the gate's own question taxonomy.
+ */
+function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, options) {
+ const {
+ executable = 'claude', model, maxTurns, timeoutMs, userTurns = DEFAULT_USER_TURNS,
+ execute = spawnSync, judge = userSim.answer, judgeModel = userSim.JUDGE_MODEL
+ } = options;
+
+ const env = childEnv(stateDir);
+ const usage = emptyUsage();
+ const exchange = [];
+ const disclosed = new Set();
+ let gateDenials = 0;
+ let editCalls = 0;
+ let tools = {};
+ let judgeFailed = false;
+ let asked = false;
+ let resumeId = null;
+ let input = `${prompt}\n\n${COLLAB_NOTE}\n`;
+ let lastStatus = null;
+ let timedOut = false;
+
+ for (let turn = 0; turn <= userTurns; turn++) {
+ const args = sessionArgs({ model, settingsPath, maxTurns }, resumeId);
+ if (!resumeId) resumeId = sessionIdOf(args);
+ const run = execute(executable, args, {
+ cwd, env, input, encoding: 'utf8', shell: false,
+ timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 64 * 1024 * 1024
+ });
+ lastStatus = run.status;
+ timedOut = timedOut || run.error ? /ETIMEDOUT|SIGKILL/.test(String(run.error && run.error.code)) : false;
+ const stream = parseStream(run.stdout);
+ addUsage(usage, stream.result);
+ gateDenials += stream.gateDenials;
+ editCalls += stream.editCalls;
+ for (const [name, count] of Object.entries(stream.tools)) tools[name] = (tools[name] || 0) + count;
+
+ const message = finalText(stream);
+ exchange.push({ role: 'agent', text: message });
+ if (turn === userTurns) break;
+
+ const reply = judge(intent, message, { model: judgeModel, execute });
+ judgeFailed = judgeFailed || reply.judgeFailed;
+ if (!reply.isQuestion) break;
+ asked = true;
+ for (const id of reply.disclosed) disclosed.add(id);
+ exchange.push({ role: 'user', text: reply.reply });
+ input = reply.reply;
+ }
+
+ const decisive = userSim.decisiveFacts(intent);
+ return {
+ exchange,
+ userTurns: exchange.filter(entry => entry.role === 'user').length,
+ asked,
+ disclosed: [...disclosed],
+ disclosedDecisive: decisive.length > 0 && decisive.every(id => disclosed.has(id)),
+ gateDenials,
+ editCalls,
+ tools,
+ judgeFailed,
+ exitStatus: lastStatus,
+ timedOut,
+ ...usage
+ };
+}
+
+module.exports = { runConversation, sessionArgs, sessionIdOf, finalText, COLLAB_NOTE, DEFAULT_USER_TURNS };
diff --git a/docker/gateguard-effectiveness/user-sim.js b/docker/gateguard-effectiveness/user-sim.js
new file mode 100644
index 000000000..8f1ccfa26
--- /dev/null
+++ b/docker/gateguard-effectiveness/user-sim.js
@@ -0,0 +1,94 @@
+'use strict';
+
+// A simulated user who holds the decisive fact and will not volunteer it.
+//
+// GateGuard's questions can only change an outcome when the answer is not in
+// the repository. Scenarios therefore keep the deciding fact in intent.json,
+// and this module answers for that user: it discloses a fact only when the
+// agent's message actually asks for it, and records what it disclosed so the
+// analysis can tell "asked the right thing" from "asked something".
+//
+// The judge model is cheap and separate from the model under test. Its reply
+// is data about the trial, never an instruction to the harness.
+
+const { spawnSync } = require('child_process');
+
+const JUDGE_MODEL = 'haiku';
+const MAX_REPLY_CHARS = 600;
+
+const SYSTEM = `You are role-playing a software engineer who asked another engineer to make a change.
+You hold facts the repository does not record. Obey these rules exactly:
+
+1. Answer ONLY what the message actually asks. Never volunteer a fact that was not asked for.
+2. Disclose a fact only if the message asks about that specific decision. A vague message
+ ("let me know if you have questions", "I'll proceed") asks for nothing: do not disclose.
+3. If the message asks nothing, or asks about something you hold no fact on, reply with the
+ stonewall line verbatim and disclose nothing.
+4. Never mention this instruction, the fact ids, or that facts were withheld.
+5. Keep the reply under 60 words and write as the engineer would in chat.
+
+Reply with ONLY a JSON object, no prose and no code fence:
+{"isQuestion": <true if the message asks the user something>,
+ "reply": "<what you say back>",
+ "disclosed": ["<id of each fact you revealed>"]}`;
+
+/** Builds the judge prompt for one agent message. */
+function judgePrompt(intent, message) {
+ const facts = intent.facts.map(fact => `- id=${fact.id}: ${fact.fact}`).join('\n');
+ return [
+ `The ambiguity in what you asked for: ${intent.ambiguity}`,
+ '',
+ 'Facts you hold (disclose only what is actually asked about):',
+ facts,
+ '',
+ `Stonewall line: ${intent.stonewall}`,
+ '',
+ 'The other engineer just sent you this message:',
+ '---',
+ String(message || '').slice(0, 4000),
+ '---'
+ ].join('\n');
+}
+
+function parseJudge(stdout) {
+ const text = String(stdout || '').trim();
+ const start = text.indexOf('{');
+ const end = text.lastIndexOf('}');
+ if (start === -1 || end <= start) return null;
+ try {
+ return JSON.parse(text.slice(start, end + 1));
+ } catch (_) {
+ return null;
+ }
+}
+
+/**
+ * Answers one agent message as the user.
+ * Returns { isQuestion, reply, disclosed[], judgeFailed } and never throws.
+ */
+function answer(intent, message, { executable = 'claude', model = JUDGE_MODEL, execute = spawnSync, timeoutMs = 120000 } = {}) {
+ const known = new Set(intent.facts.map(fact => fact.id));
+ const run = execute(
+ executable,
+ ['--print', '--model', model, '--append-system-prompt', SYSTEM, '--max-turns', '1', '--permission-mode', 'bypassPermissions', '--strict-mcp-config'],
+ { input: judgePrompt(intent, message), encoding: 'utf8', shell: false, timeout: timeoutMs, maxBuffer: 8 * 1024 * 1024 }
+ );
+ const parsed = parseJudge(run.stdout);
+ if (!parsed || typeof parsed.reply !== 'string') {
+ // A judge failure must not look like a stonewall: the trial is marked instead.
+ return { isQuestion: false, reply: intent.stonewall, disclosed: [], judgeFailed: true };
+ }
+ return {
+ isQuestion: parsed.isQuestion === true,
+ reply: parsed.reply.slice(0, MAX_REPLY_CHARS),
+ disclosed: Array.isArray(parsed.disclosed) ? parsed.disclosed.filter(id => known.has(id)) : [],
+ judgeFailed: false
+ };
+}
+
+/** The fact ids a scenario treats as deciding the outcome. */
+function decisiveFacts(intent) {
+ return intent.facts.filter(fact => fact.decisive).map(fact => fact.id);
+}
+
+module.exports = { answer, decisiveFacts, judgePrompt, parseJudge, SYSTEM, JUDGE_MODEL };
diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js
new file mode 100644
index 000000000..28f360604
--- /dev/null
+++ b/tests/docker/gateguard-intent.test.js
@@ -0,0 +1,297 @@
+/**
+ * Tests for the hidden-intent effectiveness harness. No model is called: the
+ * session executable and the simulated user are both injected.
+ *
+ * These tests deliberately assert nothing about which question the gate raises.
+ * The previous suite did, using targets copied from the gate's own taxonomy, so
+ * it passed by construction. What is asserted here are properties that can
+ * actually fail: that scenarios are authored blind, that every grader separates
+ * the start, the trap and a correct solution, that behaviour is classified into
+ * the right hole, and that a scenario carrying no information is excluded from
+ * arm comparison instead of being reported as a result.
+ */
+
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { test } = require('node:test');
+
+const ROOT = path.join(__dirname, '..', '..');
+const DIR = path.join(ROOT, 'docker', 'gateguard-effectiveness');
+const userSim = require(path.join(DIR, 'user-sim'));
+const session = require(path.join(DIR, 'session'));
+const coverage = require(path.join(DIR, 'coverage'));
+const intentEval = require(path.join(DIR, 'intent-eval'));
+const { parseArgs } = require(path.join(DIR, 'run-intent'));
+
+const INTENT = Object.freeze({
+ id: 'probe',
+ ambiguity: 'the window and the delete mode were never stated',
+ facts: [
+ { id: 'window', fact: 'The window is 400 days.', decisive: true },
+ { id: 'mode', fact: 'Rows are marked, never removed.', decisive: true },
+ { id: 'ordering', fact: 'Order does not matter.', decisive: false }
+ ],
+ stonewall: 'No strong view - use your judgement.',
+ volunteer: false
+});
+
+function row(overrides) {
+ return {
+ scenario: 'probe', arm: 'gate', rep: 1, passed: false, asked: false,
+ disclosedDecisive: false, gateDenials: 0, questionsAsked: [], ...overrides
+ };
+}
+
+// --- scenarios are authored blind to the taxonomy ---
+
+test('every shipped hidden-intent scenario loads and declares a decisive fact', () => {
+ const scenarios = intentEval.loadIntentScenarios();
+ assert.ok(scenarios.length > 0, 'no hidden-intent scenarios found');
+ for (const scenario of scenarios) {
+ assert.ok(userSim.decisiveFacts(scenario.intent).length > 0, `${scenario.id} has no decisive fact`);
+ assert.strictEqual(scenario.targetQuestions, undefined, `${scenario.id} declares targetQuestions`);
+ }
+});
+
+test('a scenario that declares targetQuestions is rejected', () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-blind-'));
+ const scenario = path.join(dir, 'leaky');
+ fs.mkdirSync(path.join(scenario, 'workspace'), { recursive: true });
+ fs.writeFileSync(
+ path.join(scenario, 'task.json'),
+ JSON.stringify({ id: 'leaky', trap: 't', prompt: 'p', evidence: ['e'], targetQuestions: ['no-duplicate'] })
+ );
+ fs.writeFileSync(path.join(scenario, 'intent.json'), JSON.stringify(INTENT));
+ fs.writeFileSync(path.join(scenario, 'grader.cjs'), 'console.log("ECC_EVAL_SCORE {\\"score\\":0}");');
+ assert.throws(() => intentEval.loadIntentScenarios(dir), /authored blind to the taxonomy/);
+});
+
+test('an intent with no decisive fact is rejected', () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-nodecisive-'));
+ const scenario = path.join(dir, 'soft');
+ fs.mkdirSync(path.join(scenario, 'workspace'), { recursive: true });
+ fs.writeFileSync(path.join(scenario, 'task.json'), JSON.stringify({ id: 'soft', trap: 't', prompt: 'p', evidence: ['e'] }));
+ fs.writeFileSync(
+ path.join(scenario, 'intent.json'),
+ JSON.stringify({ ...INTENT, facts: [{ id: 'ordering', fact: 'Order does not matter.', decisive: false }] })
+ );
+ fs.writeFileSync(path.join(scenario, 'grader.cjs'), 'console.log("ECC_EVAL_SCORE {\\"score\\":0}");');
+ assert.throws(() => intentEval.loadIntentScenarios(dir), /no decisive fact/);
+});
+
+// --- graders must discriminate, or the trial is worthless ---
+
+test('every grader separates the start, the trap and a correct solution', () => {
+ const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-sound-'));
+ for (const scenario of intentEval.loadIntentScenarios()) {
+ const { scores, sound } = intentEval.graderIsSound(scenario, workRoot);
+ assert.ok(
+ sound,
+ `${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}`
+ );
+ }
+});
+
+// --- behaviour is classified into the right hole ---
+
+test('classify maps each behaviour to its hole', () => {
+ const cases = [
+ [{ disclosedDecisive: true, passed: true, asked: true, gateDenials: 1 }, 'working'],
+ [{ disclosedDecisive: true, passed: false, asked: true, gateDenials: 1 }, 'follow-through-hole'],
+ [{ asked: true, disclosedDecisive: false, passed: false, gateDenials: 1 }, 'targeting-hole'],
+ [{ asked: true, disclosedDecisive: false, passed: true, gateDenials: 1 }, 'targeting-hole'],
+ [{ asked: false, passed: false, gateDenials: 3 }, 'silence-hole'],
+ [{ asked: false, passed: false, gateDenials: 0 }, 'coverage-hole'],
+ [{ asked: false, passed: true, gateDenials: 0 }, 'lucky']
+ ];
+ for (const [fields, expected] of cases) {
+ assert.strictEqual(coverage.classify(row(fields)), expected, JSON.stringify(fields));
+ }
+});
+
+test('a right outcome reached without asking is lucky, not working', () => {
+ // The distinction is the whole point: a scenario that can be passed without
+ // the deciding fact cannot measure the gate.
+ assert.strictEqual(coverage.classify(row({ passed: true, asked: false, disclosedDecisive: false })), 'lucky');
+});
+
+// --- non-informative scenarios are excluded, not reported as results ---
+
+test('trap strength is read from the ungated arm alone', () => {
+ const rows = [
+ row({ scenario: 'easy', arm: 'off', passed: true }),
+ row({ scenario: 'easy', arm: 'off', passed: true }),
+ row({ scenario: 'hard', arm: 'off', passed: false }),
+ row({ scenario: 'hard', arm: 'off', passed: false }),
+ row({ scenario: 'mid', arm: 'off', passed: true }),
+ row({ scenario: 'mid', arm: 'off', passed: false })
+ ];
+ const byScenario = Object.fromEntries(coverage.difficulty(rows).map(entry => [entry.scenario, entry.label]));
+ // A trap the ungated arm always beats is no trap; anything else traps at least sometimes.
+ assert.deepStrictEqual(byScenario, { easy: 'ceiling', hard: 'traps', mid: 'traps' });
+});
+
+test('usability is judged across arms, so an ungated arm that fails is not discarded', () => {
+ // This is the case the first cut got wrong: off failing every time is exactly
+ // what a working trap looks like, and must not make the scenario unusable.
+ const rows = [
+ row({ scenario: 'works', arm: 'off', passed: false }),
+ row({ scenario: 'works', arm: 'gate', passed: true }),
+ row({ scenario: 'unreached', arm: 'off', passed: false }),
+ row({ scenario: 'unreached', arm: 'gate', passed: false }),
+ row({ scenario: 'toothless', arm: 'off', passed: true }),
+ row({ scenario: 'toothless', arm: 'gate', passed: true })
+ ];
+ const byScenario = Object.fromEntries(coverage.informative(rows).map(entry => [entry.scenario, entry.label]));
+ assert.deepStrictEqual(byScenario, { works: 'informative', unreached: 'floor', toothless: 'ceiling' });
+});
+
+test('a ceiling scenario is excluded from the outcome-class table', () => {
+ const rows = [
+ row({ scenario: 'easy', arm: 'off', passed: true }),
+ row({ scenario: 'easy', arm: 'gate', passed: true, asked: true, disclosedDecisive: true, gateDenials: 1 }),
+ row({ scenario: 'mid', arm: 'off', passed: false }),
+ row({ scenario: 'mid', arm: 'gate', passed: true, asked: true, disclosedDecisive: true, gateDenials: 1 })
+ ];
+ const report = coverage.renderHoles(rows);
+ assert.match(report, /easy: ceiling/);
+ assert.match(report, /carry no information/);
+ // Only the informative scenario's gate trial is counted as working.
+ const gateLine = report.split('\n').find(line => line.startsWith('| gate |'));
+ assert.strictEqual(gateLine.split('|')[2].trim(), '1', gateLine);
+});
+
+test('unengaged reports scenarios the gate never fired on', () => {
+ const rows = [
+ row({ scenario: 'quiet', arm: 'gate', gateDenials: 0 }),
+ row({ scenario: 'quiet', arm: 'placebo', gateDenials: 0 }),
+ row({ scenario: 'noisy', arm: 'gate', gateDenials: 2 })
+ ];
+ assert.deepStrictEqual(coverage.unengaged(rows), ['quiet']);
+});
+
+test('discovered questions are observed, never declared', () => {
+ const rows = [
+ row({ scenario: 'probe', arm: 'gate', questionsAsked: ['data-schema'] }),
+ row({ scenario: 'probe', arm: 'gate', questionsAsked: ['importers', 'data-schema'] })
+ ];
+ assert.deepStrictEqual(coverage.discoveredQuestions(rows), [
+ { scenario: 'probe', questions: ['data-schema', 'importers'] }
+ ]);
+});
+
+// --- the simulated user withholds unless actually asked ---
+
+test('parseJudge tolerates prose and fences around the object', () => {
+ assert.deepStrictEqual(userSim.parseJudge('sure thing\n```json\n{"isQuestion":true,"reply":"hi","disclosed":[]}\n```'), {
+ isQuestion: true, reply: 'hi', disclosed: []
+ });
+ assert.strictEqual(userSim.parseJudge('no json here'), null);
+});
+
+test('answer keeps only fact ids the scenario declares', () => {
+ const execute = () => ({ stdout: '{"isQuestion":true,"reply":"400 days","disclosed":["window","invented"]}', status: 0 });
+ const reply = userSim.answer(INTENT, 'how long is the window?', { execute });
+ assert.deepStrictEqual(reply.disclosed, ['window']);
+ assert.strictEqual(reply.judgeFailed, false);
+});
+
+test('an unparseable judge is a flagged failure, not a silent stonewall', () => {
+ const execute = () => ({ stdout: 'the model rambled', status: 0 });
+ const reply = userSim.answer(INTENT, 'anything?', { execute });
+ assert.strictEqual(reply.judgeFailed, true);
+ assert.strictEqual(reply.isQuestion, false);
+});
+
+// --- the conversation driver ---
+
+test('session args use session-id first, resume after, and never disable persistence', () => {
+ const first = session.sessionArgs({ model: 'm', settingsPath: 's', maxTurns: 5 }, null);
+ assert.ok(first.includes('--session-id'));
+ assert.ok(!first.includes('--resume'));
+ // --no-session-persistence would make --resume impossible, which is how the
+ // one-shot runner is configured; this guards against copying that flag over.
+ assert.ok(!first.includes('--no-session-persistence'));
+ const next = session.sessionArgs({ model: 'm', settingsPath: 's', maxTurns: 5 }, 'abc');
+ assert.deepStrictEqual(next.slice(-2), ['--resume', 'abc']);
+});
+
+function streamFor(text, { denials = 0 } = {}) {
+ const lines = [];
+ for (let i = 0; i < denials; i++) {
+ lines.push(JSON.stringify({ type: 'user', message: { content: [{ type: 'tool_result', content: '[Fact-Forcing Gate] state facts' }] } }));
+ }
+ lines.push(JSON.stringify({ type: 'assistant', message: { content: [{ type: 'text', text }] } }));
+ lines.push(JSON.stringify({ type: 'result', usage: { input_tokens: 10, output_tokens: 5 }, num_turns: 2, total_cost_usd: 0.01 }));
+ return { stdout: lines.join('\n'), status: 0 };
+}
+
+test('the conversation stops as soon as the agent stops asking', () => {
+ let calls = 0;
+ const execute = () => {
+ calls++;
+ return streamFor('done, no questions');
+ };
+ const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false });
+ const result = session.runConversation(
+ { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
+ { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 3, execute, judge }
+ );
+ assert.strictEqual(calls, 1);
+ assert.strictEqual(result.asked, false);
+ assert.strictEqual(result.disclosedDecisive, false);
+});
+
+test('a question is answered and the reply is fed back as the next turn', () => {
+ const inputs = [];
+ const execute = (file, args, options) => {
+ inputs.push(options.input);
+ return streamFor(inputs.length === 1 ? 'how long is the window?' : 'done', { denials: 1 });
+ };
+ let asked = 0;
+ const judge = () => {
+ asked++;
+ return asked === 1
+ ? { isQuestion: true, reply: '400 days, and mark them', disclosed: ['window', 'mode'], judgeFailed: false }
+ : { isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false };
+ };
+ const result = session.runConversation(
+ { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'build it', intent: INTENT },
+ { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 3, execute, judge }
+ );
+ assert.match(inputs[0], /build it/);
+ assert.strictEqual(inputs[1], '400 days, and mark them');
+ assert.strictEqual(result.asked, true);
+ assert.strictEqual(result.disclosedDecisive, true);
+ assert.strictEqual(result.userTurns, 1);
+ assert.strictEqual(result.gateDenials, 2);
+});
+
+test('disclosedDecisive needs every decisive fact, not just one', () => {
+ const execute = () => streamFor('how long is the window?');
+ const judge = () => ({ isQuestion: true, reply: '400 days', disclosed: ['window'], judgeFailed: false });
+ const result = session.runConversation(
+ { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
+ { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge }
+ );
+ assert.deepStrictEqual(result.disclosed, ['window']);
+ assert.strictEqual(result.disclosedDecisive, false);
+});
+
+// --- CLI guards ---
+
+test('minus-target is refused: it needs declared targets these scenarios lack', () => {
+ assert.throws(
+ () => parseArgs(['--out', 'o', '--model', 'm', '--allow-real-provider', '--arms', 'minus-target']),
+ /not supported for hidden-intent scenarios/
+ );
+});
+
+test('real sessions require the explicit flag', () => {
+ assert.throws(() => parseArgs(['--out', 'o', '--model', 'm']), /--allow-real-provider/);
+ assert.doesNotThrow(() => parseArgs(['--out', 'o', '--dry-run']));
+});