The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix: distinguish auth failures from ordinary provider discussion
haelyra6cb9f0a1625c
diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.jsindex 228e1a308..30cf59650 100644--- a/docker/gateguard-effectiveness/lib.js+++ b/docker/gateguard-effectiveness/lib.js@@ -227,6 +227,13 @@ function parseStream(stdout) {return parsed;}+/** Recognize sign-in failure diagnostics, not ordinary authentication discussion. */+function isAuthenticationFailure(message) {+ if (typeof message !== 'string') return false;+ return /^\s*(?:(?:API\s+)?Error:\s*)?(?:Not\s+(?:logged|signed)\s+in|Authentication\s+(?:is\s+)?(?:failed|required)|OAuth(?:\s+(?:token|authentication))?\s+(?:failed|expired|invalid|required)|(?:Login|Sign-in)\s+(?:failed|required)|Invalid\s+API\s+key)(?=\s*(?:$|[.:;!()\u00b7-])|[\r\n])/i.test(message)+ || /^\s*(?:Please|Must)\s+(?:run\s+\/login|log\s+in|sign\s+in)\b/i.test(message);+}+function readMetrics(stateDir) {const file = path.join(stateDir, 'metrics.jsonl');if (!fs.existsSync(file)) return [];@@ -461,6 +468,7 @@ module.exports = {childEnv,claudeArgs,parseStream,+ isAuthenticationFailure,readMetrics,hookWasObserved,evidenceRecall,diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.jsindex 0fd0f07fa..22c3a5252 100644--- a/docker/gateguard-effectiveness/run-intent.js+++ b/docker/gateguard-effectiveness/run-intent.js@@ -13,7 +13,7 @@ const crypto = require('crypto');const os = require('os');const path = require('path');const { spawnSync } = require('child_process');-const { schedule, completedTrialKeys } = require('./lib');+const { schedule, completedTrialKeys, isAuthenticationFailure } = require('./lib');const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');const { renderHoles, difficulty, isValidTrial } = require('./coverage');@@ -356,7 +356,7 @@ function main() {fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });// An unauthenticated or hookless run must not be recorded: graded as data it// looks like a finding (the first such trial was filed as a coverage hole).- if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {+ if (row.providerError && isAuthenticationFailure(row.providerMessage)) {throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);}if (!row.hookObserved) {diff --git a/docker/gateguard-effectiveness/run.js b/docker/gateguard-effectiveness/run.jsindex e0284e943..3c6e9d2e9 100644--- a/docker/gateguard-effectiveness/run.js+++ b/docker/gateguard-effectiveness/run.js@@ -7,7 +7,7 @@ const os = require('os');const path = require('path');const { spawnSync } = require('child_process');const { ARMS, DEFAULT_ARMS, resolveRef, materializeTree, armSettings } = require('./arms');-const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys } = require('./lib');+const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys, isAuthenticationFailure } = require('./lib');const REPO_ROOT = path.join(__dirname, '..', '..');@@ -202,7 +202,7 @@ function main() {};fs.writeFileSync(path.join(options.out, row.transcript), transcript);fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });- if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {+ if (row.providerError && isAuthenticationFailure(row.providerMessage)) {throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);}if (!row.hookObserved) throw new Error(`${trial.key}: the gate did not run although the agent edited files; check the hook setup before spending more sessions`);diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.jsindex 48b33f023..9d4e4ba1c 100644--- a/tests/docker/gateguard-effectiveness.test.js+++ b/tests/docker/gateguard-effectiveness.test.js@@ -317,6 +317,24 @@ if (gnuTarDir) {});}+test('authentication abort recognizes failure diagnostics rather than ordinary discussion', () => {+ const failures = [+ 'Not logged in - Please run /login', 'Authentication failed: OAuth token expired',+ 'OAuth authentication failed', 'OAuth token expired; sign in again',+ 'Authentication is required', 'Login failed', 'Error: Not signed in.',+ 'Invalid API key · Please run /login', 'Please run /login', 'Must sign in'+ ];+ const ordinary = [+ 'Payment API key authentication uses the Authorization header.',+ 'OAuth tokens authorize payment API requests.', 'Login routes use OAuth.',+ 'Authentication uses a bearer token.', 'How to use OAuth login',+ 'Please explain login flows', 'Authentication failed examples belong in docs',+ 'Not logged in errors are covered in tests', '', null+ ];+ for (const text of failures) assert.strictEqual(lib.isAuthenticationFailure(text), true, text);+ for (const text of ordinary) assert.strictEqual(lib.isAuthenticationFailure(text), false, String(text));+});+test('pilot summaries count valid trials only and report provider errors apart', () => {const row = (arm, rep, passed, extra = {}) => ({key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,@@ -442,6 +460,24 @@ if (gitAvailable()) {} finally { fs.rmSync(root, { recursive: true, force: true }); }});+ test('ordinary authentication discussion remains a failed-attempt row on missing result', () => {+ for (const text of ['Payment API key authentication uses the Authorization header.', 'OAuth tokens authorize payment API requests.']) {+ const root = tempDir('gg-eff-auth-discussion-');+ try {+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');+ const events = [{ type: 'assistant', message: { content: [{ type: 'text', text }] } }];+ const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events);+ const result = run({});+ assert.strictEqual(result.status, 0, result.stderr);+ assert.strictEqual(results().length, 1);+ assert.strictEqual(results()[0].providerError, true);+ assert.strictEqual(results()[0].providerMessage, text);+ assert.strictEqual(fs.readdirSync(path.join(out, 'transcripts')).length, 1);+ assert.ok(!fs.existsSync(path.join(out, '.run.lock')));+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+ }+ });+test('the pilot runner stops before recording a gated trial whose hook never ran', () => {const root = tempDir('gg-eff-hookless-');try {