Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

feat: agent compression, inspection logic, governance hooks (#491, #485, #482) (#688)

Affaan Mustafa0b0b66c02fa1
diff --git a/hooks/hooks.json b/hooks/hooks.json
index 24bc248a7..a3d23262d 100644
--- a/hooks/hooks.json
+++ b/hooks/hooks.json
@@ -74,6 +74,17 @@
}
],
"description": "Optional InsAIts AI security monitor for Bash/Edit/Write flows. Enable with ECC_ENABLE_INSAITS=1. Requires: pip install insa-its"
+ },
+ {
+ "matcher": "Bash|Write|Edit|MultiEdit",
+ "hooks": [
+ {
+ "type": "command",
+ "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags.js\" \"pre:governance-capture\" \"scripts/hooks/governance-capture.js\" \"standard,strict\"",
+ "timeout": 10
+ }
+ ],
+ "description": "Capture governance events (secrets, policy violations, approval requests). Enable with ECC_GOVERNANCE_CAPTURE=1"
}
],
"PreCompact": [
@@ -165,6 +176,17 @@
],
"description": "Warn about console.log statements after edits"
},
+ {
+ "matcher": "Bash|Write|Edit|MultiEdit",
+ "hooks": [
+ {
+ "type": "command",
+ "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags.js\" \"post:governance-capture\" \"scripts/hooks/governance-capture.js\" \"standard,strict\"",
+ "timeout": 10
+ }
+ ],
+ "description": "Capture governance events from tool outputs. Enable with ECC_GOVERNANCE_CAPTURE=1"
+ },
{
"matcher": "*",
"hooks": [
diff --git a/scripts/hooks/governance-capture.js b/scripts/hooks/governance-capture.js
new file mode 100644
index 000000000..0efec36c3
--- /dev/null
+++ b/scripts/hooks/governance-capture.js
@@ -0,0 +1,280 @@
+#!/usr/bin/env node
+/**
+ * Governance Event Capture Hook
+ *
+ * PreToolUse/PostToolUse hook that detects governance-relevant events
+ * and writes them to the governance_events table in the state store.
+ *
+ * Captured event types:
+ * - secret_detected: Hardcoded secrets in tool input/output
+ * - policy_violation: Actions that violate configured policies
+ * - security_finding: Security-relevant tool invocations
+ * - approval_requested: Operations requiring explicit approval
+ *
+ * Enable: Set ECC_GOVERNANCE_CAPTURE=1
+ * Configure session: Set ECC_SESSION_ID for session correlation
+ */
+
+'use strict';
+
+const crypto = require('crypto');
+
+const MAX_STDIN = 1024 * 1024;
+
+// Patterns that indicate potential hardcoded secrets
+const SECRET_PATTERNS = [
+ { name: 'aws_key', pattern: /(?:AKIA|ASIA)[A-Z0-9]{16}/i },
+ { name: 'generic_secret', pattern: /(?:secret|password|token|api[_-]?key)\s*[:=]\s*["'][^"']{8,}/i },
+ { name: 'private_key', pattern: /-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----/ },
+ { name: 'jwt', pattern: /eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/ },
+ { name: 'github_token', pattern: /gh[pousr]_[A-Za-z0-9_]{36,}/ },
+];
+
+// Tool names that represent security-relevant operations
+const SECURITY_RELEVANT_TOOLS = new Set([
+ 'Bash', // Could execute arbitrary commands
+]);
+
+// Commands that require governance approval
+const APPROVAL_COMMANDS = [
+ /git\s+push\s+.*--force/,
+ /git\s+reset\s+--hard/,
+ /rm\s+-rf?\s/,
+ /DROP\s+(?:TABLE|DATABASE)/i,
+ /DELETE\s+FROM\s+\w+\s*(?:;|$)/i,
+];
+
+// File patterns that indicate policy-sensitive paths
+const SENSITIVE_PATHS = [
+ /\.env(?:\.|$)/,
+ /credentials/i,
+ /secrets?\./i,
+ /\.pem$/,
+ /\.key$/,
+ /id_rsa/,
+];
+
+/**
+ * Generate a unique event ID.
+ */
+function generateEventId() {
+ return `gov-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
+}
+
+/**
+ * Scan text content for hardcoded secrets.
+ * Returns array of { name, match } for each detected secret.
+ */
+function detectSecrets(text) {
+ if (!text || typeof text !== 'string') return [];
+
+ const findings = [];
+ for (const { name, pattern } of SECRET_PATTERNS) {
+ if (pattern.test(text)) {
+ findings.push({ name });
+ }
+ }
+ return findings;
+}
+
+/**
+ * Check if a command requires governance approval.
+ */
+function detectApprovalRequired(command) {
+ if (!command || typeof command !== 'string') return [];
+
+ const findings = [];
+ for (const pattern of APPROVAL_COMMANDS) {
+ if (pattern.test(command)) {
+ findings.push({ pattern: pattern.source });
+ }
+ }
+ return findings;
+}
+
+/**
+ * Check if a file path is policy-sensitive.
+ */
+function detectSensitivePath(filePath) {
+ if (!filePath || typeof filePath !== 'string') return false;
+
+ return SENSITIVE_PATHS.some(pattern => pattern.test(filePath));
+}
+
+/**
+ * Analyze a hook input payload and return governance events to capture.
+ *
+ * @param {Object} input - Parsed hook input (tool_name, tool_input, tool_output)
+ * @param {Object} [context] - Additional context (sessionId, hookPhase)
+ * @returns {Array<Object>} Array of governance event objects
+ */
+function analyzeForGovernanceEvents(input, context = {}) {
+ const events = [];
+ const toolName = input.tool_name || '';
+ const toolInput = input.tool_input || {};
+ const toolOutput = typeof input.tool_output === 'string' ? input.tool_output : '';
+ const sessionId = context.sessionId || null;
+ const hookPhase = context.hookPhase || 'unknown';
+
+ // 1. Secret detection in tool input content
+ const inputText = typeof toolInput === 'object'
+ ? JSON.stringify(toolInput)
+ : String(toolInput);
+
+ const inputSecrets = detectSecrets(inputText);
+ const outputSecrets = detectSecrets(toolOutput);
+ const allSecrets = [...inputSecrets, ...outputSecrets];
+
+ if (allSecrets.length > 0) {
+ events.push({
+ id: generateEventId(),
+ sessionId,
+ eventType: 'secret_detected',
+ payload: {
+ toolName,
+ hookPhase,
+ secretTypes: allSecrets.map(s => s.name),
+ location: inputSecrets.length > 0 ? 'input' : 'output',
+ severity: 'critical',
+ },
+ resolvedAt: null,
+ resolution: null,
+ });
+ }
+
+ // 2. Approval-required commands (Bash only)
+ if (toolName === 'Bash') {
+ const command = toolInput.command || '';
+ const approvalFindings = detectApprovalRequired(command);
+
+ if (approvalFindings.length > 0) {
+ events.push({
+ id: generateEventId(),
+ sessionId,
+ eventType: 'approval_requested',
+ payload: {
+ toolName,
+ hookPhase,
+ command: command.slice(0, 200),
+ matchedPatterns: approvalFindings.map(f => f.pattern),
+ severity: 'high',
+ },
+ resolvedAt: null,
+ resolution: null,
+ });
+ }
+ }
+
+ // 3. Policy violation: writing to sensitive paths
+ const filePath = toolInput.file_path || toolInput.path || '';
+ if (filePath && detectSensitivePath(filePath)) {
+ events.push({
+ id: generateEventId(),
+ sessionId,
+ eventType: 'policy_violation',
+ payload: {
+ toolName,
+ hookPhase,
+ filePath: filePath.slice(0, 200),
+ reason: 'sensitive_file_access',
+ severity: 'warning',
+ },
+ resolvedAt: null,
+ resolution: null,
+ });
+ }
+
+ // 4. Security-relevant tool usage tracking
+ if (SECURITY_RELEVANT_TOOLS.has(toolName) && hookPhase === 'post') {
+ const command = toolInput.command || '';
+ const hasElevated = /sudo\s/.test(command) || /chmod\s/.test(command) || /chown\s/.test(command);
+
+ if (hasElevated) {
+ events.push({
+ id: generateEventId(),
+ sessionId,
+ eventType: 'security_finding',
+ payload: {
+ toolName,
+ hookPhase,
+ command: command.slice(0, 200),
+ reason: 'elevated_privilege_command',
+ severity: 'medium',
+ },
+ resolvedAt: null,
+ resolution: null,
+ });
+ }
+ }
+
+ return events;
+}
+
+/**
+ * Core hook logic — exported so run-with-flags.js can call directly.
+ *
+ * @param {string} rawInput - Raw JSON string from stdin
+ * @returns {string} The original input (pass-through)
+ */
+function run(rawInput) {
+ // Gate on feature flag
+ if (String(process.env.ECC_GOVERNANCE_CAPTURE || '').toLowerCase() !== '1') {
+ return rawInput;
+ }
+
+ try {
+ const input = JSON.parse(rawInput);
+ const sessionId = process.env.ECC_SESSION_ID || null;
+ const hookPhase = process.env.CLAUDE_HOOK_EVENT_NAME || 'unknown';
+
+ const events = analyzeForGovernanceEvents(input, {
+ sessionId,
+ hookPhase: hookPhase.startsWith('Pre') ? 'pre' : 'post',
+ });
+
+ if (events.length > 0) {
+ // Write events to stderr as JSON-lines for the caller to capture.
+ // The state store write is async and handled by a separate process
+ // to avoid blocking the hook pipeline.
+ for (const event of events) {
+ process.stderr.write(
+ `[governance] ${JSON.stringify(event)}\n`
+ );
+ }
+ }
+ } catch {
+ // Silently ignore parse errors — never block the tool pipeline.
+ }
+
+ return rawInput;
+}
+
+// ── stdin entry point ────────────────────────────────
+if (require.main === module) {
+ let raw = '';
+ process.stdin.setEncoding('utf8');
+ process.stdin.on('data', chunk => {
+ if (raw.length < MAX_STDIN) {
+ const remaining = MAX_STDIN - raw.length;
+ raw += chunk.substring(0, remaining);
+ }
+ });
+
+ process.stdin.on('end', () => {
+ const result = run(raw);
+ process.stdout.write(result);
+ });
+}
+
+module.exports = {
+ APPROVAL_COMMANDS,
+ SECRET_PATTERNS,
+ SECURITY_RELEVANT_TOOLS,
+ SENSITIVE_PATHS,
+ analyzeForGovernanceEvents,
+ detectApprovalRequired,
+ detectSecrets,
+ detectSensitivePath,
+ generateEventId,
+ run,
+};
diff --git a/scripts/lib/agent-compress.js b/scripts/lib/agent-compress.js
new file mode 100644
index 000000000..c605a053e
--- /dev/null
+++ b/scripts/lib/agent-compress.js
@@ -0,0 +1,230 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Parse YAML frontmatter from a markdown string.
+ * Returns { frontmatter: {}, body: string }.
+ */
+function parseFrontmatter(content) {
+ const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
+ if (!match) {
+ return { frontmatter: {}, body: content };
+ }
+
+ const frontmatter = {};
+ for (const line of match[1].split('\n')) {
+ const colonIdx = line.indexOf(':');
+ if (colonIdx === -1) continue;
+
+ const key = line.slice(0, colonIdx).trim();
+ let value = line.slice(colonIdx + 1).trim();
+
+ // Handle JSON arrays (e.g. tools: ["Read", "Grep"])
+ if (value.startsWith('[') && value.endsWith(']')) {
+ try {
+ value = JSON.parse(value);
+ } catch {
+ // keep as string
+ }
+ }
+
+ // Strip surrounding quotes
+ if (typeof value === 'string' && value.startsWith('"') && value.endsWith('"')) {
+ value = value.slice(1, -1);
+ }
+
+ frontmatter[key] = value;
+ }
+
+ return { frontmatter, body: match[2] };
+}
+
+/**
+ * Extract the first meaningful paragraph from agent body as a summary.
+ * Skips headings and blank lines, returns up to maxSentences sentences.
+ */
+function extractSummary(body, maxSentences = 1) {
+ const lines = body.split('\n');
+ const paragraphs = [];
+ let current = [];
+
+ for (const line of lines) {
+ const trimmed = line.trim();
+
+ if (trimmed === '') {
+ if (current.length > 0) {
+ paragraphs.push(current.join(' '));
+ current = [];
+ }
+ continue;
+ }
+
+ // Skip headings
+ if (trimmed.startsWith('#')) {
+ if (current.length > 0) {
+ paragraphs.push(current.join(' '));
+ current = [];
+ }
+ continue;
+ }
+
+ // Skip list items, code blocks, etc.
+ if (trimmed.startsWith('```') || trimmed.startsWith('- **') || trimmed.startsWith('|')) {
+ continue;
+ }
+
+ current.push(trimmed);
+ }
+ if (current.length > 0) {
+ paragraphs.push(current.join(' '));
+ }
+
+ // Find first non-empty paragraph
+ const firstParagraph = paragraphs.find(p => p.length > 0);
+ if (!firstParagraph) {
+ return '';
+ }
+
+ // Extract up to maxSentences sentences
+ const sentences = firstParagraph.match(/[^.!?]+[.!?]+/g) || [firstParagraph];
+ return sentences.slice(0, maxSentences).join(' ').trim();
+}
+
+/**
+ * Load and parse a single agent file.
+ * Returns the full agent object with frontmatter and body.
+ */
+function loadAgent(filePath) {
+ const content = fs.readFileSync(filePath, 'utf8');
+ const { frontmatter, body } = parseFrontmatter(content);
+ const fileName = path.basename(filePath, '.md');
+
+ return {
+ fileName,
+ name: frontmatter.name || fileName,
+ description: frontmatter.description || '',
+ tools: Array.isArray(frontmatter.tools) ? frontmatter.tools : [],
+ model: frontmatter.model || 'sonnet',
+ body,
+ byteSize: Buffer.byteLength(content, 'utf8'),
+ };
+}
+
+/**
+ * Load all agents from a directory.
+ */
+function loadAgents(agentsDir) {
+ if (!fs.existsSync(agentsDir)) {
+ return [];
+ }
+
+ return fs.readdirSync(agentsDir)
+ .filter(f => f.endsWith('.md'))
+ .sort()
+ .map(f => loadAgent(path.join(agentsDir, f)));
+}
+
+/**
+ * Compress an agent to its catalog entry (metadata only).
+ * This is the minimal representation needed for agent selection.
+ */
+function compressToCatalog(agent) {
+ return {
+ name: agent.name,
+ description: agent.description,
+ tools: agent.tools,
+ model: agent.model,
+ };
+}
+
+/**
+ * Compress an agent to a summary entry (metadata + first paragraph).
+ * More context than catalog, less than full body.
+ */
+function compressToSummary(agent) {
+ return {
+ name: agent.name,
+ description: agent.description,
+ tools: agent.tools,
+ model: agent.model,
+ summary: extractSummary(agent.body),
+ };
+}
+
+/**
+ * Build a full compressed catalog from a directory of agents.
+ *
+ * Modes:
+ * - 'catalog': name, description, tools, model only (~2-3k tokens for 27 agents)
+ * - 'summary': catalog + first paragraph summary (~4-5k tokens)
+ * - 'full': no compression, full body included
+ *
+ * Returns { agents: [], stats: { totalAgents, originalBytes, compressedTokenEstimate } }
+ */
+function buildAgentCatalog(agentsDir, options = {}) {
+ const mode = options.mode || 'catalog';
+ const filter = options.filter || null;
+
+ let agents = loadAgents(agentsDir);
+
+ if (typeof filter === 'function') {
+ agents = agents.filter(filter);
+ }
+
+ const originalBytes = agents.reduce((sum, a) => sum + a.byteSize, 0);
+
+ let compressed;
+ if (mode === 'catalog') {
+ compressed = agents.map(compressToCatalog);
+ } else if (mode === 'summary') {
+ compressed = agents.map(compressToSummary);
+ } else {
+ compressed = agents.map(a => ({
+ name: a.name,
+ description: a.description,
+ tools: a.tools,
+ model: a.model,
+ body: a.body,
+ }));
+ }
+
+ const compressedJson = JSON.stringify(compressed);
+ // Rough token estimate: ~4 chars per token for English text
+ const compressedTokenEstimate = Math.ceil(compressedJson.length / 4);
+
+ return {
+ agents: compressed,
+ stats: {
+ totalAgents: agents.length,
+ originalBytes,
+ compressedBytes: Buffer.byteLength(compressedJson, 'utf8'),
+ compressedTokenEstimate,
+ mode,
+ },
+ };
+}
+
+/**
+ * Lazy-load a single agent's full content by name from a directory.
+ * Returns null if not found.
+ */
+function lazyLoadAgent(agentsDir, agentName) {
+ const filePath = path.join(agentsDir, `${agentName}.md`);
+ if (!fs.existsSync(filePath)) {
+ return null;
+ }
+ return loadAgent(filePath);
+}
+
+module.exports = {
+ buildAgentCatalog,
+ compressToCatalog,
+ compressToSummary,
+ extractSummary,
+ lazyLoadAgent,
+ loadAgent,
+ loadAgents,
+ parseFrontmatter,
+};
diff --git a/scripts/lib/inspection.js b/scripts/lib/inspection.js
new file mode 100644
index 000000000..e6f2cdf33
--- /dev/null
+++ b/scripts/lib/inspection.js
@@ -0,0 +1,212 @@
+'use strict';
+
+const DEFAULT_FAILURE_THRESHOLD = 3;
+const DEFAULT_WINDOW_SIZE = 50;
+
+const FAILURE_OUTCOMES = new Set(['failure', 'failed', 'error']);
+
+/**
+ * Normalize a failure reason string for grouping.
+ * Strips timestamps, UUIDs, file paths, and numeric suffixes.
+ */
+function normalizeFailureReason(reason) {
+ if (!reason || typeof reason !== 'string') {
+ return 'unknown';
+ }
+
+ return reason
+ .trim()
+ .toLowerCase()
+ // Strip ISO timestamps (note: already lowercased, so t/z not T/Z)
+ .replace(/\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}[.\dz]*/g, '<timestamp>')
+ // Strip UUIDs (already lowercased)
+ .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, '<uuid>')
+ // Strip file paths
+ .replace(/\/[\w./-]+/g, '<path>')
+ // Collapse whitespace
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+/**
+ * Group skill runs by skill ID and normalized failure reason.
+ *
+ * @param {Array} skillRuns - Array of skill run objects
+ * @returns {Map<string, { skillId: string, normalizedReason: string, runs: Array }>}
+ */
+function groupFailures(skillRuns) {
+ const groups = new Map();
+
+ for (const run of skillRuns) {
+ const outcome = String(run.outcome || '').toLowerCase();
+ if (!FAILURE_OUTCOMES.has(outcome)) {
+ continue;
+ }
+
+ const normalizedReason = normalizeFailureReason(run.failureReason);
+ const key = `${run.skillId}::${normalizedReason}`;
+
+ if (!groups.has(key)) {
+ groups.set(key, {
+ skillId: run.skillId,
+ normalizedReason,
+ runs: [],
+ });
+ }
+
+ groups.get(key).runs.push(run);
+ }
+
+ return groups;
+}
+
+/**
+ * Detect recurring failure patterns from skill runs.
+ *
+ * @param {Array} skillRuns - Array of skill run objects (newest first)
+ * @param {Object} [options]
+ * @param {number} [options.threshold=3] - Minimum failure count to trigger pattern detection
+ * @returns {Array<Object>} Array of detected patterns sorted by count descending
+ */
+function detectPatterns(skillRuns, options = {}) {
+ const threshold = options.threshold ?? DEFAULT_FAILURE_THRESHOLD;
+ const groups = groupFailures(skillRuns);
+ const patterns = [];
+
+ for (const [, group] of groups) {
+ if (group.runs.length < threshold) {
+ continue;
+ }
+
+ const sortedRuns = [...group.runs].sort(
+ (a, b) => (b.createdAt || '').localeCompare(a.createdAt || '')
+ );
+
+ const firstSeen = sortedRuns[sortedRuns.length - 1].createdAt || null;
+ const lastSeen = sortedRuns[0].createdAt || null;
+ const sessionIds = [...new Set(sortedRuns.map(r => r.sessionId).filter(Boolean))];
+ const versions = [...new Set(sortedRuns.map(r => r.skillVersion).filter(Boolean))];
+
+ // Collect unique raw failure reasons for this normalized group
+ const rawReasons = [...new Set(sortedRuns.map(r => r.failureReason).filter(Boolean))];
+
+ patterns.push({
+ skillId: group.skillId,
+ normalizedReason: group.normalizedReason,
+ count: group.runs.length,
+ firstSeen,
+ lastSeen,
+ sessionIds,
+ versions,
+ rawReasons,
+ runIds: sortedRuns.map(r => r.id),
+ });
+ }
+
+ // Sort by count descending, then by lastSeen descending
+ return patterns.sort((a, b) => {
+ if (b.count !== a.count) return b.count - a.count;
+ return (b.lastSeen || '').localeCompare(a.lastSeen || '');
+ });
+}
+
+/**
+ * Generate an inspection report from detected patterns.
+ *
+ * @param {Array} patterns - Output from detectPatterns()
+ * @param {Object} [options]
+ * @param {string} [options.generatedAt] - ISO timestamp for the report
+ * @returns {Object} Inspection report
+ */
+function generateReport(patterns, options = {}) {
+ const generatedAt = options.generatedAt || new Date().toISOString();
+
+ if (patterns.length === 0) {
+ return {
+ generatedAt,
+ status: 'clean',
+ patternCount: 0,
+ patterns: [],
+ summary: 'No recurring failure patterns detected.',
+ };
+ }
+
+ const totalFailures = patterns.reduce((sum, p) => sum + p.count, 0);
+ const affectedSkills = [...new Set(patterns.map(p => p.skillId))];
+
+ return {
+ generatedAt,
+ status: 'attention_needed',
+ patternCount: patterns.length,
+ totalFailures,
+ affectedSkills,
+ patterns: patterns.map(p => ({
+ skillId: p.skillId,
+ normalizedReason: p.normalizedReason,
+ count: p.count,
+ firstSeen: p.firstSeen,
+ lastSeen: p.lastSeen,
+ sessionIds: p.sessionIds,
+ versions: p.versions,
+ rawReasons: p.rawReasons.slice(0, 5),
+ suggestedAction: suggestAction(p),
+ })),
+ summary: `Found ${patterns.length} recurring failure pattern(s) across ${affectedSkills.length} skill(s) (${totalFailures} total failures).`,
+ };
+}
+
+/**
+ * Suggest a remediation action based on pattern characteristics.
+ */
+function suggestAction(pattern) {
+ const reason = pattern.normalizedReason;
+
+ if (reason.includes('timeout')) {
+ return 'Increase timeout or optimize skill execution time.';
+ }
+ if (reason.includes('permission') || reason.includes('denied') || reason.includes('auth')) {
+ return 'Check tool permissions and authentication configuration.';
+ }
+ if (reason.includes('not found') || reason.includes('missing')) {
+ return 'Verify required files/dependencies exist before skill execution.';
+ }
+ if (reason.includes('parse') || reason.includes('syntax') || reason.includes('json')) {
+ return 'Review input/output format expectations and add validation.';
+ }
+ if (pattern.versions.length > 1) {
+ return 'Failure spans multiple versions. Consider rollback to last stable version.';
+ }
+
+ return 'Investigate root cause and consider adding error handling.';
+}
+
+/**
+ * Run full inspection pipeline: query skill runs, detect patterns, generate report.
+ *
+ * @param {Object} store - State store instance with listRecentSessions, getSessionDetail
+ * @param {Object} [options]
+ * @param {number} [options.threshold] - Minimum failure count
+ * @param {number} [options.windowSize] - Number of recent skill runs to analyze
+ * @returns {Object} Inspection report
+ */
+function inspect(store, options = {}) {
+ const windowSize = options.windowSize ?? DEFAULT_WINDOW_SIZE;
+ const threshold = options.threshold ?? DEFAULT_FAILURE_THRESHOLD;
+
+ const status = store.getStatus({ recentSkillRunLimit: windowSize });
+ const skillRuns = status.skillRuns.recent || [];
+
+ const patterns = detectPatterns(skillRuns, { threshold });
+ return generateReport(patterns, { generatedAt: status.generatedAt });
+}
+
+module.exports = {
+ DEFAULT_FAILURE_THRESHOLD,
+ DEFAULT_WINDOW_SIZE,
+ detectPatterns,
+ generateReport,
+ groupFailures,
+ inspect,
+ normalizeFailureReason,
+ suggestAction,
+};
diff --git a/tests/hooks/governance-capture.test.js b/tests/hooks/governance-capture.test.js
new file mode 100644
index 000000000..d7b11e40c
--- /dev/null
+++ b/tests/hooks/governance-capture.test.js
@@ -0,0 +1,294 @@
+/**
+ * Tests for governance event capture hook.
+ */
+
+const assert = require('assert');
+
+const {
+ detectSecrets,
+ detectApprovalRequired,
+ detectSensitivePath,
+ analyzeForGovernanceEvents,
+ run,
+} = require('../../scripts/hooks/governance-capture');
+
+async function test(name, fn) {
+ try {
+ await fn();
+ console.log(` \u2713 ${name}`);
+ return true;
+ } catch (error) {
+ console.log(` \u2717 ${name}`);
+ console.log(` Error: ${error.message}`);
+ return false;
+ }
+}
+
+async function runTests() {
+ console.log('\n=== Testing governance-capture ===\n');
+
+ let passed = 0;
+ let failed = 0;
+
+ // ── detectSecrets ──────────────────────────────────────────
+
+ if (await test('detectSecrets finds AWS access keys', async () => {
+ const findings = detectSecrets('my key is AKIAIOSFODNN7EXAMPLE');
+ assert.ok(findings.length > 0);
+ assert.ok(findings.some(f => f.name === 'aws_key'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSecrets finds generic secrets', async () => {
+ const findings = detectSecrets('api_key = "sk-proj-abcdefghij1234567890"');
+ assert.ok(findings.length > 0);
+ assert.ok(findings.some(f => f.name === 'generic_secret'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSecrets finds private keys', async () => {
+ const findings = detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMIIE...');
+ assert.ok(findings.length > 0);
+ assert.ok(findings.some(f => f.name === 'private_key'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSecrets finds GitHub tokens', async () => {
+ const findings = detectSecrets('token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij');
+ assert.ok(findings.length > 0);
+ assert.ok(findings.some(f => f.name === 'github_token'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSecrets returns empty array for clean text', async () => {
+ const findings = detectSecrets('This is a normal log message with no secrets.');
+ assert.strictEqual(findings.length, 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSecrets handles null and undefined', async () => {
+ assert.deepStrictEqual(detectSecrets(null), []);
+ assert.deepStrictEqual(detectSecrets(undefined), []);
+ assert.deepStrictEqual(detectSecrets(''), []);
+ })) passed += 1; else failed += 1;
+
+ // ── detectApprovalRequired ─────────────────────────────────
+
+ if (await test('detectApprovalRequired flags force push', async () => {
+ const findings = detectApprovalRequired('git push origin main --force');
+ assert.ok(findings.length > 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectApprovalRequired flags hard reset', async () => {
+ const findings = detectApprovalRequired('git reset --hard HEAD~3');
+ assert.ok(findings.length > 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectApprovalRequired flags rm -rf', async () => {
+ const findings = detectApprovalRequired('rm -rf /tmp/important');
+ assert.ok(findings.length > 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectApprovalRequired flags DROP TABLE', async () => {
+ const findings = detectApprovalRequired('DROP TABLE users');
+ assert.ok(findings.length > 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectApprovalRequired allows safe commands', async () => {
+ const findings = detectApprovalRequired('git status');
+ assert.strictEqual(findings.length, 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectApprovalRequired handles null', async () => {
+ assert.deepStrictEqual(detectApprovalRequired(null), []);
+ assert.deepStrictEqual(detectApprovalRequired(''), []);
+ })) passed += 1; else failed += 1;
+
+ // ── detectSensitivePath ────────────────────────────────────
+
+ if (await test('detectSensitivePath identifies .env files', async () => {
+ assert.ok(detectSensitivePath('.env'));
+ assert.ok(detectSensitivePath('.env.local'));
+ assert.ok(detectSensitivePath('/project/.env.production'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSensitivePath identifies credential files', async () => {
+ assert.ok(detectSensitivePath('credentials.json'));
+ assert.ok(detectSensitivePath('/home/user/.ssh/id_rsa'));
+ assert.ok(detectSensitivePath('server.key'));
+ assert.ok(detectSensitivePath('cert.pem'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSensitivePath returns false for normal files', async () => {
+ assert.ok(!detectSensitivePath('index.js'));
+ assert.ok(!detectSensitivePath('README.md'));
+ assert.ok(!detectSensitivePath('package.json'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectSensitivePath handles null', async () => {
+ assert.ok(!detectSensitivePath(null));
+ assert.ok(!detectSensitivePath(''));
+ })) passed += 1; else failed += 1;
+
+ // ── analyzeForGovernanceEvents ─────────────────────────────
+
+ if (await test('analyzeForGovernanceEvents detects secrets in tool input', async () => {
+ const events = analyzeForGovernanceEvents({
+ tool_name: 'Write',
+ tool_input: {
+ file_path: '/tmp/config.js',
+ content: 'const key = "AKIAIOSFODNN7EXAMPLE";',
+ },
+ });
+
+ assert.ok(events.length > 0);
+ const secretEvent = events.find(e => e.eventType === 'secret_detected');
+ assert.ok(secretEvent);
+ assert.strictEqual(secretEvent.payload.severity, 'critical');
+ })) passed += 1; else failed += 1;
+
+ if (await test('analyzeForGovernanceEvents detects approval-required commands', async () => {
+ const events = analyzeForGovernanceEvents({
+ tool_name: 'Bash',
+ tool_input: {
+ command: 'git push origin main --force',
+ },
+ });
+
+ assert.ok(events.length > 0);
+ const approvalEvent = events.find(e => e.eventType === 'approval_requested');
+ assert.ok(approvalEvent);
+ assert.strictEqual(approvalEvent.payload.severity, 'high');
+ })) passed += 1; else failed += 1;
+
+ if (await test('analyzeForGovernanceEvents detects sensitive file access', async () => {
+ const events = analyzeForGovernanceEvents({
+ tool_name: 'Edit',
+ tool_input: {
+ file_path: '/project/.env.production',
+ old_string: 'DB_URL=old',
+ new_string: 'DB_URL=new',
+ },
+ });
+
+ assert.ok(events.length > 0);
+ const policyEvent = events.find(e => e.eventType === 'policy_violation');
+ assert.ok(policyEvent);
+ assert.strictEqual(policyEvent.payload.reason, 'sensitive_file_access');
+ })) passed += 1; else failed += 1;
+
+ if (await test('analyzeForGovernanceEvents detects elevated privilege commands', async () => {
+ const events = analyzeForGovernanceEvents({
+ tool_name: 'Bash',
+ tool_input: { command: 'sudo rm -rf /etc/something' },
+ }, {
+ hookPhase: 'post',
+ });
+
+ const securityEvent = events.find(e => e.eventType === 'security_finding');
+ assert.ok(securityEvent);
+ assert.strictEqual(securityEvent.payload.reason, 'elevated_privilege_command');
+ })) passed += 1; else failed += 1;
+
+ if (await test('analyzeForGovernanceEvents returns empty for clean inputs', async () => {
+ const events = analyzeForGovernanceEvents({
+ tool_name: 'Read',
+ tool_input: { file_path: '/project/src/index.js' },
+ });
+ assert.strictEqual(events.length, 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('analyzeForGovernanceEvents populates session ID from context', async () => {
+ const events = analyzeForGovernanceEvents({
+ tool_name: 'Write',
+ tool_input: {
+ file_path: '/project/.env',
+ content: 'DB_URL=test',
+ },
+ }, {
+ sessionId: 'test-session-123',
+ });
+
+ assert.ok(events.length > 0);
+ assert.strictEqual(events[0].sessionId, 'test-session-123');
+ })) passed += 1; else failed += 1;
+
+ if (await test('analyzeForGovernanceEvents generates unique event IDs', async () => {
+ const events1 = analyzeForGovernanceEvents({
+ tool_name: 'Write',
+ tool_input: { file_path: '.env', content: '' },
+ });
+ const events2 = analyzeForGovernanceEvents({
+ tool_name: 'Write',
+ tool_input: { file_path: '.env.local', content: '' },
+ });
+
+ if (events1.length > 0 && events2.length > 0) {
+ assert.notStrictEqual(events1[0].id, events2[0].id);
+ }
+ })) passed += 1; else failed += 1;
+
+ // ── run() function ─────────────────────────────────────────
+
+ if (await test('run() passes through input when feature flag is off', async () => {
+ const original = process.env.ECC_GOVERNANCE_CAPTURE;
+ delete process.env.ECC_GOVERNANCE_CAPTURE;
+
+ try {
+ const input = JSON.stringify({ tool_name: 'Bash', tool_input: { command: 'git push --force' } });
+ const result = run(input);
+ assert.strictEqual(result, input);
+ } finally {
+ if (original !== undefined) {
+ process.env.ECC_GOVERNANCE_CAPTURE = original;
+ }
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('run() passes through input when feature flag is on', async () => {
+ const original = process.env.ECC_GOVERNANCE_CAPTURE;
+ process.env.ECC_GOVERNANCE_CAPTURE = '1';
+
+ try {
+ const input = JSON.stringify({ tool_name: 'Read', tool_input: { file_path: 'index.js' } });
+ const result = run(input);
+ assert.strictEqual(result, input);
+ } finally {
+ if (original !== undefined) {
+ process.env.ECC_GOVERNANCE_CAPTURE = original;
+ } else {
+ delete process.env.ECC_GOVERNANCE_CAPTURE;
+ }
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('run() handles invalid JSON gracefully', async () => {
+ const original = process.env.ECC_GOVERNANCE_CAPTURE;
+ process.env.ECC_GOVERNANCE_CAPTURE = '1';
+
+ try {
+ const result = run('not valid json');
+ assert.strictEqual(result, 'not valid json');
+ } finally {
+ if (original !== undefined) {
+ process.env.ECC_GOVERNANCE_CAPTURE = original;
+ } else {
+ delete process.env.ECC_GOVERNANCE_CAPTURE;
+ }
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('run() can detect multiple event types in one input', async () => {
+ // Bash command with force push AND secret in command
+ const events = analyzeForGovernanceEvents({
+ tool_name: 'Bash',
+ tool_input: {
+ command: 'API_KEY="AKIAIOSFODNN7EXAMPLE" git push --force',
+ },
+ });
+
+ const eventTypes = events.map(e => e.eventType);
+ assert.ok(eventTypes.includes('secret_detected'));
+ assert.ok(eventTypes.includes('approval_requested'));
+ })) passed += 1; else failed += 1;
+
+ console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
+ process.exit(failed > 0 ? 1 : 0);
+}
+
+runTests();
diff --git a/tests/lib/agent-compress.test.js b/tests/lib/agent-compress.test.js
new file mode 100644
index 000000000..45da6668f
--- /dev/null
+++ b/tests/lib/agent-compress.test.js
@@ -0,0 +1,293 @@
+/**
+ * Tests for agent description compression and lazy loading.
+ */
+
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const {
+ parseFrontmatter,
+ extractSummary,
+ loadAgent,
+ loadAgents,
+ compressToCatalog,
+ compressToSummary,
+ buildAgentCatalog,
+ lazyLoadAgent,
+} = require('../../scripts/lib/agent-compress');
+
+function createTempDir(prefix) {
+ return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+}
+
+function cleanupTempDir(dirPath) {
+ fs.rmSync(dirPath, { recursive: true, force: true });
+}
+
+function writeAgent(dir, name, content) {
+ fs.writeFileSync(path.join(dir, `${name}.md`), content, 'utf8');
+}
+
+const SAMPLE_AGENT = `---
+name: test-agent
+description: A test agent for unit testing purposes.
+tools: ["Read", "Grep", "Glob"]
+model: sonnet
+---
+
+You are a test agent that validates compression logic.
+
+## Your Role
+
+- Run unit tests
+- Validate compression output
+- Ensure correctness
+
+## Process
+
+### 1. Setup
+- Prepare test fixtures
+- Load agent files
+
+### 2. Validate
+Check the output format and content.
+`;
+
+const MINIMAL_AGENT = `---
+name: minimal
+description: Minimal agent.
+tools: ["Read"]
+model: haiku
+---
+
+Short body.
+`;
+
+async function test(name, fn) {
+ try {
+ await fn();
+ console.log(` \u2713 ${name}`);
+ return true;
+ } catch (error) {
+ console.log(` \u2717 ${name}`);
+ console.log(` Error: ${error.message}`);
+ return false;
+ }
+}
+
+async function runTests() {
+ console.log('\n=== Testing agent-compress ===\n');
+
+ let passed = 0;
+ let failed = 0;
+
+ if (await test('parseFrontmatter extracts YAML frontmatter and body', async () => {
+ const { frontmatter, body } = parseFrontmatter(SAMPLE_AGENT);
+ assert.strictEqual(frontmatter.name, 'test-agent');
+ assert.strictEqual(frontmatter.description, 'A test agent for unit testing purposes.');
+ assert.deepStrictEqual(frontmatter.tools, ['Read', 'Grep', 'Glob']);
+ assert.strictEqual(frontmatter.model, 'sonnet');
+ assert.ok(body.includes('You are a test agent'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('parseFrontmatter handles content without frontmatter', async () => {
+ const { frontmatter, body } = parseFrontmatter('Just a plain document.');
+ assert.deepStrictEqual(frontmatter, {});
+ assert.strictEqual(body, 'Just a plain document.');
+ })) passed += 1; else failed += 1;
+
+ if (await test('extractSummary returns the first paragraph of the body', async () => {
+ const { body } = parseFrontmatter(SAMPLE_AGENT);
+ const summary = extractSummary(body);
+ assert.ok(summary.includes('test agent'));
+ assert.ok(summary.includes('compression logic'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('extractSummary returns empty string for empty body', async () => {
+ assert.strictEqual(extractSummary(''), '');
+ assert.strictEqual(extractSummary('# Just a heading'), '');
+ })) passed += 1; else failed += 1;
+
+ if (await test('loadAgent reads and parses a single agent file', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
+ const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
+ assert.strictEqual(agent.name, 'test-agent');
+ assert.strictEqual(agent.fileName, 'test-agent');
+ assert.deepStrictEqual(agent.tools, ['Read', 'Grep', 'Glob']);
+ assert.strictEqual(agent.model, 'sonnet');
+ assert.ok(agent.byteSize > 0);
+ assert.ok(agent.body.includes('You are a test agent'));
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('loadAgents reads all .md files from a directory', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
+ writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
+ const agents = loadAgents(tmpDir);
+ assert.strictEqual(agents.length, 2);
+ assert.strictEqual(agents[0].fileName, 'agent-a');
+ assert.strictEqual(agents[1].fileName, 'agent-b');
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('loadAgents returns empty array for non-existent directory', async () => {
+ const agents = loadAgents('/tmp/nonexistent-ecc-dir-12345');
+ assert.deepStrictEqual(agents, []);
+ })) passed += 1; else failed += 1;
+
+ if (await test('compressToCatalog strips body and keeps only metadata', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
+ const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
+ const catalog = compressToCatalog(agent);
+
+ assert.strictEqual(catalog.name, 'test-agent');
+ assert.strictEqual(catalog.description, 'A test agent for unit testing purposes.');
+ assert.deepStrictEqual(catalog.tools, ['Read', 'Grep', 'Glob']);
+ assert.strictEqual(catalog.model, 'sonnet');
+ assert.strictEqual(catalog.body, undefined);
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('compressToSummary includes first paragraph summary', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
+ const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
+ const summary = compressToSummary(agent);
+
+ assert.strictEqual(summary.name, 'test-agent');
+ assert.ok(summary.summary.length > 0);
+ assert.strictEqual(summary.body, undefined);
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('buildAgentCatalog in catalog mode produces minimal output with stats', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
+ writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
+
+ const result = buildAgentCatalog(tmpDir, { mode: 'catalog' });
+ assert.strictEqual(result.agents.length, 2);
+ assert.strictEqual(result.stats.totalAgents, 2);
+ assert.strictEqual(result.stats.mode, 'catalog');
+ assert.ok(result.stats.originalBytes > 0);
+ assert.ok(result.stats.compressedBytes > 0);
+ assert.ok(result.stats.compressedBytes < result.stats.originalBytes);
+ assert.ok(result.stats.compressedTokenEstimate > 0);
+
+ // Catalog entries should not have body
+ for (const agent of result.agents) {
+ assert.strictEqual(agent.body, undefined);
+ assert.ok(agent.name);
+ assert.ok(agent.description);
+ }
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('buildAgentCatalog in summary mode includes summaries', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
+
+ const result = buildAgentCatalog(tmpDir, { mode: 'summary' });
+ assert.strictEqual(result.agents.length, 1);
+ assert.ok(result.agents[0].summary);
+ assert.strictEqual(result.agents[0].body, undefined);
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('buildAgentCatalog in full mode preserves body', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
+
+ const result = buildAgentCatalog(tmpDir, { mode: 'full' });
+ assert.strictEqual(result.agents.length, 1);
+ assert.ok(result.agents[0].body.includes('You are a test agent'));
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('buildAgentCatalog supports filter function', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
+ writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
+
+ const result = buildAgentCatalog(tmpDir, {
+ mode: 'catalog',
+ filter: agent => agent.model === 'haiku',
+ });
+ assert.strictEqual(result.agents.length, 1);
+ assert.strictEqual(result.agents[0].name, 'minimal');
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('lazyLoadAgent loads a single agent by name', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
+ writeAgent(tmpDir, 'other', MINIMAL_AGENT);
+
+ const agent = lazyLoadAgent(tmpDir, 'test-agent');
+ assert.ok(agent);
+ assert.strictEqual(agent.name, 'test-agent');
+ assert.ok(agent.body.includes('You are a test agent'));
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('lazyLoadAgent returns null for non-existent agent', async () => {
+ const tmpDir = createTempDir('ecc-agent-compress-');
+ try {
+ const agent = lazyLoadAgent(tmpDir, 'nonexistent');
+ assert.strictEqual(agent, null);
+ } finally {
+ cleanupTempDir(tmpDir);
+ }
+ })) passed += 1; else failed += 1;
+
+ if (await test('buildAgentCatalog works with real agents directory', async () => {
+ const agentsDir = path.join(__dirname, '..', '..', 'agents');
+ if (!fs.existsSync(agentsDir)) {
+ // Skip if agents dir doesn't exist (shouldn't happen in this repo)
+ return;
+ }
+
+ const result = buildAgentCatalog(agentsDir, { mode: 'catalog' });
+ assert.ok(result.agents.length > 0, 'Should find at least one agent');
+ assert.ok(result.stats.originalBytes > 0);
+ assert.ok(result.stats.compressedBytes < result.stats.originalBytes,
+ 'Catalog mode should be smaller than full agent files');
+ })) passed += 1; else failed += 1;
+
+ console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
+ process.exit(failed > 0 ? 1 : 0);
+}
+
+runTests();
diff --git a/tests/lib/inspection.test.js b/tests/lib/inspection.test.js
new file mode 100644
index 000000000..f8725f37c
--- /dev/null
+++ b/tests/lib/inspection.test.js
@@ -0,0 +1,232 @@
+/**
+ * Tests for inspection logic — pattern detection from failures.
+ */
+
+const assert = require('assert');
+
+const {
+ normalizeFailureReason,
+ groupFailures,
+ detectPatterns,
+ generateReport,
+ suggestAction,
+ DEFAULT_FAILURE_THRESHOLD,
+} = require('../../scripts/lib/inspection');
+
+async function test(name, fn) {
+ try {
+ await fn();
+ console.log(` \u2713 ${name}`);
+ return true;
+ } catch (error) {
+ console.log(` \u2717 ${name}`);
+ console.log(` Error: ${error.message}`);
+ return false;
+ }
+}
+
+function makeSkillRun(overrides = {}) {
+ return {
+ id: overrides.id || `run-${Math.random().toString(36).slice(2, 8)}`,
+ skillId: overrides.skillId || 'test-skill',
+ skillVersion: overrides.skillVersion || '1.0.0',
+ sessionId: overrides.sessionId || 'session-1',
+ taskDescription: overrides.taskDescription || 'test task',
+ outcome: overrides.outcome || 'failure',
+ failureReason: overrides.failureReason || 'generic error',
+ tokensUsed: overrides.tokensUsed || 500,
+ durationMs: overrides.durationMs || 1000,
+ userFeedback: overrides.userFeedback || null,
+ createdAt: overrides.createdAt || '2026-03-15T08:00:00.000Z',
+ };
+}
+
+async function runTests() {
+ console.log('\n=== Testing inspection ===\n');
+
+ let passed = 0;
+ let failed = 0;
+
+ if (await test('normalizeFailureReason strips timestamps and UUIDs', async () => {
+ const normalized = normalizeFailureReason(
+ 'Error at 2026-03-15T08:00:00.000Z for id 550e8400-e29b-41d4-a716-446655440000'
+ );
+ assert.ok(!normalized.includes('2026'));
+ assert.ok(!normalized.includes('550e8400'));
+ assert.ok(normalized.includes('<timestamp>'));
+ assert.ok(normalized.includes('<uuid>'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('normalizeFailureReason strips file paths', async () => {
+ const normalized = normalizeFailureReason('File not found: /usr/local/bin/node');
+ assert.ok(!normalized.includes('/usr/local'));
+ assert.ok(normalized.includes('<path>'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('normalizeFailureReason handles null and empty values', async () => {
+ assert.strictEqual(normalizeFailureReason(null), 'unknown');
+ assert.strictEqual(normalizeFailureReason(''), 'unknown');
+ assert.strictEqual(normalizeFailureReason(undefined), 'unknown');
+ })) passed += 1; else failed += 1;
+
+ if (await test('groupFailures groups by skillId and normalized reason', async () => {
+ const runs = [
+ makeSkillRun({ id: 'r1', skillId: 'skill-a', failureReason: 'timeout' }),
+ makeSkillRun({ id: 'r2', skillId: 'skill-a', failureReason: 'timeout' }),
+ makeSkillRun({ id: 'r3', skillId: 'skill-b', failureReason: 'parse error' }),
+ makeSkillRun({ id: 'r4', skillId: 'skill-a', outcome: 'success' }), // should be excluded
+ ];
+
+ const groups = groupFailures(runs);
+ assert.strictEqual(groups.size, 2);
+
+ const skillAGroup = groups.get('skill-a::timeout');
+ assert.ok(skillAGroup);
+ assert.strictEqual(skillAGroup.runs.length, 2);
+
+ const skillBGroup = groups.get('skill-b::parse error');
+ assert.ok(skillBGroup);
+ assert.strictEqual(skillBGroup.runs.length, 1);
+ })) passed += 1; else failed += 1;
+
+ if (await test('groupFailures handles mixed outcome casing', async () => {
+ const runs = [
+ makeSkillRun({ id: 'r1', outcome: 'FAILURE', failureReason: 'timeout' }),
+ makeSkillRun({ id: 'r2', outcome: 'Failed', failureReason: 'timeout' }),
+ makeSkillRun({ id: 'r3', outcome: 'error', failureReason: 'timeout' }),
+ ];
+
+ const groups = groupFailures(runs);
+ assert.strictEqual(groups.size, 1);
+ const group = groups.values().next().value;
+ assert.strictEqual(group.runs.length, 3);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectPatterns returns empty array when below threshold', async () => {
+ const runs = [
+ makeSkillRun({ id: 'r1', failureReason: 'timeout' }),
+ makeSkillRun({ id: 'r2', failureReason: 'timeout' }),
+ ];
+
+ const patterns = detectPatterns(runs, { threshold: 3 });
+ assert.strictEqual(patterns.length, 0);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectPatterns detects patterns at or above threshold', async () => {
+ const runs = [
+ makeSkillRun({ id: 'r1', failureReason: 'timeout', createdAt: '2026-03-15T08:00:00Z' }),
+ makeSkillRun({ id: 'r2', failureReason: 'timeout', createdAt: '2026-03-15T08:01:00Z' }),
+ makeSkillRun({ id: 'r3', failureReason: 'timeout', createdAt: '2026-03-15T08:02:00Z' }),
+ ];
+
+ const patterns = detectPatterns(runs, { threshold: 3 });
+ assert.strictEqual(patterns.length, 1);
+ assert.strictEqual(patterns[0].count, 3);
+ assert.strictEqual(patterns[0].skillId, 'test-skill');
+ assert.strictEqual(patterns[0].normalizedReason, 'timeout');
+ assert.strictEqual(patterns[0].firstSeen, '2026-03-15T08:00:00Z');
+ assert.strictEqual(patterns[0].lastSeen, '2026-03-15T08:02:00Z');
+ assert.strictEqual(patterns[0].runIds.length, 3);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectPatterns uses default threshold', async () => {
+ const runs = Array.from({ length: DEFAULT_FAILURE_THRESHOLD }, (_, i) =>
+ makeSkillRun({ id: `r${i}`, failureReason: 'permission denied' })
+ );
+
+ const patterns = detectPatterns(runs);
+ assert.strictEqual(patterns.length, 1);
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectPatterns sorts by count descending', async () => {
+ const runs = [
+ // 4 timeouts
+ ...Array.from({ length: 4 }, (_, i) =>
+ makeSkillRun({ id: `t${i}`, skillId: 'skill-a', failureReason: 'timeout' })
+ ),
+ // 3 parse errors
+ ...Array.from({ length: 3 }, (_, i) =>
+ makeSkillRun({ id: `p${i}`, skillId: 'skill-b', failureReason: 'parse error' })
+ ),
+ ];
+
+ const patterns = detectPatterns(runs, { threshold: 3 });
+ assert.strictEqual(patterns.length, 2);
+ assert.strictEqual(patterns[0].count, 4);
+ assert.strictEqual(patterns[0].skillId, 'skill-a');
+ assert.strictEqual(patterns[1].count, 3);
+ assert.strictEqual(patterns[1].skillId, 'skill-b');
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectPatterns groups similar failure reasons with different timestamps', async () => {
+ const runs = [
+ makeSkillRun({ id: 'r1', failureReason: 'Error at 2026-03-15T08:00:00Z in /tmp/foo' }),
+ makeSkillRun({ id: 'r2', failureReason: 'Error at 2026-03-15T09:00:00Z in /tmp/bar' }),
+ makeSkillRun({ id: 'r3', failureReason: 'Error at 2026-03-15T10:00:00Z in /tmp/baz' }),
+ ];
+
+ const patterns = detectPatterns(runs, { threshold: 3 });
+ assert.strictEqual(patterns.length, 1);
+ assert.ok(patterns[0].normalizedReason.includes('<timestamp>'));
+ assert.ok(patterns[0].normalizedReason.includes('<path>'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('detectPatterns tracks unique session IDs and versions', async () => {
+ const runs = [
+ makeSkillRun({ id: 'r1', sessionId: 'sess-1', skillVersion: '1.0.0', failureReason: 'err' }),
+ makeSkillRun({ id: 'r2', sessionId: 'sess-2', skillVersion: '1.0.0', failureReason: 'err' }),
+ makeSkillRun({ id: 'r3', sessionId: 'sess-1', skillVersion: '1.1.0', failureReason: 'err' }),
+ ];
+
+ const patterns = detectPatterns(runs, { threshold: 3 });
+ assert.strictEqual(patterns.length, 1);
+ assert.deepStrictEqual(patterns[0].sessionIds.sort(), ['sess-1', 'sess-2']);
+ assert.deepStrictEqual(patterns[0].versions.sort(), ['1.0.0', '1.1.0']);
+ })) passed += 1; else failed += 1;
+
+ if (await test('generateReport returns clean status with no patterns', async () => {
+ const report = generateReport([]);
+ assert.strictEqual(report.status, 'clean');
+ assert.strictEqual(report.patternCount, 0);
+ assert.ok(report.summary.includes('No recurring'));
+ assert.ok(report.generatedAt);
+ })) passed += 1; else failed += 1;
+
+ if (await test('generateReport produces structured report from patterns', async () => {
+ const runs = [
+ ...Array.from({ length: 3 }, (_, i) =>
+ makeSkillRun({ id: `r${i}`, skillId: 'my-skill', failureReason: 'timeout' })
+ ),
+ ];
+ const patterns = detectPatterns(runs, { threshold: 3 });
+ const report = generateReport(patterns, { generatedAt: '2026-03-15T09:00:00Z' });
+
+ assert.strictEqual(report.status, 'attention_needed');
+ assert.strictEqual(report.patternCount, 1);
+ assert.strictEqual(report.totalFailures, 3);
+ assert.deepStrictEqual(report.affectedSkills, ['my-skill']);
+ assert.strictEqual(report.patterns[0].skillId, 'my-skill');
+ assert.ok(report.patterns[0].suggestedAction);
+ assert.strictEqual(report.generatedAt, '2026-03-15T09:00:00Z');
+ })) passed += 1; else failed += 1;
+
+ if (await test('suggestAction returns timeout-specific advice', async () => {
+ const action = suggestAction({ normalizedReason: 'timeout after 30s', versions: ['1.0.0'] });
+ assert.ok(action.toLowerCase().includes('timeout'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('suggestAction returns permission-specific advice', async () => {
+ const action = suggestAction({ normalizedReason: 'permission denied', versions: ['1.0.0'] });
+ assert.ok(action.toLowerCase().includes('permission'));
+ })) passed += 1; else failed += 1;
+
+ if (await test('suggestAction returns version-span advice when multiple versions affected', async () => {
+ const action = suggestAction({ normalizedReason: 'something broke', versions: ['1.0.0', '1.1.0'] });
+ assert.ok(action.toLowerCase().includes('version'));
+ })) passed += 1; else failed += 1;
+
+ console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
+ process.exit(failed > 0 ? 1 : 0);
+}
+
+runTests();