The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(skill-evolution): wire Skill PostToolUse tracker so skill-health shows real runs (#2490)
Gaurav Dubey01e15490f04e
diff --git a/hooks/hooks.json b/hooks/hooks.jsonindex 2eb1ef3ea..35d79fd5a 100644--- a/hooks/hooks.json+++ b/hooks/hooks.json@@ -171,6 +171,17 @@],"description": "Track failed MCP tool calls, mark unhealthy servers, and attempt reconnect","id": "post:mcp-health-check"+ },+ {+ "matcher": "Skill",+ "hooks": [+ {+ "type": "command",+ "command": "node -e \"const p=require('path');const r=(function(){var p=require('path'),f=require('fs'),o=require('os');var e=process.env.CLAUDE_PLUGIN_ROOT;if(e&&e.trim())return e.trim();var d=p.join(o.homedir(),'.claude');function L(x){try{return require(p.join(x,'scripts','lib','resolve-ecc-root')).resolveEccRoot()}catch(_){return null}}var r=L(d);if(r)return r;var s=['ecc','ecc@ecc','marketplaces/ecc','everything-claude-code','everything-claude-code@everything-claude-code','marketplaces/everything-claude-code'];for(var i=0;i<s.length;i++){r=L(p.join(d,'plugins',s[i]));if(r)return r}try{var g=['ecc','everything-claude-code'];for(var j=0;j<g.length;j++){var c=p.join(d,'plugins','cache',g[j]);var O=f.readdirSync(c);for(var k=0;k<O.length;k++){var q=p.join(c,O[k]);var V=f.readdirSync(q);for(var m=0;m<V.length;m++){r=L(p.join(q,V[m]));if(r)return r}}}}catch(_){}return d})();const s=p.join(r,'scripts/hooks/plugin-hook-bootstrap.js');process.env.CLAUDE_PLUGIN_ROOT=r;process.argv.splice(1,0,s);require(s)\" node scripts/hooks/run-with-flags.js post:skill:track scripts/hooks/skill-run-tracker.js standard,strict"+ }+ ],+ "description": "Record hard Skill tool failures for skill-health telemetry",+ "id": "post:skill:track"}],"Stop": [diff --git a/scripts/hooks/posttooluse-dispatcher.js b/scripts/hooks/posttooluse-dispatcher.jsindex 027ea569f..ac4345afc 100644--- a/scripts/hooks/posttooluse-dispatcher.js+++ b/scripts/hooks/posttooluse-dispatcher.js@@ -19,6 +19,7 @@ const { run: runSessionActivityTracker } = require('./session-activity-tracker')const { run: runObserve } = require('./observe-runner');const { run: runMetricsBridge } = require('./ecc-metrics-bridge');const { run: runContextMonitor } = require('./ecc-context-monitor');+const { run: runSkillRunTracker } = require('./skill-run-tracker');const MAX_STDIN = 1024 * 1024;@@ -45,7 +46,8 @@ const ASYNC_HOOKS = [}},{ id: 'post:quality-gate', matcher: 'Edit|Write|MultiEdit', profiles: 'standard,strict', script: 'scripts/hooks/quality-gate.js', run: runQualityGate },- { id: 'post:observe:continuous-learning', matcher: '*', profiles: 'standard,strict', script: 'scripts/hooks/observe-runner.js', run: runObserve }+ { id: 'post:observe:continuous-learning', matcher: '*', profiles: 'standard,strict', script: 'scripts/hooks/observe-runner.js', run: runObserve },+ { id: 'post:skill:track', matcher: 'Skill', profiles: 'standard,strict', script: 'scripts/hooks/skill-run-tracker.js', run: runSkillRunTracker }];function getPluginRoot(env = process.env) {diff --git a/scripts/hooks/skill-run-tracker.js b/scripts/hooks/skill-run-tracker.jsnew file mode 100644index 000000000..eef54b859--- /dev/null+++ b/scripts/hooks/skill-run-tracker.js@@ -0,0 +1,149 @@+#!/usr/bin/env node+'use strict';++/**+ * PostToolUse hook: record Skill tool invocations for skill-health telemetry.+ *+ * Wires the write side of the already-shipped JSONL tracker+ * (scripts/lib/skill-evolution/tracker.js). Before this hook,+ * recordSkillExecution() had zero production callers, so+ * ~/.claude/state/skill-runs.jsonl was never written and+ * `scripts/skills-health.js --dashboard` always reported 0 runs (#2463).+ *+ * Privacy: the dashboard aggregates skill/version/outcome only, so this hook+ * never persists prompt text. `task_description` is synthesized from the skill+ * id, and every persisted string is charset-restricted and length-bounded — a+ * skill id is an identifier, not free text, so anything that does not look like+ * one is dropped rather than written through.+ *+ * Best-effort: never blocks tool execution. Runs under the PostToolUse+ * dispatcher, which owns stdin, pass-through, and exit codes.+ *+ * Cross-platform (Windows, macOS, Linux); CommonJS.+ */++const { recordSkillExecution } = require('../lib/skill-evolution/tracker');++// Bounds for persisted identifiers. Long enough for any real skill id or+// semver-ish version, short enough that a stray blob cannot ride in.+const MAX_SKILL_ID = 128;+const MAX_SKILL_VERSION = 64;++// Identifiers only: letters, digits, and the separators skill ids actually use.+// Anything else (whitespace, punctuation, newlines) means it is not an id.+const SKILL_ID_PATTERN = /^[A-Za-z0-9._:@/-]+$/;+const SKILL_VERSION_PATTERN = /^[A-Za-z0-9._+-]+$/;++// Return `value` only if it is a bounded, identifier-shaped string.+function boundedIdentifier(value, maxLength, pattern) {+ if (typeof value !== 'string') {+ return null;+ }+ const trimmed = value.trim();+ if (trimmed.length === 0 || trimmed.length > maxLength) {+ return null;+ }+ return pattern.test(trimmed) ? trimmed : null;+}++function firstIdentifier(maxLength, pattern, ...values) {+ for (const value of values) {+ const identifier = boundedIdentifier(value, maxLength, pattern);+ if (identifier) {+ return identifier;+ }+ }+ return null;+}++// Extract the skill identifier from the Skill tool input across the field+// names Claude Code has used for it. The Skill tool is genuinely un-wired in+// this repo, so no single canonical field is guaranteed — probe the plausible+// ones and bail (record nothing) if none is present.+function extractSkillId(toolInput) {+ if (typeof toolInput === 'string') {+ return boundedIdentifier(toolInput, MAX_SKILL_ID, SKILL_ID_PATTERN);+ }+ if (!toolInput || typeof toolInput !== 'object') {+ return null;+ }+ return firstIdentifier(+ MAX_SKILL_ID,+ SKILL_ID_PATTERN,+ toolInput.skill_id,+ toolInput.skillId,+ toolInput.skill,+ toolInput.name,+ toolInput.command+ );+}++// Best-effort outcome: a failed tool call is recorded as "failure", everything+// else as "success". Both PostToolUseFailure routing and an error-bearing+// tool response are treated as failure.+function deriveOutcome(payload) {+ if (payload && payload.hook_event_name === 'PostToolUseFailure') {+ return 'failure';+ }++ const response = (payload && (payload.tool_response ?? payload.tool_output)) || null;+ if (response && typeof response === 'object') {+ if (response.is_error === true || response.isError === true) {+ return 'failure';+ }+ if (typeof response.status === 'string' && /error|fail/i.test(response.status)) {+ return 'failure';+ }+ if (typeof response.error === 'string' && response.error.trim().length > 0) {+ return 'failure';+ }+ }++ return 'success';+}++function buildRecord(payload) {+ const skillId = extractSkillId(payload.tool_input);+ if (!skillId) {+ return null; // cannot satisfy the tracker's required skill_id — skip+ }++ const input = payload.tool_input && typeof payload.tool_input === 'object'+ ? payload.tool_input+ : {};++ const skillVersion = firstIdentifier(+ MAX_SKILL_VERSION,+ SKILL_VERSION_PATTERN,+ input.skill_version,+ input.skillVersion,+ input.version+ ) || 'unknown';++ return {+ skill_id: skillId,+ skill_version: skillVersion,+ // Synthesized, not user content. The tracker requires a non-empty+ // task_description; the dashboard never displays it as prose.+ task_description: `Skill invocation: ${skillId}`,+ outcome: deriveOutcome(payload),+ };+}++function run(rawInput) {+ try {+ const payload = typeof rawInput === 'string'+ ? (rawInput.trim() ? JSON.parse(rawInput) : {})+ : rawInput;+ if (payload && typeof payload === 'object' && payload.tool_name === 'Skill') {+ const record = buildRecord(payload);+ if (record) {+ recordSkillExecution(record);+ }+ }+ } catch {+ // Telemetry is best-effort; never block tool execution on a failure here.+ }+}++module.exports = { buildRecord, deriveOutcome, extractSkillId, run };diff --git a/scripts/lib/skill-evolution/tracker.js b/scripts/lib/skill-evolution/tracker.jsindex 67220eb93..0ea0ba1cc 100644--- a/scripts/lib/skill-evolution/tracker.js+++ b/scripts/lib/skill-evolution/tracker.js@@ -4,11 +4,19 @@ const fs = require('fs');const os = require('os');const path = require('path');-const { appendFile } = require('../utils');+const { ensureDir } = require('../utils');const VALID_OUTCOMES = new Set(['success', 'failure', 'partial']);const VALID_FEEDBACK = new Set(['accepted', 'corrected', 'rejected']);+// Retention bound for the JSONL sink. The dashboard only aggregates recent+// runs, so an unbounded append-only file is pure cost. Trim from the front+// once the file grows past the cap.+const MAX_RUN_RECORDS = 5000;+// Owner-only. The sink lives under the user's home and is local telemetry;+// nothing else on the machine needs to read it.+const RUNS_FILE_MODE = 0o600;+function resolveHomeDir(homeDir) {return homeDir ? path.resolve(homeDir) : os.homedir();}@@ -102,6 +110,46 @@ function readJsonl(filePath) {}, []);}+// Append one record to the JSONL sink with owner-only permissions, then+// enforce the retention cap. `fs.appendFileSync`'s mode only applies when it+// creates the file, so an existing world-readable sink is chmod'd on the way+// past — cheap, and it repairs files written before this bound existed.+function appendRunRecord(runsFilePath, record, options = {}) {+ const maxRecords = Number.isInteger(options.maxRecords) && options.maxRecords > 0+ ? options.maxRecords+ : MAX_RUN_RECORDS;++ ensureDir(path.dirname(runsFilePath));+ fs.appendFileSync(runsFilePath, `${JSON.stringify(record)}\n`, { encoding: 'utf8', mode: RUNS_FILE_MODE });++ try {+ fs.chmodSync(runsFilePath, RUNS_FILE_MODE);+ } catch {+ // Windows and some mounts do not support POSIX modes; the record still lands.+ }++ pruneRunRecords(runsFilePath, maxRecords);+}++// Keep only the newest `maxRecords` lines. Rewrites the whole file, which is+// fine because the file is bounded by this very cap; it only runs on the+// appends that actually cross the line.+function pruneRunRecords(runsFilePath, maxRecords) {+ try {+ const lines = fs.readFileSync(runsFilePath, 'utf8').split('\n').filter(Boolean);+ if (lines.length <= maxRecords) {+ return;+ }+ fs.writeFileSync(+ runsFilePath,+ `${lines.slice(-maxRecords).join('\n')}\n`,+ { encoding: 'utf8', mode: RUNS_FILE_MODE }+ );+ } catch {+ // Retention is best-effort; never fail a recorded run over it.+ }+}+function recordSkillExecution(input, options = {}) {const record = normalizeExecutionRecord(input, options);@@ -119,7 +167,7 @@ function recordSkillExecution(input, options = {}) {}const runsFilePath = getRunsFilePath(options);- appendFile(runsFilePath, `${JSON.stringify(record)}\n`);+ appendRunRecord(runsFilePath, record, options);return {storage: 'jsonl',@@ -137,6 +185,8 @@ function readSkillExecutionRecords(options = {}) {}module.exports = {+ MAX_RUN_RECORDS,+ RUNS_FILE_MODE,VALID_FEEDBACK,VALID_OUTCOMES,getRunsFilePath,diff --git a/tests/hooks/skill-run-tracker.test.js b/tests/hooks/skill-run-tracker.test.jsnew file mode 100644index 000000000..bd97b50c8--- /dev/null+++ b/tests/hooks/skill-run-tracker.test.js@@ -0,0 +1,259 @@+/**+ * Tests for scripts/hooks/skill-run-tracker.js and the JSONL sink bounds in+ * scripts/lib/skill-evolution/tracker.js (#2463).+ *+ * Focus: the tracker records real runs, and it never persists prompt text,+ * unbounded strings, an unbounded file, or a world-readable sink.+ *+ * Run with: node tests/run-all.js+ */++'use strict';++const assert = require('assert');+const fs = require('fs');+const os = require('os');+const path = require('path');++const { buildRecord, deriveOutcome, extractSkillId, run } = require('../../scripts/hooks/skill-run-tracker');+const {+ MAX_RUN_RECORDS,+ RUNS_FILE_MODE,+ getRunsFilePath,+ recordSkillExecution,+ readSkillExecutionRecords,+} = require('../../scripts/lib/skill-evolution/tracker');++let passed = 0;+let failed = 0;++function test(name, fn) {+ try {+ fn();+ console.log(` ✓ ${name}`);+ passed++;+ } catch (err) {+ console.log(` ✗ ${name}`);+ console.log(` Error: ${err.message}`);+ failed++;+ }+}++function withTempHome(fn) {+ const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-skill-runs-'));+ try {+ return fn(homeDir);+ } finally {+ fs.rmSync(homeDir, { recursive: true, force: true });+ }+}++function payload(overrides = {}) {+ return {+ hook_event_name: 'PostToolUse',+ tool_name: 'Skill',+ tool_input: { skill_id: 'code-review', skill_version: '1.2.0' },+ tool_response: {},+ ...overrides,+ };+}++// ── skill id extraction and bounds ────────────────────────────────────────────++test('extractSkillId probes the field names Claude Code has used', () => {+ assert.strictEqual(extractSkillId({ skill_id: 'a' }), 'a');+ assert.strictEqual(extractSkillId({ skillId: 'b' }), 'b');+ assert.strictEqual(extractSkillId({ skill: 'c' }), 'c');+ assert.strictEqual(extractSkillId({ name: 'd' }), 'd');+ assert.strictEqual(extractSkillId({ command: 'e' }), 'e');+ assert.strictEqual(extractSkillId('bare-string'), 'bare-string');+});++test('extractSkillId returns null when no skill id is present', () => {+ assert.strictEqual(extractSkillId({}), null);+ assert.strictEqual(extractSkillId(null), null);+ assert.strictEqual(extractSkillId(42), null);+});++test('extractSkillId rejects an over-long identifier rather than truncating it', () => {+ assert.strictEqual(extractSkillId({ skill_id: 'x'.repeat(129) }), null);+ assert.strictEqual(extractSkillId({ skill_id: 'x'.repeat(128) }), 'x'.repeat(128));+});++test('extractSkillId rejects free text that is not identifier-shaped', () => {+ // A prompt smuggled into the skill field must not become a persisted id.+ assert.strictEqual(extractSkillId({ skill_id: 'summarize this: my api key is sk-abc' }), null);+ assert.strictEqual(extractSkillId({ skill_id: 'line\nbreak' }), null);+ assert.strictEqual(extractSkillId({ skill_id: ' ' }), null);+});++// ── privacy: no prompt text is persisted ─────────────────────────────────────++test('buildRecord synthesizes task_description and never copies prompt fields', () => {+ const secret = 'PROMPT-SECRET-do-not-persist';+ const record = buildRecord(payload({+ tool_input: {+ skill_id: 'code-review',+ skill_version: '1.2.0',+ task_description: secret,+ description: secret,+ prompt: secret,+ taskDescription: secret,+ },+ }));++ assert.strictEqual(record.task_description, 'Skill invocation: code-review');+ assert.ok(!JSON.stringify(record).includes(secret), 'record must not contain any prompt text');+});++test('buildRecord persists only the four dashboard fields', () => {+ const record = buildRecord(payload());+ assert.deepStrictEqual(+ Object.keys(record).sort(),+ ['outcome', 'skill_id', 'skill_version', 'task_description']+ );+});++test('buildRecord drops a non-identifier skill_version instead of persisting it', () => {+ const record = buildRecord(payload({+ tool_input: { skill_id: 'code-review', skill_version: 'v1 (as requested by the user in chat)' },+ }));+ assert.strictEqual(record.skill_version, 'unknown');+});++test('buildRecord returns null when the skill id is unusable', () => {+ assert.strictEqual(buildRecord(payload({ tool_input: {} })), null);+});++// ── outcome derivation ───────────────────────────────────────────────────────++test('deriveOutcome reports failure for PostToolUseFailure routing', () => {+ assert.strictEqual(deriveOutcome(payload({ hook_event_name: 'PostToolUseFailure' })), 'failure');+});++test('deriveOutcome reports failure for error-bearing tool responses', () => {+ assert.strictEqual(deriveOutcome(payload({ tool_response: { is_error: true } })), 'failure');+ assert.strictEqual(deriveOutcome(payload({ tool_response: { isError: true } })), 'failure');+ assert.strictEqual(deriveOutcome(payload({ tool_response: { status: 'ERROR' } })), 'failure');+ assert.strictEqual(deriveOutcome(payload({ tool_response: { error: 'boom' } })), 'failure');+});++test('deriveOutcome reports success otherwise', () => {+ assert.strictEqual(deriveOutcome(payload()), 'success');+ assert.strictEqual(deriveOutcome(payload({ tool_response: { status: 'ok' } })), 'success');+});++// ── hook behaviour ───────────────────────────────────────────────────────────++test('run ignores non-Skill tools and malformed input without throwing', () => {+ assert.doesNotThrow(() => run(JSON.stringify(payload({ tool_name: 'Bash' }))));+ assert.doesNotThrow(() => run('not json'));+ assert.doesNotThrow(() => run(''));+});++// ── JSONL sink bounds ────────────────────────────────────────────────────────++test('recordSkillExecution writes the sink owner-only', function () {+ if (process.platform === 'win32') {+ return; // POSIX modes are not meaningful on Windows+ }+ withTempHome(homeDir => {+ recordSkillExecution(+ { skill_id: 'code-review', skill_version: '1.0.0', task_description: 'Skill invocation: code-review', outcome: 'success' },+ { homeDir }+ );+ const runsFilePath = getRunsFilePath({ homeDir });+ const mode = fs.statSync(runsFilePath).mode & 0o777;+ assert.strictEqual(mode, RUNS_FILE_MODE, `expected mode ${RUNS_FILE_MODE.toString(8)}, got ${mode.toString(8)}`);+ });+});++test('recordSkillExecution re-tightens an already world-readable sink', function () {+ if (process.platform === 'win32') {+ return;+ }+ withTempHome(homeDir => {+ const runsFilePath = getRunsFilePath({ homeDir });+ fs.mkdirSync(path.dirname(runsFilePath), { recursive: true });+ fs.writeFileSync(runsFilePath, '', { mode: 0o644 });++ recordSkillExecution(+ { skill_id: 'code-review', skill_version: '1.0.0', task_description: 'Skill invocation: code-review', outcome: 'success' },+ { homeDir }+ );++ assert.strictEqual(fs.statSync(runsFilePath).mode & 0o777, RUNS_FILE_MODE);+ });+});++test('the JSONL sink is bounded by a retention cap', () => {+ withTempHome(homeDir => {+ const maxRecords = 5;+ for (let i = 0; i < maxRecords + 4; i++) {+ recordSkillExecution(+ { skill_id: `skill-${i}`, skill_version: '1.0.0', task_description: `Skill invocation: skill-${i}`, outcome: 'success' },+ { homeDir, maxRecords }+ );+ }++ const records = readSkillExecutionRecords({ homeDir });+ assert.strictEqual(records.length, maxRecords, 'sink must be trimmed to the cap');+ // Trimming keeps the newest runs, so the dashboard still reflects recent activity.+ assert.strictEqual(records[records.length - 1].skill_id, `skill-${maxRecords + 3}`);+ assert.strictEqual(records[0].skill_id, `skill-${4}`);+ });+});++test('the default retention cap is a finite bound', () => {+ assert.ok(Number.isInteger(MAX_RUN_RECORDS) && MAX_RUN_RECORDS > 0, 'MAX_RUN_RECORDS must be a positive integer');+});++test('an end-to-end Skill hook run lands exactly one non-sensitive record', () => {+ withTempHome(homeDir => {+ const previousHome = process.env.HOME;+ const previousUserProfile = process.env.USERPROFILE;+ process.env.HOME = homeDir;+ process.env.USERPROFILE = homeDir;+ try {+ run(JSON.stringify(payload({+ tool_input: { skill_id: 'code-review', skill_version: '1.2.0', prompt: 'PROMPT-SECRET' },+ })));++ const records = readSkillExecutionRecords({ homeDir });+ assert.strictEqual(records.length, 1);+ assert.strictEqual(records[0].skill_id, 'code-review');+ assert.strictEqual(records[0].skill_version, '1.2.0');+ assert.strictEqual(records[0].outcome, 'success');+ assert.ok(!JSON.stringify(records[0]).includes('PROMPT-SECRET'));+ } finally {+ if (previousHome === undefined) delete process.env.HOME; else process.env.HOME = previousHome;+ if (previousUserProfile === undefined) delete process.env.USERPROFILE; else process.env.USERPROFILE = previousUserProfile;+ }+ });+});++// deriveOutcome treats PostToolUseFailure as a hard failure. That branch is+// only reachable if the hook is actually registered for the event: the+// PostToolUse dispatcher does not fan out PostToolUseFailure, so the tracker+// needs its own hooks.json entry. Without it, hard Skill failures are silently+// dropped and the dashboard's success rate is inflated.+test('the tracker is registered for PostToolUseFailure so hard failures are recorded', () => {+ const hooksConfig = JSON.parse(+ fs.readFileSync(path.join(__dirname, '..', '..', 'hooks', 'hooks.json'), 'utf8')+ );+ const entries = (hooksConfig.hooks.PostToolUseFailure || [])+ .filter(entry => entry.id === 'post:skill:track');++ assert.strictEqual(entries.length, 1, 'expected one post:skill:track PostToolUseFailure entry');+ assert.strictEqual(entries[0].matcher, 'Skill', 'tracker must only match the Skill tool');+ assert.ok(+ entries[0].hooks[0].command.includes('scripts/hooks/skill-run-tracker.js'),+ 'entry should invoke skill-run-tracker.js'+ );+});++console.log(`\nPassed: ${passed}`);+console.log(`Failed: ${failed}`);+if (failed > 0) {+ process.exitCode = 1;+}