Skip to content

28 eval-informed mental models and critical-thinking skills for Claude Code, GitHub Copilot, Codex, Cursor, and other Agent Skills-compatible tools

15.7 KB
Raw Blame
#!/usr/bin/env node
'use strict';

/**
 * Thinking Skills structural validator (side-effect free).
 *
 * Enforces:
 * - frontmatter description < 200 chars
 * - required sections: When to Use, When NOT to Use, Procedure, Output, Verification
 * - catalog count from registry
 * - per-skill word budget from registry (survivors)
 *
 * Does NOT write quality-report.json or any tracked report.
 * Run: node scripts/validate-skills.js
 */

const fs = require('fs');
const path = require('path');
const { isDeepStrictEqual } = require('util');

const REPO_ROOT = path.join(__dirname, '..');
const SKILLS_DIR = path.join(REPO_ROOT, 'skills');

let loadRegistry;
let validateRegistry;
let getSkillBudget;
try {
  ({ loadRegistry, validateRegistry, getSkillBudget } = require(path.join(REPO_ROOT, 'evals', 'lib', 'registry')));
} catch (_) {
  loadRegistry = null;
}

const DEFAULT_REQUIRED_SECTIONS = [
  'When to Use',
  'When NOT to Use',
  'Procedure',
  'Output',
  'Verification',
];

const DEFAULT_DESCRIPTION_MAX = 200;

function listSkillDirs(skillsDir = SKILLS_DIR) {
  if (!fs.existsSync(skillsDir)) return [];
  return fs.readdirSync(skillsDir)
    .filter(d => d.startsWith('thinking-'))
    .filter(d => fs.existsSync(path.join(skillsDir, d, 'SKILL.md')))
    .sort();
}

function listUnexpectedSkillFiles(repoRoot = REPO_ROOT) {
  const skippedDirectories = new Set([
    '.git',
    '.claude',
    'backups',
    'evals/checkouts',
    'node_modules',
  ]);
  const unexpected = [];

  function visit(directory, relativeDirectory = '') {
    for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
      const relativePath = relativeDirectory
        ? `${relativeDirectory}/${entry.name}`
        : entry.name;
      if (entry.isDirectory()) {
        if (!skippedDirectories.has(relativePath)) {
          visit(path.join(directory, entry.name), relativePath);
        }
        continue;
      }
      if (!entry.isFile() || entry.name !== 'SKILL.md') continue;
      if (!/^skills\/thinking-[^/]+\/SKILL\.md$/.test(relativePath)) {
        unexpected.push(relativePath);
      }
    }
  }

  visit(repoRoot);
  return unexpected.sort();
}

function validatePluginManifests(repoRoot = REPO_ROOT) {
  const manifestPaths = [
    '.claude-plugin/plugin.json',
    '.github/plugin/plugin.json',
  ];
  const fields = [
    'name',
    'description',
    'version',
    'author',
    'homepage',
    'repository',
    'license',
    'keywords',
    'skills',
  ];
  const errors = [];
  const manifests = [];

  for (const relativePath of manifestPaths) {
    const absolutePath = path.join(repoRoot, relativePath);
    if (!fs.existsSync(absolutePath)) {
      errors.push(`manifest missing: ${relativePath}`);
      continue;
    }
    try {
      const manifest = JSON.parse(fs.readFileSync(absolutePath, 'utf8'));
      manifests.push({ relativePath, manifest });
      if (!isDeepStrictEqual(manifest.skills, ['skills/'])) {
        errors.push(`manifest skills must equal ["skills/"]: ${relativePath}`);
      }
    } catch (error) {
      errors.push(`manifest invalid JSON: ${relativePath}: ${error.message}`);
    }
  }

  if (manifests.length === manifestPaths.length) {
    const [first, second] = manifests;
    for (const field of fields) {
      if (!isDeepStrictEqual(first.manifest[field], second.manifest[field])) {
        errors.push(`manifest field mismatch: ${field}`);
      }
    }
  }

  return errors;
}

function parseFrontmatter(content) {
  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
  const frontmatter = {};
  const errors = [];
  if (!match) {
    return { frontmatter, body: content, has_frontmatter: false, errors };
  }

  const lines = match[1].split(/\r?\n/);
  for (let index = 0; index < lines.length; index += 1) {
    const line = lines[index];
    const lineNumber = index + 2;
    if (/^\s*(?:#.*)?$/.test(line)) continue;

    const keyValue = line.match(/^(\w[\w-]*):\s*(.*)$/);
    if (!keyValue || keyValue[2].trim().length === 0) {
      errors.push(`frontmatter line ${lineNumber}: expected key: scalar`);
      continue;
    }

    const key = keyValue[1];
    if (Object.prototype.hasOwnProperty.call(frontmatter, key)) {
      errors.push(`frontmatter line ${lineNumber}: duplicate key "${key}"`);
      continue;
    }

    const scalar = keyValue[2].trim();
    let value;
    if (scalar.startsWith("'")) {
      if (!scalar.endsWith("'") || scalar.length < 2) {
        errors.push(`frontmatter line ${lineNumber}: unclosed single-quoted scalar`);
        continue;
      }
      const inner = scalar.slice(1, -1);
      if (/(^|[^'])'(?:[^']|$)/.test(inner)) {
        errors.push(`frontmatter line ${lineNumber}: unclosed single-quoted scalar`);
        continue;
      }
      value = inner.replace(/''/g, "'");
    } else if (scalar.startsWith('"')) {
      try {
        value = JSON.parse(scalar);
        if (typeof value !== 'string') throw new TypeError('not a string');
      } catch (_) {
        errors.push(`frontmatter line ${lineNumber}: invalid double-quoted scalar`);
        continue;
      }
    } else {
      const ambiguousToken = [': ', ' #'].find(token => scalar.includes(token));
      if (ambiguousToken) {
        errors.push(`frontmatter line ${lineNumber}: ambiguous bare scalar contains "${ambiguousToken}"`);
        continue;
      }
      value = scalar;
    }

    if (value.length === 0) {
      errors.push(`frontmatter line ${lineNumber}: expected key: scalar`);
      continue;
    }
    frontmatter[key] = value;
  }

  const body = content.slice(match[0].length);
  return { frontmatter, body, has_frontmatter: true, errors };
}

function countWords(text) {
  return String(text || '').trim().split(/\s+/).filter(Boolean).length;
}

function sectionHeadingPattern(name) {
  // Match ## When to Use / ## When NOT to Use etc.
  const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
  return new RegExp(`^##\\s+${escaped}\\s*$`, 'mi');
}

function hasSection(content, name) {
  return sectionHeadingPattern(name).test(content);
}

function sectionBody(content, name) {
  const match = sectionHeadingPattern(name).exec(content);
  if (!match) return '';
  const start = match.index + match[0].length;
  const remainder = content.slice(start);
  const next = remainder.search(/^##\s+/m);
  return (next === -1 ? remainder : remainder.slice(0, next)).trim();
}

function numberedStepCount(content) {
  return sectionBody(content, 'Procedure')
    .split('\n')
    .filter(line => /^\s*\d+\.\s+\S/.test(line))
    .length;
}

/**
 * Pure validation of a single skill body.
 * @returns {object} structured result with pass/fail details
 */
function validateSkillContent(content, opts = {}) {
  const skillName = opts.name || 'unknown';
  const requiredSections = opts.requiredSections || DEFAULT_REQUIRED_SECTIONS;
  const descriptionMax = opts.descriptionMax ?? DEFAULT_DESCRIPTION_MAX;
  const maxWords = opts.maxWords != null ? opts.maxWords : null;
  const enforceBudget = opts.enforceBudget !== false && maxWords != null;
  const requireDisableModelInvocation = opts.requireDisableModelInvocation === true;
  const forbiddenSkillIds = opts.forbiddenSkillIds || [];

  const { frontmatter, has_frontmatter, errors: frontmatterErrors } = parseFrontmatter(content);
  const words = countWords(content);
  const description = frontmatter.description || '';
  const descriptionLen = description.length;
  const procedureSteps = numberedStepCount(content);
  const nonEmptySections = requiredSections.filter(name => sectionBody(content, name).length > 0);
  const forbiddenRefs = forbiddenSkillIds.filter(id => content.includes(`thinking-${id}`));

  const sections = requiredSections.map(name => ({
    name,
    found: hasSection(content, name),
  }));

  const checks = [];
  checks.push({
    name: 'YAML Frontmatter',
    pass: has_frontmatter &&
      frontmatterErrors.length === 0 &&
      frontmatter.name === skillName &&
      Boolean(frontmatter.description),
    detail: frontmatterErrors.length > 0
      ? frontmatterErrors.join('; ')
      : has_frontmatter
        ? `name=${frontmatter.name || 'missing'}`
        : 'missing frontmatter',
  });
  checks.push({
    name: 'Description Length',
    pass: descriptionLen > 0 && descriptionLen <= descriptionMax,
    detail: descriptionLen
      ? `${descriptionLen} chars (max ${descriptionMax})`
      : 'No description found',
  });

  for (const section of sections) {
    checks.push({
      name: `Section: ${section.name}`,
      pass: section.found,
      detail: section.found ? 'found' : 'missing',
    });
  }
  checks.push({
    name: 'Required Section Content',
    pass: nonEmptySections.length === requiredSections.length,
    detail: nonEmptySections.length === requiredSections.length
      ? 'all required sections are non-empty'
      : `empty: ${requiredSections.filter(name => !nonEmptySections.includes(name)).join(', ')}`,
  });
  checks.push({
    name: 'Procedure Steps',
    pass: procedureSteps >= 3 && procedureSteps <= 7,
    detail: `${procedureSteps} numbered steps (required 3-7)`,
  });
  if (requireDisableModelInvocation) {
    checks.push({
      name: 'Manual-only Quarantine',
      pass: frontmatter['disable-model-invocation'] === 'true',
      detail: `disable-model-invocation=${frontmatter['disable-model-invocation'] || 'missing'}`,
    });
  }
  checks.push({
    name: 'Deleted Skill References',
    pass: forbiddenRefs.length === 0,
    detail: forbiddenRefs.length === 0
      ? 'none'
      : forbiddenRefs.map(id => `thinking-${id}`).join(', '),
  });

  if (enforceBudget) {
    checks.push({
      name: 'Word Budget',
      pass: words <= maxWords,
      detail: `${words} words (max ${maxWords})`,
    });
  } else if (maxWords != null) {
    checks.push({
      name: 'Word Budget',
      pass: true,
      detail: `${words} words (budget ${maxWords}; not enforced for non-survivors)`,
    });
  }

  const failed = checks.filter(c => !c.pass);
  const score = checks.filter(c => c.pass).length;
  const maxScore = checks.length;

  return {
    name: skillName,
    description,
    description_length: descriptionLen,
    words,
    max_words: maxWords,
    procedure_steps: procedureSteps,
    forbidden_skill_refs: forbiddenRefs,
    sections,
    checks,
    failed,
    score,
    maxScore,
    pass: failed.length === 0,
    frontmatter,
  };
}

function validateSkillFile(skillPath, opts = {}) {
  const content = fs.readFileSync(skillPath, 'utf8');
  const skillName = opts.name || path.basename(path.dirname(skillPath));
  return {
    ...validateSkillContent(content, { ...opts, name: skillName }),
    path: skillPath,
  };
}

function loadCatalogExpectations(registryPath) {
  if (!loadRegistry) {
    return {
      expectedCount: null,
      requiredSections: DEFAULT_REQUIRED_SECTIONS,
      descriptionMax: DEFAULT_DESCRIPTION_MAX,
      budgets: {},
      registry: null,
      registryValidation: null,
    };
  }
  const { registry } = loadRegistry(registryPath);
  const registryValidation = validateRegistry(registry);
  const budgets = {};
  for (const [id, skill] of Object.entries(registry.skills || {})) {
    const b = getSkillBudget(registry, id);
    if (b != null) budgets[id] = b;
  }
  return {
    expectedCount: registry.catalog?.expected_count ?? registry.catalog?.baseline_count ?? null,
    requiredSections: registry.required_skill_sections || DEFAULT_REQUIRED_SECTIONS,
    descriptionMax: registry.frontmatter?.description_max_chars ?? DEFAULT_DESCRIPTION_MAX,
    budgets,
    survivors: new Set(registry.catalog?.survivors || []),
    quarantined: new Set(Object.entries(registry.skills || {})
      .filter(([, skill]) => skill.disposition?.disable_model_invocation_until_elevated === true)
      .map(([id]) => id)),
    deletions: new Set(registry.catalog?.deletions || []),
    registry,
    registryValidation,
  };
}

/**
 * Validate all skills under skillsDir. Pure: no file writes.
 */
function validateAllSkills(opts = {}) {
  const repoRoot = opts.repoRoot || REPO_ROOT;
  const skillsDir = opts.skillsDir || path.join(repoRoot, 'skills');
  const catalog = loadCatalogExpectations(opts.registryPath);
  const dirs = listSkillDirs(skillsDir);
  const results = [];

  for (const dir of dirs) {
    const id = dir.replace(/^thinking-/, '');
    const maxWords = catalog.budgets[id] != null ? catalog.budgets[id] : null;
    // Enforce budget only for declared survivors (Phase 3 lean contracts).
    // Baseline pre-rewrite bodies will fail budgets/sections — that is intentional observability.
    const enforceBudget = catalog.survivors ? catalog.survivors.has(id) : maxWords != null;
    const skillPath = path.join(skillsDir, dir, 'SKILL.md');
    results.push(validateSkillFile(skillPath, {
      name: dir,
      requiredSections: catalog.requiredSections,
      descriptionMax: catalog.descriptionMax,
      maxWords,
      enforceBudget,
      requireDisableModelInvocation: catalog.quarantined?.has(id) || false,
      forbiddenSkillIds: [...(catalog.deletions || [])],
    }));
  }

  const errors = [];
  if (catalog.expectedCount != null && results.length !== catalog.expectedCount) {
    errors.push(`catalog count ${results.length} != expected ${catalog.expectedCount}`);
  }
  if (catalog.registryValidation && !catalog.registryValidation.ok) {
    for (const e of catalog.registryValidation.errors) errors.push(`registry: ${e}`);
  }
  for (const skillPath of listUnexpectedSkillFiles(repoRoot)) {
    errors.push(`unexpected SKILL.md: ${skillPath}`);
  }
  errors.push(...validatePluginManifests(repoRoot));

  const failedSkills = results.filter(r => !r.pass);
  return {
    ok: errors.length === 0 && failedSkills.length === 0,
    catalog_errors: errors,
    expected_count: catalog.expectedCount,
    found_count: results.length,
    required_sections: catalog.requiredSections,
    results,
    failed: failedSkills,
    summary: {
      total: results.length,
      passed: results.filter(r => r.pass).length,
      failed: failedSkills.length,
    },
    registry_validation: catalog.registryValidation,
  };
}

function printResults(report) {
  const GREEN = '\x1b[32m';
  const RED = '\x1b[31m';
  const YELLOW = '\x1b[33m';
  const RESET = '\x1b[0m';
  const BOLD = '\x1b[1m';

  console.log(`\n${BOLD}=== Thinking Skills Structural Validation ===${RESET}\n`);
  if (report.expected_count != null) {
    console.log(`Catalog: found ${report.found_count}, expected ${report.expected_count}`);
  } else {
    console.log(`Catalog: found ${report.found_count}`);
  }
  for (const e of report.catalog_errors || []) console.log(`${RED}CATALOG: ${e}${RESET}`);

  for (const skill of report.results) {
    const color = skill.pass ? GREEN : RED;
    console.log(`${BOLD}${skill.name}${RESET}: ${color}${skill.score}/${skill.maxScore}${RESET} words=${skill.words}${skill.max_words != null ? `/${skill.max_words}` : ''}`);
    for (const c of skill.failed) {
      console.log(`  ${YELLOW}- ${c.name}: ${c.detail}${RESET}`);
    }
  }

  console.log(`\n${BOLD}=== Summary ===${RESET}`);
  console.log(`Passed: ${report.summary.passed}/${report.summary.total}`);
  console.log(`Failed: ${report.summary.failed}`);
  // Explicit: no tracked report write
  console.log('(side-effect free: no quality-report.json written)');
}

function main() {
  const report = validateAllSkills();
  // Sort failures first for visibility
  report.results.sort((a, b) => Number(a.pass) - Number(b.pass) || a.name.localeCompare(b.name));
  printResults(report);
  if (!report.ok) process.exit(1);
}

if (require.main === module) {
  main();
}

module.exports = {
  DEFAULT_REQUIRED_SECTIONS,
  DEFAULT_DESCRIPTION_MAX,
  listSkillDirs,
  listUnexpectedSkillFiles,
  validatePluginManifests,
  parseFrontmatter,
  countWords,
  hasSection,
  sectionBody,
  numberedStepCount,
  validateSkillContent,
  validateSkillFile,
  validateAllSkills,
  loadCatalogExpectations,
  printResults,
  main,
  SKILLS_DIR,
  REPO_ROOT,
};