Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(ci): flag SKILL.md frontmatter defects in validate-skills (#1669)

Gaurav Dubeye196f8a4cb1e
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 410f617e9..dda1e7132 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -167,6 +167,8 @@ Short version:
- [ ] Tested with Claude Code
- [ ] Links to related skills
- [ ] No sensitive data (API keys, tokens, paths)
+- [ ] Frontmatter declares `name:` matching the directory name
+- [ ] Frontmatter `description:` is an inline string or folded (`>`) scalar — not a literal block (`|`, `|-`, or `|+`), which preserves internal newlines and breaks flat-table renderers
### Example Skills
diff --git a/scripts/ci/validate-skills.js b/scripts/ci/validate-skills.js
index 7664b5d65..6ffc85376 100644
--- a/scripts/ci/validate-skills.js
+++ b/scripts/ci/validate-skills.js
@@ -1,6 +1,22 @@
#!/usr/bin/env node
/**
* Validate curated skill directories (skills/ in repo).
+ *
+ * Checks:
+ * 1. Each sub-directory of skills/ contains a SKILL.md file.
+ * 2. SKILL.md is non-empty.
+ * 3. SKILL.md frontmatter (if present) declares a `name:` field.
+ * 4. SKILL.md frontmatter `description:` uses an inline scalar — not a
+ * literal block scalar (`|` / `|-` / `|+`), which preserves internal
+ * newlines and breaks flat-table renderers keyed off `description`.
+ *
+ * Frontmatter findings default to WARN so CI does not break while
+ * pre-existing data defects are being cleaned up out of band (see #1663).
+ * Pass `--strict` or set `CI_STRICT_SKILLS=1` to promote frontmatter
+ * findings to errors (exit 1).
+ *
+ * Structural findings (missing/empty SKILL.md) are always errors.
+ *
* Scope: curated only. Learned/imported/evolved roots are out of scope.
* If skills/ does not exist, exit 0 (no curated skills to validate).
*/
@@ -10,6 +26,144 @@ const path = require('path');
const SKILLS_DIR = path.join(__dirname, '../../skills');
+const STRICT = process.argv.includes('--strict') || process.env.CI_STRICT_SKILLS === '1';
+
+/**
+ * Parse the leading YAML frontmatter of a markdown document.
+ *
+ * Returns `{ present, lines }` so callers can inspect raw lines
+ * (needed to detect block-scalar `description:` values).
+ *
+ * Tolerant of UTF-8 BOM and CRLF line endings, matching the other
+ * validators in this directory.
+ *
+ * @param {string} content
+ * @returns {{present: boolean, lines: string[]}}
+ */
+function extractFrontmatter(content) {
+ // Strip BOM if present (UTF-8 BOM: U+FEFF).
+ const clean = content.replace(/^\uFEFF/, '');
+ const match = clean.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
+ if (!match) return { present: false, lines: [] };
+ return {
+ present: true,
+ lines: match[1].split(/\r?\n/)
+ };
+}
+
+/**
+ * Extract top-level keys (with trimmed values) and flag block-scalar
+ * `description:` values.
+ *
+ * Lines that continue a block scalar (`|` or `>`) are skipped — we only
+ * care about the top-level key set and the raw indicator on the
+ * `description:` line. Block-scalar indicators accept YAML chomp and
+ * indent modifiers and trailing comments, e.g. `|`, `|-`, `|+`, `|2`,
+ * `|-2`, `>- # note`.
+ *
+ * @param {string[]} lines
+ * @returns {{values: Record<string,string>, descriptionIndicator: string|null}}
+ */
+function inspectFrontmatter(lines) {
+ const values = Object.create(null);
+ let descriptionIndicator = null;
+ let inBlockScalar = false;
+ let blockScalarIndent = -1;
+
+ for (const rawLine of lines) {
+ if (inBlockScalar) {
+ // Stay inside the block until a line with indent <= the opener's
+ // indent (or an empty continuation).
+ const leadingSpaces = rawLine.match(/^(\s*)/)[1].length;
+ if (rawLine.trim() === '' || leadingSpaces > blockScalarIndent) {
+ continue;
+ }
+ inBlockScalar = false;
+ blockScalarIndent = -1;
+ }
+
+ const match = rawLine.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
+ if (!match) continue;
+
+ const key = match[1];
+ const rawValue = match[2];
+ // Strip unquoted comments for value/indicator inspection. Handles both
+ // trailing comments (`foo: bar # note`) and comment-only values
+ // (`foo: # todo`) so the latter is treated as empty.
+ const valueNoComment = rawValue
+ .replace(/^\s*#.*$/, '')
+ .replace(/\s+#.*$/, '')
+ .trim();
+ values[key] = valueNoComment;
+
+ // Detect literal / folded block-scalar indicators. Accept chomp
+ // modifiers (`-` / `+`) and optional indent-indicator digits in
+ // either order, per YAML 1.2.
+ if (/^[|>](?:[+-]?\d+|\d+[+-]?|[+-])?$/.test(valueNoComment)) {
+ if (key === 'description') {
+ descriptionIndicator = valueNoComment;
+ }
+ inBlockScalar = true;
+ blockScalarIndent = rawLine.match(/^(\s*)/)[1].length;
+ }
+ }
+
+ return { values, descriptionIndicator };
+}
+
+/**
+ * Validate a single skill directory.
+ *
+ * Returns `{ fatal }` where `fatal` indicates a structural error that
+ * should be surfaced via `console.error` and abort CI (missing/empty
+ * SKILL.md). Frontmatter findings are routed through
+ * `reportFrontmatterFinding`, which owns the WARN/ERROR decision based
+ * on strict mode.
+ *
+ * @param {string} dir
+ * @param {string} skillsDir
+ * @param {(msg: string) => void} reportFrontmatterFinding
+ * @returns {{fatal: boolean}}
+ */
+function validateSkillDir(dir, skillsDir, reportFrontmatterFinding) {
+ const skillMd = path.join(skillsDir, dir, 'SKILL.md');
+ if (!fs.existsSync(skillMd)) {
+ console.error(`ERROR: ${dir}/ - Missing SKILL.md`);
+ return { fatal: true };
+ }
+
+ let content;
+ try {
+ content = fs.readFileSync(skillMd, 'utf-8');
+ } catch (err) {
+ console.error(`ERROR: ${dir}/SKILL.md - ${err.message}`);
+ return { fatal: true };
+ }
+ if (content.trim().length === 0) {
+ console.error(`ERROR: ${dir}/SKILL.md - Empty file`);
+ return { fatal: true };
+ }
+
+ const fm = extractFrontmatter(content);
+ if (fm.present) {
+ const { values, descriptionIndicator } = inspectFrontmatter(fm.lines);
+
+ if (!Object.prototype.hasOwnProperty.call(values, 'name')) {
+ reportFrontmatterFinding(`${dir}/SKILL.md - frontmatter missing required field: name`);
+ } else if (values.name === '') {
+ reportFrontmatterFinding(`${dir}/SKILL.md - frontmatter 'name' is empty`);
+ }
+
+ if (descriptionIndicator && descriptionIndicator.startsWith('|')) {
+ reportFrontmatterFinding(
+ `${dir}/SKILL.md - frontmatter description uses literal block scalar ` + `'${descriptionIndicator}' which preserves internal newlines; ` + `use an inline string or folded '>' scalar instead`
+ );
+ }
+ }
+
+ return { fatal: false };
+}
+
function validateSkills() {
if (!fs.existsSync(SKILLS_DIR)) {
console.log('No curated skills directory (skills/), skipping');
@@ -17,32 +171,28 @@ function validateSkills() {
}
const entries = fs.readdirSync(SKILLS_DIR, { withFileTypes: true });
- const dirs = entries.filter(e => e.isDirectory()).map(e => e.name);
+ const dirs = entries.filter(e => e.isDirectory() && !e.name.startsWith('.')).map(e => e.name);
+
let hasErrors = false;
+ let warnCount = 0;
let validCount = 0;
- for (const dir of dirs) {
- const skillMd = path.join(SKILLS_DIR, dir, 'SKILL.md');
- if (!fs.existsSync(skillMd)) {
- console.error(`ERROR: ${dir}/ - Missing SKILL.md`);
+ const reportFrontmatterFinding = msg => {
+ if (STRICT) {
+ console.error(`ERROR: ${msg}`);
hasErrors = true;
- continue;
+ } else {
+ console.warn(`WARN: ${msg}`);
+ warnCount++;
}
+ };
- let content;
- try {
- content = fs.readFileSync(skillMd, 'utf-8');
- } catch (err) {
- console.error(`ERROR: ${dir}/SKILL.md - ${err.message}`);
- hasErrors = true;
- continue;
- }
- if (content.trim().length === 0) {
- console.error(`ERROR: ${dir}/SKILL.md - Empty file`);
+ for (const dir of dirs) {
+ const { fatal } = validateSkillDir(dir, SKILLS_DIR, reportFrontmatterFinding);
+ if (fatal) {
hasErrors = true;
continue;
}
-
validCount++;
}
@@ -50,7 +200,11 @@ function validateSkills() {
process.exit(1);
}
- console.log(`Validated ${validCount} skill directories`);
+ let msg = `Validated ${validCount} skill directories`;
+ if (warnCount > 0) {
+ msg += ` (${warnCount} warning${warnCount === 1 ? '' : 's'})`;
+ }
+ console.log(msg);
}
validateSkills();
diff --git a/tests/ci/validators.test.js b/tests/ci/validators.test.js
index 1b02c41e5..a35c760eb 100644
--- a/tests/ci/validators.test.js
+++ b/tests/ci/validators.test.js
@@ -11,7 +11,7 @@ const assert = require('assert');
const path = require('path');
const fs = require('fs');
const os = require('os');
-const { execFileSync } = require('child_process');
+const { execFileSync, spawnSync } = require('child_process');
const validatorsDir = path.join(__dirname, '..', '..', 'scripts', 'ci');
const repoRoot = path.join(__dirname, '..', '..');
@@ -180,6 +180,48 @@ function runCatalogValidator(overrides = {}) {
return runSourceViaTempFile(source);
}
+// Run validate-skills.js against a fixture dir, optionally passing
+// extra argv (e.g. '--strict') and env overrides (e.g.
+// CI_STRICT_SKILLS=1) so the frontmatter finding suite can exercise
+// both warn and strict modes via argv and env code paths.
+//
+// Captures stderr on both success and failure (the shared
+// runSourceViaTempFile helper only surfaces stderr when the child
+// exits non-zero, which hides WARN lines in the default mode).
+function runSkillsValidator(testDir, argv = [], envOverrides = {}) {
+ const validatorPath = path.join(validatorsDir, 'validate-skills.js');
+ let source = fs.readFileSync(validatorPath, 'utf8');
+ source = stripShebang(source);
+ source = source.replace(
+ /const SKILLS_DIR = .*?;/,
+ `const SKILLS_DIR = ${JSON.stringify(testDir)};`,
+ );
+ if (argv.length > 0) {
+ const argvPreamble = argv
+ .map(arg => `process.argv.push(${JSON.stringify(arg)});`)
+ .join('\n');
+ source = `${argvPreamble}\n${source}`;
+ }
+ const tmpFile = path.join(repoRoot,
+ `.tmp-validator-${Date.now()}-${Math.random().toString(36).slice(2)}.js`);
+ try {
+ fs.writeFileSync(tmpFile, source, 'utf8');
+ const r = spawnSync('node', [tmpFile], {
+ encoding: 'utf8',
+ timeout: 10000,
+ cwd: repoRoot,
+ env: { ...process.env, CI_STRICT_SKILLS: '', ...envOverrides },
+ });
+ return {
+ code: typeof r.status === 'number' ? r.status : 1,
+ stdout: r.stdout || '',
+ stderr: r.stderr || '',
+ };
+ } finally {
+ try { fs.unlinkSync(tmpFile); } catch (_) { /* ignore */ }
+ }
+}
+
function writeCatalogFixture(testDir, options = {}) {
const {
readmeCounts = { agents: 1, skills: 1, commands: 1 },
@@ -801,6 +843,181 @@ function runTests() {
cleanupTestDir(testDir);
})) passed++; else failed++;
+ if (test('warns when frontmatter is missing name (default mode)', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'no-name-skill');
+ fs.mkdirSync(skillDir);
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\ndescription: "X"\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir);
+ assert.strictEqual(result.code, 0,
+ `Default mode must not fail CI; got stderr: ${result.stderr}`);
+ assert.ok(
+ result.stderr.includes('WARN') && result.stderr.includes('missing required field: name'),
+ `Should warn on missing name; got stderr: ${result.stderr}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('errors when frontmatter is missing name (strict mode)', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'no-name-skill');
+ fs.mkdirSync(skillDir);
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\ndescription: "X"\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir, ['--strict']);
+ assert.strictEqual(result.code, 1, '--strict must fail CI on missing name');
+ assert.ok(result.stderr.includes('missing required field: name'),
+ 'Should report missing name');
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('warns on literal block-scalar description (|-)', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'block-desc-skill');
+ fs.mkdirSync(skillDir);
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\nname: block-desc-skill\ndescription: |-\n line one\n line two\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir);
+ assert.strictEqual(result.code, 0, 'Default mode should not fail CI');
+ assert.ok(
+ result.stderr.includes('WARN') && result.stderr.includes('literal block scalar'),
+ `Should warn on |- description; got stderr: ${result.stderr}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('accepts folded (>) and inline descriptions', () => {
+ const testDir = createTestDir();
+ const folded = path.join(testDir, 'folded-skill');
+ fs.mkdirSync(folded);
+ fs.writeFileSync(path.join(folded, 'SKILL.md'),
+ '---\nname: folded-skill\ndescription: >\n joined\n on spaces\norigin: ECC\n---\n# Skill');
+ const inline = path.join(testDir, 'inline-skill');
+ fs.mkdirSync(inline);
+ fs.writeFileSync(path.join(inline, 'SKILL.md'),
+ '---\nname: inline-skill\ndescription: "single line"\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir, ['--strict']);
+ assert.strictEqual(result.code, 0,
+ `Folded and inline should pass strict; got stderr: ${result.stderr}`);
+ assert.ok(result.stdout.includes('Validated 2'),
+ `Should count both skills; got stdout: ${result.stdout}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('skips hidden directories under skills/', () => {
+ const testDir = createTestDir();
+ // A dot-prefixed directory (e.g. .DS_Store-adjacent junk or legacy
+ // cache) must not count as a skill and must not error.
+ fs.mkdirSync(path.join(testDir, '.cache'));
+ fs.writeFileSync(path.join(testDir, '.cache', 'SKILL.md'), '# ignored');
+ const real = path.join(testDir, 'real-skill');
+ fs.mkdirSync(real);
+ fs.writeFileSync(path.join(real, 'SKILL.md'),
+ '---\nname: real-skill\ndescription: "x"\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir, ['--strict']);
+ assert.strictEqual(result.code, 0, 'Hidden dirs should be skipped');
+ assert.ok(result.stdout.includes('Validated 1'),
+ `Should only count the non-hidden skill; got stdout: ${result.stdout}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('warns when name: value is empty or whitespace', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'empty-name-skill');
+ fs.mkdirSync(skillDir);
+ // `name:` key present but value is blank.
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\nname: \ndescription: "X"\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir);
+ assert.strictEqual(result.code, 0,
+ `Default mode must not fail CI; got stderr: ${result.stderr}`);
+ assert.ok(
+ result.stderr.includes('WARN') && result.stderr.includes("'name' is empty"),
+ `Should warn on empty name; got stderr: ${result.stderr}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('warns on literal block-scalar description with |+ chomp', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'keep-desc-skill');
+ fs.mkdirSync(skillDir);
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\nname: keep-desc-skill\ndescription: |+\n line one\n line two\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir);
+ assert.strictEqual(result.code, 0, 'Default mode should not fail CI');
+ assert.ok(result.stderr.includes('literal block scalar'),
+ `Should warn on |+ description; got stderr: ${result.stderr}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('warns on block-scalar description with indent indicator and trailing comment', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'indent-desc-skill');
+ fs.mkdirSync(skillDir);
+ // `|-2 # note` is still a literal block scalar in YAML 1.2.
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\nname: indent-desc-skill\ndescription: |-2 # trimmed two-space indent\n line one\n line two\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir);
+ assert.strictEqual(result.code, 0, 'Default mode should not fail CI');
+ assert.ok(result.stderr.includes('literal block scalar'),
+ `Should warn on |-2 description; got stderr: ${result.stderr}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('honors CI_STRICT_SKILLS=1 env flag', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'no-name-skill-env');
+ fs.mkdirSync(skillDir);
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\ndescription: "X"\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir, [], { CI_STRICT_SKILLS: '1' });
+ assert.strictEqual(result.code, 1, 'CI_STRICT_SKILLS=1 must fail CI on missing name');
+ assert.ok(result.stderr.includes('missing required field: name'),
+ 'Should report missing name');
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('flags comment-only name value as empty (strict)', () => {
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'comment-only-name');
+ fs.mkdirSync(skillDir);
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '---\nname: # todo\ndescription: "X"\norigin: ECC\n---\n# Skill');
+
+ const result = runSkillsValidator(testDir, ['--strict']);
+ assert.strictEqual(result.code, 1, 'Strict mode must fail CI on empty name');
+ assert.ok(result.stderr.includes("'name' is empty"),
+ `Should report empty name; got stderr: ${result.stderr}`);
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
+ if (test('tolerates ---trailing text outside frontmatter block', () => {
+ // A SKILL.md whose body contains a line starting with '---text'
+ // must not be parsed as frontmatter. Regression guard for
+ // closing-delimiter tightening: the old regex would greedily
+ // match '---trailing'.
+ const testDir = createTestDir();
+ const skillDir = path.join(testDir, 'no-frontmatter-dashes');
+ fs.mkdirSync(skillDir);
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'),
+ '# Skill\n\nSome body text.\n\n---trailing content\nmore body\n');
+
+ const result = runSkillsValidator(testDir, ['--strict']);
+ assert.strictEqual(result.code, 0,
+ `Should not flag frontmatter findings when no valid frontmatter exists; got stderr: ${result.stderr}`);
+ assert.ok(!result.stderr.includes('missing required field: name'),
+ 'Must not treat ---trailing as a frontmatter closer');
+ cleanupTestDir(testDir);
+ })) passed++; else failed++;
+
// ==========================================
// validate-commands.js
// ==========================================