π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
test(skills): check that every CLI command a shipped skill runs exists (#4552) (#4553)
Muhammad Zubair44675e8fd506
diff --git a/scripts/__tests__/audit-skill-cli-commands.test.mjs b/scripts/__tests__/audit-skill-cli-commands.test.mjsnew file mode 100644index 0000000000..f8bb0e8d89--- /dev/null+++ b/scripts/__tests__/audit-skill-cli-commands.test.mjs@@ -0,0 +1,102 @@+import { beforeAll, describe, expect, it } from 'vitest';+import {+ evaluateAgainstBaseline,+ extractInvocations,+ findUnknown,+ findUnknownRefs,+ loadCommandTree,+ readBaseline,+ resolveCommand,+ shippedSkillFiles,+} from '../audit-skill-cli-commands.mjs';++const fakeTree = new Map([+ ['hooks', { aliases: [], subcommands: new Map([['pre-edit', []], ['session-end', []]]), loadError: null }],+ ['performance', { aliases: ['perf'], subcommands: new Map([['benchmark', []]]), loadError: null }],+ ['version', { aliases: [], subcommands: new Map(), loadError: null }],+]);++describe('skill β CLI command reference audit', () => {+ it('extracts npx invocations anywhere, with an optional subcommand', () => {+ const text = [+ 'Run `npx @claude-flow/cli@latest hooks pre-edit --file a.js` first.',+ 'npx -y ruflo@latest perf benchmark',+ 'npx claude-flow@alpha version',+ ].join('\n');+ expect(extractInvocations(text)).toEqual([+ { command: 'hooks', subcommand: 'pre-edit', line: 1 },+ { command: 'perf', subcommand: 'benchmark', line: 2 },+ { command: 'version', subcommand: null, line: 3 },+ ]);+ });++ it('counts bare ruflo/claude-flow commands only at the start of a fenced code line', () => {+ const text = [+ 'ruflo is a harness and claude-flow runs swarms.',+ '```bash',+ '$ ruflo hooks session-end',+ 'claude-flow hook pre-task',+ '```',+ 'ruflo hook outside a fence',+ ].join('\n');+ expect(extractInvocations(text)).toEqual([+ { command: 'hooks', subcommand: 'session-end', line: 3 },+ { command: 'hook', subcommand: 'pre-task', line: 4 },+ ]);+ });++ it('stops at flags and quoted arguments', () => {+ expect(extractInvocations('npx ruflo@latest hooks --list')).toEqual([{ command: 'hooks', subcommand: null, line: 1 }]);+ expect(extractInvocations('npx ruflo@latest hooks "pre-edit"')).toEqual([{ command: 'hooks', subcommand: null, line: 1 }]);+ });++ it('resolves aliases and flags unknown commands and subcommands', () => {+ expect(resolveCommand(fakeTree, 'perf')).toBe('performance');+ const unknown = findUnknown(fakeTree, 'skills/x/SKILL.md', [+ { command: 'hook', subcommand: 'pre-edit', line: 1 },+ { command: 'hooks', subcommand: 'pre-search', line: 2 },+ { command: 'hooks', subcommand: 'pre-edit', line: 3 },+ { command: 'perf', subcommand: 'benchmark', line: 4 },+ { command: 'version', subcommand: 'extra', line: 5 },+ ]);+ expect(unknown.map((u) => u.key)).toEqual([+ 'skills/x/SKILL.md::hook',+ 'skills/x/SKILL.md::hooks pre-search',+ ]);+ });++ it('only checks the name of a command whose subcommands could not be loaded', () => {+ const tree = new Map([['memory', { aliases: [], subcommands: null, loadError: 'boom' }]]);+ expect(findUnknown(tree, 'f', [{ command: 'memory', subcommand: 'anything', line: 1 }])).toEqual([]);+ });++ it('fails a new entry even when the total count is unchanged, and reports fixed ones', () => {+ const result = evaluateAgainstBaseline([{ key: 'f::new' }], ['f::old']);+ expect(result.ok).toBe(false);+ expect(result.unexpected).toEqual(['f::new']);+ expect(result.fixed).toEqual(['f::old']);+ expect(evaluateAgainstBaseline([{ key: 'f::old' }], ['f::old']).ok).toBe(true);+ });++});++describe('skill β CLI command references in this repo', () => {+ let tree;+ // Importing every command module takes a few seconds on a cold transform cache.+ beforeAll(async () => {+ tree = await loadCommandTree();+ }, 120_000);++ it('loads the real command tree from the CLI sources', () => {+ expect(tree.get('hooks')?.subcommands?.has('pre-edit')).toBe(true);+ expect(tree.get('analyze')?.subcommands?.has('diff')).toBe(true);+ expect(resolveCommand(tree, 'perf')).toBe('performance');+ expect(resolveCommand(tree, 'hook')).toBe(null);+ });++ it('has no unknown CLI references beyond verification/skill-cli-command-baseline.json', () => {+ expect(shippedSkillFiles().length).toBeGreaterThan(0);+ const result = evaluateAgainstBaseline(findUnknownRefs(tree), readBaseline());+ expect(result.unexpected).toEqual([]);+ });+});diff --git a/scripts/audit-skill-cli-commands.mjs b/scripts/audit-skill-cli-commands.mjsnew file mode 100644index 0000000000..505702cc71--- /dev/null+++ b/scripts/audit-skill-cli-commands.mjs@@ -0,0 +1,269 @@+#!/usr/bin/env node+/**+ * Skill β CLI command reference audit. Sibling of scripts/audit-skill-mcp-tools.mjs (#4141).+ *+ * Skills tell agents to run ruflo CLI commands, for example+ * `npx @claude-flow/cli@latest hooks pre-edit --file x` or `npx ruflo@latest analyze diff`.+ * A top-level command the CLI does not have fails with `Unknown command: <name>`, and an+ * unknown subcommand prints help instead of doing the work, so an agent that follows the+ * skill silently gets nothing done.+ *+ * This guard scans every SKILL.md that ships to users:+ * - v3/@claude-flow/cli/.claude/skills/** (npm package, installed by init)+ * - plugins/** (marketplace plugins)+ * and checks each invocation against the command tree registered by+ * v3/@claude-flow/cli/src/commands/index.ts (top-level names and aliases, and each+ * command's subcommand names and aliases).+ *+ * What counts as an invocation:+ * - `npx [-y] @claude-flow/cli|claude-flow|ruflo[@tag] <command> [<subcommand>]` anywhere;+ * - a line inside a fenced code block that starts with `ruflo <command>` or+ * `claude-flow <command>` (optionally after `$ `). Bare mentions in prose are ignored.+ * Only bare lowercase words count as command or subcommand names; flags and quoted+ * arguments end the match.+ *+ * The command tree is read from the TypeScript sources, so run this with a TS loader+ * after `npm ci`:+ * node --import tsx scripts/audit-skill-cli-commands.mjs # exit 1 on a NEW unknown command+ * node --import tsx scripts/audit-skill-cli-commands.mjs --json # machine-readable report+ * node --import tsx scripts/audit-skill-cli-commands.mjs --update-baseline # after fixing some+ * The vitest wrapper in scripts/__tests__/ loads the same sources natively and runs in+ * ci.yml's test ratchet on every PR.+ *+ * Known unknown commands are carried in verification/skill-cli-command-baseline.json as+ * `<skill path>::<command>` or `<skill path>::<command> <subcommand>` entries. Like+ * audit-skill-mcp-tools.mjs the baseline is identity-based: an entry that is not in it+ * fails, even if the total count is unchanged. Fixed entries never fail; they print a+ * reminder to lower the floor.+ */++import { existsSync, readdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';+import { dirname, join, relative, resolve, sep } from 'node:path';+import { fileURLToPath, pathToFileURL } from 'node:url';++const DEFAULT_REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');++/** Skill trees that ship to users, relative to the repo root. */+export const SKILL_ROOTS = ['v3/@claude-flow/cli/.claude/skills', 'plugins'];++const NPX_RE = /npx\s+(?:-y\s+)?(?:@claude-flow\/cli|claude-flow|ruflo)(?:@[\w.-]+)?[ \t]+([a-z][a-z0-9-]*)(?:[ \t]+([a-z][a-z0-9-]*))?(?![\w-])/g;+const BARE_RE = /^\s*(?:\$\s+)?(?:ruflo|claude-flow)[ \t]+([a-z][a-z0-9-]*)(?:[ \t]+([a-z][a-z0-9-]*))?(?![\w-])/;++function toPosix(path) {+ return path.split(sep).join('/');+}++function commandsDir(repoRoot) {+ return join(repoRoot, 'v3', '@claude-flow', 'cli', 'src', 'commands');+}++/**+ * Top-level command names and the source file each one lives in, read from+ * commands/index.ts: the lazy `commandLoaders` map plus the eager imports.+ */+export function topLevelCommandFiles(repoRoot = DEFAULT_REPO_ROOT) {+ const index = readFileSync(join(commandsDir(repoRoot), 'index.ts'), 'utf8');+ const files = new Map();+ for (const m of index.matchAll(/'?([a-z][a-z0-9-]*)'?:\s*\(\)\s*=>\s*import\('\.\/([^']+)\.js'\)/g)) {+ files.set(m[1], m[2]);+ }+ for (const m of index.matchAll(/^import \{ (\w+)Command \} from '\.\/([^']+)\.js';/gm)) {+ if (![...files.values()].includes(m[2])) files.set(m[2], m[2]);+ }+ return files;+}++/**+ * Load the command tree from the TypeScript sources. Needs a TS-aware loader+ * (vitest, or `node --import tsx`). A module that fails to import is kept as a+ * known command with `subcommands: null` and a `loadError`, so only its name is+ * checked. A loader key whose module has no command of that name (for example+ * `appliance-advanced`, which only exports subcommands) falls back to the+ * module's default export, or to name-only checking.+ */+export async function loadCommandTree(repoRoot = DEFAULT_REPO_ROOT, importer = (url) => import(url)) {+ const tree = new Map();+ for (const [name, file] of topLevelCommandFiles(repoRoot)) {+ let entry = { aliases: [], subcommands: null, loadError: null };+ try {+ const mod = await importer(pathToFileURL(join(commandsDir(repoRoot), `${file}.ts`)).href);+ const isCommand = (v) => v && typeof v === 'object' && typeof v.name === 'string';+ const command = Object.values(mod).find((v) => isCommand(v) && v.name === name)+ ?? (isCommand(mod.default) && Array.isArray(mod.default.subcommands) ? mod.default : null);+ if (command) {+ const subcommands = new Map();+ for (const sub of command.subcommands ?? []) subcommands.set(sub.name, sub.aliases ?? []);+ entry = { aliases: command.aliases ?? [], subcommands, loadError: null };+ }+ } catch (error) {+ entry.loadError = String(error && error.message ? error.message : error).split('\n')[0];+ }+ tree.set(name, entry);+ }+ return tree;+}++/** CLI invocations in one SKILL.md body: [{ command, subcommand, line }]. */+export function extractInvocations(text) {+ const found = [];+ let inFence = false;+ const lines = text.split('\n');+ for (let i = 0; i < lines.length; i += 1) {+ const line = lines[i];+ if (/^\s*(```|~~~)/.test(line)) {+ inFence = !inFence;+ continue;+ }+ for (const m of line.matchAll(NPX_RE)) found.push({ command: m[1], subcommand: m[2] ?? null, line: i + 1 });+ if (inFence && !/npx\s/.test(line)) {+ const m = line.match(BARE_RE);+ if (m) found.push({ command: m[1], subcommand: m[2] ?? null, line: i + 1 });+ }+ }+ return found;+}++/** Resolve a command name or alias to its canonical top-level name, or null. */+export function resolveCommand(tree, name) {+ if (tree.has(name)) return name;+ for (const [canonical, entry] of tree) if ((entry.aliases ?? []).includes(name)) return canonical;+ return null;+}++/** Unknown commands/subcommands for one file's invocations: [{ file, line, ref, key }]. */+export function findUnknown(tree, file, invocations) {+ const unknown = [];+ for (const inv of invocations) {+ const canonical = resolveCommand(tree, inv.command);+ if (!canonical) {+ unknown.push({ file, line: inv.line, ref: inv.command, key: `${file}::${inv.command}` });+ continue;+ }+ const subs = tree.get(canonical).subcommands;+ if (!inv.subcommand || !subs || subs.size === 0) continue;+ const known = subs.has(inv.subcommand) || [...subs.values()].some((aliases) => aliases.includes(inv.subcommand));+ if (!known) {+ const ref = `${canonical} ${inv.subcommand}`;+ unknown.push({ file, line: inv.line, ref, key: `${file}::${ref}` });+ }+ }+ return unknown;+}++function walkSkillFiles(dir, out) {+ if (!existsSync(dir)) return out;+ for (const entry of readdirSync(dir, { withFileTypes: true })) {+ if (entry.name === 'node_modules' || entry.name === '.git') continue;+ const full = join(dir, entry.name);+ if (entry.isDirectory()) walkSkillFiles(full, out);+ else if (entry.isFile() && entry.name === 'SKILL.md') out.push(full);+ }+ return out;+}++/** Every shipped SKILL.md, as repo-relative POSIX paths, sorted. */+export function shippedSkillFiles(repoRoot = DEFAULT_REPO_ROOT) {+ const files = [];+ for (const root of SKILL_ROOTS) walkSkillFiles(join(repoRoot, root), files);+ return files.map((f) => toPosix(relative(repoRoot, f))).sort();+}++/** Unknown CLI references across shipped skills. */+export function findUnknownRefs(tree, repoRoot = DEFAULT_REPO_ROOT) {+ const unknown = [];+ for (const file of shippedSkillFiles(repoRoot)) {+ unknown.push(...findUnknown(tree, file, extractInvocations(readFileSync(join(repoRoot, file), 'utf8'))));+ }+ return unknown;+}++/** Same contract as audit-skill-mcp-tools.mjs: ok is false only for entries missing from the baseline. */+export function evaluateAgainstBaseline(unknown, baselineEntries) {+ const known = new Set(baselineEntries);+ const current = [...new Set(unknown.map((u) => u.key))].sort();+ const currentSet = new Set(current);+ const unexpected = current.filter((key) => !known.has(key));+ const fixed = [...known].filter((key) => !currentSet.has(key)).sort();+ return { ok: unexpected.length === 0, current, unexpected, fixed };+}++export function baselinePath(repoRoot = DEFAULT_REPO_ROOT) {+ return join(repoRoot, 'verification', 'skill-cli-command-baseline.json');+}++export function readBaseline(repoRoot = DEFAULT_REPO_ROOT) {+ const file = baselinePath(repoRoot);+ if (!existsSync(file)) return [];+ const parsed = JSON.parse(readFileSync(file, 'utf8'));+ return Array.isArray(parsed.knownUnknown) ? parsed.knownUnknown : [];+}++async function main(argv) {+ const repoRoot = DEFAULT_REPO_ROOT;+ let tree;+ try {+ tree = await loadCommandTree(repoRoot);+ } catch (error) {+ console.error(`Could not load the CLI command tree: ${error.message}`);+ return 1;+ }+ const failed = [...tree.values()].filter((e) => e.loadError).length;+ if (failed * 2 > tree.size) {+ // Plain Node cannot resolve the sources' `./x.js` specifiers to `.ts`, so most+ // imports fail; partial results would be misleading.+ console.error(`${failed} of ${tree.size} CLI command modules failed to import. Run with a TS loader after npm ci: node --import tsx scripts/audit-skill-cli-commands.mjs`);+ return 1;+ }+ const unknown = findUnknownRefs(tree, repoRoot);++ if (argv.includes('--update-baseline')) {+ const knownUnknown = [...new Set(unknown.map((u) => u.key))].sort();+ writeFileSync(baselinePath(repoRoot), JSON.stringify({+ _comment: 'Monotone-decreasing baseline of shipped SKILL.md files that tell agents to run a ruflo CLI command or subcommand the CLI does not register (entries are <skill path>::<command> or <skill path>::<command> <subcommand>). Fix one, then re-run with --update-baseline to lower the floor. NEVER add entries to hide a new reference.',+ unknownCount: knownUnknown.length,+ knownUnknown,+ updatedAt: new Date().toISOString(),+ }, null, 2) + '\n', 'utf8');+ console.log(`baseline updated β ${knownUnknown.length} unknown CLI reference(s)`);+ return 0;+ }++ const result = evaluateAgainstBaseline(unknown, readBaseline(repoRoot));+ const unloadable = [...tree].filter(([, e]) => e.loadError).map(([name, e]) => `${name}: ${e.loadError}`);+ if (argv.includes('--json')) {+ console.log(JSON.stringify({+ commandCount: tree.size,+ unloadable,+ skillFilesScanned: shippedSkillFiles(repoRoot).length,+ unknownCount: result.current.length,+ unexpected: result.unexpected,+ fixed: result.fixed,+ unknown: unknown.map((u) => ({ ref: u.ref, where: `${u.file}:${u.line}` })),+ }, null, 2));+ return result.ok ? 0 : 1;+ }++ console.log(`Skill β CLI command references: ${tree.size} top-level commands, ${shippedSkillFiles(repoRoot).length} shipped SKILL.md files, ${result.current.length} unknown reference(s) (baseline ${result.current.length - result.unexpected.length + result.fixed.length})`);+ if (unloadable.length) {+ const sample = unloadable.slice(0, 3).join('; ');+ console.log(` note: subcommands not checked for ${unloadable.length} command(s) that failed to import: ${sample}${unloadable.length > 3 ? '; ...' : ''}`);+ }+ if (result.ok) {+ console.log(' β no NEW unknown commands (known ones are listed in verification/skill-cli-command-baseline.json)');+ } else {+ console.log(' β NEW unknown command(s). Use a command registered in v3/@claude-flow/cli/src/commands/:');+ for (const key of result.unexpected) {+ for (const u of unknown.filter((x) => x.key === key)) console.log(` '${u.ref}' ${u.file}:${u.line}`);+ }+ }+ if (result.fixed.length > 0) {+ console.log(` Progress: ${result.fixed.length} baseline entr${result.fixed.length === 1 ? 'y is' : 'ies are'} fixed. Run: node --import tsx scripts/audit-skill-cli-commands.mjs --update-baseline`);+ }+ return result.ok ? 0 : 1;+}++// Real paths, as in audit-skill-mcp-tools.mjs: argv can keep a symlinked path+// (macOS /var -> /private/var) that would never equal the canonical module URL.+if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {+ process.exit(await main(process.argv.slice(2)));+}diff --git a/verification/skill-cli-command-baseline.json b/verification/skill-cli-command-baseline.jsonnew file mode 100644index 0000000000..09f29c0599--- /dev/null+++ b/verification/skill-cli-command-baseline.json@@ -0,0 +1,32 @@+{+ "_comment": "Monotone-decreasing baseline of shipped SKILL.md files that tell agents to run a ruflo CLI command or subcommand the CLI does not register (entries are <skill path>::<command> or <skill path>::<command> <subcommand>). Fix one, then re-run with --update-baseline to lower the floor. NEVER add entries to hide a new reference.",+ "unknownCount": 25,+ "knownUnknown": [+ "plugins/ruflo-intelligence/skills/neural-train/SKILL.md::neural compress",+ "v3/@claude-flow/cli/.claude/skills/github-multi-repo/SKILL.md::skill",+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::diagnostic-run",+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::github",+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::sparc",+ "v3/@claude-flow/cli/.claude/skills/github-workflow-automation/SKILL.md::github",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks agent-spawned",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks check-protected",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks mcp-initialized",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks memory-sync",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks neural-trained",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks post-search",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks pre-search",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks task-orchestrated",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::hooks validate-config",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::metrics",+ "v3/@claude-flow/cli/.claude/skills/pair-programming/SKILL.md::pair config",+ "v3/@claude-flow/cli/.claude/skills/pair-programming/SKILL.md::pair profile",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::sparc",+ "v3/@claude-flow/cli/.claude/skills/stream-chain/SKILL.md::stream-chain",+ "v3/@claude-flow/cli/.claude/skills/v3-cli-modernization/SKILL.md::learning",+ "v3/@claude-flow/cli/.claude/skills/v3-cli-modernization/SKILL.md::workflow create",+ "v3/@claude-flow/cli/.claude/skills/verification-quality/SKILL.md::report",+ "v3/@claude-flow/cli/.claude/skills/verification-quality/SKILL.md::train",+ "v3/@claude-flow/cli/.claude/skills/verification-quality/SKILL.md::truth"+ ],+ "updatedAt": "2026-10-28T07:24:38.869Z"+}