π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
test(skills): check that every MCP tool a shipped skill names exists in the registry (#4141)
Muhammad Zubair20b9451f8d31
diff --git a/scripts/__tests__/audit-skill-mcp-tools.test.mjs b/scripts/__tests__/audit-skill-mcp-tools.test.mjsnew file mode 100644index 0000000000..cdf6275d47--- /dev/null+++ b/scripts/__tests__/audit-skill-mcp-tools.test.mjs@@ -0,0 +1,67 @@+import { describe, expect, it } from 'vitest';+import {+ evaluateAgainstBaseline,+ findDanglingRefs,+ readBaseline,+ registeredToolNames,+ shippedSkillFiles,+ skillToolRefs,+} from '../audit-skill-mcp-tools.mjs';++describe('skill β MCP tool reference audit', () => {+ it('extracts ruflo tool references from both the init and the plugin prefix', () => {+ const text = [+ 'allowed-tools: mcp__plugin_ruflo-core_ruflo__memory_search Bash',+ 'Call `mcp__claude-flow__swarm_init` first.',+ ].join('\n');+ expect(skillToolRefs(text)).toEqual([+ { tool: 'memory_search', line: 1 },+ { tool: 'swarm_init', line: 2 },+ ]);+ });++ it('ignores other MCP servers, wildcards and scaffold placeholders', () => {+ const text = [+ 'mcp__github__create_issue',+ 'mcp__plugin_ruflo-ai-team_ruflo-ai-team__team_launch',+ 'allowed-tools: mcp__claude-flow__*',+ 'allowed-tools: mcp__plugin_ruflo-core_ruflo__tool1 mcp__plugin_ruflo-core_ruflo__tool2',+ ].join('\n');+ expect(skillToolRefs(text)).toEqual([]);+ });++ it('passes references that are already in the baseline', () => {+ const dangling = [{ key: 'plugins/a/skills/x/SKILL.md::old_tool' }];+ const result = evaluateAgainstBaseline(dangling, ['plugins/a/skills/x/SKILL.md::old_tool']);+ expect(result.ok).toBe(true);+ expect(result.unexpected).toEqual([]);+ });++ it('fails a new reference even when the total count is unchanged', () => {+ const dangling = [{ key: 'plugins/a/skills/x/SKILL.md::new_tool' }];+ const result = evaluateAgainstBaseline(dangling, ['plugins/a/skills/x/SKILL.md::old_tool']);+ expect(result.ok).toBe(false);+ expect(result.unexpected).toEqual(['plugins/a/skills/x/SKILL.md::new_tool']);+ expect(result.fixed).toEqual(['plugins/a/skills/x/SKILL.md::old_tool']);+ });++ it('reads a non-empty tool registry from mcp-client.ts', () => {+ const registered = registeredToolNames();+ // Spot-check tools that skills commonly reference, including ones whose+ // descriptions use double quotes or template literals.+ for (const name of ['memory_store', 'swarm_init', 'agentdb_pattern-search', 'hooks_pre-task', 'federation_bbs_sync']) {+ expect(registered.has(name)).toBe(true);+ }+ });++ it('scans the shipped skill trees', () => {+ const files = shippedSkillFiles();+ expect(files.some((f) => f.startsWith('v3/@claude-flow/cli/.claude/skills/'))).toBe(true);+ expect(files.some((f) => f.startsWith('plugins/'))).toBe(true);+ });++ it('has no dangling references beyond verification/skill-mcp-tool-baseline.json', () => {+ const result = evaluateAgainstBaseline(findDanglingRefs(), readBaseline());+ expect(result.unexpected).toEqual([]);+ });+});diff --git a/scripts/audit-skill-mcp-tools.mjs b/scripts/audit-skill-mcp-tools.mjsnew file mode 100644index 0000000000..f419bbe0ce--- /dev/null+++ b/scripts/audit-skill-mcp-tools.mjs@@ -0,0 +1,220 @@+#!/usr/bin/env node+/**+ * Skill β MCP tool reference audit. Sibling of scripts/audit-cli-mcp-tools.mjs (#1916).+ *+ * Skills tell Claude which ruflo MCP tools to call, in prose and in their+ * `allowed-tools` frontmatter:+ * - `mcp__claude-flow__<tool>` (skills copied by `ruflo init`)+ * - `mcp__plugin_ruflo-core_ruflo__<tool>` (Claude Code marketplace plugins)+ * A reference to a tool the CLI never registers fails at call time+ * (`MCP tool not found`) or silently grants nothing in `allowed-tools`.+ *+ * This guard scans every SKILL.md that ships to users:+ * - v3/@claude-flow/cli/.claude/skills/** (npm package, installed by init)+ * - plugins/** (marketplace plugins)+ * and checks each referenced tool name against the registry that+ * `mcp-client.ts` assembles, resolved the same way audit-cli-mcp-tools.mjs+ * resolves it (the `registerTools([ ... ])` spreads and their imports).+ *+ * Existing dangling references are carried in+ * verification/skill-mcp-tool-baseline.json as `<skill path>::<tool>`+ * entries. Like scripts/ci-test-ratchet.mjs the baseline is identity-based:+ * a reference that is not in it fails, even if the total count is unchanged.+ * Fixed entries never fail; they print a reminder to lower the floor.+ *+ * Usage:+ * node scripts/audit-skill-mcp-tools.mjs # exit 1 on a NEW dangling reference+ * node scripts/audit-skill-mcp-tools.mjs --json # machine-readable report+ * node scripts/audit-skill-mcp-tools.mjs --update-baseline # after fixing some (never to hide new ones)+ */++import { existsSync, readdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';+import { dirname, join, relative, resolve, sep } from 'node:path';+import { fileURLToPath } from 'node:url';++const DEFAULT_REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');++/** Skill trees that ship to users, relative to the repo root. */+export const SKILL_ROOTS = ['v3/@claude-flow/cli/.claude/skills', 'plugins'];++/** MCP server prefixes that resolve to the ruflo CLI tool registry. */+const RUFLO_REF_RE = /mcp__(?:claude-flow|plugin_ruflo-core_ruflo)__([A-Za-z0-9_-]+)(\*?)/g;++/**+ * Names that are deliberately not real tools: the scaffold example in+ * plugins/ruflo-plugin-creator/skills/create-plugin/SKILL.md uses+ * `tool1`/`tool2` as placeholders.+ */+export const PLACEHOLDER_TOOLS = new Set(['tool1', 'tool2']);++// A tool definition is `name: '<tool>',` followed by `description:` on the next+// line. Descriptions use all three quote styles (the federation_bbs_* tools in+// agentbbs-tools.ts use "..." and `...`), so accept any of them; matching only+// single quotes would report those registered tools as dangling.+const TOOL_DEF_RE = /name:\s*'([^']+)',\s*\n\s*description:\s*['"`]/g;++function toPosix(path) {+ return path.split(sep).join('/');+}++function cliSrc(repoRoot) {+ return join(repoRoot, 'v3', '@claude-flow', 'cli', 'src');+}++/** Same resolution as audit-cli-mcp-tools.mjs: '.js' import specifiers map to '.ts' sources. */+function resolveImportPath(repoRoot, spec) {+ const p = spec.replace(/^\.\//, '').replace(/\.js$/, '.ts');+ return join(cliSrc(repoRoot), p);+}++/** Files whose tool arrays are spread into `registerTools([ ... ])` in mcp-client.ts. */+export function registeredToolSourceFiles(repoRoot = DEFAULT_REPO_ROOT) {+ const mcpClient = join(cliSrc(repoRoot), 'mcp-client.ts');+ const src = readFileSync(mcpClient, 'utf8');+ const block = (src.match(/registerTools\(\[([\s\S]*?)\]\);/) || [, ''])[1];+ const idents = new Set();+ for (const m of block.matchAll(/\.\.\.\s*([A-Za-z_$][\w$]*)\s*\(?\s*\)?/g)) idents.add(m[1]);+ const files = new Set();+ for (const id of idents) {+ const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ const im = src.match(new RegExp(`import\\s*\\{[^}]*\\b${escaped}\\b[^}]*\\}\\s*from\\s*['"]([^'"]+)['"]`));+ if (im) {+ const file = resolveImportPath(repoRoot, im[1]);+ if (existsSync(file)) files.add(file);+ continue;+ }+ // `...getXTools()` factories: guess mcp-tools/<x>-tools.ts, as the sibling audit does.+ const guess = id.replace(/^get/, '').replace(/^([A-Z])/, (c) => c.toLowerCase()).replace(/([A-Z])/g, '-$1').toLowerCase();+ const candidate = join(cliSrc(repoRoot), 'mcp-tools', `${guess}.ts`);+ if (existsSync(candidate)) files.add(candidate);+ }+ return [...files];+}++/** Every tool name the CLI registers. */+export function registeredToolNames(repoRoot = DEFAULT_REPO_ROOT) {+ const names = new Set();+ for (const file of registeredToolSourceFiles(repoRoot)) {+ const src = readFileSync(file, 'utf8');+ for (const m of src.matchAll(TOOL_DEF_RE)) names.add(m[1]);+ }+ return names;+}++/** Ruflo MCP tool references in one SKILL.md body: [{ tool, line }]. */+export function skillToolRefs(text) {+ const refs = [];+ for (const m of text.matchAll(RUFLO_REF_RE)) {+ const [, tool, star] = m;+ if (star || PLACEHOLDER_TOOLS.has(tool)) continue;+ refs.push({ tool, line: text.slice(0, m.index).split('\n').length });+ }+ return refs;+}++function walkSkillFiles(dir, out) {+ if (!existsSync(dir)) return out;+ for (const entry of readdirSync(dir, { withFileTypes: true })) {+ if (entry.name === 'node_modules' || entry.name === '.git') continue;+ const full = join(dir, entry.name);+ if (entry.isDirectory()) walkSkillFiles(full, out);+ else if (entry.isFile() && entry.name === 'SKILL.md') out.push(full);+ }+ return out;+}++/** Every shipped SKILL.md, as repo-relative POSIX paths, sorted. */+export function shippedSkillFiles(repoRoot = DEFAULT_REPO_ROOT) {+ const files = [];+ for (const root of SKILL_ROOTS) walkSkillFiles(join(repoRoot, root), files);+ return files.map((f) => toPosix(relative(repoRoot, f))).sort();+}++/** Dangling references across shipped skills: [{ file, tool, line, key }]. */+export function findDanglingRefs(repoRoot = DEFAULT_REPO_ROOT, registered = registeredToolNames(repoRoot)) {+ const dangling = [];+ for (const file of shippedSkillFiles(repoRoot)) {+ const text = readFileSync(join(repoRoot, file), 'utf8');+ for (const ref of skillToolRefs(text)) {+ if (!registered.has(ref.tool)) dangling.push({ file, tool: ref.tool, line: ref.line, key: `${file}::${ref.tool}` });+ }+ }+ return dangling;+}++/**+ * Compare current dangling references with the baseline entries.+ * ok is false only when a reference is not already in the baseline.+ */+export function evaluateAgainstBaseline(dangling, baselineEntries) {+ const known = new Set(baselineEntries);+ const current = [...new Set(dangling.map((d) => d.key))].sort();+ const currentSet = new Set(current);+ const unexpected = current.filter((key) => !known.has(key));+ const fixed = [...known].filter((key) => !currentSet.has(key)).sort();+ return { ok: unexpected.length === 0, current, unexpected, fixed };+}++export function baselinePath(repoRoot = DEFAULT_REPO_ROOT) {+ return join(repoRoot, 'verification', 'skill-mcp-tool-baseline.json');+}++export function readBaseline(repoRoot = DEFAULT_REPO_ROOT) {+ const file = baselinePath(repoRoot);+ if (!existsSync(file)) return [];+ const parsed = JSON.parse(readFileSync(file, 'utf8'));+ return Array.isArray(parsed.knownDangling) ? parsed.knownDangling : [];+}++function main(argv) {+ const repoRoot = DEFAULT_REPO_ROOT;+ const registered = registeredToolNames(repoRoot);+ const dangling = findDanglingRefs(repoRoot, registered);++ if (argv.includes('--update-baseline')) {+ const knownDangling = [...new Set(dangling.map((d) => d.key))].sort();+ const next = {+ _comment: 'Monotone-decreasing baseline of shipped SKILL.md files that reference a ruflo MCP tool not registered by mcp-client.ts (entries are <skill path>::<tool>). Fix one, then re-run with --update-baseline to lower the floor. NEVER add entries to hide a new reference.',+ danglingCount: knownDangling.length,+ knownDangling,+ updatedAt: new Date().toISOString(),+ };+ writeFileSync(baselinePath(repoRoot), JSON.stringify(next, null, 2) + '\n', 'utf8');+ console.log(`baseline updated β ${knownDangling.length} dangling skill reference(s)`);+ return 0;+ }++ const result = evaluateAgainstBaseline(dangling, readBaseline(repoRoot));+ if (argv.includes('--json')) {+ console.log(JSON.stringify({+ registeredToolCount: registered.size,+ skillFilesScanned: shippedSkillFiles(repoRoot).length,+ danglingCount: result.current.length,+ unexpected: result.unexpected,+ fixed: result.fixed,+ dangling: dangling.map((d) => ({ tool: d.tool, where: `${d.file}:${d.line}` })),+ }, null, 2));+ return result.ok ? 0 : 1;+ }++ console.log(`Skill β MCP tool references: ${registered.size} registered tools, ${shippedSkillFiles(repoRoot).length} shipped SKILL.md files, ${result.current.length} dangling reference(s) (baseline ${result.current.length - result.unexpected.length + result.fixed.length})`);+ if (result.ok) {+ console.log(' β no NEW dangling references (known ones are listed in verification/skill-mcp-tool-baseline.json)');+ } else {+ console.log(' β NEW dangling reference(s). Use a tool name registered in v3/@claude-flow/cli/src/mcp-tools/*.ts:');+ for (const key of result.unexpected) {+ for (const d of dangling.filter((x) => x.key === key)) console.log(` '${d.tool}' ${d.file}:${d.line}`);+ }+ }+ if (result.fixed.length > 0) {+ console.log(` Progress: ${result.fixed.length} baseline entr${result.fixed.length === 1 ? 'y is' : 'ies are'} fixed. Run: node scripts/audit-skill-mcp-tools.mjs --update-baseline`);+ }+ return result.ok ? 0 : 1;+}++// Compare real paths, as check-guard-tool-names.mjs does: Node canonicalizes the+// module URL, while argv can keep a symlinked path (including macOS /var ->+// /private/var). A string compare would silently skip the audit there.+if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {+ process.exit(main(process.argv.slice(2)));+}diff --git a/verification/skill-mcp-tool-baseline.json b/verification/skill-mcp-tool-baseline.jsonnew file mode 100644index 0000000000..b55ffabd33--- /dev/null+++ b/verification/skill-mcp-tool-baseline.json@@ -0,0 +1,55 @@+{+ "_comment": "Monotone-decreasing baseline of shipped SKILL.md files that reference a ruflo MCP tool not registered by mcp-client.ts (entries are <skill path>::<tool>). Fix one, then re-run with --update-baseline to lower the floor. NEVER add entries to hide a new reference.",+ "danglingCount": 48,+ "knownDangling": [+ "v3/@claude-flow/cli/.claude/skills/github-multi-repo/SKILL.md::memory_usage",+ "v3/@claude-flow/cli/.claude/skills/github-multi-repo/SKILL.md::task_orchestrate",+ "v3/@claude-flow/cli/.claude/skills/github-project-management/SKILL.md::task_orchestrate",+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::memory_usage",+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::swarm_monitor",+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::task_orchestrate",+ "v3/@claude-flow/cli/.claude/skills/github-workflow-automation/SKILL.md::task_orchestrate",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::memory_persist",+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::memory_usage",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::bottleneck_analyze",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::context_restore",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::memory_persist",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::memory_usage",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::sparc_mode",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::swarm_monitor",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::task_orchestrate",+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::token_usage",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::automation_setup",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::batch_process",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::benchmark_run",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::bottleneck_analyze",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::cognitive_analyze",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::context_restore",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::cost_analysis",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::daa_fault_tolerance",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::error_analysis",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::health_check",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::learning_adapt",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::load_balance",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_backup",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_namespace",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_persist",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_usage",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::metrics_collect",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::parallel_execute",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::pattern_recognize",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::pipeline_create",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::quality_assess",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::security_scan",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::state_snapshot",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::swarm_monitor",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::swarm_scale",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::task_orchestrate",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::task_results",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::topology_optimize",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::trend_analysis",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::trigger_setup",+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::usage_stats"+ ],+ "updatedAt": "2026-10-13T08:38:37.635Z"+}