Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

test(skills): check that every MCP tool a shipped skill names exists in the registry (#4141)

Muhammad Zubair20b9451f8d31
diff --git a/scripts/__tests__/audit-skill-mcp-tools.test.mjs b/scripts/__tests__/audit-skill-mcp-tools.test.mjs
new file mode 100644
index 0000000000..cdf6275d47
--- /dev/null
+++ b/scripts/__tests__/audit-skill-mcp-tools.test.mjs
@@ -0,0 +1,67 @@
+import { describe, expect, it } from 'vitest';
+import {
+ evaluateAgainstBaseline,
+ findDanglingRefs,
+ readBaseline,
+ registeredToolNames,
+ shippedSkillFiles,
+ skillToolRefs,
+} from '../audit-skill-mcp-tools.mjs';
+
+describe('skill ↔ MCP tool reference audit', () => {
+ it('extracts ruflo tool references from both the init and the plugin prefix', () => {
+ const text = [
+ 'allowed-tools: mcp__plugin_ruflo-core_ruflo__memory_search Bash',
+ 'Call `mcp__claude-flow__swarm_init` first.',
+ ].join('\n');
+ expect(skillToolRefs(text)).toEqual([
+ { tool: 'memory_search', line: 1 },
+ { tool: 'swarm_init', line: 2 },
+ ]);
+ });
+
+ it('ignores other MCP servers, wildcards and scaffold placeholders', () => {
+ const text = [
+ 'mcp__github__create_issue',
+ 'mcp__plugin_ruflo-ai-team_ruflo-ai-team__team_launch',
+ 'allowed-tools: mcp__claude-flow__*',
+ 'allowed-tools: mcp__plugin_ruflo-core_ruflo__tool1 mcp__plugin_ruflo-core_ruflo__tool2',
+ ].join('\n');
+ expect(skillToolRefs(text)).toEqual([]);
+ });
+
+ it('passes references that are already in the baseline', () => {
+ const dangling = [{ key: 'plugins/a/skills/x/SKILL.md::old_tool' }];
+ const result = evaluateAgainstBaseline(dangling, ['plugins/a/skills/x/SKILL.md::old_tool']);
+ expect(result.ok).toBe(true);
+ expect(result.unexpected).toEqual([]);
+ });
+
+ it('fails a new reference even when the total count is unchanged', () => {
+ const dangling = [{ key: 'plugins/a/skills/x/SKILL.md::new_tool' }];
+ const result = evaluateAgainstBaseline(dangling, ['plugins/a/skills/x/SKILL.md::old_tool']);
+ expect(result.ok).toBe(false);
+ expect(result.unexpected).toEqual(['plugins/a/skills/x/SKILL.md::new_tool']);
+ expect(result.fixed).toEqual(['plugins/a/skills/x/SKILL.md::old_tool']);
+ });
+
+ it('reads a non-empty tool registry from mcp-client.ts', () => {
+ const registered = registeredToolNames();
+ // Spot-check tools that skills commonly reference, including ones whose
+ // descriptions use double quotes or template literals.
+ for (const name of ['memory_store', 'swarm_init', 'agentdb_pattern-search', 'hooks_pre-task', 'federation_bbs_sync']) {
+ expect(registered.has(name)).toBe(true);
+ }
+ });
+
+ it('scans the shipped skill trees', () => {
+ const files = shippedSkillFiles();
+ expect(files.some((f) => f.startsWith('v3/@claude-flow/cli/.claude/skills/'))).toBe(true);
+ expect(files.some((f) => f.startsWith('plugins/'))).toBe(true);
+ });
+
+ it('has no dangling references beyond verification/skill-mcp-tool-baseline.json', () => {
+ const result = evaluateAgainstBaseline(findDanglingRefs(), readBaseline());
+ expect(result.unexpected).toEqual([]);
+ });
+});
diff --git a/scripts/audit-skill-mcp-tools.mjs b/scripts/audit-skill-mcp-tools.mjs
new file mode 100644
index 0000000000..f419bbe0ce
--- /dev/null
+++ b/scripts/audit-skill-mcp-tools.mjs
@@ -0,0 +1,220 @@
+#!/usr/bin/env node
+/**
+ * Skill ↔ MCP tool reference audit. Sibling of scripts/audit-cli-mcp-tools.mjs (#1916).
+ *
+ * Skills tell Claude which ruflo MCP tools to call, in prose and in their
+ * `allowed-tools` frontmatter:
+ * - `mcp__claude-flow__<tool>` (skills copied by `ruflo init`)
+ * - `mcp__plugin_ruflo-core_ruflo__<tool>` (Claude Code marketplace plugins)
+ * A reference to a tool the CLI never registers fails at call time
+ * (`MCP tool not found`) or silently grants nothing in `allowed-tools`.
+ *
+ * This guard scans every SKILL.md that ships to users:
+ * - v3/@claude-flow/cli/.claude/skills/** (npm package, installed by init)
+ * - plugins/** (marketplace plugins)
+ * and checks each referenced tool name against the registry that
+ * `mcp-client.ts` assembles, resolved the same way audit-cli-mcp-tools.mjs
+ * resolves it (the `registerTools([ ... ])` spreads and their imports).
+ *
+ * Existing dangling references are carried in
+ * verification/skill-mcp-tool-baseline.json as `<skill path>::<tool>`
+ * entries. Like scripts/ci-test-ratchet.mjs the baseline is identity-based:
+ * a reference that is not in it fails, even if the total count is unchanged.
+ * Fixed entries never fail; they print a reminder to lower the floor.
+ *
+ * Usage:
+ * node scripts/audit-skill-mcp-tools.mjs # exit 1 on a NEW dangling reference
+ * node scripts/audit-skill-mcp-tools.mjs --json # machine-readable report
+ * node scripts/audit-skill-mcp-tools.mjs --update-baseline # after fixing some (never to hide new ones)
+ */
+
+import { existsSync, readdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const DEFAULT_REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+
+/** Skill trees that ship to users, relative to the repo root. */
+export const SKILL_ROOTS = ['v3/@claude-flow/cli/.claude/skills', 'plugins'];
+
+/** MCP server prefixes that resolve to the ruflo CLI tool registry. */
+const RUFLO_REF_RE = /mcp__(?:claude-flow|plugin_ruflo-core_ruflo)__([A-Za-z0-9_-]+)(\*?)/g;
+
+/**
+ * Names that are deliberately not real tools: the scaffold example in
+ * plugins/ruflo-plugin-creator/skills/create-plugin/SKILL.md uses
+ * `tool1`/`tool2` as placeholders.
+ */
+export const PLACEHOLDER_TOOLS = new Set(['tool1', 'tool2']);
+
+// A tool definition is `name: '<tool>',` followed by `description:` on the next
+// line. Descriptions use all three quote styles (the federation_bbs_* tools in
+// agentbbs-tools.ts use "..." and `...`), so accept any of them; matching only
+// single quotes would report those registered tools as dangling.
+const TOOL_DEF_RE = /name:\s*'([^']+)',\s*\n\s*description:\s*['"`]/g;
+
+function toPosix(path) {
+ return path.split(sep).join('/');
+}
+
+function cliSrc(repoRoot) {
+ return join(repoRoot, 'v3', '@claude-flow', 'cli', 'src');
+}
+
+/** Same resolution as audit-cli-mcp-tools.mjs: '.js' import specifiers map to '.ts' sources. */
+function resolveImportPath(repoRoot, spec) {
+ const p = spec.replace(/^\.\//, '').replace(/\.js$/, '.ts');
+ return join(cliSrc(repoRoot), p);
+}
+
+/** Files whose tool arrays are spread into `registerTools([ ... ])` in mcp-client.ts. */
+export function registeredToolSourceFiles(repoRoot = DEFAULT_REPO_ROOT) {
+ const mcpClient = join(cliSrc(repoRoot), 'mcp-client.ts');
+ const src = readFileSync(mcpClient, 'utf8');
+ const block = (src.match(/registerTools\(\[([\s\S]*?)\]\);/) || [, ''])[1];
+ const idents = new Set();
+ for (const m of block.matchAll(/\.\.\.\s*([A-Za-z_$][\w$]*)\s*\(?\s*\)?/g)) idents.add(m[1]);
+ const files = new Set();
+ for (const id of idents) {
+ const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ const im = src.match(new RegExp(`import\\s*\\{[^}]*\\b${escaped}\\b[^}]*\\}\\s*from\\s*['"]([^'"]+)['"]`));
+ if (im) {
+ const file = resolveImportPath(repoRoot, im[1]);
+ if (existsSync(file)) files.add(file);
+ continue;
+ }
+ // `...getXTools()` factories: guess mcp-tools/<x>-tools.ts, as the sibling audit does.
+ const guess = id.replace(/^get/, '').replace(/^([A-Z])/, (c) => c.toLowerCase()).replace(/([A-Z])/g, '-$1').toLowerCase();
+ const candidate = join(cliSrc(repoRoot), 'mcp-tools', `${guess}.ts`);
+ if (existsSync(candidate)) files.add(candidate);
+ }
+ return [...files];
+}
+
+/** Every tool name the CLI registers. */
+export function registeredToolNames(repoRoot = DEFAULT_REPO_ROOT) {
+ const names = new Set();
+ for (const file of registeredToolSourceFiles(repoRoot)) {
+ const src = readFileSync(file, 'utf8');
+ for (const m of src.matchAll(TOOL_DEF_RE)) names.add(m[1]);
+ }
+ return names;
+}
+
+/** Ruflo MCP tool references in one SKILL.md body: [{ tool, line }]. */
+export function skillToolRefs(text) {
+ const refs = [];
+ for (const m of text.matchAll(RUFLO_REF_RE)) {
+ const [, tool, star] = m;
+ if (star || PLACEHOLDER_TOOLS.has(tool)) continue;
+ refs.push({ tool, line: text.slice(0, m.index).split('\n').length });
+ }
+ return refs;
+}
+
+function walkSkillFiles(dir, out) {
+ if (!existsSync(dir)) return out;
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
+ if (entry.name === 'node_modules' || entry.name === '.git') continue;
+ const full = join(dir, entry.name);
+ if (entry.isDirectory()) walkSkillFiles(full, out);
+ else if (entry.isFile() && entry.name === 'SKILL.md') out.push(full);
+ }
+ return out;
+}
+
+/** Every shipped SKILL.md, as repo-relative POSIX paths, sorted. */
+export function shippedSkillFiles(repoRoot = DEFAULT_REPO_ROOT) {
+ const files = [];
+ for (const root of SKILL_ROOTS) walkSkillFiles(join(repoRoot, root), files);
+ return files.map((f) => toPosix(relative(repoRoot, f))).sort();
+}
+
+/** Dangling references across shipped skills: [{ file, tool, line, key }]. */
+export function findDanglingRefs(repoRoot = DEFAULT_REPO_ROOT, registered = registeredToolNames(repoRoot)) {
+ const dangling = [];
+ for (const file of shippedSkillFiles(repoRoot)) {
+ const text = readFileSync(join(repoRoot, file), 'utf8');
+ for (const ref of skillToolRefs(text)) {
+ if (!registered.has(ref.tool)) dangling.push({ file, tool: ref.tool, line: ref.line, key: `${file}::${ref.tool}` });
+ }
+ }
+ return dangling;
+}
+
+/**
+ * Compare current dangling references with the baseline entries.
+ * ok is false only when a reference is not already in the baseline.
+ */
+export function evaluateAgainstBaseline(dangling, baselineEntries) {
+ const known = new Set(baselineEntries);
+ const current = [...new Set(dangling.map((d) => d.key))].sort();
+ const currentSet = new Set(current);
+ const unexpected = current.filter((key) => !known.has(key));
+ const fixed = [...known].filter((key) => !currentSet.has(key)).sort();
+ return { ok: unexpected.length === 0, current, unexpected, fixed };
+}
+
+export function baselinePath(repoRoot = DEFAULT_REPO_ROOT) {
+ return join(repoRoot, 'verification', 'skill-mcp-tool-baseline.json');
+}
+
+export function readBaseline(repoRoot = DEFAULT_REPO_ROOT) {
+ const file = baselinePath(repoRoot);
+ if (!existsSync(file)) return [];
+ const parsed = JSON.parse(readFileSync(file, 'utf8'));
+ return Array.isArray(parsed.knownDangling) ? parsed.knownDangling : [];
+}
+
+function main(argv) {
+ const repoRoot = DEFAULT_REPO_ROOT;
+ const registered = registeredToolNames(repoRoot);
+ const dangling = findDanglingRefs(repoRoot, registered);
+
+ if (argv.includes('--update-baseline')) {
+ const knownDangling = [...new Set(dangling.map((d) => d.key))].sort();
+ const next = {
+ _comment: 'Monotone-decreasing baseline of shipped SKILL.md files that reference a ruflo MCP tool not registered by mcp-client.ts (entries are <skill path>::<tool>). Fix one, then re-run with --update-baseline to lower the floor. NEVER add entries to hide a new reference.',
+ danglingCount: knownDangling.length,
+ knownDangling,
+ updatedAt: new Date().toISOString(),
+ };
+ writeFileSync(baselinePath(repoRoot), JSON.stringify(next, null, 2) + '\n', 'utf8');
+ console.log(`baseline updated β†’ ${knownDangling.length} dangling skill reference(s)`);
+ return 0;
+ }
+
+ const result = evaluateAgainstBaseline(dangling, readBaseline(repoRoot));
+ if (argv.includes('--json')) {
+ console.log(JSON.stringify({
+ registeredToolCount: registered.size,
+ skillFilesScanned: shippedSkillFiles(repoRoot).length,
+ danglingCount: result.current.length,
+ unexpected: result.unexpected,
+ fixed: result.fixed,
+ dangling: dangling.map((d) => ({ tool: d.tool, where: `${d.file}:${d.line}` })),
+ }, null, 2));
+ return result.ok ? 0 : 1;
+ }
+
+ console.log(`Skill ↔ MCP tool references: ${registered.size} registered tools, ${shippedSkillFiles(repoRoot).length} shipped SKILL.md files, ${result.current.length} dangling reference(s) (baseline ${result.current.length - result.unexpected.length + result.fixed.length})`);
+ if (result.ok) {
+ console.log(' βœ“ no NEW dangling references (known ones are listed in verification/skill-mcp-tool-baseline.json)');
+ } else {
+ console.log(' βœ— NEW dangling reference(s). Use a tool name registered in v3/@claude-flow/cli/src/mcp-tools/*.ts:');
+ for (const key of result.unexpected) {
+ for (const d of dangling.filter((x) => x.key === key)) console.log(` '${d.tool}' ${d.file}:${d.line}`);
+ }
+ }
+ if (result.fixed.length > 0) {
+ console.log(` Progress: ${result.fixed.length} baseline entr${result.fixed.length === 1 ? 'y is' : 'ies are'} fixed. Run: node scripts/audit-skill-mcp-tools.mjs --update-baseline`);
+ }
+ return result.ok ? 0 : 1;
+}
+
+// Compare real paths, as check-guard-tool-names.mjs does: Node canonicalizes the
+// module URL, while argv can keep a symlinked path (including macOS /var ->
+// /private/var). A string compare would silently skip the audit there.
+if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ process.exit(main(process.argv.slice(2)));
+}
diff --git a/verification/skill-mcp-tool-baseline.json b/verification/skill-mcp-tool-baseline.json
new file mode 100644
index 0000000000..b55ffabd33
--- /dev/null
+++ b/verification/skill-mcp-tool-baseline.json
@@ -0,0 +1,55 @@
+{
+ "_comment": "Monotone-decreasing baseline of shipped SKILL.md files that reference a ruflo MCP tool not registered by mcp-client.ts (entries are <skill path>::<tool>). Fix one, then re-run with --update-baseline to lower the floor. NEVER add entries to hide a new reference.",
+ "danglingCount": 48,
+ "knownDangling": [
+ "v3/@claude-flow/cli/.claude/skills/github-multi-repo/SKILL.md::memory_usage",
+ "v3/@claude-flow/cli/.claude/skills/github-multi-repo/SKILL.md::task_orchestrate",
+ "v3/@claude-flow/cli/.claude/skills/github-project-management/SKILL.md::task_orchestrate",
+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::memory_usage",
+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::swarm_monitor",
+ "v3/@claude-flow/cli/.claude/skills/github-release-management/SKILL.md::task_orchestrate",
+ "v3/@claude-flow/cli/.claude/skills/github-workflow-automation/SKILL.md::task_orchestrate",
+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::memory_persist",
+ "v3/@claude-flow/cli/.claude/skills/hooks-automation/SKILL.md::memory_usage",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::bottleneck_analyze",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::context_restore",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::memory_persist",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::memory_usage",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::sparc_mode",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::swarm_monitor",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::task_orchestrate",
+ "v3/@claude-flow/cli/.claude/skills/sparc-methodology/SKILL.md::token_usage",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::automation_setup",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::batch_process",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::benchmark_run",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::bottleneck_analyze",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::cognitive_analyze",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::context_restore",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::cost_analysis",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::daa_fault_tolerance",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::error_analysis",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::health_check",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::learning_adapt",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::load_balance",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_backup",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_namespace",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_persist",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::memory_usage",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::metrics_collect",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::parallel_execute",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::pattern_recognize",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::pipeline_create",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::quality_assess",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::security_scan",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::state_snapshot",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::swarm_monitor",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::swarm_scale",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::task_orchestrate",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::task_results",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::topology_optimize",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::trend_analysis",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::trigger_setup",
+ "v3/@claude-flow/cli/.claude/skills/swarm-advanced/SKILL.md::usage_stats"
+ ],
+ "updatedAt": "2026-10-13T08:38:37.635Z"
+}