Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

feat(metaharness): integrate @metaharness/redblue@~0.1.1 β€” adversarial red/blue LLM testing

ruva63cdf052663
diff --git a/CLAUDE.md b/CLAUDE.md
index 1ac853046d..0b25d2863d 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -1140,6 +1140,17 @@ npx ruflo metaharness drift-from-history \ # iter 53 β€” 1-command drift (
# iter 67 β€” --baseline-file skips memory entirely (~19x faster)
# iter 78 β€” --alert-on-new-severity adds orthogonal finding-severity gate
npx ruflo metaharness mint --name foo --template vertical:coding --confirm
+npx ruflo metaharness redblue init # @metaharness/redblue β€” scaffold redblue.yaml
+npx ruflo metaharness redblue run --mock-judge --tests 10
+ # $0 marker-fixture path (CI / offline)
+npx ruflo metaharness redblue run --tests 50 --patch
+ # real model judge (needs OPENROUTER_API_KEY,
+ # capped by max_cost_usd, default $3)
+npx ruflo metaharness redblue attack prompt --count 3
+ # preview generated attack cases (no target call)
+npx ruflo metaharness redblue patch --mock-judge # baseline β†’ blue-team patch β†’ retest delta
+npx ruflo metaharness redblue report --in report.json
+ # render existing report as markdown
# Dedicated command
npx ruflo eject --name my-harness # lift ruflo project β†’ standalone harness
@@ -1161,6 +1172,7 @@ mcp__claude-flow__metaharness_drift_from_history # iter 53 β€” 1-command drift
mcp__claude-flow__metaharness_bench # ADR-153 β€” create/verify bench suites for evolve --bench
mcp__claude-flow__metaharness_evolve # MAP-Elites driver β€” evolve a harness across bench suites
mcp__claude-flow__metaharness_security_bench # security-focused benchmark suite gate
+mcp__claude-flow__metaharness_redblue # @metaharness/redblue β€” adversarial red/blue LLM testing (init|run|patch|attack|report)
```
### Routing integration (ADR-148/149)
diff --git a/plugins/ruflo-metaharness/scripts/_redblue.mjs b/plugins/ruflo-metaharness/scripts/_redblue.mjs
new file mode 100644
index 0000000000..a0866e3af7
--- /dev/null
+++ b/plugins/ruflo-metaharness/scripts/_redblue.mjs
@@ -0,0 +1,176 @@
+// _redblue.mjs β€” invocation helper for `@metaharness/redblue`.
+//
+// Sibling of `_darwin.mjs` / `_harness.mjs`. Targets the `redblue` binary
+// from the standalone `@metaharness/redblue@~0.1.1` package.
+//
+// Subcommands surfaced (matches redblue 0.1.x):
+// - `redblue init [--out redblue.yaml]`
+// - `redblue run [--config redblue.yaml] [--tests N] [--patch] [--mock-judge] [--out report.json]`
+// - `redblue attack <prompt|tools|data|all> [--count N]`
+// - `redblue patch [--config redblue.yaml] [--mock-judge]`
+// - `redblue report --in report.json`
+//
+// UPSTREAM BUG WORKAROUND
+// -----------------------
+// @metaharness/[email protected]'s CLI bootstrap is:
+// const isMain = import.meta.url === `file://${process.argv[1]}`;
+// if (isMain) { dispatch(...) }
+// When npx links `redblue` β†’ `dist/cli/index.js`, process.argv[1] is the
+// symlink path but import.meta.url is the resolved real path, so the
+// check fails silently β€” the binary exits 0 with no output, no file,
+// no error. Darwin's CLI doesn't suffer this (calls main() unconditionally).
+//
+// Workaround: install the package once into a ruflo-owned cache dir
+// (~/.ruflo/redblue-cache) and invoke `node <abs_path>/dist/cli/index.js`
+// directly. argv[1] then equals the real path and isMain becomes true.
+//
+// Track upstream fix at: github.com/ruvnet/agent-harness-generator/issues
+// (file separately β€” when fixed, we can drop the cached-install path and
+// go back to the `npx -y -p ...` pattern used by _darwin.mjs).
+//
+// CONTRACT (matches runMetaharness/runDarwin):
+// - returns `{ stdout, stderr, exitCode, durationMs, degraded, reason? }`
+// - subprocess hard timeout (default 120s; --mock-judge runs are seconds)
+// - on install failure / MODULE_NOT_FOUND / network failure, returns
+// `degraded: true, reason: 'metaharness-redblue-not-available'`
+// - never throws β€” ADR-150 graceful-degradation rule #3
+//
+// SAFETY NOTE
+// redblue itself enforces hard safety boundaries in `src/config/safety.ts`
+// (no real creds, no live targets, no shell, no eval, no arbitrary network).
+// This wrapper does NOT relax those β€” it only forwards argv with shell:false.
+// `--mock-judge` is the $0 CI path; the real model judge gates on
+// $OPENROUTER_API_KEY which we never inject.
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+const DEFAULT_TIMEOUT_MS = 120_000;
+const INSTALL_TIMEOUT_MS = 180_000; // npm install can be slow on cold cache
+
+// Pinned semver range. Bump in lock-step with optionalDependencies in
+// @claude-flow/cli/package.json + ruflo/package.json.
+const REDBLUE_PIN = '@metaharness/redblue@~0.1.1';
+const REDBLUE_PKG = '@metaharness/redblue';
+const REDBLUE_PIN_VERSION = '~0.1.1';
+
+const CACHE_DIR = join(homedir(), '.ruflo', 'redblue-cache');
+const RESOLVED_CLI = join(
+ CACHE_DIR, 'node_modules', '@metaharness', 'redblue', 'dist', 'cli', 'index.js',
+);
+
+const DEGRADED_RX = /could not determine executable|404|not installed|MODULE_NOT_FOUND|ENOTFOUND|getaddrinfo|ECONNREFUSED|ETIMEDOUT|npm ERR/i;
+
+function ensureInstalled() {
+ if (existsSync(RESOLVED_CLI)) return { ok: true };
+ try {
+ mkdirSync(CACHE_DIR, { recursive: true });
+ } catch (e) {
+ return { ok: false, reason: 'cache-dir-create-failed', error: String(e) };
+ }
+ // Use `npm install --prefix` so the package lands in a known location;
+ // this avoids npx's symlinked shim entirely (the upstream bin-bootstrap bug).
+ // shell:false; argv only.
+ const r = spawnSync('npm', [
+ 'install',
+ '--no-audit', '--no-fund', '--no-package-lock',
+ '--prefix', CACHE_DIR,
+ `${REDBLUE_PKG}@${REDBLUE_PIN_VERSION}`,
+ ], {
+ stdio: ['ignore', 'pipe', 'pipe'],
+ encoding: 'utf-8',
+ timeout: INSTALL_TIMEOUT_MS,
+ shell: process.platform === 'win32',
+ });
+ if (r.status !== 0 || !existsSync(RESOLVED_CLI)) {
+ return {
+ ok: false,
+ reason: 'install-failed',
+ stderr: (r.stderr || '').slice(0, 600),
+ stdout: (r.stdout || '').slice(0, 600),
+ };
+ }
+ return { ok: true };
+}
+
+function classifyDegraded(stderr, exitCode) {
+ if (exitCode === null) return { degraded: true, reason: 'metaharness-redblue-timeout' };
+ if (DEGRADED_RX.test(stderr)) return { degraded: true, reason: 'metaharness-redblue-not-available' };
+ return { degraded: false };
+}
+
+/**
+ * Sync invocation. `redblue` runs are bounded by `max_cost_usd` and
+ * `max_runtime_minutes` from the config, so sync is fine for the CLI
+ * wrapper. The default --mock-judge fixture path completes in seconds.
+ */
+export function runRedblue(args, opts = {}) {
+ const start = Date.now();
+ const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+
+ // 1. Ensure redblue is installed in our cache dir (one-time install).
+ const install = ensureInstalled();
+ if (!install.ok) {
+ return {
+ stdout: install.stdout ?? '',
+ stderr: install.stderr ?? install.error ?? '',
+ exitCode: 127,
+ durationMs: Date.now() - start,
+ degraded: true,
+ reason: install.reason === 'install-failed'
+ ? 'metaharness-redblue-not-available'
+ : `metaharness-redblue-${install.reason}`,
+ };
+ }
+
+ // 2. Invoke `node <real-path-to-cli> <args>` so upstream's isMain check
+ // succeeds (argv[1] matches import.meta.url).
+ const r = spawnSync('node', [RESOLVED_CLI, ...args], {
+ stdio: ['ignore', 'pipe', 'pipe'],
+ encoding: 'utf-8',
+ timeout: timeoutMs,
+ cwd: opts.cwd,
+ env: { ...process.env, ...(opts.env || {}) },
+ shell: process.platform === 'win32',
+ });
+ const durationMs = Date.now() - start;
+ const stdout = r.stdout || '';
+ const stderr = r.stderr || '';
+ const classified = classifyDegraded(stderr, r.status);
+ if (classified.degraded) {
+ return {
+ stdout, stderr,
+ exitCode: r.status ?? 127,
+ durationMs,
+ degraded: true,
+ reason: classified.reason,
+ };
+ }
+ return {
+ stdout, stderr,
+ exitCode: r.status ?? 0,
+ durationMs,
+ degraded: false,
+ };
+}
+
+/**
+ * Match the existing `emit*DegradedJsonAndExit` shape used by sibling
+ * scripts so MCP tool consumers see one contract.
+ */
+export function emitRedblueDegradedJsonAndExit(reason) {
+ const payload = {
+ degraded: true,
+ reason,
+ hint: 'Install with `npm i -D ' + REDBLUE_PIN + '` or run `npx -y ' + REDBLUE_PIN + ' --help` to verify network access.',
+ generatedAt: new Date().toISOString(),
+ };
+ console.log(JSON.stringify(payload, null, 2));
+ // Exit 0 β€” ADR-150 architectural constraint #3.
+ process.exit(0);
+}
+
+export const REDBLUE_VERSION_PIN = REDBLUE_PIN;
+export const REDBLUE_CACHE_DIR = CACHE_DIR;
diff --git a/plugins/ruflo-metaharness/scripts/redblue.mjs b/plugins/ruflo-metaharness/scripts/redblue.mjs
new file mode 100644
index 0000000000..66b43ae705
--- /dev/null
+++ b/plugins/ruflo-metaharness/scripts/redblue.mjs
@@ -0,0 +1,286 @@
+#!/usr/bin/env node
+// redblue.mjs β€” wrapper around `@metaharness/redblue` for LLM red/blue testing.
+//
+// ADR-150 sibling-package integration: `@metaharness/redblue` runs an
+// adversarial red-team β†’ judge β†’ blue-team patch loop against an LLM
+// target you own. Upstream README: packages/redblue#readme on
+// github.com/ruvnet/agent-harness-generator.
+//
+// SAFETY (enforced by upstream `src/config/safety.ts`, NOT this wrapper):
+// - no real credentials (`allow_real_credentials:true` = load-time error)
+// - no live external targets (validateTarget() rejects non-loopback)
+// - no shell, no arbitrary network, no eval
+// - sensitive outputs redacted before storage
+// - `max_cost_usd` / `max_runtime_minutes` / `max_tests` cap every run
+//
+// USAGE
+// node scripts/redblue.mjs init # write redblue.yaml
+// node scripts/redblue.mjs run --mock-judge --tests 10 # $0 marker-fixture path
+// node scripts/redblue.mjs run --tests 50 --patch # real judge (needs OPENROUTER_API_KEY)
+// node scripts/redblue.mjs attack prompt --count 5 # preview attacks
+// node scripts/redblue.mjs patch --mock-judge # baseline β†’ patch β†’ retest delta
+// node scripts/redblue.mjs report --in report.json # render summary
+//
+// EXIT CODES
+// 0 ran OK (or degraded β€” redblue not available)
+// 1 --alert-on-fail and overall verdict was FAIL (after patch retest)
+// 2 config / argv error
+//
+// REPORT FORMAT
+// `run` / `patch` write a JSON report to --out (default: ./redblue-report.json).
+// We re-emit it on stdout when --format json so MCP tools can parse it
+// without round-tripping through disk. Markdown rendering is delegated
+// to `redblue report` upstream.
+
+import { runRedblue, emitRedblueDegradedJsonAndExit } from './_redblue.mjs';
+import { existsSync, readFileSync, mkdtempSync, unlinkSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const VALID_SUBCOMMANDS = new Set(['init', 'run', 'patch', 'attack', 'report']);
+const VALID_ATTACK_FAMILIES = new Set(['prompt', 'tools', 'data', 'all']);
+
+const ARGS = (() => {
+ const a = {
+ sub: null,
+ // shared
+ config: null,
+ out: null,
+ inPath: null,
+ format: 'json',
+ // run/patch
+ tests: null,
+ patch: false,
+ mockJudge: false,
+ alertOnFail: false,
+ // attack
+ family: null,
+ count: null,
+ // budget overrides (forwarded only when set)
+ maxCostUsd: null,
+ maxRuntimeMinutes: null,
+ timeoutMs: null,
+ };
+ const argv = process.argv.slice(2);
+ if (argv.length === 0) return a;
+ a.sub = argv[0];
+ for (let i = 1; i < argv.length; i++) {
+ const v = argv[i];
+ if (v === '--config') a.config = argv[++i];
+ else if (v === '--out') a.out = argv[++i];
+ else if (v === '--in') a.inPath = argv[++i];
+ else if (v === '--format') a.format = argv[++i];
+ else if (v === '--tests') a.tests = parseInt(argv[++i], 10);
+ else if (v === '--patch') a.patch = true;
+ else if (v === '--mock-judge') a.mockJudge = true;
+ else if (v === '--alert-on-fail') a.alertOnFail = true;
+ else if (v === '--count') a.count = parseInt(argv[++i], 10);
+ else if (v === '--max-cost-usd') a.maxCostUsd = parseFloat(argv[++i]);
+ else if (v === '--max-runtime-minutes') a.maxRuntimeMinutes = parseFloat(argv[++i]);
+ else if (v === '--timeout-ms') a.timeoutMs = parseInt(argv[++i], 10);
+ else if (v === '--help' || v === '-h') a.sub = 'help';
+ else if (!v.startsWith('-') && a.sub === 'attack' && !a.family) a.family = v;
+ // any other unknown flag β€” pass through to upstream redblue
+ else if (v.startsWith('--')) {
+ // forward unknown flags verbatim so we don't bottleneck new redblue features
+ a._passthrough = a._passthrough || [];
+ a._passthrough.push(v);
+ // peek for next-token value (only consume if it's not another flag)
+ const next = argv[i + 1];
+ if (next !== undefined && !next.startsWith('-')) {
+ a._passthrough.push(next);
+ i++;
+ }
+ }
+ }
+ return a;
+})();
+
+function printHelp() {
+ console.log('npx ruflo metaharness redblue <subcommand> [options]');
+ console.log('');
+ console.log('Subcommands:');
+ console.log(' init Write a sample redblue.yaml config');
+ console.log(' run Run red-team β†’ judge β†’ (optional --patch β†’ retest) β†’ report');
+ console.log(' patch Baseline β†’ blue-team patch β†’ retest delta');
+ console.log(' attack <prompt|tools|data|all> Preview generated attack test cases (no target call)');
+ console.log(' report Render an existing report.json as markdown');
+ console.log('');
+ console.log('Common options:');
+ console.log(' --config <path> Config YAML (default: redblue.yaml in cwd)');
+ console.log(' --out <path> Output path for run/patch (default: ./redblue-report.json)');
+ console.log(' --in <path> Input report path (for `report` subcommand)');
+ console.log(' --tests N How many test cases (run/patch)');
+ console.log(' --patch After baseline run, apply blue-team patches and retest');
+ console.log(' --mock-judge $0 TEST-ONLY marker fixture β€” for CI / offline');
+ console.log(' --alert-on-fail Exit 1 when overall post-patch verdict is FAIL');
+ console.log(' --format json|markdown Output format (default: json)');
+ console.log(' --max-cost-usd N Override config max_cost_usd');
+ console.log(' --max-runtime-minutes N Override config max_runtime_minutes');
+ console.log(' --timeout-ms N Subprocess hard timeout (default: 120000)');
+ console.log('');
+ console.log('Examples:');
+ console.log(' node scripts/redblue.mjs init');
+ console.log(' node scripts/redblue.mjs run --mock-judge --tests 10');
+ console.log(' node scripts/redblue.mjs run --tests 50 --patch # needs OPENROUTER_API_KEY');
+ console.log(' node scripts/redblue.mjs attack prompt --count 3');
+ console.log('');
+ console.log('SAFETY: upstream redblue enforces no-creds, no-live-targets, no-shell,');
+ console.log('no-eval at load time (src/config/safety.ts). This wrapper does not relax');
+ console.log('those β€” it only forwards argv with shell:false.');
+}
+
+function buildUpstreamArgs() {
+ switch (ARGS.sub) {
+ case 'init': {
+ const out = ARGS.out ?? null;
+ return ['init', ...(out ? ['--out', out] : [])];
+ }
+ case 'run': {
+ const a = ['run'];
+ if (ARGS.config) a.push('--config', ARGS.config);
+ if (ARGS.tests !== null && Number.isFinite(ARGS.tests)) a.push('--tests', String(ARGS.tests));
+ if (ARGS.patch) a.push('--patch');
+ if (ARGS.mockJudge) a.push('--mock-judge');
+ if (ARGS.out) a.push('--out', ARGS.out);
+ return a;
+ }
+ case 'patch': {
+ const a = ['patch'];
+ if (ARGS.config) a.push('--config', ARGS.config);
+ if (ARGS.mockJudge) a.push('--mock-judge');
+ if (ARGS.out) a.push('--out', ARGS.out);
+ return a;
+ }
+ case 'attack': {
+ if (!ARGS.family || !VALID_ATTACK_FAMILIES.has(ARGS.family)) {
+ console.error(`redblue: --attack family must be one of ${[...VALID_ATTACK_FAMILIES].join('|')} (got: ${ARGS.family ?? 'none'})`);
+ process.exit(2);
+ }
+ const a = ['attack', ARGS.family];
+ if (ARGS.count !== null && Number.isFinite(ARGS.count)) a.push('--count', String(ARGS.count));
+ return a;
+ }
+ case 'report': {
+ if (!ARGS.inPath) {
+ console.error('redblue: report requires --in <path>');
+ process.exit(2);
+ }
+ return ['report', '--in', ARGS.inPath];
+ }
+ default:
+ console.error(`redblue: unknown subcommand "${ARGS.sub}". Valid: ${[...VALID_SUBCOMMANDS].join(', ')}`);
+ process.exit(2);
+ }
+}
+
+function main() {
+ if (!ARGS.sub || ARGS.sub === 'help' || ARGS.sub === '--help' || ARGS.sub === '-h') {
+ printHelp();
+ process.exit(0);
+ }
+ if (!VALID_SUBCOMMANDS.has(ARGS.sub)) {
+ console.error(`redblue: unknown subcommand "${ARGS.sub}". Valid: ${[...VALID_SUBCOMMANDS].join(', ')}`);
+ process.exit(2);
+ }
+
+ // For `run` and `patch` we want the report JSON in our hands even when
+ // the caller didn't pass --out (so MCP tools can read it back from stdout).
+ let synthOut = null;
+ if ((ARGS.sub === 'run' || ARGS.sub === 'patch') && !ARGS.out) {
+ const dir = mkdtempSync(join(tmpdir(), 'ruflo-redblue-'));
+ synthOut = join(dir, 'report.json');
+ ARGS.out = synthOut;
+ }
+
+ const upstreamArgs = buildUpstreamArgs();
+ if (ARGS._passthrough?.length) upstreamArgs.push(...ARGS._passthrough);
+
+ const r = runRedblue(upstreamArgs, { timeoutMs: ARGS.timeoutMs ?? undefined });
+
+ if (r.degraded) {
+ if (synthOut && existsSync(synthOut)) {
+ try { unlinkSync(synthOut); } catch { /* ignore */ }
+ }
+ emitRedblueDegradedJsonAndExit(r.reason);
+ return;
+ }
+
+ // Pass through stderr (progress) when format=markdown so users see it live
+ if (ARGS.format === 'markdown' && r.stderr) {
+ process.stderr.write(r.stderr);
+ }
+
+ // For run/patch: read the report we asked redblue to write.
+ let payload = null;
+ if (ARGS.sub === 'run' || ARGS.sub === 'patch') {
+ const reportPath = resolve(ARGS.out);
+ if (existsSync(reportPath)) {
+ try {
+ payload = JSON.parse(readFileSync(reportPath, 'utf-8'));
+ } catch (e) {
+ console.error(`redblue: failed to parse report at ${reportPath}: ${e instanceof Error ? e.message : String(e)}`);
+ if (synthOut) { try { unlinkSync(synthOut); } catch { /* ignore */ } }
+ process.exit(2);
+ }
+ }
+ }
+
+ // Decorate every payload with our own metadata + the safety boundary
+ // reminder so consumers always see "this was capability-contained".
+ const enriched = payload && typeof payload === 'object' ? {
+ ...payload,
+ _ruflo: {
+ subcommand: ARGS.sub,
+ mockJudge: ARGS.mockJudge,
+ durationMs: r.durationMs,
+ safetyBoundary: 'enforced-by-upstream-redblue (no creds, no live targets, no shell, no eval)',
+ },
+ } : payload;
+
+ // Determine overall verdict for --alert-on-fail (post-patch run only).
+ // redblue reports include a `verdict` or `gate` field; we treat absence
+ // as "no data β†’ no alert" rather than erroring.
+ let alertTriggered = false;
+ if (ARGS.alertOnFail && enriched && typeof enriched === 'object') {
+ const v = (enriched.verdict || enriched.gate || enriched.status || '').toString().toUpperCase();
+ if (v.includes('FAIL')) alertTriggered = true;
+ }
+
+ if (ARGS.format === 'json') {
+ if (enriched !== null) {
+ console.log(JSON.stringify(enriched, null, 2));
+ } else {
+ // init / attack / report subcommands β€” forward upstream stdout verbatim
+ // (it's already markdown/yaml). Wrap in a tiny JSON envelope.
+ console.log(JSON.stringify({
+ subcommand: ARGS.sub,
+ ok: r.exitCode === 0,
+ durationMs: r.durationMs,
+ stdout: r.stdout,
+ _ruflo: { safetyBoundary: 'enforced-by-upstream-redblue' },
+ }, null, 2));
+ }
+ } else {
+ // markdown / passthrough
+ if (r.stdout) process.stdout.write(r.stdout);
+ if (enriched && typeof enriched === 'object') {
+ console.log('');
+ console.log('---');
+ console.log('## redblue report (parsed)');
+ console.log('```json');
+ console.log(JSON.stringify(enriched, null, 2));
+ console.log('```');
+ }
+ }
+
+ // Clean up synthetic temp file
+ if (synthOut && existsSync(synthOut)) {
+ try { unlinkSync(synthOut); } catch { /* ignore */ }
+ }
+
+ if (alertTriggered) process.exit(1);
+ if (r.exitCode !== 0) process.exit(r.exitCode);
+}
+
+main();
diff --git a/plugins/ruflo-metaharness/scripts/smoke.sh b/plugins/ruflo-metaharness/scripts/smoke.sh
index 6f1e263dad..3d66e6a6b3 100755
--- a/plugins/ruflo-metaharness/scripts/smoke.sh
+++ b/plugins/ruflo-metaharness/scripts/smoke.sh
@@ -151,9 +151,9 @@ F="$ROOT/../../v3/@claude-flow/cli/src/commands/metaharness.ts"
miss=""
[[ -f "$F" ]] || miss="$miss command-file-missing"
grep -q "name: 'metaharness'" "$F" 2>/dev/null || miss="$miss no-name-field"
-# All 8 subcommands must each be present in the dispatch table.
+# All 9 subcommands must each be present in the dispatch table.
# Match either quoted ('mcp-scan': ...) or unquoted shorthand (score: ...) keys.
-for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend mint; do
+for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend mint redblue; do
grep -qE "(^|[[:space:]])'?${sub}'?:" "$F" 2>/dev/null || miss="$miss missing-$sub"
done
# Registered in the loader
@@ -342,7 +342,7 @@ miss=""
ROOTPJ="$ROOT/../../package.json"
RUFLOPJ="$ROOT/../../ruflo/package.json"
CLIPJ="$ROOT/../../v3/@claude-flow/cli/package.json"
-for pkg in "metaharness" "@metaharness/router" "@metaharness/kernel"; do
+for pkg in "metaharness" "@metaharness/router" "@metaharness/kernel" "@metaharness/redblue"; do
ROOT_PIN=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$ROOTPJ')).optionalDependencies?.['$pkg'] || '')" 2>/dev/null)
RUFLO_PIN=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$RUFLOPJ')).optionalDependencies?.['$pkg'] || '')" 2>/dev/null)
CLI_PIN=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$CLIPJ')).optionalDependencies?.['$pkg'] || '')" 2>/dev/null)
@@ -722,7 +722,7 @@ for k in $KEYS; do
grep -qE "npx ruflo metaharness ${k}([ \\\\]|$)" "$CMD" 2>/dev/null \
|| miss="$miss subcommand-${k}-not-in-claude-md"
done
-[[ "$COUNT" == "10" ]] || miss="$miss subcommand-count-stale:$COUNT-expected-10"
+[[ "$COUNT" == "11" ]] || miss="$miss subcommand-count-stale:$COUNT-expected-11"
[[ -z "$miss" ]] && ok || bad "$miss"
step "17z56. every MCP tool documented in CLAUDE.md (iter 93)"
@@ -742,10 +742,11 @@ for t in $TOOLS; do
grep -q "mcp__claude-flow__${t}" "$CMD" 2>/dev/null \
|| miss="$miss ${t}-not-in-claude-md"
done
-# Lock count: 12 MCP tools (mint deliberately excluded β€” see iter 73).
+# Lock count: 13 MCP tools (mint deliberately excluded β€” see iter 73).
# Bumped from 9 β†’ 12 after ADR-153 added metaharness_bench,
-# metaharness_evolve, and metaharness_security_bench.
-[[ "$COUNT" == "12" ]] || miss="$miss mcp-tool-count-stale:$COUNT-expected-12"
+# metaharness_evolve, and metaharness_security_bench. Bumped from 12 β†’ 13
+# after @metaharness/redblue integration added metaharness_redblue.
+[[ "$COUNT" == "13" ]] || miss="$miss mcp-tool-count-stale:$COUNT-expected-13"
[[ -z "$miss" ]] && ok || bad "$miss"
step "17z55. MCP enum + SEVERITY_RANK vocabulary aligned (iter 92)"
@@ -820,9 +821,10 @@ for f in $REFS; do
COUNT=$((COUNT + 1))
[[ -f "$SCRIPTS_DIR/$f" ]] || miss="$miss mcp-script-${f}-missing"
done
-# Should be 12 unique scripts (one per MCP tool; mint deliberately excluded).
+# Should be 13 unique scripts (one per MCP tool; mint deliberately excluded).
# Bumped from 9 β†’ 12 after ADR-153 added bench/evolve/security_bench tools.
-[[ "$COUNT" == "12" ]] || miss="$miss mcp-script-count-stale:$COUNT-expected-12"
+# Bumped from 12 β†’ 13 after @metaharness/redblue integration added redblue.mjs.
+[[ "$COUNT" == "13" ]] || miss="$miss mcp-script-count-stale:$COUNT-expected-13"
[[ -z "$miss" ]] && ok || bad "$miss"
step "17z52. SUBCOMMANDS map entries point at existing script files (iter 89)"
@@ -842,9 +844,9 @@ for f in $MAPPINGS; do
COUNT=$((COUNT + 1))
[[ -f "$SCRIPTS_DIR/$f" ]] || miss="$miss script-${f}-missing"
done
-# Iter 73's description string lists 10 subcommands. SUBCOMMANDS map should
-# have the same 10 entries. Lock the count at exactly 10.
-[[ "$COUNT" == "10" ]] || miss="$miss mapping-count-stale:$COUNT-expected-10"
+# Iter 73's description string + redblue addition lists 11 subcommands.
+# SUBCOMMANDS map should have the same 11 entries. Lock the count at exactly 11.
+[[ "$COUNT" == "11" ]] || miss="$miss mapping-count-stale:$COUNT-expected-11"
[[ -z "$miss" ]] && ok || bad "$miss"
step "17z51. all metaharness scripts produce parseable JSON in --format json mode (iter 88)"
@@ -1138,8 +1140,9 @@ step "17z36. CLI subcommand list current + mint anti-MCP guard (iter 73)"
miss=""
DISP="$ROOT/../../v3/@claude-flow/cli/src/commands/metaharness.ts"
WRAPPER="$ROOT/../../v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts"
-# iter-73 description string lists all 10 dispatchable subcommands
-for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend similarity drift-from-history mint; do
+# All 11 dispatchable subcommands present (iter-73 had 10; @metaharness/redblue
+# integration added redblue.mjs).
+for sub in score genome mcp-scan threat-model oia-audit audit-list audit-trend similarity drift-from-history mint redblue; do
grep -q "${sub}" "$DISP" 2>/dev/null || miss="$miss subcommand-${sub}-not-listed"
done
# ANTI-MINT GUARD: mint is intentionally CLI-only per ADR-150 Β§Sandboxing
@@ -1628,8 +1631,9 @@ WRAPPER="$ROOT/../../v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts"
# Success-semantic constant declared + appended to N descriptions = N+1 occurrences.
# Iter 46 set this at 9 (8 tools); iter 54 added the 9th β†’ 10. ADR-153
# added 3 more tools (bench/evolve/security_bench) β†’ 1 + 12 = 13.
+# @metaharness/redblue integration added 1 more β†’ 1 + 13 = 14.
COUNT=$(grep -c "MCP_SUCCESS_SEMANTIC" "$WRAPPER" 2>/dev/null; true)
-[[ "$COUNT" == "13" ]] || miss="$miss footnote-count:$COUNT-expected-13"
+[[ "$COUNT" == "14" ]] || miss="$miss footnote-count:$COUNT-expected-14"
# audit_trend now exposes baselineFile / currentFile
grep -q "baselineFile" "$WRAPPER" 2>/dev/null || miss="$miss no-baseline-file"
grep -q "currentFile" "$WRAPPER" 2>/dev/null || miss="$miss no-current-file"
@@ -1668,8 +1672,9 @@ COUNT_OLD=$(grep -c "success: !r.degraded" "$WRAPPER" 2>/dev/null; true)
[[ "$COUNT_OLD" == "0" ]] || miss="$miss old-pattern-still-present:$COUNT_OLD"
COUNT_NEW=$(grep -c "success: r.success" "$WRAPPER" 2>/dev/null; true)
# Iter 54 added a 9th tool β†’ 9. ADR-153 added 3 more (bench, evolve,
-# security_bench) β†’ 12. Future iters that add tools should bump this.
-[[ "$COUNT_NEW" == "12" ]] || miss="$miss new-pattern-count:$COUNT_NEW-expected-12"
+# security_bench) β†’ 12. @metaharness/redblue integration added 1 more β†’ 13.
+# Future iters that add tools should bump this.
+[[ "$COUNT_NEW" == "13" ]] || miss="$miss new-pattern-count:$COUNT_NEW-expected-13"
# Runtime anchors: iter 44 success assertions present
T="$ROOT/scripts/test-mcp-tools.mjs"
grep -q "iter 44 fix" "$T" 2>/dev/null || miss="$miss no-iter44-anchors"
@@ -2027,13 +2032,13 @@ grep -q "result has 'success'" "$F" || miss="$miss no-success-assertion"
grep -q "result has 'data'" "$F" || miss="$miss no-data-assertion"
grep -q "result has 'degraded'" "$F" || miss="$miss no-degraded-assertion"
grep -q "result has 'exitCode'" "$F" || miss="$miss no-exitcode-assertion"
-# All 12 tool names enumerated (similarity iter 36, drift_from_history iter 54,
-# ADR-153 added bench/evolve/security_bench)
-for tool in metaharness_score metaharness_genome metaharness_mcp_scan metaharness_threat_model metaharness_oia_audit metaharness_audit_list metaharness_audit_trend metaharness_similarity metaharness_drift_from_history metaharness_bench metaharness_evolve metaharness_security_bench; do
+# All 13 tool names enumerated (similarity iter 36, drift_from_history iter 54,
+# ADR-153 added bench/evolve/security_bench, @metaharness/redblue added redblue)
+for tool in metaharness_score metaharness_genome metaharness_mcp_scan metaharness_threat_model metaharness_oia_audit metaharness_audit_list metaharness_audit_trend metaharness_similarity metaharness_drift_from_history metaharness_bench metaharness_evolve metaharness_security_bench metaharness_redblue; do
grep -q "${tool}" "$F" || miss="$miss missing-${tool}"
done
-# Count assertion must match iter-54 expansion (8 β†’ 9), then ADR-153 (9 β†’ 12)
-grep -q "tools.length === 12" "$F" || miss="$miss tool-count-assertion-stale"
+# Count assertion must match iter-54 (8 β†’ 9), then ADR-153 (9 β†’ 12), then redblue (12 β†’ 13)
+grep -q "tools.length === 13" "$F" || miss="$miss tool-count-assertion-stale"
# Graceful skip when dist absent (so the script is smoke-runnable pre-build)
grep -q "SKIPPED" "$F" || miss="$miss no-skip-doc"
[[ -z "$miss" ]] && ok || bad "$miss"
diff --git a/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs b/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs
index 708dff1881..f78813145a 100755
--- a/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs
+++ b/plugins/ruflo-metaharness/scripts/test-mcp-tools.mjs
@@ -74,7 +74,7 @@ async function main() {
// ──────────────────────────────────────────────────────────────────
console.log('Phase 1 β€” module shape');
assert(Array.isArray(tools), 'metaharnessTools is an array');
- assert(tools.length === 12, `12 tools registered (got ${tools.length})`);
+ assert(tools.length === 13, `13 tools registered (got ${tools.length})`);
const expectedNames = new Set([
'metaharness_score',
@@ -92,6 +92,8 @@ async function main() {
'metaharness_bench',
'metaharness_evolve',
'metaharness_security_bench',
+ // @metaharness/redblue@~0.1.1 β€” adversarial red/blue LLM testing
+ 'metaharness_redblue',
]);
const actualNames = new Set(tools.map((t) => t.name));
for (const name of expectedNames) {
@@ -145,6 +147,12 @@ async function main() {
// test doesn't pollute namespaces.
input = { dryRun: true };
}
+ if (tool.name === 'metaharness_redblue') {
+ // `attack` preview is the fastest path that exercises the upstream
+ // binary without needing OPENROUTER_API_KEY or running any model
+ // calls. Count=1 keeps cold-cache npx fetch the dominant cost.
+ input = { subcommand: 'attack', family: 'prompt', count: 1 };
+ }
// iter 124 β†’ 130 β€” timeouts have crept up as CI cold-cache npx
// warmup costs got measured. Final budgets:
@@ -157,7 +165,11 @@ async function main() {
// ONNX). 180s gives 30x headroom over the local cost.
const isChainTool = tool.name === 'metaharness_drift_from_history'
|| tool.name === 'metaharness_oia_audit'
- || tool.name === 'metaharness_audit_list';
+ || tool.name === 'metaharness_audit_list'
+ // redblue: `attack prompt --count 1` is preview-only (no model
+ // calls) but the cold-cache `npx -y @metaharness/redblue@~0.1.1`
+ // fetch can take 30-60s. 180s gives 3x headroom.
+ || tool.name === 'metaharness_redblue';
const timeoutMs = isChainTool ? 180_000 : 60_000;
const handlerPromise = tool.handler(input);
const timeoutPromise = new Promise((_, reject) =>
@@ -432,7 +444,7 @@ async function main() {
for (const f of failures) console.log(` - ${f}`);
process.exit(1);
}
- console.log('\nβœ“ All 9 MCP tools satisfy the runtime contract.');
+ console.log('\nβœ“ All 13 MCP tools satisfy the runtime contract.');
}
main().catch((e) => {
diff --git a/ruflo/package.json b/ruflo/package.json
index e9f0ad8877..fbac38325f 100644
--- a/ruflo/package.json
+++ b/ruflo/package.json
@@ -46,7 +46,8 @@
"metaharness": "~0.2.6",
"@metaharness/router": "~0.3.2",
"@metaharness/kernel": "~0.1.0",
- "@metaharness/darwin": "~0.3.1"
+ "@metaharness/darwin": "~0.3.1",
+ "@metaharness/redblue": "~0.1.1"
},
"overrides": {
"ruvector": "^0.2.27",
diff --git a/v3/@claude-flow/cli/package.json b/v3/@claude-flow/cli/package.json
index 8d18387eba..a4947edfa6 100644
--- a/v3/@claude-flow/cli/package.json
+++ b/v3/@claude-flow/cli/package.json
@@ -115,6 +115,7 @@
"@claude-flow/security": "^3.0.0-alpha.10",
"@metaharness/darwin": "~0.3.1",
"@metaharness/kernel": "~0.1.0",
+ "@metaharness/redblue": "~0.1.1",
"@metaharness/router": "~0.3.2",
"metaharness": "~0.2.6",
"@ruvector/attention": "^0.1.32",
diff --git a/v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/_redblue.mjs b/v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/_redblue.mjs
new file mode 100644
index 0000000000..a0866e3af7
--- /dev/null
+++ b/v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/_redblue.mjs
@@ -0,0 +1,176 @@
+// _redblue.mjs β€” invocation helper for `@metaharness/redblue`.
+//
+// Sibling of `_darwin.mjs` / `_harness.mjs`. Targets the `redblue` binary
+// from the standalone `@metaharness/redblue@~0.1.1` package.
+//
+// Subcommands surfaced (matches redblue 0.1.x):
+// - `redblue init [--out redblue.yaml]`
+// - `redblue run [--config redblue.yaml] [--tests N] [--patch] [--mock-judge] [--out report.json]`
+// - `redblue attack <prompt|tools|data|all> [--count N]`
+// - `redblue patch [--config redblue.yaml] [--mock-judge]`
+// - `redblue report --in report.json`
+//
+// UPSTREAM BUG WORKAROUND
+// -----------------------
+// @metaharness/[email protected]'s CLI bootstrap is:
+// const isMain = import.meta.url === `file://${process.argv[1]}`;
+// if (isMain) { dispatch(...) }
+// When npx links `redblue` β†’ `dist/cli/index.js`, process.argv[1] is the
+// symlink path but import.meta.url is the resolved real path, so the
+// check fails silently β€” the binary exits 0 with no output, no file,
+// no error. Darwin's CLI doesn't suffer this (calls main() unconditionally).
+//
+// Workaround: install the package once into a ruflo-owned cache dir
+// (~/.ruflo/redblue-cache) and invoke `node <abs_path>/dist/cli/index.js`
+// directly. argv[1] then equals the real path and isMain becomes true.
+//
+// Track upstream fix at: github.com/ruvnet/agent-harness-generator/issues
+// (file separately β€” when fixed, we can drop the cached-install path and
+// go back to the `npx -y -p ...` pattern used by _darwin.mjs).
+//
+// CONTRACT (matches runMetaharness/runDarwin):
+// - returns `{ stdout, stderr, exitCode, durationMs, degraded, reason? }`
+// - subprocess hard timeout (default 120s; --mock-judge runs are seconds)
+// - on install failure / MODULE_NOT_FOUND / network failure, returns
+// `degraded: true, reason: 'metaharness-redblue-not-available'`
+// - never throws β€” ADR-150 graceful-degradation rule #3
+//
+// SAFETY NOTE
+// redblue itself enforces hard safety boundaries in `src/config/safety.ts`
+// (no real creds, no live targets, no shell, no eval, no arbitrary network).
+// This wrapper does NOT relax those β€” it only forwards argv with shell:false.
+// `--mock-judge` is the $0 CI path; the real model judge gates on
+// $OPENROUTER_API_KEY which we never inject.
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+const DEFAULT_TIMEOUT_MS = 120_000;
+const INSTALL_TIMEOUT_MS = 180_000; // npm install can be slow on cold cache
+
+// Pinned semver range. Bump in lock-step with optionalDependencies in
+// @claude-flow/cli/package.json + ruflo/package.json.
+const REDBLUE_PIN = '@metaharness/redblue@~0.1.1';
+const REDBLUE_PKG = '@metaharness/redblue';
+const REDBLUE_PIN_VERSION = '~0.1.1';
+
+const CACHE_DIR = join(homedir(), '.ruflo', 'redblue-cache');
+const RESOLVED_CLI = join(
+ CACHE_DIR, 'node_modules', '@metaharness', 'redblue', 'dist', 'cli', 'index.js',
+);
+
+const DEGRADED_RX = /could not determine executable|404|not installed|MODULE_NOT_FOUND|ENOTFOUND|getaddrinfo|ECONNREFUSED|ETIMEDOUT|npm ERR/i;
+
+function ensureInstalled() {
+ if (existsSync(RESOLVED_CLI)) return { ok: true };
+ try {
+ mkdirSync(CACHE_DIR, { recursive: true });
+ } catch (e) {
+ return { ok: false, reason: 'cache-dir-create-failed', error: String(e) };
+ }
+ // Use `npm install --prefix` so the package lands in a known location;
+ // this avoids npx's symlinked shim entirely (the upstream bin-bootstrap bug).
+ // shell:false; argv only.
+ const r = spawnSync('npm', [
+ 'install',
+ '--no-audit', '--no-fund', '--no-package-lock',
+ '--prefix', CACHE_DIR,
+ `${REDBLUE_PKG}@${REDBLUE_PIN_VERSION}`,
+ ], {
+ stdio: ['ignore', 'pipe', 'pipe'],
+ encoding: 'utf-8',
+ timeout: INSTALL_TIMEOUT_MS,
+ shell: process.platform === 'win32',
+ });
+ if (r.status !== 0 || !existsSync(RESOLVED_CLI)) {
+ return {
+ ok: false,
+ reason: 'install-failed',
+ stderr: (r.stderr || '').slice(0, 600),
+ stdout: (r.stdout || '').slice(0, 600),
+ };
+ }
+ return { ok: true };
+}
+
+function classifyDegraded(stderr, exitCode) {
+ if (exitCode === null) return { degraded: true, reason: 'metaharness-redblue-timeout' };
+ if (DEGRADED_RX.test(stderr)) return { degraded: true, reason: 'metaharness-redblue-not-available' };
+ return { degraded: false };
+}
+
+/**
+ * Sync invocation. `redblue` runs are bounded by `max_cost_usd` and
+ * `max_runtime_minutes` from the config, so sync is fine for the CLI
+ * wrapper. The default --mock-judge fixture path completes in seconds.
+ */
+export function runRedblue(args, opts = {}) {
+ const start = Date.now();
+ const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+
+ // 1. Ensure redblue is installed in our cache dir (one-time install).
+ const install = ensureInstalled();
+ if (!install.ok) {
+ return {
+ stdout: install.stdout ?? '',
+ stderr: install.stderr ?? install.error ?? '',
+ exitCode: 127,
+ durationMs: Date.now() - start,
+ degraded: true,
+ reason: install.reason === 'install-failed'
+ ? 'metaharness-redblue-not-available'
+ : `metaharness-redblue-${install.reason}`,
+ };
+ }
+
+ // 2. Invoke `node <real-path-to-cli> <args>` so upstream's isMain check
+ // succeeds (argv[1] matches import.meta.url).
+ const r = spawnSync('node', [RESOLVED_CLI, ...args], {
+ stdio: ['ignore', 'pipe', 'pipe'],
+ encoding: 'utf-8',
+ timeout: timeoutMs,
+ cwd: opts.cwd,
+ env: { ...process.env, ...(opts.env || {}) },
+ shell: process.platform === 'win32',
+ });
+ const durationMs = Date.now() - start;
+ const stdout = r.stdout || '';
+ const stderr = r.stderr || '';
+ const classified = classifyDegraded(stderr, r.status);
+ if (classified.degraded) {
+ return {
+ stdout, stderr,
+ exitCode: r.status ?? 127,
+ durationMs,
+ degraded: true,
+ reason: classified.reason,
+ };
+ }
+ return {
+ stdout, stderr,
+ exitCode: r.status ?? 0,
+ durationMs,
+ degraded: false,
+ };
+}
+
+/**
+ * Match the existing `emit*DegradedJsonAndExit` shape used by sibling
+ * scripts so MCP tool consumers see one contract.
+ */
+export function emitRedblueDegradedJsonAndExit(reason) {
+ const payload = {
+ degraded: true,
+ reason,
+ hint: 'Install with `npm i -D ' + REDBLUE_PIN + '` or run `npx -y ' + REDBLUE_PIN + ' --help` to verify network access.',
+ generatedAt: new Date().toISOString(),
+ };
+ console.log(JSON.stringify(payload, null, 2));
+ // Exit 0 β€” ADR-150 architectural constraint #3.
+ process.exit(0);
+}
+
+export const REDBLUE_VERSION_PIN = REDBLUE_PIN;
+export const REDBLUE_CACHE_DIR = CACHE_DIR;
diff --git a/v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/redblue.mjs b/v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/redblue.mjs
new file mode 100644
index 0000000000..66b43ae705
--- /dev/null
+++ b/v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/redblue.mjs
@@ -0,0 +1,286 @@
+#!/usr/bin/env node
+// redblue.mjs β€” wrapper around `@metaharness/redblue` for LLM red/blue testing.
+//
+// ADR-150 sibling-package integration: `@metaharness/redblue` runs an
+// adversarial red-team β†’ judge β†’ blue-team patch loop against an LLM
+// target you own. Upstream README: packages/redblue#readme on
+// github.com/ruvnet/agent-harness-generator.
+//
+// SAFETY (enforced by upstream `src/config/safety.ts`, NOT this wrapper):
+// - no real credentials (`allow_real_credentials:true` = load-time error)
+// - no live external targets (validateTarget() rejects non-loopback)
+// - no shell, no arbitrary network, no eval
+// - sensitive outputs redacted before storage
+// - `max_cost_usd` / `max_runtime_minutes` / `max_tests` cap every run
+//
+// USAGE
+// node scripts/redblue.mjs init # write redblue.yaml
+// node scripts/redblue.mjs run --mock-judge --tests 10 # $0 marker-fixture path
+// node scripts/redblue.mjs run --tests 50 --patch # real judge (needs OPENROUTER_API_KEY)
+// node scripts/redblue.mjs attack prompt --count 5 # preview attacks
+// node scripts/redblue.mjs patch --mock-judge # baseline β†’ patch β†’ retest delta
+// node scripts/redblue.mjs report --in report.json # render summary
+//
+// EXIT CODES
+// 0 ran OK (or degraded β€” redblue not available)
+// 1 --alert-on-fail and overall verdict was FAIL (after patch retest)
+// 2 config / argv error
+//
+// REPORT FORMAT
+// `run` / `patch` write a JSON report to --out (default: ./redblue-report.json).
+// We re-emit it on stdout when --format json so MCP tools can parse it
+// without round-tripping through disk. Markdown rendering is delegated
+// to `redblue report` upstream.
+
+import { runRedblue, emitRedblueDegradedJsonAndExit } from './_redblue.mjs';
+import { existsSync, readFileSync, mkdtempSync, unlinkSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const VALID_SUBCOMMANDS = new Set(['init', 'run', 'patch', 'attack', 'report']);
+const VALID_ATTACK_FAMILIES = new Set(['prompt', 'tools', 'data', 'all']);
+
+const ARGS = (() => {
+ const a = {
+ sub: null,
+ // shared
+ config: null,
+ out: null,
+ inPath: null,
+ format: 'json',
+ // run/patch
+ tests: null,
+ patch: false,
+ mockJudge: false,
+ alertOnFail: false,
+ // attack
+ family: null,
+ count: null,
+ // budget overrides (forwarded only when set)
+ maxCostUsd: null,
+ maxRuntimeMinutes: null,
+ timeoutMs: null,
+ };
+ const argv = process.argv.slice(2);
+ if (argv.length === 0) return a;
+ a.sub = argv[0];
+ for (let i = 1; i < argv.length; i++) {
+ const v = argv[i];
+ if (v === '--config') a.config = argv[++i];
+ else if (v === '--out') a.out = argv[++i];
+ else if (v === '--in') a.inPath = argv[++i];
+ else if (v === '--format') a.format = argv[++i];
+ else if (v === '--tests') a.tests = parseInt(argv[++i], 10);
+ else if (v === '--patch') a.patch = true;
+ else if (v === '--mock-judge') a.mockJudge = true;
+ else if (v === '--alert-on-fail') a.alertOnFail = true;
+ else if (v === '--count') a.count = parseInt(argv[++i], 10);
+ else if (v === '--max-cost-usd') a.maxCostUsd = parseFloat(argv[++i]);
+ else if (v === '--max-runtime-minutes') a.maxRuntimeMinutes = parseFloat(argv[++i]);
+ else if (v === '--timeout-ms') a.timeoutMs = parseInt(argv[++i], 10);
+ else if (v === '--help' || v === '-h') a.sub = 'help';
+ else if (!v.startsWith('-') && a.sub === 'attack' && !a.family) a.family = v;
+ // any other unknown flag β€” pass through to upstream redblue
+ else if (v.startsWith('--')) {
+ // forward unknown flags verbatim so we don't bottleneck new redblue features
+ a._passthrough = a._passthrough || [];
+ a._passthrough.push(v);
+ // peek for next-token value (only consume if it's not another flag)
+ const next = argv[i + 1];
+ if (next !== undefined && !next.startsWith('-')) {
+ a._passthrough.push(next);
+ i++;
+ }
+ }
+ }
+ return a;
+})();
+
+function printHelp() {
+ console.log('npx ruflo metaharness redblue <subcommand> [options]');
+ console.log('');
+ console.log('Subcommands:');
+ console.log(' init Write a sample redblue.yaml config');
+ console.log(' run Run red-team β†’ judge β†’ (optional --patch β†’ retest) β†’ report');
+ console.log(' patch Baseline β†’ blue-team patch β†’ retest delta');
+ console.log(' attack <prompt|tools|data|all> Preview generated attack test cases (no target call)');
+ console.log(' report Render an existing report.json as markdown');
+ console.log('');
+ console.log('Common options:');
+ console.log(' --config <path> Config YAML (default: redblue.yaml in cwd)');
+ console.log(' --out <path> Output path for run/patch (default: ./redblue-report.json)');
+ console.log(' --in <path> Input report path (for `report` subcommand)');
+ console.log(' --tests N How many test cases (run/patch)');
+ console.log(' --patch After baseline run, apply blue-team patches and retest');
+ console.log(' --mock-judge $0 TEST-ONLY marker fixture β€” for CI / offline');
+ console.log(' --alert-on-fail Exit 1 when overall post-patch verdict is FAIL');
+ console.log(' --format json|markdown Output format (default: json)');
+ console.log(' --max-cost-usd N Override config max_cost_usd');
+ console.log(' --max-runtime-minutes N Override config max_runtime_minutes');
+ console.log(' --timeout-ms N Subprocess hard timeout (default: 120000)');
+ console.log('');
+ console.log('Examples:');
+ console.log(' node scripts/redblue.mjs init');
+ console.log(' node scripts/redblue.mjs run --mock-judge --tests 10');
+ console.log(' node scripts/redblue.mjs run --tests 50 --patch # needs OPENROUTER_API_KEY');
+ console.log(' node scripts/redblue.mjs attack prompt --count 3');
+ console.log('');
+ console.log('SAFETY: upstream redblue enforces no-creds, no-live-targets, no-shell,');
+ console.log('no-eval at load time (src/config/safety.ts). This wrapper does not relax');
+ console.log('those β€” it only forwards argv with shell:false.');
+}
+
+function buildUpstreamArgs() {
+ switch (ARGS.sub) {
+ case 'init': {
+ const out = ARGS.out ?? null;
+ return ['init', ...(out ? ['--out', out] : [])];
+ }
+ case 'run': {
+ const a = ['run'];
+ if (ARGS.config) a.push('--config', ARGS.config);
+ if (ARGS.tests !== null && Number.isFinite(ARGS.tests)) a.push('--tests', String(ARGS.tests));
+ if (ARGS.patch) a.push('--patch');
+ if (ARGS.mockJudge) a.push('--mock-judge');
+ if (ARGS.out) a.push('--out', ARGS.out);
+ return a;
+ }
+ case 'patch': {
+ const a = ['patch'];
+ if (ARGS.config) a.push('--config', ARGS.config);
+ if (ARGS.mockJudge) a.push('--mock-judge');
+ if (ARGS.out) a.push('--out', ARGS.out);
+ return a;
+ }
+ case 'attack': {
+ if (!ARGS.family || !VALID_ATTACK_FAMILIES.has(ARGS.family)) {
+ console.error(`redblue: --attack family must be one of ${[...VALID_ATTACK_FAMILIES].join('|')} (got: ${ARGS.family ?? 'none'})`);
+ process.exit(2);
+ }
+ const a = ['attack', ARGS.family];
+ if (ARGS.count !== null && Number.isFinite(ARGS.count)) a.push('--count', String(ARGS.count));
+ return a;
+ }
+ case 'report': {
+ if (!ARGS.inPath) {
+ console.error('redblue: report requires --in <path>');
+ process.exit(2);
+ }
+ return ['report', '--in', ARGS.inPath];
+ }
+ default:
+ console.error(`redblue: unknown subcommand "${ARGS.sub}". Valid: ${[...VALID_SUBCOMMANDS].join(', ')}`);
+ process.exit(2);
+ }
+}
+
+function main() {
+ if (!ARGS.sub || ARGS.sub === 'help' || ARGS.sub === '--help' || ARGS.sub === '-h') {
+ printHelp();
+ process.exit(0);
+ }
+ if (!VALID_SUBCOMMANDS.has(ARGS.sub)) {
+ console.error(`redblue: unknown subcommand "${ARGS.sub}". Valid: ${[...VALID_SUBCOMMANDS].join(', ')}`);
+ process.exit(2);
+ }
+
+ // For `run` and `patch` we want the report JSON in our hands even when
+ // the caller didn't pass --out (so MCP tools can read it back from stdout).
+ let synthOut = null;
+ if ((ARGS.sub === 'run' || ARGS.sub === 'patch') && !ARGS.out) {
+ const dir = mkdtempSync(join(tmpdir(), 'ruflo-redblue-'));
+ synthOut = join(dir, 'report.json');
+ ARGS.out = synthOut;
+ }
+
+ const upstreamArgs = buildUpstreamArgs();
+ if (ARGS._passthrough?.length) upstreamArgs.push(...ARGS._passthrough);
+
+ const r = runRedblue(upstreamArgs, { timeoutMs: ARGS.timeoutMs ?? undefined });
+
+ if (r.degraded) {
+ if (synthOut && existsSync(synthOut)) {
+ try { unlinkSync(synthOut); } catch { /* ignore */ }
+ }
+ emitRedblueDegradedJsonAndExit(r.reason);
+ return;
+ }
+
+ // Pass through stderr (progress) when format=markdown so users see it live
+ if (ARGS.format === 'markdown' && r.stderr) {
+ process.stderr.write(r.stderr);
+ }
+
+ // For run/patch: read the report we asked redblue to write.
+ let payload = null;
+ if (ARGS.sub === 'run' || ARGS.sub === 'patch') {
+ const reportPath = resolve(ARGS.out);
+ if (existsSync(reportPath)) {
+ try {
+ payload = JSON.parse(readFileSync(reportPath, 'utf-8'));
+ } catch (e) {
+ console.error(`redblue: failed to parse report at ${reportPath}: ${e instanceof Error ? e.message : String(e)}`);
+ if (synthOut) { try { unlinkSync(synthOut); } catch { /* ignore */ } }
+ process.exit(2);
+ }
+ }
+ }
+
+ // Decorate every payload with our own metadata + the safety boundary
+ // reminder so consumers always see "this was capability-contained".
+ const enriched = payload && typeof payload === 'object' ? {
+ ...payload,
+ _ruflo: {
+ subcommand: ARGS.sub,
+ mockJudge: ARGS.mockJudge,
+ durationMs: r.durationMs,
+ safetyBoundary: 'enforced-by-upstream-redblue (no creds, no live targets, no shell, no eval)',
+ },
+ } : payload;
+
+ // Determine overall verdict for --alert-on-fail (post-patch run only).
+ // redblue reports include a `verdict` or `gate` field; we treat absence
+ // as "no data β†’ no alert" rather than erroring.
+ let alertTriggered = false;
+ if (ARGS.alertOnFail && enriched && typeof enriched === 'object') {
+ const v = (enriched.verdict || enriched.gate || enriched.status || '').toString().toUpperCase();
+ if (v.includes('FAIL')) alertTriggered = true;
+ }
+
+ if (ARGS.format === 'json') {
+ if (enriched !== null) {
+ console.log(JSON.stringify(enriched, null, 2));
+ } else {
+ // init / attack / report subcommands β€” forward upstream stdout verbatim
+ // (it's already markdown/yaml). Wrap in a tiny JSON envelope.
+ console.log(JSON.stringify({
+ subcommand: ARGS.sub,
+ ok: r.exitCode === 0,
+ durationMs: r.durationMs,
+ stdout: r.stdout,
+ _ruflo: { safetyBoundary: 'enforced-by-upstream-redblue' },
+ }, null, 2));
+ }
+ } else {
+ // markdown / passthrough
+ if (r.stdout) process.stdout.write(r.stdout);
+ if (enriched && typeof enriched === 'object') {
+ console.log('');
+ console.log('---');
+ console.log('## redblue report (parsed)');
+ console.log('```json');
+ console.log(JSON.stringify(enriched, null, 2));
+ console.log('```');
+ }
+ }
+
+ // Clean up synthetic temp file
+ if (synthOut && existsSync(synthOut)) {
+ try { unlinkSync(synthOut); } catch { /* ignore */ }
+ }
+
+ if (alertTriggered) process.exit(1);
+ if (r.exitCode !== 0) process.exit(r.exitCode);
+}
+
+main();
diff --git a/v3/@claude-flow/cli/src/commands/metaharness.ts b/v3/@claude-flow/cli/src/commands/metaharness.ts
index e1e29b261a..4223aee3b8 100644
--- a/v3/@claude-flow/cli/src/commands/metaharness.ts
+++ b/v3/@claude-flow/cli/src/commands/metaharness.ts
@@ -60,6 +60,9 @@ const SUBCOMMANDS: Record<string, string> = {
// iter 53 β€” one-command drift detection (compose audit-list + oia-audit + audit-trend)
'drift-from-history': 'drift-from-history.mjs',
mint: 'mint.mjs',
+ // @metaharness/redblue@~0.1.1 β€” adversarial red/blue LLM testing
+ // (init|run|patch|attack|report sub-subcommands handled by redblue.mjs)
+ redblue: 'redblue.mjs',
};
/**
@@ -68,8 +71,16 @@ const SUBCOMMANDS: Record<string, string> = {
* 1. ruflo dev tree (cwd / ../plugins/...)
* 2. ruflo wrapper (ruflo/node_modules/@claude-flow/cli/...)
* 3. npx (npm-cache/__npx/... β€” fall back to cwd-scan)
+ *
+ * `requiredScript`, if provided, narrows the match: the returned dir must
+ * contain BOTH `_harness.mjs` (proves it's a plugin-scripts dir) AND the
+ * named script. This guards against the publish-artifact mirror at
+ * `v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/` getting picked
+ * over the source `plugins/ruflo-metaharness/scripts/` when the mirror
+ * lags the source (the mirror is regenerated only at publish time by
+ * `prepublishOnly`).
*/
-function locatePluginScripts(): string | null {
+function locatePluginScripts(requiredScript?: string): string | null {
const candidates: string[] = [];
// Up from the cli dist dir
let p = resolve(__dirname);
@@ -86,7 +97,9 @@ function locatePluginScripts(): string | null {
candidates.push(join(cwd, 'node_modules', '@claude-flow', 'cli', 'plugins', 'ruflo-metaharness', 'scripts'));
for (const c of candidates) {
- if (existsSync(join(c, '_harness.mjs'))) return c;
+ if (!existsSync(join(c, '_harness.mjs'))) continue;
+ if (requiredScript && !existsSync(join(c, requiredScript))) continue;
+ return c;
}
return null;
}
@@ -115,7 +128,7 @@ export const metaharnessCommand: Command = {
// iter 73 β€” list reflects all 10 dispatchable subcommands (was
// stale at the iter-3 list of 5). Keep this synced with the
// SUBCOMMANDS map above.
- description: 'One of: score | genome | mcp-scan | threat-model | oia-audit | audit-list | audit-trend | similarity | drift-from-history | mint',
+ description: 'One of: score | genome | mcp-scan | threat-model | oia-audit | audit-list | audit-trend | similarity | drift-from-history | mint | redblue',
type: 'string' as const,
},
],
@@ -174,6 +187,7 @@ export const metaharnessCommand: Command = {
output.writeln(' similarity ADR-152 β€” weighted similarity between two harness fingerprints');
output.writeln(' drift-from-history iter 53 β€” diff current state against most recent audit (1-command drift)');
output.writeln(' mint scaffold a custom harness (dry-run by default)');
+ output.writeln(' redblue adversarial red/blue LLM testing (init|run|patch|attack|report)');
output.writeln('');
output.writeln('Each subcommand accepts --format json|table and --help.');
output.writeln('');
@@ -187,7 +201,7 @@ export const metaharnessCommand: Command = {
return { success: false, exitCode: 2, data: { subcommand } };
}
- const scriptDir = locatePluginScripts();
+ const scriptDir = locatePluginScripts(SUBCOMMANDS[subcommand]);
if (!scriptDir) {
output.writeln(
output.warning(
diff --git a/v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts b/v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts
index 10cdd413c0..d52bebb752 100644
--- a/v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts
+++ b/v3/@claude-flow/cli/src/mcp-tools/metaharness-tools.ts
@@ -18,6 +18,12 @@
* - metaharness_security_bench upstream's "Darwin Shield" (their own ADR-155)
* - metaharness_bench create/verify bench suites used by evolve --bench
*
+ * @metaharness/redblue integration adds one tool that targets the standalone
+ * `@metaharness/redblue` npm package β€” adversarial red/blue LLM testing
+ * for the agents and apps you own:
+ *
+ * - metaharness_redblue red-team β†’ judge β†’ blue-patch β†’ retest β†’ report
+ *
* Every tool resolves the corresponding plugin script
* (`plugins/ruflo-metaharness/scripts/<X>.mjs`) via the same locator
* the commands/metaharness.ts dispatcher uses, then spawns it with
@@ -46,8 +52,14 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
/**
* Walk up from this module to find plugins/ruflo-metaharness/scripts/.
* Handles three install layouts (mirrors commands/metaharness.ts).
+ *
+ * `requiredScript`, if provided, narrows the match the same way the
+ * commands dispatcher does β€” guards against the publish-artifact mirror
+ * (`v3/@claude-flow/cli/plugins/ruflo-metaharness/scripts/`, regenerated
+ * by `prepublishOnly`) shadowing the source when it's stale on a new
+ * script.
*/
-function locatePluginScripts(): string | null {
+function locatePluginScripts(requiredScript?: string): string | null {
const candidates: string[] = [];
let p = resolve(__dirname);
for (let i = 0; i < 8; i++) {
@@ -59,7 +71,9 @@ function locatePluginScripts(): string | null {
candidates.push(join(cwd, 'plugins', 'ruflo-metaharness', 'scripts'));
candidates.push(join(cwd, 'node_modules', '@claude-flow', 'cli', 'plugins', 'ruflo-metaharness', 'scripts'));
for (const c of candidates) {
- if (existsSync(join(c, '_harness.mjs'))) return c;
+ if (!existsSync(join(c, '_harness.mjs'))) continue;
+ if (requiredScript && !existsSync(join(c, requiredScript))) continue;
+ return c;
}
return null;
}
@@ -100,7 +114,7 @@ function runScript(scriptName: string, args: string[]): Promise<{
success: boolean;
}> {
return new Promise((resolve) => {
- const dir = locatePluginScripts();
+ const dir = locatePluginScripts(scriptName);
if (!dir) {
resolve({
exitCode: 0, stdout: '', json: { degraded: true, reason: 'plugin-not-found' },
@@ -464,4 +478,58 @@ export const metaharnessTools: MCPTool[] = [
return { success: r.success, data: r.json, degraded: r.degraded, exitCode: r.exitCode };
},
},
+ // ───────────────────────────────────────────────────────────────────────
+ // @metaharness/redblue integration (1 tool).
+ // Backed by `@metaharness/redblue@~0.1.1`. Plugin script shells out via
+ // _redblue.mjs. Same {success, data, degraded, exitCode} contract.
+ //
+ // SAFETY: redblue itself enforces hard boundaries (no real creds, no live
+ // targets, no shell, no arbitrary network, no eval) in upstream's
+ // src/config/safety.ts at config-load time. The wrapper does NOT relax
+ // those β€” it only forwards argv with shell:false. `--mock-judge` is the
+ // $0 marker-fixture CI path; the real model judge requires
+ // $OPENROUTER_API_KEY which we never inject.
+ // ───────────────────────────────────────────────────────────────────────
+ {
+ name: 'metaharness_redblue',
+ description: 'Adversarial red/blue LLM testing via @metaharness/redblue β€” generates attacks across OWASP LLM Top-10 / NIST AI RMF families (prompt injection, tool misuse, data leakage, jailbreaks, denial-of-wallet), runs them against an LLM target YOU OWN, judges compromise, optionally applies declarative blue-team patches, retests, and emits a board-readable report with measured failure reduction. Use when shipping an LLM-powered product and you need a repeatable security gate before exposing it to users β€” eyeballing prompts is wrong because attack surface coverage requires the OWASP/NIST taxonomy and the judge has to be model-driven for jailbreak detection. SAFETY: upstream hard-enforces no-creds / no-live-targets / no-shell / no-network / no-eval at config-load time; cannot be relaxed via flags. For CI / offline use --mockJudge=true ($0 marker fixture). For real model judging set $OPENROUTER_API_KEY and accept the per-run cost capped by max_cost_usd (default $3). ' + MCP_SUCCESS_SEMANTIC,
+ category: 'metaharness',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ subcommand: {
+ type: 'string',
+ enum: ['init', 'run', 'patch', 'attack', 'report'],
+ description: 'init = scaffold redblue.yaml; run = baseline (+ optional --patch); patch = baseline β†’ patch β†’ retest delta; attack = preview attacks; report = render existing report.json',
+ default: 'run',
+ },
+ config: { type: 'string', description: 'Path to redblue.yaml (default: ./redblue.yaml)' },
+ out: { type: 'string', description: 'Output report path for run/patch (default: temp file we read back inline)' },
+ in: { type: 'string', description: 'Input report path for `report` subcommand' },
+ tests: { type: 'number', description: 'How many test cases (run/patch only)' },
+ patch: { type: 'boolean', description: '`run` only β€” after baseline, apply blue-team patches and retest', default: false },
+ mockJudge: { type: 'boolean', description: '$0 TEST-ONLY marker fixture (no model calls). Use for CI / offline. Real judging requires OPENROUTER_API_KEY.', default: false },
+ family: { type: 'string', enum: ['prompt', 'tools', 'data', 'all'], description: '`attack` subcommand only β€” which attack family to preview' },
+ count: { type: 'number', description: '`attack` only β€” how many cases to preview' },
+ alertOnFail: { type: 'boolean', description: 'Exit 1 when post-patch verdict is FAIL (gate-style)', default: false },
+ timeoutMs: { type: 'number', description: 'Subprocess hard timeout (default 120000; mock-judge runs complete in seconds)' },
+ },
+ },
+ handler: async (input) => {
+ const args: string[] = [String(input.subcommand ?? 'run')];
+ // attack family is positional
+ if (input.subcommand === 'attack' && input.family) args.push(String(input.family));
+ if (input.config) args.push('--config', String(input.config));
+ if (input.out) args.push('--out', String(input.out));
+ if (input.in) args.push('--in', String(input.in));
+ if (input.tests !== undefined) args.push('--tests', String(input.tests));
+ if (input.patch === true) args.push('--patch');
+ if (input.mockJudge === true) args.push('--mock-judge');
+ if (input.count !== undefined) args.push('--count', String(input.count));
+ if (input.alertOnFail === true) args.push('--alert-on-fail');
+ if (input.timeoutMs !== undefined) args.push('--timeout-ms', String(input.timeoutMs));
+ const r = await runScript('redblue.mjs', args);
+ return { success: r.success, data: r.json, degraded: r.degraded, exitCode: r.exitCode };
+ },
+ },
];