π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
feat(metaharness): SelfEvolvingRouter promotion-gate analyzer (iter 10)
ruv473d2fcc8244
diff --git a/plugins/ruflo-metaharness/scripts/router-parallel-analyze.mjs b/plugins/ruflo-metaharness/scripts/router-parallel-analyze.mjsnew file mode 100755index 0000000000..cdcc7c2bc5--- /dev/null+++ b/plugins/ruflo-metaharness/scripts/router-parallel-analyze.mjs@@ -0,0 +1,250 @@+#!/usr/bin/env node+// router-parallel-analyze.mjs β ADR-150 Phase 2 SelfEvolvingRouter+// promotion gate.+//+// Reads paired routing decisions (Thompson bandit pick + SelfEvolvingRouter+// pick, both with measured outcomes) from a JSONL trajectory file and+// computes the THREE PROMOTION CRITERIA from ADR-150 review-round-1:+//+// (a) qualityScore improvement > 2% (relative)+// (b) usdPerDecision increase < 1% (relative)+// (c) p95 routing-decision latency increase < 5% (relative)+//+// ALL THREE must hold for promotion. AND, not OR β the OR form let+// quality gains mask cost regressions, exactly the failure mode ADR-149's+// Pareto framing was built to prevent.+//+// INPUT FORMAT (.swarm/router-parallel.jsonl, one JSON record per line):+// {+// "ts": "<iso>",+// "task": { ... },+// "bandit": { "pick": "<modelId>", "predictedQuality": 0.84, "predictedCostUsd": 0.003 },+// "ser": { "pick": "<modelId>", "predictedQuality": 0.87, "predictedCostUsd": 0.004 },+// "outcome": { "actualModel": "<modelId>", "actualQuality": 0.91, "actualUsd": 0.0035, "actualLatencyMs": 1240 }+// }+//+// USAGE+// node scripts/router-parallel-analyze.mjs --input .swarm/router-parallel.jsonl+// node scripts/router-parallel-analyze.mjs --input <file> --format json+// node scripts/router-parallel-analyze.mjs --input <file> --strict # exit 1 if NOT promotable+//+// EXIT CODES+// 0 analysis complete; in --strict mode β all three criteria passed+// 1 --strict requested AND at least one criterion failed (NOT promotable)+// 2 config error or input file missing/malformed++import { readFileSync, existsSync } from 'node:fs';++const ARGS = (() => {+ const a = { input: '.swarm/router-parallel.jsonl', format: 'table', strict: false };+ for (let i = 2; i < process.argv.length; i++) {+ const v = process.argv[i];+ if (v === '--input') a.input = process.argv[++i];+ else if (v === '--format') a.format = process.argv[++i];+ else if (v === '--strict') a.strict = true;+ }+ return a;+})();++function median(sorted) {+ if (sorted.length === 0) return 0;+ const n = sorted.length;+ return n % 2 === 0 ? (sorted[n / 2 - 1] + sorted[n / 2]) / 2 : sorted[(n - 1) / 2];+}++function pctile(sorted, q) {+ if (sorted.length === 0) return 0;+ const idx = Math.floor(q * (sorted.length - 1));+ return sorted[idx];+}++function mean(arr) {+ if (arr.length === 0) return 0;+ return arr.reduce((s, x) => s + x, 0) / arr.length;+}++function main() {+ if (!existsSync(ARGS.input)) {+ if (ARGS.format === 'json') {+ console.log(JSON.stringify({+ ok: true,+ sufficient: false,+ reason: 'input-file-not-found',+ input: ARGS.input,+ hint: 'Enable parallel logging (CLAUDE_FLOW_ROUTER_PARALLEL_LOG=1) and run a workload first.',+ }, null, 2));+ } else {+ console.log(`# router-parallel-analyze`);+ console.log('');+ console.log(`_Input file not found: ${ARGS.input}_`);+ console.log('');+ console.log('Enable parallel logging by setting `CLAUDE_FLOW_ROUTER_PARALLEL_LOG=1`');+ console.log('and running a real workload, then re-run this analyzer.');+ }+ process.exit(ARGS.strict ? 1 : 0);+ }++ let rows = [];+ try {+ rows = readFileSync(ARGS.input, 'utf-8')+ .split('\n')+ .filter((l) => l.trim())+ .map((l) => JSON.parse(l));+ } catch (e) {+ console.error(`router-parallel-analyze: cannot parse ${ARGS.input}: ${e.message}`);+ process.exit(2);+ }++ // Filter to records with both predictions + outcome present.+ const usable = rows.filter((r) => r.bandit && r.ser && r.outcome);++ if (usable.length < 30) {+ const payload = {+ ok: true,+ sufficient: false,+ reason: `n=${usable.length} < 30 (insufficient sample for AND-gate evaluation)`,+ hint: 'Continue collecting parallel-routing data; statistical significance needs β₯30 paired decisions per arm.',+ sampleSize: usable.length,+ generatedAt: new Date().toISOString(),+ };+ if (ARGS.format === 'json') console.log(JSON.stringify(payload, null, 2));+ else {+ console.log(`# router-parallel-analyze`);+ console.log('');+ console.log(`Sample size: **${usable.length}** paired decisions`);+ console.log('');+ console.log(`_Insufficient β need β₯30 for the 3-criteria AND-gate._`);+ }+ process.exit(ARGS.strict ? 1 : 0);+ }++ // For each record, attribute the outcome to BOTH arms when they+ // agreed on the model pick; when they disagreed, the actual outcome+ // belongs only to the bandit (which was the executing arm). The SER+ // arm gets its PREDICTED metrics for the counterfactual.+ const banditOutcomes = usable.map((r) => ({+ qualityActual: r.outcome.actualQuality,+ usdActual: r.outcome.actualUsd,+ latencyActual: r.outcome.actualLatencyMs,+ }));++ // SER counterfactual: use ser.predictedQuality / predictedCostUsd+ // when picks disagreed; use actual when they agreed. Latency: when+ // they agreed, both saw the same actual latency; when they+ // disagreed, the SER pick's latency is unknown β fall back to the+ // mean of agreed-decision latencies for that model.+ const agreedLatencyByModel = {};+ for (const r of usable) {+ if (r.bandit.pick === r.ser.pick) {+ const m = r.outcome.actualModel || r.bandit.pick;+ (agreedLatencyByModel[m] = agreedLatencyByModel[m] || []).push(r.outcome.actualLatencyMs);+ }+ }+ const modelMeanLatency = {};+ for (const [m, arr] of Object.entries(agreedLatencyByModel)) {+ modelMeanLatency[m] = mean(arr);+ }++ const serOutcomes = usable.map((r) => {+ const agreed = r.bandit.pick === r.ser.pick;+ return {+ qualityActual: agreed ? r.outcome.actualQuality : r.ser.predictedQuality,+ usdActual: agreed ? r.outcome.actualUsd : r.ser.predictedCostUsd,+ latencyActual: agreed+ ? r.outcome.actualLatencyMs+ : (modelMeanLatency[r.ser.pick] ?? r.outcome.actualLatencyMs),+ };+ });++ // Compute the three criteria.+ const banditQuality = mean(banditOutcomes.map((o) => o.qualityActual));+ const serQuality = mean(serOutcomes.map((o) => o.qualityActual));+ const qualityImprovementPct =+ banditQuality > 0 ? ((serQuality - banditQuality) / banditQuality) * 100 : 0;++ const banditUsd = mean(banditOutcomes.map((o) => o.usdActual));+ const serUsd = mean(serOutcomes.map((o) => o.usdActual));+ const usdIncreasePct = banditUsd > 0 ? ((serUsd - banditUsd) / banditUsd) * 100 : 0;++ const banditLatencyP95 = pctile(banditOutcomes.map((o) => o.latencyActual).sort((a, b) => a - b), 0.95);+ const serLatencyP95 = pctile(serOutcomes.map((o) => o.latencyActual).sort((a, b) => a - b), 0.95);+ const latencyIncreasePct =+ banditLatencyP95 > 0 ? ((serLatencyP95 - banditLatencyP95) / banditLatencyP95) * 100 : 0;++ const passes = {+ quality: qualityImprovementPct > 2,+ cost: usdIncreasePct < 1,+ latency: latencyIncreasePct < 5,+ };+ const allPass = passes.quality && passes.cost && passes.latency;++ // Disagreement rate (informational; not part of the gate).+ const disagreements = usable.filter((r) => r.bandit.pick !== r.ser.pick).length;+ const disagreementPct = (disagreements / usable.length) * 100;++ const payload = {+ ok: true,+ sufficient: true,+ sampleSize: usable.length,+ disagreement: {+ count: disagreements,+ pct: Math.round(disagreementPct * 100) / 100,+ },+ bandit: {+ meanQuality: Math.round(banditQuality * 10000) / 10000,+ meanUsd: Math.round(banditUsd * 1e6) / 1e6,+ p95LatencyMs: Math.round(banditLatencyP95),+ },+ ser: {+ meanQuality: Math.round(serQuality * 10000) / 10000,+ meanUsd: Math.round(serUsd * 1e6) / 1e6,+ p95LatencyMs: Math.round(serLatencyP95),+ },+ criteria: {+ qualityImprovementPct: Math.round(qualityImprovementPct * 100) / 100,+ qualityThresholdPct: 2,+ qualityPasses: passes.quality,+ usdIncreasePct: Math.round(usdIncreasePct * 100) / 100,+ usdThresholdPct: 1,+ costPasses: passes.cost,+ latencyIncreasePct: Math.round(latencyIncreasePct * 100) / 100,+ latencyThresholdPct: 5,+ latencyPasses: passes.latency,+ },+ verdict: {+ promotable: allPass,+ reason: allPass+ ? 'All three criteria met (quality > 2% AND cost < 1% AND latency < 5%)'+ : `Blocked by: ${[+ !passes.quality && `quality only +${(qualityImprovementPct).toFixed(2)}% (need >2%)`,+ !passes.cost && `cost +${(usdIncreasePct).toFixed(2)}% (need <1%)`,+ !passes.latency && `latency +${(latencyIncreasePct).toFixed(2)}% (need <5%)`,+ ].filter(Boolean).join(', ')}`,+ },+ generatedAt: new Date().toISOString(),+ };++ if (ARGS.format === 'json') {+ console.log(JSON.stringify(payload, null, 2));+ } else {+ console.log('# router-parallel-analyze (ADR-150 SelfEvolvingRouter promotion gate)');+ console.log('');+ console.log(`Sample size: **${payload.sampleSize}** paired decisions`);+ console.log(`Disagreement rate: ${payload.disagreement.pct}% (${payload.disagreement.count} / ${payload.sampleSize})`);+ console.log('');+ console.log(`| Metric | Bandit | SER | Delta | Threshold | Passes |`);+ console.log(`|---|---:|---:|---:|---:|:---:|`);+ console.log(`| Mean quality | ${payload.bandit.meanQuality} | ${payload.ser.meanQuality} | ${payload.criteria.qualityImprovementPct >= 0 ? '+' : ''}${payload.criteria.qualityImprovementPct}% | >+2% | ${passes.quality ? 'β' : 'β '} |`);+ console.log(`| Mean $/decision | $${payload.bandit.meanUsd.toFixed(6)} | $${payload.ser.meanUsd.toFixed(6)} | ${payload.criteria.usdIncreasePct >= 0 ? '+' : ''}${payload.criteria.usdIncreasePct}% | <+1% | ${passes.cost ? 'β' : 'β '} |`);+ console.log(`| p95 latency | ${payload.bandit.p95LatencyMs}ms | ${payload.ser.p95LatencyMs}ms | ${payload.criteria.latencyIncreasePct >= 0 ? '+' : ''}${payload.criteria.latencyIncreasePct}% | <+5% | ${passes.latency ? 'β' : 'β '} |`);+ console.log('');+ console.log(`**Verdict**: ${allPass ? 'β PROMOTABLE' : 'β NOT promotable'}`);+ console.log('');+ console.log(`_${payload.verdict.reason}_`);+ }++ if (ARGS.strict && !allPass) process.exit(1);+ process.exit(0);+}++main();diff --git a/plugins/ruflo-metaharness/scripts/smoke.sh b/plugins/ruflo-metaharness/scripts/smoke.shindex 09c29fa4bc..bdc987ecd8 100755--- a/plugins/ruflo-metaharness/scripts/smoke.sh+++ b/plugins/ruflo-metaharness/scripts/smoke.sh@@ -170,6 +170,26 @@ if (!od.metaharness) { console.error('missing metaharness in optionalDependencieif (j.dependencies && j.dependencies.metaharness) { console.error('metaharness leaked into dependencies'); process.exit(1); }" 2>/dev/null && ok || bad "ruflo wrapper missing metaharness optionalDep"+step "17d. router-parallel-analyze (ADR-150 SelfEvolvingRouter promotion gate β iter 10)"+F="$ROOT/scripts/router-parallel-analyze.mjs"+miss=""+[[ -x "$F" ]] || miss="$miss not-executable"+node --check "$F" 2>/dev/null || miss="$miss syntax-error"+# The 3-criteria AND-gate from ADR-150 review-round-1 must be explicit+grep -q "qualityImprovementPct" "$F" || miss="$miss no-quality-metric"+grep -q "usdIncreasePct" "$F" || miss="$miss no-cost-metric"+grep -q "latencyIncreasePct" "$F" || miss="$miss no-latency-metric"+# AND-semantics (not OR)+grep -q "passes.quality && passes.cost && passes.latency" "$F" || miss="$miss no-AND-gate"+# Thresholds documented in source+grep -q "qualityThresholdPct: 2" "$F" || miss="$miss no-quality-threshold"+grep -q "usdThresholdPct: 1" "$F" || miss="$miss no-cost-threshold"+grep -q "latencyThresholdPct: 5" "$F" || miss="$miss no-latency-threshold"+# Insufficient-data + strict modes both exit cleanly+grep -q "n=\${usable.length} < 30\|sufficient: false" "$F" || miss="$miss no-insufficient-guard"+grep -q "ARGS.strict" "$F" || miss="$miss no-strict-mode"+[[ -z "$miss" ]] && ok || bad "$miss"+step "17c. oia-audit composite worker (Phase 2 β iter 7)"F="$ROOT/scripts/oia-audit.mjs"miss=""diff --git a/ruflo/package.json b/ruflo/package.jsonindex 8064d5a2e1..509c0924f6 100644--- a/ruflo/package.json+++ b/ruflo/package.json@@ -44,7 +44,8 @@},"optionalDependencies": {"metaharness": "~0.1.11",- "@metaharness/router": "~0.3.2"+ "@metaharness/router": "~0.3.2",+ "@metaharness/kernel": "~0.1.0"},"overrides": {"ruvector": "^0.2.27",diff --git a/v3/@claude-flow/cli/package.json b/v3/@claude-flow/cli/package.jsonindex 3b6878b427..657bc0e04f 100644--- a/v3/@claude-flow/cli/package.json+++ b/v3/@claude-flow/cli/package.json@@ -112,6 +112,7 @@"@claude-flow/memory": "^3.0.0-alpha.20","@claude-flow/plugin-gastown-bridge": "^0.1.3","@claude-flow/security": "^3.0.0-alpha.10",+ "@metaharness/kernel": "~0.1.0","@metaharness/router": "~0.3.2","metaharness": "~0.1.11","@ruvector/attention": "^0.1.32",diff --git a/v3/docs/adr/ADR-150-metaharness-integration-surfaces.md b/v3/docs/adr/ADR-150-metaharness-integration-surfaces.mdindex c64a26c371..456adab1a6 100644--- a/v3/docs/adr/ADR-150-metaharness-integration-surfaces.md+++ b/v3/docs/adr/ADR-150-metaharness-integration-surfaces.md@@ -195,9 +195,18 @@ The integration shipped across eight `/loop` iterations on branchoia-manifest + threat-model + mcp-scan into one timestamped record;persistence to `metaharness-audit` memory namespace; weekly cronworkflow `.github/workflows/oia-audit-weekly.yml` at Sundays 04:17 UTC.-- π `SelfEvolvingRouter` parallel-logging β DEFERRED to a follow-up- ADR. Promotion criteria from review-round-1 still stand:- quality > 2% AND cost < 1% AND latency < 5%.+- π/β `SelfEvolvingRouter` parallel-logging β ANALYZER LANDED (iter 10):+ `plugins/ruflo-metaharness/scripts/router-parallel-analyze.mjs` reads+ paired routing decisions from a JSONL trajectory file and computes+ the 3-criteria AND-gate from review-round-1. Verified end-to-end with+ synthetic fixtures (β PROMOTABLE / β NOT promotable paths both work,+ insufficient-data path exits cleanly at n<30). `@metaharness/kernel`+ added to `optionalDependencies` of `@claude-flow/cli` AND+ `ruflo/package.json` so the future Recording side can dynamic-import+ `SelfEvolvingRouter` without a static dep. The RECORDING side (the+ parallel-log emitter in `model-router.ts` hot path) is the only+ remaining piece β its blast radius is high and warrants its own ADR+ before merging.### Phase-1 item #3 β Real seed corpus retraining π PENDINGRequires production trajectory data. The pipeline is wired: