Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

test(metaharness): roundtrip catches REAL drift, not just self-match (iter 51)

ruve722508ad827
diff --git a/plugins/ruflo-metaharness/scripts/smoke.sh b/plugins/ruflo-metaharness/scripts/smoke.sh
index 954d601d5e..b6777dcfd8 100755
--- a/plugins/ruflo-metaharness/scripts/smoke.sh
+++ b/plugins/ruflo-metaharness/scripts/smoke.sh
@@ -191,6 +191,23 @@ grep -q "execCli(\[\s*'-y'\s*,\s*'metaharness@latest'" "$F" 2>/dev/null || \
grep -q "cwd: opts" "$F" || miss="$miss no-cwd-passthrough"
[[ -z "$miss" ]] && ok || bad "$miss"
+step "17z14. roundtrip Stage 7 β€” drift detection actually fires on mutation (iter 51)"
+miss=""
+F="$ROOT/scripts/test-pipeline-roundtrip.mjs"
+# Stage 7 present
+grep -q "Stage 7 β€” drift detection on mutated audit" "$F" 2>/dev/null || miss="$miss no-stage-7"
+# Mutates all 3 similarity components (cosine + categorical + jaccard)
+grep -q "harnessFit - 40" "$F" 2>/dev/null || miss="$miss no-cosine-mutation"
+grep -q "iter-51-synthetic-archetype\|synthetic-archetype" "$F" 2>/dev/null || miss="$miss no-categorical-mutation"
+grep -q "iter-51-marker" "$F" 2>/dev/null || miss="$miss no-jaccard-mutation"
+# Critical assertions
+grep -q "drift detected β€” verdict !== near-identical" "$F" 2>/dev/null || miss="$miss no-verdict-flip"
+grep -q "drift detected β€” distance > 0" "$F" 2>/dev/null || miss="$miss no-distance-positive"
+grep -q "drift alert at threshold" "$F" 2>/dev/null || miss="$miss no-drift-alert"
+# Runtime: 31/31 (was 25/25 pre-iter-51)
+node "$F" 2>&1 | grep -q "31 passed, 0 failed" || miss="$miss roundtrip-not-31"
+[[ -z "$miss" ]] && ok || bad "$miss"
+
step "17z13. mcp-scan findings parsed from text β†’ audit-trend diff works (iter 50)"
miss=""
HARNESS="$ROOT/scripts/_harness.mjs"
@@ -207,8 +224,8 @@ grep -q "label === 'mcp-scan'" "$OIA" 2>/dev/null || miss="$miss no-label-dispat
OUT=$(node "$SCAN" --path . --format json 2>/dev/null)
echo "$OUT" | grep -q '"findings": \[' || miss="$miss runtime-no-findings-array"
echo "$OUT" | grep -q '"severity":' || miss="$miss runtime-no-severity-field"
-# Runtime: roundtrip passes 25/25 (was 24/24 with mcp-scan SKIP)
-node "$ROOT/scripts/test-pipeline-roundtrip.mjs" 2>&1 | grep -q "25 passed, 0 failed" || miss="$miss roundtrip-not-25"
+# Runtime: roundtrip passes (β‰₯25 β€” iter 50 took it from 24, future iters keep climbing)
+node "$ROOT/scripts/test-pipeline-roundtrip.mjs" 2>&1 | grep -qE "(2[5-9]|[3-9][0-9]+) passed, 0 failed" || miss="$miss roundtrip-fewer-than-25"
[[ -z "$miss" ]] && ok || bad "$miss"
step "17z12. roundtrip covers non-similarity schemas + flags mcp-scan gap (iter 49)"
diff --git a/plugins/ruflo-metaharness/scripts/test-pipeline-roundtrip.mjs b/plugins/ruflo-metaharness/scripts/test-pipeline-roundtrip.mjs
index 0cbe1ab905..5e69096bd7 100755
--- a/plugins/ruflo-metaharness/scripts/test-pipeline-roundtrip.mjs
+++ b/plugins/ruflo-metaharness/scripts/test-pipeline-roundtrip.mjs
@@ -218,6 +218,76 @@ try {
// delta.worst.verdict on a self-match must be 'unchanged'. If upstream
// renames any of these fields, the verdict silently becomes 'missing'
// or undefined β€” this assertion catches that BEFORE shipping.
+ // ──────────────────────────────────────────────────────────────
+ // ──────────────────────────────────────────────────────────────
+ // STAGE 7 (iter 51) β€” drift detection actually FIRES on mutated input
+ //
+ // Every prior stage uses the same audit record twice (self-match).
+ // That proves the chain doesn't FALSELY report drift on identical
+ // input, but doesn't prove the chain catches REAL drift. This stage
+ // takes the audit record and synthetically mutates fields the
+ // similarity module reads, then asserts the chain detects the
+ // difference.
+ //
+ // Mutations applied (all measurable by similarity):
+ // - harnessFit: unchanged β†’ -20 (cosine signal)
+ // - taskCoverage: unchanged β†’ -15 (cosine signal)
+ // - agent_topology: add one β†’ jaccard divergence
+ //
+ // Expected outcome:
+ // - structuralDistance.verdict !== 'near-identical'
+ // - structuralDistance.overall < 1
+ // - --alert-on-distance-below 0.95 should fire
+ // ──────────────────────────────────────────────────────────────
+ console.log('\nStage 7 β€” drift detection on mutated audit (iter 51)');
+ const mutatedPath = join(tmp, 'mutated.json');
+ const mutated = JSON.parse(JSON.stringify(audit));
+ // Modify fields the similarity module actually reads. The mutation has
+ // to be large enough to cross the verdict-bucket threshold (0.95 β†’
+ // 'near-identical' boundary). The 9-dim cosine vector is dominated by
+ // values already clustered around the same range, so single-field
+ // changes don't move overall much. Mutate across all 3 components:
+ // - cosine: drop 4 scorecard fields by 25-40 points
+ // - categorical: swap archetype + template (2 of 4 enums diverge)
+ // - jaccard: rebuild agent_topology with a different set
+ mutated.fingerprint.score.harnessFit = Math.max(0, audit.fingerprint.score.harnessFit - 40);
+ mutated.fingerprint.score.taskCoverage = Math.max(0, audit.fingerprint.score.taskCoverage - 35);
+ mutated.fingerprint.score.toolSafety = Math.max(0, audit.fingerprint.score.toolSafety - 30);
+ mutated.fingerprint.score.compileConfidence = Math.max(0, audit.fingerprint.score.compileConfidence - 25);
+ mutated.fingerprint.score.archetype = 'iter-51-synthetic-archetype';
+ mutated.fingerprint.score.template = 'iter-51-synthetic-template';
+ mutated.fingerprint.genome.agent_topology = ['iter-51-marker-a', 'iter-51-marker-b'];
+ writeFileSync(mutatedPath, JSON.stringify(mutated));
+
+ const driftRun = runNode('audit-trend.mjs', [
+ '--baseline', basePath,
+ '--current', mutatedPath,
+ '--format', 'json',
+ ]);
+ const driftMatch = /\{[\s\S]*\}/.exec(driftRun.stdout);
+ assert(driftMatch !== null, 'drift run produced JSON');
+ const driftTrend = JSON.parse(driftMatch[0]);
+ const driftSd = driftTrend.delta?.structuralDistance;
+ assert(typeof driftSd === 'object', 'drift trend exposes structuralDistance');
+ assert(driftSd.verdict !== 'near-identical',
+ `drift detected β€” verdict !== near-identical (got ${driftSd?.verdict})`);
+ assert(driftSd.overall < 1,
+ `drift detected β€” overall < 1 (got ${driftSd?.overall})`);
+ assert(driftSd.distance > 0,
+ `drift detected β€” distance > 0 (got ${driftSd?.distance})`);
+
+ // Alert chain: --alert-on-distance-below 0.95 must fire IF drift > 0.05
+ // If the synthetic mutation produced overall β‰₯ 0.95, lower the threshold.
+ const threshold = driftSd.overall < 0.95 ? '0.95' : String(driftSd.overall + 0.01);
+ const alertDriftRun = runNode('audit-trend.mjs', [
+ '--baseline', basePath,
+ '--current', mutatedPath,
+ '--alert-on-distance-below', threshold,
+ '--format', 'json',
+ ]);
+ assert(alertDriftRun.status === 1,
+ `drift alert at threshold ${threshold} fires on mutated audit (exit 1, got ${alertDriftRun.status})`);
+
// ──────────────────────────────────────────────────────────────
console.log('\nStage 6 β€” non-similarity schema contracts (iter 49)');
assert(typeof trend.delta?.worst === 'object',