🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
test: make the timing-sensitive CI tests robust
ruv3438b4085e6f
diff --git a/v3/@claude-flow/performance/__tests__/benchmarks.test.ts b/v3/@claude-flow/performance/__tests__/benchmarks.test.tsindex a1361347c0..1dbe632554 100644--- a/v3/@claude-flow/performance/__tests__/benchmarks.test.ts+++ b/v3/@claude-flow/performance/__tests__/benchmarks.test.ts@@ -418,15 +418,34 @@ describe('quickValidation()', () => {});});+const RUNS_PER_MEASUREMENT = 15;+describe('Performance Validation', () => {+ // Statistical, timing-based test. The old form compared two single runs, each+ // of which times a Flash block and then a baseline block with no warm-up, so a+ // cold JIT or one scheduler stall landing on one block of one run swung the+ // ratio past 2.0 (observed 2.23 in CI). What it guards is that the speedup+ // measurement is reproducible, so keep that exact claim ("two measurements+ // agree within 2x") but make each measurement robust: warm up, then take two+ // independent measurements that are each the MEDIAN of several runs, with the+ // runs of the two measurements interleaved so slow drift in machine load hits+ // both equally. A measurement that is genuinely irreproducible still fails.it('should demonstrate consistent speedup', () => {const runner = new AttentionBenchmarkRunner();- const result1 = runner.runComparison(256, 50, 100);- const result2 = runner.runComparison(256, 50, 100);+ for (let i = 0; i < 3; i++) runner.runComparison(256, 50, 100); // warm-up, discarded++ const a: number[] = [];+ const b: number[] = [];+ for (let i = 0; i < RUNS_PER_MEASUREMENT; i++) {+ a.push(runner.runComparison(256, 50, 100).results.speedup);+ b.push(runner.runComparison(256, 50, 100).results.speedup);+ }+ const median = (xs: number[]) => [...xs].sort((x, y) => x - y)[xs.length >> 1];// Speedup should be relatively consistent (within 50% variance)- const ratio = result1.results.speedup / result2.results.speedup;+ const ratio = median(a) / median(b);+ console.log(JSON.stringify({ medianA: median(a), medianB: median(b), medianRatio: ratio }));expect(ratio).toBeGreaterThan(0.5);expect(ratio).toBeLessThan(2.0);});diff --git a/v3/@claude-flow/performance/__tests__/resource-admission-report.test.ts b/v3/@claude-flow/performance/__tests__/resource-admission-report.test.tsindex 58213e3aa1..b6f058342e 100644--- a/v3/@claude-flow/performance/__tests__/resource-admission-report.test.ts+++ b/v3/@claude-flow/performance/__tests__/resource-admission-report.test.ts@@ -1,4 +1,5 @@import { expect, it } from 'vitest';+import { pairedTiming, percentile } from './timing-utils.js';import { decideResourceAdmission, type ResourceAdmissionInput } from '../src/resource-admission.js';const now = '2026-09-21T14:00:00.000Z';@@ -46,11 +47,6 @@ function fixedCpuBaseline(input: ResourceAdmissionInput): number {return Math.max(0, Math.min(4, input.profile.maxCpuCores, input.host.maxCpuPerTask, input.policy.maxCpuPerTask, share));}-function percentile(values: number[], fraction: number): number {- const sorted = [...values].sort((a, b) => a - b);- return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))];-}-function mean(values: number[]): number {return values.reduce((sum, value) => sum + value, 0) / values.length;}@@ -60,10 +56,18 @@ function variance(values: number[]): number {return values.reduce((sum, value) => sum + (value - center) ** 2, 0) / values.length;}-it('reports baseline and candidate resource admission overhead with frozen workload', () => {+// Gate on median(decision cost / reference cost), measured in-process (see timing-utils.ts).+// Measured over 30+ runs under a 32-process CPU stressor: ratio 5.3-6.2 (decision ~6 us, reference ~1.1 us).+// 15 is ~2.5x above the worst observed run and fails any >= ~2.7x slowdown of the decision path.+const RATIO_LIMIT = 15;+// Backstop for the documented budget: the MEDIAN decision stays far below 100 us (measured ~6 us).+const MEDIAN_MICROS_LIMIT = 100;++// Statistical, timing-based gate: one retry absorbs a run that overlapped a+// scheduler stall; a real regression fails every attempt.+it('reports baseline and candidate resource admission overhead with frozen workload', { retry: 1 }, () => {const seeds = [11, 29, 47, 71, 101];- const candidateBatchMicros: number[] = [];- const baselineBatchMicros: number[] = [];+ const batches: ResourceAdmissionInput[][] = [];let candidateCeilingViolations = 0;let baselineCeilingViolations = 0;let admits = 0;@@ -71,6 +75,7 @@ it('reports baseline and candidate resource admission overhead with frozen workllet rejects = 0;let samples = 0;+ // Frozen, deterministic workload; fixtures are built outside the timed region.for (const seed of seeds) {let state = seed >>> 0;for (let batch = 0; batch < 20; batch += 1) {@@ -79,29 +84,33 @@ it('reports baseline and candidate resource admission overhead with frozen worklstate = (Math.imul(state, 1664525) + 1013904223) >>> 0;inputs.push(inputFor(seed, batch, index, state / 0xffffffff));}+ batches.push(inputs);+ }+ }- let start = performance.now();- for (const input of inputs) {- const cpu = fixedCpuBaseline(input);- if (cpu > input.host.maxCpuPerTask || cpu > input.policy.maxCpuPerTask) baselineCeilingViolations += 1;- }- baselineBatchMicros.push(((performance.now() - start) * 1000) / inputs.length);-- start = performance.now();- for (const input of inputs) {- const decision = decideResourceAdmission(input);- samples += 1;- if (decision.verdict === 'admit') admits += 1;- if (decision.verdict === 'defer') defers += 1;- if (decision.verdict === 'reject') rejects += 1;- if (decision.recommendedCpuCores > input.host.maxCpuPerTask || decision.recommendedCpuCores > input.policy.maxCpuPerTask) {- candidateCeilingViolations += 1;- }+ // Correctness half (not timing): verdict mix and CPU ceilings.+ for (const inputs of batches) {+ for (const input of inputs) {+ const cpu = fixedCpuBaseline(input);+ if (cpu > input.host.maxCpuPerTask || cpu > input.policy.maxCpuPerTask) baselineCeilingViolations += 1;+ const decision = decideResourceAdmission(input);+ samples += 1;+ if (decision.verdict === 'admit') admits += 1;+ if (decision.verdict === 'defer') defers += 1;+ if (decision.verdict === 'reject') rejects += 1;+ if (decision.recommendedCpuCores > input.host.maxCpuPerTask || decision.recommendedCpuCores > input.policy.maxCpuPerTask) {+ candidateCeilingViolations += 1;}- candidateBatchMicros.push(((performance.now() - start) * 1000) / inputs.length);}}+ // Timing half: warm up (JIT), then interleave each batch with the fixed reference workload.+ const options = { batches: batches.length, unitsPerBatch: 100, warmupBatches: 20 };+ const baselineTiming = pairedTiming((index) => { for (const input of batches[index]) fixedCpuBaseline(input); }, options);+ const candidateTiming = pairedTiming((index) => { for (const input of batches[index]) decideResourceAdmission(input); }, options);+ const baselineBatchMicros = baselineTiming.batchMicros;+ const candidateBatchMicros = candidateTiming.batchMicros;+const baselineMean = mean(baselineBatchMicros);const candidateMean = mean(candidateBatchMicros);const report = {@@ -113,6 +122,7 @@ it('reports baseline and candidate resource admission overhead with frozen worklbaseline: {name: 'fixed_cpu_clipped_to_host_ceiling',meanMicrosPerDecision: baselineMean,+ medianMicrosPerDecision: baselineTiming.medianMicros,p95MicrosPerDecision: percentile(baselineBatchMicros, 0.95),varianceMicros2: variance(baselineBatchMicros),ceilingViolations: baselineCeilingViolations,@@ -120,8 +130,10 @@ it('reports baseline and candidate resource admission overhead with frozen worklcandidate: {name: 'task_conditioned_resource_admission',meanMicrosPerDecision: candidateMean,+ medianMicrosPerDecision: candidateTiming.medianMicros,p95MicrosPerDecision: percentile(candidateBatchMicros, 0.95),varianceMicros2: variance(candidateBatchMicros),+ medianRatioToReference: candidateTiming.medianRatio,ceilingViolations: candidateCeilingViolations,admits,defers,@@ -140,5 +152,6 @@ it('reports baseline and candidate resource admission overhead with frozen worklconsole.log(JSON.stringify(report));expect(samples).toBe(10_000);expect(candidateCeilingViolations).toBe(0);- expect(report.candidate.p95MicrosPerDecision).toBeLessThan(100);+ expect(candidateTiming.medianRatio).toBeLessThan(RATIO_LIMIT);+ expect(candidateTiming.medianMicros).toBeLessThan(MEDIAN_MICROS_LIMIT);});diff --git a/v3/@claude-flow/performance/__tests__/resource-admission.test.ts b/v3/@claude-flow/performance/__tests__/resource-admission.test.tsindex 6d72836606..6ac7b9bad0 100644--- a/v3/@claude-flow/performance/__tests__/resource-admission.test.ts+++ b/v3/@claude-flow/performance/__tests__/resource-admission.test.ts@@ -1,6 +1,14 @@import { describe, expect, it } from 'vitest';+import { pairedTiming } from './timing-utils.js';import { decideResourceAdmission, type ResourceAdmissionInput } from '../src/resource-admission.js';+// Gate on median(decision cost / reference cost), measured in-process (see timing-utils.ts).+// Measured over 30+ runs under a 32-process CPU stressor: ratio 5.3-6.2 (decision ~6 us, reference ~1.1 us).+// 15 is ~2.5x above the worst observed run and fails any >= ~2.7x slowdown of the decision path.+const RATIO_LIMIT = 15;+// Backstop for the documented budget: the MEDIAN decision stays far below 100 us (measured ~6 us).+const MEDIAN_MICROS_LIMIT = 100;+const now = '2026-09-21T14:00:00.000Z';const digest = 'a'.repeat(64);@@ -140,12 +148,15 @@ describe('resource admission policy', () => {});describe('resource admission structural benchmark', () => {- it('keeps p95 policy decision overhead below 100 microseconds', () => {+ // Statistical, timing-based gate: one retry absorbs a run that overlapped a+ // scheduler stall; a real regression fails every attempt (see timing-utils.ts).+ it('keeps policy decision overhead bounded relative to an in-process reference', { retry: 1 }, () => {const seeds = [11, 29, 47, 71, 101];- const batchMicros: number[] = [];+ const batches: ResourceAdmissionInput[][] = [];let ceilingViolations = 0;let decisions = 0;+ // Frozen, deterministic workload; fixtures are built outside the timed region.for (const seed of seeds) {let state = seed >>> 0;for (let batch = 0; batch < 20; batch += 1) {@@ -170,27 +181,37 @@ describe('resource admission structural benchmark', () => {},}));}- const start = performance.now();- for (const input of inputs) {- const result = decideResourceAdmission(input);- decisions += 1;- if (result.recommendedCpuCores > input.host.maxCpuPerTask || result.recommendedCpuCores > input.policy.maxCpuPerTask) {- ceilingViolations += 1;- }+ batches.push(inputs);+ }+ }++ // Correctness half (not timing): every decision respects the CPU ceilings.+ for (const inputs of batches) {+ for (const input of inputs) {+ const result = decideResourceAdmission(input);+ decisions += 1;+ if (result.recommendedCpuCores > input.host.maxCpuPerTask || result.recommendedCpuCores > input.policy.maxCpuPerTask) {+ ceilingViolations += 1;}- batchMicros.push(((performance.now() - start) * 1000) / inputs.length);}}- batchMicros.sort((a, b) => a - b);- const p95 = batchMicros[Math.floor(batchMicros.length * 0.95)];+ // Timing half: warm up (JIT), then interleave decision batches with the+ // fixed reference workload so machine load cancels out of the ratio.+ const timing = pairedTiming(+ (index) => { for (const input of batches[index]) decideResourceAdmission(input); },+ { batches: batches.length, unitsPerBatch: 100, warmupBatches: 20 },+ );console.log(JSON.stringify({schema: 'ruflo-resource-admission-benchmark/v1',workload: '10000 deterministic synthetic resource decisions',seeds,sampleSize: decisions,metric: 'microseconds_per_decision',- p95,+ medianMicros: timing.medianMicros,+ referenceMedianMicros: timing.referenceMedianMicros,+ medianRatio: timing.medianRatio,+ p95: timing.p95Micros,ceilingViolations,authorityExpansion: 0,energy: 'not_measured',@@ -198,6 +219,7 @@ describe('resource admission structural benchmark', () => {}));expect(decisions).toBe(10_000);expect(ceilingViolations).toBe(0);- expect(p95).toBeLessThan(100);+ expect(timing.medianRatio).toBeLessThan(RATIO_LIMIT);+ expect(timing.medianMicros).toBeLessThan(MEDIAN_MICROS_LIMIT);});});diff --git a/v3/@claude-flow/performance/__tests__/timing-utils.ts b/v3/@claude-flow/performance/__tests__/timing-utils.tsnew file mode 100644index 0000000000..5daba20917--- /dev/null+++ b/v3/@claude-flow/performance/__tests__/timing-utils.ts@@ -0,0 +1,103 @@+/**+ * Noise-robust timing helpers for the performance package's CI tests.+ *+ * Why this exists: CI runs the whole repo's test files in parallel on shared+ * runners, so absolute wall-clock thresholds ("p95 < 100 us") are measuring+ * the machine's momentary load as much as the code. Two defences are used:+ *+ * 1. Relative measurement. Every timed batch of the code under test is paired+ * with a batch of a fixed, independent reference workload, run back to+ * back (order alternates so neither side systematically goes first). Both+ * halves see the same CPU contention, so the per-batch ratio cancels it.+ * The reference does NOT call the code under test, so a slowdown of the+ * code under test still moves the ratio.+ * 2. Robust statistics. The gate is the MEDIAN of the per-batch ratios after+ * an untimed warm-up, not a tail percentile, so a few batches that were+ * pre-empted mid-run (or hit JIT/GC) cannot decide the outcome.+ */++const REF_PARTS = ['profile.ref', 'tool.ref', 'a'.repeat(64), 'host.ref', 'b'.repeat(64), '2026-09-21T13:59:30.000Z'];+let refSink = 0;++/** Fixed workload with the same flavour as the code under test (string build + hash + date parse). */+export function referenceUnit(n: number): void {+ const material = `${REF_PARTS.join('|')}|${n}`;+ let a = 2166136261 >>> 0;+ let b = 2246822519 >>> 0;+ for (let i = 0; i < material.length; i += 1) {+ const c = material.charCodeAt(i);+ a = Math.imul(a ^ c, 16777619) >>> 0;+ b = Math.imul(b ^ c, 3266489917) >>> 0;+ }+ refSink ^= a ^ b ^ Date.parse('2026-09-21T14:00:00.000Z');+}++export function median(values: number[]): number {+ const sorted = [...values].sort((x, y) => x - y);+ const mid = sorted.length >> 1;+ return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;+}++export function percentile(values: number[], fraction: number): number {+ const sorted = [...values].sort((x, y) => x - y);+ return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))];+}++export interface PairedTiming {+ /** Median microseconds per unit of the code under test. */+ medianMicros: number;+ /** Median microseconds per unit of the reference workload. */+ referenceMedianMicros: number;+ /** Median of per-batch (code under test / reference). The robust gate. */+ medianRatio: number;+ /** Tail view, reported but not gated: p95 of the per-batch microseconds. */+ p95Micros: number;+ batchMicros: number[];+ ratios: number[];+}++/**+ * Time `runBatch(i)` (which must process `unitsPerBatch` units) against the+ * reference workload, interleaved, after `warmupBatches` untimed batches.+ */+export function pairedTiming(+ runBatch: (batchIndex: number) => void,+ options: { batches: number; unitsPerBatch: number; warmupBatches: number },+): PairedTiming {+ const { batches, unitsPerBatch, warmupBatches } = options;+ for (let w = 0; w < warmupBatches; w += 1) {+ runBatch(w % batches);+ for (let n = 0; n < unitsPerBatch; n += 1) referenceUnit(n);+ }++ const batchMicros: number[] = [];+ const refMicros: number[] = [];+ const ratios: number[] = [];+ for (let batch = 0; batch < batches; batch += 1) {+ let candidate = 0;+ let reference = 0;+ const timeCandidate = () => {+ const start = performance.now();+ runBatch(batch);+ candidate = ((performance.now() - start) * 1000) / unitsPerBatch;+ };+ const timeReference = () => {+ const start = performance.now();+ for (let n = 0; n < unitsPerBatch; n += 1) referenceUnit(n);+ reference = ((performance.now() - start) * 1000) / unitsPerBatch;+ };+ if (batch % 2 === 0) { timeCandidate(); timeReference(); } else { timeReference(); timeCandidate(); }+ batchMicros.push(candidate);+ refMicros.push(reference);+ ratios.push(candidate / Math.max(reference, Number.EPSILON));+ }++ return {+ medianMicros: median(batchMicros),+ referenceMedianMicros: median(refMicros),+ medianRatio: median(ratios),+ p95Micros: percentile(batchMicros, 0.95),+ batchMicros,+ ratios,+ };+}diff --git a/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.ts b/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.tsindex 9c4a83aad4..a12662890b 100644--- a/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.ts+++ b/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.ts@@ -12,6 +12,8 @@import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';import { Readable, Writable, Transform } from 'stream';+import v8 from 'node:v8';+import vm from 'node:vm';import { pipeline } from 'stream/promises';import {randomVector,@@ -23,6 +25,32 @@ import {type MockPgPool,} from './utils/ruvector-test-utils.js';+// Heap retention helpers (see 'should release references after streaming').+// Measured with forced GC: a clean pipeline retains ~0 MB (|x| < 0.2 MB); one that keeps every+// chunk reachable retains ~16.2 MB. 8 MB sits ~50x above noise and at half the leak.+const RETAINED_LIMIT_BYTES = 8 * 1024 * 1024;+const UNCOLLECTED_LIMIT_BYTES = 50 * 1024 * 1024;++/** A forced-GC function: the global one if run with --expose-gc, else obtained at runtime. */+function getGc(): (() => void) | undefined {+ if (typeof globalThis.gc === 'function') return globalThis.gc;+ try {+ v8.setFlagsFromString('--expose-gc');+ return vm.runInNewContext('gc') as () => void;+ } catch {+ return undefined;+ }+}++/** Heap in use after repeated full collections with a short settle between them. */+async function settledHeapUsed(collect: (() => void) | undefined): Promise<number> {+ for (let i = 0; i < 4; i += 1) {+ collect?.();+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ return process.memoryUsage().heapUsed;+}+// ============================================================================// Stream Utilities// ============================================================================@@ -968,18 +996,21 @@ describe('RuVector Streaming', () => {});it('should release references after streaming', async () => {- // Force GC if available- if (global.gc) {- global.gc();- }-- const memBefore = process.memoryUsage().heapUsed;-- // Process a large stream- const generator = createVectorGeneratorStream(5000, 384);-+ // This guards against a stream pipeline that keeps the chunks it has+ // processed reachable after it has finished (a leak). Raw `heapUsed`+ // before/after, without a forced collection, cannot show that: it counts+ // garbage that simply has not been collected yet (and garbage from+ // earlier tests in this worker), and a collection that frees it made the+ // old `Math.abs` check fail too (observed 273 MB / 159 MB in CI). So+ // measure RETAINED heap: force full collections, let finalizers settle,+ // and compare signed before/after.+ const collect = getGc();++ const memBefore = await settledHeapUsed(collect);++ // Process a large stream: 5000 x 384-dim vectors, ~16 MB if all retained.await pipeline(- generator,+ createVectorGeneratorStream(5000, 384),new Writable({objectMode: true,write(chunk, encoding, callback) {@@ -988,16 +1019,17 @@ describe('RuVector Streaming', () => {}));- // Force GC if available- if (global.gc) {- global.gc();- }-- const memAfter = process.memoryUsage().heapUsed;+ const memAfter = await settledHeapUsed(collect);+ const retained = memAfter - memBefore;+ console.log(JSON.stringify({ retainedMB: retained / 1048576, gc: collect !== undefined }));- // Memory growth should be reasonable (allow 50MB variance for test environment)- // In production with proper GC, this would be much lower- expect(Math.abs(memAfter - memBefore)).toBeLessThan(50 * 1024 * 1024);+ if (collect) {+ expect(retained).toBeLessThan(RETAINED_LIMIT_BYTES);+ } else {+ // No way to force a collection on this runtime: the number is only+ // meaningful as a gross-leak backstop.+ expect(Math.abs(retained)).toBeLessThan(UNCOLLECTED_LIMIT_BYTES);+ }});});