Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

test: make the timing-sensitive CI tests robust

ruv3438b4085e6f
diff --git a/v3/@claude-flow/performance/__tests__/benchmarks.test.ts b/v3/@claude-flow/performance/__tests__/benchmarks.test.ts
index a1361347c0..1dbe632554 100644
--- a/v3/@claude-flow/performance/__tests__/benchmarks.test.ts
+++ b/v3/@claude-flow/performance/__tests__/benchmarks.test.ts
@@ -418,15 +418,34 @@ describe('quickValidation()', () => {
});
});
+const RUNS_PER_MEASUREMENT = 15;
+
describe('Performance Validation', () => {
+ // Statistical, timing-based test. The old form compared two single runs, each
+ // of which times a Flash block and then a baseline block with no warm-up, so a
+ // cold JIT or one scheduler stall landing on one block of one run swung the
+ // ratio past 2.0 (observed 2.23 in CI). What it guards is that the speedup
+ // measurement is reproducible, so keep that exact claim ("two measurements
+ // agree within 2x") but make each measurement robust: warm up, then take two
+ // independent measurements that are each the MEDIAN of several runs, with the
+ // runs of the two measurements interleaved so slow drift in machine load hits
+ // both equally. A measurement that is genuinely irreproducible still fails.
it('should demonstrate consistent speedup', () => {
const runner = new AttentionBenchmarkRunner();
- const result1 = runner.runComparison(256, 50, 100);
- const result2 = runner.runComparison(256, 50, 100);
+ for (let i = 0; i < 3; i++) runner.runComparison(256, 50, 100); // warm-up, discarded
+
+ const a: number[] = [];
+ const b: number[] = [];
+ for (let i = 0; i < RUNS_PER_MEASUREMENT; i++) {
+ a.push(runner.runComparison(256, 50, 100).results.speedup);
+ b.push(runner.runComparison(256, 50, 100).results.speedup);
+ }
+ const median = (xs: number[]) => [...xs].sort((x, y) => x - y)[xs.length >> 1];
// Speedup should be relatively consistent (within 50% variance)
- const ratio = result1.results.speedup / result2.results.speedup;
+ const ratio = median(a) / median(b);
+ console.log(JSON.stringify({ medianA: median(a), medianB: median(b), medianRatio: ratio }));
expect(ratio).toBeGreaterThan(0.5);
expect(ratio).toBeLessThan(2.0);
});
diff --git a/v3/@claude-flow/performance/__tests__/resource-admission-report.test.ts b/v3/@claude-flow/performance/__tests__/resource-admission-report.test.ts
index 58213e3aa1..b6f058342e 100644
--- a/v3/@claude-flow/performance/__tests__/resource-admission-report.test.ts
+++ b/v3/@claude-flow/performance/__tests__/resource-admission-report.test.ts
@@ -1,4 +1,5 @@
import { expect, it } from 'vitest';
+import { pairedTiming, percentile } from './timing-utils.js';
import { decideResourceAdmission, type ResourceAdmissionInput } from '../src/resource-admission.js';
const now = '2026-09-21T14:00:00.000Z';
@@ -46,11 +47,6 @@ function fixedCpuBaseline(input: ResourceAdmissionInput): number {
return Math.max(0, Math.min(4, input.profile.maxCpuCores, input.host.maxCpuPerTask, input.policy.maxCpuPerTask, share));
}
-function percentile(values: number[], fraction: number): number {
- const sorted = [...values].sort((a, b) => a - b);
- return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))];
-}
-
function mean(values: number[]): number {
return values.reduce((sum, value) => sum + value, 0) / values.length;
}
@@ -60,10 +56,18 @@ function variance(values: number[]): number {
return values.reduce((sum, value) => sum + (value - center) ** 2, 0) / values.length;
}
-it('reports baseline and candidate resource admission overhead with frozen workload', () => {
+// Gate on median(decision cost / reference cost), measured in-process (see timing-utils.ts).
+// Measured over 30+ runs under a 32-process CPU stressor: ratio 5.3-6.2 (decision ~6 us, reference ~1.1 us).
+// 15 is ~2.5x above the worst observed run and fails any >= ~2.7x slowdown of the decision path.
+const RATIO_LIMIT = 15;
+// Backstop for the documented budget: the MEDIAN decision stays far below 100 us (measured ~6 us).
+const MEDIAN_MICROS_LIMIT = 100;
+
+// Statistical, timing-based gate: one retry absorbs a run that overlapped a
+// scheduler stall; a real regression fails every attempt.
+it('reports baseline and candidate resource admission overhead with frozen workload', { retry: 1 }, () => {
const seeds = [11, 29, 47, 71, 101];
- const candidateBatchMicros: number[] = [];
- const baselineBatchMicros: number[] = [];
+ const batches: ResourceAdmissionInput[][] = [];
let candidateCeilingViolations = 0;
let baselineCeilingViolations = 0;
let admits = 0;
@@ -71,6 +75,7 @@ it('reports baseline and candidate resource admission overhead with frozen workl
let rejects = 0;
let samples = 0;
+ // Frozen, deterministic workload; fixtures are built outside the timed region.
for (const seed of seeds) {
let state = seed >>> 0;
for (let batch = 0; batch < 20; batch += 1) {
@@ -79,29 +84,33 @@ it('reports baseline and candidate resource admission overhead with frozen workl
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
inputs.push(inputFor(seed, batch, index, state / 0xffffffff));
}
+ batches.push(inputs);
+ }
+ }
- let start = performance.now();
- for (const input of inputs) {
- const cpu = fixedCpuBaseline(input);
- if (cpu > input.host.maxCpuPerTask || cpu > input.policy.maxCpuPerTask) baselineCeilingViolations += 1;
- }
- baselineBatchMicros.push(((performance.now() - start) * 1000) / inputs.length);
-
- start = performance.now();
- for (const input of inputs) {
- const decision = decideResourceAdmission(input);
- samples += 1;
- if (decision.verdict === 'admit') admits += 1;
- if (decision.verdict === 'defer') defers += 1;
- if (decision.verdict === 'reject') rejects += 1;
- if (decision.recommendedCpuCores > input.host.maxCpuPerTask || decision.recommendedCpuCores > input.policy.maxCpuPerTask) {
- candidateCeilingViolations += 1;
- }
+ // Correctness half (not timing): verdict mix and CPU ceilings.
+ for (const inputs of batches) {
+ for (const input of inputs) {
+ const cpu = fixedCpuBaseline(input);
+ if (cpu > input.host.maxCpuPerTask || cpu > input.policy.maxCpuPerTask) baselineCeilingViolations += 1;
+ const decision = decideResourceAdmission(input);
+ samples += 1;
+ if (decision.verdict === 'admit') admits += 1;
+ if (decision.verdict === 'defer') defers += 1;
+ if (decision.verdict === 'reject') rejects += 1;
+ if (decision.recommendedCpuCores > input.host.maxCpuPerTask || decision.recommendedCpuCores > input.policy.maxCpuPerTask) {
+ candidateCeilingViolations += 1;
}
- candidateBatchMicros.push(((performance.now() - start) * 1000) / inputs.length);
}
}
+ // Timing half: warm up (JIT), then interleave each batch with the fixed reference workload.
+ const options = { batches: batches.length, unitsPerBatch: 100, warmupBatches: 20 };
+ const baselineTiming = pairedTiming((index) => { for (const input of batches[index]) fixedCpuBaseline(input); }, options);
+ const candidateTiming = pairedTiming((index) => { for (const input of batches[index]) decideResourceAdmission(input); }, options);
+ const baselineBatchMicros = baselineTiming.batchMicros;
+ const candidateBatchMicros = candidateTiming.batchMicros;
+
const baselineMean = mean(baselineBatchMicros);
const candidateMean = mean(candidateBatchMicros);
const report = {
@@ -113,6 +122,7 @@ it('reports baseline and candidate resource admission overhead with frozen workl
baseline: {
name: 'fixed_cpu_clipped_to_host_ceiling',
meanMicrosPerDecision: baselineMean,
+ medianMicrosPerDecision: baselineTiming.medianMicros,
p95MicrosPerDecision: percentile(baselineBatchMicros, 0.95),
varianceMicros2: variance(baselineBatchMicros),
ceilingViolations: baselineCeilingViolations,
@@ -120,8 +130,10 @@ it('reports baseline and candidate resource admission overhead with frozen workl
candidate: {
name: 'task_conditioned_resource_admission',
meanMicrosPerDecision: candidateMean,
+ medianMicrosPerDecision: candidateTiming.medianMicros,
p95MicrosPerDecision: percentile(candidateBatchMicros, 0.95),
varianceMicros2: variance(candidateBatchMicros),
+ medianRatioToReference: candidateTiming.medianRatio,
ceilingViolations: candidateCeilingViolations,
admits,
defers,
@@ -140,5 +152,6 @@ it('reports baseline and candidate resource admission overhead with frozen workl
console.log(JSON.stringify(report));
expect(samples).toBe(10_000);
expect(candidateCeilingViolations).toBe(0);
- expect(report.candidate.p95MicrosPerDecision).toBeLessThan(100);
+ expect(candidateTiming.medianRatio).toBeLessThan(RATIO_LIMIT);
+ expect(candidateTiming.medianMicros).toBeLessThan(MEDIAN_MICROS_LIMIT);
});
diff --git a/v3/@claude-flow/performance/__tests__/resource-admission.test.ts b/v3/@claude-flow/performance/__tests__/resource-admission.test.ts
index 6d72836606..6ac7b9bad0 100644
--- a/v3/@claude-flow/performance/__tests__/resource-admission.test.ts
+++ b/v3/@claude-flow/performance/__tests__/resource-admission.test.ts
@@ -1,6 +1,14 @@
import { describe, expect, it } from 'vitest';
+import { pairedTiming } from './timing-utils.js';
import { decideResourceAdmission, type ResourceAdmissionInput } from '../src/resource-admission.js';
+// Gate on median(decision cost / reference cost), measured in-process (see timing-utils.ts).
+// Measured over 30+ runs under a 32-process CPU stressor: ratio 5.3-6.2 (decision ~6 us, reference ~1.1 us).
+// 15 is ~2.5x above the worst observed run and fails any >= ~2.7x slowdown of the decision path.
+const RATIO_LIMIT = 15;
+// Backstop for the documented budget: the MEDIAN decision stays far below 100 us (measured ~6 us).
+const MEDIAN_MICROS_LIMIT = 100;
+
const now = '2026-09-21T14:00:00.000Z';
const digest = 'a'.repeat(64);
@@ -140,12 +148,15 @@ describe('resource admission policy', () => {
});
describe('resource admission structural benchmark', () => {
- it('keeps p95 policy decision overhead below 100 microseconds', () => {
+ // Statistical, timing-based gate: one retry absorbs a run that overlapped a
+ // scheduler stall; a real regression fails every attempt (see timing-utils.ts).
+ it('keeps policy decision overhead bounded relative to an in-process reference', { retry: 1 }, () => {
const seeds = [11, 29, 47, 71, 101];
- const batchMicros: number[] = [];
+ const batches: ResourceAdmissionInput[][] = [];
let ceilingViolations = 0;
let decisions = 0;
+ // Frozen, deterministic workload; fixtures are built outside the timed region.
for (const seed of seeds) {
let state = seed >>> 0;
for (let batch = 0; batch < 20; batch += 1) {
@@ -170,27 +181,37 @@ describe('resource admission structural benchmark', () => {
},
}));
}
- const start = performance.now();
- for (const input of inputs) {
- const result = decideResourceAdmission(input);
- decisions += 1;
- if (result.recommendedCpuCores > input.host.maxCpuPerTask || result.recommendedCpuCores > input.policy.maxCpuPerTask) {
- ceilingViolations += 1;
- }
+ batches.push(inputs);
+ }
+ }
+
+ // Correctness half (not timing): every decision respects the CPU ceilings.
+ for (const inputs of batches) {
+ for (const input of inputs) {
+ const result = decideResourceAdmission(input);
+ decisions += 1;
+ if (result.recommendedCpuCores > input.host.maxCpuPerTask || result.recommendedCpuCores > input.policy.maxCpuPerTask) {
+ ceilingViolations += 1;
}
- batchMicros.push(((performance.now() - start) * 1000) / inputs.length);
}
}
- batchMicros.sort((a, b) => a - b);
- const p95 = batchMicros[Math.floor(batchMicros.length * 0.95)];
+ // Timing half: warm up (JIT), then interleave decision batches with the
+ // fixed reference workload so machine load cancels out of the ratio.
+ const timing = pairedTiming(
+ (index) => { for (const input of batches[index]) decideResourceAdmission(input); },
+ { batches: batches.length, unitsPerBatch: 100, warmupBatches: 20 },
+ );
console.log(JSON.stringify({
schema: 'ruflo-resource-admission-benchmark/v1',
workload: '10000 deterministic synthetic resource decisions',
seeds,
sampleSize: decisions,
metric: 'microseconds_per_decision',
- p95,
+ medianMicros: timing.medianMicros,
+ referenceMedianMicros: timing.referenceMedianMicros,
+ medianRatio: timing.medianRatio,
+ p95: timing.p95Micros,
ceilingViolations,
authorityExpansion: 0,
energy: 'not_measured',
@@ -198,6 +219,7 @@ describe('resource admission structural benchmark', () => {
}));
expect(decisions).toBe(10_000);
expect(ceilingViolations).toBe(0);
- expect(p95).toBeLessThan(100);
+ expect(timing.medianRatio).toBeLessThan(RATIO_LIMIT);
+ expect(timing.medianMicros).toBeLessThan(MEDIAN_MICROS_LIMIT);
});
});
diff --git a/v3/@claude-flow/performance/__tests__/timing-utils.ts b/v3/@claude-flow/performance/__tests__/timing-utils.ts
new file mode 100644
index 0000000000..5daba20917
--- /dev/null
+++ b/v3/@claude-flow/performance/__tests__/timing-utils.ts
@@ -0,0 +1,103 @@
+/**
+ * Noise-robust timing helpers for the performance package's CI tests.
+ *
+ * Why this exists: CI runs the whole repo's test files in parallel on shared
+ * runners, so absolute wall-clock thresholds ("p95 < 100 us") are measuring
+ * the machine's momentary load as much as the code. Two defences are used:
+ *
+ * 1. Relative measurement. Every timed batch of the code under test is paired
+ * with a batch of a fixed, independent reference workload, run back to
+ * back (order alternates so neither side systematically goes first). Both
+ * halves see the same CPU contention, so the per-batch ratio cancels it.
+ * The reference does NOT call the code under test, so a slowdown of the
+ * code under test still moves the ratio.
+ * 2. Robust statistics. The gate is the MEDIAN of the per-batch ratios after
+ * an untimed warm-up, not a tail percentile, so a few batches that were
+ * pre-empted mid-run (or hit JIT/GC) cannot decide the outcome.
+ */
+
+const REF_PARTS = ['profile.ref', 'tool.ref', 'a'.repeat(64), 'host.ref', 'b'.repeat(64), '2026-09-21T13:59:30.000Z'];
+let refSink = 0;
+
+/** Fixed workload with the same flavour as the code under test (string build + hash + date parse). */
+export function referenceUnit(n: number): void {
+ const material = `${REF_PARTS.join('|')}|${n}`;
+ let a = 2166136261 >>> 0;
+ let b = 2246822519 >>> 0;
+ for (let i = 0; i < material.length; i += 1) {
+ const c = material.charCodeAt(i);
+ a = Math.imul(a ^ c, 16777619) >>> 0;
+ b = Math.imul(b ^ c, 3266489917) >>> 0;
+ }
+ refSink ^= a ^ b ^ Date.parse('2026-09-21T14:00:00.000Z');
+}
+
+export function median(values: number[]): number {
+ const sorted = [...values].sort((x, y) => x - y);
+ const mid = sorted.length >> 1;
+ return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+
+export function percentile(values: number[], fraction: number): number {
+ const sorted = [...values].sort((x, y) => x - y);
+ return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))];
+}
+
+export interface PairedTiming {
+ /** Median microseconds per unit of the code under test. */
+ medianMicros: number;
+ /** Median microseconds per unit of the reference workload. */
+ referenceMedianMicros: number;
+ /** Median of per-batch (code under test / reference). The robust gate. */
+ medianRatio: number;
+ /** Tail view, reported but not gated: p95 of the per-batch microseconds. */
+ p95Micros: number;
+ batchMicros: number[];
+ ratios: number[];
+}
+
+/**
+ * Time `runBatch(i)` (which must process `unitsPerBatch` units) against the
+ * reference workload, interleaved, after `warmupBatches` untimed batches.
+ */
+export function pairedTiming(
+ runBatch: (batchIndex: number) => void,
+ options: { batches: number; unitsPerBatch: number; warmupBatches: number },
+): PairedTiming {
+ const { batches, unitsPerBatch, warmupBatches } = options;
+ for (let w = 0; w < warmupBatches; w += 1) {
+ runBatch(w % batches);
+ for (let n = 0; n < unitsPerBatch; n += 1) referenceUnit(n);
+ }
+
+ const batchMicros: number[] = [];
+ const refMicros: number[] = [];
+ const ratios: number[] = [];
+ for (let batch = 0; batch < batches; batch += 1) {
+ let candidate = 0;
+ let reference = 0;
+ const timeCandidate = () => {
+ const start = performance.now();
+ runBatch(batch);
+ candidate = ((performance.now() - start) * 1000) / unitsPerBatch;
+ };
+ const timeReference = () => {
+ const start = performance.now();
+ for (let n = 0; n < unitsPerBatch; n += 1) referenceUnit(n);
+ reference = ((performance.now() - start) * 1000) / unitsPerBatch;
+ };
+ if (batch % 2 === 0) { timeCandidate(); timeReference(); } else { timeReference(); timeCandidate(); }
+ batchMicros.push(candidate);
+ refMicros.push(reference);
+ ratios.push(candidate / Math.max(reference, Number.EPSILON));
+ }
+
+ return {
+ medianMicros: median(batchMicros),
+ referenceMedianMicros: median(refMicros),
+ medianRatio: median(ratios),
+ p95Micros: percentile(batchMicros, 0.95),
+ batchMicros,
+ ratios,
+ };
+}
diff --git a/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.ts b/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.ts
index 9c4a83aad4..a12662890b 100644
--- a/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.ts
+++ b/v3/@claude-flow/plugins/__tests__/ruvector-streaming.test.ts
@@ -12,6 +12,8 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { Readable, Writable, Transform } from 'stream';
+import v8 from 'node:v8';
+import vm from 'node:vm';
import { pipeline } from 'stream/promises';
import {
randomVector,
@@ -23,6 +25,32 @@ import {
type MockPgPool,
} from './utils/ruvector-test-utils.js';
+// Heap retention helpers (see 'should release references after streaming').
+// Measured with forced GC: a clean pipeline retains ~0 MB (|x| < 0.2 MB); one that keeps every
+// chunk reachable retains ~16.2 MB. 8 MB sits ~50x above noise and at half the leak.
+const RETAINED_LIMIT_BYTES = 8 * 1024 * 1024;
+const UNCOLLECTED_LIMIT_BYTES = 50 * 1024 * 1024;
+
+/** A forced-GC function: the global one if run with --expose-gc, else obtained at runtime. */
+function getGc(): (() => void) | undefined {
+ if (typeof globalThis.gc === 'function') return globalThis.gc;
+ try {
+ v8.setFlagsFromString('--expose-gc');
+ return vm.runInNewContext('gc') as () => void;
+ } catch {
+ return undefined;
+ }
+}
+
+/** Heap in use after repeated full collections with a short settle between them. */
+async function settledHeapUsed(collect: (() => void) | undefined): Promise<number> {
+ for (let i = 0; i < 4; i += 1) {
+ collect?.();
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ return process.memoryUsage().heapUsed;
+}
+
// ============================================================================
// Stream Utilities
// ============================================================================
@@ -968,18 +996,21 @@ describe('RuVector Streaming', () => {
});
it('should release references after streaming', async () => {
- // Force GC if available
- if (global.gc) {
- global.gc();
- }
-
- const memBefore = process.memoryUsage().heapUsed;
-
- // Process a large stream
- const generator = createVectorGeneratorStream(5000, 384);
-
+ // This guards against a stream pipeline that keeps the chunks it has
+ // processed reachable after it has finished (a leak). Raw `heapUsed`
+ // before/after, without a forced collection, cannot show that: it counts
+ // garbage that simply has not been collected yet (and garbage from
+ // earlier tests in this worker), and a collection that frees it made the
+ // old `Math.abs` check fail too (observed 273 MB / 159 MB in CI). So
+ // measure RETAINED heap: force full collections, let finalizers settle,
+ // and compare signed before/after.
+ const collect = getGc();
+
+ const memBefore = await settledHeapUsed(collect);
+
+ // Process a large stream: 5000 x 384-dim vectors, ~16 MB if all retained.
await pipeline(
- generator,
+ createVectorGeneratorStream(5000, 384),
new Writable({
objectMode: true,
write(chunk, encoding, callback) {
@@ -988,16 +1019,17 @@ describe('RuVector Streaming', () => {
})
);
- // Force GC if available
- if (global.gc) {
- global.gc();
- }
-
- const memAfter = process.memoryUsage().heapUsed;
+ const memAfter = await settledHeapUsed(collect);
+ const retained = memAfter - memBefore;
+ console.log(JSON.stringify({ retainedMB: retained / 1048576, gc: collect !== undefined }));
- // Memory growth should be reasonable (allow 50MB variance for test environment)
- // In production with proper GC, this would be much lower
- expect(Math.abs(memAfter - memBefore)).toBeLessThan(50 * 1024 * 1024);
+ if (collect) {
+ expect(retained).toBeLessThan(RETAINED_LIMIT_BYTES);
+ } else {
+ // No way to force a collection on this runtime: the number is only
+ // meaningful as a gross-leak backstop.
+ expect(Math.abs(retained)).toBeLessThan(UNCOLLECTED_LIMIT_BYTES);
+ }
});
});