AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
import { describe, expect, it } from 'vitest';
import { benchmarkUtils } from './benchmark';
const baseOpts = { url: 'http://localhost:3000/', requests: '10', concurrency: '2', apiKey: 'k', body: '{"x":1}' };
describe('benchmarkUtils.normalizeOptions', () => {
it('parses valid options and strips the trailing slash', () => {
const config = benchmarkUtils.normalizeOptions(baseOpts);
expect(config.url).toBe('http://localhost:3000');
expect(config.requests).toBe(10);
expect(config.concurrency).toBe(2);
expect(config.apiKey).toBe('k');
expect(config.body).toBe('{"x":1}');
});
it('leaves auth and load fields undefined when not provided', () => {
const config = benchmarkUtils.normalizeOptions({ url: 'http://x', body: '{}' });
expect(config.apiKey).toBeUndefined();
expect(config.requests).toBeUndefined();
expect(config.concurrency).toBeUndefined();
});
it('rejects non-positive / non-integer requests and concurrency', () => {
expect(() => benchmarkUtils.normalizeOptions({ ...baseOpts, requests: '0' })).toThrow(/requests/);
expect(() => benchmarkUtils.normalizeOptions({ ...baseOpts, requests: 'abc' })).toThrow(/requests/);
expect(() => benchmarkUtils.normalizeOptions({ ...baseOpts, concurrency: '-1' })).toThrow(/concurrency/);
});
it('rejects a body that is not valid JSON', () => {
expect(() => benchmarkUtils.normalizeOptions({ ...baseOpts, body: 'not-json' })).toThrow(/JSON/);
});
});
describe('benchmarkUtils.toSummary', () => {
it('maps an autocannon result and sums failures', () => {
const result = {
requests: { sent: 100, average: 250 },
latency: { mean: 12, p50: 10, p90: 20, p99: 40, min: 5, max: 90 },
duration: 4,
'2xx': 98,
non2xx: 1,
errors: 1,
timeouts: 0,
};
const s = benchmarkUtils.toSummary({ result, flowId: 'flow123', connections: 10 });
expect(s.throughputReqSec).toBe(250);
expect(s.p99Ms).toBe(40);
expect(s.ok2xx).toBe(98);
expect(s.failed).toBe(2);
});
});
describe('benchmarkUtils.resolvePhases', () => {
it('honours an explicit concurrency override', () => {
const phases = benchmarkUtils.resolvePhases({ concurrency: 7, slots: 3 });
expect(phases).toEqual([{ label: 'conc 7', connections: 7 }]);
});
it('derives a single matched phase from slots', () => {
const phases = benchmarkUtils.resolvePhases({ slots: 4 });
expect(phases).toHaveLength(1);
expect(phases[0].connections).toBe(4);
});
it('falls back to a default when slots are unknown', () => {
const phases = benchmarkUtils.resolvePhases({});
expect(phases[0].connections).toBe(10);
});
});
describe('benchmarkUtils.percentile', () => {
it('returns 0 for an empty set and the right order statistic otherwise', () => {
expect(benchmarkUtils.percentile([], 50)).toBe(0);
expect(benchmarkUtils.percentile([10, 1, 5, 3, 9], 50)).toBe(5);
expect(benchmarkUtils.percentile([10, 1, 5, 3, 9], 90)).toBe(10);
});
});
describe('benchmarkUtils.validateSetup', () => {
const machine = (props: Record<string, string>, cpu: number, ramGb: number) => ({
status: 'ONLINE',
information: { workerProps: props, totalCpuCores: cpu, totalAvailableRamInBytes: ramGb * 1024 * 1024 * 1024 },
});
it('passes the recommended shape and warns on drift', () => {
const good = benchmarkUtils.validateSetup([machine({ EXECUTION_MODE: 'SANDBOX_CODE_ONLY', REUSE_SANDBOX: 'true', WORKER_CONCURRENCY: '1' }, 0.5, 1)]);
expect(good.every((c) => c.status === 'PASS')).toBe(true);
const bad = benchmarkUtils.validateSetup([machine({ EXECUTION_MODE: 'SANDBOX', REUSE_SANDBOX: 'false', WORKER_CONCURRENCY: '5' }, 4, 8)]);
expect(bad.find((c) => c.dimension === 'sandbox mode')?.status).toBe('WARN');
expect(bad.find((c) => c.dimension === 'worker concurrency')?.status).toBe('WARN');
});
it('warns when no workers are connected', () => {
const checks = benchmarkUtils.validateSetup([]);
expect(checks[0].status).toBe('WARN');
});
});
describe('benchmarkUtils.aggregateTimeline', () => {
const run = (queue: number, provision: number, boot: number, runMs: number) => ({
timeline: { legs: [[
{ name: 'QUEUE' as const, durationMs: queue },
{ name: 'PROVISION' as const, durationMs: provision },
{ name: 'BOOT' as const, durationMs: boot },
{ name: 'RUN' as const, durationMs: runMs },
]] },
});
it('splits queue-wait (non-RUN phases) from service (RUN)', () => {
const agg = benchmarkUtils.aggregateTimeline([run(100, 10, 5, 50), run(300, 10, 5, 60)]);
expect(agg.sampleCount).toBe(2);
// nearest-rank p50 of [50, 60] is the lower element (rank 1)
expect(agg.serviceP50).toBe(50);
// queue-wait per run = queue + provision + boot = 115 and 315; p50 = 115
expect(agg.queueWaitP50).toBe(115);
});
it('ignores runs without a timeline', () => {
const agg = benchmarkUtils.aggregateTimeline([{ timeline: null }, run(1, 1, 1, 20)]);
expect(agg.sampleCount).toBe(1);
expect(agg.serviceP50).toBe(20);
});
it('flags runs carrying the rate-limiter backoff delay (QUEUE >= 15s)', () => {
const agg = benchmarkUtils.aggregateTimeline([run(100, 1, 1, 50), run(21_000, 1, 1, 50), run(14_999, 1, 1, 50)]);
expect(agg.rateLimitedRunsCount).toBe(1);
expect(agg.queueMax).toBe(21_000);
});
});
describe('benchmarkUtils.aggregateOutsideRuns', () => {
it('groups runs by flow with count and avg run time, busiest first', () => {
const at = (ms: number) => new Date(ms).toISOString();
const flows = benchmarkUtils.aggregateOutsideRuns([
{ flowId: 'a', projectId: 'p1', startTime: at(0), finishTime: at(100) },
{ flowId: 'a', projectId: 'p1', startTime: at(0), finishTime: at(300) },
{ flowId: 'a', projectId: 'p1' }, // still counted, excluded from avg (no timestamps)
{ flowId: 'b', projectId: 'p2', startTime: at(0), finishTime: at(50) },
{ projectId: 'p2' }, // no flowId — dropped
]);
expect(flows).toEqual([
{ flowId: 'a', projectId: 'p1', runs: 3, avgRunMs: 200 },
{ flowId: 'b', projectId: 'p2', runs: 1, avgRunMs: 50 },
]);
});
});