AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
import os from 'os';
import autocannon from 'autocannon';
import axios, { AxiosInstance, AxiosError } from 'axios';
import chalk from 'chalk';
import { Command } from 'commander';
import type { Project } from '@activepieces/shared';
const BENCHMARK_DOC = 'Load-test a deployment\'s sync-webhook path, auto-discover its shape, and attribute latency (queue-wait vs service-time) against the recommended setup.';
export const benchmarkCommand = new Command('benchmark')
.description(BENCHMARK_DOC)
.option('--url <url>', 'Activepieces base URL (dev env API port)', 'http://localhost:3000')
.option('--requests <n>', 'Total requests to fire (default: 40 x concurrency)')
.option('--concurrency <c>', 'Concurrent connections (default: auto = sum of worker execution slots)')
.option('--api-key <key>', 'Platform API key (Bearer). Or set AP_API_KEY.')
.option('--project-id <id>', 'Reuse an existing project ID (skip throwaway provision + teardown). Useful in CI where the platform plan does not allow team-project creation.')
.option('--flow-id <id>', 'Reuse an existing sync-webhook flow ID (skip flow creation). Requires --project-id.')
.option('--body <json>', 'JSON request body sent to the webhook', '{"test":true}')
.option('--json', 'Emit machine-readable JSON output')
.action(async (opts) => {
const config = benchmarkUtils.normalizeOptions(opts);
try {
const client = axios.create({ baseURL: config.url, validateStatus: () => true });
await waitForReady(client);
const auth = authenticate({ config });
const authed = axios.create({
baseURL: config.url,
headers: { Authorization: `Bearer ${auth.token}` },
validateStatus: () => true,
});
const [setup, health, diagnostics, flags, network] = await Promise.all([
discoverSetup(authed),
collectHealth(authed),
collectDiagnostics(authed),
collectFlags(authed),
measureNetwork(authed),
]);
const reusedProjectId = config.projectId;
const project = reusedProjectId
? { id: reusedProjectId }
: await provisionProject({ client: authed });
log(config, reusedProjectId ? `Reusing project ${project.id}` : `Provisioned throwaway project ${project.id}`);
const runsFailed = await (async () => {
const projectLimits = await collectProjectLimits({ client: authed, projectId: project.id, rateLimiterEnabled: flags['PROJECT_RATE_LIMITER_ENABLED'] === true });
const flowId = config.flowId ?? await createBenchmarkFlow({ client: authed, projectId: project.id });
log(config, config.flowId ? `Reusing flow ${flowId}` : `Flow ready: ${flowId}`);
const slots = setup.executionSlots;
const phases = benchmarkUtils.resolvePhases({ concurrency: config.concurrency, slots });
const diagnosticsSampler = startDiagnosticsSampler(authed);
const runs: PhaseReport[] = [];
for (const phase of phases) {
const requests = config.requests ?? Math.max(200, phase.connections * 40);
log(config, `Running ${phase.label}: ${requests} requests @ ${phase.connections} connections`);
// createdAfter is compared against the server clock, but this timestamp is the CLI's.
// The CLI runs cross-region, so its clock can lead the server's — widen the window by a
// skew buffer so runs aren't silently dropped. Safe because each benchmark builds a fresh
// flow, so the flowId filter still admits only this run's flow runs.
const startedAt = new Date(Date.now() - CLOCK_SKEW_BUFFER_MS).toISOString();
const queueSampler = startQueueSampler(authed);
const result = await autocannon({
url: `${config.url}/api/v1/webhooks/${flowId}/sync`,
connections: phase.connections,
amount: requests,
method: 'POST',
headers: { 'content-type': 'application/json' },
body: config.body,
});
const queueDepth = queueSampler.stop();
const summary = benchmarkUtils.toSummary({ result, flowId, connections: phase.connections });
const runsInWindow = await collectRuns({ client: authed, projectId: project.id, flowId, since: startedAt });
runs.push({ label: phase.label, connections: phase.connections, requests, startedAt, summary, timeline: runsInWindow.timeline, outcomes: runsInWindow.outcomes, queueDepth });
}
const diagnosticsTimeline = { intervalMs: DIAGNOSTICS_SAMPLE_INTERVAL_MS, samples: diagnosticsSampler.stop() };
log(config, 'Scanning other projects for flows that ran during the benchmark...');
const outsideFlows = await collectOutsideFlows({ client: authed, benchmarkProjectId: project.id, since: runs[0].startedAt });
const storage = await probeStorage({ client: authed, projectId: project.id, flowId });
const report: BenchmarkReport = { meta: buildMeta({ url: config.url }), flowId, project: { id: project.id, limits: projectLimits }, health, diagnostics, diagnosticsTimeline, setup, flags, network, storage, outsideFlows, runs };
if (config.json) {
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
} else {
renderReport(report);
}
return runs.some((r) => r.summary.failed > 0);
})().finally(async () => {
if (!reusedProjectId) {
log(config, `Deleting throwaway project ${project.id}`);
await deleteProject({ client: authed, id: project.id });
}
});
process.exit(runsFailed ? 1 : 0);
} catch (e) {
const message = e instanceof AxiosError ? `${e.message}${e.response ? ` (HTTP ${e.response.status}: ${JSON.stringify(e.response.data)})` : ''}` : e instanceof Error ? e.message : String(e);
if (config.json) {
process.stdout.write(JSON.stringify({ error: message }) + '\n');
} else {
console.error(chalk.red(`Benchmark failed: ${message}`));
}
process.exit(2);
}
});
function normalizeOptions(opts: Record<string, string | boolean | undefined>): BenchmarkConfig {
const requests = optionalPositiveInt(opts.requests, '--requests');
const concurrency = optionalPositiveInt(opts.concurrency, '--concurrency');
const body = String(opts.body);
try {
JSON.parse(body);
} catch {
throw new Error(`--body must be valid JSON, got "${opts.body}"`);
}
const url = String(opts.url);
const projectId = typeof opts.projectId === 'string' ? opts.projectId : undefined;
const flowId = typeof opts.flowId === 'string' ? opts.flowId : undefined;
if (Boolean(projectId) !== Boolean(flowId)) {
throw new Error('--project-id and --flow-id must be provided together, or both omitted (letting the CLI provision and delete its own throwaway project + flow).');
}
return {
url: url.endsWith('/') ? url.slice(0, -1) : url,
requests,
concurrency,
apiKey: typeof opts.apiKey === 'string' ? opts.apiKey : undefined,
projectId,
flowId,
body,
json: opts.json === true,
};
}
function resolvePhases({ concurrency, slots }: ResolvePhasesParams): Phase[] {
if (concurrency !== undefined) {
return [{ label: `conc ${concurrency}`, connections: concurrency }];
}
const base = slots ?? DEFAULT_CONCURRENCY;
return [{ label: `conc ${base} (= slots)`, connections: base }];
}
function validateSetup(machines: WorkerMachineWithStatus[]): SetupCheck[] {
if (machines.length === 0) {
return [{ dimension: 'workers', status: 'WARN', detail: 'No connected workers reported (need a platform-admin token, or no workers online).' }];
}
const props = machines.map((m) => m.information.workerProps);
const modes = unique(props.map((p) => p.EXECUTION_MODE ?? 'unset'));
const reuse = unique(props.map((p) => p.REUSE_SANDBOX ?? 'unset'));
const concs = props.map((p) => parseSlot(p.WORKER_CONCURRENCY));
const cpus = machines.map((m) => m.information.totalCpuCores);
const ramsGb = machines.map((m) => m.information.totalAvailableRamInBytes / BYTES_PER_GB);
return [
{
dimension: 'sandbox mode',
status: modes.length === 1 && modes[0] === 'SANDBOX_CODE_ONLY' ? 'PASS' : 'WARN',
detail: `EXECUTION_MODE = ${modes.join(', ')} (recommended: SANDBOX_CODE_ONLY)`,
},
{
dimension: 'reuse sandbox',
status: reuse.length === 1 && reuse[0] === 'true' ? 'PASS' : 'WARN',
detail: `REUSE_SANDBOX = ${reuse.join(', ')} (recommended: true)`,
},
{
dimension: 'worker concurrency',
status: concs.every((c) => c === 1) ? 'PASS' : 'WARN',
detail: `per-worker WORKER_CONCURRENCY = ${unique(concs.map(String)).join(', ')} (recommended: 1, scale by replicas)`,
},
{
dimension: 'worker CPU',
status: cpus.every((c) => c <= RECOMMENDED_MAX_CPU_CORES + 0.01) ? 'PASS' : 'WARN',
detail: `cores/worker = ${unique(cpus.map((c) => c.toString())).join(', ')} (recommended: ~0.5 vCPU)`,
},
{
dimension: 'worker RAM',
status: ramsGb.every((g) => g <= RECOMMENDED_MAX_RAM_GB + 0.3) ? 'PASS' : 'WARN',
detail: `GB/worker = ${unique(ramsGb.map((g) => g.toFixed(1))).join(', ')} (recommended: ~1 GB)`,
},
];
}
function aggregateTimeline(runs: FlowRunLike[]): TimelineAggregate {
const phase = (name: TimelinePhaseName): number[] =>
runs
.map((r) => r.timeline?.legs?.[0]?.find((p) => p.name === name)?.durationMs)
.filter((v): v is number => typeof v === 'number');
const queue = phase('QUEUE');
const provision = phase('PROVISION');
const boot = phase('BOOT');
const run = phase('RUN');
// A run the rate limiter rejected was re-queued with a >=20s delay (rate-limiter-interceptor.ts,
// 20s * 2^attempts), so it shows up as a QUEUE outlier of at least that size. Threshold at 15s to
// absorb app<->worker clock skew. Zero here = the rate limiter added no delay to these runs.
const rateLimitedRunsCount = queue.filter((q) => q >= RATE_LIMIT_DETECTION_MS).length;
const waitPerRun = runs
.map((r) => {
const legs = r.timeline?.legs?.[0];
if (!legs) return undefined;
return legs.filter((p) => p.name !== 'RUN').reduce((sum, p) => sum + p.durationMs, 0);
})
.filter((v): v is number => typeof v === 'number');
return {
sampleCount: run.length,
queueWaitP50: percentile(waitPerRun, 50),
queueWaitP90: percentile(waitPerRun, 90),
serviceP50: percentile(run, 50),
serviceP90: percentile(run, 90),
queueP50: percentile(queue, 50),
queueP90: percentile(queue, 90),
queueMax: queue.length > 0 ? Math.max(...queue) : 0,
provisionP50: percentile(provision, 50),
bootP50: percentile(boot, 50),
rateLimitedRunsCount,
};
}
async function discoverSetup(client: AxiosInstance): Promise<SetupDiscovery> {
const res = await client.get('/api/v1/worker-machines');
if (res.status !== 200 || !Array.isArray(res.data)) {
return { available: false, reason: `worker-machines returned HTTP ${res.status} (platform-admin token required)`, machines: [], executionSlots: undefined, checks: validateSetup([]) };
}
const machines: WorkerMachineWithStatus[] = res.data;
const usable = machines.filter((m) => m.status === 'ONLINE' && typeof m.information?.workerProps?.WORKER_CONCURRENCY === 'string');
const skipped = machines.length - usable.length;
const executionSlots = usable.reduce((sum, m) => sum + parseSlot(m.information.workerProps.WORKER_CONCURRENCY), 0);
const checks = validateSetup(usable);
if (skipped > 0) {
checks.unshift({ dimension: 'worker registry', status: 'WARN', detail: `${skipped}/${machines.length} registry entries loo
k stale or incomplete (non-ONLINE or missing worker props) and were excluded from the slot count — if a worker rollout just happen
ed, wait ~60s and re-run.` });
}
return { available: true, machines: usable, executionSlots: executionSlots > 0 ? executionSlots : undefined, checks };
}
async function measureNetwork(client: AxiosInstance): Promise<NetworkBaseline> {
const samples: number[] = [];
for (let i = 0; i < NETWORK_PROBES; i++) {
const start = process.hrtime.bigint();
const res = await client.get('/api/v1/flags').catch(() => null);
const ms = Number(process.hrtime.bigint() - start) / 1e6;
if (res && res.status === 200) samples.push(ms);
}
return { probes: samples.length, minMs: samples.length ? Math.min(...samples) : 0, p50Ms: percentile(samples, 50) };
}
async function collectRuns({ client, projectId, flowId, since }: CollectRunsParams): Promise<CollectedRuns> {
const collected: FlowRunLike[] = [];
let cursor: string | undefined;
for (let page = 0; page < MAX_RUN_PAGES; page++) {
const params: Record<string, string | number> = { projectId, flowId, createdAfter: since, limit: RUN_PAGE_SIZE };
if (cursor) params.cursor = cursor;
const res = await client.get('/api/v1/flow-runs', { params });
if (res.status !== 200 || !Array.isArray(res.data?.data)) break;
collected.push(...res.data.data);
cursor = res.data.next ?? undefined;
if (!cursor) break;
}
const outcomes: RunOutcomes = {};
for (const r of collected) {
const status = r.status ?? 'UNKNOWN';
outcomes[status] = (outcomes[status] ?? 0) + 1;
}
return { timeline: aggregateTimeline(collected), outcomes };
}
async function collectHealth(client: AxiosInstance): Promise<HealthInfo> {
const res = await client.get('/api/v1/health/system');
if (res.status !== 200 || typeof res.data !== 'object') {
return { available: false, reason: `health/system returned HTTP ${res.status} (platform-admin token required)` };
}
return { available: true, ...res.data };
}
async function collectDiagnostics(client: AxiosInstance): Promise<DiagnosticsInfo> {
const res = await client.get('/api/v1/health/diagnostics');
if (res.status !== 200 || typeof res.data !== 'object') {
return { available: false, reason: `health/diagnostics returned HTTP ${res.status} (needs platform-admin; server may predate this endpoint)` };
}
return { available: true, ...res.data };
}
async function collectFlags(client: AxiosInstance): Promise<Record<string, unknown>> {
const res = await client.get('/api/v1/flags');
const all: Record<string, unknown> = res.data && typeof res.data === 'object' && !Array.isArray(res.data) ? res.data : {};
const picked: Record<string, unknown> = {};
for (const id of DIAGNOSTIC_FLAGS) {
if (id in all) picked[id] = all[id];
}
return picked;
}
// The one-shot diagnostics block is a single die-roll: a storage probe that reads 231ms at start
// can read 455ms mid-load. This sampler re-probes every few seconds across the whole benchmark and
// keys each sample by time, so variance and load-correlation become visible. App CPU/evloop and
// worker pings only refresh on their ~60s ticks, so consecutive samples may repeat those values.
function startDiagnosticsSampler(client: AxiosInstance): DiagnosticsSampler {
const samples: Record<string, DiagnosticsSample> = {};
let stopped = false;
void (async () => {
while (!stopped) {
const sampledAt = new Date().toISOString();
const res = await client.get('/api/v1/health/diagnostics').catch(() => null);
if (!stopped && res && res.status === 200 && res.data) {
const d = res.data;
const machines = (d.workers?.machines ?? []) as Array<{ serverPingMs?: number | null }>;
const pings = machines.map((w) => w.serverPingMs).filter((v): v is number => typeof v === 'number');
samples[sampledAt] = {
databaseMs: d.database?.latencyMs ?? null,
redisMs: d.redis?.latencyMs ?? null,
storageMs: d.storage?.latencyMs ?? null,
apps: ((d.apps?.instances ?? []) as Array<{ hostname: string; cpuUsagePercentage: number; eventLoopDelayMs: number } & CpuPressureInfo>)
.map((a) => ({ hostname: a.hostname, cpuUsagePercentage: a.cpuUsagePercentage, eventLoopDelayMs: a.eventLoopDelayMs, cpuStealPercentage: a.cpuStealPercentage, cpuThrottledPercentage: a.cpuThrottledPercentage })),
workerPingP50Ms: pings.length > 0 ? percentile(pings, 50) : null,
workerPingMaxMs: pings.length > 0 ? Math.max(...pings) : null,
};
}
await sleep(DIAGNOSTICS_SAMPLE_INTERVAL_MS);
}
})();
return {
stop() {
stopped = true;
return samples;
},
};
}
function startQueueSampler(client: AxiosInstance): QueueSampler {
const samples: QueueSample[] = [];
const timer = setInterval(() => {
client.get('/api/v1/worker-machines/queue-metrics')
.then((res) => {
const queues: Array<{ waiting: number; active: number }> = Array.isArray(res.data?.queues) ? res.data.queues : [];
const waiting = queues.reduce((s, q) => s + (q.waiting ?? 0), 0);
const active = queues.reduce((s, q) => s + (q.active ?? 0), 0);
samples.push({ waiting, active });
})
.catch(() => undefined);
}, QUEUE_SAMPLE_INTERVAL_MS);
return {
stop() {
clearInterval(timer);
if (samples.length === 0) return { samples: 0, available: false };
return {
samples: samples.length,
available: true,
maxWaiting: Math.max(...samples.map((s) => s.waiting)),
maxActive: Math.max(...samples.map((s) => s.active)),
avgWaiting: Math.round(samples.reduce((s, q) => s + q.waiting, 0) / samples.length),
};
},
};
}
function buildMeta({ url }: { url: string }): RunMeta {
return {
ranAt: new Date().toISOString(),
target: url,
cli: {
node: process.version,
platform: `${os.platform()} ${os.arch()}`,
cpus: os.cpus().length,
note: 'The CLI runs from a different host/region than the API and workers; all client-side latency below includes CLI→server network. Authoritative per-run latency is server-measured (FlowRun.timeline).',
},
};
}
// Flows the benchmark does NOT own that ran inside its window. They share the same execution slots,
// so they are the answer to "why is QUEUE high on a deployment that looks idle". Scans every platform
// project except the benchmark's throwaway one.
async function collectOutsideFlows({ client, benchmarkProjectId, since }: CollectOutsideFlowsParams): Promise<OutsideFlowsReport> {
const projectsRes = await client.get('/api/v1/projects', { params: { limit: PROJECT_PAGE_SIZE } });
if (projectsRes.status !== 200 || !Array.isArray(projectsRes.data?.data)) {
return { available: false, detail: `project listing failed (HTTP ${projectsRes.status}) — cannot scan for outside flows`, flows: [] };
}
const notes: string[] = [];
if (projectsRes.data.next) notes.push(`only the first ${PROJECT_PAGE_SIZE} projects scanned`);
const projects: Array<{ id: string }> = projectsRes.data.data;
const outsideRuns: OutsideRunLike[] = [];
for (const project of projects.filter((p) => p.id !== benchmarkProjectId)) {
let cursor: string | undefined;
for (let page = 0; page < MAX_OUTSIDE_RUN_PAGES; page++) {
const params: Record<string, string | number> = { projectId: project.id, createdAfter: since, limit: RUN_PAGE_SIZE };
if (cursor) params.cursor = cursor;
const res = await client.get('/api/v1/flow-runs', { params });
if (res.status !== 200 || !Array.isArray(res.data?.data)) break;
outsideRuns.push(...res.data.data.map((r: Record<string, unknown>) => ({ ...r, projectId: project.id }) as OutsideRunLike));
cursor = res.data.next ?? undefined;
if (!cursor) break;
if (page === MAX_OUTSIDE_RUN_PAGES - 1) notes.push(`run listing truncated at ${MAX_OUTSIDE_RUN_PAGES * RUN_PAGE_SIZE} runs for project ${project.id}`);
}
}
const aggregates = aggregateOutsideRuns(outsideRuns);
if (aggregates.length > MAX_OUTSIDE_FLOWS_DETAILED) notes.push(`pieces/name resolved for the top ${MAX_OUTSIDE_FLOWS_DETAILED} of ${aggregates.length} flows only`);
const flows: OutsideFlow[] = [];
for (const [index, aggregate] of aggregates.entries()) {
const description = index < MAX_OUTSIDE_FLOWS_DETAILED
? await describeFlow({ client, flowId: aggregate.flowId, projectId: aggregate.projectId })
: { displayName: null, pieces: [] };
flows.push({ ...aggregate, ...description });
}
return { available: true, flows, ...(notes.length > 0 ? { detail: notes.join('; ') } : {}) };
}
function aggregateOutsideRuns(runs: OutsideRunLike[]): OutsideFlowAggregate[] {
const byFlow = new Map<string, { projectId: string; runs: number; totalRunMs: number; timedRuns: number }>();
for (const run of runs) {
if (typeof run.flowId !== 'string') continue;
const entry = byFlow.get(run.flowId) ?? { projectId: run.projectId, runs: 0, totalRunMs: 0, timedRuns: 0 };
entry.runs += 1;
const started = Date.parse(run.startTime ?? '');
const finished = Date.parse(run.finishTime ?? '');
if (Number.isFinite(started) && Number.isFinite(finished) && finished >= started) {
entry.totalRunMs += finished - started;
entry.timedRuns += 1;
}
byFlow.set(run.flowId, entry);
}
return [...byFlow.entries()]
.map(([flowId, agg]) => ({ flowId, projectId: agg.projectId, runs: agg.runs, avgRunMs: agg.timedRuns > 0 ? Math.round(agg.totalRunMs / agg.timedRuns) : null }))
.sort((a, b) => b.runs - a.runs);
}
async function describeFlow({ client, flowId, projectId }: DescribeFlowParams): Promise<FlowDescription> {
const res = await client.get(`/api/v1/flows/${flowId}`, { params: { projectId } }).catch(() => null);
if (!res || res.status !== 200 || isNilLike(res.data?.version)) {
return { displayName: null, pieces: [] };
}
const version = res.data.version;
const pieces = unique([...JSON.stringify(version).matchAll(/"pieceName"\s*:\s*"([^"]+)"/g)].map((m) => m[1]));
return { displayName: typeof version.displayName === 'string' ? version.displayName : null, pieces };
}
function isNilLike(value: unknown): value is null | undefined {
return value === null || value === undefined;
}
async function probeStorage({ client, projectId, flowId }: ProbeStorageParams): Promise<StorageProbe> {
const res = await client.get('/api/v1/flow-runs', { params: { projectId, flowId, limit: 50 } });
const runs: FlowRunLike[] = Array.isArray(res.data?.data) ? res.data.data : [];
if (runs.length === 0) {
return { logsPersisted: 0, sampled: 0, detail: 'No runs found to check log persistence.' };
}
const withLog = runs.filter((r) => typeof r.logsFileId === 'string' && r.logsFileId.length > 0).length;
// logsFileId is written only after the worker's end-of-run backup uploads the log to the configured
// FILE_STORAGE_LOCATION (S3/GCS or DB). A high rate proves the worker->storage write path works;
// the write LATENCY itself is inside RUN. The signed-URL fetch path needs a server-minted file token
// the CLI can't produce, so signed-URL *mode* is not client-detectable — it does not affect run latency.
return { logsPersisted: withLog, sampled: runs.length, detail: `${withLog}/${runs.length} sampled runs have a persisted log — the worker->storage write path is healthy` };
}
async function waitForReady(client: AxiosInstance): Promise<void> {
for (let i = 0; i < 60; i++) {
const res = await client.get('/api/v1/flags').catch(() => null);
if (res && res.status === 200) return;
await sleep(2000);
}
throw new Error('Server did not become ready in time (checked /api/v1/flags)');
}
function authenticate({ config }: { config: BenchmarkConfig }): AuthResult {
const apiKey = config.apiKey ?? process.env.AP_API_KEY;
if (!apiKey) {
throw new Error('Provide a platform API key via --api-key or the AP_API_KEY env var.');
}
return { token: apiKey };
}
// The benchmark always runs in a throwaway project it provisions and deletes, never the caller's own.
// A low maxConcurrentJobs on a real project would queue the load and inflate latency — a measurement
// artifact, not a server problem — so the throwaway project is created with a cap well above any
// realistic slot count, taking the project rate limiter out of the picture entirely.
async function provisionProject({ client }: { client: AxiosInstance }): Promise<ResolvedProject> {
const res = await client.post('/api/v1/projects', {
displayName: `benchmark-${Date.now()}`,
maxConcurrentJobs: EPHEMERAL_PROJECT_MAX_CONCURRENCY,
});
if (res.status >= 400 || typeof res.data?.id !== 'string') {
throw new Error(`Could not create a benchmark project (HTTP ${res.status}: ${JSON.stringify(res.data)}). The API key must be a platform admin key, and the plan must allow team projects.`);
}
return { id: res.data.id };
}
async function deleteProject({ client, id }: { client: AxiosInstance; id: string }): Promise<void> {
await client.delete(`/api/v1/projects/${id}`).catch(() => undefined);
}
// Reads the project's own concurrency cap so a queue-throttled benchmark can be told apart from a slow
// server. PROJECT_RATE_LIMITER_ENABLED gates whether the cap is enforced at all.
async function collectProjectLimits({ client, projectId, rateLimiterEnabled }: CollectProjectLimitsParams): Promise<ProjectLimits> {
const res = await client.get<Project>(`/api/v1/projects/${projectId}`);
if (res.status !== 200 || typeof res.data !== 'object') {
return { available: false, reason: `projects/${projectId} returned HTTP ${res.status}` };
}
const maxConcurrentJobs: number | null = res.data?.maxConcurrentJobs ?? null;
return { available: true, maxConcurrentJobs, rateLimiterEnabled };
}
async function createBenchmarkFlow({ client, projectId }: { client: AxiosInstance; projectId: string }): Promise<string> {
const [webhookVersion, mapperVersion] = await Promise.all([
resolvePieceVersion(client, WEBHOOK_PIECE),
resolvePieceVersion(client, DATA_MAPPER_PIECE),
]);
const created = await client.post('/api/v1/flows', { displayName: 'Benchmark Flow', projectId });
const flowId: string | undefined = created.data?.id;
if (!flowId) {
throw new Error(`Failed to create flow: ${JSON.stringify(created.data)}`);
}
await postOperation(client, flowId, {
type: 'IMPORT_FLOW',
request: buildImportRequest({ webhookVersion, mapperVersion }),
});
await postOperation(client, flowId, { type: 'LOCK_AND_PUBLISH', request: { status: 'ENABLED' } });
await waitForEnabled(client, flowId);
return flowId;
}
async function resolvePieceVersion(client: AxiosInstance, name: string): Promise<string> {
const res = await client.get(`/api/v1/pieces/${encodeURIComponent(name)}`);
const version: string | undefined = res.data?.version;
if (!version) {
throw new Error(`Piece ${name} not available on server (is it synced?): HTTP ${res.status}`);
}
return `~${version}`;
}
async function postOperation(client: AxiosInstance, flowId: string, operation: unknown): Promise<void> {
const res = await client.post(`/api/v1/flows/${flowId}`, operation);
if (res.status >= 400) {
throw new Error(`Flow operation failed (HTTP ${res.status}): ${JSON.stringify(res.data)}`);
}
}
async function waitForEnabled(client: AxiosInstance, flowId: string): Promise<void> {
for (let i = 0; i < 30; i++) {
const res = await client.get(`/api/v1/flows/${flowId}`);
if (res.data?.status === 'ENABLED') return;
await sleep(1000);
}
console.error(chalk.yellow('Flow did not report ENABLED within 30s; proceeding anyway.'));
}
// schemaVersion:null makes the server migrate this payload up to its own latest schema,
// so the same payload stays valid across server versions without CLI maintenance.
function buildImportRequest({ webhookVersion, mapperVersion }: { webhookVersion: string; mapperVersion: string }): unknown {
return {
displayName: 'Benchmark Flow',
schemaVersion: null,
notes: [],
trigger: {
name: 'trigger',
valid: true,
displayName: 'Catch Webhook',
type: 'PIECE_TRIGGER',
settings: {
pieceName: WEBHOOK_PIECE,
pieceVersion: webhookVersion,
triggerName: 'catch_webhook',
input: { authType: 'none', authFields: {} },
propertySettings: {},
sampleData: {},
},
nextAction: {
name: 'step_1',
skip: false,
type: 'PIECE',
valid: true,
displayName: 'Advanced Mapping',
settings: {
pieceName: DATA_MAPPER_PIECE,
pieceVersion: mapperVersion,
actionName: 'advanced_mapping',
input: { mapping: { echo: '{{trigger.body}}' } },
propertySettings: {},
sampleData: {},
errorHandlingOptions: { retryOnFailure: { value: false }, continueOnFailure: { value: false } },
},
nextAction: {
name: 'step_2',
skip: false,
type: 'PIECE',
valid: true,
displayName: 'Return Response',
settings: {
pieceName: WEBHOOK_PIECE,
pieceVersion: webhookVersion,
actionName: 'return_response',
input: { fields: { body: '{{step_1}}', status: 200, headers: {} }, respond: 'stop', responseType: 'json' },
propertySettings: {},
sampleData: {},
errorHandlingOptions: { retryOnFailure: { value: false }, continueOnFailure: { value: false } },
},
},
},
},
};
}
function toSummary({ result, flowId, connections }: ToSummaryParams): Summary {
const failed = result.non2xx + result.errors + result.timeouts;
return {
flowId,
requests: result.requests.sent,
connections,
durationSec: result.duration,
throughputReqSec: result.requests.average,
latencyMeanMs: result.latency.mean,
p50Ms: result.latency.p50,
p90Ms: result.latency.p90,
p99Ms: result.latency.p99,
minMs: result.latency.min,
maxMs: result.latency.max,
ok2xx: result['2xx'],
non2xx: result.non2xx,
statusCodes: { '1xx': result['1xx'] ?? 0, '2xx': result['2xx'] ?? 0, '3xx': result['3xx'] ?? 0, '4xx': result['4xx'] ?? 0, '5xx': result['5xx'] ?? 0 },
errors: result.errors,
timeouts: result.timeouts,
failed,
};
}
function renderReport(report: BenchmarkReport): void {
console.log(chalk.bold(`\nBenchmark report (flow ${report.flowId})`));
console.log(chalk.gray(` ran ${report.meta.ranAt} against ${report.meta.target}`));
console.log(chalk.gray(` CLI host: node ${report.meta.cli.node}, ${report.meta.cli.platform}, ${report.meta.cli.cpus} cpu — different region than API/workers`));
console.log(chalk.bold('\nVersion & health'));
if (report.health.available) {
const r = report.health.release;
const skew = r && r.workers.versionMismatched > 0
? chalk.yellow(`SKEW: ${r.workers.versionMismatched}/${r.workers.total} workers on ${JSON.stringify(r.workers.mismatchedVersions)}`)
: chalk.green('all workers match app version');
console.log(` app version : ${r?.current ?? '?'} (latest available: ${report.health.latestVersion ?? '?'}) — ${skew}`);
console.log(` health : app-cpu=${hb(report.health.appCpu)} app-ram=${hb(report.health.appRam)} disk=${hb(report.health.disk)} worker-cpu=${hb(report.health.workerCpu)} worker-ram=${hb(report.health.workerRam)} db=${hb(report.health.database)}`);
} else {
console.log(chalk.yellow(` skipped: ${report.health.reason}`));
}
console.log(chalk.bold('\nInfra round-trip (server-measured, in-region — authoritative, not reachable from the CLI)'));
if (report.diagnostics.available) {
const d = report.diagnostics;
console.log(` database : ${infra(d.database)}`);
console.log(` redis : ${infra(d.redis)}`);
console.log(` storage : ${infra(d.storage)} ${chalk.gray('(S3/GCS write+read round-trip; same cost is inside every RUN as the end-of-run log backup)')}`);
if (d.config) {
console.log(` config : execution=${d.config.executionMode} storage=${d.config.fileStorageLocation} signedUrls=${d.config.s3SignedUrls} sandboxMemKB=${d.config.sandboxMemoryLimitKb} s3=${d.config.s3Endpoint ?? 'n/a'}/${d.config.s3Region ?? 'n/a'} rateLimiter=${d.config.projectRateLimiterEnabled ?? 'n/a'} concurrentJobsLimit=${d.config.defaultConcurrentJobsLimit ?? 'n/a'}`);
}
if (d.apps) {
console.log(` apps : ${d.apps.count} connected`);
for (const a of d.apps.instances) {
console.log(` - ${a.hostname} v${a.version} | ${a.cpuCores} core | cpu ${a.cpuUsagePercentage.toFixed(1)}% | ram ${(a.ramTotalBytes / BYTES_PER_GB).toFixed(1)}GB ${a.ramUsagePercentage.toFixed(1)}% | disk ${a.diskPercentage.toFixed(0)}% | evloop ${a.eventLoopDelayMs}ms${cpuPressureLabel(a)}`);
}
}
if (d.workers) {
console.log(` workers : ${d.workers.count} connected`);
for (const w of d.workers.machines) {
console.log(` - ${w.status} ${w.workerId.slice(0, 8)} | ${w.cpuCores} core | cpu ${w.cpuUsagePercentage.toFixed(1)}% | ram ${w.ramUsagePercentage.toFixed(1)}% | worker→app ${w.serverPingMs == null ? 'n/a' : w.serverPingMs + 'ms'}`);
}
}
renderCpuPressureVerdict(d);
} else {
console.log(chalk.yellow(` skipped: ${report.diagnostics.reason}`));
}
console.log(chalk.bold('\nConfig flags (server-reported)'));
if (Object.keys(report.flags).length === 0) {
console.log(chalk.yellow(' none reported'));
} else {
for (const [id, value] of Object.entries(report.flags)) {
console.log(` ${id.padEnd(28)} ${JSON.stringify(value)}`);
}
}
console.log(chalk.bold('\nSetup'));
if (report.setup.available) {
console.log(` workers online : ${report.setup.machines.length}, execution slots : ${report.setup.executionSlots ?? '?'}`);
for (const m of report.setup.machines) {
const i = m.information;
console.log(` - ${m.status} v${i.workerProps.version ?? '?'} | ${i.totalCpuCores} core ${i.cpuUsagePercentage ?? '?'}% cpu | ${(i.totalAvailableRamInBytes / BYTES_PER_GB).toFixed(1)}GB ${i.ramUsagePercentage ?? '?'}% ram | conc ${i.workerProps.WORKER_CONCURRENCY ?? '?'} | ${i.workerProps.EXECUTION_MODE ?? '?'} reuse=${i.workerProps.REUSE_SANDBOX ?? '?'} | sandboxes ${i.sandboxes?.length ?? 0}`);
}
} else {
console.log(chalk.yellow(` setup discovery skipped: ${report.setup.reason}`));
}
for (const c of report.setup.checks) {
const badge = c.status === 'PASS' ? chalk.green('PASS') : chalk.yellow('WARN');
console.log(` [${badge}] ${c.dimension.padEnd(18)} ${c.detail}`);
}
console.log(chalk.bold('\nProject'));
console.log(` throwaway project (auto-created, deleted after) : ${report.project.id}`);
if (report.project.limits.available === true) {
const cap = report.project.limits.maxConcurrentJobs;
const capLabel = cap === null ? 'unset (platform default)' : `${cap} concurrent jobs`;
// The benchmark project is uncapped on purpose, so these numbers can't throttle THIS run — they are
// reported so you can see whether the platform rate limiter would throttle your real projects.
console.log(` benchmark project cap : ${capLabel} (not a bottleneck here)`);
} else {
console.log(chalk.yellow(` project limits skipped: ${report.project.limits.reason}`));
}
renderRateLimiter(report);
renderOutsideFlows(report);
renderDiagnosticsTimeline(report);
console.log(chalk.bold('\nNetwork (CLI -> server, cross-region)'));
console.log(` RTT min / p50 : ${report.network.minMs.toFixed(1)} / ${report.network.p50Ms.toFixed(1)} ms over ${report.network.probes} probes`);
for (const run of report.runs) {
const s = run.summary;
const t = run.timeline;
console.log(chalk.bold(`\n${run.label}`));
console.log(` throughput : ${s.throughputReqSec.toFixed(1)} req/s (${s.requests} reqs in ${s.durationSec.toFixed(1)}s)`);
console.log(` run outcomes : ${formatOutcomes(run.outcomes)} (server-truth: ${s.ok2xx} 2xx, ${s.non2xx} non-2xx [4xx ${s.statusCodes['4xx']}, 5xx ${s.statusCodes['5xx']}], ${s.errors} errors, ${s.timeouts} timeouts)`);
if (run.queueDepth.available) {
console.log(` queue depth : max waiting ${run.queueDepth.maxWaiting}, max active ${run.queueDepth.maxActive}, avg waiting ${run.queueDepth.avgWaiting} ${chalk.gray(`(${run.queueDepth.samples} samples, sampled server-side during load)`)}`);
}
if (t.sampleCount === 0) {
console.log(chalk.yellow(' latency: no worker timelines found (server predates the timeline feature #13991, or runs were pruned)'));
} else {
console.log(chalk.bold(` worker-measured latency (authoritative — each phase timed inside the worker, region-independent, ${t.sampleCount} runs):`));
const rateLimitNote = t.rateLimitedRunsCount > 0
? chalk.yellow(` !! ${t.rateLimitedRunsCount} runs rate-limited (QUEUE >= ${RATE_LIMIT_DETECTION_MS / 1000}s backoff signature)`)
: '';
console.log(` QUEUE p50/p90/max ${fmt(t.queueP50)} / ${fmt(t.queueP90)} / ${fmt(t.queueMax)} ms — wait for a free execution slot ${chalk.gray('(±app↔worker clock skew; cross-check the queue-depth above)')}${rateLimitNote}`);
console.log(` PROVISION p50 ${fmt(t.provisionP50)} ms — piece install / cache provision`);
console.log(` BOOT p50 ${fmt(t.bootP50)} ms — engine fork + Node boot + isolate + socket connect`);
console.log(` RUN p50/p90 ${fmt(t.serviceP50)} / ${fmt(t.serviceP90)} ms — engine executes the flow, incl. end-of-run S3 log backup`);
console.log(` => queue-wait p50 ${fmt(t.queueWaitP50)} ms (QUEUE+PROVISION+BOOT) vs service p50 ${fmt(t.serviceP50)} ms (RUN)`);
console.log(` ${verdict(t)}`);
}
console.log(chalk.gray(` observational (CLI-side, cross-region — NOT authoritative): client latency p50/p90/p99 ${s.p50Ms.toFixed(0)}/${s.p90Ms.toFixed(0)}/${s.p99Ms.toFixed(0)} ms incl. CLI→server network`));
}
console.log(chalk.bold('\nStorage (log persistence — worker→storage write cost is inside RUN above)'));
console.log(` ${report.storage.detail}`);
console.log(chalk.gray('\nShare the full machine-readable bundle for support: re-run with --json > ap-benchmark.json'));
}
// Answers "is the project rate limiter the reason QUEUE is high?" directly: prints whether the
// limiter is on, the cap real projects get, whether the driven load would exceed it, and — the
// decisive signal — whether any benchmark run carries the >=20s re-queue delay the limiter imposes.
function renderRateLimiter(report: BenchmarkReport): void {
console.log(chalk.bold('\nRate limiter (project concurrency throttle)'));
const enabled = report.flags['PROJECT_RATE_LIMITER_ENABLED'] === true;
const defaultLimit = report.flags['DEFAULT_CONCURRENT_JOBS_LIMIT'];
console.log(` PROJECT_RATE_LIMITER_ENABLED : ${enabled ? chalk.yellow('true') : chalk.green('false')}`);
console.log(` DEFAULT_CONCURRENT_JOBS_LIMIT : ${JSON.stringify(defaultLimit ?? null)} ${chalk.gray('(cap per real project; the benchmark project is exempt via its own high cap)')}`);
if (!enabled) {
console.log(chalk.green(' => rate limiter is OFF — it cannot be the source of any queue time on this deployment.'));
return;
}
const rateLimited = report.runs.reduce((sum, r) => sum + r.timeline.rateLimitedRunsCount, 0);
const sampled = report.runs.reduce((sum, r) => sum + r.timeline.sampleCount, 0);
const maxQueueMs = Math.max(0, ...report.runs.map((r) => r.timeline.queueMax));
console.log(` rate-limited runs detected : ${rateLimited > 0 ? chalk.yellow(`${rateLimited}/${sampled}`) : chalk.green(`0/${sampled}`)} ${chalk.gray(`(a limited run is re-queued with a >=${RATE_LIMIT_MIN_BACKOFF_MS / 1000}s delay, so its QUEUE >= ${RATE_LIMIT_DETECTION_MS / 1000}s; max QUEUE seen ${(maxQueueMs / 1000).toFixed(1)}s)`)}`);
if (rateLimited > 0) {
console.log(chalk.yellow(` => rate limiter FIRED during the benchmark — ${rateLimited} runs carry the backoff delay. Raise the project cap or lower concurrency.`));
return;
}
console.log(chalk.green(` => no run shows the backoff signature: the QUEUE times measured here contain zero rate-limiter delay.`));
const driven = Math.max(0, ...report.runs.map((r) => r.connections));
if (typeof defaultLimit === 'number' && driven > defaultLimit) {
console.log(chalk.yellow(` => caution for REAL projects: this load ran at concurrency ${driven}, above the ${defaultLimit}-job default cap — a real project under the same load would throttle (+${RATE_LIMIT_MIN_BACKOFF_MS / 1000}s per rejected run).`));
} else if (typeof defaultLimit === 'number') {
console.log(chalk.gray(` => real projects capped at ${defaultLimit} concurrent jobs would sustain this load (driven concurrency ${driven}).`));
}
}
// One diagnostics reading is a die-roll; the timeline shows whether a bad number (say a 455ms
// storage probe) is the norm or a blip, and whether infra latency degrades under the load phases.
function renderDiagnosticsTimeline(report: BenchmarkReport): void {
const { intervalMs, samples } = report.diagnosticsTimeline;
const entries = Object.entries(samples);
if (entries.length === 0) return;
console.log(chalk.bold(`\nDiagnostics over time (server-probed every ${intervalMs / 1000}s across the benchmark)`));
console.log(chalk.gray(' time db redis storage app cpu% evloop ms worker ping p50/max app steal/thr%'));
for (const [at, s] of entries) {
const time = at.slice(11, 19);
const appsCpu = s.apps.map((a) => a.cpuUsagePercentage.toFixed(0)).join('/') || 'n/a';
const appsLoop = s.apps.map((a) => a.eventLoopDelayMs).join('/') || 'n/a';
const ping = s.workerPingP50Ms === null ? 'n/a' : `${s.workerPingP50Ms}/${s.workerPingMaxMs}ms`;
const pct = (v: number | undefined | null) => (typeof v === 'number' ? v.toFixed(1) : '-');
const appsPressure = s.apps.map((a) => `${pct(a.cpuStealPercentage)}/${pct(a.cpuThrottledPercentage)}`).join(' ') || 'n/a';
console.log(` ${time} ${String(s.databaseMs ?? 'n/a').padEnd(5)} ${String(s.redisMs ?? 'n/a').padEnd(5)} ${String(s.storageMs ?? 'n/a').padEnd(7)} ${appsCpu.padEnd(16)} ${appsLoop.padEnd(13)} ${ping.padEnd(19)} ${appsPressure}`);
}
console.log(chalk.gray(' app cpu/evloop and worker pings refresh on their ~60s ticks, so consecutive rows can repeat them; db/redis/storage are probed fresh each sample.'));
}
// Answers "was anything else running while I benchmarked?" — outside flows share the execution
// slots, so each entry here is workload that inflated QUEUE without any config flag explaining it.
function renderOutsideFlows(report: BenchmarkReport): void {
console.log(chalk.bold('\nOutside flows (ran during the benchmark, NOT part of it)'));
const { available, flows, detail } = report.outsideFlows;
if (!available) {
console.log(chalk.gray(` ${detail ?? 'unavailable'}`));
return;
}
if (flows.length === 0) {
console.log(chalk.green(' none — no other flow ran during the benchmark window. QUEUE times are contention-free.'));
return;
}
console.log(chalk.yellow(` ${flows.length} flow(s) competed for the execution slots during the load — their runs inflate QUEUE:`));
for (const flow of flows) {
const avgRun = flow.avgRunMs === null ? 'n/a' : `${flow.avgRunMs} ms`;
console.log(` - ${flow.displayName ?? flow.flowId}: ${flow.runs} runs, avg run ${avgRun}`);
console.log(chalk.gray(` flow ${flow.flowId} project ${flow.projectId} pieces [${flow.pieces.join(', ') || 'unknown'}]`));
}
console.log(chalk.gray(` window starts ${CLOCK_SKEW_BUFFER_MS / 60_000} min before the load (clock-skew tolerance), so slightly-earlier runs can appear.`));
if (detail) console.log(chalk.gray(` note: ${detail}`));
}
function cpuPressureLabel(instance: CpuPressureInfo): string {
if (instance.cpuStealPercentage === undefined && instance.cpuThrottledPercentage === undefined) return '';
const steal = instance.cpuStealPercentage;
const throttled = instance.cpuThrottledPercentage;
const stealText = steal === undefined ? '' : ` | steal ${steal >= CPU_STEAL_WARN_PCT ? chalk.yellow(`${steal.toFixed(1)}%`) : `${steal.toFixed(1)}%`}`;
const throttledText = throttled === undefined ? '' : ` | throttled ${throttled >= CPU_THROTTLE_WARN_PCT ? chalk.yellow(`${throttled.toFixed(1)}%`) : `${throttled.toFixed(1)}%`}`;
return stealText + throttledText;
}
// Both signals are latency the CPU-usage numbers cannot show: steal is the hypervisor running a
// NEIGHBOR tenant while our vCPU was runnable (shared-CPU node types, e.g. GCE E2); throttled is
// our own k8s CPU limit stalling the container in ~100ms CFS quanta. Either one inflates every
// app touch inside QUEUE and RUN while cpu% still looks idle. App-only: workers don't report these.
function renderCpuPressureVerdict(d: DiagnosticsInfo): void {
const instances: CpuPressureInfo[] = d.apps?.instances ?? [];
const steals = instances.map((i) => i.cpuStealPercentage).filter((v): v is number => v !== undefined);
const throttles = instances.map((i) => i.cpuThrottledPercentage).filter((v): v is number => v !== undefined);
if (steals.length === 0 && throttles.length === 0) return;
const maxSteal = Math.max(0, ...steals);
const maxThrottled = Math.max(0, ...throttles);
if (maxSteal >= CPU_STEAL_WARN_PCT) {
console.log(chalk.yellow(` !! CPU steal up to ${maxSteal.toFixed(1)}% — the hypervisor is running OTHER tenants on this node's physical cores (shared-CPU machine types, e.g. GCE E2). This adds latency that cpu%% cannot show; move the app (and redis, if co-located) to dedicated-CPU nodes.`));
}
if (maxThrottled >= CPU_THROTTLE_WARN_PCT) {
console.log(chalk.yellow(` !! CFS throttling up to ${maxThrottled.toFixed(1)}% of periods — a container keeps hitting its k8s CPU limit and stalls in ~100ms quanta. Raise (or remove) the CPU limit on the affected pods.`));
}
if (maxSteal < CPU_STEAL_WARN_PCT && maxThrottled < CPU_THROTTLE_WARN_PCT) {
console.log(chalk.green(` cpu pressure ok: steal max ${maxSteal.toFixed(1)}%, throttled max ${maxThrottled.toFixed(1)}% — no neighbor contention or limit throttling detected`));
}
}
function hb(v: boolean | null | undefined): string {
if (v === null || v === undefined) return chalk.gray('n/a');
return v ? chalk.green('ok') : chalk.yellow('warn');
}
function infra(check: InfraCheckInfo | undefined): string {
if (!check) return chalk.gray('n/a');
if (!check.ok) return chalk.yellow(`FAIL${check.detail ? ` — ${check.detail}` : ''}`);
const latency = check.latencyMs === null || check.latencyMs === undefined ? (check.detail ?? 'ok') : `${check.latencyMs} ms`;
return chalk.green(latency);
}
function formatOutcomes(outcomes: RunOutcomes): string {
const entries = Object.entries(outcomes);
if (entries.length === 0) return 'none recorded';
return entries.map(([status, count]) => `${count} ${status}`).join(', ');
}
function verdict(t: TimelineAggregate): string {
if (t.serviceP50 <= 0) return chalk.gray('verdict: insufficient data');
const ratio = t.queueWaitP50 / t.serviceP50;
if (ratio > 1) {
return chalk.yellow(`verdict: queue-bound — waiting ${ratio.toFixed(1)}x longer than it runs. Add worker slots or lower concurrency, not a code fix.`);
}
return chalk.green('verdict: service-bound — queue-wait is small; latency is real engine time.');
}
function optionalPositiveInt(value: string | boolean | undefined, flag: string): number | undefined {
if (value === undefined) return undefined;
const n = Number(value);
if (!Number.isInteger(n) || n <= 0) {
throw new Error(`${flag} must be a positive integer, got "${value}"`);
}
return n;
}
function parseSlot(value: string | undefined): number {
const n = Number.parseInt(value ?? '1', 10);
return Number.isInteger(n) && n > 0 ? n : 1;
}
function percentile(values: number[], p: number): number {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
// Nearest-rank: rank = ceil(p/100 * N), 1-based. Avoids the floor()-based right bias that
// returned the max as the "p50" of a 2-element sample.
const rank = Math.ceil((p / 100) * sorted.length);
const idx = Math.min(sorted.length - 1, Math.max(0, rank - 1));
return sorted[idx];
}
function unique(values: string[]): string[] {
return [...new Set(values)];
}
function fmt(ms: number): string {
return ms.toFixed(0);
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function log(config: BenchmarkConfig, message: string): void {
if (!config.json) console.error(chalk.gray(message));
}
const WEBHOOK_PIECE = '@activepieces/piece-webhook';
const DATA_MAPPER_PIECE = '@activepieces/piece-data-mapper';
const DEFAULT_CONCURRENCY = 10;
const EPHEMERAL_PROJECT_MAX_CONCURRENCY = 1000;
const NETWORK_PROBES = 20;
const RUN_PAGE_SIZE = 100;
const MAX_RUN_PAGES = 30;
const BYTES_PER_GB = 1024 * 1024 * 1024;
const RECOMMENDED_MAX_CPU_CORES = 0.5;
const RECOMMENDED_MAX_RAM_GB = 1;
const QUEUE_SAMPLE_INTERVAL_MS = 500;
const DIAGNOSTICS_SAMPLE_INTERVAL_MS = 5_000;
const CLOCK_SKEW_BUFFER_MS = 5 * 60 * 1000;
const PROJECT_PAGE_SIZE = 100;
const MAX_OUTSIDE_RUN_PAGES = 5;
const MAX_OUTSIDE_FLOWS_DETAILED = 20;
// The server's rate-limiter re-queues a rejected job with min(600s, 20s * 2^attempts) delay
// (rate-limiter-interceptor.ts). 15s = one backoff minus clock-skew tolerance.
const RATE_LIMIT_MIN_BACKOFF_MS = 20_000;
const RATE_LIMIT_DETECTION_MS = 15_000;
// Steal above a few percent means real neighbor contention; healthy dedicated nodes sit at ~0.
// Throttling on a well-sized limit should be rare — a fifth of periods stalled is a sizing problem.
const CPU_STEAL_WARN_PCT = 5;
const CPU_THROTTLE_WARN_PCT = 20;
// Flags that matter for a perf triage — edition, version, execution mode, resource limits, and the
// two throttles (project concurrency cap + rate limiter) that silently cap throughput and emit 429s.
const DIAGNOSTIC_FLAGS = [
'EDITION', 'CURRENT_VERSION', 'ENVIRONMENT', 'PUBLIC_URL', 'PIECES_SYNC_MODE',
'FLOW_RUN_TIME_SECONDS', 'TRIGGER_TIMEOUT_SECONDS', 'WEBHOOK_TIMEOUT_SECONDS',
'FLOW_RUN_MEMORY_LIMIT_KB', 'FLOW_RUN_LOG_SIZE_LIMIT_MB', 'ALLOW_NPM_PACKAGES_IN_CODE_STEP',
'DEFAULT_CONCURRENT_JOBS_LIMIT', 'PROJECT_RATE_LIMITER_ENABLED', 'EXECUTION_DATA_RETENTION_DAYS',
];
export const benchmarkUtils = { normalizeOptions, toSummary, resolvePhases, validateSetup, aggregateTimeline, percentile, aggregateOutsideRuns };
type BenchmarkConfig = {
projectId?: string;
flowId?: string;
url: string;
requests?: number;
concurrency?: number;
apiKey?: string;
body: string;
json: boolean;
};
type AuthResult = { token: string };
type ResolvedProject = { id: string };
type CollectProjectLimitsParams = { client: AxiosInstance; projectId: string; rateLimiterEnabled: boolean };
type ProjectLimits =
| { available: false; reason: string }
| { available: true; maxConcurrentJobs: number | null; rateLimiterEnabled: boolean };
type Phase = { label: string; connections: number };
type ResolvePhasesParams = { concurrency?: number; slots?: number };
type CheckStatus = 'PASS' | 'WARN';
type SetupCheck = { dimension: string; status: CheckStatus; detail: string };
type WorkerProps = { EXECUTION_MODE?: string; WORKER_CONCURRENCY?: string; REUSE_SANDBOX?: string; version?: string };
type WorkerMachineWithStatus = {
status: string;
information: {
workerProps: WorkerProps;
totalCpuCores: number;
totalAvailableRamInBytes: number;
cpuUsagePercentage?: number;
ramUsagePercentage?: number;
sandboxes?: unknown[];
};
};
type SetupDiscovery = {
available: boolean;
reason?: string;
machines: WorkerMachineWithStatus[];
executionSlots?: number;
checks: SetupCheck[];
};
type NetworkBaseline = { probes: number; minMs: number; p50Ms: number };
type TimelinePhaseName = 'QUEUE' | 'PROVISION' | 'BOOT' | 'RUN';
type FlowRunLike = {
logsFileId?: string | null;
status?: string;
timeline?: { legs?: Array<Array<{ name: TimelinePhaseName; durationMs: number }>> } | null;
};
type TimelineAggregate = {
sampleCount: number;
queueWaitP50: number;
queueWaitP90: number;
serviceP50: number;
serviceP90: number;
queueP50: number;
queueP90: number;
queueMax: number;
provisionP50: number;
bootP50: number;
rateLimitedRunsCount: number;
};
type StorageProbe = {
logsPersisted: number;
sampled: number;
detail: string;
};
type CollectRunsParams = { client: AxiosInstance; projectId: string; flowId: string; since: string };
type RunOutcomes = Record<string, number>;
type CollectedRuns = { timeline: TimelineAggregate; outcomes: RunOutcomes };
type ProbeStorageParams = { client: AxiosInstance; projectId: string; flowId: string };
type QueueSample = { waiting: number; active: number };
type OutsideRunLike = { flowId?: string; projectId: string; startTime?: string; finishTime?: string };
type OutsideFlowAggregate = { flowId: string; projectId: string; runs: number; avgRunMs: number | null };
type FlowDescription = { displayName: string | null; pieces: string[] };
type OutsideFlow = OutsideFlowAggregate & FlowDescription;
type OutsideFlowsReport = { available: boolean; detail?: string; flows: OutsideFlow[] };
type CollectOutsideFlowsParams = { client: AxiosInstance; benchmarkProjectId: string; since: string };
type DescribeFlowParams = { client: AxiosInstance; flowId: string; projectId: string };
type QueueDepth = { samples: number; available: boolean; maxWaiting?: number; maxActive?: number; avgWaiting?: number };
type QueueSampler = { stop: () => QueueDepth };
type DiagnosticsSample = {
databaseMs: number | null;
redisMs: number | null;
storageMs: number | null;
apps: Array<{ hostname: string; cpuUsagePercentage: number; eventLoopDelayMs: number } & CpuPressureInfo>;
workerPingP50Ms: number | null;
workerPingMaxMs: number | null;
};
type DiagnosticsTimeline = { intervalMs: number; samples: Record<string, DiagnosticsSample> };
type DiagnosticsSampler = { stop: () => Record<string, DiagnosticsSample> };
type HealthInfo = {
available: boolean;
reason?: string;
latestVersion?: string;
appCpu?: boolean;
appRam?: boolean;
disk?: boolean;
workerCpu?: boolean | null;
workerRam?: boolean | null;
database?: boolean;
release?: { current: string; workers: { total: number; versionMismatched: number; mismatchedVersions: string[] } };
};
type RunMeta = {
ranAt: string;
target: string;
cli: { node: string; platform: string; cpus: number; note: string };
};
type InfraCheckInfo = { ok: boolean; latencyMs: number | null; detail?: string };
type DiagnosticsInfo = {
available: boolean;
reason?: string;
database?: InfraCheckInfo;
redis?: InfraCheckInfo;
storage?: InfraCheckInfo;
config?: {
executionMode: string | null;
fileStorageLocation: string | null;
sandboxMemoryLimitKb: number | null;
s3SignedUrls: boolean | null;
s3Endpoint: string | null;
s3Region: string | null;
projectRateLimiterEnabled?: boolean | null;
defaultConcurrentJobsLimit?: number | null;
};
apps?: {
count: number;
instances: Array<{ hostname: string; version: string; cpuCores: number; cpuUsagePercentage: number; ramTotalBytes: number; ramUsagePercentage: number; diskPercentage: number; eventLoopDelayMs: number } & CpuPressureInfo>;
};
workers?: {
count: number;
machines: Array<{ workerId: string; cpuCores: number; cpuUsagePercentage: number; ramUsagePercentage: number; serverPingMs: number | null; status: string }>;
};
};
type CpuPressureInfo = { cpuStealPercentage?: number; cpuThrottledPercentage?: number };
type LoadResult = {
requests: { sent: number; average: number };
latency: { mean: number; p50: number; p90: number; p99: number; min: number; max: number };
duration: number;
'1xx'?: number;
'2xx': number;
'3xx'?: number;
'4xx'?: number;
'5xx'?: number;
non2xx: number;
errors: number;
timeouts: number;
};
type ToSummaryParams = {
result: LoadResult;
flowId: string;
connections: number;
};
type Summary = {
flowId: string;
requests: number;
connections: number;
durationSec: number;
throughputReqSec: number;
latencyMeanMs: number;
p50Ms: number;
p90Ms: number;
p99Ms: number;
minMs: number;
maxMs: number;
ok2xx: number;
non2xx: number;
statusCodes: { '1xx': number; '2xx': number; '3xx': number; '4xx': number; '5xx': number };
errors: number;
timeouts: number;
failed: number;
};
type PhaseReport = {
label: string;
connections: number;
requests: number;
startedAt: string;
summary: Summary;
timeline: TimelineAggregate;
outcomes: RunOutcomes;
queueDepth: QueueDepth;
};
type BenchmarkReport = {
meta: RunMeta;
flowId: string;
project: { id: string; limits: ProjectLimits };
health: HealthInfo;
diagnostics: DiagnosticsInfo;
diagnosticsTimeline: DiagnosticsTimeline;
setup: SetupDiscovery;
flags: Record<string, unknown>;
network: NetworkBaseline;
storage: StorageProbe;
outsideFlows: OutsideFlowsReport;
runs: PhaseReport[];
};