Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

7.8 KB
Raw Blame
#!/usr/bin/env node
import { readFileSync } from 'node:fs'

const args = Object.fromEntries(
    process.argv.slice(2).reduce((acc, curr, i, arr) => {
        if (curr.startsWith('--')) acc.push([curr.slice(2), arr[i + 1]])
        return acc
    }, []),
)

const required = ['current', 'label', 'runUrl']
for (const key of required) {
    if (!args[key]) {
        console.error(`missing --${key}`)
        process.exit(2)
    }
}

if (!/^[a-z0-9-]+$/i.test(args.label)) {
    console.error(`--label must match /^[a-z0-9-]+$/i, got ${args.label}`)
    process.exit(2)
}

const THRESHOLD_PCT = Number(args.threshold ?? 5)
const CONSECUTIVE = Number(args.consecutive ?? 2)
const SIGNALS_URL = process.env.SIGNALS_URL
const SIGNALS_TOKEN = process.env.SIGNALS_TOKEN
const CH_HOST = process.env.CH_HOST
const CH_USER = process.env.CH_USER
const CH_PASS = process.env.CH_PASS

if (!CH_HOST || !CH_USER || !CH_PASS) {
    console.error('CH_HOST / CH_USER / CH_PASS env vars are required')
    process.exit(2)
}

const DIMENSIONS = [
    { name: 'client p90', unit: 'ms', get: (r) => r.latency?.p90Ms ?? 0 },
    { name: 'client p99', unit: 'ms', get: (r) => r.latency?.p99Ms ?? 0 },
    { name: 'throughput', unit: 'req/s', get: (r) => r.throughput?.reqSec ?? 0, invert: true },
    { name: 'queue wait p90', unit: 'ms', get: (r) => r.timeline?.queueWaitP90Ms ?? 0 },
    { name: 'service p90', unit: 'ms', get: (r) => r.timeline?.serviceP90Ms ?? 0 },
    { name: 'provision p50', unit: 'ms', get: (r) => r.timeline?.provisionP50Ms ?? 0 },
    { name: 'boot p50', unit: 'ms', get: (r) => r.timeline?.bootP50Ms ?? 0 },
    { name: 'worker cpu p95', unit: '%', get: (r) => r.resources?.worker?.cpuPct?.p95 ?? 0 },
    { name: 'worker mem max', unit: 'MB', get: (r) => r.resources?.worker?.memMb?.max ?? 0 },
    { name: 'app cpu p95', unit: '%', get: (r) => r.resources?.app?.cpuPct?.p95 ?? 0 },
    { name: 'app mem max', unit: 'MB', get: (r) => r.resources?.app?.memMb?.max ?? 0 },
]

const current = JSON.parse(readFileSync(args.current, 'utf8'))
const history = (await fetchHistory({ label: args.label })).reverse()

console.error(`Loaded ${history.length} trailing rows for label=${args.label}`)

const results = []
for (const dim of DIMENSIONS) {
    const currentValue = dim.get(current)
    const historyValues = history.map(dim.get)
    const baseline = median(historyValues)
    const pct = shiftPct({ current: currentValue, baseline, invert: dim.invert })

    const priorNeeded = Math.max(0, CONSECUTIVE - 1)
    const recentValues = priorNeeded > 0 ? history.slice(-priorNeeded).map(dim.get) : []
    const recentShifts = recentValues.map((v) => shiftPct({ current: v, baseline, invert: dim.invert }))
    const consecutiveBreach = pct > THRESHOLD_PCT
        && recentShifts.length >= priorNeeded
        && recentShifts.every((s) => s > THRESHOLD_PCT)

    results.push({ name: dim.name, unit: dim.unit, currentValue, baseline, pct, breach: consecutiveBreach })
}

const breaches = results.filter((r) => r.breach)
const summary = {
    label: args.label,
    threshold: THRESHOLD_PCT,
    consecutive: CONSECUTIVE,
    historyCount: history.length,
    dimensions: results,
    breachCount: breaches.length,
}
process.stdout.write(JSON.stringify(summary, null, 2) + '\n')

const MIN_HISTORY = Math.max(3, CONSECUTIVE)
if (history.length < MIN_HISTORY) {
    console.error(`History has ${history.length} entries (< ${MIN_HISTORY}); need more baselines before firing.`)
    process.exit(0)
}

if (breaches.length === 0) {
    console.error(`No regressions on ${args.label}.`)
    process.exit(0)
}

if (!SIGNALS_URL || !SIGNALS_TOKEN) {
    console.error(`BREACH on ${args.label} but SIGNALS_URL / SIGNALS_TOKEN unset — skipping signal.`)
    process.exit(0)
}

const lines = breaches.map((b) => `- ${b.name}: ${b.currentValue.toFixed(2)} ${b.unit} (baseline ${b.baseline.toFixed(2)}, +${b.pct.toFixed(1)}%)`)
const historyRows = history.map((h) => `${h.meta.ts}  ${h.meta.sha}  ${DIMENSIONS.map((d) => `${d.name}=${d.get(h)}`).join('  ')}`)

await sendBoxSignal({
    service: 'nightly-benchmark',
    fingerprint: `bench-regression-${args.label}`,
    message: [
        `Nightly benchmark regression on the \`${args.label}\` worker shape: ${breaches.length} metric(s) more than ${THRESHOLD_PCT}% worse than the trailing-14 median for ${CONSECUTIVE} runs in a row.`,
        '',
        ...lines,
        '',
        `Current SHA: ${current.meta.sha}`,
        `Run: ${args.runUrl}`,
        '',
        'Find the commit that caused this. The stack field lists the trailing runs oldest first with their SHA; the first run where the breached metrics step up brackets the culprit between its SHA and the previous one. Run `git log <prev>..<first-bad>` and read the diffs that touch packages/server, packages/core or benchmark/. The bench flow is built in benchmark/setup.ts (webhook trigger, math steps, a CODE step); check whether a commit changed how one of those executes. In the report give the culprit commit, why it costs this much, and whether it looks intended. Open a PR only if the regression is clearly unintended and the fix is small; this is a performance regression, so no failing test is expected and the next nightly run is the check. Linear: team SRE, project Stability, even when the culprit commit belongs to another team; the benchmark is SRE-owned and SRE decides whether to hand it off.',
    ].join('\n'),
    stack: historyRows.join('\n'),
})
console.error(`Sent Box signal for ${breaches.length} dimension(s).`)

async function sendBoxSignal(payload) {
    const res = await fetch(SIGNALS_URL, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json; charset=utf-8', Authorization: `Bearer ${SIGNALS_TOKEN}` },
        body: JSON.stringify(payload),
    })
    if (!res.ok) {
        console.error(`Box signals API responded ${res.status}: ${await res.text().catch(() => '')}`)
        process.exit(1)
    }
}

async function fetchHistory({ label }) {
    const sql = `SELECT toString(ts) AS ts, sha, p50_ms, p90_ms, p99_ms, mean_ms, req_sec, queue_p90_ms, service_p90_ms, provision_p50_ms, boot_p50_ms, app_cpu_p95, app_mem_max_mb, worker_cpu_p95, worker_mem_max_mb FROM bench.results WHERE label = '${label}' ORDER BY ts DESC LIMIT 14 FORMAT JSONEachRow`
    const url = `https://${CH_HOST}:8443/?query=${encodeURIComponent(sql)}`
    const auth = Buffer.from(`${CH_USER}:${CH_PASS}`).toString('base64')
    const res = await fetch(url, { headers: { Authorization: `Basic ${auth}` } })
    if (!res.ok) {
        const body = await res.text().catch(() => '')
        throw new Error(`ClickHouse history query failed (${res.status}): ${body}`)
    }
    const body = (await res.text()).trim()
    if (!body) return []
    return body.split('\n').map((line) => JSON.parse(line)).map(toResultShape)
}

function toResultShape(row) {
    return {
        meta: { ts: row.ts, sha: row.sha },
        latency: { p50Ms: row.p50_ms, p90Ms: row.p90_ms, p99Ms: row.p99_ms, meanMs: row.mean_ms },
        throughput: { reqSec: row.req_sec },
        timeline: {
            queueWaitP90Ms: row.queue_p90_ms,
            serviceP90Ms: row.service_p90_ms,
            provisionP50Ms: row.provision_p50_ms,
            bootP50Ms: row.boot_p50_ms,
        },
        resources: {
            app: { cpuPct: { p95: row.app_cpu_p95 }, memMb: { max: row.app_mem_max_mb } },
            worker: { cpuPct: { p95: row.worker_cpu_p95 }, memMb: { max: row.worker_mem_max_mb } },
        },
    }
}

function median(values) {
    const sorted = [...values].filter((v) => Number.isFinite(v)).sort((a, b) => a - b)
    if (sorted.length === 0) return 0
    const mid = Math.floor(sorted.length / 2)
    return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]
}

function shiftPct({ current, baseline, invert }) {
    if (baseline === 0) return 0
    const raw = ((current - baseline) / baseline) * 100
    return invert ? -raw : raw
}