Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

10.7 KB
Raw Blame
// Copyright 2026 Ix Infrastructure Inc.

// Performance gate: backend requests, response bytes and wall time per command
// on a fixed fixture, against budgets in test/perf/budgets.json.
//
//   IX_E2E=1 node scripts/perf-gate.mjs <fixture-repo> [--runs 3] [--update] [--out results.json]
//
// Runs against the real-backend harness stack (test/e2e, `npm run e2e:up`),
// with the same IX_E2E_* variables and the same refusal to touch a shared
// stack: every map run starts from POST /v1/reset. Each command runs --runs
// times through scripts/fetch-log.mjs; the medians are compared with the
// budgets. Requests and bytes are deterministic for a fixed fixture and backend
// image, so they fail above budget + 10%. Wall time depends on the machine and
// is reported, never failed, until there is CI history to set it from.
// --update rewrites the budgets from this run's medians.
//
// Needs a built CLI (npm run build).

/* global process, console, fetch, URL, performance */

import { spawnSync } from "node:child_process";
import { appendFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const HERE = dirname(fileURLToPath(import.meta.url));
const CLI = resolve(HERE, "../dist/cli/main.js");
const FETCH_LOG = resolve(HERE, "fetch-log.mjs");
const BUDGETS = resolve(HERE, "../test/perf/budgets.json");
const PROTECTED_PORTS = new Set([8090, 8091, 8099, 8100, 8190, 8529, 8530, 8539, 8540, 8629]);
const TOLERANCE = 1.1;

function parseArgs(argv) {
  const opts = { runs: 3, update: false, out: null, fixture: null };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--runs") opts.runs = Number(argv[++i]);
    else if (a === "--update") opts.update = true;
    else if (a === "--out") opts.out = argv[++i];
    else opts.fixture = a;
  }
  if (!opts.fixture || !Number.isInteger(opts.runs) || opts.runs < 1) {
    console.error("usage: IX_E2E=1 node scripts/perf-gate.mjs <fixture-repo> [--runs 3] [--update] [--out results.json]");
    process.exit(2);
  }
  return opts;
}

function endpoint() {
  if (process.env.IX_E2E !== "1") {
    throw new Error("Set IX_E2E=1: the gate resets the backend's database before every map.");
  }
  const url = new URL(process.env.IX_E2E_ENDPOINT ?? `http://127.0.0.1:${process.env.IX_E2E_BACKEND_PORT ?? "8290"}`);
  if (!["localhost", "127.0.0.1", "[::1]"].includes(url.hostname) || PROTECTED_PORTS.has(Number(url.port))) {
    throw new Error(`${url.href}: not a harness stack (loopback, and none of the shared ports ${[...PROTECTED_PORTS].join(", ")})`);
  }
  return url.href.replace(/\/$/, "");
}

async function reset(base) {
  const headers = { "content-type": "application/json" };
  if (process.env.IX_TOKEN) headers.authorization = `Bearer ${process.env.IX_TOKEN}`;
  const res = await fetch(`${base}/v1/reset`, { method: "POST", headers, body: "{}" });
  if (!res.ok) throw new Error(`POST /v1/reset: ${res.status}`);
}

/**
 * Where every copy of the fixture is made: the same absolute path on every run
 * and every machine whose temp directory is /tmp, CI included.
 *
 * Node ids are hashed from the workspace's absolute path (workspaceIdForPath),
 * and context's fan-out follows the backend's order of a node's neighbours,
 * which follows those ids. A fresh random directory per run therefore drew a
 * different set of walk seeds each time, and `context` response bytes moved
 * by tens of percent between identical runs of the same commit.
 */
const CHECKOUT_ROOT = join(tmpdir(), "ix-perf-gate-checkout");

/** A fresh copy of the fixture (same directory name) and an empty IX_HOME. */
function checkout(fixture) {
  const root = CHECKOUT_ROOT;
  rmSync(root, { recursive: true, force: true });
  mkdirSync(root, { recursive: true });
  const dir = join(root, basename(fixture));
  cpSync(fixture, dir, { recursive: true });
  const ixHome = join(root, "ix-home");
  mkdirSync(ixHome);
  return { dir, ixHome };
}

/** One CLI run: { requests, bytes, wallMs, status }. */
function measure(base, co, args, log) {
  rmSync(log, { force: true });
  const t0 = performance.now();
  const res = spawnSync(process.execPath, ["--import", FETCH_LOG, CLI, ...args], {
    cwd: co.dir,
    encoding: "utf8",
    timeout: 600_000,
    env: { ...process.env, IX_HOME: co.ixHome, IX_ENDPOINT: base, IX_NO_UPDATE_CHECK: "1", NO_COLOR: "1", IX_FETCH_LOG: log },
  });
  const wallMs = Math.round(performance.now() - t0);
  const lines = existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) : [];
  const reqs = lines.filter((l) => l.ev === "req");
  return {
    requests: reqs.length,
    bytes: reqs.reduce((n, r) => n + (r.bytes ?? 0), 0),
    wallMs,
    status: res.status,
    stderr: (res.stderr ?? "").slice(-400),
  };
}

const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];

/**
 * The scenarios. `prepare` runs unmeasured before each measured run; `args` is
 * what is measured. Map scenarios reset the backend first; read scenarios run
 * against one map of the fixture made before them.
 */
function scenarios(issueFile) {
  const editTs = (co, i) => appendFileSync(join(co.dir, "ix-cli/src/cli/format.ts"), `\n// perf-gate edit ${i}\n`);
  const addFile = (co, i) =>
    writeFileSync(join(co.dir, `ix-cli/src/cli/perf-gate-${i}.ts`), `export const perfGate${i} = ${i};\n`);
  return [
    { name: "map (fresh)", fresh: true, args: (co) => ["map", co.dir, "--silent"] },
    { name: "map (no-op)", args: (co) => ["map", co.dir, "--silent"] },
    { name: "map (one-line TS edit)", prepare: editTs, args: (co) => ["map", co.dir, "--silent"] },
    { name: "map (add one file)", prepare: addFile, args: (co) => ["map", co.dir, "--silent"] },
    { name: "context IxClient", args: () => ["context", "IxClient", "--format", "json"] },
    { name: "context --from-issue", args: () => ["context", "--from-issue", issueFile, "--format", "json"] },
    { name: "impact resolveWorkspaceRoot", args: () => ["impact", "resolveWorkspaceRoot", "--format", "json"] },
    { name: "rank dependents/function", args: () => ["rank", "--by", "dependents", "--kind", "function", "--format", "json"] },
    { name: "callers getEndpoint", args: () => ["callers", "getEndpoint", "--format", "json"] },
  ];
}

async function main() {
  const opts = parseArgs(process.argv.slice(2));
  const base = endpoint();
  if (!existsSync(CLI)) throw new Error(`${CLI} not found: run npm run build first`);
  const fixture = resolve(opts.fixture);
  const scratch = mkdtempSync(join(tmpdir(), "ix-perf-gate-"));
  const log = join(scratch, "fetch.jsonl");
  const issueFile = join(scratch, "issue.md");
  writeFileSync(issueFile, "ix map reports success but ix search finds nothing after a branch switch; the workspace root looks wrong.\n");
  const results = {};

  try {
    const all = scenarios(issueFile);
    const measureRuns = async (sc, co0) => {
      const runs = [];
      for (let i = 0; i < opts.runs; i++) {
        let co = co0;
        if (sc.fresh) {
          await reset(base);
          co = checkout(fixture);
        }
        sc.prepare?.(co, i);
        const r = measure(base, co, sc.args(co), log);
        // Every command, not only the maps: a read that errors out (a renamed
        // symbol, a crash) makes fewer requests and fewer bytes, which the
        // budgets alone would score as an improvement.
        if (r.status !== 0) throw new Error(`${sc.name} failed (exit ${r.status}): ${r.stderr}`);
        runs.push(r);
      }
      results[sc.name] = {
        requests: median(runs.map((r) => r.requests)),
        bytes: median(runs.map((r) => r.bytes)),
        wallMs: median(runs.map((r) => r.wallMs)),
      };
    };

    // Fresh maps first: each resets the backend and maps a new copy.
    for (const sc of all.filter((s) => s.fresh)) await measureRuns(sc, null);

    // Then one mapped copy shared by every other scenario, in order.
    await reset(base);
    const shared = checkout(fixture);
    const first = measure(base, shared, ["map", shared.dir, "--silent"], log);
    if (first.status !== 0) throw new Error(`initial map failed (exit ${first.status}): ${first.stderr}`);
    for (const sc of all.filter((s) => !s.fresh)) await measureRuns(sc, shared);
  } finally {
    await reset(base).catch(() => {});
    rmSync(scratch, { recursive: true, force: true });
    rmSync(CHECKOUT_ROOT, { recursive: true, force: true });
  }

  if (opts.out) writeFileSync(opts.out, JSON.stringify(results, null, 2) + "\n");
  if (opts.update) {
    // A command's own tolerance survives an update; context's fan-out is not
    // deterministic yet, so it starts looser than the default.
    let previous = {};
    try {
      previous = JSON.parse(readFileSync(BUDGETS, "utf8")).budgets ?? {};
    } catch {
      // No budgets yet: start from this run.
    }
    const budgets = Object.fromEntries(Object.entries(results).map(([name, m]) => {
      const tolerance = previous[name]?.tolerance ?? (name.startsWith("context") ? 1.25 : undefined);
      return [name, tolerance ? { ...m, tolerance } : m];
    }));
    writeFileSync(BUDGETS, JSON.stringify({ fixture: basename(fixture), budgets }, null, 2) + "\n");
    console.log(`budgets written to ${BUDGETS}`);
  }
  return report(results, opts.update ? null : JSON.parse(readFileSync(BUDGETS, "utf8")).budgets);
}

function report(results, budgets) {
  const rows = [];
  let failed = 0;
  for (const [name, m] of Object.entries(results)) {
    const b = budgets?.[name];
    const verdict = (value, budget, tolerance) => {
      if (!budget) return "";
      if (value > budget * tolerance) return "OVER";
      return "";
    };
    const tolerance = b?.tolerance ?? TOLERANCE;
    const reqV = verdict(m.requests, b?.requests, tolerance);
    const byteV = verdict(m.bytes, b?.bytes, tolerance);
    const wallV = b?.wallMs && m.wallMs > b.wallMs * 3 ? "slow (advisory)" : "";
    if (reqV || byteV) failed++;
    rows.push(
      `${name.padEnd(30)} req ${String(m.requests).padStart(5)}${b ? ` / ${b.requests}` : ""} ${reqV}`.padEnd(56) +
        `bytes ${String(m.bytes).padStart(9)}${b ? ` / ${b.bytes}` : ""} ${byteV}`.padEnd(36) +
        `wall ${m.wallMs} ms${b ? ` / ${b.wallMs}` : ""} ${wallV}`,
    );
  }
  console.log(rows.join("\n"));
  if (budgets && failed) {
    console.error(
      `\n${failed} command(s) over budget (requests or bytes above budget + its tolerance: ${Math.round((TOLERANCE - 1) * 100)}% unless budgets.json sets one). ` +
        "If the increase is intended, run with --update and commit test/perf/budgets.json.",
    );
    return 1;
  }
  return 0;
}

main().then(
  (code) => process.exit(code),
  (err) => {
    console.error(err instanceof Error ? err.message : err);
    process.exit(2);
  },
);