Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

ci(perf): gate backend requests and response bytes per command

KageBinary5140bc3db270
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2729e71a..560ef14f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -312,6 +312,32 @@ jobs:
# dist/ is already built above; test:e2e would rebuild it.
run: npx vitest run --config vitest.e2e.config.ts
+ # Performance gate (ix-cli/scripts/perf-gate.mjs): backend requests and
+ # response bytes per command on a fixed fixture -- Ix itself at a pinned
+ # tag -- against ix-cli/test/perf/budgets.json. Fails above budget + 10%,
+ # which is what an added N+1 or a doubled payload looks like. Wall time is
+ # reported only.
+ - name: Check out the perf fixture (Ix v0.12.0)
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: v0.12.0
+ path: perf-fixture
+ persist-credentials: false
+
+ - name: Performance gate (requests, bytes)
+ working-directory: ix-cli
+ env:
+ IX_E2E: "1"
+ run: node scripts/perf-gate.mjs "$GITHUB_WORKSPACE/perf-fixture" --out "$RUNNER_TEMP/perf-results.json"
+
+ - name: Upload perf results
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: perf-results
+ path: ${{ runner.temp }}/perf-results.json
+ if-no-files-found: ignore
+
- name: Upload graph diffs on failure
if: failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
diff --git a/ix-cli/scripts/perf-gate.mjs b/ix-cli/scripts/perf-gate.mjs
new file mode 100644
index 00000000..606bb7bc
--- /dev/null
+++ b/ix-cli/scripts/perf-gate.mjs
@@ -0,0 +1,225 @@
+// Copyright 2026 Ix Infrastructure Inc.
+
+// Performance gate: backend requests, response bytes and wall time per command
+// on a fixed fixture, against budgets in test/perf/budgets.json.
+//
+// IX_E2E=1 node scripts/perf-gate.mjs <fixture-repo> [--runs 3] [--update] [--out results.json]
+//
+// Runs against the real-backend harness stack (test/e2e, `npm run e2e:up`),
+// with the same IX_E2E_* variables and the same refusal to touch a shared
+// stack: every map run starts from POST /v1/reset. Each command runs --runs
+// times through scripts/fetch-log.mjs; the medians are compared with the
+// budgets. Requests and bytes are deterministic for a fixed fixture and backend
+// image, so they fail above budget + 10%. Wall time depends on the machine and
+// is reported, never failed, until there is CI history to set it from.
+// --update rewrites the budgets from this run's medians.
+//
+// Needs a built CLI (npm run build).
+
+/* global process, console, fetch, URL, performance */
+
+import { spawnSync } from "node:child_process";
+import { appendFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const CLI = resolve(HERE, "../dist/cli/main.js");
+const FETCH_LOG = resolve(HERE, "fetch-log.mjs");
+const BUDGETS = resolve(HERE, "../test/perf/budgets.json");
+const PROTECTED_PORTS = new Set([8090, 8091, 8099, 8100, 8190, 8529, 8530, 8539, 8540, 8629]);
+const TOLERANCE = 1.1;
+
+function parseArgs(argv) {
+ const opts = { runs: 3, update: false, out: null, fixture: null };
+ for (let i = 0; i < argv.length; i++) {
+ const a = argv[i];
+ if (a === "--runs") opts.runs = Number(argv[++i]);
+ else if (a === "--update") opts.update = true;
+ else if (a === "--out") opts.out = argv[++i];
+ else opts.fixture = a;
+ }
+ if (!opts.fixture || !Number.isInteger(opts.runs) || opts.runs < 1) {
+ console.error("usage: IX_E2E=1 node scripts/perf-gate.mjs <fixture-repo> [--runs 3] [--update] [--out results.json]");
+ process.exit(2);
+ }
+ return opts;
+}
+
+function endpoint() {
+ if (process.env.IX_E2E !== "1") {
+ throw new Error("Set IX_E2E=1: the gate resets the backend's database before every map.");
+ }
+ const url = new URL(process.env.IX_E2E_ENDPOINT ?? `http://127.0.0.1:${process.env.IX_E2E_BACKEND_PORT ?? "8290"}`);
+ if (!["localhost", "127.0.0.1", "[::1]"].includes(url.hostname) || PROTECTED_PORTS.has(Number(url.port))) {
+ throw new Error(`${url.href}: not a harness stack (loopback, and none of the shared ports ${[...PROTECTED_PORTS].join(", ")})`);
+ }
+ return url.href.replace(/\/$/, "");
+}
+
+async function reset(base) {
+ const headers = { "content-type": "application/json" };
+ if (process.env.IX_TOKEN) headers.authorization = `Bearer ${process.env.IX_TOKEN}`;
+ const res = await fetch(`${base}/v1/reset`, { method: "POST", headers, body: "{}" });
+ if (!res.ok) throw new Error(`POST /v1/reset: ${res.status}`);
+}
+
+/** A fresh copy of the fixture (same directory name) and an empty IX_HOME. */
+function checkout(fixture, scratch) {
+ const root = mkdtempSync(join(scratch, "run-"));
+ const dir = join(root, basename(fixture));
+ cpSync(fixture, dir, { recursive: true });
+ const ixHome = join(root, "ix-home");
+ mkdirSync(ixHome);
+ return { dir, ixHome };
+}
+
+/** One CLI run: { requests, bytes, wallMs, status }. */
+function measure(base, co, args, log) {
+ rmSync(log, { force: true });
+ const t0 = performance.now();
+ const res = spawnSync(process.execPath, ["--import", FETCH_LOG, CLI, ...args], {
+ cwd: co.dir,
+ encoding: "utf8",
+ timeout: 600_000,
+ env: { ...process.env, IX_HOME: co.ixHome, IX_ENDPOINT: base, IX_NO_UPDATE_CHECK: "1", NO_COLOR: "1", IX_FETCH_LOG: log },
+ });
+ const wallMs = Math.round(performance.now() - t0);
+ const lines = existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) : [];
+ const reqs = lines.filter((l) => l.ev === "req");
+ return {
+ requests: reqs.length,
+ bytes: reqs.reduce((n, r) => n + (r.bytes ?? 0), 0),
+ wallMs,
+ status: res.status,
+ stderr: (res.stderr ?? "").slice(-400),
+ };
+}
+
+const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];
+
+/**
+ * The scenarios. `prepare` runs unmeasured before each measured run; `args` is
+ * what is measured. Map scenarios reset the backend first; read scenarios run
+ * against one map of the fixture made before them.
+ */
+function scenarios(issueFile) {
+ const editTs = (co, i) => appendFileSync(join(co.dir, "ix-cli/src/cli/format.ts"), `\n// perf-gate edit ${i}\n`);
+ const addFile = (co, i) =>
+ writeFileSync(join(co.dir, `ix-cli/src/cli/perf-gate-${i}.ts`), `export const perfGate${i} = ${i};\n`);
+ return [
+ { name: "map (fresh)", fresh: true, args: (co) => ["map", co.dir, "--silent"] },
+ { name: "map (no-op)", args: (co) => ["map", co.dir, "--silent"] },
+ { name: "map (one-line TS edit)", prepare: editTs, args: (co) => ["map", co.dir, "--silent"] },
+ { name: "map (add one file)", prepare: addFile, args: (co) => ["map", co.dir, "--silent"] },
+ { name: "context IxClient", args: () => ["context", "IxClient", "--format", "json"] },
+ { name: "context --from-issue", args: () => ["context", "--from-issue", issueFile, "--format", "json"] },
+ { name: "impact resolveWorkspaceRoot", args: () => ["impact", "resolveWorkspaceRoot", "--format", "json"] },
+ { name: "rank dependents/function", args: () => ["rank", "--by", "dependents", "--kind", "function", "--format", "json"] },
+ { name: "callers getEndpoint", args: () => ["callers", "getEndpoint", "--format", "json"] },
+ ];
+}
+
+async function main() {
+ const opts = parseArgs(process.argv.slice(2));
+ const base = endpoint();
+ if (!existsSync(CLI)) throw new Error(`${CLI} not found: run npm run build first`);
+ const fixture = resolve(opts.fixture);
+ const scratch = mkdtempSync(join(tmpdir(), "ix-perf-gate-"));
+ const log = join(scratch, "fetch.jsonl");
+ const issueFile = join(scratch, "issue.md");
+ writeFileSync(issueFile, "ix map reports success but ix search finds nothing after a branch switch; the workspace root looks wrong.\n");
+ const results = {};
+
+ try {
+ const all = scenarios(issueFile);
+ const measureRuns = async (sc, co0) => {
+ const runs = [];
+ for (let i = 0; i < opts.runs; i++) {
+ let co = co0;
+ if (sc.fresh) {
+ await reset(base);
+ co = checkout(fixture, scratch);
+ }
+ sc.prepare?.(co, i);
+ const r = measure(base, co, sc.args(co), log);
+ if (sc.name.startsWith("map") && r.status !== 0) throw new Error(`${sc.name} failed (exit ${r.status}): ${r.stderr}`);
+ runs.push(r);
+ }
+ results[sc.name] = {
+ requests: median(runs.map((r) => r.requests)),
+ bytes: median(runs.map((r) => r.bytes)),
+ wallMs: median(runs.map((r) => r.wallMs)),
+ };
+ };
+
+ // Fresh maps first: each resets the backend and maps a new copy.
+ for (const sc of all.filter((s) => s.fresh)) await measureRuns(sc, null);
+
+ // Then one mapped copy shared by every other scenario, in order.
+ await reset(base);
+ const shared = checkout(fixture, scratch);
+ const first = measure(base, shared, ["map", shared.dir, "--silent"], log);
+ if (first.status !== 0) throw new Error(`initial map failed (exit ${first.status}): ${first.stderr}`);
+ for (const sc of all.filter((s) => !s.fresh)) await measureRuns(sc, shared);
+ } finally {
+ await reset(base).catch(() => {});
+ rmSync(scratch, { recursive: true, force: true });
+ }
+
+ if (opts.out) writeFileSync(opts.out, JSON.stringify(results, null, 2) + "\n");
+ if (opts.update) {
+ // A command's own tolerance survives an update; context's fan-out is not
+ // deterministic yet, so it starts looser than the default.
+ const previous = existsSync(BUDGETS) ? JSON.parse(readFileSync(BUDGETS, "utf8")).budgets ?? {} : {};
+ const budgets = Object.fromEntries(Object.entries(results).map(([name, m]) => {
+ const tolerance = previous[name]?.tolerance ?? (name.startsWith("context") ? 1.25 : undefined);
+ return [name, tolerance ? { ...m, tolerance } : m];
+ }));
+ writeFileSync(BUDGETS, JSON.stringify({ fixture: basename(fixture), budgets }, null, 2) + "\n");
+ console.log(`budgets written to ${BUDGETS}`);
+ }
+ return report(results, opts.update ? null : JSON.parse(readFileSync(BUDGETS, "utf8")).budgets);
+}
+
+function report(results, budgets) {
+ const rows = [];
+ let failed = 0;
+ for (const [name, m] of Object.entries(results)) {
+ const b = budgets?.[name];
+ const verdict = (value, budget, tolerance) => {
+ if (!budget) return "";
+ if (value > budget * tolerance) return "OVER";
+ return "";
+ };
+ const tolerance = b?.tolerance ?? TOLERANCE;
+ const reqV = verdict(m.requests, b?.requests, tolerance);
+ const byteV = verdict(m.bytes, b?.bytes, tolerance);
+ const wallV = b?.wallMs && m.wallMs > b.wallMs * 3 ? "slow (advisory)" : "";
+ if (reqV || byteV) failed++;
+ rows.push(
+ `${name.padEnd(30)} req ${String(m.requests).padStart(5)}${b ? ` / ${b.requests}` : ""} ${reqV}`.padEnd(56) +
+ `bytes ${String(m.bytes).padStart(9)}${b ? ` / ${b.bytes}` : ""} ${byteV}`.padEnd(36) +
+ `wall ${m.wallMs} ms${b ? ` / ${b.wallMs}` : ""} ${wallV}`,
+ );
+ }
+ console.log(rows.join("\n"));
+ if (budgets && failed) {
+ console.error(
+ `\n${failed} command(s) over budget (requests or bytes above budget + its tolerance: ${Math.round((TOLERANCE - 1) * 100)}% unless budgets.json sets one). ` +
+ "If the increase is intended, run with --update and commit test/perf/budgets.json.",
+ );
+ return 1;
+ }
+ return 0;
+}
+
+main().then(
+ (code) => process.exit(code),
+ (err) => {
+ console.error(err instanceof Error ? err.message : err);
+ process.exit(2);
+ },
+);
+
diff --git a/ix-cli/test/e2e/README.md b/ix-cli/test/e2e/README.md
index 02ab2231..f81f1f3d 100644
--- a/ix-cli/test/e2e/README.md
+++ b/ix-cli/test/e2e/README.md
@@ -121,6 +121,28 @@ the smoke test, against the pinned image on the default ports, and is part of
stacks' logs. A backend fix reaches this job only once its image is released
and the pin is moved to it, normally by the Dependabot PR.
+## Performance gate
+
+`scripts/perf-gate.mjs` runs on the same stack. For a fixed fixture, Ix itself
+at `v0.12.0`, it measures backend requests, response bytes and wall time for
+these commands: a fresh map, a no-op map, a one-line edit then map, an
+added file then map, `context`, `context --from-issue`, `impact`, `rank` and
+`callers`. Each runs three times, and it compares the medians with
+`test/perf/budgets.json`.
+
+```sh
+git clone --depth 1 --branch v0.12.0 https://github.com/ix-infrastructure/Ix /tmp/perf-fixture
+IX_E2E=1 node scripts/perf-gate.mjs /tmp/perf-fixture # check against the budgets
+IX_E2E=1 node scripts/perf-gate.mjs /tmp/perf-fixture --update # rewrite them from this run
+```
+
+Requests and bytes are deterministic for a fixed fixture and backend image, so
+the gate fails when either is more than 10% over budget. That is what an added
+N+1 or a doubled payload looks like. Wall time is only reported. A change that
+makes a command cheaper should lower its budget in the same PR (`--update`,
+then commit the file), so the gain can't be lost again unnoticed. In CI it is a
+step of job `e2e`, and its results are uploaded as `perf-results`.
+
## Fixtures
`fixtures/polyglot/` has 23 files: Python with cross-file calls (`app/`),
diff --git a/ix-cli/test/perf/budgets.json b/ix-cli/test/perf/budgets.json
new file mode 100644
index 00000000..78d1bdbc
--- /dev/null
+++ b/ix-cli/test/perf/budgets.json
@@ -0,0 +1,52 @@
+{
+ "fixture": "perf-fixture",
+ "budgets": {
+ "map (fresh)": {
+ "requests": 9,
+ "bytes": 940272,
+ "wallMs": 8319
+ },
+ "map (no-op)": {
+ "requests": 6,
+ "bytes": 914,
+ "wallMs": 282
+ },
+ "map (one-line TS edit)": {
+ "requests": 10,
+ "bytes": 925866,
+ "wallMs": 2192
+ },
+ "map (add one file)": {
+ "requests": 459,
+ "bytes": 966736,
+ "wallMs": 6555
+ },
+ "context IxClient": {
+ "requests": 110,
+ "bytes": 5630913,
+ "wallMs": 1004,
+ "tolerance": 1.25
+ },
+ "context --from-issue": {
+ "requests": 115,
+ "bytes": 2770336,
+ "wallMs": 843,
+ "tolerance": 1.25
+ },
+ "impact resolveWorkspaceRoot": {
+ "requests": 26,
+ "bytes": 43144,
+ "wallMs": 304
+ },
+ "rank dependents/function": {
+ "requests": 1819,
+ "bytes": 5494180,
+ "wallMs": 1840
+ },
+ "callers getEndpoint": {
+ "requests": 3,
+ "bytes": 41825,
+ "wallMs": 234
+ }
+ }
+}