Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
ci(perf): gate backend requests and response bytes per command
KageBinary5140bc3db270
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 2729e71a..560ef14f 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -312,6 +312,32 @@ jobs:# dist/ is already built above; test:e2e would rebuild it.run: npx vitest run --config vitest.e2e.config.ts+ # Performance gate (ix-cli/scripts/perf-gate.mjs): backend requests and+ # response bytes per command on a fixed fixture -- Ix itself at a pinned+ # tag -- against ix-cli/test/perf/budgets.json. Fails above budget + 10%,+ # which is what an added N+1 or a doubled payload looks like. Wall time is+ # reported only.+ - name: Check out the perf fixture (Ix v0.12.0)+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: v0.12.0+ path: perf-fixture+ persist-credentials: false++ - name: Performance gate (requests, bytes)+ working-directory: ix-cli+ env:+ IX_E2E: "1"+ run: node scripts/perf-gate.mjs "$GITHUB_WORKSPACE/perf-fixture" --out "$RUNNER_TEMP/perf-results.json"++ - name: Upload perf results+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: perf-results+ path: ${{ runner.temp }}/perf-results.json+ if-no-files-found: ignore+- name: Upload graph diffs on failureif: failure()uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1diff --git a/ix-cli/scripts/perf-gate.mjs b/ix-cli/scripts/perf-gate.mjsnew file mode 100644index 00000000..606bb7bc--- /dev/null+++ b/ix-cli/scripts/perf-gate.mjs@@ -0,0 +1,225 @@+// Copyright 2026 Ix Infrastructure Inc.++// Performance gate: backend requests, response bytes and wall time per command+// on a fixed fixture, against budgets in test/perf/budgets.json.+//+// IX_E2E=1 node scripts/perf-gate.mjs <fixture-repo> [--runs 3] [--update] [--out results.json]+//+// Runs against the real-backend harness stack (test/e2e, `npm run e2e:up`),+// with the same IX_E2E_* variables and the same refusal to touch a shared+// stack: every map run starts from POST /v1/reset. Each command runs --runs+// times through scripts/fetch-log.mjs; the medians are compared with the+// budgets. Requests and bytes are deterministic for a fixed fixture and backend+// image, so they fail above budget + 10%. Wall time depends on the machine and+// is reported, never failed, until there is CI history to set it from.+// --update rewrites the budgets from this run's medians.+//+// Needs a built CLI (npm run build).++/* global process, console, fetch, URL, performance */++import { spawnSync } from "node:child_process";+import { appendFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { basename, dirname, join, resolve } from "node:path";+import { fileURLToPath } from "node:url";++const HERE = dirname(fileURLToPath(import.meta.url));+const CLI = resolve(HERE, "../dist/cli/main.js");+const FETCH_LOG = resolve(HERE, "fetch-log.mjs");+const BUDGETS = resolve(HERE, "../test/perf/budgets.json");+const PROTECTED_PORTS = new Set([8090, 8091, 8099, 8100, 8190, 8529, 8530, 8539, 8540, 8629]);+const TOLERANCE = 1.1;++function parseArgs(argv) {+ const opts = { runs: 3, update: false, out: null, fixture: null };+ for (let i = 0; i < argv.length; i++) {+ const a = argv[i];+ if (a === "--runs") opts.runs = Number(argv[++i]);+ else if (a === "--update") opts.update = true;+ else if (a === "--out") opts.out = argv[++i];+ else opts.fixture = a;+ }+ if (!opts.fixture || !Number.isInteger(opts.runs) || opts.runs < 1) {+ console.error("usage: IX_E2E=1 node scripts/perf-gate.mjs <fixture-repo> [--runs 3] [--update] [--out results.json]");+ process.exit(2);+ }+ return opts;+}++function endpoint() {+ if (process.env.IX_E2E !== "1") {+ throw new Error("Set IX_E2E=1: the gate resets the backend's database before every map.");+ }+ const url = new URL(process.env.IX_E2E_ENDPOINT ?? `http://127.0.0.1:${process.env.IX_E2E_BACKEND_PORT ?? "8290"}`);+ if (!["localhost", "127.0.0.1", "[::1]"].includes(url.hostname) || PROTECTED_PORTS.has(Number(url.port))) {+ throw new Error(`${url.href}: not a harness stack (loopback, and none of the shared ports ${[...PROTECTED_PORTS].join(", ")})`);+ }+ return url.href.replace(/\/$/, "");+}++async function reset(base) {+ const headers = { "content-type": "application/json" };+ if (process.env.IX_TOKEN) headers.authorization = `Bearer ${process.env.IX_TOKEN}`;+ const res = await fetch(`${base}/v1/reset`, { method: "POST", headers, body: "{}" });+ if (!res.ok) throw new Error(`POST /v1/reset: ${res.status}`);+}++/** A fresh copy of the fixture (same directory name) and an empty IX_HOME. */+function checkout(fixture, scratch) {+ const root = mkdtempSync(join(scratch, "run-"));+ const dir = join(root, basename(fixture));+ cpSync(fixture, dir, { recursive: true });+ const ixHome = join(root, "ix-home");+ mkdirSync(ixHome);+ return { dir, ixHome };+}++/** One CLI run: { requests, bytes, wallMs, status }. */+function measure(base, co, args, log) {+ rmSync(log, { force: true });+ const t0 = performance.now();+ const res = spawnSync(process.execPath, ["--import", FETCH_LOG, CLI, ...args], {+ cwd: co.dir,+ encoding: "utf8",+ timeout: 600_000,+ env: { ...process.env, IX_HOME: co.ixHome, IX_ENDPOINT: base, IX_NO_UPDATE_CHECK: "1", NO_COLOR: "1", IX_FETCH_LOG: log },+ });+ const wallMs = Math.round(performance.now() - t0);+ const lines = existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) : [];+ const reqs = lines.filter((l) => l.ev === "req");+ return {+ requests: reqs.length,+ bytes: reqs.reduce((n, r) => n + (r.bytes ?? 0), 0),+ wallMs,+ status: res.status,+ stderr: (res.stderr ?? "").slice(-400),+ };+}++const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];++/**+ * The scenarios. `prepare` runs unmeasured before each measured run; `args` is+ * what is measured. Map scenarios reset the backend first; read scenarios run+ * against one map of the fixture made before them.+ */+function scenarios(issueFile) {+ const editTs = (co, i) => appendFileSync(join(co.dir, "ix-cli/src/cli/format.ts"), `\n// perf-gate edit ${i}\n`);+ const addFile = (co, i) =>+ writeFileSync(join(co.dir, `ix-cli/src/cli/perf-gate-${i}.ts`), `export const perfGate${i} = ${i};\n`);+ return [+ { name: "map (fresh)", fresh: true, args: (co) => ["map", co.dir, "--silent"] },+ { name: "map (no-op)", args: (co) => ["map", co.dir, "--silent"] },+ { name: "map (one-line TS edit)", prepare: editTs, args: (co) => ["map", co.dir, "--silent"] },+ { name: "map (add one file)", prepare: addFile, args: (co) => ["map", co.dir, "--silent"] },+ { name: "context IxClient", args: () => ["context", "IxClient", "--format", "json"] },+ { name: "context --from-issue", args: () => ["context", "--from-issue", issueFile, "--format", "json"] },+ { name: "impact resolveWorkspaceRoot", args: () => ["impact", "resolveWorkspaceRoot", "--format", "json"] },+ { name: "rank dependents/function", args: () => ["rank", "--by", "dependents", "--kind", "function", "--format", "json"] },+ { name: "callers getEndpoint", args: () => ["callers", "getEndpoint", "--format", "json"] },+ ];+}++async function main() {+ const opts = parseArgs(process.argv.slice(2));+ const base = endpoint();+ if (!existsSync(CLI)) throw new Error(`${CLI} not found: run npm run build first`);+ const fixture = resolve(opts.fixture);+ const scratch = mkdtempSync(join(tmpdir(), "ix-perf-gate-"));+ const log = join(scratch, "fetch.jsonl");+ const issueFile = join(scratch, "issue.md");+ writeFileSync(issueFile, "ix map reports success but ix search finds nothing after a branch switch; the workspace root looks wrong.\n");+ const results = {};++ try {+ const all = scenarios(issueFile);+ const measureRuns = async (sc, co0) => {+ const runs = [];+ for (let i = 0; i < opts.runs; i++) {+ let co = co0;+ if (sc.fresh) {+ await reset(base);+ co = checkout(fixture, scratch);+ }+ sc.prepare?.(co, i);+ const r = measure(base, co, sc.args(co), log);+ if (sc.name.startsWith("map") && r.status !== 0) throw new Error(`${sc.name} failed (exit ${r.status}): ${r.stderr}`);+ runs.push(r);+ }+ results[sc.name] = {+ requests: median(runs.map((r) => r.requests)),+ bytes: median(runs.map((r) => r.bytes)),+ wallMs: median(runs.map((r) => r.wallMs)),+ };+ };++ // Fresh maps first: each resets the backend and maps a new copy.+ for (const sc of all.filter((s) => s.fresh)) await measureRuns(sc, null);++ // Then one mapped copy shared by every other scenario, in order.+ await reset(base);+ const shared = checkout(fixture, scratch);+ const first = measure(base, shared, ["map", shared.dir, "--silent"], log);+ if (first.status !== 0) throw new Error(`initial map failed (exit ${first.status}): ${first.stderr}`);+ for (const sc of all.filter((s) => !s.fresh)) await measureRuns(sc, shared);+ } finally {+ await reset(base).catch(() => {});+ rmSync(scratch, { recursive: true, force: true });+ }++ if (opts.out) writeFileSync(opts.out, JSON.stringify(results, null, 2) + "\n");+ if (opts.update) {+ // A command's own tolerance survives an update; context's fan-out is not+ // deterministic yet, so it starts looser than the default.+ const previous = existsSync(BUDGETS) ? JSON.parse(readFileSync(BUDGETS, "utf8")).budgets ?? {} : {};+ const budgets = Object.fromEntries(Object.entries(results).map(([name, m]) => {+ const tolerance = previous[name]?.tolerance ?? (name.startsWith("context") ? 1.25 : undefined);+ return [name, tolerance ? { ...m, tolerance } : m];+ }));+ writeFileSync(BUDGETS, JSON.stringify({ fixture: basename(fixture), budgets }, null, 2) + "\n");+ console.log(`budgets written to ${BUDGETS}`);+ }+ return report(results, opts.update ? null : JSON.parse(readFileSync(BUDGETS, "utf8")).budgets);+}++function report(results, budgets) {+ const rows = [];+ let failed = 0;+ for (const [name, m] of Object.entries(results)) {+ const b = budgets?.[name];+ const verdict = (value, budget, tolerance) => {+ if (!budget) return "";+ if (value > budget * tolerance) return "OVER";+ return "";+ };+ const tolerance = b?.tolerance ?? TOLERANCE;+ const reqV = verdict(m.requests, b?.requests, tolerance);+ const byteV = verdict(m.bytes, b?.bytes, tolerance);+ const wallV = b?.wallMs && m.wallMs > b.wallMs * 3 ? "slow (advisory)" : "";+ if (reqV || byteV) failed++;+ rows.push(+ `${name.padEnd(30)} req ${String(m.requests).padStart(5)}${b ? ` / ${b.requests}` : ""} ${reqV}`.padEnd(56) ++ `bytes ${String(m.bytes).padStart(9)}${b ? ` / ${b.bytes}` : ""} ${byteV}`.padEnd(36) ++ `wall ${m.wallMs} ms${b ? ` / ${b.wallMs}` : ""} ${wallV}`,+ );+ }+ console.log(rows.join("\n"));+ if (budgets && failed) {+ console.error(+ `\n${failed} command(s) over budget (requests or bytes above budget + its tolerance: ${Math.round((TOLERANCE - 1) * 100)}% unless budgets.json sets one). ` ++ "If the increase is intended, run with --update and commit test/perf/budgets.json.",+ );+ return 1;+ }+ return 0;+}++main().then(+ (code) => process.exit(code),+ (err) => {+ console.error(err instanceof Error ? err.message : err);+ process.exit(2);+ },+);+diff --git a/ix-cli/test/e2e/README.md b/ix-cli/test/e2e/README.mdindex 02ab2231..f81f1f3d 100644--- a/ix-cli/test/e2e/README.md+++ b/ix-cli/test/e2e/README.md@@ -121,6 +121,28 @@ the smoke test, against the pinned image on the default ports, and is part ofstacks' logs. A backend fix reaches this job only once its image is releasedand the pin is moved to it, normally by the Dependabot PR.+## Performance gate++`scripts/perf-gate.mjs` runs on the same stack. For a fixed fixture, Ix itself+at `v0.12.0`, it measures backend requests, response bytes and wall time for+these commands: a fresh map, a no-op map, a one-line edit then map, an+added file then map, `context`, `context --from-issue`, `impact`, `rank` and+`callers`. Each runs three times, and it compares the medians with+`test/perf/budgets.json`.++```sh+git clone --depth 1 --branch v0.12.0 https://github.com/ix-infrastructure/Ix /tmp/perf-fixture+IX_E2E=1 node scripts/perf-gate.mjs /tmp/perf-fixture # check against the budgets+IX_E2E=1 node scripts/perf-gate.mjs /tmp/perf-fixture --update # rewrite them from this run+```++Requests and bytes are deterministic for a fixed fixture and backend image, so+the gate fails when either is more than 10% over budget. That is what an added+N+1 or a doubled payload looks like. Wall time is only reported. A change that+makes a command cheaper should lower its budget in the same PR (`--update`,+then commit the file), so the gain can't be lost again unnoticed. In CI it is a+step of job `e2e`, and its results are uploaded as `perf-results`.+## Fixtures`fixtures/polyglot/` has 23 files: Python with cross-file calls (`app/`),diff --git a/ix-cli/test/perf/budgets.json b/ix-cli/test/perf/budgets.jsonnew file mode 100644index 00000000..78d1bdbc--- /dev/null+++ b/ix-cli/test/perf/budgets.json@@ -0,0 +1,52 @@+{+ "fixture": "perf-fixture",+ "budgets": {+ "map (fresh)": {+ "requests": 9,+ "bytes": 940272,+ "wallMs": 8319+ },+ "map (no-op)": {+ "requests": 6,+ "bytes": 914,+ "wallMs": 282+ },+ "map (one-line TS edit)": {+ "requests": 10,+ "bytes": 925866,+ "wallMs": 2192+ },+ "map (add one file)": {+ "requests": 459,+ "bytes": 966736,+ "wallMs": 6555+ },+ "context IxClient": {+ "requests": 110,+ "bytes": 5630913,+ "wallMs": 1004,+ "tolerance": 1.25+ },+ "context --from-issue": {+ "requests": 115,+ "bytes": 2770336,+ "wallMs": 843,+ "tolerance": 1.25+ },+ "impact resolveWorkspaceRoot": {+ "requests": 26,+ "bytes": 43144,+ "wallMs": 304+ },+ "rank dependents/function": {+ "requests": 1819,+ "bytes": 5494180,+ "wallMs": 1840+ },+ "callers getEndpoint": {+ "requests": 3,+ "bytes": 41825,+ "wallMs": 234+ }+ }+}