Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
#!/usr/bin/env node
/**
* check-install-upgrade — proves the two install paths a real user takes, BEFORE publishing.
*
* `check:pack-boot` already proves a fresh install boots. It does NOT prove the path that
* actually broke us: installing the new version OVER an existing one, where ~110 SQLite
* migrations run against a populated database. v3.8.48 shipped as a hotfix precisely because
* the published 3.8.47 crashed on boot, and a v3.8.49 manual test on a real 3.8.48 box was
* what first exercised the upgrade path end to end.
*
* Phase A — clean install: fresh prefix + fresh DATA_DIR, install the packed tarball, boot.
* Phase B — upgrade install: fresh prefix + fresh DATA_DIR, install the PREVIOUS published
* version, boot it (creates + migrates the DB), stop, install the
* packed tarball over the SAME prefix, boot against the SAME DATA_DIR.
*
* Schema convergence is the third assertion, and its DIRECTION is what matters:
*
* fresh − upgraded ≠ ∅ → FAIL. A table a clean install creates but an upgrade does not
* means every existing user is missing structure the code expects.
* This is the failure mode that only ever bites upgraders.
* upgraded − fresh ≠ ∅ → WARN. Residue: a table whose CREATE left the migration set in
* some past cycle but survives in databases that already had it.
* Harmless, but it means the two paths do not converge — allowlist
* it explicitly so a NEW divergence is still visible.
*
* Usage:
* node scripts/check/check-install-upgrade.mjs [--from <version>] [--skip-upgrade]
*
* `--from` pins the previous version (default: the current `latest` dist-tag on npm).
* Requires `npm run build:cli` first — this is a --with-build gate, like check:pack-boot.
*/
import { execFileSync, spawn, spawnSync } from "node:child_process";
import crypto from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { DatabaseSync } from "node:sqlite";
const BOOT_DEADLINE_MS = 180_000;
const POLL_INTERVAL_MS = 2_000;
const ALLOWLIST_PATH = "config/quality/install-upgrade-allowlist.json";
const log = (msg) => console.log(`[install-upgrade] ${msg}`);
const warn = (msg) => console.log(`[install-upgrade] ⚠️ ${msg}`);
/** Root of the installed package inside an `npm install -g --prefix` tree. */
function packageRootFor(prefix) {
return path.join(prefix, "lib", "node_modules", "omniroute");
}
/**
* Credential for the health probe.
*
* GHSA-mvf8-qc78-5mxm hardened /api/monitoring/health: an ANONYMOUS caller now gets only
* `{ status }` — the version, node version, pid and provider config are reserved for a
* management principal (src/app/api/monitoring/health/route.ts → publicHealthView). An
* unauthenticated probe therefore reads `body.version === undefined`, and this gate's
* version assertion could never pass again; the v3.8.50 publish run failed with
* "clean: health reports version undefined, expected 3.8.50" for exactly that reason.
*
* The gate spawns the server itself, so it can mint the credential instead of guessing one:
* `OMNIROUTE_INTERNAL_SERVICE_TOKEN` + the `x-omniroute-internal-service-token` header is
* accepted by requireManagementAuth() via isTrustedLoopbackInternalServiceRequest(), and the
* probe is loopback by construction. Unlike the machine token `check:pack-boot` derives, this
* does not depend on a readable machine-id, and an older PREVIOUS version that never gated
* health simply ignores the header. The assertion keeps its full strength — it just presents
* a credential.
*/
const INTERNAL_SERVICE_TOKEN = crypto.randomBytes(32).toString("hex");
const INTERNAL_SERVICE_HEADER = "x-omniroute-internal-service-token";
/**
* Secondary credential: the same loopback machine token `check:pack-boot` derives from the
* packaged CLI. Sent alongside the internal-service token so a build that only honours one
* of the two still answers with the full payload.
*/
// The CLI token salt lives under <DATA_DIR> (#13679/#13909): derive against the DATA_DIR the
// probed server boots on, or the token never matches.
function derivePackagedCliToken(prefix, dataDir) {
const cliModuleUrl = pathToFileURL(
path.join(packageRootFor(prefix), "bin", "cli", "utils", "cliToken.mjs")
).href;
try {
return execFileSync(
process.execPath,
[
"--input-type=module",
"--eval",
"import(process.argv[1]).then(async m => process.stdout.write(await m.getCliToken()))",
cliModuleUrl,
],
{ encoding: "utf8", env: { ...process.env, DATA_DIR: dataDir } }
).trim();
} catch {
// A truncated/broken install cannot derive a token. Returning null keeps the boot
// probe running (it will fail loudly on its own) instead of crashing the gate here.
return null;
}
}
function pickTarball(packJson) {
const filename = JSON.parse(packJson)?.[0]?.filename;
if (!filename) throw new Error("npm pack --json returned no filename");
return filename;
}
/** Free-ish port per phase so a leaked child from a previous run cannot collide. */
function pickPort(offset) {
return 21000 + offset + (process.pid % 500);
}
function loadAllowlist(root) {
const file = path.join(root, ALLOWLIST_PATH);
if (!fs.existsSync(file)) return { residualTables: {} };
return JSON.parse(fs.readFileSync(file, "utf8"));
}
/**
* Pure verdict on schema convergence — exported so the asymmetry can be tested without
* building, packing and booting anything (the same reason check-test-masking exports its
* helpers: reproducing the deterministic part must not cost a full gate run).
*
* The two directions are NOT symmetric:
* onlyFresh → always a failure. Upgraders would be missing structure.
* onlyUpgraded → residue. Fails only when not recorded in the allowlist.
*/
export function evaluateConvergence({ freshTables, upgradedTables, residualAllowlist = {} }) {
const fresh = freshTables instanceof Set ? freshTables : new Set(freshTables ?? []);
const upgraded = upgradedTables instanceof Set ? upgradedTables : new Set(upgradedTables ?? []);
const onlyFresh = [...fresh].filter((t) => !upgraded.has(t)).sort();
const onlyUpgraded = [...upgraded].filter((t) => !fresh.has(t)).sort();
const unknownResidue = onlyUpgraded.filter((t) => !(t in residualAllowlist));
const failures = [];
if (onlyFresh.length) {
failures.push(
`schema divergence — tables a CLEAN install creates but an UPGRADE does not: ${onlyFresh.join(", ")}. ` +
"Every existing user would be missing these; add the migration."
);
}
if (unknownResidue.length) {
failures.push(
`NEW residual table(s) not in ${ALLOWLIST_PATH}: ${unknownResidue.join(", ")}. ` +
"Either drop them in a migration or record them with a justification."
);
}
return { ok: failures.length === 0, failures, onlyFresh, onlyUpgraded, unknownResidue };
}
/** Table names in a SQLite file, excluding sqlite_* internals. */
function readTables(dbPath) {
if (!fs.existsSync(dbPath)) return null;
const db = new DatabaseSync(dbPath, { readOnly: true });
try {
const rows = db
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")
.all();
return new Set(rows.map((r) => r.name));
} finally {
db.close();
}
}
function findDb(dataDir) {
const candidates = ["storage.sqlite", "omniroute.sqlite", "data.sqlite"];
for (const name of candidates) {
const p = path.join(dataDir, name);
if (fs.existsSync(p)) return p;
}
const found = fs.readdirSync(dataDir).find((f) => f.endsWith(".sqlite"));
return found ? path.join(dataDir, found) : null;
}
/** Boot an installed CLI and poll health. Returns { ok, version, failures, tail }. */
async function bootAndProbe({ prefix, dataDir, port, expectVersion, label }) {
const binPath = path.join(prefix, "bin", "omniroute");
if (!fs.existsSync(binPath)) {
return { ok: false, failures: [`${label}: bin not found at ${binPath}`], tail: [] };
}
const cliToken = derivePackagedCliToken(prefix, dataDir);
const probeHeaders = {
[INTERNAL_SERVICE_HEADER]: INTERNAL_SERVICE_TOKEN,
...(cliToken ? { "x-omniroute-cli-token": cliToken } : {}),
};
const child = spawn(binPath, ["serve", "--port", String(port)], {
env: {
...process.env,
PORT: String(port),
DATA_DIR: dataDir,
JWT_SECRET: "install-upgrade-gate-secret-with-sufficient-length",
API_KEY_SECRET: "install-upgrade-gate-api-key-secret-long",
DISABLE_SQLITE_AUTO_BACKUP: "true",
OMNIROUTE_SKIP_SYSTEM_TRUST: "1",
OMNIROUTE_INTERNAL_SERVICE_TOKEN: INTERNAL_SERVICE_TOKEN,
},
stdio: ["ignore", "pipe", "pipe"],
detached: true,
});
const tail = [];
const keepTail = (chunk) => {
tail.push(String(chunk));
while (tail.length > 80) tail.shift();
};
child.stdout.on("data", keepTail);
child.stderr.on("data", keepTail);
let childExit = null;
child.on("exit", (code) => {
childExit = code ?? -1;
});
const deadline = Date.now() + BOOT_DEADLINE_MS;
let result = { ok: false, failures: [`${label}: never became healthy`], tail };
while (Date.now() < deadline) {
if (childExit !== null) {
result = {
ok: false,
failures: [`${label}: exited with code ${childExit} before serving`],
tail,
};
break;
}
try {
const res = await fetch(`http://127.0.0.1:${port}/api/monitoring/health`, {
headers: probeHeaders,
});
const body = await res.json().catch(() => null);
if (res.status === 200 && body && typeof body === "object") {
const failures = [];
// `status` may legitimately report degraded (no providers configured) — the gate
// targets boot crashes and version mismatches, not health of a bare install.
const reportedVersion = body.version ?? body.system?.version;
if (expectVersion && reportedVersion !== expectVersion) {
failures.push(
`${label}: health reports version ${reportedVersion}, expected ${expectVersion}` +
(reportedVersion === undefined
? " — the payload carries no version at all, which is the ANONYMOUS health " +
"view: the probe's credentials were not accepted (see GHSA-mvf8-qc78-5mxm)"
: "")
);
}
result = { ok: failures.length === 0, version: reportedVersion, failures, tail };
break;
}
} catch {
// not listening yet
}
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
}
try {
if (childExit === null) process.kill(-child.pid, "SIGTERM");
} catch {
/* already gone */
}
// Give the process a moment to flush and release the SQLite handle before we read the file.
await new Promise((r) => setTimeout(r, 3_000));
return result;
}
/**
* `npm install` reports ENOSPC as a *warning* per failed tar entry and still exits 0.
*
* That is not a theoretical concern: on the v3.8.50 publish run the Phase B upgrade install
* emitted 5611 `npm warn tar TAR_ENTRY_ERROR ENOSPC: no space left on device` lines, exited
* 0, and left a truncated package behind. `omniroute serve` then "exited with code 0 before
* serving", no migration ever ran, and the gate concluded the release was missing 15 tables
* — a full false alarm produced by a full disk. Each install tree is ~3 GB, and the run
* builds two of them plus a ~275 MB tarball.
*
* So: surface the truncation at the install, where it is unambiguous.
*/
function npmInstallInto(prefix, spec, label = spec) {
// spawnSync (not execFileSync): execFileSync forwards the child's stderr straight to the
// parent's, so the ENOSPC warnings scrolled past in CI without the script ever seeing
// them. spawnSync hands both streams back.
const run = spawnSync(
"npm",
["install", "-g", "--prefix", prefix, "--no-audit", "--no-fund", spec],
{ encoding: "utf8", maxBuffer: 512 * 1024 * 1024 }
);
const output = `${run.stdout ?? ""}${run.stderr ?? ""}`;
// Keep the install log visible, but a truncated package emits thousands of identical
// warnings — collapse them so the real message is not buried.
const stderrLines = String(run.stderr ?? "").split("\n");
const shown = stderrLines.length > 60 ? stderrLines.slice(0, 40) : stderrLines;
if (String(run.stderr ?? "").trim()) {
process.stderr.write(shown.join("\n") + "\n");
if (stderrLines.length > 60) {
process.stderr.write(`[install-upgrade] … ${stderrLines.length - 40} more npm line(s)\n`);
}
}
assertNoDiskExhaustion(output, label);
if (run.error) throw run.error;
if (run.status !== 0) {
throw new Error(`${label}: npm install exited with code ${run.status}`);
}
}
export function assertNoDiskExhaustion(output, label) {
if (!/ENOSPC|no space left on device/i.test(output)) return;
const count = (output.match(/ENOSPC/g) ?? []).length;
throw new Error(
`${label}: the install ran out of disk space (${count} ENOSPC error(s) from npm). ` +
`The package tree is truncated, so anything measured from it — boot, schema, ` +
`migrations — is meaningless. Free space in ${workDirForMessages} (each install tree is ` +
`~3 GB) and re-run. This is an environment failure, NOT a schema divergence.`
);
}
/** Best-effort free bytes on the filesystem backing `dir`, or null when unavailable. */
function freeBytes(dir) {
try {
return fs.statfsSync(dir).bavail * fs.statfsSync(dir).bsize;
} catch {
return null;
}
}
const GB = 1024 ** 3;
// Set once the work directory exists, so the ENOSPC message names the filesystem that
// actually ran out — pointing at /tmp when the gate works elsewhere sends the reader to
// free space on the wrong volume (which is what happened during the v3.8.50 publish).
let workDirForMessages = os.tmpdir();
function resolvePreviousVersion(current, explicit) {
if (explicit) return explicit;
const out = execFileSync("npm", ["view", "omniroute", "dist-tags.latest"], { encoding: "utf8" });
const latest = out.trim();
if (!latest) throw new Error("could not resolve omniroute@latest from npm");
if (latest === current) {
// The version under test is already published (re-run of a shipped release): step back
// to the highest published version strictly below it.
const all = JSON.parse(
execFileSync("npm", ["view", "omniroute", "versions", "--json"], { encoding: "utf8" })
);
const stable = all.filter((v) => !/-(rc|alpha|beta|pre|next)/.test(v) && v !== current);
return stable[stable.length - 1];
}
return latest;
}
async function main() {
const ROOT = process.cwd();
const args = process.argv.slice(2);
const fromIdx = args.indexOf("--from");
const explicitFrom = fromIdx >= 0 ? args[fromIdx + 1] : null;
const skipUpgrade = args.includes("--skip-upgrade");
if (!fs.existsSync(path.join(ROOT, "dist", "server.js"))) {
console.error("[install-upgrade] dist/server.js missing — run `npm run build:cli` first");
process.exit(2);
}
const version = JSON.parse(fs.readFileSync(path.join(ROOT, "package.json"), "utf8")).version;
const allowlist = loadAllowlist(ROOT);
// NOT os.tmpdir(): on the self-hosted runner /tmp is a 12 GB tmpfs backed by RAM, while
// the root filesystem has ~66 GB free. This gate needs ~12 GB, so it exhausted the tmpfs
// and npm truncated the package — 58269 ENOSPC errors on the v3.8.50 publish, which the
// previous code could only report as a crash. Freeing disk did not help because the disk
// was never the constraint. Work on real disk beside the repo instead.
const workRoot =
process.env.OMNIROUTE_INSTALL_UPGRADE_WORKDIR || path.join(ROOT, ".install-upgrade");
fs.mkdirSync(workRoot, { recursive: true });
const tmp = fs.mkdtempSync(path.join(workRoot, "omniroute-install-upgrade-"));
workDirForMessages = tmp;
const failures = [];
const warnings = [];
try {
// Timed, because this turned out to be the expensive part: on the 2026-08-27
// v3.8.50 publish `npm pack` alone took 24m37s, leaving 5 of the step's 30-minute
// budget for two installs and two boots. Without a duration here the log showed
// only "packing…" then a timeout, which reads like a hang and is not.
const packStarted = Date.now();
log(`packing v${version}…`);
const packOut = execFileSync("npm", ["pack", "--json", "--pack-destination", tmp], {
cwd: ROOT,
encoding: "utf8",
maxBuffer: 128 * 1024 * 1024,
});
const tarball = path.join(tmp, pickTarball(packOut));
const packMb = (fs.statSync(tarball).size / 1024 / 1024).toFixed(1);
log(`packed in ${Math.round((Date.now() - packStarted) / 1000)}s (${packMb} MB)`);
// Each install tree is ~3 GB and this run builds two of them, side by side, plus the
// ~275 MB tarball. On the v3.8.50 publish run that overflowed the runner disk mid-way
// through the Phase B upgrade install; npm warned per truncated tar entry and still
// exited 0, and every later measurement was taken from a broken tree.
const availableBytes = freeBytes(tmp);
if (availableBytes !== null) {
log(`free space in ${tmp}: ${(availableBytes / GB).toFixed(1)} GB`);
if (availableBytes < 12 * GB) {
warn(
`only ${(availableBytes / GB).toFixed(1)} GB free — this gate needs roughly 12 GB ` +
`(two ~3 GB install trees, the second installed over twice, plus the tarball). ` +
`An install truncated by ENOSPC looks like a schema divergence.`
);
}
}
// ---- Phase A: clean install -------------------------------------------------
log("PHASE A — clean install of the packed tarball");
const aPrefix = path.join(tmp, "a-prefix");
const aData = path.join(tmp, "a-data");
fs.mkdirSync(aData, { recursive: true });
npmInstallInto(aPrefix, tarball, "clean install");
const a = await bootAndProbe({
prefix: aPrefix,
dataDir: aData,
port: pickPort(0),
expectVersion: version,
label: "clean",
});
failures.push(...a.failures);
const cleanBooted = a.ok;
if (a.ok) log(`clean install healthy on v${a.version}`);
else if (a.tail?.length) {
console.error("[install-upgrade] last output from the clean-install server:");
console.error(a.tail.join("").split("\n").slice(-40).join("\n"));
}
const aDb = findDb(aData);
const freshTables = aDb ? readTables(aDb) : null;
if (!freshTables) failures.push("clean: no SQLite database was created");
else log(`clean install schema: ${freshTables.size} tables`);
// Phase A is fully measured (boot verdict + schema snapshot); its ~3 GB install tree is
// dead weight from here on and Phase B needs the room. The DATA_DIR stays — only the
// node_modules tree goes.
if (!skipUpgrade) {
fs.rmSync(aPrefix, { recursive: true, force: true });
const reclaimed = freeBytes(tmp);
log(
"released the clean-install tree before the upgrade phase" +
(reclaimed !== null ? ` (${(reclaimed / GB).toFixed(1)} GB free)` : "")
);
}
// ---- Phase B: upgrade over the previous published version -------------------
let upgradedTables = null;
// `--skip-upgrade` never reaches the convergence block (upgradedTables stays null), so
// defaulting this to true keeps that path unchanged.
let upgradeBooted = true;
if (skipUpgrade) {
warn("PHASE B skipped (--skip-upgrade)");
} else {
const previous = resolvePreviousVersion(version, explicitFrom);
log(`PHASE B — upgrade path: omniroute@${previous} → v${version}`);
const bPrefix = path.join(tmp, "b-prefix");
const bData = path.join(tmp, "b-data");
fs.mkdirSync(bData, { recursive: true });
npmInstallInto(bPrefix, `omniroute@${previous}`, `previous(${previous}) install`);
const before = await bootAndProbe({
prefix: bPrefix,
dataDir: bData,
port: pickPort(1),
expectVersion: previous,
label: `previous(${previous})`,
});
if (!before.ok) {
// A broken PREVIOUS version is not this release's fault — degrade to a warning so a
// historically bad publish cannot block the current one.
warnings.push(
`previous version ${previous} did not boot cleanly — upgrade path unverified`
);
for (const f of before.failures) warn(f);
} else {
const beforeDb = findDb(bData);
const beforeTables = beforeDb ? readTables(beforeDb) : new Set();
log(`previous(${previous}) schema: ${beforeTables.size} tables — upgrading in place`);
npmInstallInto(bPrefix, tarball, "upgrade install");
const after = await bootAndProbe({
prefix: bPrefix,
dataDir: bData,
port: pickPort(2),
expectVersion: version,
label: "upgraded",
});
failures.push(...after.failures);
if (after.ok) log(`upgrade healthy on v${after.version}`);
upgradeBooted = after.ok;
if (!after.ok && after.tail?.length) {
console.error("[install-upgrade] last output from the upgraded server:");
console.error(after.tail.join("").split("\n").slice(-40).join("\n"));
}
const afterDb = findDb(bData);
upgradedTables = afterDb ? readTables(afterDb) : null;
if (!upgradedTables) {
failures.push("upgraded: database disappeared after the upgrade");
} else {
log(`upgraded schema: ${upgradedTables.size} tables`);
const dropped = [...beforeTables].filter((t) => !upgradedTables.has(t));
if (dropped.length) {
failures.push(`upgrade DROPPED tables that existed before: ${dropped.join(", ")}`);
}
}
}
}
// ---- Schema convergence -----------------------------------------------------
// Only meaningful when BOTH servers actually served. A boot that died before serving
// never ran a migration, so its database still holds the PREVIOUS release's schema and
// every post-baseline table shows up as "a clean install creates but an upgrade does
// not" — which is what the v3.8.50 publish run reported after ENOSPC truncated the
// upgrade install. Comparing there does not add information, it manufactures a
// 15-table false alarm on top of the real failure. The run still fails: the boot
// failure is already in `failures`.
if (freshTables && upgradedTables && !(cleanBooted && upgradeBooted)) {
warn(
"schema convergence NOT evaluated — a phase failed to boot, so its database was " +
"never migrated and any table difference would describe the broken boot, not the schema"
);
} else if (freshTables && upgradedTables) {
const verdict = evaluateConvergence({
freshTables,
upgradedTables,
residualAllowlist: allowlist.residualTables ?? {},
});
if (verdict.onlyUpgraded.length) {
warn(`residual tables present only after upgrade: ${verdict.onlyUpgraded.join(", ")}`);
}
failures.push(...verdict.failures);
if (verdict.ok) log("schema convergence OK (no new divergence)");
}
if (warnings.length) for (const w of warnings) warn(w);
if (failures.length) {
console.error(`[install-upgrade] FAIL — ${failures.length} problem(s):`);
for (const f of failures) console.error(` ✗ ${f}`);
process.exit(1);
}
log("PASS — clean install and upgrade path both boot; schema converges.");
process.exit(0);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
}
// Only run the (expensive) gate when invoked directly — importing this module for the pure
// helper above must not pack, install or boot anything.
if (
process.argv[1] &&
path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname)
) {
main().catch((err) => {
console.error(`[install-upgrade] crashed: ${err?.message ?? err}`);
process.exit(1);
});
}