Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
#!/usr/bin/env bun
/**
* scripts/ci-ubicloud.ts — the ci:local gate fanned out across ephemeral
* Ubicloud VMs.
*
* bun run ci:ubicloud # gitleaks + verify + serial + slow + unit + pg-arms + ALL E2E
* bun run ci:ubicloud:diff # doc-only diff: doc checks + gitleaks; any other diff: the full gate
*
* Options:
* --vms N VMs to provision in parallel (default: burst-sized from
* `ubi-runner.sh usage`; 4 when the quota is busy)
* --size SIZE Ubicloud size (default standard-16)
* --slots N concurrent work slots per VM (default: three per eight of --size's vCPUs)
* --location LOC Ubicloud location (default eu-central-h1). A full gate
* runs only there; another location needs --lanes and
* is a diagnostic run, never gate evidence
* --image IMAGE boot image (default: UBI_CI_IMAGE, else stock Ubuntu 24.04);
* gbrain-ci@latest is the prebaked machine image
* scripts/ubicloud/build-ci-image.sh builds
* --lanes a,b subset of gitleaks,verify,serial,slow,unit,pg-arms,e2e (default all)
* --pg-image IMAGE Postgres server image of every slot (default
* pgvector/pgvector:pg16). Another image needs --lanes and
* is a diagnostic run, never gate evidence: run
* --lanes pg-arms,e2e --pg-image pgvector/pgvector:pg15 to
* audit Postgres 15 (#7112); the catalog goldens hold
* Postgres 16 output and fail on 15 by design
* --diff doc-only diffs run only the doc checks and gitleaks (ci:local --diff)
* --list print the test files this selection runs (every selected
* lane's inventory, keyed-only E2E files left out), one per
* line, and exit; exit 1 when it is empty. Needs no key or
* VM (scripts/pregate-remote-gate.sh --list)
* --record-weights write measured durations to scripts/ubicloud/weights.json
* --keep leave the VMs running (destroy with ubi-runner.sh down NAME)
* --ttl-hours N VM TTL written into each name as -t<N> (default 2,
* at most 999). The operator reaper destroys VMs past their
* expiry, so a VM a sleeping machine never tore down
* dies within hours (scripts/ubicloud/reap-expired.ts)
*
* Needs UBI_OWNER (your thread code; every VM name carries it, and the run
* refuses without it) and UBICLOUD_API_KEY (or UBICLOUD_API_TOKEN) plus bash, curl, python3,
* ssh, ssh-keygen and tar locally. The checkout (tracked + untracked-unignored
* files + .git) is packed once and streamed to every VM, so uncommitted edits
* are tested.
*
* Each VM runs scripts/ubicloud/setup-ci-vm.sh (bun, test prerequisites, one
* pgvector server + PgBouncer per slot, PGLite snapshots), streamed over SSH
* while the checkout uploads. The first VM to
* finish setup runs the machine-level items in order (gitleaks, verify, the
* serial and slow lanes' machine-exclusive files) and then joins the pool. Every other
* test file is an item in one global heaviest-first queue that idle slots pull
* from (scripts/ubicloud/schedule.ts); items run through the same wrappers
* ci:local uses (scripts/ubicloud/ci-item.sh). The pg-arms lane runs every
* file in test/postgres-unit-arms.txt with DATABASE_URL set, the way
* persistence-validation's unit-postgres-arms job does, and fails the run when
* a shard of that job's partition measures over its step timeout; with the unit
* and slow lanes it fails a test.yml unit shard whose mined weight is over
* UNIT_SHARD_BUDGET_MS (unitShardCheck) and reports each shard's measured time. Durations of every run are
* merged into .context/ci-ubicloud/weights.json, which weights the next run.
*
* VMs are named ubirun-<owner>-<epoch>-ciNN<hex>-t<ttl hours> (owner:
* UBI_OWNER), gstack's runner grammar, so
* gstack's reap reads the same TTL. Each name is appended to <run>/vms.txt before its
* create request is sent. On exit, including SIGINT, SIGTERM, SIGHUP (a
* dropped terminal, a cancelled background operation) and SIGQUIT, in-flight
* `up` calls are allowed to finish (they destroy their own VM), then every
* recorded name is destroyed and polled until it is confirmed gone. Then the
* project's VM list is read once more and the run prints gstack's leftover line,
* UBICLOUD: left=<n|unknown> owner=<owner> vms=<requested> others=<n>
* (left: this run's VMs still listed; others: this owner's other VMs, for
* information), also appended to UBI_LEFT_FILE when set
* (scripts/pregate-remote-gate.sh reads it). A left other than 0 fails a run
* that would otherwise pass; --keep reports its VMs as left=<n> kept=1. A run
* that exits before it requests any VM (a missing key or owner, a broken
* inventory) prints left=0 vms=0, so pregate reads its failure as the gate's,
* and a missing line still means a crash after VMs were requested.
*/
import { spawnSync, type ChildProcess } from "node:child_process";
import { appendFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import {
buildItems,
burstVms,
parseItemLog,
parseUsageTotal,
readWeightTable,
sortQueue,
takeBatch,
weightKey,
type Item,
type ItemResult,
type Lane,
type WeightTable,
} from "./ubicloud/schedule.ts";
import { runScript, type RunScriptOpts } from "./ubicloud/run-script.ts";
import { expiringVmName } from "./ubicloud/reap-expired.ts";
import { buildTypecheckState, portableBuildInfo, typecheckBuildInfo } from "./ubicloud/typecheck-state.ts";
import { ARM_SHARDS, ARM_STEP_TIMEOUT_MS, ARM_WEIGHTS, ARM_WORKERS, armMakespan, armShards, readArmsList } from "./postgres-unit-arms.ts";
import { UNIT_SHARD_BUDGET_MS, UNIT_SHARDS, unitShardPlan } from "./sharding.ts";
const ROOT = resolve(import.meta.dir, "..");
const RUNNER = join(ROOT, "scripts/ubicloud/ubi-runner.sh");
const REMOTE_DIR = "work/gbrain";
// Written once the checkout is unpacked; setup-ci-vm.sh waits for it before the steps that need the checkout.
const UNPACKED_MARKER = "work/gbrain.unpacked";
const SETUP_SCRIPT = join(ROOT, "scripts/ubicloud/setup-ci-vm.sh");
const COMMITTED_WEIGHTS = join(ROOT, "scripts/ubicloud/weights.json");
const LOCAL_WEIGHTS = join(ROOT, ".context/ci-ubicloud/weights.json");
const ALL_LANES = ["gitleaks", "verify", "serial", "slow", "unit", "pg-arms", "e2e"] as const;
const BATCH_TIMEOUT_MS = 20 * 60 * 1000;
const VM_LIST_TIMEOUT_MS = 120_000;
// Test timeouts and the mined shard weights are calibrated on this location's VMs.
const GATE_LOCATION = "eu-central-h1";
// Keyed-only E2E files: e2e.yml's Tier 2 job runs them with the provider keys and
// openclaw on the nightly schedule. A gate VM has neither (setup-ci-vm.sh drops the
// keys), so every test in them would skip; the gate leaves them out instead of
// counting a run that tested nothing (GBRA-237 removed the same keyless run from CI).
export const KEYED_E2E = ["test/e2e/skills.test.ts"];
// Items this heavy are the run's long poles: spread them one per VM so two of
// them never compete for the same VM's CPUs.
const HEAVY_MS = 60_000;
// VMs still in setup count toward heavy spreading for this long after the
// first VM is ready, so the first VM does not claim every long pole.
const HEAVY_SETUP_GRACE_MS = 45_000;
// Fleet size without --vms: the default when the shared quota is busy, up to
// BURST_MAX_VMS when it is free, never past BURST_CEILING_VCPUS of the
// project's 512 (the rest is headroom for pull-request runners `usage` can't see).
const DEFAULT_VMS = 4;
const BURST_MAX_VMS = 10;
const BURST_CEILING_VCPUS = 448;
// A VM whose slots would each get less than this much weighted work is not worth its setup.
const MIN_SLOT_WORK_S = 120;
const vcpusOf = (size: string) => Number(/(\d+)$/.exec(size)?.[1] ?? 16);
interface Opts {
/** 0 until sized: --vms N, or the burst decision from `ubi-runner.sh usage`. */
vms: number;
size: string;
slots: number;
location: string;
/** A machine image (NAME@VERSION) or "stock". */
image: string;
lanes: Set<string>;
/** The Postgres server image of every slot; empty keeps setup-ci-vm.sh's default. */
pgImage: string;
diff: boolean;
recordWeights: boolean;
keep: boolean;
ttlHours: number;
/** Print the gate's test files and exit; starts nothing. */
list: boolean;
}
function parseArgs(argv: string[]): Opts {
const opts: Opts = {
vms: 0,
size: "standard-16",
slots: 0,
location: GATE_LOCATION,
image: process.env.UBI_CI_IMAGE || "stock",
lanes: new Set(ALL_LANES),
pgImage: "",
diff: false,
recordWeights: false,
keep: false,
ttlHours: 2,
list: false,
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i]!;
const value = () => {
const v = argv[++i];
if (!v) usage(`${arg} needs a value`);
return v!;
};
if (arg === "--vms") opts.vms = Number(value());
else if (arg === "--size") opts.size = value();
else if (arg === "--slots") opts.slots = Number(value());
else if (arg === "--location") opts.location = value();
else if (arg === "--image") opts.image = value();
else if (arg === "--lanes") opts.lanes = new Set(value().split(","));
else if (arg === "--pg-image") opts.pgImage = value();
else if (arg === "--diff") opts.diff = true;
else if (arg === "--record-weights") opts.recordWeights = true;
else if (arg === "--keep") opts.keep = true;
else if (arg === "--ttl-hours") opts.ttlHours = Number(value());
else if (arg === "--list") opts.list = true;
else usage(`unknown argument ${arg}`);
}
for (const lane of opts.lanes) if (!(ALL_LANES as readonly string[]).includes(lane)) usage(`unknown lane ${lane}`);
if (opts.location !== GATE_LOCATION && !argv.includes("--lanes")) usage(`a full gate runs only in ${GATE_LOCATION}. Test timeouts and the mined shard weights are calibrated there; on 2026-10-19 us-east-a2 ran about half as fast and timed out tests that pass in ${GATE_LOCATION}, so a full run in ${opts.location} is not gate evidence. Wait for ${GATE_LOCATION} capacity or run bun run ci:local (and report it as local). To run a diagnostic subset in ${opts.location}, pass --lanes.`);
if (opts.pgImage && !/^[a-z0-9][a-z0-9._\/-]*(?::[A-Za-z0-9._-]+)?$/.test(opts.pgImage)) usage(`--pg-image ${opts.pgImage} is not an image reference`);
if (opts.pgImage && !argv.includes("--lanes")) usage("a full gate runs on the default Postgres image; --pg-image is a diagnostic run, so pass --lanes (for example --lanes pg-arms,e2e).");
if (!Number.isInteger(opts.ttlHours) || opts.ttlHours < 1 || opts.ttlHours > 999) usage("--ttl-hours must be a whole number of hours from 1 to 999 (every gate VM carries a TTL; 999 is the longest)");
if (!Number.isInteger(opts.vms) || opts.vms < 0 || (opts.vms === 0 && argv.includes("--vms"))) usage("--vms must be a positive integer");
// Three slots per eight vCPUs: each slot's bun process, its children and its
// own PostgreSQL and PgBouncer keep about 2.5 threads runnable. One slot per
// two vCPUs measured a 1-minute load of 1.11 per vCPU at p50 and 1.39 at p90
// and timed out a file that passes at this density (0.79 p50, 0.95 p90).
if (!opts.slots) opts.slots = Math.max(1, Math.floor((vcpusOf(opts.size) * 3) / 8));
if (!Number.isInteger(opts.slots) || opts.slots < 1) usage("--slots must be a positive integer");
return opts;
}
function usage(message: string): never {
console.error(`ci-ubicloud: ${message}`);
console.error("usage: bun run scripts/ci-ubicloud.ts [--vms N] [--size SIZE] [--slots N] [--location LOC] [--image IMAGE] [--lanes a,b] [--pg-image IMAGE] [--diff] [--record-weights] [--keep] [--ttl-hours N] [--list]");
process.exit(2);
}
const t0 = Date.now();
const clock = () => {
const s = Math.round((Date.now() - t0) / 1000);
return `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s`;
};
const log = (message: string) => console.log(`[ci-ubicloud ${clock()}] ${message}`);
function sh(cmd: string, args: string[]): string {
const r = spawnSync(cmd, args, { cwd: ROOT, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
if (r.status !== 0) throw new Error(`${cmd} ${args.join(" ")} failed (${r.status}): ${r.stderr}`);
return r.stdout;
}
const lines = (text: string) => text.split(/\s+/).map((l) => l.trim()).filter(Boolean);
const children = new Map<ChildProcess, string>();
let vmRequested = false;
let leftReported = false;
/** Run the runner script; output goes to `out` (a path) or is returned (scripts/ubicloud/run-script.ts). */
function runner(args: string[], opts: Omit<RunScriptOpts, "cwd" | "onSpawn" | "onClose"> = {}): Promise<{ code: number; stdout: string }> {
return runScript(RUNNER, args, { ...opts, cwd: ROOT, onSpawn: (child) => children.set(child, args[0]!), onClose: (child) => children.delete(child) });
}
const quote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;
interface Vm {
name: string;
state: "provisioning" | "setup" | "ready" | "failed" | "destroyed";
requested: boolean;
infraErrors: number;
setupMs?: number;
busySlots: number;
heavy: number;
/** Running the machine-level control items; takes no queue items meanwhile. */
control: boolean;
}
/**
* The gate's test selection, the same discovery the ci:local wrappers use: the
* lanes that run, each lane's files, and the serial and slow lanes'
* machine-exclusive files. --list prints exactly these files.
*/
function gateInventory(opts: Opts, note: (message: string) => void) {
if (opts.diff) {
const classification = spawnSync("bun", ["run", "scripts/select-e2e.ts", "--classify-only"], { cwd: ROOT, encoding: "utf8" }).stdout?.trim();
if (classification === "DOC_ONLY") {
note("--diff: doc-only diff — running the doc checks locally, then gitleaks (ci:local doc-only fast-path)");
if (!opts.list && spawnSync("bash", ["scripts/ci-doc-checks.sh"], { cwd: ROOT, stdio: "inherit" }).status !== 0) {
process.exit(1);
}
opts.lanes = new Set(["gitleaks"]);
opts.vms = 1;
} else {
note(`--diff: E2E narrowing is retired; running the full E2E corpus (see docs/TESTING.md#e2e-selection). Classification: ${classification || "unknown"}`);
}
}
const exclusive = lines(sh("bash", ["scripts/run-serial-tests.sh", "--dry-run-list-exclusive"]));
const slowExclusive = lines(sh("bash", ["scripts/run-slow-tests.sh", "--dry-run-list-exclusive"]));
const arms = readArmsList(ROOT);
if (arms.errors.length) throw new Error(`test/postgres-unit-arms.txt: ${arms.errors.join("; ")}`);
const inventory: Record<Lane, string[]> = {
unit: lines(sh("bash", ["scripts/run-unit-shard.sh", "--dry-run-list"])),
serial: lines(sh("bash", ["scripts/run-serial-tests.sh", "--dry-run-list"])).filter((f) => !exclusive.includes(f)),
slow: lines(sh("bash", ["scripts/run-slow-tests.sh", "--dry-run-list"])).filter((f) => !slowExclusive.includes(f)),
"pg-arms": arms.files,
e2e: lines(sh("bash", ["scripts/run-e2e.sh", "--dry-run-list"])).filter((f) => !KEYED_E2E.includes(f)),
};
return { inventory, exclusive, slowExclusive };
}
async function main() {
const opts = parseArgs(process.argv.slice(2));
if (opts.list) {
const { inventory, exclusive, slowExclusive } = gateInventory(opts, (message) => console.error(`ci-ubicloud --list: ${message}`));
const files = new Set<string>();
for (const [lane, laneFiles] of Object.entries(inventory)) if (opts.lanes.has(lane)) laneFiles.forEach((f) => files.add(f));
if (opts.lanes.has("serial")) exclusive.forEach((f) => files.add(f));
if (opts.lanes.has("slow")) slowExclusive.forEach((f) => files.add(f));
if (!files.size) {
console.error(`ci-ubicloud --list: the gate runs no test file on this tree (lanes: ${[...opts.lanes].join(",")})`);
process.exit(1);
}
console.log([...files].sort().join("\n"));
return;
}
if (!process.env.UBICLOUD_API_KEY && !process.env.UBICLOUD_API_TOKEN) {
console.error("ci-ubicloud: UBICLOUD_API_KEY (or UBICLOUD_API_TOKEN) is not set");
process.exit(2);
}
if (!process.env.UBI_OWNER) {
console.error("ci-ubicloud: UBI_OWNER is not set; set it to your thread code in lowercase (e.g. UBI_OWNER=gbra106) so every VM carries its owner (UBI_OWNER_REQUIRED)");
process.exit(2);
}
const owner = sh("bash", [RUNNER, "owner"]).trim();
const sha = sh("git", ["rev-parse", "HEAD"]).trim();
const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${sha.slice(0, 8)}`;
const runDir = join(ROOT, ".context/ci-ubicloud", runId);
mkdirSync(join(runDir, "logs"), { recursive: true });
mkdirSync(join(runDir, "failures"), { recursive: true });
const { inventory, exclusive, slowExclusive } = gateInventory(opts, log);
const tables: WeightTable[] = [
readWeightTable(LOCAL_WEIGHTS),
readWeightTable(COMMITTED_WEIGHTS),
readWeightTable(join(ROOT, "scripts/e2e-weights.json"), 1, "e2e:"),
readWeightTable(join(ROOT, ARM_WEIGHTS), 1, "pg-arms:"),
readWeightTable(join(ROOT, "scripts/serial-weights.json"), 1000, "serial:"),
readWeightTable(join(ROOT, "scripts/test-weights.json"), 1, "unit:"),
readWeightTable(join(ROOT, "scripts/test-weights.json"), 1, "slow:"),
];
const queue: Item[] = [];
for (const lane of ["unit", "serial", "slow", "pg-arms", "e2e"] as Lane[]) {
if (opts.lanes.has(lane)) queue.push(...buildItems(lane, inventory[lane], tables));
}
sortQueue(queue);
const control: { lane: string; files: string[] }[] = [];
if (opts.lanes.has("gitleaks")) control.push({ lane: "gitleaks", files: [] });
if (opts.lanes.has("verify")) control.push({ lane: "verify", files: [] });
if (opts.lanes.has("serial") && exclusive.length) control.push({ lane: "serial", files: exclusive });
if (opts.lanes.has("slow") && slowExclusive.length) control.push({ lane: "slow", files: slowExclusive });
const totalItems = queue.length + control.reduce((n, c) => n + Math.max(c.files.length, 1), 0);
const estimate = queue.reduce((s, i) => s + i.weight, 0) / 1000;
log(`${totalItems} items (${Object.entries(inventory).filter(([l]) => opts.lanes.has(l)).map(([l, f]) => `${l} ${f.length}`).join(", ")}, control ${control.map((c) => c.lane).join("+") || "none"}); ~${Math.round(estimate)}s of weighted work`);
if (!opts.vms) {
const usageReport = await runner(["usage"], { timeoutMs: 120_000 });
const decision = burstVms({
usedVcpus: usageReport.code === 0 ? parseUsageTotal(usageReport.stdout) : null,
vmVcpus: vcpusOf(opts.size),
defaultVms: DEFAULT_VMS,
maxVms: BURST_MAX_VMS,
ceilingVcpus: BURST_CEILING_VCPUS,
workVms: Math.max(1, Math.ceil(estimate / (opts.slots * MIN_SLOT_WORK_S))),
});
opts.vms = decision.vms;
log(`fleet: ${decision.reason} (pass --vms N to override)`);
}
log(`provisioning ${opts.vms} × ${opts.size} in ${opts.location} (${opts.image === "stock" ? "stock Ubuntu 24.04" : `image ${opts.image}`}), ${opts.slots} slots each, names expiring ${opts.ttlHours}h after creation; logs in ${runDir}`);
// ── Checkout tarball, packed once while the VMs boot ─────────────────────
const tarball = join(tmpdir(), `gbrain-ci-ubicloud-${process.pid}.tgz`);
const packed = runner(["pack", ROOT], { out: tarball, stdoutOnly: true });
// verify's typecheck runs warm from a buildinfo of this tree, built while the VMs boot (scripts/ubicloud/typecheck-state.ts).
const buildInfo = opts.lanes.has("verify") ? typecheckBuildInfo(ROOT) : null;
const typecheckState = buildInfo
? buildTypecheckState(ROOT, join(runDir, "logs", "local-typecheck.log"), (child) => {
children.set(child, "typecheck");
child.once("close", () => children.delete(child));
})
: null;
// Faster create and SSH polling than the runner's interactive default.
process.env.UBI_POLL_SECONDS ??= "2";
const bunVersion = process.env.GBRAIN_CI_BUN_TAG
?? /oven\/bun:\$\{GBRAIN_CI_BUN_TAG:-([^}]+)\}/.exec(readFileSync(join(ROOT, "docker-compose.ci.yml"), "utf8"))?.[1]
?? "1.4.4";
// ── Teardown on every exit path ──────────────────────────────────────────
const ledger = join(runDir, "vms.txt");
const createdAt = Math.floor(Date.now() / 1000);
const vms: Vm[] = Array.from({ length: opts.vms }, (_, i) => ({
name: expiringVmName({ owner, createdAt, ttlHours: opts.ttlHours, suffix: `ci${String(i + 1).padStart(2, "0")}${Math.random().toString(16).slice(2, 6).padEnd(4, "0")}` }),
state: "provisioning",
requested: false,
infraErrors: 0,
busySlots: 0,
heavy: 0,
control: false,
}));
// Memoized: the signal handler and main's finally must await the same
// teardown, or main exits while its `down` calls are still running.
let tornDown: Promise<void> | null = null;
const teardown = () => (tornDown ??= destroyAll());
const destroyAll = async () => {
// An `up` gets SIGTERM, records its create answer and destroys its own VM;
// waiting for it means no create request is in flight when `down` runs.
const ups = [...children].filter(([, sub]) => sub === "up").map(([child]) => child);
for (const [child, sub] of children) child.kill(sub === "up" ? "SIGTERM" : "SIGKILL");
await Promise.all(ups.map((child) => new Promise((done) => (child.exitCode !== null || child.signalCode !== null ? done(null) : child.once("exit", done)))));
rmSync(tarball, { force: true });
const live = vms.filter((vm) => vm.requested && vm.state !== "destroyed");
const requested = vms.filter((vm) => vm.requested).map((vm) => vm.name);
if (opts.keep) {
log(`--keep: leaving ${live.map((vm) => vm.name).join(" ")} running; destroy with: scripts/ubicloud/ubi-runner.sh down NAME (the operator reaper destroys them ${opts.ttlHours}h after creation; pass --ttl-hours up to 999 for longer)`);
await leftover(owner, requested, true);
return;
}
log(`destroying ${live.length} VM(s): ${live.map((vm) => vm.name).join(" ")}`);
await Promise.all(live.map(async (vm) => {
for (let attempt = 1; attempt <= 3; attempt++) {
if ((await runner(["down", vm.name, "-l", opts.location])).code === 0) {
vm.state = "destroyed";
return;
}
}
console.error(`ci-ubicloud: WARNING could not confirm ${vm.name} is gone; run: scripts/ubicloud/ubi-runner.sh down ${vm.name} -l ${opts.location}`);
}));
const left = live.filter((vm) => vm.state !== "destroyed").length;
log(left ? `teardown left ${left} VM(s) unconfirmed (names in ${ledger})` : `teardown confirmed ${live.length} VM(s) gone`);
leftClean = await leftover(owner, requested, false);
};
let leftClean = true;
let interrupted = false;
// After SIGHUP the terminal may be gone: a failed log write must not abort teardown.
for (const stream of [process.stdout, process.stderr]) stream.on("error", () => {});
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP", "SIGQUIT"] as const) {
process.on(signal, () => {
if (interrupted) return;
interrupted = true;
log(`${signal}: tearing down`);
teardown().finally(() => process.exit(130));
});
}
// ── Scheduling state ─────────────────────────────────────────────────────
const results = new Map<string, ItemResult & { vm: string; slot: number }>();
let inFlight = 0;
let controlDone = control.length === 0;
let controlClaimed = false;
let batchSeq = 0;
const totalSlots = () => vms.filter((vm) => vm.state === "ready" && !vm.control).length * opts.slots;
let firstReadyAt = 0;
const skipHeavy = (vm: Vm) => (item: Item) => {
if (item.weight < HEAVY_MS) return false;
const inGrace = Date.now() - firstReadyAt < HEAVY_SETUP_GRACE_MS;
// The control VM holds no heavy items until it joins the pool; counting it would cap every other VM at one.
const peers = vms.filter((v) => !v.control && (v.state === "ready" || (inGrace && (v.state === "setup" || v.state === "provisioning"))));
return vm.heavy > Math.min(...peers.map((v) => v.heavy));
};
// Every item has a result: VMs still waiting for capacity have nothing left to do, and teardown reaps them.
let allRecorded = () => {};
const recorded = new Promise<void>((done) => (allRecorded = done));
const record = (vm: Vm, slot: number, result: ItemResult) => {
results.set(weightKey(result.lane as Lane, result.file), { ...result, vm: vm.name, slot });
if (results.size === totalItems) allRecorded();
if (result.rc !== 0) {
const path = join(runDir, "failures", `${result.lane}__${result.file.replace(/[\\/]/g, "__")}.log`);
writeFileSync(path, result.output + "\n");
log(`FAIL ${result.lane} ${result.file} (rc=${result.rc}, ${Math.round(result.ms / 1000)}s) → ${path}`);
}
};
const runBatch = async (vm: Vm, slot: number, lane: string, files: string[]) => {
const logPath = join(runDir, "logs", `${String(++batchSeq).padStart(5, "0")}-${vm.name.replace(/-t\d+$/, "").slice(-8)}-s${slot}-${lane}.log`);
const cmd = `cd ${REMOTE_DIR} && SLOT=${slot} bash scripts/ubicloud/ci-item.sh ${lane} ${files.map(quote).join(" ")}`;
const r = await runner(["ssh", vm.name, cmd], { out: logPath, timeoutMs: BATCH_TIMEOUT_MS });
return { code: r.code, parsed: parseItemLog(readFileSync(logPath, "utf8")), logPath };
};
const slotWorker = async (vm: Vm, slot: number) => {
while (!interrupted && vm.state === "ready") {
const batch = takeBatch(queue, { slots: Math.max(totalSlots(), 1), skip: skipHeavy(vm) });
if (batch.length === 0) {
if (inFlight === 0) return;
await Bun.sleep(1000);
continue;
}
inFlight++;
vm.busySlots++;
const heavy = batch.filter((i) => i.weight >= HEAVY_MS).length;
vm.heavy += heavy;
const lane = batch[0]!.lane;
const { code, parsed, logPath } = await runBatch(vm, slot, lane, batch.map((i) => i.file));
vm.heavy -= heavy;
vm.busySlots--;
const done = new Set(parsed.map((r) => r.file));
for (const result of parsed) record(vm, slot, result);
const lost = batch.filter((item) => !done.has(item.file));
// The teardown killed this batch: its lost items say nothing about the VM.
if (lost.length && !interrupted) {
vm.infraErrors++;
log(`batch on ${vm.name} slot ${slot} ended (exit ${code}) with ${lost.length} unfinished item(s); log ${logPath}`);
for (const item of lost) {
item.attempts++;
if (item.attempts < 2) queue.push(item);
else record(vm, slot, { lane: item.lane, file: item.file, rc: 255, ms: 0, output: `no result after ${item.attempts} attempts; last batch log: ${logPath}` });
}
sortQueue(queue);
if (vm.infraErrors >= 3) {
log(`retiring ${vm.name} after ${vm.infraErrors} infrastructure errors`);
vm.state = "failed";
}
}
inFlight--;
}
};
const runControl = async (vm: Vm) => {
vm.control = true;
for (const step of control) {
if (step.lane === "verify" && typecheckState) {
const { code, ms } = await typecheckState;
const local = `local incremental typecheck (exit ${code}, ${Math.round(ms / 1000)}s)`;
const state = join(runDir, "typecheck.tsbuildinfo");
let portable: ReturnType<typeof portableBuildInfo> = null;
try { portable = portableBuildInfo(ROOT, buildInfo!, state); } catch { /* absent or unreadable: reported below */ }
const sent = portable && portable.outside === 0
? await runner(["ssh", vm.name, `mkdir -p ${quote(dirname(`${REMOTE_DIR}/${buildInfo}`))} && cat > ${quote(`${REMOTE_DIR}/${buildInfo}`)}`], { input: state })
: null;
const rerooted = portable?.moved ? `, ${portable.moved} entries re-rooted from the symlinked node_modules` : "";
log(sent?.code === 0
? `typecheck state: ${local} uploaded to ${vm.name}${rerooted}; verify's typecheck runs warm`
: `typecheck state: ${!portable ? `no readable ${buildInfo} after the ${local}`
: portable.outside ? `${portable.outside} entries of the ${local} resolve outside the checkout, not uploaded`
: `upload to ${vm.name} failed (exit ${sent!.code})`}; verify's typecheck runs cold`);
}
log(`control on ${vm.name}: ${step.lane}${step.files.length ? ` (${step.files.length} exclusive files, sequential)` : ""}`);
const { parsed, logPath } = await runBatch(vm, 1, step.lane, step.files);
const done = new Set(parsed.map((r) => r.file));
for (const result of parsed) record(vm, 1, result);
for (const file of step.files.length ? step.files : ["-"]) {
if (!done.has(file)) record(vm, 1, { lane: step.lane, file, rc: 255, ms: 0, output: `no result; batch log: ${logPath}` });
}
}
controlDone = true;
vm.control = false;
};
const vmLifecycle = async (vm: Vm, index: number) => {
const started = Date.now();
const setupLog = join(runDir, "logs", `setup-${vm.name}.log`);
if (tornDown) return;
vm.requested = true;
vmRequested = true;
appendFileSync(ledger, `${vm.name} ${opts.location}\n`);
const up = await runner(["up", "-n", vm.name, "-s", opts.size, "-l", opts.location, ...(opts.image === "stock" ? [] : ["-b", opts.image])], { out: setupLog });
if (tornDown) return;
if (up.code !== 0) {
vm.state = "failed";
const setupText = readFileSync(setupLog, "utf8").trim();
const quota = setupText.search(/ubi-runner: quota (preflight )?refused/);
if (quota >= 0) log(`VM ${index + 1} refused by the Ubicloud vCPU quota; the run continues on the VMs that started:\n${setupText.slice(quota)}`);
else log(`VM ${index + 1} failed to provision (see ${setupLog}): ${setupText.split("\n").slice(-2).join(" | ")}`);
return;
}
vm.state = "setup";
// System packages, Bun and image pulls need no checkout: setup starts now
// and waits for the marker the upload writes when it lands.
const env = `SLOTS=${opts.slots} BUN_VERSION=${quote(bunVersion)} GITLEAKS=${opts.lanes.has("gitleaks") ? 1 : 0} CHECKOUT=${REMOTE_DIR} CHECKOUT_MARKER=${UNPACKED_MARKER}${opts.pgImage ? ` PG_IMAGE=${quote(opts.pgImage)}` : ""}`;
const setupRun = runner(["ssh", vm.name, `${env} bash -s`], { input: SETUP_SCRIPT, out: `${setupLog}.bootstrap`, timeoutMs: 15 * 60 * 1000 });
const pack = await packed;
if (pack.code !== 0) throw new Error(`packing the checkout failed: ${pack.stdout.trim().split("\n").slice(-3).join(" | ")}`);
const unpack = await runner(["unpack", vm.name, REMOTE_DIR], { input: tarball });
const marked = unpack.code === 0 ? await runner(["ssh", vm.name, `touch ${UNPACKED_MARKER}`]) : unpack;
if (marked.code !== 0) {
vm.state = "failed";
log(`VM ${vm.name} checkout upload failed: ${marked.stdout.trim().split("\n").slice(-2).join(" | ")}`);
return;
}
const setup = await setupRun;
if (setup.code !== 0) {
vm.state = "failed";
log(`VM ${vm.name} setup failed (see ${setupLog}.bootstrap)`);
return;
}
vm.setupMs = Date.now() - started;
vm.state = "ready";
firstReadyAt ||= Date.now();
log(`${vm.name} ready in ${Math.round(vm.setupMs / 1000)}s`);
if (!controlClaimed && control.length) {
controlClaimed = true;
await runControl(vm);
}
await Promise.all(Array.from({ length: opts.slots }, (_, slot) => slotWorker(vm, slot + 1)));
};
const progress = setInterval(() => {
const failed = [...results.values()].filter((r) => r.rc !== 0).length;
const ready = vms.filter((vm) => vm.state === "ready").length;
const busy = vms.reduce((n, vm) => n + vm.busySlots, 0);
log(`${results.size}/${totalItems} done, ${failed} failed | queue ${queue.length} | VMs ready ${ready}/${opts.vms} | busy slots ${busy}`);
}, 15000);
let exitCode = 0;
try {
await Promise.race([Promise.all(vms.map((vm, i) => vmLifecycle(vm, i))), recorded]);
if (control.length && !controlDone) throw new Error("no VM became ready to run gitleaks/verify/exclusive items");
if (interrupted) throw new Error(`interrupted: ${totalItems - results.size} item(s) did not run`);
if (queue.length) throw new Error(`${queue.length} items never ran: every VM failed`);
} catch (error) {
console.error(`ci-ubicloud: ${error instanceof Error ? error.message : String(error)}`);
exitCode = 1;
} finally {
clearInterval(progress);
await teardown();
}
if (!leftClean) {
console.error(`ci-ubicloud: VM cleanup not confirmed (see the UBICLOUD: line above); check with: UBI_OWNER=${owner} scripts/ubicloud/ubi-runner.sh list --mine`);
exitCode = 1;
}
// ── Report ───────────────────────────────────────────────────────────────
const all = [...results.values()];
const failures = all.filter((r) => r.rc !== 0);
const byLane = new Map<string, { pass: number; fail: number; ms: number }>();
for (const r of all) {
const entry = byLane.get(r.lane) ?? { pass: 0, fail: 0, ms: 0 };
if (r.rc === 0) entry.pass++;
else entry.fail++;
entry.ms += r.ms;
byLane.set(r.lane, entry);
}
const wallMs = Date.now() - t0;
const computeMs = all.reduce((s, r) => s + r.ms, 0);
console.log("");
console.log("=== ci-ubicloud summary ===");
for (const [lane, e] of byLane) console.log(` ${lane.padEnd(8)} ${e.pass} passed, ${e.fail} failed, ${Math.round(e.ms / 1000)}s compute`);
const ready = vms.filter((vm) => vm.setupMs !== undefined);
console.log(` VMs: ${ready.length}/${opts.vms} ready${ready.length ? `, setup ${Math.round(Math.min(...ready.map((v) => v.setupMs!)) / 1000)}-${Math.round(Math.max(...ready.map((v) => v.setupMs!)) / 1000)}s` : ""}`);
console.log(` wall ${Math.round(wallMs / 1000)}s, ${Math.round(computeMs / 1000)}s of test compute`);
const loads = all.flatMap((r) => (r.load === undefined ? [] : [r.load])).sort((a, b) => a - b);
if (loads.length) {
const at = (q: number) => loads[Math.min(loads.length - 1, Math.floor(q * loads.length))]!.toFixed(2);
console.log(` VM load (1-minute load average per vCPU as each item ended, ${opts.slots} slots per VM): p50 ${at(0.5)}, p90 ${at(0.9)}, max ${at(1)}`);
if (Number(at(0.9)) > 1.1) console.log(` WARNING: VMs ran oversubscribed (p90 ${at(0.9)} runnable per vCPU), so timing-sensitive tests ran slower than the weights and file caps assume. Lower --slots.`);
}
console.log(" slowest items:");
for (const r of [...all].sort((a, b) => b.ms - a.ms).slice(0, 8)) console.log(` ${Math.round(r.ms / 1000)}s ${r.lane} ${r.file}`);
// The pg-arms lane's proof that unit-postgres-arms fits: each shard of the
// workflow's own partition, its ARM_WORKERS workers' makespan over this run's
// measured file durations (both arms, as this lane runs them).
const armShardReport = opts.lanes.has("pg-arms")
? armShards(ROOT, inventory["pg-arms"], ARM_SHARDS).map((files, i) => ({
shard: i + 1,
files: files.length,
ms: armMakespan(files, (file) => results.get(weightKey("pg-arms", file))?.ms ?? 0),
}))
: [];
if (armShardReport.length) {
console.log(` pg-arms shards (unit-postgres-arms partition, ${ARM_SHARDS} shards of ${ARM_WORKERS} workers, step timeout ${ARM_STEP_TIMEOUT_MS / 1000}s each):`);
for (const s of armShardReport) console.log(` ${s.shard}/${ARM_SHARDS}: ${Math.round(s.ms / 1000)}s across ${s.files} files`);
}
const overBudget = armShardReport.filter((s) => s.ms > ARM_STEP_TIMEOUT_MS);
// test.yml's unit shards: the plan test/scripts/unit-shard-budget.test.ts holds (unitShardCheck).
const unitCheck = opts.lanes.has("unit") && opts.lanes.has("slow")
? unitShardCheck(ROOT, (file) => results.get(weightKey("unit", file))?.ms ?? results.get(weightKey("slow", file))?.ms)
: null;
const unitShardReport = unitCheck?.shards ?? [];
const unitOverBudget = unitCheck?.overBudget ?? [];
if (unitShardReport.length) {
console.log(` unit shards (test.yml partition, ${UNIT_SHARDS} shards, mined weight at most ${UNIT_SHARD_BUDGET_MS / 1000}s each; measured here for drift only):`);
for (const s of unitShardReport) console.log(` ${s.shard}/${UNIT_SHARDS}: mined ${Math.round(s.mined / 1000)}s, measured ${Math.round(s.ms / 1000)}s across ${s.files} files${s.unmeasured ? ` (${s.unmeasured} not run by this gate)` : ""}`);
}
const timings: Record<string, number> = {};
for (const r of all) if (r.rc === 0 && r.file !== "-") timings[weightKey(r.lane as Lane, r.file)] = r.ms;
const merged = { ...Object.fromEntries(readWeightTable(LOCAL_WEIGHTS)), ...timings };
mkdirSync(dirname(LOCAL_WEIGHTS), { recursive: true });
writeFileSync(LOCAL_WEIGHTS, JSON.stringify(sortedObject(merged), null, 2) + "\n");
if (opts.recordWeights) {
const committed = { ...Object.fromEntries(readWeightTable(COMMITTED_WEIGHTS)), ...timings };
const current = new Set(Object.entries(inventory).flatMap(([lane, files]) => files.map((f) => weightKey(lane as Lane, f))));
const pruned = Object.fromEntries(Object.entries(committed).filter(([key]) => current.has(key)));
writeFileSync(COMMITTED_WEIGHTS, JSON.stringify(sortedObject(pruned), null, 2) + "\n");
log(`recorded ${Object.keys(timings).length} durations into scripts/ubicloud/weights.json`);
}
writeFileSync(join(runDir, "summary.json"), JSON.stringify({
sha, runId, wallMs, computeMs, opts: { ...opts, lanes: [...opts.lanes] },
vms: vms.map(({ name, state, setupMs, infraErrors }) => ({ name, state, setupMs, infraErrors })),
lanes: Object.fromEntries(byLane), failures: failures.map(({ lane, file, rc, ms, vm }) => ({ lane, file, rc, ms, vm })),
armShards: armShardReport,
unitShards: unitShardReport,
}, null, 2) + "\n");
if (failures.length) {
console.log("");
console.log(`${failures.length} failing item(s):`);
for (const f of failures) console.log(` - ${f.lane} ${f.file} (rc=${f.rc})`);
for (const f of failures.slice(0, 5)) {
console.log(`\n--- ${f.lane} ${f.file} (last 30 lines) ---`);
console.log(f.output.split("\n").slice(-30).join("\n"));
}
console.log(`\nFull per-item logs: ${join(runDir, "failures")}`);
exitCode = 1;
}
if (overBudget.length) {
console.log(`\npg-arms: ${overBudget.map((s) => `shard ${s.shard}/${ARM_SHARDS} measured ${Math.round(s.ms / 1000)}s`).join(", ")}, over the ${ARM_STEP_TIMEOUT_MS / 1000}s step timeout of persistence-validation's unit-postgres-arms job, which would cut that shard off mid-list. Fix: raise ARM_SHARDS (and the workflow matrix) or speed up the heaviest files, then re-mine scripts/postgres-arm-weights.json. Docs: docs/TESTING.md#postgres-arm-lanes`);
exitCode = 1;
}
if (unitOverBudget.length) {
console.log(`\nunit: ${unitOverBudget.map((s) => `shard ${s.shard}/${UNIT_SHARDS} mined ${Math.round(s.mined / 1000)}s`).join(", ")}, over UNIT_SHARD_BUDGET_MS (${UNIT_SHARD_BUDGET_MS / 1000}s), which test/scripts/unit-shard-budget.test.ts also enforces. Fix: re-mine scripts/test-weights.json, then raise UNIT_SHARDS (and both workflow matrices) or speed up the heaviest files. Docs: docs/TESTING.md#keeping-ci-partitions-balanced`);
exitCode = 1;
}
const expected = totalItems;
if (exitCode === 0 && results.size !== expected) {
console.error(`ci-ubicloud: ${results.size} results for ${expected} items`);
exitCode = 1;
}
console.log(exitCode === 0 ? `\n[ci-ubicloud] All checks passed in ${clock()}.` : `\n[ci-ubicloud] FAILED after ${clock()}.`);
if (opts.location !== GATE_LOCATION) console.log(`[ci-ubicloud] diagnostic run in ${opts.location}: not gate evidence (a full gate runs only in ${GATE_LOCATION}).`);
process.exit(exitCode);
}
/**
* Print gstack's leftover line after teardown (and append it to UBI_LEFT_FILE):
* left counts `names` still in the project's VM list ("unknown" when the list
* cannot be read), others this owner's other ubirun VMs. Returns whether left
* is 0; a --keep run reports its live VMs as left=<n> kept=1.
*/
async function leftover(owner: string, names: string[], kept: boolean): Promise<boolean> {
const list = await runner(["cli", "vm", "list", "-N", "-f", "location,name"], { timeoutMs: VM_LIST_TIMEOUT_MS });
const listed = list.code === 0 ? new Set(list.stdout.split("\n").map((line) => line.trim().split(/\s+/)[1]).filter(Boolean)) : null;
const left = listed ? names.filter((name) => listed.has(name)).length : "unknown";
const others = listed ? [...listed].filter((name) => name!.startsWith(`ubirun-${owner}-`) && !names.includes(name!)).length : "unknown";
const line = `UBICLOUD: left=${left} owner=${owner} vms=${names.length} ${kept ? "kept=1" : `others=${others}`}`;
console.log(line);
if (process.env.UBI_LEFT_FILE) appendFileSync(process.env.UBI_LEFT_FILE, line + "\n");
leftReported = true;
return left === 0;
}
/**
* test.yml's unit shards as this gate checks them: the partition and mined weights of unitShardPlan, failing a
* shard over UNIT_SHARD_BUDGET_MS exactly as test/scripts/unit-shard-budget.test.ts does. `measured` (this run's
* per-file durations) is reported beside the plan and never gates: each item here is its own wrapper and Bun
* process (~0.6 s of startup per file) on a shared 16-vCPU VM, not one Bun process per shard on a 4-vCPU runner,
* so recent gates measured the same partition at 1.2-2.5x its mined weight.
*/
export function unitShardCheck(root: string, measured: (file: string) => number | undefined) {
const shards = unitShardPlan(root).map(({ shard, files, ms }) => {
const times = files.map(measured);
return { shard, files: files.length, mined: ms, unmeasured: times.filter((t) => t === undefined).length, ms: times.reduce<number>((sum, t) => sum + (t ?? 0), 0) };
});
return { budgetMs: UNIT_SHARD_BUDGET_MS, shards, overBudget: shards.filter((s) => s.mined > UNIT_SHARD_BUDGET_MS) };
}
function sortedObject(obj: Record<string, number>): Record<string, number> {
return Object.fromEntries(Object.entries(obj).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
}
if (import.meta.main) {
process.on("exit", () => {
if (leftReported || vmRequested || process.argv.includes("--list")) return;
const line = `UBICLOUD: left=0 owner=${process.env.UBI_OWNER ?? ""} vms=0 others=unknown`;
console.log(line);
if (process.env.UBI_LEFT_FILE) appendFileSync(process.env.UBI_LEFT_FILE, line + "\n");
});
main().catch((error) => {
console.error(error);
process.exit(1);
});
}