Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.5 KB
Raw Blame
// ubi-runner.test.ts — scripts/ubicloud/ubi-runner.sh against a mock Ubicloud
// API: owner-tagged VM names, the stale sweep (off unless UBI_GC_HOURS is set,
// and never another owner's VMs), list --mine / usage, and teardown that waits
// out in-flight creates when `up` gets SIGTERM or SIGKILL mid-provision.

import { afterEach, beforeEach, describe, expect, it } from "bun:test";
import { execFileSync } from "node:child_process";
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { startMockUbicloud, waitForEvent, type MockUbicloud, type MockUbicloudOptions } from "../helpers/mock-ubicloud-api.ts";

const RUNNER = resolve(import.meta.dir, "..", "..", "scripts/ubicloud/ubi-runner.sh");
const NOW = Math.floor(Date.now() / 1000);
const OLD = NOW - 5 * 3600;

let dir = "";
let mock: MockUbicloud | null = null;

beforeEach(() => {
  dir = mkdtempSync(join(tmpdir(), "ubi-runner-test-"));
});
afterEach(() => {
  mock?.stop();
  mock = null;
  rmSync(dir, { recursive: true, force: true });
});

function serve(opts: MockUbicloudOptions = {}): MockUbicloud {
  mock = startMockUbicloud(opts);
  return mock;
}

function env(extra: Record<string, string> = {}): Record<string, string> {
  return {
    PATH: process.env.PATH ?? "/usr/bin:/bin",
    HOME: dir,
    UBICLOUD_API_KEY: "test-token",
    UBICLOUD_API_URL: mock?.url ?? "http://127.0.0.1:9",
    UBI_RUNNER_STATE: join(dir, "state"),
    UBI_POLL_SECONDS: "0.2",
    ...extra,
  };
}

function spawnRunner(args: string[], extra: Record<string, string> = {}) {
  return Bun.spawn(["bash", RUNNER, ...args], { env: env(extra), stdout: "pipe", stderr: "pipe" });
}

async function run(args: string[], extra: Record<string, string> = {}) {
  const proc = spawnRunner(args, extra);
  const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
  return { stdout, stderr, code };
}

describe("owner tag", () => {
  it("sanitizes UBI_OWNER to at most 12 lowercase letters and digits, starting with a letter", async () => {
    expect((await run(["owner"], { UBI_OWNER: "GBRA-41 lane!" })).stdout.trim()).toBe("gbra41lane");
    expect((await run(["owner"], { UBI_OWNER: "12345" })).stdout.trim()).toBe("u12345");
    expect((await run(["owner"], { UBI_OWNER: "abcdefghijklmnopq" })).stdout.trim()).toBe("abcdefghijkl");
    expect((await run(["owner"], { UBI_OWNER: "--" })).code).toBe(1);
  });

  it("defaults to a stable per-machine id", async () => {
    const first = (await run(["owner"])).stdout.trim();
    expect(first).toMatch(/^m[0-9a-f]{8}$/);
    expect((await run(["owner"])).stdout.trim()).toBe(first);
  });
});

describe("teardown waits for in-flight creates", () => {
  for (const signal of ["SIGTERM", "SIGHUP", "SIGINT", "SIGQUIT"] as const) {
    it(`${signal} during the create request: the create finishes, then the VM is destroyed and confirmed gone`, async () => {
      const api = serve({ createDelayMs: () => 1500 });
      const proc = spawnRunner(["up"], { UBI_OWNER: "t1" });
      const started = await waitForEvent(api, /^create-start /);
      const name = started.split(" ")[1]!;
      expect(name).toMatch(/^ubirun-t1-\d{10}-[0-9a-f]{8}-t3$/);
      proc.kill(signal);
      expect(await proc.exited).toBe(130);
      expect(api.events).toEqual([`create-start ${name}`, `created ${name}`, `destroy ${name}`, `removed ${name}`]);
      expect(api.vms.size).toBe(0);
    });
  }

  it("a second signal during teardown does not cut it short", async () => {
    const api = serve({ destroyDelayMs: 1500 });
    const proc = spawnRunner(["up"], { UBI_OWNER: "t1" });
    const name = (await waitForEvent(api, /^created /)).split(" ")[1]!;
    proc.kill("SIGTERM");
    await waitForEvent(api, /^destroy /);
    proc.kill("SIGHUP");
    proc.kill("SIGINT");
    expect(await proc.exited).toBe(130);
    expect(api.events.slice(-2)).toEqual([`destroy ${name}`, `removed ${name}`]);
    expect(api.vms.size).toBe(0);
  });

  it("SIGTERM while the VM is still provisioning destroys it before exiting", async () => {
    const api = serve();
    const proc = spawnRunner(["up"], { UBI_OWNER: "t1" });
    const name = (await waitForEvent(api, /^created /)).split(" ")[1]!;
    proc.kill("SIGTERM");
    expect(await proc.exited).toBe(130);
    expect(api.events.slice(-2)).toEqual([`destroy ${name}`, `removed ${name}`]);
    expect(api.vms.size).toBe(0);
  });

  // Bash 5.2 runs a pending signal trap inside the parse of the next $(...),
  // where the trap fails to parse and the shell exits 2 without running it.
  // BASH_ENV turns on xtrace with a PS4 whose first $(...) sends the runner
  // SIGTERM, so the second is always parsed with that trap pending.
  for (const args of [["up"], ["run", "--", "true"]]) {
    it(`${args[0]}: a SIGTERM that lands while bash parses a $(...) still destroys the VM and exits 130`, async () => {
      const api = serve();
      const bashEnv = join(dir, "bash-env");
      writeFileSync(bashEnv, `PS4='$([ "$BASH_SUBSHELL" = 1 ] && [ -e "$PROBE_DIR/go" ] && mkdir "$PROBE_DIR/fired" 2>/dev/null && kill -TERM $$)$(:)+ '\nset -x\n`);
      const proc = spawnRunner(args, { UBI_OWNER: "t1", BASH_ENV: bashEnv, PROBE_DIR: dir });
      const output = Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text()]);
      const name = (await waitForEvent(api, /^created /)).split(" ")[1]!;
      writeFileSync(join(dir, "go"), "");
      expect(await proc.exited).toBe(130);
      await output;
      expect(existsSync(join(dir, "fired"))).toBe(true);
      expect(api.events.slice(-2)).toEqual([`destroy ${name}`, `removed ${name}`]);
      expect(api.vms.size).toBe(0);
    });
  }

  it("after SIGKILL mid-create, down waits for the create to land and destroys the VM", async () => {
    const api = serve({ createDelayMs: () => 1500 });
    const proc = spawnRunner(["up"], { UBI_OWNER: "t1" });
    const name = (await waitForEvent(api, /^create-start /)).split(" ")[1]!;
    proc.kill("SIGKILL");
    await proc.exited;
    expect(api.vms.size).toBe(0);
    const down = await run(["down", name], { UBI_CREATE_GRACE: "20" });
    expect(down.code).toBe(0);
    expect(api.events).toEqual([`create-start ${name}`, `created ${name}`, `destroy ${name}`, `removed ${name}`]);
    expect(api.vms.size).toBe(0);
  });

  it("a quota refusal reports usage by owner and needs no grace wait; down of an unknown name reports it gone", async () => {
    const api = serve({ rejectCreates: true, seed: [{ name: `ubirun-lane2-${NOW}-aaaaaaaa` }, { name: `ubirun-lane2-${NOW}-bbbbbbbb` }] });
    const t0 = Date.now();
    const up = await run(["up"], { UBI_OWNER: "t1", UBI_CREATE_GRACE: "60" });
    expect(up.code).toBe(1);
    expect(up.stderr).toMatch(/quota refused eu-central-h1\/ubirun-t1-\S+ \(standard-16\): it needs 16 vCPUs and the project already uses 252 of 256/);
    expect(up.stderr).toMatch(/^lane2\s+2\s+32$/m);
    expect(up.stderr).toContain("create failed: vCPU quota exhausted");
    expect(Date.now() - t0).toBeLessThan(15_000);
    const down = await run(["down", `ubirun-t1-${NOW}-deadbeef`, "-l", "eu-central-h1"]);
    expect(down.code).toBe(0);
    expect(down.stderr).toContain("never existed or already destroyed");
    expect(api.vms.size).toBe(2);
    expect(api.events.some((e) => e.startsWith("destroy "))).toBe(false);
  });
});

describe("stale sweep and ownership", () => {
  const seed = [
    { name: `ubirun-t1-${OLD}-aaaaaaaa`, size: "standard-16" },
    { name: `ubirun-t1-${NOW}-bbbbbbbb`, size: "standard-8" },
    { name: `ubirun-other-${OLD}-cccccccc`, size: "standard-16" },
    { name: `ubirun-${OLD}-dddddddd`, size: "standard-4" },
    { name: "prod-db", size: "standard-2" },
  ];

  it("up never sweeps when UBI_GC_HOURS is unset", async () => {
    const api = serve({ seed });
    const proc = spawnRunner(["up"], { UBI_OWNER: "t1" });
    const name = (await waitForEvent(api, /^created /)).split(" ")[1]!;
    proc.kill("SIGTERM");
    await proc.exited;
    expect(api.events.filter((e) => e.startsWith("destroy "))).toEqual([`destroy ${name}`]);
    expect(api.vms.size).toBe(seed.length);
  });

  it("with UBI_GC_HOURS set, up sweeps only the caller's own stale VMs", async () => {
    const api = serve({ seed });
    const proc = spawnRunner(["up"], { UBI_OWNER: "t1", UBI_GC_HOURS: "1" });
    const name = (await waitForEvent(api, /^created /)).split(" ")[1]!;
    proc.kill("SIGTERM");
    await proc.exited;
    expect(api.events.filter((e) => e.startsWith("destroy "))).toEqual([`destroy ubirun-t1-${OLD}-aaaaaaaa`, `destroy ${name}`]);
    expect([...api.vms.keys()].sort()).toEqual(seed.slice(1).map((s) => s.name).sort());
  });

  it("gc HOURS destroys only the caller's VMs older than HOURS; without HOURS it destroys nothing", async () => {
    const api = serve({ seed });
    const none = await run(["gc"], { UBI_OWNER: "t1" });
    expect(none.code).toBe(0);
    expect(none.stderr).toContain("gc: no sweep");
    expect(api.events).toEqual([]);
    const gc = await run(["gc", "1"], { UBI_OWNER: "t1" });
    expect(gc.code).toBe(0);
    expect(api.events).toEqual([`destroy ubirun-t1-${OLD}-aaaaaaaa`, `removed ubirun-t1-${OLD}-aaaaaaaa`]);
  });

  it("list --mine and gc also cover VMs this machine created under another owner tag", async () => {
    const api = serve({ seed });
    const other = `ubirun-other-${OLD}-cccccccc`;
    execFileSync("mkdir", ["-p", join(dir, "state", other)]);
    writeFileSync(join(dir, "state", other, "env"), `NAME=${other}\nLOCATION=eu-central-h1\n`);
    const mine = await run(["list", "--mine"], { UBI_OWNER: "t1" });
    expect(mine.stdout.trim().split("\n").map((l) => l.split(/\s+/)[1])).toEqual([`ubirun-t1-${OLD}-aaaaaaaa`, `ubirun-t1-${NOW}-bbbbbbbb`, other]);
    expect(mine.stderr).toContain("list --mine: owner tag t1 plus VMs this machine created (also under: other)");
    const gc = await run(["gc", "1"], { UBI_OWNER: "t1" });
    expect(gc.code).toBe(0);
    expect(api.events.filter((e) => e.startsWith("destroy ")).sort()).toEqual([`destroy ubirun-t1-${OLD}-aaaaaaaa`, `destroy ${other}`].sort());
    expect(api.vms.has(`ubirun-${OLD}-dddddddd`)).toBe(true);
  });

  it("list --mine shows only the caller's VMs; usage sums vCPUs by owner", async () => {
    serve({ seed });
    const mine = await run(["list", "--mine"], { UBI_OWNER: "t1" });
    expect(mine.stdout.trim().split("\n").map((l) => l.split(/\s+/)[1])).toEqual([`ubirun-t1-${OLD}-aaaaaaaa`, `ubirun-t1-${NOW}-bbbbbbbb`]);
    const usage = (await run(["usage"], { UBI_OWNER: "t1" })).stdout;
    expect(usage).toMatch(/^t1\s+2\s+24\s+\(you\)$/m);
    expect(usage).toMatch(/^other\s+1\s+16$/m);
    expect(usage).toMatch(/^\(untagged\)\s+1\s+4$/m);
    expect(usage).toMatch(/^\(other\)\s+1\s+2$/m);
    expect(usage).toMatch(/^total\s+5\s+46$/m);
  });

  // macOS runs the one-true-awk, which rejected usage's vCPU loop (a comparison in an unparenthesized printf list).
  it("usage prints the same table under every awk on PATH, the one-true-awk included where installed", async () => {
    const awks = [["original-awk"], ["mawk"], ["gawk"], ["busybox", "awk"]].filter(([bin]) => Bun.which(bin!));
    expect(awks.length).toBeGreaterThan(0);
    const tables: string[] = [];
    for (const awk of awks) {
      serve({ seed });
      const shim = join(dir, `awk-${awk[0]}`);
      execFileSync("mkdir", ["-p", shim]);
      writeFileSync(join(shim, "awk"), `#!/bin/sh\nexec ${awk.map((a) => Bun.which(a) ?? a).join(" ")} "$@"\n`, { mode: 0o755 });
      const usage = await run(["usage"], { UBI_OWNER: "t1", PATH: `${shim}:${process.env.PATH ?? "/usr/bin:/bin"}` });
      expect({ awk: awk.join(" "), code: usage.code, stderr: usage.stderr }).toEqual({ awk: awk.join(" "), code: 0, stderr: "" });
      tables.push(usage.stdout);
      mock?.stop();
    }
    expect(tables[0]).toMatch(/^t1\s+2\s+24\s+\(you\)$/m);
    expect(new Set(tables).size).toBe(1);
  });
});

// gstack #3746 (#3678): Ubicloud answers `vm list` with a bare newline for a
// project with no VMs, which vm_rows read as a VM named "-"; and a usage whose
// list or size reads failed printed a partial total the quota preflight trusted.
describe("empty project and unreadable usage", () => {
  it("an empty project (vm list answers a bare or blank line) has no VMs: reap destroys nothing, usage prints a zero total, list --mine prints nothing", async () => {
    for (const listBody of ["\n", "  \n"]) {
      const api = serve({ listBody });
      const reap = await run(["reap"], { UBI_OWNER: "t1" });
      expect(reap.stderr).toContain("reap: no expired VMs");
      expect(reap.stderr).not.toContain("invalid VM name");
      expect(reap.stderr).not.toContain("expired VMs of other owners");
      expect(reap.code).toBe(0);
      const usage = await run(["usage"], { UBI_OWNER: "t1" });
      expect({ code: usage.code, stdout: usage.stdout }).toEqual({ code: 0, stdout: "OWNER           VMS  VCPUS\ntotal             0      0\n" });
      expect((await run(["list", "--mine"], { UBI_OWNER: "t1" })).stdout).toBe("");
      expect(api.destroyRequests).toBe(0);
      mock?.stop();
    }
  });

  it("usage skips a VM destroyed since the list (404), and exits 1 with no total when a size read or the list fails", async () => {
    const api = serve({ seed: [{ name: `ubirun-peer-${NOW}-cccc-t3` }], ghosts: [`ubirun-gone-${NOW}-dddd-t3`] });
    const ghost = await run(["usage"], { UBI_OWNER: "t1" });
    expect({ code: ghost.code, stdout: ghost.stdout }).toEqual({ code: 0, stdout: "OWNER           VMS  VCPUS\npeer              1     16\ntotal             1     16\n" });
    api.faults.show = true;
    const noShow = await run(["usage"], { UBI_OWNER: "t1" });
    expect(noShow.code).toBe(1);
    expect(noShow.stdout).not.toContain("total");
    expect(noShow.stderr).toContain("usage: could not read the size of 2 of 2 VMs");
    api.faults.list = true;
    const noList = await run(["usage"], { UBI_OWNER: "t1" });
    expect(noList.code).toBe(1);
    expect(noList.stdout).not.toContain("total");
    expect(noList.stderr).toContain("usage: could not list the project's VMs");
  });
});

// `pack` from a checkout on a branch named like `git init`'s default branch
// (master on this repo). slim_git fetches HEAD into that branch inside a fresh
// repository whose unborn HEAD names the same branch; without --update-head-ok
// git refuses ("refusing to fetch into branch ... checked out"), pack falls
// back to the full .git and prints that error on stderr, which the gate wrote
// into the tarball it streamed to every VM.
describe("pack", () => {
  it("packs a checkout on init's default branch with a slim .git and nothing on stderr", async () => {
    const repo = join(dir, "repo");
    const git = (...args: string[]) => execFileSync("git", ["-C", repo, ...args], { encoding: "utf8" }).trim();
    execFileSync("git", ["init", "-q", repo]);
    git("config", "user.email", "[email protected]");
    git("config", "user.name", "t");
    writeFileSync(join(repo, "tracked.txt"), "tracked\n");
    git("add", "tracked.txt");
    git("commit", "-q", "-m", "one");
    writeFileSync(join(repo, "untracked.txt"), "untracked\n");
    // An unreachable object: the full .git carries it, the slim copy (a fetch of HEAD) does not.
    writeFileSync(join(dir, "dangling.txt"), "dangling blob\n");
    const dangling = git("hash-object", "-w", join(dir, "dangling.txt"));
    const branch = git("symbolic-ref", "HEAD");
    expect(branch).toMatch(/^refs\/heads\//);
    const proc = Bun.spawn(["bash", RUNNER, "pack", repo], { env: env(), stdout: "pipe", stderr: "pipe" });
    const [archive, stderr, code] = await Promise.all([new Response(proc.stdout).arrayBuffer(), new Response(proc.stderr).text(), proc.exited]);
    expect({ code, stderr }).toEqual({ code: 0, stderr: "" });
    const out = join(dir, "out");
    execFileSync("mkdir", ["-p", out]);
    execFileSync("tar", ["-xzf", "-", "-C", out], { input: Buffer.from(archive) });
    expect(readFileSync(join(out, "untracked.txt"), "utf8")).toBe("untracked\n");
    expect(execFileSync("git", ["-C", out, "symbolic-ref", "HEAD"], { encoding: "utf8" }).trim()).toBe(branch);
    expect(execFileSync("git", ["-C", out, "log", "--format=%s"], { encoding: "utf8" }).trim()).toBe("one");
    expect(() => execFileSync("git", ["-C", out, "cat-file", "-e", dangling], { stdio: "ignore" })).toThrow();
  });

  // gstack #3746: only Apple's tar reads COPYFILE_DISABLE; without it, files with
  // extended attributes gain ._* entries the VM sees as untracked files.
  it("runs every tar with COPYFILE_DISABLE=1, for a checkout and for a plain directory", () => {
    const realTar = Bun.which("tar");
    expect(realTar).toBeTruthy();
    const bin = join(dir, "tar-shim");
    execFileSync("mkdir", ["-p", bin]);
    writeFileSync(join(bin, "tar"), `#!/usr/bin/env bash\necho "COPYFILE_DISABLE=${"$"}{COPYFILE_DISABLE:-unset}" >> ${JSON.stringify(join(dir, "tar.log"))}\nexec ${JSON.stringify(realTar)} "$@"\n`, { mode: 0o755 });
    const repo = join(dir, "repo");
    execFileSync("git", ["init", "-q", repo]);
    execFileSync("git", ["-C", repo, "-c", "[email protected]", "-c", "user.name=t", "commit", "-q", "--allow-empty", "-m", "one"]);
    writeFileSync(join(repo, "a"), "a\n");
    execFileSync("mkdir", ["-p", join(dir, "plain")]);
    writeFileSync(join(dir, "plain/p"), "p\n");
    for (const src of [repo, join(dir, "plain")]) execFileSync("bash", [RUNNER, "pack", src], { env: env({ PATH: `${bin}:${process.env.PATH ?? "/usr/bin:/bin"}` }), stdio: ["ignore", "ignore", "pipe"] });
    expect(readFileSync(join(dir, "tar.log"), "utf8").trim().split("\n")).toEqual(["COPYFILE_DISABLE=1", "COPYFILE_DISABLE=1", "COPYFILE_DISABLE=1"]);
  });
});