Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

ubi-runner: down retries a destroy request that timed out or got a 5xx (#8103 item 3)

garrytan7e3d729b9608
diff --git a/scripts/ubicloud/ubi-runner.sh b/scripts/ubicloud/ubi-runner.sh
index 9922704a50..c2208b8a43 100755
--- a/scripts/ubicloud/ubi-runner.sh
+++ b/scripts/ubicloud/ubi-runner.sh
@@ -22,6 +22,10 @@
# - Bash 5.2 lost-signal-trap guard (set -E, FAILED, finish).
# - UBI_REAP takes the usual false spellings (0/false/no/off) and refuses
# anything it cannot read, instead of treating every value but 0 as on.
+# - down retries a destroy request that timed out or got a 5xx (also 408,
+# 429 and 404) inside its poll loop until UBI_DOWN_TIMEOUT, and gives up
+# early only on another 4xx; reap bounds each VM's wait with
+# UBI_REAP_DOWN_TIMEOUT. Both are pending upstream (GSTA-36).
#
# Scratch record: every VM this machine creates is appended to a VM-id file
# (UBI_VM_RECORD; default ~/.capy/work/ubi-runner/vms.tsv when ~/.capy/work
@@ -102,6 +106,7 @@ GC_HOURS="${UBI_GC_HOURS:-0}"
API_TIMEOUT="${UBI_API_TIMEOUT:-120}"
CREATE_GRACE="${UBI_CREATE_GRACE:-180}"
DOWN_TIMEOUT="${UBI_DOWN_TIMEOUT:-900}"
+REAP_DOWN_TIMEOUT="${UBI_REAP_DOWN_TIMEOUT:-180}"
POLL="${UBI_POLL_SECONDS:-5}"
ALLOWED_STORAGE="${UBI_ALLOWED_STORAGE:-320 640}"
TTL_HOURS="${UBI_TTL_HOURS:-3}"
@@ -150,6 +155,9 @@ for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT; do
[[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"
done
[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"
+for v in DOWN_TIMEOUT REAP_DOWN_TIMEOUT; do
+ [[ "${!v}" =~ ^[1-9][0-9]*$ ]] || die "UBI_$v must be a whole number of seconds from 1"
+done
# gbrain: UBI_REAP=false (or no/off) must not turn reaping on.
case $(printf %s "$REAP" | tr '[:upper:]' '[:lower:]') in
0|false|no|off) REAP=0 ;;
@@ -385,6 +393,8 @@ mine_rows() {
# outside the prefix are never touched: only a TTL lets reap judge another lane's VM.
cmd_reap() {
local now rows loc name owner expires created n=0 failed=0
+ # cmd_down reads DOWN_TIMEOUT; a reap waits at most REAP_DOWN_TIMEOUT per VM.
+ local DOWN_TIMEOUT=$REAP_DOWN_TIMEOUT
[[ "$LEGACY_HOURS" =~ ^[1-9][0-9]{0,3}$ ]] || die "UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999 with no leading zero (how old a TTL-less ubirun-* VM must be before reap destroys it)"
now=$(date +%s)
rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
@@ -672,7 +682,7 @@ cmd_down() {
fi
fi
- local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now
+ local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now failed=""
out=$(mktemp)
while :; do
now=$(date +%s)
@@ -687,14 +697,19 @@ cmd_down() {
seen=1
if [ $(( now - asked )) -ge 60 ]; then
code=$(api "$out" vm "$loc/$name" destroy -f)
- if [ "$code" != 200 ]; then
+ if [ "$code" = 200 ]; then
+ asked=$now
+ failed=""
+ log "destroy requested: $loc/$name"
+ elif [[ "$code" =~ ^4[0-9][0-9]$ ]] && [ "$code" != 404 ] && [ "$code" != 408 ] && [ "$code" != 429 ]; then
cat "$out" >&2
rm -f "$out"
log "FAILED to destroy $loc/$name (HTTP $code); retry: $0 down $name"
return 1
+ else
+ [ "$code" = "$failed" ] || log "destroy request for $loc/$name got HTTP $code; retrying until ${DOWN_TIMEOUT}s have passed"
+ failed=$code
fi
- asked=$now
- log "destroy requested: $loc/$name"
fi
elif [ "$code" = 404 ]; then
if [ "$seen" = 1 ] || [ "$CREATED" != unknown ] || { [ $(( now - CREATE_AT )) -ge "$CREATE_GRACE" ] \
@@ -709,7 +724,7 @@ cmd_down() {
fi
if [ "$now" -ge "$deadline" ]; then
rm -f "$out"
- log "WARNING: could not confirm ${loc:-?}/$name is gone after ${DOWN_TIMEOUT}s; rerun: $0 down $name"
+ log "WARNING: could not confirm ${loc:-?}/$name is gone after ${DOWN_TIMEOUT}s${failed:+ (last destroy request: HTTP $failed)}; rerun: $0 down $name"
return 1
fi
sleep "$POLL"
@@ -1076,7 +1091,7 @@ env: UBI_TTL_HOURS (${TTL_HOURS}h in the name, 1-999), UBI_REAP (default 0 in gb
UBI_QUOTA_VCPUS (project vCPU ceiling for the preflight; else the last refusal's maximum),
UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),
UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
- UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
+ UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_REAP_DOWN_TIMEOUT (${REAP_DOWN_TIMEOUT}s per VM in reap), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640")
EOF
}
diff --git a/test/helpers/mock-ubicloud-api.ts b/test/helpers/mock-ubicloud-api.ts
index 45a0421156..31df109d6c 100644
--- a/test/helpers/mock-ubicloud-api.ts
+++ b/test/helpers/mock-ubicloud-api.ts
@@ -5,7 +5,9 @@
// VMs never reach `running`, so every `up` stays mid-provision until killed,
// unless `running` is set: then a created VM shows as running at 192.0.2.1
// (pair it with a fake `ssh` on PATH). `faults` makes `destroy` or `list`
-// answer HTTP 500 / 503 from the next request on.
+// answer HTTP 500 / 503 from the next request on; `destroyFaults` answers the
+// next destroy requests one by one with a status, or holds one past the
+// client's timeout without acting on it ("hang").
export interface MockVm {
location: string;
@@ -23,6 +25,8 @@ export interface MockUbicloudOptions {
rejectCreates?: boolean;
/** Created VMs show as running at 192.0.2.1. */
running?: boolean;
+ /** Answers for the next destroy requests, in order: an HTTP status, or "hang" (3 s, then a 504 nobody reads). */
+ destroyFaults?: Array<number | "hang">;
seed?: Array<{ location?: string; name: string; size?: string }>;
}
@@ -74,6 +78,12 @@ export function startMockUbicloud(opts: MockUbicloudOptions = {}): MockUbicloud
return text(`id: vm-${name}\nname: ${name}\nstate: ${vm.state}\nlocation: ${vm.location}\nsize: ${vm.size}\n${vm.state === "running" ? "ip4: 192.0.2.1\n" : ""}`);
}
if (verb === "destroy") {
+ const fault = opts.destroyFaults?.shift();
+ if (fault === "hang") {
+ await Bun.sleep(3000);
+ return text("! Unexpected response status: 504", 504);
+ }
+ if (fault !== undefined) return text(`! Unexpected response status: ${fault}`, fault);
if (faults.destroy) return text("! Unexpected response status: 500", 500);
if (vm && vm.state !== "destroying") {
vm.state = "destroying";
diff --git a/test/scripts/ubi-runner-run.test.ts b/test/scripts/ubi-runner-run.test.ts
index e31269a9ec..2934813a95 100644
--- a/test/scripts/ubi-runner-run.test.ts
+++ b/test/scripts/ubi-runner-run.test.ts
@@ -180,7 +180,7 @@ describe("owner and reap", () => {
it("reap reports a failed list and a failed destroy; bad UBI_TTL_HOURS refuses creates and bad UBI_LEGACY_HOURS refuses reap, never read-only commands", async () => {
const api = serve({ seed: [{ name: "ubirun-other-1000000000-bbbb-t1" }] });
api.faults.destroy = true;
- const destroyFails = await run(["reap"]);
+ const destroyFails = await run(["reap"], { UBI_REAP_DOWN_TIMEOUT: "2" });
expect(destroyFails.code).toBe(1);
expect(destroyFails.stderr).toContain("failed to reap eu-central-h1/ubirun-other-1000000000-bbbb-t1");
expect(destroyFails.stderr).toContain("reap: 1 of 1 expired VMs could not be destroyed");
@@ -225,6 +225,51 @@ describe("owner and reap", () => {
}, 90_000);
});
+// #8103 item 3: down gave up on the first destroy request that was not HTTP 200,
+// so one API timeout or 5xx left the VM up until its TTL and read as left=1.
+describe("down retries a failed destroy request", () => {
+ it("a destroy that timed out (000) or got a 5xx is retried inside the poll loop until the VM is gone", async () => {
+ const api = serve({ seed: [{ name: "vm-a" }], destroyFaults: [503, "hang", 502] });
+ const r = await run(["down", "vm-a"], { UBI_API_TIMEOUT: "1", UBI_DOWN_TIMEOUT: "60" });
+ expect(r.code).toBe(0);
+ expect(r.stderr).toContain("destroy request for eu-central-h1/vm-a got HTTP 503; retrying until 60s have passed");
+ expect(r.stderr).toContain("destroy request for eu-central-h1/vm-a got HTTP 000; retrying until 60s have passed");
+ expect(r.stderr).toContain("destroy request for eu-central-h1/vm-a got HTTP 502; retrying until 60s have passed");
+ expect(r.stderr).toContain("destroyed eu-central-h1/vm-a");
+ expect(r.stderr).not.toContain("FAILED");
+ expect(destroyed()).toEqual(["vm-a"]);
+ expect(api.vms.size).toBe(0);
+ }, 60_000);
+
+ it("a definite 4xx other than 404 fails at once; a destroy that keeps failing gives up at UBI_DOWN_TIMEOUT naming the last status", async () => {
+ const api = serve({ seed: [{ name: "vm-a" }], destroyFaults: [403] });
+ const started = Date.now();
+ const refused = await run(["down", "vm-a"], { UBI_DOWN_TIMEOUT: "60" });
+ expect(refused.code).toBe(1);
+ expect(Date.now() - started).toBeLessThan(15_000);
+ expect(refused.stderr).toContain("FAILED to destroy eu-central-h1/vm-a (HTTP 403); retry:");
+ expect(api.vms.has("vm-a")).toBe(true);
+ api.faults.destroy = true;
+ const stuck = await run(["down", "vm-a"], { UBI_DOWN_TIMEOUT: "2" });
+ expect(stuck.code).toBe(1);
+ expect(stuck.stderr).toContain("could not confirm eu-central-h1/vm-a is gone after 2s (last destroy request: HTTP 500); rerun:");
+ expect(stuck.stderr.match(/got HTTP 500; retrying/g)).toHaveLength(1);
+ expect(api.vms.has("vm-a")).toBe(true);
+ }, 60_000);
+
+ it("reap waits at most UBI_REAP_DOWN_TIMEOUT per VM, not UBI_DOWN_TIMEOUT", async () => {
+ const api = serve({ seed: [{ name: "ubirun-other-1000000000-bbbb-t1" }, { name: "ubirun-peer-1000000000-cccc-t1" }] });
+ api.faults.destroy = true;
+ const started = Date.now();
+ const r = await run(["reap"], { UBI_DOWN_TIMEOUT: "600", UBI_REAP_DOWN_TIMEOUT: "2" });
+ expect(r.code).toBe(1);
+ expect(Date.now() - started).toBeLessThan(30_000);
+ expect(r.stderr).toContain("could not confirm eu-central-h1/ubirun-other-1000000000-bbbb-t1 is gone after 2s (last destroy request: HTTP 500)");
+ expect(r.stderr).toContain("reap: 2 of 2 expired VMs could not be destroyed");
+ for (const bad of ["0", "x"]) expect((await run(["reap"], { UBI_REAP_DOWN_TIMEOUT: bad })).stderr).toContain("UBI_REAP_DOWN_TIMEOUT must be a whole number of seconds from 1");
+ }, 60_000);
+});
+
describe("run leftover line", () => {
it("ends with UBICLOUD: left=0 for its own VM, also in UBI_LEFT_FILE; a same-owner sibling is information only; --keep reports left=1 kept=1", async () => {
const api = serve({ seed: [{ name: "ubirun-t1-9999999999-sib1-t3" }] });