Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
ubi-runner: down retries a destroy request that timed out or got a 5xx (#8103 item 3)
garrytan7e3d729b9608
diff --git a/scripts/ubicloud/ubi-runner.sh b/scripts/ubicloud/ubi-runner.shindex 9922704a50..c2208b8a43 100755--- a/scripts/ubicloud/ubi-runner.sh+++ b/scripts/ubicloud/ubi-runner.sh@@ -22,6 +22,10 @@# - Bash 5.2 lost-signal-trap guard (set -E, FAILED, finish).# - UBI_REAP takes the usual false spellings (0/false/no/off) and refuses# anything it cannot read, instead of treating every value but 0 as on.+# - down retries a destroy request that timed out or got a 5xx (also 408,+# 429 and 404) inside its poll loop until UBI_DOWN_TIMEOUT, and gives up+# early only on another 4xx; reap bounds each VM's wait with+# UBI_REAP_DOWN_TIMEOUT. Both are pending upstream (GSTA-36).## Scratch record: every VM this machine creates is appended to a VM-id file# (UBI_VM_RECORD; default ~/.capy/work/ubi-runner/vms.tsv when ~/.capy/work@@ -102,6 +106,7 @@ GC_HOURS="${UBI_GC_HOURS:-0}"API_TIMEOUT="${UBI_API_TIMEOUT:-120}"CREATE_GRACE="${UBI_CREATE_GRACE:-180}"DOWN_TIMEOUT="${UBI_DOWN_TIMEOUT:-900}"+REAP_DOWN_TIMEOUT="${UBI_REAP_DOWN_TIMEOUT:-180}"POLL="${UBI_POLL_SECONDS:-5}"ALLOWED_STORAGE="${UBI_ALLOWED_STORAGE:-320 640}"TTL_HOURS="${UBI_TTL_HOURS:-3}"@@ -150,6 +155,9 @@ for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT; do[[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"done[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"+for v in DOWN_TIMEOUT REAP_DOWN_TIMEOUT; do+ [[ "${!v}" =~ ^[1-9][0-9]*$ ]] || die "UBI_$v must be a whole number of seconds from 1"+done# gbrain: UBI_REAP=false (or no/off) must not turn reaping on.case $(printf %s "$REAP" | tr '[:upper:]' '[:lower:]') in0|false|no|off) REAP=0 ;;@@ -385,6 +393,8 @@ mine_rows() {# outside the prefix are never touched: only a TTL lets reap judge another lane's VM.cmd_reap() {local now rows loc name owner expires created n=0 failed=0+ # cmd_down reads DOWN_TIMEOUT; a reap waits at most REAP_DOWN_TIMEOUT per VM.+ local DOWN_TIMEOUT=$REAP_DOWN_TIMEOUT[[ "$LEGACY_HOURS" =~ ^[1-9][0-9]{0,3}$ ]] || die "UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999 with no leading zero (how old a TTL-less ubirun-* VM must be before reap destroys it)"now=$(date +%s)rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }@@ -672,7 +682,7 @@ cmd_down() {fifi- local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now+ local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now failed=""out=$(mktemp)while :; donow=$(date +%s)@@ -687,14 +697,19 @@ cmd_down() {seen=1if [ $(( now - asked )) -ge 60 ]; thencode=$(api "$out" vm "$loc/$name" destroy -f)- if [ "$code" != 200 ]; then+ if [ "$code" = 200 ]; then+ asked=$now+ failed=""+ log "destroy requested: $loc/$name"+ elif [[ "$code" =~ ^4[0-9][0-9]$ ]] && [ "$code" != 404 ] && [ "$code" != 408 ] && [ "$code" != 429 ]; thencat "$out" >&2rm -f "$out"log "FAILED to destroy $loc/$name (HTTP $code); retry: $0 down $name"return 1+ else+ [ "$code" = "$failed" ] || log "destroy request for $loc/$name got HTTP $code; retrying until ${DOWN_TIMEOUT}s have passed"+ failed=$codefi- asked=$now- log "destroy requested: $loc/$name"fielif [ "$code" = 404 ]; thenif [ "$seen" = 1 ] || [ "$CREATED" != unknown ] || { [ $(( now - CREATE_AT )) -ge "$CREATE_GRACE" ] \@@ -709,7 +724,7 @@ cmd_down() {fiif [ "$now" -ge "$deadline" ]; thenrm -f "$out"- log "WARNING: could not confirm ${loc:-?}/$name is gone after ${DOWN_TIMEOUT}s; rerun: $0 down $name"+ log "WARNING: could not confirm ${loc:-?}/$name is gone after ${DOWN_TIMEOUT}s${failed:+ (last destroy request: HTTP $failed)}; rerun: $0 down $name"return 1fisleep "$POLL"@@ -1076,7 +1091,7 @@ env: UBI_TTL_HOURS (${TTL_HOURS}h in the name, 1-999), UBI_REAP (default 0 in gbUBI_QUOTA_VCPUS (project vCPU ceiling for the preflight; else the last refusal's maximum),UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),- UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),+ UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_REAP_DOWN_TIMEOUT (${REAP_DOWN_TIMEOUT}s per VM in reap), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640")EOF}diff --git a/test/helpers/mock-ubicloud-api.ts b/test/helpers/mock-ubicloud-api.tsindex 45a0421156..31df109d6c 100644--- a/test/helpers/mock-ubicloud-api.ts+++ b/test/helpers/mock-ubicloud-api.ts@@ -5,7 +5,9 @@// VMs never reach `running`, so every `up` stays mid-provision until killed,// unless `running` is set: then a created VM shows as running at 192.0.2.1// (pair it with a fake `ssh` on PATH). `faults` makes `destroy` or `list`-// answer HTTP 500 / 503 from the next request on.+// answer HTTP 500 / 503 from the next request on; `destroyFaults` answers the+// next destroy requests one by one with a status, or holds one past the+// client's timeout without acting on it ("hang").export interface MockVm {location: string;@@ -23,6 +25,8 @@ export interface MockUbicloudOptions {rejectCreates?: boolean;/** Created VMs show as running at 192.0.2.1. */running?: boolean;+ /** Answers for the next destroy requests, in order: an HTTP status, or "hang" (3 s, then a 504 nobody reads). */+ destroyFaults?: Array<number | "hang">;seed?: Array<{ location?: string; name: string; size?: string }>;}@@ -74,6 +78,12 @@ export function startMockUbicloud(opts: MockUbicloudOptions = {}): MockUbicloudreturn text(`id: vm-${name}\nname: ${name}\nstate: ${vm.state}\nlocation: ${vm.location}\nsize: ${vm.size}\n${vm.state === "running" ? "ip4: 192.0.2.1\n" : ""}`);}if (verb === "destroy") {+ const fault = opts.destroyFaults?.shift();+ if (fault === "hang") {+ await Bun.sleep(3000);+ return text("! Unexpected response status: 504", 504);+ }+ if (fault !== undefined) return text(`! Unexpected response status: ${fault}`, fault);if (faults.destroy) return text("! Unexpected response status: 500", 500);if (vm && vm.state !== "destroying") {vm.state = "destroying";diff --git a/test/scripts/ubi-runner-run.test.ts b/test/scripts/ubi-runner-run.test.tsindex e31269a9ec..2934813a95 100644--- a/test/scripts/ubi-runner-run.test.ts+++ b/test/scripts/ubi-runner-run.test.ts@@ -180,7 +180,7 @@ describe("owner and reap", () => {it("reap reports a failed list and a failed destroy; bad UBI_TTL_HOURS refuses creates and bad UBI_LEGACY_HOURS refuses reap, never read-only commands", async () => {const api = serve({ seed: [{ name: "ubirun-other-1000000000-bbbb-t1" }] });api.faults.destroy = true;- const destroyFails = await run(["reap"]);+ const destroyFails = await run(["reap"], { UBI_REAP_DOWN_TIMEOUT: "2" });expect(destroyFails.code).toBe(1);expect(destroyFails.stderr).toContain("failed to reap eu-central-h1/ubirun-other-1000000000-bbbb-t1");expect(destroyFails.stderr).toContain("reap: 1 of 1 expired VMs could not be destroyed");@@ -225,6 +225,51 @@ describe("owner and reap", () => {}, 90_000);});+// #8103 item 3: down gave up on the first destroy request that was not HTTP 200,+// so one API timeout or 5xx left the VM up until its TTL and read as left=1.+describe("down retries a failed destroy request", () => {+ it("a destroy that timed out (000) or got a 5xx is retried inside the poll loop until the VM is gone", async () => {+ const api = serve({ seed: [{ name: "vm-a" }], destroyFaults: [503, "hang", 502] });+ const r = await run(["down", "vm-a"], { UBI_API_TIMEOUT: "1", UBI_DOWN_TIMEOUT: "60" });+ expect(r.code).toBe(0);+ expect(r.stderr).toContain("destroy request for eu-central-h1/vm-a got HTTP 503; retrying until 60s have passed");+ expect(r.stderr).toContain("destroy request for eu-central-h1/vm-a got HTTP 000; retrying until 60s have passed");+ expect(r.stderr).toContain("destroy request for eu-central-h1/vm-a got HTTP 502; retrying until 60s have passed");+ expect(r.stderr).toContain("destroyed eu-central-h1/vm-a");+ expect(r.stderr).not.toContain("FAILED");+ expect(destroyed()).toEqual(["vm-a"]);+ expect(api.vms.size).toBe(0);+ }, 60_000);++ it("a definite 4xx other than 404 fails at once; a destroy that keeps failing gives up at UBI_DOWN_TIMEOUT naming the last status", async () => {+ const api = serve({ seed: [{ name: "vm-a" }], destroyFaults: [403] });+ const started = Date.now();+ const refused = await run(["down", "vm-a"], { UBI_DOWN_TIMEOUT: "60" });+ expect(refused.code).toBe(1);+ expect(Date.now() - started).toBeLessThan(15_000);+ expect(refused.stderr).toContain("FAILED to destroy eu-central-h1/vm-a (HTTP 403); retry:");+ expect(api.vms.has("vm-a")).toBe(true);+ api.faults.destroy = true;+ const stuck = await run(["down", "vm-a"], { UBI_DOWN_TIMEOUT: "2" });+ expect(stuck.code).toBe(1);+ expect(stuck.stderr).toContain("could not confirm eu-central-h1/vm-a is gone after 2s (last destroy request: HTTP 500); rerun:");+ expect(stuck.stderr.match(/got HTTP 500; retrying/g)).toHaveLength(1);+ expect(api.vms.has("vm-a")).toBe(true);+ }, 60_000);++ it("reap waits at most UBI_REAP_DOWN_TIMEOUT per VM, not UBI_DOWN_TIMEOUT", async () => {+ const api = serve({ seed: [{ name: "ubirun-other-1000000000-bbbb-t1" }, { name: "ubirun-peer-1000000000-cccc-t1" }] });+ api.faults.destroy = true;+ const started = Date.now();+ const r = await run(["reap"], { UBI_DOWN_TIMEOUT: "600", UBI_REAP_DOWN_TIMEOUT: "2" });+ expect(r.code).toBe(1);+ expect(Date.now() - started).toBeLessThan(30_000);+ expect(r.stderr).toContain("could not confirm eu-central-h1/ubirun-other-1000000000-bbbb-t1 is gone after 2s (last destroy request: HTTP 500)");+ expect(r.stderr).toContain("reap: 2 of 2 expired VMs could not be destroyed");+ for (const bad of ["0", "x"]) expect((await run(["reap"], { UBI_REAP_DOWN_TIMEOUT: bad })).stderr).toContain("UBI_REAP_DOWN_TIMEOUT must be a whole number of seconds from 1");+ }, 60_000);+});+describe("run leftover line", () => {it("ends with UBICLOUD: left=0 for its own VM, also in UBI_LEFT_FILE; a same-owner sibling is information only; --keep reports left=1 kept=1", async () => {const api = serve({ seed: [{ name: "ubirun-t1-9999999999-sib1-t3" }] });