Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: restart sandbox VM when ssh stays down

ogsamrat26cbd41f0960
diff --git a/apps/server/src/box/backends/lima.ts b/apps/server/src/box/backends/lima.ts
index a1f515a5..f7f7acbe 100644
--- a/apps/server/src/box/backends/lima.ts
+++ b/apps/server/src/box/backends/lima.ts
@@ -535,21 +535,47 @@ type Ensure = (() => Promise<{ ok: boolean }>) | null;
/**
* Run a command in the VM as root. After a failure that says the VM is away
* (sleep, a stopped VM, a dead ssh master), the VM is started and the command
- * tried once more.
+ * tried again; a VM listed as running whose ssh still refuses is restarted once.
*/
-export async function vmCall(runtime: LimaRuntime, argv: string[], { run = runCommand, env = process.env, timeoutMs = 60_000, input, ensure = null, sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms)), tries = 3, onLine }: { run?: RunCommand; env?: Env; timeoutMs?: number; input?: string; ensure?: Ensure; sleep?: (ms: number) => Promise<void>; tries?: number; onLine?: OnLine } = {}): Promise<RunResult> {
+export async function vmCall(runtime: LimaRuntime, argv: string[], { run = runCommand, env = process.env, timeoutMs = 60_000, input, ensure = null, sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms)), tries = 4, onLine, lock = lockVm }: { run?: RunCommand; env?: Env; timeoutMs?: number; input?: string; ensure?: Ensure; sleep?: (ms: number) => Promise<void>; tries?: number; onLine?: OnLine; lock?: typeof lockVm } = {}): Promise<RunResult> {
const call = () => run(runtime.bin, inVm(runtime, argv), { env: limaEnv(runtime.home, env), timeoutMs, ...(input !== undefined ? { input } : {}), ...(onLine ? { onLine } : {}) });
let res = await call();
if (res.code === 0 || !vmAway(res) || !ensure) return res;
- const started = await ensure();
- if (!started?.ok) return res;
- // A VM that is up but whose sshd drops the first connections (just woken, many at once): a few more tries.
- for (let attempt = 1; attempt <= tries; attempt += 1) {
- res = await call();
- if (res.code === 0 || !vmAway(res) || attempt === tries) return res;
- await sleep(1_000 * attempt);
+ const again = async (): Promise<RunResult> => {
+ const started = await ensure();
+ if (!started?.ok) return res;
+ // A VM that is up but whose sshd drops the first connections (just woken, many at once): a few more tries.
+ for (let attempt = 1; attempt <= tries; attempt += 1) {
+ res = await call();
+ if (res.code === 0 || !vmAway(res) || attempt === tries) return res;
+ await sleep(1_000 * attempt);
+ }
+ return res;
+ };
+ res = await again();
+ // Listed as running but its ssh still refuses (sshd gone after a sleep, an idle stop half done): restart it once.
+ if (res.code === 0 || !vmAway(res) || res.error === "timeout") return res;
+ if (!(await restartVm(runtime, { run, env, lock })).ok) return res;
+ return again();
+}
+
+/**
+ * Stop a VM whose ssh does not answer, under the owner lock, so the next
+ * ensure starts it again; never while a box runs in it.
+ */
+export async function restartVm(runtime: LimaRuntime, { run = runCommand, env = process.env, lock = lockVm, pidAlive = alive }: { run?: RunCommand; env?: Env; lock?: typeof lockVm; pidAlive?: PidAlive } = {}): Promise<{ ok: boolean }> {
+ // Running boxes (another app's, this one's) keep running without ssh (ports go through Lima's agent): never taken down for this.
+ if (liveLeases(runtime.home, { pidAlive, except: process.pid }).length || [...keepers.values()].some((k) => k.child && k.child.exitCode === null)) return { ok: false };
+ const release = await lock(runtime.home, { what: "restart" });
+ if (!release) return { ok: false };
+ try {
+ const check = await run(runtime.bin, inVm(runtime, ["true"]), { env: limaEnv(runtime.home, env), timeoutMs: 30_000 });
+ // ssh answers again (another app restarted it, or the failure was the command's own): nothing to do.
+ if (check.code === 0) return { ok: false };
+ return stopVm(runtime, { run, env });
+ } finally {
+ release();
}
- return res;
}
/** Start the VM (made before). */
diff --git a/apps/server/src/box/box-lima.test.ts b/apps/server/src/box/box-lima.test.ts
index 58f6ba8e..1133767a 100644
--- a/apps/server/src/box/box-lima.test.ts
+++ b/apps/server/src/box/box-lima.test.ts
@@ -263,6 +263,23 @@ describe("lima: limactl calls", () => {
let ensured = 0;
const res = await vmCall(runtime, ["true"], { run: async () => (++n === 1 ? fail("kex_exchange_identification: read: Connection reset by peer", 255) : ok("fine")), ensure: async () => { ensured += 1; return { ok: true }; } });
expect({ code: res.code, n, ensured }).toEqual({ code: 0, n: 2, ensured: 1 });
+ // Listed as running, but its sshd resets every connection: stopped once (no box runs), started again, the call passes.
+ const home = tmp();
+ const vm = { status: "Running", sshDead: true, stops: 0 };
+ const dead: RunCommand = async (_bin, args) => {
+ if (args.includes("stop")) { vm.stops += 1; vm.status = "Stopped"; vm.sshDead = false; return ok(""); }
+ if (args[0] === "shell") return vm.sshDead || vm.status !== "Running" ? fail("kex_exchange_identification: read: Connection reset by peer", 255) : ok("in");
+ return fail("unexpected");
+ };
+ const lock = async () => () => {};
+ const startIt = async () => { vm.status = "Running"; return { ok: true }; };
+ const healed = await vmCall({ ...runtime, home }, ["true"], { run: dead, ensure: startIt, sleep: async () => {}, lock });
+ expect({ code: healed.code, stops: vm.stops }).toEqual({ code: 0, stops: 1 });
+ // Never while another app's box holds a lease: the error stays (the box keeps running).
+ Object.assign(vm, { sshDead: true, stops: 0 });
+ takeLease(home, "other-box", { pid: process.ppid, client: "cli" });
+ const kept = await vmCall({ ...runtime, home }, ["true"], { run: dead, ensure: startIt, sleep: async () => {}, lock });
+ expect({ code: kept.code, stops: vm.stops }).toEqual({ code: 255, stops: 0 });
expect(limaError(fail('time="2026" level=fatal msg="failed to start: \\"vz\\" requires macOS 13"'))).toBe('failed to start: "vz" requires macOS 13');
const gone = await vmRow(runtime, { run: async () => fail('level=warning msg="No instance matching omnirush-box found."\nlevel=fatal msg="unmatched instances"') });
expect({ row: gone.row, missing: gone.missing }).toEqual({ row: null, missing: true });