Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: restart the sandbox helper when it stops

ogsamratb6919dd359be
diff --git a/apps/desktop/box/go/cmd/boxrun/main.go b/apps/desktop/box/go/cmd/boxrun/main.go
index c6a2ce73..f499204e 100644
--- a/apps/desktop/box/go/cmd/boxrun/main.go
+++ b/apps/desktop/box/go/cmd/boxrun/main.go
@@ -626,9 +626,11 @@ func writeJSONFile(file string, v any) error {
return os.Rename(tmp, file)
}
-// alive: the pid runs and is our launcher (not a reused pid).
+// alive: the pid runs and is our launcher (not a reused pid: after a
+// restart of the machine or of the WSL distro, pids start over, and the
+// recorded one can be this very boxrun, which would then kill itself).
func alive(s *state) bool {
- if s == nil || s.Pid <= 0 {
+ if s == nil || s.Pid <= 0 || s.Pid == os.Getpid() || s.Pid == os.Getppid() {
return false
}
if syscall.Kill(s.Pid, 0) != nil {
diff --git a/apps/desktop/box/go/cmd/boxsh/main.go b/apps/desktop/box/go/cmd/boxsh/main.go
index 5f2b46cc..736c4004 100644
--- a/apps/desktop/box/go/cmd/boxsh/main.go
+++ b/apps/desktop/box/go/cmd/boxsh/main.go
@@ -53,6 +53,10 @@ type Record struct {
Shell string `json:"shell"`
PathPre string `json:"path_prefix"`
EnvFile string `json:"env_file"`
+ // Revive: the app that wrote the record restarts boxd when asked
+ // (boxes/<key>.revive); Updated is when it last wrote the record (ms).
+ Revive bool `json:"revive"`
+ Updated int64 `json:"updated"`
}
func main() {
@@ -90,12 +94,33 @@ func main() {
req := proto.Request{Op: "exec", Token: rec.Token, Argv: argv, Cwd: mapPath(cwd, rec), Env: forwardEnv(os.Environ())}
start := time.Now()
code, timing, err := runRemote(rec, req)
- if err != nil {
- if rec.Runtime == "" && rec.Backend != "" {
- // A backend with no `exec` of its own (bubblewrap, the WSL distro): boxd is the only way in.
- fmt.Fprintf(os.Stderr, "omnirush sandbox: the sandbox is not reachable (%v); `omnirush sandbox status` says why\n", err)
+ if err != nil && rec.Runtime == "" && rec.Backend != "" {
+ // A backend with no `exec` of its own (bubblewrap, the WSL distro): boxd is the only
+ // way in. A boxd that stopped (idle limit, crash, the distro restarted) is started
+ // again by the app that owns the session, and the command runs then. Never on this
+ // machine instead.
+ if !isDialError(err) {
+ fmt.Fprintf(os.Stderr, "omnirush sandbox: the connection to the sandbox failed (%v); %s\n", err, recoverHint)
+ os.Exit(125)
+ }
+ again, why := revive(key, rec, err)
+ if again == nil {
+ if strings.Contains(why, "Restart OmniRush") {
+ fmt.Fprintf(os.Stderr, "omnirush sandbox: %s\n", why)
+ } else {
+ fmt.Fprintf(os.Stderr, "omnirush sandbox: the sandbox stopped and was not restarted (%s); %s\n", why, recoverHint)
+ }
os.Exit(125)
}
+ rec = again
+ req.Token = rec.Token
+ code, timing, err = runRemote(rec, req)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, retryFailed(err))
+ os.Exit(125)
+ }
+ }
+ if err != nil {
// boxd is not reachable (restarting, crashed): run it with the runtime's exec instead.
code = runtimeExec(rec, req)
timing.Fallback = true
@@ -106,6 +131,87 @@ func main() {
os.Exit(code)
}
+// recoverHint: what the user can do when the sandbox cannot be reached.
+const recoverHint = "restart OmniRush (installs are kept), `omnirush sandbox reset` for a fresh sandbox, or `omnirush sandbox off` to run this folder on this machine"
+
+// isDialError: nothing reached boxd (no listener, refused, timed out): the
+// command did not start, so running it again cannot run it twice.
+func isDialError(err error) bool {
+ var op *net.OpError
+ return errors.As(err, &op) && op.Op == "dial"
+}
+
+// retryFailed: the line for a command that failed again after boxd was started again.
+func retryFailed(err error) string {
+ if isDialError(err) {
+ return fmt.Sprintf("omnirush sandbox: the sandbox is not reachable after a restart (%v); %s", err, recoverHint)
+ }
+ return fmt.Sprintf("omnirush sandbox: the connection to the sandbox broke during the command (%v); %s", err, recoverHint)
+}
+
+func reviveFile(key string) string {
+ return filepath.Join(boxHome(), "boxes", key+".revive")
+}
+
+// reviveWait: how long a command waits for the sandbox to be started again.
+func reviveWait() time.Duration {
+ if v, err := strconv.Atoi(os.Getenv("OMNIRUSH_BOX_REVIVE_WAIT_MS")); err == nil && v >= 0 {
+ return time.Duration(v) * time.Millisecond
+ }
+ return 60 * time.Second
+}
+
+// revive asks the app that owns the session to start boxd again
+// (boxes/<key>.revive) and waits for the record to say where it is now.
+// Returns the new record, or nil and why not. A request nobody took within
+// the wait (no app restarts this box) is not made again: later commands
+// fail at once instead of each waiting.
+func revive(key string, rec *Record, cause error) (*Record, string) {
+ if !rec.Revive {
+ return nil, fmt.Sprintf("%v", cause)
+ }
+ wait := reviveWait()
+ file := reviveFile(key)
+ if st, err := os.Stat(file); err == nil && time.Since(st.ModTime()) > wait {
+ return nil, fmt.Sprintf("%v; no answer to the restart request", cause)
+ }
+ asked := time.Now()
+ body, _ := json.Marshal(map[string]any{"at": asked.UnixMilli(), "port": rec.Port, "error": cause.Error()})
+ if _, err := os.Stat(file); err != nil {
+ tmp := fmt.Sprintf("%s.%d.tmp", file, os.Getpid())
+ if os.WriteFile(tmp, body, 0o600) == nil {
+ _ = os.Rename(tmp, file)
+ }
+ }
+ fmt.Fprintln(os.Stderr, "omnirush sandbox: the sandbox stopped; starting it again...")
+ deadline := asked.Add(wait)
+ for time.Now().Before(deadline) {
+ time.Sleep(150 * time.Millisecond)
+ now, err := readRecord(key)
+ if err != nil {
+ continue
+ }
+ switch now.State {
+ case "failed":
+ msg := now.Error
+ if msg == "" {
+ msg = "it could not start"
+ }
+ return nil, msg
+ case "off":
+ return nil, "the sandbox was turned off for this session"
+ case "ready":
+ // Started again (a new port or a newer record), or boxd answered after all
+ // (the app took the request and found it running).
+ _, pending := os.Stat(file)
+ if now.Port != rec.Port || now.Updated > rec.Updated || pending != nil {
+ return now, ""
+ }
+ }
+ }
+ return nil, fmt.Sprintf("%v; it did not start again in %v", cause, wait.Round(time.Second))
+}
+
func (r *Record) shell() string {
if r.Shell != "" {
return r.Shell
@@ -196,6 +302,9 @@ func waitReady(key string) (*Record, error) {
if msg == "" {
msg = "it could not start"
}
+ if strings.Contains(msg, "Restart OmniRush") {
+ return nil, errors.New(msg) // says what to do already
+ }
return nil, fmt.Errorf("the sandbox is not available (%s); run `omnirush sandbox status`", msg)
}
}
diff --git a/apps/desktop/box/go/cmd/boxsh/main_test.go b/apps/desktop/box/go/cmd/boxsh/main_test.go
index c3e4d8d2..54a9222d 100644
--- a/apps/desktop/box/go/cmd/boxsh/main_test.go
+++ b/apps/desktop/box/go/cmd/boxsh/main_test.go
@@ -1,10 +1,14 @@
package main
import (
+ "encoding/json"
+ "errors"
+ "net"
"os"
"path/filepath"
"reflect"
"runtime"
+ "strconv"
"strings"
"testing"
"time"
@@ -105,3 +109,109 @@ func TestWaitReadyThroughDownload(t *testing.T) {
t.Fatalf("a start that hangs still times out: %v", err)
}
}
+
+func writeRec(t *testing.T, home, key string, rec map[string]any) {
+ t.Helper()
+ data, _ := json.Marshal(rec)
+ if err := os.MkdirAll(filepath.Join(home, "boxes"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(home, "boxes", key+".json"), data, 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestReviveAsksAndWaitsForTheNewRecord(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("OMNIRUSH_BOX_HOME", home)
+ t.Setenv("OMNIRUSH_BOX_REVIVE_WAIT_MS", "3000")
+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "updated": 100, "revive": true, "backend": "bwrap", "token": "t1"})
+ rec, _ := readRecord("k1")
+ go func() {
+ // The session sees the request, starts boxd again and writes where it is.
+ for i := 0; i < 100; i++ {
+ if _, err := os.Stat(reviveFile("k1")); err == nil {
+ break
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 45001, "updated": 200, "revive": true, "backend": "bwrap", "token": "t2"})
+ _ = os.Remove(reviveFile("k1"))
+ }()
+ again, why := revive("k1", rec, errors.New("dial tcp 127.0.0.1:44897: connect: connection refused"))
+ if again == nil || again.Port != 45001 || again.Token != "t2" {
+ t.Fatalf("got %+v (%s)", again, why)
+ }
+}
+
+func TestReviveFailsWithTheSessionsReason(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("OMNIRUSH_BOX_HOME", home)
+ t.Setenv("OMNIRUSH_BOX_REVIVE_WAIT_MS", "3000")
+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "updated": 100, "revive": true, "backend": "bwrap"})
+ rec, _ := readRecord("k1")
+ go func() {
+ time.Sleep(100 * time.Millisecond)
+ writeRec(t, home, "k1", map[string]any{"state": "failed", "error": "the sandbox stopped and could not start again (x). Restart OmniRush (installs are kept), or `omnirush sandbox off`", "revive": true, "backend": "bwrap"})
+ }()
+ again, why := revive("k1", rec, errors.New("refused"))
+ if again != nil || !strings.Contains(why, "omnirush sandbox off") {
+ t.Fatalf("got %+v %q", again, why)
+ }
+}
+
+func TestReviveOnlyWhenTheAppRestartsAndNotTwice(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("OMNIRUSH_BOX_HOME", home)
+ t.Setenv("OMNIRUSH_BOX_REVIVE_WAIT_MS", "200")
+ // An app that does not restart boxd (no "revive"): no request, no wait.
+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "backend": "bwrap"})
+ rec, _ := readRecord("k1")
+ start := time.Now()
+ if again, _ := revive("k1", rec, errors.New("refused")); again != nil || time.Since(start) > 100*time.Millisecond {
+ t.Fatal("waited for an app that does not restart")
+ }
+ if _, err := os.Stat(reviveFile("k1")); err == nil {
+ t.Fatal("asked an app that does not restart")
+ }
+ // Nobody answers: the first call waits once; a later one (request older than the wait) fails at once.
+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "revive": true, "backend": "bwrap"})
+ rec, _ = readRecord("k1")
+ if again, why := revive("k1", rec, errors.New("refused")); again != nil || !strings.Contains(why, "did not start again") {
+ t.Fatalf("got %+v %q", again, why)
+ }
+ time.Sleep(50 * time.Millisecond)
+ start = time.Now()
+ if again, why := revive("k1", rec, errors.New("refused")); again != nil || !strings.Contains(why, "no answer") || time.Since(start) > 100*time.Millisecond {
+ t.Fatalf("waited again: %q", why)
+ }
+}
+
+func TestIsDialError(t *testing.T) {
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ port := ln.Addr().(*net.TCPAddr).Port
+ ln.Close()
+ _, err = net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), time.Second)
+ if err == nil || !isDialError(err) {
+ t.Fatalf("refused dial: %v", err)
+ }
+ if isDialError(errors.New("EOF")) {
+ t.Fatal("a read error is not a dial error")
+ }
+}
+
+func TestRetryFailedSaysWhatBroke(t *testing.T) {
+ dial := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")}
+ if got := retryFailed(dial); !strings.Contains(got, "not reachable after a restart") {
+ t.Fatal(got)
+ }
+ for _, err := range []error{errors.New("EOF"), &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}} {
+ got := retryFailed(err)
+ if !strings.Contains(got, "broke during the command") || strings.Contains(got, "not reachable") {
+ t.Fatal(got)
+ }
+ }
+}
diff --git a/apps/server/src/box/backends/types.ts b/apps/server/src/box/backends/types.ts
index 4471394e..8e76bef3 100644
--- a/apps/server/src/box/backends/types.ts
+++ b/apps/server/src/box/backends/types.ts
@@ -37,6 +37,8 @@ export type OpenContext = {
/** The download's output lines as they come (its progress). */
onLine?: RunOptions["onLine"];
spawnImpl?: SpawnLike;
+ /** boxd's ping (tests); client.ping otherwise. */
+ ping?: (endpoint: BoxEndpoint, options?: { timeoutMs?: number }) => Promise<Record<string, unknown> | null>;
};
export type Opened =
diff --git a/apps/server/src/box/backends/wsl.ts b/apps/server/src/box/backends/wsl.ts
index 5181fd68..486e0c26 100644
--- a/apps/server/src/box/backends/wsl.ts
+++ b/apps/server/src/box/backends/wsl.ts
@@ -41,6 +41,7 @@ import bundledPins from "../wsl.json" with { type: "json" };
import { IDLE_MINUTES, newMeta, upArgs, type BoxMeta } from "./bwrap.js";
import { boxshExec, num, parseOut, str } from "./shared.js";
import type { BoxDriver, OpenContext, Opened, PresentImage, SpawnLike } from "./types.js";
+import { ping } from "../client.js";
type Env = NodeJS.ProcessEnv;
type WslRuntime = Pick<BoxRuntime, "bin" | "distro" | "arch" | "installed">;
@@ -324,7 +325,16 @@ async function open(ctx: OpenContext): Promise<Opened> {
if (meta && meta.root !== root) return { ok: false, error: `box ${key} belongs to another folder` };
const live = keepers.get(key);
if (live && live.child.exitCode === null && meta) {
- return { ok: true, created: null, endpoint: live.endpoint, alternates: live.alternates, uid: 0, gid: 0, containerId: `wsl_${key}_${meta.instance}`, imageId: meta.config || meta.image, mounts: meta.mounts ?? [] };
+ // The keeper outlives boxd (its idle limit, a crash) and Windows' forward of the
+ // distro's loopback (sleep): reused only when boxd answers, else started again.
+ const asks = ctx.ping ?? ping;
+ const answers = (await asks(live.endpoint, { timeoutMs: 1_500 })) || (live.alternates.length > 0 && (await asks(live.alternates[0]!, { timeoutMs: 1_500 })));
+ if (answers) {
+ return { ok: true, created: null, endpoint: live.endpoint, alternates: live.alternates, uid: 0, gid: 0, containerId: `wsl_${key}_${meta.instance}`, imageId: meta.config || meta.image, mounts: meta.mounts ?? [] };
+ }
+ log("info", "box keeper running but boxd not answering; starting it again", { key });
+ try { live.child.kill(); } catch { /* gone */ }
+ keepers.delete(key);
}
let present = await imagePresent(runtime, image, { run, env });
if (!present && !meta) {
diff --git a/apps/server/src/box/box-revive.test.ts b/apps/server/src/box/box-revive.test.ts
new file mode 100644
index 00000000..f54d7ab2
--- /dev/null
+++ b/apps/server/src/box/box-revive.test.ts
@@ -0,0 +1,162 @@
+// A box whose boxd stopped while the app ran (its idle limit, a crash, the WSL
+// distro restarted): the project's box run starts it again when a shell call
+// asks (boxsh writes boxes/<key>.revive) or at a turn's checkpoint, and says
+// once when it cannot. Never this computer instead. No container runtime here.
+import { describe, expect, test } from "bun:test";
+import { EventEmitter } from "node:events";
+import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { PassThrough } from "node:stream";
+
+import { reviveFile, type Box } from "./box.js";
+import { wslDriver } from "./backends/wsl.js";
+import type { OpenContext } from "./backends/types.js";
+import { boxRun, reviveError, type ReadyPlan } from "./launch.js";
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+const tmp = () => mkdtempSync(join(tmpdir(), "box-revive-"));
+async function until(fn: () => boolean, ms = 3_000): Promise<boolean> {
+ const end = Date.now() + ms;
+ while (Date.now() < end) { if (fn()) return true; await sleep(20); }
+ return false;
+}
+
+type FakeBox = { ready: boolean; error: string | null; endpoint: { host: string; port: number; token: string }; turns: number[]; failed: string | null; setTurn: (turn: number) => void; fail: (message: string) => FakeBox; checkpoint: () => Promise<unknown> };
+function fakeBox(port: number, { ready = true, error = null }: { ready?: boolean; error?: string | null } = {}): FakeBox {
+ const box: FakeBox = {
+ ready, error, endpoint: { host: "127.0.0.1", port, token: "t" }, turns: [], failed: null,
+ setTurn(turn) { box.turns.push(turn); },
+ fail(message) { box.ready = false; box.failed = message; box.error = message; return box; },
+ checkpoint: async () => ({ summary: { stats: { copied: 0, kept: 0 }, delta: { changed: [] }, bytes: 0 }, commit() {}, discard() {} }),
+ };
+ return box;
+}
+
+function fakePlan(first: FakeBox, next: FakeBox[]): ReadyPlan & { restarts: number } {
+ const home = tmp();
+ mkdirSync(join(home, "boxes"), { recursive: true });
+ const plan = {
+ box: { key: "k1", home, root: "/p", runtime: { name: "bwrap", backend: "bwrap", bin: "/x/boxrun" } },
+ notice: null,
+ shell: "/x/boxsh",
+ restarts: 0,
+ phase: "ready",
+ start: async () => first as unknown as Box,
+ restart: async () => { plan.restarts += 1; return next.shift() as unknown as Box; },
+ };
+ return plan as unknown as ReadyPlan & { restarts: number };
+}
+
+const pinger = (live: Set<number>) => async (endpoint: { port: number }) => (live.has(endpoint.port) ? { ok: true } : null);
+
+describe("box revive", () => {
+ test("a restart request from boxsh: a stopped boxd is started again, the request answered, the turn carried", async () => {
+ const live = new Set([41001]);
+ const plan = fakePlan(fakeBox(41001), [fakeBox(41002)]);
+ const said: unknown[] = [];
+ const run = boxRun(plan, { pollMs: 20, ping: pinger(live) as never, onRevive: (e) => said.push(e) });
+ try {
+ expect((await run.ready())?.endpoint?.port).toBe(41001);
+ run.setTurn(3);
+ live.clear();
+ live.add(41002);
+ const request = reviveFile(plan.box.home, "k1");
+ writeFileSync(request, "{}");
+ expect(await until(() => !existsSync(request))).toBe(true);
+ expect(plan.restarts).toBe(1);
+ expect(run.box?.endpoint?.port).toBe(41002);
+ expect((run.box as unknown as FakeBox).turns).toEqual([3]);
+ expect(said).toEqual([{ ok: true, why: "request" }]);
+ expect(run.native).toBe(false);
+ } finally {
+ run.close();
+ }
+ });
+
+ test("a box that cannot start again: failed with what to do, said once, never this computer", async () => {
+ const down = () => fakeBox(0, { ready: false, error: "boxd did not answer in 30s" });
+ const plan = fakePlan(fakeBox(41021), [down(), down()]);
+ const said: Array<{ ok: boolean }> = [];
+ const natives: string[] = [];
+ const run = boxRun(plan, { pollMs: 20, ping: pinger(new Set()) as never, onRevive: (e) => said.push(e), onNative: (r) => natives.push(r) });
+ try {
+ await run.ready();
+ const request = reviveFile(plan.box.home, "k1");
+ writeFileSync(request, "{}");
+ expect(await until(() => !existsSync(request))).toBe(true);
+ expect((run.box as unknown as FakeBox).failed).toContain("could not start again (boxd did not answer in 30s)");
+ expect((run.box as unknown as FakeBox).failed).toContain("Restart OmniRush");
+ expect(await run.freeze("turn-2", { sessionId: "s1", turn: 2 })).toBeNull();
+ expect(plan.restarts).toBe(2);
+ expect(said.length).toBe(1);
+ expect(said[0]!.ok).toBe(false);
+ expect(natives).toEqual([]);
+ expect(run.native).toBe(false);
+ } finally {
+ run.close();
+ }
+ });
+
+ test("each turn starts with a box that answers: a stopped boxd is started before the checkpoint", async () => {
+ const live = new Set([41031]);
+ const plan = fakePlan(fakeBox(41031), [fakeBox(41032)]);
+ const run = boxRun(plan, { pollMs: 10_000, ping: pinger(live) as never });
+ try {
+ expect(await run.freeze("turn-1", { sessionId: "s1", turn: 1 })).not.toBeNull();
+ expect(plan.restarts).toBe(0);
+ live.clear();
+ expect(await run.freeze("turn-2", { sessionId: "s1", turn: 2 })).not.toBeNull();
+ expect(plan.restarts).toBe(1);
+ expect(run.box?.endpoint?.port).toBe(41032);
+ } finally {
+ run.close();
+ }
+ });
+
+ test("reviveError names why and what to do", () => {
+ const text = reviveError("dial tcp 127.0.0.1:44897: connect: connection refused");
+ expect(text.startsWith("the sandbox stopped and could not start again (dial tcp 127.0.0.1:44897: connect: connection refused).")).toBe(true);
+ expect(text).toContain("Restart OmniRush (installs are kept)");
+ expect(reviveError("")).toContain("(it did not answer)");
+ });
+
+ test("WSL: a keeper still running whose boxd does not answer is replaced, not reused", async () => {
+ const home = tmp();
+ const bin = join(home, "bin");
+ mkdirSync(join(bin, "linux-amd64"), { recursive: true });
+ for (const name of ["boxrun", "boxd"]) writeFileSync(join(bin, "linux-amd64", name), "");
+ const env = { OMNIRUSH_BOX_BIN_DIR: bin };
+ const runtime = { name: "wsl", backend: "wsl", bin: "C:\\Windows\\System32\\wsl.exe", distro: "OmniRushBox", installed: true, arch: "amd64" };
+ const run = async (_bin: string, args: string[]) => (args.includes("image")
+ ? { code: 0, stdout: '{"present":true,"id":"img1","digest":"sha256:x","arch":"amd64","config":"cfg1"}\n', stderr: "" }
+ : { code: 0, stdout: "", stderr: "" });
+ let port = 40100;
+ const spawned: Array<EventEmitter & { killed: boolean }> = [];
+ const spawnImpl = (_bin: string, args: string[]) => {
+ const child = Object.assign(new EventEmitter(), { stdout: new PassThrough(), stderr: new PassThrough(), stdin: new PassThrough(), exitCode: null, args, unref() {}, killed: false, kill() { child.killed = true; return true; } });
+ const p = ++port;
+ setTimeout(() => child.stdout.write(`{"ok":true,"reused":false,"pid":7,"port":${p}}\n`), 5);
+ spawned.push(child);
+ return child;
+ };
+ const live = new Set<number>();
+ const ctx = {
+ key: "wk1", name: "omnirush-box-wk1", root: "C:\\p", croot: "/c/p", runtime, run, home, runFolder: join(home, "run"), image: { ref: "r", pull: "r", stack: "base" },
+ pull: false, shims: null, pkglog: null, env, platform: "win32", log: () => {}, onPulling: () => {}, spawnImpl, ping: pinger(live),
+ } as unknown as OpenContext;
+ const first = await wslDriver.open(ctx);
+ if (!first.ok) throw new Error(first.error);
+ live.add(first.endpoint.port);
+ const again = await wslDriver.open(ctx);
+ if (!again.ok) throw new Error(again.error);
+ expect(again.endpoint.port).toBe(first.endpoint.port);
+ expect(spawned.length).toBe(1);
+ live.clear();
+ const third = await wslDriver.open(ctx);
+ if (!third.ok) throw new Error(third.error);
+ expect(spawned.length).toBe(2);
+ expect(spawned[0]!.killed).toBe(true);
+ expect(third.endpoint.port).not.toBe(first.endpoint.port);
+ });
+});
diff --git a/apps/server/src/box/box.test.ts b/apps/server/src/box/box.test.ts
index a63330eb..0b1a7786 100644
--- a/apps/server/src/box/box.test.ts
+++ b/apps/server/src/box/box.test.ts
@@ -334,7 +334,7 @@ describe("box service", () => {
key: "k", container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,
ready: async () => null,
label: () => null,
- setTurn: () => {},
+ setTurn: () => {}, heal: async () => null, close: () => {},
freeze: async (label, { sessionId, turn = null, sessionTurn = null }) => {
frozen.push(`${sessionId} ${label} ${sessionTurn}`);
return { label, turn, copied: 0, carried: 0, save: async () => ({ label, changed: 0, bytes: 0 }), discard: async () => null };
@@ -378,7 +378,7 @@ describe("box service", () => {
key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,
ready: async () => null,
label: () => null,
- setTurn: () => {},
+ setTurn: () => {}, heal: async () => null, close: () => {},
freeze: async (label, { sessionId, turn = null }) => {
frozen.push(`${sessionId} ${label}`);
return { label, turn, copied: 0, carried: 0, save: async () => ({ label, changed: 0, bytes: 0 }), discard: async () => null };
@@ -420,7 +420,7 @@ describe("box service", () => {
key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,
ready: async () => null,
label: () => null,
- setTurn: () => {},
+ setTurn: () => {}, heal: async () => null, close: () => {},
freeze: async (label, { sessionId, turn = null }) => {
frozen.push(`${sessionId} ${label}`);
if (failNext) { failNext = false; throw new Error("checkpoint failed"); }
@@ -456,7 +456,7 @@ describe("box service", () => {
const workspace: BoxWorkspace = { id: "ws_o", path: root, workspaceType: "local" };
const run: BoxRun = {
key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,
- ready: async () => null, label: () => null, setTurn: () => {},
+ ready: async () => null, label: () => null, setTurn: () => {}, heal: async () => null, close: () => {},
freeze: async (label, { turn = null }) => ({ label, turn, copied: 0, carried: 0, save: async () => ({ label, changed: 0, bytes: 0 }), discard: async () => null }),
};
rememberServerDefault(stateDir, { local_box: { default: "off" } });
@@ -508,7 +508,7 @@ describe("box service", () => {
const boxes = service({ stateDir, workspaces: [workspace], settings: new Map(), env: { OMNIRUSH_BOX_HOME: await temp("boxhome") } });
const run: BoxRun = {
key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,
- ready: async () => null, label: () => null, setTurn: () => {}, freeze: async () => null,
+ ready: async () => null, label: () => null, setTurn: () => {}, heal: async () => null, close: () => {}, freeze: async () => null,
};
const projects = boxes["projects"];
projects.set(boxKey(root), { run, notice: null, planning: null, nextPlanAt: Date.now() + 60_000, waitingImage: false });
@@ -543,7 +543,7 @@ describe("box service", () => {
box.setState("ready", null);
const run: BoxRun = {
key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box,
- ready: async () => box, label: () => null, setTurn: () => {}, freeze: async () => null,
+ ready: async () => box, label: () => null, setTurn: () => {}, heal: async () => null, close: () => {}, freeze: async () => null,
};
boxes["projects"].set(boxKey(root), { run, notice: null, planning: null, nextPlanAt: 0, waitingImage: false });
expect(await boxes.ensure(workspace)).toBe(run);
diff --git a/apps/server/src/box/box.ts b/apps/server/src/box/box.ts
index dfa8e4f1..c20ec026 100644
--- a/apps/server/src/box/box.ts
+++ b/apps/server/src/box/box.ts
@@ -198,7 +198,7 @@ type OnState = (state: BoxState, error: string | null) => void;
* (the record boxsh reads says the same). Throws only for programming
* errors; a box that cannot start is a Box with state "failed" and `error`.
*/
-export async function openBox({ root, runtime, env = process.env, platform = process.platform, home = boxHome(env), run = runCommand, log = () => {}, shims = null, pkglog = null, pull = true, image: asked = null, onState = () => {}, onLine, spawnImpl, readyMs = 15_000 }: { root: string; runtime: BoxRuntime; env?: Env; platform?: NodeJS.Platform; home?: string; run?: RunCommand; log?: BoxLog; shims?: string | null; pkglog?: BoxInstallLog | null; pull?: boolean; image?: BoxImage | null; onState?: OnState; onLine?: RunOptions["onLine"]; spawnImpl?: SpawnLike; readyMs?: number }): Promise<Box> {
+export async function openBox({ root, runtime, env = process.env, platform = process.platform, home = boxHome(env), run = runCommand, log = () => {}, shims = null, pkglog = null, pull = true, image: asked = null, onState = () => {}, onLine, spawnImpl, readyMs = 15_000, revive = false }: { root: string; runtime: BoxRuntime; env?: Env; platform?: NodeJS.Platform; home?: string; run?: RunCommand; log?: BoxLog; shims?: string | null; pkglog?: BoxInstallLog | null; pull?: boolean; image?: BoxImage | null; onState?: OnState; onLine?: RunOptions["onLine"]; spawnImpl?: SpawnLike; readyMs?: number; revive?: boolean }): Promise<Box> {
const projectRoot = path.resolve(root);
const key = boxKey(projectRoot, { platform });
const name = containerName(key, env);
@@ -206,6 +206,8 @@ export async function openBox({ root, runtime, env = process.env, platform = pro
const runFolder = runDir(home, key);
mkdirSync(path.join(runFolder, "state"), { recursive: true });
const box = new Box({ key, name, root: projectRoot, croot, runtime, home, runFolder, run, log, platform, env });
+ // The app restarts boxd when a shell call finds it stopped (boxsh asks through boxes/<key>.revive).
+ box.revive = revive;
const driver = driverFor(runtime);
const lock = await acquireLock(home, key);
try {
@@ -354,6 +356,8 @@ export class Box {
readonly container: string;
state: BoxState = "starting";
error: string | null = null;
+ /** The record tells boxsh the app restarts boxd when asked. */
+ revive = false;
endpoint: BoxEndpoint | null = null;
identity: Record<string, unknown> | null = null;
pathPrefix = "";
@@ -395,7 +399,7 @@ export class Box {
runtime: fallback ? this.runtime.bin ?? "" : "", runtime_args: fallback ? this.runtime.prefix ?? [] : [],
host: this.endpoint?.host ?? "127.0.0.1", port: this.endpoint?.port ?? 0, token: this.endpoint?.token ?? "",
state: this.state, error: this.error ?? "", uid: this.uid ?? 0, gid: this.gid ?? 0, home: HOME_IN_BOX, shell: "/bin/bash",
- path_prefix: this.pathPrefix, env_file: `${RUN_DIR}/env`, updated: Date.now(), pid: process.pid,
+ path_prefix: this.pathPrefix, env_file: `${RUN_DIR}/env`, updated: Date.now(), pid: process.pid, revive: this.revive,
};
writeJson(recordFile(this.home, this.key), record);
}
@@ -683,6 +687,11 @@ export class Box {
type Removed = Array<{ kind: string; name: string; reason: string }>;
/** Boxes on this machine: { key, name, root, running, image, created }[] */
+/** Where boxsh asks the app to start a stopped boxd again (launch.ts boxRun). */
+export function reviveFile(home: string, key: string): string {
+ return path.join(home, "boxes", `${key}.revive`);
+}
+
export async function listBoxes(runtime: BoxRuntime, { run = runCommand, env = process.env, home = boxHome(env) }: { run?: RunCommand; env?: Env; home?: string } = {}): Promise<ListedBox[]> {
return driverFor(runtime).list(runtime, { run, env, home });
}
diff --git a/apps/server/src/box/common.ts b/apps/server/src/box/common.ts
index 77ca9427..92defd44 100644
--- a/apps/server/src/box/common.ts
+++ b/apps/server/src/box/common.ts
@@ -94,6 +94,8 @@ export type BoxRecord = {
v: 1; key: string; root: string; croot: string; container: string; backend: string; runtime: string; runtime_args: string[];
host: string; port: number; token: string; state: string; error: string; uid: number; gid: number;
home: string; shell: string; path_prefix: string; env_file: string; updated: number; pid: number;
+ /** The app restarts boxd when boxsh asks (boxes/<key>.revive). */
+ revive?: boolean;
};
export function readRecord(home: string, key: string): BoxRecord | null {
diff --git a/apps/server/src/box/launch.ts b/apps/server/src/box/launch.ts
index 7f8b3db1..7032fcbc 100644
--- a/apps/server/src/box/launch.ts
+++ b/apps/server/src/box/launch.ts
@@ -18,10 +18,11 @@
// background, and a shell call waits for it (boxsh).
import { spawn, type ChildProcess } from "node:child_process";
-import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import path from "node:path";
-import { boxKey, containerName, gcBoxes, installShell, openBox, recordFile, savedTurns, type Box, type BoxInstallLog, type BoxLabel, type BoxLog, type BoxRuntime, type CheckpointSummary } from "./box.js";
+import * as client from "./client.js";
+import { boxKey, containerName, gcBoxes, installShell, openBox, recordFile, reviveFile, savedTurns, type Box, type BoxInstallLog, type BoxLabel, type BoxLog, type BoxRuntime, type CheckpointSummary } from "./box.js";
import { driverFor, normalizeBackend } from "./backends/index.js";
import type { BoxExec } from "./backends/types.js";
import { detectStack, imageFor, loadPins, type BoxImage } from "./images.js";
@@ -235,6 +236,11 @@ export type ReadyPlan = {
* POSIX wrappers), mounted into a new box and put on its PATH.
*/
start: (options?: { pkglog?: BoxInstallLog | null }) => Promise<Box>;
+ /**
+ * Open the box again (boxd stopped: its idle limit, a crash, the WSL distro
+ * restarted). The backend starts boxd again when it does not answer; installs are kept.
+ */
+ restart?: () => Promise<Box>;
/** What the box start is doing ("starting", "pulling", "ready", "failed"; null before it starts). */
readonly phase: string | null;
};
@@ -274,12 +280,20 @@ export async function planBox({ env = process.env, home, root, log = () => {}, r
if (env.OMNIRUSH_BOX_GC !== "off") setTimeout(() => gcInBackground(runtime, home, { log }), 30_000).unref?.();
let opening: Promise<Box> | null = null;
let phase: string | null = null;
+ let startedWith: BoxInstallLog | null = null;
+ const start = ({ pkglog = null }: { pkglog?: BoxInstallLog | null } = {}): Promise<Box> => {
+ if (!opening) startedWith = pkglog;
+ opening ??= openBox({ root, runtime, env, home, log, pull: false, shims, pkglog, revive: true, onState: (state) => { phase = state; } });
+ return opening;
+ };
return {
box: { key, home, root, runtime },
notice: null,
shell,
- start: ({ pkglog = null } = {}) => {
- opening ??= openBox({ root, runtime, env, home, log, pull: false, shims, pkglog, onState: (state) => { phase = state; } });
+ start,
+ restart: () => {
+ if (!opening) return start();
+ opening = openBox({ root, runtime, env, home, log, pull: false, shims, pkglog: startedWith, revive: true });
return opening;
},
get phase() { return phase; },
@@ -320,6 +334,15 @@ export function engineConfig(bits: { shell: string; plugin: string } | null, con
};
}
+/** How often a project's box looks for a restart request from boxsh. */
+export const REVIVE_POLL_MS = 400;
+
+/** The record's error when boxd stopped and could not start again: why, and what the user can do. */
+export function reviveError(error: string | null | undefined): string {
+ const why = String(error || "it did not answer").replace(/\s+/g, " ").slice(0, 240);
+ return `the sandbox stopped and could not start again (${why}). Restart OmniRush (installs are kept), reset this folder's sandbox for a fresh one, or turn the sandbox off for this folder to run it on this computer`;
+}
+
/** Turn the record off (the project went on without the box). */
export function recordOff(home: string, key: string, reason: string): void {
try {
@@ -357,6 +380,10 @@ export type BoxRun = {
/** environment.sandbox for a chat's uploads; null while it is not (or no longer) the chat's. */
label: (sessionId: string) => BoxLabel | null;
setTurn: (turn: number) => void;
+ /** Start boxd again when it does not answer (resolves the Box now in use). */
+ heal: (why: string) => Promise<Box | null>;
+ /** Stop watching for restart requests (the project's box is let go). */
+ close: () => void;
/** `sessionTurn`: the chat's own number of the prompt the checkpoint comes before (written as session_turn). */
freeze: (label: string, options: { sessionId: string; turn?: number | null; sessionTurn?: number | null; chain?: string | null }) => Promise<Frozen | null>;
};
@@ -369,11 +396,14 @@ export type BoxRun = {
* "off"; the shell runs on this machine, its chats are not labelled); later
* failures are logged and that turn has no checkpoint.
*/
-export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000, onNative = () => {}, tmpRoot, pkglog = null }: { log?: BoxLog; readyWaitMs?: number; onNative?: (reason: string) => void; tmpRoot?: string; pkglog?: BoxInstallLog | null } = {}): BoxRun {
+export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000, onNative = () => {}, onRevive = () => {}, tmpRoot, pkglog = null, pollMs = REVIVE_POLL_MS, ping = client.ping }: { log?: BoxLog; readyWaitMs?: number; onNative?: (reason: string) => void; onRevive?: (event: { ok: boolean; why: string; error?: string }) => void; tmpRoot?: string; pkglog?: BoxInstallLog | null; pollMs?: number; ping?: typeof client.ping } = {}): BoxRun {
const { key, home, runtime, root } = plan.box;
let box: Box | null = null;
let native = false;
const checkpoints = new Map<string, number>();
+ const requestFile = reviveFile(home, key);
+ // A restart request nobody answered before (the app quit): this start answers it.
+ try { rmSync(requestFile, { force: true }); } catch { /* not there */ }
const opening = plan.start({ pkglog }).then((opened) => { box = opened; return opened; });
const goNative = (reason: string) => {
if (native) return;
@@ -397,10 +427,64 @@ export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000,
};
tick();
});
+ // boxd stopped while the app ran (its idle limit, a crash, OOM, the WSL distro
+ // restarted, sleep): start it again, once at a time. Asked by a shell call that
+ // could not reach it (boxsh writes boxes/<key>.revive) and checked at each
+ // turn's checkpoint. A box that cannot start again stays "failed" with the
+ // reason and what to do (every shell call says it); the app says it once.
+ // Never this computer instead.
+ let healing: Promise<Box | null> | null = null;
+ let lostSaid = false;
+ let lastTurn: number | null = null;
+ const heal = (why: string): Promise<Box | null> => {
+ if (healing) return healing;
+ const since = Date.now();
+ healing = (async () => {
+ try {
+ const first = await opening;
+ if (native || !first.ready || !plan.restart) return box;
+ const current = box ?? first;
+ if (current.ready && current.endpoint && (await ping(current.endpoint, { timeoutMs: 1_500 }))) return current;
+ log("warn", "box not answering; starting it again", { why, key });
+ const again = await plan.restart();
+ box = again;
+ if (lastTurn !== null) again.setTurn(lastTurn);
+ if (again.ready) {
+ log("info", "box started again", { why, key });
+ lostSaid = false;
+ onRevive({ ok: true, why });
+ } else {
+ log("warn", "box could not start again", { why, key, error: again.error });
+ const error = reviveError(again.error);
+ again.fail(error);
+ if (!lostSaid) {
+ lostSaid = true;
+ onRevive({ ok: false, why, error });
+ }
+ }
+ return again;
+ } catch (error) {
+ log("warn", "box restart failed", { why, error: error instanceof Error ? error.message : String(error) });
+ return box;
+ } finally {
+ healing = null;
+ // Answered: boxsh reads the record now. A request made after this check started waits for the next one.
+ try { if (statSync(requestFile).mtimeMs <= since) rmSync(requestFile, { force: true }); } catch { /* none */ }
+ }
+ })();
+ return healing;
+ };
+ const watcher = setInterval(() => {
+ if (!native && !healing && existsSync(requestFile)) heal("request").catch(() => {});
+ }, pollMs);
+ watcher.unref?.();
const ready = async (): Promise<Box | null> => {
if (native) return null;
const opened = await Promise.race([opening, limit()]);
- if (opened?.ready) return opened;
+ if (opened?.ready) {
+ if (healing) await healing;
+ return box?.ready ? box : null;
+ }
if ([...checkpoints.values()].every((count) => count === 0)) goNative(opened?.error || "the sandbox did not start in time");
return null;
};
@@ -417,15 +501,21 @@ export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000,
get box() { return box; },
ready,
label: (sessionId) => (box?.ready && !native && (checkpoints.get(sessionId) ?? 0) > 0 ? box.label() : null),
- setTurn: (turn) => box?.setTurn(turn),
+ setTurn: (turn) => { lastTurn = turn; box?.setTurn(turn); },
+ heal,
+ close: () => clearInterval(watcher),
/**
* The old sandbox's freeze(label, { turn, chain }) -> { save(dir), discard() } | null:
* the checkpoint is taken now and placed by save(); `final` (the state a
* chat ends with) is a leaf, so a resumed chat chains on from its last turn.
*/
freeze: async (label, { sessionId, turn = null, sessionTurn = null, chain = null }) => {
- const opened = await ready();
- if (!opened) return null;
+ await ready();
+ if (native || !box) return null;
+ // Each turn starts with a box that answers (started again when it stopped since the last turn,
+ // or tried again when it could not start at the last try).
+ const opened = await heal("turn");
+ if (!opened?.ready) return null;
const n = Number(turn ?? /^turn-(\d+)$/.exec(label)?.[1] ?? 0) || null;
const name = n ? `turn-${n}` : label;
const dir = chain ?? path.join(pending, `${process.pid}-${Date.now()}`);
diff --git a/apps/server/src/box/service.ts b/apps/server/src/box/service.ts
index e3906840..09100c7f 100644
--- a/apps/server/src/box/service.ts
+++ b/apps/server/src/box/service.ts
@@ -233,7 +233,7 @@ export class BoxService {
const chosen = now.ok ? now.backend ?? now.name : null;
for (const project of this.projects.values()) {
if (project.run && project.run.backend === chosen) continue;
- project.run = null;
+ this.dropRun(project);
project.notice = null;
project.nextPlanAt = 0;
}
@@ -253,7 +253,7 @@ export class BoxService {
await Promise.allSettled([...this.projects.values()].map((project) => project.planning));
for (const project of this.projects.values()) {
if (!project.run || !allow || allow.includes(project.run.backend)) continue;
- project.run = null;
+ this.dropRun(project);
project.notice = null;
project.nextPlanAt = 0;
}
@@ -341,7 +341,7 @@ export class BoxService {
const { on, hostBuilds: gate } = await this.mode(workspace);
if (!on) {
if (gate?.native && project.notice !== gate.notice) this.log("info", "box: the server's default not applied, the folder has this computer's builds", { key, things: gate.things });
- project.run = null;
+ this.dropRun(project);
project.notice = gate?.native ? gate.notice : null;
await this.writeProjects();
return null;
@@ -352,7 +352,7 @@ export class BoxService {
// one, where every command fails. Planned again, the box restarts on the new folder (openBox).
if (!box?.ready || !box.endpoint || !(await this.folderReplaced({ endpoint: box.endpoint, croot: box.croot }).catch(() => false))) return project.run;
this.log("info", "box folder replaced while it ran: starting it again", { key });
- project.run = null;
+ this.dropRun(project);
project.nextPlanAt = 0;
}
if (Date.now() < project.nextPlanAt) return null;
@@ -363,17 +363,27 @@ export class BoxService {
: plan.runtime && !plan.runtime.ok ? { reason: "no_runtime", runtime: runtimeWhy(plan.runtime) }
: { reason: "helpers_missing" };
if (!plan.box) {
- project.run = null;
+ this.dropRun(project);
project.notice = plan.notice;
project.nextPlanAt = Date.now() + REPLAN_MS;
await this.writeProjects();
return null;
}
project.notice = null;
+ project.run?.close();
project.run = boxRun(plan, {
log: this.log,
pkglog: this.installLog ? { root: this.installLog.root, bin: this.installLog.bin, py: this.installLog.py } : null,
tmpRoot: join(this.home, "pending"),
+ // boxd stopped while the app ran: started again (installs kept), or once why not and what to do.
+ onRevive: (event) => {
+ if (event.ok) {
+ this.log("info", "box started again after it stopped", { key, why: event.why });
+ if (project.notice?.startsWith("Sandbox stopped:")) project.notice = null;
+ } else {
+ project.notice = `Sandbox stopped: ${event.error}.`;
+ }
+ },
onNative: (reason) => {
project.notice = `Sandbox off for now: ${reason}.`;
project.nextPlanAt = Date.now() + REPLAN_MS;
@@ -524,7 +534,7 @@ export class BoxService {
*/
async retry(workspace: BoxWorkspace): Promise<BoxStatus> {
const project = this.projects.get(boxKey(boxRoot(workspace)));
- if (project && (project.run?.native || project.run?.box?.state === "failed")) project.run = null;
+ if (project && (project.run?.native || project.run?.box?.state === "failed")) this.dropRun(project);
if (project) {
project.notice = null;
project.nextPlanAt = 0;
@@ -544,7 +554,7 @@ export class BoxService {
const runtime = await this.detect();
const project = this.projects.get(key);
if (project) {
- project.run = null;
+ this.dropRun(project);
project.nextPlanAt = 0;
}
await this.writeProjects();
@@ -639,6 +649,17 @@ export class BoxService {
}
/** At quit: every chat with checkpoints gets its `final` leaf (bounded); the sessions that got one. */
+ /** Let a project's box run go (no more restart requests answered for it). */
+ private dropRun(project: Project): void {
+ project.run?.close();
+ project.run = null;
+ }
+
+ /** The app quits: stop watching every project's box for restart requests. */
+ close(): void {
+ for (const project of this.projects.values()) project.run?.close();
+ }
+
async finals({ budgetMs = 20_000 }: { budgetMs?: number } = {}): Promise<string[]> {
const done: string[] = [];
const work = (async () => {
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index de1ce92c..1f78ab7f 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -1673,6 +1673,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
if (boxes.hasChats() && await Promise.race([archiveFinals.catch(() => false), new Promise<false>((resolvePromise) => setTimeout(() => resolvePromise(false), 1_000).unref?.())])) {
await boxes.finals({ budgetMs: 10_000 }).catch(() => []);
}
+ boxes.close();
const captureStopped = capture.stop({ archiveFinals });
bestPracticesReloads.stop();
let recoveryError: unknown;