A desktop coding agent with free access to frontier models.
TypeScript
fix: restart the sandbox helper when it stops
ogsamratb6919dd359be
diff --git a/apps/desktop/box/go/cmd/boxrun/main.go b/apps/desktop/box/go/cmd/boxrun/main.goindex c6a2ce73..f499204e 100644--- a/apps/desktop/box/go/cmd/boxrun/main.go+++ b/apps/desktop/box/go/cmd/boxrun/main.go@@ -626,9 +626,11 @@ func writeJSONFile(file string, v any) error {return os.Rename(tmp, file)}-// alive: the pid runs and is our launcher (not a reused pid).+// alive: the pid runs and is our launcher (not a reused pid: after a+// restart of the machine or of the WSL distro, pids start over, and the+// recorded one can be this very boxrun, which would then kill itself).func alive(s *state) bool {- if s == nil || s.Pid <= 0 {+ if s == nil || s.Pid <= 0 || s.Pid == os.Getpid() || s.Pid == os.Getppid() {return false}if syscall.Kill(s.Pid, 0) != nil {diff --git a/apps/desktop/box/go/cmd/boxsh/main.go b/apps/desktop/box/go/cmd/boxsh/main.goindex 5f2b46cc..736c4004 100644--- a/apps/desktop/box/go/cmd/boxsh/main.go+++ b/apps/desktop/box/go/cmd/boxsh/main.go@@ -53,6 +53,10 @@ type Record struct {Shell string `json:"shell"`PathPre string `json:"path_prefix"`EnvFile string `json:"env_file"`+ // Revive: the app that wrote the record restarts boxd when asked+ // (boxes/<key>.revive); Updated is when it last wrote the record (ms).+ Revive bool `json:"revive"`+ Updated int64 `json:"updated"`}func main() {@@ -90,12 +94,33 @@ func main() {req := proto.Request{Op: "exec", Token: rec.Token, Argv: argv, Cwd: mapPath(cwd, rec), Env: forwardEnv(os.Environ())}start := time.Now()code, timing, err := runRemote(rec, req)- if err != nil {- if rec.Runtime == "" && rec.Backend != "" {- // A backend with no `exec` of its own (bubblewrap, the WSL distro): boxd is the only way in.- fmt.Fprintf(os.Stderr, "omnirush sandbox: the sandbox is not reachable (%v); `omnirush sandbox status` says why\n", err)+ if err != nil && rec.Runtime == "" && rec.Backend != "" {+ // A backend with no `exec` of its own (bubblewrap, the WSL distro): boxd is the only+ // way in. A boxd that stopped (idle limit, crash, the distro restarted) is started+ // again by the app that owns the session, and the command runs then. Never on this+ // machine instead.+ if !isDialError(err) {+ fmt.Fprintf(os.Stderr, "omnirush sandbox: the connection to the sandbox failed (%v); %s\n", err, recoverHint)+ os.Exit(125)+ }+ again, why := revive(key, rec, err)+ if again == nil {+ if strings.Contains(why, "Restart OmniRush") {+ fmt.Fprintf(os.Stderr, "omnirush sandbox: %s\n", why)+ } else {+ fmt.Fprintf(os.Stderr, "omnirush sandbox: the sandbox stopped and was not restarted (%s); %s\n", why, recoverHint)+ }os.Exit(125)}+ rec = again+ req.Token = rec.Token+ code, timing, err = runRemote(rec, req)+ if err != nil {+ fmt.Fprintln(os.Stderr, retryFailed(err))+ os.Exit(125)+ }+ }+ if err != nil {// boxd is not reachable (restarting, crashed): run it with the runtime's exec instead.code = runtimeExec(rec, req)timing.Fallback = true@@ -106,6 +131,87 @@ func main() {os.Exit(code)}+// recoverHint: what the user can do when the sandbox cannot be reached.+const recoverHint = "restart OmniRush (installs are kept), `omnirush sandbox reset` for a fresh sandbox, or `omnirush sandbox off` to run this folder on this machine"++// isDialError: nothing reached boxd (no listener, refused, timed out): the+// command did not start, so running it again cannot run it twice.+func isDialError(err error) bool {+ var op *net.OpError+ return errors.As(err, &op) && op.Op == "dial"+}++// retryFailed: the line for a command that failed again after boxd was started again.+func retryFailed(err error) string {+ if isDialError(err) {+ return fmt.Sprintf("omnirush sandbox: the sandbox is not reachable after a restart (%v); %s", err, recoverHint)+ }+ return fmt.Sprintf("omnirush sandbox: the connection to the sandbox broke during the command (%v); %s", err, recoverHint)+}++func reviveFile(key string) string {+ return filepath.Join(boxHome(), "boxes", key+".revive")+}++// reviveWait: how long a command waits for the sandbox to be started again.+func reviveWait() time.Duration {+ if v, err := strconv.Atoi(os.Getenv("OMNIRUSH_BOX_REVIVE_WAIT_MS")); err == nil && v >= 0 {+ return time.Duration(v) * time.Millisecond+ }+ return 60 * time.Second+}++// revive asks the app that owns the session to start boxd again+// (boxes/<key>.revive) and waits for the record to say where it is now.+// Returns the new record, or nil and why not. A request nobody took within+// the wait (no app restarts this box) is not made again: later commands+// fail at once instead of each waiting.+func revive(key string, rec *Record, cause error) (*Record, string) {+ if !rec.Revive {+ return nil, fmt.Sprintf("%v", cause)+ }+ wait := reviveWait()+ file := reviveFile(key)+ if st, err := os.Stat(file); err == nil && time.Since(st.ModTime()) > wait {+ return nil, fmt.Sprintf("%v; no answer to the restart request", cause)+ }+ asked := time.Now()+ body, _ := json.Marshal(map[string]any{"at": asked.UnixMilli(), "port": rec.Port, "error": cause.Error()})+ if _, err := os.Stat(file); err != nil {+ tmp := fmt.Sprintf("%s.%d.tmp", file, os.Getpid())+ if os.WriteFile(tmp, body, 0o600) == nil {+ _ = os.Rename(tmp, file)+ }+ }+ fmt.Fprintln(os.Stderr, "omnirush sandbox: the sandbox stopped; starting it again...")+ deadline := asked.Add(wait)+ for time.Now().Before(deadline) {+ time.Sleep(150 * time.Millisecond)+ now, err := readRecord(key)+ if err != nil {+ continue+ }+ switch now.State {+ case "failed":+ msg := now.Error+ if msg == "" {+ msg = "it could not start"+ }+ return nil, msg+ case "off":+ return nil, "the sandbox was turned off for this session"+ case "ready":+ // Started again (a new port or a newer record), or boxd answered after all+ // (the app took the request and found it running).+ _, pending := os.Stat(file)+ if now.Port != rec.Port || now.Updated > rec.Updated || pending != nil {+ return now, ""+ }+ }+ }+ return nil, fmt.Sprintf("%v; it did not start again in %v", cause, wait.Round(time.Second))+}+func (r *Record) shell() string {if r.Shell != "" {return r.Shell@@ -196,6 +302,9 @@ func waitReady(key string) (*Record, error) {if msg == "" {msg = "it could not start"}+ if strings.Contains(msg, "Restart OmniRush") {+ return nil, errors.New(msg) // says what to do already+ }return nil, fmt.Errorf("the sandbox is not available (%s); run `omnirush sandbox status`", msg)}}diff --git a/apps/desktop/box/go/cmd/boxsh/main_test.go b/apps/desktop/box/go/cmd/boxsh/main_test.goindex c3e4d8d2..54a9222d 100644--- a/apps/desktop/box/go/cmd/boxsh/main_test.go+++ b/apps/desktop/box/go/cmd/boxsh/main_test.go@@ -1,10 +1,14 @@package mainimport (+ "encoding/json"+ "errors"+ "net""os""path/filepath""reflect""runtime"+ "strconv""strings""testing""time"@@ -105,3 +109,109 @@ func TestWaitReadyThroughDownload(t *testing.T) {t.Fatalf("a start that hangs still times out: %v", err)}}++func writeRec(t *testing.T, home, key string, rec map[string]any) {+ t.Helper()+ data, _ := json.Marshal(rec)+ if err := os.MkdirAll(filepath.Join(home, "boxes"), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(home, "boxes", key+".json"), data, 0o600); err != nil {+ t.Fatal(err)+ }+}++func TestReviveAsksAndWaitsForTheNewRecord(t *testing.T) {+ home := t.TempDir()+ t.Setenv("OMNIRUSH_BOX_HOME", home)+ t.Setenv("OMNIRUSH_BOX_REVIVE_WAIT_MS", "3000")+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "updated": 100, "revive": true, "backend": "bwrap", "token": "t1"})+ rec, _ := readRecord("k1")+ go func() {+ // The session sees the request, starts boxd again and writes where it is.+ for i := 0; i < 100; i++ {+ if _, err := os.Stat(reviveFile("k1")); err == nil {+ break+ }+ time.Sleep(10 * time.Millisecond)+ }+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 45001, "updated": 200, "revive": true, "backend": "bwrap", "token": "t2"})+ _ = os.Remove(reviveFile("k1"))+ }()+ again, why := revive("k1", rec, errors.New("dial tcp 127.0.0.1:44897: connect: connection refused"))+ if again == nil || again.Port != 45001 || again.Token != "t2" {+ t.Fatalf("got %+v (%s)", again, why)+ }+}++func TestReviveFailsWithTheSessionsReason(t *testing.T) {+ home := t.TempDir()+ t.Setenv("OMNIRUSH_BOX_HOME", home)+ t.Setenv("OMNIRUSH_BOX_REVIVE_WAIT_MS", "3000")+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "updated": 100, "revive": true, "backend": "bwrap"})+ rec, _ := readRecord("k1")+ go func() {+ time.Sleep(100 * time.Millisecond)+ writeRec(t, home, "k1", map[string]any{"state": "failed", "error": "the sandbox stopped and could not start again (x). Restart OmniRush (installs are kept), or `omnirush sandbox off`", "revive": true, "backend": "bwrap"})+ }()+ again, why := revive("k1", rec, errors.New("refused"))+ if again != nil || !strings.Contains(why, "omnirush sandbox off") {+ t.Fatalf("got %+v %q", again, why)+ }+}++func TestReviveOnlyWhenTheAppRestartsAndNotTwice(t *testing.T) {+ home := t.TempDir()+ t.Setenv("OMNIRUSH_BOX_HOME", home)+ t.Setenv("OMNIRUSH_BOX_REVIVE_WAIT_MS", "200")+ // An app that does not restart boxd (no "revive"): no request, no wait.+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "backend": "bwrap"})+ rec, _ := readRecord("k1")+ start := time.Now()+ if again, _ := revive("k1", rec, errors.New("refused")); again != nil || time.Since(start) > 100*time.Millisecond {+ t.Fatal("waited for an app that does not restart")+ }+ if _, err := os.Stat(reviveFile("k1")); err == nil {+ t.Fatal("asked an app that does not restart")+ }+ // Nobody answers: the first call waits once; a later one (request older than the wait) fails at once.+ writeRec(t, home, "k1", map[string]any{"state": "ready", "port": 44897, "revive": true, "backend": "bwrap"})+ rec, _ = readRecord("k1")+ if again, why := revive("k1", rec, errors.New("refused")); again != nil || !strings.Contains(why, "did not start again") {+ t.Fatalf("got %+v %q", again, why)+ }+ time.Sleep(50 * time.Millisecond)+ start = time.Now()+ if again, why := revive("k1", rec, errors.New("refused")); again != nil || !strings.Contains(why, "no answer") || time.Since(start) > 100*time.Millisecond {+ t.Fatalf("waited again: %q", why)+ }+}++func TestIsDialError(t *testing.T) {+ ln, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ port := ln.Addr().(*net.TCPAddr).Port+ ln.Close()+ _, err = net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), time.Second)+ if err == nil || !isDialError(err) {+ t.Fatalf("refused dial: %v", err)+ }+ if isDialError(errors.New("EOF")) {+ t.Fatal("a read error is not a dial error")+ }+}++func TestRetryFailedSaysWhatBroke(t *testing.T) {+ dial := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")}+ if got := retryFailed(dial); !strings.Contains(got, "not reachable after a restart") {+ t.Fatal(got)+ }+ for _, err := range []error{errors.New("EOF"), &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}} {+ got := retryFailed(err)+ if !strings.Contains(got, "broke during the command") || strings.Contains(got, "not reachable") {+ t.Fatal(got)+ }+ }+}diff --git a/apps/server/src/box/backends/types.ts b/apps/server/src/box/backends/types.tsindex 4471394e..8e76bef3 100644--- a/apps/server/src/box/backends/types.ts+++ b/apps/server/src/box/backends/types.ts@@ -37,6 +37,8 @@ export type OpenContext = {/** The download's output lines as they come (its progress). */onLine?: RunOptions["onLine"];spawnImpl?: SpawnLike;+ /** boxd's ping (tests); client.ping otherwise. */+ ping?: (endpoint: BoxEndpoint, options?: { timeoutMs?: number }) => Promise<Record<string, unknown> | null>;};export type Opened =diff --git a/apps/server/src/box/backends/wsl.ts b/apps/server/src/box/backends/wsl.tsindex 5181fd68..486e0c26 100644--- a/apps/server/src/box/backends/wsl.ts+++ b/apps/server/src/box/backends/wsl.ts@@ -41,6 +41,7 @@ import bundledPins from "../wsl.json" with { type: "json" };import { IDLE_MINUTES, newMeta, upArgs, type BoxMeta } from "./bwrap.js";import { boxshExec, num, parseOut, str } from "./shared.js";import type { BoxDriver, OpenContext, Opened, PresentImage, SpawnLike } from "./types.js";+import { ping } from "../client.js";type Env = NodeJS.ProcessEnv;type WslRuntime = Pick<BoxRuntime, "bin" | "distro" | "arch" | "installed">;@@ -324,7 +325,16 @@ async function open(ctx: OpenContext): Promise<Opened> {if (meta && meta.root !== root) return { ok: false, error: `box ${key} belongs to another folder` };const live = keepers.get(key);if (live && live.child.exitCode === null && meta) {- return { ok: true, created: null, endpoint: live.endpoint, alternates: live.alternates, uid: 0, gid: 0, containerId: `wsl_${key}_${meta.instance}`, imageId: meta.config || meta.image, mounts: meta.mounts ?? [] };+ // The keeper outlives boxd (its idle limit, a crash) and Windows' forward of the+ // distro's loopback (sleep): reused only when boxd answers, else started again.+ const asks = ctx.ping ?? ping;+ const answers = (await asks(live.endpoint, { timeoutMs: 1_500 })) || (live.alternates.length > 0 && (await asks(live.alternates[0]!, { timeoutMs: 1_500 })));+ if (answers) {+ return { ok: true, created: null, endpoint: live.endpoint, alternates: live.alternates, uid: 0, gid: 0, containerId: `wsl_${key}_${meta.instance}`, imageId: meta.config || meta.image, mounts: meta.mounts ?? [] };+ }+ log("info", "box keeper running but boxd not answering; starting it again", { key });+ try { live.child.kill(); } catch { /* gone */ }+ keepers.delete(key);}let present = await imagePresent(runtime, image, { run, env });if (!present && !meta) {diff --git a/apps/server/src/box/box-revive.test.ts b/apps/server/src/box/box-revive.test.tsnew file mode 100644index 00000000..f54d7ab2--- /dev/null+++ b/apps/server/src/box/box-revive.test.ts@@ -0,0 +1,162 @@+// A box whose boxd stopped while the app ran (its idle limit, a crash, the WSL+// distro restarted): the project's box run starts it again when a shell call+// asks (boxsh writes boxes/<key>.revive) or at a turn's checkpoint, and says+// once when it cannot. Never this computer instead. No container runtime here.+import { describe, expect, test } from "bun:test";+import { EventEmitter } from "node:events";+import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { PassThrough } from "node:stream";++import { reviveFile, type Box } from "./box.js";+import { wslDriver } from "./backends/wsl.js";+import type { OpenContext } from "./backends/types.js";+import { boxRun, reviveError, type ReadyPlan } from "./launch.js";++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));+const tmp = () => mkdtempSync(join(tmpdir(), "box-revive-"));+async function until(fn: () => boolean, ms = 3_000): Promise<boolean> {+ const end = Date.now() + ms;+ while (Date.now() < end) { if (fn()) return true; await sleep(20); }+ return false;+}++type FakeBox = { ready: boolean; error: string | null; endpoint: { host: string; port: number; token: string }; turns: number[]; failed: string | null; setTurn: (turn: number) => void; fail: (message: string) => FakeBox; checkpoint: () => Promise<unknown> };+function fakeBox(port: number, { ready = true, error = null }: { ready?: boolean; error?: string | null } = {}): FakeBox {+ const box: FakeBox = {+ ready, error, endpoint: { host: "127.0.0.1", port, token: "t" }, turns: [], failed: null,+ setTurn(turn) { box.turns.push(turn); },+ fail(message) { box.ready = false; box.failed = message; box.error = message; return box; },+ checkpoint: async () => ({ summary: { stats: { copied: 0, kept: 0 }, delta: { changed: [] }, bytes: 0 }, commit() {}, discard() {} }),+ };+ return box;+}++function fakePlan(first: FakeBox, next: FakeBox[]): ReadyPlan & { restarts: number } {+ const home = tmp();+ mkdirSync(join(home, "boxes"), { recursive: true });+ const plan = {+ box: { key: "k1", home, root: "/p", runtime: { name: "bwrap", backend: "bwrap", bin: "/x/boxrun" } },+ notice: null,+ shell: "/x/boxsh",+ restarts: 0,+ phase: "ready",+ start: async () => first as unknown as Box,+ restart: async () => { plan.restarts += 1; return next.shift() as unknown as Box; },+ };+ return plan as unknown as ReadyPlan & { restarts: number };+}++const pinger = (live: Set<number>) => async (endpoint: { port: number }) => (live.has(endpoint.port) ? { ok: true } : null);++describe("box revive", () => {+ test("a restart request from boxsh: a stopped boxd is started again, the request answered, the turn carried", async () => {+ const live = new Set([41001]);+ const plan = fakePlan(fakeBox(41001), [fakeBox(41002)]);+ const said: unknown[] = [];+ const run = boxRun(plan, { pollMs: 20, ping: pinger(live) as never, onRevive: (e) => said.push(e) });+ try {+ expect((await run.ready())?.endpoint?.port).toBe(41001);+ run.setTurn(3);+ live.clear();+ live.add(41002);+ const request = reviveFile(plan.box.home, "k1");+ writeFileSync(request, "{}");+ expect(await until(() => !existsSync(request))).toBe(true);+ expect(plan.restarts).toBe(1);+ expect(run.box?.endpoint?.port).toBe(41002);+ expect((run.box as unknown as FakeBox).turns).toEqual([3]);+ expect(said).toEqual([{ ok: true, why: "request" }]);+ expect(run.native).toBe(false);+ } finally {+ run.close();+ }+ });++ test("a box that cannot start again: failed with what to do, said once, never this computer", async () => {+ const down = () => fakeBox(0, { ready: false, error: "boxd did not answer in 30s" });+ const plan = fakePlan(fakeBox(41021), [down(), down()]);+ const said: Array<{ ok: boolean }> = [];+ const natives: string[] = [];+ const run = boxRun(plan, { pollMs: 20, ping: pinger(new Set()) as never, onRevive: (e) => said.push(e), onNative: (r) => natives.push(r) });+ try {+ await run.ready();+ const request = reviveFile(plan.box.home, "k1");+ writeFileSync(request, "{}");+ expect(await until(() => !existsSync(request))).toBe(true);+ expect((run.box as unknown as FakeBox).failed).toContain("could not start again (boxd did not answer in 30s)");+ expect((run.box as unknown as FakeBox).failed).toContain("Restart OmniRush");+ expect(await run.freeze("turn-2", { sessionId: "s1", turn: 2 })).toBeNull();+ expect(plan.restarts).toBe(2);+ expect(said.length).toBe(1);+ expect(said[0]!.ok).toBe(false);+ expect(natives).toEqual([]);+ expect(run.native).toBe(false);+ } finally {+ run.close();+ }+ });++ test("each turn starts with a box that answers: a stopped boxd is started before the checkpoint", async () => {+ const live = new Set([41031]);+ const plan = fakePlan(fakeBox(41031), [fakeBox(41032)]);+ const run = boxRun(plan, { pollMs: 10_000, ping: pinger(live) as never });+ try {+ expect(await run.freeze("turn-1", { sessionId: "s1", turn: 1 })).not.toBeNull();+ expect(plan.restarts).toBe(0);+ live.clear();+ expect(await run.freeze("turn-2", { sessionId: "s1", turn: 2 })).not.toBeNull();+ expect(plan.restarts).toBe(1);+ expect(run.box?.endpoint?.port).toBe(41032);+ } finally {+ run.close();+ }+ });++ test("reviveError names why and what to do", () => {+ const text = reviveError("dial tcp 127.0.0.1:44897: connect: connection refused");+ expect(text.startsWith("the sandbox stopped and could not start again (dial tcp 127.0.0.1:44897: connect: connection refused).")).toBe(true);+ expect(text).toContain("Restart OmniRush (installs are kept)");+ expect(reviveError("")).toContain("(it did not answer)");+ });++ test("WSL: a keeper still running whose boxd does not answer is replaced, not reused", async () => {+ const home = tmp();+ const bin = join(home, "bin");+ mkdirSync(join(bin, "linux-amd64"), { recursive: true });+ for (const name of ["boxrun", "boxd"]) writeFileSync(join(bin, "linux-amd64", name), "");+ const env = { OMNIRUSH_BOX_BIN_DIR: bin };+ const runtime = { name: "wsl", backend: "wsl", bin: "C:\\Windows\\System32\\wsl.exe", distro: "OmniRushBox", installed: true, arch: "amd64" };+ const run = async (_bin: string, args: string[]) => (args.includes("image")+ ? { code: 0, stdout: '{"present":true,"id":"img1","digest":"sha256:x","arch":"amd64","config":"cfg1"}\n', stderr: "" }+ : { code: 0, stdout: "", stderr: "" });+ let port = 40100;+ const spawned: Array<EventEmitter & { killed: boolean }> = [];+ const spawnImpl = (_bin: string, args: string[]) => {+ const child = Object.assign(new EventEmitter(), { stdout: new PassThrough(), stderr: new PassThrough(), stdin: new PassThrough(), exitCode: null, args, unref() {}, killed: false, kill() { child.killed = true; return true; } });+ const p = ++port;+ setTimeout(() => child.stdout.write(`{"ok":true,"reused":false,"pid":7,"port":${p}}\n`), 5);+ spawned.push(child);+ return child;+ };+ const live = new Set<number>();+ const ctx = {+ key: "wk1", name: "omnirush-box-wk1", root: "C:\\p", croot: "/c/p", runtime, run, home, runFolder: join(home, "run"), image: { ref: "r", pull: "r", stack: "base" },+ pull: false, shims: null, pkglog: null, env, platform: "win32", log: () => {}, onPulling: () => {}, spawnImpl, ping: pinger(live),+ } as unknown as OpenContext;+ const first = await wslDriver.open(ctx);+ if (!first.ok) throw new Error(first.error);+ live.add(first.endpoint.port);+ const again = await wslDriver.open(ctx);+ if (!again.ok) throw new Error(again.error);+ expect(again.endpoint.port).toBe(first.endpoint.port);+ expect(spawned.length).toBe(1);+ live.clear();+ const third = await wslDriver.open(ctx);+ if (!third.ok) throw new Error(third.error);+ expect(spawned.length).toBe(2);+ expect(spawned[0]!.killed).toBe(true);+ expect(third.endpoint.port).not.toBe(first.endpoint.port);+ });+});diff --git a/apps/server/src/box/box.test.ts b/apps/server/src/box/box.test.tsindex a63330eb..0b1a7786 100644--- a/apps/server/src/box/box.test.ts+++ b/apps/server/src/box/box.test.ts@@ -334,7 +334,7 @@ describe("box service", () => {key: "k", container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,ready: async () => null,label: () => null,- setTurn: () => {},+ setTurn: () => {}, heal: async () => null, close: () => {},freeze: async (label, { sessionId, turn = null, sessionTurn = null }) => {frozen.push(`${sessionId} ${label} ${sessionTurn}`);return { label, turn, copied: 0, carried: 0, save: async () => ({ label, changed: 0, bytes: 0 }), discard: async () => null };@@ -378,7 +378,7 @@ describe("box service", () => {key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,ready: async () => null,label: () => null,- setTurn: () => {},+ setTurn: () => {}, heal: async () => null, close: () => {},freeze: async (label, { sessionId, turn = null }) => {frozen.push(`${sessionId} ${label}`);return { label, turn, copied: 0, carried: 0, save: async () => ({ label, changed: 0, bytes: 0 }), discard: async () => null };@@ -420,7 +420,7 @@ describe("box service", () => {key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,ready: async () => null,label: () => null,- setTurn: () => {},+ setTurn: () => {}, heal: async () => null, close: () => {},freeze: async (label, { sessionId, turn = null }) => {frozen.push(`${sessionId} ${label}`);if (failNext) { failNext = false; throw new Error("checkpoint failed"); }@@ -456,7 +456,7 @@ describe("box service", () => {const workspace: BoxWorkspace = { id: "ws_o", path: root, workspaceType: "local" };const run: BoxRun = {key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,- ready: async () => null, label: () => null, setTurn: () => {},+ ready: async () => null, label: () => null, setTurn: () => {}, heal: async () => null, close: () => {},freeze: async (label, { turn = null }) => ({ label, turn, copied: 0, carried: 0, save: async () => ({ label, changed: 0, bytes: 0 }), discard: async () => null }),};rememberServerDefault(stateDir, { local_box: { default: "off" } });@@ -508,7 +508,7 @@ describe("box service", () => {const boxes = service({ stateDir, workspaces: [workspace], settings: new Map(), env: { OMNIRUSH_BOX_HOME: await temp("boxhome") } });const run: BoxRun = {key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box: null,- ready: async () => null, label: () => null, setTurn: () => {}, freeze: async () => null,+ ready: async () => null, label: () => null, setTurn: () => {}, heal: async () => null, close: () => {}, freeze: async () => null,};const projects = boxes["projects"];projects.set(boxKey(root), { run, notice: null, planning: null, nextPlanAt: Date.now() + 60_000, waitingImage: false });@@ -543,7 +543,7 @@ describe("box service", () => {box.setState("ready", null);const run: BoxRun = {key: boxKey(root), container: "c", runtimeBin: "docker", runtimeArgs: [], backend: "docker", exec: null, root, native: false, box,- ready: async () => box, label: () => null, setTurn: () => {}, freeze: async () => null,+ ready: async () => box, label: () => null, setTurn: () => {}, heal: async () => null, close: () => {}, freeze: async () => null,};boxes["projects"].set(boxKey(root), { run, notice: null, planning: null, nextPlanAt: 0, waitingImage: false });expect(await boxes.ensure(workspace)).toBe(run);diff --git a/apps/server/src/box/box.ts b/apps/server/src/box/box.tsindex dfa8e4f1..c20ec026 100644--- a/apps/server/src/box/box.ts+++ b/apps/server/src/box/box.ts@@ -198,7 +198,7 @@ type OnState = (state: BoxState, error: string | null) => void;* (the record boxsh reads says the same). Throws only for programming* errors; a box that cannot start is a Box with state "failed" and `error`.*/-export async function openBox({ root, runtime, env = process.env, platform = process.platform, home = boxHome(env), run = runCommand, log = () => {}, shims = null, pkglog = null, pull = true, image: asked = null, onState = () => {}, onLine, spawnImpl, readyMs = 15_000 }: { root: string; runtime: BoxRuntime; env?: Env; platform?: NodeJS.Platform; home?: string; run?: RunCommand; log?: BoxLog; shims?: string | null; pkglog?: BoxInstallLog | null; pull?: boolean; image?: BoxImage | null; onState?: OnState; onLine?: RunOptions["onLine"]; spawnImpl?: SpawnLike; readyMs?: number }): Promise<Box> {+export async function openBox({ root, runtime, env = process.env, platform = process.platform, home = boxHome(env), run = runCommand, log = () => {}, shims = null, pkglog = null, pull = true, image: asked = null, onState = () => {}, onLine, spawnImpl, readyMs = 15_000, revive = false }: { root: string; runtime: BoxRuntime; env?: Env; platform?: NodeJS.Platform; home?: string; run?: RunCommand; log?: BoxLog; shims?: string | null; pkglog?: BoxInstallLog | null; pull?: boolean; image?: BoxImage | null; onState?: OnState; onLine?: RunOptions["onLine"]; spawnImpl?: SpawnLike; readyMs?: number; revive?: boolean }): Promise<Box> {const projectRoot = path.resolve(root);const key = boxKey(projectRoot, { platform });const name = containerName(key, env);@@ -206,6 +206,8 @@ export async function openBox({ root, runtime, env = process.env, platform = proconst runFolder = runDir(home, key);mkdirSync(path.join(runFolder, "state"), { recursive: true });const box = new Box({ key, name, root: projectRoot, croot, runtime, home, runFolder, run, log, platform, env });+ // The app restarts boxd when a shell call finds it stopped (boxsh asks through boxes/<key>.revive).+ box.revive = revive;const driver = driverFor(runtime);const lock = await acquireLock(home, key);try {@@ -354,6 +356,8 @@ export class Box {readonly container: string;state: BoxState = "starting";error: string | null = null;+ /** The record tells boxsh the app restarts boxd when asked. */+ revive = false;endpoint: BoxEndpoint | null = null;identity: Record<string, unknown> | null = null;pathPrefix = "";@@ -395,7 +399,7 @@ export class Box {runtime: fallback ? this.runtime.bin ?? "" : "", runtime_args: fallback ? this.runtime.prefix ?? [] : [],host: this.endpoint?.host ?? "127.0.0.1", port: this.endpoint?.port ?? 0, token: this.endpoint?.token ?? "",state: this.state, error: this.error ?? "", uid: this.uid ?? 0, gid: this.gid ?? 0, home: HOME_IN_BOX, shell: "/bin/bash",- path_prefix: this.pathPrefix, env_file: `${RUN_DIR}/env`, updated: Date.now(), pid: process.pid,+ path_prefix: this.pathPrefix, env_file: `${RUN_DIR}/env`, updated: Date.now(), pid: process.pid, revive: this.revive,};writeJson(recordFile(this.home, this.key), record);}@@ -683,6 +687,11 @@ export class Box {type Removed = Array<{ kind: string; name: string; reason: string }>;/** Boxes on this machine: { key, name, root, running, image, created }[] */+/** Where boxsh asks the app to start a stopped boxd again (launch.ts boxRun). */+export function reviveFile(home: string, key: string): string {+ return path.join(home, "boxes", `${key}.revive`);+}+export async function listBoxes(runtime: BoxRuntime, { run = runCommand, env = process.env, home = boxHome(env) }: { run?: RunCommand; env?: Env; home?: string } = {}): Promise<ListedBox[]> {return driverFor(runtime).list(runtime, { run, env, home });}diff --git a/apps/server/src/box/common.ts b/apps/server/src/box/common.tsindex 77ca9427..92defd44 100644--- a/apps/server/src/box/common.ts+++ b/apps/server/src/box/common.ts@@ -94,6 +94,8 @@ export type BoxRecord = {v: 1; key: string; root: string; croot: string; container: string; backend: string; runtime: string; runtime_args: string[];host: string; port: number; token: string; state: string; error: string; uid: number; gid: number;home: string; shell: string; path_prefix: string; env_file: string; updated: number; pid: number;+ /** The app restarts boxd when boxsh asks (boxes/<key>.revive). */+ revive?: boolean;};export function readRecord(home: string, key: string): BoxRecord | null {diff --git a/apps/server/src/box/launch.ts b/apps/server/src/box/launch.tsindex 7f8b3db1..7032fcbc 100644--- a/apps/server/src/box/launch.ts+++ b/apps/server/src/box/launch.ts@@ -18,10 +18,11 @@// background, and a shell call waits for it (boxsh).import { spawn, type ChildProcess } from "node:child_process";-import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs";+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";import path from "node:path";-import { boxKey, containerName, gcBoxes, installShell, openBox, recordFile, savedTurns, type Box, type BoxInstallLog, type BoxLabel, type BoxLog, type BoxRuntime, type CheckpointSummary } from "./box.js";+import * as client from "./client.js";+import { boxKey, containerName, gcBoxes, installShell, openBox, recordFile, reviveFile, savedTurns, type Box, type BoxInstallLog, type BoxLabel, type BoxLog, type BoxRuntime, type CheckpointSummary } from "./box.js";import { driverFor, normalizeBackend } from "./backends/index.js";import type { BoxExec } from "./backends/types.js";import { detectStack, imageFor, loadPins, type BoxImage } from "./images.js";@@ -235,6 +236,11 @@ export type ReadyPlan = {* POSIX wrappers), mounted into a new box and put on its PATH.*/start: (options?: { pkglog?: BoxInstallLog | null }) => Promise<Box>;+ /**+ * Open the box again (boxd stopped: its idle limit, a crash, the WSL distro+ * restarted). The backend starts boxd again when it does not answer; installs are kept.+ */+ restart?: () => Promise<Box>;/** What the box start is doing ("starting", "pulling", "ready", "failed"; null before it starts). */readonly phase: string | null;};@@ -274,12 +280,20 @@ export async function planBox({ env = process.env, home, root, log = () => {}, rif (env.OMNIRUSH_BOX_GC !== "off") setTimeout(() => gcInBackground(runtime, home, { log }), 30_000).unref?.();let opening: Promise<Box> | null = null;let phase: string | null = null;+ let startedWith: BoxInstallLog | null = null;+ const start = ({ pkglog = null }: { pkglog?: BoxInstallLog | null } = {}): Promise<Box> => {+ if (!opening) startedWith = pkglog;+ opening ??= openBox({ root, runtime, env, home, log, pull: false, shims, pkglog, revive: true, onState: (state) => { phase = state; } });+ return opening;+ };return {box: { key, home, root, runtime },notice: null,shell,- start: ({ pkglog = null } = {}) => {- opening ??= openBox({ root, runtime, env, home, log, pull: false, shims, pkglog, onState: (state) => { phase = state; } });+ start,+ restart: () => {+ if (!opening) return start();+ opening = openBox({ root, runtime, env, home, log, pull: false, shims, pkglog: startedWith, revive: true });return opening;},get phase() { return phase; },@@ -320,6 +334,15 @@ export function engineConfig(bits: { shell: string; plugin: string } | null, con};}+/** How often a project's box looks for a restart request from boxsh. */+export const REVIVE_POLL_MS = 400;++/** The record's error when boxd stopped and could not start again: why, and what the user can do. */+export function reviveError(error: string | null | undefined): string {+ const why = String(error || "it did not answer").replace(/\s+/g, " ").slice(0, 240);+ return `the sandbox stopped and could not start again (${why}). Restart OmniRush (installs are kept), reset this folder's sandbox for a fresh one, or turn the sandbox off for this folder to run it on this computer`;+}+/** Turn the record off (the project went on without the box). */export function recordOff(home: string, key: string, reason: string): void {try {@@ -357,6 +380,10 @@ export type BoxRun = {/** environment.sandbox for a chat's uploads; null while it is not (or no longer) the chat's. */label: (sessionId: string) => BoxLabel | null;setTurn: (turn: number) => void;+ /** Start boxd again when it does not answer (resolves the Box now in use). */+ heal: (why: string) => Promise<Box | null>;+ /** Stop watching for restart requests (the project's box is let go). */+ close: () => void;/** `sessionTurn`: the chat's own number of the prompt the checkpoint comes before (written as session_turn). */freeze: (label: string, options: { sessionId: string; turn?: number | null; sessionTurn?: number | null; chain?: string | null }) => Promise<Frozen | null>;};@@ -369,11 +396,14 @@ export type BoxRun = {* "off"; the shell runs on this machine, its chats are not labelled); later* failures are logged and that turn has no checkpoint.*/-export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000, onNative = () => {}, tmpRoot, pkglog = null }: { log?: BoxLog; readyWaitMs?: number; onNative?: (reason: string) => void; tmpRoot?: string; pkglog?: BoxInstallLog | null } = {}): BoxRun {+export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000, onNative = () => {}, onRevive = () => {}, tmpRoot, pkglog = null, pollMs = REVIVE_POLL_MS, ping = client.ping }: { log?: BoxLog; readyWaitMs?: number; onNative?: (reason: string) => void; onRevive?: (event: { ok: boolean; why: string; error?: string }) => void; tmpRoot?: string; pkglog?: BoxInstallLog | null; pollMs?: number; ping?: typeof client.ping } = {}): BoxRun {const { key, home, runtime, root } = plan.box;let box: Box | null = null;let native = false;const checkpoints = new Map<string, number>();+ const requestFile = reviveFile(home, key);+ // A restart request nobody answered before (the app quit): this start answers it.+ try { rmSync(requestFile, { force: true }); } catch { /* not there */ }const opening = plan.start({ pkglog }).then((opened) => { box = opened; return opened; });const goNative = (reason: string) => {if (native) return;@@ -397,10 +427,64 @@ export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000,};tick();});+ // boxd stopped while the app ran (its idle limit, a crash, OOM, the WSL distro+ // restarted, sleep): start it again, once at a time. Asked by a shell call that+ // could not reach it (boxsh writes boxes/<key>.revive) and checked at each+ // turn's checkpoint. A box that cannot start again stays "failed" with the+ // reason and what to do (every shell call says it); the app says it once.+ // Never this computer instead.+ let healing: Promise<Box | null> | null = null;+ let lostSaid = false;+ let lastTurn: number | null = null;+ const heal = (why: string): Promise<Box | null> => {+ if (healing) return healing;+ const since = Date.now();+ healing = (async () => {+ try {+ const first = await opening;+ if (native || !first.ready || !plan.restart) return box;+ const current = box ?? first;+ if (current.ready && current.endpoint && (await ping(current.endpoint, { timeoutMs: 1_500 }))) return current;+ log("warn", "box not answering; starting it again", { why, key });+ const again = await plan.restart();+ box = again;+ if (lastTurn !== null) again.setTurn(lastTurn);+ if (again.ready) {+ log("info", "box started again", { why, key });+ lostSaid = false;+ onRevive({ ok: true, why });+ } else {+ log("warn", "box could not start again", { why, key, error: again.error });+ const error = reviveError(again.error);+ again.fail(error);+ if (!lostSaid) {+ lostSaid = true;+ onRevive({ ok: false, why, error });+ }+ }+ return again;+ } catch (error) {+ log("warn", "box restart failed", { why, error: error instanceof Error ? error.message : String(error) });+ return box;+ } finally {+ healing = null;+ // Answered: boxsh reads the record now. A request made after this check started waits for the next one.+ try { if (statSync(requestFile).mtimeMs <= since) rmSync(requestFile, { force: true }); } catch { /* none */ }+ }+ })();+ return healing;+ };+ const watcher = setInterval(() => {+ if (!native && !healing && existsSync(requestFile)) heal("request").catch(() => {});+ }, pollMs);+ watcher.unref?.();const ready = async (): Promise<Box | null> => {if (native) return null;const opened = await Promise.race([opening, limit()]);- if (opened?.ready) return opened;+ if (opened?.ready) {+ if (healing) await healing;+ return box?.ready ? box : null;+ }if ([...checkpoints.values()].every((count) => count === 0)) goNative(opened?.error || "the sandbox did not start in time");return null;};@@ -417,15 +501,21 @@ export function boxRun(plan: ReadyPlan, { log = () => {}, readyWaitMs = 30_000,get box() { return box; },ready,label: (sessionId) => (box?.ready && !native && (checkpoints.get(sessionId) ?? 0) > 0 ? box.label() : null),- setTurn: (turn) => box?.setTurn(turn),+ setTurn: (turn) => { lastTurn = turn; box?.setTurn(turn); },+ heal,+ close: () => clearInterval(watcher),/*** The old sandbox's freeze(label, { turn, chain }) -> { save(dir), discard() } | null:* the checkpoint is taken now and placed by save(); `final` (the state a* chat ends with) is a leaf, so a resumed chat chains on from its last turn.*/freeze: async (label, { sessionId, turn = null, sessionTurn = null, chain = null }) => {- const opened = await ready();- if (!opened) return null;+ await ready();+ if (native || !box) return null;+ // Each turn starts with a box that answers (started again when it stopped since the last turn,+ // or tried again when it could not start at the last try).+ const opened = await heal("turn");+ if (!opened?.ready) return null;const n = Number(turn ?? /^turn-(\d+)$/.exec(label)?.[1] ?? 0) || null;const name = n ? `turn-${n}` : label;const dir = chain ?? path.join(pending, `${process.pid}-${Date.now()}`);diff --git a/apps/server/src/box/service.ts b/apps/server/src/box/service.tsindex e3906840..09100c7f 100644--- a/apps/server/src/box/service.ts+++ b/apps/server/src/box/service.ts@@ -233,7 +233,7 @@ export class BoxService {const chosen = now.ok ? now.backend ?? now.name : null;for (const project of this.projects.values()) {if (project.run && project.run.backend === chosen) continue;- project.run = null;+ this.dropRun(project);project.notice = null;project.nextPlanAt = 0;}@@ -253,7 +253,7 @@ export class BoxService {await Promise.allSettled([...this.projects.values()].map((project) => project.planning));for (const project of this.projects.values()) {if (!project.run || !allow || allow.includes(project.run.backend)) continue;- project.run = null;+ this.dropRun(project);project.notice = null;project.nextPlanAt = 0;}@@ -341,7 +341,7 @@ export class BoxService {const { on, hostBuilds: gate } = await this.mode(workspace);if (!on) {if (gate?.native && project.notice !== gate.notice) this.log("info", "box: the server's default not applied, the folder has this computer's builds", { key, things: gate.things });- project.run = null;+ this.dropRun(project);project.notice = gate?.native ? gate.notice : null;await this.writeProjects();return null;@@ -352,7 +352,7 @@ export class BoxService {// one, where every command fails. Planned again, the box restarts on the new folder (openBox).if (!box?.ready || !box.endpoint || !(await this.folderReplaced({ endpoint: box.endpoint, croot: box.croot }).catch(() => false))) return project.run;this.log("info", "box folder replaced while it ran: starting it again", { key });- project.run = null;+ this.dropRun(project);project.nextPlanAt = 0;}if (Date.now() < project.nextPlanAt) return null;@@ -363,17 +363,27 @@ export class BoxService {: plan.runtime && !plan.runtime.ok ? { reason: "no_runtime", runtime: runtimeWhy(plan.runtime) }: { reason: "helpers_missing" };if (!plan.box) {- project.run = null;+ this.dropRun(project);project.notice = plan.notice;project.nextPlanAt = Date.now() + REPLAN_MS;await this.writeProjects();return null;}project.notice = null;+ project.run?.close();project.run = boxRun(plan, {log: this.log,pkglog: this.installLog ? { root: this.installLog.root, bin: this.installLog.bin, py: this.installLog.py } : null,tmpRoot: join(this.home, "pending"),+ // boxd stopped while the app ran: started again (installs kept), or once why not and what to do.+ onRevive: (event) => {+ if (event.ok) {+ this.log("info", "box started again after it stopped", { key, why: event.why });+ if (project.notice?.startsWith("Sandbox stopped:")) project.notice = null;+ } else {+ project.notice = `Sandbox stopped: ${event.error}.`;+ }+ },onNative: (reason) => {project.notice = `Sandbox off for now: ${reason}.`;project.nextPlanAt = Date.now() + REPLAN_MS;@@ -524,7 +534,7 @@ export class BoxService {*/async retry(workspace: BoxWorkspace): Promise<BoxStatus> {const project = this.projects.get(boxKey(boxRoot(workspace)));- if (project && (project.run?.native || project.run?.box?.state === "failed")) project.run = null;+ if (project && (project.run?.native || project.run?.box?.state === "failed")) this.dropRun(project);if (project) {project.notice = null;project.nextPlanAt = 0;@@ -544,7 +554,7 @@ export class BoxService {const runtime = await this.detect();const project = this.projects.get(key);if (project) {- project.run = null;+ this.dropRun(project);project.nextPlanAt = 0;}await this.writeProjects();@@ -639,6 +649,17 @@ export class BoxService {}/** At quit: every chat with checkpoints gets its `final` leaf (bounded); the sessions that got one. */+ /** Let a project's box run go (no more restart requests answered for it). */+ private dropRun(project: Project): void {+ project.run?.close();+ project.run = null;+ }++ /** The app quits: stop watching every project's box for restart requests. */+ close(): void {+ for (const project of this.projects.values()) project.run?.close();+ }+async finals({ budgetMs = 20_000 }: { budgetMs?: number } = {}): Promise<string[]> {const done: string[] = [];const work = (async () => {diff --git a/apps/server/src/server.ts b/apps/server/src/server.tsindex de1ce92c..1f78ab7f 100644--- a/apps/server/src/server.ts+++ b/apps/server/src/server.ts@@ -1673,6 +1673,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {if (boxes.hasChats() && await Promise.race([archiveFinals.catch(() => false), new Promise<false>((resolvePromise) => setTimeout(() => resolvePromise(false), 1_000).unref?.())])) {await boxes.finals({ budgetMs: 10_000 }).catch(() => []);}+ boxes.close();const captureStopped = capture.stop({ archiveFinals });bestPracticesReloads.stop();let recoveryError: unknown;