A desktop coding agent with free access to frontier models.
TypeScript
fix: keep left-out box paths in the chain
ogsamrata67556876d1b
diff --git a/apps/desktop/box/go/proto/proto.go b/apps/desktop/box/go/proto/proto.goindex 64f3c860..545d12ba 100644--- a/apps/desktop/box/go/proto/proto.go+++ b/apps/desktop/box/go/proto/proto.go@@ -37,7 +37,7 @@ const MaxFrame = 1 << 20// MaxRequestLine bounds the first line (a snapshot's tar or stat lists every// path). Older boxd builds took 4 MiB: clients send long path lists in// batches well under that, whatever boxd a box runs.-const MaxRequestLine = 64 << 20+const MaxRequestLine = 16 << 20// Request is the first line of a connection.type Request struct {diff --git a/apps/server/src/box/box.ts b/apps/server/src/box/box.tsindex eed5e04a..d0587194 100644--- a/apps/server/src/box/box.ts+++ b/apps/server/src/box/box.ts@@ -51,6 +51,11 @@ export function boxSnapshotsDir(stateDir: string): string {// --- snapshot rules -------------------------------------------------------------+/** The chain's signature of a path a checkpoint left out: never equal to a real one. */+export const UNKNOWN_SIGNATURE = "?";+/** How many paths of a failed tar a checkpoint lists in `skipped` (the rest: `skipped_omitted`). */+export const SKIPPED_LISTED = 1000;+const H = HOME_IN_BOX;/** Never part of a turn's environment: runtime state, logs, download caches the ledger replays. */export const NOISE_PREFIXES = [@@ -314,6 +319,8 @@ export type CheckpointSummary = {delta: { parent: string | null; changed: string[]; removed: string[] };checkpoint: Record<string, unknown>;skipped: Array<{ path: string; size: number; reason: string }>;+ /** Paths left out but not listed in `skipped`, counted by reason. */+ skipped_omitted?: Record<string, number>;deps?: Record<string, unknown>;stats: { source: string; diff_entries: number; kept: number; copied: number; removed: number; diff_ms: number; stat_ms: number; tar_ms: number; total_ms: number; warning?: string; tar_error?: string };};@@ -619,14 +626,23 @@ export class Box {mkdirSync(stage, { recursive: true });let tarInfo: client.TarResult | null = null;let tarError: string | null = null;- // Paths the tar does not carry after all.+ // Paths the tar does not carry after all. A failed tar lists its first+ // SKIPPED_LISTED paths (a failed npm install would make the summary huge), the+ // rest as a count; every one still gets the "unknown" signature below.+ const leftOut = new Set<string>();+ const omitted: Record<string, number> = {};const leaveOut = (items: client.SkippedPath[]) => {+ let listed = skipped.filter((item) => item.reason === "tar_failed").length;for (const { path: p, reason } of items) {const entry = sized.get(p);- skipped.push({ path: p, size: entry?.type === "file" ? entry.size ?? 0 : 0, reason });+ const size = entry?.type === "file" ? entry.size ?? 0 : 0;+ leftOut.add(p);+ if (isDep(p)) depBytes -= size;+ if (reason === "tar_failed" && listed >= SKIPPED_LISTED) { omitted[reason] = (omitted[reason] ?? 0) + 1; continue; }+ if (reason === "tar_failed") listed += 1;+ skipped.push({ path: p, size, reason });}- const out = new Set(items.map((item) => item.path));- changed = changed.filter((p) => !out.has(p));+ changed = changed.filter((p) => !leftOut.has(p));};if (changed.length) {try {@@ -645,11 +661,13 @@ export class Box {rmSync(path.join(stage, `${label}.tar`), { force: true });}}- const skippedSet = new Set(skipped.map((item) => item.path));+ for (const item of skipped) leftOut.add(item.path);const tTar = Date.now();- // What a later checkpoint compares with: a path left out stays "unknown", so it is tried again.+ // What a later checkpoint compares with. A path left out keeps an "unknown"+ // signature (not none): the next checkpoint looks at it again, so it is copied+ // then if it is still there, or listed as removed if it is gone.const signatures: Record<string, string> = {};- for (const [p, sig] of Object.entries(now)) if (!skippedSet.has(p)) signatures[p] = sig;+ for (const [p, sig] of Object.entries(now)) signatures[p] = leftOut.has(p) ? UNKNOWN_SIGNATURE : sig;const image = this.image();const parent = full ? null : chain.last;const id = `cp_${createHash("sha256").update(canonical({ schema: RECORD_SCHEMA, box: this.key, container: this.containerId, label, turn, parent, image: image.digest || image.id, changed, removed, sha256: tarInfo?.sha256 ?? null })).digest("hex")}`;@@ -668,6 +686,7 @@ export class Box {delta: { parent, changed, removed },checkpoint: { schema: RECORD_SCHEMA, id, parent, turn: turn ?? null, image: { ref: image.ref ?? null, digest: image.digest ?? null }, status: tarError ? "incomplete" : "complete", capturedAt: new Date().toISOString() },skipped,+ ...(Object.keys(omitted).length ? { skipped_omitted: omitted } : {}),...(depsInfo ? { deps: { ...depsInfo, bytes: depBytes, cut: depsCut } } : {}),stats: { source, diff_entries: entries.length, kept: Object.keys(now).length, copied: changed.length, removed: removed.length, diff_ms: tDiff - t0, stat_ms: tStat - tDiff, tar_ms: tTar - tStat, total_ms: Date.now() - t0, ...(tarInfo?.warning ? { warning: tarInfo.warning } : {}), ...(tarError ? { tar_error: tarError } : {}) },};diff --git a/apps/server/src/box/client-tar-batch.test.ts b/apps/server/src/box/client-tar-batch.test.tsindex 609fad7e..44a1a645 100644--- a/apps/server/src/box/client-tar-batch.test.ts+++ b/apps/server/src/box/client-tar-batch.test.ts@@ -94,10 +94,10 @@ function members(buf: Buffer): { members: Array<{ name: string; data: Buffer }>;// --- a fake boxd ----------------------------------------------------------------------------type Change = { kind: "A"; path: string; type: string; size: number; mode: number; mtime_ns: number; ctime_ns: number; ino: number; uid: number; gid: number };+type Change = { kind: "A" | "C" | "D"; path: string; type: string; size: number; mode: number; mtime_ns: number; ctime_ns: number; ino: number; uid: number; gid: number };/** boxd's request-line limit and its tar/stat/walk/changes ops. `fail(paths)` may answer a tar request with an error instead. */-async function fakeBoxd({ padding = 0, fail = () => null, changes = [], limit = REQUEST_LINE_MAX }: { padding?: number; fail?: (paths: string[]) => string | null; changes?: Change[]; limit?: number } = {}): Promise<{ box: BoxEndpoint; seen: { lines: number[]; ops: Array<{ op: string; count: number }> } }> {+async function fakeBoxd({ padding = 0, fail = () => null, changes = [], gone = () => false, limit = REQUEST_LINE_MAX }: { padding?: number; fail?: (paths: string[]) => string | null; changes?: Change[] | (() => Change[]); gone?: (path: string) => boolean; limit?: number } = {}): Promise<{ box: BoxEndpoint; seen: { lines: number[]; ops: Array<{ op: string; count: number }> } }> {const seen = { lines: [] as number[], ops: [] as Array<{ op: string; count: number }> };const server = net.createServer((socket) => {socket.on("error", () => undefined);@@ -130,13 +130,13 @@ async function fakeBoxd({ padding = 0, fail = () => null, changes = [], limit =for (let i = 0; i < tar.length; i += 7777) out.push(frame(0x64, tar.subarray(i, i + 7777)));socket.end(Buffer.concat([...out, frame(0x65, Buffer.from("busy: a command was running during the snapshot\n")), exit()]));} else if (req.op === "stat") {- socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(paths.map((p) => ({ path: p, type: "file", size: 1, uid: 0, gid: 0 }))))), exit()]));+ socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(paths.map((p) => (gone(p) ? { path: p, type: "missing", uid: 0, gid: 0 } : { path: p, type: "file", size: 1, uid: 0, gid: 0 }))))), exit()]));} else if (req.op === "walk") {const entries = paths.map((p) => ({ path: `${p}/x`, type: "file", size: 1, uid: 0, gid: 0 }));const wait = req.wait ?? 400_000;socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify({ entries: entries.slice(0, wait), truncated: entries.length > wait }))), exit()]));} else if (req.op === "changes") {- socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(changes))), exit()]));+ socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(typeof changes === "function" ? changes() : changes))), exit()]));} else {socket.end(exit(0));}@@ -294,10 +294,11 @@ describe("box path lists in requests", () => {// --- the checkpoint -------------------------------------------------------------------------function fakeBox(endpoint: BoxEndpoint) {+function fakeBox(endpoint: BoxEndpoint, { deps = false }: { deps?: boolean } = {}) {const work = tmp();const root = path.join(work, "proj");mkdirSync(root);+ if (deps) mkdirSync(path.join(root, "node_modules"));const logs: Array<{ level: string; message: string }> = [];const box = new Box({key: "k1", name: "omnirush-box-k1", root, croot: root, runtime: { name: "docker" } as BoxRuntime, home: path.join(work, "home"), runFolder: path.join(work, "run"),@@ -334,7 +335,7 @@ describe("box checkpoint with many changed files", () => {expect(s.checkpoint.status).toBe("incomplete");expect(s.tar).toBeNull();expect(s.delta.changed).toEqual([]);- expect(s.skipped.filter((item) => item.reason === "tar_failed").length).toBe(paths.length);+ expect(s.skipped.filter((item) => item.reason === "tar_failed").length + (s.skipped_omitted?.tar_failed ?? 0)).toBe(paths.length);expect(s.stats.tar_error).toMatch(/request line too long/);expect(logs.some((line) => line.level === "warn" && line.message === "box checkpoint tar failed")).toBe(true);first.commit();@@ -344,4 +345,56 @@ describe("box checkpoint with many changed files", () => {expect(second.summary.delta.parent).toBe(s.checkpoint.id as string);expect(second.summary.delta.changed.length).toBe(paths.length);});++ test("a path whose tar failed is looked at again: deleted later, it is listed as removed", async () => {+ // turn-2 carries x; turn-3 changes x and its tar fails; turn-4 finds x deleted.+ const x = "/usr/local/lib/node_modules/pkg-0/lib/x.js";+ let state: { changes: Change[]; gone: boolean } = { changes: [{ ...changeOf(x), mtime_ns: 1 }], gone: false };+ let failing = false;+ const { box: endpoint } = await fakeBoxd({ changes: () => state.changes, gone: (p) => state.gone && p === x, fail: () => (failing ? "request line too long" : null) });+ const { box, snaps } = fakeBox(endpoint);+ const t2 = await box.checkpoint(snaps, 2);+ expect(t2.summary.delta.changed).toEqual([x]);+ t2.commit();+ state = { changes: [{ ...changeOf(x), kind: "C", mtime_ns: 2 }], gone: false };+ failing = true;+ const t3 = await box.checkpoint(snaps, 3);+ expect(t3.summary.checkpoint.status).toBe("incomplete");+ expect(t3.summary.skipped.map((item) => item.reason)).toEqual(["tar_failed"]);+ t3.commit();+ expect(JSON.parse(readFileSync(path.join(snaps, ".chain.json"), "utf8")).signatures[x]).toBe("?");+ state = { changes: [{ ...changeOf(x), kind: "D", type: "missing" }], gone: true };+ failing = false;+ const t4 = await box.checkpoint(snaps, 4);+ expect(t4.summary.checkpoint.status).toBe("complete");+ expect(t4.summary.delta.removed).toEqual([x]);+ expect(t4.summary.delta.parent).toBe(t3.summary.checkpoint.id as string);+ });++ test("a path whose tar failed and is still there is copied by the next checkpoint", async () => {+ const x = "/usr/local/lib/node_modules/pkg-0/lib/x.js";+ let failing = true;+ const { box: endpoint } = await fakeBoxd({ changes: [changeOf(x)], fail: () => (failing ? "boom" : null) });+ const { box, snaps } = fakeBox(endpoint);+ (await box.checkpoint(snaps, 2)).commit();+ failing = false;+ const t3 = await box.checkpoint(snaps, 3);+ expect(t3.summary.delta.changed).toEqual([x]);+ expect(t3.summary.checkpoint.status).toBe("complete");+ });++ test("a failed tar lists its first 1000 paths, counts the rest, and leaves their bytes out of deps", async () => {+ const { box: endpoint } = await fakeBoxd({ changes: manyPaths(3000, "/usr/local/lib/node_modules").map(changeOf), fail: () => "boom" });+ const { box, snaps } = fakeBox(endpoint, { deps: true });+ const taken = await box.checkpoint(snaps, 2);+ const s = taken.summary;+ // 3003 paths and the project's node_modules/x (the fake walk's one entry).+ expect(s.skipped.filter((item) => item.reason === "tar_failed").length).toBe(1000);+ expect(s.skipped_omitted?.tar_failed).toBe(3004 - 1000);+ expect(s.deps?.entries).toBe(1);+ expect(s.deps?.bytes).toBe(0);+ taken.commit();+ const signatures: Record<string, string> = JSON.parse(readFileSync(path.join(snaps, ".chain.json"), "utf8")).signatures;+ expect(Object.values(signatures).filter((sig) => sig === "?").length).toBe(3004);+ });});diff --git a/apps/server/src/box/client.ts b/apps/server/src/box/client.tsindex 77763d53..8f606d2e 100644--- a/apps/server/src/box/client.ts+++ b/apps/server/src/box/client.ts@@ -234,7 +234,7 @@ export async function tarTo(box: BoxEndpoint, paths: string[], file: string, { t// --- path lists in requests -------------------------------------------------------/**- * boxd reads a request line of at most 4 MiB (box/go/proto ReadRequest; 64+ * boxd reads a request line of at most 4 MiB (box/go/proto ReadRequest; 16* MiB from this version on, but a box may run an older boxd): a list of* paths goes in requests whose line stays well under it.*/