Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: keep left-out box paths in the chain

ogsamrata67556876d1b
diff --git a/apps/desktop/box/go/proto/proto.go b/apps/desktop/box/go/proto/proto.go
index 64f3c860..545d12ba 100644
--- a/apps/desktop/box/go/proto/proto.go
+++ b/apps/desktop/box/go/proto/proto.go
@@ -37,7 +37,7 @@ const MaxFrame = 1 << 20
// MaxRequestLine bounds the first line (a snapshot's tar or stat lists every
// path). Older boxd builds took 4 MiB: clients send long path lists in
// batches well under that, whatever boxd a box runs.
-const MaxRequestLine = 64 << 20
+const MaxRequestLine = 16 << 20
// Request is the first line of a connection.
type Request struct {
diff --git a/apps/server/src/box/box.ts b/apps/server/src/box/box.ts
index eed5e04a..d0587194 100644
--- a/apps/server/src/box/box.ts
+++ b/apps/server/src/box/box.ts
@@ -51,6 +51,11 @@ export function boxSnapshotsDir(stateDir: string): string {
// --- snapshot rules -------------------------------------------------------------
+/** The chain's signature of a path a checkpoint left out: never equal to a real one. */
+export const UNKNOWN_SIGNATURE = "?";
+/** How many paths of a failed tar a checkpoint lists in `skipped` (the rest: `skipped_omitted`). */
+export const SKIPPED_LISTED = 1000;
+
const H = HOME_IN_BOX;
/** Never part of a turn's environment: runtime state, logs, download caches the ledger replays. */
export const NOISE_PREFIXES = [
@@ -314,6 +319,8 @@ export type CheckpointSummary = {
delta: { parent: string | null; changed: string[]; removed: string[] };
checkpoint: Record<string, unknown>;
skipped: Array<{ path: string; size: number; reason: string }>;
+ /** Paths left out but not listed in `skipped`, counted by reason. */
+ skipped_omitted?: Record<string, number>;
deps?: Record<string, unknown>;
stats: { source: string; diff_entries: number; kept: number; copied: number; removed: number; diff_ms: number; stat_ms: number; tar_ms: number; total_ms: number; warning?: string; tar_error?: string };
};
@@ -619,14 +626,23 @@ export class Box {
mkdirSync(stage, { recursive: true });
let tarInfo: client.TarResult | null = null;
let tarError: string | null = null;
- // Paths the tar does not carry after all.
+ // Paths the tar does not carry after all. A failed tar lists its first
+ // SKIPPED_LISTED paths (a failed npm install would make the summary huge), the
+ // rest as a count; every one still gets the "unknown" signature below.
+ const leftOut = new Set<string>();
+ const omitted: Record<string, number> = {};
const leaveOut = (items: client.SkippedPath[]) => {
+ let listed = skipped.filter((item) => item.reason === "tar_failed").length;
for (const { path: p, reason } of items) {
const entry = sized.get(p);
- skipped.push({ path: p, size: entry?.type === "file" ? entry.size ?? 0 : 0, reason });
+ const size = entry?.type === "file" ? entry.size ?? 0 : 0;
+ leftOut.add(p);
+ if (isDep(p)) depBytes -= size;
+ if (reason === "tar_failed" && listed >= SKIPPED_LISTED) { omitted[reason] = (omitted[reason] ?? 0) + 1; continue; }
+ if (reason === "tar_failed") listed += 1;
+ skipped.push({ path: p, size, reason });
}
- const out = new Set(items.map((item) => item.path));
- changed = changed.filter((p) => !out.has(p));
+ changed = changed.filter((p) => !leftOut.has(p));
};
if (changed.length) {
try {
@@ -645,11 +661,13 @@ export class Box {
rmSync(path.join(stage, `${label}.tar`), { force: true });
}
}
- const skippedSet = new Set(skipped.map((item) => item.path));
+ for (const item of skipped) leftOut.add(item.path);
const tTar = Date.now();
- // What a later checkpoint compares with: a path left out stays "unknown", so it is tried again.
+ // What a later checkpoint compares with. A path left out keeps an "unknown"
+ // signature (not none): the next checkpoint looks at it again, so it is copied
+ // then if it is still there, or listed as removed if it is gone.
const signatures: Record<string, string> = {};
- for (const [p, sig] of Object.entries(now)) if (!skippedSet.has(p)) signatures[p] = sig;
+ for (const [p, sig] of Object.entries(now)) signatures[p] = leftOut.has(p) ? UNKNOWN_SIGNATURE : sig;
const image = this.image();
const parent = full ? null : chain.last;
const id = `cp_${createHash("sha256").update(canonical({ schema: RECORD_SCHEMA, box: this.key, container: this.containerId, label, turn, parent, image: image.digest || image.id, changed, removed, sha256: tarInfo?.sha256 ?? null })).digest("hex")}`;
@@ -668,6 +686,7 @@ export class Box {
delta: { parent, changed, removed },
checkpoint: { schema: RECORD_SCHEMA, id, parent, turn: turn ?? null, image: { ref: image.ref ?? null, digest: image.digest ?? null }, status: tarError ? "incomplete" : "complete", capturedAt: new Date().toISOString() },
skipped,
+ ...(Object.keys(omitted).length ? { skipped_omitted: omitted } : {}),
...(depsInfo ? { deps: { ...depsInfo, bytes: depBytes, cut: depsCut } } : {}),
stats: { source, diff_entries: entries.length, kept: Object.keys(now).length, copied: changed.length, removed: removed.length, diff_ms: tDiff - t0, stat_ms: tStat - tDiff, tar_ms: tTar - tStat, total_ms: Date.now() - t0, ...(tarInfo?.warning ? { warning: tarInfo.warning } : {}), ...(tarError ? { tar_error: tarError } : {}) },
};
diff --git a/apps/server/src/box/client-tar-batch.test.ts b/apps/server/src/box/client-tar-batch.test.ts
index 609fad7e..44a1a645 100644
--- a/apps/server/src/box/client-tar-batch.test.ts
+++ b/apps/server/src/box/client-tar-batch.test.ts
@@ -94,10 +94,10 @@ function members(buf: Buffer): { members: Array<{ name: string; data: Buffer }>;
// --- a fake boxd ---------------------------------------------------------------------------
-type Change = { kind: "A"; path: string; type: string; size: number; mode: number; mtime_ns: number; ctime_ns: number; ino: number; uid: number; gid: number };
+type Change = { kind: "A" | "C" | "D"; path: string; type: string; size: number; mode: number; mtime_ns: number; ctime_ns: number; ino: number; uid: number; gid: number };
/** boxd's request-line limit and its tar/stat/walk/changes ops. `fail(paths)` may answer a tar request with an error instead. */
-async function fakeBoxd({ padding = 0, fail = () => null, changes = [], limit = REQUEST_LINE_MAX }: { padding?: number; fail?: (paths: string[]) => string | null; changes?: Change[]; limit?: number } = {}): Promise<{ box: BoxEndpoint; seen: { lines: number[]; ops: Array<{ op: string; count: number }> } }> {
+async function fakeBoxd({ padding = 0, fail = () => null, changes = [], gone = () => false, limit = REQUEST_LINE_MAX }: { padding?: number; fail?: (paths: string[]) => string | null; changes?: Change[] | (() => Change[]); gone?: (path: string) => boolean; limit?: number } = {}): Promise<{ box: BoxEndpoint; seen: { lines: number[]; ops: Array<{ op: string; count: number }> } }> {
const seen = { lines: [] as number[], ops: [] as Array<{ op: string; count: number }> };
const server = net.createServer((socket) => {
socket.on("error", () => undefined);
@@ -130,13 +130,13 @@ async function fakeBoxd({ padding = 0, fail = () => null, changes = [], limit =
for (let i = 0; i < tar.length; i += 7777) out.push(frame(0x64, tar.subarray(i, i + 7777)));
socket.end(Buffer.concat([...out, frame(0x65, Buffer.from("busy: a command was running during the snapshot\n")), exit()]));
} else if (req.op === "stat") {
- socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(paths.map((p) => ({ path: p, type: "file", size: 1, uid: 0, gid: 0 }))))), exit()]));
+ socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(paths.map((p) => (gone(p) ? { path: p, type: "missing", uid: 0, gid: 0 } : { path: p, type: "file", size: 1, uid: 0, gid: 0 }))))), exit()]));
} else if (req.op === "walk") {
const entries = paths.map((p) => ({ path: `${p}/x`, type: "file", size: 1, uid: 0, gid: 0 }));
const wait = req.wait ?? 400_000;
socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify({ entries: entries.slice(0, wait), truncated: entries.length > wait }))), exit()]));
} else if (req.op === "changes") {
- socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(changes))), exit()]));
+ socket.end(Buffer.concat([frame(0x64, Buffer.from(JSON.stringify(typeof changes === "function" ? changes() : changes))), exit()]));
} else {
socket.end(exit(0));
}
@@ -294,10 +294,11 @@ describe("box path lists in requests", () => {
// --- the checkpoint ------------------------------------------------------------------------
-function fakeBox(endpoint: BoxEndpoint) {
+function fakeBox(endpoint: BoxEndpoint, { deps = false }: { deps?: boolean } = {}) {
const work = tmp();
const root = path.join(work, "proj");
mkdirSync(root);
+ if (deps) mkdirSync(path.join(root, "node_modules"));
const logs: Array<{ level: string; message: string }> = [];
const box = new Box({
key: "k1", name: "omnirush-box-k1", root, croot: root, runtime: { name: "docker" } as BoxRuntime, home: path.join(work, "home"), runFolder: path.join(work, "run"),
@@ -334,7 +335,7 @@ describe("box checkpoint with many changed files", () => {
expect(s.checkpoint.status).toBe("incomplete");
expect(s.tar).toBeNull();
expect(s.delta.changed).toEqual([]);
- expect(s.skipped.filter((item) => item.reason === "tar_failed").length).toBe(paths.length);
+ expect(s.skipped.filter((item) => item.reason === "tar_failed").length + (s.skipped_omitted?.tar_failed ?? 0)).toBe(paths.length);
expect(s.stats.tar_error).toMatch(/request line too long/);
expect(logs.some((line) => line.level === "warn" && line.message === "box checkpoint tar failed")).toBe(true);
first.commit();
@@ -344,4 +345,56 @@ describe("box checkpoint with many changed files", () => {
expect(second.summary.delta.parent).toBe(s.checkpoint.id as string);
expect(second.summary.delta.changed.length).toBe(paths.length);
});
+
+ test("a path whose tar failed is looked at again: deleted later, it is listed as removed", async () => {
+ // turn-2 carries x; turn-3 changes x and its tar fails; turn-4 finds x deleted.
+ const x = "/usr/local/lib/node_modules/pkg-0/lib/x.js";
+ let state: { changes: Change[]; gone: boolean } = { changes: [{ ...changeOf(x), mtime_ns: 1 }], gone: false };
+ let failing = false;
+ const { box: endpoint } = await fakeBoxd({ changes: () => state.changes, gone: (p) => state.gone && p === x, fail: () => (failing ? "request line too long" : null) });
+ const { box, snaps } = fakeBox(endpoint);
+ const t2 = await box.checkpoint(snaps, 2);
+ expect(t2.summary.delta.changed).toEqual([x]);
+ t2.commit();
+ state = { changes: [{ ...changeOf(x), kind: "C", mtime_ns: 2 }], gone: false };
+ failing = true;
+ const t3 = await box.checkpoint(snaps, 3);
+ expect(t3.summary.checkpoint.status).toBe("incomplete");
+ expect(t3.summary.skipped.map((item) => item.reason)).toEqual(["tar_failed"]);
+ t3.commit();
+ expect(JSON.parse(readFileSync(path.join(snaps, ".chain.json"), "utf8")).signatures[x]).toBe("?");
+ state = { changes: [{ ...changeOf(x), kind: "D", type: "missing" }], gone: true };
+ failing = false;
+ const t4 = await box.checkpoint(snaps, 4);
+ expect(t4.summary.checkpoint.status).toBe("complete");
+ expect(t4.summary.delta.removed).toEqual([x]);
+ expect(t4.summary.delta.parent).toBe(t3.summary.checkpoint.id as string);
+ });
+
+ test("a path whose tar failed and is still there is copied by the next checkpoint", async () => {
+ const x = "/usr/local/lib/node_modules/pkg-0/lib/x.js";
+ let failing = true;
+ const { box: endpoint } = await fakeBoxd({ changes: [changeOf(x)], fail: () => (failing ? "boom" : null) });
+ const { box, snaps } = fakeBox(endpoint);
+ (await box.checkpoint(snaps, 2)).commit();
+ failing = false;
+ const t3 = await box.checkpoint(snaps, 3);
+ expect(t3.summary.delta.changed).toEqual([x]);
+ expect(t3.summary.checkpoint.status).toBe("complete");
+ });
+
+ test("a failed tar lists its first 1000 paths, counts the rest, and leaves their bytes out of deps", async () => {
+ const { box: endpoint } = await fakeBoxd({ changes: manyPaths(3000, "/usr/local/lib/node_modules").map(changeOf), fail: () => "boom" });
+ const { box, snaps } = fakeBox(endpoint, { deps: true });
+ const taken = await box.checkpoint(snaps, 2);
+ const s = taken.summary;
+ // 3003 paths and the project's node_modules/x (the fake walk's one entry).
+ expect(s.skipped.filter((item) => item.reason === "tar_failed").length).toBe(1000);
+ expect(s.skipped_omitted?.tar_failed).toBe(3004 - 1000);
+ expect(s.deps?.entries).toBe(1);
+ expect(s.deps?.bytes).toBe(0);
+ taken.commit();
+ const signatures: Record<string, string> = JSON.parse(readFileSync(path.join(snaps, ".chain.json"), "utf8")).signatures;
+ expect(Object.values(signatures).filter((sig) => sig === "?").length).toBe(3004);
+ });
});
diff --git a/apps/server/src/box/client.ts b/apps/server/src/box/client.ts
index 77763d53..8f606d2e 100644
--- a/apps/server/src/box/client.ts
+++ b/apps/server/src/box/client.ts
@@ -234,7 +234,7 @@ export async function tarTo(box: BoxEndpoint, paths: string[], file: string, { t
// --- path lists in requests -------------------------------------------------------
/**
- * boxd reads a request line of at most 4 MiB (box/go/proto ReadRequest; 64
+ * boxd reads a request line of at most 4 MiB (box/go/proto ReadRequest; 16
* MiB from this version on, but a box may run an older boxd): a list of
* paths goes in requests whose line stays well under it.
*/