Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(uploader): last snapshot waits briefly for the toolchain; excluded list fits the budget

ogsamratce2b3be12703
diff --git a/apps/server/src/excluded-files.ts b/apps/server/src/excluded-files.ts
index 1f860fb5..56e036ba 100644
--- a/apps/server/src/excluded-files.ts
+++ b/apps/server/src/excluded-files.ts
@@ -38,7 +38,7 @@ export const MAX_EXCLUDED_HASH_FILE_BYTES = 64 * 1024 * 1024;
/** Bytes read for hashing per snapshot; past it, entries go without a SHA-256. */
export const EXCLUDED_HASH_BUDGET_BYTES = 256 * 1024 * 1024;
const MAX_GITIGNORED_LISTED = 20_000;
-/** The serialized block's cap; the snapshot reserves this much of its budget. */
+/** The serialized block's cap (the snapshot may allow less: what its budget has left). */
export const MAX_EXCLUDED_FILES_BYTES = 1024 * 1024;
type Pending = { path: string; reason: ExcludedReason; size?: number | null; sha256?: string | null };
@@ -66,7 +66,8 @@ export class ExcludedFiles {
}
/** The block, with sizes and digests filled in (bounded; never throws). */
- async finish(): Promise<ExcludedFilesBlock> {
+ async finish(maxBytes = MAX_EXCLUDED_FILES_BYTES): Promise<ExcludedFilesBlock> {
+ const limit = Math.min(maxBytes, MAX_EXCLUDED_FILES_BYTES);
const listed = [...this.pending.values()].slice(0, MAX_EXCLUDED_ENTRIES);
let budget = this.options.hashBudgetBytes ?? EXCLUDED_HASH_BUDGET_BYTES;
const entries: ExcludedEntry[] = [];
@@ -99,7 +100,7 @@ export class ExcludedFiles {
const uploadPath = this.options.uploadPath(directory ? item.path.slice(0, -1) : item.path);
const entry: ExcludedEntry = { path: directory ? `${uploadPath}/` : uploadPath, size, sha256, reason: item.reason };
bytes += JSON.stringify(entry).length + 1;
- if (bytes > MAX_EXCLUDED_FILES_BYTES) break;
+ if (bytes > limit) break;
entries.push(entry);
}
return { schema: 1, entries, truncated_count: Math.max(0, this.pending.size - entries.length), counts: { ...this.counts } };
diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.ts
index 38e9014f..c226f1b0 100644
--- a/apps/server/src/session-uploader.test.ts
+++ b/apps/server/src/session-uploader.test.ts
@@ -2910,7 +2910,11 @@ describe("session uploader incremental snapshots", () => {
expect(contentPaths(batchedChange)).toContain("kept.txt");
expect(batchedChange.manifest.map((entry) => entry.path)).not.toContain("scratch-0.log");
await batcher.stop();
- expect(JSON.stringify(batched.uploads)).not.toContain("scratch-");
+ // Only excluded_files may name an ignored file (as left out, with no content).
+ const withoutExclusions = batched.uploads.map((envelope) => ({ ...envelope, excluded_files: undefined }));
+ expect(JSON.stringify(withoutExclusions)).not.toContain("scratch-");
+ const listed = batched.uploads.flatMap((envelope) => (((envelope as Record<string, unknown>).excluded_files as { entries?: Array<{ path: string; reason: string }> } | undefined)?.entries ?? []));
+ expect(listed.filter((entry) => entry.path.startsWith("scratch-")).every((entry) => entry.reason === "gitignored")).toBe(true);
// Past the cap the tree is polled: no watchers, and a milestone rescans it.
const polled = makeUploads();
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index b1ef09c0..770f61f6 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -23,7 +23,7 @@ import { TurnBaseStore, TurnDiffBuilder, type TurnDiffInput } from "./turn-diff.
import { SUBAGENT_MODEL_FALLBACK_TRACE } from "./omnirush-swarm.js";
import { writeFileAtomic } from "./atomic-write.js";
import { XCODE_CLT_MISSING, gitSkipReason } from "./command-guard.js";
-import { ExcludedFiles, MAX_EXCLUDED_FILES_BYTES, listGitIgnored, type HashCache } from "./excluded-files.js";
+import { ExcludedFiles, listGitIgnored, type HashCache } from "./excluded-files.js";
import { ToolchainCache, type CollectOptions as ToolchainOptions, type UploadToolchain } from "./toolchain.js";
const execFileAsync = promisify(execFile);
@@ -3703,6 +3703,12 @@ type EnvelopeBody = {
schemaVersion?: 2 | 3;
};
+/** The part of SNAPSHOT_WRAPPER_MARGIN_BYTES excluded_files may use beyond the content budget's leftover. */
+const EXCLUDED_FILES_MARGIN_BYTES = 256 * 1024;
+
+/** How long a session's last snapshot waits for a toolchain collection still running. */
+const FINAL_TOOLCHAIN_WAIT_MS = 3_000;
+
/** `{ git_skipped: "xcode_clt_missing" }` when capture may not run git here (command-guard.ts), else nothing. */
async function gitSkippedField(): Promise<{ git_skipped?: string }> {
const reason = await gitSkipReason();
@@ -4057,7 +4063,7 @@ export class SessionUploader {
* denied, binary, oversized, gitignored and capped files; a targeted one
* only what it inspected.
*/
- private async excludedFiles(root: string, scan: ScanResult, omitted: readonly SnapshotCandidate[], full: boolean): Promise<unknown> {
+ private async excludedFiles(root: string, scan: ScanResult, omitted: readonly SnapshotCandidate[], full: boolean, maxBytes: number): Promise<unknown> {
const excluded = new ExcludedFiles(root, { uploadPath: portablePathForUpload, hashes: this.excludedHashes });
for (const path of scan.deniedPaths ?? []) excluded.add(path, "privacy");
for (const [path, size] of scan.tooLargePaths ?? []) excluded.add(path, "too_large", { size });
@@ -4067,13 +4073,13 @@ export class SessionUploader {
}
for (const path of scan.cappedPaths ?? []) excluded.add(path, "snapshot_cap");
for (const candidate of omitted) excluded.add(candidate.path, "snapshot_cap", { size: candidate.bytes, sha256: scan.manifest.get(candidate.path)?.sha256 ?? null });
- return excluded.finish();
+ return excluded.finish(maxBytes);
}
/** The environment block plus the toolchain of the session's project root, when one was captured. */
- private async sessionEnvironment(root: string): Promise<UploadEnvironment> {
+ private async sessionEnvironment(root: string, waitMs?: number): Promise<UploadEnvironment> {
const environment = await this.environment();
- const toolchain = this.toolchains ? await this.toolchains.get(root).catch(() => null) : null;
+ const toolchain = this.toolchains ? await this.toolchains.get(root, waitMs).catch(() => null) : null;
return toolchain ? { ...environment, toolchain } : environment;
}
@@ -5345,7 +5351,8 @@ export class SessionUploader {
...snapshotPriorityPaths([], git),
]);
const candidates = candidatesFor(scan, state.relevance, gitPaths);
- const environment = await this.sessionEnvironment(state.root);
+ // The last snapshot waits briefly for a collection still running (toolchain.ts).
+ const environment = await this.sessionEnvironment(state.root, type === "end" ? FINAL_TOOLCHAIN_WAIT_MS : undefined);
const rootName = workspaceRootName(state.root);
const touchedPaths = this.touchedPathsForUpload(state);
const metadata = JSON.stringify({
@@ -5383,9 +5390,12 @@ export class SessionUploader {
let fixedBytes = serializedBytes(extras) + SNAPSHOT_WRAPPER_MARGIN_BYTES;
for (const file of leading) fixedBytes += serializedBytes(file) + 1;
for (const entry of scan.manifest.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;
- const budget = Math.max(0, this.snapshotMaxBytes - fixedBytes - MAX_EXCLUDED_FILES_BYTES);
+ const budget = Math.max(0, this.snapshotMaxBytes - fixedBytes);
const selection = selectSnapshotContent(candidates, budget);
const capOmitted: SnapshotCandidate[] = [];
+ // excluded_files gets what the content budget left plus a slice of the
+ // wrapper margin (EXCLUDED_FILES_MARGIN_BYTES), so a full snapshot still lists some.
+ let contentUsed = 0;
let omittedCount = selection.omittedCount;
let omittedBytes = selection.omittedBytes;
const manifest = scan.manifest;
@@ -5420,6 +5430,7 @@ export class SessionUploader {
if (read.entry.sha256 !== manifest.get(candidate.path)?.sha256) manifest.set(candidate.path, read.entry);
await emit({ path: candidate.uploadPath, content: read.content, sha256: read.entry.sha256 });
used += cost;
+ contentUsed = used;
this.bases.put(read.entry.sha256, read.content, read.entry.bytes);
await this.bases.writable();
}
@@ -5440,7 +5451,7 @@ export class SessionUploader {
excluded_files: await this.excludedFiles(state.root, scan, [
...candidates.filter((candidate) => !selection.selected.has(candidate.path)),
...capOmitted,
- ], !(targeted && previous !== null)),
+ ], !(targeted && previous !== null), budget - contentUsed + EXCLUDED_FILES_MARGIN_BYTES),
}),
});
if (omittedCount > 0) {
diff --git a/apps/server/src/toolchain.test.ts b/apps/server/src/toolchain.test.ts
index f94256fe..d6d1619e 100644
--- a/apps/server/src/toolchain.test.ts
+++ b/apps/server/src/toolchain.test.ts
@@ -332,3 +332,16 @@ test.skipIf(!hasZstd)("an upload never waits for a slow collection: the start en
assert.deepEqual(end.environment.toolchain.versions, { node: "v22.19.0" });
assert.ok(Date.now() - started < 5_000);
});
+
+test.skipIf(!hasZstd)("a short session still records it: the last snapshot waits briefly for a collection in flight", async () => {
+ const envelopes = await envelopesFrom({
+ toolchain: {
+ resolve: (command) => (command === "node" ? "/usr/bin/node" : null),
+ run: () => new Promise((resolve) => setTimeout(() => resolve(ok("v22.19.0\n")), 800)),
+ },
+ });
+ const start = envelopes.find((envelope) => envelope.snapshot_type === "start");
+ const end = envelopes.find((envelope) => envelope.snapshot_type === "end");
+ assert.equal("toolchain" in start.environment, false);
+ assert.deepEqual(end.environment.toolchain.versions, { node: "v22.19.0" });
+});
diff --git a/apps/server/src/toolchain.ts b/apps/server/src/toolchain.ts
index a96c4eee..fe78a616 100644
--- a/apps/server/src/toolchain.ts
+++ b/apps/server/src/toolchain.ts
@@ -623,9 +623,11 @@ type CacheEntry = { signature: string; pending: Promise<UploadToolchain | null>;
/**
* One toolchain per project root, collected when a session starts (or at its
* first upload) and again only when the root's manifest/lockfile set
- * changes. An upload never waits for a collection in flight: it carries the
+ * changes. An upload does not wait for a collection in flight: it carries the
* cached block once there is one, so the session's first envelope may go
- * without it and the next ones carry it. `waitMs` (tests) waits that long.
+ * without it and the next ones carry it. `waitMs` bounds a wait: the
+ * session's last snapshot waits a little, so a short session still records
+ * its toolchain.
*/
export class ToolchainCache {
private readonly entries = new Map<string, CacheEntry>();
@@ -633,7 +635,7 @@ export class ToolchainCache {
constructor(private readonly options: CollectOptions & { waitMs?: number; scrub?: (toolchain: UploadToolchain) => UploadToolchain } = {}) {}
- async get(root: string): Promise<UploadToolchain | null> {
+ async get(root: string, waitMs = this.options.waitMs ?? 0): Promise<UploadToolchain | null> {
const files = await projectFiles(root);
let entry = this.entries.get(root);
if (!entry || entry.signature !== files.signature) {
@@ -650,13 +652,13 @@ export class ToolchainCache {
this.entries.set(root, next);
entry = next;
}
- const waitMs = this.options.waitMs ?? 0;
let timer: ReturnType<typeof setTimeout> | undefined;
await Promise.race([
entry.pending,
new Promise<void>((resolvePromise) => {
+ // Not unref'd: the wait is bounded, and an unref'd timer could let
+ // the process exit with an upload still waiting on it.
timer = setTimeout(resolvePromise, waitMs);
- timer.unref?.();
}),
]);
clearTimeout(timer);