A desktop coding agent with free access to frontier models.
TypeScript
fix(uploader): last snapshot waits briefly for the toolchain; excluded list fits the budget
ogsamratce2b3be12703
diff --git a/apps/server/src/excluded-files.ts b/apps/server/src/excluded-files.tsindex 1f860fb5..56e036ba 100644--- a/apps/server/src/excluded-files.ts+++ b/apps/server/src/excluded-files.ts@@ -38,7 +38,7 @@ export const MAX_EXCLUDED_HASH_FILE_BYTES = 64 * 1024 * 1024;/** Bytes read for hashing per snapshot; past it, entries go without a SHA-256. */export const EXCLUDED_HASH_BUDGET_BYTES = 256 * 1024 * 1024;const MAX_GITIGNORED_LISTED = 20_000;-/** The serialized block's cap; the snapshot reserves this much of its budget. */+/** The serialized block's cap (the snapshot may allow less: what its budget has left). */export const MAX_EXCLUDED_FILES_BYTES = 1024 * 1024;type Pending = { path: string; reason: ExcludedReason; size?: number | null; sha256?: string | null };@@ -66,7 +66,8 @@ export class ExcludedFiles {}/** The block, with sizes and digests filled in (bounded; never throws). */- async finish(): Promise<ExcludedFilesBlock> {+ async finish(maxBytes = MAX_EXCLUDED_FILES_BYTES): Promise<ExcludedFilesBlock> {+ const limit = Math.min(maxBytes, MAX_EXCLUDED_FILES_BYTES);const listed = [...this.pending.values()].slice(0, MAX_EXCLUDED_ENTRIES);let budget = this.options.hashBudgetBytes ?? EXCLUDED_HASH_BUDGET_BYTES;const entries: ExcludedEntry[] = [];@@ -99,7 +100,7 @@ export class ExcludedFiles {const uploadPath = this.options.uploadPath(directory ? item.path.slice(0, -1) : item.path);const entry: ExcludedEntry = { path: directory ? `${uploadPath}/` : uploadPath, size, sha256, reason: item.reason };bytes += JSON.stringify(entry).length + 1;- if (bytes > MAX_EXCLUDED_FILES_BYTES) break;+ if (bytes > limit) break;entries.push(entry);}return { schema: 1, entries, truncated_count: Math.max(0, this.pending.size - entries.length), counts: { ...this.counts } };diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.tsindex 38e9014f..c226f1b0 100644--- a/apps/server/src/session-uploader.test.ts+++ b/apps/server/src/session-uploader.test.ts@@ -2910,7 +2910,11 @@ describe("session uploader incremental snapshots", () => {expect(contentPaths(batchedChange)).toContain("kept.txt");expect(batchedChange.manifest.map((entry) => entry.path)).not.toContain("scratch-0.log");await batcher.stop();- expect(JSON.stringify(batched.uploads)).not.toContain("scratch-");+ // Only excluded_files may name an ignored file (as left out, with no content).+ const withoutExclusions = batched.uploads.map((envelope) => ({ ...envelope, excluded_files: undefined }));+ expect(JSON.stringify(withoutExclusions)).not.toContain("scratch-");+ const listed = batched.uploads.flatMap((envelope) => (((envelope as Record<string, unknown>).excluded_files as { entries?: Array<{ path: string; reason: string }> } | undefined)?.entries ?? []));+ expect(listed.filter((entry) => entry.path.startsWith("scratch-")).every((entry) => entry.reason === "gitignored")).toBe(true);// Past the cap the tree is polled: no watchers, and a milestone rescans it.const polled = makeUploads();diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex b1ef09c0..770f61f6 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -23,7 +23,7 @@ import { TurnBaseStore, TurnDiffBuilder, type TurnDiffInput } from "./turn-diff.import { SUBAGENT_MODEL_FALLBACK_TRACE } from "./omnirush-swarm.js";import { writeFileAtomic } from "./atomic-write.js";import { XCODE_CLT_MISSING, gitSkipReason } from "./command-guard.js";-import { ExcludedFiles, MAX_EXCLUDED_FILES_BYTES, listGitIgnored, type HashCache } from "./excluded-files.js";+import { ExcludedFiles, listGitIgnored, type HashCache } from "./excluded-files.js";import { ToolchainCache, type CollectOptions as ToolchainOptions, type UploadToolchain } from "./toolchain.js";const execFileAsync = promisify(execFile);@@ -3703,6 +3703,12 @@ type EnvelopeBody = {schemaVersion?: 2 | 3;};+/** The part of SNAPSHOT_WRAPPER_MARGIN_BYTES excluded_files may use beyond the content budget's leftover. */+const EXCLUDED_FILES_MARGIN_BYTES = 256 * 1024;++/** How long a session's last snapshot waits for a toolchain collection still running. */+const FINAL_TOOLCHAIN_WAIT_MS = 3_000;+/** `{ git_skipped: "xcode_clt_missing" }` when capture may not run git here (command-guard.ts), else nothing. */async function gitSkippedField(): Promise<{ git_skipped?: string }> {const reason = await gitSkipReason();@@ -4057,7 +4063,7 @@ export class SessionUploader {* denied, binary, oversized, gitignored and capped files; a targeted one* only what it inspected.*/- private async excludedFiles(root: string, scan: ScanResult, omitted: readonly SnapshotCandidate[], full: boolean): Promise<unknown> {+ private async excludedFiles(root: string, scan: ScanResult, omitted: readonly SnapshotCandidate[], full: boolean, maxBytes: number): Promise<unknown> {const excluded = new ExcludedFiles(root, { uploadPath: portablePathForUpload, hashes: this.excludedHashes });for (const path of scan.deniedPaths ?? []) excluded.add(path, "privacy");for (const [path, size] of scan.tooLargePaths ?? []) excluded.add(path, "too_large", { size });@@ -4067,13 +4073,13 @@ export class SessionUploader {}for (const path of scan.cappedPaths ?? []) excluded.add(path, "snapshot_cap");for (const candidate of omitted) excluded.add(candidate.path, "snapshot_cap", { size: candidate.bytes, sha256: scan.manifest.get(candidate.path)?.sha256 ?? null });- return excluded.finish();+ return excluded.finish(maxBytes);}/** The environment block plus the toolchain of the session's project root, when one was captured. */- private async sessionEnvironment(root: string): Promise<UploadEnvironment> {+ private async sessionEnvironment(root: string, waitMs?: number): Promise<UploadEnvironment> {const environment = await this.environment();- const toolchain = this.toolchains ? await this.toolchains.get(root).catch(() => null) : null;+ const toolchain = this.toolchains ? await this.toolchains.get(root, waitMs).catch(() => null) : null;return toolchain ? { ...environment, toolchain } : environment;}@@ -5345,7 +5351,8 @@ export class SessionUploader {...snapshotPriorityPaths([], git),]);const candidates = candidatesFor(scan, state.relevance, gitPaths);- const environment = await this.sessionEnvironment(state.root);+ // The last snapshot waits briefly for a collection still running (toolchain.ts).+ const environment = await this.sessionEnvironment(state.root, type === "end" ? FINAL_TOOLCHAIN_WAIT_MS : undefined);const rootName = workspaceRootName(state.root);const touchedPaths = this.touchedPathsForUpload(state);const metadata = JSON.stringify({@@ -5383,9 +5390,12 @@ export class SessionUploader {let fixedBytes = serializedBytes(extras) + SNAPSHOT_WRAPPER_MARGIN_BYTES;for (const file of leading) fixedBytes += serializedBytes(file) + 1;for (const entry of scan.manifest.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;- const budget = Math.max(0, this.snapshotMaxBytes - fixedBytes - MAX_EXCLUDED_FILES_BYTES);+ const budget = Math.max(0, this.snapshotMaxBytes - fixedBytes);const selection = selectSnapshotContent(candidates, budget);const capOmitted: SnapshotCandidate[] = [];+ // excluded_files gets what the content budget left plus a slice of the+ // wrapper margin (EXCLUDED_FILES_MARGIN_BYTES), so a full snapshot still lists some.+ let contentUsed = 0;let omittedCount = selection.omittedCount;let omittedBytes = selection.omittedBytes;const manifest = scan.manifest;@@ -5420,6 +5430,7 @@ export class SessionUploader {if (read.entry.sha256 !== manifest.get(candidate.path)?.sha256) manifest.set(candidate.path, read.entry);await emit({ path: candidate.uploadPath, content: read.content, sha256: read.entry.sha256 });used += cost;+ contentUsed = used;this.bases.put(read.entry.sha256, read.content, read.entry.bytes);await this.bases.writable();}@@ -5440,7 +5451,7 @@ export class SessionUploader {excluded_files: await this.excludedFiles(state.root, scan, [...candidates.filter((candidate) => !selection.selected.has(candidate.path)),...capOmitted,- ], !(targeted && previous !== null)),+ ], !(targeted && previous !== null), budget - contentUsed + EXCLUDED_FILES_MARGIN_BYTES),}),});if (omittedCount > 0) {diff --git a/apps/server/src/toolchain.test.ts b/apps/server/src/toolchain.test.tsindex f94256fe..d6d1619e 100644--- a/apps/server/src/toolchain.test.ts+++ b/apps/server/src/toolchain.test.ts@@ -332,3 +332,16 @@ test.skipIf(!hasZstd)("an upload never waits for a slow collection: the start enassert.deepEqual(end.environment.toolchain.versions, { node: "v22.19.0" });assert.ok(Date.now() - started < 5_000);});++test.skipIf(!hasZstd)("a short session still records it: the last snapshot waits briefly for a collection in flight", async () => {+ const envelopes = await envelopesFrom({+ toolchain: {+ resolve: (command) => (command === "node" ? "/usr/bin/node" : null),+ run: () => new Promise((resolve) => setTimeout(() => resolve(ok("v22.19.0\n")), 800)),+ },+ });+ const start = envelopes.find((envelope) => envelope.snapshot_type === "start");+ const end = envelopes.find((envelope) => envelope.snapshot_type === "end");+ assert.equal("toolchain" in start.environment, false);+ assert.deepEqual(end.environment.toolchain.versions, { node: "v22.19.0" });+});diff --git a/apps/server/src/toolchain.ts b/apps/server/src/toolchain.tsindex a96c4eee..fe78a616 100644--- a/apps/server/src/toolchain.ts+++ b/apps/server/src/toolchain.ts@@ -623,9 +623,11 @@ type CacheEntry = { signature: string; pending: Promise<UploadToolchain | null>;/*** One toolchain per project root, collected when a session starts (or at its* first upload) and again only when the root's manifest/lockfile set- * changes. An upload never waits for a collection in flight: it carries the+ * changes. An upload does not wait for a collection in flight: it carries the* cached block once there is one, so the session's first envelope may go- * without it and the next ones carry it. `waitMs` (tests) waits that long.+ * without it and the next ones carry it. `waitMs` bounds a wait: the+ * session's last snapshot waits a little, so a short session still records+ * its toolchain.*/export class ToolchainCache {private readonly entries = new Map<string, CacheEntry>();@@ -633,7 +635,7 @@ export class ToolchainCache {constructor(private readonly options: CollectOptions & { waitMs?: number; scrub?: (toolchain: UploadToolchain) => UploadToolchain } = {}) {}- async get(root: string): Promise<UploadToolchain | null> {+ async get(root: string, waitMs = this.options.waitMs ?? 0): Promise<UploadToolchain | null> {const files = await projectFiles(root);let entry = this.entries.get(root);if (!entry || entry.signature !== files.signature) {@@ -650,13 +652,13 @@ export class ToolchainCache {this.entries.set(root, next);entry = next;}- const waitMs = this.options.waitMs ?? 0;let timer: ReturnType<typeof setTimeout> | undefined;await Promise.race([entry.pending,new Promise<void>((resolvePromise) => {+ // Not unref'd: the wait is bounded, and an unref'd timer could let+ // the process exit with an upload still waiting on it.timer = setTimeout(resolvePromise, waitMs);- timer.unref?.();}),]);clearTimeout(timer);