A desktop coding agent with free access to frontier models.
TypeScript
fix(collector): spool an upload that ran out its deadline; shutdown aborts served uploads
ogsamrat03768f206aa8
diff --git a/apps/server/src/capture-client.test.ts b/apps/server/src/capture-client.test.tsindex e0bed8cb..11f16303 100644--- a/apps/server/src/capture-client.test.ts+++ b/apps/server/src/capture-client.test.ts@@ -1,6 +1,6 @@import { afterEach, describe, expect, test } from "bun:test";import { execFile } from "node:child_process";-import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";+import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";import { promisify } from "node:util";@@ -227,6 +227,69 @@ describe("capture worker", () => {expect(sink.envelopes().map((item) => item.snapshot_type)).toContain("end");}, 60_000);+ test("a stop that runs out of time aborts the collector upload this thread still serves for the worker, and does not wait for it", async () => {+ const root = await syntheticWorkspace(5);+ const signals: AbortSignal[] = [];+ let started!: () => void;+ const inFlight = new Promise<void>((resolvePromise) => { started = resolvePromise; });+ const capture = service({+ stateDir: await tempDir("state"),+ stopTimeoutMs: 300,+ collector: {+ // The gateway never answers and this hook ignores its signal, as a stalled upload on the main thread would.+ upload: (_sessionId, _bytes, signal) => {+ signals.push(signal!);+ started();+ return new Promise<Response>(() => undefined);+ },+ },+ });+ capture.startSession("session-served-0001", "workspace-served", root);+ await inFlight;+ expect(signals[0]!.aborted).toBe(false);+ const stopping = performance.now();+ await capture.stop();+ expect(performance.now() - stopping).toBeLessThan(5_000);+ expect(capture.mode()).toBe("down");+ // Terminating the worker ended the request it had asked this thread to make.+ expect(signals).toHaveLength(1);+ expect(signals[0]!.aborted).toBe(true);+ }, 60_000);++ test("in-process, a stop that runs out of time aborts the upload in flight and spools it for the next start", async () => {+ const root = await syntheticWorkspace(5);+ const stateDir = await tempDir("state");+ const signals: AbortSignal[] = [];+ let started!: () => void;+ const inFlight = new Promise<void>((resolvePromise) => { started = resolvePromise; });+ const capture = service({+ stateDir,+ worker: false,+ stopTimeoutMs: 300,+ collector: {+ upload: (_sessionId, _bytes, signal) => {+ signals.push(signal!);+ started();+ return new Promise<Response>(() => undefined);+ },+ },+ });+ capture.startSession("session-local-stop-01", "workspace-local-stop", root);+ await inFlight;+ const stopping = performance.now();+ await capture.stop();+ expect(performance.now() - stopping).toBeLessThan(5_000);+ expect(signals[0]!.aborted).toBe(true);+ // The start snapshot and the session's end snapshot wait in the spool, the end one never sent.+ const spoolDir = join(stateDir, "omnirush-collector-spool");+ const spooledTypes = async () => (await Promise.all((await readdir(spoolDir).catch(() => [] as string[]))+ .filter((name) => name.endsWith(".json"))+ .map(async (name) => (JSON.parse(await readFile(join(spoolDir, name), "utf8")) as { snapshot_type: string }).snapshot_type))).sort();+ await until(async () => (await spooledTypes()).length === 2, 10_000, "the spooled snapshots");+ expect(await spooledTypes()).toEqual(["end", "start"]);+ expect(signals).toHaveLength(1);+ }, 60_000);+test("stopping packs the project archive's final archives on the worker, unless the account is gone", async () => {const results: string[][] = [];for (const archiveFinals of [undefined, Promise.resolve(false)]) {diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.tsindex 902eb0f3..c14e706f 100644--- a/apps/server/src/capture-client.ts+++ b/apps/server/src/capture-client.ts@@ -41,6 +41,8 @@ const SESSION_ID_PATTERN = /^[A-Za-z0-9._:-]{8,128}$/;export type CaptureServiceOptions = Omit<CaptureHostOptions, "onSessionClosed"> & {/** Run capture on a worker thread; default on unless OMNIRUSH_CAPTURE_WORKER is 0, false, no or off. */worker?: boolean;+ /** How long shutdown waits for the capture to stop; STOP_TIMEOUT_MS unless a test lowers it. */+ stopTimeoutMs?: number;};export type CaptureService = {@@ -184,20 +186,26 @@ class CaptureClient implements CaptureService {for (const { controller, channel } of this.served.values()) if (channel === "archive") controller.abort();const finals = await withinTimeout(archiveFinals, ACCOUNT_CHECK_TIMEOUT_MS);const done = this.dispatch({ kind: "call", id: null, method: "stop", args: [{ archiveFinals: finals }] }, [], true);+ const timeoutMs = this.options.stopTimeoutMs ?? STOP_TIMEOUT_MS;let timer: ReturnType<typeof setTimeout> | undefined;- const timedOut = new Promise<void>((resolvePromise) => {+ const timedOut = new Promise<true>((resolvePromise) => {timer = setTimeout(() => {- this.options.log("warn", "OmniRush capture did not stop in time; the capture worker is terminated", { timeoutMs: STOP_TIMEOUT_MS });- resolvePromise();- }, STOP_TIMEOUT_MS);+ this.options.log("warn", this.local+ ? "OmniRush capture did not stop in time; its uploads in flight are aborted"+ : "OmniRush capture did not stop in time; the capture worker is terminated", { timeoutMs });+ resolvePromise(true);+ }, timeoutMs);timer.unref?.();});- await Promise.race([done, timedOut]);+ const late = await Promise.race([done.then(() => false), timedOut]);clearTimeout(timer);+ // In-process there is no worker to terminate: its uploads end here instead (spooled for the next start).+ if (late) this.local?.collector.abortUploads();const worker = this.worker;this.worker = null;this.current = "down";if (worker) await worker.terminate().catch(() => undefined);+ // Also aborts the requests this thread was still serving for the worker.this.settleOutstanding();}@@ -328,7 +336,6 @@ class CaptureClient implements CaptureService {private onExit(worker: Worker, code: number): void {if (worker !== this.worker) return;this.worker = null;- for (const { controller } of this.served.values()) controller.abort();this.settleOutstanding();if (this.stopping) return;if (this.current === "starting") {@@ -347,10 +354,16 @@ class CaptureClient implements CaptureService {this.options.log("warn", "OmniRush capture worker exited too often; capture is off until the app restarts", { code });}- /** Every caller waiting on the worker gets an empty result. */+ /**+ * The worker is gone: every caller waiting on it gets an empty result, and+ * every request this thread still serves for it (a collector upload that+ * outlasted the stop, an archive or egress fetch) is aborted, so none runs+ * on for up to its own deadline with no one left to take the answer.+ */private settleOutstanding(): void {for (const resolvePromise of this.replies.values()) resolvePromise(null);this.replies.clear();+ for (const { controller } of this.served.values()) controller.abort(new DOMException("The capture worker has stopped", "AbortError"));this.served.clear();}@@ -359,7 +372,7 @@ class CaptureClient implements CaptureService {if (reason !== null) {this.options.log("warn", "OmniRush capture worker could not start; capturing in-process", { error: errorSummary(reason) });}- const { worker: _worker, ...hostOptions } = this.options;+ const { worker: _worker, stopTimeoutMs: _stopTimeoutMs, ...hostOptions } = this.options;try {this.local = new CaptureHost(hostOptions);} catch (error) {diff --git a/apps/server/src/omnirush-gateway-broker.test.ts b/apps/server/src/omnirush-gateway-broker.test.tsindex 18ed590c..a09c2cb1 100644--- a/apps/server/src/omnirush-gateway-broker.test.ts+++ b/apps/server/src/omnirush-gateway-broker.test.ts@@ -933,3 +933,97 @@ describe("OmniRush gateway broker model catalog requests", () => {expect(await response.json()).toEqual({ error: "omnirush_account_required" });});});++describe("OmniRush gateway broker collect deadline", () => {+ const gatewayUrl = "https://gateway.example/omnirush/v1";+ // 200 ms, the body at 1 MiB/s, 200 ms for the answer: 401 ms for a few bytes, 650 ms for 256 KiB.+ const collectUploadBudget = { baseMs: 200, bytesPerSecond: 1024 * 1024, maxSendMs: 5_000, responseMs: 200 };++ /** Resolves after `ms`, or rejects with the signal's reason once it aborts. */+ const sleep = (ms: number, signal: AbortSignal) => new Promise<void>((resolvePromise, reject) => {+ if (signal.aborted) return reject(signal.reason);+ const timer = setTimeout(resolvePromise, ms);+ signal.addEventListener("abort", () => {+ clearTimeout(timer);+ reject(signal.reason);+ }, { once: true });+ });++ type Attempt = { authorization: string | null; signal: AbortSignal };++ /** The gateway accepts `access-2` only; `answer` decides how each collect attempt goes. */+ function collectBroker(attempts: Attempt[], refreshCalls: string[], answer: (attempt: Attempt) => Promise<Response>) {+ return new OmniRushGatewayBroker({+ credentials: { gatewayUrl, accessToken: "access-1", refreshToken: "refresh-1" },+ engineToken: "local-engine-token",+ collectUploadBudget,+ fetch: async (input, init) => {+ if (String(input).endsWith("/device/refresh")) {+ refreshCalls.push((JSON.parse(String(init?.body)) as { refresh_token: string }).refresh_token);+ return Response.json({ access_token: "access-2", refresh_token: "refresh-2", gateway_url: gatewayUrl });+ }+ const attempt = { authorization: new Headers(init?.headers).get("authorization"), signal: init!.signal! };+ attempts.push(attempt);+ return answer(attempt);+ },+ });+ }++ test("ends at the deadline its body's size sets", async () => {+ const attempts: Attempt[] = [];+ const broker = collectBroker(attempts, [], async ({ signal }) => {+ // Accepted bearer, but the answer never comes.+ await new Promise((_resolvePromise, reject) => signal.addEventListener("abort", () => reject(signal.reason), { once: true }));+ return Response.json({ ok: true });+ });+ const started = performance.now();+ const ended = async (bytes: number) => {+ const error = await broker.collect("session-deadline-1", new Uint8Array(bytes)).catch((reason: unknown) => reason);+ return { name: (error as Error).name, afterMs: performance.now() - started };+ };+ const [small, large] = await Promise.all([ended(64), ended(256 * 1024)]);+ expect(small.name).toBe("TimeoutError");+ expect(large.name).toBe("TimeoutError");+ expect(small.afterMs).toBeGreaterThanOrEqual(401 - 20);+ expect(large.afterMs).toBeGreaterThanOrEqual(650 - 20);+ expect(small.afterMs).toBeLessThan(large.afterMs);+ expect(attempts).toHaveLength(2);+ });++ test("gives the retry with a refreshed bearer a deadline of its own", async () => {+ const attempts: Attempt[] = [];+ const refreshCalls: string[] = [];+ const broker = collectBroker(attempts, refreshCalls, async ({ authorization, signal }) => {+ // Each try takes 250 ms of its 401: together they outlast one shared deadline.+ await sleep(250, signal);+ return authorization === "Bearer access-2"+ ? Response.json({ ok: true }, { status: 201 })+ : Response.json({ detail: "invalid_token" }, { status: 401 });+ });+ const caller = new AbortController();+ const response = await broker.collect("session-deadline-2", new Uint8Array(64), caller.signal);+ expect(response.status).toBe(201);+ expect(refreshCalls).toEqual(["refresh-1"]);+ expect(attempts.map((attempt) => attempt.authorization)).toEqual(["Bearer access-1", "Bearer access-2"]);+ expect(attempts[1]!.signal).not.toBe(attempts[0]!.signal);+ expect(attempts[1]!.signal.aborted).toBe(false);+ });++ test("ends at once when the caller's signal aborts, on the retry with a refreshed bearer too", async () => {+ const attempts: Attempt[] = [];+ const caller = new AbortController();+ const reason = new DOMException("The collector's deadline", "TimeoutError");+ const broker = collectBroker(attempts, [], async ({ authorization, signal }) => {+ if (authorization !== "Bearer access-2") return Response.json({ detail: "invalid_token" }, { status: 401 });+ setTimeout(() => caller.abort(reason), 50);+ await sleep(10_000, signal);+ return Response.json({ ok: true }, { status: 201 });+ });+ const started = performance.now();+ const error = await broker.collect("session-deadline-3", new Uint8Array(64), caller.signal).catch((cause: unknown) => cause);+ expect(error).toBe(reason);+ expect(performance.now() - started).toBeLessThan(401 - 100);+ expect(attempts.map((attempt) => attempt.authorization)).toEqual(["Bearer access-1", "Bearer access-2"]);+ expect(attempts[1]!.signal.reason).toBe(reason);+ });+});diff --git a/apps/server/src/omnirush-gateway-broker.ts b/apps/server/src/omnirush-gateway-broker.tsindex 3de1980a..a970024e 100644--- a/apps/server/src/omnirush-gateway-broker.ts+++ b/apps/server/src/omnirush-gateway-broker.ts@@ -1,6 +1,6 @@import { timingSafeEqual } from "node:crypto";-import { collectUploadTimeoutMs } from "./collect-upload-budget.js";+import { COLLECT_UPLOAD_BUDGET, collectUploadTimeoutMs, type CollectUploadBudget } from "./collect-upload-budget.js";import { externalFetch } from "./server-fetch.js";import type { OmniRushGatewayCredentialBundle, OmniRushGatewayCredentials } from "./types.js";@@ -9,6 +9,8 @@ type BrokerOptions = {engineToken?: string;fetch?: typeof externalFetch;log?: (level: "info" | "warn" | "error", message: string, attributes?: Record<string, unknown>) => void;+ /** collect()'s deadline parameters; COLLECT_UPLOAD_BUDGET unless a test shrinks it. */+ collectUploadBudget?: CollectUploadBudget;};/**@@ -515,6 +517,7 @@ export class OmniRushGatewayBroker {private readonly latest?: OmniRushGatewayCredentials["latest"];private readonly fetcher: typeof externalFetch;private readonly log?: BrokerOptions["log"];+ private readonly collectUploadBudget: CollectUploadBudget;private refreshInFlight: Promise<boolean> | null = null;/*** The pair this broker held before it adopted one from the store. Spent as@@ -539,6 +542,7 @@ export class OmniRushGatewayBroker {this.latest = options.credentials?.latest;this.fetcher = options.fetch ?? externalFetch;this.log = options.log;+ this.collectUploadBudget = options.collectUploadBudget ?? COLLECT_UPLOAD_BUDGET;}get enabled(): boolean {@@ -601,11 +605,12 @@ export class OmniRushGatewayBroker {/*** One collector envelope. The deadline grows with the envelope's size- * (collect-upload-budget.ts); `signal`, the collector's own deadline or- * cancel, ends it sooner.+ * (collect-upload-budget.ts), and the retry with a refreshed bearer gets a+ * deadline of its own; `signal`, the collector's own deadline or cancel,+ * ends either sooner.*/collect(sessionId: string, body: Uint8Array, signal?: AbortSignal): Promise<Response> {- const timeoutMs = collectUploadTimeoutMs(body.byteLength);+ const timeoutMs = collectUploadTimeoutMs(body.byteLength, this.collectUploadBudget);return this.withDeviceBearer((state) => this.fetcher(apiUrl(state.gatewayUrl, "collect"), {method: "POST",headers: {diff --git a/apps/server/src/workspace-collector.test.ts b/apps/server/src/workspace-collector.test.tsindex 3913c086..2f1cbe3d 100644--- a/apps/server/src/workspace-collector.test.ts+++ b/apps/server/src/workspace-collector.test.ts@@ -994,6 +994,124 @@ describe("workspace collector upload deadline and spool drain", () => {await collector.clearSpool();await collector.stop();});++ test("spools a live upload once an attempt runs out its deadline, and still retries quick failures in band", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-deadline-live-"));+ roots.push(root);+ const stateDir = await drainState();+ await writeFile(join(root, "app.txt"), "v0\n");+ // What the gateway does with each call, in order; "drop" once the plan runs out.+ const plan: Array<"hang" | "drop" | "timeout"> = [];+ const calls: string[] = [];+ const spooled: unknown[] = [];+ const collector = new WorkspaceCollector({+ stateDir,+ upload: async () => {+ const step = plan.shift() ?? "drop";+ calls.push(step);+ // No answer ever, and the hook ignores its signal: only the deadline ends the attempt.+ if (step === "hang") return new Promise<Response>(() => undefined);+ // The gateway broker's own deadline, reported by name across the worker boundary.+ if (step === "timeout") throw new DOMException("The operation timed out.", "TimeoutError");+ throw new Error("network down");+ },+ log: (level, message, attributes) => {+ if (level === "warn" && message === "OmniRush collection artifact spooled for retry") spooled.push(attributes?.reason);+ },+ uploadBudget: { baseMs: 50, bytesPerSecond: 128 * 1024 * 1000, maxSendMs: 900, responseMs: 100 },+ fallbackScanMs: 60_000,+ changeDebounceMs: 60_000,+ uploadRetryDelayMs: 1,+ retryBaseMs: 60_000,+ });+ const sessionId = "session-deadline-live-1";++ // The start snapshot's first attempt runs out its deadline: spooled, not tried twice more in band.+ plan.push("hang");+ collector.startSession(sessionId, "workspace-deadline-live", root);+ await collector.idle(sessionId);+ expect(calls).toEqual(["hang"]);+ expect(spooled).toHaveLength(1);+ expect(await collector.spoolStatus()).toMatchObject({ entries: 1 });++ // The session's queue moved on to its next snapshot, whose quick failures keep their in-band retries.+ plan.push("drop", "drop", "drop");+ await writeFile(join(root, "app.txt"), "v1\n");+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(calls).toEqual(["hang", "drop", "drop", "drop"]);+ expect(await collector.spoolStatus()).toMatchObject({ entries: 2 });++ // A quick failure, then an attempt out of time: no third attempt.+ plan.push("drop", "timeout");+ await writeFile(join(root, "app.txt"), "v2\n");+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(calls).toEqual(["hang", "drop", "drop", "drop", "drop", "timeout"]);+ expect(spooled.slice(1)).toEqual(["network down", "The operation timed out."]);+ expect(await collector.spoolStatus()).toMatchObject({ entries: 3 });+ await collector.clearSpool();+ await collector.stop();+ });++ test("a stop mid-drain leaves the spooled entry in flight as it was", async () => {+ const stateDir = await drainState();+ const id = await spoolEntry(stateDir, 1, 1_024, { attempts: 3, last_attempt_at: new Date(Date.now() - 60 * 60_000).toISOString() });+ const before = await spoolMeta(stateDir, id);+ let started!: () => void;+ const inFlight = new Promise<void>((resolvePromise) => { started = resolvePromise; });+ const collector = new WorkspaceCollector({+ stateDir,+ // Never answers and ignores its signal: the stop ends it all the same.+ upload: () => {+ started();+ return new Promise<Response>(() => undefined);+ },+ retryBaseMs: 60_000,+ });+ const drain = collector.drainSpool();+ await inFlight;+ await collector.stop();+ expect(await drain).toEqual({ delivered: 0, pending: 1 });+ expect(await spoolMeta(stateDir, id)).toEqual(before);+ expect(before).toMatchObject({ attempts: 3, last_attempt_at: expect.any(String) });+ expect(await collector.spoolStatus()).toEqual({ entries: 1, bytes: 1_024 });+ });++ test("a drain asked for while one runs picks up the entries spooled meanwhile", async () => {+ const stateDir = await drainState();+ await spoolEntry(stateDir, 1, 512);+ let release!: () => void;+ const held = new Promise<void>((resolvePromise) => { release = resolvePromise; });+ let started!: () => void;+ const inFlight = new Promise<void>((resolvePromise) => { started = resolvePromise; });+ const delivered: string[] = [];+ const collector = new WorkspaceCollector({+ stateDir,+ upload: async (sessionId) => {+ if (sessionId === "session-drain-1") {+ started();+ await held;+ }+ delivered.push(sessionId);+ return Response.json({ ok: true }, { status: 201 });+ },+ retryBaseMs: 60_000,+ });+ const running = collector.drainSpool();+ await inFlight;+ // Spooled after the running drain listed the spool.+ await spoolEntry(stateDir, 2, 512);+ const next = collector.drainSpool();+ // Every call during the run shares the one drain after it.+ expect(collector.drainSpool()).toBe(next);+ release();+ expect(await running).toEqual({ delivered: 1, pending: 0 });+ expect(await next).toEqual({ delivered: 1, pending: 0 });+ expect(delivered).toEqual(["session-drain-1", "session-drain-2"]);+ expect(await collector.spoolStatus()).toEqual({ entries: 0, bytes: 0 });+ await collector.stop();+ });});describe("workspace collector git helpers", () => {diff --git a/apps/server/src/workspace-collector.ts b/apps/server/src/workspace-collector.tsindex 27f8f106..6f1a94f9 100644--- a/apps/server/src/workspace-collector.ts+++ b/apps/server/src/workspace-collector.ts@@ -2948,6 +2948,8 @@ export class WorkspaceCollector {* upload carried is not spooled.*/private account = new AbortController();+ /** Aborted once by abortUploads(): a shutdown out of time ends every upload in flight and sends nothing more. */+ private readonly halted = new AbortController();private stopped = false;private readonly minChangeIntervalMs: number;private readonly maxWatchedFiles: number;@@ -3954,6 +3956,18 @@ export class WorkspaceCollector {if (!this.ledgerPath) await rm(this.tempDir, { recursive: true, force: true }).catch(() => undefined);}+ /**+ * A shutdown that ran out of time and has no worker thread to terminate+ * (capture-client.ts, capture in-process): every upload in flight, live or+ * from the spool, ends now, and any envelope still to go out is spooled+ * without being sent, all of it for the next start. stop() still settles+ * the sessions; this only keeps their uploads from outliving it by up to+ * an upload deadline.+ */+ abortUploads(): void {+ this.halted.abort(new DOMException("The capture is shutting down", "AbortError"));+ }+private scheduleChange(state: SessionState, trigger: Extract<ChangeTrigger, "fs_change" | "periodic">): void {if (state.finished) return;if (!state.turnInProgress && this.turnOnRootElsewhere(state)) state.changesHeld = true;@@ -4385,13 +4399,12 @@ export class WorkspaceCollector {}/**- * One POST of an envelope. Its deadline grows with the envelope's size- * (collect-upload-budget.ts: fetch reports no upload progress, so sending- * the body and the gateway's answer share one size-scaled deadline);- * `cancel` ends it sooner.+ * One POST of an envelope, ended by `deadline` (the attempt's size-scaled+ * deadline, collect-upload-budget.ts: fetch reports no upload progress, so+ * sending the body and the gateway's answer share one deadline) or sooner+ * by `cancel`.*/- private send(sessionId: string, compressed: Uint8Array, cancel?: AbortSignal): Promise<Response> {- const deadline = AbortSignal.timeout(collectUploadTimeoutMs(compressed.byteLength, this.uploadBudget));+ private send(sessionId: string, compressed: Uint8Array, deadline: AbortSignal, cancel?: AbortSignal): Promise<Response> {const signal = cancel ? AbortSignal.any([deadline, cancel]) : deadline;if (signal.aborted) return Promise.reject(signal.reason);if (this.uploader) return untilAborted(this.uploader(sessionId, compressed, signal), signal);@@ -4420,13 +4433,23 @@ export class WorkspaceCollector {}}+ /**+ * Sends an envelope, up to `attempts` times. A quick failure (a dropped+ * connection, a 5xx) is tried again after a short delay; an attempt that+ * ran out its deadline is not: at a size-scaled deadline one more attempt+ * could hold the session's queue for many minutes, so the envelope goes to+ * the spool at once and the spool drain retries it on its own backoff.+ */private async transmit(sessionId: string, compressed: Uint8Array, attempts = UPLOAD_ATTEMPTS, cancel?: AbortSignal): Promise<TransmitOutcome> {+ const timeoutMs = collectUploadTimeoutMs(compressed.byteLength, this.uploadBudget);let lastReason = "collector upload unavailable";let refreshAttempted = false;let attempt = 0;while (attempt < attempts) {+ // Each attempt, the retry with a refreshed bearer too, gets a deadline of its own.+ const deadline = AbortSignal.timeout(timeoutMs);try {- const response = await this.send(sessionId, compressed, cancel);+ const response = await this.send(sessionId, compressed, deadline, cancel);if (response.ok) return { ok: true };lastReason = `collector upload failed with status ${response.status}`;if (UNAUTHORIZED_STATUSES.has(response.status)) {@@ -4455,6 +4478,11 @@ export class WorkspaceCollector {} catch (error) {lastReason = error instanceof Error ? error.message : "collector upload unavailable";if (cancel?.aborted) return { ok: false, retryable: true, reason: lastReason };+ // Out of time: this attempt's deadline, or the gateway broker's own+ // (the same size-scaled budget, reported across the worker boundary by name).+ if (deadline.aborted || (error instanceof Error && error.name === "TimeoutError")) {+ return { ok: false, retryable: true, reason: lastReason };+ }}attempt += 1;if (attempt < attempts && !cancel?.aborted) {@@ -4557,7 +4585,7 @@ export class WorkspaceCollector {}const compressed = await readFile(path);const account = this.account.signal;- const outcome = await this.transmit(state.id, compressed, UPLOAD_ATTEMPTS, account);+ const outcome = await this.transmit(state.id, compressed, UPLOAD_ATTEMPTS, AbortSignal.any([account, this.halted.signal]));if (!outcome.ok) {if ((!outcome.retryable || !this.spoolDir) && !account.aborted) {await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);@@ -4745,7 +4773,7 @@ export class WorkspaceCollector {return this.drainQueued;}const controller = new AbortController();- const run = this.runDrain(controller.signal).finally(() => {+ const run = this.runDrain(AbortSignal.any([controller.signal, this.halted.signal])).finally(() => {if (this.draining?.controller === controller) this.draining = null;});this.draining = { run, controller };diff --git a/docs/workspace-collector-privacy.md b/docs/workspace-collector-privacy.mdindex 1c32b26b..35409884 100644--- a/docs/workspace-collector-privacy.md+++ b/docs/workspace-collector-privacy.md@@ -17,7 +17,7 @@ Neither field changes what may leave the machine: `files[]`, `changed_paths` andThere is no per-session storage limit: a session keeps uploading snapshots and traces for as long as it runs, however much it has already sent, and the privacy block carries no `max_session_bytes`. Every remaining limit applies to one file or one request and matches what the backend accepts per request: 4 MiB per file, 64 MiB per snapshot envelope (uncompressed, and again once compressed), 16 MiB per trace document, 2 MiB of git diff and 50,000 manifest entries. These bound the collector's memory and never stop a session: a snapshot that would not fit still goes out with the files it has room for (touched and git-changed files first, smallest first within each tier), the full manifest, and the rest counted in `snapshot_cap_omitted_count` and `snapshot_cap_omitted_bytes`. The collector still counts the bytes each session has sent, in its local session ledger, for diagnostics only.-The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. An upload's deadline grows with its envelope: 30 seconds, plus the compressed body at 128 KiB/s (at most 15 minutes for that part), plus 2 minutes for the gateway's answer (`apps/server/src/collect-upload-budget.ts`); fetch reports no upload progress, so the three parts form one deadline for the whole request. Spooled uploads are retried oldest first, each on a backoff of its own (5 seconds after its first retry, doubling up to 5 minutes, dropped after 24 attempts), and one that keeps failing holds back none of the uploads behind it: a retry pass moves past it and ends after three failures, and the gateway orders a session's snapshots by their sequence rather than by arrival. A sign-out cancels every upload in flight, live or from the spool, and an envelope it catches is deleted rather than spooled, so nothing captured for one account waits on disk for the next. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path outside the workspace root (a parent folder, another Windows drive or a network share), and none reached through a symlinked directory, is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once. When several chats are open on the same folder, the edits made while one of them runs a turn are that turn's: only a chat with a turn in progress uploads filesystem- and reconcile-driven change snapshots for them, and a chat without one leaves them to its own next prompt or turn snapshot (or its end snapshot), so one agent's edits are not uploaded again by every chat open on the folder. Each chat's own prompt and turn milestones capture as before, and a chat alone on its folder is unaffected. This concerns the collector's scrubbed snapshots only: the project archive keeps its own copies of the folder for each chat ([project-archive.md](project-archive.md)).+The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. An upload's deadline grows with its envelope: 30 seconds, plus the compressed body at 128 KiB/s (at most 15 minutes for that part), plus 2 minutes for the gateway's answer (`apps/server/src/collect-upload-budget.ts`); fetch reports no upload progress, so the three parts form one deadline for the whole request. An upload that runs out its deadline goes to the spool at once instead of being tried again on the spot, so the chat's later snapshots are not held up behind it; a quick failure (a dropped connection, a 5xx answer) is still tried twice more first. Spooled uploads are retried oldest first, each on a backoff of its own (5 seconds after its first retry, doubling up to 5 minutes, dropped after 24 attempts), and one that keeps failing holds back none of the uploads behind it: a retry pass moves past it and ends after three failures, and the gateway orders a session's snapshots by their sequence rather than by arrival. A sign-out cancels every upload in flight, live or from the spool, and an envelope it catches is deleted rather than spooled, so nothing captured for one account waits on disk for the next. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path outside the workspace root (a parent folder, another Windows drive or a network share), and none reached through a symlinked directory, is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once. When several chats are open on the same folder, the edits made while one of them runs a turn are that turn's: only a chat with a turn in progress uploads filesystem- and reconcile-driven change snapshots for them, and a chat without one leaves them to its own next prompt or turn snapshot (or its end snapshot), so one agent's edits are not uploaded again by every chat open on the folder. Each chat's own prompt and turn milestones capture as before, and a chat alone on its folder is unaffected. This concerns the collector's scrubbed snapshots only: the project archive keeps its own copies of the folder for each chat ([project-archive.md](project-archive.md)).## Per-turn diffs: the `turn.diff` trace event