A desktop coding agent with free access to frontier models.
TypeScript
fix(capture): keep uploads refused with 5xx/507 or Retry-After in the spool and send them at the next start
ogsamratfdddc07a4db5
diff --git a/apps/server/src/session-archive/upload.test.ts b/apps/server/src/session-archive/upload.test.tsindex 2b5b09fe..811d1346 100644--- a/apps/server/src/session-archive/upload.test.ts+++ b/apps/server/src/session-archive/upload.test.ts@@ -201,6 +201,32 @@ describe("archive upload client", () => {expect((await pending).status).toBe("unavailable");});+ test("a full disk (507), another 5xx and a 413 with Retry-After leave the job queued; a plain 413 ends the chain", async () => {+ const cases: Array<{ status: number; headers?: Record<string, string>; expected: "retry_later" | "stop_session"; calls: number }> = [+ { status: 507, expected: "retry_later", calls: 1 },+ { status: 508, expected: "retry_later", calls: 3 },+ { status: 413, headers: { "Retry-After": "60" }, expected: "retry_later", calls: 1 },+ { status: 503, headers: { "Retry-After": "60" }, expected: "retry_later", calls: 1 },+ { status: 413, expected: "stop_session", calls: 1 },+ { status: 501, expected: "retry_later", calls: 1 },+ ];+ for (const { status, headers, expected, calls } of cases) {+ const server = new FakeArchiveServer();+ let creates = 0;+ const fetch: ArchiveUploaderOptions["fetch"] = async (url, init) => {+ if (String(url).endsWith("/archives") && init?.method === "POST") {+ creates += 1;+ return new Response(JSON.stringify({ detail: "insufficient_storage" }), { status, headers: { "Content-Type": "application/json", ...headers } });+ }+ return server.respond(url, init);+ };+ const { job, persist } = await sealedJob();+ const outcome = await uploader(server, { fetch }).upload(job, persist);+ expect({ status, outcome: outcome.status }).toEqual({ status, outcome: expected });+ expect({ status, creates }).toEqual({ status, creates: calls });+ }+ });+test("428 and 503 archive_disabled on create turn archiving off without an error", async () => {for (const gate of [{ status: 428, detail: "archive_consent_required" }, { status: 503, detail: "archive_disabled" }]) {const server = new FakeArchiveServer();diff --git a/apps/server/src/session-archive/upload.ts b/apps/server/src/session-archive/upload.tsindex e5b97204..a49a5295 100644--- a/apps/server/src/session-archive/upload.ts+++ b/apps/server/src/session-archive/upload.ts@@ -66,6 +66,22 @@ const PART_MIN_BYTES_PER_MS = 32;/** A part is read from the sealed file in slices this big, checking for an abort between them. */const PART_READ_SLICE_BYTES = 1024 * 1024;const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);++/**+ * An API answer worth trying again later (the job stays queued, also for the+ * next start): the statuses above, any other 5xx but 501/505 (507: the+ * server's disk is full), and a 413 that carries a Retry-After (load+ * shedding, not a too-large archive). `soon` is false when another attempt+ * within this drain is pointless (507, or the server said when to return).+ */+function retryableApiStatus(response: Response): { retry: boolean; soon: boolean } {+ const retryAfter = response.headers?.get?.("retry-after")?.trim() || null;+ if (RETRYABLE_STATUSES.has(response.status)) return { retry: true, soon: retryAfter === null };+ if (response.status >= 500 && response.status <= 599 && response.status !== 501 && response.status !== 505) {+ return { retry: true, soon: retryAfter === null && response.status !== 507 };+ }+ return { retry: response.status === 413 && retryAfter !== null, soon: false };+}const PART_RETRYABLE_STATUSES = new Set([400, 408, 429, 500, 502, 503, 504]);const MAX_PARTS_PER_REQUEST = 100;const MAX_UPLOAD_ROUNDS = 6;@@ -287,10 +303,13 @@ export class ArchiveUploader {refreshed = true;if (await this.refresh()) continue;}- if (!RETRYABLE_STATUSES.has(response.status) || (response.status === 503 && code === "archive_disabled")) {+ const retryable = retryableApiStatus(response);+ if (!retryable.retry || (response.status === 503 && code === "archive_disabled")) {return { kind: "error", status: response.status, code };}failure = `status ${response.status}`;+ // The job waits in the queue for a later drain (or the next start).+ if (!retryable.soon) return { kind: "unavailable", reason: failure };} catch (error) {if (signal?.aborted) return { kind: "aborted" };failure = error instanceof Error ? error.name : "network error";@@ -323,7 +342,7 @@ export class ArchiveUploader {const response = await this.send("GET", path, undefined, signal, true);if (response.ok) return { kind: "ok", status: response.status, body: await response.json().catch(() => null) };const code = await errorCode(response);- if (!RETRYABLE_STATUSES.has(response.status) || (response.status === 503 && code === "archive_disabled")) return { kind: "error", status: response.status, code };+ if (!retryableApiStatus(response).retry || (response.status === 503 && code === "archive_disabled")) return { kind: "error", status: response.status, code };return { kind: "unavailable", reason: `status ${response.status}` };} catch (error) {if (signal?.aborted) return { kind: "aborted" };diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.tsindex 47624b49..395c8530 100644--- a/apps/server/src/session-uploader.test.ts+++ b/apps/server/src/session-uploader.test.ts@@ -17,7 +17,9 @@ import {MAX_UPLOAD_TRACE_EVENTS,MAX_UPLOAD_WEB_VISIT_TEXT_BYTES,SessionUploader,+ isRetryableUploadStatus,mapBounded,+ retryAfterMs,clampUploadBytes,clampUploadText,collectGitBlock,@@ -811,6 +813,78 @@ describe("session uploader durable retry", () => {await second.stop();});+ test("a 507 on the end upload is spooled after one attempt, and the next start sends it", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-507-"));+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-507-state-"));+ roots.push(root, stateDir);+ await writeFile(join(root, "app.txt"), "hello\n");+ const sessionId = "session-507-1234";+ const sent: string[] = [];+ const first = new SessionUploader({+ stateDir,+ upload: async (_sessionId, compressed) => {+ const envelope = JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Envelope;+ sent.push(String(envelope.snapshot_type));+ // The server's disk filled up before the session ended.+ return envelope.snapshot_type === "end" ? Response.json({ detail: "insufficient storage" }, { status: 507 }) : Response.json({ ok: true }, { status: 201 });+ },+ fallbackScanMs: 60_000,+ uploadRetryDelayMs: 1,+ retryBaseMs: 60_000,+ });+ first.startSession(sessionId, "workspace-507", root);+ await first.idle(sessionId);+ await first.stop();+ expect(sent.filter((type) => type === "end")).toHaveLength(1);+ const spoolDir = join(stateDir, "omnirush-upload-spool");+ const metas = await Promise.all((await readdir(spoolDir)).filter((name) => name.endsWith(".json"))+ .map(async (name) => JSON.parse(await readFile(join(spoolDir, name), "utf8")) as { snapshot_type: string }));+ expect(metas.map((meta) => meta.snapshot_type)).toEqual(["end"]);++ const { uploads, upload } = makeUploads();+ const second = new SessionUploader({ stateDir, upload, fallbackScanMs: 60_000, retryBaseMs: 10, retryMaxMs: 20 });+ for (let attempt = 0; attempt < 200 && uploads.length < 1; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 10));+ expect(uploads.map((item) => [item.snapshot_type, item.trigger])).toEqual([["end", "session_end"]]);+ expect(await second.spoolStatus()).toEqual({ entries: 0, bytes: 0 });+ await second.stop();+ });++ test("a 413 or 429 with Retry-After is spooled and not sent before it; a plain 413 or a 501 is final", async () => {+ expect(retryAfterMs("30", 0)).toBe(30_000);+ expect(retryAfterMs("Thu, 01 Jan 1970 00:01:00 GMT", 0)).toBe(60_000);+ expect(retryAfterMs("later", 0)).toBeNull();+ expect([507, 599, 501, 505].map((status) => isRetryableUploadStatus(status))).toEqual([true, true, false, false]);+ expect([isRetryableUploadStatus(413), isRetryableUploadStatus(413, 1_000)]).toEqual([false, true]);+ for (const [status, headers, spooled] of [[413, { "Retry-After": "3600" }, true], [429, { "Retry-After": "3600" }, true], [413, {}, false], [501, {}, false]] as const) {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-ra-"));+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-ra-state-"));+ roots.push(root, stateDir);+ await writeFile(join(root, "app.txt"), "hello\n");+ let calls = 0;+ let healthy = false;+ const uploader = new SessionUploader({+ stateDir,+ upload: async () => {+ calls += 1;+ return healthy ? Response.json({ ok: true }, { status: 201 }) : Response.json({ detail: "busy" }, { status, headers });+ },+ fallbackScanMs: 60_000,+ uploadRetryDelayMs: 1,+ retryBaseMs: 60_000,+ });+ uploader.startSession("session-ra-1234", "workspace-ra", root);+ await uploader.idle("session-ra-1234");+ expect({ status, calls }).toEqual({ status, calls: 1 });+ expect({ status, entries: (await uploader.spoolStatus()).entries }).toEqual({ status, entries: spooled ? 1 : 0 });+ healthy = true;+ // Not before the Retry-After: the drain leaves it.+ if (spooled) expect(await uploader.drainSpool()).toEqual({ delivered: 0, pending: 1 });+ expect(calls).toBe(1);+ await uploader.clearSpool();+ await uploader.stop();+ }+ });+test("bounds the spool, drops permanently rejected uploads and clears on request", async () => {const root = await mkdtemp(join(tmpdir(), "omnirush-upload-spoolcap-"));const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-spoolcap-state-"));diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex 93dc17df..f5e16bfc 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -143,7 +143,36 @@ const MAX_RESPONSE_BODY_BYTES = 64 * 1024;// next one: enough to get past a stuck entry, few enough not to hammer a link// that is down for every entry in the spool.const MAX_DRAIN_FAILURES = 3;-const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);+const RETRYABLE_STATUSES = new Set([408, 425, 429]);+/** The longest Retry-After honoured: a server asking for more is tried again after this. */+const MAX_RETRY_AFTER_MS = 60 * 60_000;++/** A Retry-After header (delta-seconds or an HTTP date) in ms from now, capped; null when absent or unreadable. */+export function retryAfterMs(value: string | null | undefined, now = Date.now()): number | null {+ const text = value?.trim();+ if (!text) return null;+ let ms: number;+ if (/^\d+(?:\.\d+)?$/.test(text)) ms = Number(text) * 1000;+ else {+ const at = Date.parse(text);+ if (!Number.isFinite(at)) return null;+ ms = at - now;+ }+ return Number.isFinite(ms) ? Math.min(MAX_RETRY_AFTER_MS, Math.max(0, Math.round(ms))) : null;+}++/**+ * Whether an upload refused with `status` may go through later, so the+ * envelope is kept (spooled) rather than dropped: a timeout or rate limit+ * (408, 425, 429), any server-side failure (5xx: a 507 is the server's disk+ * filling up) except 501 and 505, which no retry changes, and a 413 that+ * says when to come back (Retry-After: the server is shedding load).+ */+export function isRetryableUploadStatus(status: number, retryAfter: number | null = null): boolean {+ if (RETRYABLE_STATUSES.has(status)) return true;+ if (status >= 500 && status <= 599) return status !== 501 && status !== 505;+ return status === 413 && retryAfter !== null;+}// A rejected bearer. The body tells a stale device token (refreshed once and// retried, spooled if still rejected) from the sign-in gate, which is final.const UNAUTHORIZED_STATUSES = new Set([401, 403]);@@ -402,6 +431,8 @@ type SpoolMeta = {created_at: string;attempts: number;last_attempt_at?: string;+ /** Not sent again before this (a Retry-After the server gave). */+ not_before?: string;};type SessionState = {@@ -3916,6 +3947,7 @@ function parseSpoolMeta(value: unknown): SpoolMeta | null {created_at: record.created_at,attempts: optionalCount(record.attempts) ?? 0,...(optionalString(record.last_attempt_at) ? { last_attempt_at: record.last_attempt_at } : {}),+ ...(optionalString(record.not_before) ? { not_before: record.not_before } : {}),};}@@ -5866,7 +5898,14 @@ export class SessionUploader {return { ok: false, retryable: false, reason: lastReason, unsupportedSchema: true };}await response.body?.cancel().catch(() => undefined);- if (!RETRYABLE_STATUSES.has(response.status)) return { ok: false, retryable: false, reason: lastReason };+ const retryAfter = retryAfterMs(response.headers?.get?.("retry-after"));+ if (!isRetryableUploadStatus(response.status, retryAfter)) return { ok: false, retryable: false, reason: lastReason };+ // A full disk (507) or a server that said when to come back is not+ // asked again within the second: the spool sends it after its backoff+ // (and not before the Retry-After), now or at the next start.+ if (retryAfter !== null || response.status === 507 || response.status === 413) {+ return { ok: false, retryable: true, reason: lastReason, ...(retryAfter !== null ? { retryAfterMs: retryAfter } : {}) };+ }} catch (error) {lastReason = error instanceof Error ? error.message : "session upload unavailable";if (cancel?.aborted) return { ok: false, retryable: true, reason: lastReason };@@ -5993,8 +6032,11 @@ export class SessionUploader {const compressed: CompressedArtifact = { path, size: compressedBytes };const account = this.account.signal;const uploadSignal = AbortSignal.any([account, this.halted.signal]);- const spoolArtifact = async (reason: string): Promise<boolean> => {- const spooled = await this.spoolEnvelopeFile({ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressedBytes, created_at: new Date().toISOString(), attempts: 1 }, path, account);+ const spoolArtifact = async (reason: string, retryAfter?: number): Promise<boolean> => {+ const spooled = await this.spoolEnvelopeFile({+ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressedBytes, created_at: new Date().toISOString(), attempts: 1,+ ...(retryAfter !== undefined ? { not_before: new Date(Date.now() + retryAfter).toISOString() } : {}),+ }, path, account);if (!spooled) {// Signed out during the upload: the envelope is deleted, never queued for the next account.this.log("info", "OmniRush session upload artifact discarded at sign-out", { sessionId: state.id, snapshotType, trigger, sequence });@@ -6177,11 +6219,15 @@ export class SessionUploader {* spooled it).*/private spoolEntryDueAt(entry: SpoolMeta): number {+ const now = Date.now();+ // Never before a Retry-After the server gave (unless the clock went back past it by more than the cap).+ const notBefore = entry.not_before ? Date.parse(entry.not_before) : Number.NaN;+ const floor = Number.isFinite(notBefore) && notBefore - now <= MAX_RETRY_AFTER_MS ? notBefore : 0;const last = entry.last_attempt_at ? Date.parse(entry.last_attempt_at) : Number.NaN;- if (!Number.isFinite(last)) return 0;+ if (!Number.isFinite(last)) return floor;// A last attempt after now means the clock went back: due at once, and// the attempt then records a sane time.- return last > Date.now() ? 0 : last + this.backoffMs(entry.attempts - 1);+ return Math.max(floor, last > now ? 0 : last + this.backoffMs(entry.attempts - 1));}/**