Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-10089 Resume stalled agent downloads instead of timing out (#7961)

Andrew Mikofalvyb454e8b550e3
diff --git a/.changeset/resumable-agent-downloads.md b/.changeset/resumable-agent-downloads.md
new file mode 100644
index 000000000..8a36d70a6
--- /dev/null
+++ b/.changeset/resumable-agent-downloads.md
@@ -0,0 +1,6 @@
+---
+'@inkeep/open-knowledge': patch
+'@inkeep/open-knowledge-server': patch
+---
+
+Agent installs such as OpenCode no longer fail on slower connections after two minutes. Downloads resume after a dropped or stalled connection, and give up only when repeated attempts make no progress.
diff --git a/packages/server/src/acp/archive.test.ts b/packages/server/src/acp/archive.test.ts
new file mode 100644
index 000000000..f25650b63
--- /dev/null
+++ b/packages/server/src/acp/archive.test.ts
@@ -0,0 +1,697 @@
+import { createHash } from 'node:crypto';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import { downloadToFileWithSha } from './archive.ts';
+
+const URL_UNDER_TEST = 'https://example.com/agent.zip';
+
+let dirs: string[] = [];
+afterEach(() => {
+ for (const d of dirs) rmSync(d, { recursive: true, force: true });
+ dirs = [];
+});
+
+function destination(): string {
+ const dir = mkdtempSync(join(tmpdir(), 'ok-archive-download-'));
+ dirs.push(dir);
+ return join(dir, 'archive.zip');
+}
+
+function payloadOf(size: number): Uint8Array {
+ const bytes = new Uint8Array(size);
+ for (let i = 0; i < size; i++) bytes[i] = (i * 31 + 7) % 251;
+ return bytes;
+}
+
+function sha256(bytes: Uint8Array): string {
+ return createHash('sha256').update(bytes).digest('hex');
+}
+
+type StreamEnd = 'close' | 'error' | 'hang';
+
+function streamOf(
+ bytes: Uint8Array,
+ chunkSize: number,
+ end: StreamEnd,
+ failure: Error = new Error('socket hang up'),
+): ReadableStream<Uint8Array> {
+ let offset = 0;
+ return new ReadableStream<Uint8Array>({
+ pull(controller) {
+ if (offset < bytes.length) {
+ controller.enqueue(bytes.slice(offset, offset + chunkSize));
+ offset += chunkSize;
+ return;
+ }
+ if (end === 'close') controller.close();
+ else if (end === 'error') controller.error(failure);
+ else return new Promise<void>(() => {});
+ },
+ });
+}
+
+interface RecordedRequest {
+ range: string | null;
+ ifRange: string | null;
+}
+
+function scriptedFetch(responses: Array<(request: RecordedRequest) => Response>): {
+ fetchImpl: typeof fetch;
+ requests: RecordedRequest[];
+} {
+ const requests: RecordedRequest[] = [];
+ const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const headers = new Headers(init?.headers);
+ const request = { range: headers.get('range'), ifRange: headers.get('if-range') };
+ requests.push(request);
+ const respond = responses[Math.min(requests.length, responses.length) - 1];
+ if (respond === undefined) throw new Error('no scripted response');
+ return respond(request);
+ }) as typeof fetch;
+ return { fetchImpl, requests };
+}
+
+function stillRunningAfter(ms: number): Promise<string> {
+ return new Promise((resolve) => setTimeout(() => resolve('still running'), ms).unref());
+}
+
+describe('downloadToFileWithSha', () => {
+ test('resumes from the received offset when the connection drops mid-transfer', async () => {
+ const payload = payloadOf(4_096);
+ const half = 2_048;
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, half), 512, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(payload.slice(half), 512, 'close'), {
+ status: 206,
+ headers: {
+ 'content-length': String(payload.length - half),
+ 'content-range': `bytes ${half}-${payload.length - 1}/${payload.length}`,
+ },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ });
+
+ expect(sha).toBe(sha256(payload));
+ expect(new Uint8Array(readFileSync(dest))).toEqual(payload);
+ expect(requests).toEqual([
+ { range: null, ifRange: null },
+ { range: `bytes=${half}-`, ifRange: '"v1"' },
+ ]);
+ });
+
+ test('restarts from the beginning when the server ignores the range request', async () => {
+ const payload = payloadOf(3_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(payload, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ });
+
+ expect(sha).toBe(sha256(payload));
+ expect(new Uint8Array(readFileSync(dest))).toEqual(payload);
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-']);
+ });
+
+ test('retries a body that ends before its declared length', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_500), 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(payload.slice(1_500), 500, 'close'), {
+ status: 206,
+ headers: { 'content-range': `bytes 1500-1999/${payload.length}` },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1500-']);
+ });
+
+ test('fails a transfer that stops sending data instead of waiting on it', async () => {
+ const payload = payloadOf(1_024);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 512), 512, 'hang'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(new Uint8Array(0), 512, 'hang'), {
+ status: 206,
+ headers: { 'content-range': `bytes 512-1023/${payload.length}` },
+ }),
+ ]);
+
+ const download = downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ stallTimeoutMs: 50,
+ maxAttemptsWithoutProgress: 2,
+ retryDelayMs: 0,
+ });
+ const outcome = await Promise.race([
+ download.then(
+ () => 'resolved',
+ (err: Error) => err.message,
+ ),
+ stillRunningAfter(5_000),
+ ]);
+
+ expect(outcome).toMatch(/no data for 0\.05s/);
+ expect(outcome).toMatch(/received 512 B of 1\.0 KB after 3 attempts/);
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=512-', 'bytes=512-']);
+ });
+
+ test('a transfer that keeps delivering data is never cut off by elapsed time', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ let offset = 0;
+ const slowBody = new ReadableStream<Uint8Array>({
+ async pull(controller) {
+ if (offset >= payload.length) {
+ controller.close();
+ return;
+ }
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ controller.enqueue(payload.slice(offset, offset + 100));
+ offset += 100;
+ },
+ });
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(slowBody, {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ stallTimeoutMs: 200,
+ retryDelayMs: 0,
+ });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests).toHaveLength(1);
+ });
+
+ test('does not retry a request the server rejected', async () => {
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () => new Response('not found', { status: 404 }),
+ ]);
+
+ await expect(
+ downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 }),
+ ).rejects.toThrow(/HTTP 404/);
+ expect(requests).toHaveLength(1);
+ });
+
+ test('retries a server error response', async () => {
+ const payload = payloadOf(1_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () => new Response('unavailable', { status: 503 }),
+ () =>
+ new Response(streamOf(payload, 250, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests).toHaveLength(2);
+ });
+
+ test('stops without retrying once the caller aborts', async () => {
+ const payload = payloadOf(1_000);
+ const dest = destination();
+ const caller = new AbortController();
+ const { fetchImpl, requests } = scriptedFetch([
+ () => {
+ caller.abort(new Error('thread closed'));
+ return new Response(streamOf(payload.slice(0, 500), 250, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ });
+ },
+ ]);
+
+ await expect(
+ downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ signal: caller.signal,
+ retryDelayMs: 0,
+ }),
+ ).rejects.toThrow();
+ expect(requests).toHaveLength(1);
+ });
+
+ test('restarts when a resumed response describes a different file', async () => {
+ const payload = payloadOf(2_000);
+ const replaced = payloadOf(2_400).reverse();
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(replaced.slice(1_000), 500, 'close'), {
+ status: 206,
+ headers: { 'content-range': `bytes 1000-2399/${replaced.length}` },
+ }),
+ () =>
+ new Response(streamOf(replaced, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(replaced.length) },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ });
+
+ expect(sha).toBe(sha256(replaced));
+ expect(new Uint8Array(readFileSync(dest))).toEqual(replaced);
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);
+ });
+
+ test('treats a range past the end of a complete body as done', async () => {
+ const payload = payloadOf(1_500);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload, 500, 'error'), { status: 200, headers: { etag: '"v1"' } }),
+ () =>
+ new Response(null, {
+ status: 416,
+ headers: { 'content-range': `bytes */${payload.length}` },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1500-']);
+ });
+
+ test('gives up when restarts never get past the furthest point reached', async () => {
+ const payload = payloadOf(4_000);
+ const dest = destination();
+ const dropAfter = [2_000, 1_500, 1_800, 1_200, 1_900];
+ const { fetchImpl, requests } = scriptedFetch(
+ dropAfter.map(
+ (bytes) => () =>
+ new Response(streamOf(payload.slice(0, bytes), 100, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ }),
+ ),
+ );
+
+ await expect(
+ downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ maxAttemptsWithoutProgress: 3,
+ retryDelayMs: 0,
+ }),
+ ).rejects.toThrow(/after 4 attempts/);
+ expect(requests).toHaveLength(4);
+ });
+
+ test('gives up after repeated server errors', async () => {
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () => new Response('unavailable', { status: 503 }),
+ ]);
+
+ await expect(
+ downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ maxAttemptsWithoutProgress: 3,
+ retryDelayMs: 0,
+ }),
+ ).rejects.toThrow(/HTTP 503 .*received 0 B of an unknown size after 3 attempts/);
+ expect(requests).toHaveLength(3);
+ });
+
+ test('restarts instead of resuming when the server gave no validator', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ }),
+ () =>
+ new Response(streamOf(payload, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests).toEqual([
+ { range: null, ifRange: null },
+ { range: null, ifRange: null },
+ ]);
+ });
+
+ test('never resumes from a weak entity tag', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: {
+ 'content-length': String(payload.length),
+ etag: 'W/"v1"',
+ 'last-modified': 'Tue, 27 Oct 2026 09:04:42 GMT',
+ },
+ }),
+ () =>
+ new Response(streamOf(payload, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length) },
+ }),
+ ]);
+
+ await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(requests.map((r) => r.range)).toEqual([null, null]);
+ });
+
+ test('uses Last-Modified as the resume validator when there is no entity tag', async () => {
+ const payload = payloadOf(2_000);
+ const lastModified = 'Tue, 27 Oct 2026 09:04:42 GMT';
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), 'last-modified': lastModified },
+ }),
+ () =>
+ new Response(streamOf(payload.slice(1_000), 500, 'close'), {
+ status: 206,
+ headers: { 'content-range': `bytes 1000-1999/${payload.length}` },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests[1]).toEqual({ range: 'bytes=1000-', ifRange: lastModified });
+ });
+
+ test('restarts when a resumed response carries a different entity tag', async () => {
+ const payload = payloadOf(2_000);
+ const replaced = payloadOf(2_000).reverse();
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(replaced.slice(1_000), 500, 'close'), {
+ status: 206,
+ headers: { 'content-range': `bytes 1000-1999/${replaced.length}`, etag: '"v2"' },
+ }),
+ () =>
+ new Response(streamOf(replaced, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(replaced.length), etag: '"v2"' },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(sha).toBe(sha256(replaced));
+ expect(new Uint8Array(readFileSync(dest))).toEqual(replaced);
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);
+ });
+
+ test('restarts when a range is unsatisfiable for a file of a different size', async () => {
+ const payload = payloadOf(2_000);
+ const smaller = payloadOf(800);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(null, {
+ status: 416,
+ headers: { 'content-range': `bytes */${smaller.length}` },
+ }),
+ () =>
+ new Response(streamOf(smaller, 400, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(smaller.length), etag: '"v2"' },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(sha).toBe(sha256(smaller));
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);
+ });
+
+ test('never resumes a content-encoded response', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-encoding': 'gzip', etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(payload, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ ]);
+
+ await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(requests.map((r) => r.range)).toEqual([null, null]);
+ });
+
+ test('reports the furthest progress after a restart that fails before any data', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(null, {
+ status: 416,
+ headers: { 'content-range': 'bytes */1500' },
+ }),
+ () => new Response('unavailable', { status: 503 }),
+ ]);
+ const retries: Array<{
+ receivedBytes: number;
+ totalBytes: number | null;
+ resumeFromBytes: number;
+ }> = [];
+
+ await expect(
+ downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ maxAttemptsWithoutProgress: 3,
+ retryDelayMs: 0,
+ onRetry: ({ receivedBytes, totalBytes, resumeFromBytes }) =>
+ retries.push({ receivedBytes, totalBytes, resumeFromBytes }),
+ }),
+ ).rejects.toThrow(/HTTP 503 .*received 1000 B of 2\.0 KB after 4 attempts/);
+ expect(retries).toEqual([
+ { receivedBytes: 1_000, totalBytes: 2_000, resumeFromBytes: 1_000 },
+ { receivedBytes: 1_000, totalBytes: 2_000, resumeFromBytes: 0 },
+ { receivedBytes: 1_000, totalBytes: 2_000, resumeFromBytes: 0 },
+ ]);
+ });
+
+ test('reports the offset the next attempt resumes from', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(streamOf(payload.slice(1_000), 500, 'close'), {
+ status: 206,
+ headers: { 'content-range': `bytes 1000-1999/${payload.length}`, etag: '"v1"' },
+ }),
+ ]);
+ const resumeOffsets: number[] = [];
+
+ await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ onRetry: ({ resumeFromBytes }) => resumeOffsets.push(resumeFromBytes),
+ });
+
+ expect(resumeOffsets).toEqual([1_000]);
+ });
+
+ test('never resumes from an entity tag that is not a quoted strong tag', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: {
+ 'content-length': String(payload.length),
+ etag: 'abc123',
+ 'last-modified': 'Tue, 27 Oct 2026 18:00:00 GMT',
+ },
+ }),
+ () =>
+ new Response(streamOf(payload, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: 'abc123' },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests).toEqual([
+ { range: null, ifRange: null },
+ { range: null, ifRange: null },
+ ]);
+ });
+
+ test('restarts when a range is unsatisfiable at a length short of the declared size', async () => {
+ const payload = payloadOf(2_000);
+ const dest = destination();
+ const { fetchImpl, requests } = scriptedFetch([
+ () =>
+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ () =>
+ new Response(null, {
+ status: 416,
+ headers: { 'content-range': 'bytes */1000' },
+ }),
+ () =>
+ new Response(streamOf(payload, 500, 'close'), {
+ status: 200,
+ headers: { 'content-length': String(payload.length), etag: '"v1"' },
+ }),
+ ]);
+
+ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });
+
+ expect(sha).toBe(sha256(payload));
+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);
+ });
+
+ test('names the network error code behind a dropped connection', async () => {
+ const dest = destination();
+ const socketError = Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' });
+ const { fetchImpl } = scriptedFetch([
+ () =>
+ new Response(
+ streamOf(
+ new Uint8Array(0),
+ 500,
+ 'error',
+ new TypeError('terminated', { cause: socketError }),
+ ),
+ { status: 200 },
+ ),
+ ]);
+
+ await expect(
+ downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ maxAttemptsWithoutProgress: 1,
+ retryDelayMs: 0,
+ }),
+ ).rejects.toThrow(/^terminated \(UND_ERR_SOCKET\) \(received 0 B/);
+ });
+
+ test('names a local write failure code once', async () => {
+ const dest = join(destination(), 'missing', 'archive.zip');
+ const { fetchImpl } = scriptedFetch([
+ () => new Response(streamOf(payloadOf(100), 100, 'close'), { status: 200 }),
+ ]);
+
+ const message = await downloadToFileWithSha(URL_UNDER_TEST, dest, {
+ fetchImpl,
+ retryDelayMs: 0,
+ }).then(
+ () => 'resolved',
+ (err: Error) => err.message,
+ );
+
+ expect(message).toMatch(/could not write/);
+ expect(message.match(/ENOENT/g)).toHaveLength(1);
+ });
+});
diff --git a/packages/server/src/acp/archive.ts b/packages/server/src/acp/archive.ts
index a74f82339..94a80f0ca 100644
--- a/packages/server/src/acp/archive.ts
+++ b/packages/server/src/acp/archive.ts
@@ -1,8 +1,8 @@
import { type ChildProcess, spawn } from 'node:child_process';
-import { createHash } from 'node:crypto';
-import { createWriteStream } from 'node:fs';
+import { createHash, type Hash } from 'node:crypto';
+import { open } from 'node:fs/promises';
import { normalize, sep } from 'node:path';
-import { Readable } from 'node:stream';
+import { setTimeout as sleep } from 'node:timers/promises';
export function isWithin(root: string, candidate: string): boolean {
const normalizedRoot = normalize(root) + sep;
@@ -64,7 +64,32 @@ export interface DownloadProgress {
export interface DownloadOptions {
signal?: AbortSignal;
onProgress?: (p: DownloadProgress) => void;
+ onRetry?: (
+ r: DownloadProgress & { attempt: number; resumeFromBytes: number; error: Error },
+ ) => void;
fetchImpl?: typeof fetch;
+ stallTimeoutMs?: number;
+ maxAttemptsWithoutProgress?: number;
+ retryDelayMs?: number;
+}
+
+const DEFAULT_STALL_TIMEOUT_MS = 30_000;
+const DEFAULT_ATTEMPTS_WITHOUT_PROGRESS = 3;
+const DEFAULT_RETRY_DELAY_MS = 1_000;
+const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
+
+class FatalDownloadError extends Error {}
+
+interface ResumeValidator {
+ header: 'etag' | 'last-modified';
+ value: string;
+}
+
+interface Transfer {
+ hash: Hash;
+ receivedBytes: number;
+ totalBytes: number | null;
+ validator: ResumeValidator | null;
}
export async function downloadToFileWithSha(
@@ -72,35 +97,224 @@ export async function downloadToFileWithSha(
destPath: string,
opts: DownloadOptions = {},
): Promise<string> {
+ const attemptLimit = opts.maxAttemptsWithoutProgress ?? DEFAULT_ATTEMPTS_WITHOUT_PROGRESS;
+ const retryDelayMs = opts.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+ const transfer: Transfer = {
+ hash: createHash('sha256'),
+ receivedBytes: 0,
+ totalBytes: null,
+ validator: null,
+ };
+ let attempt = 0;
+ let attemptsWithoutProgress = 0;
+ let furthestBytes = 0;
+ let knownTotalBytes: number | null = null;
+ for (;;) {
+ attempt += 1;
+ try {
+ await downloadAttempt(url, destPath, transfer, opts);
+ return transfer.hash.digest('hex');
+ } catch (err) {
+ const error = err instanceof Error ? err : new Error(String(err));
+ attemptsWithoutProgress =
+ transfer.receivedBytes > furthestBytes ? 0 : attemptsWithoutProgress + 1;
+ furthestBytes = Math.max(furthestBytes, transfer.receivedBytes);
+ knownTotalBytes = transfer.totalBytes ?? knownTotalBytes;
+ const progress = { receivedBytes: furthestBytes, totalBytes: knownTotalBytes };
+ if (
+ opts.signal?.aborted === true ||
+ error instanceof FatalDownloadError ||
+ attemptsWithoutProgress >= attemptLimit
+ ) {
+ throw new Error(`${describeError(error)} (${describeProgress(progress, attempt)})`, {
+ cause: error,
+ });
+ }
+ opts.onRetry?.({
+ attempt,
+ ...progress,
+ resumeFromBytes: transfer.validator === null ? 0 : transfer.receivedBytes,
+ error,
+ });
+ await sleep(retryDelayMs * 2 ** attemptsWithoutProgress, undefined, {
+ signal: opts.signal,
+ });
+ }
+ }
+}
+
+async function downloadAttempt(
+ url: string,
+ destPath: string,
+ transfer: Transfer,
+ opts: DownloadOptions,
+): Promise<void> {
const fetchImpl = opts.fetchImpl ?? fetch;
- const res = await fetchImpl(url, { signal: opts.signal });
+ const stallTimeoutMs = opts.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
+ const resumeFrom = transfer.receivedBytes;
+ if (resumeFrom > 0 && resumeFrom === transfer.totalBytes) return;
+ const validator = resumeFrom > 0 ? transfer.validator : null;
+ const headers: Record<string, string> = { 'accept-encoding': 'identity' };
+ if (validator !== null) {
+ headers.range = `bytes=${resumeFrom}-`;
+ headers['if-range'] = validator.value;
+ }
+ const stall = new AbortController();
+ const signal =
+ opts.signal === undefined ? stall.signal : AbortSignal.any([opts.signal, stall.signal]);
+ const res = await withinStallWindow(fetchImpl(url, { signal, headers }), stallTimeoutMs, stall);
+ if (res.status === 416 && validator !== null) {
+ await res.body?.cancel().catch(() => {});
+ if (transfer.totalBytes === null && unsatisfiedRangeTotal(res) === resumeFrom) return;
+ resetTransfer(transfer, null, null);
+ throw new Error(`download failed: server no longer has the requested byte range for ${url}`);
+ }
if (!res.ok || res.body === null) {
- throw new Error(`download failed: HTTP ${res.status} for ${url}`);
+ await res.body?.cancel().catch(() => {});
+ const message = `download failed: HTTP ${res.status} for ${url}`;
+ throw RETRYABLE_HTTP_STATUSES.has(res.status)
+ ? new Error(message)
+ : new FatalDownloadError(message);
}
- const lenHeader = res.headers.get('content-length');
- const totalBytes = lenHeader !== null && lenHeader !== '' ? Number(lenHeader) : null;
- const hash = createHash('sha256');
- let receivedBytes = 0;
- const sink = createWriteStream(destPath);
- const source = Readable.fromWeb(
- res.body as unknown as import('node:stream/web').ReadableStream<Uint8Array>,
- );
- source.on('data', (chunk: Buffer) => {
- hash.update(chunk);
- receivedBytes += chunk.length;
- opts.onProgress?.({
- receivedBytes,
- totalBytes: Number.isFinite(totalBytes) ? totalBytes : null,
- });
+
+ const range = res.status === 206 ? parseContentRange(res.headers.get('content-range')) : null;
+ const encoded = isEncoded(res);
+ const responseEtag = nonEmpty(res.headers.get('etag'));
+ const resumed =
+ range !== null &&
+ validator !== null &&
+ !encoded &&
+ range.start === resumeFrom &&
+ range.total !== null &&
+ (transfer.totalBytes === null || range.total === transfer.totalBytes) &&
+ (responseEtag === null || validator.header !== 'etag' || responseEtag === validator.value);
+ if (res.status === 206 && !resumed) {
+ await res.body.cancel().catch(() => {});
+ resetTransfer(transfer, null, null);
+ throw new Error(`download failed: server answered with an unexpected byte range for ${url}`);
+ }
+ if (resumed) {
+ transfer.totalBytes = range.total;
+ } else {
+ resetTransfer(
+ transfer,
+ encoded ? null : declaredLength(res.headers.get('content-length')),
+ encoded ? null : resumeValidator(responseEtag, nonEmpty(res.headers.get('last-modified'))),
+ );
+ }
+
+ const sink = await open(destPath, resumed ? 'a' : 'w').catch(async (err: Error) => {
+ await res.body?.cancel().catch(() => {});
+ throw new FatalDownloadError(`could not write ${destPath}: ${err.message}`, { cause: err });
});
- await new Promise<void>((resolvePromise, rejectPromise) => {
- source
- .pipe(sink)
- .on('finish', () => resolvePromise())
- .on('error', rejectPromise);
- source.on('error', rejectPromise);
+ const reader = res.body.getReader();
+ try {
+ for (;;) {
+ const { done, value } = await withinStallWindow(reader.read(), stallTimeoutMs, stall);
+ if (done) break;
+ transfer.hash.update(value);
+ transfer.receivedBytes += value.length;
+ await sink.write(value).catch((err: Error) => {
+ throw new FatalDownloadError(`could not write ${destPath}: ${err.message}`, {
+ cause: err,
+ });
+ });
+ opts.onProgress?.({
+ receivedBytes: transfer.receivedBytes,
+ totalBytes: transfer.totalBytes,
+ });
+ }
+ } catch (err) {
+ await reader.cancel().catch(() => {});
+ throw err;
+ } finally {
+ await sink.close();
+ }
+ if (transfer.totalBytes !== null && transfer.receivedBytes < transfer.totalBytes) {
+ throw new Error(`download ended early for ${url}`);
+ }
+}
+
+async function withinStallWindow<T>(
+ pending: Promise<T>,
+ stallTimeoutMs: number,
+ stall: AbortController,
+): Promise<T> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const expired = new Promise<never>((_, reject) => {
+ timer = setTimeout(() => {
+ const err = new Error(`download stalled: no data for ${stallTimeoutMs / 1000}s`);
+ stall.abort(err);
+ reject(err);
+ }, stallTimeoutMs);
});
- return hash.digest('hex');
+ try {
+ return await Promise.race([pending, expired]);
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+function resetTransfer(
+ transfer: Transfer,
+ totalBytes: number | null,
+ validator: ResumeValidator | null,
+): void {
+ transfer.hash = createHash('sha256');
+ transfer.receivedBytes = 0;
+ transfer.totalBytes = totalBytes;
+ transfer.validator = validator;
+}
+
+function resumeValidator(etag: string | null, lastModified: string | null): ResumeValidator | null {
+ if (etag !== null) return /^"[^"]*"$/.test(etag) ? { header: 'etag', value: etag } : null;
+ return lastModified === null ? null : { header: 'last-modified', value: lastModified };
+}
+
+function isEncoded(res: Response): boolean {
+ const encoding = res.headers.get('content-encoding')?.trim().toLowerCase() ?? '';
+ return encoding !== '' && encoding !== 'identity';
+}
+
+function describeError(error: Error): string {
+ const cause = error.cause;
+ if (!(cause instanceof Error)) return error.message;
+ const code = (cause as NodeJS.ErrnoException).code;
+ const detail = typeof code === 'string' ? code : cause.message;
+ return error.message.includes(detail) ? error.message : `${error.message} (${detail})`;
+}
+
+function nonEmpty(header: string | null): string | null {
+ return header === null || header === '' ? null : header;
+}
+
+function unsatisfiedRangeTotal(res: Response): number | null {
+ const match = /^bytes \*\/(\d+)$/.exec(res.headers.get('content-range')?.trim() ?? '');
+ return match === null ? null : Number(match[1]);
+}
+
+function declaredLength(header: string | null): number | null {
+ if (header === null || header === '') return null;
+ const n = Number(header);
+ return Number.isSafeInteger(n) && n >= 0 ? n : null;
+}
+
+function parseContentRange(header: string | null): { start: number; total: number | null } | null {
+ const match = header === null ? null : /^bytes (\d+)-\d+\/(\d+|\*)$/.exec(header.trim());
+ if (match === null) return null;
+ return { start: Number(match[1]), total: match[2] === '*' ? null : Number(match[2]) };
+}
+
+function describeProgress(progress: DownloadProgress, attempt: number): string {
+ const total = progress.totalBytes === null ? 'an unknown size' : formatBytes(progress.totalBytes);
+ const attempts = attempt === 1 ? '1 attempt' : `${attempt} attempts`;
+ return `received ${formatBytes(progress.receivedBytes)} of ${total} after ${attempts}`;
+}
+
+function formatBytes(bytes: number): string {
+ if (bytes < 1024) return `${bytes} B`;
+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+ return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
export async function fetchText(
diff --git a/packages/server/src/acp/launch.test.ts b/packages/server/src/acp/launch.test.ts
index 4dd01ff1d..780c2081e 100644
--- a/packages/server/src/acp/launch.test.ts
+++ b/packages/server/src/acp/launch.test.ts
@@ -465,6 +465,67 @@ describe('ensureBinaryInstalled', () => {
expect(existsSync(staleDir)).toBe(false);
});
+ test('a second install of the same binary joins the download already running', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const target = { ...artifact.target, sha256: artifact.sha };
+ let downloads = 0;
+ const baseFetch = binaryFetch(artifact.bytes);
+ const countingFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ downloads += 1;
+ return baseFetch(input, init);
+ }) as typeof fetch;
+ const started: string[] = [];
+
+ const installed = await Promise.all([
+ ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: countingFetch,
+ onDownloadStart: () => started.push('first'),
+ }),
+ ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: countingFetch,
+ onDownloadStart: () => started.push('second'),
+ }),
+ ]);
+
+ expect(downloads).toBe(1);
+ expect(installed[0]).toBe(installed[1]);
+ expect(started.sort()).toEqual(['first', 'second']);
+ });
+
+ test('an install that joins after the download started is told it is installing', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const target = { ...artifact.target, sha256: artifact.sha };
+ const { promise: responseGate, resolve: releaseResponse } = Promise.withResolvers<void>();
+ const baseFetch = binaryFetch(artifact.bytes);
+ const gatedFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ await responseGate;
+ return baseFetch(input, init);
+ }) as typeof fetch;
+ const { promise: firstStarted, resolve: markFirstStarted } = Promise.withResolvers<void>();
+ const started: string[] = [];
+
+ const first = ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: gatedFetch,
+ onDownloadStart: () => {
+ started.push('first');
+ markFirstStarted();
+ },
+ });
+ await firstStarted;
+ const second = ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: gatedFetch,
+ onDownloadStart: () => started.push('second'),
+ });
+
+ expect(started).toEqual(['first', 'second']);
+ releaseResponse();
+ expect(await second).toBe(await first);
+ });
+
test('concurrent installers adopt the first completed binary', async () => {
const firstSource = tmp();
const secondSource = tmp();
@@ -1402,7 +1463,7 @@ describe('startup package policy classification', () => {
const generic = 'No solution found when resolving tool dependencies';
expect(packageAcquisitionFailure(uv, generic)).toBeNull();
expect(
- packageAcquisitionFailure(uv, `${generic}\nversions were filtered by ` + '`exclude-newer`'),
+ packageAcquisitionFailure(uv, `${generic}\nversions were filtered by \`exclude-newer\``),
).toMatchObject({ code: 'install-failed' });
});
});
diff --git a/packages/server/src/acp/launch.ts b/packages/server/src/acp/launch.ts
index 2050835f4..f375b96b8 100644
--- a/packages/server/src/acp/launch.ts
+++ b/packages/server/src/acp/launch.ts
@@ -879,13 +879,61 @@ async function writeInstallFailureMarker(
});
}
-export async function ensureBinaryInstalled(
+interface InFlightBinaryInstall {
+ result: Promise<string>;
+ downloading: boolean;
+ downloadStartListeners: Array<() => void>;
+}
+
+const inFlightBinaryInstalls = new Map<string, InFlightBinaryInstall>();
+
+export function ensureBinaryInstalled(
id: string,
version: string,
target: RegistryBinaryTarget,
cacheDir: string,
log: PinoLogger,
opts: EnsureBinaryInstallOptions = {},
+): Promise<string> {
+ const key = JSON.stringify([
+ resolve(cacheDir),
+ id,
+ version,
+ target.archive,
+ target.cmd,
+ target.sha256 ?? null,
+ ]);
+ const running = inFlightBinaryInstalls.get(key);
+ if (running !== undefined) {
+ if (opts.onDownloadStart !== undefined) {
+ if (running.downloading) opts.onDownloadStart();
+ else running.downloadStartListeners.push(opts.onDownloadStart);
+ }
+ return running.result;
+ }
+ const entry: InFlightBinaryInstall = {
+ result: Promise.resolve(''),
+ downloading: false,
+ downloadStartListeners: opts.onDownloadStart === undefined ? [] : [opts.onDownloadStart],
+ };
+ entry.result = installBinary(id, version, target, cacheDir, log, {
+ ...opts,
+ onDownloadStart: () => {
+ entry.downloading = true;
+ for (const notify of entry.downloadStartListeners) notify();
+ },
+ }).finally(() => inFlightBinaryInstalls.delete(key));
+ inFlightBinaryInstalls.set(key, entry);
+ return entry.result;
+}
+
+async function installBinary(
+ id: string,
+ version: string,
+ target: RegistryBinaryTarget,
+ cacheDir: string,
+ log: PinoLogger,
+ opts: EnsureBinaryInstallOptions,
): Promise<string> {
const versionDir = join(cacheDir, sanitizeSegment(id), sanitizeSegment(version));
const markerPath = installFailureMarkerPath(dirname(versionDir), version);
@@ -906,10 +954,19 @@ export async function ensureBinaryInstalled(
log.info({ id, version, archive: target.archive }, '[acp-launch] downloading agent binary');
const isZip = /\.zip$/i.test(new URL(target.archive).pathname);
const archivePath = join(stagingDir, isZip ? 'archive.zip' : 'archive.tar.gz');
+ const downloadStartedAt = Date.now();
const sha = await downloadToFileWithSha(target.archive, archivePath, {
- signal: AbortSignal.timeout(120_000),
fetchImpl: opts.fetchImpl,
+ onRetry: ({ attempt, receivedBytes, totalBytes, resumeFromBytes, error }) =>
+ log.warn(
+ { id, version, attempt, receivedBytes, totalBytes, resumeFromBytes, err: error },
+ '[acp-launch] agent binary download interrupted, retrying',
+ ),
});
+ log.info(
+ { id, version, elapsedMs: Date.now() - downloadStartedAt },
+ '[acp-launch] agent binary downloaded',
+ );
if (target.sha256 !== undefined) {
if (sha !== target.sha256.toLowerCase()) {
throw new Error(`archive checksum mismatch: expected ${target.sha256}, got ${sha}`);
diff --git a/packages/server/src/acp/managed-runtime.test.ts b/packages/server/src/acp/managed-runtime.test.ts
index 99375ddd8..b3db10bda 100644
--- a/packages/server/src/acp/managed-runtime.test.ts
+++ b/packages/server/src/acp/managed-runtime.test.ts
@@ -309,6 +309,33 @@ describe('ensureManagedRuntime', () => {
expect(await findManagedRuntime('node', root)).toBeNull();
});
+ test('stops the archive download when the checksum list cannot be fetched', async () => {
+ const root = tmp();
+ let archiveSignal: AbortSignal | undefined;
+ const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => {
+ if (String(url).includes('SHASUMS256') || String(url).endsWith('.sha256')) {
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ return new Response('missing', { status: 404 });
+ }
+ archiveSignal = init?.signal ?? undefined;
+ const body = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(new Uint8Array(64));
+ init?.signal?.addEventListener('abort', () => controller.error(init.signal?.reason));
+ },
+ });
+ return new Response(body, { status: 200, headers: { 'content-length': '1048576' } });
+ }) as unknown as typeof fetch;
+
+ const error = await ensureManagedRuntime('node', log, { root, fetchImpl }).catch(
+ (err: unknown) => err,
+ );
+
+ expect(error).toBeInstanceOf(RuntimeInstallError);
+ expect((error as Error).message).toMatch(/HTTP 404/);
+ expect(archiveSignal?.aborted).toBe(true);
+ });
+
test('preserves the underlying failure as the error cause', async () => {
const root = tmp();
const failure = new Error('network unreachable');
diff --git a/packages/server/src/acp/managed-runtime.ts b/packages/server/src/acp/managed-runtime.ts
index 64a8e9e9a..601ef21ca 100644
--- a/packages/server/src/acp/managed-runtime.ts
+++ b/packages/server/src/acp/managed-runtime.ts
@@ -274,14 +274,39 @@ export async function ensureManagedRuntime(
{ kind, version: versionOf(kind), url: spec.archiveUrl },
'[managed-runtime] downloading runtime',
);
- const [actualSha, checksums] = await Promise.all([
+ const paired = new AbortController();
+ const signal =
+ opts.signal === undefined ? paired.signal : AbortSignal.any([opts.signal, paired.signal]);
+ const abortPair = (error: unknown): never => {
+ paired.abort(error);
+ throw error;
+ };
+ const [download, checksumFetch] = await Promise.allSettled([
downloadToFileWithSha(spec.archiveUrl, archivePath, {
- signal: opts.signal,
+ signal,
onProgress: opts.onProgress,
+ onRetry: ({ attempt, receivedBytes, totalBytes, resumeFromBytes, error }) =>
+ log.warn(
+ {
+ kind,
+ version: versionOf(kind),
+ attempt,
+ receivedBytes,
+ totalBytes,
+ resumeFromBytes,
+ err: error,
+ },
+ '[managed-runtime] runtime download interrupted, retrying',
+ ),
fetchImpl: opts.fetchImpl,
- }),
- fetchText(spec.checksumUrl, { signal: opts.signal, fetchImpl: opts.fetchImpl }),
+ }).catch(abortPair),
+ fetchText(spec.checksumUrl, { signal, fetchImpl: opts.fetchImpl }).catch(abortPair),
]);
+ if (download.status === 'rejected' || checksumFetch.status === 'rejected') {
+ throw paired.signal.reason;
+ }
+ const actualSha = download.value;
+ const checksums = checksumFetch.value;
const expectedSha = shaForFile(checksums, spec.archiveName);
if (expectedSha === null) {
throw new RuntimeInstallError(`no published checksum for ${spec.archiveName}`);