Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
PRD-10089 Resume stalled agent downloads instead of timing out (#7961)
Andrew Mikofalvyb454e8b550e3
diff --git a/.changeset/resumable-agent-downloads.md b/.changeset/resumable-agent-downloads.mdnew file mode 100644index 000000000..8a36d70a6--- /dev/null+++ b/.changeset/resumable-agent-downloads.md@@ -0,0 +1,6 @@+---+'@inkeep/open-knowledge': patch+'@inkeep/open-knowledge-server': patch+---++Agent installs such as OpenCode no longer fail on slower connections after two minutes. Downloads resume after a dropped or stalled connection, and give up only when repeated attempts make no progress.diff --git a/packages/server/src/acp/archive.test.ts b/packages/server/src/acp/archive.test.tsnew file mode 100644index 000000000..f25650b63--- /dev/null+++ b/packages/server/src/acp/archive.test.ts@@ -0,0 +1,697 @@+import { createHash } from 'node:crypto';+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, describe, expect, test } from 'vitest';+import { downloadToFileWithSha } from './archive.ts';++const URL_UNDER_TEST = 'https://example.com/agent.zip';++let dirs: string[] = [];+afterEach(() => {+ for (const d of dirs) rmSync(d, { recursive: true, force: true });+ dirs = [];+});++function destination(): string {+ const dir = mkdtempSync(join(tmpdir(), 'ok-archive-download-'));+ dirs.push(dir);+ return join(dir, 'archive.zip');+}++function payloadOf(size: number): Uint8Array {+ const bytes = new Uint8Array(size);+ for (let i = 0; i < size; i++) bytes[i] = (i * 31 + 7) % 251;+ return bytes;+}++function sha256(bytes: Uint8Array): string {+ return createHash('sha256').update(bytes).digest('hex');+}++type StreamEnd = 'close' | 'error' | 'hang';++function streamOf(+ bytes: Uint8Array,+ chunkSize: number,+ end: StreamEnd,+ failure: Error = new Error('socket hang up'),+): ReadableStream<Uint8Array> {+ let offset = 0;+ return new ReadableStream<Uint8Array>({+ pull(controller) {+ if (offset < bytes.length) {+ controller.enqueue(bytes.slice(offset, offset + chunkSize));+ offset += chunkSize;+ return;+ }+ if (end === 'close') controller.close();+ else if (end === 'error') controller.error(failure);+ else return new Promise<void>(() => {});+ },+ });+}++interface RecordedRequest {+ range: string | null;+ ifRange: string | null;+}++function scriptedFetch(responses: Array<(request: RecordedRequest) => Response>): {+ fetchImpl: typeof fetch;+ requests: RecordedRequest[];+} {+ const requests: RecordedRequest[] = [];+ const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => {+ const headers = new Headers(init?.headers);+ const request = { range: headers.get('range'), ifRange: headers.get('if-range') };+ requests.push(request);+ const respond = responses[Math.min(requests.length, responses.length) - 1];+ if (respond === undefined) throw new Error('no scripted response');+ return respond(request);+ }) as typeof fetch;+ return { fetchImpl, requests };+}++function stillRunningAfter(ms: number): Promise<string> {+ return new Promise((resolve) => setTimeout(() => resolve('still running'), ms).unref());+}++describe('downloadToFileWithSha', () => {+ test('resumes from the received offset when the connection drops mid-transfer', async () => {+ const payload = payloadOf(4_096);+ const half = 2_048;+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, half), 512, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(payload.slice(half), 512, 'close'), {+ status: 206,+ headers: {+ 'content-length': String(payload.length - half),+ 'content-range': `bytes ${half}-${payload.length - 1}/${payload.length}`,+ },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ });++ expect(sha).toBe(sha256(payload));+ expect(new Uint8Array(readFileSync(dest))).toEqual(payload);+ expect(requests).toEqual([+ { range: null, ifRange: null },+ { range: `bytes=${half}-`, ifRange: '"v1"' },+ ]);+ });++ test('restarts from the beginning when the server ignores the range request', async () => {+ const payload = payloadOf(3_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(payload, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ });++ expect(sha).toBe(sha256(payload));+ expect(new Uint8Array(readFileSync(dest))).toEqual(payload);+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-']);+ });++ test('retries a body that ends before its declared length', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_500), 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(payload.slice(1_500), 500, 'close'), {+ status: 206,+ headers: { 'content-range': `bytes 1500-1999/${payload.length}` },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ });++ expect(sha).toBe(sha256(payload));+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1500-']);+ });++ test('fails a transfer that stops sending data instead of waiting on it', async () => {+ const payload = payloadOf(1_024);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 512), 512, 'hang'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(new Uint8Array(0), 512, 'hang'), {+ status: 206,+ headers: { 'content-range': `bytes 512-1023/${payload.length}` },+ }),+ ]);++ const download = downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ stallTimeoutMs: 50,+ maxAttemptsWithoutProgress: 2,+ retryDelayMs: 0,+ });+ const outcome = await Promise.race([+ download.then(+ () => 'resolved',+ (err: Error) => err.message,+ ),+ stillRunningAfter(5_000),+ ]);++ expect(outcome).toMatch(/no data for 0\.05s/);+ expect(outcome).toMatch(/received 512 B of 1\.0 KB after 3 attempts/);+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=512-', 'bytes=512-']);+ });++ test('a transfer that keeps delivering data is never cut off by elapsed time', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ let offset = 0;+ const slowBody = new ReadableStream<Uint8Array>({+ async pull(controller) {+ if (offset >= payload.length) {+ controller.close();+ return;+ }+ await new Promise((resolve) => setTimeout(resolve, 20));+ controller.enqueue(payload.slice(offset, offset + 100));+ offset += 100;+ },+ });+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(slowBody, {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ stallTimeoutMs: 200,+ retryDelayMs: 0,+ });++ expect(sha).toBe(sha256(payload));+ expect(requests).toHaveLength(1);+ });++ test('does not retry a request the server rejected', async () => {+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () => new Response('not found', { status: 404 }),+ ]);++ await expect(+ downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 }),+ ).rejects.toThrow(/HTTP 404/);+ expect(requests).toHaveLength(1);+ });++ test('retries a server error response', async () => {+ const payload = payloadOf(1_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () => new Response('unavailable', { status: 503 }),+ () =>+ new Response(streamOf(payload, 250, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ });++ expect(sha).toBe(sha256(payload));+ expect(requests).toHaveLength(2);+ });++ test('stops without retrying once the caller aborts', async () => {+ const payload = payloadOf(1_000);+ const dest = destination();+ const caller = new AbortController();+ const { fetchImpl, requests } = scriptedFetch([+ () => {+ caller.abort(new Error('thread closed'));+ return new Response(streamOf(payload.slice(0, 500), 250, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ });+ },+ ]);++ await expect(+ downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ signal: caller.signal,+ retryDelayMs: 0,+ }),+ ).rejects.toThrow();+ expect(requests).toHaveLength(1);+ });++ test('restarts when a resumed response describes a different file', async () => {+ const payload = payloadOf(2_000);+ const replaced = payloadOf(2_400).reverse();+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(replaced.slice(1_000), 500, 'close'), {+ status: 206,+ headers: { 'content-range': `bytes 1000-2399/${replaced.length}` },+ }),+ () =>+ new Response(streamOf(replaced, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(replaced.length) },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ });++ expect(sha).toBe(sha256(replaced));+ expect(new Uint8Array(readFileSync(dest))).toEqual(replaced);+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);+ });++ test('treats a range past the end of a complete body as done', async () => {+ const payload = payloadOf(1_500);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload, 500, 'error'), { status: 200, headers: { etag: '"v1"' } }),+ () =>+ new Response(null, {+ status: 416,+ headers: { 'content-range': `bytes */${payload.length}` },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ });++ expect(sha).toBe(sha256(payload));+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1500-']);+ });++ test('gives up when restarts never get past the furthest point reached', async () => {+ const payload = payloadOf(4_000);+ const dest = destination();+ const dropAfter = [2_000, 1_500, 1_800, 1_200, 1_900];+ const { fetchImpl, requests } = scriptedFetch(+ dropAfter.map(+ (bytes) => () =>+ new Response(streamOf(payload.slice(0, bytes), 100, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ }),+ ),+ );++ await expect(+ downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ maxAttemptsWithoutProgress: 3,+ retryDelayMs: 0,+ }),+ ).rejects.toThrow(/after 4 attempts/);+ expect(requests).toHaveLength(4);+ });++ test('gives up after repeated server errors', async () => {+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () => new Response('unavailable', { status: 503 }),+ ]);++ await expect(+ downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ maxAttemptsWithoutProgress: 3,+ retryDelayMs: 0,+ }),+ ).rejects.toThrow(/HTTP 503 .*received 0 B of an unknown size after 3 attempts/);+ expect(requests).toHaveLength(3);+ });++ test('restarts instead of resuming when the server gave no validator', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ }),+ () =>+ new Response(streamOf(payload, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(sha).toBe(sha256(payload));+ expect(requests).toEqual([+ { range: null, ifRange: null },+ { range: null, ifRange: null },+ ]);+ });++ test('never resumes from a weak entity tag', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: {+ 'content-length': String(payload.length),+ etag: 'W/"v1"',+ 'last-modified': 'Tue, 27 Oct 2026 09:04:42 GMT',+ },+ }),+ () =>+ new Response(streamOf(payload, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length) },+ }),+ ]);++ await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(requests.map((r) => r.range)).toEqual([null, null]);+ });++ test('uses Last-Modified as the resume validator when there is no entity tag', async () => {+ const payload = payloadOf(2_000);+ const lastModified = 'Tue, 27 Oct 2026 09:04:42 GMT';+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), 'last-modified': lastModified },+ }),+ () =>+ new Response(streamOf(payload.slice(1_000), 500, 'close'), {+ status: 206,+ headers: { 'content-range': `bytes 1000-1999/${payload.length}` },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(sha).toBe(sha256(payload));+ expect(requests[1]).toEqual({ range: 'bytes=1000-', ifRange: lastModified });+ });++ test('restarts when a resumed response carries a different entity tag', async () => {+ const payload = payloadOf(2_000);+ const replaced = payloadOf(2_000).reverse();+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(replaced.slice(1_000), 500, 'close'), {+ status: 206,+ headers: { 'content-range': `bytes 1000-1999/${replaced.length}`, etag: '"v2"' },+ }),+ () =>+ new Response(streamOf(replaced, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(replaced.length), etag: '"v2"' },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(sha).toBe(sha256(replaced));+ expect(new Uint8Array(readFileSync(dest))).toEqual(replaced);+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);+ });++ test('restarts when a range is unsatisfiable for a file of a different size', async () => {+ const payload = payloadOf(2_000);+ const smaller = payloadOf(800);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(null, {+ status: 416,+ headers: { 'content-range': `bytes */${smaller.length}` },+ }),+ () =>+ new Response(streamOf(smaller, 400, 'close'), {+ status: 200,+ headers: { 'content-length': String(smaller.length), etag: '"v2"' },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(sha).toBe(sha256(smaller));+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);+ });++ test('never resumes a content-encoded response', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-encoding': 'gzip', etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(payload, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ ]);++ await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(requests.map((r) => r.range)).toEqual([null, null]);+ });++ test('reports the furthest progress after a restart that fails before any data', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(null, {+ status: 416,+ headers: { 'content-range': 'bytes */1500' },+ }),+ () => new Response('unavailable', { status: 503 }),+ ]);+ const retries: Array<{+ receivedBytes: number;+ totalBytes: number | null;+ resumeFromBytes: number;+ }> = [];++ await expect(+ downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ maxAttemptsWithoutProgress: 3,+ retryDelayMs: 0,+ onRetry: ({ receivedBytes, totalBytes, resumeFromBytes }) =>+ retries.push({ receivedBytes, totalBytes, resumeFromBytes }),+ }),+ ).rejects.toThrow(/HTTP 503 .*received 1000 B of 2\.0 KB after 4 attempts/);+ expect(retries).toEqual([+ { receivedBytes: 1_000, totalBytes: 2_000, resumeFromBytes: 1_000 },+ { receivedBytes: 1_000, totalBytes: 2_000, resumeFromBytes: 0 },+ { receivedBytes: 1_000, totalBytes: 2_000, resumeFromBytes: 0 },+ ]);+ });++ test('reports the offset the next attempt resumes from', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(streamOf(payload.slice(1_000), 500, 'close'), {+ status: 206,+ headers: { 'content-range': `bytes 1000-1999/${payload.length}`, etag: '"v1"' },+ }),+ ]);+ const resumeOffsets: number[] = [];++ await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ onRetry: ({ resumeFromBytes }) => resumeOffsets.push(resumeFromBytes),+ });++ expect(resumeOffsets).toEqual([1_000]);+ });++ test('never resumes from an entity tag that is not a quoted strong tag', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: {+ 'content-length': String(payload.length),+ etag: 'abc123',+ 'last-modified': 'Tue, 27 Oct 2026 18:00:00 GMT',+ },+ }),+ () =>+ new Response(streamOf(payload, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: 'abc123' },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(sha).toBe(sha256(payload));+ expect(requests).toEqual([+ { range: null, ifRange: null },+ { range: null, ifRange: null },+ ]);+ });++ test('restarts when a range is unsatisfiable at a length short of the declared size', async () => {+ const payload = payloadOf(2_000);+ const dest = destination();+ const { fetchImpl, requests } = scriptedFetch([+ () =>+ new Response(streamOf(payload.slice(0, 1_000), 500, 'error'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ () =>+ new Response(null, {+ status: 416,+ headers: { 'content-range': 'bytes */1000' },+ }),+ () =>+ new Response(streamOf(payload, 500, 'close'), {+ status: 200,+ headers: { 'content-length': String(payload.length), etag: '"v1"' },+ }),+ ]);++ const sha = await downloadToFileWithSha(URL_UNDER_TEST, dest, { fetchImpl, retryDelayMs: 0 });++ expect(sha).toBe(sha256(payload));+ expect(requests.map((r) => r.range)).toEqual([null, 'bytes=1000-', null]);+ });++ test('names the network error code behind a dropped connection', async () => {+ const dest = destination();+ const socketError = Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' });+ const { fetchImpl } = scriptedFetch([+ () =>+ new Response(+ streamOf(+ new Uint8Array(0),+ 500,+ 'error',+ new TypeError('terminated', { cause: socketError }),+ ),+ { status: 200 },+ ),+ ]);++ await expect(+ downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ maxAttemptsWithoutProgress: 1,+ retryDelayMs: 0,+ }),+ ).rejects.toThrow(/^terminated \(UND_ERR_SOCKET\) \(received 0 B/);+ });++ test('names a local write failure code once', async () => {+ const dest = join(destination(), 'missing', 'archive.zip');+ const { fetchImpl } = scriptedFetch([+ () => new Response(streamOf(payloadOf(100), 100, 'close'), { status: 200 }),+ ]);++ const message = await downloadToFileWithSha(URL_UNDER_TEST, dest, {+ fetchImpl,+ retryDelayMs: 0,+ }).then(+ () => 'resolved',+ (err: Error) => err.message,+ );++ expect(message).toMatch(/could not write/);+ expect(message.match(/ENOENT/g)).toHaveLength(1);+ });+});diff --git a/packages/server/src/acp/archive.ts b/packages/server/src/acp/archive.tsindex a74f82339..94a80f0ca 100644--- a/packages/server/src/acp/archive.ts+++ b/packages/server/src/acp/archive.ts@@ -1,8 +1,8 @@import { type ChildProcess, spawn } from 'node:child_process';-import { createHash } from 'node:crypto';-import { createWriteStream } from 'node:fs';+import { createHash, type Hash } from 'node:crypto';+import { open } from 'node:fs/promises';import { normalize, sep } from 'node:path';-import { Readable } from 'node:stream';+import { setTimeout as sleep } from 'node:timers/promises';export function isWithin(root: string, candidate: string): boolean {const normalizedRoot = normalize(root) + sep;@@ -64,7 +64,32 @@ export interface DownloadProgress {export interface DownloadOptions {signal?: AbortSignal;onProgress?: (p: DownloadProgress) => void;+ onRetry?: (+ r: DownloadProgress & { attempt: number; resumeFromBytes: number; error: Error },+ ) => void;fetchImpl?: typeof fetch;+ stallTimeoutMs?: number;+ maxAttemptsWithoutProgress?: number;+ retryDelayMs?: number;+}++const DEFAULT_STALL_TIMEOUT_MS = 30_000;+const DEFAULT_ATTEMPTS_WITHOUT_PROGRESS = 3;+const DEFAULT_RETRY_DELAY_MS = 1_000;+const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);++class FatalDownloadError extends Error {}++interface ResumeValidator {+ header: 'etag' | 'last-modified';+ value: string;+}++interface Transfer {+ hash: Hash;+ receivedBytes: number;+ totalBytes: number | null;+ validator: ResumeValidator | null;}export async function downloadToFileWithSha(@@ -72,35 +97,224 @@ export async function downloadToFileWithSha(destPath: string,opts: DownloadOptions = {},): Promise<string> {+ const attemptLimit = opts.maxAttemptsWithoutProgress ?? DEFAULT_ATTEMPTS_WITHOUT_PROGRESS;+ const retryDelayMs = opts.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;+ const transfer: Transfer = {+ hash: createHash('sha256'),+ receivedBytes: 0,+ totalBytes: null,+ validator: null,+ };+ let attempt = 0;+ let attemptsWithoutProgress = 0;+ let furthestBytes = 0;+ let knownTotalBytes: number | null = null;+ for (;;) {+ attempt += 1;+ try {+ await downloadAttempt(url, destPath, transfer, opts);+ return transfer.hash.digest('hex');+ } catch (err) {+ const error = err instanceof Error ? err : new Error(String(err));+ attemptsWithoutProgress =+ transfer.receivedBytes > furthestBytes ? 0 : attemptsWithoutProgress + 1;+ furthestBytes = Math.max(furthestBytes, transfer.receivedBytes);+ knownTotalBytes = transfer.totalBytes ?? knownTotalBytes;+ const progress = { receivedBytes: furthestBytes, totalBytes: knownTotalBytes };+ if (+ opts.signal?.aborted === true ||+ error instanceof FatalDownloadError ||+ attemptsWithoutProgress >= attemptLimit+ ) {+ throw new Error(`${describeError(error)} (${describeProgress(progress, attempt)})`, {+ cause: error,+ });+ }+ opts.onRetry?.({+ attempt,+ ...progress,+ resumeFromBytes: transfer.validator === null ? 0 : transfer.receivedBytes,+ error,+ });+ await sleep(retryDelayMs * 2 ** attemptsWithoutProgress, undefined, {+ signal: opts.signal,+ });+ }+ }+}++async function downloadAttempt(+ url: string,+ destPath: string,+ transfer: Transfer,+ opts: DownloadOptions,+): Promise<void> {const fetchImpl = opts.fetchImpl ?? fetch;- const res = await fetchImpl(url, { signal: opts.signal });+ const stallTimeoutMs = opts.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;+ const resumeFrom = transfer.receivedBytes;+ if (resumeFrom > 0 && resumeFrom === transfer.totalBytes) return;+ const validator = resumeFrom > 0 ? transfer.validator : null;+ const headers: Record<string, string> = { 'accept-encoding': 'identity' };+ if (validator !== null) {+ headers.range = `bytes=${resumeFrom}-`;+ headers['if-range'] = validator.value;+ }+ const stall = new AbortController();+ const signal =+ opts.signal === undefined ? stall.signal : AbortSignal.any([opts.signal, stall.signal]);+ const res = await withinStallWindow(fetchImpl(url, { signal, headers }), stallTimeoutMs, stall);+ if (res.status === 416 && validator !== null) {+ await res.body?.cancel().catch(() => {});+ if (transfer.totalBytes === null && unsatisfiedRangeTotal(res) === resumeFrom) return;+ resetTransfer(transfer, null, null);+ throw new Error(`download failed: server no longer has the requested byte range for ${url}`);+ }if (!res.ok || res.body === null) {- throw new Error(`download failed: HTTP ${res.status} for ${url}`);+ await res.body?.cancel().catch(() => {});+ const message = `download failed: HTTP ${res.status} for ${url}`;+ throw RETRYABLE_HTTP_STATUSES.has(res.status)+ ? new Error(message)+ : new FatalDownloadError(message);}- const lenHeader = res.headers.get('content-length');- const totalBytes = lenHeader !== null && lenHeader !== '' ? Number(lenHeader) : null;- const hash = createHash('sha256');- let receivedBytes = 0;- const sink = createWriteStream(destPath);- const source = Readable.fromWeb(- res.body as unknown as import('node:stream/web').ReadableStream<Uint8Array>,- );- source.on('data', (chunk: Buffer) => {- hash.update(chunk);- receivedBytes += chunk.length;- opts.onProgress?.({- receivedBytes,- totalBytes: Number.isFinite(totalBytes) ? totalBytes : null,- });++ const range = res.status === 206 ? parseContentRange(res.headers.get('content-range')) : null;+ const encoded = isEncoded(res);+ const responseEtag = nonEmpty(res.headers.get('etag'));+ const resumed =+ range !== null &&+ validator !== null &&+ !encoded &&+ range.start === resumeFrom &&+ range.total !== null &&+ (transfer.totalBytes === null || range.total === transfer.totalBytes) &&+ (responseEtag === null || validator.header !== 'etag' || responseEtag === validator.value);+ if (res.status === 206 && !resumed) {+ await res.body.cancel().catch(() => {});+ resetTransfer(transfer, null, null);+ throw new Error(`download failed: server answered with an unexpected byte range for ${url}`);+ }+ if (resumed) {+ transfer.totalBytes = range.total;+ } else {+ resetTransfer(+ transfer,+ encoded ? null : declaredLength(res.headers.get('content-length')),+ encoded ? null : resumeValidator(responseEtag, nonEmpty(res.headers.get('last-modified'))),+ );+ }++ const sink = await open(destPath, resumed ? 'a' : 'w').catch(async (err: Error) => {+ await res.body?.cancel().catch(() => {});+ throw new FatalDownloadError(`could not write ${destPath}: ${err.message}`, { cause: err });});- await new Promise<void>((resolvePromise, rejectPromise) => {- source- .pipe(sink)- .on('finish', () => resolvePromise())- .on('error', rejectPromise);- source.on('error', rejectPromise);+ const reader = res.body.getReader();+ try {+ for (;;) {+ const { done, value } = await withinStallWindow(reader.read(), stallTimeoutMs, stall);+ if (done) break;+ transfer.hash.update(value);+ transfer.receivedBytes += value.length;+ await sink.write(value).catch((err: Error) => {+ throw new FatalDownloadError(`could not write ${destPath}: ${err.message}`, {+ cause: err,+ });+ });+ opts.onProgress?.({+ receivedBytes: transfer.receivedBytes,+ totalBytes: transfer.totalBytes,+ });+ }+ } catch (err) {+ await reader.cancel().catch(() => {});+ throw err;+ } finally {+ await sink.close();+ }+ if (transfer.totalBytes !== null && transfer.receivedBytes < transfer.totalBytes) {+ throw new Error(`download ended early for ${url}`);+ }+}++async function withinStallWindow<T>(+ pending: Promise<T>,+ stallTimeoutMs: number,+ stall: AbortController,+): Promise<T> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ const expired = new Promise<never>((_, reject) => {+ timer = setTimeout(() => {+ const err = new Error(`download stalled: no data for ${stallTimeoutMs / 1000}s`);+ stall.abort(err);+ reject(err);+ }, stallTimeoutMs);});- return hash.digest('hex');+ try {+ return await Promise.race([pending, expired]);+ } finally {+ clearTimeout(timer);+ }+}++function resetTransfer(+ transfer: Transfer,+ totalBytes: number | null,+ validator: ResumeValidator | null,+): void {+ transfer.hash = createHash('sha256');+ transfer.receivedBytes = 0;+ transfer.totalBytes = totalBytes;+ transfer.validator = validator;+}++function resumeValidator(etag: string | null, lastModified: string | null): ResumeValidator | null {+ if (etag !== null) return /^"[^"]*"$/.test(etag) ? { header: 'etag', value: etag } : null;+ return lastModified === null ? null : { header: 'last-modified', value: lastModified };+}++function isEncoded(res: Response): boolean {+ const encoding = res.headers.get('content-encoding')?.trim().toLowerCase() ?? '';+ return encoding !== '' && encoding !== 'identity';+}++function describeError(error: Error): string {+ const cause = error.cause;+ if (!(cause instanceof Error)) return error.message;+ const code = (cause as NodeJS.ErrnoException).code;+ const detail = typeof code === 'string' ? code : cause.message;+ return error.message.includes(detail) ? error.message : `${error.message} (${detail})`;+}++function nonEmpty(header: string | null): string | null {+ return header === null || header === '' ? null : header;+}++function unsatisfiedRangeTotal(res: Response): number | null {+ const match = /^bytes \*\/(\d+)$/.exec(res.headers.get('content-range')?.trim() ?? '');+ return match === null ? null : Number(match[1]);+}++function declaredLength(header: string | null): number | null {+ if (header === null || header === '') return null;+ const n = Number(header);+ return Number.isSafeInteger(n) && n >= 0 ? n : null;+}++function parseContentRange(header: string | null): { start: number; total: number | null } | null {+ const match = header === null ? null : /^bytes (\d+)-\d+\/(\d+|\*)$/.exec(header.trim());+ if (match === null) return null;+ return { start: Number(match[1]), total: match[2] === '*' ? null : Number(match[2]) };+}++function describeProgress(progress: DownloadProgress, attempt: number): string {+ const total = progress.totalBytes === null ? 'an unknown size' : formatBytes(progress.totalBytes);+ const attempts = attempt === 1 ? '1 attempt' : `${attempt} attempts`;+ return `received ${formatBytes(progress.receivedBytes)} of ${total} after ${attempts}`;+}++function formatBytes(bytes: number): string {+ if (bytes < 1024) return `${bytes} B`;+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;+ return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;}export async function fetchText(diff --git a/packages/server/src/acp/launch.test.ts b/packages/server/src/acp/launch.test.tsindex 4dd01ff1d..780c2081e 100644--- a/packages/server/src/acp/launch.test.ts+++ b/packages/server/src/acp/launch.test.ts@@ -465,6 +465,67 @@ describe('ensureBinaryInstalled', () => {expect(existsSync(staleDir)).toBe(false);});+ test('a second install of the same binary joins the download already running', async () => {+ const source = tmp();+ const cacheDir = tmp();+ const artifact = buildBinaryTarball(source);+ const target = { ...artifact.target, sha256: artifact.sha };+ let downloads = 0;+ const baseFetch = binaryFetch(artifact.bytes);+ const countingFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {+ downloads += 1;+ return baseFetch(input, init);+ }) as typeof fetch;+ const started: string[] = [];++ const installed = await Promise.all([+ ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {+ fetchImpl: countingFetch,+ onDownloadStart: () => started.push('first'),+ }),+ ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {+ fetchImpl: countingFetch,+ onDownloadStart: () => started.push('second'),+ }),+ ]);++ expect(downloads).toBe(1);+ expect(installed[0]).toBe(installed[1]);+ expect(started.sort()).toEqual(['first', 'second']);+ });++ test('an install that joins after the download started is told it is installing', async () => {+ const source = tmp();+ const cacheDir = tmp();+ const artifact = buildBinaryTarball(source);+ const target = { ...artifact.target, sha256: artifact.sha };+ const { promise: responseGate, resolve: releaseResponse } = Promise.withResolvers<void>();+ const baseFetch = binaryFetch(artifact.bytes);+ const gatedFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {+ await responseGate;+ return baseFetch(input, init);+ }) as typeof fetch;+ const { promise: firstStarted, resolve: markFirstStarted } = Promise.withResolvers<void>();+ const started: string[] = [];++ const first = ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {+ fetchImpl: gatedFetch,+ onDownloadStart: () => {+ started.push('first');+ markFirstStarted();+ },+ });+ await firstStarted;+ const second = ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {+ fetchImpl: gatedFetch,+ onDownloadStart: () => started.push('second'),+ });++ expect(started).toEqual(['first', 'second']);+ releaseResponse();+ expect(await second).toBe(await first);+ });+test('concurrent installers adopt the first completed binary', async () => {const firstSource = tmp();const secondSource = tmp();@@ -1402,7 +1463,7 @@ describe('startup package policy classification', () => {const generic = 'No solution found when resolving tool dependencies';expect(packageAcquisitionFailure(uv, generic)).toBeNull();expect(- packageAcquisitionFailure(uv, `${generic}\nversions were filtered by ` + '`exclude-newer`'),+ packageAcquisitionFailure(uv, `${generic}\nversions were filtered by \`exclude-newer\``),).toMatchObject({ code: 'install-failed' });});});diff --git a/packages/server/src/acp/launch.ts b/packages/server/src/acp/launch.tsindex 2050835f4..f375b96b8 100644--- a/packages/server/src/acp/launch.ts+++ b/packages/server/src/acp/launch.ts@@ -879,13 +879,61 @@ async function writeInstallFailureMarker(});}-export async function ensureBinaryInstalled(+interface InFlightBinaryInstall {+ result: Promise<string>;+ downloading: boolean;+ downloadStartListeners: Array<() => void>;+}++const inFlightBinaryInstalls = new Map<string, InFlightBinaryInstall>();++export function ensureBinaryInstalled(id: string,version: string,target: RegistryBinaryTarget,cacheDir: string,log: PinoLogger,opts: EnsureBinaryInstallOptions = {},+): Promise<string> {+ const key = JSON.stringify([+ resolve(cacheDir),+ id,+ version,+ target.archive,+ target.cmd,+ target.sha256 ?? null,+ ]);+ const running = inFlightBinaryInstalls.get(key);+ if (running !== undefined) {+ if (opts.onDownloadStart !== undefined) {+ if (running.downloading) opts.onDownloadStart();+ else running.downloadStartListeners.push(opts.onDownloadStart);+ }+ return running.result;+ }+ const entry: InFlightBinaryInstall = {+ result: Promise.resolve(''),+ downloading: false,+ downloadStartListeners: opts.onDownloadStart === undefined ? [] : [opts.onDownloadStart],+ };+ entry.result = installBinary(id, version, target, cacheDir, log, {+ ...opts,+ onDownloadStart: () => {+ entry.downloading = true;+ for (const notify of entry.downloadStartListeners) notify();+ },+ }).finally(() => inFlightBinaryInstalls.delete(key));+ inFlightBinaryInstalls.set(key, entry);+ return entry.result;+}++async function installBinary(+ id: string,+ version: string,+ target: RegistryBinaryTarget,+ cacheDir: string,+ log: PinoLogger,+ opts: EnsureBinaryInstallOptions,): Promise<string> {const versionDir = join(cacheDir, sanitizeSegment(id), sanitizeSegment(version));const markerPath = installFailureMarkerPath(dirname(versionDir), version);@@ -906,10 +954,19 @@ export async function ensureBinaryInstalled(log.info({ id, version, archive: target.archive }, '[acp-launch] downloading agent binary');const isZip = /\.zip$/i.test(new URL(target.archive).pathname);const archivePath = join(stagingDir, isZip ? 'archive.zip' : 'archive.tar.gz');+ const downloadStartedAt = Date.now();const sha = await downloadToFileWithSha(target.archive, archivePath, {- signal: AbortSignal.timeout(120_000),fetchImpl: opts.fetchImpl,+ onRetry: ({ attempt, receivedBytes, totalBytes, resumeFromBytes, error }) =>+ log.warn(+ { id, version, attempt, receivedBytes, totalBytes, resumeFromBytes, err: error },+ '[acp-launch] agent binary download interrupted, retrying',+ ),});+ log.info(+ { id, version, elapsedMs: Date.now() - downloadStartedAt },+ '[acp-launch] agent binary downloaded',+ );if (target.sha256 !== undefined) {if (sha !== target.sha256.toLowerCase()) {throw new Error(`archive checksum mismatch: expected ${target.sha256}, got ${sha}`);diff --git a/packages/server/src/acp/managed-runtime.test.ts b/packages/server/src/acp/managed-runtime.test.tsindex 99375ddd8..b3db10bda 100644--- a/packages/server/src/acp/managed-runtime.test.ts+++ b/packages/server/src/acp/managed-runtime.test.ts@@ -309,6 +309,33 @@ describe('ensureManagedRuntime', () => {expect(await findManagedRuntime('node', root)).toBeNull();});+ test('stops the archive download when the checksum list cannot be fetched', async () => {+ const root = tmp();+ let archiveSignal: AbortSignal | undefined;+ const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => {+ if (String(url).includes('SHASUMS256') || String(url).endsWith('.sha256')) {+ await new Promise((resolve) => setTimeout(resolve, 20));+ return new Response('missing', { status: 404 });+ }+ archiveSignal = init?.signal ?? undefined;+ const body = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new Uint8Array(64));+ init?.signal?.addEventListener('abort', () => controller.error(init.signal?.reason));+ },+ });+ return new Response(body, { status: 200, headers: { 'content-length': '1048576' } });+ }) as unknown as typeof fetch;++ const error = await ensureManagedRuntime('node', log, { root, fetchImpl }).catch(+ (err: unknown) => err,+ );++ expect(error).toBeInstanceOf(RuntimeInstallError);+ expect((error as Error).message).toMatch(/HTTP 404/);+ expect(archiveSignal?.aborted).toBe(true);+ });+test('preserves the underlying failure as the error cause', async () => {const root = tmp();const failure = new Error('network unreachable');diff --git a/packages/server/src/acp/managed-runtime.ts b/packages/server/src/acp/managed-runtime.tsindex 64a8e9e9a..601ef21ca 100644--- a/packages/server/src/acp/managed-runtime.ts+++ b/packages/server/src/acp/managed-runtime.ts@@ -274,14 +274,39 @@ export async function ensureManagedRuntime({ kind, version: versionOf(kind), url: spec.archiveUrl },'[managed-runtime] downloading runtime',);- const [actualSha, checksums] = await Promise.all([+ const paired = new AbortController();+ const signal =+ opts.signal === undefined ? paired.signal : AbortSignal.any([opts.signal, paired.signal]);+ const abortPair = (error: unknown): never => {+ paired.abort(error);+ throw error;+ };+ const [download, checksumFetch] = await Promise.allSettled([downloadToFileWithSha(spec.archiveUrl, archivePath, {- signal: opts.signal,+ signal,onProgress: opts.onProgress,+ onRetry: ({ attempt, receivedBytes, totalBytes, resumeFromBytes, error }) =>+ log.warn(+ {+ kind,+ version: versionOf(kind),+ attempt,+ receivedBytes,+ totalBytes,+ resumeFromBytes,+ err: error,+ },+ '[managed-runtime] runtime download interrupted, retrying',+ ),fetchImpl: opts.fetchImpl,- }),- fetchText(spec.checksumUrl, { signal: opts.signal, fetchImpl: opts.fetchImpl }),+ }).catch(abortPair),+ fetchText(spec.checksumUrl, { signal, fetchImpl: opts.fetchImpl }).catch(abortPair),]);+ if (download.status === 'rejected' || checksumFetch.status === 'rejected') {+ throw paired.signal.reason;+ }+ const actualSha = download.value;+ const checksums = checksumFetch.value;const expectedSha = shaForFile(checksums, spec.archiveName);if (expectedSha === null) {throw new RuntimeInstallError(`no published checksum for ${spec.archiveName}`);