Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Fail a download when the server's file changes mid-transfer
Anmol Kapil5874f1bc8c91
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.tsindex cfae7db..21d5ea8 100644--- a/e2e/integrity.spec.ts+++ b/e2e/integrity.spec.ts@@ -1,6 +1,5 @@-import { readFile } from 'node:fs/promises'import { BLOCK, expect, test } from './fixtures'-import { seededBytes, sha256, type Fault } from './origin'+import { seededBytes, type Fault } from './origin'// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the// automatic checks in fixtures.ts — is that a download never ends `completed` with wrong bytes:@@ -69,28 +68,47 @@ test.describe('permanent faults end in a clean error @smoke', () => {}})-test('file changes on the server mid-download (same size, new ETag) @smoke', async ({- plexo,- serve-}) => {- test.fail(- true,- 'known gap: chunk responses are not checked against the probed ETag, so old and new bytes get stitched together'- )- const origin = await serve({ size: SIZE, seed: 1 })- const original = origin.sha256- const replacement = seededBytes(SIZE, 2)+test.describe('the file changes on the server mid-download @smoke', () => {+ const cases: [string, Omit<Parameters<typeof mutate>[0], 'origin'>][] = [+ ['same size, new ETag', { etag: '"v2"' }],+ ['same size, new Last-Modified (no ETag)', { lastModified: 'Thu, 02 Jan 2025 00:00:00 GMT' }],+ ['different size, no validators at all', { size: SIZE + BLOCK }]+ ]++ /** Republishes the file on the server, as a CDN rolling out a new version would. */+ function mutate({+ origin,+ etag,+ lastModified,+ size+ }: {+ origin: import('./origin').Origin+ etag?: string+ lastModified?: string+ size?: number+ }): void {+ origin.setContent(seededBytes(size ?? SIZE, 2), etag ?? origin.etag)+ if (lastModified) origin.lastModified = lastModified+ }- const reached = origin.hold(3 * BLOCK + 100)- await plexo.start(origin.url(), original, { connections: 2 })- await reached- origin.setContent(replacement, '"v2"')- origin.release()+ for (const [label, change] of cases) {+ test(label, async ({ plexo, serve }) => {+ const origin = await serve({+ size: SIZE,+ seed: 1,+ etag: change.etag ? '"v1"' : null,+ lastModified: change.lastModified ? 'Wed, 01 Jan 2025 00:00:00 GMT' : null+ })+ const reached = origin.hold(3 * BLOCK + 100)+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })+ await reached+ mutate({ origin, ...change })+ origin.release()- const state = await plexo.waitForStatus(['completed', 'error'])- // Acceptable outcomes: an error, or a file that is entirely one version. Never a mix.- if (state.status === 'completed') {- const got = sha256(await readFile(state.destinationPath))- expect([original, sha256(replacement)]).toContain(got)+ // Never a file stitched from two versions: the download stops with a clear reason.+ const state = await plexo.waitForStatus(['completed', 'error'])+ expect(state.status).toBe('error')+ expect(state.error).toMatch(/changed during the download/)+ })}})diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex 3e92dc0..5f54785 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -16,8 +16,19 @@ export interface ChunkDownloadOptions {append: booleanonProgress: (bytesDownloadedThisRun: number) => voidsignal: AbortSignal+ /** What the probe saw. A response that disagrees comes from a different version of the file. */+ expected: {+ etag: string | null+ lastModified: string | null+ /** 0 = unknown. */+ totalBytes: number+ }}+/** The server is now serving a different file than the one this download started on. Retrying+ * can't help — every retry would fetch the new version — so this fails the whole download. */+export class RemoteChangedError extends Error {}+// A server that accepts the connection and then goes silent (no data, no// error, no close) would otherwise hang the chunk forever with no way to// detect or retry it.@@ -31,18 +42,56 @@ const MAX_REDIRECTS = 5interface ServedRange {start: numberend: number+ /** null when the server sent `*`. */+ total: number | null}/** Parses `Content-Range: bytes <start>-<end>/<total>`. Returns null if it isn't in that form. */function parseContentRange(value: string | string[] | undefined): ServedRange | null {const raw = Array.isArray(value) ? value[0] : valueif (!raw) return null- const match = /^\s*bytes\s+(\d+)-(\d+)\/(?:\d+|\*)\s*$/i.exec(raw)+ const match = /^\s*bytes\s+(\d+)-(\d+)\/(\d+|\*)\s*$/i.exec(raw)if (!match) return nullconst start = Number(match[1])const end = Number(match[2])if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null- return { start, end }+ return { start, end, total: match[3] === '*' ? null : Number(match[3]) }+}++function header(res: IncomingMessage, name: string): string | undefined {+ const value = res.headers[name]+ return Array.isArray(value) ? value[0] : value+}++/** Why this response can't belong to the file the download started on, or null if it can. Each+ * part is fetched separately, so without this a file republished mid-download would be stitched+ * together from two versions and still pass every length check. */+function versionMismatch(+ res: IncomingMessage,+ served: ServedRange | null,+ expected: ChunkDownloadOptions['expected']+): string | null {+ const etag = header(res, 'etag')+ if (expected.etag && etag && etag !== expected.etag) return `its ETag is now ${etag}`+ const lastModified = header(res, 'last-modified')+ if (+ !expected.etag &&+ expected.lastModified &&+ lastModified &&+ lastModified !== expected.lastModified+ ) {+ return `it was modified at ${lastModified}`+ }+ // A 206 states the file's total size in Content-Range; a 200 is the whole file, so its length.+ const total =+ served?.total ??+ (res.statusCode === 200 && header(res, 'content-length')+ ? Number(header(res, 'content-length'))+ : null)+ if (expected.totalBytes > 0 && total !== null && total !== expected.totalBytes) {+ return `it is now ${total} bytes instead of ${expected.totalBytes}`+ }+ return null}/**@@ -55,8 +104,17 @@ function parseContentRange(value: string | string[] | undefined): ServedRange |* puts the block back on the queue for a retry instead of marking it done.*/export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {- const { url, rangeStart, rangeEnd, localAddress, destinationPath, append, onProgress, signal } =- options+ const {+ url,+ rangeStart,+ rangeEnd,+ localAddress,+ destinationPath,+ append,+ onProgress,+ signal,+ expected+ } = optionsreturn new Promise((resolve, reject) => {if (signal.aborted) {@@ -145,10 +203,21 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {return}+ const served = parseContentRange(res.headers['content-range'])+ const mismatch = versionMismatch(res, served, expected)+ if (mismatch) {+ fail(+ new RemoteChangedError(+ `The file on the server changed during the download (${mismatch}). Start the download over.`+ )+ )+ res.resume()+ return+ }+// A 206 says where in the file these bytes belong — check it lines up// with what we asked for before writing any of them into the part file.if (status === 206) {- const served = parseContentRange(res.headers['content-range'])if (!served) {fail(new Error('Server sent a 206 without a usable Content-Range header'))res.resume()diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex 4e56b15..683eda7 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -25,7 +25,7 @@ import type {StartDownloadRequest,StartSimulatedDownloadRequest} from '../../shared/types'-import { downloadChunk } from './chunkDownloader'+import { downloadChunk, RemoteChangedError } from './chunkDownloader'import { testKnobs } from '../testKnobs'import { reserveDestinationPath } from './paths'import { isResourceUnchanged } from './probe'@@ -839,6 +839,11 @@ export class DownloadManager {destinationPath: partPath,append: resumeOffset > 0,signal: controller.signal,+ expected: {+ etag: runtime.requestPayload.etag,+ lastModified: runtime.requestPayload.lastModified,+ totalBytes: runtime.requestPayload.totalBytes+ },onProgress: (bytesThisRun) => {const delta = bytesThisRun - lastReportedThisRunlastReportedThisRun = bytesThisRun@@ -876,6 +881,19 @@ export class DownloadManager {const message = error instanceof Error ? error.message : String(error)chunk.error = message++ if (error instanceof RemoteChangedError) {+ // Every other worker would hit the same new version, so stop them all now rather+ // than let each burn through its retries first.+ block.status = 'pending'+ chunk.status = 'error'+ chunk.speedBytesPerSec = 0+ runtime.state.status = 'error'+ runtime.state.error = message+ for (const cr of runtime.chunkRuntimes.values()) cr.controller.abort()+ break+ }+attempt += 1chunk.retryCount += 1