Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Fall back to a plain GET when a server refuses range requests with 403 (#139)
Arpit Khandelwal7a9b1b489a54
diff --git a/e2e/download.spec.ts b/e2e/download.spec.tsindex 5634d67..47c09a4 100644--- a/e2e/download.spec.ts+++ b/e2e/download.spec.ts@@ -91,6 +91,30 @@ test.describe('happy paths @smoke', () => {expect(state.streams).toHaveLength(1)})+ test('server that refuses range requests with 403: one stream, whole file', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: 10 * BLOCK })+ origin.setRule(({ range }) => (range ? { status: 403 } : 'ok'))+ const probe = await plexo.api.probeUrl(origin.url())+ expect(probe).toMatchObject({ kind: 'http', supportsRanges: false, totalBytes: 10 * BLOCK })+ await plexo.start(origin.url(), origin.sha256)+ const state = await plexo.waitForHttpStatus('completed')+ expect(state.streams).toHaveLength(1)+ // After the probe's ranged GET, nothing else asks for a range the server would refuse.+ expect(origin.chunkRequests().filter((request) => request.range !== null)).toEqual([])+ })++ test('a link that answers 403 with or without a range is still an error', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: 3 * BLOCK })+ origin.setRule(() => ({ status: 403 }))+ await expect(plexo.api.probeUrl(origin.url())).rejects.toThrow(/status 403/)+ })+test('unknown size (no Content-Length)', async ({ plexo, serve }) => {const origin = await serve({ size: 10 * BLOCK, ranges: false, contentLength: false })await plexo.start(origin.url(), origin.sha256)diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex fff22fb..6f90cd5 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -13,6 +13,9 @@ export interface ChunkDownloadOptions {rangeStart: number/** null = open-ended range, download to end of file. */rangeEnd: number | null+ /** The server doesn't take range requests, so ask for the whole file without a Range header+ * (only valid from byte 0). Some servers refuse any Range with 403 (#131). */+ wholeFile?: boolean/** The stream's connection, through the network it's bound to. */connection: StreamConnectioncreateDestination: () => Writable@@ -131,6 +134,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {browser,rangeStart,rangeEnd,+ wholeFile = false,connection,createDestination,onNetworkProgress,@@ -206,7 +210,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {// The whole file from the start needs no Range at all — and an empty file would answer// `bytes=0-` with 416, since it has no byte 0 to start from.const range: Record<string, string> =- rangeStart > 0 || rangeEnd !== null+ !(wholeFile && rangeStart === 0) && (rangeStart > 0 || rangeEnd !== null)? { Range: rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}` }: {}diff --git a/src/main/download/httpTransfer.ts b/src/main/download/httpTransfer.tsindex d680878..4981d39 100644--- a/src/main/download/httpTransfer.ts+++ b/src/main/download/httpTransfer.ts@@ -717,6 +717,7 @@ export class HttpTransfer implements Transfer {browser: this.runtime.requestPayload.browser,rangeStart: block.rangeStart + attempt.startOffset,rangeEnd: block.rangeEnd,+ wholeFile: !this.runtime.requestPayload.supportsRanges,connection: self.connection,createDestination: () =>this.runtime.file.writer(block.rangeStart + (attempt.startOffset ?? 0)),diff --git a/src/main/download/probe.ts b/src/main/download/probe.tsindex 37232bb..de17b58 100644--- a/src/main/download/probe.ts+++ b/src/main/download/probe.ts@@ -34,11 +34,13 @@ interface ProbeResponse {/** GET with a small range: cheaper than fetching the body, and unlike HEAD it also tells us (via* the 206 status) whether range requests actually work. Not 1 byte: some file hosts answer a range- * that ends at byte 0 or 1 with the whole file, while serving every real chunk as asked (#101). */+ * that ends at byte 0 or 1 with the whole file, while serving every real chunk as asked (#101).+ * `ranged: false` sends a plain GET instead; the body is still dropped once the headers arrive. */function requestFirstBytes(url: URL,deadline: number,- browser: BrowserContext | undefined+ browser: BrowserContext | undefined,+ ranged: boolean): Promise<ProbeResponse> {return new Promise((resolve, reject) => {const requester = url.protocol === 'https:' ? httpsRequest : httpRequest@@ -48,7 +50,9 @@ function requestFirstBytes(hostname: url.hostname.replace(/^\[|\]$/g, ''),port: url.port || undefined,path: `${url.pathname}${url.search}`,- headers: { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }+ headers: ranged+ ? { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }+ : requestHeaders(url, browser)},(res) => {clearTimeout(timer)@@ -92,8 +96,16 @@ async function requestFollowingRedirects(let response: ProbeResponse | null = nullconst deadline = Date.now() + PROBE_TIMEOUT_MS+ let ranged = truefor (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {- response = await requestFirstBytes(current, deadline, browser)+ response = await requestFirstBytes(current, deadline, browser, ranged)+ // Some servers refuse any Range request with 403 while serving the same URL in full (#131).+ // Ask once more without one: if that works, the file downloads as one stream from byte zero,+ // which sends no Range either — the same as on a server without range support.+ if (response.statusCode === 403 && ranged) {+ ranged = false+ response = await requestFirstBytes(current, deadline, browser, ranged)+ }if (response.statusCode >= 300 && response.statusCode < 400) {const location = headerValue(response.headers, 'location')if (!location) break