Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
fix: handle range-reporting hosts and fallback when range end is ignored (#188) (#192)
SITHESWAR K2d2132f35046
diff --git a/e2e/download.spec.ts b/e2e/download.spec.tsindex f711114..c15092c 100644--- a/e2e/download.spec.ts+++ b/e2e/download.spec.ts@@ -106,6 +106,45 @@ test.describe('happy paths @smoke', () => {expect(origin.chunkRequests().filter((request) => request.range !== null)).toEqual([])})+ test('server that ignores the end of a range: one stream, whole file', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: 10 * BLOCK })+ origin.setRule(({ range }) => (range ? 'wholeRange' : 'ok'))+ const probe = await plexo.api.probeUrl(origin.url())+ expect(probe).toMatchObject({+ kind: 'http',+ supportsRanges: false,+ totalBytes: 10 * BLOCK+ })+ await plexo.start(origin.url(), origin.sha256)+ const state = await plexo.waitForHttpStatus('completed')+ expect(state.streams).toHaveLength(1)+ // The transfer asks for the whole file, not a range the server would overrun.+ expect(origin.chunkRequests().filter((request) => request.range !== null)).toEqual([])+ })++ test('server that names the file end but sends only the range: streams still split', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: 10 * BLOCK })+ // Cloudflare-fronted hosts: Content-Range ends at the file, Content-Length is the range.+ origin.setRule(({ range }) => (range ? 'rangeToEof' : 'ok'))+ const probe = await plexo.api.probeUrl(origin.url())+ expect(probe).toMatchObject({+ kind: 'http',+ supportsRanges: true,+ totalBytes: 10 * BLOCK+ })+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })+ await plexo.waitForHttpStatus('completed')+ expect(+ origin.chunkRequests().filter((request) => request.range !== null).length+ ).toBeGreaterThan(0)+ })+test('a link that answers 403 with or without a range is still an error', async ({plexo,servediff --git a/e2e/origin.ts b/e2e/origin.tsindex 7ab8dcf..401a88f 100644--- a/e2e/origin.ts+++ b/e2e/origin.ts@@ -14,6 +14,10 @@ export type Fault =| 'ok'/** Replies 200 with the whole file, as a server without range support does. */| 'ignoreRange'+ /** 206 whose Content-Range and body run to the end of the file, ignoring the asked end. */+ | 'wholeRange'+ /** 206 like a Cloudflare-fronted host: Content-Range ends at the file, but the body is the range. */+ | 'rangeToEof'/** 206 whose Content-Range starts one byte later than asked. */| 'wrongStart'/** 206 with a correct header but one byte more body than the range. */@@ -248,7 +252,7 @@ export class Origin {let status = 200if (rangesOn && request.range) {start = request.range.start- end = Math.min(request.range.end ?? total - 1, total - 1)+ end = fault === 'wholeRange' ? total - 1 : Math.min(request.range.end ?? total - 1, total - 1)if (start >= total) {entry.status = 416res.writeHead(416, { 'Content-Range': `bytes */${total}` }).end()@@ -257,7 +261,8 @@ export class Origin {status = 206if (fault !== 'noContentRange') {const shownStart = fault === 'wrongStart' ? start + 1 : start- headers['Content-Range'] = `bytes ${shownStart}-${end}/${total}`+ const shownEnd = fault === 'rangeToEof' ? total - 1 : end+ headers['Content-Range'] = `bytes ${shownStart}-${shownEnd}/${total}`}}diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex 2fe78a3..e779a85 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -299,7 +299,15 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {res.resume()return}- if (rangeEnd !== null && served.end > rangeEnd) {+ // Some hosts name the end of the file in Content-Range while sending Content-Length+ // bytes for the range. The header alone would look like an overrun; the length says+ // the body is exactly what we asked for, so take it rather than fail every block.+ const declared = Number(header(res, 'content-length') ?? NaN)+ if (+ rangeEnd !== null &&+ served.end > rangeEnd &&+ !(Number.isFinite(declared) && declared === expectedBytes)+ ) {fail(new Error(`Server returned more than the requested range: through byte ${served.end}, asked through ${rangeEnd}`diff --git a/src/main/download/probe.ts b/src/main/download/probe.tsindex 064ba72..b9a1c20 100644--- a/src/main/download/probe.ts+++ b/src/main/download/probe.ts@@ -21,6 +21,9 @@ const MAX_REDIRECTS = 5// redirects included, so a chain of slow hops can't stretch it.const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMsconst NO_RESPONSE = 'The server did not respond — check the link and try again'+/** The last byte the probe asks for. Not 0 or 1: some file hosts answer a range that ends there+ * with the whole file while serving every real chunk as asked (#101). */+const PROBE_RANGE_END = 1023type Headers = Record<string, string | string[] | undefined>@@ -43,7 +46,7 @@ function requestFirstBytes(deadline: number,browser: BrowserContext | undefined,ranged: boolean,- range = 'bytes=0-1023'+ range = `bytes=0-${PROBE_RANGE_END}`): Promise<ProbeResponse> {return new Promise((resolve, reject) => {const requester = url.protocol === 'https:' ? httpsRequest : httpRequest@@ -248,7 +251,18 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis// A server that supports range requests must answer our range GET with 206 Partial Content.// If it returned 200 OK, it ignored the Range header and sent the whole file — even if its headers// statically claim `Accept-Ranges: bytes`.- const supportsRanges = response.statusCode === 206+ //+ // A 206 is only a promise if it sends us the bytes we asked for. Most say so in Content-Range,+ // but some hosts (Cloudflare-fronted file hosts among them) put the end of the *file* there+ // while setting Content-Length to the range they actually send. Trust whichever says the body+ // is bounded, so those keep their parallel streams instead of falling back to one.+ const served = /^bytes\s+(\d+)-(\d+)\//i.exec(contentRange ?? '')+ const contentLength = Number(headerValue(response.headers, 'content-length') ?? NaN)+ const servesOnlyProbe =+ served !== null &&+ Number(served[1]) === 0 &&+ (Number(served[2]) <= PROBE_RANGE_END || contentLength === PROBE_RANGE_END + 1)+ const supportsRanges = response.statusCode === 206 && servesOnlyProbelet totalBytes: number | null = nullif (contentRange) {@@ -262,7 +276,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promisif (contentLength) totalBytes = Number(contentLength)}- const perRange = await etagIsPerRange(current, browser, response)+ const perRange = supportsRanges ? await etagIsPerRange(current, browser, response) : falsereturn {kind: 'http',