Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Follow HTTP redirects mid-chunk-download
Anmol Kapilc2e9281fc945
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex fbc662e..6a6f6d5 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -22,6 +22,11 @@ export interface ChunkDownloadOptions {// detect or retry it.const STALL_TIMEOUT_MS = 20_000+// Only the initial probe resolves redirects today — if a CDN reissues a+// redirect mid-download (e.g. a signed URL rotates), a chunk needs to be+// able to follow it too instead of failing outright.+const MAX_REDIRECTS = 5+/** Downloads a single byte range of a URL, bound to one network interface, into a part file. */export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {const { url, rangeStart, rangeEnd, localAddress, destinationPath, append, onProgress, signal } =@@ -33,10 +38,9 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {return}- const parsed = new URL(url)- const requester = parsed.protocol === 'https:' ? httpsRequest : httpRequestlet bytesDownloaded = 0let settled = false+ let currentReq: ClientRequest | null = nullconst finish = (fn: () => void): void => {if (settled) return@@ -47,7 +51,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {const fail = (error: Error): void =>finish(() => {- req.destroy()+ currentReq?.destroy()reject(error)})@@ -57,47 +61,68 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {const rangeHeader =rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}`- const req: ClientRequest = requester(- {- method: 'GET',- hostname: parsed.hostname,- port: parsed.port || undefined,- path: `${parsed.pathname}${parsed.search}`,- localAddress,- // localAddress is always an IPv4 interface address — force the remote- // host to resolve to IPv4 too, or binding fails with EINVAL when DNS- // hands back an IPv6 address for it instead.- family: 4,- headers: { 'User-Agent': 'Plexo/1.0', Range: rangeHeader }- },- (res: IncomingMessage) => {- // A 200 is only correct here if we asked for the whole file from byte- // 0 — otherwise the server ignored our Range and we'd silently write- // the wrong bytes into this chunk's slot.- const isValidFullBody = res.statusCode === 200 && rangeStart === 0- if (res.statusCode !== 206 && !isValidFullBody) {- fail(new Error(`Unexpected status ${res.statusCode} for range request`))- res.resume()- return- }+ const attempt = (targetUrl: URL, redirectsLeft: number): void => {+ const requester = targetUrl.protocol === 'https:' ? httpsRequest : httpRequest++ const req: ClientRequest = requester(+ {+ method: 'GET',+ hostname: targetUrl.hostname,+ port: targetUrl.port || undefined,+ path: `${targetUrl.pathname}${targetUrl.search}`,+ localAddress,+ // localAddress is always an IPv4 interface address — force the remote+ // host to resolve to IPv4 too, or binding fails with EINVAL when DNS+ // hands back an IPv6 address for it instead.+ family: 4,+ headers: { 'User-Agent': 'Plexo/1.0', Range: rangeHeader }+ },+ (res: IncomingMessage) => {+ const status = res.statusCode ?? 0++ if (status >= 300 && status < 400) {+ res.resume()+ if (!res.headers.location || redirectsLeft <= 0) {+ fail(new Error('Too many redirects for range request'))+ return+ }+ attempt(new URL(res.headers.location, targetUrl), redirectsLeft - 1)+ return+ }- const fileStream = createWriteStream(destinationPath, { flags: append ? 'a' : 'w' })+ // A 200 is only correct here if we asked for the whole file from byte+ // 0 — otherwise the server ignored our Range and we'd silently write+ // the wrong bytes into this chunk's slot.+ const isValidFullBody = status === 200 && rangeStart === 0+ if (status !== 206 && !isValidFullBody) {+ fail(new Error(`Unexpected status ${status} for range request`))+ res.resume()+ return+ }- res.on('data', (chunk: Buffer) => {- bytesDownloaded += chunk.length- onProgress(bytesDownloaded)- })+ const fileStream = createWriteStream(destinationPath, { flags: append ? 'a' : 'w' })- res.on('error', fail)- fileStream.on('error', fail)- fileStream.on('finish', () => finish(resolve))+ res.on('data', (chunk: Buffer) => {+ bytesDownloaded += chunk.length+ onProgress(bytesDownloaded)+ })- res.pipe(fileStream)- }- )+ res.on('error', fail)+ fileStream.on('error', fail)+ fileStream.on('finish', () => finish(resolve))++ res.pipe(fileStream)+ }+ )++ currentReq = req+ req.on('error', fail)+ req.setTimeout(STALL_TIMEOUT_MS, () =>+ fail(new Error('Connection stalled: no response from server'))+ )+ req.end()+ }- req.on('error', fail)- req.setTimeout(STALL_TIMEOUT_MS, () => fail(new Error('Connection stalled: no response from server')))- req.end()+ attempt(new URL(url), MAX_REDIRECTS)})}