Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Follow HTTP redirects mid-chunk-download

Anmol Kapilc2e9281fc945
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.ts
index fbc662e..6a6f6d5 100644
--- a/src/main/download/chunkDownloader.ts
+++ b/src/main/download/chunkDownloader.ts
@@ -22,6 +22,11 @@ export interface ChunkDownloadOptions {
// detect or retry it.
const STALL_TIMEOUT_MS = 20_000
+// Only the initial probe resolves redirects today — if a CDN reissues a
+// redirect mid-download (e.g. a signed URL rotates), a chunk needs to be
+// able to follow it too instead of failing outright.
+const MAX_REDIRECTS = 5
+
/** Downloads a single byte range of a URL, bound to one network interface, into a part file. */
export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
const { url, rangeStart, rangeEnd, localAddress, destinationPath, append, onProgress, signal } =
@@ -33,10 +38,9 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
return
}
- const parsed = new URL(url)
- const requester = parsed.protocol === 'https:' ? httpsRequest : httpRequest
let bytesDownloaded = 0
let settled = false
+ let currentReq: ClientRequest | null = null
const finish = (fn: () => void): void => {
if (settled) return
@@ -47,7 +51,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
const fail = (error: Error): void =>
finish(() => {
- req.destroy()
+ currentReq?.destroy()
reject(error)
})
@@ -57,47 +61,68 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
const rangeHeader =
rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}`
- const req: ClientRequest = requester(
- {
- method: 'GET',
- hostname: parsed.hostname,
- port: parsed.port || undefined,
- path: `${parsed.pathname}${parsed.search}`,
- localAddress,
- // localAddress is always an IPv4 interface address — force the remote
- // host to resolve to IPv4 too, or binding fails with EINVAL when DNS
- // hands back an IPv6 address for it instead.
- family: 4,
- headers: { 'User-Agent': 'Plexo/1.0', Range: rangeHeader }
- },
- (res: IncomingMessage) => {
- // A 200 is only correct here if we asked for the whole file from byte
- // 0 — otherwise the server ignored our Range and we'd silently write
- // the wrong bytes into this chunk's slot.
- const isValidFullBody = res.statusCode === 200 && rangeStart === 0
- if (res.statusCode !== 206 && !isValidFullBody) {
- fail(new Error(`Unexpected status ${res.statusCode} for range request`))
- res.resume()
- return
- }
+ const attempt = (targetUrl: URL, redirectsLeft: number): void => {
+ const requester = targetUrl.protocol === 'https:' ? httpsRequest : httpRequest
+
+ const req: ClientRequest = requester(
+ {
+ method: 'GET',
+ hostname: targetUrl.hostname,
+ port: targetUrl.port || undefined,
+ path: `${targetUrl.pathname}${targetUrl.search}`,
+ localAddress,
+ // localAddress is always an IPv4 interface address — force the remote
+ // host to resolve to IPv4 too, or binding fails with EINVAL when DNS
+ // hands back an IPv6 address for it instead.
+ family: 4,
+ headers: { 'User-Agent': 'Plexo/1.0', Range: rangeHeader }
+ },
+ (res: IncomingMessage) => {
+ const status = res.statusCode ?? 0
+
+ if (status >= 300 && status < 400) {
+ res.resume()
+ if (!res.headers.location || redirectsLeft <= 0) {
+ fail(new Error('Too many redirects for range request'))
+ return
+ }
+ attempt(new URL(res.headers.location, targetUrl), redirectsLeft - 1)
+ return
+ }
- const fileStream = createWriteStream(destinationPath, { flags: append ? 'a' : 'w' })
+ // A 200 is only correct here if we asked for the whole file from byte
+ // 0 — otherwise the server ignored our Range and we'd silently write
+ // the wrong bytes into this chunk's slot.
+ const isValidFullBody = status === 200 && rangeStart === 0
+ if (status !== 206 && !isValidFullBody) {
+ fail(new Error(`Unexpected status ${status} for range request`))
+ res.resume()
+ return
+ }
- res.on('data', (chunk: Buffer) => {
- bytesDownloaded += chunk.length
- onProgress(bytesDownloaded)
- })
+ const fileStream = createWriteStream(destinationPath, { flags: append ? 'a' : 'w' })
- res.on('error', fail)
- fileStream.on('error', fail)
- fileStream.on('finish', () => finish(resolve))
+ res.on('data', (chunk: Buffer) => {
+ bytesDownloaded += chunk.length
+ onProgress(bytesDownloaded)
+ })
- res.pipe(fileStream)
- }
- )
+ res.on('error', fail)
+ fileStream.on('error', fail)
+ fileStream.on('finish', () => finish(resolve))
+
+ res.pipe(fileStream)
+ }
+ )
+
+ currentReq = req
+ req.on('error', fail)
+ req.setTimeout(STALL_TIMEOUT_MS, () =>
+ fail(new Error('Connection stalled: no response from server'))
+ )
+ req.end()
+ }
- req.on('error', fail)
- req.setTimeout(STALL_TIMEOUT_MS, () => fail(new Error('Connection stalled: no response from server')))
- req.end()
+ attempt(new URL(url), MAX_REDIRECTS)
})
}