Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Fix seven smaller download and resume bugs
Anmol Kapil68b9939fcf00
diff --git a/e2e/disk.spec.ts b/e2e/disk.spec.tsindex a43af11..f0a762e 100644--- a/e2e/disk.spec.ts+++ b/e2e/disk.spec.ts@@ -130,10 +130,6 @@ test.describe('destination folder problems @smoke', () => {dirs}) => {test.skip(process.platform === 'win32', 'chmod does not make a folder read-only on Windows')- test.fail(- true,- 'known bug: parts/ is created before the destination name is reserved, and never removed when that fails'- )const origin = await serve({ size: 4 * BLOCK })await chmod(dirs.dest, 0o555)try {diff --git a/e2e/download.spec.ts b/e2e/download.spec.tsindex cd7fcc0..2c44682 100644--- a/e2e/download.spec.ts+++ b/e2e/download.spec.ts@@ -123,7 +123,6 @@ test.describe('file names @smoke', () => {})test('a malformed %-escape in the URL path still downloads', async ({ plexo, serve }) => {- test.fail(true, 'known bug: probe.ts decodeURIComponent(url.pathname) throws a URIError')const origin = await serve({ size: 2 * BLOCK })await plexo.start(origin.url('/files/100%25%E0%A4%A.bin'), origin.sha256)await plexo.waitForStatus('completed')@@ -132,10 +131,6 @@ test.describe('file names @smoke', () => {test.describe('edge cases', () => {test('a 0-byte file @smoke', async ({ plexo, serve }) => {- test.fail(- true,- 'known bug: the probe asks for bytes=0-0, an empty file answers 416, probe throws'- )const origin = await serve({ size: 0 })await plexo.start(origin.url(), origin.sha256)await plexo.waitForStatus('completed', 5000)diff --git a/e2e/lifecycle.spec.ts b/e2e/lifecycle.spec.tsindex 17956e8..feb99e1 100644--- a/e2e/lifecycle.spec.ts+++ b/e2e/lifecycle.spec.ts@@ -167,10 +167,6 @@ test.describe('resume safety checks @smoke', () => {})test('resume against a server without range support', async ({ plexo, serve }) => {- test.fail(- true,- 'known bug: resumes with Range: bytes=N-, the server answers 200, every retry fails'- )const origin = await serve({ size: SIZE, ranges: false })const reached = origin.hold(5 * BLOCK)const id = await plexo.start(origin.url(), origin.sha256)@@ -183,7 +179,6 @@ test.describe('resume safety checks @smoke', () => {})test('resume while the server hangs on the check request', async ({ plexo, serve }) => {- test.fail(true, 'known bug: probe.ts has no timeout, so resume waits forever')const origin = await serve({ size: SIZE })const reached = origin.hold(5 * BLOCK)const id = await plexo.start(origin.url(), origin.sha256)diff --git a/e2e/recovery.spec.ts b/e2e/recovery.spec.tsindex 088d47d..95033ab 100644--- a/e2e/recovery.spec.ts+++ b/e2e/recovery.spec.ts@@ -169,10 +169,6 @@ test.describe('persisted state on disk @smoke', () => {serve,dirs}) => {- test.fail(- true,- 'known gap: resume trusts the manifest, never recreates parts/, and fails with ENOENT instead of re-fetching the missing blocks'- )const { id } = await pausedDownload(plexo, serve)await plexo.quit()await rm(join(dirs.userData, 'downloads', id, 'parts'), { recursive: true, force: true })diff --git a/e2e/ui.spec.ts b/e2e/ui.spec.tsindex b908802..08a5a55 100644--- a/e2e/ui.spec.ts+++ b/e2e/ui.spec.ts@@ -62,10 +62,6 @@ test.describe('UI journeys @smoke', () => {serve,dirs}) => {- test.fail(- true,- 'known bug: remove() does not stop the cancelled run from pushing more updates, so a late "cancelled" event brings the removed download back on screen'- )const origin = await serve({ size: SIZE })await stubNativeUi(plexo, dirs.dest)const page = plexo.pagediff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex 5f54785..97cd246 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -159,8 +159,13 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {const onAbort = (): void => fail(new DOMException('Aborted', 'AbortError'))signal.addEventListener('abort', onAbort)- const rangeHeader =- rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}`+ // The whole file from the start needs no Range at all — and an empty file would answer+ // `bytes=0-` with 416, since it has no byte 0 to start from.+ const headers: Record<string, string> = { 'User-Agent': 'Plexo/1.0' }+ if (rangeStart > 0 || rangeEnd !== null) {+ headers['Range'] =+ rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}`+ }const attempt = (targetUrl: URL, redirectsLeft: number): void => {const requester = targetUrl.protocol === 'https:' ? httpsRequest : httpRequest@@ -176,7 +181,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {// host to resolve to IPv4 too, or binding fails with EINVAL when DNS// hands back an IPv6 address for it instead.family: 4,- headers: { 'User-Agent': 'Plexo/1.0', Range: rangeHeader }+ headers},(res: IncomingMessage) => {const status = res.statusCode ?? 0diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex c0d2aca..d68304f 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -423,17 +423,24 @@ export class DownloadManager {requestPayload.totalBytes)- const id = randomUUID()- const tempDir = join(this.downloadDir(id), 'parts')- await mkdir(tempDir, { recursive: true })-// Claimed on disk, not just picked, so a second download of the same file- // name can't pick it too and overwrite this one at reassembly time.+ // name can't pick it too and overwrite this one at reassembly time. Done+ // before anything else is created, so a destination we can't write to+ // leaves nothing behind.const destinationPath = await reserveDestinationPath(requestPayload.destinationDir,requestPayload.suggestedFileName)+ const id = randomUUID()+ const tempDir = join(this.downloadDir(id), 'parts')+ try {+ await mkdir(tempDir, { recursive: true })+ } catch (error) {+ await rm(destinationPath, { force: true })+ throw error+ }+const connectionsPerNetwork = Math.max(1,Math.min(@@ -645,6 +652,23 @@ export class DownloadManager {return}+ // The part files are the real record of what's downloaded, not the manifest. If one went+ // missing or came up short while paused (userData cleaned out, a crash before a write hit+ // the disk), fetch that block again instead of failing at assembly.+ await mkdir(runtime.tempDir, { recursive: true })+ await Promise.all(+ runtime.blocks.map(async (block) => {+ if (block.status !== 'completed' || block.rangeEnd === null) return+ const partPath = join(runtime.tempDir, `part-${block.index}`)+ const size = await stat(partPath).then(+ (stats) => stats.size,+ () => -1+ )+ if (size !== block.rangeEnd - block.rangeStart + 1) block.status = 'pending'+ })+ )+ if (runtime.state.status !== 'paused') return+for (let index = 0; index < runtime.state.chunks.length; index++) {const chunk = runtime.state.chunks[index]const iface =@@ -815,7 +839,12 @@ export class DownloadManager {chunkRuntime.partPath = partPath}- const resumeOffset = await reconcilePartFileSize(partPath, block.bytesDownloaded)+ // Without range support the server can only send the file from the start, so a retry or+ // resume begins again at byte 0 rather than asking for a Range it will ignore.+ const resumeOffset = await reconcilePartFileSize(+ partPath,+ runtime.requestPayload.supportsRanges ? block.bytesDownloaded : 0+ )if (resumeOffset !== block.bytesDownloaded) {block.bytesDownloaded = resumeOffsettrimBlockAttribution(block, resumeOffset, previousWriter)@@ -986,6 +1015,9 @@ export class DownloadManager {}private pushUpdate(runtime: DownloadRuntime, persist = true): void {+ // A removed download can still be winding down (workers finishing, cleanup). Its updates+ // would put it back on screen after the renderer has already moved on.+ if (this.runtimes.get(runtime.state.id) !== runtime) returnif (persist) this.schedulePersistence(runtime)const window = this.getWindow()if (!window || window.isDestroyed()) returndiff --git a/src/main/download/probe.ts b/src/main/download/probe.tsindex b8bf1f1..9365cbf 100644--- a/src/main/download/probe.ts+++ b/src/main/download/probe.ts@@ -2,9 +2,13 @@ import { request as httpRequest } from 'node:http'import { request as httpsRequest } from 'node:https'import { URL } from 'node:url'import type { ProbeResult } from '../../shared/types'+import { testKnobs } from '../testKnobs'const MAX_REDIRECTS = 5const USER_AGENT = 'Plexo/1.0'+// A server that accepts the connection and never answers would otherwise hang the probe — and+// with it resume, which re-probes before continuing — forever. Same budget as a stalled chunk.+const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMstype Headers = Record<string, string | string[] | undefined>@@ -37,6 +41,9 @@ function requestOneByte(url: URL): Promise<ProbeResponse> {})req.on('error', reject)+ req.setTimeout(PROBE_TIMEOUT_MS, () =>+ req.destroy(new Error('The server did not respond — check the link and try again'))+ )req.end()})}@@ -53,7 +60,12 @@ function fileNameFromHeaders(headers: Headers, url: URL): string {}}}- const pathname = decodeURIComponent(url.pathname)+ let pathname = url.pathname+ try {+ pathname = decodeURIComponent(pathname)+ } catch {+ // A malformed %-escape: the raw path still names the file well enough.+ }const base = pathname.split('/').filter(Boolean).pop()return base && base.length > 0 ? base : 'download'}@@ -111,6 +123,24 @@ export async function isResourceUnchanged(export async function probeUrl(rawUrl: string): Promise<ProbeResult> {const { current, response } = await requestFollowingRedirects(rawUrl)+ // An empty file can't satisfy a request for its first byte: the server answers 416 and gives+ // the size as `bytes */0`. That's a valid, empty download, not an error.+ if (+ response?.statusCode === 416 &&+ /^\s*bytes\s+\*\/0\s*$/i.test(headerValue(response.headers, 'content-range') ?? '')+ ) {+ return {+ requestedUrl: rawUrl,+ finalUrl: current.toString(),+ supportsRanges: false,+ totalBytes: 0,+ suggestedFileName: fileNameFromHeaders(response.headers, current),+ contentType: headerValue(response.headers, 'content-type') ?? null,+ etag: headerValue(response.headers, 'etag') ?? null,+ lastModified: headerValue(response.headers, 'last-modified') ?? null+ }+ }+if (!response || response.statusCode === 0 || response.statusCode >= 400) {throw new Error(`Server responded with status ${response?.statusCode || 'unknown'}`)}