Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
fix: reconnect connections stuck at a crawl so downloads don't hang at 99%
Anmol Kapil6eddb0ec5169
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.tsindex f15dccf..4dc3f16 100644--- a/e2e/integrity.spec.ts+++ b/e2e/integrity.spec.ts@@ -1,5 +1,5 @@import { BLOCK, expect, LAN_ADDRESS, test } from './fixtures'-import { seededBytes, type Fault } from './origin'+import { seededBytes, type Fault, type LoggedRequest } from './origin'// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the// automatic checks in fixtures.ts — is that a download never ends `completed` with wrong bytes:@@ -48,6 +48,61 @@ test.describe('transient server faults are retried to a correct file @smoke', ()})})+test.describe('a connection stuck at a crawl @smoke', () => {+ test.use({ appEnv: { PLEXO_E2E_SLOW_WARMUP_MS: '500', PLEXO_E2E_SLOW_FOR_MS: '1500' } })++ const BLOCKS = 48+ const tookFullBlock = (entry: LoggedRequest): boolean =>+ entry.bytesSent === entry.range!.end! - entry.range!.start + 1++ for (const [label, crawlStart] of [+ ['mid-download', BLOCK],+ ['on the last block (the 99% case)', (BLOCKS - 1) * BLOCK]+ ] as const) {+ test(`${label}: reconnected and resumed`, async ({ plexo, serve }) => {+ const origin = await serve({ size: BLOCKS * BLOCK, bytesPerSecond: 256 * 1024 })+ let crawled = false+ // 2 KB/s: this one block alone would take ~32 s.+ origin.setRule(({ range }) =>+ range?.start === crawlStart && !crawled ? ((crawled = true), { crawl: 2048 }) : 'ok'+ )++ await plexo.start(origin.url(), origin.sha256, { connections: 4 })+ const state = await plexo.waitForStatus('completed', 15_000)++ const requests = origin.chunkRequests()+ const slow = requests.find((entry) => typeof entry.fault === 'object')!+ expect(tookFullBlock(slow), 'the slow request was cut off').toBe(false)+ const resumed = requests.find(+ (entry) =>+ entry.n > slow.n &&+ entry.range!.start > slow.range!.start &&+ entry.range!.start <= slow.range!.end!+ )+ expect(resumed, 'the block resumed from where the slow connection stopped').toBeTruthy()+ expect(state.chunks.reduce((sum, chunk) => sum + chunk.retryCount, 0)).toBe(0)+ })+ }++ test('connections that are all equally slow are left alone', async ({ plexo, serve }) => {+ const origin = await serve({ size: BLOCKS * BLOCK, bytesPerSecond: 256 * 1024 })+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })+ await plexo.waitForStatus('completed', 15_000)+ expect(origin.chunkRequests().every(tookFullBlock), 'no request was cut off').toBe(true)+ })++ test('a block is refreshed at most twice, then left to finish', async ({ plexo, serve }) => {+ const origin = await serve({ size: 3 * BLOCK })+ // Two fast blocks set the reference; every request for the third crawls.+ origin.setRule(({ range }) => (range && range.start >= 2 * BLOCK ? { crawl: 8192 } : 'ok'))++ await plexo.start(origin.url(), origin.sha256, { connections: 1 })+ await plexo.waitForStatus('completed', 30_000)+ const lastBlock = origin.chunkRequests().filter((entry) => entry.range!.start >= 2 * BLOCK)+ expect(lastBlock).toHaveLength(3)+ })+})+test.describe('servers without range support @smoke', () => {// With no ranges the only way to recover a dropped connection is to start the file over.for (const [label, contentLength] of [diff --git a/e2e/origin.ts b/e2e/origin.tsindex 5d64aa0..74ae47f 100644--- a/e2e/origin.ts+++ b/e2e/origin.ts@@ -29,6 +29,8 @@ export type Fault =| { cutAfter: number }/** Sends this many body bytes, then ends the response cleanly (a short body). */| { endAfter: number }+ /** Trickles the body at this rate in small pieces — slow, but never silent long enough to stall. */+ | { crawl: number }| { redirect: string }export interface OriginRequest {@@ -245,6 +247,8 @@ export class Origin {bodyEnd = Math.min(bodyEnd, start + fault.endAfter)}const cutAfter = typeof fault === 'object' && 'cutAfter' in fault ? fault.cutAfter : null+ const crawl = typeof fault === 'object' && 'crawl' in fault ? fault.crawl : null+ const rate = crawl ?? this.options.bytesPerSecondif (this.options.contentLength !== false && cutAfter === null) {headers['Content-Length'] = bodyEnd - start@@ -268,7 +272,7 @@ export class Origin {return}- let next = Math.min(bodyEnd, position + PIECE_BYTES)+ let next = Math.min(bodyEnd, position + (crawl ? 256 : PIECE_BYTES))if (cutAfter !== null) next = Math.min(next, start + cutAfter)if (holdable && this.holdAt !== null) {if (position === this.holdAt) {@@ -296,10 +300,8 @@ export class Origin {// Yield so other connections — and the app's pause/cancel — interleave with this one.await new Promise((resolve) => setImmediate(resolve))}- if (this.options.bytesPerSecond) {- await new Promise((resolve) =>- setTimeout(resolve, (piece.length / this.options.bytesPerSecond!) * 1000)- )+ if (rate) {+ await new Promise((resolve) => setTimeout(resolve, (piece.length / rate) * 1000))}}diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex ac337e7..dbd8413 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -40,6 +40,13 @@ import {interface ChunkRuntime {controller: AbortControllerpartPath: string+ /** Aborts only the in-flight block request, to reconnect a slow connection. Null between requests. */+ refresh: AbortController | null+ /** When this connection last (re)connected; it isn't judged until SLOW_WARMUP_MS after. */+ warmSince: number+ slowSince: number | null+ /** Speed of its last finished block, start to end — 0 until it finishes one. */+ lastBlockSpeed: number}interface SpeedSample {@@ -63,6 +70,8 @@ interface DownloadRuntime {/** The version this download started on, plus any since confirmed to serve identical bytes* (see confirmSameBytes). Not persisted: after a restart, a confirmation is simply redone. */acceptedVersions: FileVersion[]+ /** Slow-connection refreshes per block index, capped at MAX_REFRESHES_PER_BLOCK. */+ refreshesByBlock: Map<number, number>}interface PersistedDownload {@@ -120,6 +129,26 @@ function retryDelayMs(attempt: number): number {return Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)}+// A single TCP stream can get stuck at a crawl (loss-collapsed congestion window, a bad CDN node+// or route) while its siblings on the same network run fine. It never goes silent, so the stall+// watchdog can't catch it; a fresh connection usually lands somewhere healthy. Only relative+// thresholds — nothing here knows how fast the network ought to be.+const SLOW_RATIO = 0.1+const SLOW_WARMUP_MS = testKnobs.slowWarmupMs+const SLOW_FOR_MS = testKnobs.slowForMs+// Bounds every case where refreshing can't help (the whole network got slower, a stale+// reference): at worst a block pays for a couple of cheap reconnects, then is left alone.+const MAX_REFRESHES_PER_BLOCK = 2+// Idle connections look for work every 250 ms, so waiting a bit longer hands a refreshed block+// to a connection that is already proven fast, if there is one.+const REFRESH_HANDOFF_MS = 300++function median(values: number[]): number {+ const sorted = [...values].sort((a, b) => a - b)+ const mid = sorted.length >> 1+ return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2+}+/** Records that `interfaceId` delivered `deltaBytes` of this block. Attribution is per-network* rather than a single winner because one block can be started on one network and finished on* another (a retry after a dropped connection, or a pause/resume), and the grid colors blocks@@ -342,7 +371,8 @@ export class DownloadManager {totalBlocks: state.totalBlocks ?? blocks.length,persistenceChain: Promise.resolve(),removed: false,- acceptedVersions: [requestedVersion(persisted.requestPayload)]+ acceptedVersions: [requestedVersion(persisted.requestPayload)],+ refreshesByBlock: new Map()})} catch {// Ignore incomplete or corrupt manifests; other downloads can still be restored.@@ -597,7 +627,8 @@ export class DownloadManager {totalBlocks: blocks.length,persistenceChain: Promise.resolve(),removed: false,- acceptedVersions: [requestedVersion(requestPayload)]+ acceptedVersions: [requestedVersion(requestPayload)],+ refreshesByBlock: new Map()}this.runtimes.set(id, runtime)await this.persistNow(runtime)@@ -798,6 +829,7 @@ export class DownloadManager {runtime.state.speedBytesPerSec = newSpeedthis.scheduleUpdate(runtime)}+ this.refreshSlowConnections(runtime, now)}, 500)speedTicker.unref()@@ -842,6 +874,47 @@ export class DownloadManager {await this.discardUnfinishedDestination(runtime)}+ /**+ * Reconnects a connection running under SLOW_RATIO of its network's reference speed for+ * SLOW_FOR_MS. The reference is the median of what connections on the same network are doing+ * now and did on their last finished block — its own included, which covers the tail (everyone+ * else is done) and a network with a single connection. Networks are never compared with each+ * other: cellular is expected to be slower than Wi-Fi.+ */+ private refreshSlowConnections(runtime: DownloadRuntime, now: number): void {+ // Without ranges a reconnect restarts the whole file from byte 0.+ if (!runtime.requestPayload.supportsRanges) return++ const warm = (self: ChunkRuntime): boolean => now - self.warmSince >= SLOW_WARMUP_MS+ for (const chunk of runtime.state.chunks) {+ const self = runtime.chunkRuntimes.get(chunk.id)+ const blockIndex = chunk.currentBlockIndex+ if (!self?.refresh || !warm(self) || blockIndex === undefined) continue+ const refreshes = runtime.refreshesByBlock.get(blockIndex) ?? 0+ if (refreshes >= MAX_REFRESHES_PER_BLOCK) continue++ const reference: number[] = []+ for (const other of runtime.state.chunks) {+ if (other.interfaceId !== chunk.interfaceId) continue+ const peer = runtime.chunkRuntimes.get(other.id)+ if (!peer) continue+ if (peer.lastBlockSpeed > 0) reference.push(peer.lastBlockSpeed)+ if (other !== chunk && peer.refresh && warm(peer)) reference.push(other.speedBytesPerSec)+ }++ if (reference.length === 0 || chunk.speedBytesPerSec >= median(reference) * SLOW_RATIO) {+ self.slowSince = null+ continue+ }+ self.slowSince ??= now+ if (now - self.slowSince >= SLOW_FOR_MS) {+ runtime.refreshesByBlock.set(blockIndex, refreshes + 1)+ self.refresh.abort()+ self.refresh = null+ }+ }+ }+private notify(title: string, body: string): void {if (testKnobs.userDataDir || !Notification.isSupported()) returntry {@@ -874,7 +947,15 @@ export class DownloadManager {}const controller = new AbortController()- runtime.chunkRuntimes.set(chunk.id, { controller, partPath: '' })+ const self: ChunkRuntime = {+ controller,+ partPath: '',+ refresh: null,+ warmSince: Date.now(),+ slowSince: null,+ lastBlockSpeed: 0+ }+ runtime.chunkRuntimes.set(chunk.id, self)let attempt = 0@@ -910,10 +991,7 @@ export class DownloadManager {this.scheduleUpdate(runtime)const partPath = join(runtime.tempDir, `part-${block.index}`)- const chunkRuntime = runtime.chunkRuntimes.get(chunk.id)- if (chunkRuntime) {- chunkRuntime.partPath = partPath- }+ self.partPath = partPath// Without range support the server can only send the file from the start, so a retry or// resume begins again at byte 0 rather than asking for a Range it will ignore.@@ -936,6 +1014,9 @@ export class DownloadManager {}let lastReportedThisRun = 0+ const refresh = new AbortController()+ self.refresh = refresh+ const attemptStartedAt = Date.now()try {const runDownload = isSimulatedUrl(runtime.requestPayload.url)@@ -948,7 +1029,7 @@ export class DownloadManager {localAddress: iface.address,destinationPath: partPath,append: resumeOffset > 0,- signal: controller.signal,+ signal: AbortSignal.any([controller.signal, refresh.signal]),acceptedVersions: runtime.acceptedVersions,onProgress: (bytesThisRun) => {const delta = bytesThisRun - lastReportedThisRun@@ -968,6 +1049,8 @@ export class DownloadManager {const blockBytes = block.rangeEnd - block.rangeStart + 1creditBlockBytes(block, iface.id, blockBytes - block.bytesDownloaded)block.bytesDownloaded = blockBytes+ self.lastBlockSpeed =+ ((blockBytes - resumeOffset) / Math.max(1, Date.now() - attemptStartedAt)) * 1000}attempt = 0this.recomputeAggregates(runtime)@@ -985,6 +1068,16 @@ export class DownloadManager {break}+ if (refresh.signal.aborted) {+ // Not a failure: no retry counted, no backoff. Whoever leases the block next resumes+ // it from its part file on a new connection.+ block.status = 'pending'+ await delay(REFRESH_HANDOFF_MS, controller.signal)+ self.warmSince = Date.now()+ self.slowSince = null+ continue+ }+const message = error instanceof Error ? error.message : String(error)chunk.error = message@@ -1058,6 +1151,10 @@ export class DownloadManager {break}chunk.status = 'downloading'+ self.warmSince = Date.now()+ self.slowSince = null+ } finally {+ self.refresh = null}}diff --git a/src/main/testKnobs.ts b/src/main/testKnobs.tsindex da1ed64..b1913d8 100644--- a/src/main/testKnobs.ts+++ b/src/main/testKnobs.ts@@ -17,6 +17,8 @@ export const testKnobs = {blockBytes: positiveNumber('PLEXO_E2E_BLOCK_BYTES', 8 * 1024 * 1024),retryBaseDelayMs: positiveNumber('PLEXO_E2E_RETRY_BASE_MS', 1000),stallTimeoutMs: positiveNumber('PLEXO_E2E_STALL_MS', 20_000),+ slowWarmupMs: positiveNumber('PLEXO_E2E_SLOW_WARMUP_MS', 5_000),+ slowForMs: positiveNumber('PLEXO_E2E_SLOW_FOR_MS', 10_000),/** Skips the real GitHub check and pretends this version is available, for exercising the* update banner without needing an actual newer release published. */forceUpdateVersion: env['PLEXO_FORCE_UPDATE_VERSION']