Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Cap downloads at once at 4 and share Auto streams per network (#104)
Anmol Kapild4d316849ee1
diff --git a/README.md b/README.mdindex 0b0ca9f..1887f79 100644--- a/README.md+++ b/README.md@@ -21,7 +21,7 @@ Plexo splits each download across every network you select, so a second connecti### Direct downloads-- **Several networks, many streams each**: a file is split into blocks of up to 8 MB that streams on every network fetch in parallel. Each network starts with 8 streams and doubles to 16 and 32 once they're all receiving. You can also pick a fixed 4, 8, 16, or 32 in the **New download** dialog.+- **Several networks, many streams each**: a file is split into blocks of up to 8 MB that streams on every network fetch in parallel. Each network starts with 8 streams and doubles to 16 and 32 once they're all receiving. Downloads running at once share those 32. You can also pick a fixed 4, 8, 16, or 32 in the **New download** dialog.- **Faster networks do more of the work**: a stream takes the next block as soon as it's free, so a fast network fetches more blocks than a slow one.- **No slow finish**: near the end, a free stream races a slow block from where it got to, and the first to finish wins- **Pause and resume**: progress is saved as you go. Before resuming, Plexo checks the file on the server hasn't changed, and refuses rather than mixing two versions.diff --git a/e2e/concurrency.spec.ts b/e2e/concurrency.spec.tsindex 837aac0..967f59a 100644--- a/e2e/concurrency.spec.ts+++ b/e2e/concurrency.spec.ts@@ -196,6 +196,20 @@ test.describe('stream count', () => {).toEqual([])})+ test('downloads running at once share a network: one starting retires the extra streams', () => {+ let downloads = 1+ const controller = new ConcurrencyController(MAX, true, () => Math.floor(MAX / downloads))+ expect(controller.limit('a')).toBe(32)+ downloads = 4+ expect(controller.tick({ now: 0, networks: [network('a', 32)], spareWork: 1000 })).toEqual([+ { kind: 'retire', networkId: 'a', count: 24 }+ ])+ downloads = 2+ expect(controller.tick({ now: 0, networks: [network('a', 8)], spareWork: 1000 })).toEqual([+ { kind: 'add', networkId: 'a', count: 8 }+ ])+ })+test('a count the user picked is kept, not grown, and still backs off when refused', () => {const controller = new ConcurrencyController(4, false)expect(controller.tick({ now: 0, networks: [network('a', 4)], spareWork: 1000 })).toEqual([])diff --git a/src/main/download/concurrency.ts b/src/main/download/concurrency.tsindex 0c10753..59e3a38 100644--- a/src/main/download/concurrency.ts+++ b/src/main/download/concurrency.ts@@ -30,6 +30,10 @@// refusals keep their ceiling and pace beside it, and each network runs the lower of the two. A// download the disk keeps up with is never behind, so this costs nothing anywhere else. The// user's own pick is kept, as it is for everything but refusals.+// - Downloads running at once share a network's limit: each may run its even share of it (see+// TransferHost.downloadsOn), so four downloads on one network run 32 streams between them, not+// 128. A download that starts lowers the others' share, and their extra streams retire; one that+// ends raises it, and they grow back.// - Otherwise each network is decided on its own, so one that drops out or comes back never// disturbs the others.//@@ -136,7 +140,9 @@ export class ConcurrencyController {/** The most streams a network may run. */private readonly maxPerNetwork: number,/** false: keep each network at maxPerNetwork (the user's pick) rather than doubling to it. */- private readonly grows = true+ private readonly grows = true,+ /** The most a network may run for this download's share of it now (see the rules above). */+ private readonly share: (networkId: string) => number = () => Infinity) {}tick(snapshot: Snapshot): Action[] {@@ -172,11 +178,13 @@ export class ConcurrencyController {return actions}- /** The most streams a network may run now: the lower of a server's ceiling and the disk's cap,- * and never more than the limit. Also what a network joining the download starts with at most. */+ /** The most streams a network may run now: the lowest of a server's ceiling, the disk's cap and+ * this download's share, and never more than the limit. Also what a network joining the download+ * starts with at most. */limit(networkId: string): number {return Math.min(this.maxPerNetwork,+ this.share(networkId),this.ceilings.get(networkId)?.limit ?? Infinity,this.disk.cap ?? Infinity)diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex b27b52a..4632ac7 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -37,7 +37,7 @@ import { loadSettings } from '../settings'import { addToHistory, findInHistory, removeFromHistory } from './history'import { testKnobs } from '../testKnobs'import { DownloadFile } from './downloadFile'-import { compareSamples, HttpTransfer, splittable } from './httpTransfer'+import { compareSamples, HttpTransfer, pickedStreams, splittable } from './httpTransfer'import { normalizeEtag } from './fileVersion'import { ensureDirectory, pathExists, reserveDestinationPath } from './paths'import { planBlocks, planDownload, planPieces } from './plan'@@ -397,6 +397,26 @@ export class DownloadManager {}}+ /** Running downloads on Auto with `networkId` on: how many share its streams (see+ * concurrency.ts). One that can't split runs a single stream, and a picked count is kept, so+ * neither takes a share. */+ private httpDownloadsOn(networkId: string): number {+ let count = 0+ for (const runtime of this.runtimes.values()) {+ if (+ runtime.kind === 'http' &&+ runtime.state.status === 'downloading' &&+ splittable(runtime.requestPayload) &&+ pickedStreams(runtime.requestPayload) === undefined &&+ runtime.state.networks.some(+ (network) => network.id === networkId && network.status === 'on'+ )+ )+ count+++ }+ return count+ }+private runningCount(): number {let running = 0for (const runtime of this.runtimes.values()) {@@ -479,6 +499,7 @@ export class DownloadManager {const host: TransferHost = {networks: this.networks,limits: this.limits,+ downloadsOn: (networkId) => this.httpDownloadsOn(networkId),reconcile: () => this.reconcile(runtime),failDownload: (message, discard) => this.failDownload(runtime, message, discard),failNetwork: (network, message) => this.failNetwork(runtime, network, message),@@ -510,6 +531,7 @@ export class DownloadManager {const host: TransferHost = {networks: this.networks,limits: this.limits,+ downloadsOn: (networkId) => this.httpDownloadsOn(networkId),reconcile: () => this.reconcile(runtime),failDownload: (message, discard) => this.failDownload(runtime, message, discard),failNetwork: (network, message) => this.failNetwork(runtime, network, message),diff --git a/src/main/download/httpTransfer.ts b/src/main/download/httpTransfer.tsindex 10c5291..d680878 100644--- a/src/main/download/httpTransfer.ts+++ b/src/main/download/httpTransfer.ts@@ -165,12 +165,17 @@ export function splittable(request: StartHttpDownloadRequest): boolean {}/** How many streams a download runs is only for it to work out when there can be more than one,- * and unless a test fixes it. */-function concurrencyFor(request: StartHttpDownloadRequest): ConcurrencyController | null {+ * and unless a test fixes it. On Auto, it shares each network with the others running on it. */+function concurrencyFor(+ request: StartHttpDownloadRequest,+ host: TransferHost+): ConcurrencyController | null {if (!splittable(request) || testStreamsPerNetwork() !== null) return nullconst picked = pickedStreams(request)return picked === undefined- ? new ConcurrencyController(MAX_STREAMS_PER_NETWORK)+ ? new ConcurrencyController(MAX_STREAMS_PER_NETWORK, true, (networkId) =>+ Math.floor(MAX_STREAMS_PER_NETWORK / Math.max(1, host.downloadsOn(networkId)))+ ): new ConcurrencyController(picked, false)}@@ -311,7 +316,7 @@ export class HttpTransfer implements Transfer {private readonly host: TransferHost) {this.acceptedVersions = [requestedVersion(runtime.requestPayload)]- this.concurrency = concurrencyFor(runtime.requestPayload)+ this.concurrency = concurrencyFor(runtime.requestPayload, host)}/**diff --git a/src/main/download/transfer.ts b/src/main/download/transfer.tsindex c667576..4bbecef 100644--- a/src/main/download/transfer.ts+++ b/src/main/download/transfer.ts@@ -72,6 +72,9 @@ export interface TransferHost {networks: NetworkMonitor/** Every byte received goes through it (see Limits.take). */limits: Limits+ /** Running downloads on Auto that have `networkId` on, this one included: how many share its+ * streams (see concurrency.ts). */+ downloadsOn(networkId: string): number/** The manager's reconcile: networks first, then the transfer's own. */reconcile(): voidfailDownload(message: string, discard?: boolean): voiddiff --git a/src/main/settings.ts b/src/main/settings.tsindex 16e6489..2964a47 100644--- a/src/main/settings.ts+++ b/src/main/settings.ts@@ -70,12 +70,9 @@ function sanitizeSettings(parsed: unknown): AppSettings {if (typeof destinationDir === 'string' && isAbsolute(destinationDir)) {settings.destinationDir = destinationDir}- if (- Number.isInteger(downloadsAtOnce) &&- (downloadsAtOnce as number) >= DOWNLOADS_AT_ONCE.min &&- (downloadsAtOnce as number) <= DOWNLOADS_AT_ONCE.max- ) {- settings.downloadsAtOnce = downloadsAtOnce as number+ // Clamped rather than dropped: a count saved under an older, higher limit keeps the most allowed.+ if (Number.isInteger(downloadsAtOnce) && (downloadsAtOnce as number) >= DOWNLOADS_AT_ONCE.min) {+ settings.downloadsAtOnce = Math.min(downloadsAtOnce as number, DOWNLOADS_AT_ONCE.max)}if (parsed.networkPreferences !== undefined) {settings.networkPreferences = sanitizeNetworkPreferences(parsed.networkPreferences)diff --git a/src/shared/types.ts b/src/shared/types.tsindex 39b9d59..c82a063 100644--- a/src/shared/types.ts+++ b/src/shared/types.ts@@ -341,7 +341,7 @@ export interface AppSettings {/** bytes: MB/s. bits: Mbps, as internet plans and speed tests count them. */export type SpeedUnit = 'bytes' | 'bits'-export const DOWNLOADS_AT_ONCE = { default: 2, min: 1, max: 8 }+export const DOWNLOADS_AT_ONCE = { default: 2, min: 1, max: 4 }/** Seconds of speed history a download keeps (a sample a second), and so the span the throughput* chart always shows. */export const SPEED_HISTORY_SECONDS = 60