Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Cap downloads at once at 4 and share Auto streams per network (#104)

Anmol Kapild4d316849ee1
diff --git a/README.md b/README.md
index 0b0ca9f..1887f79 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ Plexo splits each download across every network you select, so a second connecti
### Direct downloads
-- **Several networks, many streams each**: a file is split into blocks of up to 8 MB that streams on every network fetch in parallel. Each network starts with 8 streams and doubles to 16 and 32 once they're all receiving. You can also pick a fixed 4, 8, 16, or 32 in the **New download** dialog.
+- **Several networks, many streams each**: a file is split into blocks of up to 8 MB that streams on every network fetch in parallel. Each network starts with 8 streams and doubles to 16 and 32 once they're all receiving. Downloads running at once share those 32. You can also pick a fixed 4, 8, 16, or 32 in the **New download** dialog.
- **Faster networks do more of the work**: a stream takes the next block as soon as it's free, so a fast network fetches more blocks than a slow one.
- **No slow finish**: near the end, a free stream races a slow block from where it got to, and the first to finish wins
- **Pause and resume**: progress is saved as you go. Before resuming, Plexo checks the file on the server hasn't changed, and refuses rather than mixing two versions.
diff --git a/e2e/concurrency.spec.ts b/e2e/concurrency.spec.ts
index 837aac0..967f59a 100644
--- a/e2e/concurrency.spec.ts
+++ b/e2e/concurrency.spec.ts
@@ -196,6 +196,20 @@ test.describe('stream count', () => {
).toEqual([])
})
+ test('downloads running at once share a network: one starting retires the extra streams', () => {
+ let downloads = 1
+ const controller = new ConcurrencyController(MAX, true, () => Math.floor(MAX / downloads))
+ expect(controller.limit('a')).toBe(32)
+ downloads = 4
+ expect(controller.tick({ now: 0, networks: [network('a', 32)], spareWork: 1000 })).toEqual([
+ { kind: 'retire', networkId: 'a', count: 24 }
+ ])
+ downloads = 2
+ expect(controller.tick({ now: 0, networks: [network('a', 8)], spareWork: 1000 })).toEqual([
+ { kind: 'add', networkId: 'a', count: 8 }
+ ])
+ })
+
test('a count the user picked is kept, not grown, and still backs off when refused', () => {
const controller = new ConcurrencyController(4, false)
expect(controller.tick({ now: 0, networks: [network('a', 4)], spareWork: 1000 })).toEqual([])
diff --git a/src/main/download/concurrency.ts b/src/main/download/concurrency.ts
index 0c10753..59e3a38 100644
--- a/src/main/download/concurrency.ts
+++ b/src/main/download/concurrency.ts
@@ -30,6 +30,10 @@
// refusals keep their ceiling and pace beside it, and each network runs the lower of the two. A
// download the disk keeps up with is never behind, so this costs nothing anywhere else. The
// user's own pick is kept, as it is for everything but refusals.
+// - Downloads running at once share a network's limit: each may run its even share of it (see
+// TransferHost.downloadsOn), so four downloads on one network run 32 streams between them, not
+// 128. A download that starts lowers the others' share, and their extra streams retire; one that
+// ends raises it, and they grow back.
// - Otherwise each network is decided on its own, so one that drops out or comes back never
// disturbs the others.
//
@@ -136,7 +140,9 @@ export class ConcurrencyController {
/** The most streams a network may run. */
private readonly maxPerNetwork: number,
/** false: keep each network at maxPerNetwork (the user's pick) rather than doubling to it. */
- private readonly grows = true
+ private readonly grows = true,
+ /** The most a network may run for this download's share of it now (see the rules above). */
+ private readonly share: (networkId: string) => number = () => Infinity
) {}
tick(snapshot: Snapshot): Action[] {
@@ -172,11 +178,13 @@ export class ConcurrencyController {
return actions
}
- /** The most streams a network may run now: the lower of a server's ceiling and the disk's cap,
- * and never more than the limit. Also what a network joining the download starts with at most. */
+ /** The most streams a network may run now: the lowest of a server's ceiling, the disk's cap and
+ * this download's share, and never more than the limit. Also what a network joining the download
+ * starts with at most. */
limit(networkId: string): number {
return Math.min(
this.maxPerNetwork,
+ this.share(networkId),
this.ceilings.get(networkId)?.limit ?? Infinity,
this.disk.cap ?? Infinity
)
diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.ts
index b27b52a..4632ac7 100644
--- a/src/main/download/downloadManager.ts
+++ b/src/main/download/downloadManager.ts
@@ -37,7 +37,7 @@ import { loadSettings } from '../settings'
import { addToHistory, findInHistory, removeFromHistory } from './history'
import { testKnobs } from '../testKnobs'
import { DownloadFile } from './downloadFile'
-import { compareSamples, HttpTransfer, splittable } from './httpTransfer'
+import { compareSamples, HttpTransfer, pickedStreams, splittable } from './httpTransfer'
import { normalizeEtag } from './fileVersion'
import { ensureDirectory, pathExists, reserveDestinationPath } from './paths'
import { planBlocks, planDownload, planPieces } from './plan'
@@ -397,6 +397,26 @@ export class DownloadManager {
}
}
+ /** Running downloads on Auto with `networkId` on: how many share its streams (see
+ * concurrency.ts). One that can't split runs a single stream, and a picked count is kept, so
+ * neither takes a share. */
+ private httpDownloadsOn(networkId: string): number {
+ let count = 0
+ for (const runtime of this.runtimes.values()) {
+ if (
+ runtime.kind === 'http' &&
+ runtime.state.status === 'downloading' &&
+ splittable(runtime.requestPayload) &&
+ pickedStreams(runtime.requestPayload) === undefined &&
+ runtime.state.networks.some(
+ (network) => network.id === networkId && network.status === 'on'
+ )
+ )
+ count++
+ }
+ return count
+ }
+
private runningCount(): number {
let running = 0
for (const runtime of this.runtimes.values()) {
@@ -479,6 +499,7 @@ export class DownloadManager {
const host: TransferHost = {
networks: this.networks,
limits: this.limits,
+ downloadsOn: (networkId) => this.httpDownloadsOn(networkId),
reconcile: () => this.reconcile(runtime),
failDownload: (message, discard) => this.failDownload(runtime, message, discard),
failNetwork: (network, message) => this.failNetwork(runtime, network, message),
@@ -510,6 +531,7 @@ export class DownloadManager {
const host: TransferHost = {
networks: this.networks,
limits: this.limits,
+ downloadsOn: (networkId) => this.httpDownloadsOn(networkId),
reconcile: () => this.reconcile(runtime),
failDownload: (message, discard) => this.failDownload(runtime, message, discard),
failNetwork: (network, message) => this.failNetwork(runtime, network, message),
diff --git a/src/main/download/httpTransfer.ts b/src/main/download/httpTransfer.ts
index 10c5291..d680878 100644
--- a/src/main/download/httpTransfer.ts
+++ b/src/main/download/httpTransfer.ts
@@ -165,12 +165,17 @@ export function splittable(request: StartHttpDownloadRequest): boolean {
}
/** How many streams a download runs is only for it to work out when there can be more than one,
- * and unless a test fixes it. */
-function concurrencyFor(request: StartHttpDownloadRequest): ConcurrencyController | null {
+ * and unless a test fixes it. On Auto, it shares each network with the others running on it. */
+function concurrencyFor(
+ request: StartHttpDownloadRequest,
+ host: TransferHost
+): ConcurrencyController | null {
if (!splittable(request) || testStreamsPerNetwork() !== null) return null
const picked = pickedStreams(request)
return picked === undefined
- ? new ConcurrencyController(MAX_STREAMS_PER_NETWORK)
+ ? new ConcurrencyController(MAX_STREAMS_PER_NETWORK, true, (networkId) =>
+ Math.floor(MAX_STREAMS_PER_NETWORK / Math.max(1, host.downloadsOn(networkId)))
+ )
: new ConcurrencyController(picked, false)
}
@@ -311,7 +316,7 @@ export class HttpTransfer implements Transfer {
private readonly host: TransferHost
) {
this.acceptedVersions = [requestedVersion(runtime.requestPayload)]
- this.concurrency = concurrencyFor(runtime.requestPayload)
+ this.concurrency = concurrencyFor(runtime.requestPayload, host)
}
/**
diff --git a/src/main/download/transfer.ts b/src/main/download/transfer.ts
index c667576..4bbecef 100644
--- a/src/main/download/transfer.ts
+++ b/src/main/download/transfer.ts
@@ -72,6 +72,9 @@ export interface TransferHost {
networks: NetworkMonitor
/** Every byte received goes through it (see Limits.take). */
limits: Limits
+ /** Running downloads on Auto that have `networkId` on, this one included: how many share its
+ * streams (see concurrency.ts). */
+ downloadsOn(networkId: string): number
/** The manager's reconcile: networks first, then the transfer's own. */
reconcile(): void
failDownload(message: string, discard?: boolean): void
diff --git a/src/main/settings.ts b/src/main/settings.ts
index 16e6489..2964a47 100644
--- a/src/main/settings.ts
+++ b/src/main/settings.ts
@@ -70,12 +70,9 @@ function sanitizeSettings(parsed: unknown): AppSettings {
if (typeof destinationDir === 'string' && isAbsolute(destinationDir)) {
settings.destinationDir = destinationDir
}
- if (
- Number.isInteger(downloadsAtOnce) &&
- (downloadsAtOnce as number) >= DOWNLOADS_AT_ONCE.min &&
- (downloadsAtOnce as number) <= DOWNLOADS_AT_ONCE.max
- ) {
- settings.downloadsAtOnce = downloadsAtOnce as number
+ // Clamped rather than dropped: a count saved under an older, higher limit keeps the most allowed.
+ if (Number.isInteger(downloadsAtOnce) && (downloadsAtOnce as number) >= DOWNLOADS_AT_ONCE.min) {
+ settings.downloadsAtOnce = Math.min(downloadsAtOnce as number, DOWNLOADS_AT_ONCE.max)
}
if (parsed.networkPreferences !== undefined) {
settings.networkPreferences = sanitizeNetworkPreferences(parsed.networkPreferences)
diff --git a/src/shared/types.ts b/src/shared/types.ts
index 39b9d59..c82a063 100644
--- a/src/shared/types.ts
+++ b/src/shared/types.ts
@@ -341,7 +341,7 @@ export interface AppSettings {
/** bytes: MB/s. bits: Mbps, as internet plans and speed tests count them. */
export type SpeedUnit = 'bytes' | 'bits'
-export const DOWNLOADS_AT_ONCE = { default: 2, min: 1, max: 8 }
+export const DOWNLOADS_AT_ONCE = { default: 2, min: 1, max: 4 }
/** Seconds of speed history a download keeps (a sample a second), and so the span the throughput
* chart always shows. */
export const SPEED_HISTORY_SECONDS = 60