Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

fix: reconnect connections stuck at a crawl so downloads don't hang at 99%

Anmol Kapil6eddb0ec5169
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.ts
index f15dccf..4dc3f16 100644
--- a/e2e/integrity.spec.ts
+++ b/e2e/integrity.spec.ts
@@ -1,5 +1,5 @@
import { BLOCK, expect, LAN_ADDRESS, test } from './fixtures'
-import { seededBytes, type Fault } from './origin'
+import { seededBytes, type Fault, type LoggedRequest } from './origin'
// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the
// automatic checks in fixtures.ts — is that a download never ends `completed` with wrong bytes:
@@ -48,6 +48,61 @@ test.describe('transient server faults are retried to a correct file @smoke', ()
})
})
+test.describe('a connection stuck at a crawl @smoke', () => {
+ test.use({ appEnv: { PLEXO_E2E_SLOW_WARMUP_MS: '500', PLEXO_E2E_SLOW_FOR_MS: '1500' } })
+
+ const BLOCKS = 48
+ const tookFullBlock = (entry: LoggedRequest): boolean =>
+ entry.bytesSent === entry.range!.end! - entry.range!.start + 1
+
+ for (const [label, crawlStart] of [
+ ['mid-download', BLOCK],
+ ['on the last block (the 99% case)', (BLOCKS - 1) * BLOCK]
+ ] as const) {
+ test(`${label}: reconnected and resumed`, async ({ plexo, serve }) => {
+ const origin = await serve({ size: BLOCKS * BLOCK, bytesPerSecond: 256 * 1024 })
+ let crawled = false
+ // 2 KB/s: this one block alone would take ~32 s.
+ origin.setRule(({ range }) =>
+ range?.start === crawlStart && !crawled ? ((crawled = true), { crawl: 2048 }) : 'ok'
+ )
+
+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })
+ const state = await plexo.waitForStatus('completed', 15_000)
+
+ const requests = origin.chunkRequests()
+ const slow = requests.find((entry) => typeof entry.fault === 'object')!
+ expect(tookFullBlock(slow), 'the slow request was cut off').toBe(false)
+ const resumed = requests.find(
+ (entry) =>
+ entry.n > slow.n &&
+ entry.range!.start > slow.range!.start &&
+ entry.range!.start <= slow.range!.end!
+ )
+ expect(resumed, 'the block resumed from where the slow connection stopped').toBeTruthy()
+ expect(state.chunks.reduce((sum, chunk) => sum + chunk.retryCount, 0)).toBe(0)
+ })
+ }
+
+ test('connections that are all equally slow are left alone', async ({ plexo, serve }) => {
+ const origin = await serve({ size: BLOCKS * BLOCK, bytesPerSecond: 256 * 1024 })
+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })
+ await plexo.waitForStatus('completed', 15_000)
+ expect(origin.chunkRequests().every(tookFullBlock), 'no request was cut off').toBe(true)
+ })
+
+ test('a block is refreshed at most twice, then left to finish', async ({ plexo, serve }) => {
+ const origin = await serve({ size: 3 * BLOCK })
+ // Two fast blocks set the reference; every request for the third crawls.
+ origin.setRule(({ range }) => (range && range.start >= 2 * BLOCK ? { crawl: 8192 } : 'ok'))
+
+ await plexo.start(origin.url(), origin.sha256, { connections: 1 })
+ await plexo.waitForStatus('completed', 30_000)
+ const lastBlock = origin.chunkRequests().filter((entry) => entry.range!.start >= 2 * BLOCK)
+ expect(lastBlock).toHaveLength(3)
+ })
+})
+
test.describe('servers without range support @smoke', () => {
// With no ranges the only way to recover a dropped connection is to start the file over.
for (const [label, contentLength] of [
diff --git a/e2e/origin.ts b/e2e/origin.ts
index 5d64aa0..74ae47f 100644
--- a/e2e/origin.ts
+++ b/e2e/origin.ts
@@ -29,6 +29,8 @@ export type Fault =
| { cutAfter: number }
/** Sends this many body bytes, then ends the response cleanly (a short body). */
| { endAfter: number }
+ /** Trickles the body at this rate in small pieces — slow, but never silent long enough to stall. */
+ | { crawl: number }
| { redirect: string }
export interface OriginRequest {
@@ -245,6 +247,8 @@ export class Origin {
bodyEnd = Math.min(bodyEnd, start + fault.endAfter)
}
const cutAfter = typeof fault === 'object' && 'cutAfter' in fault ? fault.cutAfter : null
+ const crawl = typeof fault === 'object' && 'crawl' in fault ? fault.crawl : null
+ const rate = crawl ?? this.options.bytesPerSecond
if (this.options.contentLength !== false && cutAfter === null) {
headers['Content-Length'] = bodyEnd - start
@@ -268,7 +272,7 @@ export class Origin {
return
}
- let next = Math.min(bodyEnd, position + PIECE_BYTES)
+ let next = Math.min(bodyEnd, position + (crawl ? 256 : PIECE_BYTES))
if (cutAfter !== null) next = Math.min(next, start + cutAfter)
if (holdable && this.holdAt !== null) {
if (position === this.holdAt) {
@@ -296,10 +300,8 @@ export class Origin {
// Yield so other connections — and the app's pause/cancel — interleave with this one.
await new Promise((resolve) => setImmediate(resolve))
}
- if (this.options.bytesPerSecond) {
- await new Promise((resolve) =>
- setTimeout(resolve, (piece.length / this.options.bytesPerSecond!) * 1000)
- )
+ if (rate) {
+ await new Promise((resolve) => setTimeout(resolve, (piece.length / rate) * 1000))
}
}
diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.ts
index ac337e7..dbd8413 100644
--- a/src/main/download/downloadManager.ts
+++ b/src/main/download/downloadManager.ts
@@ -40,6 +40,13 @@ import {
interface ChunkRuntime {
controller: AbortController
partPath: string
+ /** Aborts only the in-flight block request, to reconnect a slow connection. Null between requests. */
+ refresh: AbortController | null
+ /** When this connection last (re)connected; it isn't judged until SLOW_WARMUP_MS after. */
+ warmSince: number
+ slowSince: number | null
+ /** Speed of its last finished block, start to end — 0 until it finishes one. */
+ lastBlockSpeed: number
}
interface SpeedSample {
@@ -63,6 +70,8 @@ interface DownloadRuntime {
/** The version this download started on, plus any since confirmed to serve identical bytes
* (see confirmSameBytes). Not persisted: after a restart, a confirmation is simply redone. */
acceptedVersions: FileVersion[]
+ /** Slow-connection refreshes per block index, capped at MAX_REFRESHES_PER_BLOCK. */
+ refreshesByBlock: Map<number, number>
}
interface PersistedDownload {
@@ -120,6 +129,26 @@ function retryDelayMs(attempt: number): number {
return Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
}
+// A single TCP stream can get stuck at a crawl (loss-collapsed congestion window, a bad CDN node
+// or route) while its siblings on the same network run fine. It never goes silent, so the stall
+// watchdog can't catch it; a fresh connection usually lands somewhere healthy. Only relative
+// thresholds — nothing here knows how fast the network ought to be.
+const SLOW_RATIO = 0.1
+const SLOW_WARMUP_MS = testKnobs.slowWarmupMs
+const SLOW_FOR_MS = testKnobs.slowForMs
+// Bounds every case where refreshing can't help (the whole network got slower, a stale
+// reference): at worst a block pays for a couple of cheap reconnects, then is left alone.
+const MAX_REFRESHES_PER_BLOCK = 2
+// Idle connections look for work every 250 ms, so waiting a bit longer hands a refreshed block
+// to a connection that is already proven fast, if there is one.
+const REFRESH_HANDOFF_MS = 300
+
+function median(values: number[]): number {
+ const sorted = [...values].sort((a, b) => a - b)
+ const mid = sorted.length >> 1
+ return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2
+}
+
/** Records that `interfaceId` delivered `deltaBytes` of this block. Attribution is per-network
* rather than a single winner because one block can be started on one network and finished on
* another (a retry after a dropped connection, or a pause/resume), and the grid colors blocks
@@ -342,7 +371,8 @@ export class DownloadManager {
totalBlocks: state.totalBlocks ?? blocks.length,
persistenceChain: Promise.resolve(),
removed: false,
- acceptedVersions: [requestedVersion(persisted.requestPayload)]
+ acceptedVersions: [requestedVersion(persisted.requestPayload)],
+ refreshesByBlock: new Map()
})
} catch {
// Ignore incomplete or corrupt manifests; other downloads can still be restored.
@@ -597,7 +627,8 @@ export class DownloadManager {
totalBlocks: blocks.length,
persistenceChain: Promise.resolve(),
removed: false,
- acceptedVersions: [requestedVersion(requestPayload)]
+ acceptedVersions: [requestedVersion(requestPayload)],
+ refreshesByBlock: new Map()
}
this.runtimes.set(id, runtime)
await this.persistNow(runtime)
@@ -798,6 +829,7 @@ export class DownloadManager {
runtime.state.speedBytesPerSec = newSpeed
this.scheduleUpdate(runtime)
}
+ this.refreshSlowConnections(runtime, now)
}, 500)
speedTicker.unref()
@@ -842,6 +874,47 @@ export class DownloadManager {
await this.discardUnfinishedDestination(runtime)
}
+ /**
+ * Reconnects a connection running under SLOW_RATIO of its network's reference speed for
+ * SLOW_FOR_MS. The reference is the median of what connections on the same network are doing
+ * now and did on their last finished block — its own included, which covers the tail (everyone
+ * else is done) and a network with a single connection. Networks are never compared with each
+ * other: cellular is expected to be slower than Wi-Fi.
+ */
+ private refreshSlowConnections(runtime: DownloadRuntime, now: number): void {
+ // Without ranges a reconnect restarts the whole file from byte 0.
+ if (!runtime.requestPayload.supportsRanges) return
+
+ const warm = (self: ChunkRuntime): boolean => now - self.warmSince >= SLOW_WARMUP_MS
+ for (const chunk of runtime.state.chunks) {
+ const self = runtime.chunkRuntimes.get(chunk.id)
+ const blockIndex = chunk.currentBlockIndex
+ if (!self?.refresh || !warm(self) || blockIndex === undefined) continue
+ const refreshes = runtime.refreshesByBlock.get(blockIndex) ?? 0
+ if (refreshes >= MAX_REFRESHES_PER_BLOCK) continue
+
+ const reference: number[] = []
+ for (const other of runtime.state.chunks) {
+ if (other.interfaceId !== chunk.interfaceId) continue
+ const peer = runtime.chunkRuntimes.get(other.id)
+ if (!peer) continue
+ if (peer.lastBlockSpeed > 0) reference.push(peer.lastBlockSpeed)
+ if (other !== chunk && peer.refresh && warm(peer)) reference.push(other.speedBytesPerSec)
+ }
+
+ if (reference.length === 0 || chunk.speedBytesPerSec >= median(reference) * SLOW_RATIO) {
+ self.slowSince = null
+ continue
+ }
+ self.slowSince ??= now
+ if (now - self.slowSince >= SLOW_FOR_MS) {
+ runtime.refreshesByBlock.set(blockIndex, refreshes + 1)
+ self.refresh.abort()
+ self.refresh = null
+ }
+ }
+ }
+
private notify(title: string, body: string): void {
if (testKnobs.userDataDir || !Notification.isSupported()) return
try {
@@ -874,7 +947,15 @@ export class DownloadManager {
}
const controller = new AbortController()
- runtime.chunkRuntimes.set(chunk.id, { controller, partPath: '' })
+ const self: ChunkRuntime = {
+ controller,
+ partPath: '',
+ refresh: null,
+ warmSince: Date.now(),
+ slowSince: null,
+ lastBlockSpeed: 0
+ }
+ runtime.chunkRuntimes.set(chunk.id, self)
let attempt = 0
@@ -910,10 +991,7 @@ export class DownloadManager {
this.scheduleUpdate(runtime)
const partPath = join(runtime.tempDir, `part-${block.index}`)
- const chunkRuntime = runtime.chunkRuntimes.get(chunk.id)
- if (chunkRuntime) {
- chunkRuntime.partPath = partPath
- }
+ self.partPath = partPath
// Without range support the server can only send the file from the start, so a retry or
// resume begins again at byte 0 rather than asking for a Range it will ignore.
@@ -936,6 +1014,9 @@ export class DownloadManager {
}
let lastReportedThisRun = 0
+ const refresh = new AbortController()
+ self.refresh = refresh
+ const attemptStartedAt = Date.now()
try {
const runDownload = isSimulatedUrl(runtime.requestPayload.url)
@@ -948,7 +1029,7 @@ export class DownloadManager {
localAddress: iface.address,
destinationPath: partPath,
append: resumeOffset > 0,
- signal: controller.signal,
+ signal: AbortSignal.any([controller.signal, refresh.signal]),
acceptedVersions: runtime.acceptedVersions,
onProgress: (bytesThisRun) => {
const delta = bytesThisRun - lastReportedThisRun
@@ -968,6 +1049,8 @@ export class DownloadManager {
const blockBytes = block.rangeEnd - block.rangeStart + 1
creditBlockBytes(block, iface.id, blockBytes - block.bytesDownloaded)
block.bytesDownloaded = blockBytes
+ self.lastBlockSpeed =
+ ((blockBytes - resumeOffset) / Math.max(1, Date.now() - attemptStartedAt)) * 1000
}
attempt = 0
this.recomputeAggregates(runtime)
@@ -985,6 +1068,16 @@ export class DownloadManager {
break
}
+ if (refresh.signal.aborted) {
+ // Not a failure: no retry counted, no backoff. Whoever leases the block next resumes
+ // it from its part file on a new connection.
+ block.status = 'pending'
+ await delay(REFRESH_HANDOFF_MS, controller.signal)
+ self.warmSince = Date.now()
+ self.slowSince = null
+ continue
+ }
+
const message = error instanceof Error ? error.message : String(error)
chunk.error = message
@@ -1058,6 +1151,10 @@ export class DownloadManager {
break
}
chunk.status = 'downloading'
+ self.warmSince = Date.now()
+ self.slowSince = null
+ } finally {
+ self.refresh = null
}
}
diff --git a/src/main/testKnobs.ts b/src/main/testKnobs.ts
index da1ed64..b1913d8 100644
--- a/src/main/testKnobs.ts
+++ b/src/main/testKnobs.ts
@@ -17,6 +17,8 @@ export const testKnobs = {
blockBytes: positiveNumber('PLEXO_E2E_BLOCK_BYTES', 8 * 1024 * 1024),
retryBaseDelayMs: positiveNumber('PLEXO_E2E_RETRY_BASE_MS', 1000),
stallTimeoutMs: positiveNumber('PLEXO_E2E_STALL_MS', 20_000),
+ slowWarmupMs: positiveNumber('PLEXO_E2E_SLOW_WARMUP_MS', 5_000),
+ slowForMs: positiveNumber('PLEXO_E2E_SLOW_FOR_MS', 10_000),
/** Skips the real GitHub check and pretends this version is available, for exercising the
* update banner without needing an actual newer release published. */
forceUpdateVersion: env['PLEXO_FORCE_UPDATE_VERSION']