Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
perf: let hedges write straight to the staging file
Anmol Kapilbdfa59ffb09e
diff --git a/e2e/blockProgress.spec.ts b/e2e/blockProgress.spec.tsindex 8df518a..29a0438 100644--- a/e2e/blockProgress.spec.ts+++ b/e2e/blockProgress.spec.ts@@ -98,13 +98,20 @@ test.describe('destination-side staging file', () => {await expect(readFile(second)).rejects.toMatchObject({ code: 'ENOENT' })})- test('out-of-order ranges and a hedge overwrite produce one exact file', async () => {+ test('out-of-order ranges and a racing hedge produce one exact file', async () => {const destination = join(dir, 'result.bin')const file = new DownloadFile(`${destination}.plexo`)await writeFile(file.path, '')- await file.writeBuffers(500, [bytes(500, LENGTH)])- await file.writeBuffers(0, [bytes(0, 600)])- await file.writeBuffers(400, [bytes(400, LENGTH)])+ const write = (position: number, data: Buffer): Promise<void> =>+ new Promise((resolve, reject) => {+ const writer = file.writer(position)+ writer.once('error', reject).once('close', resolve)+ writer.end(data)+ })+ // A later block lands first; then a primary and a hedge that began further in write over the+ // same range at once, the same bytes in whichever order the disk takes them.+ await write(500, bytes(500, LENGTH))+ await Promise.all([write(0, bytes(0, 600)), write(400, bytes(400, 600))])expect(await file.read(390, 20)).toEqual(bytes(390, 410))expect(await readFile(file.path)).toEqual(bytes(0, LENGTH))expect(await file.publish(destination, LENGTH, async () => {})).toBe(destination)diff --git a/e2e/hedge.spec.ts b/e2e/hedge.spec.tsindex 5ab627c..8b06d2e 100644--- a/e2e/hedge.spec.ts+++ b/e2e/hedge.spec.ts@@ -28,7 +28,8 @@ test.describe('racing a slow block', () => {}) => {const origin = await serve({ size: BLOCKS * BLOCK })// 64 KB at 2 KB/s would take half a minute. The hedge is quick but not instant, so the slow- // holder keeps writing past the point the hedge began at — which the merge must discard.+ // holder keeps writing over the range the hedge is writing too: the same bytes, so the file+ // still comes out exact.origin.setRule(answerRequestsInSlowBlock((n) => (n === 0 ? { crawl: 2000 } : { crawl: 100_000 }))())@@ -83,6 +84,29 @@ test.describe('racing a slow block', () => {expect(origin.chunkRequests().filter((r) => inSlowBlock(r.range))).toHaveLength(2)})+ test('a hedge cut off partway keeps what it wrote: the next one starts after it', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: BLOCKS * BLOCK })+ origin.setRule(+ answerRequestsInSlowBlock((n) =>+ // The drop comes a moment after the data, as a real one does: by then it's on disk.+ n === 0 ? { crawl: 2000 } : n === 1 ? { cutAfter: BLOCK / 2, afterMs: 300 } : 'ok'+ )()+ )++ await plexo.start(origin.url(), origin.sha256, { connections: 4 })+ await plexo.waitForStatus('completed', 10_000)++ const [, cut, next] = origin.chunkRequests().filter((r) => inSlowBlock(r.range))+ expect(next, 'a second hedge finished the block').toBeDefined()+ expect(+ next.range!.start,+ 'the second hedge began after what the first had written, not back where the holder was'+ ).toBeGreaterThanOrEqual(cut.range!.start + BLOCK / 4)+ })+test('paused while racing, then resumed: no half-hedge is left behind', async ({plexo,servediff --git a/e2e/origin.ts b/e2e/origin.tsindex 9aadd5e..efc6946 100644--- a/e2e/origin.ts+++ b/e2e/origin.ts@@ -25,8 +25,8 @@ export type Fault =/** Never answers at all. */| 'stallHeaders'| { status: number }- /** Sends this many body bytes, then drops the connection. */- | { cutAfter: number }+ /** Sends this many body bytes, then drops the connection — `afterMs` later, if given. */+ | { cutAfter: number; afterMs?: number }/** Sends this many body bytes, then ends the response cleanly (a short body). */| { endAfter: number }/** Trickles the body at this rate in small pieces — slow, but never silent long enough to stall. */@@ -257,6 +257,7 @@ export class Origin {bodyEnd = Math.min(bodyEnd, start + fault.endAfter)}const cutAfter = typeof fault === 'object' && 'cutAfter' in fault ? fault.cutAfter : null+ const cutDelayMs = typeof fault === 'object' && 'cutAfter' in fault ? fault.afterMs : undefinedconst crawl = typeof fault === 'object' && 'crawl' in fault ? fault.crawl : nullconst rate = crawl ?? this.options.bytesPerSecond@@ -278,6 +279,7 @@ export class Origin {while (position < bodyEnd) {if (res.destroyed) returnif (cutAfter !== null && position - start >= cutAfter) {+ if (cutDelayMs) await new Promise((resolve) => setTimeout(resolve, cutDelayMs))req.socket.destroy()return}diff --git a/src/main/download/downloadFile.ts b/src/main/download/downloadFile.tsindex f08eeec..2cbaef1 100644--- a/src/main/download/downloadFile.ts+++ b/src/main/download/downloadFile.ts@@ -23,23 +23,6 @@ export class DownloadFile {}}- async writeBuffers(position: number, buffers: readonly Buffer[]): Promise<void> {- const handle = await open(this.path, 'r+')- try {- for (const buffer of buffers) {- let written = 0- while (written < buffer.length) {- const result = await handle.write(buffer, written, buffer.length - written, position)- if (result.bytesWritten === 0) throw new Error('Could not write the download file')- written += result.bytesWritten- position += result.bytesWritten- }- }- } finally {- await handle.close()- }- }-/** Flushes the staging file before a recovery checkpoint or final publication. */async sync(): Promise<void> {const handle = await open(this.path, 'r+')diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex e2347b1..32a623a 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -1,7 +1,6 @@import { randomUUID } from 'node:crypto'import { readFile, readdir, rename, rm, stat, statfs, writeFile } from 'node:fs/promises'import { basename, join } from 'node:path'-import { Writable } from 'node:stream'import type { BrowserWindow } from 'electron'import { app, Notification } from 'electron'import { IpcChannels } from '../../shared/ipc-channels'@@ -43,6 +42,10 @@ type AbortReason = 'refresh' | 'lost'* One request for one block, by one stream. A block normally has a single (primary) attempt.* Near the end of a download a second (hedge) one may race it — see scheduler.ts — and then the* block is finished by whichever gets there first.+ *+ * Both write straight into the staging file at the block's own offsets. Every response is checked+ * to be the download's version before any of it is written, so racing attempts write identical+ * bytes and it doesn't matter which lands last: whatever either has written stays secured.*/interface Attempt {kind: 'primary' | 'hedge'@@ -58,21 +61,13 @@ interface Attempt {lastNetworkAt: number/** When the request was sent. */startedAt: number- /** A hedge stays in bounded memory until it wins; only primaries write to the staging file. */- hedgeBuffers: Buffer[]/** The network previously credited for this block's prefix. */previousWriter: string | undefined/** Aborts this request alone; ChunkRuntime.controller aborts the whole stream. */abort: AbortControllerabortReason: AbortReason | null- /** Set once this attempt has been chosen to finish the block, so a near-simultaneous finisher- * can tell it lost. */- won: boolean/** What the server's answer cost, for diagnosing slow connections (see PLEXO_DEBUG). */response: { ttfbMs: number; reusedSocket: boolean } | null- /** Resolves once the request is over and its writer closed. */- settled: Promise<void>- settle: () => void}/** What became of an attempt's request. */@@ -222,7 +217,6 @@ const SCHEDULER_POLICY: SchedulerPolicy = {hedgeAfterMs: testKnobs.hedgeAfterMs,maxHedgesPerBlock: 2}-const MAX_ACTIVE_HEDGES = 2 // At most two block-sized buffers in memory.// Idle connections look for work every 250 ms, so waiting a bit longer hands a refreshed block// to a connection that is already proven fast, if there is one.const REFRESH_HANDOFF_MS = 300@@ -692,13 +686,6 @@ export class DownloadManager {}}runtime.avoidNetworkByBlock.clear()- // A hedge's file is discarded, so its progress must not be in the saved manifest.- for (const attempts of runtime.attempts.values()) {- for (const hedge of attempts) {- const chunk = runtime.state.chunks.find((entry) => entry.id === hedge.streamId)- if (hedge.kind === 'hedge' && chunk) this.retractHedge(runtime, chunk, hedge)- }- }for (const chunk of runtime.state.chunks) chunk.hedge = undefinedfor (const chunkRuntime of runtime.chunkRuntimes.values()) {chunkRuntime.controller.abort()@@ -1092,8 +1079,6 @@ export class DownloadManager {work: Work): Attempt {const { block } = work- let settle!: () => void- const settled = new Promise<void>((resolve) => (settle = resolve))const attempt: Attempt = {kind: work.kind,block,@@ -1104,16 +1089,12 @@ export class DownloadManager {networkReceived: 0,lastNetworkAt: 0,startedAt: Date.now(),- hedgeBuffers: [],// Whoever held this block before now is the one whose tail bytes a truncation would// discard — captured before the lease overwrites the field.previousWriter: block.interfaceId,abort: new AbortController(),abortReason: null,- won: false,- response: null,- settled,- settle+ response: null}if (work.kind === 'primary') {@@ -1184,15 +1165,7 @@ export class DownloadManager {rangeStart: block.rangeStart + attempt.startOffset,rangeEnd: block.rangeEnd,connection: self.connection,- createDestination: () =>- attempt.kind === 'primary'- ? runtime.file.writer(block.rangeStart + (attempt.startOffset ?? 0))- : new Writable({- write: (chunk: Buffer, _encoding, callback) => {- attempt.hedgeBuffers.push(Buffer.from(chunk))- callback()- }- }),+ createDestination: () => runtime.file.writer(block.rangeStart + (attempt.startOffset ?? 0)),signal: AbortSignal.any([self.controller.signal, attempt.abort.signal]),acceptedVersions: runtime.acceptedVersions,onResponse: (info) => (attempt.response = info),@@ -1218,8 +1191,6 @@ export class DownloadManager {if (self.controller.signal.aborted || status !== 'downloading') return { type: 'stopped' }if (attempt.abortReason) return { type: 'aborted', reason: attempt.abortReason }return { type: 'failed', error }- } finally {- attempt.settle()}}@@ -1288,53 +1259,19 @@ export class DownloadManager {}}- private async finishCompleted(+ private finishCompleted(runtime: DownloadRuntime,chunk: ChunkState,self: ChunkRuntime,attempt: Attempt- ): Promise<'continue' | 'stop'> {+ ): 'continue' {const { block } = attempt- // Another attempt already finished this block, or is finishing it: these bytes are surplus.- if (block.status === 'completed' || runtime.attempts.get(block.index)?.some((a) => a.won)) {- await this.letGo(runtime, chunk, self, attempt, false)+ // Another attempt already finished this block: these bytes are surplus.+ if (block.status === 'completed') {+ this.letGo(runtime, chunk, self, attempt, false)return 'continue'}- attempt.won = true-- if (attempt.kind === 'hedge') {- // The primary writer must close before the winning bytes overwrite its range.- const rivals = (runtime.attempts.get(block.index) ?? []).filter((a) => a !== attempt)- for (const rival of rivals) this.abortAttempt(rival, 'lost')- await Promise.all(rivals.map((rival) => rival.settled))- const expected = block.rangeEnd === null ? 0 : block.rangeEnd - block.rangeStart + 1- const hedgeBytes = attempt.hedgeBuffers.reduce((sum, buffer) => sum + buffer.length, 0)- let merged = hedgeBytes === expected - (attempt.startOffset ?? 0)- if (merged) {- try {- await runtime.file.writeBuffers(- block.rangeStart + (attempt.startOffset ?? 0),- attempt.hedgeBuffers- )- } catch {- merged = false- }- }- attempt.hedgeBuffers.length = 0-- if (!merged) {- // A failed hedge commit leaves the primary's prefix; the next worker overwrites the rest.- this.endAttempt(runtime, self, attempt)- const removed = retractBlock(block, attempt.startOffset ?? 0, attempt.networkId)- chunk.bytesDownloaded = Math.max(0, chunk.bytesDownloaded - removed)- block.status = 'pending'- this.recomputeAggregates(runtime)- this.goIdle(chunk)- this.scheduleUpdate(runtime)- return 'continue'- }- }this.endAttempt(runtime, self, attempt)block.status = 'completed'@@ -1358,25 +1295,23 @@ export class DownloadManager {}/** The download was paused or cancelled while the request was in flight. */- private async finishStopped(+ private finishStopped(runtime: DownloadRuntime,chunk: ChunkState,self: ChunkRuntime,attempt: Attempt- ): Promise<'stop'> {+ ): 'stop' {const status = runtime.state.status as DownloadStatus- let cleanup: Promise<void> = Promise.resolve()if (attempt.kind === 'primary') {this.endAttempt(runtime, self, attempt)if (status === 'paused') attempt.block.status = 'pending'} else {- cleanup = this.letGo(runtime, chunk, self, attempt, false)+ this.letGo(runtime, chunk, self, attempt, false)}chunk.status = status === 'paused' ? 'paused' : 'cancelled'chunk.currentBlockIndex = undefinedchunk.hedge = undefinedchunk.speedBytesPerSec = 0- await cleanupreturn 'stop'}@@ -1390,13 +1325,7 @@ export class DownloadManager {// 'lost': another attempt decided the block, so its state is no longer this one's to touch.// 'refresh': not a failure — no retry counted, no backoff. Whoever takes the block next// resumes it from the staging file on a new connection.- await this.letGo(- runtime,- chunk,- self,- attempt,- reason === 'refresh' && attempt.networkReceived === 0- )+ this.letGo(runtime, chunk, self, attempt, reason === 'refresh' && attempt.networkReceived === 0)if (reason === 'refresh' && attempt.kind === 'primary') {// A moment before this stream asks again, so that a connection already proven fast, if// there is one, gets to the block before this one does.@@ -1435,18 +1364,17 @@ export class DownloadManager {runtime.acceptedVersions.push(error.seen)}chunk.error = undefined- await this.letGo(runtime, chunk, self, attempt, false)+ this.letGo(runtime, chunk, self, attempt, false)return 'continue'}if (verdict === 'different') {// Every other worker would hit the same new version, so stop them all now rather than// let each burn through its retries first.- const cleanup = this.letGo(runtime, chunk, self, attempt, false)+ this.letGo(runtime, chunk, self, attempt, false)chunk.status = 'error'runtime.state.status = 'error'runtime.state.error = messagefor (const other of runtime.chunkRuntimes.values()) other.controller.abort()- await cleanupreturn 'stop'}// 'unknown' — nothing to compare yet, or the check itself failed: retry like any other@@ -1456,7 +1384,7 @@ export class DownloadManager {if (error instanceof NoCompatibleRouteError) {// A redirect can move this worker to a host its network cannot reach. Return its block// to the shared queue so another compatible worker can finish it.- await this.letGo(runtime, chunk, self, attempt, deliveredNothing)+ this.letGo(runtime, chunk, self, attempt, deliveredNothing)chunk.status = 'error'if (runtime.state.chunks.every((entry) => entry.status === 'error')) {runtime.state.status = 'error'@@ -1466,16 +1394,16 @@ export class DownloadManager {return 'stop'}- // A hedge is optional work: when it fails, the block is exactly where it was.+ // A hedge is optional work: its failure isn't retried, and what it did write stays.if (attempt.kind === 'hedge') {- await this.letGo(runtime, chunk, self, attempt, deliveredNothing)+ this.letGo(runtime, chunk, self, attempt, deliveredNothing)return 'continue'}self.failures += 1chunk.retryCount += 1// Back to the queue, so any available worker can pick it up.- void this.letGo(runtime, chunk, self, attempt, deliveredNothing) // nothing to clean up: a primary+ this.letGo(runtime, chunk, self, attempt, deliveredNothing)if (self.failures > MAX_CHUNK_RETRIES) {chunk.status = 'error'@@ -1506,11 +1434,8 @@ export class DownloadManager {/*** The stream stops working on the attempt's block without having finished it. A primary hands- * the block back to the queue; a hedge just drops out, taking with it any progress that only it- * had made. When the attempt delivered nothing, its network is remembered (see scheduler.ts).- *- * Every change to the download's state is made before this returns, so no other stream can see- * it half done; the promise it returns is only the removal of a hedge's file, to be awaited.+ * the block back to the queue; a hedge just drops out. Either way what it wrote stays counted.+ * When the attempt delivered nothing, its network is remembered (see scheduler.ts).*/private letGo(runtime: DownloadRuntime,@@ -1518,32 +1443,13 @@ export class DownloadManager {self: ChunkRuntime,attempt: Attempt,deliveredNothing: boolean- ): Promise<void> {+ ): void {this.endAttempt(runtime, self, attempt)const { block } = attempt- if (attempt.kind === 'primary') {- // Not if another attempt has decided the block already: it is that one's to finish.- const decided = runtime.attempts.get(block.index)?.some((other) => other.won)- if (block.status === 'downloading' && !decided) block.status = 'pending'- } else {- this.retractHedge(runtime, chunk, attempt)- }+ // A block another attempt has finished stays finished.+ if (attempt.kind === 'primary' && block.status === 'downloading') block.status = 'pending'if (deliveredNothing) runtime.avoidNetworkByBlock.set(block.index, attempt.networkId)this.goIdle(chunk)- attempt.hedgeBuffers.length = 0- return Promise.resolve()- }-- /** Takes back the progress a hedge alone had made, now that it is not going to finish. */- private retractHedge(runtime: DownloadRuntime, chunk: ChunkState, hedge: Attempt): void {- // A finished block is finished: whoever lost the race for it has nothing to take back.- if (hedge.block.status === 'completed') return- const keep = Math.max(hedge.startOffset ?? 0, this.otherAttemptsPosition(runtime, hedge))- const removed = retractBlock(hedge.block, keep, hedge.networkId)- if (removed > 0) {- chunk.bytesDownloaded = Math.max(0, chunk.bytesDownloaded - removed)- this.recomputeAggregates(runtime)- }}private async runWorker(runtime: DownloadRuntime, chunk: ChunkState): Promise<void> {@@ -1589,15 +1495,6 @@ export class DownloadManager {Date.now(),SCHEDULER_POLICY)- if (- work?.kind === 'hedge' &&- [...runtime.attempts.values()].flat().filter((attempt) => attempt.kind === 'hedge')- .length >= MAX_ACTIVE_HEDGES- ) {- this.goIdle(chunk)- await delay(250, controller.signal)- continue- }if (!work) {this.goIdle(chunk)// While blocks are still in flight, stay available in case one fails and is handed back,