Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Add stall detection, chunk retry, and resume integrity checks
Anmol Kapil6893072ca911
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex e66b9be..fbc662e 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -17,6 +17,11 @@ export interface ChunkDownloadOptions {signal: AbortSignal}+// A server that accepts the connection and then goes silent (no data, no+// error, no close) would otherwise hang the chunk forever with no way to+// detect or retry it.+const STALL_TIMEOUT_MS = 20_000+/** Downloads a single byte range of a URL, bound to one network interface, into a part file. */export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {const { url, rangeStart, rangeEnd, localAddress, destinationPath, append, onProgress, signal } =@@ -92,6 +97,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {)req.on('error', fail)+ req.setTimeout(STALL_TIMEOUT_MS, () => fail(new Error('Connection stalled: no response from server')))req.end()})}diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex a8065eb..0384b91 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -1,6 +1,6 @@import { randomUUID } from 'node:crypto'import { createReadStream, createWriteStream } from 'node:fs'-import { mkdir, rm } from 'node:fs/promises'+import { mkdir, rm, stat, truncate } from 'node:fs/promises'import { basename, join } from 'node:path'import type { BrowserWindow } from 'electron'import { app } from 'electron'@@ -71,6 +71,55 @@ function splitIntoRanges(return ranges}+const MAX_CHUNK_RETRIES = 5+const RETRY_BASE_DELAY_MS = 1000+const RETRY_MAX_DELAY_MS = 15_000++function retryDelayMs(attempt: number): number {+ return Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)+}++/** Waits, but returns early if the signal aborts (pause/cancel shouldn't wait out a retry backoff). */+function delay(ms: number, signal: AbortSignal): Promise<void> {+ return new Promise((resolve) => {+ if (signal.aborted) {+ resolve()+ return+ }+ const onAbort = (): void => {+ clearTimeout(timer)+ resolve()+ }+ const timer = setTimeout(() => {+ signal.removeEventListener('abort', onAbort)+ resolve()+ }, ms)+ signal.addEventListener('abort', onAbort, { once: true })+ })+}++// A part file's on-disk size is the only thing we can actually trust across+// a retry or resume — appending opens the file with flag 'a', which always+// writes at the real end-of-file regardless of what byte count we think+// we're at, so any mismatch (a crash, a write that hadn't flushed yet)+// would otherwise silently shift every byte after it. Truncating to the+// smaller of the two counts keeps the part file's length and our own+// bookkeeping in agreement before we append another byte to it.+async function reconcilePartFileSize(partPath: string, expectedBytes: number): Promise<number> {+ let actualBytes = 0+ try {+ actualBytes = (await stat(partPath)).size+ } catch {+ actualBytes = 0+ }++ const safeBytes = Math.min(expectedBytes, actualBytes)+ if (actualBytes !== safeBytes) {+ await truncate(partPath, safeBytes)+ }+ return safeBytes+}+/** Appends one file's bytes onto an already-open writable, without ending it. */function appendFileToStream(sourcePath: string, output: NodeJS.WritableStream): Promise<void> {return new Promise((resolve, reject) => {@@ -249,7 +298,6 @@ export class DownloadManager {private async runChunk(runtime: DownloadRuntime, chunk: ChunkState): Promise<void> {const iface = runtime.activeInterfaces[chunk.id]- const resumeOffset = chunk.bytesDownloadedconst controller = new AbortController()const existing = runtime.chunkRuntimes.get(chunk.id)const partPath = existing?.partPath ?? join(runtime.tempDir, `part-${chunk.id}`)@@ -257,39 +305,68 @@ export class DownloadManager {chunk.status = 'downloading'this.scheduleUpdate(runtime)- try {- await downloadChunk({- url: runtime.requestPayload.url,- rangeStart: chunk.rangeStart + resumeOffset,- rangeEnd: chunk.rangeEnd,- localAddress: iface.address,- destinationPath: partPath,- append: resumeOffset > 0,- signal: controller.signal,- onProgress: (bytesThisRun) =>- this.onChunkProgress(runtime, chunk.id, resumeOffset + bytesThisRun)- })- chunk.status = 'completed'- } catch (error) {- if (controller.signal.aborted) {- chunk.status = runtime.state.status === 'paused' ? 'paused' : 'cancelled'- } else {+ // A dropped connection shouldn't fail the whole download outright — most+ // network blips are transient, so give the chunk a few tries with+ // backoff before giving up on it (and, by extension, the download).+ let attempt = 0+ for (;;) {+ const resumeOffset = await reconcilePartFileSize(partPath, chunk.bytesDownloaded)+ if (resumeOffset !== chunk.bytesDownloaded) {+ chunk.bytesDownloaded = resumeOffset+ runtime.speedSamplesByChunk.delete(chunk.id)+ this.recomputeAggregates(runtime)+ }++ try {+ await downloadChunk({+ url: runtime.requestPayload.url,+ rangeStart: chunk.rangeStart + resumeOffset,+ rangeEnd: chunk.rangeEnd,+ localAddress: iface.address,+ destinationPath: partPath,+ append: resumeOffset > 0,+ signal: controller.signal,+ onProgress: (bytesThisRun) =>+ this.onChunkProgress(runtime, chunk.id, resumeOffset + bytesThisRun)+ })+ chunk.status = 'completed'+ break+ } catch (error) {+ if (controller.signal.aborted) {+ chunk.status = runtime.state.status === 'paused' ? 'paused' : 'cancelled'+ break+ }+const message = error instanceof Error ? error.message : String(error)- chunk.status = 'error'chunk.error = message- // Fail fast: one broken connection shouldn't leave the others- // downloading a file we're about to discard anyway.- if (runtime.state.status === 'downloading') {- runtime.state.status = 'error'- runtime.state.error = message- for (const chunkRuntime of runtime.chunkRuntimes.values()) {- chunkRuntime.controller.abort()+ attempt += 1++ if (attempt > MAX_CHUNK_RETRIES) {+ chunk.status = 'error'+ // Fail fast: one broken connection shouldn't leave the others+ // downloading a file we're about to discard anyway.+ if (runtime.state.status === 'downloading') {+ runtime.state.status = 'error'+ runtime.state.error = message+ for (const chunkRuntime of runtime.chunkRuntimes.values()) {+ chunkRuntime.controller.abort()+ }}+ break+ }++ chunk.status = 'retrying'+ this.scheduleUpdate(runtime)+ await delay(retryDelayMs(attempt), controller.signal)+ if (controller.signal.aborted) {+ chunk.status = runtime.state.status === 'paused' ? 'paused' : 'cancelled'+ break}+ chunk.status = 'downloading'}- } finally {- this.scheduleUpdate(runtime)}++ this.scheduleUpdate(runtime)}private onChunkProgress(@@ -309,6 +386,11 @@ export class DownloadManager {chunk.speedBytesPerSec = pushSpeedSample(samples, bytesDownloaded, now)chunk.bytesDownloaded = bytesDownloaded+ this.recomputeAggregates(runtime)+ this.scheduleUpdate(runtime)+ }++ private recomputeAggregates(runtime: DownloadRuntime): void {runtime.state.bytesDownloaded = runtime.state.chunks.reduce((sum, entry) => sum + entry.bytesDownloaded,0@@ -317,8 +399,6 @@ export class DownloadManager {(sum, entry) => sum + entry.speedBytesPerSec,0)-- this.scheduleUpdate(runtime)}private scheduleUpdate(runtime: DownloadRuntime): void {diff --git a/src/shared/types.ts b/src/shared/types.tsindex 1a976e0..ffdde16 100644--- a/src/shared/types.ts+++ b/src/shared/types.ts@@ -23,7 +23,14 @@ export interface ProbeResult {export type DownloadStatus = 'downloading' | 'paused' | 'completed' | 'error' | 'cancelled'-export type ChunkStatus = 'pending' | 'downloading' | 'paused' | 'completed' | 'error' | 'cancelled'+export type ChunkStatus =+ | 'pending'+ | 'downloading'+ | 'retrying'+ | 'paused'+ | 'completed'+ | 'error'+ | 'cancelled'export interface ChunkState {id: number