Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
fix: retry timing: busy servers, network changes, sleep, and a disk-stall false alarm
Claude34541f15181c
diff --git a/README.md b/README.mdindex b9e3953..41facfc 100644--- a/README.md+++ b/README.md@@ -73,8 +73,9 @@ File ──→ Split ─────┤ ├─- ⏸️ **Resumable downloads** — cleanly pause and resume downloads with progress saved in a destination-side staging file.- 💾 **Relaunch recovery** — interrupted downloads are restored as paused after Plexo restarts, with a small manifest in application data.- 🛡️ **Safe, integrity-checked resume** — re-verifies remote `ETag` and `Last-Modified` validators before resuming, refusing to resume (rather than corrupting the file) if the server-side file has changed.-- 🔁 **Automatic retry with backoff** — failed chunks are automatically returned to the queue and retried with exponential backoff (up to 5 retries, 1s–15s backoff).-- 💤 **Stall detection & watchdog** — automatically drops and re-queues connections that remain open but silent (>20s without incoming data).+- 🔁 **Automatic retry with backoff** — failed chunks go back to the queue and are retried with jittered exponential backoff (1s–15s). A dropped connection is retried for as long as its network is there; a busy server (429, 503, …) is waited out, honouring `Retry-After`; a server that answers wrongly gets 5 retries.+- 🔄 **Network changes and sleep** — a network that gets a new address, or a computer that wakes from sleep, gets its connections going again at once instead of waiting out a backoff; the computer is kept awake while a download runs.+- 💤 **Stall detection & watchdog** — automatically drops and re-queues connections that remain open but silent (>20s without incoming data, not counting time spent waiting on the disk).- 🔔 **Desktop notifications** — native desktop alerts when downloads complete or encounter errors.- 💾 **Upfront disk-space verification** — checks the destination volume before writing the staging file.- 🔀 **Mid-download redirect handling** — transparently follows 3xx HTTP redirects (up to 5 hops) during probing and individual chunk downloads.diff --git a/e2e/fixtures.ts b/e2e/fixtures.tsindex 504930d..f6fedf9 100644--- a/e2e/fixtures.ts+++ b/e2e/fixtures.ts@@ -96,6 +96,9 @@ export class PlexoApp {PLEXO_E2E_BLOCK_BYTES: String(BLOCK),PLEXO_E2E_RETRY_BASE_MS: '20',PLEXO_E2E_STALL_MS: '1500',+ // A busy server gives up after its retries alone, as any other wrong answer does,+ // unless a test waits it out on purpose.+ PLEXO_E2E_SERVER_BUSY_MS: '1',// Off unless a test asks for it: a hedge is an extra request, and most tests count them.PLEXO_E2E_HEDGE_MS: '600000',// Fixed for the same reason, and for downloads started through the UI.diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.tsindex 3537885..87d2cb8 100644--- a/e2e/integrity.spec.ts+++ b/e2e/integrity.spec.ts@@ -150,6 +150,37 @@ test('one network dies for good mid-download; the other finishes it @smoke', asyexpect(failedOverB.length, 'network b really did fail').toBeGreaterThan(0)})+test.describe('a busy server @smoke', () => {+ test.use({ appEnv: { PLEXO_E2E_SERVER_BUSY_MS: '60000' } })++ test('is waited out for as long as it asks, past the retries a wrong answer gets', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: SIZE })+ let busy = 0+ // More busy answers than both streams' retries together: a wrong answer this many times+ // would end the download.+ origin.setRule(({ range }) =>+ range && range.start > 0 && busy++ < 14+ ? { status: 503, headers: { 'Retry-After': '1' } }+ : 'ok'+ )+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })+ await plexo.waitForStatus('completed')+ expect(busy).toBeGreaterThan(14)++ // It asked the network to wait, so no stream asked again within the second, save a request+ // already on its way.+ for (const answer of origin.log.filter((entry) => entry.status === 503)) {+ const tooSoon = origin.log.filter(+ (entry) => entry.at > answer.at + 100 && entry.at < answer.at + 950+ )+ expect(tooSoon, `asked again within a second of request ${answer.n}`).toEqual([])+ }+ })+})+test.describe('permanent faults end in a clean error @smoke', () => {const PERMANENT: [string, Fault][] = [['every chunk request fails with 500', { status: 500 }],diff --git a/e2e/liveNetworks.spec.ts b/e2e/liveNetworks.spec.tsindex 9c66911..ae2101d 100644--- a/e2e/liveNetworks.spec.ts+++ b/e2e/liveNetworks.spec.ts@@ -149,3 +149,57 @@ test.describe('a network that can’t reach the server @smoke', () => {expect(network((await plexo.current())!, 'b')!.bytesDownloaded).toBeGreaterThan(0)})})++test.describe('a network that changes address', () => {+ // Far longer than the test waits: a stream only retries in time if it is woken.+ test.use({ appEnv: { PLEXO_E2E_RETRY_BASE_MS: '10000' } })++ test('gets its streams going again at once, not when their backoff runs out @smoke', async ({+ plexo,+ serve+ }) => {+ test.skip(!LAN_ADDRESS, 'needs a LAN address to act as the second network')+ const origin = await serve({ size: 192 * BLOCK, bytesPerSecond: 256 * 1024 })+ // Every connection over b's first address drops before a byte of the file.+ origin.setRule((request) =>+ overB(request) && request.range && request.range.end !== 0 ? { cutAfter: 0 } : 'ok'+ )+ await plexo.start(origin.url(), origin.sha256, { networks: ['a', 'b'], connections: 2 })+ await plexo.waitUntil((state) =>+ state.chunks.some((chunk) => chunk.interfaceId === 'b' && chunk.status === 'retrying')+ )++ // A new DHCP lease, say: b is now reached at another address, which gets through.+ await setNetworks(plexo, { a: NETWORKS['a'], b: '127.0.0.1' })+ const changedAt = Date.now()+ await plexo.waitUntil((state) => (network(state, 'b')?.bytesDownloaded ?? 0) > 0, 5000)+ expect(Date.now() - changedAt, 'well before a backoff of 8 s or more').toBeLessThan(5000)+ await plexo.waitForStatus('completed')+ })+})++test.describe('the computer wakes from sleep', () => {+ // Nothing else would notice the dead connection for a long while.+ test.use({ appEnv: { PLEXO_E2E_STALL_MS: '60000', PLEXO_E2E_SILENT_MS: '60000' } })++ test('a connection that died in its sleep is replaced at once @smoke', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: 16 * BLOCK })+ let stalled = 0+ // One block's answer stops after its headers, as a connection does across a sleep.+ origin.setRule(({ range }) =>+ range?.start === 3 * BLOCK && stalled++ === 0 ? 'stallBody' : 'ok'+ )+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })+ await plexo.waitUntil((state) => state.bytesDownloaded >= 15 * BLOCK)+ expect((await plexo.current())!.status).toBe('downloading')++ await plexo.evaluateMain((electron) => {+ electron.powerMonitor.emit('resume')+ }, null)+ await plexo.waitForStatus('completed', 5000)+ expect(stalled, 'the block was asked for again').toBeGreaterThan(1)+ })+})diff --git a/e2e/network.spec.ts b/e2e/network.spec.tsindex 8fdcd0b..4ff9c20 100644--- a/e2e/network.spec.ts+++ b/e2e/network.spec.ts@@ -35,8 +35,7 @@ test('AAAA-only hostname connects over IPv6 and keeps its HTTP Host header @smokawait new Promise<void>((resolve) => server.listen(0, '::1', resolve))const connection = new StreamConnection(() => ({ ...iface, addresses: [{ address: '::1', family: 6 }] }),- 2000,- async () => [{ address: '::1', family: 6 }]+ { timeoutMs: 2000, resolveHost: async () => [{ address: '::1', family: 6 }] })try {const port = (server.address() as AddressInfo).port@@ -62,12 +61,14 @@ test('a working route gets time to answer after an earlier route fails @smoke',{ address: '::1', family: 6 }]}),- 3000,- async () => [- { address: '127.0.0.1', family: 4 },- { address: '::1', family: 6 },- { address: '127.0.0.2', family: 4 }- ]+ {+ timeoutMs: 3000,+ resolveHost: async () => [+ { address: '127.0.0.1', family: 4 },+ { address: '::1', family: 6 },+ { address: '127.0.0.2', family: 4 }+ ]+ })try {const port = (server.address() as AddressInfo).port@@ -80,11 +81,10 @@ test('a working route gets time to answer after an earlier route fails @smoke',})test('DNS resolution obeys the request deadline @smoke', async () => {- const connection = new StreamConnection(- () => iface,- 100,- () => new Promise(() => {})- )+ const connection = new StreamConnection(() => iface, {+ timeoutMs: 100,+ resolveHost: () => new Promise(() => {})+ })await expect(connection.request(new URL('http://unresolved.example.test/file'), {})).rejects.toThrow('Could not resolve the download host in time')@@ -102,7 +102,7 @@ test('a kept-alive connection the server dropped is replaced without failing the})server.on('connection', () => connections++)await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))- const connection = new StreamConnection(() => loopback, 2000)+ const connection = new StreamConnection(() => loopback, { timeoutMs: 2000 })try {const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)expect(await body((await connection.request(url, {})).res)).toBe('answer 1')@@ -117,11 +117,10 @@ test('a kept-alive connection the server dropped is replaced without failing thetest('a request abandoned while still connecting gives up the connection with it @smoke', async () => {// DNS that never answers stands in for any connect that hangs: the abort has to end it, not the// deadline.- const connection = new StreamConnection(- () => iface,- 5000,- () => new Promise(() => {})- )+ const connection = new StreamConnection(() => iface, {+ timeoutMs: 5000,+ resolveHost: () => new Promise(() => {})+ })const abort = new AbortController()const started = Date.now()setTimeout(() => abort.abort(), 100)@@ -137,7 +136,7 @@ test('a network that comes back with a new address is reached at that one @smokeawait new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))// Gone, and then back: every new socket asks for the network as it is now.let current: NetworkInterfaceInfo | undefined- const connection = new StreamConnection(() => current, 2000)+ const connection = new StreamConnection(() => current, { timeoutMs: 2000 })try {const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)await expect(connection.request(url, {})).rejects.toThrow(/not connected/)@@ -149,3 +148,23 @@ test('a network that comes back with a new address is reached at that one @smokeawait new Promise<void>((resolve) => server.close(() => resolve()))}})++test('reconnect drops its sockets, and the next request opens a new one @smoke', async () => {+ let connections = 0+ const server = createServer((_req, res) => res.end('ok'))+ server.on('connection', () => connections++)+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))+ const connection = new StreamConnection(() => loopback, { timeoutMs: 2000 })+ try {+ const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)+ expect(await body((await connection.request(url, {})).res)).toBe('ok')+ expect(await body((await connection.request(url, {})).res)).toBe('ok')+ expect(connections, 'kept alive until then').toBe(1)+ connection.reconnect()+ expect(await body((await connection.request(url, {})).res)).toBe('ok')+ expect(connections).toBe(2)+ } finally {+ connection.close()+ await new Promise<void>((resolve) => server.close(() => resolve()))+ }+})diff --git a/e2e/origin.ts b/e2e/origin.tsindex 69557f9..52efc44 100644--- a/e2e/origin.ts+++ b/e2e/origin.ts@@ -24,7 +24,7 @@ export type Fault =| 'stallBody'/** Never answers at all. */| 'stallHeaders'- | { status: number }+ | { status: number; headers?: Record<string, string> }/** Sends this many body bytes, then drops the connection — `afterMs` later, if given. */| { cutAfter: number; afterMs?: number }/** Sends this many body bytes, then ends the response cleanly (a short body). */@@ -45,6 +45,8 @@ export interface OriginRequest {}export interface LoggedRequest extends OriginRequest {+ /** When it arrived (Date.now()). */+ at: numberfault: Faultstatus: numberbytesSent: number@@ -201,13 +203,13 @@ export class Origin {range: parseRange(req.headers.range)}const fault = this.rule(request) ?? 'ok'- const entry: LoggedRequest = { ...request, fault, status: 0, bytesSent: 0 }+ const entry: LoggedRequest = { ...request, at: Date.now(), fault, status: 0, bytesSent: 0 }this.log.push(entry)if (fault === 'stallHeaders') returnif (typeof fault === 'object' && 'status' in fault) {entry.status = fault.status- res.writeHead(fault.status).end()+ res.writeHead(fault.status, fault.headers).end()return}if (typeof fault === 'object' && 'redirect' in fault) {diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex 0d8b703..486c5e7 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -38,6 +38,32 @@ export class RemoteChangedError extends Error {}}+/** The server answered with a status that isn't the range asked for. */+export class HttpStatusError extends Error {+ constructor(+ readonly status: number,+ /** How long the server asked to be left alone (Retry-After), if it said. */+ readonly retryAfterMs: number | null+ ) {+ super(`Unexpected status ${status} for range request`)+ }++ /** A server that is busy, briefly broken or limiting requests: worth waiting out, not a sign+ * that asking again will never work. The statuses curl's --retry treats as transient. */+ get transient(): boolean {+ return [408, 429, 500, 502, 503, 504].includes(this.status)+ }+}++/** Retry-After (RFC 9110 §10.2.3), in seconds or as an HTTP date, as milliseconds from now. */+export function retryAfterMs(value: string | undefined, now = Date.now()): number | null {+ if (!value) return null+ const trimmed = value.trim()+ if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000+ const at = Date.parse(trimmed)+ return Number.isNaN(at) ? null : Math.max(0, at - now)+}+// A server that accepts the connection and then goes silent (no data, no// error, no close) would otherwise hang the chunk forever with no way to// detect or retry it.@@ -201,7 +227,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {// sent this chunk rather than the whole file.const isValidFullBody = status === 200 && rangeStart === 0if (status !== 206 && !isValidFullBody) {- fail(new Error(`Unexpected status ${status} for range request`))+ fail(new HttpStatusError(status, retryAfterMs(header(res, 'retry-after'))))res.resume()return}@@ -278,6 +304,9 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {else if (!settled) onProgress(progress)})) {+ // Waiting on the disk says nothing about the network: the watchdog stops until+ // the writer has caught up.+ clearWatchdog()res.pause()fileStream.once('drain', () => {resetWatchdog()diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex 5e61c82..17e9af7 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto'import { readFile, readdir, rename, rm, stat, statfs, writeFile } from 'node:fs/promises'import { basename, join } from 'node:path'import type { BrowserWindow } from 'electron'-import { app, Notification } from 'electron'+import { app, Notification, powerSaveBlocker } from 'electron'import { IpcChannels } from '../../shared/ipc-channels'import type {BlockState,@@ -23,7 +23,7 @@ import {type ConcurrencyPolicy,type Snapshot} from './concurrency'-import { downloadChunk, fetchRange, RemoteChangedError } from './chunkDownloader'+import { downloadChunk, fetchRange, HttpStatusError, RemoteChangedError } from './chunkDownloader'import { DownloadFile } from './downloadFile'import { compareVersion, type FileVersion } from './fileVersion'import { ensureDirectory, reserveDestinationPath } from './paths'@@ -96,6 +96,9 @@ interface ChunkRuntime {/** Cuts its waits short — a backoff, a look for work — when the stream stops, or when the run* has nothing left for it to do. */wait: AbortSignal+ /** Cuts a backoff short without stopping the stream: what it was waiting out has changed (see+ * wake). Replaced once used. */+ nudge: AbortController/** Its one connection to the server, reused from block to block. */connection: StreamConnection/** What the stream is fetching right now, if anything. */@@ -113,6 +116,9 @@ interface ChunkRuntime {/** Of those, the ones the server answered wrongly: MAX_CHUNK_RETRIES of them and it gives up.* A connection that failed isn't one — see finishFailed. */strikes: number+ /** When the server first answered busy (see HttpStatusError.transient) since this stream last+ * made progress: a busy server is waited out for SERVER_BUSY_FOR_MS, not MAX_CHUNK_RETRIES. */+ busySince: number | null/** Stopped for good, and to leave the list once its worker has (see retireStreams). */retiring: boolean}@@ -163,6 +169,9 @@ interface DownloadRuntime {* judged by (see concurrency.ts), and when it last showed it was alive — received something,* or came into use. */traffic: Map<string, { received: number; aliveAt: number }>+ /** By network id: until when the server asked (Retry-After) not to be sent new requests over+ * it. */+ holdUntil: Map<string, number>/** Decides how many streams each network runs; kept for the whole download, so a pause and* resume doesn't forget what it has found out. */concurrency: ConcurrencyController | null@@ -242,9 +251,20 @@ function calculateCurrentSpeed(samples: SpeedSample[] | undefined, time: number)const MAX_CHUNK_RETRIES = 5const RETRY_BASE_DELAY_MS = testKnobs.retryBaseDelayMsconst RETRY_MAX_DELAY_MS = 15_000-+// A network that can't reach the server keeps one stream asking (see reconcile). That is one+// cheap connection, so it asks often enough to find out soon after the network gets through.+const UNREACHABLE_RETRY_MS = 5_000+// A server that answers busy (429, 503, …) is waited out this long before a stream gives up on+// it, however many retries that takes; what it asks for in Retry-After is waited, up to+// RETRY_AFTER_MAX_MS.+const SERVER_BUSY_FOR_MS = testKnobs.serverBusyForMs+const RETRY_AFTER_MAX_MS = 120_000++/** Doubling from RETRY_BASE_DELAY_MS, with ±20% jitter as gRPC's backoff has: a network that+ * drops fails all its streams at once, and they shouldn't all come back at the same instant. */function retryDelayMs(attempt: number): number {- return Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)+ const exact = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)+ return exact * (0.8 + 0.4 * Math.random())}// A single TCP stream can get stuck at a crawl (loss-collapsed congestion window, a bad CDN node@@ -286,9 +306,11 @@ function concurrencyFor(request: StartDownloadRequest): ConcurrencyController |? new ConcurrencyController(CONCURRENCY_POLICY): null}-// Idle connections look for work every 250 ms, so waiting a bit longer hands a refreshed block-// to a connection that is already proven fast, if there is one.-const REFRESH_HANDOFF_MS = 300+/** How often a stream with nothing to do looks for work again. */+const IDLE_POLL_MS = 250+// Just longer than an idle stream takes to look for work, so a refreshed block goes to a+// connection that is already proven fast, if there is one.+const REFRESH_HANDOFF_MS = IDLE_POLL_MS + 50function median(values: number[]): number {const sorted = [...values].sort((a, b) => a - b)@@ -396,6 +418,7 @@ function newRuntime(attempts: new Map(),hedgesByBlock: new Map(),traffic: new Map(),+ holdUntil: new Map(),concurrency: concurrencyFor(requestPayload),sentUpdates: 0,sentBlocks: []@@ -434,6 +457,10 @@ export class DownloadManager {private runtimes = new Map<string, DownloadRuntime>()private readonly initialization: Promise<void>private suspending = false+ /** Each network's addresses as last seen, by id: what tells a network that has changed. */+ private seenAddresses = new Map<string, string[]>()+ /** The powerSaveBlocker keeping the computer awake while a download runs (see keepAwake). */+ private awakeBlocker: number | null = nullconstructor(private getWindow: () => BrowserWindow | null,@@ -796,16 +823,107 @@ export class DownloadManager {this.pushUpdate(runtime)}- /** The computer's networks changed (see NetworkMonitor). */+ /** The computer's networks changed (see NetworkMonitor). A network whose addresses changed+ * gets its streams going again at once: what they were waiting out may be what changed. One+ * that lost an address also drops its sockets, which may be bound to it. Chrome does the same+ * when its IP address changes (ERR_NETWORK_CHANGED). */networksChanged(): void {+ const changed = new Set<string>()+ const moved = new Set<string>()+ const seen = new Map<string, string[]>()+ for (const iface of this.networks.current ?? []) {+ const addresses = iface.addresses.map((entry) => entry.address)+ const before = this.seenAddresses.get(iface.id)+ seen.set(iface.id, addresses)+ if (before?.length === addresses.length && before.every((a) => addresses.includes(a))) {+ continue+ }+ changed.add(iface.id)+ if (before?.some((address) => !addresses.includes(address))) moved.add(iface.id)+ }+ this.seenAddresses = seen+for (const runtime of this.runtimes.values()) {const { status } = runtime.stateif (status !== 'downloading' && status !== 'paused') continuethis.reconcile(runtime)+ this.wake(+ runtime,+ (id) => changed.has(id),+ (id) => moved.has(id)+ )this.scheduleUpdate(runtime)}}+ /** The computer woke from sleep. Its sockets are likely dead, though nothing will say so until+ * a stall watchdog runs out, and every judgement made by the clock (a silent network, a+ * crawling connection, a step of the stream-count controller) spans the sleep. All of it starts+ * over. */+ systemResumed(): void {+ const now = Date.now()+ for (const runtime of this.runtimes.values()) {+ if (runtime.state.status !== 'downloading') continue+ for (const traffic of runtime.traffic.values()) traffic.aliveAt = now+ for (const self of runtime.chunkRuntimes.values()) {+ self.warmSince = now+ self.slowSince = null+ }+ this.adjustStreams(runtime, runtime.concurrency?.interrupt())+ this.wake(+ runtime,+ () => true,+ () => true+ )+ }+ }++ /**+ * Gets streams going again now, rather than when their backoff runs out: what held them back+ * has changed. The ones `reconnect` picks also drop their sockets, and what they were fetching+ * is taken up again on a new one: a refresh, so nothing is lost and no retry is counted.+ */+ private wake(+ runtime: DownloadRuntime,+ pick: (networkId: string) => boolean,+ reconnect: (networkId: string) => boolean+ ): void {+ for (const chunk of runtime.state.chunks) {+ const self = runtime.chunkRuntimes.get(chunk.id)+ if (!self || self.retiring || !pick(chunk.interfaceId)) continue+ self.failures = 0+ if (reconnect(chunk.interfaceId)) {+ if (self.attempt) this.abortAttempt(self.attempt, 'refresh')+ self.connection.reconnect()+ }+ self.nudge.abort()+ }+ }++ /** Waits out a backoff; cut short when the stream stops, or is woken (see wake). */+ private async backOff(self: ChunkRuntime, ms: number): Promise<void> {+ await delay(ms, AbortSignal.any([self.wait, self.nudge.signal]))+ if (self.nudge.signal.aborted) self.nudge = new AbortController()+ }++ /** Keeps the computer from sleeping while a download runs, which would stop it — as+ * qBittorrent and Transmission offer to. The display can still sleep. */+ private keepAwake(): void {+ const running = [...this.runtimes.values()].some(+ (runtime) => runtime.state.status === 'downloading'+ )+ try {+ if (running && this.awakeBlocker === null) {+ this.awakeBlocker = powerSaveBlocker.start('prevent-app-suspension')+ } else if (!running && this.awakeBlocker !== null) {+ powerSaveBlocker.stop(this.awakeBlocker)+ this.awakeBlocker = null+ }+ } catch {+ // Not every desktop can be kept awake; the download runs regardless.+ }+ }+async cancel(id: string): Promise<void> {const runtime = this.runtimes.get(id)if (@@ -1130,25 +1248,45 @@ export class DownloadManager {// Without ranges a reconnect restarts the whole file from byte 0.if (!runtime.requestPayload.supportsRanges) return+ let unreachable = falsefor (const chunk of runtime.state.chunks) {const self = runtime.chunkRuntimes.get(chunk.id)const attempt = self?.attemptif (!self || !attempt || attempt.abortReason) continue+ const silent = this.isSilent(runtime, attempt, now)+ // A refresh isn't a failure, so no failed request would say that the network can't get+ // through; a connection gone silent with the rest of its network says it instead, as soon+ // as SILENT_AFTER_MS rather than once a connect or stall timeout has run out.+ if (silent && this.markUnreachable(runtime, this.network(runtime, attempt.networkId), now)) {+ unreachable = true+ }+const index = attempt.block.indexconst refreshes = runtime.refreshesByBlock.get(index) ?? 0// A hedge is optional work: it is only ever dropped, never counted against its block.const isPrimary = attempt.kind === 'primary'if (isPrimary && refreshes >= MAX_REFRESHES_PER_BLOCK) continue- if (- this.isSilent(runtime, attempt, now) ||- (isPrimary && this.isCrawling(runtime, chunk, self, now))- ) {+ if (silent || (isPrimary && this.isCrawling(runtime, chunk, self, now))) {if (isPrimary) runtime.refreshesByBlock.set(index, refreshes + 1)this.abortAttempt(attempt, 'refresh')}}+ if (unreachable) this.reconcile(runtime)+ }++ /** Marks a network in use that has received nothing for SILENT_AFTER_MS as unable to reach the+ * server. Whether it did. */+ private markUnreachable(+ runtime: DownloadRuntime,+ network: DownloadNetwork,+ now: number+ ): boolean {+ if (network.status !== 'on') return false+ if (now - this.traffic(runtime, network.id).aliveAt < SILENT_AFTER_MS) return false+ network.status = 'unreachable'+ return true}private isSilent(runtime: DownloadRuntime, attempt: Attempt, now: number): boolean {@@ -1553,12 +1691,7 @@ export class DownloadManager {// What the server's answer says about the network: a connection failing while the whole// network has gone quiet means the network can't reach the server — dropped, or connected// with no way through. It stops being used, bar one stream that keeps trying (see reconcile).- if (- error instanceof ConnectionError &&- network.status === 'on' &&- Date.now() - this.traffic(runtime, network.id).aliveAt >= SILENT_AFTER_MS- ) {- network.status = 'unreachable'+ if (error instanceof ConnectionError && this.markUnreachable(runtime, network, Date.now())) {this.reconcile(runtime)}@@ -1570,14 +1703,26 @@ export class DownloadManager {// A network that can't get through retries as a matter of course: that's not news.if (network.status === 'on') network.retries += 1+ // What arrived before it failed shows the network works, and what was written shows the+ // server does: the count in a row starts over, and so does the backoff.+ if (attempt.networkReceived > 0) self.failures = 0+ if (attempt.received > 0) {+ self.strikes = 0+ self.busySince = null+ }self.failures += 1// Back to the queue, so any available worker can pick it up.this.letGo(runtime, chunk, self, attempt, deliveredNothing)// A connection that failed is the network's doing, and is tried again for as long as the- // network is there. A server that answered wrongly gets MAX_CHUNK_RETRIES in a row; then this- // stream gives up, and once all its network's have, so does the network.- if (!(error instanceof ConnectionError) && ++self.strikes > MAX_CHUNK_RETRIES) {+ // network is there. A server that answered wrongly gets MAX_CHUNK_RETRIES in a row, and one+ // that answered busy is waited out for SERVER_BUSY_FOR_MS as well; then this stream gives up,+ // and once all its network's have, so does the network.+ const now = Date.now()+ const busy = error instanceof HttpStatusError && error.transient+ if (busy) self.busySince ??= now+ const waitingOut = busy && now - (self.busySince ?? now) < SERVER_BUSY_FOR_MS+ if (!(error instanceof ConnectionError) && ++self.strikes > MAX_CHUNK_RETRIES && !waitingOut) {self.retiring = trueif (this.liveStreams(runtime, network.id).length === 0) {this.failNetwork(runtime, network, message)@@ -1585,9 +1730,22 @@ export class DownloadManager {return 'stop'}+ let wait = retryDelayMs(self.failures)+ if (network.status === 'unreachable') wait = Math.min(wait, UNREACHABLE_RETRY_MS)+ if (error instanceof HttpStatusError && error.transient && error.retryAfterMs !== null) {+ // Asked of whoever sent it — this network's address, to the server — not of this one+ // request: none of the network's streams sends another until then.+ const asked = Math.min(error.retryAfterMs, RETRY_AFTER_MAX_MS)+ wait = Math.max(wait, asked)+ runtime.holdUntil.set(+ network.id,+ Math.max(runtime.holdUntil.get(network.id) ?? 0, now + asked)+ )+ }+chunk.status = 'retrying'this.scheduleUpdate(runtime)- await delay(retryDelayMs(self.failures), self.wait)+ await this.backOff(self, wait)const statusAfterDelay = runtime.state.status as DownloadStatusif (self.controller.signal.aborted || statusAfterDelay !== 'downloading') {chunk.status = statusAfterDelay === 'paused' ? 'paused' : 'cancelled'@@ -1685,10 +1843,11 @@ export class DownloadManager {const self: ChunkRuntime = {controller,wait: AbortSignal.any([controller.signal, runtime.stop.signal]),- connection: new StreamConnection(- () => this.networks.find(chunk.interfaceId),- testKnobs.stallTimeoutMs- ),+ nudge: new AbortController(),+ connection: new StreamConnection(() => this.networks.find(chunk.interfaceId), {+ timeoutMs: testKnobs.stallTimeoutMs,+ connectTimeoutMs: testKnobs.connectTimeoutMs+ }),attempt: null,warmSince: Date.now(),slowSince: null,@@ -1696,6 +1855,7 @@ export class DownloadManager {receivedBytes: 0,failures: 0,strikes: 0,+ busySince: null,retiring: false}runtime.chunkRuntimes.set(chunk.id, self)@@ -1704,6 +1864,17 @@ export class DownloadManager {while (runtime.state.status === 'downloading') {if (controller.signal.aborted || self.retiring) break+ // The server asked this network to hold off: no new request over it until then. Each+ // stream comes back a little apart from the others.+ const holdFor = (runtime.holdUntil.get(chunk.interfaceId) ?? 0) - Date.now()+ if (holdFor > 0) {+ this.goIdle(chunk)+ chunk.status = 'retrying'+ this.scheduleUpdate(runtime)+ await this.backOff(self, holdFor + Math.random() * Math.min(1000, holdFor / 10))+ continue+ }+// Taken atomically: nothing between choosing the work and registering it can yield.const work = pickWork({@@ -1723,7 +1894,7 @@ export class DownloadManager {// or turns out to be slow enough to be worth racing.if (runtime.blocks.some((b) => b.status === 'pending' || b.status === 'downloading')) {this.scheduleUpdate(runtime)- await delay(250, self.wait)+ await delay(IDLE_POLL_MS, self.wait)continue}chunk.status = 'completed'@@ -1836,6 +2007,7 @@ export class DownloadManager {}private pushUpdate(runtime: DownloadRuntime, persist = true): void {+ this.keepAwake()// A removed download can still be winding down (workers finishing, cleanup). Its updates// would put it back on screen after the renderer has already moved on.if (this.runtimes.get(runtime.state.id) !== runtime) returndiff --git a/src/main/download/probe.ts b/src/main/download/probe.tsindex e4e6f94..713624e 100644--- a/src/main/download/probe.ts+++ b/src/main/download/probe.ts@@ -7,8 +7,10 @@ import { testKnobs } from '../testKnobs'const MAX_REDIRECTS = 5const USER_AGENT = 'Plexo/1.0'// A server that accepts the connection and never answers would otherwise hang the probe — and-// the link field's "Checking…" — forever. Same budget as a stalled chunk.+// the link field's "Checking…" — forever. Same budget as a stalled chunk, for the whole probe:+// redirects included, so a chain of slow hops can't stretch it.const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMs+const NO_RESPONSE = 'The server did not respond — check the link and try again'type Headers = Record<string, string | string[] | undefined>@@ -24,7 +26,7 @@ interface ProbeResponse {/** GET with a 1-byte range: cheaper than fetching the body, and unlike HEAD it* also tells us (via the 206 status) whether range requests actually work. */-function requestOneByte(url: URL): Promise<ProbeResponse> {+function requestOneByte(url: URL, deadline: number): Promise<ProbeResponse> {return new Promise((resolve, reject) => {const requester = url.protocol === 'https:' ? httpsRequest : httpRequestconst req = requester(@@ -42,8 +44,8 @@ function requestOneByte(url: URL): Promise<ProbeResponse> {})const timer = setTimeout(- () => req.destroy(new Error('The server did not respond — check the link and try again')),- PROBE_TIMEOUT_MS+ () => req.destroy(new Error(NO_RESPONSE)),+ Math.max(0, deadline - Date.now()))req.on('error', (error) => {clearTimeout(timer)@@ -110,9 +112,10 @@ async function requestFollowingRedirects(): Promise<{ current: URL; response: ProbeResponse | null }> {let current = new URL(rawUrl)let response: ProbeResponse | null = null+ const deadline = Date.now() + PROBE_TIMEOUT_MSfor (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {- response = await requestOneByte(current)+ response = await requestOneByte(current, deadline)if (response.statusCode >= 300 && response.statusCode < 400) {const location = headerValue(response.headers, 'location')if (!location) breakdiff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.tsindex 35acb68..9ddb0f1 100644--- a/src/main/ipc/handlers.ts+++ b/src/main/ipc/handlers.ts@@ -5,6 +5,7 @@ import {dialog,ipcMain,nativeTheme,+ powerMonitor,shell,type BrowserWindow,type IpcMainInvokeEvent@@ -63,6 +64,11 @@ export function registerIpcHandlers(getWindow: () => BrowserWindow | null): Downif (window && !window.isDestroyed()) window.webContents.send(IpcChannels.networksChanged, list)})const manager = new DownloadManager(getWindow, networks)+ // Waking from sleep, the networks may have changed without a poll in between to see it.+ powerMonitor.on('resume', () => {+ manager.systemResumed()+ void networks.refresh()+ })handle('listInterfaces', () => networks.refresh())diff --git a/src/main/network/routes.ts b/src/main/network/routes.tsindex ab0df62..f40e784 100644--- a/src/main/network/routes.ts+++ b/src/main/network/routes.ts@@ -118,10 +118,35 @@ export function resolveTargetWithin(const abortError = (): DOMException => new DOMException('Aborted', 'AbortError')+// RFC 8305 (Happy Eyeballs): each next address starts this long after the one before, unless+// that one has already failed, and the first to finish its handshake wins. An address that+// doesn't answer (IPv6 with no way out, a stale record) then costs a new connection 250 ms, not+// its whole timeout. It is also Node's autoSelectFamilyAttemptTimeout.+const ATTEMPT_DELAY_MS = 250++/** By network and host, the route that last connected: where the next connection starts. */+const lastGoodRoute = new Map<string, string>()+const routeKey = (route: NetworkRoute): string => `${route.localAddress} ${route.remoteAddress}`++/** The route that last worked first, then the rest alternating between address families, as+ * RFC 8305 §4 orders them. */+function attemptOrder(routes: NetworkRoute[], preferred: string | undefined): NetworkRoute[] {+ const first = routes.find((route) => routeKey(route) === preferred)+ const rest = routes.filter((route) => route !== first)+ const families = [...new Set(rest.map((route) => route.family))]+ const byFamily = families.map((family) => rest.filter((route) => route.family === family))+ const ordered: NetworkRoute[] = first ? [first] : []+ for (let i = 0; byFamily.some((list) => i < list.length); i++) {+ for (const list of byFamily) if (i < list.length) ordered.push(list[i])+ }+ return ordered+}+/**- * A socket to `host` through `iface`, trying each compatible route in DNS order until one- * completes its handshake. `secure` wraps the TCP socket in TLS before that point, so a route- * whose TLS handshake never finishes is given up on just like one that never connects.+ * A socket to `host` through `iface`, racing its compatible routes as RFC 8305 does: whichever+ * completes its handshake first is kept, and the others are dropped. `secure` wraps the TCP+ * socket in TLS before that point, so a route whose TLS handshake never finishes is given up on+ * just like one that never connects.*/async function connectOnInterface(iface: NetworkInterfaceInfo,@@ -133,52 +158,80 @@ async function connectOnInterface(resolveHost?: ResolveHost): Promise<Socket> {const deadline = Date.now() + timeoutMs- const routes = routesFor(iface, await resolveTargetWithin(host, timeoutMs, signal, resolveHost))- if (routes.length === 0) throw new NoCompatibleRouteError(host)- let lastError: Error = new Error('Connection failed')-- for (const [index, route] of routes.entries()) {- const remaining = deadline - Date.now()- if (remaining <= 0) break- // A stale address gets a short window, so the next can still be tried; the last route gets- // all that remains.- const window =- index === routes.length - 1 ? remaining : Math.min(3_000, Math.max(1, remaining / 2))- const tcp = connectRoute(route, port)- const socket = secure ? secure(tcp) : tcp- try {- await new Promise<void>((resolve, reject) => {- const settle = (error?: Error): void => {- clearTimeout(timer)- signal.removeEventListener('abort', onAbort)- socket.off(secure ? 'secureConnect' : 'connect', onConnected)- socket.off('error', settle)- tcp.off('error', settle)- if (error) reject(error)- else resolve()- }- const onConnected = (): void => settle()- const onAbort = (): void => settle(abortError())- const timer = setTimeout(- () => settle(new Error('Connection stalled: could not connect')),- window- )- socket.once(secure ? 'secureConnect' : 'connect', onConnected)- // A refused TCP connection is reported on the raw socket, not always on its TLS wrapper.- socket.once('error', settle)- tcp.once('error', settle)- if (signal.aborted) onAbort()- else signal.addEventListener('abort', onAbort, { once: true })- })- return socket- } catch (error) {- socket.destroy()- tcp.destroy()- if (signal.aborted) throw abortError()- lastError = error instanceof Error ? error : new Error(String(error))+ const found = routesFor(iface, await resolveTargetWithin(host, timeoutMs, signal, resolveHost))+ if (found.length === 0) throw new NoCompatibleRouteError(host)+ const memory = `${iface.id} ${host}:${port}`+ const routes = attemptOrder(found, lastGoodRoute.get(memory))++ return new Promise<Socket>((resolve, reject) => {+ const racing = new Set<{ tcp: Socket; socket: Socket }>()+ let next = 0+ let done = false+ let lastError: Error = new Error('Connection failed')+ let stagger: NodeJS.Timeout | undefined++ const finish = (error: Error | null, winner?: Socket): void => {+ if (done) return+ done = true+ clearTimeout(stagger)+ clearTimeout(timer)+ signal.removeEventListener('abort', onAbort)+ for (const { tcp, socket } of racing) {+ socket.destroy()+ tcp.destroy()+ }+ racing.clear()+ if (winner) resolve(winner)+ else reject(error)}- }- throw lastError+ const onAbort = (): void => finish(abortError())+ const timer = setTimeout(+ () => finish(new Error('Connection stalled: could not connect')),+ Math.max(1, deadline - Date.now())+ )++ const launch = (): void => {+ clearTimeout(stagger)+ if (done || next >= routes.length) return+ const route = routes[next++]+ const tcp = connectRoute(route, port)+ const socket = secure ? secure(tcp) : tcp+ const entry = { tcp, socket }+ racing.add(entry)+ const connected = secure ? 'secureConnect' : 'connect'++ const detach = (): void => {+ socket.off(connected, onConnected)+ socket.off('error', onError)+ tcp.off('error', onError)+ racing.delete(entry)+ }+ const onConnected = (): void => {+ detach()+ lastGoodRoute.set(memory, routeKey(route))+ finish(null, socket)+ }+ const onError = (error: Error): void => {+ detach()+ socket.destroy()+ tcp.destroy()+ lastError = error+ if (lastGoodRoute.get(memory) === routeKey(route)) lastGoodRoute.delete(memory)+ // A route that fails hands over at once rather than after its delay.+ if (next < routes.length) launch()+ else if (racing.size === 0) finish(lastError)+ }+ socket.once(connected, onConnected)+ // A refused TCP connection is reported on the raw socket, not always on its TLS wrapper.+ socket.once('error', onError)+ tcp.once('error', onError)+ if (next < routes.length) stagger = setTimeout(launch, ATTEMPT_DELAY_MS)+ }++ if (signal.aborted) return onAbort()+ signal.addEventListener('abort', onAbort, { once: true })+ launch()+ })}/** What a request passes its agent: Node hands a request's options to the agent's@@ -249,13 +302,25 @@ export class StreamConnection {private readonly http: HttpAgentprivate readonly https: HttpsAgent+ private readonly timeoutMs: number+constructor(/** The network, or undefined while it isn't connected. */network: () => NetworkInterfaceInfo | undefined,- /** How long a request may take to get its response headers, connecting included. */- private readonly timeoutMs: number,- resolveHost?: ResolveHost+ {+ timeoutMs,+ connectTimeoutMs = timeoutMs,+ resolveHost+ }: {+ /** How long a request may take to get its response headers, connecting included. */+ timeoutMs: number+ /** How long opening a socket may take: DNS, TCP and TLS. Shorter than `timeoutMs`, so a+ * network that can't get through is found out before a slow server would be. */+ connectTimeoutMs?: number+ resolveHost?: ResolveHost+ }) {+ this.timeoutMs = timeoutMsconst open: Open = async (options, secure) => {const iface = network()if (!iface) throw new Error('The network is not connected')@@ -264,7 +329,7 @@ export class StreamConnection {options.host ?? '',Number(options.port),secure,- timeoutMs,+ Math.min(connectTimeoutMs, timeoutMs),// Given up on when its request is, not only when the stream closes: a request abandoned// mid-connect (a stuck connection being replaced) shouldn't leave a handshake running.options.connectSignal@@ -285,6 +350,14 @@ export class StreamConnection {return this.send(target, headers, signal, true)}+ /** Drops its sockets, in use or kept for reuse, so the next request opens a new one: they may+ * be bound to an address the network no longer has, or have died while the computer slept.+ * Unlike close, it stays usable. */+ reconnect(): void {+ this.http.destroy()+ this.https.destroy()+ }+/** Closes its sockets, and gives up on any still connecting. */close(): void {this.lifetime.abort()diff --git a/src/main/testKnobs.ts b/src/main/testKnobs.tsindex 6c3e8e4..1ba61b2 100644--- a/src/main/testKnobs.ts+++ b/src/main/testKnobs.ts@@ -18,6 +18,10 @@ export const testKnobs = {blockBytes: positiveNumber('PLEXO_E2E_BLOCK_BYTES', 8 * 1024 * 1024),retryBaseDelayMs: positiveNumber('PLEXO_E2E_RETRY_BASE_MS', 1000),stallTimeoutMs: positiveNumber('PLEXO_E2E_STALL_MS', 20_000),+ connectTimeoutMs: positiveNumber('PLEXO_E2E_CONNECT_MS', 10_000),+ /** How long a server that keeps answering busy (429, 503, …) is waited out; see+ * downloadManager.ts. */+ serverBusyForMs: positiveNumber('PLEXO_E2E_SERVER_BUSY_MS', 5 * 60_000),slowWarmupMs: positiveNumber('PLEXO_E2E_SLOW_WARMUP_MS', 5_000),slowForMs: positiveNumber('PLEXO_E2E_SLOW_FOR_MS', 10_000),silentAfterMs: positiveNumber('PLEXO_E2E_SILENT_MS', 5_000),