Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

fix: retry timing: busy servers, network changes, sleep, and a disk-stall false alarm

Claude34541f15181c
diff --git a/README.md b/README.md
index b9e3953..41facfc 100644
--- a/README.md
+++ b/README.md
@@ -73,8 +73,9 @@ File ──→ Split ─────┤ ├─
- ⏸️ **Resumable downloads** — cleanly pause and resume downloads with progress saved in a destination-side staging file.
- 💾 **Relaunch recovery** — interrupted downloads are restored as paused after Plexo restarts, with a small manifest in application data.
- 🛡️ **Safe, integrity-checked resume** — re-verifies remote `ETag` and `Last-Modified` validators before resuming, refusing to resume (rather than corrupting the file) if the server-side file has changed.
-- 🔁 **Automatic retry with backoff** — failed chunks are automatically returned to the queue and retried with exponential backoff (up to 5 retries, 1s–15s backoff).
-- 💤 **Stall detection & watchdog** — automatically drops and re-queues connections that remain open but silent (>20s without incoming data).
+- 🔁 **Automatic retry with backoff** — failed chunks go back to the queue and are retried with jittered exponential backoff (1s–15s). A dropped connection is retried for as long as its network is there; a busy server (429, 503, …) is waited out, honouring `Retry-After`; a server that answers wrongly gets 5 retries.
+- 🔄 **Network changes and sleep** — a network that gets a new address, or a computer that wakes from sleep, gets its connections going again at once instead of waiting out a backoff; the computer is kept awake while a download runs.
+- 💤 **Stall detection & watchdog** — automatically drops and re-queues connections that remain open but silent (>20s without incoming data, not counting time spent waiting on the disk).
- 🔔 **Desktop notifications** — native desktop alerts when downloads complete or encounter errors.
- 💾 **Upfront disk-space verification** — checks the destination volume before writing the staging file.
- 🔀 **Mid-download redirect handling** — transparently follows 3xx HTTP redirects (up to 5 hops) during probing and individual chunk downloads.
diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts
index 504930d..f6fedf9 100644
--- a/e2e/fixtures.ts
+++ b/e2e/fixtures.ts
@@ -96,6 +96,9 @@ export class PlexoApp {
PLEXO_E2E_BLOCK_BYTES: String(BLOCK),
PLEXO_E2E_RETRY_BASE_MS: '20',
PLEXO_E2E_STALL_MS: '1500',
+ // A busy server gives up after its retries alone, as any other wrong answer does,
+ // unless a test waits it out on purpose.
+ PLEXO_E2E_SERVER_BUSY_MS: '1',
// Off unless a test asks for it: a hedge is an extra request, and most tests count them.
PLEXO_E2E_HEDGE_MS: '600000',
// Fixed for the same reason, and for downloads started through the UI.
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.ts
index 3537885..87d2cb8 100644
--- a/e2e/integrity.spec.ts
+++ b/e2e/integrity.spec.ts
@@ -150,6 +150,37 @@ test('one network dies for good mid-download; the other finishes it @smoke', asy
expect(failedOverB.length, 'network b really did fail').toBeGreaterThan(0)
})
+test.describe('a busy server @smoke', () => {
+ test.use({ appEnv: { PLEXO_E2E_SERVER_BUSY_MS: '60000' } })
+
+ test('is waited out for as long as it asks, past the retries a wrong answer gets', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: SIZE })
+ let busy = 0
+ // More busy answers than both streams' retries together: a wrong answer this many times
+ // would end the download.
+ origin.setRule(({ range }) =>
+ range && range.start > 0 && busy++ < 14
+ ? { status: 503, headers: { 'Retry-After': '1' } }
+ : 'ok'
+ )
+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })
+ await plexo.waitForStatus('completed')
+ expect(busy).toBeGreaterThan(14)
+
+ // It asked the network to wait, so no stream asked again within the second, save a request
+ // already on its way.
+ for (const answer of origin.log.filter((entry) => entry.status === 503)) {
+ const tooSoon = origin.log.filter(
+ (entry) => entry.at > answer.at + 100 && entry.at < answer.at + 950
+ )
+ expect(tooSoon, `asked again within a second of request ${answer.n}`).toEqual([])
+ }
+ })
+})
+
test.describe('permanent faults end in a clean error @smoke', () => {
const PERMANENT: [string, Fault][] = [
['every chunk request fails with 500', { status: 500 }],
diff --git a/e2e/liveNetworks.spec.ts b/e2e/liveNetworks.spec.ts
index 9c66911..ae2101d 100644
--- a/e2e/liveNetworks.spec.ts
+++ b/e2e/liveNetworks.spec.ts
@@ -149,3 +149,57 @@ test.describe('a network that can’t reach the server @smoke', () => {
expect(network((await plexo.current())!, 'b')!.bytesDownloaded).toBeGreaterThan(0)
})
})
+
+test.describe('a network that changes address', () => {
+ // Far longer than the test waits: a stream only retries in time if it is woken.
+ test.use({ appEnv: { PLEXO_E2E_RETRY_BASE_MS: '10000' } })
+
+ test('gets its streams going again at once, not when their backoff runs out @smoke', async ({
+ plexo,
+ serve
+ }) => {
+ test.skip(!LAN_ADDRESS, 'needs a LAN address to act as the second network')
+ const origin = await serve({ size: 192 * BLOCK, bytesPerSecond: 256 * 1024 })
+ // Every connection over b's first address drops before a byte of the file.
+ origin.setRule((request) =>
+ overB(request) && request.range && request.range.end !== 0 ? { cutAfter: 0 } : 'ok'
+ )
+ await plexo.start(origin.url(), origin.sha256, { networks: ['a', 'b'], connections: 2 })
+ await plexo.waitUntil((state) =>
+ state.chunks.some((chunk) => chunk.interfaceId === 'b' && chunk.status === 'retrying')
+ )
+
+ // A new DHCP lease, say: b is now reached at another address, which gets through.
+ await setNetworks(plexo, { a: NETWORKS['a'], b: '127.0.0.1' })
+ const changedAt = Date.now()
+ await plexo.waitUntil((state) => (network(state, 'b')?.bytesDownloaded ?? 0) > 0, 5000)
+ expect(Date.now() - changedAt, 'well before a backoff of 8 s or more').toBeLessThan(5000)
+ await plexo.waitForStatus('completed')
+ })
+})
+
+test.describe('the computer wakes from sleep', () => {
+ // Nothing else would notice the dead connection for a long while.
+ test.use({ appEnv: { PLEXO_E2E_STALL_MS: '60000', PLEXO_E2E_SILENT_MS: '60000' } })
+
+ test('a connection that died in its sleep is replaced at once @smoke', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: 16 * BLOCK })
+ let stalled = 0
+ // One block's answer stops after its headers, as a connection does across a sleep.
+ origin.setRule(({ range }) =>
+ range?.start === 3 * BLOCK && stalled++ === 0 ? 'stallBody' : 'ok'
+ )
+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })
+ await plexo.waitUntil((state) => state.bytesDownloaded >= 15 * BLOCK)
+ expect((await plexo.current())!.status).toBe('downloading')
+
+ await plexo.evaluateMain((electron) => {
+ electron.powerMonitor.emit('resume')
+ }, null)
+ await plexo.waitForStatus('completed', 5000)
+ expect(stalled, 'the block was asked for again').toBeGreaterThan(1)
+ })
+})
diff --git a/e2e/network.spec.ts b/e2e/network.spec.ts
index 8fdcd0b..4ff9c20 100644
--- a/e2e/network.spec.ts
+++ b/e2e/network.spec.ts
@@ -35,8 +35,7 @@ test('AAAA-only hostname connects over IPv6 and keeps its HTTP Host header @smok
await new Promise<void>((resolve) => server.listen(0, '::1', resolve))
const connection = new StreamConnection(
() => ({ ...iface, addresses: [{ address: '::1', family: 6 }] }),
- 2000,
- async () => [{ address: '::1', family: 6 }]
+ { timeoutMs: 2000, resolveHost: async () => [{ address: '::1', family: 6 }] }
)
try {
const port = (server.address() as AddressInfo).port
@@ -62,12 +61,14 @@ test('a working route gets time to answer after an earlier route fails @smoke',
{ address: '::1', family: 6 }
]
}),
- 3000,
- async () => [
- { address: '127.0.0.1', family: 4 },
- { address: '::1', family: 6 },
- { address: '127.0.0.2', family: 4 }
- ]
+ {
+ timeoutMs: 3000,
+ resolveHost: async () => [
+ { address: '127.0.0.1', family: 4 },
+ { address: '::1', family: 6 },
+ { address: '127.0.0.2', family: 4 }
+ ]
+ }
)
try {
const port = (server.address() as AddressInfo).port
@@ -80,11 +81,10 @@ test('a working route gets time to answer after an earlier route fails @smoke',
})
test('DNS resolution obeys the request deadline @smoke', async () => {
- const connection = new StreamConnection(
- () => iface,
- 100,
- () => new Promise(() => {})
- )
+ const connection = new StreamConnection(() => iface, {
+ timeoutMs: 100,
+ resolveHost: () => new Promise(() => {})
+ })
await expect(
connection.request(new URL('http://unresolved.example.test/file'), {})
).rejects.toThrow('Could not resolve the download host in time')
@@ -102,7 +102,7 @@ test('a kept-alive connection the server dropped is replaced without failing the
})
server.on('connection', () => connections++)
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
- const connection = new StreamConnection(() => loopback, 2000)
+ const connection = new StreamConnection(() => loopback, { timeoutMs: 2000 })
try {
const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)
expect(await body((await connection.request(url, {})).res)).toBe('answer 1')
@@ -117,11 +117,10 @@ test('a kept-alive connection the server dropped is replaced without failing the
test('a request abandoned while still connecting gives up the connection with it @smoke', async () => {
// DNS that never answers stands in for any connect that hangs: the abort has to end it, not the
// deadline.
- const connection = new StreamConnection(
- () => iface,
- 5000,
- () => new Promise(() => {})
- )
+ const connection = new StreamConnection(() => iface, {
+ timeoutMs: 5000,
+ resolveHost: () => new Promise(() => {})
+ })
const abort = new AbortController()
const started = Date.now()
setTimeout(() => abort.abort(), 100)
@@ -137,7 +136,7 @@ test('a network that comes back with a new address is reached at that one @smoke
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
// Gone, and then back: every new socket asks for the network as it is now.
let current: NetworkInterfaceInfo | undefined
- const connection = new StreamConnection(() => current, 2000)
+ const connection = new StreamConnection(() => current, { timeoutMs: 2000 })
try {
const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)
await expect(connection.request(url, {})).rejects.toThrow(/not connected/)
@@ -149,3 +148,23 @@ test('a network that comes back with a new address is reached at that one @smoke
await new Promise<void>((resolve) => server.close(() => resolve()))
}
})
+
+test('reconnect drops its sockets, and the next request opens a new one @smoke', async () => {
+ let connections = 0
+ const server = createServer((_req, res) => res.end('ok'))
+ server.on('connection', () => connections++)
+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+ const connection = new StreamConnection(() => loopback, { timeoutMs: 2000 })
+ try {
+ const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)
+ expect(await body((await connection.request(url, {})).res)).toBe('ok')
+ expect(await body((await connection.request(url, {})).res)).toBe('ok')
+ expect(connections, 'kept alive until then').toBe(1)
+ connection.reconnect()
+ expect(await body((await connection.request(url, {})).res)).toBe('ok')
+ expect(connections).toBe(2)
+ } finally {
+ connection.close()
+ await new Promise<void>((resolve) => server.close(() => resolve()))
+ }
+})
diff --git a/e2e/origin.ts b/e2e/origin.ts
index 69557f9..52efc44 100644
--- a/e2e/origin.ts
+++ b/e2e/origin.ts
@@ -24,7 +24,7 @@ export type Fault =
| 'stallBody'
/** Never answers at all. */
| 'stallHeaders'
- | { status: number }
+ | { status: number; headers?: Record<string, string> }
/** Sends this many body bytes, then drops the connection — `afterMs` later, if given. */
| { cutAfter: number; afterMs?: number }
/** Sends this many body bytes, then ends the response cleanly (a short body). */
@@ -45,6 +45,8 @@ export interface OriginRequest {
}
export interface LoggedRequest extends OriginRequest {
+ /** When it arrived (Date.now()). */
+ at: number
fault: Fault
status: number
bytesSent: number
@@ -201,13 +203,13 @@ export class Origin {
range: parseRange(req.headers.range)
}
const fault = this.rule(request) ?? 'ok'
- const entry: LoggedRequest = { ...request, fault, status: 0, bytesSent: 0 }
+ const entry: LoggedRequest = { ...request, at: Date.now(), fault, status: 0, bytesSent: 0 }
this.log.push(entry)
if (fault === 'stallHeaders') return
if (typeof fault === 'object' && 'status' in fault) {
entry.status = fault.status
- res.writeHead(fault.status).end()
+ res.writeHead(fault.status, fault.headers).end()
return
}
if (typeof fault === 'object' && 'redirect' in fault) {
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.ts
index 0d8b703..486c5e7 100644
--- a/src/main/download/chunkDownloader.ts
+++ b/src/main/download/chunkDownloader.ts
@@ -38,6 +38,32 @@ export class RemoteChangedError extends Error {
}
}
+/** The server answered with a status that isn't the range asked for. */
+export class HttpStatusError extends Error {
+ constructor(
+ readonly status: number,
+ /** How long the server asked to be left alone (Retry-After), if it said. */
+ readonly retryAfterMs: number | null
+ ) {
+ super(`Unexpected status ${status} for range request`)
+ }
+
+ /** A server that is busy, briefly broken or limiting requests: worth waiting out, not a sign
+ * that asking again will never work. The statuses curl's --retry treats as transient. */
+ get transient(): boolean {
+ return [408, 429, 500, 502, 503, 504].includes(this.status)
+ }
+}
+
+/** Retry-After (RFC 9110 §10.2.3), in seconds or as an HTTP date, as milliseconds from now. */
+export function retryAfterMs(value: string | undefined, now = Date.now()): number | null {
+ if (!value) return null
+ const trimmed = value.trim()
+ if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000
+ const at = Date.parse(trimmed)
+ return Number.isNaN(at) ? null : Math.max(0, at - now)
+}
+
// A server that accepts the connection and then goes silent (no data, no
// error, no close) would otherwise hang the chunk forever with no way to
// detect or retry it.
@@ -201,7 +227,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
// sent this chunk rather than the whole file.
const isValidFullBody = status === 200 && rangeStart === 0
if (status !== 206 && !isValidFullBody) {
- fail(new Error(`Unexpected status ${status} for range request`))
+ fail(new HttpStatusError(status, retryAfterMs(header(res, 'retry-after'))))
res.resume()
return
}
@@ -278,6 +304,9 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
else if (!settled) onProgress(progress)
})
) {
+ // Waiting on the disk says nothing about the network: the watchdog stops until
+ // the writer has caught up.
+ clearWatchdog()
res.pause()
fileStream.once('drain', () => {
resetWatchdog()
diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.ts
index 5e61c82..17e9af7 100644
--- a/src/main/download/downloadManager.ts
+++ b/src/main/download/downloadManager.ts
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto'
import { readFile, readdir, rename, rm, stat, statfs, writeFile } from 'node:fs/promises'
import { basename, join } from 'node:path'
import type { BrowserWindow } from 'electron'
-import { app, Notification } from 'electron'
+import { app, Notification, powerSaveBlocker } from 'electron'
import { IpcChannels } from '../../shared/ipc-channels'
import type {
BlockState,
@@ -23,7 +23,7 @@ import {
type ConcurrencyPolicy,
type Snapshot
} from './concurrency'
-import { downloadChunk, fetchRange, RemoteChangedError } from './chunkDownloader'
+import { downloadChunk, fetchRange, HttpStatusError, RemoteChangedError } from './chunkDownloader'
import { DownloadFile } from './downloadFile'
import { compareVersion, type FileVersion } from './fileVersion'
import { ensureDirectory, reserveDestinationPath } from './paths'
@@ -96,6 +96,9 @@ interface ChunkRuntime {
/** Cuts its waits short — a backoff, a look for work — when the stream stops, or when the run
* has nothing left for it to do. */
wait: AbortSignal
+ /** Cuts a backoff short without stopping the stream: what it was waiting out has changed (see
+ * wake). Replaced once used. */
+ nudge: AbortController
/** Its one connection to the server, reused from block to block. */
connection: StreamConnection
/** What the stream is fetching right now, if anything. */
@@ -113,6 +116,9 @@ interface ChunkRuntime {
/** Of those, the ones the server answered wrongly: MAX_CHUNK_RETRIES of them and it gives up.
* A connection that failed isn't one — see finishFailed. */
strikes: number
+ /** When the server first answered busy (see HttpStatusError.transient) since this stream last
+ * made progress: a busy server is waited out for SERVER_BUSY_FOR_MS, not MAX_CHUNK_RETRIES. */
+ busySince: number | null
/** Stopped for good, and to leave the list once its worker has (see retireStreams). */
retiring: boolean
}
@@ -163,6 +169,9 @@ interface DownloadRuntime {
* judged by (see concurrency.ts), and when it last showed it was alive — received something,
* or came into use. */
traffic: Map<string, { received: number; aliveAt: number }>
+ /** By network id: until when the server asked (Retry-After) not to be sent new requests over
+ * it. */
+ holdUntil: Map<string, number>
/** Decides how many streams each network runs; kept for the whole download, so a pause and
* resume doesn't forget what it has found out. */
concurrency: ConcurrencyController | null
@@ -242,9 +251,20 @@ function calculateCurrentSpeed(samples: SpeedSample[] | undefined, time: number)
const MAX_CHUNK_RETRIES = 5
const RETRY_BASE_DELAY_MS = testKnobs.retryBaseDelayMs
const RETRY_MAX_DELAY_MS = 15_000
-
+// A network that can't reach the server keeps one stream asking (see reconcile). That is one
+// cheap connection, so it asks often enough to find out soon after the network gets through.
+const UNREACHABLE_RETRY_MS = 5_000
+// A server that answers busy (429, 503, …) is waited out this long before a stream gives up on
+// it, however many retries that takes; what it asks for in Retry-After is waited, up to
+// RETRY_AFTER_MAX_MS.
+const SERVER_BUSY_FOR_MS = testKnobs.serverBusyForMs
+const RETRY_AFTER_MAX_MS = 120_000
+
+/** Doubling from RETRY_BASE_DELAY_MS, with ±20% jitter as gRPC's backoff has: a network that
+ * drops fails all its streams at once, and they shouldn't all come back at the same instant. */
function retryDelayMs(attempt: number): number {
- return Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
+ const exact = Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), RETRY_MAX_DELAY_MS)
+ return exact * (0.8 + 0.4 * Math.random())
}
// A single TCP stream can get stuck at a crawl (loss-collapsed congestion window, a bad CDN node
@@ -286,9 +306,11 @@ function concurrencyFor(request: StartDownloadRequest): ConcurrencyController |
? new ConcurrencyController(CONCURRENCY_POLICY)
: null
}
-// Idle connections look for work every 250 ms, so waiting a bit longer hands a refreshed block
-// to a connection that is already proven fast, if there is one.
-const REFRESH_HANDOFF_MS = 300
+/** How often a stream with nothing to do looks for work again. */
+const IDLE_POLL_MS = 250
+// Just longer than an idle stream takes to look for work, so a refreshed block goes to a
+// connection that is already proven fast, if there is one.
+const REFRESH_HANDOFF_MS = IDLE_POLL_MS + 50
function median(values: number[]): number {
const sorted = [...values].sort((a, b) => a - b)
@@ -396,6 +418,7 @@ function newRuntime(
attempts: new Map(),
hedgesByBlock: new Map(),
traffic: new Map(),
+ holdUntil: new Map(),
concurrency: concurrencyFor(requestPayload),
sentUpdates: 0,
sentBlocks: []
@@ -434,6 +457,10 @@ export class DownloadManager {
private runtimes = new Map<string, DownloadRuntime>()
private readonly initialization: Promise<void>
private suspending = false
+ /** Each network's addresses as last seen, by id: what tells a network that has changed. */
+ private seenAddresses = new Map<string, string[]>()
+ /** The powerSaveBlocker keeping the computer awake while a download runs (see keepAwake). */
+ private awakeBlocker: number | null = null
constructor(
private getWindow: () => BrowserWindow | null,
@@ -796,16 +823,107 @@ export class DownloadManager {
this.pushUpdate(runtime)
}
- /** The computer's networks changed (see NetworkMonitor). */
+ /** The computer's networks changed (see NetworkMonitor). A network whose addresses changed
+ * gets its streams going again at once: what they were waiting out may be what changed. One
+ * that lost an address also drops its sockets, which may be bound to it. Chrome does the same
+ * when its IP address changes (ERR_NETWORK_CHANGED). */
networksChanged(): void {
+ const changed = new Set<string>()
+ const moved = new Set<string>()
+ const seen = new Map<string, string[]>()
+ for (const iface of this.networks.current ?? []) {
+ const addresses = iface.addresses.map((entry) => entry.address)
+ const before = this.seenAddresses.get(iface.id)
+ seen.set(iface.id, addresses)
+ if (before?.length === addresses.length && before.every((a) => addresses.includes(a))) {
+ continue
+ }
+ changed.add(iface.id)
+ if (before?.some((address) => !addresses.includes(address))) moved.add(iface.id)
+ }
+ this.seenAddresses = seen
+
for (const runtime of this.runtimes.values()) {
const { status } = runtime.state
if (status !== 'downloading' && status !== 'paused') continue
this.reconcile(runtime)
+ this.wake(
+ runtime,
+ (id) => changed.has(id),
+ (id) => moved.has(id)
+ )
this.scheduleUpdate(runtime)
}
}
+ /** The computer woke from sleep. Its sockets are likely dead, though nothing will say so until
+ * a stall watchdog runs out, and every judgement made by the clock (a silent network, a
+ * crawling connection, a step of the stream-count controller) spans the sleep. All of it starts
+ * over. */
+ systemResumed(): void {
+ const now = Date.now()
+ for (const runtime of this.runtimes.values()) {
+ if (runtime.state.status !== 'downloading') continue
+ for (const traffic of runtime.traffic.values()) traffic.aliveAt = now
+ for (const self of runtime.chunkRuntimes.values()) {
+ self.warmSince = now
+ self.slowSince = null
+ }
+ this.adjustStreams(runtime, runtime.concurrency?.interrupt())
+ this.wake(
+ runtime,
+ () => true,
+ () => true
+ )
+ }
+ }
+
+ /**
+ * Gets streams going again now, rather than when their backoff runs out: what held them back
+ * has changed. The ones `reconnect` picks also drop their sockets, and what they were fetching
+ * is taken up again on a new one: a refresh, so nothing is lost and no retry is counted.
+ */
+ private wake(
+ runtime: DownloadRuntime,
+ pick: (networkId: string) => boolean,
+ reconnect: (networkId: string) => boolean
+ ): void {
+ for (const chunk of runtime.state.chunks) {
+ const self = runtime.chunkRuntimes.get(chunk.id)
+ if (!self || self.retiring || !pick(chunk.interfaceId)) continue
+ self.failures = 0
+ if (reconnect(chunk.interfaceId)) {
+ if (self.attempt) this.abortAttempt(self.attempt, 'refresh')
+ self.connection.reconnect()
+ }
+ self.nudge.abort()
+ }
+ }
+
+ /** Waits out a backoff; cut short when the stream stops, or is woken (see wake). */
+ private async backOff(self: ChunkRuntime, ms: number): Promise<void> {
+ await delay(ms, AbortSignal.any([self.wait, self.nudge.signal]))
+ if (self.nudge.signal.aborted) self.nudge = new AbortController()
+ }
+
+ /** Keeps the computer from sleeping while a download runs, which would stop it — as
+ * qBittorrent and Transmission offer to. The display can still sleep. */
+ private keepAwake(): void {
+ const running = [...this.runtimes.values()].some(
+ (runtime) => runtime.state.status === 'downloading'
+ )
+ try {
+ if (running && this.awakeBlocker === null) {
+ this.awakeBlocker = powerSaveBlocker.start('prevent-app-suspension')
+ } else if (!running && this.awakeBlocker !== null) {
+ powerSaveBlocker.stop(this.awakeBlocker)
+ this.awakeBlocker = null
+ }
+ } catch {
+ // Not every desktop can be kept awake; the download runs regardless.
+ }
+ }
+
async cancel(id: string): Promise<void> {
const runtime = this.runtimes.get(id)
if (
@@ -1130,25 +1248,45 @@ export class DownloadManager {
// Without ranges a reconnect restarts the whole file from byte 0.
if (!runtime.requestPayload.supportsRanges) return
+ let unreachable = false
for (const chunk of runtime.state.chunks) {
const self = runtime.chunkRuntimes.get(chunk.id)
const attempt = self?.attempt
if (!self || !attempt || attempt.abortReason) continue
+ const silent = this.isSilent(runtime, attempt, now)
+ // A refresh isn't a failure, so no failed request would say that the network can't get
+ // through; a connection gone silent with the rest of its network says it instead, as soon
+ // as SILENT_AFTER_MS rather than once a connect or stall timeout has run out.
+ if (silent && this.markUnreachable(runtime, this.network(runtime, attempt.networkId), now)) {
+ unreachable = true
+ }
+
const index = attempt.block.index
const refreshes = runtime.refreshesByBlock.get(index) ?? 0
// A hedge is optional work: it is only ever dropped, never counted against its block.
const isPrimary = attempt.kind === 'primary'
if (isPrimary && refreshes >= MAX_REFRESHES_PER_BLOCK) continue
- if (
- this.isSilent(runtime, attempt, now) ||
- (isPrimary && this.isCrawling(runtime, chunk, self, now))
- ) {
+ if (silent || (isPrimary && this.isCrawling(runtime, chunk, self, now))) {
if (isPrimary) runtime.refreshesByBlock.set(index, refreshes + 1)
this.abortAttempt(attempt, 'refresh')
}
}
+ if (unreachable) this.reconcile(runtime)
+ }
+
+ /** Marks a network in use that has received nothing for SILENT_AFTER_MS as unable to reach the
+ * server. Whether it did. */
+ private markUnreachable(
+ runtime: DownloadRuntime,
+ network: DownloadNetwork,
+ now: number
+ ): boolean {
+ if (network.status !== 'on') return false
+ if (now - this.traffic(runtime, network.id).aliveAt < SILENT_AFTER_MS) return false
+ network.status = 'unreachable'
+ return true
}
private isSilent(runtime: DownloadRuntime, attempt: Attempt, now: number): boolean {
@@ -1553,12 +1691,7 @@ export class DownloadManager {
// What the server's answer says about the network: a connection failing while the whole
// network has gone quiet means the network can't reach the server — dropped, or connected
// with no way through. It stops being used, bar one stream that keeps trying (see reconcile).
- if (
- error instanceof ConnectionError &&
- network.status === 'on' &&
- Date.now() - this.traffic(runtime, network.id).aliveAt >= SILENT_AFTER_MS
- ) {
- network.status = 'unreachable'
+ if (error instanceof ConnectionError && this.markUnreachable(runtime, network, Date.now())) {
this.reconcile(runtime)
}
@@ -1570,14 +1703,26 @@ export class DownloadManager {
// A network that can't get through retries as a matter of course: that's not news.
if (network.status === 'on') network.retries += 1
+ // What arrived before it failed shows the network works, and what was written shows the
+ // server does: the count in a row starts over, and so does the backoff.
+ if (attempt.networkReceived > 0) self.failures = 0
+ if (attempt.received > 0) {
+ self.strikes = 0
+ self.busySince = null
+ }
self.failures += 1
// Back to the queue, so any available worker can pick it up.
this.letGo(runtime, chunk, self, attempt, deliveredNothing)
// A connection that failed is the network's doing, and is tried again for as long as the
- // network is there. A server that answered wrongly gets MAX_CHUNK_RETRIES in a row; then this
- // stream gives up, and once all its network's have, so does the network.
- if (!(error instanceof ConnectionError) && ++self.strikes > MAX_CHUNK_RETRIES) {
+ // network is there. A server that answered wrongly gets MAX_CHUNK_RETRIES in a row, and one
+ // that answered busy is waited out for SERVER_BUSY_FOR_MS as well; then this stream gives up,
+ // and once all its network's have, so does the network.
+ const now = Date.now()
+ const busy = error instanceof HttpStatusError && error.transient
+ if (busy) self.busySince ??= now
+ const waitingOut = busy && now - (self.busySince ?? now) < SERVER_BUSY_FOR_MS
+ if (!(error instanceof ConnectionError) && ++self.strikes > MAX_CHUNK_RETRIES && !waitingOut) {
self.retiring = true
if (this.liveStreams(runtime, network.id).length === 0) {
this.failNetwork(runtime, network, message)
@@ -1585,9 +1730,22 @@ export class DownloadManager {
return 'stop'
}
+ let wait = retryDelayMs(self.failures)
+ if (network.status === 'unreachable') wait = Math.min(wait, UNREACHABLE_RETRY_MS)
+ if (error instanceof HttpStatusError && error.transient && error.retryAfterMs !== null) {
+ // Asked of whoever sent it — this network's address, to the server — not of this one
+ // request: none of the network's streams sends another until then.
+ const asked = Math.min(error.retryAfterMs, RETRY_AFTER_MAX_MS)
+ wait = Math.max(wait, asked)
+ runtime.holdUntil.set(
+ network.id,
+ Math.max(runtime.holdUntil.get(network.id) ?? 0, now + asked)
+ )
+ }
+
chunk.status = 'retrying'
this.scheduleUpdate(runtime)
- await delay(retryDelayMs(self.failures), self.wait)
+ await this.backOff(self, wait)
const statusAfterDelay = runtime.state.status as DownloadStatus
if (self.controller.signal.aborted || statusAfterDelay !== 'downloading') {
chunk.status = statusAfterDelay === 'paused' ? 'paused' : 'cancelled'
@@ -1685,10 +1843,11 @@ export class DownloadManager {
const self: ChunkRuntime = {
controller,
wait: AbortSignal.any([controller.signal, runtime.stop.signal]),
- connection: new StreamConnection(
- () => this.networks.find(chunk.interfaceId),
- testKnobs.stallTimeoutMs
- ),
+ nudge: new AbortController(),
+ connection: new StreamConnection(() => this.networks.find(chunk.interfaceId), {
+ timeoutMs: testKnobs.stallTimeoutMs,
+ connectTimeoutMs: testKnobs.connectTimeoutMs
+ }),
attempt: null,
warmSince: Date.now(),
slowSince: null,
@@ -1696,6 +1855,7 @@ export class DownloadManager {
receivedBytes: 0,
failures: 0,
strikes: 0,
+ busySince: null,
retiring: false
}
runtime.chunkRuntimes.set(chunk.id, self)
@@ -1704,6 +1864,17 @@ export class DownloadManager {
while (runtime.state.status === 'downloading') {
if (controller.signal.aborted || self.retiring) break
+ // The server asked this network to hold off: no new request over it until then. Each
+ // stream comes back a little apart from the others.
+ const holdFor = (runtime.holdUntil.get(chunk.interfaceId) ?? 0) - Date.now()
+ if (holdFor > 0) {
+ this.goIdle(chunk)
+ chunk.status = 'retrying'
+ this.scheduleUpdate(runtime)
+ await this.backOff(self, holdFor + Math.random() * Math.min(1000, holdFor / 10))
+ continue
+ }
+
// Taken atomically: nothing between choosing the work and registering it can yield.
const work = pickWork(
{
@@ -1723,7 +1894,7 @@ export class DownloadManager {
// or turns out to be slow enough to be worth racing.
if (runtime.blocks.some((b) => b.status === 'pending' || b.status === 'downloading')) {
this.scheduleUpdate(runtime)
- await delay(250, self.wait)
+ await delay(IDLE_POLL_MS, self.wait)
continue
}
chunk.status = 'completed'
@@ -1836,6 +2007,7 @@ export class DownloadManager {
}
private pushUpdate(runtime: DownloadRuntime, persist = true): void {
+ this.keepAwake()
// A removed download can still be winding down (workers finishing, cleanup). Its updates
// would put it back on screen after the renderer has already moved on.
if (this.runtimes.get(runtime.state.id) !== runtime) return
diff --git a/src/main/download/probe.ts b/src/main/download/probe.ts
index e4e6f94..713624e 100644
--- a/src/main/download/probe.ts
+++ b/src/main/download/probe.ts
@@ -7,8 +7,10 @@ import { testKnobs } from '../testKnobs'
const MAX_REDIRECTS = 5
const USER_AGENT = 'Plexo/1.0'
// A server that accepts the connection and never answers would otherwise hang the probe — and
-// the link field's "Checking…" — forever. Same budget as a stalled chunk.
+// the link field's "Checking…" — forever. Same budget as a stalled chunk, for the whole probe:
+// redirects included, so a chain of slow hops can't stretch it.
const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMs
+const NO_RESPONSE = 'The server did not respond — check the link and try again'
type Headers = Record<string, string | string[] | undefined>
@@ -24,7 +26,7 @@ interface ProbeResponse {
/** GET with a 1-byte range: cheaper than fetching the body, and unlike HEAD it
* also tells us (via the 206 status) whether range requests actually work. */
-function requestOneByte(url: URL): Promise<ProbeResponse> {
+function requestOneByte(url: URL, deadline: number): Promise<ProbeResponse> {
return new Promise((resolve, reject) => {
const requester = url.protocol === 'https:' ? httpsRequest : httpRequest
const req = requester(
@@ -42,8 +44,8 @@ function requestOneByte(url: URL): Promise<ProbeResponse> {
}
)
const timer = setTimeout(
- () => req.destroy(new Error('The server did not respond — check the link and try again')),
- PROBE_TIMEOUT_MS
+ () => req.destroy(new Error(NO_RESPONSE)),
+ Math.max(0, deadline - Date.now())
)
req.on('error', (error) => {
clearTimeout(timer)
@@ -110,9 +112,10 @@ async function requestFollowingRedirects(
): Promise<{ current: URL; response: ProbeResponse | null }> {
let current = new URL(rawUrl)
let response: ProbeResponse | null = null
+ const deadline = Date.now() + PROBE_TIMEOUT_MS
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
- response = await requestOneByte(current)
+ response = await requestOneByte(current, deadline)
if (response.statusCode >= 300 && response.statusCode < 400) {
const location = headerValue(response.headers, 'location')
if (!location) break
diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.ts
index 35acb68..9ddb0f1 100644
--- a/src/main/ipc/handlers.ts
+++ b/src/main/ipc/handlers.ts
@@ -5,6 +5,7 @@ import {
dialog,
ipcMain,
nativeTheme,
+ powerMonitor,
shell,
type BrowserWindow,
type IpcMainInvokeEvent
@@ -63,6 +64,11 @@ export function registerIpcHandlers(getWindow: () => BrowserWindow | null): Down
if (window && !window.isDestroyed()) window.webContents.send(IpcChannels.networksChanged, list)
})
const manager = new DownloadManager(getWindow, networks)
+ // Waking from sleep, the networks may have changed without a poll in between to see it.
+ powerMonitor.on('resume', () => {
+ manager.systemResumed()
+ void networks.refresh()
+ })
handle('listInterfaces', () => networks.refresh())
diff --git a/src/main/network/routes.ts b/src/main/network/routes.ts
index ab0df62..f40e784 100644
--- a/src/main/network/routes.ts
+++ b/src/main/network/routes.ts
@@ -118,10 +118,35 @@ export function resolveTargetWithin(
const abortError = (): DOMException => new DOMException('Aborted', 'AbortError')
+// RFC 8305 (Happy Eyeballs): each next address starts this long after the one before, unless
+// that one has already failed, and the first to finish its handshake wins. An address that
+// doesn't answer (IPv6 with no way out, a stale record) then costs a new connection 250 ms, not
+// its whole timeout. It is also Node's autoSelectFamilyAttemptTimeout.
+const ATTEMPT_DELAY_MS = 250
+
+/** By network and host, the route that last connected: where the next connection starts. */
+const lastGoodRoute = new Map<string, string>()
+const routeKey = (route: NetworkRoute): string => `${route.localAddress} ${route.remoteAddress}`
+
+/** The route that last worked first, then the rest alternating between address families, as
+ * RFC 8305 §4 orders them. */
+function attemptOrder(routes: NetworkRoute[], preferred: string | undefined): NetworkRoute[] {
+ const first = routes.find((route) => routeKey(route) === preferred)
+ const rest = routes.filter((route) => route !== first)
+ const families = [...new Set(rest.map((route) => route.family))]
+ const byFamily = families.map((family) => rest.filter((route) => route.family === family))
+ const ordered: NetworkRoute[] = first ? [first] : []
+ for (let i = 0; byFamily.some((list) => i < list.length); i++) {
+ for (const list of byFamily) if (i < list.length) ordered.push(list[i])
+ }
+ return ordered
+}
+
/**
- * A socket to `host` through `iface`, trying each compatible route in DNS order until one
- * completes its handshake. `secure` wraps the TCP socket in TLS before that point, so a route
- * whose TLS handshake never finishes is given up on just like one that never connects.
+ * A socket to `host` through `iface`, racing its compatible routes as RFC 8305 does: whichever
+ * completes its handshake first is kept, and the others are dropped. `secure` wraps the TCP
+ * socket in TLS before that point, so a route whose TLS handshake never finishes is given up on
+ * just like one that never connects.
*/
async function connectOnInterface(
iface: NetworkInterfaceInfo,
@@ -133,52 +158,80 @@ async function connectOnInterface(
resolveHost?: ResolveHost
): Promise<Socket> {
const deadline = Date.now() + timeoutMs
- const routes = routesFor(iface, await resolveTargetWithin(host, timeoutMs, signal, resolveHost))
- if (routes.length === 0) throw new NoCompatibleRouteError(host)
- let lastError: Error = new Error('Connection failed')
-
- for (const [index, route] of routes.entries()) {
- const remaining = deadline - Date.now()
- if (remaining <= 0) break
- // A stale address gets a short window, so the next can still be tried; the last route gets
- // all that remains.
- const window =
- index === routes.length - 1 ? remaining : Math.min(3_000, Math.max(1, remaining / 2))
- const tcp = connectRoute(route, port)
- const socket = secure ? secure(tcp) : tcp
- try {
- await new Promise<void>((resolve, reject) => {
- const settle = (error?: Error): void => {
- clearTimeout(timer)
- signal.removeEventListener('abort', onAbort)
- socket.off(secure ? 'secureConnect' : 'connect', onConnected)
- socket.off('error', settle)
- tcp.off('error', settle)
- if (error) reject(error)
- else resolve()
- }
- const onConnected = (): void => settle()
- const onAbort = (): void => settle(abortError())
- const timer = setTimeout(
- () => settle(new Error('Connection stalled: could not connect')),
- window
- )
- socket.once(secure ? 'secureConnect' : 'connect', onConnected)
- // A refused TCP connection is reported on the raw socket, not always on its TLS wrapper.
- socket.once('error', settle)
- tcp.once('error', settle)
- if (signal.aborted) onAbort()
- else signal.addEventListener('abort', onAbort, { once: true })
- })
- return socket
- } catch (error) {
- socket.destroy()
- tcp.destroy()
- if (signal.aborted) throw abortError()
- lastError = error instanceof Error ? error : new Error(String(error))
+ const found = routesFor(iface, await resolveTargetWithin(host, timeoutMs, signal, resolveHost))
+ if (found.length === 0) throw new NoCompatibleRouteError(host)
+ const memory = `${iface.id} ${host}:${port}`
+ const routes = attemptOrder(found, lastGoodRoute.get(memory))
+
+ return new Promise<Socket>((resolve, reject) => {
+ const racing = new Set<{ tcp: Socket; socket: Socket }>()
+ let next = 0
+ let done = false
+ let lastError: Error = new Error('Connection failed')
+ let stagger: NodeJS.Timeout | undefined
+
+ const finish = (error: Error | null, winner?: Socket): void => {
+ if (done) return
+ done = true
+ clearTimeout(stagger)
+ clearTimeout(timer)
+ signal.removeEventListener('abort', onAbort)
+ for (const { tcp, socket } of racing) {
+ socket.destroy()
+ tcp.destroy()
+ }
+ racing.clear()
+ if (winner) resolve(winner)
+ else reject(error)
}
- }
- throw lastError
+ const onAbort = (): void => finish(abortError())
+ const timer = setTimeout(
+ () => finish(new Error('Connection stalled: could not connect')),
+ Math.max(1, deadline - Date.now())
+ )
+
+ const launch = (): void => {
+ clearTimeout(stagger)
+ if (done || next >= routes.length) return
+ const route = routes[next++]
+ const tcp = connectRoute(route, port)
+ const socket = secure ? secure(tcp) : tcp
+ const entry = { tcp, socket }
+ racing.add(entry)
+ const connected = secure ? 'secureConnect' : 'connect'
+
+ const detach = (): void => {
+ socket.off(connected, onConnected)
+ socket.off('error', onError)
+ tcp.off('error', onError)
+ racing.delete(entry)
+ }
+ const onConnected = (): void => {
+ detach()
+ lastGoodRoute.set(memory, routeKey(route))
+ finish(null, socket)
+ }
+ const onError = (error: Error): void => {
+ detach()
+ socket.destroy()
+ tcp.destroy()
+ lastError = error
+ if (lastGoodRoute.get(memory) === routeKey(route)) lastGoodRoute.delete(memory)
+ // A route that fails hands over at once rather than after its delay.
+ if (next < routes.length) launch()
+ else if (racing.size === 0) finish(lastError)
+ }
+ socket.once(connected, onConnected)
+ // A refused TCP connection is reported on the raw socket, not always on its TLS wrapper.
+ socket.once('error', onError)
+ tcp.once('error', onError)
+ if (next < routes.length) stagger = setTimeout(launch, ATTEMPT_DELAY_MS)
+ }
+
+ if (signal.aborted) return onAbort()
+ signal.addEventListener('abort', onAbort, { once: true })
+ launch()
+ })
}
/** What a request passes its agent: Node hands a request's options to the agent's
@@ -249,13 +302,25 @@ export class StreamConnection {
private readonly http: HttpAgent
private readonly https: HttpsAgent
+ private readonly timeoutMs: number
+
constructor(
/** The network, or undefined while it isn't connected. */
network: () => NetworkInterfaceInfo | undefined,
- /** How long a request may take to get its response headers, connecting included. */
- private readonly timeoutMs: number,
- resolveHost?: ResolveHost
+ {
+ timeoutMs,
+ connectTimeoutMs = timeoutMs,
+ resolveHost
+ }: {
+ /** How long a request may take to get its response headers, connecting included. */
+ timeoutMs: number
+ /** How long opening a socket may take: DNS, TCP and TLS. Shorter than `timeoutMs`, so a
+ * network that can't get through is found out before a slow server would be. */
+ connectTimeoutMs?: number
+ resolveHost?: ResolveHost
+ }
) {
+ this.timeoutMs = timeoutMs
const open: Open = async (options, secure) => {
const iface = network()
if (!iface) throw new Error('The network is not connected')
@@ -264,7 +329,7 @@ export class StreamConnection {
options.host ?? '',
Number(options.port),
secure,
- timeoutMs,
+ Math.min(connectTimeoutMs, timeoutMs),
// Given up on when its request is, not only when the stream closes: a request abandoned
// mid-connect (a stuck connection being replaced) shouldn't leave a handshake running.
options.connectSignal
@@ -285,6 +350,14 @@ export class StreamConnection {
return this.send(target, headers, signal, true)
}
+ /** Drops its sockets, in use or kept for reuse, so the next request opens a new one: they may
+ * be bound to an address the network no longer has, or have died while the computer slept.
+ * Unlike close, it stays usable. */
+ reconnect(): void {
+ this.http.destroy()
+ this.https.destroy()
+ }
+
/** Closes its sockets, and gives up on any still connecting. */
close(): void {
this.lifetime.abort()
diff --git a/src/main/testKnobs.ts b/src/main/testKnobs.ts
index 6c3e8e4..1ba61b2 100644
--- a/src/main/testKnobs.ts
+++ b/src/main/testKnobs.ts
@@ -18,6 +18,10 @@ export const testKnobs = {
blockBytes: positiveNumber('PLEXO_E2E_BLOCK_BYTES', 8 * 1024 * 1024),
retryBaseDelayMs: positiveNumber('PLEXO_E2E_RETRY_BASE_MS', 1000),
stallTimeoutMs: positiveNumber('PLEXO_E2E_STALL_MS', 20_000),
+ connectTimeoutMs: positiveNumber('PLEXO_E2E_CONNECT_MS', 10_000),
+ /** How long a server that keeps answering busy (429, 503, …) is waited out; see
+ * downloadManager.ts. */
+ serverBusyForMs: positiveNumber('PLEXO_E2E_SERVER_BUSY_MS', 5 * 60_000),
slowWarmupMs: positiveNumber('PLEXO_E2E_SLOW_WARMUP_MS', 5_000),
slowForMs: positiveNumber('PLEXO_E2E_SLOW_FOR_MS', 10_000),
silentAfterMs: positiveNumber('PLEXO_E2E_SILENT_MS', 5_000),