Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Probe ranges with 1 KiB instead of 1 byte (#105)
Anmol Kapil1b805b8b642f
diff --git a/e2e/chaos.spec.ts b/e2e/chaos.spec.tsindex d62eaaa..c7f61af 100644--- a/e2e/chaos.spec.ts+++ b/e2e/chaos.spec.ts@@ -10,7 +10,7 @@ import {PlexoApp,test} from './fixtures'-import { Origin, type Fault } from './origin'+import { isProbe, Origin, type Fault } from './origin'// H. Model-based chaos. fast-check generates random sequences of user actions and failures —// pause, resume, crash, quit, flaky server, waiting — and runs each against a real app and a@@ -152,9 +152,7 @@ test('any sequence of pauses, crashes and faults still ends in the exact file @c// Kept automatic across the relaunches a crash or a quit brings.const app = new PlexoApp(dirs, connections === 'auto' ? { PLEXO_E2E_STREAMS: '' } : {})const real: Real = { app, origin, id: '', faults: [] }- origin.setRule(({ range }) =>- range && !(range.start === 0 && range.end === 0) ? real.faults.shift() : undefined- )+ origin.setRule(({ range }) => (range && !isProbe(range) ? real.faults.shift() : undefined))try {await app.launch()diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.tsindex a60295e..9e2f39b 100644--- a/e2e/integrity.spec.ts+++ b/e2e/integrity.spec.ts@@ -1,5 +1,5 @@import { BLOCK, expect, LAN_ADDRESS, test } from './fixtures'-import { seededBytes, type Fault, type LoggedRequest } from './origin'+import { isProbe, seededBytes, type Fault, type LoggedRequest } from './origin'// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the// automatic checks in fixtures.ts — is that a download never ends `completed` with wrong bytes:@@ -91,11 +91,7 @@ test.describe('servers without range support @smoke', () => {const origin = await serve({ size: 20 * BLOCK, ranges: false })let transfers = 0origin.setRule(({ range }) =>- range?.start === 0 && range.end === 0- ? 'ok'- : transfers++ === 0- ? { cutAfter: 7 * BLOCK + 3 }- : 'ok'+ isProbe(range) ? 'ok' : transfers++ === 0 ? { cutAfter: 7 * BLOCK + 3 } : 'ok')await plexo.start(origin.url(), origin.sha256)await plexo.waitForStatus('completed')@@ -112,9 +108,7 @@ test('one network dies for good mid-download; the other finishes it @smoke', asylet fromB = 0// Network b serves two blocks, then every request over it fails.origin.setRule(({ from, range }) =>- from !== '127.0.0.1' && range && !(range.start === 0 && range.end === 0) && fromB++ >= 2- ? { status: 503 }- : 'ok'+ from !== '127.0.0.1' && range && !isProbe(range) && fromB++ >= 2 ? { status: 503 } : 'ok')await plexo.start(origin.url(), origin.sha256, { networks: ['a', 'b'], connections: 2 })await plexo.waitForStatus('completed')@@ -163,9 +157,7 @@ test.describe('permanent faults end in a clean error @smoke', () => {for (const [label, fault] of PERMANENT) {test(label, async ({ plexo, serve }) => {const origin = await serve({ size: SIZE })- origin.setRule(({ range }) =>- range && !(range.start === 0 && range.end === 0) ? fault : 'ok'- )+ origin.setRule(({ range }) => (range && !isProbe(range) ? fault : 'ok'))await plexo.start(origin.url(), origin.sha256, { connections: 2 })const state = await plexo.waitForStatus('error')expect(state.error).toBeTruthy()diff --git a/e2e/origin.ts b/e2e/origin.tsindex 52efc44..0c309d7 100644--- a/e2e/origin.ts+++ b/e2e/origin.ts@@ -33,6 +33,10 @@ export type Fault =| { crawl: number }| { redirect: string }+/** The app's probe (src/main/download/probe.ts asks for bytes=0-1023), as opposed to a chunk. */+export const isProbe = (range: OriginRequest['range']): boolean =>+ range?.start === 0 && range.end === 1023+export interface OriginRequest {/** 1-based count of requests this server has seen. */n: number@@ -189,9 +193,9 @@ export class Origin {this.holdRelease = null}- /** Requests that asked for byte ranges, i.e. the app's chunk requests (not the 1-byte probe). */+ /** Requests that asked for byte ranges, i.e. the app's chunk requests (not the probe). */chunkRequests(): LoggedRequest[] {- return this.log.filter((entry) => !(entry.range?.start === 0 && entry.range.end === 0))+ return this.log.filter((entry) => !isProbe(entry.range))}private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {@@ -218,8 +222,8 @@ export class Origin {return}- // The app's 1-byte probes (bytes=0-0) are never held — only real transfers are.- const holdable = !(request.range?.start === 0 && request.range.end === 0)+ // The app's probes are never held — only real transfers are.+ const holdable = !isProbe(request.range)if (holdable && this.holdAt !== null && request.range && request.range.start > this.holdAt) {await this.releasePromise}diff --git a/e2e/streams.spec.ts b/e2e/streams.spec.tsindex 90e8e8a..1979814 100644--- a/e2e/streams.spec.ts+++ b/e2e/streams.spec.ts@@ -1,5 +1,6 @@import { BLOCK, expect, test } from './fixtures'import type { DownloadState } from '../src/shared/types'+import { isProbe } from './origin'// N. How many streams a download runs, decided while it runs (see concurrency.ts), against a// server that limits it in each of the ways that matter.@@ -17,7 +18,7 @@ test.describe('automatic stream count', () => {const origin = await serve({ size: 96 * BLOCK, bytesPerSecond: 256 * 1024 })const accepted = new Set<number>()origin.setRule(({ connection, range }) => {- if (range?.start === 0 && range.end === 0) return undefined // the probe+ if (isProbe(range)) return undefined // the probeif (accepted.has(connection) || accepted.size < 4) {accepted.add(connection)return undefined@@ -43,7 +44,7 @@ test.describe('automatic stream count', () => {const origin = await serve({ size: 256 * BLOCK, bytesPerSecond: 256 * 1024 })const accepted = new Set<number>()origin.setRule(({ connection, range }) => {- if (range?.start === 0 && range.end === 0) return undefined // the probe+ if (isProbe(range)) return undefined // the probeif (accepted.has(connection) || accepted.size < 4) {accepted.add(connection)return undefined@@ -68,7 +69,7 @@ test.describe('automatic stream count', () => {const origin = await serve({ size: 96 * BLOCK, bytesPerSecond: 256 * 1024 })let busyUntil = 0origin.setRule(({ range }) => {- if (range?.start === 0 && range.end === 0) return undefined // the probe+ if (isProbe(range)) return undefined // the probebusyUntil ||= Date.now() + 1500return Date.now() < busyUntil ? { status: 503 } : undefined})diff --git a/src/main/download/probe.ts b/src/main/download/probe.tsindex 036d0ac..37232bb 100644--- a/src/main/download/probe.ts+++ b/src/main/download/probe.ts@@ -32,9 +32,10 @@ interface ProbeResponse {headers: Headers}-/** GET with a 1-byte range: cheaper than fetching the body, and unlike HEAD it- * also tells us (via the 206 status) whether range requests actually work. */-function requestOneByte(+/** GET with a small range: cheaper than fetching the body, and unlike HEAD it also tells us (via+ * the 206 status) whether range requests actually work. Not 1 byte: some file hosts answer a range+ * that ends at byte 0 or 1 with the whole file, while serving every real chunk as asked (#101). */+function requestFirstBytes(url: URL,deadline: number,browser: BrowserContext | undefined@@ -47,7 +48,7 @@ function requestOneByte(hostname: url.hostname.replace(/^\[|\]$/g, ''),port: url.port || undefined,path: `${url.pathname}${url.search}`,- headers: { ...requestHeaders(url, browser), Range: 'bytes=0-0' }+ headers: { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }},(res) => {clearTimeout(timer)@@ -92,7 +93,7 @@ async function requestFollowingRedirects(const deadline = Date.now() + PROBE_TIMEOUT_MSfor (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {- response = await requestOneByte(current, deadline, browser)+ response = await requestFirstBytes(current, deadline, browser)if (response.statusCode >= 300 && response.statusCode < 400) {const location = headerValue(response.headers, 'location')if (!location) break@@ -122,7 +123,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promisconst { current, response } = await requestFollowingRedirects(rawUrl, browser)- // An empty file can't satisfy a request for its first byte: the server answers 416 and gives+ // An empty file can't satisfy a request for its first bytes: the server answers 416 and gives// the size as `bytes */0`. That's a valid, empty download, not an error.if (response?.statusCode === 416 &&@@ -156,7 +157,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis}const contentRange = headerValue(response.headers, 'content-range')- // A server that supports range requests must answer our 1-byte range GET with 206 Partial Content.+ // A server that supports range requests must answer our range GET with 206 Partial Content.// If it returned 200 OK, it ignored the Range header and sent the whole file — even if its headers// statically claim `Accept-Ranges: bytes`.const supportsRanges = response.statusCode === 206@@ -168,7 +169,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis}if (totalBytes === null && response.statusCode === 200) {// Only trust Content-Length on a full (200) response — on a 206 it- // describes the single byte we asked for, not the whole file.+ // describes the bytes we asked for, not the whole file.const contentLength = headerValue(response.headers, 'content-length')if (contentLength) totalBytes = Number(contentLength)}