Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Download from servers that give each byte range its own ETag (pCloud) (#158)
Arpit Khandelwal39475030007d
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.tsindex 9e2f39b..674c7bb 100644--- a/e2e/integrity.spec.ts+++ b/e2e/integrity.spec.ts@@ -221,6 +221,16 @@ test.describe('servers that label the same file differently @smoke', () => {expect(origin.log.filter((entry) => entry.n % 2 === 0).length).toBeGreaterThan(0)})+ test('every byte range labelled with an ETag of its own (pCloud)', async ({ plexo, serve }) => {+ const origin = await serve({ size: SIZE, etag: '"whole"' })+ origin.setVersionRule(({ range }) => ({+ content: origin.content,+ etag: range ? `"r${range.start}-${range.end ?? 'end'}"` : '"whole"'+ }))+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })+ await plexo.waitForStatus('completed')+ })+test('the same ETag, differing only by W/ or a -gzip suffix', async ({ plexo, serve }) => {const origin = await serve({ size: SIZE, etag: '"v1"' })const variants = ['W/"v1"', '"v1-gzip"', '"v1"']diff --git a/e2e/origin.ts b/e2e/origin.tsindex 0c309d7..7ab8dcf 100644--- a/e2e/origin.ts+++ b/e2e/origin.ts@@ -35,7 +35,7 @@ export type Fault =/** The app's probe (src/main/download/probe.ts asks for bytes=0-1023), as opposed to a chunk. */export const isProbe = (range: OriginRequest['range']): boolean =>- range?.start === 0 && range.end === 1023+ (range?.start === 0 && range.end === 1023) || (range?.start === 1024 && range.end === 2047)export interface OriginRequest {/** 1-based count of requests this server has seen. */diff --git a/src/main/download/probe.ts b/src/main/download/probe.tsindex 6d578f4..2e637d6 100644--- a/src/main/download/probe.ts+++ b/src/main/download/probe.ts@@ -6,6 +6,7 @@ import type { BrowserContext, ProbeResult } from '../../shared/types'import { testKnobs } from '../testKnobs'import { requestHeaders } from './browserContext'import { parseContentDispositionFilename } from './contentDisposition'+import { normalizeEtag } from './fileVersion'import {describeTorrent,downloadTorrentFile,@@ -40,7 +41,8 @@ function requestFirstBytes(url: URL,deadline: number,browser: BrowserContext | undefined,- ranged: boolean+ ranged: boolean,+ range = 'bytes=0-1023'): Promise<ProbeResponse> {return new Promise((resolve, reject) => {const requester = url.protocol === 'https:' ? httpsRequest : httpRequest@@ -51,7 +53,7 @@ function requestFirstBytes(port: url.port || undefined,path: `${url.pathname}${url.search}`,headers: ranged- ? { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }+ ? { ...requestHeaders(url, browser), Range: range }: requestHeaders(url, browser)},(res) => {@@ -121,6 +123,35 @@ async function requestFollowingRedirects(return { current, response }}+/** Whether the server labels each byte range with an ETag of its own (pCloud does, #156): then+ * no chunk's ETag ever matches the probe's, and every one would be taken for a changed file.+ * The first range asked again keeps its ETag while the next range gets another. A load balancer+ * alternating labels per request changes it on the repeat instead, and keeps its ETag checked. */+async function etagIsPerRange(+ url: URL,+ browser: BrowserContext | undefined,+ first: ProbeResponse+): Promise<boolean> {+ const etag = headerValue(first.headers, 'etag')+ const total = /\/(\d+)$/.exec(headerValue(first.headers, 'content-range') ?? '')?.[1]+ if (first.statusCode !== 206 || !etag || !total || Number(total) <= 2048) return false+ const deadline = Date.now() + PROBE_TIMEOUT_MS+ try {+ const again = await requestFirstBytes(url, deadline, browser, true)+ const next = await requestFirstBytes(url, deadline, browser, true, 'bytes=1024-2047')+ const etagOf = (r: ProbeResponse): string => normalizeEtag(headerValue(r.headers, 'etag') ?? '')+ return (+ again.statusCode === 206 &&+ next.statusCode === 206 &&+ etagOf(again) === normalizeEtag(etag) &&+ etagOf(next) !== '' &&+ etagOf(next) !== normalizeEtag(etag)+ )+ } catch {+ return false+ }+}+/** A link the server says is a torrent, or that is named like one. */function isTorrentLink(headers: Headers, url: URL): boolean {const contentType = headerValue(headers, 'content-type') ?? ''@@ -189,6 +220,8 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promisif (contentLength) totalBytes = Number(contentLength)}+ const perRange = await etagIsPerRange(current, browser, response)+return {kind: 'http',requestedUrl: rawUrl,@@ -197,7 +230,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): PromistotalBytes,suggestedFileName: fileNameFromHeaders(response.headers, current),contentType: headerValue(response.headers, 'content-type') ?? null,- etag: headerValue(response.headers, 'etag') ?? null,+ etag: perRange ? null : (headerValue(response.headers, 'etag') ?? null),lastModified: headerValue(response.headers, 'last-modified') ?? null}}