Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Download from servers that give each byte range its own ETag (pCloud) (#158)

Arpit Khandelwal39475030007d
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.ts
index 9e2f39b..674c7bb 100644
--- a/e2e/integrity.spec.ts
+++ b/e2e/integrity.spec.ts
@@ -221,6 +221,16 @@ test.describe('servers that label the same file differently @smoke', () => {
expect(origin.log.filter((entry) => entry.n % 2 === 0).length).toBeGreaterThan(0)
})
+ test('every byte range labelled with an ETag of its own (pCloud)', async ({ plexo, serve }) => {
+ const origin = await serve({ size: SIZE, etag: '"whole"' })
+ origin.setVersionRule(({ range }) => ({
+ content: origin.content,
+ etag: range ? `"r${range.start}-${range.end ?? 'end'}"` : '"whole"'
+ }))
+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })
+ await plexo.waitForStatus('completed')
+ })
+
test('the same ETag, differing only by W/ or a -gzip suffix', async ({ plexo, serve }) => {
const origin = await serve({ size: SIZE, etag: '"v1"' })
const variants = ['W/"v1"', '"v1-gzip"', '"v1"']
diff --git a/e2e/origin.ts b/e2e/origin.ts
index 0c309d7..7ab8dcf 100644
--- a/e2e/origin.ts
+++ b/e2e/origin.ts
@@ -35,7 +35,7 @@ export type Fault =
/** The app's probe (src/main/download/probe.ts asks for bytes=0-1023), as opposed to a chunk. */
export const isProbe = (range: OriginRequest['range']): boolean =>
- range?.start === 0 && range.end === 1023
+ (range?.start === 0 && range.end === 1023) || (range?.start === 1024 && range.end === 2047)
export interface OriginRequest {
/** 1-based count of requests this server has seen. */
diff --git a/src/main/download/probe.ts b/src/main/download/probe.ts
index 6d578f4..2e637d6 100644
--- a/src/main/download/probe.ts
+++ b/src/main/download/probe.ts
@@ -6,6 +6,7 @@ import type { BrowserContext, ProbeResult } from '../../shared/types'
import { testKnobs } from '../testKnobs'
import { requestHeaders } from './browserContext'
import { parseContentDispositionFilename } from './contentDisposition'
+import { normalizeEtag } from './fileVersion'
import {
describeTorrent,
downloadTorrentFile,
@@ -40,7 +41,8 @@ function requestFirstBytes(
url: URL,
deadline: number,
browser: BrowserContext | undefined,
- ranged: boolean
+ ranged: boolean,
+ range = 'bytes=0-1023'
): Promise<ProbeResponse> {
return new Promise((resolve, reject) => {
const requester = url.protocol === 'https:' ? httpsRequest : httpRequest
@@ -51,7 +53,7 @@ function requestFirstBytes(
port: url.port || undefined,
path: `${url.pathname}${url.search}`,
headers: ranged
- ? { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }
+ ? { ...requestHeaders(url, browser), Range: range }
: requestHeaders(url, browser)
},
(res) => {
@@ -121,6 +123,35 @@ async function requestFollowingRedirects(
return { current, response }
}
+/** Whether the server labels each byte range with an ETag of its own (pCloud does, #156): then
+ * no chunk's ETag ever matches the probe's, and every one would be taken for a changed file.
+ * The first range asked again keeps its ETag while the next range gets another. A load balancer
+ * alternating labels per request changes it on the repeat instead, and keeps its ETag checked. */
+async function etagIsPerRange(
+ url: URL,
+ browser: BrowserContext | undefined,
+ first: ProbeResponse
+): Promise<boolean> {
+ const etag = headerValue(first.headers, 'etag')
+ const total = /\/(\d+)$/.exec(headerValue(first.headers, 'content-range') ?? '')?.[1]
+ if (first.statusCode !== 206 || !etag || !total || Number(total) <= 2048) return false
+ const deadline = Date.now() + PROBE_TIMEOUT_MS
+ try {
+ const again = await requestFirstBytes(url, deadline, browser, true)
+ const next = await requestFirstBytes(url, deadline, browser, true, 'bytes=1024-2047')
+ const etagOf = (r: ProbeResponse): string => normalizeEtag(headerValue(r.headers, 'etag') ?? '')
+ return (
+ again.statusCode === 206 &&
+ next.statusCode === 206 &&
+ etagOf(again) === normalizeEtag(etag) &&
+ etagOf(next) !== '' &&
+ etagOf(next) !== normalizeEtag(etag)
+ )
+ } catch {
+ return false
+ }
+}
+
/** A link the server says is a torrent, or that is named like one. */
function isTorrentLink(headers: Headers, url: URL): boolean {
const contentType = headerValue(headers, 'content-type') ?? ''
@@ -189,6 +220,8 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis
if (contentLength) totalBytes = Number(contentLength)
}
+ const perRange = await etagIsPerRange(current, browser, response)
+
return {
kind: 'http',
requestedUrl: rawUrl,
@@ -197,7 +230,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis
totalBytes,
suggestedFileName: fileNameFromHeaders(response.headers, current),
contentType: headerValue(response.headers, 'content-type') ?? null,
- etag: headerValue(response.headers, 'etag') ?? null,
+ etag: perRange ? null : (headerValue(response.headers, 'etag') ?? null),
lastModified: headerValue(response.headers, 'last-modified') ?? null
}
}