Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Fall back to a plain GET when a server refuses range requests with 403 (#139)

Arpit Khandelwal7a9b1b489a54
diff --git a/e2e/download.spec.ts b/e2e/download.spec.ts
index 5634d67..47c09a4 100644
--- a/e2e/download.spec.ts
+++ b/e2e/download.spec.ts
@@ -91,6 +91,30 @@ test.describe('happy paths @smoke', () => {
expect(state.streams).toHaveLength(1)
})
+ test('server that refuses range requests with 403: one stream, whole file', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: 10 * BLOCK })
+ origin.setRule(({ range }) => (range ? { status: 403 } : 'ok'))
+ const probe = await plexo.api.probeUrl(origin.url())
+ expect(probe).toMatchObject({ kind: 'http', supportsRanges: false, totalBytes: 10 * BLOCK })
+ await plexo.start(origin.url(), origin.sha256)
+ const state = await plexo.waitForHttpStatus('completed')
+ expect(state.streams).toHaveLength(1)
+ // After the probe's ranged GET, nothing else asks for a range the server would refuse.
+ expect(origin.chunkRequests().filter((request) => request.range !== null)).toEqual([])
+ })
+
+ test('a link that answers 403 with or without a range is still an error', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: 3 * BLOCK })
+ origin.setRule(() => ({ status: 403 }))
+ await expect(plexo.api.probeUrl(origin.url())).rejects.toThrow(/status 403/)
+ })
+
test('unknown size (no Content-Length)', async ({ plexo, serve }) => {
const origin = await serve({ size: 10 * BLOCK, ranges: false, contentLength: false })
await plexo.start(origin.url(), origin.sha256)
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.ts
index fff22fb..6f90cd5 100644
--- a/src/main/download/chunkDownloader.ts
+++ b/src/main/download/chunkDownloader.ts
@@ -13,6 +13,9 @@ export interface ChunkDownloadOptions {
rangeStart: number
/** null = open-ended range, download to end of file. */
rangeEnd: number | null
+ /** The server doesn't take range requests, so ask for the whole file without a Range header
+ * (only valid from byte 0). Some servers refuse any Range with 403 (#131). */
+ wholeFile?: boolean
/** The stream's connection, through the network it's bound to. */
connection: StreamConnection
createDestination: () => Writable
@@ -131,6 +134,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
browser,
rangeStart,
rangeEnd,
+ wholeFile = false,
connection,
createDestination,
onNetworkProgress,
@@ -206,7 +210,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
// The whole file from the start needs no Range at all — and an empty file would answer
// `bytes=0-` with 416, since it has no byte 0 to start from.
const range: Record<string, string> =
- rangeStart > 0 || rangeEnd !== null
+ !(wholeFile && rangeStart === 0) && (rangeStart > 0 || rangeEnd !== null)
? { Range: rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}` }
: {}
diff --git a/src/main/download/httpTransfer.ts b/src/main/download/httpTransfer.ts
index d680878..4981d39 100644
--- a/src/main/download/httpTransfer.ts
+++ b/src/main/download/httpTransfer.ts
@@ -717,6 +717,7 @@ export class HttpTransfer implements Transfer {
browser: this.runtime.requestPayload.browser,
rangeStart: block.rangeStart + attempt.startOffset,
rangeEnd: block.rangeEnd,
+ wholeFile: !this.runtime.requestPayload.supportsRanges,
connection: self.connection,
createDestination: () =>
this.runtime.file.writer(block.rangeStart + (attempt.startOffset ?? 0)),
diff --git a/src/main/download/probe.ts b/src/main/download/probe.ts
index 37232bb..de17b58 100644
--- a/src/main/download/probe.ts
+++ b/src/main/download/probe.ts
@@ -34,11 +34,13 @@ interface ProbeResponse {
/** GET with a small range: cheaper than fetching the body, and unlike HEAD it also tells us (via
* the 206 status) whether range requests actually work. Not 1 byte: some file hosts answer a range
- * that ends at byte 0 or 1 with the whole file, while serving every real chunk as asked (#101). */
+ * that ends at byte 0 or 1 with the whole file, while serving every real chunk as asked (#101).
+ * `ranged: false` sends a plain GET instead; the body is still dropped once the headers arrive. */
function requestFirstBytes(
url: URL,
deadline: number,
- browser: BrowserContext | undefined
+ browser: BrowserContext | undefined,
+ ranged: boolean
): Promise<ProbeResponse> {
return new Promise((resolve, reject) => {
const requester = url.protocol === 'https:' ? httpsRequest : httpRequest
@@ -48,7 +50,9 @@ function requestFirstBytes(
hostname: url.hostname.replace(/^\[|\]$/g, ''),
port: url.port || undefined,
path: `${url.pathname}${url.search}`,
- headers: { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }
+ headers: ranged
+ ? { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }
+ : requestHeaders(url, browser)
},
(res) => {
clearTimeout(timer)
@@ -92,8 +96,16 @@ async function requestFollowingRedirects(
let response: ProbeResponse | null = null
const deadline = Date.now() + PROBE_TIMEOUT_MS
+ let ranged = true
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
- response = await requestFirstBytes(current, deadline, browser)
+ response = await requestFirstBytes(current, deadline, browser, ranged)
+ // Some servers refuse any Range request with 403 while serving the same URL in full (#131).
+ // Ask once more without one: if that works, the file downloads as one stream from byte zero,
+ // which sends no Range either — the same as on a server without range support.
+ if (response.statusCode === 403 && ranged) {
+ ranged = false
+ response = await requestFirstBytes(current, deadline, browser, ranged)
+ }
if (response.statusCode >= 300 && response.statusCode < 400) {
const location = headerValue(response.headers, 'location')
if (!location) break