Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

fix: handle range-reporting hosts and fallback when range end is ignored (#188) (#192)

SITHESWAR K2d2132f35046
diff --git a/e2e/download.spec.ts b/e2e/download.spec.ts
index f711114..c15092c 100644
--- a/e2e/download.spec.ts
+++ b/e2e/download.spec.ts
@@ -106,6 +106,45 @@ test.describe('happy paths @smoke', () => {
expect(origin.chunkRequests().filter((request) => request.range !== null)).toEqual([])
})
+ test('server that ignores the end of a range: one stream, whole file', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: 10 * BLOCK })
+ origin.setRule(({ range }) => (range ? 'wholeRange' : 'ok'))
+ const probe = await plexo.api.probeUrl(origin.url())
+ expect(probe).toMatchObject({
+ kind: 'http',
+ supportsRanges: false,
+ totalBytes: 10 * BLOCK
+ })
+ await plexo.start(origin.url(), origin.sha256)
+ const state = await plexo.waitForHttpStatus('completed')
+ expect(state.streams).toHaveLength(1)
+ // The transfer asks for the whole file, not a range the server would overrun.
+ expect(origin.chunkRequests().filter((request) => request.range !== null)).toEqual([])
+ })
+
+ test('server that names the file end but sends only the range: streams still split', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: 10 * BLOCK })
+ // Cloudflare-fronted hosts: Content-Range ends at the file, Content-Length is the range.
+ origin.setRule(({ range }) => (range ? 'rangeToEof' : 'ok'))
+ const probe = await plexo.api.probeUrl(origin.url())
+ expect(probe).toMatchObject({
+ kind: 'http',
+ supportsRanges: true,
+ totalBytes: 10 * BLOCK
+ })
+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })
+ await plexo.waitForHttpStatus('completed')
+ expect(
+ origin.chunkRequests().filter((request) => request.range !== null).length
+ ).toBeGreaterThan(0)
+ })
+
test('a link that answers 403 with or without a range is still an error', async ({
plexo,
serve
diff --git a/e2e/origin.ts b/e2e/origin.ts
index 7ab8dcf..401a88f 100644
--- a/e2e/origin.ts
+++ b/e2e/origin.ts
@@ -14,6 +14,10 @@ export type Fault =
| 'ok'
/** Replies 200 with the whole file, as a server without range support does. */
| 'ignoreRange'
+ /** 206 whose Content-Range and body run to the end of the file, ignoring the asked end. */
+ | 'wholeRange'
+ /** 206 like a Cloudflare-fronted host: Content-Range ends at the file, but the body is the range. */
+ | 'rangeToEof'
/** 206 whose Content-Range starts one byte later than asked. */
| 'wrongStart'
/** 206 with a correct header but one byte more body than the range. */
@@ -248,7 +252,7 @@ export class Origin {
let status = 200
if (rangesOn && request.range) {
start = request.range.start
- end = Math.min(request.range.end ?? total - 1, total - 1)
+ end = fault === 'wholeRange' ? total - 1 : Math.min(request.range.end ?? total - 1, total - 1)
if (start >= total) {
entry.status = 416
res.writeHead(416, { 'Content-Range': `bytes */${total}` }).end()
@@ -257,7 +261,8 @@ export class Origin {
status = 206
if (fault !== 'noContentRange') {
const shownStart = fault === 'wrongStart' ? start + 1 : start
- headers['Content-Range'] = `bytes ${shownStart}-${end}/${total}`
+ const shownEnd = fault === 'rangeToEof' ? total - 1 : end
+ headers['Content-Range'] = `bytes ${shownStart}-${shownEnd}/${total}`
}
}
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.ts
index 2fe78a3..e779a85 100644
--- a/src/main/download/chunkDownloader.ts
+++ b/src/main/download/chunkDownloader.ts
@@ -299,7 +299,15 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
res.resume()
return
}
- if (rangeEnd !== null && served.end > rangeEnd) {
+ // Some hosts name the end of the file in Content-Range while sending Content-Length
+ // bytes for the range. The header alone would look like an overrun; the length says
+ // the body is exactly what we asked for, so take it rather than fail every block.
+ const declared = Number(header(res, 'content-length') ?? NaN)
+ if (
+ rangeEnd !== null &&
+ served.end > rangeEnd &&
+ !(Number.isFinite(declared) && declared === expectedBytes)
+ ) {
fail(
new Error(
`Server returned more than the requested range: through byte ${served.end}, asked through ${rangeEnd}`
diff --git a/src/main/download/probe.ts b/src/main/download/probe.ts
index 064ba72..b9a1c20 100644
--- a/src/main/download/probe.ts
+++ b/src/main/download/probe.ts
@@ -21,6 +21,9 @@ const MAX_REDIRECTS = 5
// redirects included, so a chain of slow hops can't stretch it.
const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMs
const NO_RESPONSE = 'The server did not respond — check the link and try again'
+/** The last byte the probe asks for. Not 0 or 1: some file hosts answer a range that ends there
+ * with the whole file while serving every real chunk as asked (#101). */
+const PROBE_RANGE_END = 1023
type Headers = Record<string, string | string[] | undefined>
@@ -43,7 +46,7 @@ function requestFirstBytes(
deadline: number,
browser: BrowserContext | undefined,
ranged: boolean,
- range = 'bytes=0-1023'
+ range = `bytes=0-${PROBE_RANGE_END}`
): Promise<ProbeResponse> {
return new Promise((resolve, reject) => {
const requester = url.protocol === 'https:' ? httpsRequest : httpRequest
@@ -248,7 +251,18 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis
// A server that supports range requests must answer our range GET with 206 Partial Content.
// If it returned 200 OK, it ignored the Range header and sent the whole file — even if its headers
// statically claim `Accept-Ranges: bytes`.
- const supportsRanges = response.statusCode === 206
+ //
+ // A 206 is only a promise if it sends us the bytes we asked for. Most say so in Content-Range,
+ // but some hosts (Cloudflare-fronted file hosts among them) put the end of the *file* there
+ // while setting Content-Length to the range they actually send. Trust whichever says the body
+ // is bounded, so those keep their parallel streams instead of falling back to one.
+ const served = /^bytes\s+(\d+)-(\d+)\//i.exec(contentRange ?? '')
+ const contentLength = Number(headerValue(response.headers, 'content-length') ?? NaN)
+ const servesOnlyProbe =
+ served !== null &&
+ Number(served[1]) === 0 &&
+ (Number(served[2]) <= PROBE_RANGE_END || contentLength === PROBE_RANGE_END + 1)
+ const supportsRanges = response.statusCode === 206 && servesOnlyProbe
let totalBytes: number | null = null
if (contentRange) {
@@ -262,7 +276,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis
if (contentLength) totalBytes = Number(contentLength)
}
- const perRange = await etagIsPerRange(current, browser, response)
+ const perRange = supportsRanges ? await etagIsPerRange(current, browser, response) : false
return {
kind: 'http',