Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Probe ranges with 1 KiB instead of 1 byte (#105)

Anmol Kapil1b805b8b642f
diff --git a/e2e/chaos.spec.ts b/e2e/chaos.spec.ts
index d62eaaa..c7f61af 100644
--- a/e2e/chaos.spec.ts
+++ b/e2e/chaos.spec.ts
@@ -10,7 +10,7 @@ import {
PlexoApp,
test
} from './fixtures'
-import { Origin, type Fault } from './origin'
+import { isProbe, Origin, type Fault } from './origin'
// H. Model-based chaos. fast-check generates random sequences of user actions and failures —
// pause, resume, crash, quit, flaky server, waiting — and runs each against a real app and a
@@ -152,9 +152,7 @@ test('any sequence of pauses, crashes and faults still ends in the exact file @c
// Kept automatic across the relaunches a crash or a quit brings.
const app = new PlexoApp(dirs, connections === 'auto' ? { PLEXO_E2E_STREAMS: '' } : {})
const real: Real = { app, origin, id: '', faults: [] }
- origin.setRule(({ range }) =>
- range && !(range.start === 0 && range.end === 0) ? real.faults.shift() : undefined
- )
+ origin.setRule(({ range }) => (range && !isProbe(range) ? real.faults.shift() : undefined))
try {
await app.launch()
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.ts
index a60295e..9e2f39b 100644
--- a/e2e/integrity.spec.ts
+++ b/e2e/integrity.spec.ts
@@ -1,5 +1,5 @@
import { BLOCK, expect, LAN_ADDRESS, test } from './fixtures'
-import { seededBytes, type Fault, type LoggedRequest } from './origin'
+import { isProbe, seededBytes, type Fault, type LoggedRequest } from './origin'
// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the
// automatic checks in fixtures.ts — is that a download never ends `completed` with wrong bytes:
@@ -91,11 +91,7 @@ test.describe('servers without range support @smoke', () => {
const origin = await serve({ size: 20 * BLOCK, ranges: false })
let transfers = 0
origin.setRule(({ range }) =>
- range?.start === 0 && range.end === 0
- ? 'ok'
- : transfers++ === 0
- ? { cutAfter: 7 * BLOCK + 3 }
- : 'ok'
+ isProbe(range) ? 'ok' : transfers++ === 0 ? { cutAfter: 7 * BLOCK + 3 } : 'ok'
)
await plexo.start(origin.url(), origin.sha256)
await plexo.waitForStatus('completed')
@@ -112,9 +108,7 @@ test('one network dies for good mid-download; the other finishes it @smoke', asy
let fromB = 0
// Network b serves two blocks, then every request over it fails.
origin.setRule(({ from, range }) =>
- from !== '127.0.0.1' && range && !(range.start === 0 && range.end === 0) && fromB++ >= 2
- ? { status: 503 }
- : 'ok'
+ from !== '127.0.0.1' && range && !isProbe(range) && fromB++ >= 2 ? { status: 503 } : 'ok'
)
await plexo.start(origin.url(), origin.sha256, { networks: ['a', 'b'], connections: 2 })
await plexo.waitForStatus('completed')
@@ -163,9 +157,7 @@ test.describe('permanent faults end in a clean error @smoke', () => {
for (const [label, fault] of PERMANENT) {
test(label, async ({ plexo, serve }) => {
const origin = await serve({ size: SIZE })
- origin.setRule(({ range }) =>
- range && !(range.start === 0 && range.end === 0) ? fault : 'ok'
- )
+ origin.setRule(({ range }) => (range && !isProbe(range) ? fault : 'ok'))
await plexo.start(origin.url(), origin.sha256, { connections: 2 })
const state = await plexo.waitForStatus('error')
expect(state.error).toBeTruthy()
diff --git a/e2e/origin.ts b/e2e/origin.ts
index 52efc44..0c309d7 100644
--- a/e2e/origin.ts
+++ b/e2e/origin.ts
@@ -33,6 +33,10 @@ export type Fault =
| { crawl: number }
| { redirect: string }
+/** The app's probe (src/main/download/probe.ts asks for bytes=0-1023), as opposed to a chunk. */
+export const isProbe = (range: OriginRequest['range']): boolean =>
+ range?.start === 0 && range.end === 1023
+
export interface OriginRequest {
/** 1-based count of requests this server has seen. */
n: number
@@ -189,9 +193,9 @@ export class Origin {
this.holdRelease = null
}
- /** Requests that asked for byte ranges, i.e. the app's chunk requests (not the 1-byte probe). */
+ /** Requests that asked for byte ranges, i.e. the app's chunk requests (not the probe). */
chunkRequests(): LoggedRequest[] {
- return this.log.filter((entry) => !(entry.range?.start === 0 && entry.range.end === 0))
+ return this.log.filter((entry) => !isProbe(entry.range))
}
private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
@@ -218,8 +222,8 @@ export class Origin {
return
}
- // The app's 1-byte probes (bytes=0-0) are never held — only real transfers are.
- const holdable = !(request.range?.start === 0 && request.range.end === 0)
+ // The app's probes are never held — only real transfers are.
+ const holdable = !isProbe(request.range)
if (holdable && this.holdAt !== null && request.range && request.range.start > this.holdAt) {
await this.releasePromise
}
diff --git a/e2e/streams.spec.ts b/e2e/streams.spec.ts
index 90e8e8a..1979814 100644
--- a/e2e/streams.spec.ts
+++ b/e2e/streams.spec.ts
@@ -1,5 +1,6 @@
import { BLOCK, expect, test } from './fixtures'
import type { DownloadState } from '../src/shared/types'
+import { isProbe } from './origin'
// N. How many streams a download runs, decided while it runs (see concurrency.ts), against a
// server that limits it in each of the ways that matter.
@@ -17,7 +18,7 @@ test.describe('automatic stream count', () => {
const origin = await serve({ size: 96 * BLOCK, bytesPerSecond: 256 * 1024 })
const accepted = new Set<number>()
origin.setRule(({ connection, range }) => {
- if (range?.start === 0 && range.end === 0) return undefined // the probe
+ if (isProbe(range)) return undefined // the probe
if (accepted.has(connection) || accepted.size < 4) {
accepted.add(connection)
return undefined
@@ -43,7 +44,7 @@ test.describe('automatic stream count', () => {
const origin = await serve({ size: 256 * BLOCK, bytesPerSecond: 256 * 1024 })
const accepted = new Set<number>()
origin.setRule(({ connection, range }) => {
- if (range?.start === 0 && range.end === 0) return undefined // the probe
+ if (isProbe(range)) return undefined // the probe
if (accepted.has(connection) || accepted.size < 4) {
accepted.add(connection)
return undefined
@@ -68,7 +69,7 @@ test.describe('automatic stream count', () => {
const origin = await serve({ size: 96 * BLOCK, bytesPerSecond: 256 * 1024 })
let busyUntil = 0
origin.setRule(({ range }) => {
- if (range?.start === 0 && range.end === 0) return undefined // the probe
+ if (isProbe(range)) return undefined // the probe
busyUntil ||= Date.now() + 1500
return Date.now() < busyUntil ? { status: 503 } : undefined
})
diff --git a/src/main/download/probe.ts b/src/main/download/probe.ts
index 036d0ac..37232bb 100644
--- a/src/main/download/probe.ts
+++ b/src/main/download/probe.ts
@@ -32,9 +32,10 @@ interface ProbeResponse {
headers: Headers
}
-/** GET with a 1-byte range: cheaper than fetching the body, and unlike HEAD it
- * also tells us (via the 206 status) whether range requests actually work. */
-function requestOneByte(
+/** GET with a small range: cheaper than fetching the body, and unlike HEAD it also tells us (via
+ * the 206 status) whether range requests actually work. Not 1 byte: some file hosts answer a range
+ * that ends at byte 0 or 1 with the whole file, while serving every real chunk as asked (#101). */
+function requestFirstBytes(
url: URL,
deadline: number,
browser: BrowserContext | undefined
@@ -47,7 +48,7 @@ function requestOneByte(
hostname: url.hostname.replace(/^\[|\]$/g, ''),
port: url.port || undefined,
path: `${url.pathname}${url.search}`,
- headers: { ...requestHeaders(url, browser), Range: 'bytes=0-0' }
+ headers: { ...requestHeaders(url, browser), Range: 'bytes=0-1023' }
},
(res) => {
clearTimeout(timer)
@@ -92,7 +93,7 @@ async function requestFollowingRedirects(
const deadline = Date.now() + PROBE_TIMEOUT_MS
for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
- response = await requestOneByte(current, deadline, browser)
+ response = await requestFirstBytes(current, deadline, browser)
if (response.statusCode >= 300 && response.statusCode < 400) {
const location = headerValue(response.headers, 'location')
if (!location) break
@@ -122,7 +123,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis
const { current, response } = await requestFollowingRedirects(rawUrl, browser)
- // An empty file can't satisfy a request for its first byte: the server answers 416 and gives
+ // An empty file can't satisfy a request for its first bytes: the server answers 416 and gives
// the size as `bytes */0`. That's a valid, empty download, not an error.
if (
response?.statusCode === 416 &&
@@ -156,7 +157,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis
}
const contentRange = headerValue(response.headers, 'content-range')
- // A server that supports range requests must answer our 1-byte range GET with 206 Partial Content.
+ // A server that supports range requests must answer our range GET with 206 Partial Content.
// If it returned 200 OK, it ignored the Range header and sent the whole file — even if its headers
// statically claim `Accept-Ranges: bytes`.
const supportsRanges = response.statusCode === 206
@@ -168,7 +169,7 @@ export async function probeUrl(rawUrl: string, browser?: BrowserContext): Promis
}
if (totalBytes === null && response.statusCode === 200) {
// Only trust Content-Length on a full (200) response — on a 206 it
- // describes the single byte we asked for, not the whole file.
+ // describes the bytes we asked for, not the whole file.
const contentLength = headerValue(response.headers, 'content-length')
if (contentLength) totalBytes = Number(contentLength)
}