Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Fail a download when the server's file changes mid-transfer

Anmol Kapil5874f1bc8c91
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.ts
index cfae7db..21d5ea8 100644
--- a/e2e/integrity.spec.ts
+++ b/e2e/integrity.spec.ts
@@ -1,6 +1,5 @@
-import { readFile } from 'node:fs/promises'
import { BLOCK, expect, test } from './fixtures'
-import { seededBytes, sha256, type Fault } from './origin'
+import { seededBytes, type Fault } from './origin'
// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the
// automatic checks in fixtures.ts — is that a download never ends `completed` with wrong bytes:
@@ -69,28 +68,47 @@ test.describe('permanent faults end in a clean error @smoke', () => {
}
})
-test('file changes on the server mid-download (same size, new ETag) @smoke', async ({
- plexo,
- serve
-}) => {
- test.fail(
- true,
- 'known gap: chunk responses are not checked against the probed ETag, so old and new bytes get stitched together'
- )
- const origin = await serve({ size: SIZE, seed: 1 })
- const original = origin.sha256
- const replacement = seededBytes(SIZE, 2)
+test.describe('the file changes on the server mid-download @smoke', () => {
+ const cases: [string, Omit<Parameters<typeof mutate>[0], 'origin'>][] = [
+ ['same size, new ETag', { etag: '"v2"' }],
+ ['same size, new Last-Modified (no ETag)', { lastModified: 'Thu, 02 Jan 2025 00:00:00 GMT' }],
+ ['different size, no validators at all', { size: SIZE + BLOCK }]
+ ]
+
+ /** Republishes the file on the server, as a CDN rolling out a new version would. */
+ function mutate({
+ origin,
+ etag,
+ lastModified,
+ size
+ }: {
+ origin: import('./origin').Origin
+ etag?: string
+ lastModified?: string
+ size?: number
+ }): void {
+ origin.setContent(seededBytes(size ?? SIZE, 2), etag ?? origin.etag)
+ if (lastModified) origin.lastModified = lastModified
+ }
- const reached = origin.hold(3 * BLOCK + 100)
- await plexo.start(origin.url(), original, { connections: 2 })
- await reached
- origin.setContent(replacement, '"v2"')
- origin.release()
+ for (const [label, change] of cases) {
+ test(label, async ({ plexo, serve }) => {
+ const origin = await serve({
+ size: SIZE,
+ seed: 1,
+ etag: change.etag ? '"v1"' : null,
+ lastModified: change.lastModified ? 'Wed, 01 Jan 2025 00:00:00 GMT' : null
+ })
+ const reached = origin.hold(3 * BLOCK + 100)
+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })
+ await reached
+ mutate({ origin, ...change })
+ origin.release()
- const state = await plexo.waitForStatus(['completed', 'error'])
- // Acceptable outcomes: an error, or a file that is entirely one version. Never a mix.
- if (state.status === 'completed') {
- const got = sha256(await readFile(state.destinationPath))
- expect([original, sha256(replacement)]).toContain(got)
+ // Never a file stitched from two versions: the download stops with a clear reason.
+ const state = await plexo.waitForStatus(['completed', 'error'])
+ expect(state.status).toBe('error')
+ expect(state.error).toMatch(/changed during the download/)
+ })
}
})
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.ts
index 3e92dc0..5f54785 100644
--- a/src/main/download/chunkDownloader.ts
+++ b/src/main/download/chunkDownloader.ts
@@ -16,8 +16,19 @@ export interface ChunkDownloadOptions {
append: boolean
onProgress: (bytesDownloadedThisRun: number) => void
signal: AbortSignal
+ /** What the probe saw. A response that disagrees comes from a different version of the file. */
+ expected: {
+ etag: string | null
+ lastModified: string | null
+ /** 0 = unknown. */
+ totalBytes: number
+ }
}
+/** The server is now serving a different file than the one this download started on. Retrying
+ * can't help — every retry would fetch the new version — so this fails the whole download. */
+export class RemoteChangedError extends Error {}
+
// A server that accepts the connection and then goes silent (no data, no
// error, no close) would otherwise hang the chunk forever with no way to
// detect or retry it.
@@ -31,18 +42,56 @@ const MAX_REDIRECTS = 5
interface ServedRange {
start: number
end: number
+ /** null when the server sent `*`. */
+ total: number | null
}
/** Parses `Content-Range: bytes <start>-<end>/<total>`. Returns null if it isn't in that form. */
function parseContentRange(value: string | string[] | undefined): ServedRange | null {
const raw = Array.isArray(value) ? value[0] : value
if (!raw) return null
- const match = /^\s*bytes\s+(\d+)-(\d+)\/(?:\d+|\*)\s*$/i.exec(raw)
+ const match = /^\s*bytes\s+(\d+)-(\d+)\/(\d+|\*)\s*$/i.exec(raw)
if (!match) return null
const start = Number(match[1])
const end = Number(match[2])
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null
- return { start, end }
+ return { start, end, total: match[3] === '*' ? null : Number(match[3]) }
+}
+
+function header(res: IncomingMessage, name: string): string | undefined {
+ const value = res.headers[name]
+ return Array.isArray(value) ? value[0] : value
+}
+
+/** Why this response can't belong to the file the download started on, or null if it can. Each
+ * part is fetched separately, so without this a file republished mid-download would be stitched
+ * together from two versions and still pass every length check. */
+function versionMismatch(
+ res: IncomingMessage,
+ served: ServedRange | null,
+ expected: ChunkDownloadOptions['expected']
+): string | null {
+ const etag = header(res, 'etag')
+ if (expected.etag && etag && etag !== expected.etag) return `its ETag is now ${etag}`
+ const lastModified = header(res, 'last-modified')
+ if (
+ !expected.etag &&
+ expected.lastModified &&
+ lastModified &&
+ lastModified !== expected.lastModified
+ ) {
+ return `it was modified at ${lastModified}`
+ }
+ // A 206 states the file's total size in Content-Range; a 200 is the whole file, so its length.
+ const total =
+ served?.total ??
+ (res.statusCode === 200 && header(res, 'content-length')
+ ? Number(header(res, 'content-length'))
+ : null)
+ if (expected.totalBytes > 0 && total !== null && total !== expected.totalBytes) {
+ return `it is now ${total} bytes instead of ${expected.totalBytes}`
+ }
+ return null
}
/**
@@ -55,8 +104,17 @@ function parseContentRange(value: string | string[] | undefined): ServedRange |
* puts the block back on the queue for a retry instead of marking it done.
*/
export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
- const { url, rangeStart, rangeEnd, localAddress, destinationPath, append, onProgress, signal } =
- options
+ const {
+ url,
+ rangeStart,
+ rangeEnd,
+ localAddress,
+ destinationPath,
+ append,
+ onProgress,
+ signal,
+ expected
+ } = options
return new Promise((resolve, reject) => {
if (signal.aborted) {
@@ -145,10 +203,21 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
return
}
+ const served = parseContentRange(res.headers['content-range'])
+ const mismatch = versionMismatch(res, served, expected)
+ if (mismatch) {
+ fail(
+ new RemoteChangedError(
+ `The file on the server changed during the download (${mismatch}). Start the download over.`
+ )
+ )
+ res.resume()
+ return
+ }
+
// A 206 says where in the file these bytes belong — check it lines up
// with what we asked for before writing any of them into the part file.
if (status === 206) {
- const served = parseContentRange(res.headers['content-range'])
if (!served) {
fail(new Error('Server sent a 206 without a usable Content-Range header'))
res.resume()
diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.ts
index 4e56b15..683eda7 100644
--- a/src/main/download/downloadManager.ts
+++ b/src/main/download/downloadManager.ts
@@ -25,7 +25,7 @@ import type {
StartDownloadRequest,
StartSimulatedDownloadRequest
} from '../../shared/types'
-import { downloadChunk } from './chunkDownloader'
+import { downloadChunk, RemoteChangedError } from './chunkDownloader'
import { testKnobs } from '../testKnobs'
import { reserveDestinationPath } from './paths'
import { isResourceUnchanged } from './probe'
@@ -839,6 +839,11 @@ export class DownloadManager {
destinationPath: partPath,
append: resumeOffset > 0,
signal: controller.signal,
+ expected: {
+ etag: runtime.requestPayload.etag,
+ lastModified: runtime.requestPayload.lastModified,
+ totalBytes: runtime.requestPayload.totalBytes
+ },
onProgress: (bytesThisRun) => {
const delta = bytesThisRun - lastReportedThisRun
lastReportedThisRun = bytesThisRun
@@ -876,6 +881,19 @@ export class DownloadManager {
const message = error instanceof Error ? error.message : String(error)
chunk.error = message
+
+ if (error instanceof RemoteChangedError) {
+ // Every other worker would hit the same new version, so stop them all now rather
+ // than let each burn through its retries first.
+ block.status = 'pending'
+ chunk.status = 'error'
+ chunk.speedBytesPerSec = 0
+ runtime.state.status = 'error'
+ runtime.state.error = message
+ for (const cr of runtime.chunkRuntimes.values()) cr.controller.abort()
+ break
+ }
+
attempt += 1
chunk.retryCount += 1