Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Tell a changed file from a relabelled one instead of failing on any ETag change

Anmol Kapil194433a5c7d2
diff --git a/e2e/download.spec.ts b/e2e/download.spec.ts
index 2c44682..9964e98 100644
--- a/e2e/download.spec.ts
+++ b/e2e/download.spec.ts
@@ -136,6 +136,17 @@ test.describe('edge cases', () => {
await plexo.waitForStatus('completed', 5000)
})
+ test('a server that never answers the link check → an error, not endless "Checking…" @smoke', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: BLOCK })
+ origin.setRule(() => 'stallHeaders')
+ const started = Date.now()
+ await expect(plexo.api.probeUrl(origin.url())).rejects.toThrow(/did not respond/)
+ expect(Date.now() - started).toBeLessThan(10_000)
+ })
+
test('starting a second download while one is active is refused @smoke', async ({
plexo,
serve
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.ts
index eb0c601..f15dccf 100644
--- a/e2e/integrity.spec.ts
+++ b/e2e/integrity.spec.ts
@@ -158,3 +158,56 @@ test.describe('the file changes on the server mid-download @smoke', () => {
})
}
})
+
+test.describe('servers that label the same file differently @smoke', () => {
+ // None of these is a changed file, so none may fail the download.
+ test('load balancer: identical bytes, two different ETags', async ({ plexo, serve }) => {
+ const origin = await serve({ size: SIZE, etag: '"server-a"' })
+ origin.setVersionRule(({ n }) =>
+ n % 2 === 0 ? { content: origin.content, etag: '"server-b"' } : undefined
+ )
+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })
+ await plexo.waitForStatus('completed')
+ expect(origin.log.filter((entry) => entry.n % 2 === 0).length).toBeGreaterThan(0)
+ })
+
+ test('the same ETag, differing only by W/ or a -gzip suffix', async ({ plexo, serve }) => {
+ const origin = await serve({ size: SIZE, etag: '"v1"' })
+ const variants = ['W/"v1"', '"v1-gzip"', '"v1"']
+ origin.setVersionRule(({ n }) => ({ content: origin.content, etag: variants[n % 3] }))
+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })
+ await plexo.waitForStatus('completed')
+ })
+
+ test('load balancer: identical bytes, Last-Modified differs per server (no ETag)', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({
+ size: SIZE,
+ etag: null,
+ lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT'
+ })
+ // lastModified is shared, so alternate it from the rule instead.
+ origin.setVersionRule(({ n }) => {
+ origin.lastModified =
+ n % 2 === 0 ? 'Wed, 01 Jan 2025 00:00:07 GMT' : 'Wed, 01 Jan 2025 00:00:00 GMT'
+ return undefined
+ })
+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })
+ await plexo.waitForStatus('completed')
+ })
+
+ test('half-rolled-out new version: some servers new, some old → error, never a mix', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: SIZE, seed: 1, etag: '"old"' })
+ const next = seededBytes(SIZE, 2)
+ origin.setVersionRule(({ n }) => (n % 2 === 0 ? { content: next, etag: '"new"' } : undefined))
+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })
+ const state = await plexo.waitForStatus(['completed', 'error'])
+ expect(state.status).toBe('error')
+ expect(state.error).toMatch(/changed during the download/)
+ })
+})
diff --git a/e2e/lifecycle.spec.ts b/e2e/lifecycle.spec.ts
index feb99e1..04a5358 100644
--- a/e2e/lifecycle.spec.ts
+++ b/e2e/lifecycle.spec.ts
@@ -92,39 +92,78 @@ test.describe('pause during a retry backoff', () => {
})
test.describe('resume safety checks @smoke', () => {
- test('ETag changed while paused → error, nothing kept', async ({ plexo, serve }) => {
- const origin = await serve({ size: SIZE })
+ /** Starts a download, pauses it partway, and lets `change` alter the server while paused. */
+ async function pauseThenChange(
+ plexo: import('./fixtures').PlexoApp,
+ origin: import('./origin').Origin,
+ change: () => void
+ ): Promise<{
+ id: string
+ resumedAt: number
+ paused: import('../src/shared/types').DownloadState
+ }> {
const reached = origin.hold(5 * BLOCK)
const id = await plexo.start(origin.url(), origin.sha256)
await reached
await plexo.api.pauseDownload(id)
- await plexo.waitForStatus('paused')
- origin.setContent(seededBytes(SIZE, 9), '"v2"')
+ const paused = await plexo.waitForStatus('paused')
+ change()
origin.release()
-
+ const resumedAt = origin.log.length
await plexo.api.resumeDownload(id)
+ return { id, resumedAt, paused }
+ }
+
+ test('a new version published while paused (new ETag) → error, nothing kept', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: SIZE })
+ await pauseThenChange(plexo, origin, () => origin.setContent(seededBytes(SIZE, 9), '"v2"'))
const state = await plexo.waitForStatus('error')
expect(state.error).toMatch(/changed/)
})
- test('Last-Modified changed while paused → error', async ({ plexo, serve }) => {
+ test('a new version published while paused (new Last-Modified) → error', async ({
+ plexo,
+ serve
+ }) => {
const origin = await serve({
size: SIZE,
etag: null,
lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT'
})
- const reached = origin.hold(5 * BLOCK)
- const id = await plexo.start(origin.url(), origin.sha256)
- await reached
- await plexo.api.pauseDownload(id)
- await plexo.waitForStatus('paused')
- origin.lastModified = 'Thu, 02 Jan 2025 00:00:00 GMT'
- origin.release()
-
- await plexo.api.resumeDownload(id)
+ await pauseThenChange(plexo, origin, () => {
+ origin.setContent(seededBytes(SIZE, 9), null)
+ origin.lastModified = 'Thu, 02 Jan 2025 00:00:00 GMT'
+ })
await plexo.waitForStatus('error')
})
+ test('same bytes under a new ETag while paused (server migrated) → resumes, progress kept', async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: SIZE })
+ const { resumedAt, paused } = await pauseThenChange(plexo, origin, () =>
+ origin.setContent(origin.content, '"migrated"')
+ )
+ await plexo.waitForStatus('completed')
+
+ // Kept, not restarted: after resuming, no block that was already complete was fetched again
+ // — only the small samples that confirmed the bytes match.
+ const done = (paused.blocks ?? []).filter((block) => block.status === 'completed')
+ const afterResume = origin.log.slice(resumedAt)
+ const refetchedDone = afterResume.filter((entry) =>
+ done.some((block) => entry.range?.start === block.rangeStart && entry.bytesSent > 16 * 1024)
+ )
+ expect(done.length).toBeGreaterThan(0)
+ expect(refetchedDone).toEqual([])
+ expect(afterResume.some((entry) => entry.bytesSent > 0 && entry.bytesSent <= 16 * 1024)).toBe(
+ true
+ )
+ })
+
test('no validators at all → resumes', async ({ plexo, serve }) => {
const origin = await serve({ size: SIZE, etag: null, lastModified: null })
const reached = origin.hold(5 * BLOCK)
@@ -177,21 +216,6 @@ test.describe('resume safety checks @smoke', () => {
await plexo.api.resumeDownload(id)
await plexo.waitForStatus('completed', 10_000)
})
-
- test('resume while the server hangs on the check request', async ({ plexo, serve }) => {
- const origin = await serve({ size: SIZE })
- const reached = origin.hold(5 * BLOCK)
- const id = await plexo.start(origin.url(), origin.sha256)
- await reached
- await plexo.api.pauseDownload(id)
- await plexo.waitForStatus('paused')
- origin.release()
- origin.setRule(({ range }) => (range?.start === 0 && range.end === 0 ? 'stallHeaders' : 'ok'))
-
- await plexo.api.resumeDownload(id)
- // Either outcome is fine — resuming, or a clear error. Sitting silently paused is not.
- await plexo.waitUntil((state) => state.status !== 'paused' || Boolean(state.error), 15_000)
- })
})
test.describe('cancel and remove @smoke', () => {
diff --git a/e2e/origin.ts b/e2e/origin.ts
index 4769ffe..d5afcd8 100644
--- a/e2e/origin.ts
+++ b/e2e/origin.ts
@@ -92,6 +92,9 @@ export class Origin {
lastModified: string | null
readonly log: LoggedRequest[] = []
private rule: (request: OriginRequest) => Fault | undefined = () => 'ok'
+ private version: (
+ request: OriginRequest
+ ) => { content: Buffer; etag: string | null } | undefined = () => undefined
private holdAt: number | null = null
private holdReached: (() => void) | null = null
private holdRelease: (() => void) | null = null
@@ -137,6 +140,14 @@ export class Origin {
this.rule = rule
}
+ /** Answers some requests as if from another server behind a load balancer, with its own
+ * content and ETag. Returning undefined serves the default version. */
+ setVersionRule(
+ rule: (request: OriginRequest) => { content: Buffer; etag: string | null } | undefined
+ ): void {
+ this.version = rule
+ }
+
/** Swaps the served file, as a server publishing a new version would. */
setContent(content: Buffer, etag: string | null): void {
this.content = content
@@ -190,10 +201,12 @@ export class Origin {
}
// Snapshot, so a setContent() mid-response doesn't splice two versions into one reply.
- const content = this.content
+ const served = this.version(request)
+ const content = served?.content ?? this.content
+ const etag = served ? served.etag : this.etag
const total = content.length
const headers: Record<string, string | number> = {}
- if (this.etag) headers['ETag'] = this.etag
+ if (etag) headers['ETag'] = etag
if (this.lastModified) headers['Last-Modified'] = this.lastModified
if (this.options.contentDisposition) {
headers['Content-Disposition'] = this.options.contentDisposition
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.ts
index 97cd246..9c39cd9 100644
--- a/src/main/download/chunkDownloader.ts
+++ b/src/main/download/chunkDownloader.ts
@@ -3,6 +3,7 @@ import { request as httpRequest, type ClientRequest, type IncomingMessage } from
import { request as httpsRequest } from 'node:https'
import { URL } from 'node:url'
import { testKnobs } from '../testKnobs'
+import { compareVersion, type FileVersion, type VersionCheck } from './fileVersion'
export interface ChunkDownloadOptions {
url: string
@@ -16,18 +17,22 @@ export interface ChunkDownloadOptions {
append: boolean
onProgress: (bytesDownloadedThisRun: number) => void
signal: AbortSignal
- /** What the probe saw. A response that disagrees comes from a different version of the file. */
- expected: {
- etag: string | null
- lastModified: string | null
- /** 0 = unknown. */
- totalBytes: number
- }
+ /** The version the download started on, plus any confirmed to serve identical bytes. */
+ acceptedVersions: FileVersion[]
}
-/** The server is now serving a different file than the one this download started on. Retrying
- * can't help — every retry would fetch the new version — so this fails the whole download. */
-export class RemoteChangedError extends Error {}
+/** A response whose version doesn't match the download's. Nothing from it was written; the
+ * caller decides whether it's a new file (fail) or the same bytes under another label (accept). */
+export class RemoteChangedError extends Error {
+ constructor(
+ readonly check: Exclude<VersionCheck, { kind: 'same' }>,
+ readonly seen: FileVersion
+ ) {
+ super(
+ `The file on the server changed during the download (${check.detail}). Start the download over.`
+ )
+ }
+}
// A server that accepts the connection and then goes silent (no data, no
// error, no close) would otherwise hang the chunk forever with no way to
@@ -63,35 +68,15 @@ function header(res: IncomingMessage, name: string): string | undefined {
return Array.isArray(value) ? value[0] : value
}
-/** Why this response can't belong to the file the download started on, or null if it can. Each
- * part is fetched separately, so without this a file republished mid-download would be stitched
- * together from two versions and still pass every length check. */
-function versionMismatch(
- res: IncomingMessage,
- served: ServedRange | null,
- expected: ChunkDownloadOptions['expected']
-): string | null {
- const etag = header(res, 'etag')
- if (expected.etag && etag && etag !== expected.etag) return `its ETag is now ${etag}`
- const lastModified = header(res, 'last-modified')
- if (
- !expected.etag &&
- expected.lastModified &&
- lastModified &&
- lastModified !== expected.lastModified
- ) {
- return `it was modified at ${lastModified}`
- }
- // A 206 states the file's total size in Content-Range; a 200 is the whole file, so its length.
- const total =
- served?.total ??
- (res.statusCode === 200 && header(res, 'content-length')
- ? Number(header(res, 'content-length'))
- : null)
- if (expected.totalBytes > 0 && total !== null && total !== expected.totalBytes) {
- return `it is now ${total} bytes instead of ${expected.totalBytes}`
+/** The version a response says it's from. A 206 states the file's total size in
+ * Content-Range; a 200 is the whole file, so its length is the size. */
+function versionOf(res: IncomingMessage, served: ServedRange | null): FileVersion {
+ const length = header(res, 'content-length')
+ return {
+ etag: header(res, 'etag') ?? null,
+ lastModified: header(res, 'last-modified') ?? null,
+ totalBytes: served?.total ?? (res.statusCode === 200 && length ? Number(length) : 0)
}
- return null
}
/**
@@ -113,7 +98,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
append,
onProgress,
signal,
- expected
+ acceptedVersions
} = options
return new Promise((resolve, reject) => {
@@ -208,14 +193,13 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
return
}
+ // Each part is fetched separately, so a file republished mid-download would otherwise
+ // be stitched together from two versions and still pass every length check.
const served = parseContentRange(res.headers['content-range'])
- const mismatch = versionMismatch(res, served, expected)
- if (mismatch) {
- fail(
- new RemoteChangedError(
- `The file on the server changed during the download (${mismatch}). Start the download over.`
- )
- )
+ const seen = versionOf(res, served)
+ const check = compareVersion(acceptedVersions, seen)
+ if (check.kind !== 'same') {
+ fail(new RemoteChangedError(check, seen))
res.resume()
return
}
@@ -309,3 +293,53 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
attempt(new URL(url), MAX_REDIRECTS)
})
}
+
+/** Fetches `start`..`end` into memory, following redirects, and reports which version served
+ * it — used to compare a few bytes already on disk against what a server serves now. */
+export function fetchRange(
+ url: string,
+ start: number,
+ end: number,
+ localAddress: string
+): Promise<{ body: Buffer; version: FileVersion }> {
+ return new Promise((resolve, reject) => {
+ const attempt = (target: URL, redirectsLeft: number): void => {
+ const requester = target.protocol === 'https:' ? httpsRequest : httpRequest
+ const req = requester(
+ {
+ method: 'GET',
+ hostname: target.hostname,
+ port: target.port || undefined,
+ path: `${target.pathname}${target.search}`,
+ localAddress,
+ family: 4,
+ headers: { 'User-Agent': 'Plexo/1.0', Range: `bytes=${start}-${end}` }
+ },
+ (res) => {
+ const status = res.statusCode ?? 0
+ if (status >= 300 && status < 400 && res.headers.location && redirectsLeft > 0) {
+ res.resume()
+ attempt(new URL(res.headers.location, target), redirectsLeft - 1)
+ return
+ }
+ const served = parseContentRange(res.headers['content-range'])
+ if (status !== 206 || served?.start !== start) {
+ res.resume()
+ reject(new Error(`Unexpected response to a sample request (status ${status})`))
+ return
+ }
+ const chunks: Buffer[] = []
+ res.on('data', (chunk: Buffer) => chunks.push(chunk))
+ res.on('error', reject)
+ res.on('end', () =>
+ resolve({ body: Buffer.concat(chunks), version: versionOf(res, served) })
+ )
+ }
+ )
+ req.on('error', reject)
+ req.setTimeout(STALL_TIMEOUT_MS, () => req.destroy(new Error('Sample request stalled')))
+ req.end()
+ }
+ attempt(new URL(url), MAX_REDIRECTS)
+ })
+}
diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.ts
index d68304f..3b31f28 100644
--- a/src/main/download/downloadManager.ts
+++ b/src/main/download/downloadManager.ts
@@ -25,10 +25,10 @@ import type {
StartDownloadRequest,
StartSimulatedDownloadRequest
} from '../../shared/types'
-import { downloadChunk, RemoteChangedError } from './chunkDownloader'
+import { downloadChunk, fetchRange, RemoteChangedError } from './chunkDownloader'
+import { compareVersion, type FileVersion } from './fileVersion'
import { testKnobs } from '../testKnobs'
import { reserveDestinationPath } from './paths'
-import { isResourceUnchanged } from './probe'
import {
createSimSession,
downloadChunkSimulated,
@@ -60,6 +60,9 @@ interface DownloadRuntime {
persistenceTimer?: NodeJS.Timeout
persistenceChain: Promise<void>
removed: boolean
+ /** The version this download started on, plus any since confirmed to serve identical bytes
+ * (see confirmSameBytes). Not persisted: after a restart, a confirmation is simply redone. */
+ acceptedVersions: FileVersion[]
}
interface PersistedDownload {
@@ -192,6 +195,18 @@ async function reconcilePartFileSize(partPath: string, expectedBytes: number): P
return safeBytes
}
+function requestedVersion(request: StartDownloadRequest): FileVersion {
+ return { etag: request.etag, lastModified: request.lastModified, totalBytes: request.totalBytes }
+}
+
+// How much of the already-downloaded file to re-fetch and compare when a server labels a
+// response with an ETag or Last-Modified this download hasn't seen before.
+const SAMPLE_BYTES = 16 * 1024
+const MAX_SAMPLES = 8
+/** A sample must come from a server presenting the new label; behind a load balancer the next
+ * request may land on another one, so take a few tries at reaching it. */
+const SAMPLE_TRIES = 4
+
function formatGigabytes(bytes: number): string {
return `${(bytes / 1024 ** 3).toFixed(2)} GB`
}
@@ -308,7 +323,8 @@ export class DownloadManager {
blocks,
totalBlocks: state.totalBlocks ?? blocks.length,
persistenceChain: Promise.resolve(),
- removed: false
+ removed: false,
+ acceptedVersions: [requestedVersion(persisted.requestPayload)]
})
} catch {
// Ignore incomplete or corrupt manifests; other downloads can still be restored.
@@ -561,7 +577,8 @@ export class DownloadManager {
blocks,
totalBlocks: blocks.length,
persistenceChain: Promise.resolve(),
- removed: false
+ removed: false,
+ acceptedVersions: [requestedVersion(requestPayload)]
}
this.runtimes.set(id, runtime)
await this.persistNow(runtime)
@@ -604,28 +621,11 @@ export class DownloadManager {
void this.resumeAfterVerifying(runtime)
}
- // Appending onto part files assumes the remote file hasn't changed since
- // it was probed — if the server's ETag/Last-Modified moved on while this
- // download sat paused, resuming would silently stitch old and new bytes
- // together. Check first, and refuse to resume rather than corrupt the
- // output (the user can always start the download over from scratch).
+ // A file that changed on the server while this download was paused is caught by the first
+ // chunk request after resuming: its response is checked against the version the download
+ // started on (see runWorker), which can tell a real change from a relabelled server.
private async resumeAfterVerifying(runtime: DownloadRuntime): Promise<void> {
- const { url, etag, lastModified } = runtime.requestPayload
- // A simulated download has no server to re-probe — the source file sitting on disk is
- // exactly what it was when the sim session was created, so there's nothing to verify.
- const unchanged = isSimulatedUrl(url) || (await isResourceUnchanged(url, etag, lastModified))
-
- if (runtime.state.status !== 'paused') return // cancelled while we were checking
-
- if (!unchanged) {
- runtime.state.status = 'error'
- runtime.state.error =
- 'The remote file changed while this download was paused, so resuming would corrupt it. Start the download over instead.'
- this.pushUpdate(runtime)
- await this.cleanupTempDir(runtime)
- await this.discardUnfinishedDestination(runtime)
- return
- }
+ const { url } = runtime.requestPayload
let availableInterfaces: NetworkInterfaceInfo[]
if (isSimulatedUrl(url)) {
@@ -873,11 +873,7 @@ export class DownloadManager {
destinationPath: partPath,
append: resumeOffset > 0,
signal: controller.signal,
- expected: {
- etag: runtime.requestPayload.etag,
- lastModified: runtime.requestPayload.lastModified,
- totalBytes: runtime.requestPayload.totalBytes
- },
+ acceptedVersions: runtime.acceptedVersions,
onProgress: (bytesThisRun) => {
const delta = bytesThisRun - lastReportedThisRun
lastReportedThisRun = bytesThisRun
@@ -917,15 +913,41 @@ export class DownloadManager {
chunk.error = message
if (error instanceof RemoteChangedError) {
- // Every other worker would hit the same new version, so stop them all now rather
- // than let each burn through its retries first.
- block.status = 'pending'
- chunk.status = 'error'
- chunk.speedBytesPerSec = 0
- runtime.state.status = 'error'
- runtime.state.error = message
- for (const cr of runtime.chunkRuntimes.values()) cr.controller.abort()
- break
+ const verdict =
+ error.check.kind === 'size'
+ ? 'different'
+ : await this.confirmSameBytes(runtime, iface, error.seen)
+ // The check takes a moment; the download may have been paused or stopped meanwhile.
+ const statusNow = runtime.state.status as DownloadStatus
+ if (statusNow !== 'downloading') {
+ block.status = 'pending'
+ chunk.status = statusNow === 'paused' ? 'paused' : 'cancelled'
+ chunk.speedBytesPerSec = 0
+ break
+ }
+ if (verdict === 'same') {
+ // Same bytes under another label — accept it and fetch this block again straight
+ // away; nothing from the rejected response was written.
+ if (compareVersion(runtime.acceptedVersions, error.seen).kind !== 'same') {
+ runtime.acceptedVersions.push(error.seen)
+ }
+ chunk.error = undefined
+ block.status = 'pending'
+ continue
+ }
+ if (verdict === 'different') {
+ // Every other worker would hit the same new version, so stop them all now rather
+ // than let each burn through its retries first.
+ block.status = 'pending'
+ chunk.status = 'error'
+ chunk.speedBytesPerSec = 0
+ runtime.state.status = 'error'
+ runtime.state.error = message
+ for (const cr of runtime.chunkRuntimes.values()) cr.controller.abort()
+ break
+ }
+ // 'unknown' — nothing to compare yet, or the check itself failed: retry like any
+ // other failed request. Nothing wrong was written either way.
}
attempt += 1
@@ -997,6 +1019,63 @@ export class DownloadManager {
this.scheduleUpdate(runtime)
}
+ /**
+ * Settles whether a server labelling the file differently (new ETag or Last-Modified, same
+ * size) is serving the same bytes — load-balanced servers often disagree on labels for
+ * identical files — or a new version. Re-fetches a spread of bytes this download already has
+ * on disk, from a server presenting the new label, and compares them.
+ *
+ * 'unknown' when there's nothing on disk to compare yet or the samples couldn't be fetched;
+ * the caller then treats the response as an ordinary failed request and retries.
+ */
+ private async confirmSameBytes(
+ runtime: DownloadRuntime,
+ iface: NetworkInterfaceInfo,
+ seen: FileVersion
+ ): Promise<'same' | 'different' | 'unknown'> {
+ if (compareVersion(runtime.acceptedVersions, seen).kind === 'same') return 'same'
+
+ // Every byte on disk came from an accepted version: mismatched responses are rejected
+ // before anything is written.
+ const withData = runtime.blocks.filter((block) => block.bytesDownloaded > 0)
+ const step = Math.max(1, withData.length / MAX_SAMPLES)
+ const picks = Array.from(
+ { length: Math.min(MAX_SAMPLES, withData.length) },
+ (_, i) => withData[Math.floor(i * step)]
+ )
+
+ let compared = 0
+ for (const block of picks) {
+ const partPath = join(runtime.tempDir, `part-${block.index}`)
+ let local: Buffer
+ try {
+ local = (await readFile(partPath)).subarray(0, SAMPLE_BYTES)
+ } catch {
+ continue
+ }
+ if (local.length === 0) continue
+
+ for (let tries = 0; tries < SAMPLE_TRIES; tries++) {
+ try {
+ const remote = await fetchRange(
+ runtime.requestPayload.url,
+ block.rangeStart,
+ block.rangeStart + local.length - 1,
+ iface.address
+ )
+ // A reply from a server still presenting an accepted label proves nothing here.
+ if (compareVersion([seen], remote.version).kind !== 'same') continue
+ if (!remote.body.equals(local)) return 'different'
+ compared += 1
+ break
+ } catch {
+ return 'unknown'
+ }
+ }
+ }
+ return compared > 0 ? 'same' : 'unknown'
+ }
+
private recomputeAggregates(runtime: DownloadRuntime): void {
runtime.state.bytesDownloaded = runtime.blocks.reduce(
(sum, entry) => sum + entry.bytesDownloaded,
diff --git a/src/main/download/fileVersion.ts b/src/main/download/fileVersion.ts
new file mode 100644
index 0000000..ee462f8
--- /dev/null
+++ b/src/main/download/fileVersion.ts
@@ -0,0 +1,49 @@
+/** What a server says about which version of the file it's serving. */
+export interface FileVersion {
+ etag: string | null
+ lastModified: string | null
+ /** 0 = unknown. */
+ totalBytes: number
+}
+
+/**
+ * How a response's version compares with the ones a download has accepted:
+ * - 'same': nothing suggests a different file.
+ * - 'size': the size differs. Identical bytes can't have a different size, so this is proof.
+ * - 'validators': the ETag or Last-Modified differs. That's only a hint — load-balanced servers
+ * can label the same bytes differently — so it has to be settled by comparing actual bytes.
+ */
+export type VersionCheck =
+ { kind: 'same' } | { kind: 'size'; detail: string } | { kind: 'validators'; detail: string }
+
+/** Drops what varies between servers without the bytes changing: the weak-validator marker,
+ * the quotes, and the suffix some servers add for a compressed variant (Apache's "-gzip"). */
+export function normalizeEtag(etag: string): string {
+ return etag
+ .trim()
+ .replace(/^W\//i, '')
+ .replace(/^"(.*)"$/, '$1')
+ .replace(/[-;](gzip|br|deflate|zstd)$/i, '')
+}
+
+/** `accepted` is the version the download started on, plus any whose bytes were since
+ * confirmed identical to it. */
+export function compareVersion(accepted: FileVersion[], seen: FileVersion): VersionCheck {
+ const expectedSize = accepted[0]?.totalBytes ?? 0
+ if (expectedSize > 0 && seen.totalBytes > 0 && seen.totalBytes !== expectedSize) {
+ return { kind: 'size', detail: `it is now ${seen.totalBytes} bytes instead of ${expectedSize}` }
+ }
+
+ const etags = accepted.flatMap((version) => (version.etag ? [normalizeEtag(version.etag)] : []))
+ if (seen.etag && etags.length > 0) {
+ return etags.includes(normalizeEtag(seen.etag))
+ ? { kind: 'same' }
+ : { kind: 'validators', detail: `its ETag is now ${seen.etag}` }
+ }
+
+ const dates = accepted.flatMap((version) => (version.lastModified ? [version.lastModified] : []))
+ if (seen.lastModified && dates.length > 0 && !dates.includes(seen.lastModified)) {
+ return { kind: 'validators', detail: `it was modified at ${seen.lastModified}` }
+ }
+ return { kind: 'same' }
+}
diff --git a/src/main/download/probe.ts b/src/main/download/probe.ts
index 9365cbf..d63a863 100644
--- a/src/main/download/probe.ts
+++ b/src/main/download/probe.ts
@@ -7,7 +7,7 @@ import { testKnobs } from '../testKnobs'
const MAX_REDIRECTS = 5
const USER_AGENT = 'Plexo/1.0'
// A server that accepts the connection and never answers would otherwise hang the probe — and
-// with it resume, which re-probes before continuing — forever. Same budget as a stalled chunk.
+// the link field's "Checking…" — forever. Same budget as a stalled chunk.
const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMs
type Headers = Record<string, string | string[] | undefined>
@@ -90,36 +90,6 @@ async function requestFollowingRedirects(
return { current, response }
}
-/**
- * Re-checks a previously probed URL's strong validators before a paused
- * download resumes. Appending onto part files assumes the remote content
- * hasn't changed since it was probed — if it has (a different ETag or
- * Last-Modified), stitching old and new bytes together would silently
- * produce a corrupt file. Returns true when unchanged *or* when we can't
- * tell (no validators, or the check itself failed) — a probe failure isn't
- * proof the file changed, so it shouldn't block a resume on its own.
- */
-export async function isResourceUnchanged(
- rawUrl: string,
- etag: string | null,
- lastModified: string | null
-): Promise<boolean> {
- if (!etag && !lastModified) return true
-
- try {
- const { response } = await requestFollowingRedirects(rawUrl)
- if (!response || response.statusCode >= 400) return true
-
- const currentEtag = headerValue(response.headers, 'etag')
- const currentLastModified = headerValue(response.headers, 'last-modified')
- if (etag && currentEtag) return currentEtag === etag
- if (lastModified && currentLastModified) return currentLastModified === lastModified
- return true
- } catch {
- return true
- }
-}
-
export async function probeUrl(rawUrl: string): Promise<ProbeResult> {
const { current, response } = await requestFollowingRedirects(rawUrl)