Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Tell a changed file from a relabelled one instead of failing on any ETag change
Anmol Kapil194433a5c7d2
diff --git a/e2e/download.spec.ts b/e2e/download.spec.tsindex 2c44682..9964e98 100644--- a/e2e/download.spec.ts+++ b/e2e/download.spec.ts@@ -136,6 +136,17 @@ test.describe('edge cases', () => {await plexo.waitForStatus('completed', 5000)})+ test('a server that never answers the link check → an error, not endless "Checking…" @smoke', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: BLOCK })+ origin.setRule(() => 'stallHeaders')+ const started = Date.now()+ await expect(plexo.api.probeUrl(origin.url())).rejects.toThrow(/did not respond/)+ expect(Date.now() - started).toBeLessThan(10_000)+ })+test('starting a second download while one is active is refused @smoke', async ({plexo,servediff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.tsindex eb0c601..f15dccf 100644--- a/e2e/integrity.spec.ts+++ b/e2e/integrity.spec.ts@@ -158,3 +158,56 @@ test.describe('the file changes on the server mid-download @smoke', () => {})}})++test.describe('servers that label the same file differently @smoke', () => {+ // None of these is a changed file, so none may fail the download.+ test('load balancer: identical bytes, two different ETags', async ({ plexo, serve }) => {+ const origin = await serve({ size: SIZE, etag: '"server-a"' })+ origin.setVersionRule(({ n }) =>+ n % 2 === 0 ? { content: origin.content, etag: '"server-b"' } : undefined+ )+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })+ await plexo.waitForStatus('completed')+ expect(origin.log.filter((entry) => entry.n % 2 === 0).length).toBeGreaterThan(0)+ })++ test('the same ETag, differing only by W/ or a -gzip suffix', async ({ plexo, serve }) => {+ const origin = await serve({ size: SIZE, etag: '"v1"' })+ const variants = ['W/"v1"', '"v1-gzip"', '"v1"']+ origin.setVersionRule(({ n }) => ({ content: origin.content, etag: variants[n % 3] }))+ await plexo.start(origin.url(), origin.sha256, { connections: 2 })+ await plexo.waitForStatus('completed')+ })++ test('load balancer: identical bytes, Last-Modified differs per server (no ETag)', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({+ size: SIZE,+ etag: null,+ lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT'+ })+ // lastModified is shared, so alternate it from the rule instead.+ origin.setVersionRule(({ n }) => {+ origin.lastModified =+ n % 2 === 0 ? 'Wed, 01 Jan 2025 00:00:07 GMT' : 'Wed, 01 Jan 2025 00:00:00 GMT'+ return undefined+ })+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })+ await plexo.waitForStatus('completed')+ })++ test('half-rolled-out new version: some servers new, some old → error, never a mix', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: SIZE, seed: 1, etag: '"old"' })+ const next = seededBytes(SIZE, 2)+ origin.setVersionRule(({ n }) => (n % 2 === 0 ? { content: next, etag: '"new"' } : undefined))+ await plexo.start(origin.url(), origin.sha256, { connections: 4 })+ const state = await plexo.waitForStatus(['completed', 'error'])+ expect(state.status).toBe('error')+ expect(state.error).toMatch(/changed during the download/)+ })+})diff --git a/e2e/lifecycle.spec.ts b/e2e/lifecycle.spec.tsindex feb99e1..04a5358 100644--- a/e2e/lifecycle.spec.ts+++ b/e2e/lifecycle.spec.ts@@ -92,39 +92,78 @@ test.describe('pause during a retry backoff', () => {})test.describe('resume safety checks @smoke', () => {- test('ETag changed while paused → error, nothing kept', async ({ plexo, serve }) => {- const origin = await serve({ size: SIZE })+ /** Starts a download, pauses it partway, and lets `change` alter the server while paused. */+ async function pauseThenChange(+ plexo: import('./fixtures').PlexoApp,+ origin: import('./origin').Origin,+ change: () => void+ ): Promise<{+ id: string+ resumedAt: number+ paused: import('../src/shared/types').DownloadState+ }> {const reached = origin.hold(5 * BLOCK)const id = await plexo.start(origin.url(), origin.sha256)await reachedawait plexo.api.pauseDownload(id)- await plexo.waitForStatus('paused')- origin.setContent(seededBytes(SIZE, 9), '"v2"')+ const paused = await plexo.waitForStatus('paused')+ change()origin.release()-+ const resumedAt = origin.log.lengthawait plexo.api.resumeDownload(id)+ return { id, resumedAt, paused }+ }++ test('a new version published while paused (new ETag) → error, nothing kept', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: SIZE })+ await pauseThenChange(plexo, origin, () => origin.setContent(seededBytes(SIZE, 9), '"v2"'))const state = await plexo.waitForStatus('error')expect(state.error).toMatch(/changed/)})- test('Last-Modified changed while paused → error', async ({ plexo, serve }) => {+ test('a new version published while paused (new Last-Modified) → error', async ({+ plexo,+ serve+ }) => {const origin = await serve({size: SIZE,etag: null,lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT'})- const reached = origin.hold(5 * BLOCK)- const id = await plexo.start(origin.url(), origin.sha256)- await reached- await plexo.api.pauseDownload(id)- await plexo.waitForStatus('paused')- origin.lastModified = 'Thu, 02 Jan 2025 00:00:00 GMT'- origin.release()-- await plexo.api.resumeDownload(id)+ await pauseThenChange(plexo, origin, () => {+ origin.setContent(seededBytes(SIZE, 9), null)+ origin.lastModified = 'Thu, 02 Jan 2025 00:00:00 GMT'+ })await plexo.waitForStatus('error')})+ test('same bytes under a new ETag while paused (server migrated) → resumes, progress kept', async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: SIZE })+ const { resumedAt, paused } = await pauseThenChange(plexo, origin, () =>+ origin.setContent(origin.content, '"migrated"')+ )+ await plexo.waitForStatus('completed')++ // Kept, not restarted: after resuming, no block that was already complete was fetched again+ // — only the small samples that confirmed the bytes match.+ const done = (paused.blocks ?? []).filter((block) => block.status === 'completed')+ const afterResume = origin.log.slice(resumedAt)+ const refetchedDone = afterResume.filter((entry) =>+ done.some((block) => entry.range?.start === block.rangeStart && entry.bytesSent > 16 * 1024)+ )+ expect(done.length).toBeGreaterThan(0)+ expect(refetchedDone).toEqual([])+ expect(afterResume.some((entry) => entry.bytesSent > 0 && entry.bytesSent <= 16 * 1024)).toBe(+ true+ )+ })+test('no validators at all → resumes', async ({ plexo, serve }) => {const origin = await serve({ size: SIZE, etag: null, lastModified: null })const reached = origin.hold(5 * BLOCK)@@ -177,21 +216,6 @@ test.describe('resume safety checks @smoke', () => {await plexo.api.resumeDownload(id)await plexo.waitForStatus('completed', 10_000)})-- test('resume while the server hangs on the check request', async ({ plexo, serve }) => {- const origin = await serve({ size: SIZE })- const reached = origin.hold(5 * BLOCK)- const id = await plexo.start(origin.url(), origin.sha256)- await reached- await plexo.api.pauseDownload(id)- await plexo.waitForStatus('paused')- origin.release()- origin.setRule(({ range }) => (range?.start === 0 && range.end === 0 ? 'stallHeaders' : 'ok'))-- await plexo.api.resumeDownload(id)- // Either outcome is fine — resuming, or a clear error. Sitting silently paused is not.- await plexo.waitUntil((state) => state.status !== 'paused' || Boolean(state.error), 15_000)- })})test.describe('cancel and remove @smoke', () => {diff --git a/e2e/origin.ts b/e2e/origin.tsindex 4769ffe..d5afcd8 100644--- a/e2e/origin.ts+++ b/e2e/origin.ts@@ -92,6 +92,9 @@ export class Origin {lastModified: string | nullreadonly log: LoggedRequest[] = []private rule: (request: OriginRequest) => Fault | undefined = () => 'ok'+ private version: (+ request: OriginRequest+ ) => { content: Buffer; etag: string | null } | undefined = () => undefinedprivate holdAt: number | null = nullprivate holdReached: (() => void) | null = nullprivate holdRelease: (() => void) | null = null@@ -137,6 +140,14 @@ export class Origin {this.rule = rule}+ /** Answers some requests as if from another server behind a load balancer, with its own+ * content and ETag. Returning undefined serves the default version. */+ setVersionRule(+ rule: (request: OriginRequest) => { content: Buffer; etag: string | null } | undefined+ ): void {+ this.version = rule+ }+/** Swaps the served file, as a server publishing a new version would. */setContent(content: Buffer, etag: string | null): void {this.content = content@@ -190,10 +201,12 @@ export class Origin {}// Snapshot, so a setContent() mid-response doesn't splice two versions into one reply.- const content = this.content+ const served = this.version(request)+ const content = served?.content ?? this.content+ const etag = served ? served.etag : this.etagconst total = content.lengthconst headers: Record<string, string | number> = {}- if (this.etag) headers['ETag'] = this.etag+ if (etag) headers['ETag'] = etagif (this.lastModified) headers['Last-Modified'] = this.lastModifiedif (this.options.contentDisposition) {headers['Content-Disposition'] = this.options.contentDispositiondiff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.tsindex 97cd246..9c39cd9 100644--- a/src/main/download/chunkDownloader.ts+++ b/src/main/download/chunkDownloader.ts@@ -3,6 +3,7 @@ import { request as httpRequest, type ClientRequest, type IncomingMessage } fromimport { request as httpsRequest } from 'node:https'import { URL } from 'node:url'import { testKnobs } from '../testKnobs'+import { compareVersion, type FileVersion, type VersionCheck } from './fileVersion'export interface ChunkDownloadOptions {url: string@@ -16,18 +17,22 @@ export interface ChunkDownloadOptions {append: booleanonProgress: (bytesDownloadedThisRun: number) => voidsignal: AbortSignal- /** What the probe saw. A response that disagrees comes from a different version of the file. */- expected: {- etag: string | null- lastModified: string | null- /** 0 = unknown. */- totalBytes: number- }+ /** The version the download started on, plus any confirmed to serve identical bytes. */+ acceptedVersions: FileVersion[]}-/** The server is now serving a different file than the one this download started on. Retrying- * can't help — every retry would fetch the new version — so this fails the whole download. */-export class RemoteChangedError extends Error {}+/** A response whose version doesn't match the download's. Nothing from it was written; the+ * caller decides whether it's a new file (fail) or the same bytes under another label (accept). */+export class RemoteChangedError extends Error {+ constructor(+ readonly check: Exclude<VersionCheck, { kind: 'same' }>,+ readonly seen: FileVersion+ ) {+ super(+ `The file on the server changed during the download (${check.detail}). Start the download over.`+ )+ }+}// A server that accepts the connection and then goes silent (no data, no// error, no close) would otherwise hang the chunk forever with no way to@@ -63,35 +68,15 @@ function header(res: IncomingMessage, name: string): string | undefined {return Array.isArray(value) ? value[0] : value}-/** Why this response can't belong to the file the download started on, or null if it can. Each- * part is fetched separately, so without this a file republished mid-download would be stitched- * together from two versions and still pass every length check. */-function versionMismatch(- res: IncomingMessage,- served: ServedRange | null,- expected: ChunkDownloadOptions['expected']-): string | null {- const etag = header(res, 'etag')- if (expected.etag && etag && etag !== expected.etag) return `its ETag is now ${etag}`- const lastModified = header(res, 'last-modified')- if (- !expected.etag &&- expected.lastModified &&- lastModified &&- lastModified !== expected.lastModified- ) {- return `it was modified at ${lastModified}`- }- // A 206 states the file's total size in Content-Range; a 200 is the whole file, so its length.- const total =- served?.total ??- (res.statusCode === 200 && header(res, 'content-length')- ? Number(header(res, 'content-length'))- : null)- if (expected.totalBytes > 0 && total !== null && total !== expected.totalBytes) {- return `it is now ${total} bytes instead of ${expected.totalBytes}`+/** The version a response says it's from. A 206 states the file's total size in+ * Content-Range; a 200 is the whole file, so its length is the size. */+function versionOf(res: IncomingMessage, served: ServedRange | null): FileVersion {+ const length = header(res, 'content-length')+ return {+ etag: header(res, 'etag') ?? null,+ lastModified: header(res, 'last-modified') ?? null,+ totalBytes: served?.total ?? (res.statusCode === 200 && length ? Number(length) : 0)}- return null}/**@@ -113,7 +98,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {append,onProgress,signal,- expected+ acceptedVersions} = optionsreturn new Promise((resolve, reject) => {@@ -208,14 +193,13 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {return}+ // Each part is fetched separately, so a file republished mid-download would otherwise+ // be stitched together from two versions and still pass every length check.const served = parseContentRange(res.headers['content-range'])- const mismatch = versionMismatch(res, served, expected)- if (mismatch) {- fail(- new RemoteChangedError(- `The file on the server changed during the download (${mismatch}). Start the download over.`- )- )+ const seen = versionOf(res, served)+ const check = compareVersion(acceptedVersions, seen)+ if (check.kind !== 'same') {+ fail(new RemoteChangedError(check, seen))res.resume()return}@@ -309,3 +293,53 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {attempt(new URL(url), MAX_REDIRECTS)})}++/** Fetches `start`..`end` into memory, following redirects, and reports which version served+ * it — used to compare a few bytes already on disk against what a server serves now. */+export function fetchRange(+ url: string,+ start: number,+ end: number,+ localAddress: string+): Promise<{ body: Buffer; version: FileVersion }> {+ return new Promise((resolve, reject) => {+ const attempt = (target: URL, redirectsLeft: number): void => {+ const requester = target.protocol === 'https:' ? httpsRequest : httpRequest+ const req = requester(+ {+ method: 'GET',+ hostname: target.hostname,+ port: target.port || undefined,+ path: `${target.pathname}${target.search}`,+ localAddress,+ family: 4,+ headers: { 'User-Agent': 'Plexo/1.0', Range: `bytes=${start}-${end}` }+ },+ (res) => {+ const status = res.statusCode ?? 0+ if (status >= 300 && status < 400 && res.headers.location && redirectsLeft > 0) {+ res.resume()+ attempt(new URL(res.headers.location, target), redirectsLeft - 1)+ return+ }+ const served = parseContentRange(res.headers['content-range'])+ if (status !== 206 || served?.start !== start) {+ res.resume()+ reject(new Error(`Unexpected response to a sample request (status ${status})`))+ return+ }+ const chunks: Buffer[] = []+ res.on('data', (chunk: Buffer) => chunks.push(chunk))+ res.on('error', reject)+ res.on('end', () =>+ resolve({ body: Buffer.concat(chunks), version: versionOf(res, served) })+ )+ }+ )+ req.on('error', reject)+ req.setTimeout(STALL_TIMEOUT_MS, () => req.destroy(new Error('Sample request stalled')))+ req.end()+ }+ attempt(new URL(url), MAX_REDIRECTS)+ })+}diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex d68304f..3b31f28 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -25,10 +25,10 @@ import type {StartDownloadRequest,StartSimulatedDownloadRequest} from '../../shared/types'-import { downloadChunk, RemoteChangedError } from './chunkDownloader'+import { downloadChunk, fetchRange, RemoteChangedError } from './chunkDownloader'+import { compareVersion, type FileVersion } from './fileVersion'import { testKnobs } from '../testKnobs'import { reserveDestinationPath } from './paths'-import { isResourceUnchanged } from './probe'import {createSimSession,downloadChunkSimulated,@@ -60,6 +60,9 @@ interface DownloadRuntime {persistenceTimer?: NodeJS.TimeoutpersistenceChain: Promise<void>removed: boolean+ /** The version this download started on, plus any since confirmed to serve identical bytes+ * (see confirmSameBytes). Not persisted: after a restart, a confirmation is simply redone. */+ acceptedVersions: FileVersion[]}interface PersistedDownload {@@ -192,6 +195,18 @@ async function reconcilePartFileSize(partPath: string, expectedBytes: number): Preturn safeBytes}+function requestedVersion(request: StartDownloadRequest): FileVersion {+ return { etag: request.etag, lastModified: request.lastModified, totalBytes: request.totalBytes }+}++// How much of the already-downloaded file to re-fetch and compare when a server labels a+// response with an ETag or Last-Modified this download hasn't seen before.+const SAMPLE_BYTES = 16 * 1024+const MAX_SAMPLES = 8+/** A sample must come from a server presenting the new label; behind a load balancer the next+ * request may land on another one, so take a few tries at reaching it. */+const SAMPLE_TRIES = 4+function formatGigabytes(bytes: number): string {return `${(bytes / 1024 ** 3).toFixed(2)} GB`}@@ -308,7 +323,8 @@ export class DownloadManager {blocks,totalBlocks: state.totalBlocks ?? blocks.length,persistenceChain: Promise.resolve(),- removed: false+ removed: false,+ acceptedVersions: [requestedVersion(persisted.requestPayload)]})} catch {// Ignore incomplete or corrupt manifests; other downloads can still be restored.@@ -561,7 +577,8 @@ export class DownloadManager {blocks,totalBlocks: blocks.length,persistenceChain: Promise.resolve(),- removed: false+ removed: false,+ acceptedVersions: [requestedVersion(requestPayload)]}this.runtimes.set(id, runtime)await this.persistNow(runtime)@@ -604,28 +621,11 @@ export class DownloadManager {void this.resumeAfterVerifying(runtime)}- // Appending onto part files assumes the remote file hasn't changed since- // it was probed — if the server's ETag/Last-Modified moved on while this- // download sat paused, resuming would silently stitch old and new bytes- // together. Check first, and refuse to resume rather than corrupt the- // output (the user can always start the download over from scratch).+ // A file that changed on the server while this download was paused is caught by the first+ // chunk request after resuming: its response is checked against the version the download+ // started on (see runWorker), which can tell a real change from a relabelled server.private async resumeAfterVerifying(runtime: DownloadRuntime): Promise<void> {- const { url, etag, lastModified } = runtime.requestPayload- // A simulated download has no server to re-probe — the source file sitting on disk is- // exactly what it was when the sim session was created, so there's nothing to verify.- const unchanged = isSimulatedUrl(url) || (await isResourceUnchanged(url, etag, lastModified))-- if (runtime.state.status !== 'paused') return // cancelled while we were checking-- if (!unchanged) {- runtime.state.status = 'error'- runtime.state.error =- 'The remote file changed while this download was paused, so resuming would corrupt it. Start the download over instead.'- this.pushUpdate(runtime)- await this.cleanupTempDir(runtime)- await this.discardUnfinishedDestination(runtime)- return- }+ const { url } = runtime.requestPayloadlet availableInterfaces: NetworkInterfaceInfo[]if (isSimulatedUrl(url)) {@@ -873,11 +873,7 @@ export class DownloadManager {destinationPath: partPath,append: resumeOffset > 0,signal: controller.signal,- expected: {- etag: runtime.requestPayload.etag,- lastModified: runtime.requestPayload.lastModified,- totalBytes: runtime.requestPayload.totalBytes- },+ acceptedVersions: runtime.acceptedVersions,onProgress: (bytesThisRun) => {const delta = bytesThisRun - lastReportedThisRunlastReportedThisRun = bytesThisRun@@ -917,15 +913,41 @@ export class DownloadManager {chunk.error = messageif (error instanceof RemoteChangedError) {- // Every other worker would hit the same new version, so stop them all now rather- // than let each burn through its retries first.- block.status = 'pending'- chunk.status = 'error'- chunk.speedBytesPerSec = 0- runtime.state.status = 'error'- runtime.state.error = message- for (const cr of runtime.chunkRuntimes.values()) cr.controller.abort()- break+ const verdict =+ error.check.kind === 'size'+ ? 'different'+ : await this.confirmSameBytes(runtime, iface, error.seen)+ // The check takes a moment; the download may have been paused or stopped meanwhile.+ const statusNow = runtime.state.status as DownloadStatus+ if (statusNow !== 'downloading') {+ block.status = 'pending'+ chunk.status = statusNow === 'paused' ? 'paused' : 'cancelled'+ chunk.speedBytesPerSec = 0+ break+ }+ if (verdict === 'same') {+ // Same bytes under another label — accept it and fetch this block again straight+ // away; nothing from the rejected response was written.+ if (compareVersion(runtime.acceptedVersions, error.seen).kind !== 'same') {+ runtime.acceptedVersions.push(error.seen)+ }+ chunk.error = undefined+ block.status = 'pending'+ continue+ }+ if (verdict === 'different') {+ // Every other worker would hit the same new version, so stop them all now rather+ // than let each burn through its retries first.+ block.status = 'pending'+ chunk.status = 'error'+ chunk.speedBytesPerSec = 0+ runtime.state.status = 'error'+ runtime.state.error = message+ for (const cr of runtime.chunkRuntimes.values()) cr.controller.abort()+ break+ }+ // 'unknown' — nothing to compare yet, or the check itself failed: retry like any+ // other failed request. Nothing wrong was written either way.}attempt += 1@@ -997,6 +1019,63 @@ export class DownloadManager {this.scheduleUpdate(runtime)}+ /**+ * Settles whether a server labelling the file differently (new ETag or Last-Modified, same+ * size) is serving the same bytes — load-balanced servers often disagree on labels for+ * identical files — or a new version. Re-fetches a spread of bytes this download already has+ * on disk, from a server presenting the new label, and compares them.+ *+ * 'unknown' when there's nothing on disk to compare yet or the samples couldn't be fetched;+ * the caller then treats the response as an ordinary failed request and retries.+ */+ private async confirmSameBytes(+ runtime: DownloadRuntime,+ iface: NetworkInterfaceInfo,+ seen: FileVersion+ ): Promise<'same' | 'different' | 'unknown'> {+ if (compareVersion(runtime.acceptedVersions, seen).kind === 'same') return 'same'++ // Every byte on disk came from an accepted version: mismatched responses are rejected+ // before anything is written.+ const withData = runtime.blocks.filter((block) => block.bytesDownloaded > 0)+ const step = Math.max(1, withData.length / MAX_SAMPLES)+ const picks = Array.from(+ { length: Math.min(MAX_SAMPLES, withData.length) },+ (_, i) => withData[Math.floor(i * step)]+ )++ let compared = 0+ for (const block of picks) {+ const partPath = join(runtime.tempDir, `part-${block.index}`)+ let local: Buffer+ try {+ local = (await readFile(partPath)).subarray(0, SAMPLE_BYTES)+ } catch {+ continue+ }+ if (local.length === 0) continue++ for (let tries = 0; tries < SAMPLE_TRIES; tries++) {+ try {+ const remote = await fetchRange(+ runtime.requestPayload.url,+ block.rangeStart,+ block.rangeStart + local.length - 1,+ iface.address+ )+ // A reply from a server still presenting an accepted label proves nothing here.+ if (compareVersion([seen], remote.version).kind !== 'same') continue+ if (!remote.body.equals(local)) return 'different'+ compared += 1+ break+ } catch {+ return 'unknown'+ }+ }+ }+ return compared > 0 ? 'same' : 'unknown'+ }+private recomputeAggregates(runtime: DownloadRuntime): void {runtime.state.bytesDownloaded = runtime.blocks.reduce((sum, entry) => sum + entry.bytesDownloaded,diff --git a/src/main/download/fileVersion.ts b/src/main/download/fileVersion.tsnew file mode 100644index 0000000..ee462f8--- /dev/null+++ b/src/main/download/fileVersion.ts@@ -0,0 +1,49 @@+/** What a server says about which version of the file it's serving. */+export interface FileVersion {+ etag: string | null+ lastModified: string | null+ /** 0 = unknown. */+ totalBytes: number+}++/**+ * How a response's version compares with the ones a download has accepted:+ * - 'same': nothing suggests a different file.+ * - 'size': the size differs. Identical bytes can't have a different size, so this is proof.+ * - 'validators': the ETag or Last-Modified differs. That's only a hint — load-balanced servers+ * can label the same bytes differently — so it has to be settled by comparing actual bytes.+ */+export type VersionCheck =+ { kind: 'same' } | { kind: 'size'; detail: string } | { kind: 'validators'; detail: string }++/** Drops what varies between servers without the bytes changing: the weak-validator marker,+ * the quotes, and the suffix some servers add for a compressed variant (Apache's "-gzip"). */+export function normalizeEtag(etag: string): string {+ return etag+ .trim()+ .replace(/^W\//i, '')+ .replace(/^"(.*)"$/, '$1')+ .replace(/[-;](gzip|br|deflate|zstd)$/i, '')+}++/** `accepted` is the version the download started on, plus any whose bytes were since+ * confirmed identical to it. */+export function compareVersion(accepted: FileVersion[], seen: FileVersion): VersionCheck {+ const expectedSize = accepted[0]?.totalBytes ?? 0+ if (expectedSize > 0 && seen.totalBytes > 0 && seen.totalBytes !== expectedSize) {+ return { kind: 'size', detail: `it is now ${seen.totalBytes} bytes instead of ${expectedSize}` }+ }++ const etags = accepted.flatMap((version) => (version.etag ? [normalizeEtag(version.etag)] : []))+ if (seen.etag && etags.length > 0) {+ return etags.includes(normalizeEtag(seen.etag))+ ? { kind: 'same' }+ : { kind: 'validators', detail: `its ETag is now ${seen.etag}` }+ }++ const dates = accepted.flatMap((version) => (version.lastModified ? [version.lastModified] : []))+ if (seen.lastModified && dates.length > 0 && !dates.includes(seen.lastModified)) {+ return { kind: 'validators', detail: `it was modified at ${seen.lastModified}` }+ }+ return { kind: 'same' }+}diff --git a/src/main/download/probe.ts b/src/main/download/probe.tsindex 9365cbf..d63a863 100644--- a/src/main/download/probe.ts+++ b/src/main/download/probe.ts@@ -7,7 +7,7 @@ import { testKnobs } from '../testKnobs'const MAX_REDIRECTS = 5const USER_AGENT = 'Plexo/1.0'// A server that accepts the connection and never answers would otherwise hang the probe — and-// with it resume, which re-probes before continuing — forever. Same budget as a stalled chunk.+// the link field's "Checking…" — forever. Same budget as a stalled chunk.const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMstype Headers = Record<string, string | string[] | undefined>@@ -90,36 +90,6 @@ async function requestFollowingRedirects(return { current, response }}-/**- * Re-checks a previously probed URL's strong validators before a paused- * download resumes. Appending onto part files assumes the remote content- * hasn't changed since it was probed — if it has (a different ETag or- * Last-Modified), stitching old and new bytes together would silently- * produce a corrupt file. Returns true when unchanged *or* when we can't- * tell (no validators, or the check itself failed) — a probe failure isn't- * proof the file changed, so it shouldn't block a resume on its own.- */-export async function isResourceUnchanged(- rawUrl: string,- etag: string | null,- lastModified: string | null-): Promise<boolean> {- if (!etag && !lastModified) return true-- try {- const { response } = await requestFollowingRedirects(rawUrl)- if (!response || response.statusCode >= 400) return true-- const currentEtag = headerValue(response.headers, 'etag')- const currentLastModified = headerValue(response.headers, 'last-modified')- if (etag && currentEtag) return currentEtag === etag- if (lastModified && currentLastModified) return currentLastModified === lastModified- return true- } catch {- return true- }-}-export async function probeUrl(rawUrl: string): Promise<ProbeResult> {const { current, response } = await requestFollowingRedirects(rawUrl)