Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Fix seven smaller download and resume bugs

Anmol Kapil68b9939fcf00
diff --git a/e2e/disk.spec.ts b/e2e/disk.spec.ts
index a43af11..f0a762e 100644
--- a/e2e/disk.spec.ts
+++ b/e2e/disk.spec.ts
@@ -130,10 +130,6 @@ test.describe('destination folder problems @smoke', () => {
dirs
}) => {
test.skip(process.platform === 'win32', 'chmod does not make a folder read-only on Windows')
- test.fail(
- true,
- 'known bug: parts/ is created before the destination name is reserved, and never removed when that fails'
- )
const origin = await serve({ size: 4 * BLOCK })
await chmod(dirs.dest, 0o555)
try {
diff --git a/e2e/download.spec.ts b/e2e/download.spec.ts
index cd7fcc0..2c44682 100644
--- a/e2e/download.spec.ts
+++ b/e2e/download.spec.ts
@@ -123,7 +123,6 @@ test.describe('file names @smoke', () => {
})
test('a malformed %-escape in the URL path still downloads', async ({ plexo, serve }) => {
- test.fail(true, 'known bug: probe.ts decodeURIComponent(url.pathname) throws a URIError')
const origin = await serve({ size: 2 * BLOCK })
await plexo.start(origin.url('/files/100%25%E0%A4%A.bin'), origin.sha256)
await plexo.waitForStatus('completed')
@@ -132,10 +131,6 @@ test.describe('file names @smoke', () => {
test.describe('edge cases', () => {
test('a 0-byte file @smoke', async ({ plexo, serve }) => {
- test.fail(
- true,
- 'known bug: the probe asks for bytes=0-0, an empty file answers 416, probe throws'
- )
const origin = await serve({ size: 0 })
await plexo.start(origin.url(), origin.sha256)
await plexo.waitForStatus('completed', 5000)
diff --git a/e2e/lifecycle.spec.ts b/e2e/lifecycle.spec.ts
index 17956e8..feb99e1 100644
--- a/e2e/lifecycle.spec.ts
+++ b/e2e/lifecycle.spec.ts
@@ -167,10 +167,6 @@ test.describe('resume safety checks @smoke', () => {
})
test('resume against a server without range support', async ({ plexo, serve }) => {
- test.fail(
- true,
- 'known bug: resumes with Range: bytes=N-, the server answers 200, every retry fails'
- )
const origin = await serve({ size: SIZE, ranges: false })
const reached = origin.hold(5 * BLOCK)
const id = await plexo.start(origin.url(), origin.sha256)
@@ -183,7 +179,6 @@ test.describe('resume safety checks @smoke', () => {
})
test('resume while the server hangs on the check request', async ({ plexo, serve }) => {
- test.fail(true, 'known bug: probe.ts has no timeout, so resume waits forever')
const origin = await serve({ size: SIZE })
const reached = origin.hold(5 * BLOCK)
const id = await plexo.start(origin.url(), origin.sha256)
diff --git a/e2e/recovery.spec.ts b/e2e/recovery.spec.ts
index 088d47d..95033ab 100644
--- a/e2e/recovery.spec.ts
+++ b/e2e/recovery.spec.ts
@@ -169,10 +169,6 @@ test.describe('persisted state on disk @smoke', () => {
serve,
dirs
}) => {
- test.fail(
- true,
- 'known gap: resume trusts the manifest, never recreates parts/, and fails with ENOENT instead of re-fetching the missing blocks'
- )
const { id } = await pausedDownload(plexo, serve)
await plexo.quit()
await rm(join(dirs.userData, 'downloads', id, 'parts'), { recursive: true, force: true })
diff --git a/e2e/ui.spec.ts b/e2e/ui.spec.ts
index b908802..08a5a55 100644
--- a/e2e/ui.spec.ts
+++ b/e2e/ui.spec.ts
@@ -62,10 +62,6 @@ test.describe('UI journeys @smoke', () => {
serve,
dirs
}) => {
- test.fail(
- true,
- 'known bug: remove() does not stop the cancelled run from pushing more updates, so a late "cancelled" event brings the removed download back on screen'
- )
const origin = await serve({ size: SIZE })
await stubNativeUi(plexo, dirs.dest)
const page = plexo.page
diff --git a/src/main/download/chunkDownloader.ts b/src/main/download/chunkDownloader.ts
index 5f54785..97cd246 100644
--- a/src/main/download/chunkDownloader.ts
+++ b/src/main/download/chunkDownloader.ts
@@ -159,8 +159,13 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
const onAbort = (): void => fail(new DOMException('Aborted', 'AbortError'))
signal.addEventListener('abort', onAbort)
- const rangeHeader =
- rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}`
+ // The whole file from the start needs no Range at all — and an empty file would answer
+ // `bytes=0-` with 416, since it has no byte 0 to start from.
+ const headers: Record<string, string> = { 'User-Agent': 'Plexo/1.0' }
+ if (rangeStart > 0 || rangeEnd !== null) {
+ headers['Range'] =
+ rangeEnd === null ? `bytes=${rangeStart}-` : `bytes=${rangeStart}-${rangeEnd}`
+ }
const attempt = (targetUrl: URL, redirectsLeft: number): void => {
const requester = targetUrl.protocol === 'https:' ? httpsRequest : httpRequest
@@ -176,7 +181,7 @@ export function downloadChunk(options: ChunkDownloadOptions): Promise<void> {
// host to resolve to IPv4 too, or binding fails with EINVAL when DNS
// hands back an IPv6 address for it instead.
family: 4,
- headers: { 'User-Agent': 'Plexo/1.0', Range: rangeHeader }
+ headers
},
(res: IncomingMessage) => {
const status = res.statusCode ?? 0
diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.ts
index c0d2aca..d68304f 100644
--- a/src/main/download/downloadManager.ts
+++ b/src/main/download/downloadManager.ts
@@ -423,17 +423,24 @@ export class DownloadManager {
requestPayload.totalBytes
)
- const id = randomUUID()
- const tempDir = join(this.downloadDir(id), 'parts')
- await mkdir(tempDir, { recursive: true })
-
// Claimed on disk, not just picked, so a second download of the same file
- // name can't pick it too and overwrite this one at reassembly time.
+ // name can't pick it too and overwrite this one at reassembly time. Done
+ // before anything else is created, so a destination we can't write to
+ // leaves nothing behind.
const destinationPath = await reserveDestinationPath(
requestPayload.destinationDir,
requestPayload.suggestedFileName
)
+ const id = randomUUID()
+ const tempDir = join(this.downloadDir(id), 'parts')
+ try {
+ await mkdir(tempDir, { recursive: true })
+ } catch (error) {
+ await rm(destinationPath, { force: true })
+ throw error
+ }
+
const connectionsPerNetwork = Math.max(
1,
Math.min(
@@ -645,6 +652,23 @@ export class DownloadManager {
return
}
+ // The part files are the real record of what's downloaded, not the manifest. If one went
+ // missing or came up short while paused (userData cleaned out, a crash before a write hit
+ // the disk), fetch that block again instead of failing at assembly.
+ await mkdir(runtime.tempDir, { recursive: true })
+ await Promise.all(
+ runtime.blocks.map(async (block) => {
+ if (block.status !== 'completed' || block.rangeEnd === null) return
+ const partPath = join(runtime.tempDir, `part-${block.index}`)
+ const size = await stat(partPath).then(
+ (stats) => stats.size,
+ () => -1
+ )
+ if (size !== block.rangeEnd - block.rangeStart + 1) block.status = 'pending'
+ })
+ )
+ if (runtime.state.status !== 'paused') return
+
for (let index = 0; index < runtime.state.chunks.length; index++) {
const chunk = runtime.state.chunks[index]
const iface =
@@ -815,7 +839,12 @@ export class DownloadManager {
chunkRuntime.partPath = partPath
}
- const resumeOffset = await reconcilePartFileSize(partPath, block.bytesDownloaded)
+ // Without range support the server can only send the file from the start, so a retry or
+ // resume begins again at byte 0 rather than asking for a Range it will ignore.
+ const resumeOffset = await reconcilePartFileSize(
+ partPath,
+ runtime.requestPayload.supportsRanges ? block.bytesDownloaded : 0
+ )
if (resumeOffset !== block.bytesDownloaded) {
block.bytesDownloaded = resumeOffset
trimBlockAttribution(block, resumeOffset, previousWriter)
@@ -986,6 +1015,9 @@ export class DownloadManager {
}
private pushUpdate(runtime: DownloadRuntime, persist = true): void {
+ // A removed download can still be winding down (workers finishing, cleanup). Its updates
+ // would put it back on screen after the renderer has already moved on.
+ if (this.runtimes.get(runtime.state.id) !== runtime) return
if (persist) this.schedulePersistence(runtime)
const window = this.getWindow()
if (!window || window.isDestroyed()) return
diff --git a/src/main/download/probe.ts b/src/main/download/probe.ts
index b8bf1f1..9365cbf 100644
--- a/src/main/download/probe.ts
+++ b/src/main/download/probe.ts
@@ -2,9 +2,13 @@ import { request as httpRequest } from 'node:http'
import { request as httpsRequest } from 'node:https'
import { URL } from 'node:url'
import type { ProbeResult } from '../../shared/types'
+import { testKnobs } from '../testKnobs'
const MAX_REDIRECTS = 5
const USER_AGENT = 'Plexo/1.0'
+// A server that accepts the connection and never answers would otherwise hang the probe — and
+// with it resume, which re-probes before continuing — forever. Same budget as a stalled chunk.
+const PROBE_TIMEOUT_MS = testKnobs.stallTimeoutMs
type Headers = Record<string, string | string[] | undefined>
@@ -37,6 +41,9 @@ function requestOneByte(url: URL): Promise<ProbeResponse> {
}
)
req.on('error', reject)
+ req.setTimeout(PROBE_TIMEOUT_MS, () =>
+ req.destroy(new Error('The server did not respond — check the link and try again'))
+ )
req.end()
})
}
@@ -53,7 +60,12 @@ function fileNameFromHeaders(headers: Headers, url: URL): string {
}
}
}
- const pathname = decodeURIComponent(url.pathname)
+ let pathname = url.pathname
+ try {
+ pathname = decodeURIComponent(pathname)
+ } catch {
+ // A malformed %-escape: the raw path still names the file well enough.
+ }
const base = pathname.split('/').filter(Boolean).pop()
return base && base.length > 0 ? base : 'download'
}
@@ -111,6 +123,24 @@ export async function isResourceUnchanged(
export async function probeUrl(rawUrl: string): Promise<ProbeResult> {
const { current, response } = await requestFollowingRedirects(rawUrl)
+ // An empty file can't satisfy a request for its first byte: the server answers 416 and gives
+ // the size as `bytes */0`. That's a valid, empty download, not an error.
+ if (
+ response?.statusCode === 416 &&
+ /^\s*bytes\s+\*\/0\s*$/i.test(headerValue(response.headers, 'content-range') ?? '')
+ ) {
+ return {
+ requestedUrl: rawUrl,
+ finalUrl: current.toString(),
+ supportsRanges: false,
+ totalBytes: 0,
+ suggestedFileName: fileNameFromHeaders(response.headers, current),
+ contentType: headerValue(response.headers, 'content-type') ?? null,
+ etag: headerValue(response.headers, 'etag') ?? null,
+ lastModified: headerValue(response.headers, 'last-modified') ?? null
+ }
+ }
+
if (!response || response.statusCode === 0 || response.statusCode >= 400) {
throw new Error(`Server responded with status ${response?.statusCode || 'unknown'}`)
}