Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Stop assembly hanging on write errors; count parts in the space check
Anmol Kapil31fb2541f9a2
diff --git a/e2e/disk.spec.ts b/e2e/disk.spec.tsindex 1649f45..a43af11 100644--- a/e2e/disk.spec.ts+++ b/e2e/disk.spec.ts@@ -80,11 +80,9 @@ test.describe('disk space @disk', () => {}})- test('a disk that fills up while assembling ends in an error, not a hang', async ({ serve }) => {- test.fail(- true,- 'known bugs: the space check ignores the part files (needs ~2× the size), and appendFileToStream hangs forever on a write error'- )+ test('a file that fits once but not twice (file + its parts) is refused upfront', async ({+ serve+ }) => {const volume = await smallVolume(12)test.skip(!volume, 'cannot create a small volume on this machine')// userData on the same small volume, so the part files and the final file compete for it.@@ -97,18 +95,10 @@ test.describe('disk space @disk', () => {await mkdir(app.dirs.dest)await app.launch()const origin = await serve({ size: 112 * BLOCK }) // 7 MB: fits once, not twice- let refusedUpfront = false- try {- await app.start(origin.url(), origin.sha256)- } catch (error) {- refusedUpfront = /disk space/.test(String(error))- if (!refusedUpfront) throw error- }- if (!refusedUpfront) {- const state = await app.waitForStatus(['completed', 'error'], 20_000)- expect(state.status).toBe('error')- expect(existsSync(state.destinationPath)).toBe(false)- }+ // Without counting the parts, this would start, fill the disk while assembling, and+ // fail there — after downloading the whole file.+ await expect(app.start(origin.url(), origin.sha256)).rejects.toThrow(/Not enough disk space/)+ expect(await readdir(app.dirs.dest)).toEqual([])} finally {if (app.alive) await app.kill()await volume!.dispose()@@ -117,15 +107,13 @@ test.describe('disk space @disk', () => {})test.describe('destination folder problems @smoke', () => {+ // Also the test for assembly write errors: the output stream fails to open, and assembly must+ // surface that instead of waiting forever for a 'drain' that never comes.test('destination folder deleted mid-download → error, nothing left', async ({plexo,serve,dirs}) => {- test.fail(- true,- 'known bug: appendFileToStream never settles once the output stream errors — stuck in assembling'- )const origin = await serve({ size: 24 * BLOCK })const reached = origin.hold(10 * BLOCK)await plexo.start(origin.url(), origin.sha256)diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex 683eda7..c0d2aca 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -12,7 +12,7 @@ import {writeFile} from 'node:fs/promises'import { basename, join } from 'node:path'-import { finished } from 'node:stream/promises'+import { finished, pipeline } from 'node:stream/promises'import type { BrowserWindow } from 'electron'import { app } from 'electron'import { IpcChannels } from '../../shared/ipc-channels'@@ -196,36 +196,36 @@ function formatGigabytes(bytes: number): string {return `${(bytes / 1024 ** 3).toFixed(2)} GB`}-/** Throws if the destination volume doesn't have room for the download — a full disk should- * fail upfront with a clear reason, not partway through as a confusing ENOSPC write error. */-async function ensureDiskSpace(destinationDir: string, requiredBytes: number): Promise<void> {+/** Throws if there isn't room for the download — a full disk should fail upfront with a clear+ * reason, not partway through as a confusing ENOSPC write error. The part files and the assembled+ * file both exist until assembly finishes, so when they share a volume it needs room for both. */+async function ensureDiskSpace(+ destinationDir: string,+ partsRoot: string,+ requiredBytes: number+): Promise<void> {if (requiredBytes <= 0) return // unknown size — nothing to check against- const stats = await statfs(destinationDir)- const availableBytes = stats.bavail * stats.bsize- if (availableBytes < requiredBytes) {- throw new Error(- `Not enough disk space: this download needs ${formatGigabytes(requiredBytes)} but only ${formatGigabytes(availableBytes)} is free`- )+ const [destination, parts] = await Promise.all([stat(destinationDir), stat(partsRoot)])+ const needs: [string, number][] =+ destination.dev === parts.dev+ ? [[destinationDir, requiredBytes * 2]]+ : [+ [destinationDir, requiredBytes],+ [partsRoot, requiredBytes]+ ]++ for (const [dir, bytes] of needs) {+ const stats = await statfs(dir)+ const availableBytes = stats.bavail * stats.bsize+ if (availableBytes < bytes) {+ throw new Error(+ `Not enough disk space: this download needs ${formatGigabytes(bytes)} (the file plus its temporary parts) but only ${formatGigabytes(availableBytes)} is free`+ )+ }}}-/** Appends one file's bytes onto an already-open writable, without ending it. */-function appendFileToStream(sourcePath: string, output: NodeJS.WritableStream): Promise<void> {- return new Promise((resolve, reject) => {- const input = createReadStream(sourcePath)- input.on('error', reject)- input.on('data', (chunk) => {- const canContinue = output.write(chunk)- if (!canContinue) {- input.pause()- output.once('drain', () => input.resume())- }- })- input.on('close', resolve)- })-}-export class DownloadManager {private runtimes = new Map<string, DownloadRuntime>()private readonly initialization: Promise<void>@@ -416,7 +416,12 @@ export class DownloadManager {requestPayload: StartDownloadRequest,interfaces: NetworkInterfaceInfo[]): Promise<string> {- await ensureDiskSpace(requestPayload.destinationDir, requestPayload.totalBytes)+ await mkdir(this.downloadsRoot(), { recursive: true })+ await ensureDiskSpace(+ requestPayload.destinationDir,+ this.downloadsRoot(),+ requestPayload.totalBytes+ )const id = randomUUID()const tempDir = join(this.downloadDir(id), 'parts')@@ -1034,7 +1039,9 @@ export class DownloadManager {)}- await appendFileToStream(partPath, output)+ // pipeline rejects on an error from either side; a hand-rolled pause/'drain' loop here+ // would wait forever for a 'drain' that an errored output never emits.+ await pipeline(createReadStream(partPath), output, { end: false })if (outputErrors.length > 0) throw outputErrors[0]bytesWritten += actualBytes