Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
fix: listener leak warning while assembling large downloads
Anmol Kapilaa3c256d00f2
diff --git a/e2e/fixtures.ts b/e2e/fixtures.tsindex 8d07130..a4f2233 100644--- a/e2e/fixtures.ts+++ b/e2e/fixtures.ts@@ -408,6 +408,10 @@ export const test = base.extend<Fixtures>({// A test already failing (or expected to fail) has said what it needed to.if (testInfo.status !== 'passed' || testInfo.expectedStatus !== 'passed') returncheckEvents(plexo.sessions)+ // Listeners piling up on one stream or emitter: harmless today, a leak tomorrow.+ expect(plexo.output.join(''), 'the app warned of a listener leak').not.toContain(+ 'MaxListenersExceededWarning'+ )if (plexo.alive) await checkFinalState(plexo)},{ auto: true }diff --git a/e2e/sim.spec.ts b/e2e/sim.spec.tsindex f037f85..a4d8ca0 100644--- a/e2e/sim.spec.ts+++ b/e2e/sim.spec.ts@@ -1,4 +1,4 @@-import { writeFile } from 'node:fs/promises'+import { truncate, writeFile } from 'node:fs/promises'import { join } from 'node:path'import type { SimulatedNetworkConfig } from '../src/shared/types'import { BLOCK, expect, test } from './fixtures'@@ -42,6 +42,31 @@ test.describe('simulated downloads @smoke', () => {expect(state.chunks.reduce((sum, chunk) => sum + chunk.retryCount, 0)).toBeGreaterThan(0)})+ test('a part that goes wrong during assembly fails the download and leaves nothing behind', async ({+ plexo,+ dirs+ }) => {+ const source = await sourceFile(dirs.userData, 5)+ const id = await plexo.startSimulated(+ {+ sourceFilePath: source.path,+ networks: [network('one')],+ chunkCount: 2,+ connectionsPerNetwork: 2,+ assembleSpeedBytesPerSec: SIZE / 2+ },+ source.sha+ )+ await plexo.waitForStatus('assembling')+ // The last part is still waiting its turn; cut a byte off it.+ const last = join(dirs.userData, 'downloads', id, 'parts', `part-${SIZE / BLOCK - 1}`)+ await truncate(last, BLOCK - 1)++ const state = await plexo.waitForStatus('error')+ expect(state.error).toMatch(/refusing to write a corrupt file/)+ // (the automatic checks then confirm there is no file at the destination and no parts left)+ })+test('pause and cancel are ignored while assembling', async ({ plexo, dirs }) => {const source = await sourceFile(dirs.userData, 2)const id = await plexo.startSimulated(diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.tsindex d78f132..8ea756b 100644--- a/src/main/download/downloadManager.ts+++ b/src/main/download/downloadManager.ts@@ -12,6 +12,7 @@ import {writeFile} from 'node:fs/promises'import { basename, join } from 'node:path'+import { Readable } from 'node:stream'import { finished, pipeline } from 'node:stream/promises'import type { BrowserWindow } from 'electron'import { app, Notification } from 'electron'@@ -1303,57 +1304,30 @@ export class DownloadManager {const output = createWriteStream(runtime.state.destinationPath, {highWaterMark: ASSEMBLE_STREAM_BUFFER_BYTES})- let bytesWritten = 0-- // Set only for a dev-tool simulated download that asked for a slowed-down assemble — real- // downloads always reassemble at full disk speed. Throttling here (rather than faking it in- // the renderer) exercises the exact same assembledBytes/IPC path a real assemble uses.- const assembleSpeedBytesPerSec = getSimAssembleSpeed(runtime.requestPayload.url)-- // Attached before the first write, and kept for the stream's whole life:- // destroying the output after a failed check can surface an in-flight- // write as an 'error' event, and an unhandled 'error' on a stream takes- // down the main process rather than failing this one download.- const outputErrors: Error[] = []- output.on('error', (error: Error) => outputErrors.push(error))-- try {- for (let i = 0; i < runtime.totalBlocks; i++) {- const partPath = join(runtime.tempDir, `part-${i}`)- const block = runtime.blocks[i]- const expectedBytes = block.rangeEnd === null ? null : block.rangeEnd - block.rangeStart + 1- const actualBytes = (await stat(partPath)).size-- if (expectedBytes !== null && actualBytes !== expectedBytes) {- throw new Error(- `Part ${i} is ${actualBytes} bytes but should be ${expectedBytes} — refusing to write a corrupt file`- )- }-- // pipeline rejects on an error from either side; a hand-rolled pause/'drain' loop here- // would wait forever for a 'drain' that an errored output never emits.- await pipeline(- createReadStream(partPath, { highWaterMark: ASSEMBLE_STREAM_BUFFER_BYTES }),- output,- { end: false }- )- if (outputErrors.length > 0) throw outputErrors[0]- bytesWritten += actualBytes-- if (assembleSpeedBytesPerSec) {- await sleep(Math.max(1, (actualBytes / assembleSpeedBytesPerSec) * 1000))- }+ // pipeline reports a failure by rejecting, but only listens while it runs: destroying the+ // output afterwards can still surface an in-flight write as an 'error' event, and an+ // unhandled 'error' on a stream takes down the main process rather than failing this one+ // download. So one listener stays for the stream's whole life.+ output.on('error', () => {})+ let bytesWritten = 0+ // A single pipeline for the whole file, not one per part: pipeline leaves its listeners on+ // a destination it doesn't end, so one per part would pile up on the output — five per part.+ const source = Readable.from(+ this.readParts(runtime, ASSEMBLE_STREAM_BUFFER_BYTES, (partBytes) => {+ bytesWritten += partBytes// Reported per part rather than per underlying write so the assembling visualization// advances in the same units the block grid already shows — one step per chunk, not a// byte stream.runtime.state.assembledBytes = bytesWrittenthis.scheduleUpdate(runtime)- }+ }),+ { objectMode: false, highWaterMark: ASSEMBLE_STREAM_BUFFER_BYTES }+ )- output.end()- await finished(output)- if (outputErrors.length > 0) throw outputErrors[0]+ try {+ // Rejects on an error from either side, and destroys both.+ await pipeline(source, output)if (runtime.state.totalBytes > 0 && bytesWritten !== runtime.state.totalBytes) {throw new Error(@@ -1370,6 +1344,48 @@ export class DownloadManager {}}+ /** The download's bytes in file order: each part checked against the size its range says, then+ * read through. `onPartRead` is told how many bytes each part held, once it has been read. */+ private async *readParts(+ runtime: DownloadRuntime,+ bufferBytes: number,+ onPartRead: (partBytes: number) => void+ ): AsyncGenerator<Buffer> {+ // Set only for a dev-tool simulated download that asked for a slowed-down assemble — real+ // downloads always reassemble at full disk speed. Throttling here (rather than faking it in+ // the renderer) exercises the exact same assembledBytes/IPC path a real assemble uses.+ const assembleSpeedBytesPerSec = getSimAssembleSpeed(runtime.requestPayload.url)++ for (let i = 0; i < runtime.totalBlocks; i++) {+ const partPath = join(runtime.tempDir, `part-${i}`)+ const block = runtime.blocks[i]+ const expectedBytes = block.rangeEnd === null ? null : block.rangeEnd - block.rangeStart + 1+ const actualBytes = (await stat(partPath)).size++ if (expectedBytes !== null && actualBytes !== expectedBytes) {+ throw new Error(+ `Part ${i} is ${actualBytes} bytes but should be ${expectedBytes} — refusing to write a corrupt file`+ )+ }++ let read = 0+ for await (const chunk of createReadStream(partPath, { highWaterMark: bufferBytes })) {+ read += (chunk as Buffer).length+ yield chunk as Buffer+ }+ if (read !== actualBytes) {+ throw new Error(+ `Part ${i} changed while it was being assembled — refusing to write a corrupt file`+ )+ }++ if (assembleSpeedBytesPerSec) {+ await sleep(Math.max(1, (read / assembleSpeedBytesPerSec) * 1000))+ }+ onPartRead(read)+ }+ }+/*** Releases the placeholder file reserved at start when the download won't be* filling it in, so its name is free for the next attempt. Only ever removes