Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Stop assembly hanging on write errors; count parts in the space check

Anmol Kapil31fb2541f9a2
diff --git a/e2e/disk.spec.ts b/e2e/disk.spec.ts
index 1649f45..a43af11 100644
--- a/e2e/disk.spec.ts
+++ b/e2e/disk.spec.ts
@@ -80,11 +80,9 @@ test.describe('disk space @disk', () => {
}
})
- test('a disk that fills up while assembling ends in an error, not a hang', async ({ serve }) => {
- test.fail(
- true,
- 'known bugs: the space check ignores the part files (needs ~2× the size), and appendFileToStream hangs forever on a write error'
- )
+ test('a file that fits once but not twice (file + its parts) is refused upfront', async ({
+ serve
+ }) => {
const volume = await smallVolume(12)
test.skip(!volume, 'cannot create a small volume on this machine')
// userData on the same small volume, so the part files and the final file compete for it.
@@ -97,18 +95,10 @@ test.describe('disk space @disk', () => {
await mkdir(app.dirs.dest)
await app.launch()
const origin = await serve({ size: 112 * BLOCK }) // 7 MB: fits once, not twice
- let refusedUpfront = false
- try {
- await app.start(origin.url(), origin.sha256)
- } catch (error) {
- refusedUpfront = /disk space/.test(String(error))
- if (!refusedUpfront) throw error
- }
- if (!refusedUpfront) {
- const state = await app.waitForStatus(['completed', 'error'], 20_000)
- expect(state.status).toBe('error')
- expect(existsSync(state.destinationPath)).toBe(false)
- }
+ // Without counting the parts, this would start, fill the disk while assembling, and
+ // fail there — after downloading the whole file.
+ await expect(app.start(origin.url(), origin.sha256)).rejects.toThrow(/Not enough disk space/)
+ expect(await readdir(app.dirs.dest)).toEqual([])
} finally {
if (app.alive) await app.kill()
await volume!.dispose()
@@ -117,15 +107,13 @@ test.describe('disk space @disk', () => {
})
test.describe('destination folder problems @smoke', () => {
+ // Also the test for assembly write errors: the output stream fails to open, and assembly must
+ // surface that instead of waiting forever for a 'drain' that never comes.
test('destination folder deleted mid-download → error, nothing left', async ({
plexo,
serve,
dirs
}) => {
- test.fail(
- true,
- 'known bug: appendFileToStream never settles once the output stream errors — stuck in assembling'
- )
const origin = await serve({ size: 24 * BLOCK })
const reached = origin.hold(10 * BLOCK)
await plexo.start(origin.url(), origin.sha256)
diff --git a/src/main/download/downloadManager.ts b/src/main/download/downloadManager.ts
index 683eda7..c0d2aca 100644
--- a/src/main/download/downloadManager.ts
+++ b/src/main/download/downloadManager.ts
@@ -12,7 +12,7 @@ import {
writeFile
} from 'node:fs/promises'
import { basename, join } from 'node:path'
-import { finished } from 'node:stream/promises'
+import { finished, pipeline } from 'node:stream/promises'
import type { BrowserWindow } from 'electron'
import { app } from 'electron'
import { IpcChannels } from '../../shared/ipc-channels'
@@ -196,36 +196,36 @@ function formatGigabytes(bytes: number): string {
return `${(bytes / 1024 ** 3).toFixed(2)} GB`
}
-/** Throws if the destination volume doesn't have room for the download — a full disk should
- * fail upfront with a clear reason, not partway through as a confusing ENOSPC write error. */
-async function ensureDiskSpace(destinationDir: string, requiredBytes: number): Promise<void> {
+/** Throws if there isn't room for the download — a full disk should fail upfront with a clear
+ * reason, not partway through as a confusing ENOSPC write error. The part files and the assembled
+ * file both exist until assembly finishes, so when they share a volume it needs room for both. */
+async function ensureDiskSpace(
+ destinationDir: string,
+ partsRoot: string,
+ requiredBytes: number
+): Promise<void> {
if (requiredBytes <= 0) return // unknown size — nothing to check against
- const stats = await statfs(destinationDir)
- const availableBytes = stats.bavail * stats.bsize
- if (availableBytes < requiredBytes) {
- throw new Error(
- `Not enough disk space: this download needs ${formatGigabytes(requiredBytes)} but only ${formatGigabytes(availableBytes)} is free`
- )
+ const [destination, parts] = await Promise.all([stat(destinationDir), stat(partsRoot)])
+ const needs: [string, number][] =
+ destination.dev === parts.dev
+ ? [[destinationDir, requiredBytes * 2]]
+ : [
+ [destinationDir, requiredBytes],
+ [partsRoot, requiredBytes]
+ ]
+
+ for (const [dir, bytes] of needs) {
+ const stats = await statfs(dir)
+ const availableBytes = stats.bavail * stats.bsize
+ if (availableBytes < bytes) {
+ throw new Error(
+ `Not enough disk space: this download needs ${formatGigabytes(bytes)} (the file plus its temporary parts) but only ${formatGigabytes(availableBytes)} is free`
+ )
+ }
}
}
-/** Appends one file's bytes onto an already-open writable, without ending it. */
-function appendFileToStream(sourcePath: string, output: NodeJS.WritableStream): Promise<void> {
- return new Promise((resolve, reject) => {
- const input = createReadStream(sourcePath)
- input.on('error', reject)
- input.on('data', (chunk) => {
- const canContinue = output.write(chunk)
- if (!canContinue) {
- input.pause()
- output.once('drain', () => input.resume())
- }
- })
- input.on('close', resolve)
- })
-}
-
export class DownloadManager {
private runtimes = new Map<string, DownloadRuntime>()
private readonly initialization: Promise<void>
@@ -416,7 +416,12 @@ export class DownloadManager {
requestPayload: StartDownloadRequest,
interfaces: NetworkInterfaceInfo[]
): Promise<string> {
- await ensureDiskSpace(requestPayload.destinationDir, requestPayload.totalBytes)
+ await mkdir(this.downloadsRoot(), { recursive: true })
+ await ensureDiskSpace(
+ requestPayload.destinationDir,
+ this.downloadsRoot(),
+ requestPayload.totalBytes
+ )
const id = randomUUID()
const tempDir = join(this.downloadDir(id), 'parts')
@@ -1034,7 +1039,9 @@ export class DownloadManager {
)
}
- await appendFileToStream(partPath, output)
+ // pipeline rejects on an error from either side; a hand-rolled pause/'drain' loop here
+ // would wait forever for a 'drain' that an errored output never emits.
+ await pipeline(createReadStream(partPath), output, { end: false })
if (outputErrors.length > 0) throw outputErrors[0]
bytesWritten += actualBytes