Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Rebalance the e2e suite now that the known bugs are fixed
Anmol Kapil655d123834c6
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.tsindex 21d5ea8..eb0c601 100644--- a/e2e/integrity.spec.ts+++ b/e2e/integrity.spec.ts@@ -1,4 +1,4 @@-import { BLOCK, expect, test } from './fixtures'+import { BLOCK, expect, LAN_ADDRESS, test } from './fixtures'import { seededBytes, type Fault } from './origin'// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the@@ -15,7 +15,6 @@ const TRANSIENT: [string, Fault][] = [['200 (whole file) for a range that does not start at 0', 'ignoreRange'],['416 Range Not Satisfiable', { status: 416 }],['500 Internal Server Error', { status: 500 }],- ['503 Service Unavailable', { status: 503 }],['short body, then a clean end', { endAfter: 1000 }],['connection reset partway through the body', { cutAfter: 5000 }],['connection reset before any body', { cutAfter: 0 }],@@ -49,6 +48,53 @@ test.describe('transient server faults are retried to a correct file @smoke', ()})})+test.describe('servers without range support @smoke', () => {+ // With no ranges the only way to recover a dropped connection is to start the file over.+ for (const [label, contentLength] of [+ ['known size', true],+ ['unknown size', false]+ ] as const) {+ test(`a dropped connection restarts from the beginning (${label})`, async ({+ plexo,+ serve+ }) => {+ const origin = await serve({ size: 20 * BLOCK, ranges: false, contentLength })+ let transfers = 0+ origin.setRule(({ range }) =>+ range?.start === 0 && range.end === 0+ ? 'ok'+ : transfers++ === 0+ ? { cutAfter: 7 * BLOCK + 3 }+ : 'ok'+ )+ await plexo.start(origin.url(), origin.sha256)+ await plexo.waitForStatus('completed')+ expect(transfers, 'the second attempt fetched the whole file again').toBe(2)+ })+ }+})++test('one network dies for good mid-download; the other finishes it @smoke', async ({+ plexo,+ serve+}) => {+ test.skip(!LAN_ADDRESS, 'needs a LAN address to act as the second network')+ const origin = await serve({ size: 40 * BLOCK })+ let fromB = 0+ // Network b serves two blocks, then every request over it fails.+ origin.setRule(({ from, range }) =>+ from !== '127.0.0.1' && range && !(range.start === 0 && range.end === 0) && fromB++ >= 2+ ? { status: 503 }+ : 'ok'+ )+ await plexo.start(origin.url(), origin.sha256, { networks: ['a', 'b'], connections: 2 })+ await plexo.waitForStatus('completed')+ const failedOverB = origin.log.filter(+ (entry) => entry.from !== '127.0.0.1' && entry.status === 503+ )+ expect(failedOverB.length, 'network b really did fail').toBeGreaterThan(0)+})+test.describe('permanent faults end in a clean error @smoke', () => {const PERMANENT: [string, Fault][] = [['every chunk request fails with 500', { status: 500 }],diff --git a/e2e/recovery.spec.ts b/e2e/recovery.spec.tsindex 95033ab..c5ff8ae 100644--- a/e2e/recovery.spec.ts+++ b/e2e/recovery.spec.ts@@ -1,4 +1,4 @@-import { cp, readFile, rm, writeFile } from 'node:fs/promises'+import { cp, readFile, rm, truncate, writeFile } from 'node:fs/promises'import { existsSync } from 'node:fs'import { join } from 'node:path'import { BLOCK, expect, test } from './fixtures'@@ -43,9 +43,10 @@ test.describe('restart @smoke', () => {})test.describe('crash (SIGKILL) and recover @smoke', () => {- // Offsets chosen to land in different places: first block, mid-block, a block boundary,- // and the tail — each one leaves different part-file and manifest states behind.- for (const offset of [100, 3 * BLOCK + 777, 12 * BLOCK, 20 * BLOCK + 1, SIZE - 10]) {+ // Offsets chosen to land in different places: so early no progress has been saved yet,+ // mid-block, a block boundary, and the tail — each leaves different part-file and manifest+ // states behind. Chaos covers the moments in between.+ for (const offset of [100, 3 * BLOCK + 777, 12 * BLOCK, SIZE - 10]) {test(`killed with a response held at byte ${offset}`, async ({ plexo, serve }) => {const origin = await serve({ size: SIZE, seed: offset })const reached = origin.hold(offset)@@ -177,4 +178,22 @@ test.describe('persisted state on disk @smoke', () => {const state = await plexo.waitForStatus(['completed', 'error'])expect(state.error).toBeUndefined()})++ test('a finished part file cut short while the app was closed is fetched again', async ({+ plexo,+ serve,+ dirs+ }) => {+ // What a power cut can do: the manifest says a block is done, but its data never all+ // reached the disk.+ const { id } = await pausedDownload(plexo, serve)+ const done = (await plexo.current())!.blocks!.find((block) => block.status === 'completed')!+ await plexo.quit()+ const part = join(dirs.userData, 'downloads', id, 'parts', `part-${done.index}`)+ await truncate(part, 1000)++ await plexo.launch()+ await plexo.api.resumeDownload(id)+ await plexo.waitForStatus('completed')+ })})