Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Rebalance the e2e suite now that the known bugs are fixed

Anmol Kapil655d123834c6
diff --git a/e2e/integrity.spec.ts b/e2e/integrity.spec.ts
index 21d5ea8..eb0c601 100644
--- a/e2e/integrity.spec.ts
+++ b/e2e/integrity.spec.ts
@@ -1,4 +1,4 @@
-import { BLOCK, expect, test } from './fixtures'
+import { BLOCK, expect, LAN_ADDRESS, test } from './fixtures'
import { seededBytes, type Fault } from './origin'
// B. A misbehaving server or network. The rule every case here must satisfy — enforced by the
@@ -15,7 +15,6 @@ const TRANSIENT: [string, Fault][] = [
['200 (whole file) for a range that does not start at 0', 'ignoreRange'],
['416 Range Not Satisfiable', { status: 416 }],
['500 Internal Server Error', { status: 500 }],
- ['503 Service Unavailable', { status: 503 }],
['short body, then a clean end', { endAfter: 1000 }],
['connection reset partway through the body', { cutAfter: 5000 }],
['connection reset before any body', { cutAfter: 0 }],
@@ -49,6 +48,53 @@ test.describe('transient server faults are retried to a correct file @smoke', ()
})
})
+test.describe('servers without range support @smoke', () => {
+ // With no ranges the only way to recover a dropped connection is to start the file over.
+ for (const [label, contentLength] of [
+ ['known size', true],
+ ['unknown size', false]
+ ] as const) {
+ test(`a dropped connection restarts from the beginning (${label})`, async ({
+ plexo,
+ serve
+ }) => {
+ const origin = await serve({ size: 20 * BLOCK, ranges: false, contentLength })
+ let transfers = 0
+ origin.setRule(({ range }) =>
+ range?.start === 0 && range.end === 0
+ ? 'ok'
+ : transfers++ === 0
+ ? { cutAfter: 7 * BLOCK + 3 }
+ : 'ok'
+ )
+ await plexo.start(origin.url(), origin.sha256)
+ await plexo.waitForStatus('completed')
+ expect(transfers, 'the second attempt fetched the whole file again').toBe(2)
+ })
+ }
+})
+
+test('one network dies for good mid-download; the other finishes it @smoke', async ({
+ plexo,
+ serve
+}) => {
+ test.skip(!LAN_ADDRESS, 'needs a LAN address to act as the second network')
+ const origin = await serve({ size: 40 * BLOCK })
+ let fromB = 0
+ // Network b serves two blocks, then every request over it fails.
+ origin.setRule(({ from, range }) =>
+ from !== '127.0.0.1' && range && !(range.start === 0 && range.end === 0) && fromB++ >= 2
+ ? { status: 503 }
+ : 'ok'
+ )
+ await plexo.start(origin.url(), origin.sha256, { networks: ['a', 'b'], connections: 2 })
+ await plexo.waitForStatus('completed')
+ const failedOverB = origin.log.filter(
+ (entry) => entry.from !== '127.0.0.1' && entry.status === 503
+ )
+ expect(failedOverB.length, 'network b really did fail').toBeGreaterThan(0)
+})
+
test.describe('permanent faults end in a clean error @smoke', () => {
const PERMANENT: [string, Fault][] = [
['every chunk request fails with 500', { status: 500 }],
diff --git a/e2e/recovery.spec.ts b/e2e/recovery.spec.ts
index 95033ab..c5ff8ae 100644
--- a/e2e/recovery.spec.ts
+++ b/e2e/recovery.spec.ts
@@ -1,4 +1,4 @@
-import { cp, readFile, rm, writeFile } from 'node:fs/promises'
+import { cp, readFile, rm, truncate, writeFile } from 'node:fs/promises'
import { existsSync } from 'node:fs'
import { join } from 'node:path'
import { BLOCK, expect, test } from './fixtures'
@@ -43,9 +43,10 @@ test.describe('restart @smoke', () => {
})
test.describe('crash (SIGKILL) and recover @smoke', () => {
- // Offsets chosen to land in different places: first block, mid-block, a block boundary,
- // and the tail — each one leaves different part-file and manifest states behind.
- for (const offset of [100, 3 * BLOCK + 777, 12 * BLOCK, 20 * BLOCK + 1, SIZE - 10]) {
+ // Offsets chosen to land in different places: so early no progress has been saved yet,
+ // mid-block, a block boundary, and the tail — each leaves different part-file and manifest
+ // states behind. Chaos covers the moments in between.
+ for (const offset of [100, 3 * BLOCK + 777, 12 * BLOCK, SIZE - 10]) {
test(`killed with a response held at byte ${offset}`, async ({ plexo, serve }) => {
const origin = await serve({ size: SIZE, seed: offset })
const reached = origin.hold(offset)
@@ -177,4 +178,22 @@ test.describe('persisted state on disk @smoke', () => {
const state = await plexo.waitForStatus(['completed', 'error'])
expect(state.error).toBeUndefined()
})
+
+ test('a finished part file cut short while the app was closed is fetched again', async ({
+ plexo,
+ serve,
+ dirs
+ }) => {
+ // What a power cut can do: the manifest says a block is done, but its data never all
+ // reached the disk.
+ const { id } = await pausedDownload(plexo, serve)
+ const done = (await plexo.current())!.blocks!.find((block) => block.status === 'completed')!
+ await plexo.quit()
+ const part = join(dirs.userData, 'downloads', id, 'parts', `part-${done.index}`)
+ await truncate(part, 1000)
+
+ await plexo.launch()
+ await plexo.api.resumeDownload(id)
+ await plexo.waitForStatus('completed')
+ })
})