Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

feat: scheduler, block progress and part-file modules for racing slow blocks

Anmol Kapilbe75817c8509
diff --git a/e2e/blockProgress.spec.ts b/e2e/blockProgress.spec.ts
new file mode 100644
index 0000000..4b1d61b
--- /dev/null
+++ b/e2e/blockProgress.spec.ts
@@ -0,0 +1,134 @@
+import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { expect, test } from '@playwright/test'
+import fc from 'fast-check'
+import { advanceBlock, retractBlock } from '../src/main/download/blockProgress'
+import { hedgeFile, mergeHedge, partFile, removeHedgeFiles } from '../src/main/download/partFiles'
+import type { BlockState } from '../src/shared/types'
+
+// K. What a block's byte counts mean when two attempts race for it, and how a racing attempt's
+// file joins the block's own.
+
+const LENGTH = 1000
+const fresh = (): BlockState => ({
+ index: 0,
+ rangeStart: 0,
+ rangeEnd: LENGTH - 1,
+ status: 'downloading',
+ bytesDownloaded: 0,
+ bytesByInterface: {}
+})
+const attributed = (block: BlockState): number =>
+ Object.values(block.bytesByInterface).reduce((sum, bytes) => sum + bytes, 0)
+
+test.describe('block progress', () => {
+ test('two attempts racing count only the ground the furthest one covers', () => {
+ const block = fresh()
+ expect(advanceBlock(block, 'a', 300)).toBe(300)
+ expect(advanceBlock(block, 'b', 200)).toBe(0) // behind: nothing new
+ expect(advanceBlock(block, 'b', 500)).toBe(200) // only what goes past a
+ expect(block.bytesDownloaded).toBe(500)
+ expect(block.bytesByInterface).toEqual({ a: 300, b: 200 })
+ })
+
+ test('the frontier never passes the block’s end', () => {
+ const block = fresh()
+ advanceBlock(block, 'a', LENGTH + 500)
+ expect(block.bytesDownloaded).toBe(LENGTH)
+ })
+
+ test('retracting takes the bytes off the network they came from first', () => {
+ const block = fresh()
+ advanceBlock(block, 'a', 300)
+ advanceBlock(block, 'b', 500)
+ expect(retractBlock(block, 300, 'b')).toBe(200)
+ expect(block.bytesByInterface).toEqual({ a: 300 })
+ })
+
+ test('however attempts advance and retract, attribution adds up to the frontier', () => {
+ const step = fc.oneof(
+ fc.record({
+ op: fc.constant('advance' as const),
+ network: fc.constantFrom('a', 'b', 'c'),
+ position: fc.integer({ min: 0, max: LENGTH + 200 })
+ }),
+ fc.record({
+ op: fc.constant('retract' as const),
+ network: fc.constantFrom('a', 'b', 'c'),
+ position: fc.integer({ min: 0, max: LENGTH })
+ })
+ )
+ fc.assert(
+ fc.property(fc.array(step, { maxLength: 40 }), (steps) => {
+ const block = fresh()
+ for (const { op, network, position } of steps) {
+ if (op === 'advance') advanceBlock(block, network, position)
+ else retractBlock(block, position, network)
+ expect(attributed(block)).toBe(block.bytesDownloaded)
+ expect(block.bytesDownloaded).toBeGreaterThanOrEqual(0)
+ expect(block.bytesDownloaded).toBeLessThanOrEqual(LENGTH)
+ }
+ })
+ )
+ })
+})
+
+test.describe('joining a racing attempt onto its block', () => {
+ let dir: string
+ test.beforeEach(async () => {
+ dir = await mkdtemp(join(tmpdir(), 'plexo-merge-'))
+ })
+ test.afterEach(async () => {
+ await rm(dir, { recursive: true, force: true })
+ })
+
+ const bytes = (from: number, to: number): Buffer =>
+ Buffer.from(Array.from({ length: to - from }, (_, i) => (from + i) % 251))
+
+ test('keeps the block’s first bytes and appends the hedge’s, dropping what the block’s writer got past the join', async () => {
+ const part = partFile(dir, 3)
+ const hedge = hedgeFile(dir, 3, 7)
+ // The block's writer got to 600 before it was stopped; the hedge began at 400.
+ await writeFile(part, bytes(0, 600))
+ await writeFile(hedge, bytes(400, LENGTH))
+ expect(await mergeHedge(part, hedge, 400, LENGTH)).toBe(true)
+ expect(await readFile(part)).toEqual(bytes(0, LENGTH))
+ })
+
+ test('a hedge that began at the very start becomes the block', async () => {
+ const part = partFile(dir, 0)
+ const hedge = hedgeFile(dir, 0, 1)
+ await writeFile(hedge, bytes(0, LENGTH)) // no part file at all
+ expect(await mergeHedge(part, hedge, 0, LENGTH)).toBe(true)
+ expect(await readFile(part)).toEqual(bytes(0, LENGTH))
+ })
+
+ test('refuses, changing nothing, when the block’s file is shorter than the join', async () => {
+ const part = partFile(dir, 0)
+ const hedge = hedgeFile(dir, 0, 1)
+ await writeFile(part, bytes(0, 300)) // its writer never flushed the last 100
+ await writeFile(hedge, bytes(400, LENGTH))
+ expect(await mergeHedge(part, hedge, 400, LENGTH)).toBe(false)
+ expect((await stat(part)).size).toBe(300)
+ })
+
+ test('refuses, changing nothing, when the hedge’s file is not the size its range says', async () => {
+ const part = partFile(dir, 0)
+ const hedge = hedgeFile(dir, 0, 1)
+ await writeFile(part, bytes(0, 500))
+ await writeFile(hedge, bytes(400, 900))
+ expect(await mergeHedge(part, hedge, 400, LENGTH)).toBe(false)
+ expect(await readFile(part)).toEqual(bytes(0, 500))
+ })
+
+ test('leftover hedge files are cleared without touching the blocks’ own', async () => {
+ await writeFile(partFile(dir, 1), 'keep')
+ await writeFile(hedgeFile(dir, 1, 4), 'stale')
+ await writeFile(hedgeFile(dir, 2, 0), 'stale')
+ await removeHedgeFiles(dir)
+ expect((await stat(partFile(dir, 1))).size).toBe(4)
+ await expect(stat(hedgeFile(dir, 1, 4))).rejects.toThrow()
+ await expect(stat(hedgeFile(dir, 2, 0))).rejects.toThrow()
+ })
+})
diff --git a/e2e/scheduler.spec.ts b/e2e/scheduler.spec.ts
new file mode 100644
index 0000000..e24893e
--- /dev/null
+++ b/e2e/scheduler.spec.ts
@@ -0,0 +1,310 @@
+import { expect, test } from '@playwright/test'
+import fc from 'fast-check'
+import {
+ pickWork,
+ type AttemptView,
+ type SchedulerPolicy,
+ type SchedulerState
+} from '../src/main/download/scheduler'
+import type { BlockState, BlockStatus, ChunkStatus } from '../src/shared/types'
+
+// J. Who fetches what. Pure, so the races that decide it (which stream asked first, how far
+// along the holder is) are pinned down as inputs instead of left to timing.
+
+const POLICY: SchedulerPolicy = { hedgeAfterMs: 5000, maxHedgesPerBlock: 2 }
+const NOW = 100_000
+const LENGTH = 1000
+
+const block = (index: number, status: BlockStatus = 'pending', bytes = 0): BlockState => ({
+ index,
+ rangeStart: index * LENGTH,
+ rangeEnd: index * LENGTH + LENGTH - 1,
+ status,
+ bytesDownloaded: bytes,
+ bytesByInterface: {}
+})
+
+const stream = (
+ id: number,
+ interfaceId: string,
+ status: ChunkStatus,
+ speedBytesPerSec = 0
+): SchedulerState['streams'][number] => ({ id, interfaceId, status, speedBytesPerSec })
+
+const attempt = (
+ streamId: number,
+ networkId: string,
+ kind: AttemptView['kind'] = 'primary',
+ ageMs = 10_000
+): AttemptView => ({ kind, streamId, networkId, startedAt: NOW - ageMs })
+
+function state(parts: {
+ blocks: BlockState[]
+ streams?: SchedulerState['streams']
+ attempts?: [number, AttemptView[]][]
+ avoid?: [number, string][]
+ hedgesUsed?: [number, number][]
+}): SchedulerState {
+ return {
+ blocks: parts.blocks,
+ streams: parts.streams ?? [],
+ attempts: new Map(parts.attempts),
+ avoid: new Map(parts.avoid),
+ hedgesUsed: new Map(parts.hedgesUsed)
+ }
+}
+
+const pick = (s: SchedulerState, id: number, networkId: string): string | undefined => {
+ const work = pickWork(s, { id, networkId }, NOW, POLICY)
+ return work && `${work.kind}:${work.block.index}`
+}
+
+test.describe('scheduler: taking the next block', () => {
+ test('takes the first waiting block', () => {
+ const blocks = [block(0, 'completed'), block(1, 'downloading'), block(2), block(3)]
+ expect(pick(state({ blocks }), 9, 'a')).toBe('primary:2')
+ })
+
+ test('nothing waiting and nothing to race → nothing to do', () => {
+ const blocks = [block(0, 'completed'), block(1, 'completed')]
+ expect(pick(state({ blocks }), 9, 'a')).toBeUndefined()
+ })
+
+ test('a block a network got nothing from goes to another network that is free', () => {
+ const s = state({
+ blocks: [block(0), block(1)],
+ streams: [stream(0, 'a', 'pending'), stream(1, 'b', 'pending')],
+ avoid: [[0, 'b']]
+ })
+ // b skips the block it just failed on; a, free at the same moment, takes it.
+ expect(pick(s, 1, 'b')).toBe('primary:1')
+ expect(pick(s, 0, 'a')).toBe('primary:0')
+ })
+
+ test('…but takes it back when no other network is free, so it cannot be stranded', () => {
+ const s = state({
+ blocks: [block(0)],
+ streams: [
+ stream(0, 'a', 'downloading'),
+ stream(1, 'a', 'retrying'),
+ stream(2, 'b', 'pending')
+ ],
+ avoid: [[0, 'b']]
+ })
+ expect(pick(s, 2, 'b')).toBe('primary:0')
+ })
+
+ test('another network being free only matters for the network that failed', () => {
+ const s = state({
+ blocks: [block(0)],
+ streams: [stream(0, 'a', 'pending'), stream(1, 'b', 'pending')],
+ avoid: [[0, 'b']]
+ })
+ expect(pick(s, 0, 'a')).toBe('primary:0')
+ })
+
+ test('never a deadlock: with a block waiting and a stream idle, some idle stream takes it', () => {
+ const networks = ['a', 'b', 'c']
+ fc.assert(
+ fc.property(
+ fc.array(fc.constantFrom<BlockStatus>('pending', 'downloading', 'completed'), {
+ minLength: 1,
+ maxLength: 8
+ }),
+ fc.array(
+ fc.record({
+ interfaceId: fc.constantFrom(...networks),
+ status: fc.constantFrom<ChunkStatus>('pending', 'downloading', 'retrying', 'error')
+ }),
+ { minLength: 1, maxLength: 8 }
+ ),
+ fc.array(fc.tuple(fc.nat(7), fc.constantFrom(...networks))),
+ (blockStatuses, chunks, avoided) => {
+ const s = state({
+ blocks: blockStatuses.map((status, index) => block(index, status)),
+ streams: chunks.map((c, id) => stream(id, c.interfaceId, c.status)),
+ avoid: avoided
+ })
+ const idle = s.streams.filter((c) => c.status === 'pending')
+ fc.pre(idle.length > 0 && s.blocks.some((b) => b.status === 'pending'))
+ expect(idle.some((c) => pick(s, c.id, c.interfaceId)?.startsWith('primary'))).toBe(true)
+ }
+ )
+ )
+ })
+})
+
+test.describe('scheduler: racing a slow block', () => {
+ // Block 0 is 1000 bytes with 200 done, held by stream 0 on network a at `speed`.
+ const laggard = (
+ speed: number,
+ over: Partial<Parameters<typeof state>[0]> = {}
+ ): SchedulerState =>
+ state({
+ blocks: [block(0, 'downloading', 200), block(1, 'completed')],
+ streams: [stream(0, 'a', 'downloading', speed), stream(1, 'a', 'pending')],
+ attempts: [[0, [attempt(0, 'a')]]],
+ ...over
+ })
+
+ test('a free stream races the block whose holder needs longest', () => {
+ // 800 bytes left at 10 B/s: 80 s to go, against a 5 s threshold.
+ expect(pick(laggard(10), 1, 'a')).toBe('hedge:0')
+ })
+
+ test('a holder that has gone quiet has no finish time, so it is raced', () => {
+ expect(pick(laggard(0), 1, 'a')).toBe('hedge:0')
+ })
+
+ test('a holder about to finish is left alone', () => {
+ // 800 bytes left at 1000 B/s: 0.8 s.
+ expect(pick(laggard(1000), 1, 'a')).toBeUndefined()
+ })
+
+ test('the threshold is the finish time, not the speed: a slow holder nearly done is left alone', () => {
+ const s = state({
+ blocks: [block(0, 'downloading', 990), block(1, 'completed')],
+ streams: [stream(0, 'a', 'downloading', 10), stream(1, 'a', 'pending')],
+ attempts: [[0, [attempt(0, 'a')]]]
+ })
+ expect(pick(s, 1, 'a')).toBeUndefined() // 10 bytes left at 10 B/s: 1 s
+ })
+
+ test('a holder that has only just started has no measured speed yet, so it is left alone', () => {
+ const s = laggard(0, { attempts: [[0, [attempt(0, 'a', 'primary', 1000)]]] })
+ expect(pick(s, 1, 'a')).toBeUndefined()
+ })
+
+ test('never while any block is still waiting: a hedge must not take work from the queue', () => {
+ const s = laggard(0, { blocks: [block(0, 'downloading', 200), block(1)] })
+ expect(pick(s, 1, 'a')).toBe('primary:1')
+ // A stream that has to decline the waiting block does not race instead: b sits it out, and a
+ // takes the block.
+ const avoided = laggard(0, {
+ blocks: [block(0, 'downloading', 200), block(1)],
+ streams: [
+ stream(0, 'a', 'downloading'),
+ stream(1, 'b', 'pending'),
+ stream(2, 'a', 'pending')
+ ],
+ avoid: [[1, 'b']]
+ })
+ expect(pick(avoided, 1, 'b')).toBeUndefined()
+ expect(pick(avoided, 2, 'a')).toBe('primary:1')
+ })
+
+ test('one hedge at a time, and at most a few over a block’s life', () => {
+ const hedged = laggard(0, { attempts: [[0, [attempt(0, 'a'), attempt(1, 'a', 'hedge')]]] })
+ expect(pick(hedged, 2, 'a')).toBeUndefined()
+ expect(pick(laggard(0, { hedgesUsed: [[0, 2]] }), 1, 'a')).toBeUndefined()
+ expect(pick(laggard(0, { hedgesUsed: [[0, 1]] }), 1, 'a')).toBe('hedge:0')
+ })
+
+ test('a stream never races its own block', () => {
+ expect(pick(laggard(0), 0, 'a')).toBeUndefined()
+ })
+
+ test('a block whose holder let go is not raced: it goes back to the queue', () => {
+ const s = laggard(0, {
+ blocks: [block(0, 'pending', 200), block(1, 'completed')],
+ attempts: [[0, [attempt(1, 'b', 'hedge')]]]
+ })
+ expect(pick(s, 2, 'a')).toBe('primary:0')
+ })
+
+ test('the block with the longest way to go is raced first', () => {
+ const s = state({
+ blocks: [block(0, 'downloading', 200), block(1, 'downloading', 200)],
+ streams: [
+ stream(0, 'a', 'downloading', 100),
+ stream(1, 'a', 'downloading', 20),
+ stream(2, 'a', 'pending')
+ ],
+ attempts: [
+ [0, [attempt(0, 'a')]],
+ [1, [attempt(1, 'a')]]
+ ]
+ })
+ expect(pick(s, 2, 'a')).toBe('hedge:1')
+ })
+
+ test('a network that got nothing from a block is not sent to race it', () => {
+ expect(pick(laggard(0, { avoid: [[0, 'b']] }), 2, 'b')).toBeUndefined()
+ })
+
+ test('another network gets the first go, since the holder’s network may be what is slow', () => {
+ const s = laggard(0, {
+ streams: [
+ stream(0, 'a', 'downloading', 0),
+ stream(1, 'a', 'pending'),
+ stream(2, 'b', 'pending')
+ ]
+ })
+ expect(pick(s, 1, 'a')).toBeUndefined() // b is free, and would take it
+ expect(pick(s, 2, 'b')).toBe('hedge:0')
+ })
+
+ test('…unless that other network would not take it, or none is free', () => {
+ const streams = [
+ stream(0, 'a', 'downloading', 0),
+ stream(1, 'a', 'pending'),
+ stream(2, 'b', 'pending')
+ ]
+ // b already got nothing from this block, so it will decline; a must not wait for it.
+ expect(pick(laggard(0, { streams, avoid: [[0, 'b']] }), 1, 'a')).toBe('hedge:0')
+ // b is busy.
+ const busy = [streams[0], streams[1], stream(2, 'b', 'downloading')]
+ expect(pick(laggard(0, { streams: busy }), 1, 'a')).toBe('hedge:0')
+ })
+
+ test('a block of unknown length is never raced', () => {
+ const open = { ...block(0, 'downloading', 200), rangeEnd: null }
+ const s = laggard(0, { blocks: [open] })
+ expect(pick(s, 1, 'a')).toBeUndefined()
+ })
+
+ test('whatever is chosen is real work: a waiting block, or a lone primary’s block', () => {
+ const networks = ['a', 'b']
+ fc.assert(
+ fc.property(
+ fc.array(
+ fc.record({
+ status: fc.constantFrom<BlockStatus>('pending', 'downloading', 'completed'),
+ bytes: fc.integer({ min: 0, max: LENGTH }),
+ holderNetwork: fc.constantFrom(...networks),
+ holderAge: fc.integer({ min: 0, max: 20_000 }),
+ speed: fc.integer({ min: 0, max: 500 }),
+ hedged: fc.boolean()
+ }),
+ { minLength: 1, maxLength: 6 }
+ ),
+ fc.constantFrom(...networks),
+ (rows, myNetwork) => {
+ const blocks = rows.map((r, i) => block(i, r.status, r.bytes))
+ const streams = rows.map((r, i) => stream(i, r.holderNetwork, 'downloading', r.speed))
+ const attempts: [number, AttemptView[]][] = []
+ rows.forEach((r, i) => {
+ if (r.status !== 'downloading') return
+ const held = [attempt(i, r.holderNetwork, 'primary', r.holderAge)]
+ if (r.hedged) held.push(attempt(i, 'b', 'hedge'))
+ attempts.push([i, held])
+ })
+ const s = state({
+ blocks,
+ streams: [...streams, stream(99, myNetwork, 'pending')],
+ attempts
+ })
+ const work = pickWork(s, { id: 99, networkId: myNetwork }, NOW, POLICY)
+ if (!work) return
+ if (work.kind === 'primary') {
+ expect(work.block.status).toBe('pending')
+ } else {
+ expect(blocks.some((b) => b.status === 'pending')).toBe(false)
+ expect(work.block.status).toBe('downloading')
+ expect(s.attempts.get(work.block.index)).toHaveLength(1)
+ }
+ }
+ )
+ )
+ })
+})
diff --git a/src/main/download/blockProgress.ts b/src/main/download/blockProgress.ts
new file mode 100644
index 0000000..886ace0
--- /dev/null
+++ b/src/main/download/blockProgress.ts
@@ -0,0 +1,58 @@
+import type { BlockState } from '../../shared/types'
+
+// A block's `bytesDownloaded` is its frontier: how much of it, from its first byte, is secured.
+// Near the end of a download two attempts can race for one block, so the frontier is the furthest
+// either has reached, not the sum of what they received. `bytesByInterface` splits it by network
+// and always adds up to it — these three functions are the only ways either changes.
+
+/** Records that `networkId` delivered `bytes` more of this block. */
+function credit(block: BlockState, networkId: string, bytes: number): void {
+ block.bytesByInterface[networkId] = (block.bytesByInterface[networkId] ?? 0) + bytes
+}
+
+/** Drops attribution for bytes that turned out not to be secured, so the per-network tallies
+ * keep summing to the block's real byte count. The lost bytes are always at the tail, so they
+ * come off `blamed` first (the network that wrote last) before spilling over to the rest. */
+function trim(block: BlockState, keepBytes: number, blamed?: string): void {
+ let attributed = 0
+ for (const bytes of Object.values(block.bytesByInterface)) attributed += bytes
+
+ let excess = attributed - keepBytes
+ if (excess <= 0) return
+
+ const order = Object.keys(block.bytesByInterface).sort((a, b) =>
+ a === blamed ? -1 : b === blamed ? 1 : 0
+ )
+
+ for (const networkId of order) {
+ if (excess <= 0) break
+ const taken = Math.min(block.bytesByInterface[networkId], excess)
+ const remaining = block.bytesByInterface[networkId] - taken
+ excess -= taken
+ if (remaining > 0) block.bytesByInterface[networkId] = remaining
+ else delete block.bytesByInterface[networkId]
+ }
+}
+
+/** Moves the frontier forward to `position` (a byte offset into the block), crediting the bytes
+ * it gains to `networkId`. Does nothing if something already got further. Returns the gain. */
+export function advanceBlock(block: BlockState, networkId: string, position: number): number {
+ const length = block.rangeEnd === null ? Infinity : block.rangeEnd - block.rangeStart + 1
+ const next = Math.min(position, length)
+ const gained = next - block.bytesDownloaded
+ if (gained <= 0) return 0
+ block.bytesDownloaded = next
+ credit(block, networkId, gained)
+ return gained
+}
+
+/** Pulls the frontier back to `position`, for bytes that turned out not to be there — a part file
+ * shorter than counted, a racing attempt that lost. `blamed` is the network they came from.
+ * Returns how much it removed. */
+export function retractBlock(block: BlockState, position: number, blamed?: string): number {
+ const removed = block.bytesDownloaded - position
+ if (removed <= 0) return 0
+ trim(block, position, blamed)
+ block.bytesDownloaded = position
+ return removed
+}
diff --git a/src/main/download/partFiles.ts b/src/main/download/partFiles.ts
new file mode 100644
index 0000000..e8eea4c
--- /dev/null
+++ b/src/main/download/partFiles.ts
@@ -0,0 +1,84 @@
+import { createReadStream, createWriteStream } from 'node:fs'
+import { readdir, rm, stat, truncate } from 'node:fs/promises'
+import { join } from 'node:path'
+import { pipeline } from 'node:stream/promises'
+
+// Where a block's bytes live while a download is in progress. A block's own file (`part-N`) holds
+// it from its first byte on and only ever grows by appending, so its size is always a valid
+// prefix of the block. A racing attempt (see scheduler.ts) writes its own file, and only becomes
+// part of `part-N` if it wins.
+
+export const partFile = (dir: string, blockIndex: number): string => join(dir, `part-${blockIndex}`)
+
+export const hedgeFile = (dir: string, blockIndex: number, streamId: number): string =>
+ join(dir, `part-${blockIndex}.hedge-${streamId}`)
+
+/** Bytes on disk in `path`; 0 if it doesn't exist. */
+export async function fileSize(path: string): Promise<number> {
+ try {
+ return (await stat(path)).size
+ } catch {
+ return 0
+ }
+}
+
+// A part file's on-disk size is the only thing we can actually trust across a retry or resume —
+// appending opens the file with flag 'a', which always writes at the real end-of-file regardless
+// of what byte count we think we're at, so any mismatch (a crash, a write that hadn't flushed
+// yet) would otherwise silently shift every byte after it. Truncating to the smaller of the two
+// counts keeps the part file's length and our own bookkeeping in agreement before we append
+// another byte to it.
+export async function reconcilePartFileSize(
+ partPath: string,
+ expectedBytes: number
+): Promise<number> {
+ const actualBytes = await fileSize(partPath)
+ const safeBytes = Math.min(expectedBytes, actualBytes)
+ if (actualBytes !== safeBytes) {
+ await truncate(partPath, safeBytes)
+ }
+ return safeBytes
+}
+
+/**
+ * Finishes a block from a racing attempt's file: keeps the block's first `hedgeStart` bytes and
+ * appends what the hedge fetched from there to the end. Only valid once nothing else is writing
+ * the part file.
+ *
+ * Resolves true when the part file then holds all `blockBytes`. Resolves false when the two
+ * don't fit together — the part file holds less than `hedgeStart` (bytes its writer never
+ * flushed), the hedge's file isn't the size its range says, or the joined file comes out the
+ * wrong size — and in every such case the part file is left holding exactly the bytes it had
+ * before the join point and nothing after it, which are known good. If the copy itself fails it
+ * throws, leaving the part file a valid prefix of the block, the same state a crash would.
+ */
+export async function mergeHedge(
+ partPath: string,
+ hedgePath: string,
+ hedgeStart: number,
+ blockBytes: number
+): Promise<boolean> {
+ if ((await fileSize(hedgePath)) !== blockBytes - hedgeStart) return false
+ if ((await fileSize(partPath)) < hedgeStart) return false
+
+ if (hedgeStart > 0) await truncate(partPath, hedgeStart)
+ await pipeline(
+ createReadStream(hedgePath),
+ createWriteStream(partPath, { flags: hedgeStart > 0 ? 'a' : 'w' })
+ )
+ if ((await fileSize(partPath)) === blockBytes) return true
+
+ // Something else wrote to the file mid-join, so nothing past the join can be trusted.
+ await truncate(partPath, hedgeStart)
+ return false
+}
+
+/** Removes racing attempts' files left in `dir` — by a crash, or a pause that abandoned them. */
+export async function removeHedgeFiles(dir: string): Promise<void> {
+ const names = await readdir(dir).catch(() => [] as string[])
+ await Promise.all(
+ names
+ .filter((name) => name.includes('.hedge-'))
+ .map((name) => rm(join(dir, name), { force: true }))
+ )
+}
diff --git a/src/main/download/scheduler.ts b/src/main/download/scheduler.ts
new file mode 100644
index 0000000..aa17c6c
--- /dev/null
+++ b/src/main/download/scheduler.ts
@@ -0,0 +1,130 @@
+import type { BlockState, ChunkState } from '../../shared/types'
+
+// Who should fetch what, decided from a snapshot of the download and nothing else — no I/O, no
+// clock of its own — so every rule here can be checked against exact situations.
+//
+// A free stream is given one of two kinds of work:
+//
+// primary A block nobody is fetching. The normal case, and the only one until the queue runs dry.
+// hedge A second attempt at a block someone else is fetching too slowly. Only handed out once
+// no block is left waiting, so it can never take bandwidth from work that still needs
+// doing: what a hedge costs is bytes fetched twice at the very end, and what it buys is
+// that one slow connection can no longer hold the whole download back. Whichever attempt
+// finishes first wins; the other is dropped.
+
+export interface AttemptView {
+ kind: 'primary' | 'hedge'
+ streamId: number
+ networkId: string
+ /** When its request was sent. */
+ startedAt: number
+}
+
+export interface SchedulerState {
+ blocks: readonly BlockState[]
+ streams: readonly Pick<ChunkState, 'id' | 'interfaceId' | 'status' | 'speedBytesPerSec'>[]
+ /** Attempts in flight, by block index. */
+ attempts: ReadonlyMap<number, readonly AttemptView[]>
+ /** For a block whose last attempt on some network delivered nothing, that network. */
+ avoid: ReadonlyMap<number, string>
+ /** How many hedges each block has had so far. */
+ hedgesUsed: ReadonlyMap<number, number>
+}
+
+export interface SchedulerPolicy {
+ /** A block is hedged once its holder has been at it this long and still needs at least as long
+ * again. Long enough to have measured a real speed, and far more than a new connection costs. */
+ hedgeAfterMs: number
+ /** Bounds the duplicate work on a block whose hedges keep failing. */
+ maxHedgesPerBlock: number
+}
+
+export interface Requester {
+ id: number
+ networkId: string
+}
+
+export interface Work {
+ kind: 'primary' | 'hedge'
+ block: BlockState
+}
+
+/** The first waiting block — except that one whose last attempt on this network delivered
+ * nothing is left to another network, for as long as a stream there is free to take it.
+ * Otherwise a network that isn't answering would be handed the same block again and again, by
+ * whichever of its streams asked first, while healthy networks sat idle beside it. With no other
+ * network free the block is taken anyway, so it can never be stranded. */
+function nextWaitingBlock(state: SchedulerState, networkId: string): BlockState | undefined {
+ // A pending stream is one waiting for work (see ChunkStatus).
+ const otherNetworkFree = state.streams.some(
+ (stream) => stream.interfaceId !== networkId && stream.status === 'pending'
+ )
+ return state.blocks.find(
+ (block) =>
+ block.status === 'pending' &&
+ !(otherNetworkFree && state.avoid.get(block.index) === networkId)
+ )
+}
+
+/** The block most worth a second attempt: the one whose holder will be longest yet. */
+function nextHedgeTarget(
+ state: SchedulerState,
+ who: Requester,
+ now: number,
+ policy: SchedulerPolicy
+): BlockState | undefined {
+ // Only when everything left is already being fetched.
+ if (state.blocks.some((block) => block.status === 'pending')) return undefined
+
+ let target: BlockState | undefined
+ let latest = 0
+ for (const [index, attempts] of state.attempts) {
+ const block = state.blocks[index]
+ if (block?.index !== index || block.rangeEnd === null || block.status !== 'downloading')
+ continue
+ // Lone primary only: one hedge at a time, and nothing to race if the holder let go.
+ if (attempts.length !== 1 || attempts[0].kind !== 'primary') continue
+ const holder = attempts[0]
+
+ if (holder.streamId === who.id) continue
+ if ((state.hedgesUsed.get(index) ?? 0) >= policy.maxHedgesPerBlock) continue
+ // A network that got nothing from this block won't do better on a second try.
+ if (state.avoid.get(index) === who.networkId) continue
+ if (now - holder.startedAt < policy.hedgeAfterMs) continue
+
+ const remaining = block.rangeEnd - block.rangeStart + 1 - block.bytesDownloaded
+ if (remaining <= 0) continue
+ const speed = state.streams.find((stream) => stream.id === holder.streamId)?.speedBytesPerSec
+ // A holder that has gone quiet has no finish time at all.
+ const eta = speed ? (remaining / speed) * 1000 : Infinity
+ if (eta < policy.hedgeAfterMs) continue
+
+ // The holder's network may be what is slow, so a stream on another network gets the first
+ // go — unless none of those would take it either.
+ const otherNetworkFree = state.streams.some(
+ (stream) =>
+ stream.interfaceId !== holder.networkId &&
+ stream.status === 'pending' &&
+ state.avoid.get(index) !== stream.interfaceId
+ )
+ if (holder.networkId === who.networkId && otherNetworkFree) continue
+
+ if (eta > latest) {
+ target = block
+ latest = eta
+ }
+ }
+ return target
+}
+
+export function pickWork(
+ state: SchedulerState,
+ who: Requester,
+ now: number,
+ policy: SchedulerPolicy
+): Work | undefined {
+ const waiting = nextWaitingBlock(state, who.networkId)
+ if (waiting) return { kind: 'primary', block: waiting }
+ const slow = nextHedgeTarget(state, who, now, policy)
+ return slow && { kind: 'hedge', block: slow }
+}