Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Say "No peers found yet", search again, and stop after 30 s when the DHT finds no one for a magnet link (#159)
Arpit Khandelwal2ffd68177f52
diff --git a/e2e/torrentProbe.spec.ts b/e2e/torrentProbe.spec.tsindex 61f4d6d..7aedde4 100644--- a/e2e/torrentProbe.spec.ts+++ b/e2e/torrentProbe.spec.ts@@ -1,6 +1,6 @@import { mkdtemp, writeFile } from 'node:fs/promises'import { createServer, type Server } from 'node:http'-import type { AddressInfo } from 'node:net'+import { createServer as createNetServer, type AddressInfo } from 'node:net'import { tmpdir } from 'node:os'import { join } from 'node:path'import type WebTorrent from 'webtorrent'@@ -130,14 +130,118 @@ test.describe('getting a torrent in', () => {await expect(plexo.api.probeUrl(url)).rejects.toThrow(/BitTorrent v2 only/)})- test.describe('with nobody to ask', () => {- test.use({ appEnv: { PLEXO_E2E_MAGNET_MS: '1500' } })+ test('a magnet link with only an xs= source, its details fetched over HTTP', async ({+ plexo+ }) => {+ const { client, torrent } = await seed([named(seededBytes(200_000, 7), 'notes.pdf')])+ // Only the .torrent is served; the seeding client takes no part.+ await new Promise((done) => client.destroy(done))+ const { url, server } = await serveTorrent(torrent.torrentFile)+ servers.push(server)+ const magnet = `magnet:?xt=urn:btih:${torrent.infoHash}&xs=${encodeURIComponent(url)}`- test('a magnet link nobody answers gives up', async ({ plexo }) => {+ const probe = await plexo.api.probeUrl(magnet)+ expect(probe.kind).toBe('torrent')+ if (probe.kind !== 'torrent') throw new Error('Expected torrent metadata')+ expect(probe.suggestedFileName).toBe('notes.pdf')+ })++ test.describe('with nobody to ask', () => {+ test('a magnet link with no trackers and no DHT gives up at once', async ({ plexo }) => {const magnet = `magnet:?xt=urn:btih:${'ab'.repeat(20)}&dn=nothing`- await expect(plexo.api.probeUrl(magnet)).rejects.toThrow(- /No peers responded to this magnet link/+ const started = Date.now()+ await expect(plexo.api.probeUrl(magnet)).rejects.toThrow(/Couldn’t find anyone sharing/)+ // Well inside the 3-minute wait for peers: there was nothing to wait for.+ expect(Date.now() - started).toBeLessThan(10_000)+ })+ })++ test.describe('with a DHT that reaches no one', () => {+ // A bootstrap node nobody listens on: the DHT's lookup ends with no nodes to ask, as it does+ // behind a VPN or firewall that drops BitTorrent's UDP.+ test.use({+ appEnv: {+ PLEXO_E2E_DHT: '1',+ PLEXO_E2E_DHT_BOOTSTRAP: '127.0.0.1:9',+ PLEXO_E2E_MAGNET_GRACE_MS: '3000'+ }+ })++ test('a magnet link says no peers are found yet, then gives up after a short wait', async ({+ plexo+ }) => {+ const link = await plexo.newDownload()+ await link.fill(`magnet:?xt=urn:btih:${'cd'.repeat(20)}&dn=nothing`)+ await expect(plexo.page.getByText('No peers found yet, still looking…')).toBeVisible()+ const noPeersYetAt = Date.now()+ await expect(plexo.page.getByText(/Couldn’t find anyone sharing/)).toBeVisible({+ timeout: 20_000+ })+ // It kept looking for the whole window, not one moment longer than it needed.+ expect(Date.now() - noPeersYetAt).toBeGreaterThan(2_000)+ })+ })++ test('a second search finds a peer the first one missed', async ({ plexo }) => {+ // The DHT's only node. It knows of no peer until the test announces the seeder to it, after+ // the first search came back empty.+ const { default: DHT } = (await import('bittorrent-dht' as string)) as {+ default: new (options: object) => {+ listen(port: number, address: string, cb: () => void): void+ address(): { port: number }+ announce(infoHash: string, port: number, cb: (error?: Error) => void): void+ destroy(cb?: () => void): void+ }+ }+ // `host`: what it announces is also kept in its own peer list, for a get_peers to find.+ const node = new DHT({ bootstrap: false, host: '127.0.0.1' })+ await new Promise<void>((resolve) => node.listen(0, '127.0.0.1', resolve))+ const bootstrap = `127.0.0.1:${node.address().port}`+ try {+ await plexo.relaunch({+ PLEXO_E2E_DHT: '1',+ PLEXO_E2E_DHT_BOOTSTRAP: bootstrap,+ PLEXO_E2E_MAGNET_GRACE_MS: '15000'+ })+ // Off the DHT, with no tracker: it never learns of Plexo, so it can't connect to it. Plexo+ // can only find it by asking the DHT node again.+ const { client: seeder, torrent } = await seed([named(seededBytes(300_000, 8), 'late.mkv')])+ clients.push(seeder)++ const link = await plexo.newDownload()+ await link.fill(`magnet:?xt=urn:btih:${torrent.infoHash}&dn=late`)+ await expect(plexo.page.getByText('No peers found yet, still looking…')).toBeVisible()++ await new Promise<void>((resolve, reject) =>+ node.announce(torrent.infoHash, seeder.address().port, (error) =>+ error ? reject(error) : resolve()+ ))++ // Its details arrived: the name, and the file list.+ await expect(plexo.page.getByText('late.mkv').first()).toBeVisible({ timeout: 20_000 })+ await expect(plexo.page.getByText('No peers found yet, still looking…')).toBeHidden()+ } finally {+ await new Promise<void>((resolve) => node.destroy(() => resolve()))+ }+ })++ test.describe('with a peer that never answers', () => {+ test.use({ appEnv: { PLEXO_E2E_MAGNET_MS: '1500' } })++ test('a magnet link still waits for its peers, then gives up', async ({ plexo }) => {+ // Accepts the connection and says nothing, so no metadata ever comes.+ const silent = createNetServer(() => {})+ await new Promise<void>((resolve) => silent.listen(0, '127.0.0.1', resolve))+ try {+ const port = (silent.address() as AddressInfo).port+ const magnet = `magnet:?xt=urn:btih:${'ef'.repeat(20)}&dn=quiet&x.pe=127.0.0.1:${port}`+ await expect(plexo.api.probeUrl(magnet)).rejects.toThrow(+ /No peers responded to this magnet link/+ )+ } finally {+ silent.close()+ }})})})diff --git a/src/main/download/torrent/engine.ts b/src/main/download/torrent/engine.tsindex 584d1fe..9c174c5 100644--- a/src/main/download/torrent/engine.ts+++ b/src/main/download/torrent/engine.ts@@ -74,7 +74,10 @@ export async function createClient(natUpnp: false,natPmp: false,lsd: false,- dht: testKnobs.torrentDht,+ dht:+ testKnobs.torrentDht && testKnobs.torrentDhtBootstrap+ ? { bootstrap: testKnobs.torrentDhtBootstrap }+ : testKnobs.torrentDht,...options})}diff --git a/src/main/download/torrent/metadata.ts b/src/main/download/torrent/metadata.tsindex d2e8b02..fdf7f4d 100644--- a/src/main/download/torrent/metadata.ts+++ b/src/main/download/torrent/metadata.ts@@ -2,7 +2,7 @@ import { open } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'import type WebTorrent from 'webtorrent'-import type { ProbeResult } from '../../../shared/types'+import type { MagnetCheckStatus, ProbeResult } from '../../../shared/types'import { testKnobs } from '../../testKnobs'import { createClient } from './engine'import { safeTorrentPaths } from './paths'@@ -12,6 +12,9 @@ const MAX_TORRENT_FILE_BYTES = 10 * 1024 * 1024const TOO_LARGE = 'Plexo supports .torrent files up to 10 MB.'const V2_ONLY = 'This torrent uses BitTorrent v2 only, which Plexo doesn’t support yet'const NO_PEERS = 'No peers responded to this magnet link. Try again later or open a .torrent file.'+const NO_ONE_FOUND =+ 'Couldn’t find anyone sharing this magnet link. A VPN or firewall may be blocking BitTorrent; ' ++ 'otherwise try again later or open a .torrent file.'/** The .torrent of each torrent probed lately, by info hash: what a download of it starts from,* so the window never has to hold or send it. */@@ -93,12 +96,26 @@ let probeClient: Promise<WebTorrent> | null = null/** The magnet link being looked up. A newer one replaces it: the window only shows the latest. */let current: { infoHash: string; metadata: Promise<Uint8Array>; cancel: () => void } | null = null+const statusListeners = new Set<(status: MagnetCheckStatus) => void>()++/** Hears how each magnet link's check is going, for the window to show while it waits. */+export function onMagnetCheckStatus(listener: (status: MagnetCheckStatus) => void): () => void {+ statusListeners.add(listener)+ return () => statusListeners.delete(listener)+}++function tellStatus(status: MagnetCheckStatus): void {+ for (const listener of statusListeners) listener(status)+}+/** A magnet link's .torrent, from whichever of its peers answers first. */export async function fetchMagnetMetadata(magnet: string): Promise<Uint8Array> {const { default: parseTorrent } = await import('parse-torrent')- const { infoHash } = await parseTorrent(magnet).catch(() => {+ const { infoHash, announce, peerAddresses, xs } = await parseTorrent(magnet).catch(() => {throw new Error('This magnet link isn’t valid')})+ // Only the DHT can help a link that names no tracker, no peer and no .torrent to fetch (xs).+ const dhtOnly = !announce?.length && !peerAddresses?.length && !xs?.lengthif (current?.infoHash === infoHash) return current.metadatacurrent?.cancel()@@ -111,16 +128,57 @@ export async function fetchMagnetMetadata(magnet: string): Promise<Uint8Array> {// Deselected: only the metadata is wanted, none of the files.const torrent = client.add(magnet, { path: join(tmpdir(), 'plexo-magnet'), deselect: true })let done = false+ // Set while it keeps looking after a first search that found no one.+ let lookingLonger: { giveUp: NodeJS.Timeout; searchAgain: NodeJS.Timeout } | null = null+ let stopSearch = (): void => {}+ const stopLookingLonger = (): void => {+ if (!lookingLonger) return+ clearTimeout(lookingLonger.giveUp)+ clearTimeout(lookingLonger.searchAgain)+ lookingLonger = null+ }const finish = (error: Error | null, torrentFile?: Uint8Array): void => {if (done) returndone = trueclearTimeout(timer)+ stopLookingLonger()+ stopSearch()if (current === lookup) current = nulltorrent.destroy({ destroyStore: true })if (error) reject(error)else resolve(torrentFile!)}const timer = setTimeout(() => finish(new Error(NO_PEERS)), testKnobs.magnetTimeoutMs)++ // A link only the DHT can help: when its first search comes back empty, its next is 15+ // minutes off, so waiting out the timeout would only delay the same answer. It looks a little+ // longer instead: the first search after launch can miss peers while the DHT's table is still+ // filling, so it searches once more, and a peer that found this one through its announce can+ // still connect. Anyone turning up puts it back on the usual wait.+ let found = 0+ const foundSomeone = (): void => {+ found+++ if (!lookingLonger) return+ stopLookingLonger()+ tellStatus({ link: magnet, noPeersYet: false })+ }+ torrent.on('peer', foundSomeone)+ torrent.on('wire', foundSomeone)+ const firstSearchDone = (): void => {+ if (!dhtOnly || found > 0 || done) return+ const dht = client.dht+ // No DHT: nothing at all can find a peer for it.+ if (!dht) return finish(new Error(NO_ONE_FOUND))+ tellStatus({ link: magnet, noPeersYet: true })+ lookingLonger = {+ giveUp: setTimeout(() => finish(new Error(NO_ONE_FOUND)), testKnobs.magnetGraceMs),+ searchAgain: setTimeout(() => {+ stopSearch = dht.lookup(infoHash)+ }, testKnobs.magnetGraceMs / 3)+ }+ }+ if (client.dht) torrent.once('dhtAnnounce', firstSearchDone)+ else setImmediate(firstSearchDone)torrent.once('metadata', () => finish(null, torrent.torrentFile))torrent.once('error', (error: Error) => finish(error))lookup.cancel = () => finish(new Error('Replaced by a newer link'))diff --git a/src/main/download/torrent/modules.d.ts b/src/main/download/torrent/modules.d.tsindex 27f944a..8185cbe 100644--- a/src/main/download/torrent/modules.d.ts+++ b/src/main/download/torrent/modules.d.ts@@ -5,7 +5,8 @@ declare module 'webtorrent' {import type { Socket } from 'node:net'export interface ClientOptions {- dht?: boolean+ /** Off, on, or on and started from these nodes ("host:port") instead of the internet's. */+ dht?: boolean | { bootstrap?: string[] }lsd?: booleanutp?: booleannatUpnp?: boolean@@ -80,6 +81,12 @@ declare module 'webtorrent' {constructor(options?: ClientOptions)maxConns: numberdestroyed: boolean+ /** Its DHT (bittorrent-dht); false when turned off. `lookup` searches it for a torrent's+ * peers again, handing them to the torrent; it returns a function that stops the search. */+ dht:+ | (EventEmitter & { lookup(infoHash: string, callback?: () => void): () => void })+ | false+ | nulltorrents: Torrent[]add(torrentId: string | Uint8Array, options?: AddOptions): Torrentseed(@@ -117,6 +124,10 @@ declare module 'parse-torrent' {length?: numberpieceLength?: numberfiles?: { path: string; name: string; length: number; offset: number }[]+ /** A magnet link's trackers (tr), peers (x.pe) and .torrent sources (xs). */+ announce?: string[]+ peerAddresses?: string[]+ xs?: string | string[]}export default function parseTorrent(torrentId: string | Uint8Array): Promise<ParsedTorrent>}diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.tsindex 5ed3eb7..7fe3515 100644--- a/src/main/ipc/handlers.ts+++ b/src/main/ipc/handlers.ts@@ -21,6 +21,7 @@ import { DownloadManager } from '../download/downloadManager'import { getDefaultDownloadsDir, getHomeDir } from '../download/paths'import { listHistory } from '../download/history'import { probeUrl } from '../download/probe'+import { onMagnetCheckStatus } from '../download/torrent/metadata'import { deviceBindingSupported } from '../network/deviceBinding'import { pendingLink, pendingLinks, releaseLink } from '../openLinks'import { NetworkMonitor } from '../network/interfaces'@@ -157,6 +158,11 @@ export function registerIpcHandlers(getWindow: () => BrowserWindow | null): Down})handle('probeUrl', async (_event, url) => probeUrl(url))+ onMagnetCheckStatus((status) => {+ const window = getWindow()+ if (window && !window.isDestroyed())+ window.webContents.send(IpcChannels.magnetCheckStatus, status)+ })handle('chooseDestinationFolder', async (_event, defaultPath) => {const window = getWindow()diff --git a/src/main/testKnobs.ts b/src/main/testKnobs.tsindex 8d75eca..d5bcb05 100644--- a/src/main/testKnobs.ts+++ b/src/main/testKnobs.ts@@ -28,8 +28,14 @@ export const testKnobs = {hedgeAfterMs: positiveNumber('PLEXO_E2E_HEDGE_MS', 2_000),/** How long a magnet link may take to find peers that send its metadata. */magnetTimeoutMs: positiveNumber('PLEXO_E2E_MAGNET_MS', 3 * 60_000),+ /** How long a magnet link only the DHT can help keeps looking once its first search finds no+ * one (Deluge's magnet prefetch also waits 30 s). */+ magnetGraceMs: positiveNumber('PLEXO_E2E_MAGNET_GRACE_MS', 30_000),/** Tests turn the DHT off, so a run never reaches out to the internet's DHT nodes. */- torrentDht: env['PLEXO_E2E_DHT'] !== '0'+ torrentDht: env['PLEXO_E2E_DHT'] !== '0',+ /** `PLEXO_E2E_DHT_BOOTSTRAP=127.0.0.1:9` starts the DHT from these nodes instead of the+ * internet's, so a test can have a DHT that reaches no one. */+ torrentDhtBootstrap: env['PLEXO_E2E_DHT_BOOTSTRAP']?.split(',').filter(Boolean)}/** `PLEXO_E2E_STREAMS=2` fixes how many streams each network runs and turns the automaticdiff --git a/src/preload/index.ts b/src/preload/index.tsindex e11a3e1..504f47e 100644--- a/src/preload/index.ts+++ b/src/preload/index.ts@@ -5,6 +5,7 @@ import type {AppSettings,DownloadUpdate,InitialState,+ MagnetCheckStatus,NetworkInterfaceInfo,UpdateState} from '../shared/types'@@ -93,6 +94,12 @@ const plexoApi = {callback(networks)ipcRenderer.on(IpcChannels.networksChanged, listener)return () => ipcRenderer.removeListener(IpcChannels.networksChanged, listener)+ },++ onMagnetCheckStatus: (callback: (status: MagnetCheckStatus) => void): (() => void) => {+ const listener = (_event: IpcRendererEvent, status: MagnetCheckStatus): void => callback(status)+ ipcRenderer.on(IpcChannels.magnetCheckStatus, listener)+ return () => ipcRenderer.removeListener(IpcChannels.magnetCheckStatus, listener)}}diff --git a/src/renderer/src/components/NewDownloadDialog.tsx b/src/renderer/src/components/NewDownloadDialog.tsxindex ac93ecc..430f7d1 100644--- a/src/renderer/src/components/NewDownloadDialog.tsx+++ b/src/renderer/src/components/NewDownloadDialog.tsx@@ -23,7 +23,8 @@ import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip'type ProbeState =| { status: 'idle' }- | { status: 'probing' }+ /** `noPeersYet`: a magnet link whose first search found no one, still being looked for. */+ | { status: 'probing'; noPeersYet?: boolean }| { status: 'ready'; result: ProbeResult }| { status: 'error'; message: string }@@ -141,6 +142,12 @@ function NewDownloadForm({ onDone }: { onDone: () => void }): React.JSX.Element// answer is no longer wanted then.let stale = falsesetProbe({ status: 'probing' })+ const stopListening = window.plexo.onMagnetCheckStatus((status) => {+ if (stale || status.link !== trimmed) return+ setProbe((now) =>+ now.status === 'probing' ? { status: 'probing', noPeersYet: status.noPeersYet } : now+ )+ })const timer = setTimeout(async () => {try {const result = await window.plexo.probeUrl(trimmed)@@ -155,6 +162,7 @@ function NewDownloadForm({ onDone }: { onDone: () => void }): React.JSX.Elementreturn () => {stale = trueclearTimeout(timer)+ stopListening()}}, [url, dropLink])@@ -453,7 +461,9 @@ function NewDownloadForm({ onDone }: { onDone: () => void }): React.JSX.Element)}{probe.status === 'probing' && (- <div className="font-mono text-[11.5px] text-muted-foreground">Checking the link…</div>+ <div className="font-mono text-[11.5px] text-muted-foreground">+ {probe.noPeersYet ? 'No peers found yet, still looking…' : 'Checking the link…'}+ </div>)}{probe.status === 'error' && (<div className="flex items-center gap-2 text-[12.5px] text-[var(--color-danger)]">diff --git a/src/shared/ipc-channels.ts b/src/shared/ipc-channels.tsindex ee38396..b9b4529 100644--- a/src/shared/ipc-channels.ts+++ b/src/shared/ipc-channels.ts@@ -28,6 +28,7 @@ export const IpcChannels = {removeDownload: 'download:remove',downloadUpdated: 'download:updated',networksChanged: 'network:changed',+ magnetCheckStatus: 'download:magnet-check-status',linkReceived: 'app:link-received',pendingLinks: 'app:pending-links',resumeFromLink: 'app:resume-from-link',diff --git a/src/shared/ipc-contract.ts b/src/shared/ipc-contract.tsindex 5041be7..d8f186f 100644--- a/src/shared/ipc-contract.ts+++ b/src/shared/ipc-contract.ts@@ -11,8 +11,8 @@ import type {} from './types'/** The request/response half of the IPC surface (every IpcChannels entry except the- * main->renderer push events, downloadUpdated, networksChanged, historyChanged, linkReceived and- * updateStateChanged) — one source of truth for+ * main->renderer push events, downloadUpdated, networksChanged, magnetCheckStatus, historyChanged,+ * linkReceived and updateStateChanged) — one source of truth for* both plexoApi (preload) and registerIpcHandlers (main), so a signature drift between the two* is a compile error instead of a runtime one. */export interface IpcContract {diff --git a/src/shared/types.ts b/src/shared/types.tsindex 5f08203..ab0eff2 100644--- a/src/shared/types.ts+++ b/src/shared/types.ts@@ -46,6 +46,14 @@ export interface TorrentProbeResult extends ProbeResultBase {export type ProbeResult = HttpProbeResult | TorrentProbeResult+/** How the check of a magnet link is going, while it's still going: `noPeersYet` once its first+ * search came back empty and it's looking a little longer. */+export interface MagnetCheckStatus {+ /** The link as it was given to probeUrl. */+ link: string+ noPeersYet: boolean+}+export interface TorrentInfo {infoHash: stringpieceLength: number