Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Say "No peers found yet", search again, and stop after 30 s when the DHT finds no one for a magnet link (#159)

Arpit Khandelwal2ffd68177f52
diff --git a/e2e/torrentProbe.spec.ts b/e2e/torrentProbe.spec.ts
index 61f4d6d..7aedde4 100644
--- a/e2e/torrentProbe.spec.ts
+++ b/e2e/torrentProbe.spec.ts
@@ -1,6 +1,6 @@
import { mkdtemp, writeFile } from 'node:fs/promises'
import { createServer, type Server } from 'node:http'
-import type { AddressInfo } from 'node:net'
+import { createServer as createNetServer, type AddressInfo } from 'node:net'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type WebTorrent from 'webtorrent'
@@ -130,14 +130,118 @@ test.describe('getting a torrent in', () => {
await expect(plexo.api.probeUrl(url)).rejects.toThrow(/BitTorrent v2 only/)
})
- test.describe('with nobody to ask', () => {
- test.use({ appEnv: { PLEXO_E2E_MAGNET_MS: '1500' } })
+ test('a magnet link with only an xs= source, its details fetched over HTTP', async ({
+ plexo
+ }) => {
+ const { client, torrent } = await seed([named(seededBytes(200_000, 7), 'notes.pdf')])
+ // Only the .torrent is served; the seeding client takes no part.
+ await new Promise((done) => client.destroy(done))
+ const { url, server } = await serveTorrent(torrent.torrentFile)
+ servers.push(server)
+ const magnet = `magnet:?xt=urn:btih:${torrent.infoHash}&xs=${encodeURIComponent(url)}`
- test('a magnet link nobody answers gives up', async ({ plexo }) => {
+ const probe = await plexo.api.probeUrl(magnet)
+ expect(probe.kind).toBe('torrent')
+ if (probe.kind !== 'torrent') throw new Error('Expected torrent metadata')
+ expect(probe.suggestedFileName).toBe('notes.pdf')
+ })
+
+ test.describe('with nobody to ask', () => {
+ test('a magnet link with no trackers and no DHT gives up at once', async ({ plexo }) => {
const magnet = `magnet:?xt=urn:btih:${'ab'.repeat(20)}&dn=nothing`
- await expect(plexo.api.probeUrl(magnet)).rejects.toThrow(
- /No peers responded to this magnet link/
+ const started = Date.now()
+ await expect(plexo.api.probeUrl(magnet)).rejects.toThrow(/Couldn’t find anyone sharing/)
+ // Well inside the 3-minute wait for peers: there was nothing to wait for.
+ expect(Date.now() - started).toBeLessThan(10_000)
+ })
+ })
+
+ test.describe('with a DHT that reaches no one', () => {
+ // A bootstrap node nobody listens on: the DHT's lookup ends with no nodes to ask, as it does
+ // behind a VPN or firewall that drops BitTorrent's UDP.
+ test.use({
+ appEnv: {
+ PLEXO_E2E_DHT: '1',
+ PLEXO_E2E_DHT_BOOTSTRAP: '127.0.0.1:9',
+ PLEXO_E2E_MAGNET_GRACE_MS: '3000'
+ }
+ })
+
+ test('a magnet link says no peers are found yet, then gives up after a short wait', async ({
+ plexo
+ }) => {
+ const link = await plexo.newDownload()
+ await link.fill(`magnet:?xt=urn:btih:${'cd'.repeat(20)}&dn=nothing`)
+ await expect(plexo.page.getByText('No peers found yet, still looking…')).toBeVisible()
+ const noPeersYetAt = Date.now()
+ await expect(plexo.page.getByText(/Couldn’t find anyone sharing/)).toBeVisible({
+ timeout: 20_000
+ })
+ // It kept looking for the whole window, not one moment longer than it needed.
+ expect(Date.now() - noPeersYetAt).toBeGreaterThan(2_000)
+ })
+ })
+
+ test('a second search finds a peer the first one missed', async ({ plexo }) => {
+ // The DHT's only node. It knows of no peer until the test announces the seeder to it, after
+ // the first search came back empty.
+ const { default: DHT } = (await import('bittorrent-dht' as string)) as {
+ default: new (options: object) => {
+ listen(port: number, address: string, cb: () => void): void
+ address(): { port: number }
+ announce(infoHash: string, port: number, cb: (error?: Error) => void): void
+ destroy(cb?: () => void): void
+ }
+ }
+ // `host`: what it announces is also kept in its own peer list, for a get_peers to find.
+ const node = new DHT({ bootstrap: false, host: '127.0.0.1' })
+ await new Promise<void>((resolve) => node.listen(0, '127.0.0.1', resolve))
+ const bootstrap = `127.0.0.1:${node.address().port}`
+ try {
+ await plexo.relaunch({
+ PLEXO_E2E_DHT: '1',
+ PLEXO_E2E_DHT_BOOTSTRAP: bootstrap,
+ PLEXO_E2E_MAGNET_GRACE_MS: '15000'
+ })
+ // Off the DHT, with no tracker: it never learns of Plexo, so it can't connect to it. Plexo
+ // can only find it by asking the DHT node again.
+ const { client: seeder, torrent } = await seed([named(seededBytes(300_000, 8), 'late.mkv')])
+ clients.push(seeder)
+
+ const link = await plexo.newDownload()
+ await link.fill(`magnet:?xt=urn:btih:${torrent.infoHash}&dn=late`)
+ await expect(plexo.page.getByText('No peers found yet, still looking…')).toBeVisible()
+
+ await new Promise<void>((resolve, reject) =>
+ node.announce(torrent.infoHash, seeder.address().port, (error) =>
+ error ? reject(error) : resolve()
+ )
)
+
+ // Its details arrived: the name, and the file list.
+ await expect(plexo.page.getByText('late.mkv').first()).toBeVisible({ timeout: 20_000 })
+ await expect(plexo.page.getByText('No peers found yet, still looking…')).toBeHidden()
+ } finally {
+ await new Promise<void>((resolve) => node.destroy(() => resolve()))
+ }
+ })
+
+ test.describe('with a peer that never answers', () => {
+ test.use({ appEnv: { PLEXO_E2E_MAGNET_MS: '1500' } })
+
+ test('a magnet link still waits for its peers, then gives up', async ({ plexo }) => {
+ // Accepts the connection and says nothing, so no metadata ever comes.
+ const silent = createNetServer(() => {})
+ await new Promise<void>((resolve) => silent.listen(0, '127.0.0.1', resolve))
+ try {
+ const port = (silent.address() as AddressInfo).port
+ const magnet = `magnet:?xt=urn:btih:${'ef'.repeat(20)}&dn=quiet&x.pe=127.0.0.1:${port}`
+ await expect(plexo.api.probeUrl(magnet)).rejects.toThrow(
+ /No peers responded to this magnet link/
+ )
+ } finally {
+ silent.close()
+ }
})
})
})
diff --git a/src/main/download/torrent/engine.ts b/src/main/download/torrent/engine.ts
index 584d1fe..9c174c5 100644
--- a/src/main/download/torrent/engine.ts
+++ b/src/main/download/torrent/engine.ts
@@ -74,7 +74,10 @@ export async function createClient(
natUpnp: false,
natPmp: false,
lsd: false,
- dht: testKnobs.torrentDht,
+ dht:
+ testKnobs.torrentDht && testKnobs.torrentDhtBootstrap
+ ? { bootstrap: testKnobs.torrentDhtBootstrap }
+ : testKnobs.torrentDht,
...options
})
}
diff --git a/src/main/download/torrent/metadata.ts b/src/main/download/torrent/metadata.ts
index d2e8b02..fdf7f4d 100644
--- a/src/main/download/torrent/metadata.ts
+++ b/src/main/download/torrent/metadata.ts
@@ -2,7 +2,7 @@ import { open } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type WebTorrent from 'webtorrent'
-import type { ProbeResult } from '../../../shared/types'
+import type { MagnetCheckStatus, ProbeResult } from '../../../shared/types'
import { testKnobs } from '../../testKnobs'
import { createClient } from './engine'
import { safeTorrentPaths } from './paths'
@@ -12,6 +12,9 @@ const MAX_TORRENT_FILE_BYTES = 10 * 1024 * 1024
const TOO_LARGE = 'Plexo supports .torrent files up to 10 MB.'
const V2_ONLY = 'This torrent uses BitTorrent v2 only, which Plexo doesn’t support yet'
const NO_PEERS = 'No peers responded to this magnet link. Try again later or open a .torrent file.'
+const NO_ONE_FOUND =
+ 'Couldn’t find anyone sharing this magnet link. A VPN or firewall may be blocking BitTorrent; ' +
+ 'otherwise try again later or open a .torrent file.'
/** The .torrent of each torrent probed lately, by info hash: what a download of it starts from,
* so the window never has to hold or send it. */
@@ -93,12 +96,26 @@ let probeClient: Promise<WebTorrent> | null = null
/** The magnet link being looked up. A newer one replaces it: the window only shows the latest. */
let current: { infoHash: string; metadata: Promise<Uint8Array>; cancel: () => void } | null = null
+const statusListeners = new Set<(status: MagnetCheckStatus) => void>()
+
+/** Hears how each magnet link's check is going, for the window to show while it waits. */
+export function onMagnetCheckStatus(listener: (status: MagnetCheckStatus) => void): () => void {
+ statusListeners.add(listener)
+ return () => statusListeners.delete(listener)
+}
+
+function tellStatus(status: MagnetCheckStatus): void {
+ for (const listener of statusListeners) listener(status)
+}
+
/** A magnet link's .torrent, from whichever of its peers answers first. */
export async function fetchMagnetMetadata(magnet: string): Promise<Uint8Array> {
const { default: parseTorrent } = await import('parse-torrent')
- const { infoHash } = await parseTorrent(magnet).catch(() => {
+ const { infoHash, announce, peerAddresses, xs } = await parseTorrent(magnet).catch(() => {
throw new Error('This magnet link isn’t valid')
})
+ // Only the DHT can help a link that names no tracker, no peer and no .torrent to fetch (xs).
+ const dhtOnly = !announce?.length && !peerAddresses?.length && !xs?.length
if (current?.infoHash === infoHash) return current.metadata
current?.cancel()
@@ -111,16 +128,57 @@ export async function fetchMagnetMetadata(magnet: string): Promise<Uint8Array> {
// Deselected: only the metadata is wanted, none of the files.
const torrent = client.add(magnet, { path: join(tmpdir(), 'plexo-magnet'), deselect: true })
let done = false
+ // Set while it keeps looking after a first search that found no one.
+ let lookingLonger: { giveUp: NodeJS.Timeout; searchAgain: NodeJS.Timeout } | null = null
+ let stopSearch = (): void => {}
+ const stopLookingLonger = (): void => {
+ if (!lookingLonger) return
+ clearTimeout(lookingLonger.giveUp)
+ clearTimeout(lookingLonger.searchAgain)
+ lookingLonger = null
+ }
const finish = (error: Error | null, torrentFile?: Uint8Array): void => {
if (done) return
done = true
clearTimeout(timer)
+ stopLookingLonger()
+ stopSearch()
if (current === lookup) current = null
torrent.destroy({ destroyStore: true })
if (error) reject(error)
else resolve(torrentFile!)
}
const timer = setTimeout(() => finish(new Error(NO_PEERS)), testKnobs.magnetTimeoutMs)
+
+ // A link only the DHT can help: when its first search comes back empty, its next is 15
+ // minutes off, so waiting out the timeout would only delay the same answer. It looks a little
+ // longer instead: the first search after launch can miss peers while the DHT's table is still
+ // filling, so it searches once more, and a peer that found this one through its announce can
+ // still connect. Anyone turning up puts it back on the usual wait.
+ let found = 0
+ const foundSomeone = (): void => {
+ found++
+ if (!lookingLonger) return
+ stopLookingLonger()
+ tellStatus({ link: magnet, noPeersYet: false })
+ }
+ torrent.on('peer', foundSomeone)
+ torrent.on('wire', foundSomeone)
+ const firstSearchDone = (): void => {
+ if (!dhtOnly || found > 0 || done) return
+ const dht = client.dht
+ // No DHT: nothing at all can find a peer for it.
+ if (!dht) return finish(new Error(NO_ONE_FOUND))
+ tellStatus({ link: magnet, noPeersYet: true })
+ lookingLonger = {
+ giveUp: setTimeout(() => finish(new Error(NO_ONE_FOUND)), testKnobs.magnetGraceMs),
+ searchAgain: setTimeout(() => {
+ stopSearch = dht.lookup(infoHash)
+ }, testKnobs.magnetGraceMs / 3)
+ }
+ }
+ if (client.dht) torrent.once('dhtAnnounce', firstSearchDone)
+ else setImmediate(firstSearchDone)
torrent.once('metadata', () => finish(null, torrent.torrentFile))
torrent.once('error', (error: Error) => finish(error))
lookup.cancel = () => finish(new Error('Replaced by a newer link'))
diff --git a/src/main/download/torrent/modules.d.ts b/src/main/download/torrent/modules.d.ts
index 27f944a..8185cbe 100644
--- a/src/main/download/torrent/modules.d.ts
+++ b/src/main/download/torrent/modules.d.ts
@@ -5,7 +5,8 @@ declare module 'webtorrent' {
import type { Socket } from 'node:net'
export interface ClientOptions {
- dht?: boolean
+ /** Off, on, or on and started from these nodes ("host:port") instead of the internet's. */
+ dht?: boolean | { bootstrap?: string[] }
lsd?: boolean
utp?: boolean
natUpnp?: boolean
@@ -80,6 +81,12 @@ declare module 'webtorrent' {
constructor(options?: ClientOptions)
maxConns: number
destroyed: boolean
+ /** Its DHT (bittorrent-dht); false when turned off. `lookup` searches it for a torrent's
+ * peers again, handing them to the torrent; it returns a function that stops the search. */
+ dht:
+ | (EventEmitter & { lookup(infoHash: string, callback?: () => void): () => void })
+ | false
+ | null
torrents: Torrent[]
add(torrentId: string | Uint8Array, options?: AddOptions): Torrent
seed(
@@ -117,6 +124,10 @@ declare module 'parse-torrent' {
length?: number
pieceLength?: number
files?: { path: string; name: string; length: number; offset: number }[]
+ /** A magnet link's trackers (tr), peers (x.pe) and .torrent sources (xs). */
+ announce?: string[]
+ peerAddresses?: string[]
+ xs?: string | string[]
}
export default function parseTorrent(torrentId: string | Uint8Array): Promise<ParsedTorrent>
}
diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.ts
index 5ed3eb7..7fe3515 100644
--- a/src/main/ipc/handlers.ts
+++ b/src/main/ipc/handlers.ts
@@ -21,6 +21,7 @@ import { DownloadManager } from '../download/downloadManager'
import { getDefaultDownloadsDir, getHomeDir } from '../download/paths'
import { listHistory } from '../download/history'
import { probeUrl } from '../download/probe'
+import { onMagnetCheckStatus } from '../download/torrent/metadata'
import { deviceBindingSupported } from '../network/deviceBinding'
import { pendingLink, pendingLinks, releaseLink } from '../openLinks'
import { NetworkMonitor } from '../network/interfaces'
@@ -157,6 +158,11 @@ export function registerIpcHandlers(getWindow: () => BrowserWindow | null): Down
})
handle('probeUrl', async (_event, url) => probeUrl(url))
+ onMagnetCheckStatus((status) => {
+ const window = getWindow()
+ if (window && !window.isDestroyed())
+ window.webContents.send(IpcChannels.magnetCheckStatus, status)
+ })
handle('chooseDestinationFolder', async (_event, defaultPath) => {
const window = getWindow()
diff --git a/src/main/testKnobs.ts b/src/main/testKnobs.ts
index 8d75eca..d5bcb05 100644
--- a/src/main/testKnobs.ts
+++ b/src/main/testKnobs.ts
@@ -28,8 +28,14 @@ export const testKnobs = {
hedgeAfterMs: positiveNumber('PLEXO_E2E_HEDGE_MS', 2_000),
/** How long a magnet link may take to find peers that send its metadata. */
magnetTimeoutMs: positiveNumber('PLEXO_E2E_MAGNET_MS', 3 * 60_000),
+ /** How long a magnet link only the DHT can help keeps looking once its first search finds no
+ * one (Deluge's magnet prefetch also waits 30 s). */
+ magnetGraceMs: positiveNumber('PLEXO_E2E_MAGNET_GRACE_MS', 30_000),
/** Tests turn the DHT off, so a run never reaches out to the internet's DHT nodes. */
- torrentDht: env['PLEXO_E2E_DHT'] !== '0'
+ torrentDht: env['PLEXO_E2E_DHT'] !== '0',
+ /** `PLEXO_E2E_DHT_BOOTSTRAP=127.0.0.1:9` starts the DHT from these nodes instead of the
+ * internet's, so a test can have a DHT that reaches no one. */
+ torrentDhtBootstrap: env['PLEXO_E2E_DHT_BOOTSTRAP']?.split(',').filter(Boolean)
}
/** `PLEXO_E2E_STREAMS=2` fixes how many streams each network runs and turns the automatic
diff --git a/src/preload/index.ts b/src/preload/index.ts
index e11a3e1..504f47e 100644
--- a/src/preload/index.ts
+++ b/src/preload/index.ts
@@ -5,6 +5,7 @@ import type {
AppSettings,
DownloadUpdate,
InitialState,
+ MagnetCheckStatus,
NetworkInterfaceInfo,
UpdateState
} from '../shared/types'
@@ -93,6 +94,12 @@ const plexoApi = {
callback(networks)
ipcRenderer.on(IpcChannels.networksChanged, listener)
return () => ipcRenderer.removeListener(IpcChannels.networksChanged, listener)
+ },
+
+ onMagnetCheckStatus: (callback: (status: MagnetCheckStatus) => void): (() => void) => {
+ const listener = (_event: IpcRendererEvent, status: MagnetCheckStatus): void => callback(status)
+ ipcRenderer.on(IpcChannels.magnetCheckStatus, listener)
+ return () => ipcRenderer.removeListener(IpcChannels.magnetCheckStatus, listener)
}
}
diff --git a/src/renderer/src/components/NewDownloadDialog.tsx b/src/renderer/src/components/NewDownloadDialog.tsx
index ac93ecc..430f7d1 100644
--- a/src/renderer/src/components/NewDownloadDialog.tsx
+++ b/src/renderer/src/components/NewDownloadDialog.tsx
@@ -23,7 +23,8 @@ import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip'
type ProbeState =
| { status: 'idle' }
- | { status: 'probing' }
+ /** `noPeersYet`: a magnet link whose first search found no one, still being looked for. */
+ | { status: 'probing'; noPeersYet?: boolean }
| { status: 'ready'; result: ProbeResult }
| { status: 'error'; message: string }
@@ -141,6 +142,12 @@ function NewDownloadForm({ onDone }: { onDone: () => void }): React.JSX.Element
// answer is no longer wanted then.
let stale = false
setProbe({ status: 'probing' })
+ const stopListening = window.plexo.onMagnetCheckStatus((status) => {
+ if (stale || status.link !== trimmed) return
+ setProbe((now) =>
+ now.status === 'probing' ? { status: 'probing', noPeersYet: status.noPeersYet } : now
+ )
+ })
const timer = setTimeout(async () => {
try {
const result = await window.plexo.probeUrl(trimmed)
@@ -155,6 +162,7 @@ function NewDownloadForm({ onDone }: { onDone: () => void }): React.JSX.Element
return () => {
stale = true
clearTimeout(timer)
+ stopListening()
}
}, [url, dropLink])
@@ -453,7 +461,9 @@ function NewDownloadForm({ onDone }: { onDone: () => void }): React.JSX.Element
)}
{probe.status === 'probing' && (
- <div className="font-mono text-[11.5px] text-muted-foreground">Checking the link…</div>
+ <div className="font-mono text-[11.5px] text-muted-foreground">
+ {probe.noPeersYet ? 'No peers found yet, still looking…' : 'Checking the link…'}
+ </div>
)}
{probe.status === 'error' && (
<div className="flex items-center gap-2 text-[12.5px] text-[var(--color-danger)]">
diff --git a/src/shared/ipc-channels.ts b/src/shared/ipc-channels.ts
index ee38396..b9b4529 100644
--- a/src/shared/ipc-channels.ts
+++ b/src/shared/ipc-channels.ts
@@ -28,6 +28,7 @@ export const IpcChannels = {
removeDownload: 'download:remove',
downloadUpdated: 'download:updated',
networksChanged: 'network:changed',
+ magnetCheckStatus: 'download:magnet-check-status',
linkReceived: 'app:link-received',
pendingLinks: 'app:pending-links',
resumeFromLink: 'app:resume-from-link',
diff --git a/src/shared/ipc-contract.ts b/src/shared/ipc-contract.ts
index 5041be7..d8f186f 100644
--- a/src/shared/ipc-contract.ts
+++ b/src/shared/ipc-contract.ts
@@ -11,8 +11,8 @@ import type {
} from './types'
/** The request/response half of the IPC surface (every IpcChannels entry except the
- * main->renderer push events, downloadUpdated, networksChanged, historyChanged, linkReceived and
- * updateStateChanged) — one source of truth for
+ * main->renderer push events, downloadUpdated, networksChanged, magnetCheckStatus, historyChanged,
+ * linkReceived and updateStateChanged) — one source of truth for
* both plexoApi (preload) and registerIpcHandlers (main), so a signature drift between the two
* is a compile error instead of a runtime one. */
export interface IpcContract {
diff --git a/src/shared/types.ts b/src/shared/types.ts
index 5f08203..ab0eff2 100644
--- a/src/shared/types.ts
+++ b/src/shared/types.ts
@@ -46,6 +46,14 @@ export interface TorrentProbeResult extends ProbeResultBase {
export type ProbeResult = HttpProbeResult | TorrentProbeResult
+/** How the check of a magnet link is going, while it's still going: `noPeersYet` once its first
+ * search came back empty and it's looking a little longer. */
+export interface MagnetCheckStatus {
+ /** The link as it was given to probeUrl. */
+ link: string
+ noPeersYet: boolean
+}
+
export interface TorrentInfo {
infoHash: string
pieceLength: number