Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
Torrents: reach each peer over every network (#161)
Yashwanth Sai9ce20c7151fb
diff --git a/e2e/torrent.spec.ts b/e2e/torrent.spec.tsindex 7388ab5..9da468d 100644--- a/e2e/torrent.spec.ts+++ b/e2e/torrent.spec.ts@@ -202,6 +202,25 @@ test.describe('torrents over two networks', () => {const done = await plexo.waitForTorrentStatus('completed', 60_000)expect(done.networks.find((network) => network.id === 'b')?.enabled).toBe(false)})++ test('one seeder is reached over both networks, and fills both', async ({ plexo }) => {+ const data = seededBytes(8 * MB, 44)+ const torrent = await swarm.seed([named(data, 'twin.bin')], {+ pieceLength: PIECE,+ uploadLimit: 1 * MB+ })++ await plexo.start(await torrentFileOnDisk(torrent), sha256(data), { networks: ['a', 'b'] })+ const done = await plexo.waitForTorrentStatus('completed', 30_000)+ for (const id of ['a', 'b']) {+ const network = done.networks.find((entry) => entry.id === id)!+ expect(network.bytesDownloaded, `network ${id} delivered pieces`).toBeGreaterThan(0)+ }+ const seen = new Set(+ plexo.sessions.flat().flatMap((state) => (state.kind === 'torrent' ? state.peers : []))+ )+ expect(new Set([...seen].map((peer) => peer.interfaceId))).toEqual(new Set(['a', 'b']))+ })})test.describe('uploading', () => {diff --git a/patches/webtorrent+3.0.21.patch b/patches/webtorrent+3.0.21.patchindex 21f5ec3..a5ab699 100644--- a/patches/webtorrent+3.0.21.patch+++ b/patches/webtorrent+3.0.21.patch@@ -10,16 +10,89 @@ index 8d3d156..08e8cf7 100644this.utp = WebTorrent.UTP_SUPPORT && opts.utp !== falsethis.seedOutgoingConnections = opts.seedOutgoingConnections ?? true+diff --git a/node_modules/webtorrent/lib/peer.js b/node_modules/webtorrent/lib/peer.js+index 212f038..0fc41b3 100644+--- a/node_modules/webtorrent/lib/peer.js++++ b/node_modules/webtorrent/lib/peer.js+@@ -188,7 +188,7 @@ export default class Peer extends EventEmitter {+ if (!addr && this.conn.remoteAddress && this.conn.remotePort) {+ addr = `${this.conn.remoteAddress}:${this.conn.remotePort}`+ }+- this.swarm._onWire(this.wire, addr)++ this.swarm._onWire(this.wire, addr, this.id)++ // swarm could be destroyed in user's 'wire' event handler+ if (!this.swarm || this.swarm.destroyed) return+@@ -201,7 +201,7 @@ export default class Peer extends EventEmitter {+ dht: this.swarm.private ? false : !!this.swarm.client.dht,+ fast: true+ }+- this.wire.handshake(this.swarm.infoHash, this.swarm.client.peerId, opts)++ this.wire.handshake(this.swarm.infoHash, this.twinPeerId || this.swarm.client.peerId, opts)+ this.sentHandshake = true+ }+diff --git a/node_modules/webtorrent/lib/torrent.js b/node_modules/webtorrent/lib/torrent.js-index 771a87f..1eafb73 100644+index 771a87f..88f4893 100644--- a/node_modules/webtorrent/lib/torrent.js+++ b/node_modules/webtorrent/lib/torrent.js-@@ -2122,7 +2122,7 @@ export default class Torrent extends EventEmitter {+@@ -1120,6 +1120,23 @@ export default class Torrent extends EventEmitter {+ return newPeer+ }+++ /**++ * Plexo: dials `addr` again, under `key` rather than the address, so it doesn't count as a++ * duplicate of the peer already there, and shakes hands with `peerId` (hex) rather than the++ * client's, so the remote end sees a client of its own. No reconnects: the caller re-adds it.++ * Dialled next: the peer is known to answer, unlike most of a queue fed by trackers and DHT.++ */++ addTwinPeer (addr, key, peerId) {++ if (this.destroyed || this.paused || this._peers.has(key) || !this._validAddr(addr)) return null++ const newPeer = Peer.createTCPOutgoingPeer(addr, this, this.client.throttleGroups, 'twin', this.client.secure)++ newPeer.id = key++ newPeer.twinPeerId = peerId++ this._registerPeer(newPeer)++ this._queue.unshift(newPeer)++ this._drain()++ return newPeer++ }+++ addWebSeed (urlOrConn) {+ if (this.destroyed) throw new Error('torrent is destroyed')++@@ -1287,7 +1304,7 @@ export default class Torrent extends EventEmitter {+ this._updateSelections()+ }++- _onWire (wire, addr) {++ _onWire (wire, addr, key = addr) {+ this._debug('got wire %s (%s)', wire._debugId, addr || 'Unknown')++ this.wires.push(wire)+@@ -1380,7 +1397,7 @@ export default class Torrent extends EventEmitter {++ // Hook to allow user-defined `bittorrent-protocol` extensions+ // More info: https://github.com/webtorrent/bittorrent-protocol#extension-api+- this.emit('wire', wire, addr)++ this.emit('wire', wire, addr, key)++ if (this.ready) {+ queueMicrotask(() => {+@@ -2122,7 +2139,7 @@ export default class Torrent extends EventEmitter {const utpServer = this.client._utpServerpeer.conn = utpServer ? utpServer.connect(opts.port, opts.host) : utp.connect(opts.port, opts.host)} else {- peer.conn = net.connect(opts)-+ peer.conn = this.client._connect ? this.client._connect(opts) : net.connect(opts)++ peer.conn = this.client._connect ? this.client._connect({ ...opts, key: peer.id }) : net.connect(opts)}const conn = peer.conn+@@ -2143,7 +2160,7 @@ export default class Torrent extends EventEmitter {++ // When connection closes, attempt reconnect after timeout (with exponential backoff)+ conn.on('close', () => {+- if (this.destroyed) return++ if (this.destroyed || peer.twinPeerId) return++ if (peer.retries >= RECONNECT_WAIT.length) {+ if (this.client.utp) {diff --git a/src/main/download/torrent/modules.d.ts b/src/main/download/torrent/modules.d.tsindex 8185cbe..9a7c789 100644--- a/src/main/download/torrent/modules.d.ts+++ b/src/main/download/torrent/modules.d.ts@@ -17,8 +17,8 @@ declare module 'webtorrent' {/** Bytes per second this client uploads at most. */uploadLimit?: number/** Added by Plexo's patch (patches/webtorrent+3.0.21.patch): opens every outgoing TCP peer- * connection. */- connect?: (options: { host: string; port: number }) => Socket+ * connection. `key` is the peer's key in the swarm: its address, or a twin's own key. */+ connect?: (options: { host: string; port: number; key: string }) => Socket}/** What webtorrent hands a store: the torrent's files, paths relative to `path`. */@@ -75,6 +75,9 @@ declare module 'webtorrent' {bitfield: { get(index: number): boolean }destroy(options?: { destroyStore?: boolean }, callback?: (error?: Error) => void): voidaddPeer(address: string): boolean+ /** Added by Plexo's patch: dials `address` again under `key`, shaking hands as `peerId` (hex),+ * so one peer can be reached over more than one network. Null if `key` is already there. */+ addTwinPeer(address: string, key: string, peerId: string): object | null}export default class WebTorrent extends EventEmitter {diff --git a/src/main/download/torrent/torrentTransfer.ts b/src/main/download/torrent/torrentTransfer.tsindex d5ca66b..7de22d4 100644--- a/src/main/download/torrent/torrentTransfer.ts+++ b/src/main/download/torrent/torrentTransfer.ts@@ -20,6 +20,16 @@ const PEERS_PER_NETWORK = 30// network has peers, can't reach the swarm. It keeps one dial going, to see when it can again.const UNREACHABLE_AFTER_ATTEMPTS = 5const UNREACHABLE_AFTER_MS = 60_000+// A twin that failed (refused, dropped) is tried again on its network after this long.+const TWIN_RETRY_MS = 30_000+const TWIN_MARK = '#twin:'++/** A twin's key in the swarm: the peer's address and the network it is dialled over. */+const twinKey = (address: string, networkId: string): string => `${address}${TWIN_MARK}${networkId}`+const twinNetwork = (key: string): string | null => {+ const at = key.indexOf(TWIN_MARK)+ return at < 0 ? null : key.slice(at + TWIN_MARK.length)+}const message = (error: unknown): string => (error instanceof Error ? error.message : String(error))@@ -65,6 +75,14 @@ export class TorrentTransfer implements Transfer {private reach = new Map<string, { dials: number; answeredAt: number }>()/** Bytes of each piece not verified yet, by network. */private unverified = new Map<number, Record<string, number>>()+ /** This run's torrent from the moment it is added, for dialling twins before 'ready'. */+ private engineTorrent: Torrent | null = null+ /** Addresses this run dialled out to: only these listen there, so only these get twins. */+ private dialledOut = new Set<string>()+ /** The peer id each network's twins shake hands with, so a peer sees one client per network. */+ private twinIds = new Map<string, string>()+ /** By twin key: when it may be dialled again. */+ private twinRetryAt = new Map<string, number>()constructor(private readonly runtime: TorrentTransferTarget,@@ -99,6 +117,7 @@ export class TorrentTransfer implements Transfer {this.host.scheduleUpdate()}}+ this.dialTwins(now)for (const peer of this.peers.values()) {const status = peer.state.speedBytesPerSec > 0 ? 'receiving' : 'connected'if (peer.state.status !== status) {@@ -118,6 +137,8 @@ export class TorrentTransfer implements Transfer {}reset(): void {+ this.dialledOut.clear()+ this.twinRetryAt.clear()this.chosen.clear()this.dialling.clear()this.reach.clear()@@ -180,6 +201,7 @@ export class TorrentTransfer implements Transfer {for (const peer of this.peers.values()) this.removePeer(peer)this.peers.clear()this.torrent = null+ this.engineTorrent = nullthis.ended = null}}@@ -220,7 +242,10 @@ export class TorrentTransfer implements Transfer {// Nothing until webtorrent is ready to be told which files (below), as on a change.deselect: true})- torrent.on('wire', (wire: Wire, address: string) => this.onWire(wire, address))+ this.engineTorrent = torrent+ torrent.on('wire', (wire: Wire, address: string, key?: string) =>+ this.onWire(wire, address, key)+ )torrent.on('verified', (index: number) => this.onVerified(index))torrent.once('ready', () => {this.torrent = torrent@@ -244,7 +269,18 @@ export class TorrentTransfer implements Transfer {}/** webtorrent's `connect` hook: every outgoing peer comes through here. */- private connect({ host: address, port }: { host: string; port: number }): Socket {+ private connect({+ host: address,+ port,+ key+ }: {+ host: string+ port: number+ key?: string+ }): Socket {+ // A twin goes over the network its key names, or nowhere.+ const twinOn = key ? twinNetwork(key) : null+ if (twinOn) return this.connectTwin(address, port, key!, twinOn)// A peer named by hostname (a magnet's x.pe can be) has no family: no network takes it.const family = isIP(address) as 0 | 4 | 6const peersOn = (id: string): number =>@@ -267,9 +303,77 @@ export class TorrentTransfer implements Transfer {}this.chosen.set(addressOf(address, port), networkId)+ this.dialledOut.add(addressOf(address, port))+ return this.dial(routes.get(networkId)!, port, networkId)+ }++ /** A twin's dial: over `networkId` only, if it is on and has a route to the peer. */+ private connectTwin(address: string, port: number, key: string, networkId: string): Socket {+ const family = isIP(address) as 0 | 4 | 6+ const network = this.usable().find((entry) => entry.id === networkId)+ const iface = network && this.host.networks.find(network.id)+ const route = family && iface ? routesFor(iface, [{ address, family }])[0] : undefined+ if (!route || network?.status !== 'on') {+ const socket = new Socket()+ process.nextTick(() => socket.destroy(new Error('Twin network cannot reach this peer')))+ return socket+ }+ this.chosen.set(key, networkId)+ return this.dial(route, port, networkId)+ }++ /**+ * Dials a peer again over every other network that is on, each as a client of its own (see+ * addTwinPeer in patches/webtorrent+3.0.21.patch): a swarm of one fast peer then fills more+ * than one network. Only peers this run dialled out to, whose address is where they listen.+ */+ private dialTwins(now: number): void {+ const torrent = this.engineTorrent+ if (!torrent) return+ const on = this.usable().filter((network) => network.status === 'on')+ if (on.length < 2) return+ const originals = new Map<string, string>()+ for (const [wire, peer] of this.peers) {+ const address = wire as Wire & { remoteAddress?: string; remotePort?: number }+ if (!address.remoteAddress || !address.remotePort) continue+ const key = addressOf(address.remoteAddress, address.remotePort)+ if (this.dialledOut.has(key) && this.chosen.get(key) === peer.state.interfaceId) {+ originals.set(key, peer.state.interfaceId)+ }+ }+ for (const [address, networkId] of originals) {+ for (const network of on) {+ if (network.id === networkId) continue+ const key = twinKey(address, network.id)+ if ((this.twinRetryAt.get(key) ?? 0) > now) continue+ this.twinRetryAt.set(key, now + TWIN_RETRY_MS)+ torrent.addTwinPeer(address, key, this.twinIdFor(network.id))+ }+ }+ }++ /** A peer id for `networkId`'s twins: the client's prefix (its name and version), the rest its+ * own. */+ private twinIdFor(networkId: string): string {+ let id = this.twinIds.get(networkId)+ if (!id) {+ const prefix = (this.client as WebTorrent & { peerId?: string })?.peerId?.slice(0, 16) ?? ''+ const rest = Array.from({ length: (40 - prefix.length) / 2 }, () =>+ Math.floor(Math.random() * 256)+ .toString(16)+ .padStart(2, '0')+ ).join('')+ id = prefix + rest+ this.twinIds.set(networkId, id)+ }+ return id+ }++ /** Opens a socket to a peer over a network, counted as a dial on it until it settles. */+ private dial(route: NetworkRoute, port: number, networkId: string): Socket {this.dialling.set(networkId, (this.dialling.get(networkId) ?? 0) + 1)this.reachOf(networkId).dials += 1- const socket = connectRoute(routes.get(networkId)!, port)+ const socket = connectRoute(route, port)let settled = falseconst settle = (): void => {if (settled) return@@ -313,8 +417,9 @@ export class TorrentTransfer implements Transfer {this.limit(socket, network.id)}- private onWire(wire: Wire, address: string): void {- const networkId = this.chosen.get(address)+ private onWire(wire: Wire, address: string, key = address): void {+ // A twin is filed under its own key; every other peer under its address.+ const networkId = this.chosen.get(twinNetwork(key) ? key : address)const network = this.runtime.state.networks.find((entry) => entry.id === networkId)if (!network) {wire.destroy()