Skip to content

armgit / plexo

Public Imported from GitHub
0

Speed up downloads by combining multiple network connections in parallel

Torrents: reach each peer over every network (#161)

Yashwanth Sai9ce20c7151fb
diff --git a/e2e/torrent.spec.ts b/e2e/torrent.spec.ts
index 7388ab5..9da468d 100644
--- a/e2e/torrent.spec.ts
+++ b/e2e/torrent.spec.ts
@@ -202,6 +202,25 @@ test.describe('torrents over two networks', () => {
const done = await plexo.waitForTorrentStatus('completed', 60_000)
expect(done.networks.find((network) => network.id === 'b')?.enabled).toBe(false)
})
+
+ test('one seeder is reached over both networks, and fills both', async ({ plexo }) => {
+ const data = seededBytes(8 * MB, 44)
+ const torrent = await swarm.seed([named(data, 'twin.bin')], {
+ pieceLength: PIECE,
+ uploadLimit: 1 * MB
+ })
+
+ await plexo.start(await torrentFileOnDisk(torrent), sha256(data), { networks: ['a', 'b'] })
+ const done = await plexo.waitForTorrentStatus('completed', 30_000)
+ for (const id of ['a', 'b']) {
+ const network = done.networks.find((entry) => entry.id === id)!
+ expect(network.bytesDownloaded, `network ${id} delivered pieces`).toBeGreaterThan(0)
+ }
+ const seen = new Set(
+ plexo.sessions.flat().flatMap((state) => (state.kind === 'torrent' ? state.peers : []))
+ )
+ expect(new Set([...seen].map((peer) => peer.interfaceId))).toEqual(new Set(['a', 'b']))
+ })
})
test.describe('uploading', () => {
diff --git a/patches/webtorrent+3.0.21.patch b/patches/webtorrent+3.0.21.patch
index 21f5ec3..a5ab699 100644
--- a/patches/webtorrent+3.0.21.patch
+++ b/patches/webtorrent+3.0.21.patch
@@ -10,16 +10,89 @@ index 8d3d156..08e8cf7 100644
this.utp = WebTorrent.UTP_SUPPORT && opts.utp !== false
this.seedOutgoingConnections = opts.seedOutgoingConnections ?? true
+diff --git a/node_modules/webtorrent/lib/peer.js b/node_modules/webtorrent/lib/peer.js
+index 212f038..0fc41b3 100644
+--- a/node_modules/webtorrent/lib/peer.js
++++ b/node_modules/webtorrent/lib/peer.js
+@@ -188,7 +188,7 @@ export default class Peer extends EventEmitter {
+ if (!addr && this.conn.remoteAddress && this.conn.remotePort) {
+ addr = `${this.conn.remoteAddress}:${this.conn.remotePort}`
+ }
+- this.swarm._onWire(this.wire, addr)
++ this.swarm._onWire(this.wire, addr, this.id)
+
+ // swarm could be destroyed in user's 'wire' event handler
+ if (!this.swarm || this.swarm.destroyed) return
+@@ -201,7 +201,7 @@ export default class Peer extends EventEmitter {
+ dht: this.swarm.private ? false : !!this.swarm.client.dht,
+ fast: true
+ }
+- this.wire.handshake(this.swarm.infoHash, this.swarm.client.peerId, opts)
++ this.wire.handshake(this.swarm.infoHash, this.twinPeerId || this.swarm.client.peerId, opts)
+ this.sentHandshake = true
+ }
+
diff --git a/node_modules/webtorrent/lib/torrent.js b/node_modules/webtorrent/lib/torrent.js
-index 771a87f..1eafb73 100644
+index 771a87f..88f4893 100644
--- a/node_modules/webtorrent/lib/torrent.js
+++ b/node_modules/webtorrent/lib/torrent.js
-@@ -2122,7 +2122,7 @@ export default class Torrent extends EventEmitter {
+@@ -1120,6 +1120,23 @@ export default class Torrent extends EventEmitter {
+ return newPeer
+ }
+
++ /**
++ * Plexo: dials `addr` again, under `key` rather than the address, so it doesn't count as a
++ * duplicate of the peer already there, and shakes hands with `peerId` (hex) rather than the
++ * client's, so the remote end sees a client of its own. No reconnects: the caller re-adds it.
++ * Dialled next: the peer is known to answer, unlike most of a queue fed by trackers and DHT.
++ */
++ addTwinPeer (addr, key, peerId) {
++ if (this.destroyed || this.paused || this._peers.has(key) || !this._validAddr(addr)) return null
++ const newPeer = Peer.createTCPOutgoingPeer(addr, this, this.client.throttleGroups, 'twin', this.client.secure)
++ newPeer.id = key
++ newPeer.twinPeerId = peerId
++ this._registerPeer(newPeer)
++ this._queue.unshift(newPeer)
++ this._drain()
++ return newPeer
++ }
++
+ addWebSeed (urlOrConn) {
+ if (this.destroyed) throw new Error('torrent is destroyed')
+
+@@ -1287,7 +1304,7 @@ export default class Torrent extends EventEmitter {
+ this._updateSelections()
+ }
+
+- _onWire (wire, addr) {
++ _onWire (wire, addr, key = addr) {
+ this._debug('got wire %s (%s)', wire._debugId, addr || 'Unknown')
+
+ this.wires.push(wire)
+@@ -1380,7 +1397,7 @@ export default class Torrent extends EventEmitter {
+
+ // Hook to allow user-defined `bittorrent-protocol` extensions
+ // More info: https://github.com/webtorrent/bittorrent-protocol#extension-api
+- this.emit('wire', wire, addr)
++ this.emit('wire', wire, addr, key)
+
+ if (this.ready) {
+ queueMicrotask(() => {
+@@ -2122,7 +2139,7 @@ export default class Torrent extends EventEmitter {
const utpServer = this.client._utpServer
peer.conn = utpServer ? utpServer.connect(opts.port, opts.host) : utp.connect(opts.port, opts.host)
} else {
- peer.conn = net.connect(opts)
-+ peer.conn = this.client._connect ? this.client._connect(opts) : net.connect(opts)
++ peer.conn = this.client._connect ? this.client._connect({ ...opts, key: peer.id }) : net.connect(opts)
}
const conn = peer.conn
+@@ -2143,7 +2160,7 @@ export default class Torrent extends EventEmitter {
+
+ // When connection closes, attempt reconnect after timeout (with exponential backoff)
+ conn.on('close', () => {
+- if (this.destroyed) return
++ if (this.destroyed || peer.twinPeerId) return
+
+ if (peer.retries >= RECONNECT_WAIT.length) {
+ if (this.client.utp) {
diff --git a/src/main/download/torrent/modules.d.ts b/src/main/download/torrent/modules.d.ts
index 8185cbe..9a7c789 100644
--- a/src/main/download/torrent/modules.d.ts
+++ b/src/main/download/torrent/modules.d.ts
@@ -17,8 +17,8 @@ declare module 'webtorrent' {
/** Bytes per second this client uploads at most. */
uploadLimit?: number
/** Added by Plexo's patch (patches/webtorrent+3.0.21.patch): opens every outgoing TCP peer
- * connection. */
- connect?: (options: { host: string; port: number }) => Socket
+ * connection. `key` is the peer's key in the swarm: its address, or a twin's own key. */
+ connect?: (options: { host: string; port: number; key: string }) => Socket
}
/** What webtorrent hands a store: the torrent's files, paths relative to `path`. */
@@ -75,6 +75,9 @@ declare module 'webtorrent' {
bitfield: { get(index: number): boolean }
destroy(options?: { destroyStore?: boolean }, callback?: (error?: Error) => void): void
addPeer(address: string): boolean
+ /** Added by Plexo's patch: dials `address` again under `key`, shaking hands as `peerId` (hex),
+ * so one peer can be reached over more than one network. Null if `key` is already there. */
+ addTwinPeer(address: string, key: string, peerId: string): object | null
}
export default class WebTorrent extends EventEmitter {
diff --git a/src/main/download/torrent/torrentTransfer.ts b/src/main/download/torrent/torrentTransfer.ts
index d5ca66b..7de22d4 100644
--- a/src/main/download/torrent/torrentTransfer.ts
+++ b/src/main/download/torrent/torrentTransfer.ts
@@ -20,6 +20,16 @@ const PEERS_PER_NETWORK = 30
// network has peers, can't reach the swarm. It keeps one dial going, to see when it can again.
const UNREACHABLE_AFTER_ATTEMPTS = 5
const UNREACHABLE_AFTER_MS = 60_000
+// A twin that failed (refused, dropped) is tried again on its network after this long.
+const TWIN_RETRY_MS = 30_000
+const TWIN_MARK = '#twin:'
+
+/** A twin's key in the swarm: the peer's address and the network it is dialled over. */
+const twinKey = (address: string, networkId: string): string => `${address}${TWIN_MARK}${networkId}`
+const twinNetwork = (key: string): string | null => {
+ const at = key.indexOf(TWIN_MARK)
+ return at < 0 ? null : key.slice(at + TWIN_MARK.length)
+}
const message = (error: unknown): string => (error instanceof Error ? error.message : String(error))
@@ -65,6 +75,14 @@ export class TorrentTransfer implements Transfer {
private reach = new Map<string, { dials: number; answeredAt: number }>()
/** Bytes of each piece not verified yet, by network. */
private unverified = new Map<number, Record<string, number>>()
+ /** This run's torrent from the moment it is added, for dialling twins before 'ready'. */
+ private engineTorrent: Torrent | null = null
+ /** Addresses this run dialled out to: only these listen there, so only these get twins. */
+ private dialledOut = new Set<string>()
+ /** The peer id each network's twins shake hands with, so a peer sees one client per network. */
+ private twinIds = new Map<string, string>()
+ /** By twin key: when it may be dialled again. */
+ private twinRetryAt = new Map<string, number>()
constructor(
private readonly runtime: TorrentTransferTarget,
@@ -99,6 +117,7 @@ export class TorrentTransfer implements Transfer {
this.host.scheduleUpdate()
}
}
+ this.dialTwins(now)
for (const peer of this.peers.values()) {
const status = peer.state.speedBytesPerSec > 0 ? 'receiving' : 'connected'
if (peer.state.status !== status) {
@@ -118,6 +137,8 @@ export class TorrentTransfer implements Transfer {
}
reset(): void {
+ this.dialledOut.clear()
+ this.twinRetryAt.clear()
this.chosen.clear()
this.dialling.clear()
this.reach.clear()
@@ -180,6 +201,7 @@ export class TorrentTransfer implements Transfer {
for (const peer of this.peers.values()) this.removePeer(peer)
this.peers.clear()
this.torrent = null
+ this.engineTorrent = null
this.ended = null
}
}
@@ -220,7 +242,10 @@ export class TorrentTransfer implements Transfer {
// Nothing until webtorrent is ready to be told which files (below), as on a change.
deselect: true
})
- torrent.on('wire', (wire: Wire, address: string) => this.onWire(wire, address))
+ this.engineTorrent = torrent
+ torrent.on('wire', (wire: Wire, address: string, key?: string) =>
+ this.onWire(wire, address, key)
+ )
torrent.on('verified', (index: number) => this.onVerified(index))
torrent.once('ready', () => {
this.torrent = torrent
@@ -244,7 +269,18 @@ export class TorrentTransfer implements Transfer {
}
/** webtorrent's `connect` hook: every outgoing peer comes through here. */
- private connect({ host: address, port }: { host: string; port: number }): Socket {
+ private connect({
+ host: address,
+ port,
+ key
+ }: {
+ host: string
+ port: number
+ key?: string
+ }): Socket {
+ // A twin goes over the network its key names, or nowhere.
+ const twinOn = key ? twinNetwork(key) : null
+ if (twinOn) return this.connectTwin(address, port, key!, twinOn)
// A peer named by hostname (a magnet's x.pe can be) has no family: no network takes it.
const family = isIP(address) as 0 | 4 | 6
const peersOn = (id: string): number =>
@@ -267,9 +303,77 @@ export class TorrentTransfer implements Transfer {
}
this.chosen.set(addressOf(address, port), networkId)
+ this.dialledOut.add(addressOf(address, port))
+ return this.dial(routes.get(networkId)!, port, networkId)
+ }
+
+ /** A twin's dial: over `networkId` only, if it is on and has a route to the peer. */
+ private connectTwin(address: string, port: number, key: string, networkId: string): Socket {
+ const family = isIP(address) as 0 | 4 | 6
+ const network = this.usable().find((entry) => entry.id === networkId)
+ const iface = network && this.host.networks.find(network.id)
+ const route = family && iface ? routesFor(iface, [{ address, family }])[0] : undefined
+ if (!route || network?.status !== 'on') {
+ const socket = new Socket()
+ process.nextTick(() => socket.destroy(new Error('Twin network cannot reach this peer')))
+ return socket
+ }
+ this.chosen.set(key, networkId)
+ return this.dial(route, port, networkId)
+ }
+
+ /**
+ * Dials a peer again over every other network that is on, each as a client of its own (see
+ * addTwinPeer in patches/webtorrent+3.0.21.patch): a swarm of one fast peer then fills more
+ * than one network. Only peers this run dialled out to, whose address is where they listen.
+ */
+ private dialTwins(now: number): void {
+ const torrent = this.engineTorrent
+ if (!torrent) return
+ const on = this.usable().filter((network) => network.status === 'on')
+ if (on.length < 2) return
+ const originals = new Map<string, string>()
+ for (const [wire, peer] of this.peers) {
+ const address = wire as Wire & { remoteAddress?: string; remotePort?: number }
+ if (!address.remoteAddress || !address.remotePort) continue
+ const key = addressOf(address.remoteAddress, address.remotePort)
+ if (this.dialledOut.has(key) && this.chosen.get(key) === peer.state.interfaceId) {
+ originals.set(key, peer.state.interfaceId)
+ }
+ }
+ for (const [address, networkId] of originals) {
+ for (const network of on) {
+ if (network.id === networkId) continue
+ const key = twinKey(address, network.id)
+ if ((this.twinRetryAt.get(key) ?? 0) > now) continue
+ this.twinRetryAt.set(key, now + TWIN_RETRY_MS)
+ torrent.addTwinPeer(address, key, this.twinIdFor(network.id))
+ }
+ }
+ }
+
+ /** A peer id for `networkId`'s twins: the client's prefix (its name and version), the rest its
+ * own. */
+ private twinIdFor(networkId: string): string {
+ let id = this.twinIds.get(networkId)
+ if (!id) {
+ const prefix = (this.client as WebTorrent & { peerId?: string })?.peerId?.slice(0, 16) ?? ''
+ const rest = Array.from({ length: (40 - prefix.length) / 2 }, () =>
+ Math.floor(Math.random() * 256)
+ .toString(16)
+ .padStart(2, '0')
+ ).join('')
+ id = prefix + rest
+ this.twinIds.set(networkId, id)
+ }
+ return id
+ }
+
+ /** Opens a socket to a peer over a network, counted as a dial on it until it settles. */
+ private dial(route: NetworkRoute, port: number, networkId: string): Socket {
this.dialling.set(networkId, (this.dialling.get(networkId) ?? 0) + 1)
this.reachOf(networkId).dials += 1
- const socket = connectRoute(routes.get(networkId)!, port)
+ const socket = connectRoute(route, port)
let settled = false
const settle = (): void => {
if (settled) return
@@ -313,8 +417,9 @@ export class TorrentTransfer implements Transfer {
this.limit(socket, network.id)
}
- private onWire(wire: Wire, address: string): void {
- const networkId = this.chosen.get(address)
+ private onWire(wire: Wire, address: string, key = address): void {
+ // A twin is filed under its own key; every other peer under its address.
+ const networkId = this.chosen.get(twinNetwork(key) ? key : address)
const network = this.runtime.state.networks.find((entry) => entry.id === networkId)
if (!network) {
wire.destroy()