Speed up downloads by combining multiple network connections in parallel
TypeScript bandwidth-aggregationcross-platformdesktop-appdownloaderdownloadmanagerlinuxmacosmulticonnectionparallel-downloadswindows
import { expect, test } from '@playwright/test'
import { createServer, type IncomingMessage } from 'node:http'
import {
connect as netConnect,
createServer as createTcpServer,
type AddressInfo,
type Socket
} from 'node:net'
import type { NetworkInterfaceInfo, ProxyConfig } from '../src/shared/types'
import { connectThrough, StreamConnection } from '../src/main/network/routes'
const iface: NetworkInterfaceInfo = {
id: 'wifi',
device: 'wifi',
displayName: 'Wi-Fi',
kind: 'wifi',
addresses: [
{ address: '192.0.2.2', family: 4 },
{ address: '2001:db8::2', family: 6 }
]
}
const loopback: NetworkInterfaceInfo = {
...iface,
addresses: [{ address: '127.0.0.1', family: 4 }]
}
async function body(res: IncomingMessage): Promise<string> {
let text = ''
for await (const chunk of res) text += chunk
return text
}
test('AAAA-only hostname connects over IPv6 and keeps its HTTP Host header @smoke', async () => {
let seenHost = ''
const server = createServer((req, res) => {
seenHost = req.headers.host ?? ''
res.end('ok')
})
await new Promise<void>((resolve) => server.listen(0, '::1', resolve))
const connection = new StreamConnection(
() => ({ ...iface, addresses: [{ address: '::1', family: 6 }] }),
{ timeoutMs: 2000, resolveHost: async () => [{ address: '::1', family: 6 }] }
)
try {
const port = (server.address() as AddressInfo).port
const { res } = await connection.request(new URL(`http://ipv6.example.test:${port}/file`), {})
expect(await body(res)).toBe('ok')
expect(seenHost).toBe(`ipv6.example.test:${port}`)
} finally {
connection.close()
await new Promise<void>((resolve) => server.close(() => resolve()))
}
})
test('a working route gets time to answer after an earlier route fails @smoke', async () => {
const server = createServer((_req, res) => {
setTimeout(() => res.end('ipv6'), 1800)
})
await new Promise<void>((resolve) => server.listen(0, '::1', resolve))
const connection = new StreamConnection(
() => ({
...iface,
addresses: [
{ address: '127.0.0.1', family: 4 },
{ address: '::1', family: 6 }
]
}),
{
timeoutMs: 3000,
resolveHost: async () => [
{ address: '127.0.0.1', family: 4 },
{ address: '::1', family: 6 },
{ address: '127.0.0.2', family: 4 }
]
}
)
try {
const port = (server.address() as AddressInfo).port
const { res } = await connection.request(new URL(`http://dual.example.test:${port}/`), {})
expect(await body(res)).toBe('ipv6')
} finally {
connection.close()
await new Promise<void>((resolve) => server.close(() => resolve()))
}
})
test('DNS resolution obeys the request deadline @smoke', async () => {
const connection = new StreamConnection(() => iface, {
timeoutMs: 100,
resolveHost: () => new Promise(() => {})
})
await expect(
connection.request(new URL('http://unresolved.example.test/file'), {})
).rejects.toThrow('Could not resolve the download host in time')
connection.close()
})
test('a kept-alive connection the server dropped is replaced without failing the request @smoke', async () => {
let requests = 0
let connections = 0
// The second request lands on the reused connection and is dropped unanswered, as happens when
// a server times out an idle connection just as the next request goes out.
const server = createServer((req, res) => {
if (++requests === 2) req.socket.destroy()
else res.end(`answer ${requests}`)
})
server.on('connection', () => connections++)
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const connection = new StreamConnection(() => loopback, { timeoutMs: 2000 })
try {
const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)
expect(await body((await connection.request(url, {})).res)).toBe('answer 1')
expect(await body((await connection.request(url, {})).res)).toBe('answer 3')
expect(connections).toBe(2)
} finally {
connection.close()
await new Promise<void>((resolve) => server.close(() => resolve()))
}
})
test('a request abandoned while still connecting gives up the connection with it @smoke', async () => {
// DNS that never answers stands in for any connect that hangs: the abort has to end it, not the
// deadline.
const connection = new StreamConnection(() => iface, {
timeoutMs: 5000,
resolveHost: () => new Promise(() => {})
})
const abort = new AbortController()
const started = Date.now()
setTimeout(() => abort.abort(), 100)
await expect(
connection.request(new URL('http://slow.example.test/'), {}, abort.signal)
).rejects.toMatchObject({ name: 'AbortError' })
expect(Date.now() - started).toBeLessThan(1000)
connection.close()
})
test('a network that comes back with a new address is reached at that one @smoke', async () => {
const server = createServer((_req, res) => res.end('ok'))
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
// Gone, and then back: every new socket asks for the network as it is now.
let current: NetworkInterfaceInfo | undefined
const connection = new StreamConnection(() => current, { timeoutMs: 2000 })
try {
const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)
await expect(connection.request(url, {})).rejects.toThrow(/not connected/)
current = loopback
const { res } = await connection.request(url, {})
expect(await body(res)).toBe('ok')
} finally {
connection.close()
await new Promise<void>((resolve) => server.close(() => resolve()))
}
})
test('reconnect drops its sockets, and the next request opens a new one @smoke', async () => {
let connections = 0
const server = createServer((_req, res) => res.end('ok'))
server.on('connection', () => connections++)
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const connection = new StreamConnection(() => loopback, { timeoutMs: 2000 })
try {
const url = new URL(`http://127.0.0.1:${(server.address() as AddressInfo).port}/`)
expect(await body((await connection.request(url, {})).res)).toBe('ok')
expect(await body((await connection.request(url, {})).res)).toBe('ok')
expect(connections, 'kept alive until then').toBe(1)
connection.reconnect()
expect(await body((await connection.request(url, {})).res)).toBe('ok')
expect(connections).toBe(2)
} finally {
connection.close()
await new Promise<void>((resolve) => server.close(() => resolve()))
}
})
test('invalid proxy details fail and never fall back to direct download @smoke', async () => {
let directServerHit = false
const targetServer = createServer((_req, res) => {
directServerHit = true
res.end('direct hit')
})
await new Promise<void>((resolve) => targetServer.listen(0, '127.0.0.1', resolve))
const targetPort = (targetServer.address() as AddressInfo).port
// Proxy pointing to an unused port that will fail connection
const proxyConfig: ProxyConfig = {
enabled: true,
type: 'http',
host: '127.0.0.1',
port: 59998
}
const connection = new StreamConnection(() => loopback, {
timeoutMs: 1500,
proxy: () => proxyConfig
})
try {
const url = new URL(`http://127.0.0.1:${targetPort}/download`)
await expect(connection.request(url, {})).rejects.toThrow()
// CRITICAL: The direct server must NOT be contacted! Direct fallback is strictly forbidden!
expect(directServerHit).toBe(false)
} finally {
connection.close()
await new Promise<void>((resolve) => targetServer.close(() => resolve()))
}
})
test('proxy authentication failure fails and never falls back to direct download @smoke', async () => {
let targetServerHit = false
const targetServer = createServer((_req, res) => {
targetServerHit = true
res.end('target payload')
})
await new Promise<void>((resolve) => targetServer.listen(0, '127.0.0.1', resolve))
const targetPort = (targetServer.address() as AddressInfo).port
// Proxy requiring auth and rejecting wrong credentials with HTTP 407
const proxyServer = createServer()
proxyServer.on('connect', (_req, clientSocket) => {
clientSocket.write('HTTP/1.1 407 Proxy Authentication Required\r\n\r\n')
clientSocket.end()
})
await new Promise<void>((resolve) => proxyServer.listen(0, '127.0.0.1', resolve))
const proxyPort = (proxyServer.address() as AddressInfo).port
const proxyConfig: ProxyConfig = {
enabled: true,
type: 'http',
host: '127.0.0.1',
port: proxyPort,
username: 'invalidUser',
password: 'wrongPassword'
}
const connection = new StreamConnection(() => loopback, {
timeoutMs: 2000,
proxy: () => proxyConfig
})
try {
const url = new URL(`http://127.0.0.1:${targetPort}/download`)
await expect(connection.request(url, {})).rejects.toThrow(
/Proxy authentication required|HTTP 407/i
)
expect(targetServerHit).toBe(false)
} finally {
connection.close()
await new Promise<void>((resolve) => proxyServer.close(() => resolve()))
await new Promise<void>((resolve) => targetServer.close(() => resolve()))
}
})
test('valid HTTP proxy tunnels requests through the proxy @smoke', async () => {
let proxyConnectCount = 0
const targetServer = createServer((_req, res) => {
res.end('proxied data')
})
await new Promise<void>((resolve) => targetServer.listen(0, '127.0.0.1', resolve))
const targetPort = (targetServer.address() as AddressInfo).port
const proxyServer = createServer()
proxyServer.on('connect', (req, clientSocket, head) => {
proxyConnectCount++
const [tHost, tPort] = req.url!.split(':')
const targetSocket = netConnect(Number(tPort), tHost, () => {
clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
if (head && head.length > 0) targetSocket.write(head)
targetSocket.pipe(clientSocket)
clientSocket.pipe(targetSocket)
})
targetSocket.on('error', () => clientSocket.destroy())
clientSocket.on('error', () => targetSocket.destroy())
})
await new Promise<void>((resolve) => proxyServer.listen(0, '127.0.0.1', resolve))
const proxyPort = (proxyServer.address() as AddressInfo).port
const proxyConfig: ProxyConfig = {
enabled: true,
type: 'http',
host: '127.0.0.1',
port: proxyPort
}
const connection = new StreamConnection(() => loopback, {
timeoutMs: 2000,
proxy: () => proxyConfig
})
try {
const url = new URL(`http://127.0.0.1:${targetPort}/download`)
const { res } = await connection.request(url, {})
expect(await body(res)).toBe('proxied data')
expect(proxyConnectCount).toBe(1)
} finally {
connection.close()
await new Promise<void>((resolve) => proxyServer.close(() => resolve()))
await new Promise<void>((resolve) => targetServer.close(() => resolve()))
}
})
// A news server speaks first, so bytes that arrive with the proxy's answer must not be lost.
for (const type of ['http', 'socks5'] as const) {
test(`a non-HTTP connection (NNTP) goes through the network’s ${type} proxy @smoke`, async () => {
const tunneled: string[] = []
const newsServer = createTcpServer((socket) => socket.end('200 news ready\r\n'))
await new Promise<void>((resolve) => newsServer.listen(0, '127.0.0.1', resolve))
const newsPort = (newsServer.address() as AddressInfo).port
const tunnel = (client: Socket, target: string, established: Buffer): void => {
tunneled.push(target)
const [tHost, tPort] = target.split(':')
const targetSocket = netConnect(Number(tPort), tHost, () => {
client.write(established)
targetSocket.pipe(client)
client.pipe(targetSocket)
})
targetSocket.on('error', () => client.destroy())
client.on('error', () => targetSocket.destroy())
}
const proxyServer =
type === 'http'
? createServer().on('connect', (req, client: Socket) =>
tunnel(client, req.url!, Buffer.from('HTTP/1.1 200 Connection Established\r\n\r\n'))
)
: // SOCKS5 without auth, IPv4 targets only: greeting, then CONNECT.
createTcpServer((client) => {
client.once('data', () => {
client.write(Buffer.from([0x05, 0x00]))
client.once('data', (request: Buffer) => {
const target = `${[...request.subarray(4, 8)].join('.')}:${request.readUInt16BE(8)}`
tunnel(client, target, Buffer.from([0x05, 0x00, 0x00, 0x01, 0, 0, 0, 0, 0, 0]))
})
})
})
await new Promise<void>((resolve) => proxyServer.listen(0, '127.0.0.1', resolve))
const proxyPort = (proxyServer.address() as AddressInfo).port
try {
const socket = await connectThrough(
loopback,
'127.0.0.1',
newsPort,
false,
2000,
new AbortController().signal,
{ enabled: true, type, host: '127.0.0.1', port: proxyPort }
)
let greeting = ''
for await (const chunk of socket) greeting += chunk
expect(greeting).toBe('200 news ready\r\n')
expect(tunneled).toEqual([`127.0.0.1:${newsPort}`])
} finally {
await new Promise<void>((resolve) => proxyServer.close(() => resolve()))
await new Promise<void>((resolve) => newsServer.close(() => resolve()))
}
})
}
test('proxy settings update dynamically on reconnect @smoke', async () => {
const targetServer = createServer((_req, res) => {
res.end('dynamic ok')
})
await new Promise<void>((resolve) => targetServer.listen(0, '127.0.0.1', resolve))
const targetPort = (targetServer.address() as AddressInfo).port
const proxyServer = createServer()
proxyServer.on('connect', (req, clientSocket, head) => {
const [tHost, tPort] = req.url!.split(':')
const targetSocket = netConnect(Number(tPort), tHost, () => {
clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
if (head && head.length > 0) targetSocket.write(head)
targetSocket.pipe(clientSocket)
clientSocket.pipe(targetSocket)
})
targetSocket.on('error', () => clientSocket.destroy())
clientSocket.on('error', () => targetSocket.destroy())
})
await new Promise<void>((resolve) => proxyServer.listen(0, '127.0.0.1', resolve))
const proxyPort = (proxyServer.address() as AddressInfo).port
// Initially wrong proxy
let activeProxy: ProxyConfig = {
enabled: true,
type: 'http',
host: '127.0.0.1',
port: 59997
}
const connection = new StreamConnection(() => loopback, {
timeoutMs: 1500,
proxy: () => activeProxy
})
try {
const url = new URL(`http://127.0.0.1:${targetPort}/download`)
// Fails with invalid proxy
await expect(connection.request(url, {})).rejects.toThrow()
// Switch to working proxy and reconnect
activeProxy = {
enabled: true,
type: 'http',
host: '127.0.0.1',
port: proxyPort
}
connection.reconnect()
// Succeeds through working proxy
const { res } = await connection.request(url, {})
expect(await body(res)).toBe('dynamic ok')
} finally {
connection.close()
await new Promise<void>((resolve) => proxyServer.close(() => resolve()))
await new Promise<void>((resolve) => targetServer.close(() => resolve()))
}
})