Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: implement STUN server racing to bypass werift's single-server limitation

0xken15b035f7cbc290
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index 185cf6656..ae470d9db 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -27,6 +27,7 @@ import {
type TerminalP2pData,
type TerminalP2pPolicy,
} from './terminalP2p.js'
+import { warmStunUrls } from './stunSelect.js'
const CONNECT_TIMEOUT_MS = 15_000
const LINGER_MS = 30_000
@@ -218,6 +219,12 @@ export class RemoteRelayPool {
if (frame.type === 'connected' && payload?.machineId === machineId) {
selected = true
entry.p2pPolicy = p2pPolicy(payload.p2p)
+ // Earliest instant the url list exists. startP2p() fires one round trip later on
+ // e2e_welcome, so the cache is usually still cold — what actually pays off is the
+ // in-flight dedupe: begin() joins THIS race instead of starting a second one, and its
+ // cost hides behind the welcome round trip. Nothing here is load-bearing for
+ // correctness; drop it and the only change is a slightly later first terminal.
+ if (entry.p2pPolicy) warmStunUrls(entry.p2pPolicy.stunUrls)
try { ws.send(JSON.stringify(crypto.helloFrame())) } catch { /* the close handler below rejects */ }
return
}
diff --git a/cli/src/lib/stunSelect.spec.ts b/cli/src/lib/stunSelect.spec.ts
new file mode 100644
index 000000000..ea6eb48df
--- /dev/null
+++ b/cli/src/lib/stunSelect.spec.ts
@@ -0,0 +1,253 @@
+import dgram from 'node:dgram'
+import { describe, expect, it, vi } from 'vitest'
+import {
+ buildBindingRequest,
+ createStunSelector,
+ parseBindingResponse,
+ parseStunUrl,
+ type StunProbe,
+} from './stunSelect.js'
+
+const MAGIC = 0x2112a442
+
+/** Builds a success response for `tid`, optionally with a bogus attribute set. */
+function successResponse(tid: Buffer, opts: {
+ type?: number
+ attrType?: number
+ family?: number
+ attrLen?: number
+ /** Length written INTO the attribute header, independent of how many bytes actually follow. */
+ declaredAttrLen?: number
+ declaredLength?: number
+ omitAttribute?: boolean
+} = {}): Buffer {
+ const attrLen = opts.attrLen ?? (opts.family === 0x02 ? 20 : 8)
+ const body = opts.omitAttribute ? Buffer.alloc(0) : Buffer.alloc(4 + attrLen)
+ if (!opts.omitAttribute) {
+ body.writeUInt16BE(opts.attrType ?? 0x0020, 0)
+ body.writeUInt16BE(opts.declaredAttrLen ?? attrLen, 2)
+ body.writeUInt8(0, 4)
+ body.writeUInt8(opts.family ?? 0x01, 5)
+ body.writeUInt16BE(0x1234 ^ (MAGIC >>> 16), 6)
+ body.writeUInt32BE(0x0a000001 ^ MAGIC, 8)
+ }
+ const buf = Buffer.alloc(20 + body.length)
+ buf.writeUInt16BE(opts.type ?? 0x0101, 0)
+ buf.writeUInt16BE(opts.declaredLength ?? body.length, 2)
+ buf.writeUInt32BE(MAGIC, 4)
+ tid.copy(buf, 8)
+ body.copy(buf, 20)
+ return buf
+}
+
+describe('STUN url parsing', () => {
+ it('matches werift: default ports, explicit ports, brackets, and query strings', () => {
+ expect(parseStunUrl('stun:stun.example.com')).toEqual({ scheme: 'stun', host: 'stun.example.com', port: 3478 })
+ expect(parseStunUrl('stun:stun.example.com:19302')).toEqual({ scheme: 'stun', host: 'stun.example.com', port: 19302 })
+ expect(parseStunUrl('stuns:stun.example.com')).toEqual({ scheme: 'stuns', host: 'stun.example.com', port: 5349 })
+ expect(parseStunUrl('stun://stun.example.com:1234')).toEqual({ scheme: 'stun', host: 'stun.example.com', port: 1234 })
+ expect(parseStunUrl('stun:stun.example.com:3478?transport=udp')?.port).toBe(3478)
+ expect(parseStunUrl('stun:[::1]:3478')).toEqual({ scheme: 'stun', host: '::1', port: 3478 })
+ expect(parseStunUrl(' STUN:Stun.Example.com ')?.host).toBe('Stun.Example.com')
+ })
+
+ it('rejects junk without throwing', () => {
+ for (const bad of ['', 'stun:', 'turn:example.com', 'https://example.com', 'stun:[::1', 'nonsense']) {
+ expect(parseStunUrl(bad)).toBeNull()
+ }
+ })
+})
+
+describe('STUN binding request/response codec', () => {
+ it('builds a well-formed 20-byte request with a fresh transaction id each time', () => {
+ const a = buildBindingRequest()
+ const b = buildBindingRequest()
+ expect(a.packet.length).toBe(20)
+ expect(a.packet.readUInt16BE(0)).toBe(0x0001)
+ expect(a.packet.readUInt16BE(2)).toBe(0)
+ expect(a.packet.readUInt32BE(4)).toBe(MAGIC)
+ expect(a.tid.length).toBe(12)
+ expect(a.packet.subarray(8, 20)).toEqual(a.tid)
+ expect(a.tid.equals(b.tid)).toBe(false)
+ })
+
+ it('accepts a success response carrying a mapped address', () => {
+ const { tid } = buildBindingRequest()
+ expect(parseBindingResponse(successResponse(tid), tid)).toBe(true)
+ expect(parseBindingResponse(successResponse(tid, { attrType: 0x0001 }), tid)).toBe(true)
+ expect(parseBindingResponse(successResponse(tid, { family: 0x02 }), tid)).toBe(true)
+ })
+
+ it('rejects every malformed or foreign response, and never throws', () => {
+ const { tid } = buildBindingRequest()
+ const other = buildBindingRequest().tid
+ const cases: Array<[string, Buffer]> = [
+ ['transaction id from another probe', successResponse(other)],
+ ['error class 0x0111', successResponse(tid, { type: 0x0111 })],
+ ['truncated header', successResponse(tid).subarray(0, 12)],
+ ['declared length disagrees with buffer', successResponse(tid, { declaredLength: 99 })],
+ ['attribute claims more bytes than the buffer holds', successResponse(tid, { attrLen: 8, declaredAttrLen: 20 })],
+ ['success with no mapped address', successResponse(tid, { omitAttribute: true })],
+ ['mapped address with a length that does not match its family', successResponse(tid, { attrLen: 12 })],
+ ['empty buffer', Buffer.alloc(0)],
+ ['random bytes', Buffer.from([0xff, 0x00, 0x13, 0x37])],
+ ]
+ for (const [name, buf] of cases) {
+ expect(parseBindingResponse(buf, tid), name).toBe(false)
+ }
+ })
+
+ it('rejects a response whose magic cookie is wrong', () => {
+ const { tid } = buildBindingRequest()
+ const buf = successResponse(tid)
+ buf.writeUInt32BE(0xdeadbeef, 4)
+ expect(parseBindingResponse(buf, tid)).toBe(false)
+ })
+})
+
+describe('STUN selection race', () => {
+ const URLS = ['stun:a.example:3478', 'stun:b.example:3478', 'stun:c.example:3478']
+
+ /** Answers after `delays[url]` ms; a url absent from the map never answers at all. */
+ function fakeProbe(delays: Record<string, number>, address = '10.0.0.1'): StunProbe & { calls: string[] } {
+ const calls: string[] = []
+ const probe = ((url, _timeoutMs, signal) => {
+ calls.push(url)
+ const delay = delays[url]
+ if (delay === undefined) return new Promise<string | null>(() => { /* never settles */ })
+ return new Promise<string | null>((resolve) => {
+ const timer = setTimeout(() => resolve(address), delay)
+ signal.addEventListener('abort', () => { clearTimeout(timer); resolve(null) }, { once: true })
+ })
+ }) as StunProbe & { calls: string[] }
+ probe.calls = calls
+ return probe
+ }
+
+ it('puts the fastest responder first and keeps the losers in their original order', async () => {
+ const probe = fakeProbe({ [URLS[2]!]: 10, [URLS[0]!]: 30 })
+ const select = createStunSelector({ probe })
+ expect(await select(URLS)).toEqual(['stun:10.0.0.1:3478', URLS[0], URLS[1]])
+ })
+
+ it('returns as soon as one answers, aborting the probes still outstanding', async () => {
+ let aborted = false
+ const probe: StunProbe = (url, _timeoutMs, signal) => {
+ if (url === URLS[0]) return Promise.resolve('10.0.0.1')
+ signal.addEventListener('abort', () => { aborted = true }, { once: true })
+ return new Promise(() => { /* would hang forever without the abort */ })
+ }
+ const select = createStunSelector({ probe })
+ expect((await select(URLS))[0]).toBe('stun:10.0.0.1:3478')
+ expect(aborted).toBe(true)
+ })
+
+ it('is never worse than today: when every server fails it returns the list untouched', async () => {
+ const probe = vi.fn<StunProbe>(async () => null)
+ const select = createStunSelector({ probe })
+ expect(await select(URLS)).toEqual(URLS)
+ })
+
+ it('does not probe at all for a list of zero or one url', async () => {
+ const probe = vi.fn<StunProbe>(async () => '10.0.0.1')
+ const select = createStunSelector({ probe })
+ expect(await select([])).toEqual([])
+ expect(await select(['stun:only.example:3478'])).toEqual(['stun:only.example:3478'])
+ expect(probe).not.toHaveBeenCalled()
+ })
+
+ it('never rejects, whether a probe throws synchronously or rejects', async () => {
+ const thrower: StunProbe = (url) => {
+ if (url === URLS[0]) throw new Error('sync boom')
+ if (url === URLS[1]) return Promise.reject(new Error('async boom'))
+ return Promise.resolve(null)
+ }
+ const select = createStunSelector({ probe: thrower })
+ await expect(select(URLS)).resolves.toEqual(URLS)
+ })
+
+ it('leaves a winner that is already an IP literal alone, and never pins the losers', async () => {
+ const urls = ['stun:198.51.100.7:3478', 'stun:b.example:3478']
+ const probe: StunProbe = async (url) => (url === urls[0] ? '198.51.100.7' : null)
+ const select = createStunSelector({ probe })
+ expect(await select(urls)).toEqual(urls)
+ })
+
+ it('joins an in-flight race instead of starting a second one', async () => {
+ const probe = fakeProbe({ [URLS[1]!]: 5 })
+ const select = createStunSelector({ probe })
+ const [first, second] = await Promise.all([select(URLS), select(URLS)])
+ expect(first).toEqual(second)
+ expect(probe.calls).toEqual(URLS) // one call per url, total
+ })
+
+ it('serves a hit from cache until the positive TTL expires', async () => {
+ const probe = fakeProbe({ [URLS[0]!]: 1 })
+ let clock = 1_000
+ const select = createStunSelector({ probe, now: () => clock, ttlMs: 5_000, negativeTtlMs: 100 })
+ await select(URLS)
+ expect(probe.calls.length).toBe(3)
+ clock += 4_999
+ await select(URLS)
+ expect(probe.calls.length).toBe(3)
+ clock += 2
+ await select(URLS)
+ expect(probe.calls.length).toBe(6)
+ })
+
+ it('expires a total failure on the shorter negative TTL', async () => {
+ const probe = vi.fn<StunProbe>(async () => null)
+ let clock = 1_000
+ const select = createStunSelector({ probe, now: () => clock, ttlMs: 5_000, negativeTtlMs: 100 })
+ await select(URLS)
+ expect(probe).toHaveBeenCalledTimes(3)
+ clock += 99
+ await select(URLS)
+ expect(probe).toHaveBeenCalledTimes(3)
+ clock += 2 // past the negative TTL but nowhere near the positive one
+ await select(URLS)
+ expect(probe).toHaveBeenCalledTimes(6)
+ })
+
+ it('keys the cache on the whole list, so a different list is decided on its own merits', async () => {
+ const probe = fakeProbe({ [URLS[0]!]: 1, 'stun:d.example:3478': 1 })
+ const select = createStunSelector({ probe })
+ await select(URLS)
+ await select(['stun:d.example:3478', 'stun:e.example:3478'])
+ expect(probe.calls).toContain('stun:d.example:3478')
+ })
+})
+
+describe('STUN probe over a real socket', () => {
+ it('skips a closed port and picks the server that actually answers', async () => {
+ const server = dgram.createSocket('udp4')
+ await new Promise<void>((resolve) => server.bind(0, '127.0.0.1', resolve))
+ server.on('message', (msg, rinfo) => {
+ const tid = msg.subarray(8, 20)
+ server.send(successResponse(tid), rinfo.port, rinfo.address)
+ })
+ const livePort = server.address().port
+
+ // Take a port and release it, so nothing is listening there: on loopback that surfaces as an
+ // ICMP-unreachable -> dgram 'error' EVENT, which is the path that crashes the daemon if unhandled.
+ const scratch = dgram.createSocket('udp4')
+ await new Promise<void>((resolve) => scratch.bind(0, '127.0.0.1', resolve))
+ const deadPort = scratch.address().port
+ await new Promise<void>((resolve) => scratch.close(resolve))
+
+ try {
+ const select = createStunSelector({ probeTimeoutMs: 1_000 })
+ const ordered = await select([`stun:127.0.0.1:${deadPort}`, `stun:127.0.0.1:${livePort}`])
+ expect(ordered[0]).toBe(`stun:127.0.0.1:${livePort}`)
+ } finally {
+ await new Promise<void>((resolve) => server.close(resolve))
+ }
+ }, 10_000)
+
+ it('resolves to the untouched list when nothing answers, without leaking the process', async () => {
+ const select = createStunSelector({ probeTimeoutMs: 300 })
+ // TEST-NET-3, guaranteed unroutable, so both probes hit the deadline rather than any error path.
+ const urls = ['stun:203.0.113.1:3478', 'stun:203.0.113.2:3478']
+ expect(await select(urls)).toEqual(urls)
+ }, 10_000)
+})
diff --git a/cli/src/lib/stunSelect.ts b/cli/src/lib/stunSelect.ts
new file mode 100644
index 000000000..f4dcc166c
--- /dev/null
+++ b/cli/src/lib/stunSelect.ts
@@ -0,0 +1,284 @@
+/**
+ * Picks ONE reachable STUN server out of the list the backend hands us, by racing them all at once
+ * with a raw RFC 5389 Binding Request and putting the first one that answers at the front.
+ *
+ * This exists because werift does not do STUN failover and never did. `parseIceServers()`
+ * (werift/lib/webrtc/src/utils.js:96) folds the whole `iceServers` array into a SINGLE `stunServer`
+ * — `if (!options.stunServer && parsed.kind === "stun") options.stunServer = parsed.address` — so the
+ * first `stun:` url wins and every url behind it is dropped without a word. Measured on werift 0.24.4
+ * against a blackholed first entry (198.51.100.1) with two live servers behind it: ZERO srflx
+ * candidates, and gathering stalled 5.0s on werift's gather timeout instead of finishing in ~60ms.
+ * Both halves hurt. We do not trickle ICE (nothing subscribes to onIceCandidate; the candidates ride
+ * in the SDP because `setLocalDescription` awaits `gatherCandidates()`), so that 5s is paid on the
+ * initiator AND again on the responder, inside one 10s TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS budget. A
+ * single dead STUN server therefore does not merely lose srflx — it blows the whole negotiation.
+ *
+ * The peers do NOT have to agree on a server: a srflx candidate answers "what is my own public
+ * address", so each side may ask whoever it likes. That is why nothing here touches the wire format —
+ * `p2p_offer` keeps carrying the raw policy list and the responder races it independently.
+ *
+ * Two properties are load-bearing and must survive any edit:
+ * - `select()` NEVER rejects. It runs inside a long-lived daemon; an unhandled rejection here kills
+ * the process. Every failure path degrades to "return the list exactly as given", which is
+ * byte-for-byte today's behaviour — the cheapest possible proof this can never be worse.
+ * - A list of 0 or 1 urls is returned untouched with no probe at all. Production currently ships a
+ * single STUN url, so this module is provably a no-op there until the backend ships a longer one.
+ */
+import { randomBytes } from 'node:crypto'
+import dgram from 'node:dgram'
+import { lookup } from 'node:dns'
+import { isIP } from 'node:net'
+
+const STUN_MAGIC_COOKIE = 0x2112a442
+const STUN_BINDING_REQUEST = 0x0001
+const STUN_BINDING_SUCCESS = 0x0101
+const STUN_ATTR_MAPPED_ADDRESS = 0x0001
+const STUN_ATTR_XOR_MAPPED_ADDRESS = 0x0020
+const STUN_HEADER_BYTES = 20
+
+/** Measured RTT to cloudflare/google STUN is ~117ms, so this is ~3.4x headroom. It is only ever paid
+ * in full when EVERY server is dead — i.e. exactly when p2p was doomed anyway. */
+const PROBE_TIMEOUT_MS = 400
+/** One retransmit with the same transaction id absorbs a single lost datagram inside that budget. */
+const PROBE_RETRANSMIT_MS = 150
+const POSITIVE_TTL_MS = 5 * 60_000
+/** Short, so a laptop coming off a captive portal heals on its own without any invalidation hook. */
+const NEGATIVE_TTL_MS = 30_000
+
+/** Resolves to the winner's IPv4 address, or null if this server did not answer. Never rejects. */
+export type StunProbe = (url: string, timeoutMs: number, signal: AbortSignal) => Promise<string | null>
+
+export type StunSelector = (urls: string[]) => Promise<string[]>
+
+export interface StunSelectorOptions {
+ probe?: StunProbe
+ now?: () => number
+ ttlMs?: number
+ negativeTtlMs?: number
+ probeTimeoutMs?: number
+}
+
+interface StunTarget {
+ scheme: 'stun' | 'stuns'
+ host: string
+ port: number
+}
+
+interface RaceResult {
+ ordered: string[]
+ /** false = every server failed, so `ordered` IS the input list and the result is cached briefly. */
+ ok: boolean
+}
+
+interface CacheEntry {
+ ordered: string[]
+ ok: boolean
+ at: number
+}
+
+/** Mirrors `parseIceServerUrl`/`parseAddress` in werift/lib/webrtc/src/utils.js:133-210 on purpose:
+ * disagreeing with werift about which host:port a url means would make the whole race meaningless. */
+export function parseStunUrl(url: string): StunTarget | null {
+ const matched = /^(stun|stuns):(.+)$/i.exec(url.trim())
+ if (!matched) return null
+ const scheme = matched[1]!.toLowerCase() as 'stun' | 'stuns'
+ const fallbackPort = scheme === 'stuns' ? 5349 : 3478
+ const [rest] = matched[2]!.split('?', 1)
+ const authority = rest!.startsWith('//') ? rest!.slice(2) : rest!
+ if (!authority) return null
+ if (authority.startsWith('[')) {
+ const closing = authority.indexOf(']')
+ if (closing === -1) return null
+ return { scheme, host: authority.slice(1, closing), port: parsePort(authority.slice(closing + 1), fallbackPort) }
+ }
+ const firstColon = authority.indexOf(':')
+ const lastColon = authority.lastIndexOf(':')
+ if (firstColon !== -1 && firstColon === lastColon) {
+ return { scheme, host: authority.slice(0, firstColon), port: parsePort(authority.slice(firstColon + 1), fallbackPort) }
+ }
+ return { scheme, host: authority, port: fallbackPort }
+}
+
+function parsePort(value: string, fallbackPort: number): number {
+ const port = Number.parseInt(value.startsWith(':') ? value.slice(1) : value, 10)
+ return Number.isFinite(port) ? port : fallbackPort
+}
+
+/** Exported so the wire format can be unit-tested without a socket. */
+export function buildBindingRequest(): { packet: Buffer; tid: Buffer } {
+ const packet = Buffer.alloc(STUN_HEADER_BYTES)
+ packet.writeUInt16BE(STUN_BINDING_REQUEST, 0)
+ packet.writeUInt16BE(0, 2)
+ packet.writeUInt32BE(STUN_MAGIC_COOKIE, 4)
+ // Crypto-random, not counter-based: an off-path spoofer must not be able to guess the id and win
+ // the race with a forged success response pointing us at a server that never answered.
+ const tid = randomBytes(12)
+ tid.copy(packet, 8)
+ return { packet, tid }
+}
+
+/**
+ * True only for a success response that carries a well-formed mapped address. That is deliberately
+ * the same bar werift's own gathering applies: `serverReflexiveCandidate`
+ * (werift/lib/ice/src/iceBase.js:271-288) reads XOR-MAPPED-ADDRESS unconditionally and throws without
+ * it, so "answered" alone would be a weaker predicate than "werift will succeed here".
+ *
+ * The mapped value itself is discarded — our source port differs from werift's by construction; all
+ * we are establishing is reachability. Malformed input returns false and never throws.
+ */
+export function parseBindingResponse(buf: Buffer, tid: Buffer): boolean {
+ if (buf.length < STUN_HEADER_BYTES) return false
+ if (buf.readUInt16BE(0) !== STUN_BINDING_SUCCESS) return false // also rejects 0x0111 error responses
+ if (buf.readUInt32BE(4) !== STUN_MAGIC_COOKIE) return false
+ if (!buf.subarray(8, STUN_HEADER_BYTES).equals(tid)) return false
+ if (buf.readUInt16BE(2) !== buf.length - STUN_HEADER_BYTES) return false
+ let off = STUN_HEADER_BYTES
+ while (off + 4 <= buf.length) {
+ const type = buf.readUInt16BE(off)
+ const len = buf.readUInt16BE(off + 2)
+ if (off + 4 + len > buf.length) return false
+ if (type === STUN_ATTR_XOR_MAPPED_ADDRESS || type === STUN_ATTR_MAPPED_ADDRESS) {
+ const family = buf.readUInt8(off + 5)
+ if ((family === 0x01 && len === 8) || (family === 0x02 && len === 20)) return true
+ }
+ off += 4 + len + ((4 - (len % 4)) % 4)
+ }
+ return false
+}
+
+/** werift resolves STUN hosts as IPv4 (ice/src/stun/transaction.js:18-24) and only ever gathers srflx
+ * from IPv4 host candidates (ice/src/ice.js:861), so probing anything else would measure a path it
+ * cannot use. Note stun.cloudflare.com is dual-stack, so an unqualified lookup can hand back AAAA. */
+function resolveIpv4(host: string, done: (address: string | null) => void): void {
+ const literal = isIP(host)
+ if (literal === 4) { done(host); return }
+ if (literal === 6) { done(null); return }
+ lookup(host, { family: 4 }, (err, address) => done(err ? null : address))
+}
+
+const probeStun: StunProbe = (url, timeoutMs, signal) => {
+ const target = parseStunUrl(url)
+ // stuns: is STUN-over-TLS/TCP. werift asks over UDP, so such an entry is unusable there anyway and
+ // a UDP probe could not measure it honestly either.
+ if (!target || target.scheme !== 'stun') return Promise.resolve(null)
+ return new Promise<string | null>((resolve) => {
+ if (signal.aborted) { resolve(null); return }
+ let socket: dgram.Socket | null = null
+ let deadline: ReturnType<typeof setTimeout> | null = null
+ let retransmit: ReturnType<typeof setTimeout> | null = null
+ let done = false
+ const { packet, tid } = buildBindingRequest()
+
+ const finish = (address: string | null): void => {
+ if (done) return
+ done = true
+ if (deadline) clearTimeout(deadline)
+ if (retransmit) clearTimeout(retransmit)
+ signal.removeEventListener('abort', onAbort)
+ try { socket?.close() } catch { /* never opened, or already closed */ }
+ resolve(address)
+ }
+ function onAbort(): void { finish(null) }
+ signal.addEventListener('abort', onAbort, { once: true })
+
+ deadline = setTimeout(() => finish(null), timeoutMs)
+ deadline.unref?.()
+
+ resolveIpv4(target.host, (address) => {
+ if (done) return
+ if (!address) { finish(null); return }
+ try {
+ socket = dgram.createSocket('udp4')
+ } catch { finish(null); return }
+ // A probe must never be the reason the daemon stays alive.
+ socket.unref()
+ // Mandatory: a closed port answers with ICMP unreachable, which dgram raises as an 'error'
+ // EVENT. Unhandled, that throws out of the event loop and takes the process with it.
+ socket.on('error', () => finish(null))
+ socket.on('message', (msg) => { if (parseBindingResponse(msg, tid)) finish(address) })
+ socket.send(packet, target.port, address, (err) => { if (err) finish(null) })
+ retransmit = setTimeout(() => {
+ // Errors are ignored here: the first send already succeeded, so a failing retransmit should
+ // let the deadline decide rather than cut short a response still in flight.
+ if (!done) socket?.send(packet, target.port, address, () => { /* deadline decides */ })
+ }, PROBE_RETRANSMIT_MS)
+ retransmit.unref?.()
+ })
+ })
+}
+
+/** Only the winner, only for `stun:`, only when it was a name: we measured THAT address, and a host
+ * with several A records could resolve elsewhere inside werift's own lookup, quietly invalidating the
+ * race we just ran. Bounded by the positive TTL; a stale pin costs one slow gather, i.e. today. */
+function pinWinner(url: string, address: string): string {
+ const target = parseStunUrl(url)
+ if (!target || target.scheme !== 'stun' || isIP(target.host)) return url
+ return `stun:${address}:${target.port}`
+}
+
+export function createStunSelector(opts: StunSelectorOptions = {}): StunSelector {
+ const probe = opts.probe ?? probeStun
+ const now = opts.now ?? ((): number => Date.now())
+ const ttlMs = opts.ttlMs ?? POSITIVE_TTL_MS
+ const negativeTtlMs = opts.negativeTtlMs ?? NEGATIVE_TTL_MS
+ const probeTimeoutMs = opts.probeTimeoutMs ?? PROBE_TIMEOUT_MS
+ const cache = new Map<string, CacheEntry>()
+ const pending = new Map<string, Promise<string[]>>()
+
+ const race = (urls: string[]): Promise<RaceResult> => new Promise<RaceResult>((resolve) => {
+ const controller = new AbortController()
+ let settled = false
+ let outstanding = urls.length
+ const finish = (ordered: string[], ok: boolean): void => {
+ if (settled) return
+ settled = true
+ controller.abort() // closes every losing socket and clears its timers at once
+ resolve({ ordered, ok })
+ }
+ const lost = (): void => { if (--outstanding <= 0) finish(urls, false) }
+ urls.forEach((url, index) => {
+ let result: Promise<string | null>
+ try {
+ result = probe(url, probeTimeoutMs, controller.signal)
+ } catch {
+ lost() // a probe that throws synchronously is just a failed probe
+ return
+ }
+ result.then(
+ (address) => {
+ if (!address) { lost(); return }
+ finish([pinWinner(url, address), ...urls.filter((_, i) => i !== index)], true)
+ },
+ () => lost(),
+ )
+ })
+ })
+
+ return async (urls) => {
+ // Zero or one url: nothing to choose between, so do not spend a single packet deciding.
+ if (urls.length <= 1) return urls
+ const key = urls.map((url) => url.trim().toLowerCase()).join(' ')
+ const hit = cache.get(key)
+ if (hit && now() - hit.at < (hit.ok ? ttlMs : negativeTtlMs)) return hit.ordered
+ const inflight = pending.get(key)
+ if (inflight) return inflight
+ const run = race(urls)
+ .then(({ ordered, ok }) => {
+ cache.set(key, { ordered, ok, at: now() })
+ return ordered
+ })
+ .catch(() => urls) // the contract is that this function never rejects
+ .finally(() => { pending.delete(key) })
+ pending.set(key, run)
+ return run
+ }
+}
+
+/** Process-wide selector: one shared cache, so a warm-up on one relay pays for every later peer. */
+export const selectStunUrls: StunSelector = createStunSelector()
+
+/** Fire-and-forget: start the race the moment the url list is known, so its cost hides behind the
+ * round trip that follows. Callers join the same in-flight promise; nothing waits on this. */
+export function warmStunUrls(urls: string[]): void {
+ void selectStunUrls(urls).catch(() => { /* selectStunUrls never rejects; this is belt and braces */ })
+}
diff --git a/cli/src/lib/terminalP2p.spec.ts b/cli/src/lib/terminalP2p.spec.ts
index 2d39111ea..141a63046 100644
--- a/cli/src/lib/terminalP2p.spec.ts
+++ b/cli/src/lib/terminalP2p.spec.ts
@@ -1,5 +1,10 @@
-import { describe, expect, it } from 'vitest'
-import { TerminalP2pInitiator, TerminalP2pResponderPool } from './terminalP2p.js'
+import { describe, expect, it, vi } from 'vitest'
+import {
+ TerminalP2pInitiator,
+ TerminalP2pResponderPool,
+ type TerminalP2pSignal,
+} from './terminalP2p.js'
+import type { StunSelector } from './stunSelect.js'
describe('terminal WebRTC data channel', () => {
it('negotiates locally and carries ordered text and binary frames in both directions', async () => {
@@ -31,4 +36,124 @@ describe('terminal WebRTC data channel', () => {
await responder.stop()
}
}, 10_000)
+
+ it('races the policy stun urls but still offers the raw list, since peers need not agree', async () => {
+ const policyUrls = ['stun:a.example:3478', 'stun:b.example:3478']
+ // Returning [] keeps werift on host candidates, so the offer is emitted immediately instead of
+ // waiting out a gather against hosts that do not resolve.
+ const selectStunUrls = vi.fn<StunSelector>(async () => [])
+ let resolveOffer!: (payload: TerminalP2pSignal) => void
+ const offered = new Promise<TerminalP2pSignal>((resolve) => { resolveOffer = resolve })
+ const initiator = new TerminalP2pInitiator({
+ policy: { enabled: true, protocolVersion: 1, stunUrls: policyUrls, openWaitMs: 1_500 },
+ sendSignal: (type, payload) => { if (type === 'p2p_offer') resolveOffer(payload) },
+ onData: () => { /* no peer in this test */ },
+ selectStunUrls,
+ })
+
+ try {
+ initiator.start()
+ const offer = await offered
+ expect(selectStunUrls).toHaveBeenCalledTimes(1)
+ expect(selectStunUrls).toHaveBeenCalledWith(policyUrls)
+ // The winner is deliberately NOT pinned into the offer: the responder races the same list for
+ // itself, and a srflx candidate is each peer's own public address, so they need not agree.
+ expect(offer.stunUrls).toEqual(policyUrls)
+ } finally {
+ await initiator.stop('test_complete', false)
+ }
+ }, 15_000)
+
+ it('builds one peer connection when start() is called twice during the stun race', async () => {
+ let release!: (urls: string[]) => void
+ const gate = new Promise<string[]>((resolve) => { release = resolve })
+ const selectStunUrls = vi.fn<StunSelector>(() => gate)
+ const initiator = new TerminalP2pInitiator({
+ policy: { enabled: true, protocolVersion: 1, stunUrls: ['stun:a.example:3478', 'stun:b.example:3478'], openWaitMs: 1_500 },
+ sendSignal: () => { /* nothing to signal in this test */ },
+ onData: () => { /* no peer in this test */ },
+ selectStunUrls,
+ })
+
+ try {
+ initiator.start()
+ initiator.start()
+ expect(selectStunUrls).toHaveBeenCalledTimes(1)
+ } finally {
+ release([])
+ await initiator.stop('test_complete', false)
+ }
+ })
+
+ it('builds nothing at all when stop() lands while the stun race is still in flight', async () => {
+ let release!: (urls: string[]) => void
+ const gate = new Promise<string[]>((resolve) => { release = resolve })
+ const states: string[] = []
+ const signals: string[] = []
+ const initiator = new TerminalP2pInitiator({
+ policy: { enabled: true, protocolVersion: 1, stunUrls: ['stun:a.example:3478', 'stun:b.example:3478'], openWaitMs: 1_500 },
+ sendSignal: (type) => { signals.push(type) },
+ onData: () => { /* no peer in this test */ },
+ onState: (state) => { states.push(state) },
+ selectStunUrls: () => gate,
+ })
+
+ initiator.start()
+ const ready = initiator.waitUntilReady(5_000)
+ await initiator.stop('test_complete', false)
+ release([])
+ await new Promise((resolve) => setTimeout(resolve, 20))
+
+ // No offer was ever sent, which is the observable proof no RTCPeerConnection was constructed —
+ // one built after stop() would be an orphan with live UDP sockets nothing would ever close.
+ expect(signals).not.toContain('p2p_offer')
+ expect(await ready).toBe(false)
+ expect(states).toEqual(['connecting', 'closed'])
+ })
+
+ it('fails on the negotiation timeout even if the stun race never settles', async () => {
+ vi.useFakeTimers()
+ const states: Array<{ state: string; reason?: string }> = []
+ const initiator = new TerminalP2pInitiator({
+ policy: { enabled: true, protocolVersion: 1, stunUrls: ['stun:a.example:3478', 'stun:b.example:3478'], openWaitMs: 1_500 },
+ sendSignal: () => { /* nothing to signal in this test */ },
+ onData: () => { /* no peer in this test */ },
+ onState: (state, _setupMs, reason) => { states.push({ state, reason }) },
+ selectStunUrls: () => new Promise<string[]>(() => { /* wedged on purpose */ }),
+ })
+
+ try {
+ initiator.start()
+ await vi.advanceTimersByTimeAsync(10_001)
+ expect(states).toContainEqual({ state: 'failed', reason: 'negotiation_timeout' })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('hands the responder the offered urls, filtered and capped, to race on its own', async () => {
+ const selectStunUrls = vi.fn<StunSelector>(async (urls) => urls)
+ const responder = new TerminalP2pResponderPool({
+ sendSignal: () => { /* the offer below is deliberately unanswerable */ },
+ onData: () => { /* no data in this test */ },
+ selectStunUrls,
+ })
+
+ try {
+ await responder.handleSignal('source-1', 'p2p_offer', {
+ sessionId: '00000000-0000-4000-8000-000000000000',
+ protocolVersion: 1,
+ sdp: 'not-a-real-sdp',
+ stunUrls: [
+ 'stun:1.example:3478', 'stun:2.example:3478', 'stun:3.example:3478',
+ 'stun:4.example:3478', 'stun:5.example:3478', 'turn:6.example:3478', 42,
+ ] as unknown as string[],
+ })
+ expect(selectStunUrls).toHaveBeenCalledWith([
+ 'stun:1.example:3478', 'stun:2.example:3478', 'stun:3.example:3478', 'stun:4.example:3478',
+ ])
+ } finally {
+ await responder.stop()
+ }
+ })
})
diff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.ts
index d93626dec..e4c0bb9e3 100644
--- a/cli/src/lib/terminalP2p.ts
+++ b/cli/src/lib/terminalP2p.ts
@@ -1,5 +1,6 @@
import { randomUUID } from 'node:crypto'
import { RTCPeerConnection, type RTCDataChannel, type RTCIceCandidateInit } from 'werift'
+import { selectStunUrls, type StunSelector } from './stunSelect.js'
export const TERMINAL_P2P_PROTOCOL_VERSION = 1
export const TERMINAL_P2P_CHANNEL = 'terminal-v1'
@@ -49,6 +50,7 @@ export interface TerminalP2pInitiatorDeps {
onState?: (state: TerminalP2pState, setupMs: number, reason?: string) => void
onUnavailable?: (reason: string) => void
now?: () => number
+ selectStunUrls?: StunSelector
}
type ReadyWaiter = (ready: boolean) => void
@@ -84,31 +86,51 @@ export class TerminalP2pInitiator {
readonly sessionId = randomUUID()
private readonly startedAt: number
private readonly now: () => number
+ private readonly selectStunUrls: StunSelector
private pc: RTCPeerConnection | null = null
private channel: RTCDataChannel | null = null
private ready = false
+ private starting = false
private finished = false
private timeout: ReturnType<typeof setTimeout> | null = null
private waiters: ReadyWaiter[] = []
constructor(private readonly deps: TerminalP2pInitiatorDeps) {
this.now = deps.now ?? (() => Date.now())
+ this.selectStunUrls = deps.selectStunUrls ?? selectStunUrls
this.startedAt = this.now()
}
get isReady(): boolean { return this.ready && channelCanSend(this.channel) }
start(): void {
- if (this.pc || this.finished || !this.deps.policy.enabled) return
- const pc = new RTCPeerConnection(peerConfig(this.deps.policy.stunUrls))
+ // `starting`, not `this.pc`: the peer connection is only built after the STUN race below, so for
+ // those few hundred ms `this.pc` is still null and would let a second start() build a second one.
+ if (this.starting || this.pc || this.finished || !this.deps.policy.enabled) return
+ this.starting = true
+ // Armed here rather than in begin() so the race runs INSIDE the 10s budget instead of on top of
+ // it, and so a selector that somehow never settles still fails this initiator instead of hanging
+ // it forever. It also keeps setupMs on the same origin as before (startedAt, set in the ctor).
+ this.timeout = setTimeout(() => this.fail('negotiation_timeout'), TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS)
+ this.timeout.unref?.()
+ this.deps.onState?.('connecting', 0)
+ void this.begin()
+ }
+
+ private async begin(): Promise<void> {
+ let stunUrls = this.deps.policy.stunUrls
+ try {
+ stunUrls = await this.selectStunUrls(stunUrls)
+ } catch { /* the selector contract is that it never rejects; keep the policy order regardless */ }
+ // stop()/fail() may have run while the race was in flight. Building the peer connection now would
+ // strand it: nothing holds a reference any more, so its UDP sockets would never be closed.
+ if (this.finished) return
+ const pc = new RTCPeerConnection(peerConfig(stunUrls))
const channel = pc.createDataChannel(TERMINAL_P2P_CHANNEL, { ordered: true })
this.pc = pc
this.channel = channel
this.wirePeer(pc)
this.wireChannel(channel)
- this.timeout = setTimeout(() => this.fail('negotiation_timeout'), TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS)
- this.timeout.unref?.()
- this.deps.onState?.('connecting', 0)
void this.createOffer(pc)
}
@@ -245,6 +267,7 @@ export interface TerminalP2pResponderPoolDeps {
sendSignal: (connId: string, type: string, payload: TerminalP2pSignal) => void
onData: (connId: string, data: TerminalP2pData) => void
onUnavailable?: (connId: string, reason: string) => void
+ selectStunUrls?: StunSelector
}
interface ResponderEntry {
@@ -259,8 +282,16 @@ interface ResponderEntry {
/** Target side: one responder per authenticated source connId. */
export class TerminalP2pResponderPool {
private readonly entries = new Map<string, ResponderEntry>()
-
- constructor(private readonly deps: TerminalP2pResponderPoolDeps) {}
+ /** Per-connId offer generation. acceptOffer() awaits twice before it publishes its entry, so two
+ * offers arriving back to back can interleave (handleSignal is dispatched, not serialised) and the
+ * loser's peer connection would be orphaned by the winner's entries.set(). Bump on entry, re-check
+ * after every await. Pre-existing race — the STUN selection below only widens the window. */
+ private readonly offerSeq = new Map<string, number>()
+ private readonly selectStunUrls: StunSelector
+
+ constructor(private readonly deps: TerminalP2pResponderPoolDeps) {
+ this.selectStunUrls = deps.selectStunUrls ?? selectStunUrls
+ }
async handleSignal(connId: string, type: string, value: unknown): Promise<boolean> {
if (!TERMINAL_P2P_SIGNAL_TYPES.has(type)) return false
@@ -299,6 +330,11 @@ export class TerminalP2pResponderPool {
if (!entry || entry.closing) return
entry.closing = true
this.entries.delete(connId)
+ // Every reason but 'superseded' retires the connId for good, so drop its generation counter too or
+ // the map grows for the life of the daemon. 'superseded' is excluded because that call comes from
+ // acceptOffer itself, which has already claimed the current generation and still needs it as its
+ // own liveness check across the awaits that follow.
+ if (reason !== 'superseded') this.offerSeq.delete(connId)
clearTimeout(entry.timeout)
if (notifyPeer) {
this.deps.sendSignal(connId, 'p2p_abort', {
@@ -313,14 +349,24 @@ export class TerminalP2pResponderPool {
async stop(): Promise<void> {
await Promise.all([...this.entries.keys()].map((connId) => this.closeConnection(connId, 'shutdown', false)))
+ // Also drops generations for offers still inside their STUN race, which makes them bail instead of
+ // building a peer connection nothing would ever close.
+ this.offerSeq.clear()
}
private async acceptOffer(connId: string, payload: TerminalP2pSignal): Promise<void> {
+ const seq = (this.offerSeq.get(connId) ?? 0) + 1
+ this.offerSeq.set(connId, seq)
await this.closeConnection(connId, 'superseded', false)
+ if (this.offerSeq.get(connId) !== seq) return
const offeredStunUrls = Array.isArray(payload.stunUrls)
? payload.stunUrls.filter((url): url is string => typeof url === 'string' && /^stuns?:/i.test(url)).slice(0, 4)
: []
- const pc = new RTCPeerConnection(peerConfig(offeredStunUrls))
+ // The offerer raced these too, and may well have landed on a different server. That is fine: a
+ // srflx candidate is each peer's own public address, so the two sides need not agree on who to ask.
+ const ordered = await this.selectStunUrls(offeredStunUrls)
+ if (this.offerSeq.get(connId) !== seq) return
+ const pc = new RTCPeerConnection(peerConfig(ordered))
const entry: ResponderEntry = {
sessionId: payload.sessionId,
pc,