Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(p2p): give negotiation a budget that covers the peer's gather, and see where it goes

0xken15aad11bf08544
diff --git a/cli/src/config/env.ts b/cli/src/config/env.ts
index 4bf04c908..bab9cec15 100644
--- a/cli/src/config/env.ts
+++ b/cli/src/config/env.ts
@@ -344,6 +344,10 @@ const envSchema = z.object({
// encrypted on the wire, so a packet capture cannot answer "what did the adapter actually send".
// Never prints message text, transcripts, tokens or payload bodies.
LOG_FRAMES: z.string().default('false').transform((v) => v === 'true'),
+ // TEMPORARY / diagnostic. Forces iceTransportPolicy:'relay', so the ONLY candidate either peer can
+ // offer is a Cloudflare TURN allocation — the way to exercise the relay path on a pair that would
+ // otherwise always connect directly. Costs money per GB while it is on; never ship it enabled.
+ TERMINAL_P2P_FORCE_RELAY: z.string().default('false').transform((v) => v === 'true'),
// ── OpenRouter gateway agents (`ori claude`, `ori codex`, …) ───────────────────────────────────
// An agent whose CLI is pointed at OpenRouter has no vendor credential to spend, so its recap and
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.ts
index 3c95a1e15..696578259 100644
--- a/cli/src/lib/remoteRelay.spec.ts
+++ b/cli/src/lib/remoteRelay.spec.ts
@@ -45,6 +45,36 @@ describe('RemoteRelayPool reports terminal_link_mode to the local sink', () => {
})
})
+ it.each([
+ ['relay', 'turn'],
+ ['direct', 'p2p'],
+ [null, 'p2p'],
+ ])('a p2p terminal_ready whose ICE pair is %s reports mode:%s', (transport, expected) => {
+ // The data channel is up either way; what differs is which candidate pair ICE nominated. A relay
+ // pair means every byte is going through Cloudflare TURN, which is the state that costs money —
+ // and the one the badge could not tell apart before. A null pair keeps the pre-TURN optimism.
+ const entry = fakeEntry({ p2pPendingOpens: new Set(['req-turn']), p2p: { transport } })
+ pool.noteTerminalResponse(entry, { type: 'terminal_ready', payload: { requestId: 'req-turn', streamId: 'stream-turn' } }, 'p2p')
+
+ expect(entry.p2pStreams.has('stream-turn')).toBe(true)
+ expect(entry.sink.sendFrame).toHaveBeenCalledWith({
+ type: 'terminal_link_mode',
+ payload: { streamId: 'stream-turn', mode: expected },
+ })
+ })
+
+ it('a stream with no data channel stays mode:relay however the ICE pair reads', () => {
+ // Guards the additive rename: 'relay' must keep meaning "on the backend WebSocket" so an older
+ // Desktop build can never read a TURN session as a WS one.
+ const entry = fakeEntry({ p2pPendingOpens: new Set(['req-ws']), p2p: { transport: 'relay' } })
+ pool.noteTerminalResponse(entry, { type: 'terminal_ready', payload: { requestId: 'req-ws', streamId: 'stream-ws' } }, 'relay')
+
+ expect(entry.sink.sendFrame).toHaveBeenCalledWith({
+ type: 'terminal_link_mode',
+ payload: { streamId: 'stream-ws', mode: 'relay' },
+ })
+ })
+
it('a terminal_ready delivered over relay reports mode:relay', () => {
const entry = fakeEntry({ p2pPendingOpens: new Set(['req-2']) })
pool.noteTerminalResponse(entry, { type: 'terminal_ready', payload: { requestId: 'req-2', streamId: 'stream-2' } }, 'relay')
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index 8fa29fa83..66a0647d4 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -16,6 +16,7 @@ import { WebSocket, type RawData } from 'ws'
import type { Frame, LocalClientSink } from '../backendSocket.js'
import type { AuthSessionManager } from './authSession.js'
import { b64d, type Identity } from './e2ee/core.js'
+import { sid } from './log.js'
import type { MachinePeerStore } from './e2ee/machinePeers.js'
import { RelaySessionCrypto } from './e2ee/relayClient.js'
import { encodeTerminalLocal, TerminalBinaryKind, type TerminalBinaryClear } from './terminalBinary.js'
@@ -232,7 +233,17 @@ export class RemoteRelayPool {
// in-flight dedupe: begin() joins THIS race instead of starting a second one, and its
// cost hides behind the welcome round trip. Nothing here is load-bearing for
// correctness; drop it and the only change is a slightly later first terminal.
- if (entry.p2pPolicy) warmStunUrls(entry.p2pPolicy.stunUrls)
+ // Unconditional, one line per machine select. Whether the backend actually handed us a
+ // TURN credential is invisible everywhere else — not in LOG_FRAMES (which only covers
+ // backendSocket, never this relay client), not in the badge, not in p2p_result — and its
+ // absence is indistinguishable from "TURN is configured but ICE preferred direct".
+ // Names only, never the credential itself.
+ if (entry.p2pPolicy) {
+ const p = entry.p2pPolicy
+ console.log(`[p2p] policy · machine=${sid(machineId)} stun=${p.stunUrls.length}`
+ + ` turn=${p.turn ? `${p.turn.urls.length} urls` : 'NONE'} openWait=${p.openWaitMs}ms`)
+ warmStunUrls(p.stunUrls)
+ }
try { ws.send(JSON.stringify(crypto.helloFrame())) } catch { /* the close handler below rejects */ }
return
}
@@ -248,8 +259,17 @@ export class RemoteRelayPool {
if (frame.type === 'e2e_welcome') {
const ok = crypto.handleWelcome((frame.payload ?? {}) as Record<string, unknown>)
if (!ok) { if (!settled) { settled = true; clearTimeout(timeout); reject(new RelayConnectError('E2EE_WELCOME_INVALID')) } ; return }
- if (entry.p2pPolicy && crypto.terminalP2pVersion === TERMINAL_P2P_PROTOCOL_VERSION) {
- this.startP2p(entry)
+ // Three ways p2p never even starts, and until now all three looked identical from outside —
+ // the terminal just quietly stayed on the ws relay. The peer-version case is the important
+ // one: a machine whose CLI predates p2p answers no offer, so NO amount of STUN or TURN can
+ // help it. That is a very different problem from "ICE tried and failed".
+ if (!entry.p2pPolicy) {
+ console.log(`[p2p] off · machine=${sid(machineId)} backend sent no policy (rollout or kill switch)`)
+ } else if (crypto.terminalP2pVersion !== TERMINAL_P2P_PROTOCOL_VERSION) {
+ console.log(`[p2p] off · machine=${sid(machineId)} peer speaks p2p v${crypto.terminalP2pVersion},`
+ + ` we speak v${TERMINAL_P2P_PROTOCOL_VERSION} — no data channel is possible, ws relay only`)
+ } else {
+ this.startP2p(machineId, entry)
}
if (!settled) { settled = true; clearTimeout(timeout); resolve() }
return
@@ -379,7 +399,7 @@ export class RemoteRelayPool {
}
}
- private startP2p(entry: Entry): void {
+ private startP2p(machineId: string, entry: Entry): void {
const policy = entry.p2pPolicy
if (!policy || entry.p2p) return
let wasDirect = false
@@ -390,13 +410,24 @@ export class RemoteRelayPool {
try { entry.ws.send(JSON.stringify(wrapped)) } catch { /* relay close handles cleanup */ }
},
onData: (data) => this.handleP2pData(entry, data),
+ // Milestones, so a slow setup can be attributed instead of guessed at: our own gather runs from
+ // start to `offer-sent` (setLocalDescription awaits gathering — nothing here trickles), the peer's
+ // gather plus one relay round trip lands on `answer-in`, and everything after that is ICE
+ // connectivity checks. A 10s failure looks completely different depending on which gap owns it.
+ onStep: (step, elapsedMs) => {
+ console.log(`[p2p] step · machine=${sid(machineId)} ${step} +${Math.round(elapsedMs)}ms`)
+ },
onState: (state, setupMs, reason) => {
if (state === 'direct') {
wasDirect = true
// 'relayed' distinguishes a Cloudflare TURN path from a truly direct one. Both are "p2p" as far
// as the terminal is concerned, but only one of them is billed per GB.
- this.reportP2pResult(entry, 'direct', setupMs, p2p.transport === 'relay' ? 'relayed' : undefined)
+ const relayed = p2p.transport === 'relay'
+ console.log(`[p2p] connected · machine=${sid(machineId)} via=${relayed ? 'turn' : 'direct'} setup=${Math.round(setupMs)}ms`)
+ this.reportP2pResult(entry, 'direct', setupMs, relayed ? 'relayed' : undefined)
} else if (state === 'failed' && !wasDirect) {
+ console.log(`[p2p] gave up · machine=${sid(machineId)} reason=${reason ?? 'unknown'} after=${Math.round(setupMs)}ms`
+ + ` · ${p2p.negotiationDetail} — terminals stay on the ws relay`)
this.reportP2pResult(entry, reason === 'negotiation_timeout' ? 'timeout' : 'failed', setupMs, reason)
}
},
@@ -425,6 +456,25 @@ export class RemoteRelayPool {
this.noteTerminalResponse(entry, plain, 'p2p')
}
+ /**
+ * Which of the three paths this stream's bytes are on, for the local app's badge.
+ *
+ * 'p2p' — ICE nominated a direct candidate pair.
+ * 'turn' — there IS a data channel, but ICE could only nominate a relay pair, so every byte goes
+ * through Cloudflare TURN. Still WebRTC, still E2EE, but billed per GB.
+ * 'relay' — no data channel at all; the bytes are riding the backend WebSocket.
+ *
+ * 'turn' is additive: 'relay' keeps the exact meaning it has always had, so an older Desktop simply
+ * drops the unknown value and shows no badge rather than mislabelling a TURN session as a WS one.
+ *
+ * A null `transport` (channel open but werift exposed no candidate pair yet) reads as 'p2p' — the
+ * same optimistic answer this frame already gave before TURN existed.
+ */
+ private linkMode(entry: Entry, streamId: string): 'p2p' | 'turn' | 'relay' {
+ if (!entry.p2pStreams.has(streamId)) return 'relay'
+ return entry.p2p?.transport === 'relay' ? 'turn' : 'p2p'
+ }
+
private noteTerminalResponse(entry: Entry, frame: Frame, transport: 'p2p' | 'relay'): void {
const payload = framePayload(frame)
const streamId = typeof payload.streamId === 'string' ? payload.streamId : ''
@@ -435,7 +485,7 @@ export class RemoteRelayPool {
// entry.p2pStreams' own post-update membership (the routing source of truth just above), not a
// naive echo of `transport`, so a stale/duplicate terminal_ready can never report a mode that
// doesn't match what's actually routing.
- entry.sink?.sendFrame({ type: 'terminal_link_mode', payload: { streamId, mode: entry.p2pStreams.has(streamId) ? 'p2p' : 'relay' } })
+ entry.sink?.sendFrame({ type: 'terminal_link_mode', payload: { streamId, mode: this.linkMode(entry, streamId) } })
} else if (frame.type === 'terminal_error' && requestId) {
entry.p2pPendingOpens.delete(requestId)
} else if (frame.type === 'terminal_closed' && streamId) {
diff --git a/cli/src/lib/terminalP2p.spec.ts b/cli/src/lib/terminalP2p.spec.ts
index b8d06973b..7804bcaf0 100644
--- a/cli/src/lib/terminalP2p.spec.ts
+++ b/cli/src/lib/terminalP2p.spec.ts
@@ -1,6 +1,8 @@
import { describe, expect, it, vi } from 'vitest'
import {
+ isRelayedPair,
readTurn,
+ TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS,
TerminalP2pInitiator,
TerminalP2pResponderPool,
type TerminalP2pSignal,
@@ -125,7 +127,7 @@ describe('terminal WebRTC data channel', () => {
try {
initiator.start()
- await vi.advanceTimersByTimeAsync(10_001)
+ await vi.advanceTimersByTimeAsync(TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS + 1)
expect(states).toContainEqual({ state: 'failed', reason: 'negotiation_timeout' })
} finally {
vi.useRealTimers()
@@ -217,4 +219,29 @@ describe('terminal WebRTC data channel', () => {
await responder.stop()
}
}, 15_000)
+
+ // Real candidate lines, captured from a forced relay-only run against Cloudflare.
+ const RELAY = 'candidate:856fe30cc 1 udp 16777215 104.30.136.14 29000 typ relay raddr 14.161.43.75 rport 55397'
+ const SRFLX = 'candidate:2b1f0a4c9 1 udp 1686052607 14.161.43.75 55397 typ srflx raddr 192.168.1.16 rport 55397'
+ const HOST = 'candidate:9d3e77bb1 1 udp 2130706431 192.168.1.16 55397 typ host'
+
+ it('counts a pair as relayed when EITHER end is a turn allocation', () => {
+ // The half that used to be missed: our side is srflx, the peer allocated the relay, and every byte
+ // still crosses Cloudflare — reading only the local candidate reported that as direct and
+ // under-counted the traffic that gets billed.
+ expect(isRelayedPair(SRFLX, RELAY)).toBe(true)
+ expect(isRelayedPair(RELAY, SRFLX)).toBe(true)
+ expect(isRelayedPair(RELAY, RELAY)).toBe(true)
+ })
+
+ it('counts a pair as direct only when neither end relays', () => {
+ expect(isRelayedPair(HOST, HOST)).toBe(false)
+ expect(isRelayedPair(SRFLX, SRFLX)).toBe(false)
+ expect(isRelayedPair(HOST, SRFLX)).toBe(false)
+ expect(isRelayedPair(SRFLX)).toBe(false) // remote unknown
+ })
+
+ it('does not mistake a host address that merely contains the word', () => {
+ expect(isRelayedPair('candidate:1 1 udp 1 10.0.0.1 1 typ host raddr relay.example')).toBe(false)
+ })
})
diff --git a/cli/src/lib/terminalP2p.ts b/cli/src/lib/terminalP2p.ts
index b5cb598e4..e3719b0d2 100644
--- a/cli/src/lib/terminalP2p.ts
+++ b/cli/src/lib/terminalP2p.ts
@@ -1,10 +1,25 @@
import { randomUUID } from 'node:crypto'
import { RTCPeerConnection, type RTCDataChannel, type RTCIceCandidateInit } from 'werift'
+import { env } from '../config/env.js'
import { pickTurnUrl, selectStunUrls, type StunSelector } from './stunSelect.js'
export const TERMINAL_P2P_PROTOCOL_VERSION = 1
export const TERMINAL_P2P_CHANNEL = 'terminal-v1'
-export const TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS = 10_000
+/**
+ * The budget covers BOTH peers' gathering, not just ours, and 10s was too tight for that.
+ *
+ * Measured: our own gather is 100-400ms, but a peer whose network drops UDP to the STUN server stalls
+ * the full 5s werift gather timeout (`getCandidatePromises(addresses, timeout = 5)`, hard-coded, not
+ * exposed through RTCPeerConnection) before it can answer — 5027ms with a dead STUN in the list versus
+ * 111ms with a live one. Add a TURN allocation over TLS and one relay round trip and the answer lands
+ * around 7.2s, leaving under 3s for relay-to-relay connectivity checks, which is not enough. Observed
+ * exactly that: `answer-in +7208ms` then death at +10001ms still in `connecting`.
+ *
+ * Raising it costs nothing user-visible: a terminal that opens before the channel is ready falls back
+ * to the ws relay after `openWaitMs` (2.5s) regardless, and negotiation continuing in the background
+ * only means LATER opens get p2p. Whichever side fires first aborts for both, so both ends need this.
+ */
+export const TERMINAL_P2P_NEGOTIATION_TIMEOUT_MS = 25_000
export const TERMINAL_P2P_MAX_BUFFERED_BYTES = 2 * 1024 * 1024
export const TERMINAL_P2P_SIGNAL_TYPES = new Set([
@@ -65,6 +80,8 @@ export interface TerminalP2pInitiatorDeps {
onUnavailable?: (reason: string) => void
now?: () => number
selectStunUrls?: StunSelector
+ /** Coarse negotiation milestones, for working out WHERE a slow setup spends its time. */
+ onStep?: (step: string, elapsedMs: number) => void
}
type ReadyWaiter = (ready: boolean) => void
@@ -93,6 +110,10 @@ function peerConfig(stunUrls: string[], turn?: TurnChoice): ConstructorParameter
if (turn) iceServers.push({ urls: [turn.url], username: turn.username, credential: turn.credential })
return {
iceServers,
+ // Diagnostic only (TERMINAL_P2P_FORCE_RELAY): drops host and srflx so nothing but a TURN
+ // allocation can be nominated. Note it also skips STUN gathering entirely inside werift, which is
+ // why a forced run is fast even on a network where STUN is unreachable.
+ ...(env.TERMINAL_P2P_FORCE_RELAY ? { iceTransportPolicy: 'relay' as const } : {}),
// Left at the default 'all' deliberately: that is what produces "direct first, relay only if
// nothing else works". werift scores candidates host 126 > srflx 100 > relay 0, so a TURN relay
// pair is only ever nominated once every direct pair has failed. Setting 'relay' here would force
@@ -123,6 +144,20 @@ function turnChoice(turn: TerminalP2pTurn | undefined, udpReachable: boolean | n
return url ? { url, username: turn.username, credential: turn.credential } : undefined
}
+/**
+ * Does the nominated pair send its bytes through a TURN allocation?
+ *
+ * BOTH ends decide this, not just ours. One relay candidate is enough for ICE to connect, so a pair of
+ * our srflx with the peer's relay is fully relayed by Cloudflare — and reading only the local side
+ * (which is what this did at first) reports it as 'direct' and under-counts exactly the traffic that
+ * gets billed per GB. werift's RTCIceCandidate carries only the SDP line, so the type comes out of
+ * the string.
+ */
+export function isRelayedPair(local: string, remote?: string): boolean {
+ const relay = /\btyp relay\b/
+ return relay.test(local) || (typeof remote === 'string' && relay.test(remote))
+}
+
function channelCanSend(channel: RTCDataChannel | null): channel is RTCDataChannel {
return channel?.readyState === 'open'
&& channel.bufferedAmount < TERMINAL_P2P_MAX_BUFFERED_BYTES
@@ -141,6 +176,7 @@ export class TerminalP2pInitiator {
private finished = false
private timeout: ReturnType<typeof setTimeout> | null = null
private waiters: ReadyWaiter[] = []
+ private sawAnswer = false
constructor(private readonly deps: TerminalP2pInitiatorDeps) {
this.now = deps.now ?? (() => Date.now())
@@ -148,6 +184,10 @@ export class TerminalP2pInitiator {
this.startedAt = this.now()
}
+ private step(label: string): void {
+ this.deps.onStep?.(label, this.now() - this.startedAt)
+ }
+
get isReady(): boolean { return this.ready && channelCanSend(this.channel) }
/**
@@ -155,13 +195,30 @@ export class TerminalP2pInitiator {
* through Cloudflare TURN, which is billed per GB — without reporting this we would have no idea how
* much of the traffic costs money. `null` while still negotiating, or if werift exposes no pair.
*/
+ /**
+ * Why a negotiation went nowhere, in one line. The distinction that matters is `answer=no`: it means
+ * the peer never replied to our offer, so no candidate of ours — STUN or TURN — was ever going to be
+ * tried. That is a peer problem, not a connectivity one, and no amount of relay fixes it.
+ */
+ get negotiationDetail(): string {
+ // Both sides gather before signalling (nothing subscribes to onIceCandidate), so the candidate
+ // types live in the descriptions, NOT in the trickle frames — counting those would always read 0
+ // and hide exactly the asymmetry that matters: whether the PEER managed a relay candidate too.
+ const types = (sdp: string): string =>
+ [...new Set([...sdp.matchAll(/typ (\w+)/g)].map((m) => m[1]))].join('/') || 'none'
+ return `answer=${this.sawAnswer ? 'yes' : 'no'}`
+ + ` ours=${types(this.pc?.localDescription?.sdp ?? '')}`
+ + ` peer=${types(this.pc?.remoteDescription?.sdp ?? '')}`
+ + ` ice=${this.pc?.connectionState ?? '-'}`
+ }
+
get transport(): 'direct' | 'relay' | null {
const pc = this.pc
if (!pc || !this.ready) return null
for (const iceTransport of pc.iceTransports) {
- // werift's RTCIceCandidate carries only the SDP line, so the type comes out of the string.
- const line = iceTransport.getSelectedCandidatePair?.()?.local?.candidate
- if (typeof line === 'string' && line) return /\btyp relay\b/.test(line) ? 'relay' : 'direct'
+ const pair = iceTransport.getSelectedCandidatePair?.()
+ if (!pair?.local?.candidate) continue
+ return isRelayedPair(pair.local.candidate, pair.remote?.candidate) ? 'relay' : 'direct'
}
return null
}
@@ -191,6 +248,7 @@ export class TerminalP2pInitiator {
// stop()/fail() may have run while the race was in flight. Building the peer connection now would
// strand it: nothing holds a reference any more, so its UDP sockets would never be closed.
if (this.finished) return
+ this.step('stun-raced')
const pc = new RTCPeerConnection(peerConfig(stunUrls, turnChoice(this.deps.policy.turn, udpReachable)))
const channel = pc.createDataChannel(TERMINAL_P2P_CHANNEL, { ordered: true })
this.pc = pc
@@ -216,6 +274,7 @@ export class TerminalP2pInitiator {
stunUrls: this.deps.policy.stunUrls,
...(this.deps.policy.turn ? { turn: this.deps.policy.turn } : {}),
})
+ this.step('offer-sent')
} catch {
this.fail('offer_failed')
}
@@ -229,6 +288,8 @@ export class TerminalP2pInitiator {
if (!pc) return true
try {
if (type === 'p2p_answer' && typeof payload.sdp === 'string') {
+ this.sawAnswer = true
+ this.step('answer-in')
await pc.setRemoteDescription({ type: 'answer', sdp: payload.sdp })
} else if (type === 'p2p_ice_candidate') {
await pc.addIceCandidate(payload.candidate ?? null)
@@ -298,6 +359,7 @@ export class TerminalP2pInitiator {
private wirePeer(pc: RTCPeerConnection): void {
pc.connectionStateChange.subscribe((state) => {
if (this.pc !== pc || this.finished) return
+ this.step(`ice-${state}`)
if (state === 'failed' || state === 'disconnected') this.fail(`peer_${state}`)
})
}